const config = require('./config.json'); const { DOMAIN, PRIVKEY_PATH, CERT_PATH, PORT_HTTP, PORT_HTTPS } = config; const Database = require('better-sqlite3'); const db = new Database('bot-node.db'), Parser = require('rss-parser'), request = require('request'), crypto = require('crypto'), parser = new Parser({timeout: 2000}); const Jackd = require('jackd'); const beanstalkd = new Jackd(); beanstalkd.connect() async function foo() { while (true) { try { const { id, payload } = await beanstalkd.reserve() console.log(payload) /* ... process job here ... */ await beanstalkd.delete(id) await doFeed(payload) } catch (err) { // Log error somehow console.error(err) } } } foo() function doFeed(feedUrl) { return new Promise((resolve, reject) => { // fetch new RSS for each feed parser.parseURL(feedUrl, function(err, feedData) { if (err) { reject('error fetching ' + feedUrl + '; ' + err); } else { let feed = db.prepare('select * from feeds where feed = ?').get(feedUrl); // get the old feed data from the database let oldFeed = JSON.parse(feed.content); // compare the feed item contents. if there's one or more whole new items (aka a new item with a unique guid), // add the items to a list like // [ { items: [], username }, {}, ... ] let oldItems = oldFeed.items; let newItems = feedData.items; // find the difference of the sets of guids (fall back to title or // description since guid is not required by spec) in the old and new feeds let oldGuidSet = new Set(oldItems.map(el => el.guid || el.title || el.description)); let newGuidSet = new Set(newItems.map(el => el.guid || el.title || el.description)); // find things in the new set that aren't in the old set let difference = new Set( [...newGuidSet].filter(x => !oldGuidSet.has(x))); difference = [...difference]; console.log('diff', feed.feed, difference.length, difference); if (difference.length > 0) { // get a list of new items in the diff let brandNewItems = newItems.filter(el => difference.includes(el.guid) || difference.includes(el.title) || difference.includes(el.description)); let acct = feed.username; let domain = DOMAIN; //console.log(acct, brandNewItems); // send the message to everyone for each item! for (var item of brandNewItems) { item = transformContent(item); let message; if(item.link.match('/twitter.com/')) { message = `${item.content}`; } else { message = `
${item.content || ''}
`; } if (item.enclosure && item.enclosure.url && item.enclosure.url.includes('.mp3')) { message += ``; } sendCreateMessage(message, acct, domain, null, null, item); } // update the DB with new contents let content = JSON.stringify(feedData); db.prepare('insert or replace into feeds(feed, username, content) values(?, ?, ?)').run(feed.feed, acct, content); return resolve('done with ' + feedUrl) } else { return resolve('done with ' + feedUrl + ', no change') } } }); }).catch((e) => console.log(e)); } // TODO: update the display name of a feed if the feed title has changed // This is a function with a bunch of custom rules for different kinds of content I've found in the wild in things like Reddit rss feeds. Right now we just use the first image we find, if any. function transformContent(item) { let cheerio = require('cheerio'); console.log(JSON.stringify(item)); if (item.content === undefined) { item.urls = []; return item; } let $ = cheerio.load(item.content); // look through all the links to find images let links = $('a'); let urls = []; //console.log('links', links.length); links.each((i,e) => { let url = $(e).attr('href'); // if there's an image, add it as a media attachment if (url && url.match(/(http)?s?:?(\/\/[^"']*\.(?:png|jpg|jpeg|gif|png|svg))/)) { //console.log(url); urls.push(url); } }); // look through all the images let images = $('img'); images.each((i,e) => { let url = $(e).attr('src'); // if there's an image, add it as a media attachment if (url) { //console.log(url); urls.push(url); // remove the image from the post body since it's in the attachment now $(e).remove(); } }); item.urls = urls; // find iframe embeds and turn them into links let iframes = $('iframe'); iframes.each((i,e) => { console.log('iframe',i,e); let url = $(e).attr('src'); $(e).replaceWith($(`[embedded content]`)); }); // remove multiple line breaks //$('br').remove(); $('p').each((i, el) => { if($(el).html().replace(/\s| /g, '').length === 0) {$(el).remove();} }); // couple of hacky regexes to make sure we clean up everything item.content = $('body').html().replace(/^(\n|\r)/g,'').replace(/>\r+<').replace(/ +/g, ''); item.content = item.content.replace(/^(\n|\r)/g,'').replace(/>\r+<').replace(/>\s*<').replace(//g,'').replace(/>\u200B+<').replace(/ +/g, '').replace(/<\/p>/g,'').replace(/(
)+/g,'
');
return item;
}
// for each item in the list, get the account corresponding to the username
// for each item in the ITEMS list, send a message to all followers
// TODO import these form a helper
function signAndSend(message, name, domain, req, res, targetDomain, inbox) {
// get the private key
console.log('sending to ', name, targetDomain, inbox);
let inboxFragment = inbox.replace('https://'+targetDomain,'');
let result = db.prepare('select privkey from accounts where name = ?').get(`${name}@${domain}`);
//console.log('got key', result === undefined, `${name}@${domain}`);
if (result === undefined) {
console.log(`No record found for ${name}.`);
}
else {
let privkey = result.privkey;
const signer = crypto.createSign('sha256');
let d = new Date();
let stringToSign = `(request-target): post ${inboxFragment}\nhost: ${targetDomain}\ndate: ${d.toUTCString()}`;
signer.update(stringToSign);
signer.end();
const signature = signer.sign(privkey);
const signature_b64 = signature.toString('base64');
let header = `keyId="https://${domain}/u/${name}",headers="(request-target) host date",signature="${signature_b64}"`;
//console.log('signature:',header);
request({
url: inbox,
headers: {
'Host': targetDomain,
'Date': d.toUTCString(),
'Signature': header
},
method: 'POST',
json: true,
body: message
}, function (error, response, body){
});
}
}
function createMessage(text, name, domain, item, follower, guidNote) {
const guidCreate = crypto.randomBytes(16).toString('hex');
let d = new Date();
let out = {
'@context': 'https://www.w3.org/ns/activitystreams',
'id': `https://${domain}/m/${guidCreate}`,
'type': 'Create',
'actor': `https://${domain}/u/${name}`,
'to': [ follower ],
'object': {
'id': `https://${domain}/m/${guidNote}`,
'type': 'Note',
'published': d.toISOString(),
'attributedTo': `https://${domain}/u/${name}`,
'content': text,
'link': item.link,
'cc': 'https://www.w3.org/ns/activitystreams#Public'
}
};
// add image attachment
let attachment;
console.log('NUM IMAGES',item.urls.length);
if (item.enclosure && item.enclosure.url && item.enclosure.url.includes('.mp3')) {
attachment = {
'type': 'Document',
'mediaType': 'audio/mpeg',
'url': item.enclosure.url,
'name': null
};
out.object.attachment = attachment;
}
else if (item.urls.length > 0) {
attachment = {
'type': 'Document',
'mediaType': 'image/png', // TODO: update the mediaType to match jpeg,gif,etc
'url': item.urls[0],
'name': null
};
out.object.attachment = attachment;
}
else if (item.urls.length > 1) {
attachment = [];
let lengthFourMax = Math.min(item.urls.length, 4);
for (var i=0; i