diff --git a/.gitignore b/.gitignore index a8d9779..0167d21 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ node_modules/ *.db package-lock.json -config.json +config.js diff --git a/README.md b/README.md index 6f40b3b..c2f0e4a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ As of the `v2.0.0` release of this project, only users who are authenticated wit ## Requirements -This requires Node.js v10.10.0 or above. +This requires Node.js v20 or above. You also need `beanstalkd` running. This is a simple and fast queueing system we use to manage polling RSS feeds. [Here are installation instructions](https://beanstalkd.github.io/download.html). On a production server you'll want to [install it as a background process](https://github.com/beanstalkd/beanstalkd/tree/master/adm). @@ -20,46 +20,46 @@ Clone the repository, then `cd` into its root directory. Install dependencies: `npm i` -Then copy `config.json.template` to `config.json`: +Then copy `config.js.template` to `config.js`: -`cp config.json.template config.json` +`cp config.js.template config.js` -Update your new `config.json` file: +Update your new `config.js` file: ```js -{ - "DOMAIN": "mydomain.com", - "PORT_HTTP": "3000", - "PORT_HTTPS": "8443", - "PRIVKEY_PATH": "/path/to/your/ssl/privkey.pem", - "CERT_PATH": "/path/to/your/ssl/cert.pem", - "OAUTH": { - "client_id": "abc123def456", - "client_secret": "zyx987wvu654", - "redirect_uri": "https://rss.example.social/convert", - "domain": "example.social", - "domain_human": "Example Online Community", - "authorize_path": "/oauth/authorize", - "token_path": "/oauth/token", - "token_verification_path": "/some/path/to/verify/token" - } -} +export default { + DOMAIN: "mydomain.com", + PORT_HTTP: "3000", + PORT_HTTPS: "8443", + PRIVKEY_PATH: "/path/to/your/ssl/privkey.pem", + CERT_PATH: "/path/to/your/ssl/cert.pem", + OAUTH: { + client_id: "abc123def456", + client_secret: "zyx987wvu654", + redirect_uri: "https://rss.example.social/convert", + domain: "example.social", + domain_human: "Example Online Community", + authorize_path: "/oauth/authorize", + token_path: "/oauth/token", + token_verification_path: "/some/path/to/verify/token", + }, +}; ``` -* `DOMAIN`: your domain! this should be a discoverable domain of some kind like "example.com" or "rss.example.com" -* `PORT_HTTP`: the http port that Express runs on -* `PORT_HTTPS`: the https port that Express runs on -* `PRIVKEY_PATH`: point this to your private key you got from Certbot or similar -* `CERT_PATH`: point this to your cert you got from Certbot or similar -* `OAUTH`: this object contains properties related to OAuth login. See the section below on "Running with OAuth" for more details. - * `client_id`: also known as the "client key". A long series of characters. You generate this when you register this application with an OAuth provider. - * `client_secret`: Another long series of characters that you generate when you register this application with an OAuth provider. - * `redirect_uri`: This is the URI that people get redirected to after they authorize the application on the OAuth server. Must point to the server where THIS service is running, and must point to the `/convert` page. This uri has to match what you put in the application info on the OAuth provider. - * `domain`: The domain of the OAuth provider. Not necessarily the same as this server (for example, you could host this at rss.mydomain.com and then handle all OAuth through some other server you control, like a Mastodon server). - * `domain_human`: The human-readable name of the OAuth provider. This will appear in various messages, so if you say "Example Online Community" here then the user will see a message like "Click here to log in via Example Online Community". - * `authorize_path`: This will generally be `/oauth/authorize/` but you can change it here if your OAuth provider uses a nonstandard authorization path. - * `token_path`: This will generally be `/oauth/token/` but you can change it here if your OAuth provider uses a nonstandard token path. - * `token_verification_path`: This should be the path to any URL at the OAuth server that responds with an HTTP status code 200 when you are correctly logged in (and with a non-200 value when you are not). This is the path relative to the `domain` you set, so if your `domain` is `example.social` and you set `token_verification_path` to `/foo/bar/` then the full path that this service will run a GET on to verify you are logged in is `https://example.social/foo/bar`. +- `DOMAIN`: your domain! this should be a discoverable domain of some kind like "example.com" or "rss.example.com" +- `PORT_HTTP`: the http port that Express runs on +- `PORT_HTTPS`: the https port that Express runs on +- `PRIVKEY_PATH`: point this to your private key you got from Certbot or similar +- `CERT_PATH`: point this to your cert you got from Certbot or similar +- `OAUTH`: this object contains properties related to OAuth login. See the section below on "Running with OAuth" for more details. + - `client_id`: also known as the "client key". A long series of characters. You generate this when you register this application with an OAuth provider. + - `client_secret`: Another long series of characters that you generate when you register this application with an OAuth provider. + - `redirect_uri`: This is the URI that people get redirected to after they authorize the application on the OAuth server. Must point to the server where THIS service is running, and must point to the `/convert` page. This uri has to match what you put in the application info on the OAuth provider. + - `domain`: The domain of the OAuth provider. Not necessarily the same as this server (for example, you could host this at rss.mydomain.com and then handle all OAuth through some other server you control, like a Mastodon server). + - `domain_human`: The human-readable name of the OAuth provider. This will appear in various messages, so if you say "Example Online Community" here then the user will see a message like "Click here to log in via Example Online Community". + - `authorize_path`: This will generally be `/oauth/authorize/` but you can change it here if your OAuth provider uses a nonstandard authorization path. + - `token_path`: This will generally be `/oauth/token/` but you can change it here if your OAuth provider uses a nonstandard token path. + - `token_verification_path`: This should be the path to any URL at the OAuth server that responds with an HTTP status code 200 when you are correctly logged in (and with a non-200 value when you are not). This is the path relative to the `domain` you set, so if your `domain` is `example.social` and you set `token_verification_path` to `/foo/bar/` then the full path that this service will run a GET on to verify you are logged in is `https://example.social/foo/bar`. Run the server! @@ -75,22 +75,22 @@ There is also a file called `queueFeeds.js` that needs to be run on a cron job o OAuth is unfortunately a bit underspecified so there are a lot of funky implementations out there. Here I will include an example of using a Mastodon server as the OAuth provider. This is how I have my RSS service set up: I run friend.camp as my Mastodon server, and I use my admin powers on friend.camp to register rss.friend.camp as an application. The steps for this, for Mastodon, are: -* log in as an admin user -* go to Preferences -* select Development -* select New Application -* type in an application name, and the URL where this service is running -* type in the redirect URI, which will be whatever base domain this service is running at with the `/convert` path appended. So something like `https://rss.example.social/convert` -* uncheck all scopes, and check `read:accounts` (this is the minimum required access, simply so this RSS converter can confirm someone is truly logged in) -* once you're done, save -* you will now have access to a "client key" and "client secret" for this app. -* open `config.js` in an editor -* fill in `client_id` with the client key, and `client_secret` with the client secret. -* set the `redirect_uri` to be identical to the one you put in Mastodon. It should look like `https://rss.example.social/convert` (the `/convert` part is important, this software won't work if you point to a different path) -* set `domain` to the domain of your Mastodon server, and `domain_human` to its human-friendly name -* leave `authorize_path` and `token_path` on their defaults -* set `token_verification_path` to `/api/v1/accounts/verify_credentials` -* cross your fingers and start up this server +- log in as an admin user +- go to Preferences +- select Development +- select New Application +- type in an application name, and the URL where this service is running +- type in the redirect URI, which will be whatever base domain this service is running at with the `/convert` path appended. So something like `https://rss.example.social/convert` +- uncheck all scopes, and check `read:accounts` (this is the minimum required access, simply so this RSS converter can confirm someone is truly logged in) +- once you're done, save +- you will now have access to a "client key" and "client secret" for this app. +- open `config.js` in an editor +- fill in `client_id` with the client key, and `client_secret` with the client secret. +- set the `redirect_uri` to be identical to the one you put in Mastodon. It should look like `https://rss.example.social/convert` (the `/convert` part is important, this software won't work if you point to a different path) +- set `domain` to the domain of your Mastodon server, and `domain_human` to its human-friendly name +- leave `authorize_path` and `token_path` on their defaults +- set `token_verification_path` to `/api/v1/accounts/verify_credentials` +- cross your fingers and start up this server ## Local testing @@ -112,22 +112,22 @@ There are two tables in the database: `accounts` and `feeds`. This table keeps track of all the data needed for the accounts. Columns: -* `name` `TEXT PRIMARY KEY`: the account name, in the form `thename@example.com` -* `privkey` `TEXT`: the RSA private key for the account -* `pubkey` `TEXT`: the RSA public key for the account -* `webfinger` `TEXT`: the entire contents of the webfinger JSON served for this account -* `actor` `TEXT`: the entire contents of the actor JSON served for this account -* `apikey` `TEXT`: the API key associated with this account -* `followers` `TEXT`: a JSON-formatted array of the URL for the Actor JSON of all followers, in the form `["https://remote.server/users/somePerson", "https://another.remote.server/ourUsers/anotherPerson"]` -* `messages` `TEXT`: not yet used but will eventually store all messages so we can render them on a "profile" page +- `name` `TEXT PRIMARY KEY`: the account name, in the form `thename@example.com` +- `privkey` `TEXT`: the RSA private key for the account +- `pubkey` `TEXT`: the RSA public key for the account +- `webfinger` `TEXT`: the entire contents of the webfinger JSON served for this account +- `actor` `TEXT`: the entire contents of the actor JSON served for this account +- `apikey` `TEXT`: the API key associated with this account +- `followers` `TEXT`: a JSON-formatted array of the URL for the Actor JSON of all followers, in the form `["https://remote.server/users/somePerson", "https://another.remote.server/ourUsers/anotherPerson"]` +- `messages` `TEXT`: not yet used but will eventually store all messages so we can render them on a "profile" page ### `feeds` This table keeps track of all the data needed for the feeds. Columns: -* `feed` `TEXT PRIMARY KEY`: the URI of the RSS feed -* `username` `TEXT`: the username associated with the RSS feed -* `content` `TEXT`: the most recent copy fetched of the RSS feed's contents +- `feed` `TEXT PRIMARY KEY`: the URI of the RSS feed +- `username` `TEXT`: the username associated with the RSS feed +- `content` `TEXT`: the most recent copy fetched of the RSS feed's contents ## License diff --git a/config.json.template b/config.js.template similarity index 84% rename from config.json.template rename to config.js.template index f60417a..a6c3ad0 100644 --- a/config.json.template +++ b/config.js.template @@ -1,7 +1,7 @@ -{ +export default { "DOMAIN": "", "PORT_HTTP": "3000", "PORT_HTTPS": "8443", "PRIVKEY_PATH": "", "CERT_PATH": "" -} +}; \ No newline at end of file diff --git a/index.js b/index.js index df07b3e..f86f659 100644 --- a/index.js +++ b/index.js @@ -1,14 +1,17 @@ -const config = require('./config.json'); +import express from 'express'; +import bodyParser from 'body-parser'; +import cors from 'cors'; +import http from 'http'; +import Database from 'better-sqlite3'; +import fs from 'fs'; +import config from './config.js'; +import routes from './routes/index.js'; + const { DOMAIN, PRIVKEY_PATH, CERT_PATH, PORT_HTTP, PORT_HTTPS, OAUTH } = config; -const express = require('express'); + const app = express(); -const Database = require('better-sqlite3'); const db = new Database('bot-node.db'); -const fs = require('fs'); -const routes = require('./routes'), - bodyParser = require('body-parser'), - cors = require('cors'), - http = require('http'); + let sslOptions; try { diff --git a/package.json b/package.json index 65c4873..a3daac9 100644 --- a/package.json +++ b/package.json @@ -1,28 +1,28 @@ { - "name": "rss-to-activitypub", - "version": "2.0.0", - "description": "", - "main": "index.js", - "dependencies": { - "better-sqlite3": "^5.0.1", - "body-parser": "^1.18.3", - "cheerio": "^1.0.0-rc.2", - "cors": "^2.8.4", - "express": "^4.16.3", - "generate-rsa-keypair": "^0.1.2", - "jackd": "^1.2.4", - "parse-favicon": "^2.0.0", - "pug": "^2.0.3", - "request": "^2.87.0", - "rss-parser": "^3.6.3" - }, - "engines": { - "node": ">=10.10.0" - }, - "devDependencies": {}, - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "", - "license": "MIT" + "name": "rss-to-activitypub", + "version": "2.0.0", + "description": "", + "main": "index.js", + "type": "module", + "dependencies": { + "better-sqlite3": "^11.3.0", + "body-parser": "^1.20.3", + "cheerio": "^1.0.0", + "cors": "^2.8.5", + "express": "^4.21.0", + "generate-rsa-keypair": "^0.2.1", + "jackd": "^2.2.2", + "parse-favicon": "^7.0.1", + "pug": "^3.0.3", + "rss-parser": "^3.13.0" + }, + "engines": { + "node": ">=10.10.0" + }, + "devDependencies": {}, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "MIT" } diff --git a/queueFeeds.js b/queueFeeds.js index b5e08bf..85dd67f 100644 --- a/queueFeeds.js +++ b/queueFeeds.js @@ -1,8 +1,8 @@ -const Database = require('better-sqlite3'); -const db = new Database('bot-node.db'); -const Jackd = require('jackd'); -const beanstalkd = new Jackd(); +import Database from 'better-sqlite3'; +import Jackd from 'jackd'; +const beanstalkd = new Jackd(); +const db = new Database('bot-node.db'); async function foo() { @@ -15,7 +15,7 @@ async function foo() { await beanstalkd.connect() - for (feed of feeds) { + for (let feed of feeds) { await beanstalkd.put(feed.feed) } @@ -23,4 +23,4 @@ async function foo() { } -foo() +foo(); diff --git a/routes/api.js b/routes/api.js index 86e7683..dd1a463 100644 --- a/routes/api.js +++ b/routes/api.js @@ -1,15 +1,16 @@ -'use strict'; -const express = require('express'), - router = express.Router(), - cors = require('cors'), - crypto = require('crypto'), - request = require('request'), - Parser = require('rss-parser'), - parseFavicon = require('parse-favicon').parseFavicon, - generateRSAKeypair = require('generate-rsa-keypair'), - oauth = require('../config.json').OAUTH; +import express from 'express'; +import cors from 'cors'; +import crypto from 'crypto'; +import Parser from 'rss-parser'; +import { parseFavicon } from 'parse-favicon'; +import generateRSAKeypair from 'generate-rsa-keypair'; +import config from '../config.js'; -router.get('/request-token', cors(), (req, res) => { +const router = express.Router(); +const oauth = config.OAUTH; +export default router; + +router.get('/request-token', cors(), async (req, res) => { if (!oauth) { return res.status(501).json({message: `OAuth is not enabled on this server.`}); } @@ -25,36 +26,37 @@ router.get('/request-token', cors(), (req, res) => { params.client_secret = oauth.client_secret; params.redirect_uri = oauth.redirect_uri; params.grant_type = 'authorization_code'; - request.post(`https://${oauth.domain}${oauth.token_path}`, {form: params}, (err,httpResponse,body) => { - body = JSON.parse(body); - if (body.access_token) { - return res.json({ access_token: body.access_token, domain: oauth.domain}); - } - else { - return res.status(401).json(body); - } + let response = await fetch(`https://${oauth.domain}${oauth.token_path}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded' + }, + body: new URLSearchParams(params) }); + let body = await response.json(); + if (body.access_token) { + return res.json({ access_token: body.access_token, domain: oauth.domain}); + } else { + return res.status(401).json(body); + } }); // if oauth is enabled, this function checks to see if we've been sent an access token and validates it with the server // otherwise we simply skip verification function isAuthenticated(req, res, next) { if (oauth) { - request.get({ - url: `https://${oauth.domain}${oauth.token_verification_path}`, + fetch(`https://${oauth.domain}${oauth.token_verification_path}`, { headers: { 'Authorization': `Bearer ${req.query.token}` - }, - }, (err, resp, body) => { - if (resp.statusCode === 200) { - return next(); } - else { + }).then(response => { + if (response.status === 200) { + return next(); + } else { res.redirect('/'); } }); - } - else { + } else { return next(); } } @@ -85,6 +87,7 @@ router.get('/convert', isAuthenticated, function (req, res) { res.status(400).json({err: err.message}); } else { + console.log(feedData); res.status(200).json(feedData); let displayName = feedData.title; let description = feedData.description; @@ -98,9 +101,9 @@ router.get('/convert', isAuthenticated, function (req, res) { let actorRecord = createActor(account, domain, pair.public, displayName, imageUrl, description); let webfingerRecord = createWebfinger(account, domain); const apikey = crypto.randomBytes(16).toString('hex'); - db.prepare('insert or replace into accounts(name, actor, apikey, pubkey, privkey, webfinger) values(?, ?, ?, ?, ?, ?)').run( `${account}@${domain}`, JSON.stringify(actorRecord), apikey, pair.public, pair.private, JSON.stringify(webfingerRecord)); + let result = db.prepare('insert or replace into accounts(name, actor, apikey, pubkey, privkey, webfinger) values(?, ?, ?, ?, ?, ?)').run( `${account}@${domain}`, JSON.stringify(actorRecord), apikey, pair.public, pair.private, JSON.stringify(webfingerRecord)); let content = JSON.stringify(feedData); - db.prepare('insert or replace into feeds(feed, username, content) values(?, ?, ?)').run( feed, username, content); + result = db.prepare('insert or replace into feeds(feed, username, content) values(?, ?, ?)').run( feed, username, content); }); } }); @@ -120,15 +123,14 @@ function getImage(feed, feedData, cb) { // otherwise parse the HTML for the favicon else { let favUrl = new URL(feed); - request(favUrl.origin, (err, resp, body) => { - parseFavicon(body, {baseURI: favUrl.origin}).then(result => { - if (result && result.length) { - return cb(result[0].url); - } - else { - return cb(null); - } - }); + fetch(favUrl.origin).then(response => response.body).then(body => { + const result = parseFavicon(body, {baseURI: favUrl.origin}); + if (result && result.length) { + return cb(result[0].url); + } + else { + return cb(null); + } }); } } @@ -177,6 +179,4 @@ function createWebfinger(name, domain) { } ] }; -} - -module.exports = router; +} \ No newline at end of file diff --git a/routes/inbox.js b/routes/inbox.js index 7deafac..2f4e76e 100644 --- a/routes/inbox.js +++ b/routes/inbox.js @@ -1,9 +1,9 @@ -'use strict'; -const express = require('express'), - crypto = require('crypto'), - request = require('request'), - fs = require('fs'), - router = express.Router(); +import express from 'express'; +import crypto from 'crypto'; +import fs from 'fs'; + +const router = express.Router(); +export default router; function signAndSend(message, name, domain, req, res, targetDomain) { // get the URI of the actor object and append 'inbox' to it @@ -29,8 +29,7 @@ function signAndSend(message, name, domain, req, res, targetDomain) { const signature_b64 = signature.toString('base64'); const algorithm = 'rsa-sha256'; let header = `keyId="https://${domain}/u/${name}",algorithm="${algorithm}",headers="(request-target) host date digest",signature="${signature_b64}"`; - request({ - url: inbox, + fetch(inbox, { headers: { 'Host': targetDomain, 'Date': d.toUTCString(), @@ -40,9 +39,7 @@ function signAndSend(message, name, domain, req, res, targetDomain) { 'Accept': 'application/activity+json' }, method: 'POST', - json: true, body: message - }, function (error, response, body){ }); res.json('done'); } @@ -102,5 +99,3 @@ router.post('/', function (req, res) { } } }); - -module.exports = router; diff --git a/routes/index.js b/routes/index.js index c90af8b..38cfe9c 100644 --- a/routes/index.js +++ b/routes/index.js @@ -1,9 +1,13 @@ -'use strict'; +import api from './api.js'; +import inbox from './inbox.js'; +import message from './message.js'; +import user from './user.js'; +import webfinger from './webfinger.js'; -module.exports = { - api: require('./api'), - inbox: require('./inbox'), - message: require('./message'), - user: require('./user'), - webfinger: require('./webfinger'), +export default { + api, + inbox, + message, + user, + webfinger, }; diff --git a/routes/message.js b/routes/message.js index a62a9f1..4c5de3c 100644 --- a/routes/message.js +++ b/routes/message.js @@ -1,8 +1,7 @@ -'use strict'; -const express = require('express'), - router = express.Router(), - Parser = require('rss-parser'), - parser = new Parser(); +import express from 'express'; + +const router = express.Router(); +export default router; router.get('/:guid', function (req, res) { let guid = req.params.guid; @@ -40,5 +39,3 @@ router.get('/:guid', function (req, res) { } } }); - -module.exports = router; diff --git a/routes/user.js b/routes/user.js index 18dba79..363e84b 100644 --- a/routes/user.js +++ b/routes/user.js @@ -1,6 +1,7 @@ -'use strict'; -const express = require('express'), - router = express.Router(); +import express from 'express'; + +const router = express.Router(); +export default router; router.get('/:name', function (req, res) { let name = req.params.name; @@ -78,5 +79,3 @@ router.get('/:name/followers', function (req, res) { //res.json(JSON.parse(result.actor)); } }); - -module.exports = router; diff --git a/routes/webfinger.js b/routes/webfinger.js index c743d84..f5b1677 100644 --- a/routes/webfinger.js +++ b/routes/webfinger.js @@ -1,6 +1,7 @@ -'use strict'; -const express = require('express'), - router = express.Router(); +import express from 'express'; + +const router = express.Router(); +export default router; router.get('/', function (req, res) { let resource = req.query.resource; @@ -19,5 +20,3 @@ router.get('/', function (req, res) { } } }); - -module.exports = router; diff --git a/updateFeeds.js b/updateFeeds.js index f823b0b..abca6f5 100644 --- a/updateFeeds.js +++ b/updateFeeds.js @@ -1,13 +1,13 @@ -const config = require('./config.json'); -const { DOMAIN, PRIVKEY_PATH, CERT_PATH, PORT_HTTP, PORT_HTTPS } = config; -const Database = require('better-sqlite3'); -const db = new Database('bot-node.db'), - Parser = require('rss-parser'), - request = require('request'), - crypto = require('crypto'), - parser = new Parser({timeout: 2000}); +import Database from 'better-sqlite3'; +import Parser from 'rss-parser'; +import crypto from 'crypto'; +import Jackd from 'jackd'; +import config from './config.js'; -const Jackd = require('jackd'); +const { DOMAIN, PRIVKEY_PATH, CERT_PATH, PORT_HTTP, PORT_HTTPS } = config; + +const db = new Database('bot-node.db'); +const parser = new Parser({timeout: 2000}); const beanstalkd = new Jackd(); beanstalkd.connect() @@ -15,10 +15,10 @@ beanstalkd.connect() async function processQueue() { while (true) { try { - const { id, payload } = await beanstalkd.reserve() + const job = await beanstalkd.reserve() /* ... process job here ... */ - await beanstalkd.delete(id) - await doFeed(payload) + await beanstalkd.delete(job.id); + await doFeed(job.payload.toString()); } catch (err) { // Log error somehow console.error(err) @@ -26,7 +26,7 @@ async function processQueue() { } } -processQueue() +processQueue(); function doFeed(feedUrl) { return new Promise((resolve, reject) => { @@ -175,8 +175,7 @@ function signAndSend(message, name, domain, req, res, targetDomain, inbox) { const signature_b64 = signature.toString('base64'); const algorithm = 'rsa-sha256'; let header = `keyId="https://${domain}/u/${name}",algorithm="${algorithm}",headers="(request-target) host date digest",signature="${signature_b64}"`; - request({ - url: inbox, + fetch(inbox, { headers: { 'Host': targetDomain, 'Date': d.toUTCString(), @@ -186,9 +185,7 @@ function signAndSend(message, name, domain, req, res, targetDomain, inbox) { 'Accept': 'application/activity+json' }, method: 'POST', - json: true, body: message - }, function (error, response, body){ }); } }