Skip to content

Node.js based forum software built for the modern web

20.2 KB
Raw Blame
'use strict';

const nconf = require('nconf');
const winston = require('winston');
const { createHash } = require('crypto');

const request = require('../request');
const db = require('../database');
const SendPool = require('./send');
const pubsub = require('../pubsub');
const meta = require('../meta');
const categories = require('../categories');
const posts = require('../posts');
const messaging = require('../messaging');
const user = require('../user');
const utils = require('../utils');
const ttl = require('../cache/ttl');
const lru = require('../cache/lru');
const batch = require('../batch');
const requestCache = ttl({
	name: 'ap-request-cache',
	max: 5000,
	ttl: 1000 * 60 * 5, // 5 minutes
});
const serveCache = ttl({
	name: 'ap-serve-cache',
	max: 5000,
	ttl: 1000 * 60, // 1 minute
});
const probeCache = ttl({
	name: 'ap-probe-cache',
	max: 500,
	ttl: 1000 * 60 * 60, // 1 hour
});
const probeRateLimit = ttl({
	name: 'ap-probe-rate-limit-cache',
	ttl: 1000 * 3, // 3 seconds
});
const publicKeyFetchRateLimit = ttl({
	name: 'ap-public-key-fetch-rate-limit-cache',
	ttl: 60 * 1000, // 60 seconds
	max: 1000,
});
const publicKeyCache = lru({
	name: 'ap-public-key-cache',
	ttl: 1000 * 60 * 60 * 6, // 6 hours
	max: 1000,
});

const ActivityPub = module.exports;

ActivityPub._constants = Object.freeze({
	uid: -2,
	publicAddress: 'https://www.w3.org/ns/activitystreams#Public',
	acceptablePublicAddresses: ['https://www.w3.org/ns/activitystreams#Public', 'as:Public', 'Public'],
	acceptableTypes: [
		'application/activity+json',
		'application/ld+json;profile="https://www.w3.org/ns/activitystreams"',
	],
	acceptedPostTypes: [
		'Note', 'Page', 'Article', 'Question', 'Video',
	],
	acceptableActorTypes: new Set(['Application', 'Organization', 'Person', 'Service']),
	acceptableGroupTypes: new Set(['Group']),
	requiredActorProps: ['inbox'],
	acceptedProtocols: ['https', ...(process.env.CI === 'true' ? ['http'] : [])],
	acceptable: {
		customFields: new Set(['PropertyValue', 'Link', 'Note']),
		contextTypes: new Set(['Collection', 'CollectionPage', 'OrderedCollection', 'OrderedCollectionPage']),
	},
});
ActivityPub._cache = requestCache;
ActivityPub.serveCache = serveCache;
ActivityPub._sent = new Map(); // used only in local tests

// Invalidate serve cache on post lifecycle events
pubsub.on('post:edit', pid => ActivityPub.serveCache.del(`/post/${pid}`));
pubsub.on('post:delete', pid => ActivityPub.serveCache.del(`/post/${pid}`));
pubsub.on('post:purge', pid => ActivityPub.serveCache.del(`/post/${pid}`));


ActivityPub.helpers = require('./helpers');
ActivityPub.inbox = require('./inbox');
ActivityPub.mocks = require('./mocks');
ActivityPub.notes = require('./notes');
ActivityPub.contexts = require('./contexts');
ActivityPub.actors = require('./actors');
ActivityPub.instances = require('./instances');
ActivityPub.blocklists = require('./blocklists');
ActivityPub.feps = require('./feps');
ActivityPub.rules = require('./rules');
ActivityPub.relays = require('./relays');
ActivityPub.hashtags = require('./hashtags');
ActivityPub.out = require('./out');
ActivityPub.jobs = require('./jobs');
ActivityPub.analytics = require('./analytics');
ActivityPub.signatures = require('./signatures');
ActivityPub.emoji = require('./emoji');

ActivityPub.resolveId = async (uid, id) => {
	try {
		const query = new URL(id);
		({ id } = await ActivityPub.get('uid', uid, id));
		const response = new URL(id);

		if (query.host !== response.host) {
			winston.warn(`[activitypub/resolveId] id resolution domain mismatch: ${query.href} != ${response.href}`);
			return null;
		}

		return id;
	} catch (e) {
		return null;
	}
};

ActivityPub.resolveInboxes = async (ids) => {
	const inboxes = new Set();

	if (!meta.config.activitypubAllowLoopback) {
		ids = ids.filter((id) => {
			try {
				const { hostname } = new URL(id);
				return hostname !== nconf.get('url_parsed').hostname;
			} catch (err) {
				winston.error(`[activitypub/resolveInboxes] Invalid id: ${id}`);
				return false;
			}
		});
	}

	await ActivityPub.actors.assert(ids);

	// Remove non-asserted targets
	const exists = await db.isSortedSetMembers('usersRemote:lastCrawled', ids);
	ids = ids.filter((_, idx) => exists[idx]);

	await batch.processArray(ids, async (currentIds) => {
		const isCategory = await db.exists(currentIds.map(id => `categoryRemote:${id}`));
		const [cids, uids] = currentIds.reduce(([cids, uids], id, idx) => {
			const array = isCategory[idx] ? cids : uids;
			array.push(id);
			return [cids, uids];
		}, [[], []]);
		const categoryData = await categories.getCategoriesFields(cids, ['inbox', 'sharedInbox']);
		const userData = await user.getUsersFields(uids, ['inbox', 'sharedInbox']);
		currentIds.forEach((id) => {
			if (cids.includes(id)) {
				const data = categoryData[cids.indexOf(id)];
				inboxes.add(data.sharedInbox || data.inbox);
			} else if (uids.includes(id)) {
				const data = userData[uids.indexOf(id)];
				inboxes.add(data.sharedInbox || data.inbox);
			}
		});
	}, {
		batch: 500,
	});

	let inboxArr = Array.from(inboxes);

	// Filter out blocked instances
	const blocked = [];
	const allowed = await Promise.all(inboxArr.map(async (inbox) => {
		let allowed = false;
		try {
			const { hostname } = new URL(inbox);
			({ allowed } = await ActivityPub.instances.isAllowed(hostname));
			if (!allowed) {
				blocked.push(inbox);
			}
		} catch (e) {
			winston.warn(`[activitypub/resolveInboxes] Malformed URL encountered while filtering out blocked instances: ${inbox}`);
		}

		return allowed;
	}));
	inboxArr = inboxArr.filter((_, idx) => allowed[idx]);
	if (blocked.length) {
		ActivityPub.helpers.log(`[activitypub/resolveInboxes] Not delivering to blocked instances: ${blocked.join(', ')}`);
	}

	return inboxArr;
};

ActivityPub.getPublicKey = async (type, id) => {
	let publicKey;

	try {
		({ publicKey } = await db.getObject(`${type}:${id}:keys`));
	} catch (e) {
		({ publicKey } = await ActivityPub.helpers.generateKeys(type, id));
	}

	return publicKey;
};

ActivityPub.getPrivateKey = async (type, id) => {
	// Sanity checking
	if (!['cid', 'uid'].includes(type) || !utils.isNumber(id) || parseInt(id, 10) < 0) {
		throw new Error('[[error:invalid-data]]');
	}
	id = parseInt(id, 10);
	let privateKey;

	try {
		({ privateKey } = await db.getObject(`${type}:${id}:keys`));
	} catch (e) {
		({ privateKey } = await ActivityPub.helpers.generateKeys(type, id));
	}

	let keyId;
	if (type === 'uid') {
		keyId = `${nconf.get('url')}${id > 0 ? `/uid/${id}` : '/actor'}#key`;
	} else {
		keyId = `${nconf.get('url')}${id > 0 ? `/category/${id}` : '/actor'}#key`;
	}

	return { key: privateKey, keyId };
};

ActivityPub.fetchPublicKey = async (uri, ip) => {
	const cached = publicKeyCache.get(uri);
	if (cached !== undefined) {
		if (cached === null) {
			// Failed request was cached
			throw new Error('[[error:activitypub.pubKey-not-found]]');
		}
		return cached;
	}

	// Validate URI format
	const isValidHttpUri = typeof uri === 'string' &&
		(
			uri.startsWith('https://') ||
			(uri.startsWith('http://') && process.env.CI === 'true')
		);
	if (!isValidHttpUri) {
		throw new Error('[[error:activitypub.invalid-id]]');
	}

	try {
		// Validate URI is well-formed
		new URL(uri);
	} catch (err) {
		throw new Error('[[error:activitypub.invalid-id]]');
	}

	// Check rate limit for this IP (0 = disabled)
	if (meta.config.activitypubPublicKeyFetchRateLimit > 0) {
		const lockId = `pubkey:${ip}`;
		const currentCount = publicKeyFetchRateLimit.get(lockId) || 0;
		if (currentCount >= meta.config.activitypubPublicKeyFetchRateLimit) {
			winston.warn(`[activitypub/fetchPublicKey] Rate limit exceeded for IP ${ip}`);
			throw new Error('[[error:activitypub.rate-limited]]');
		}
		publicKeyFetchRateLimit.set(lockId, currentCount + 1, 60 * 1000);
	}

	try {
		// Use request.get with built-in SSRF protections
		// Set reasonable timeout and response size limit
		const { body } = await request.get(uri, {
			timeout: 5000, // 5 seconds
			sizeLimit: 1024 * 1024, // 1MB — public keys are typically <10KB
			headers: {
				'accept': ActivityPub._constants.acceptableTypes.at(1),
			},
			redirect: 'manual',
		});

		// Process response and cache
		if (body.hasOwnProperty('publicKeyPem')) {
			// Validate public key
			if (!utils.isPEM(body.publicKeyPem)) {
				throw new Error('[[error:invalid-data]]');
			}
			// CryptographicKey returned (correct)
			publicKeyCache.set(uri, body.publicKeyPem);
			return body.publicKeyPem;
		} else if (body?.publicKey?.publicKeyPem) {
			// Validate public key
			if (!utils.isPEM(body.publicKey.publicKeyPem)) {
				throw new Error('[[error:invalid-data]]');
			}
			// Actor object returned (less correct)
			publicKeyCache.set(uri, body.publicKey.publicKeyPem);
			return body.publicKey.publicKeyPem;
		}

		// Response didn't contain expected public key
		publicKeyCache.set(uri, null);
		throw new Error('[[error:activitypub.pubKey-not-found]]');
	} catch (err) {
		// Cache the failed request
		publicKeyCache.set(uri, null);

		// Re-throw with context if needed
		if (err.message.includes('reserved-ip-address')) {
			throw new Error('[[error:activitypub.invalid-id]]');
		}
		throw err;
	}
};

ActivityPub.sign = async ({ key, keyId }, url, digest) => {
	// Determines HTTP method based on digest presence
	const method = digest ? 'POST' : 'GET';
	return await ActivityPub.signatures.sign({ key, keyId }, url, method, digest);
};

ActivityPub.verify = async (req) => {
	ActivityPub.helpers.log('[activitypub/verify] Starting signature verification...');

	const verified = await ActivityPub.signatures.verify(req, ActivityPub.fetchPublicKey);
	if (!verified) {
		ActivityPub.helpers.log('[activitypub/verify] Signature verification failed.');
	} else {
		ActivityPub.helpers.log('[activitypub/verify] Signature verification succeeded.');
	}

	return verified;
};

async function _checkFederationPolicy(hostname, url) {
	const { allowed } = await ActivityPub.instances.isAllowed(hostname);
	if (!allowed) {
		ActivityPub.helpers.log(`[activitypub/get] Not retrieving ${url}, domain is blocked.`);
		const e = new Error(`[[error:activitypub.get-failed]]`);
		e.code = `ap_get_domain_blocked`;
		throw e;
	}
};

ActivityPub.get = async (type, id, uri, options) => {
	if (!meta.config.activitypubEnabled) {
		throw new Error('[[error:activitypub.not-enabled]]');
	}

	const { hostname } = new URL(uri);
	await _checkFederationPolicy(hostname, uri);

	options = {
		cache: true,
		...options,
	};
	const cacheKey = [id, uri].join(';');
	const cached = requestCache.get(cacheKey);
	if (options.cache && cached !== undefined) {
		return cached;
	}

	const keyData = await ActivityPub.getPrivateKey(type, id);
	const headers = id >= 0 ? await ActivityPub.sign(keyData, uri) : {};
	ActivityPub.helpers.log(`[activitypub/get] ${uri}`);
	try {
		const { response, body, url: resolvedUrl } = await request.get(uri, {
			headers: {
				...headers,
				...options.headers,
				Accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
			},
			timeout: 5000,
		});

		const { hostname: resolvedHostname } = new URL(resolvedUrl);
		await _checkFederationPolicy(resolvedHostname, resolvedUrl);

		if (!String(response.statusCode).startsWith('2')) {
			ActivityPub.helpers.log(`[activitypub/get] Received ${response.statusCode} when querying ${uri}`);
			if (body.hasOwnProperty('error')) {
				ActivityPub.helpers.log(`[activitypub/get] Error received: ${body.error}`);
			}

			const e = new Error(`[[error:activitypub.get-failed]]`);
			e.code = `ap_get_${response.statusCode}`;
			throw e;
		}

		if (!body || typeof body !== 'object' || Array.isArray(body)) {
			ActivityPub.helpers.log(`[activitypub/get] Received non-object response from ${uri}`);
			const e = new Error(`[[error:activitypub.get-failed]]`);
			e.code = 'ap_get_invalid_response';
			throw e;
		}

		requestCache.set(cacheKey, body);
		return body;
	} catch (err) {
		if (String(err.code).startsWith('ap_get_')) {
			throw err;
		}

		// Handle things like non-json body, etc.
		throw new Error(`[[error:activitypub.get-failed]]`, { cause: err });
	}
};

ActivityPub.send = async (type, id, targets, payload, { delayMs = 0 } = {}) => {
	if (!meta.config.activitypubEnabled) {
		return ActivityPub.helpers.log('[activitypub/send] Federation not enabled; not sending.');
	}

	ActivityPub.helpers.log(`[activitypub/send] ${payload.id}`);

	if (!Array.isArray(targets)) {
		targets = [targets];
	}

	if (process.env.hasOwnProperty('CI')) {
		ActivityPub._sent.set(payload.id, { payload, targets });
		return;
	}

	const inboxes = await ActivityPub.resolveInboxes(targets);

	const actor = ActivityPub.helpers.resolveActor(type, id);

	payload = {
		'@context': 'https://www.w3.org/ns/activitystreams',
		actor,
		...payload,
	};
	const payloadHash = createHash('sha256');
	payloadHash.update(JSON.stringify(payload));
	const digest = `SHA-256=${payloadHash.digest('base64')}`;

	// Push all inboxes to the Redis retry queue and dispatch to workers
	setImmediate(() => {
		const retryQueueAdd = [];
		const retryQueuedSet = [];
		const nextTryOn = Date.now() + (delayMs || 0);

		inboxes.forEach((uri) => {
			const queueId = createHash('sha256').update(`${type}:${id}:${uri}`).digest('hex');
			retryQueueAdd.push(['ap:retry:queue', nextTryOn, queueId]);
			retryQueuedSet.push([`ap:retry:queue:${queueId}`, {
				queueId,
				uri,
				id,
				type,
				attempts: 1,
				timestamp: nextTryOn,
				digest,
				payload: JSON.stringify(payload),
			}]);
		});

		if (retryQueueAdd.length) {
			db.sortedSetAddBulk(retryQueueAdd);
			db.setObjectBulk(retryQueuedSet);

			// Start the drain loop if not already draining
			if (SendPool.pool > 0) {
				SendPool.drainLoop();
			}
		}
	});
};

ActivityPub.buildRecipients = async function (object, options) {
	/**
	 * - Builds a list of targets for activitypub.send to consume
	 * - Extends to and cc since the activity can be addressed more widely
	 * - Optional parameters:
	 *     - `cid`: includes followers of the passed-in cid (local only, can also be an array)
	 *     - `uid`: includes followers of the passed-in uid (local only)
	 *     - `pid`: includes post announcers and all topic participants
	 *     - `targets`: boolean; whether to calculate targets (default: true)
	 */

	let { to, cc } = object;
	to = new Set(to);
	cc = new Set(cc);

	let { pid, uid, cid } = options;
	options.targets = options.targets ?? true;

	let followers = [];
	if (uid) {
		({ uids: followers } = await ActivityPub.actors.getFollowers(uid));
		const followersUrl = `${nconf.get('url')}/uid/${uid}/followers`;
		if (!to.has(followersUrl)) {
			cc.add(followersUrl);
		}
	}

	if (cid) {
		cid = Array.isArray(cid) ? cid : [cid];
		await Promise.all(cid.map(async (cid) => {
			const cidFollowers = await ActivityPub.notes.getCategoryFollowers(cid);
			followers = followers.concat(cidFollowers);
			const followersUrl = `${nconf.get('url')}/category/${cid}/followers`;
			if (!to.has(followersUrl)) {
				cc.add(followersUrl);
			}
		}));
	}

	let targets = new Set();
	if (options.targets) {
		targets = new Set([...followers, ...to, ...cc]);

		// Remove local uris, public addresses, and any ids that aren't asserted actors
		targets.forEach((address) => {
			if (utils.isNumber(address) || address.startsWith(nconf.get('url'))) {
				targets.delete(address);
			}
		});
		ActivityPub._constants.acceptablePublicAddresses.forEach((address) => {
			targets.delete(address);
		});
		if (targets.size) {
			const exists = await db.isSortedSetMembers('usersRemote:lastCrawled', [...targets]);
			Array.from(targets).forEach((uri, idx) => {
				if (!exists[idx]) {
					targets.delete(uri);
				}
			});
		}
	}

	// Topic posters, post announcers and their followers
	if (pid) {
		const { tid, mainPid } = await posts.getPostFields(pid, ['tid', 'mainPid']);
		const participants = (await db.getSortedSetMembers(`tid:${tid}:posters`))
			.filter(uid => !utils.isNumber(uid)); // remote users only
		const announcers = (await ActivityPub.notes.announce.list({ pid })).map(({ actor }) => actor);
		const mainAnnouncers = (await ActivityPub.notes.announce.list({ pid: mainPid })).map(({ actor }) => actor);
		const auxiliaries = Array.from(new Set([...participants, ...announcers, ...mainAnnouncers]));

		const auxiliaryFollowers = (await user.getUsersFields(auxiliaries, ['followersUrl']))
			.filter(o => o.hasOwnProperty('followersUrl'))
			.map(({ followersUrl }) => followersUrl);
		[...auxiliaries].forEach(uri => uri && targets.add(uri));
		[...auxiliaries, ...auxiliaryFollowers].forEach(uri => uri && cc.add(uri));
	}

	return {
		to: [...to],
		cc: [...cc],
		targets,
	};
};

ActivityPub.checkHeader = async (url, timeout) => {
	timeout = timeout || meta.config.activitypubProbeTimeout || 2000;

	try {
		const { hostname } = new URL(url);
		const { response } = await request.head(url, {
			timeout,
		});
		const { headers } = response;

		// headers.link =
		if (headers && headers.link) {
			// Multiple link headers could be combined
			const links = headers.link.split(',');
			let apLink = false;

			links.forEach((link) => {
				let parts = link.split(';');
				const url = parts.shift().match(/<(.+)>/)[1];
				if (!url || apLink) {
					return;
				}

				parts = parts
					.map(p => p.trim())
					.reduce((memo, cur) => {
						cur = cur.split('=');
						if (cur.length < 2) {
							cur.push('');
						}
						memo[cur[0]] = cur[1].slice(1, -1);
						return memo;
					}, {});

				if (parts.rel === 'alternate' && parts.type === 'application/activity+json') {
					apLink = url;
				}
			});

			if (apLink) {
				const { hostname: compare } = new URL(apLink);
				if (hostname !== compare) {
					apLink = false;
				}
			}

			return apLink;
		}

		return false;
	} catch (e) {
		ActivityPub.helpers.log(`[activitypub/checkHeader] Failed on ${url}: ${e.message}`);
		return false;
	}
};

ActivityPub.probe = async ({ uid, url }) => {
	/**
	 * Checks whether a passed-in id or URL is an ActivityPub object and can be mapped to a local representation
	 *   - `uid` is optional (links to private messages won't match without uid)
	 *   - Returns a relative path if already available, true if not, and false otherwise.
	 */

	// Disable on config setting; restrict lookups to HTTPS-enabled URLs only
	const { activitypubProbe } = meta.config;
	const { protocol, host } = new URL(url);
	if (!activitypubProbe || protocol !== 'https:' || host === nconf.get('url_parsed').host) {
		return false;
	}

	// Known resources
	const [isNote, isMessage, isActor, isActorUrl] = await Promise.all([
		posts.exists(url),
		messaging.messageExists(url),
		db.isSortedSetMember('usersRemote:lastCrawled', url), // if url is same as id
		db.isObjectField('remoteUrl:uid', url),
	]);
	switch (true) {
		case isNote: {
			return `/post/${encodeURIComponent(url)}`;
		}

		case isMessage: {
			if (uid) {
				const { roomId } = await messaging.getMessageFields(url, ['roomId']);
				const canView = await messaging.canViewMessage(url, roomId, uid);
				if (canView) {
					return `/message/${encodeURIComponent(url)}`;
				}
			}
			break;
		}

		case isActor: {
			const slug = await user.getUserField(url, 'userslug');
			return `/user/${slug}`;
		}

		case isActorUrl: {
			const uid = await db.getObjectField('remoteUrl:uid', url);
			const slug = await user.getUserField(uid, 'userslug');
			return `/user/${slug}`;
		}
	}

	// Guests not allowed to use expensive logic path
	if (!uid) {
		return false;
	}

	// One request allowed every 3 seconds (configured at top)
	const limited = probeRateLimit.get(uid);
	if (limited) {
		return false;
	}

	// Cached result
	if (probeCache.has(url)) {
		return probeCache.get(url);
	}

	// Opportunistic HEAD
	try {
		probeRateLimit.set(uid, true);
		const probe = await ActivityPub.checkHeader(url).then((result) => {
			probeCache.set(url, result);
			return !!result;
		});

		return !!probe;
	} catch (e) {
		if (e.name === 'TimeoutError') {
			// Return early but retry for caching purposes
			ActivityPub.checkHeader(url, 1000 * 60).then((result) => {
				probeCache.set(url, result);
			}).catch(err => ActivityPub.helpers.log(err.stack));
			return false;
		}
	}

	probeCache.set(url, false);
	return false;
};

// ---------------------------------------------------------------------------
// Shutdown — called during graceful NodeBB shutdown
// ---------------------------------------------------------------------------

ActivityPub.shutdown = function () {
	if (SendPool.pool > 0) {
		SendPool.shutdown();
	}
};

// Initialize the send pool now that ActivityPub is fully defined
SendPool.init(ActivityPub);
ActivityPub.SendPool = SendPool;