Skip to content

Node.js based forum software built for the modern web

31.7 KB
Raw Blame
'use strict';

const winston = require('winston');
const nconf = require('nconf');
const tokenizer = require('sbd');
const pretty = require('pretty');

const db = require('../database');
const batch = require('../batch');
const meta = require('../meta');
const categories = require('../categories');
const messaging = require('../messaging');
const notifications = require('../notifications');
const user = require('../user');
const topics = require('../topics');
const posts = require('../posts');
const ttlCache = require('../cache/ttl');
const websockets = require('../socket.io');
const utils = require('../utils');
const translator = require('../translator');

const activitypub = module.parent.exports;
const Notes = module.exports;

const backfillCache = ttlCache({
	name: 'ap-backfill-cache',
	max: 500,
	ttl: 1000 * 60 * 2, // 2 minutes
});

Notes._normalizeTags = async (tag, cid) => {
	const systemTags = (meta.config.systemTags || '').split(',');
	const maxTags = await categories.getCategoryField(cid, 'maxTags');
	let tags = tag || [];

	if (!Array.isArray(tags)) { // the "|| []" should handle null/undefined values... #famouslastwords
		tags = [tags];
	}

	tags = tags
		.filter(({ type }) => type === 'Hashtag')
		.map((tag) => {
			tag.name = tag.name.startsWith('#') ? tag.name.slice(1) : tag.name;
			return tag;
		})
		.filter(({ name }) => !systemTags.includes(name))
		.map(t => t.name);

	if (tags.length > maxTags) {
		tags.length = maxTags;
	}

	return tags;
};

Notes.assert = async (uid, input, options = { skipChecks: false, queue: false }) => {
	/**
	 * Given the id or object of any as:Note, either retrieves the full context (if resolvable),
	 * or traverses up the reply chain to build a context.
	 */

	if (!input) {
		return null;
	}

	let id = !activitypub.helpers.isUri(input) ? input.id : input;

	let lockStatus = await db.incrObjectField('locks', id);
	lockStatus = lockStatus <= 1;
	if (!lockStatus) { // unable to achieve lock, stop processing.
		winston.warn(`[activitypub/notes.assert] Unable to acquire lock, skipping processing of ${id}`);
		return null;
	}

	try {
		if (!(options.skipChecks || process.env.hasOwnProperty('CI'))) {
			id = (await activitypub.checkHeader(id)) || id;
		}

		let chain;
		let context = await activitypub.contexts.get(uid, id);
		if (context.tid) {
			const { tid } = context;
			return { tid, count: 0 };
		} else if (context.context) {
			const { type } = await activitypub.helpers.resolveLocalId(context.context);
			// Remote contexts only, if context is local, prefer parent chain traversal instead (to catch out-of-band replies)
			if (type !== 'topic') {
				chain = Array.from(await activitypub.contexts.getItems(uid, context.context, { input }));
				if (chain && chain.length) {
					// Deduplicate by id (just in case, also a buggy NodeBB impl. sent dupes)
					const ids = new Set();
					chain = chain.filter((item) => {
						const seen = ids.has(item.pid);
						ids.add(item.pid);
						return !seen;
					});

					// Context resolves, use in later topic creation
					context = context.context;
				}
			}
		} else {
			context = undefined;
		}

		if (!chain || !chain.length) {
			// Fall back to inReplyTo traversal on context retrieval failure
			chain = Array.from(await Notes.getParentChain(uid, input));
			chain.reverse();
		}

		// Can't resolve — give up.
		if (!chain.length) {
			return null;
		}

		// Reorder chain items by timestamp
		chain = chain.sort((a, b) => a.timestamp - b.timestamp);

		const mainPost = chain[0];
		let {
			pid: mainPid,
			tid,
			uid: authorId,
			timestamp,
			title,
			content,
			sourceContent,
			contentWarning,
			_activitypub,
		} = mainPost;
		const hasTid = !!tid;

		const authorBanned = await user.bans.isBanned(authorId);
		if (!hasTid && authorBanned) { // New topics only
			activitypub.helpers.log('[notes/assert] OP is banned, not asserting topic.');
			return null;
		}

		const cid = hasTid ? await topics.getTopicField(tid, 'cid') : options.cid || -1;
		let crosspostCid = false;

		if (options.cid && cid === -1) {
			// Move topic if currently uncategorized
			await topics.tools.move(tid, { cid: options.cid, uid: 'system' });
		}

		const exists = await posts.exists(chain.map(p => p.pid));
		if (tid && exists.every(Boolean)) {
			// All cached, return early.
			activitypub.helpers.log('[notes/assert] No new notes to process.');
			return { tid, count: 0 };
		}

		let generatedTitle;
		if (hasTid) {
			mainPid = await topics.getTopicField(tid, 'mainPid');
		} else {
			// Check recipients/audience for category (local or remote)
			const set = activitypub.helpers.makeSet(_activitypub, ['to', 'cc', 'audience']);
			await activitypub.actors.assert(Array.from(set));

			// Local (set if found)
			const resolved = await Promise.all(Array.from(set).map(async id => await activitypub.helpers.resolveLocalId(id)));
			const recipientCids = resolved
				.filter(Boolean)
				.filter(({ type }) => type === 'category')
				.map(obj => obj.id);

			// Remote (set only if cid and object domains match or cid is set in `audience`)
			let remoteCid;
			const assertedGroups = await categories.exists(Array.from(set));
			try {
				const { hostname } = new URL(mainPid);
				remoteCid = Array.from(set).filter((id, idx) => {
					const { hostname: cidHostname } = new URL(id);
					const explicitAudience = Array.isArray(_activitypub.audience) ?
						_activitypub.audience.includes(id) :
						_activitypub.audience === id;

					return assertedGroups[idx] && (explicitAudience || cidHostname === hostname);
				}).shift();
			} catch (e) {
				// noop
				winston.error('[activitypub/notes.assert] Could not parse URL of mainPid', e.stack);
			}

			if (remoteCid || recipientCids.length) {
				// Overrides passed-in value, respect addressing from main post over booster
				options.cid = remoteCid || recipientCids.shift();
			}

			// Auto-categorization (takes place only if all other categorization efforts fail)
			const { cid, action: ruleAction } = await assignCategory(mainPost);
			if (ruleAction === 2) {
				activitypub.helpers.log(`[activitypub/notes.assert] Rule rejected post (${mainPid})`);
				return null;
			}
			options.queue = ruleAction === 1;
			crosspostCid = cid;
			if (!options.cid) {
				options.cid = crosspostCid;
				crosspostCid = false;
			}
			// action is used below to decide whether to queue or add the crosspost

			// mainPid ok to leave as-is
			if (!title) {
				let prettified = pretty(content || sourceContent);

				// Remove any lines that contain quote-post fallbacks
				prettified = prettified.split('\n').filter(line => !line.startsWith('<p class="quote-inline"')).join('\n');
				const sentences = tokenizer.sentences(prettified, { sanitize: true, newline_boundaries: true });
				title = sentences.shift();
				generatedTitle = 1;
			}

			if (contentWarning) {
				title = contentWarning;
				generatedTitle = 1;
			}

			// Extract emoji metadata for title rendering
			const titleEmoji = [];
			const rawTags = _activitypub.tag;
			const tagList = Array.isArray(rawTags) ? rawTags : (rawTags ? [rawTags] : []);
			const emojiTags = tagList.filter(tag => tag?.type === 'Emoji');
			for (const tag of emojiTags) {
				const { name, icon } = tag;
				if (!name || !icon?.url) continue;
				let code = name;
				if (!code.startsWith(':')) code = `:${code}`;
				if (!code.endsWith(':')) code = `${code}:`;
				if (title.includes(code)) {
					try {
						const { hostname } = new URL(icon.url);
						const clean = code.replace(/^:+|:+$/g, '');
						titleEmoji.push({ code, hostname, clean });
					} catch {
						// skip invalid icon URLs
					}
				}
			}
			if (titleEmoji.length) {
				mainPost.titleEmoji = JSON.stringify(titleEmoji);
			}
		}
		mainPid = utils.isNumber(mainPid) ? parseInt(mainPid, 10) : mainPid;

		// Relation & privilege check for local categories
		const inputIndex = chain.map(n => n.pid).indexOf(id);
		const hasRelation =
			uid || hasTid ||
			options.skipChecks || options.cid ||
			await assertRelation(chain[inputIndex !== -1 ? inputIndex : 0]);
		if (!hasRelation) {
			if (!hasRelation) {
				activitypub.helpers.log(`[activitypub/notes.assert] Not asserting ${id} as it has no relation to existing tracked content.`);
			}

			return null;
		}

		tid = tid || utils.generateUUID();
		mainPost.tid = tid;

		const urlMap = chain.reduce((map, post) => (post.url ? map.set(post.url, post.id) : map), new Map());
		let unprocessed = chain.map((post) => {
			post.tid = tid; // add tid to post hash

			// Ensure toPids in replies are ids
			if (urlMap.has(post.toPid)) {
				post.toPid = urlMap.get(post.toPid);
			}

			// Filter image attachments out if they are in content
			const attachment = post?._activitypub?.attachment;
			if (attachment && attachment.length) {
				post._activitypub.attachment = attachment.filter((attachment) => {
					const inContent = attachment.type === 'Image' && (post.sourceContent || post.content).includes(attachment.url);
					return !inContent;
				});
			}

			return post;
		}).filter((p, idx) => !exists[idx]);
		const count = unprocessed.length;
		activitypub.helpers.log(`[notes/assert] ${count} new note(s) found.`);

		if (!hasTid) {
			activitypub.helpers.log(`[activitypub/notes.assert] hasTid=${hasTid}, skipChecks=${options.skipChecks}, mainPid=${mainPid}`);
			const { hostname: mainHostname } = new URL(mainPid);
			const mainResult = await activitypub.instances.isAllowed(mainHostname);

			if (!mainResult.allowed) {
				activitypub.helpers.log(`[activitypub/notes.assert] Not asserting ${mainPid}, domain is blocked.`);
				return null;
			}

			if ((mainResult.severity === 3 || (utils.isNumber(options.cid) && options.queue)) && meta.config.postQueue) {
				activitypub.helpers.log(`[activitypub/notes.assert] Queuing main post (${mainPid}) due to blocklist severity 3${options.queue ? ' or explicit queue option' : ''}`);
				if (utils.isNumber(mainPid) || (await posts.exists([mainPid]))[0]) {
					activitypub.helpers.log(`[activitypub/notes.assert] Rejecting to-be-queued main post (${mainPid}): pid is local or already exists`);
					return null;
				}

				const queueData = {
					uid: authorId,
					cid: options.cid || cid,
					pid: mainPid,
					title,
					titleEmoji: mainPost.titleEmoji,
					timestamp,
					content: mainPost.content,
					sourceContent: mainPost.sourceContent,
					contentWarning,
					generatedTitle,
					_activitypub: mainPost._activitypub,
				};

				await posts.addToQueue(queueData);

				// Drop the rest of the chain — replies without OP don't make sense
				return { tid: null, queued: 1 };
			}

			const { to, cc } = mainPost._activitypub;
			const tags = await Notes._normalizeTags(mainPost._activitypub.tag || []);

			try {
				await topics.post({
					tid,
					uid: authorId,
					cid: options.cid || cid,
					pid: mainPid,
					title,
					titleEmoji: mainPost.titleEmoji,
					timestamp,
					tags,
					content: mainPost.content,
					sourceContent: mainPost.sourceContent,
					contentWarning,
					generatedTitle,
					_activitypub: mainPost._activitypub,
				});
				unprocessed.shift();
			} catch (e) {
				activitypub.helpers.log(`[activitypub/notes.assert] Could not post topic (${mainPost.pid}): ${e.message}`);
				return null;
			}

			// These must come after topic is posted
			await Promise.all([
				Notes.updateLocalRecipients(mainPid, { to, cc }),
				mainPost._activitypub.image ? topics.thumbs.associate({
					id: tid,
					path: mainPost._activitypub.image,
				}) : null,
			]);

			if (context) {
				activitypub.helpers.log(`[activitypub/notes.assert] Associating tid ${tid} with context ${context}`);
				await topics.setTopicField(tid, 'context', context);
			}
		}

		const uids = Array.from(unprocessed.reduce((uids, post) => {
			uids.add(post.uid);
			return uids;
		}, new Set()));
		const isBanned = await user.bans.isBanned(uids);
		const banned = uids.filter((_, idx) => isBanned[idx]);
		unprocessed = unprocessed.filter(post => !banned.includes(post.uid));

		let added = [];
		let queued = 0;
		await Promise.all(unprocessed.map(async (post) => {
			const { to, cc } = post._activitypub;

			try {
				const { hostname: postHostname } = new URL(post.pid);
				const postResult = await activitypub.instances.isAllowed(postHostname);

				if (!postResult.allowed) {
					activitypub.helpers.log(`[activitypub/notes.assert] Not asserting ${post.pid}, domain is blocked.`);
					return;
				}

				if (postResult.severity === 3 && meta.config.postQueue) {
					activitypub.helpers.log(`[activitypub/notes.assert] Queuing reply (${post.pid}) due to blocklist severity 3`);
					if (utils.isNumber(post.pid) || await posts.exists(post.pid)) {
						activitypub.helpers.log(`[activitypub/notes.assert] Rejecting to-be-queued reply (${post.pid}): pid or tid is local or already exists`);
						return;
					}

					await posts.addToQueue({
						uid: post.uid,
						tid,
						pid: post.pid,
						content: post.content,
						sourceContent: post.sourceContent,
						contentWarning: post.contentWarning,
						timestamp: post.timestamp,
						_activitypub: post._activitypub,
					});
					queued += 1;
					return;
				}

				const postData = await topics.reply(post);
				added.push(postData);
				await Notes.updateLocalRecipients(post.pid, { to, cc });
			} catch (e) {
				activitypub.helpers.log(`[activitypub/notes.assert] Could not add reply (${post.pid}): ${e.message}`);
			}
		}));

		if (added.length) {
			// Because replies are added in parallel, `index` is calculated incorrectly
			const indices = await posts.getPostIndices(added, uid);
			added = added.map((post, idx) => {
				post.index = indices[idx];
				return post;
			});
			websockets.in(`topic_${tid}`).emit('event:new_post', { posts: added });
		}

		await Notes.syncUserInboxes(tid, uid);

		if (crosspostCid) {
			if (options.queue) {
				await topics.crossposts.queue(tid, crosspostCid, 0);
			} else {
				await topics.crossposts.add(tid, crosspostCid, 0);
			}
		}

		if (!hasTid && uid && options.cid) {
			// New topic, have category announce it
			await activitypub.out.announce.topic(tid);
		}

		return { tid, count, queued };
	} catch (e) {
		console.log(e.stack);
		winston.warn(`[activitypub/notes.assert] Could not assert ${id} (${e.message}).`);
		return null;
	} finally {
		activitypub.helpers.log(`[activitypub/notes.assert] Releasing lock (${id})`);
		await db.deleteObjectField('locks', id);
	}
};

Notes.assertPrivate = async (object) => {
	// Given an object, adds it to an existing chat or creates a new chat otherwise
	// todo: context stuff

	if (!object || !object.id || !activitypub.helpers.isUri(object.id)) {
		return null;
	}

	const localUids = [];
	const recipients = new Set([...(object.to || []), ...(object.cc || [])]);
	await Promise.all(Array.from(recipients).map(async (value) => {
		const { type, id } = await activitypub.helpers.resolveLocalId(value);
		if (type === 'user') {
			localUids.push(id);
			recipients.delete(value);
			recipients.add(parseInt(id, 10));
		}
	}));

	// Trim recipient list down to asserted actors (and local users) only
	await activitypub.actors.assert([...recipients]);
	const exists = await user.exists([...recipients]);
	Array.from(recipients).forEach((uid, idx) => {
		if (!exists[idx]) {
			recipients.delete(uid);
		}
	});

	// Locate the `roomId` and set `toMid` based on `inReplyTo`
	let roomId;
	let toMid;
	const resolved = await activitypub.helpers.resolveLocalId(object.inReplyTo);
	if (resolved.type === 'message' && resolved.id) {
		toMid = resolved.id;
	} else {
		toMid = object.inReplyTo;
	}
	if (toMid && await messaging.messageExists(toMid)) {
		roomId = await messaging.getMessageField(toMid, 'roomId');
	} else {
		toMid = undefined;
		// roomId stays undefined
	}

	// Compare room members with object recipients; if someone in-room is omitted, start new chat
	const participants = await messaging.getUsersInRoom(roomId, 0, -1);
	const participantUids = participants.map(user => user.uid);
	if (roomId) {
		const omitted = participants.filter((user) => {
			const { uid } = user;
			return !recipients.has(uid) && uid !== object.attributedTo;
		});
		if (omitted.length) {
			toMid = undefined; // message creation logic fails if toMid is not in room
			roomId = null;
		}
	}

	let timestamp;
	try {
		timestamp = new Date(object.published).getTime() || Date.now();
	} catch (e) {
		timestamp = Date.now();
	}

	let payload;
	try {
		payload = await activitypub.mocks.message(object);
	} catch (e) {
		activitypub.helpers.log(`[activitypub/notes.assertPrivate] Failed to mock message: ${e.message}`);
		return null;
	}

	// Naive image appending (using src/posts/attachments.js is likely better, but not worth the effort)
	const attachments = payload._activitypub.attachment;
	if (attachments && Array.isArray(attachments)) {
		const images = attachments.filter((attachment) => {
			return attachment.mediaType.startsWith('image/');
		}).map(({ url, href }) => url || href);
		images.forEach((url) => {
			payload.content += `<p><img class="img-fluid img-thumbnail" src="${url}" /></p>`;
		});
	}

	try {
		await messaging.checkContent(payload.content, false);
	} catch (e) {
		const [displayname, userslug] = await Promise.all([
			user.getNotificationDisplayname(payload.uid),
			user.getUserField(payload.uid, 'userslug'),
		]);
		const notification = await notifications.create({
			bodyShort: translator.compile('error:remote-chat-received-too-long', displayname),
			path: `/user/${userslug}`,
			nid: `error:chat:uid:${payload.uid}`,
			from: payload.uid,
		});
		notifications.push(notification, Array.from(recipients).filter(uid => utils.isNumber(uid)));
		return null;
	}

	// Local recipients who cannot be messaged should be removed
	const recipientUids = Array.from(recipients).filter(uid => utils.isNumber(uid));
	const results = await Promise.all(recipientUids.map(async (uid) => {
		try {
			await messaging.canMessageUser(payload.uid, uid);
			return { uid, valid: true };
		} catch (e) {
			return { uid, valid: false };
		}
	}));
	const validUids = results.filter(r => r.valid).map(r => r.uid);
	if (validUids.length === 0) {
		return null;
	}
	results.filter(r => !r.valid).forEach(r => recipients.delete(r.uid));

	if (!roomId) {
		roomId = await messaging.newRoom(payload.uid, { uids: [...recipients] });
	}

	// Add any new members to the chat
	const added = Array.from(recipients).filter(uid => !participantUids.includes(uid));
	const assertion = await activitypub.actors.assert(added);
	if (assertion) {
		await Promise.all([
			messaging.addUsersToRoom(payload.uid, added, roomId),
			...added.map(uid => messaging.addSystemMessage('user-join', uid, roomId)),
		]);
	}

	// Add message to room
	const message = await messaging.sendMessage({
		...payload,
		timestamp: Date.now(),
		roomId: roomId,
		toMid: toMid,
	});
	messaging.notifyUsersInRoom(payload.uid, roomId, message);

	// Set real timestamp back so that the message shows even though it predates room joining
	await messaging.setMessageField(payload.mid, 'timestamp', timestamp);

	return { roomId };
};

async function assertRelation(post) {
	/**
	 * Given a mocked post object, ensures that it is related to some other object in database
	 * This check ensures that random content isn't added to the database just because it is received.
	 */

	// Is followed by at least one local user
	const { followers } = await activitypub.actors.getLocalFollowCounts(post.uid);

	// Local user is mentioned
	const { tag } = post._activitypub;
	let uids = [];
	if (tag && tag.length) {
		const slugs = tag.reduce((slugs, tag) => {
			if (tag.type === 'Mention') {
				const [slug, hostname] = tag.name.slice(1).split('@');
				if (hostname === nconf.get('url_parsed').hostname) {
					slugs.push(slug);
				}
			}
			return slugs;
		}, []);

		uids = slugs.length ? await db.sortedSetScores('userslug:uid', slugs) : [];
		uids = uids.filter(Boolean);
	}

	return followers > 0 || uids.length;
}

async function assignCategory(post) {
	const ageLimitDays = meta.config.activitypubRulesCutoffDays || 0;
	if (ageLimitDays > 0 && post.timestamp) {
		const ageDays = (Date.now() - post.timestamp) / (1000 * 60 * 60 * 24);
		if (ageDays > ageLimitDays) {
			activitypub.helpers.log(`[activitypub] Post ${post.pid} is ${ageDays.toFixed(1)} days old, skipping categorization (limit: ${ageLimitDays} days)`);
			return { cid: undefined, filter: false };
		}
	}

	activitypub.helpers.log('[activitypub] Checking auto-categorization rules.');
	const rules = await activitypub.rules.list();
	const tags = await Notes._normalizeTags(post._activitypub.tag || []);

	const matched = rules.reduce((matched, { type, value, cid: target, action: ruleAction }) => {
		if (!matched.cid) {
			switch (type) {
				case 'hashtag': {
					const lowerValue = value.toLowerCase();
					if (tags.some(tag => tag.toLowerCase() === lowerValue)) {
						activitypub.helpers.log(`[activitypub]   - Rule match: #${value}; cid: ${target}, action: ${ruleAction}`);
						return { cid: target, action: ruleAction };
					}
					break;
				}

				case 'user': {
					if (post.uid === value) {
						activitypub.helpers.log(`[activitypub]   - Rule match: user ${value}; cid: ${target}, action: ${ruleAction}`);
						return { cid: target, action: ruleAction };
					}
				}
			}
		}

		return matched;
	}, { cid: undefined, action: 0 });

	return matched;
}

Notes.updateLocalRecipients = async (id, { to, cc }) => {
	const recipients = new Set([...(to || []), ...(cc || [])]);
	const uids = new Set();
	await Promise.all(Array.from(recipients).map(async (recipient) => {
		const { type, id } = await activitypub.helpers.resolveLocalId(recipient);
		if (type === 'user' && await user.exists(id)) {
			uids.add(parseInt(id, 10));
			return;
		}

		const followedUid = await db.getObjectField('followersUrl:uid', recipient);
		if (followedUid) {
			const { uids: followers } = await activitypub.actors.getFollowers(followedUid);
			if (followers.size > 0) {
				followers.forEach((uid) => {
					uids.add(uid);
				});
			}
		}
	}));

	if (uids.size > 0) {
		await db.setAdd(`post:${id}:recipients`, Array.from(uids));
	}
};

Notes.getParentChain = async (uid, input) => {
	// Traverse upwards via `inReplyTo` until you find the root-level Note
	const id = activitypub.helpers.isUri(input) ? input : input.id;

	const chain = new Set();
	const visited = new Set();
	let depth = 0;
	const traverse = async (uid, id) => {
		const maxDepth = meta.config.activitypubParentTraversalDepth || 50;
		if (depth >= maxDepth) {
			activitypub.helpers.log(`[activitypub/notes/getParentChain] Depth limit reached (${maxDepth}), terminating.`);
			return;
		}
		if (visited.has(id)) {
			return;
		}
		visited.add(id);
		depth += 1;

		// Handle remote reference to local post
		const { type, id: localId } = await activitypub.helpers.resolveLocalId(id);
		if (type === 'post' && localId) {
			return await traverse(uid, localId);
		}

		const postData = await posts.getPostData(id);
		if (postData && postData.pid) {
			chain.add(postData);
			if (postData.toPid) {
				await traverse(uid, postData.toPid);
			} else if (utils.isNumber(id)) { // local pid without toPid, could be OP or reply to OP
				const mainPid = await topics.getTopicField(postData.tid, 'mainPid');
				if (mainPid !== parseInt(id, 10)) {
					await traverse(uid, mainPid);
				}
			}
		} else {
			let object = !activitypub.helpers.isUri(input) && input.id === id ? input : undefined;
			try {
				object = object || await activitypub.get('uid', uid, id);

				// Handle incorrect id passed in
				if (id !== object.id) {
					return await traverse(uid, object.id);
				}

				object = await activitypub.mocks.post(object);
				if (object) {
					chain.add(object);
					if (object.toPid) {
						await traverse(uid, object.toPid);
					}
				}
			} catch (e) {
				activitypub.helpers.log(`[activitypub/notes/getParentChain] Cannot retrieve ${id}, terminating here.`);
			}
		}
	};

	await traverse(uid, id);
	return chain;
};

Notes.syncUserInboxes = async function (tid, uid) {
	const [pids, { cid, mainPid, tags }] = await Promise.all([
		db.getSortedSetMembers(`tid:${tid}:posts`),
		topics.getTopicFields(tid, ['tid', 'cid', 'mainPid', 'tags']),
	]);
	pids.unshift(mainPid);

	const recipients = await db.getSetsMembers(pids.map(id => `post:${id}:recipients`));
	const uids = recipients.reduce((set, uids) => new Set([...set, ...uids.map(u => parseInt(u, 10))]), new Set());
	if (uid) {
		uids.add(parseInt(uid, 10));
	}

	// Tag followers
	const tagsFollowers = await topics.getTagsFollowers(tags.map(tag => tag.value));
	new Set(tagsFollowers.flat()).forEach((uid) => {
		uids.add(uid);
	});

	// Category followers
	const categoryFollowers = await activitypub.actors.getFollowers(cid);
	categoryFollowers.uids.forEach((uid) => {
		uids.add(uid);
	});

	const keys = Array.from(uids).map(uid => `uid:${uid}:inbox`);
	const score = await db.sortedSetScore(`cid:${cid}:tids`, tid);

	const removeKeys = (await db.getSetMembers(`tid:${tid}:recipients`))
		.filter(uid => !uids.has(parseInt(uid, 10)))
		.map((uid => `uid:${uid}:inbox`));

	activitypub.helpers.log(`[activitypub/syncUserInboxes] Syncing tid ${tid} with ${uids.size} inboxes`);
	await Promise.all([
		db.sortedSetsRemove(removeKeys, tid),
		db.sortedSetsAdd(keys, keys.map(() => score || Date.now()), tid),
		db.setAdd(`tid:${tid}:recipients`, Array.from(uids)),
	]);
};

Notes.getCategoryFollowers = async (cid) => {
	// Retrieves remote users who have followed a category; used to build recipient list
	let uids = await db.getSortedSetRangeByScore(`cid:${cid}:uid:watch:state`, 0, -1, categories.watchStates.tracking, categories.watchStates.tracking);
	uids = uids.filter(uid => !utils.isNumber(uid));

	return uids;
};

Notes.backfill = async (pids) => {
	if (!Array.isArray(pids)) {
		pids = [pids];
	}

	return Promise.all(pids.map(async (pid) => {
		if (backfillCache.has(pid)) {
			return;
		}

		await Notes.assert(0, pid, { skipChecks: 1 });
		backfillCache.set(pid, 1);
	}));
};

Notes.announce = {};

Notes.announce._cache = ttlCache({
	name: 'ap-note-announce-cache',
	max: 500,
	ttl: 1000 * 60 * 60, // 1 hour
});

Notes.announce.list = async ({ pid, tid }) => {
	let pids = [];
	if (pid) {
		pids = [pid];
	} else if (tid) {
		let mainPid;
		([pids, mainPid] = await Promise.all([
			db.getSortedSetMembers(`tid:${tid}:posts`),
			topics.getTopicField(tid, 'mainPid'),
		]));
		pids.unshift(mainPid);
	}

	if (!pids.length) {
		return [];
	}

	const missing = [];
	let announces = pids.map((pid, idx) => {
		const cached = Notes.announce._cache.get(pid);
		if (!cached) {
			missing.push(idx);
		}
		return cached;
	});

	if (missing.length) {
		const toCache = await db.getSortedSetsMembersWithScores(missing.map(idx => `pid:${pids[idx]}:announces`));
		toCache.forEach((value, idx) => {
			const pid = pids[missing[idx]];
			Notes.announce._cache.set(pid, value);
			announces[missing[idx]] = value;
		});
	}

	announces = announces.reduce((memo, cur, idx) => {
		if (cur.length) {
			const pid = pids[idx];
			cur.forEach(({ value: actor, score: timestamp }) => {
				memo.push({ pid, actor, timestamp });
			});
		}
		return memo;
	}, []);

	return announces;
};

Notes.announce.add = async (pid, actor, timestamp = Date.now()) => {
	const exists = await posts.exists(pid);
	if (!exists) {
		return;
	}

	const [tid] = await Promise.all([
		posts.getPostField(pid, 'tid'),
		db.sortedSetAdd(`pid:${pid}:announces`, timestamp, actor),
	]);
	Notes.announce._cache.del(`pid:${pid}:announces`);
	await Promise.all([
		posts.setPostField(pid, 'announces', await db.sortedSetCard(`pid:${pid}:announces`)),
		topics.tools.share(tid, actor, timestamp),
	]);
};

Notes.announce.remove = async (pid, actor) => {
	const exists = await posts.exists(pid);
	if (!exists) {
		return;
	}

	await db.sortedSetRemove(`pid:${pid}:announces`, actor);
	Notes.announce._cache.del(`pid:${pid}:announces`);

	const count = await db.sortedSetCard(`pid:${pid}:announces`);
	if (count > 0) {
		await posts.setPostField(pid, 'announces', count);
	} else {
		await db.deleteObjectField(`post:${pid}`, 'announces');
	}
};

Notes.announce.removeAll = async (pid) => {
	await Promise.all([
		db.delete(`pid:${pid}:announces`),
		db.deleteObjectField(`post:${pid}`, 'announces'),
	]);
	Notes.announce._cache.del(`pid:${pid}:announces`);
};

Notes.delete = async (pids) => {
	if (!Array.isArray(pids)) {
		pids = [pids];
	}

	const exists = await posts.exists(pids);
	pids = pids.filter((_, idx) => exists[idx]);

	let tids = await posts.getPostsFields(pids, ['tid']);
	tids = new Set(tids.map(obj => obj.tid));

	const recipientSets = pids.map(id => `post:${id}:recipients`);
	const announcerSets = pids.map(id => `pid:${id}:announces`);

	await db.deleteAll([...recipientSets, ...announcerSets]);
	await Promise.all(Array.from(tids).map(async tid => Notes.syncUserInboxes(tid)));
};

Notes.prune = async () => {
	/**
	 * Prune topics in cid -1 and handle:cid that have received no engagement.
	 * Categories in cid:0:children (shown on forum index) are skipped.
	 * Engagement is defined as:
	 *   - Replied to (contains a local reply)
	 *   - Post within is liked
	 * Cutoffs:
	 *   - cid -1: activitypubContentPruneDays
	 *   - remote cid, no followers: max(60, activitypubContentPruneDays * 2)
	 *   - remote cid, has followers: max(730, activitypubContentPruneDays)
	 */

	const allCids = await db.getObjectValues('handle:cid');
	const indexCids = await db.getSortedSetMembers('cid:0:children');
	const cids = allCids.filter(cid => !indexCids.includes(cid));
	winston.info(`[notes/prune] Starting scheduled pruning of topics in ${cids.length} categories (${allCids.length - cids.length} skipped: shown on index)`);

	const cutoff = Date.now() - (1000 * 60 * 60 * 24 * meta.config.activitypubContentPruneDays);
	const remoteCutoff = Date.now() - (1000 * 60 * 60 * 24 * Math.max(60, meta.config.activitypubContentPruneDays * 2));
	const remoteCutoffWithFollowers = Date.now() - (
		1000 * 60 * 60 * 24 * Math.max(730, meta.config.activitypubContentPruneDays)
	);
	await pruneCidTids(-1, cutoff);
	const followerCounts = await db.sortedSetsCard(cids.map(cid => `followersRemote:${cid}`));
	const cidCutoffs = new Map(
		cids.map((cid, idx) => [cid, followerCounts[idx] > 0 ? remoteCutoffWithFollowers : remoteCutoff]),
	);
	await batch.processArray(cids, async function (batch) {
		await Promise.all(batch.map(cid => pruneCidTids(cid, cidCutoffs.get(cid))));
	}, {
		batch: 10,
	});
	winston.info(`[notes/prune] Scheduled pruning of topics in ${cids.length} categories complete`);
};


async function pruneCidTids(cid, cutoff) {
	if (utils.isNumber(cid) && parseInt(cid, 10) !== -1) {
		// safety incase a local cid is in handle:cid
		return;
	}
	const tidsWithNoEngagement = [];

	await batch.processSortedSet(`cid:${cid}:tids`, async function (tids) {
		const [hasLocalVoters, posters] = await Promise.all([
			hasLocalVoter(tids),
			db.getSortedSetsMembers(tids.map(tid => `tid:${tid}:posters`)),
		]);

		tidsWithNoEngagement.push(...tids.filter((_, idx) => {
			const localPoster = posters[idx].some(uid => utils.isNumber(uid));
			const localVoter = hasLocalVoters[idx];
			return !localPoster && !localVoter;
		}));
	}, {
		min: '-inf',
		max: cutoff,
		batch: 500,
	});
	if (!tidsWithNoEngagement.length) {
		return;
	}
	winston.info(`[notes/prune] ${tidsWithNoEngagement.length} topics eligible in cid:${cid} for pruning`);

	await batch.processArray(tidsWithNoEngagement, async (tids) => {
		await topics.purgePostsAndTopic(tids, 0);
	}, { batch: 100 });
}

async function hasLocalVoter(tids) {
	const [topicData, topicPids] = await Promise.all([
		db.getObjectsFields(tids.map(tid => `topic:${tid}`), ['mainPid']),
		db.getSortedSetsMembers(tids.map(tid => `tid:${tid}:posts`)),
	]);

	const topicPidsCombined = topicData.map((t, idx) => {
		return t && t.mainPid ? [t.mainPid, ...topicPids[idx]] : topicPids[idx];
	});

	return await Promise.all(topicPidsCombined.map(async (topicPids) => {
		const upvote = topicPids.map(pid => `pid:${pid}:upvote`);
		const downvote = topicPids.map(pid => `pid:${pid}:downvote`);
		const voteSets = upvote.concat(downvote);
		const voters = new Set((await db.getSetsMembers(voteSets)).flat());
		return Array.from(voters).some(uid => utils.isNumber(uid));
	}));
}