Node.js based forum software built for the modern web
'use strict';
const winston = require('winston');
const nconf = require('nconf');
const tokenizer = require('sbd');
const pretty = require('pretty');
const db = require('../database');
const batch = require('../batch');
const meta = require('../meta');
const categories = require('../categories');
const messaging = require('../messaging');
const notifications = require('../notifications');
const user = require('../user');
const topics = require('../topics');
const posts = require('../posts');
const ttlCache = require('../cache/ttl');
const websockets = require('../socket.io');
const utils = require('../utils');
const translator = require('../translator');
const activitypub = module.parent.exports;
const Notes = module.exports;
const backfillCache = ttlCache({
name: 'ap-backfill-cache',
max: 500,
ttl: 1000 * 60 * 2, // 2 minutes
});
Notes._normalizeTags = async (tag, cid) => {
const systemTags = (meta.config.systemTags || '').split(',');
const maxTags = await categories.getCategoryField(cid, 'maxTags');
let tags = tag || [];
if (!Array.isArray(tags)) { // the "|| []" should handle null/undefined values... #famouslastwords
tags = [tags];
}
tags = tags
.filter(({ type }) => type === 'Hashtag')
.map((tag) => {
tag.name = tag.name.startsWith('#') ? tag.name.slice(1) : tag.name;
return tag;
})
.filter(({ name }) => !systemTags.includes(name))
.map(t => t.name);
if (tags.length > maxTags) {
tags.length = maxTags;
}
return tags;
};
Notes.assert = async (uid, input, options = { skipChecks: false, queue: false }) => {
/**
* Given the id or object of any as:Note, either retrieves the full context (if resolvable),
* or traverses up the reply chain to build a context.
*/
if (!input) {
return null;
}
let id = !activitypub.helpers.isUri(input) ? input.id : input;
let lockStatus = await db.incrObjectField('locks', id);
lockStatus = lockStatus <= 1;
if (!lockStatus) { // unable to achieve lock, stop processing.
winston.warn(`[activitypub/notes.assert] Unable to acquire lock, skipping processing of ${id}`);
return null;
}
try {
if (!(options.skipChecks || process.env.hasOwnProperty('CI'))) {
id = (await activitypub.checkHeader(id)) || id;
}
let chain;
let context = await activitypub.contexts.get(uid, id);
if (context.tid) {
const { tid } = context;
return { tid, count: 0 };
} else if (context.context) {
const { type } = await activitypub.helpers.resolveLocalId(context.context);
// Remote contexts only, if context is local, prefer parent chain traversal instead (to catch out-of-band replies)
if (type !== 'topic') {
chain = Array.from(await activitypub.contexts.getItems(uid, context.context, { input }));
if (chain && chain.length) {
// Deduplicate by id (just in case, also a buggy NodeBB impl. sent dupes)
const ids = new Set();
chain = chain.filter((item) => {
const seen = ids.has(item.pid);
ids.add(item.pid);
return !seen;
});
// Context resolves, use in later topic creation
context = context.context;
}
}
} else {
context = undefined;
}
if (!chain || !chain.length) {
// Fall back to inReplyTo traversal on context retrieval failure
chain = Array.from(await Notes.getParentChain(uid, input));
chain.reverse();
}
// Can't resolve — give up.
if (!chain.length) {
return null;
}
// Reorder chain items by timestamp
chain = chain.sort((a, b) => a.timestamp - b.timestamp);
const mainPost = chain[0];
let {
pid: mainPid,
tid,
uid: authorId,
timestamp,
title,
content,
sourceContent,
contentWarning,
_activitypub,
} = mainPost;
const hasTid = !!tid;
const authorBanned = await user.bans.isBanned(authorId);
if (!hasTid && authorBanned) { // New topics only
activitypub.helpers.log('[notes/assert] OP is banned, not asserting topic.');
return null;
}
const cid = hasTid ? await topics.getTopicField(tid, 'cid') : options.cid || -1;
let crosspostCid = false;
if (options.cid && cid === -1) {
// Move topic if currently uncategorized
await topics.tools.move(tid, { cid: options.cid, uid: 'system' });
}
const exists = await posts.exists(chain.map(p => p.pid));
if (tid && exists.every(Boolean)) {
// All cached, return early.
activitypub.helpers.log('[notes/assert] No new notes to process.');
return { tid, count: 0 };
}
let generatedTitle;
if (hasTid) {
mainPid = await topics.getTopicField(tid, 'mainPid');
} else {
// Check recipients/audience for category (local or remote)
const set = activitypub.helpers.makeSet(_activitypub, ['to', 'cc', 'audience']);
await activitypub.actors.assert(Array.from(set));
// Local (set if found)
const resolved = await Promise.all(Array.from(set).map(async id => await activitypub.helpers.resolveLocalId(id)));
const recipientCids = resolved
.filter(Boolean)
.filter(({ type }) => type === 'category')
.map(obj => obj.id);
// Remote (set only if cid and object domains match or cid is set in `audience`)
let remoteCid;
const assertedGroups = await categories.exists(Array.from(set));
try {
const { hostname } = new URL(mainPid);
remoteCid = Array.from(set).filter((id, idx) => {
const { hostname: cidHostname } = new URL(id);
const explicitAudience = Array.isArray(_activitypub.audience) ?
_activitypub.audience.includes(id) :
_activitypub.audience === id;
return assertedGroups[idx] && (explicitAudience || cidHostname === hostname);
}).shift();
} catch (e) {
// noop
winston.error('[activitypub/notes.assert] Could not parse URL of mainPid', e.stack);
}
if (remoteCid || recipientCids.length) {
// Overrides passed-in value, respect addressing from main post over booster
options.cid = remoteCid || recipientCids.shift();
}
// Auto-categorization (takes place only if all other categorization efforts fail)
const { cid, action: ruleAction } = await assignCategory(mainPost);
if (ruleAction === 2) {
activitypub.helpers.log(`[activitypub/notes.assert] Rule rejected post (${mainPid})`);
return null;
}
options.queue = ruleAction === 1;
crosspostCid = cid;
if (!options.cid) {
options.cid = crosspostCid;
crosspostCid = false;
}
// action is used below to decide whether to queue or add the crosspost
// mainPid ok to leave as-is
if (!title) {
let prettified = pretty(content || sourceContent);
// Remove any lines that contain quote-post fallbacks
prettified = prettified.split('\n').filter(line => !line.startsWith('<p class="quote-inline"')).join('\n');
const sentences = tokenizer.sentences(prettified, { sanitize: true, newline_boundaries: true });
title = sentences.shift();
generatedTitle = 1;
}
if (contentWarning) {
title = contentWarning;
generatedTitle = 1;
}
// Extract emoji metadata for title rendering
const titleEmoji = [];
const rawTags = _activitypub.tag;
const tagList = Array.isArray(rawTags) ? rawTags : (rawTags ? [rawTags] : []);
const emojiTags = tagList.filter(tag => tag?.type === 'Emoji');
for (const tag of emojiTags) {
const { name, icon } = tag;
if (!name || !icon?.url) continue;
let code = name;
if (!code.startsWith(':')) code = `:${code}`;
if (!code.endsWith(':')) code = `${code}:`;
if (title.includes(code)) {
try {
const { hostname } = new URL(icon.url);
const clean = code.replace(/^:+|:+$/g, '');
titleEmoji.push({ code, hostname, clean });
} catch {
// skip invalid icon URLs
}
}
}
if (titleEmoji.length) {
mainPost.titleEmoji = JSON.stringify(titleEmoji);
}
}
mainPid = utils.isNumber(mainPid) ? parseInt(mainPid, 10) : mainPid;
// Relation & privilege check for local categories
const inputIndex = chain.map(n => n.pid).indexOf(id);
const hasRelation =
uid || hasTid ||
options.skipChecks || options.cid ||
await assertRelation(chain[inputIndex !== -1 ? inputIndex : 0]);
if (!hasRelation) {
if (!hasRelation) {
activitypub.helpers.log(`[activitypub/notes.assert] Not asserting ${id} as it has no relation to existing tracked content.`);
}
return null;
}
tid = tid || utils.generateUUID();
mainPost.tid = tid;
const urlMap = chain.reduce((map, post) => (post.url ? map.set(post.url, post.id) : map), new Map());
let unprocessed = chain.map((post) => {
post.tid = tid; // add tid to post hash
// Ensure toPids in replies are ids
if (urlMap.has(post.toPid)) {
post.toPid = urlMap.get(post.toPid);
}
// Filter image attachments out if they are in content
const attachment = post?._activitypub?.attachment;
if (attachment && attachment.length) {
post._activitypub.attachment = attachment.filter((attachment) => {
const inContent = attachment.type === 'Image' && (post.sourceContent || post.content).includes(attachment.url);
return !inContent;
});
}
return post;
}).filter((p, idx) => !exists[idx]);
const count = unprocessed.length;
activitypub.helpers.log(`[notes/assert] ${count} new note(s) found.`);
if (!hasTid) {
activitypub.helpers.log(`[activitypub/notes.assert] hasTid=${hasTid}, skipChecks=${options.skipChecks}, mainPid=${mainPid}`);
const { hostname: mainHostname } = new URL(mainPid);
const mainResult = await activitypub.instances.isAllowed(mainHostname);
if (!mainResult.allowed) {
activitypub.helpers.log(`[activitypub/notes.assert] Not asserting ${mainPid}, domain is blocked.`);
return null;
}
if ((mainResult.severity === 3 || (utils.isNumber(options.cid) && options.queue)) && meta.config.postQueue) {
activitypub.helpers.log(`[activitypub/notes.assert] Queuing main post (${mainPid}) due to blocklist severity 3${options.queue ? ' or explicit queue option' : ''}`);
if (utils.isNumber(mainPid) || (await posts.exists([mainPid]))[0]) {
activitypub.helpers.log(`[activitypub/notes.assert] Rejecting to-be-queued main post (${mainPid}): pid is local or already exists`);
return null;
}
const queueData = {
uid: authorId,
cid: options.cid || cid,
pid: mainPid,
title,
titleEmoji: mainPost.titleEmoji,
timestamp,
content: mainPost.content,
sourceContent: mainPost.sourceContent,
contentWarning,
generatedTitle,
_activitypub: mainPost._activitypub,
};
await posts.addToQueue(queueData);
// Drop the rest of the chain — replies without OP don't make sense
return { tid: null, queued: 1 };
}
const { to, cc } = mainPost._activitypub;
const tags = await Notes._normalizeTags(mainPost._activitypub.tag || []);
try {
await topics.post({
tid,
uid: authorId,
cid: options.cid || cid,
pid: mainPid,
title,
titleEmoji: mainPost.titleEmoji,
timestamp,
tags,
content: mainPost.content,
sourceContent: mainPost.sourceContent,
contentWarning,
generatedTitle,
_activitypub: mainPost._activitypub,
});
unprocessed.shift();
} catch (e) {
activitypub.helpers.log(`[activitypub/notes.assert] Could not post topic (${mainPost.pid}): ${e.message}`);
return null;
}
// These must come after topic is posted
await Promise.all([
Notes.updateLocalRecipients(mainPid, { to, cc }),
mainPost._activitypub.image ? topics.thumbs.associate({
id: tid,
path: mainPost._activitypub.image,
}) : null,
]);
if (context) {
activitypub.helpers.log(`[activitypub/notes.assert] Associating tid ${tid} with context ${context}`);
await topics.setTopicField(tid, 'context', context);
}
}
const uids = Array.from(unprocessed.reduce((uids, post) => {
uids.add(post.uid);
return uids;
}, new Set()));
const isBanned = await user.bans.isBanned(uids);
const banned = uids.filter((_, idx) => isBanned[idx]);
unprocessed = unprocessed.filter(post => !banned.includes(post.uid));
let added = [];
let queued = 0;
await Promise.all(unprocessed.map(async (post) => {
const { to, cc } = post._activitypub;
try {
const { hostname: postHostname } = new URL(post.pid);
const postResult = await activitypub.instances.isAllowed(postHostname);
if (!postResult.allowed) {
activitypub.helpers.log(`[activitypub/notes.assert] Not asserting ${post.pid}, domain is blocked.`);
return;
}
if (postResult.severity === 3 && meta.config.postQueue) {
activitypub.helpers.log(`[activitypub/notes.assert] Queuing reply (${post.pid}) due to blocklist severity 3`);
if (utils.isNumber(post.pid) || await posts.exists(post.pid)) {
activitypub.helpers.log(`[activitypub/notes.assert] Rejecting to-be-queued reply (${post.pid}): pid or tid is local or already exists`);
return;
}
await posts.addToQueue({
uid: post.uid,
tid,
pid: post.pid,
content: post.content,
sourceContent: post.sourceContent,
contentWarning: post.contentWarning,
timestamp: post.timestamp,
_activitypub: post._activitypub,
});
queued += 1;
return;
}
const postData = await topics.reply(post);
added.push(postData);
await Notes.updateLocalRecipients(post.pid, { to, cc });
} catch (e) {
activitypub.helpers.log(`[activitypub/notes.assert] Could not add reply (${post.pid}): ${e.message}`);
}
}));
if (added.length) {
// Because replies are added in parallel, `index` is calculated incorrectly
const indices = await posts.getPostIndices(added, uid);
added = added.map((post, idx) => {
post.index = indices[idx];
return post;
});
websockets.in(`topic_${tid}`).emit('event:new_post', { posts: added });
}
await Notes.syncUserInboxes(tid, uid);
if (crosspostCid) {
if (options.queue) {
await topics.crossposts.queue(tid, crosspostCid, 0);
} else {
await topics.crossposts.add(tid, crosspostCid, 0);
}
}
if (!hasTid && uid && options.cid) {
// New topic, have category announce it
await activitypub.out.announce.topic(tid);
}
return { tid, count, queued };
} catch (e) {
console.log(e.stack);
winston.warn(`[activitypub/notes.assert] Could not assert ${id} (${e.message}).`);
return null;
} finally {
activitypub.helpers.log(`[activitypub/notes.assert] Releasing lock (${id})`);
await db.deleteObjectField('locks', id);
}
};
Notes.assertPrivate = async (object) => {
// Given an object, adds it to an existing chat or creates a new chat otherwise
// todo: context stuff
if (!object || !object.id || !activitypub.helpers.isUri(object.id)) {
return null;
}
const localUids = [];
const recipients = new Set([...(object.to || []), ...(object.cc || [])]);
await Promise.all(Array.from(recipients).map(async (value) => {
const { type, id } = await activitypub.helpers.resolveLocalId(value);
if (type === 'user') {
localUids.push(id);
recipients.delete(value);
recipients.add(parseInt(id, 10));
}
}));
// Trim recipient list down to asserted actors (and local users) only
await activitypub.actors.assert([...recipients]);
const exists = await user.exists([...recipients]);
Array.from(recipients).forEach((uid, idx) => {
if (!exists[idx]) {
recipients.delete(uid);
}
});
// Locate the `roomId` and set `toMid` based on `inReplyTo`
let roomId;
let toMid;
const resolved = await activitypub.helpers.resolveLocalId(object.inReplyTo);
if (resolved.type === 'message' && resolved.id) {
toMid = resolved.id;
} else {
toMid = object.inReplyTo;
}
if (toMid && await messaging.messageExists(toMid)) {
roomId = await messaging.getMessageField(toMid, 'roomId');
} else {
toMid = undefined;
// roomId stays undefined
}
// Compare room members with object recipients; if someone in-room is omitted, start new chat
const participants = await messaging.getUsersInRoom(roomId, 0, -1);
const participantUids = participants.map(user => user.uid);
if (roomId) {
const omitted = participants.filter((user) => {
const { uid } = user;
return !recipients.has(uid) && uid !== object.attributedTo;
});
if (omitted.length) {
toMid = undefined; // message creation logic fails if toMid is not in room
roomId = null;
}
}
let timestamp;
try {
timestamp = new Date(object.published).getTime() || Date.now();
} catch (e) {
timestamp = Date.now();
}
let payload;
try {
payload = await activitypub.mocks.message(object);
} catch (e) {
activitypub.helpers.log(`[activitypub/notes.assertPrivate] Failed to mock message: ${e.message}`);
return null;
}
// Naive image appending (using src/posts/attachments.js is likely better, but not worth the effort)
const attachments = payload._activitypub.attachment;
if (attachments && Array.isArray(attachments)) {
const images = attachments.filter((attachment) => {
return attachment.mediaType.startsWith('image/');
}).map(({ url, href }) => url || href);
images.forEach((url) => {
payload.content += `<p><img class="img-fluid img-thumbnail" src="${url}" /></p>`;
});
}
try {
await messaging.checkContent(payload.content, false);
} catch (e) {
const [displayname, userslug] = await Promise.all([
user.getNotificationDisplayname(payload.uid),
user.getUserField(payload.uid, 'userslug'),
]);
const notification = await notifications.create({
bodyShort: translator.compile('error:remote-chat-received-too-long', displayname),
path: `/user/${userslug}`,
nid: `error:chat:uid:${payload.uid}`,
from: payload.uid,
});
notifications.push(notification, Array.from(recipients).filter(uid => utils.isNumber(uid)));
return null;
}
// Local recipients who cannot be messaged should be removed
const recipientUids = Array.from(recipients).filter(uid => utils.isNumber(uid));
const results = await Promise.all(recipientUids.map(async (uid) => {
try {
await messaging.canMessageUser(payload.uid, uid);
return { uid, valid: true };
} catch (e) {
return { uid, valid: false };
}
}));
const validUids = results.filter(r => r.valid).map(r => r.uid);
if (validUids.length === 0) {
return null;
}
results.filter(r => !r.valid).forEach(r => recipients.delete(r.uid));
if (!roomId) {
roomId = await messaging.newRoom(payload.uid, { uids: [...recipients] });
}
// Add any new members to the chat
const added = Array.from(recipients).filter(uid => !participantUids.includes(uid));
const assertion = await activitypub.actors.assert(added);
if (assertion) {
await Promise.all([
messaging.addUsersToRoom(payload.uid, added, roomId),
...added.map(uid => messaging.addSystemMessage('user-join', uid, roomId)),
]);
}
// Add message to room
const message = await messaging.sendMessage({
...payload,
timestamp: Date.now(),
roomId: roomId,
toMid: toMid,
});
messaging.notifyUsersInRoom(payload.uid, roomId, message);
// Set real timestamp back so that the message shows even though it predates room joining
await messaging.setMessageField(payload.mid, 'timestamp', timestamp);
return { roomId };
};
async function assertRelation(post) {
/**
* Given a mocked post object, ensures that it is related to some other object in database
* This check ensures that random content isn't added to the database just because it is received.
*/
// Is followed by at least one local user
const { followers } = await activitypub.actors.getLocalFollowCounts(post.uid);
// Local user is mentioned
const { tag } = post._activitypub;
let uids = [];
if (tag && tag.length) {
const slugs = tag.reduce((slugs, tag) => {
if (tag.type === 'Mention') {
const [slug, hostname] = tag.name.slice(1).split('@');
if (hostname === nconf.get('url_parsed').hostname) {
slugs.push(slug);
}
}
return slugs;
}, []);
uids = slugs.length ? await db.sortedSetScores('userslug:uid', slugs) : [];
uids = uids.filter(Boolean);
}
return followers > 0 || uids.length;
}
async function assignCategory(post) {
const ageLimitDays = meta.config.activitypubRulesCutoffDays || 0;
if (ageLimitDays > 0 && post.timestamp) {
const ageDays = (Date.now() - post.timestamp) / (1000 * 60 * 60 * 24);
if (ageDays > ageLimitDays) {
activitypub.helpers.log(`[activitypub] Post ${post.pid} is ${ageDays.toFixed(1)} days old, skipping categorization (limit: ${ageLimitDays} days)`);
return { cid: undefined, filter: false };
}
}
activitypub.helpers.log('[activitypub] Checking auto-categorization rules.');
const rules = await activitypub.rules.list();
const tags = await Notes._normalizeTags(post._activitypub.tag || []);
const matched = rules.reduce((matched, { type, value, cid: target, action: ruleAction }) => {
if (!matched.cid) {
switch (type) {
case 'hashtag': {
const lowerValue = value.toLowerCase();
if (tags.some(tag => tag.toLowerCase() === lowerValue)) {
activitypub.helpers.log(`[activitypub] - Rule match: #${value}; cid: ${target}, action: ${ruleAction}`);
return { cid: target, action: ruleAction };
}
break;
}
case 'user': {
if (post.uid === value) {
activitypub.helpers.log(`[activitypub] - Rule match: user ${value}; cid: ${target}, action: ${ruleAction}`);
return { cid: target, action: ruleAction };
}
}
}
}
return matched;
}, { cid: undefined, action: 0 });
return matched;
}
Notes.updateLocalRecipients = async (id, { to, cc }) => {
const recipients = new Set([...(to || []), ...(cc || [])]);
const uids = new Set();
await Promise.all(Array.from(recipients).map(async (recipient) => {
const { type, id } = await activitypub.helpers.resolveLocalId(recipient);
if (type === 'user' && await user.exists(id)) {
uids.add(parseInt(id, 10));
return;
}
const followedUid = await db.getObjectField('followersUrl:uid', recipient);
if (followedUid) {
const { uids: followers } = await activitypub.actors.getFollowers(followedUid);
if (followers.size > 0) {
followers.forEach((uid) => {
uids.add(uid);
});
}
}
}));
if (uids.size > 0) {
await db.setAdd(`post:${id}:recipients`, Array.from(uids));
}
};
Notes.getParentChain = async (uid, input) => {
// Traverse upwards via `inReplyTo` until you find the root-level Note
const id = activitypub.helpers.isUri(input) ? input : input.id;
const chain = new Set();
const visited = new Set();
let depth = 0;
const traverse = async (uid, id) => {
const maxDepth = meta.config.activitypubParentTraversalDepth || 50;
if (depth >= maxDepth) {
activitypub.helpers.log(`[activitypub/notes/getParentChain] Depth limit reached (${maxDepth}), terminating.`);
return;
}
if (visited.has(id)) {
return;
}
visited.add(id);
depth += 1;
// Handle remote reference to local post
const { type, id: localId } = await activitypub.helpers.resolveLocalId(id);
if (type === 'post' && localId) {
return await traverse(uid, localId);
}
const postData = await posts.getPostData(id);
if (postData && postData.pid) {
chain.add(postData);
if (postData.toPid) {
await traverse(uid, postData.toPid);
} else if (utils.isNumber(id)) { // local pid without toPid, could be OP or reply to OP
const mainPid = await topics.getTopicField(postData.tid, 'mainPid');
if (mainPid !== parseInt(id, 10)) {
await traverse(uid, mainPid);
}
}
} else {
let object = !activitypub.helpers.isUri(input) && input.id === id ? input : undefined;
try {
object = object || await activitypub.get('uid', uid, id);
// Handle incorrect id passed in
if (id !== object.id) {
return await traverse(uid, object.id);
}
object = await activitypub.mocks.post(object);
if (object) {
chain.add(object);
if (object.toPid) {
await traverse(uid, object.toPid);
}
}
} catch (e) {
activitypub.helpers.log(`[activitypub/notes/getParentChain] Cannot retrieve ${id}, terminating here.`);
}
}
};
await traverse(uid, id);
return chain;
};
Notes.syncUserInboxes = async function (tid, uid) {
const [pids, { cid, mainPid, tags }] = await Promise.all([
db.getSortedSetMembers(`tid:${tid}:posts`),
topics.getTopicFields(tid, ['tid', 'cid', 'mainPid', 'tags']),
]);
pids.unshift(mainPid);
const recipients = await db.getSetsMembers(pids.map(id => `post:${id}:recipients`));
const uids = recipients.reduce((set, uids) => new Set([...set, ...uids.map(u => parseInt(u, 10))]), new Set());
if (uid) {
uids.add(parseInt(uid, 10));
}
// Tag followers
const tagsFollowers = await topics.getTagsFollowers(tags.map(tag => tag.value));
new Set(tagsFollowers.flat()).forEach((uid) => {
uids.add(uid);
});
// Category followers
const categoryFollowers = await activitypub.actors.getFollowers(cid);
categoryFollowers.uids.forEach((uid) => {
uids.add(uid);
});
const keys = Array.from(uids).map(uid => `uid:${uid}:inbox`);
const score = await db.sortedSetScore(`cid:${cid}:tids`, tid);
const removeKeys = (await db.getSetMembers(`tid:${tid}:recipients`))
.filter(uid => !uids.has(parseInt(uid, 10)))
.map((uid => `uid:${uid}:inbox`));
activitypub.helpers.log(`[activitypub/syncUserInboxes] Syncing tid ${tid} with ${uids.size} inboxes`);
await Promise.all([
db.sortedSetsRemove(removeKeys, tid),
db.sortedSetsAdd(keys, keys.map(() => score || Date.now()), tid),
db.setAdd(`tid:${tid}:recipients`, Array.from(uids)),
]);
};
Notes.getCategoryFollowers = async (cid) => {
// Retrieves remote users who have followed a category; used to build recipient list
let uids = await db.getSortedSetRangeByScore(`cid:${cid}:uid:watch:state`, 0, -1, categories.watchStates.tracking, categories.watchStates.tracking);
uids = uids.filter(uid => !utils.isNumber(uid));
return uids;
};
Notes.backfill = async (pids) => {
if (!Array.isArray(pids)) {
pids = [pids];
}
return Promise.all(pids.map(async (pid) => {
if (backfillCache.has(pid)) {
return;
}
await Notes.assert(0, pid, { skipChecks: 1 });
backfillCache.set(pid, 1);
}));
};
Notes.announce = {};
Notes.announce._cache = ttlCache({
name: 'ap-note-announce-cache',
max: 500,
ttl: 1000 * 60 * 60, // 1 hour
});
Notes.announce.list = async ({ pid, tid }) => {
let pids = [];
if (pid) {
pids = [pid];
} else if (tid) {
let mainPid;
([pids, mainPid] = await Promise.all([
db.getSortedSetMembers(`tid:${tid}:posts`),
topics.getTopicField(tid, 'mainPid'),
]));
pids.unshift(mainPid);
}
if (!pids.length) {
return [];
}
const missing = [];
let announces = pids.map((pid, idx) => {
const cached = Notes.announce._cache.get(pid);
if (!cached) {
missing.push(idx);
}
return cached;
});
if (missing.length) {
const toCache = await db.getSortedSetsMembersWithScores(missing.map(idx => `pid:${pids[idx]}:announces`));
toCache.forEach((value, idx) => {
const pid = pids[missing[idx]];
Notes.announce._cache.set(pid, value);
announces[missing[idx]] = value;
});
}
announces = announces.reduce((memo, cur, idx) => {
if (cur.length) {
const pid = pids[idx];
cur.forEach(({ value: actor, score: timestamp }) => {
memo.push({ pid, actor, timestamp });
});
}
return memo;
}, []);
return announces;
};
Notes.announce.add = async (pid, actor, timestamp = Date.now()) => {
const exists = await posts.exists(pid);
if (!exists) {
return;
}
const [tid] = await Promise.all([
posts.getPostField(pid, 'tid'),
db.sortedSetAdd(`pid:${pid}:announces`, timestamp, actor),
]);
Notes.announce._cache.del(`pid:${pid}:announces`);
await Promise.all([
posts.setPostField(pid, 'announces', await db.sortedSetCard(`pid:${pid}:announces`)),
topics.tools.share(tid, actor, timestamp),
]);
};
Notes.announce.remove = async (pid, actor) => {
const exists = await posts.exists(pid);
if (!exists) {
return;
}
await db.sortedSetRemove(`pid:${pid}:announces`, actor);
Notes.announce._cache.del(`pid:${pid}:announces`);
const count = await db.sortedSetCard(`pid:${pid}:announces`);
if (count > 0) {
await posts.setPostField(pid, 'announces', count);
} else {
await db.deleteObjectField(`post:${pid}`, 'announces');
}
};
Notes.announce.removeAll = async (pid) => {
await Promise.all([
db.delete(`pid:${pid}:announces`),
db.deleteObjectField(`post:${pid}`, 'announces'),
]);
Notes.announce._cache.del(`pid:${pid}:announces`);
};
Notes.delete = async (pids) => {
if (!Array.isArray(pids)) {
pids = [pids];
}
const exists = await posts.exists(pids);
pids = pids.filter((_, idx) => exists[idx]);
let tids = await posts.getPostsFields(pids, ['tid']);
tids = new Set(tids.map(obj => obj.tid));
const recipientSets = pids.map(id => `post:${id}:recipients`);
const announcerSets = pids.map(id => `pid:${id}:announces`);
await db.deleteAll([...recipientSets, ...announcerSets]);
await Promise.all(Array.from(tids).map(async tid => Notes.syncUserInboxes(tid)));
};
Notes.prune = async () => {
/**
* Prune topics in cid -1 and handle:cid that have received no engagement.
* Categories in cid:0:children (shown on forum index) are skipped.
* Engagement is defined as:
* - Replied to (contains a local reply)
* - Post within is liked
* Cutoffs:
* - cid -1: activitypubContentPruneDays
* - remote cid, no followers: max(60, activitypubContentPruneDays * 2)
* - remote cid, has followers: max(730, activitypubContentPruneDays)
*/
const allCids = await db.getObjectValues('handle:cid');
const indexCids = await db.getSortedSetMembers('cid:0:children');
const cids = allCids.filter(cid => !indexCids.includes(cid));
winston.info(`[notes/prune] Starting scheduled pruning of topics in ${cids.length} categories (${allCids.length - cids.length} skipped: shown on index)`);
const cutoff = Date.now() - (1000 * 60 * 60 * 24 * meta.config.activitypubContentPruneDays);
const remoteCutoff = Date.now() - (1000 * 60 * 60 * 24 * Math.max(60, meta.config.activitypubContentPruneDays * 2));
const remoteCutoffWithFollowers = Date.now() - (
1000 * 60 * 60 * 24 * Math.max(730, meta.config.activitypubContentPruneDays)
);
await pruneCidTids(-1, cutoff);
const followerCounts = await db.sortedSetsCard(cids.map(cid => `followersRemote:${cid}`));
const cidCutoffs = new Map(
cids.map((cid, idx) => [cid, followerCounts[idx] > 0 ? remoteCutoffWithFollowers : remoteCutoff]),
);
await batch.processArray(cids, async function (batch) {
await Promise.all(batch.map(cid => pruneCidTids(cid, cidCutoffs.get(cid))));
}, {
batch: 10,
});
winston.info(`[notes/prune] Scheduled pruning of topics in ${cids.length} categories complete`);
};
async function pruneCidTids(cid, cutoff) {
if (utils.isNumber(cid) && parseInt(cid, 10) !== -1) {
// safety incase a local cid is in handle:cid
return;
}
const tidsWithNoEngagement = [];
await batch.processSortedSet(`cid:${cid}:tids`, async function (tids) {
const [hasLocalVoters, posters] = await Promise.all([
hasLocalVoter(tids),
db.getSortedSetsMembers(tids.map(tid => `tid:${tid}:posters`)),
]);
tidsWithNoEngagement.push(...tids.filter((_, idx) => {
const localPoster = posters[idx].some(uid => utils.isNumber(uid));
const localVoter = hasLocalVoters[idx];
return !localPoster && !localVoter;
}));
}, {
min: '-inf',
max: cutoff,
batch: 500,
});
if (!tidsWithNoEngagement.length) {
return;
}
winston.info(`[notes/prune] ${tidsWithNoEngagement.length} topics eligible in cid:${cid} for pruning`);
await batch.processArray(tidsWithNoEngagement, async (tids) => {
await topics.purgePostsAndTopic(tids, 0);
}, { batch: 100 });
}
async function hasLocalVoter(tids) {
const [topicData, topicPids] = await Promise.all([
db.getObjectsFields(tids.map(tid => `topic:${tid}`), ['mainPid']),
db.getSortedSetsMembers(tids.map(tid => `tid:${tid}:posts`)),
]);
const topicPidsCombined = topicData.map((t, idx) => {
return t && t.mainPid ? [t.mainPid, ...topicPids[idx]] : topicPids[idx];
});
return await Promise.all(topicPidsCombined.map(async (topicPids) => {
const upvote = topicPids.map(pid => `pid:${pid}:upvote`);
const downvote = topicPids.map(pid => `pid:${pid}:downvote`);
const voteSets = upvote.concat(downvote);
const voters = new Set((await db.getSetsMembers(voteSets)).flat());
return Array.from(voters).some(uid => utils.isNumber(uid));
}));
}