Skip to content

Node.js based forum software built for the modern web

feat: prune topics remote cids, closes #13461

Barış Soner Uşaklı20eb02f15821
diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js
index d8fb26bfbb..3a042aa2e9 100644
--- a/src/activitypub/notes.js
+++ b/src/activitypub/notes.js
@@ -738,47 +738,74 @@ Notes.delete = async (pids) => {
Notes.prune = async () => {
/**
- * Prune topics in cid -1 that have received no engagement.
+ * Prune topics in cid -1 and handle:cid that have received no engagement.
* Engagement is defined as:
* - Replied to (contains a local reply)
* - Post within is liked
*/
- winston.info('[notes/prune] Starting scheduled pruning of topics');
- const start = '-inf';
- const stop = Date.now() - (1000 * 60 * 60 * 24 * meta.config.activitypubContentPruneDays);
- let tids = await db.getSortedSetRangeByScore('cid:-1:tids', 0, -1, start, stop);
- winston.info(`[notes/prune] Found ${tids.length} topics older than 30 days (since last activity).`);
+ const cids = await db.getObjectValues('handle:cid');
+ winston.info(`[notes/prune] Starting scheduled pruning of topics in ${cids.length} categories`);
- const posters = await db.getSortedSetsMembers(tids.map(tid => `tid:${tid}:posters`));
- const hasLocalVoter = await Promise.all(tids.map(async (tid) => {
- const mainPid = await db.getObjectField(`topic:${tid}`, 'mainPid');
- const pids = await db.getSortedSetMembers(`tid:${tid}:posts`);
- pids.unshift(mainPid);
-
- // Check voters of each pid for a local uid
- const voters = new Set();
- await Promise.all(pids.map(async (pid) => {
- const [upvoters, downvoters] = await db.getSetsMembers([`pid:${pid}:upvote`, `pid:${pid}:downvote`]);
- upvoters.forEach(uid => voters.add(uid));
- downvoters.forEach(uid => voters.add(uid));
- }));
-
- return Array.from(voters).some(uid => utils.isNumber(uid));
- }));
+ const cuttoff = Date.now() - (1000 * 60 * 60 * 24 * meta.config.activitypubContentPruneDays);
+ const remoteCutoff = Date.now() - (1000 * 60 * 60 * 24 * Math.max(60, meta.config.activitypubContentPruneDays * 2));
+ await pruneCidTids(-1, cuttoff);
+ await batch.processArray(cids, async function (cids) {
+ await Promise.all(cids.map(cid => pruneCidTids(cid, remoteCutoff)));
+ }, {
+ batch: 100,
+ });
+};
- tids = tids.filter((_, idx) => {
- const localPoster = posters[idx].some(uid => utils.isNumber(uid));
- const localVoter = hasLocalVoter[idx];
- return !localPoster && !localVoter;
+async function pruneCidTids(cid, cuttoff) {
+ if (utils.isNumber(cid) && cid !== -1) {
+ // safety incase a local cid is in handle:cid
+ return;
+ }
+ const tidsWithNoEngagement = [];
+
+ await batch.processSortedSet(`cid:${cid}:tids`, async function (tids) {
+ const [hasLocalVoters, posters] = await Promise.all([
+ hasLocalVoter(tids),
+ db.getSortedSetsMembers(tids.map(tid => `tid:${tid}:posters`)),
+ ]);
+
+ tidsWithNoEngagement.push(...tids.filter((_, idx) => {
+ const localPoster = posters[idx].some(uid => utils.isNumber(uid));
+ const localVoter = hasLocalVoters[idx];
+ return !localPoster && !localVoter;
+ }));
+ }, {
+ min: '-inf',
+ max: cuttoff,
+ batch: 500,
});
- winston.info(`[notes/prune] ${tids.length} topics eligible for pruning`);
+ winston.info(`[notes/prune] ${tidsWithNoEngagement.length} topics eligible in cid:${cid} for pruning`);
- await batch.processArray(tids, async (tids) => {
+ await batch.processArray(tidsWithNoEngagement, async (tids) => {
await Promise.all(tids.map(async tid => await topics.purgePostsAndTopic(tid, 0)));
}, { batch: 100 });
- winston.info('[notes/prune] Scheduled pruning of topics complete.');
-};
+ winston.info(`[notes/prune] Scheduled pruning of topics in cid:${cid} complete.`);
+}
+
+async function hasLocalVoter(tids) {
+ const [topicData, topicPids] = await Promise.all([
+ db.getObjectsFields(tids.map(tid => `topic:${tid}`), ['mainPid']),
+ db.getSortedSetsMembers(tids.map(tid => `tid:${tid}:posts`)),
+ ]);
+
+ const topicPidsCombined = topicData.map((t, idx) => {
+ return t && t.mainPid ? [t.mainPid, ...topicPids[idx]] : topicPids[idx];
+ });
+
+ return await Promise.all(topicPidsCombined.map(async (topicPids) => {
+ const upvote = topicPids.map(pid => `pid:${pid}:upvote`);
+ const downvote = topicPids.map(pid => `pid:${pid}:downvote`);
+ const voteSets = upvote.concat(downvote);
+ const voters = new Set(await db.getSetsMembers(voteSets).flat());
+ return Array.from(voters).some(uid => utils.isNumber(uid));
+ }));
+}
\ No newline at end of file