Node.js based forum software built for the modern web
feat: track user cids (#14114)
Julian Lam781ed3447bac
diff --git a/src/categories/recentreplies.js b/src/categories/recentreplies.jsindex 8e21647c70..92eed3be56 100644--- a/src/categories/recentreplies.js+++ b/src/categories/recentreplies.js@@ -191,10 +191,15 @@ module.exports = function (Categories) {const bulkRemove = [];const bulkAdd = [];+ const bulkIncr = [];postData.forEach((post) => {bulkRemove.push([`cid:${oldCid}:uid:${post.uid}:pids`, post.pid]);bulkRemove.push([`cid:${oldCid}:uid:${post.uid}:pids:votes`, post.pid]);bulkAdd.push([`cid:${cid}:uid:${post.uid}:pids`, post.timestamp, post.pid]);+ bulkIncr.push(+ [`uid:${post.uid}:cids`, -1, oldCid],+ [`uid:${post.uid}:cids`, 1, cid],+ );if (post.votes > 0 || post.votes < 0) {bulkAdd.push([`cid:${cid}:uid:${post.uid}:pids:votes`, post.votes, post.pid]);}@@ -207,6 +212,7 @@ module.exports = function (Categories) {db.sortedSetAdd(`cid:${cid}:pids`, timestamps, postsToReAdd.map(p => p.pid)),db.sortedSetRemoveBulk(bulkRemove),db.sortedSetAddBulk(bulkAdd),+ db.sortedSetIncrByBulk(bulkIncr),]);}, { batch: 500 });};diff --git a/src/controllers/accounts/posts.js b/src/controllers/accounts/posts.jsindex 9ce048bd28..1535ed0737 100644--- a/src/controllers/accounts/posts.js+++ b/src/controllers/accounts/posts.js@@ -31,7 +31,8 @@ const templateToData = {noItemsFoundKey: '[[user:has-no-posts]]',crumb: '[[global:posts]]',getSets: async function (callerUid, userData) {- const cids = await categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read');+ let cids = await db.getSortedSetRangeByScore(`uid:${userData.uid}:cids`, 0, -1, 1, '+inf');+ cids = await privileges.categories.filterCids('topics:read', cids, callerUid);return cids.map(c => `cid:${c}:uid:${userData.uid}:pids`);},},@@ -143,7 +144,8 @@ const templateToData = {noItemsFoundKey: '[[user:has-no-topics]]',crumb: '[[global:topics]]',getSets: async function (callerUid, userData) {- const cids = await categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read');+ let cids = await db.getSortedSetRangeByScore(`uid:${userData.uid}:cids`, 0, -1, 1, '+inf');+ cids = await privileges.categories.filterCids('topics:read', cids, callerUid);return cids.map(c => `cid:${c}:uid:${userData.uid}:tids`);},},diff --git a/src/database/helpers.js b/src/database/helpers.jsindex eb8206f783..d66bfb87a6 100644--- a/src/database/helpers.js+++ b/src/database/helpers.js@@ -42,3 +42,21 @@ helpers.globToRegex = function (match) {}return _match;};++helpers.aggregateIncrByBulk = function (data) {+ const buckets = Object.create(null);++ for (const [key, incr, val] of data) {+ buckets[key] = buckets[key] || {};+ buckets[key][val] = (buckets[key][val] || 0) + incr;+ }++ const result = [];+ for (const [key, vals] of Object.entries(buckets)) {+ for (const [val, incr] of Object.entries(vals)) {+ result.push([key, incr, val]);+ }+ }++ return result;+};diff --git a/src/database/mongo/sorted.js b/src/database/mongo/sorted.jsindex c6f1ee90ce..71c4f4c273 100644--- a/src/database/mongo/sorted.js+++ b/src/database/mongo/sorted.js@@ -459,16 +459,35 @@ module.exports = function (module) {};module.sortedSetIncrByBulk = async function (data) {+ if (!Array.isArray(data) || !data.length) {+ return [];+ }+ const aggregated = dbHelpers.aggregateIncrByBulk(data);const bulk = module.client.collection('objects').initializeUnorderedBulkOp();- data.forEach((item) => {+ aggregated.forEach((item) => {bulk.find({ _key: item[0], value: helpers.valueToString(item[2]) }).upsert().update({ $inc: { score: parseFloat(item[1]) } });});- await bulk.execute();++ try {+ await bulk.execute();+ } catch (err) {+ // retry failed e11000 operations+ if (err.code === 11000 || (err.writeErrors && err.writeErrors.some(e => e.code === 11000))) {+ const failedIndices = err.writeErrors.map(e => e.index);+ const retryData = failedIndices.map(idx => aggregated[idx]);+ await Promise.all(retryData.map(+ item => module.sortedSetIncrBy(item[0], item[1], item[2])+ ));+ } else {+ throw err;+ }+ }+const result = await module.client.collection('objects').find({- _key: { $in: _.uniq(data.map(i => i[0])) },- value: { $in: _.uniq(data.map(i => i[2])) },+ _key: { $in: _.uniq(aggregated.map(i => i[0])) },+ value: { $in: _.uniq(aggregated.map(i => i[2])) },}, {projection: { _id: 0, _key: 1, value: 1, score: 1 },}).toArray();diff --git a/src/database/postgres/sorted.js b/src/database/postgres/sorted.jsindex 351fe3e059..6332907e43 100644--- a/src/database/postgres/sorted.js+++ b/src/database/postgres/sorted.js@@ -1,8 +1,10 @@'use strict';module.exports = function (module) {- const helpers = require('./helpers');const util = require('util');++ const helpers = require('./helpers');+ const dbHelpers = require('../helpers');const Cursor = require('pg-cursor');Cursor.prototype.readAsync = util.promisify(Cursor.prototype.read);const sleep = util.promisify(setTimeout);@@ -547,18 +549,19 @@ RETURNING "score" s`,};module.sortedSetIncrByBulk = async function (data) {- if (!data.length) {+ if (!Array.isArray(data) || !data.length) {return [];}+ const aggregated = dbHelpers.aggregateIncrByBulk(data);return await module.transaction(async (client) => {- await helpers.ensureLegacyObjectsType(client, data.map(item => item[0]), 'zset');+ await helpers.ensureLegacyObjectsType(client, aggregated.map(item => item[0]), 'zset');const values = [];const queryParams = [];let paramIndex = 1;- data.forEach(([key, increment, value]) => {+ aggregated.forEach(([key, increment, value]) => {value = helpers.valueToString(value);increment = parseFloat(increment);values.push(key, value, increment);diff --git a/src/database/redis/sorted.js b/src/database/redis/sorted.jsindex 8433133f15..a29ef75a20 100644--- a/src/database/redis/sorted.js+++ b/src/database/redis/sorted.js@@ -261,8 +261,12 @@ module.exports = function (module) {};module.sortedSetIncrByBulk = async function (data) {+ if (!Array.isArray(data) || !data.length) {+ return [];+ }+ const aggregated = dbHelpers.aggregateIncrByBulk(data);const multi = module.client.multi();- data.forEach((item) => {+ aggregated.forEach((item) => {multi.zIncrBy(item[0], item[1], String(item[2]));});const result = await multi.exec();diff --git a/src/posts/delete.js b/src/posts/delete.jsindex f6b7aafb37..9235f37111 100644--- a/src/posts/delete.js+++ b/src/posts/delete.js@@ -96,14 +96,19 @@ module.exports = function (Posts) {async function deleteFromTopicUserNotification(postData) {const bulkRemove = [];+ const bulkIncr = [];postData.forEach((p) => {bulkRemove.push([`tid:${p.tid}:posts`, p.pid]);bulkRemove.push([`tid:${p.tid}:posts:votes`, p.pid]);bulkRemove.push([`uid:${p.uid}:posts`, p.pid]);bulkRemove.push([`cid:${p.cid}:uid:${p.uid}:pids`, p.pid]);bulkRemove.push([`cid:${p.cid}:uid:${p.uid}:pids:votes`, p.pid]);+ bulkIncr.push([`uid:${p.uid}:cids`, -1, p.cid]);});- await db.sortedSetRemoveBulk(bulkRemove);+ await Promise.all([+ db.sortedSetRemoveBulk(bulkRemove),+ db.sortedSetIncrByBulk(bulkIncr),+ ]);const localCount = postData.filter(p => utils.isNumber(p.pid)).length;const incrObjectBulk = [['global', { postCount: -localCount }]];diff --git a/src/posts/user.js b/src/posts/user.jsindex d096b0fa50..cdb9836bdb 100644--- a/src/posts/user.js+++ b/src/posts/user.js@@ -150,6 +150,7 @@ module.exports = function (Posts) {const bulkRemove = [];const bulkAdd = [];+ const bulkIncr = [];let repChange = 0;const postsByUser = {};postData.forEach((post, i) => {@@ -164,6 +165,11 @@ module.exports = function (Posts) {if (post.votes > 0 || post.votes < 0) {bulkAdd.push([`cid:${post.cid}:uid:${toUid}:pids:votes`, post.votes, post.pid]);}++ bulkIncr.push(+ [`uid:${post.uid}:cids`, -1, post.cid],+ [`uid:${toUid}:cids`, 1, post.cid],+ );postsByUser[post.uid] = postsByUser[post.uid] || [];postsByUser[post.uid].push(post);});@@ -172,6 +178,7 @@ module.exports = function (Posts) {db.setObjectField(pids.map(pid => `post:${pid}`), 'uid', toUid),db.sortedSetRemoveBulk(bulkRemove),db.sortedSetAddBulk(bulkAdd),+ db.sortedSetIncrByBulk(bulkIncr),user.incrementUserReputationBy(toUid, repChange),handleMainPidOwnerChange(postData, toUid),updateTopicPosters(postData, toUid),diff --git a/src/topics/fork.js b/src/topics/fork.jsindex fc8b030f63..22903d112e 100644--- a/src/topics/fork.js+++ b/src/topics/fork.js@@ -169,6 +169,10 @@ module.exports = function (Topics) {db.sortedSetRemove(removeFrom, postData.pid),db.sortedSetAdd(`cid:${topicData[1].cid}:pids`, postData.timestamp, postData.pid),db.sortedSetAdd(`cid:${topicData[1].cid}:uid:${postData.uid}:pids`, postData.timestamp, postData.pid),+ db.sortedSetIncrByBulk([+ [`uid:${postData.uid}:cids`, -1, topicData[0].cid],+ [`uid:${postData.uid}:cids`, 1, topicData[1].cid],+ ]),];if (postData.votes > 0 || postData.votes < 0) {tasks.push(db.sortedSetAdd(`cid:${topicData[1].cid}:uid:${postData.uid}:pids:votes`, postData.votes, postData.pid));diff --git a/src/upgrades/4.11.0/backfill-user-cids.js b/src/upgrades/4.11.0/backfill-user-cids.jsnew file mode 100644index 0000000000..58db2fc194--- /dev/null+++ b/src/upgrades/4.11.0/backfill-user-cids.js@@ -0,0 +1,38 @@+'use strict';++const db = require('../../database');+const posts = require('../../posts');+const batch = require('../../batch');++module.exports = {+ name: 'Backfill user posted categories',+ timestamp: Date.UTC(2026, 2, 20),+ method: async function () {+ const { progress } = this;+ await batch.processSortedSet('posts:pid', async (pids) => {+ const postData = await db.getObjectsFields(pids.map(pid => `post:${pid}`), ['uid']);+ const cids = await posts.getCidsByPids(pids);+ const uidPostCountByCid = Object.create(null);+ postData.forEach((post, idx) => {+ const cid = cids[idx];+ uidPostCountByCid[post.uid] = uidPostCountByCid[post.uid] || {};+ uidPostCountByCid[post.uid][cid] = (uidPostCountByCid[post.uid][cid] || 0) + 1;+ });+ const bulkIncr = [];+ Object.keys(uidPostCountByCid).forEach((uid) => {+ Object.keys(uidPostCountByCid[uid]).forEach((cid) => {+ bulkIncr.push([`uid:${uid}:cids`, uidPostCountByCid[uid][cid], cid]);+ });+ });++ if (bulkIncr.length) {+ await db.sortedSetIncrByBulk(bulkIncr);+ }++ progress.incr(pids.length);+ }, {+ batch: 500,+ progress,+ });+ },+};diff --git a/src/user/delete.js b/src/user/delete.jsindex 5c15c5c660..b13c197f7f 100644--- a/src/user/delete.js+++ b/src/user/delete.js@@ -126,6 +126,7 @@ module.exports = function (User) {`user:${uid}:usernames`,`user:${uid}:emails`,`uid:${uid}:topics`, `uid:${uid}:posts`,+ `uid:${uid}:cids`,`uid:${uid}:chats`, `uid:${uid}:chats:unread`,`uid:${uid}:chat:rooms`,`uid:${uid}:chat:rooms:unread`,diff --git a/src/user/posts.js b/src/user/posts.jsindex 30ddeaadfc..c8be6f81cd 100644--- a/src/user/posts.js+++ b/src/user/posts.js@@ -101,6 +101,7 @@ module.exports = function (User) {`uid:${postData.uid}:posts`,`cid:${postData.cid}:uid:${postData.uid}:pids`,], postData.timestamp, postData.pid);+ await db.sortedSetIncrBy(`uid:${postData.uid}:cids`, 1, postData.cid);await User.updatePostCount(postData.uid);};diff --git a/test/database/sorted.js b/test/database/sorted.jsindex fde2bafb0b..f1eca291bf 100644--- a/test/database/sorted.js+++ b/test/database/sorted.js@@ -2,6 +2,7 @@const assert = require('assert');const db = require('../mocks/databasemock');+const utils = require('../../src/utils');describe('Sorted Set methods', () => {before(async () => {@@ -1070,31 +1071,19 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea});});- describe('sortedSetIncrBy()', () => {- it('should create a sorted set with a field set to 1', (done) => {- db.sortedSetIncrBy('sortedIncr', 1, 'field1', function (err, newValue) {- assert.equal(err, null);- assert.equal(arguments.length, 2);- assert.strictEqual(newValue, 1);- db.sortedSetScore('sortedIncr', 'field1', (err, score) => {- assert.equal(err, null);- assert.strictEqual(score, 1);- done();- });- });+ describe('sortedSetIncrBy()/sortedSetIncrByBulk()', () => {+ it('should create a sorted set with a field set to 1', async () => {+ const newValue = await db.sortedSetIncrBy('sortedIncr', 1, 'field1');+ assert.strictEqual(newValue, 1);+ const score = await db.sortedSetScore('sortedIncr', 'field1');+ assert.strictEqual(score, 1);});- it('should increment a field of a sorted set by 5', (done) => {- db.sortedSetIncrBy('sortedIncr', 5, 'field1', function (err, newValue) {- assert.equal(err, null);- assert.equal(arguments.length, 2);- assert.strictEqual(newValue, 6);- db.sortedSetScore('sortedIncr', 'field1', (err, score) => {- assert.equal(err, null);- assert.strictEqual(score, 6);- done();- });- });+ it('should increment a field of a sorted set by 5', async () => {+ const newValue = await db.sortedSetIncrBy('sortedIncr', 5, 'field1');+ assert.strictEqual(newValue, 6);+ const score = await db.sortedSetScore('sortedIncr', 'field1');+ assert.strictEqual(score, 6);});it('should increment fields of sorted sets with a single call', async () => {@@ -1122,12 +1111,27 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea);});+ it('should increment the same zset twice', async () => {+ const zset = utils.generateUUID();+ const value1 = utils.generateUUID();+ const value2 = utils.generateUUID();+ await db.sortedSetIncrByBulk([+ [zset, 1, value1],+ [zset, 1, value2],+ ]);+ const scores = await Promise.all([+ db.sortedSetScore(zset, value1),+ db.sortedSetScore(zset, value2),+ ]);+ assert.deepStrictEqual(scores, [1, 1]);+ });+it('should increment the same field', async () => {- const data1 = await db.sortedSetIncrByBulk([+ await db.sortedSetIncrByBulk([['sortedIncrBulk5', 5, 'value5'],]);- const data2 = await db.sortedSetIncrByBulk([+ await db.sortedSetIncrByBulk([['sortedIncrBulk5', 5, 'value5'],]);assert.deepStrictEqual(@@ -1137,6 +1141,41 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea],);});++ it('should return empty array', async function () {+ const zset = utils.generateUUID();+ const response = await db.sortedSetIncrByBulk(zset, []);+ assert(Array.isArray(response));+ assert.strictEqual(response.length, 0);+ });++ it('should aggregate increments to the same key/value pair', async function () {+ const zset = utils.generateUUID();+ await db.sortedSetIncrByBulk([+ [zset, 1, 'baz'],+ [zset, 1, 'baz'],+ [zset, 7, 'baz'],+ [zset, 1, 'foo'],+ [zset, 3, 'foo'],+ [zset, 4, 'foo'],+ [zset, 2, 'fizz'],+ [zset, 1, 'fizz'],+ [zset, -3, 'fizz'],+ ]);+ const score = await db.sortedSetScores(zset, ['foo', 'baz', 'fizz']);+ assert.deepStrictEqual(score, [8, 9, 0]);+ });++ it('should handle parallel increments with same key/value pairs', async function () {+ const zset = utils.generateUUID();+ await Promise.all([+ db.sortedSetIncrByBulk([[zset, 1, 'baz']]),+ db.sortedSetIncrByBulk([[zset, 1, 'baz']]),+ db.sortedSetIncrByBulk([[zset, 1, 'baz']]),+ ]);+ const score = await db.sortedSetScore(zset, 'baz');+ assert.deepStrictEqual(score, 3);+ });});diff --git a/test/topics.js b/test/topics.jsindex f67e0fc40e..767af84901 100644--- a/test/topics.js+++ b/test/topics.js@@ -1425,10 +1425,13 @@ describe('Topic\'s', () => {const unreadTids = await topics.getUnreadTids({ cid: 0, uid: uid });await sleep(2000);- const _unreadTids = await topics.getUnreadTids({ cid: 0, uid: uid });+ const [_unreadTids, topicData] = await Promise.all([+ topics.getUnreadTids({ cid: 0, uid: uid }),+ topics.getTopicData(result.topicData.tid),+ ]);assert(!unreadTids.includes(result.topicData.tid),- JSON.stringify({ unreadTids, _unreadTids, tid: result.topicData.tid })+ JSON.stringify({ unreadTids, _unreadTids, topic: topicData }));});});