Node.js based forum software built for the modern web
Node redis (#13500)
Barış Uşaklı14043ab0fd6e
diff --git a/install/package.json b/install/package.jsonindex d817989349..a42eb52182 100644--- a/install/package.json+++ b/install/package.json@@ -98,7 +98,7 @@"nconf": "0.13.0","nodebb-plugin-2factor": "7.5.10","nodebb-plugin-composer-default": "10.2.51",- "nodebb-plugin-dbsearch": "6.2.19",+ "nodebb-plugin-dbsearch": "6.3.0","nodebb-plugin-emoji": "6.0.3","nodebb-plugin-emoji-android": "4.1.1","nodebb-plugin-markdown": "13.2.1",@@ -122,7 +122,7 @@"postcss-clean": "1.2.0","progress-webpack-plugin": "1.0.16","prompt": "1.3.0",- "ioredis": "5.6.1",+ "redis": "5.5.6","rimraf": "6.0.1","rss": "1.2.2","rtlcss": "4.3.0",diff --git a/src/activitypub/index.js b/src/activitypub/index.jsindex 80ec4a40f5..78cb3f26f6 100644--- a/src/activitypub/index.js+++ b/src/activitypub/index.js@@ -132,7 +132,6 @@ ActivityPub.resolveInboxes = async (ids) => {}, [[], []]);const categoryData = await categories.getCategoriesFields(cids, ['inbox', 'sharedInbox']);const userData = await user.getUsersFields(uids, ['inbox', 'sharedInbox']);-currentIds.forEach((id) => {if (cids.includes(id)) {const data = categoryData[cids.indexOf(id)];diff --git a/src/controllers/activitypub/index.js b/src/controllers/activitypub/index.jsindex de478a6021..6751cb30cd 100644--- a/src/controllers/activitypub/index.js+++ b/src/controllers/activitypub/index.js@@ -145,14 +145,15 @@ Controller.postInbox = async (req, res) => {const method = String(req.body.type).toLowerCase();if (!activitypub.inbox.hasOwnProperty(method)) {winston.warn(`[activitypub/inbox] Received Activity of type ${method} but unable to handle. Ignoring.`);+ console.log('[activitypub/inbox] method not found', method, req.body);return res.sendStatus(200);}try {await activitypub.inbox[method](req);await activitypub.record(req.body);- helpers.formatApiResponse(202, res);+ await helpers.formatApiResponse(202, res);} catch (e) {- helpers.formatApiResponse(500, res, e);+ helpers.formatApiResponse(500, res, e).catch(err => winston.error(err.stack));}};diff --git a/src/controllers/helpers.js b/src/controllers/helpers.jsindex a6ade8c73b..e11692867e 100644--- a/src/controllers/helpers.js+++ b/src/controllers/helpers.js@@ -448,6 +448,10 @@ helpers.getHomePageRoutes = async function (uid) {};helpers.formatApiResponse = async (statusCode, res, payload) => {+ if (!res.hasOwnProperty('req')) {+ console.log('formatApiResponse', statusCode, payload);+ }+if (res.req.method === 'HEAD') {return res.sendStatus(statusCode);}diff --git a/src/database/redis.js b/src/database/redis.jsindex f73ee79313..472dae8de6 100644--- a/src/database/redis.js+++ b/src/database/redis.js@@ -61,7 +61,7 @@ redisModule.checkCompatibilityVersion = function (version, callback) {};redisModule.close = async function () {- await redisModule.client.quit();+ await redisModule.client.close();if (redisModule.objectCache) {redisModule.objectCache.reset();}diff --git a/src/database/redis/connection.js b/src/database/redis/connection.jsindex a4ba757ef6..2a38cf1a79 100644--- a/src/database/redis/connection.js+++ b/src/database/redis/connection.js@@ -1,7 +1,7 @@'use strict';const nconf = require('nconf');-const Redis = require('ioredis');+const { createClient, createCluster, createSentinel } = require('redis');const winston = require('winston');const connection = module.exports;@@ -13,28 +13,40 @@ connection.connect = async function (options) {let cxn;if (options.cluster) {- cxn = new Redis.Cluster(options.cluster, options.options);+ const rootNodes = options.cluster.map(node => ({ url : `redis://${node.host}:${node.port}` }));+ cxn = createCluster({+ ...options.options,+ rootNodes: rootNodes,+ });} else if (options.sentinels) {- cxn = new Redis({- sentinels: options.sentinels,+ const sentinelRootNodes = options.sentinels.map(sentinel => ({ host: sentinel.host, port: sentinel.port }));+ cxn = createSentinel({...options.options,+ name: 'sentinel-db',+ sentinelRootNodes,});} else if (redis_socket_or_host && String(redis_socket_or_host).indexOf('/') >= 0) {// If redis.host contains a path name character, use the unix dom sock connection. ie, /tmp/redis.sock- cxn = new Redis({+ cxn = createClient({...options.options,- path: redis_socket_or_host,password: options.password,- db: options.database,+ database: options.database,+ socket: {+ path: redis_socket_or_host,+ reconnectStrategy: 3000,+ },});} else {// Else, connect over tcp/ip- cxn = new Redis({+ cxn = createClient({...options.options,host: redis_socket_or_host,port: options.port,password: options.password,- db: options.database,+ database: options.database,+ socket: {+ reconnectStrategy: 3000,+ },});}@@ -49,9 +61,14 @@ connection.connect = async function (options) {});cxn.on('ready', () => {// back-compat with node_redis- cxn.batch = cxn.pipeline;+ cxn.batch = cxn.multi;resolve(cxn);});+ cxn.connect().then(() => {+ winston.info('Connected to Redis successfully');+ }).catch((err) => {+ winston.error('Error connecting to Redis:', err);+ });if (options.password) {cxn.auth(options.password);diff --git a/src/database/redis/hash.js b/src/database/redis/hash.jsindex 4c6e7b374f..c03bff055b 100644--- a/src/database/redis/hash.js+++ b/src/database/redis/hash.js@@ -25,12 +25,13 @@ module.exports = function (module) {if (!Object.keys(data).length) {return;}+ const strObj = helpers.objectFieldsToString(data);if (Array.isArray(key)) {const batch = module.client.batch();- key.forEach(k => batch.hmset(k, data));+ key.forEach(k => batch.hSet(k, strObj));await helpers.execBatch(batch);} else {- await module.client.hmset(key, data);+ await module.client.hSet(key, strObj);}cache.del(key);@@ -49,10 +50,16 @@ module.exports = function (module) {const batch = module.client.batch();data.forEach((item) => {+ Object.keys(item[1]).forEach((key) => {+ if (item[1][key] === undefined || item[1][key] === null) {+ delete item[1][key];+ }+ });if (Object.keys(item[1]).length) {- batch.hmset(item[0], item[1]);+ batch.hSet(item[0], helpers.objectFieldsToString(item[1]));}});+await helpers.execBatch(batch);cache.del(data.map(item => item[0]));};@@ -61,12 +68,15 @@ module.exports = function (module) {if (!field) {return;}+ if (value === null || value === undefined) {+ return;+ }if (Array.isArray(key)) {const batch = module.client.batch();- key.forEach(k => batch.hset(k, field, value));+ key.forEach(k => batch.hSet(k, field, String(value)));await helpers.execBatch(batch);} else {- await module.client.hset(key, field, value);+ await module.client.hSet(key, field, String(value));}cache.del(key);@@ -92,9 +102,9 @@ module.exports = function (module) {const cachedData = {};cache.getUnCachedKeys([key], cachedData);if (cachedData[key]) {- return cachedData[key].hasOwnProperty(field) ? cachedData[key][field] : null;+ return Object.hasOwn(cachedData[key], field) ? cachedData[key][field] : null;}- return await module.client.hget(key, String(field));+ return await module.client.hGet(key, String(field));};module.getObjectFields = async function (key, fields) {@@ -116,10 +126,10 @@ module.exports = function (module) {let data = [];if (unCachedKeys.length > 1) {const batch = module.client.batch();- unCachedKeys.forEach(k => batch.hgetall(k));+ unCachedKeys.forEach(k => batch.hGetAll(k));data = await helpers.execBatch(batch);} else if (unCachedKeys.length === 1) {- data = [await module.client.hgetall(unCachedKeys[0])];+ data = [await module.client.hGetAll(unCachedKeys[0])];}// convert empty objects into null for back-compat with node_redis@@ -149,21 +159,21 @@ module.exports = function (module) {};module.getObjectKeys = async function (key) {- return await module.client.hkeys(key);+ return await module.client.hKeys(key);};module.getObjectValues = async function (key) {- return await module.client.hvals(key);+ return await module.client.hVals(key);};module.isObjectField = async function (key, field) {- const exists = await module.client.hexists(key, field);+ const exists = await module.client.hExists(key, String(field));return exists === 1;};module.isObjectFields = async function (key, fields) {const batch = module.client.batch();- fields.forEach(f => batch.hexists(String(key), String(f)));+ fields.forEach(f => batch.hExists(String(key), String(f)));const results = await helpers.execBatch(batch);return Array.isArray(results) ? helpers.resultsToBool(results) : null;};@@ -174,7 +184,7 @@ module.exports = function (module) {}field = field.toString();if (field) {- await module.client.hdel(key, field);+ await module.client.hDel(key, field);cache.del(key);}};@@ -189,10 +199,10 @@ module.exports = function (module) {}if (Array.isArray(key)) {const batch = module.client.batch();- key.forEach(k => batch.hdel(k, fields));+ key.forEach(k => batch.hDel(k, fields));await helpers.execBatch(batch);} else {- await module.client.hdel(key, fields);+ await module.client.hDel(key, fields);}cache.del(key);@@ -214,10 +224,10 @@ module.exports = function (module) {let result;if (Array.isArray(key)) {const batch = module.client.batch();- key.forEach(k => batch.hincrby(k, field, value));+ key.forEach(k => batch.hIncrBy(k, field, value));result = await helpers.execBatch(batch);} else {- result = await module.client.hincrby(key, field, value);+ result = await module.client.hIncrBy(key, field, value);}cache.del(key);return Array.isArray(result) ? result.map(value => parseInt(value, 10)) : parseInt(result, 10);@@ -231,7 +241,7 @@ module.exports = function (module) {const batch = module.client.batch();data.forEach((item) => {for (const [field, value] of Object.entries(item[1])) {- batch.hincrby(item[0], field, value);+ batch.hIncrBy(item[0], field, value);}});await helpers.execBatch(batch);diff --git a/src/database/redis/helpers.js b/src/database/redis/helpers.jsindex 8961da8255..39585b1f88 100644--- a/src/database/redis/helpers.js+++ b/src/database/redis/helpers.js@@ -5,13 +5,8 @@ const helpers = module.exports;helpers.noop = function () {};helpers.execBatch = async function (batch) {- const results = await batch.exec();- return results.map(([err, res]) => {- if (err) {- throw err;- }- return res;- });+ const results = await batch.execAsPipeline();+ return results;};helpers.resultsToBool = function (results) {@@ -21,10 +16,29 @@ helpers.resultsToBool = function (results) {return results;};-helpers.zsetToObjectArray = function (data) {- const objects = new Array(data.length / 2);- for (let i = 0, k = 0; i < objects.length; i += 1, k += 2) {- objects[i] = { value: data[k], score: parseFloat(data[k + 1]) };+helpers.objectFieldsToString = function (obj) {+ const stringified = Object.fromEntries(+ Object.entries(obj).map(([key, value]) => [key, String(value)])+ );+ return stringified;+};++helpers.normalizeLexRange = function (min, max, reverse) {+ let minmin;+ let maxmax;+ if (reverse) {+ minmin = '+';+ maxmax = '-';+ } else {+ minmin = '-';+ maxmax = '+';+ }++ if (min !== minmin && !min.match(/^[[(]/)) {+ min = `[${min}`;+ }+ if (max !== maxmax && !max.match(/^[[(]/)) {+ max = `[${max}`;}- return objects;+ return { lmin: min, lmax: max };};diff --git a/src/database/redis/list.js b/src/database/redis/list.jsindex 101ef178e3..229069b6ed 100644--- a/src/database/redis/list.js+++ b/src/database/redis/list.js@@ -1,27 +1,25 @@'use strict';module.exports = function (module) {- const helpers = require('./helpers');-module.listPrepend = async function (key, value) {if (!key) {return;}- await module.client.lpush(key, value);+ await module.client.lPush(key, Array.isArray(value) ? value.map(String) : String(value));};module.listAppend = async function (key, value) {if (!key) {return;}- await module.client.rpush(key, value);+ await module.client.rPush(key, Array.isArray(value) ? value.map(String) : String(value));};module.listRemoveLast = async function (key) {if (!key) {return;}- return await module.client.rpop(key);+ return await module.client.rPop(key);};module.listRemoveAll = async function (key, value) {@@ -29,11 +27,11 @@ module.exports = function (module) {return;}if (Array.isArray(value)) {- const batch = module.client.batch();- value.forEach(value => batch.lrem(key, 0, value));- await helpers.execBatch(batch);+ const batch = module.client.multi();+ value.forEach(value => batch.lRem(key, 0, value));+ await batch.execAsPipeline();} else {- await module.client.lrem(key, 0, value);+ await module.client.lRem(key, 0, value);}};@@ -41,17 +39,17 @@ module.exports = function (module) {if (!key) {return;}- await module.client.ltrim(key, start, stop);+ await module.client.lTrim(key, start, stop);};module.getListRange = async function (key, start, stop) {if (!key) {return;}- return await module.client.lrange(key, start, stop);+ return await module.client.lRange(key, start, stop);};module.listLength = async function (key) {- return await module.client.llen(key);+ return await module.client.lLen(key);};};diff --git a/src/database/redis/main.js b/src/database/redis/main.jsindex b849361a8e..a6506ba701 100644--- a/src/database/redis/main.js+++ b/src/database/redis/main.js@@ -4,7 +4,7 @@ module.exports = function (module) {const helpers = require('./helpers');module.flushdb = async function () {- await module.client.send_command('flushdb', []);+ await module.client.sendCommand(['FLUSHDB']);};module.emptydb = async function () {@@ -32,9 +32,9 @@ module.exports = function (module) {const seen = Object.create(null);do {/* eslint-disable no-await-in-loop */- const res = await module.client.scan(cursor, 'MATCH', params.match, 'COUNT', 10000);- cursor = res[0];- const values = res[1].filter((value) => {+ const res = await module.client.scan(cursor, { MATCH: params.match, COUNT: 10000 });+ cursor = res.cursor;+ const values = res.keys.filter((value) => {const isSeen = !!seen[value];if (!isSeen) {seen[value] = 1;@@ -67,7 +67,7 @@ module.exports = function (module) {if (!keys || !Array.isArray(keys) || !keys.length) {return [];}- return await module.client.mget(keys);+ return await module.client.mGet(keys);};module.set = async function (key, value) {@@ -96,26 +96,26 @@ module.exports = function (module) {};module.expire = async function (key, seconds) {- await module.client.expire(key, seconds);+ await module.client.EXPIRE(key, seconds);};module.expireAt = async function (key, timestamp) {- await module.client.expireat(key, timestamp);+ await module.client.EXPIREAT(key, timestamp);};module.pexpire = async function (key, ms) {- await module.client.pexpire(key, ms);+ await module.client.PEXPIRE(key, ms);};module.pexpireAt = async function (key, timestamp) {- await module.client.pexpireat(key, timestamp);+ await module.client.PEXPIREAT(key, timestamp);};module.ttl = async function (key) {- return await module.client.ttl(key);+ return await module.client.TTL(key);};module.pttl = async function (key) {- return await module.client.pttl(key);+ return await module.client.PTTL(key);};};diff --git a/src/database/redis/sets.js b/src/database/redis/sets.jsindex b2b390598b..dd7e484325 100644--- a/src/database/redis/sets.js+++ b/src/database/redis/sets.js@@ -10,7 +10,7 @@ module.exports = function (module) {if (!value.length) {return;}- await module.client.sadd(key, value);+ await module.client.sAdd(key, value.map(String));};module.setsAdd = async function (keys, value) {@@ -18,7 +18,7 @@ module.exports = function (module) {return;}const batch = module.client.batch();- keys.forEach(k => batch.sadd(String(k), String(value)));+ keys.forEach(k => batch.sAdd(String(k), String(value)));await helpers.execBatch(batch);};@@ -34,57 +34,57 @@ module.exports = function (module) {}const batch = module.client.batch();- key.forEach(k => batch.srem(String(k), value));+ key.forEach(k => batch.sRem(String(k), value.map(String)));await helpers.execBatch(batch);};module.setsRemove = async function (keys, value) {const batch = module.client.batch();- keys.forEach(k => batch.srem(String(k), value));+ keys.forEach(k => batch.sRem(String(k), String(value)));await helpers.execBatch(batch);};module.isSetMember = async function (key, value) {- const result = await module.client.sismember(key, value);+ const result = await module.client.sIsMember(key, String(value));return result === 1;};module.isSetMembers = async function (key, values) {const batch = module.client.batch();- values.forEach(v => batch.sismember(String(key), String(v)));+ values.forEach(v => batch.sIsMember(String(key), String(v)));const results = await helpers.execBatch(batch);return results ? helpers.resultsToBool(results) : null;};module.isMemberOfSets = async function (sets, value) {const batch = module.client.batch();- sets.forEach(s => batch.sismember(String(s), String(value)));+ sets.forEach(s => batch.sIsMember(String(s), String(value)));const results = await helpers.execBatch(batch);return results ? helpers.resultsToBool(results) : null;};module.getSetMembers = async function (key) {- return await module.client.smembers(key);+ return await module.client.sMembers(key);};module.getSetsMembers = async function (keys) {const batch = module.client.batch();- keys.forEach(k => batch.smembers(String(k)));+ keys.forEach(k => batch.sMembers(String(k)));return await helpers.execBatch(batch);};module.setCount = async function (key) {- return await module.client.scard(key);+ return await module.client.sCard(key);};module.setsCount = async function (keys) {const batch = module.client.batch();- keys.forEach(k => batch.scard(String(k)));+ keys.forEach(k => batch.sCard(String(k)));return await helpers.execBatch(batch);};module.setRemoveRandom = async function (key) {- return await module.client.spop(key);+ return await module.client.sPop(key);};return module;diff --git a/src/database/redis/sorted.js b/src/database/redis/sorted.jsindex 013477da5a..d8613a3a68 100644--- a/src/database/redis/sorted.js+++ b/src/database/redis/sorted.js@@ -11,34 +11,74 @@ module.exports = function (module) {require('./sorted/intersect')(module);module.getSortedSetRange = async function (key, start, stop) {- return await sortedSetRange('zrange', key, start, stop, '-inf', '+inf', false);+ return await sortedSetRange(key, start, stop, '-inf', '+inf', false, false, false);};module.getSortedSetRevRange = async function (key, start, stop) {- return await sortedSetRange('zrevrange', key, start, stop, '-inf', '+inf', false);+ return await sortedSetRange(key, start, stop, '-inf', '+inf', false, true, false);};module.getSortedSetRangeWithScores = async function (key, start, stop) {- return await sortedSetRange('zrange', key, start, stop, '-inf', '+inf', true);+ return await sortedSetRange(key, start, stop, '-inf', '+inf', true, false, false);};module.getSortedSetRevRangeWithScores = async function (key, start, stop) {- return await sortedSetRange('zrevrange', key, start, stop, '-inf', '+inf', true);+ return await sortedSetRange(key, start, stop, '-inf', '+inf', true, true, false);};- async function sortedSetRange(method, key, start, stop, min, max, withScores) {+ module.getSortedSetRangeByScore = async function (key, start, count, min, max) {+ return await sortedSetRangeByScore(key, start, count, min, max, false, false);+ };++ module.getSortedSetRevRangeByScore = async function (key, start, count, max, min) {+ return await sortedSetRangeByScore(key, start, count, max, min, false, true);+ };++ module.getSortedSetRangeByScoreWithScores = async function (key, start, count, min, max) {+ return await sortedSetRangeByScore(key, start, count, min, max, true, false);+ };++ module.getSortedSetRevRangeByScoreWithScores = async function (key, start, count, max, min) {+ return await sortedSetRangeByScore(key, start, count, max, min, true, true);+ };++ async function sortedSetRangeByScore(key, start, count, min, max, withScores, rev) {+ if (parseInt(count, 10) === 0) {+ return [];+ }+ const stop = (parseInt(count, 10) === -1) ? -1 : (start + count - 1);+ return await sortedSetRange(key, start, stop, min, max, withScores, rev, true);+ }++ async function sortedSetRange(key, start, stop, min, max, withScores, rev, byScore) {+ const opts = {};+ const cmd = withScores ? 'zRangeWithScores' : 'zRange';+ if (byScore) {+ opts.BY = 'SCORE';+ opts.LIMIT = { offset: start, count: stop !== -1 ? stop + 1 : stop };+ }+ if (rev) {+ opts.REV = true;+ }+if (Array.isArray(key)) {if (!key.length) {return [];}const batch = module.client.batch();- key.forEach(key => batch[method](genParams(method, key, 0, stop, min, max, true)));- const data = await helpers.execBatch(batch);- const batchData = data.map(setData => helpers.zsetToObjectArray(setData));-- let objects = dbHelpers.mergeBatch(batchData, 0, stop, method === 'zrange' ? 1 : -1);+ if (byScore) {+ key.forEach(key => batch.zRangeWithScores(key, min, max, {+ ...opts,+ LIMIT: { offset: 0, count: stop !== -1 ? stop + 1 : stop },+ }));+ } else {+ key.forEach(key => batch.zRangeWithScores(key, 0, stop, { ...opts }));+ }+ const data = await helpers.execBatch(batch);+ const batchData = data;+ let objects = dbHelpers.mergeBatch(batchData, 0, stop, rev ? -1 : 1);if (start > 0) {objects = objects.slice(start, stop !== -1 ? stop + 1 : undefined);}@@ -48,63 +88,25 @@ module.exports = function (module) {return objects;}- const params = genParams(method, key, start, stop, min, max, withScores);- const data = await module.client[method](params);- if (!withScores) {- return data;- }- const objects = helpers.zsetToObjectArray(data);- return objects;- }-- function genParams(method, key, start, stop, min, max, withScores) {- const params = {- zrevrange: [key, start, stop],- zrange: [key, start, stop],- zrangebyscore: [key, min, max],- zrevrangebyscore: [key, max, min],- };- if (withScores) {- params[method].push('WITHSCORES');- }-- if (method === 'zrangebyscore' || method === 'zrevrangebyscore') {- const count = stop !== -1 ? stop - start + 1 : stop;- params[method].push('LIMIT', start, count);+ let data;+ if (byScore) {+ data = await module.client[cmd](key, min, max, opts);+ } else {+ data = await module.client[cmd](key, start, stop, opts);}- return params[method];- }-- module.getSortedSetRangeByScore = async function (key, start, count, min, max) {- return await sortedSetRangeByScore('zrangebyscore', key, start, count, min, max, false);- };-- module.getSortedSetRevRangeByScore = async function (key, start, count, max, min) {- return await sortedSetRangeByScore('zrevrangebyscore', key, start, count, min, max, false);- };- module.getSortedSetRangeByScoreWithScores = async function (key, start, count, min, max) {- return await sortedSetRangeByScore('zrangebyscore', key, start, count, min, max, true);- };-- module.getSortedSetRevRangeByScoreWithScores = async function (key, start, count, max, min) {- return await sortedSetRangeByScore('zrevrangebyscore', key, start, count, min, max, true);- };-- async function sortedSetRangeByScore(method, key, start, count, min, max, withScores) {- if (parseInt(count, 10) === 0) {- return [];+ if (!withScores) {+ return data;}- const stop = (parseInt(count, 10) === -1) ? -1 : (start + count - 1);- return await sortedSetRange(method, key, start, stop, min, max, withScores);+ return data;}module.sortedSetCount = async function (key, min, max) {- return await module.client.zcount(key, min, max);+ return await module.client.zCount(key, min, max);};module.sortedSetCard = async function (key) {- return await module.client.zcard(key);+ return await module.client.zCard(key);};module.sortedSetsCard = async function (keys) {@@ -112,7 +114,7 @@ module.exports = function (module) {return [];}const batch = module.client.batch();- keys.forEach(k => batch.zcard(String(k)));+ keys.forEach(k => batch.zCard(String(k)));return await helpers.execBatch(batch);};@@ -125,26 +127,26 @@ module.exports = function (module) {}const batch = module.client.batch();if (min !== '-inf' || max !== '+inf') {- keys.forEach(k => batch.zcount(String(k), min, max));+ keys.forEach(k => batch.zCount(String(k), min, max));} else {- keys.forEach(k => batch.zcard(String(k)));+ keys.forEach(k => batch.zCard(String(k)));}const counts = await helpers.execBatch(batch);return counts.reduce((acc, val) => acc + val, 0);};module.sortedSetRank = async function (key, value) {- return await module.client.zrank(key, value);+ return await module.client.zRank(key, String(value));};module.sortedSetRevRank = async function (key, value) {- return await module.client.zrevrank(key, value);+ return await module.client.zRevRank(key, String(value));};module.sortedSetsRanks = async function (keys, values) {const batch = module.client.batch();for (let i = 0; i < values.length; i += 1) {- batch.zrank(keys[i], String(values[i]));+ batch.zRank(keys[i], String(values[i]));}return await helpers.execBatch(batch);};@@ -152,7 +154,7 @@ module.exports = function (module) {module.sortedSetsRevRanks = async function (keys, values) {const batch = module.client.batch();for (let i = 0; i < values.length; i += 1) {- batch.zrevrank(keys[i], String(values[i]));+ batch.zRevRank(keys[i], String(values[i]));}return await helpers.execBatch(batch);};@@ -160,7 +162,7 @@ module.exports = function (module) {module.sortedSetRanks = async function (key, values) {const batch = module.client.batch();for (let i = 0; i < values.length; i += 1) {- batch.zrank(key, String(values[i]));+ batch.zRank(key, String(values[i]));}return await helpers.execBatch(batch);};@@ -168,7 +170,7 @@ module.exports = function (module) {module.sortedSetRevRanks = async function (key, values) {const batch = module.client.batch();for (let i = 0; i < values.length; i += 1) {- batch.zrevrank(key, String(values[i]));+ batch.zRevRank(key, String(values[i]));}return await helpers.execBatch(batch);};@@ -177,8 +179,7 @@ module.exports = function (module) {if (!key || value === undefined) {return null;}-- const score = await module.client.zscore(key, value);+ const score = await module.client.zScore(key, String(value));return score === null ? score : parseFloat(score);};@@ -187,7 +188,7 @@ module.exports = function (module) {return [];}const batch = module.client.batch();- keys.forEach(key => batch.zscore(String(key), String(value)));+ keys.forEach(key => batch.zScore(String(key), String(value)));const scores = await helpers.execBatch(batch);return scores.map(d => (d === null ? d : parseFloat(d)));};@@ -197,7 +198,7 @@ module.exports = function (module) {return [];}const batch = module.client.batch();- values.forEach(value => batch.zscore(String(key), String(value)));+ values.forEach(value => batch.zScore(String(key), String(value)));const scores = await helpers.execBatch(batch);return scores.map(d => (d === null ? d : parseFloat(d)));};@@ -211,9 +212,9 @@ module.exports = function (module) {if (!values.length) {return [];}- const batch = module.client.batch();- values.forEach(v => batch.zscore(key, String(v)));- const results = await helpers.execBatch(batch);+ const batch = module.client.multi();+ values.forEach(v => batch.zScore(key, String(v)));+ const results = await batch.execAsPipeline();return results.map(utils.isNumber);};@@ -221,20 +222,18 @@ module.exports = function (module) {if (!Array.isArray(keys) || !keys.length) {return [];}- const batch = module.client.batch();- keys.forEach(k => batch.zscore(k, String(value)));- const results = await helpers.execBatch(batch);+ const batch = module.client.multi();+ keys.forEach(k => batch.zScore(k, String(value)));+ const results = await batch.execAsPipeline();return results.map(utils.isNumber);};module.getSortedSetMembers = async function (key) {- return await module.client.zrange(key, 0, -1);+ return await module.client.zRange(key, 0, -1);};module.getSortedSetMembersWithScores = async function (key) {- return helpers.zsetToObjectArray(- await module.client.zrange(key, 0, -1, 'WITHSCORES')- );+ return await module.client.zRangeWithScores(key, 0, -1);};module.getSortedSetsMembers = async function (keys) {@@ -242,7 +241,7 @@ module.exports = function (module) {return [];}const batch = module.client.batch();- keys.forEach(k => batch.zrange(k, 0, -1));+ keys.forEach(k => batch.zRange(k, 0, -1));return await helpers.execBatch(batch);};@@ -251,65 +250,52 @@ module.exports = function (module) {return [];}const batch = module.client.batch();- keys.forEach(k => batch.zrange(k, 0, -1, 'WITHSCORES'));+ keys.forEach(k => batch.zRangeWithScores(k, 0, -1));const res = await helpers.execBatch(batch);- return res.map(helpers.zsetToObjectArray);+ return res;};module.sortedSetIncrBy = async function (key, increment, value) {- const newValue = await module.client.zincrby(key, increment, value);+ const newValue = await module.client.zIncrBy(key, increment, String(value));return parseFloat(newValue);};module.sortedSetIncrByBulk = async function (data) {const multi = module.client.multi();data.forEach((item) => {- multi.zincrby(item[0], item[1], item[2]);+ multi.zIncrBy(item[0], item[1], String(item[2]));});const result = await multi.exec();- return result.map(item => item && parseFloat(item[1]));+ return result;};- module.getSortedSetRangeByLex = async function (key, min, max, start, count) {- return await sortedSetLex('zrangebylex', false, key, min, max, start, count);+ module.getSortedSetRangeByLex = async function (key, min, max, start = 0, count = -1) {+ const { lmin, lmax } = helpers.normalizeLexRange(min, max, false);+ return await module.client.zRange(key, lmin, lmax, {+ BY: 'LEX',+ LIMIT: { offset: start, count: count },+ });};- module.getSortedSetRevRangeByLex = async function (key, max, min, start, count) {- return await sortedSetLex('zrevrangebylex', true, key, max, min, start, count);+ module.getSortedSetRevRangeByLex = async function (key, max, min, start = 0, count = -1) {+ const { lmin, lmax } = helpers.normalizeLexRange(max, min, true);+ return await module.client.zRange(key, lmin, lmax, {+ REV: true,+ BY: 'LEX',+ LIMIT: { offset: start, count: count },+ });};module.sortedSetRemoveRangeByLex = async function (key, min, max) {- await sortedSetLex('zremrangebylex', false, key, min, max);+ const { lmin, lmax } = helpers.normalizeLexRange(min, max, false);+ await module.client.zRemRangeByLex(key, lmin, lmax);};module.sortedSetLexCount = async function (key, min, max) {- return await sortedSetLex('zlexcount', false, key, min, max);+ const { lmin, lmax } = helpers.normalizeLexRange(min, max, false);+ return await module.client.zLexCount(key, lmin, lmax);};- async function sortedSetLex(method, reverse, key, min, max, start, count) {- let minmin;- let maxmax;- if (reverse) {- minmin = '+';- maxmax = '-';- } else {- minmin = '-';- maxmax = '+';- }-- if (min !== minmin && !min.match(/^[[(]/)) {- min = `[${min}`;- }- if (max !== maxmax && !max.match(/^[[(]/)) {- max = `[${max}`;- }- const args = [key, min, max];- if (count) {- args.push('LIMIT', start, count);- }- return await module.client[method](args);- }-module.getSortedSetScan = async function (params) {let cursor = '0';@@ -318,20 +304,19 @@ module.exports = function (module) {const seen = Object.create(null);do {/* eslint-disable no-await-in-loop */- const res = await module.client.zscan(params.key, cursor, 'MATCH', params.match, 'COUNT', 5000);- cursor = res[0];+ const res = await module.client.zScan(params.key, cursor, { MATCH: params.match, COUNT: 5000 });+ cursor = res.cursor;done = cursor === '0';- const data = res[1];- for (let i = 0; i < data.length; i += 2) {- const value = data[i];- if (!seen[value]) {- seen[value] = 1;+ for (let i = 0; i < res.members.length; i ++) {+ const item = res.members[i];+ if (!seen[item.value]) {+ seen[item.value] = 1;if (params.withScores) {- returnData.push({ value: value, score: parseFloat(data[i + 1]) });+ returnData.push({ value: item.value, score: parseFloat(item.score) });} else {- returnData.push(value);+ returnData.push(item.value);}if (params.limit && returnData.length >= params.limit) {done = true;diff --git a/src/database/redis/sorted/add.js b/src/database/redis/sorted/add.jsindex 660618b8a4..264c877845 100644--- a/src/database/redis/sorted/add.js+++ b/src/database/redis/sorted/add.js@@ -1,7 +1,6 @@'use strict';module.exports = function (module) {- const helpers = require('../helpers');const utils = require('../../../utils');module.sortedSetAdd = async function (key, score, value) {@@ -14,7 +13,8 @@ module.exports = function (module) {if (!utils.isNumber(score)) {throw new Error(`[[error:invalid-score, ${score}]]`);}- await module.client.zadd(key, score, String(value));++ await module.client.zAdd(key, { score, value: String(value) });};async function sortedSetAddMulti(key, scores, values) {@@ -30,11 +30,8 @@ module.exports = function (module) {throw new Error(`[[error:invalid-score, ${scores[i]}]]`);}}- const args = [key];- for (let i = 0; i < scores.length; i += 1) {- args.push(scores[i], String(values[i]));- }- await module.client.zadd(args);+ const members = scores.map((score, i) => ({ score, value: String(values[i])}));+ await module.client.zAdd(key, members);}module.sortedSetsAdd = async function (keys, scores, value) {@@ -51,13 +48,16 @@ module.exports = function (module) {throw new Error('[[error:invalid-data]]');}- const batch = module.client.batch();+ const batch = module.client.multi();for (let i = 0; i < keys.length; i += 1) {if (keys[i]) {- batch.zadd(keys[i], isArrayOfScores ? scores[i] : scores, String(value));+ batch.zAdd(keys[i], {+ score: isArrayOfScores ? scores[i] : scores,+ value: String(value),+ });}}- await helpers.execBatch(batch);+ await batch.execAsPipeline();};module.sortedSetAddBulk = async function (data) {@@ -69,8 +69,8 @@ module.exports = function (module) {if (!utils.isNumber(item[1])) {throw new Error(`[[error:invalid-score, ${item[1]}]]`);}- batch.zadd(item[0], item[1], item[2]);+ batch.zAdd(item[0], { score: item[1], value: String(item[2]) });});- await helpers.execBatch(batch);+ await batch.execAsPipeline();};};diff --git a/src/database/redis/sorted/intersect.js b/src/database/redis/sorted/intersect.jsindex 2b2ed1fe90..983c11abc4 100644--- a/src/database/redis/sorted/intersect.js+++ b/src/database/redis/sorted/intersect.js@@ -8,52 +8,46 @@ module.exports = function (module) {return 0;}const tempSetName = `temp_${Date.now()}`;-- const interParams = [tempSetName, keys.length].concat(keys);-const multi = module.client.multi();- multi.zinterstore(interParams);- multi.zcard(tempSetName);+ multi.zInterStore(tempSetName, keys);+ multi.zCard(tempSetName);multi.del(tempSetName);const results = await helpers.execBatch(multi);return results[1] || 0;};module.getSortedSetIntersect = async function (params) {- params.method = 'zrange';+ params.reverse = false;return await getSortedSetRevIntersect(params);};module.getSortedSetRevIntersect = async function (params) {- params.method = 'zrevrange';+ params.reverse = true;return await getSortedSetRevIntersect(params);};async function getSortedSetRevIntersect(params) {- const { sets } = params;+ let { sets } = params;const start = params.hasOwnProperty('start') ? params.start : 0;const stop = params.hasOwnProperty('stop') ? params.stop : -1;const weights = params.weights || [];const tempSetName = `temp_${Date.now()}`;- let interParams = [tempSetName, sets.length].concat(sets);+ const interParams = {};if (weights.length) {- interParams = interParams.concat(['WEIGHTS'].concat(weights));+ sets = sets.map((set, index) => ({ key: set, weight: weights[index] }));}if (params.aggregate) {- interParams = interParams.concat(['AGGREGATE', params.aggregate]);+ interParams['AGGREGATE'] = params.aggregate.toUpperCase();}- const rangeParams = [tempSetName, start, stop];- if (params.withScores) {- rangeParams.push('WITHSCORES');- }+ const rangeCmd = params.withScores ? 'zRangeWithScores' : 'zRange';const multi = module.client.multi();- multi.zinterstore(interParams);- multi[params.method](rangeParams);+ multi.zInterStore(tempSetName, sets, interParams);+ multi[rangeCmd](tempSetName, start, stop, { REV: params.reverse});multi.del(tempSetName);let results = await helpers.execBatch(multi);@@ -61,6 +55,6 @@ module.exports = function (module) {return results ? results[1] : null;}results = results[1] || [];- return helpers.zsetToObjectArray(results);+ return results;}};diff --git a/src/database/redis/sorted/remove.js b/src/database/redis/sorted/remove.jsindex 0c2b0164b0..df4c980b11 100644--- a/src/database/redis/sorted/remove.js+++ b/src/database/redis/sorted/remove.js@@ -18,10 +18,10 @@ module.exports = function (module) {if (Array.isArray(key)) {const batch = module.client.batch();- key.forEach(k => batch.zrem(k, value));+ key.forEach(k => batch.zRem(k, value.map(String)));await helpers.execBatch(batch);} else {- await module.client.zrem(key, value);+ await module.client.zRem(key, value.map(String));}};@@ -31,7 +31,7 @@ module.exports = function (module) {module.sortedSetsRemoveRangeByScore = async function (keys, min, max) {const batch = module.client.batch();- keys.forEach(k => batch.zremrangebyscore(k, min, max));+ keys.forEach(k => batch.zRemRangeByScore(k, min, max));await helpers.execBatch(batch);};@@ -40,7 +40,7 @@ module.exports = function (module) {return;}const batch = module.client.batch();- data.forEach(item => batch.zrem(item[0], item[1]));+ data.forEach(item => batch.zRem(item[0], String(item[1])));await helpers.execBatch(batch);};};diff --git a/src/database/redis/sorted/union.js b/src/database/redis/sorted/union.jsindex acd57c2db0..329c5f125f 100644--- a/src/database/redis/sorted/union.js+++ b/src/database/redis/sorted/union.js@@ -4,25 +4,20 @@module.exports = function (module) {const helpers = require('../helpers');module.sortedSetUnionCard = async function (keys) {- const tempSetName = `temp_${Date.now()}`;if (!keys.length) {return 0;}- const multi = module.client.multi();- multi.zunionstore([tempSetName, keys.length].concat(keys));- multi.zcard(tempSetName);- multi.del(tempSetName);- const results = await helpers.execBatch(multi);- return Array.isArray(results) && results.length ? results[1] : 0;+ const results = await module.client.zUnion(keys);+ return results ? results.length : 0;};module.getSortedSetUnion = async function (params) {- params.method = 'zrange';+ params.reverse = false;return await module.sortedSetUnion(params);};module.getSortedSetRevUnion = async function (params) {- params.method = 'zrevrange';+ params.reverse = true;return await module.sortedSetUnion(params);};@@ -32,21 +27,16 @@ module.exports = function (module) {}const tempSetName = `temp_${Date.now()}`;-- const rangeParams = [tempSetName, params.start, params.stop];- if (params.withScores) {- rangeParams.push('WITHSCORES');- }-+ const rangeCmd = params.withScores ? 'zRangeWithScores' : 'zRange';const multi = module.client.multi();- multi.zunionstore([tempSetName, params.sets.length].concat(params.sets));- multi[params.method](rangeParams);+ multi.zUnionStore(tempSetName, params.sets);+ multi[rangeCmd](tempSetName, params.start, params.stop, { REV: params.reverse });multi.del(tempSetName);let results = await helpers.execBatch(multi);if (!params.withScores) {return results ? results[1] : null;}results = results[1] || [];- return helpers.zsetToObjectArray(results);+ return results;};};diff --git a/src/topics/posts.js b/src/topics/posts.jsindex e32c18e727..8201bcad02 100644--- a/src/topics/posts.js+++ b/src/topics/posts.js@@ -442,7 +442,7 @@ module.exports = function (Topics) {let { content } = postData;// ignore lines that start with `>`- content = content.split('\n').filter(line => !line.trim().startsWith('>')).join('\n');+ content = (content || '').split('\n').filter(line => !line.trim().startsWith('>')).join('\n');// Scan post content for topic linksconst matches = [...content.matchAll(backlinkRegex)];if (!matches) {diff --git a/test/activitypub/actors.js b/test/activitypub/actors.jsindex 37fc74280d..ac2d11e5da 100644--- a/test/activitypub/actors.js+++ b/test/activitypub/actors.js@@ -449,7 +449,8 @@ describe('Inbox resolution', () => {await activitypub.actors.assert(id);const inboxes = await activitypub.resolveInboxes([id]);-+ console.log('inboxes', inboxes);+ console.log('actor', actor);assert(inboxes && Array.isArray(inboxes));assert.strictEqual(inboxes.length, 1);assert.strictEqual(inboxes[0], actor.inbox);diff --git a/test/activitypub/notes.js b/test/activitypub/notes.jsindex fbbf0c59ec..57ffce4259 100644--- a/test/activitypub/notes.js+++ b/test/activitypub/notes.js@@ -650,7 +650,7 @@ describe('Notes', () => {it('should upvote an asserted remote post', async () => {const { id } = helpers.mocks.note();- await activitypub.notes.assert(0, [id], { skipChecks: true });+ await activitypub.notes.assert(0, id, { skipChecks: true });const { activity: like } = helpers.mocks.like({object: id,});@@ -672,7 +672,7 @@ describe('Notes', () => {it('should update a note\'s content', async () => {const { id: actor } = helpers.mocks.person();const { id, note } = helpers.mocks.note({ attributedTo: actor });- await activitypub.notes.assert(0, [id], { skipChecks: true });+ await activitypub.notes.assert(0, id, { skipChecks: true });note.content = utils.generateUUID();const { activity: update } = helpers.mocks.update({ object: note });const { activity } = helpers.mocks.announce({ object: update });diff --git a/test/database/sorted.js b/test/database/sorted.jsindex b98d969730..a375b2ec48 100644--- a/test/database/sorted.js+++ b/test/database/sorted.js@@ -501,7 +501,9 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea['byScoreWithScoresKeys1', 1, 'value1'],['byScoreWithScoresKeys2', 2, 'value2'],]);- const data = await db.getSortedSetRevRangeByScoreWithScores(['byScoreWithScoresKeys1', 'byScoreWithScoresKeys2'], 0, -1, 5, -5);+ const data = await db.getSortedSetRevRangeByScoreWithScores([+ 'byScoreWithScoresKeys1', 'byScoreWithScoresKeys2',+ ], 0, -1, 5, -5);assert.deepStrictEqual(data, [{ value: 'value2', score: 2 }, { value: 'value1', score: 1 }]);});});@@ -1144,23 +1146,17 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:reaassert.strictEqual(await db.exists('sorted3'), false);});- it('should remove multiple values from multiple keys', (done) => {- db.sortedSetAdd('multiTest1', [1, 2, 3, 4], ['one', 'two', 'three', 'four'], (err) => {- assert.ifError(err);- db.sortedSetAdd('multiTest2', [3, 4, 5, 6], ['three', 'four', 'five', 'six'], (err) => {- assert.ifError(err);- db.sortedSetRemove(['multiTest1', 'multiTest2'], ['two', 'three', 'four', 'five', 'doesnt exist'], (err) => {- assert.ifError(err);- db.getSortedSetsMembers(['multiTest1', 'multiTest2'], (err, members) => {- assert.ifError(err);- assert.equal(members[0].length, 1);- assert.equal(members[1].length, 1);- assert.deepEqual(members, [['one'], ['six']]);- done();- });- });- });- });+ it('should remove multiple values from multiple keys', async () => {+ await db.sortedSetAdd('multiTest1', [1, 2, 3, 4], ['one', 'two', 'three', 'four']);+ await db.sortedSetAdd('multiTest2', [3, 4, 5, 6], ['three', 'four', 'five', 'six']);++ await db.sortedSetRemove(['multiTest1', 'multiTest2'], ['two', 'three', 'four', 'five', 'doesnt exist']);++ const members = await db.getSortedSetsMembers(['multiTest1', 'multiTest2']);++ assert.equal(members[0].length, 1);+ assert.equal(members[1].length, 1);+ assert.deepEqual(members, [['one'], ['six']]);});it('should remove value from multiple keys', async () => {@@ -1171,24 +1167,15 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:reaassert.deepStrictEqual(await db.getSortedSetRange('multiTest4', 0, -1), ['four', 'five', 'six']);});- it('should remove multiple values from multiple keys', (done) => {- db.sortedSetAdd('multiTest5', [1], ['one'], (err) => {- assert.ifError(err);- db.sortedSetAdd('multiTest6', [2], ['two'], (err) => {- assert.ifError(err);- db.sortedSetAdd('multiTest7', [3], [333], (err) => {- assert.ifError(err);- db.sortedSetRemove(['multiTest5', 'multiTest6', 'multiTest7'], ['one', 'two', 333], (err) => {- assert.ifError(err);- db.getSortedSetsMembers(['multiTest5', 'multiTest6', 'multiTest7'], (err, members) => {- assert.ifError(err);- assert.deepEqual(members, [[], [], []]);- done();- });- });- });- });- });+ it('should remove multiple values from multiple keys', async () => {+ await db.sortedSetAdd('multiTest5', [1], ['one']);+ await db.sortedSetAdd('multiTest6', [2], ['two']);+ await db.sortedSetAdd('multiTest7', [3], [333]);++ await db.sortedSetRemove(['multiTest5', 'multiTest6', 'multiTest7'], ['one', 'two', 333]);++ const members = await db.getSortedSetsMembers(['multiTest5', 'multiTest6', 'multiTest7']);+ assert.deepEqual(members, [[], [], []]);});it('should not remove anything if values is empty array', (done) => {@@ -1379,7 +1366,10 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:reaweights: [1, 0.5],}, (err, data) => {assert.ifError(err);- assert.deepEqual([{ value: 'value2', score: 4 }, { value: 'value3', score: 5.5 }], data);+ assert.deepEqual([+ { value: 'value2', score: 4 },+ { value: 'value3', score: 5.5 },+ ], data);done();});});diff --git a/test/mocks/databasemock.js b/test/mocks/databasemock.jsindex 6a568109b7..9416962b07 100644--- a/test/mocks/databasemock.js+++ b/test/mocks/databasemock.js@@ -171,7 +171,6 @@ before(async function () {require('../../src/user').startJobs();await webserver.listen();-// Iterate over all of the test suites/contextsthis.test.parent.suites.forEach((suite) => {// Attach an afterAll listener that resets the defaults