Node.js based forum software built for the modern web
refactor: update lru/ttl cache to use a version map (#14786)
Barış Uşaklı4828d899a3ab
diff --git a/src/cache/lru.js b/src/cache/lru.jsindex 208df96de0..f6fb7f04eb 100644--- a/src/cache/lru.js+++ b/src/cache/lru.js@@ -41,7 +41,6 @@ module.exports = function (opts) {cache.hits = 0;cache.misses = 0;cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true;- const cacheSet = lruCache.set;// expose properties while keeping backwards compatibilityconst propertyMap = new Map([@@ -63,6 +62,10 @@ module.exports = function (opts) {});});+ const invalidationVersions = new LRUCache({+ max: Math.max(10000, opts.max ? Math.floor(opts.max / 5) : 10000),+ });+cache.has = function (key) {if (!cache.enabled) {return false;@@ -79,20 +82,74 @@ module.exports = function (opts) {if (ttl) {opts.ttl = ttl;}- cacheSet.apply(lruCache, [key, value, opts]);+ lruCache.set(key, value, opts);};- cache.get = function (key) {+ cache.get = function (key, loader) {if (!cache.enabled) {- return undefined;+ return loader ?+ Promise.resolve().then(loader) :+ undefined;}const data = lruCache.get(key);- if (data === undefined) {- cache.misses += 1;- } else {+ if (data !== undefined) {cache.hits += 1;+ return data;+ }+ cache.misses += 1;+ if (!loader) {+ return undefined;}- return data;+ const version = getInvalidationVersion(key);+ return Promise.resolve()+ .then(() => loader())+ .then((value) => {+ if (getInvalidationVersion(key) === version) {+ cache.set(key, value);+ }+ return value;+ });+ };++ cache.getMany = function (keys, loader) {+ if (!cache.enabled) {+ return loader ?+ Promise.resolve().then(() => loader(keys)) :+ keys.map(() => undefined);+ }++ const data = new Array(keys.length);+ const uncachedKeys = [];+ const uncachedIndexes = [];+ const getManyVersions = new Map();++ keys.forEach((key, index) => {+ data[index] = cache.get(key);++ if (data[index] === undefined) {+ uncachedKeys.push(key);+ uncachedIndexes.push(index);+ getManyVersions.set(key, getInvalidationVersion(key));+ }+ });++ if (!loader || !uncachedKeys.length) {+ return data;+ }++ return Promise.resolve()+ .then(() => loader(uncachedKeys, uncachedIndexes))+ .then((values) => {+ uncachedKeys.forEach((key, index) => {+ const value = values[index];+ data[uncachedIndexes[index]] = value;+ if (getInvalidationVersion(key) === getManyVersions.get(key)) {+ cache.set(key, value);+ }+ });++ return data;+ });};cache.del = function (keys) {@@ -102,8 +159,11 @@ module.exports = function (opts) {if (!Array.isArray(keys)) {keys = [keys];}- pubsub.publish(`${cache.name}:lruCache:del`, keys);- keys.forEach(key => lruCache.delete(key));+ localDel(keys);+ pubsub.publish(`${cache.name}:lruCache:del`, {+ id: `${os.hostname()}:${process.pid}`,+ keys,+ });};cache.delete = cache.del;@@ -121,40 +181,29 @@ module.exports = function (opts) {cache.misses = 0;}+ function getInvalidationVersion(key) {+ return invalidationVersions.get(key) || 0;+ }++ function localDel(keys) {+ keys.forEach((key) => {+ invalidationVersions.set(key, getInvalidationVersion(key) + 1);+ lruCache.delete(key);+ });+ }+pubsub.on(`${cache.name}:lruCache:reset`, ({ id }) => {if (id !== `${os.hostname()}:${process.pid}`) {localReset();}});- pubsub.on(`${cache.name}:lruCache:del`, (keys) => {- if (Array.isArray(keys)) {- keys.forEach(key => lruCache.delete(key));+ pubsub.on(`${cache.name}:lruCache:del`, ({ id, keys }) => {+ if (id !== `${os.hostname()}:${process.pid}` && Array.isArray(keys)) {+ localDel(keys);}});- cache.getUnCachedKeys = function (keys, cachedData) {- if (!cache.enabled) {- return keys;- }- let data;- let isCached;- const unCachedKeys = keys.filter((key) => {- data = cache.get(key);- isCached = data !== undefined;- if (isCached) {- cachedData[key] = data;- }- return !isCached;- });-- const hits = keys.length - unCachedKeys.length;- const misses = keys.length - hits;- cache.hits += hits;- cache.misses += misses;- return unCachedKeys;- };-cache.dump = function () {return lruCache.dump();};diff --git a/src/cache/ttl.js b/src/cache/ttl.jsindex d56cfec042..8659e0eefa 100644--- a/src/cache/ttl.js+++ b/src/cache/ttl.js@@ -2,6 +2,7 @@module.exports = function (opts) {const { TTLCache } = require('@isaacs/ttlcache');+ const { LRUCache } = require('lru-cache');const os = require('os');const winston = require('winston');const chalk = require('chalk').default;@@ -19,7 +20,6 @@ module.exports = function (opts) {cache.hits = 0;cache.misses = 0;cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true;- const cacheSet = ttlCache.set;// expose propertiesconst propertyMap = new Map([@@ -38,6 +38,10 @@ module.exports = function (opts) {});});+ const invalidationVersions = new LRUCache({+ max: Math.max(10000, opts.max ? Math.floor(opts.max / 5) : 10000),+ });+cache.has = function (key) {if (!cache.enabled) {return false;@@ -54,20 +58,74 @@ module.exports = function (opts) {if (ttl) {opts.ttl = ttl;}- cacheSet.apply(ttlCache, [key, value, opts]);+ ttlCache.set(key, value, opts);};- cache.get = function (key) {+ cache.get = function (key, loader) {if (!cache.enabled) {- return undefined;+ return loader ?+ Promise.resolve().then(loader) :+ undefined;}const data = ttlCache.get(key);- if (data === undefined) {- cache.misses += 1;- } else {+ if (data !== undefined) {cache.hits += 1;+ return data;+ }+ cache.misses += 1;+ if (!loader) {+ return undefined;+ }+ const version = getInvalidationVersion(key);+ return Promise.resolve()+ .then(() => loader())+ .then((value) => {+ if (getInvalidationVersion(key) === version) {+ cache.set(key, value);+ }+ return value;+ });+ };++ cache.getMany = function (keys, loader) {+ if (!cache.enabled) {+ return loader ?+ Promise.resolve().then(() => loader(keys)) :+ keys.map(() => undefined);+ }++ const data = new Array(keys.length);+ const uncachedKeys = [];+ const uncachedIndexes = [];+ const getManyVersions = new Map();++ keys.forEach((key, index) => {+ data[index] = cache.get(key);++ if (data[index] === undefined) {+ uncachedKeys.push(key);+ uncachedIndexes.push(index);+ getManyVersions.set(key, getInvalidationVersion(key));+ }+ });++ if (!loader || !uncachedKeys.length) {+ return data;}- return data;++ return Promise.resolve()+ .then(() => loader(uncachedKeys, uncachedIndexes))+ .then((values) => {+ uncachedKeys.forEach((key, index) => {+ const value = values[index];+ data[uncachedIndexes[index]] = value;+ if (getInvalidationVersion(key) === getManyVersions.get(key)) {+ cache.set(key, value);+ }+ });++ return data;+ });};cache.del = function (keys) {@@ -77,8 +135,9 @@ module.exports = function (opts) {if (!Array.isArray(keys)) {keys = [keys];}- pubsub.publish(`${cache.name}:ttlCache:del`, keys);- keys.forEach(key => ttlCache.delete(key));++ localDel(keys);+ pubsub.publish(`${cache.name}:ttlCache:del`, { id: `${os.hostname()}:${process.pid}`, keys });};cache.delete = cache.del;@@ -96,40 +155,29 @@ module.exports = function (opts) {cache.misses = 0;}+ function getInvalidationVersion(key) {+ return invalidationVersions.get(key) || 0;+ }++ function localDel(keys) {+ keys.forEach((key) => {+ invalidationVersions.set(key, (invalidationVersions.get(key) || 0) + 1);+ ttlCache.delete(key);+ });+ }+pubsub.on(`${cache.name}:ttlCache:reset`, ({ id }) => {if (id !== `${os.hostname()}:${process.pid}`) {localReset();}});- pubsub.on(`${cache.name}:ttlCache:del`, (keys) => {- if (Array.isArray(keys)) {- keys.forEach(key => ttlCache.delete(key));+ pubsub.on(`${cache.name}:ttlCache:del`, ({ id, keys }) => {+ if (id !== `${os.hostname()}:${process.pid}` && Array.isArray(keys)) {+ localDel(keys);}});- cache.getUnCachedKeys = function (keys, cachedData) {- if (!cache.enabled) {- return keys;- }- let data;- let isCached;- const unCachedKeys = keys.filter((key) => {- data = cache.get(key);- isCached = data !== undefined;- if (isCached) {- cachedData[key] = data;- }- return !isCached;- });-- const hits = keys.length - unCachedKeys.length;- const misses = keys.length - hits;- cache.hits += hits;- cache.misses += misses;- return unCachedKeys;- };-cache.dump = function () {return Array.from(ttlCache.entries());};diff --git a/src/categories/index.js b/src/categories/index.jsindex 87e37d0b11..6c365536f0 100644--- a/src/categories/index.js+++ b/src/categories/index.js@@ -98,13 +98,7 @@ Categories.getCidByHandle = async function (handle) {};Categories.getAllCidsFromSet = async function (key) {- let cids = cache.get(key);- if (cids) {- return cids.slice();- }-- cids = await db.getSortedSetRange(key, 0, -1);- cache.set(key, cids);+ const cids = await cache.get(key, () => db.getSortedSetRange(key, 0, -1));return cids.slice();};@@ -185,29 +179,11 @@ Categories.setUnread = async function (tree, cids, uid) {};Categories.getTagWhitelist = async function (cids) {- const cachedData = {};-- const nonCachedCids = cids.filter((cid) => {- const data = cache.get(`cid:${cid}:tag:whitelist`);- const isInCache = data !== undefined;- if (isInCache) {- cachedData[cid] = data;- }- return !isInCache;- });-- if (!nonCachedCids.length) {- return cids.map(cid => cachedData[cid]);- }-- const keys = nonCachedCids.map(cid => `cid:${cid}:tag:whitelist`);- const data = await db.getSortedSetsMembers(keys);-- nonCachedCids.forEach((cid, index) => {- cachedData[cid] = data[index];- cache.set(`cid:${cid}:tag:whitelist`, data[index]);- });- return cids.map(cid => cachedData[cid]);+ const cachedData = await cache.getMany(+ cids.map(cid => `cid:${cid}:tag:whitelist`),+ uncachedKeys => db.getSortedSetsMembers(uncachedKeys)+ );+ return cachedData;};// remove system tags from tag whitelist for non privileged user@@ -286,29 +262,25 @@ Categories.getParentCids = async function (currentCid) {};Categories.getChildrenCids = async function (rootCid) {- let allCids = [];+ const allCids = new Set();async function recursive(keys) {let childrenCids = await db.getSortedSetRange(keys, 0, -1);- childrenCids = childrenCids.filter(cid => !allCids.includes(cid));+ childrenCids = childrenCids.filter(cid => !allCids.has(cid));if (!childrenCids.length) {return;}- allCids.push(...childrenCids);+ childrenCids.forEach(cid => allCids.add(cid));keys = childrenCids.map(cid => `cid:${cid}:children`);await recursive(keys);}const key = `cid:${rootCid}:children`;const cacheKey = `${key}:all`;- const childrenCids = cache.get(cacheKey);- if (childrenCids) {- return childrenCids.slice();- }-- await recursive(key);- allCids = _.uniq(allCids);- cache.set(cacheKey, allCids);- return allCids.slice();+ const childrenCids = await cache.get(cacheKey, async () => {+ await recursive(key);+ return Array.from(allCids);+ });+ return childrenCids.slice();};Categories.flattenCategories = function (allCategories, categoryData) {diff --git a/src/controllers/topics.js b/src/controllers/topics.jsindex c3df881eab..e549f036d4 100644--- a/src/controllers/topics.js+++ b/src/controllers/topics.js@@ -227,12 +227,10 @@ async function markAsRead(req, tid) {async function loadCrosspostPrivilege(req, excludeCid) {excludeCid = String(excludeCid || '');- let cidsUserCanCrosspost = crosspostCache.get(`uid:${req.uid}`);- if (cidsUserCanCrosspost === undefined) {+ const cidsUserCanCrosspost = await crosspostCache.get(`uid:${req.uid}`, async () => {const cids = await categories.getAllCidsFromSet('categories:cid');- cidsUserCanCrosspost = await privileges.categories.filterCids('topics:crosspost', cids, req.uid);- crosspostCache.set(`uid:${req.uid}`, cidsUserCanCrosspost);- }+ return await privileges.categories.filterCids('topics:crosspost', cids, req.uid);+ });return cidsUserCanCrosspost.some(cid => cid !== excludeCid);}diff --git a/src/database/mongo/hash.js b/src/database/mongo/hash.jsindex 8dabf11831..bdac3f5bc8 100644--- a/src/database/mongo/hash.js+++ b/src/database/mongo/hash.js@@ -98,11 +98,12 @@ module.exports = function (module) {if (!key || !field) {return null;}- const cachedData = {};- cache.getUnCachedKeys([key], cachedData);- if (cachedData[key]) {- return cachedData[key].hasOwnProperty(field) ? cachedData[key][field] : null;++ const cachedData = cache.get(key);+ if (cachedData !== undefined) {+ return cachedData?.hasOwnProperty(field) ? cachedData[field] : null;}+field = helpers.fieldToString(field);const item = await module.client.collection('objects').findOne({_key: key,@@ -127,28 +128,22 @@ module.exports = function (module) {if (!Array.isArray(keys) || !keys.length) {return [];}- const cachedData = {};- const unCachedKeys = cache.getUnCachedKeys(keys, cachedData);-- if (unCachedKeys.length >= 1) {+ const cachedData = await cache.getMany(keys, async (unCachedKeys) => {let data = await module.client.collection('objects').find({ _key: unCachedKeys.length === 1 ? unCachedKeys[0] : { $in: unCachedKeys } },{ projection: { _id: 0 } }).toArray();- data = data.map(helpers.deserializeData);+ data = data.map(helpers.deserializeData);const map = helpers.toMap(data);- unCachedKeys.forEach((key) => {- cachedData[key] = map[key] || null;- cache.set(key, cachedData[key]);- });- }+ return unCachedKeys.map(key => map[key] || null);+ });if (!Array.isArray(fields) || !fields.length) {- return keys.map(key => (cachedData[key] ? { ...cachedData[key] } : null));+ return cachedData.map(data => (data ? { ...data } : null));}- return keys.map((key) => {- const item = cachedData[key] || {};+ return cachedData.map((data) => {+ const item = data || {};const result = {};fields.forEach((field) => {result[field] = item[field] !== undefined ? item[field] : null;diff --git a/src/database/postgres/hash.js b/src/database/postgres/hash.jsindex f967e4ae4b..46e792ba17 100644--- a/src/database/postgres/hash.js+++ b/src/database/postgres/hash.js@@ -132,10 +132,9 @@ module.exports = function (module) {if (!key || !field) {return null;}- const cachedData = {};- cache.getUnCachedKeys([key], cachedData);- if (cachedData[key]) {- return cachedData[key].hasOwnProperty(field) ? cachedData[key][field] : null;+ const cachedData = cache.get(key);+ if (cachedData !== undefined) {+ return cachedData?.hasOwnProperty(field) ? cachedData[field] : null;}const res = await module.pool.query({@@ -167,10 +166,7 @@ SELECT h."data"->>$2::TEXT freturn [];}- const cachedData = {};- const unCachedKeys = cache.getUnCachedKeys(keys, cachedData);-- if (unCachedKeys.length) {+ const cachedData = await cache.getMany(keys, async (unCachedKeys) => {const res = await module.pool.query({name: 'getObjectsFields',text: `@@ -185,18 +181,14 @@ SELECT h."data"values: [unCachedKeys],});const data = res.rows.map(row => row.data);-- unCachedKeys.forEach((key, i) => {- cachedData[key] = data[i] || null;- cache.set(key, cachedData[key]);- });- }+ return data;+ });if (!Array.isArray(fields) || !fields.length) {- return keys.map(key => (cachedData[key] ? { ...cachedData[key] } : null));+ return cachedData.map(data => (data ? { ...data } : null));}- return keys.map((key) => {- const item = cachedData[key] || {};+ return cachedData.map((data) => {+ const item = data || {};const result = {};fields.forEach((field) => {result[field] = item[field] !== undefined ? item[field] : null;diff --git a/src/database/redis/hash.js b/src/database/redis/hash.jsindex f046e62180..1f0b1104ea 100644--- a/src/database/redis/hash.js+++ b/src/database/redis/hash.js@@ -99,10 +99,12 @@ module.exports = function (module) {if (!key || !field) {return null;}- const cachedData = {};- cache.getUnCachedKeys([key], cachedData);- if (cachedData[key]) {- return Object.hasOwn(cachedData[key], field) ? cachedData[key][field] : null;+ const cachedData = cache.get(key);+ if (cachedData !== undefined) {+ if (cachedData !== null && typeof cachedData === 'object') {+ return Object.hasOwn(cachedData, field) ? cachedData[field] : null;+ }+ return null;}return await module.client.hGet(key, String(field));};@@ -120,36 +122,31 @@ module.exports = function (module) {return [];}- const cachedData = {};- const unCachedKeys = cache.getUnCachedKeys(keys, cachedData);-- let data = [];- if (unCachedKeys.length > 1) {- const batch = module.client.batch();- unCachedKeys.forEach(k => batch.hGetAll(k));- data = await helpers.execBatch(batch);- } else if (unCachedKeys.length === 1) {- data = [await module.client.hGetAll(unCachedKeys[0])];- }+ let cachedData = await cache.getMany(keys, async (unCachedKeys) => {+ let data = [];+ if (unCachedKeys.length > 1) {+ const batch = module.client.batch();+ unCachedKeys.forEach(k => batch.hGetAll(k));+ data = await helpers.execBatch(batch);+ } else if (unCachedKeys.length === 1) {+ data = [await module.client.hGetAll(unCachedKeys[0])];+ }+ return data;+ });// convert empty objects into null for back-compat with node_redis- data = data.map((elem) => {+ cachedData = cachedData.map((elem) => {if (!Object.keys(elem).length) {return null;}return elem;});- unCachedKeys.forEach((key, i) => {- cachedData[key] = data[i] || null;- cache.set(key, cachedData[key]);- });-if (!Array.isArray(fields) || !fields.length) {- return keys.map(key => (cachedData[key] ? { ...cachedData[key] } : null));+ return cachedData.map(data => (data ? { ...data } : null));}- return keys.map((key) => {- const item = cachedData[key] || {};+ return cachedData.map((data) => {+ const item = data || {};const result = {};fields.forEach((field) => {result[field] = item[field] !== undefined ? item[field] : null;diff --git a/src/groups/index.js b/src/groups/index.jsindex 1f3d824839..e0597340b5 100644--- a/src/groups/index.js+++ b/src/groups/index.js@@ -114,13 +114,7 @@ Groups.getNonPrivilegeGroups = async function (set, start, stop, flags) {};Groups.getAllGroupNames = async function (set) {- const cacheKey = `zset:${set}`;- let names = cache.get(cacheKey);- if (names !== undefined) {- return [...names];- }- names = await db.getSortedSetRevRange(set, 0, -1);- cache.set(cacheKey, names);+ const names = await cache.get(`zset:${set}`, () => db.getSortedSetRevRange(set, 0, -1));return [...names];};diff --git a/src/groups/membership.js b/src/groups/membership.jsindex b0bc10697c..70c187c4c6 100644--- a/src/groups/membership.js+++ b/src/groups/membership.js@@ -29,13 +29,9 @@ module.exports = function (Groups) {return isMemberOfEphemeralGroup(uid, groupName);}- const cacheKey = `${uid}:${groupName}`;- let isMember = Groups.cache.get(cacheKey);- if (isMember !== undefined) {- return isMember;- }- isMember = await db.isSortedSetMember(`group:${groupName}:members`, uid);- Groups.cache.set(cacheKey, isMember);+ const isMember = await Groups.cache.get(+ `${uid}:${groupName}`, () => db.isSortedSetMember(`group:${groupName}:members`, uid)+ );return isMember;};@@ -47,40 +43,30 @@ module.exports = function (Groups) {if (groupName === 'guests' || groupName === 'spiders') {return uids.map(uid => isMemberOfEphemeralGroup(uid, groupName));}-- const cachedData = {};- const nonCachedUids = uids.filter(uid => filterNonCached(cachedData, uid, groupName));-- if (!nonCachedUids.length) {- return uids.map(uid => cachedData[`${uid}:${groupName}`]);- }-- const isMembers = await db.isSortedSetMembers(`group:${groupName}:members`, nonCachedUids);- nonCachedUids.forEach((uid, index) => {- cachedData[`${uid}:${groupName}`] = isMembers[index];- Groups.cache.set(`${uid}:${groupName}`, isMembers[index]);- });- return uids.map(uid => cachedData[`${uid}:${groupName}`]);+ const cacheKeys = uids.map(uid => `${uid}:${groupName}`);+ const isMembers = await Groups.cache.getMany(+ cacheKeys,+ async (uncachedKeys, uncachedIndexes) => {+ const uncachedUids = uncachedIndexes.map(index => uids[index]);+ return await db.isSortedSetMembers(`group:${groupName}:members`, uncachedUids);+ }+ );+ return isMembers.slice();};Groups.isMemberOfGroups = async function (uid, groups) {if (!uid || (utils.isNumber(uid) && parseInt(uid, 10) <= 0) || !groups.length) {return groups.map(groupName => isMemberOfEphemeralGroup(uid, groupName));}- const cachedData = {};- const nonCachedGroups = groups.filter(groupName => filterNonCached(cachedData, uid, groupName));+ const keys = groups.map(groupName => `${uid}:${groupName}`);- if (!nonCachedGroups.length) {- return groups.map(groupName => cachedData[`${uid}:${groupName}`]);- }- const nonCachedGroupsMemberSets = nonCachedGroups.map(groupName => `group:${groupName}:members`);- const isMembers = await db.isMemberOfSortedSets(nonCachedGroupsMemberSets, uid);- nonCachedGroups.forEach((groupName, index) => {- cachedData[`${uid}:${groupName}`] = isMembers[index];- Groups.cache.set(`${uid}:${groupName}`, isMembers[index]);- });+ return await Groups.cache.getMany(keys, async (uncachedKeys, uncachedIndexes) => {+ const nonCachedGroupsMemberSets = uncachedIndexes.map(index => (+ `group:${groups[index]}:members`+ ));- return groups.map(groupName => cachedData[`${uid}:${groupName}`]);+ return await db.isMemberOfSortedSets(nonCachedGroupsMemberSets, uid);+ });};function isMemberOfEphemeralGroup(uid, groupName) {@@ -90,15 +76,6 @@ module.exports = function (Groups) {(groupName === 'spiders' && parsedUid === -1);}- function filterNonCached(cachedData, uid, groupName) {- const isMember = Groups.cache.get(`${uid}:${groupName}`);- const isInCache = isMember !== undefined;- if (isInCache) {- cachedData[`${uid}:${groupName}`] = isMember;- }- return !isInCache;- }-Groups.isMemberOfAny = async function (uid, groups) {if (!Array.isArray(groups) || !groups.length) {return false;@@ -159,27 +136,11 @@ module.exports = function (Groups) {const isArray = Array.isArray(keys);keys = isArray ? keys : [keys];- const cachedData = {};- const nonCachedKeys = keys.filter((groupName) => {- const groupMembers = cache.get(`group:${groupName}:members`);- const isInCache = groupMembers !== undefined;- if (isInCache) {- cachedData[groupName] = groupMembers;- }- return !isInCache;- });-- if (!nonCachedKeys.length) {- return isArray ? keys.map(groupName => cachedData[groupName]) : cachedData[keys[0]];- }- const groupMembers = await db.getSortedSetsMembers(- nonCachedKeys.map(name => `group:${name}:members`)+ const cacheKeys = keys.map(groupName => `group:${groupName}:members`);+ const groupMembers = await cache.getMany(+ cacheKeys, uncachedKeys => db.getSortedSetsMembers(uncachedKeys));- nonCachedKeys.forEach((groupName, index) => {- cachedData[groupName] = groupMembers[index];- cache.set(`group:${groupName}:members`, groupMembers[index]);- });- return isArray ? keys.map(groupName => cachedData[groupName]) : cachedData[keys[0]];+ return isArray ? groupMembers : groupMembers[0];}};diff --git a/src/messaging/index.js b/src/messaging/index.jsindex 26d4810cc1..d1a970c13b 100644--- a/src/messaging/index.js+++ b/src/messaging/index.js@@ -115,11 +115,7 @@ Messaging.isNewSet = async (uid, roomId, timestamp) => {Messaging.getPublicRoomIdsFromSet = async function (set) {const cacheKey = `${set}:all`;- let allRoomIds = cache.get(cacheKey);- if (allRoomIds === undefined) {- allRoomIds = await db.getSortedSetRange(set, 0, -1);- cache.set(cacheKey, allRoomIds);- }+ const allRoomIds = await cache.get(cacheKey, () => db.getSortedSetRange(set, 0, -1));return allRoomIds.slice();};diff --git a/src/messaging/rooms.js b/src/messaging/rooms.jsindex ad7b26eab4..dbfa09b08b 100644--- a/src/messaging/rooms.js+++ b/src/messaging/rooms.js@@ -406,14 +406,11 @@ module.exports = function (Messaging) {}Messaging.getAllUidsInRoomFromSet = async function (set) {- const cacheKey = `${set}:all`;- let uids = roomUidCache.get(cacheKey);- if (uids !== undefined) {- return uids;- }- uids = await Messaging.getUidsInRoomFromSet(set, 0, -1);- roomUidCache.set(cacheKey, uids);- return uids;+ const uids = await roomUidCache.get(+ `${set}:all`,+ () => Messaging.getUidsInRoomFromSet(set, 0, -1)+ );+ return uids.slice();};Messaging.getUidsInRoomFromSet = async (set, start, stop, reverse = false) => db[diff --git a/src/meta/settings.js b/src/meta/settings.jsindex 213b61be11..89e460a3ff 100644--- a/src/meta/settings.js+++ b/src/meta/settings.js@@ -11,30 +11,28 @@ const cache = require('../cache');const Settings = module.exports;Settings.get = async function (hash) {- const cached = cache.get(`settings:${hash}`);- if (cached) {- return _.cloneDeep(cached);- }- const [data, sortedLists] = await Promise.all([- db.getObject(`settings:${hash}`),- db.getSetMembers(`settings:${hash}:sorted-lists`),- ]);- const values = data || {};- await Promise.all(sortedLists.map(async (list) => {- const members = await db.getSortedSetRange(`settings:${hash}:sorted-list:${list}`, 0, -1);- const keys = members.map(order => `settings:${hash}:sorted-list:${list}:${order}`);-- values[list] = [];-- const objects = await db.getObjects(keys);- objects.forEach((obj) => {- values[list].push(obj);- });- }));+ const cached = await cache.get(`settings:${hash}`, async () => {+ const [data, sortedLists] = await Promise.all([+ db.getObject(`settings:${hash}`),+ db.getSetMembers(`settings:${hash}:sorted-lists`),+ ]);+ const values = data || {};+ await Promise.all(sortedLists.map(async (list) => {+ const members = await db.getSortedSetRange(`settings:${hash}:sorted-list:${list}`, 0, -1);+ const keys = members.map(order => `settings:${hash}:sorted-list:${list}:${order}`);++ values[list] = [];- const result = await plugins.hooks.fire('filter:settings.get', { plugin: hash, values: values });- cache.set(`settings:${hash}`, result.values);- return _.cloneDeep(result.values);+ const objects = await db.getObjects(keys);+ objects.forEach((obj) => {+ values[list].push(obj);+ });+ }));++ const result = await plugins.hooks.fire('filter:settings.get', { plugin: hash, values: values });+ return result.values;+ });+ return _.cloneDeep(cached);};Settings.getOne = async function (hash, field) {diff --git a/src/plugins/install.js b/src/plugins/install.jsindex 4c5cd2031f..b8df9a8fe8 100644--- a/src/plugins/install.js+++ b/src/plugins/install.js@@ -222,12 +222,9 @@ module.exports = function (Plugins) {if (nconf.get('plugins:active')) {return nconf.get('plugins:active').includes(id);}- const cached = cache.get(`plugin:isActive:${id}`);- if (cached !== undefined) {- return cached;- }- const isActive = await db.isSortedSetMember('plugins:active', id);- cache.set(`plugin:isActive:${id}`, isActive);+ const isActive = await cache.get(`plugin:isActive:${id}`, async () => {+ return await db.isSortedSetMember('plugins:active', id);+ });return isActive;};diff --git a/src/socket.io/topics.js b/src/socket.io/topics.jsindex 3df9cdc1a2..1e30ebf65a 100644--- a/src/socket.io/topics.js+++ b/src/socket.io/topics.js@@ -81,25 +81,19 @@ SocketTopics.getMyNextPostIndex = async function (socket, data) {const topicSet = data.sort === 'most_votes' ? `tid:${data.tid}:posts:votes` : `tid:${data.tid}:posts`;const reverse = data.sort === 'newest_to_oldest' || data.sort === 'most_votes';const cacheKey = `np:s:${topicSet}:r:${String(reverse)}:tid:${data.tid}:pids`;- const topicPids = cache.get(cacheKey);- if (topicPids) {- return topicPids.slice(index - 1);- }- const pids = await db[reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'](topicSet, 0, -1);- cache.set(cacheKey, pids, 30000);- return pids.slice(index - 1);+ const topicPids = await cache.get(cacheKey, async () => {+ return await db[reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'](topicSet, 0, -1);+ });+ return topicPids.slice(index - 1);}async function getUserPids() {const cid = await topics.getTopicField(data.tid, 'cid');const cacheKey = `np:cid:${cid}:uid:${socket.uid}:pids`;- const userPids = cache.get(cacheKey);- if (userPids) {- return userPids;- }- const pids = await db.getSortedSetRange(`cid:${cid}:uid:${socket.uid}:pids`, 0, -1);- cache.set(cacheKey, pids, 30000);- return pids;+ const userPids = await cache.get(cacheKey, async () => {+ return await db.getSortedSetRange(`cid:${cid}:uid:${socket.uid}:pids`, 0, -1);+ });+ return userPids;}const postCountInTopic = await db.sortedSetScore(`tid:${data.tid}:posters`, socket.uid);if (postCountInTopic <= 0) {diff --git a/src/topics/tags.js b/src/topics/tags.jsindex e7a903214c..388e2818ed 100644--- a/src/topics/tags.js+++ b/src/topics/tags.js@@ -476,13 +476,11 @@ module.exports = function (Topics) {};async function getAllTags() {- const cached = cache.get('tags:topic:count');- if (cached !== undefined) {- return cached;- }- const tags = await db.getSortedSetRevRangeWithScores('tags:topic:count', 0, -1);- cache.set('tags:topic:count', tags);- return tags;+ const cached = await cache.get(+ 'tags:topic:count',+ () => db.getSortedSetRevRangeWithScores('tags:topic:count', 0, -1)+ );+ return cached;}async function findMatches(data) {diff --git a/src/user/blocks.js b/src/user/blocks.jsindex 4292e5dfcd..a10c8b47c6 100644--- a/src/user/blocks.js+++ b/src/user/blocks.js@@ -49,17 +49,14 @@ module.exports = function (User) {User.blocks.list = async function (uids) {const isArray = Array.isArray(uids);uids = (isArray ? uids : [uids]).map(uid => String(uid));- const cachedData = {};- const unCachedUids = User.blocks._cache.getUnCachedKeys(uids, cachedData);- if (unCachedUids.length) {- const unCachedData = await db.getSortedSetsMembers(unCachedUids.map(uid => `uid:${uid}:blocked_uids`));- unCachedUids.forEach((uid, index) => {- cachedData[uid] = (unCachedData[index] || []);- User.blocks._cache.set(String(uid), cachedData[uid]);- });- }- const result = uids.map(uid => cachedData[uid] || []);- return isArray ? result.slice() : result[0];++ const cachedData = await User.blocks._cache.getMany(uids, async (unCachedUids) => {+ const unCachedData = await db.getSortedSetsMembers(+ unCachedUids.map(uid => `uid:${uid}:blocked_uids`)+ );+ return unCachedUids.map((uid, index) => unCachedData[index]);+ });+ return isArray ? cachedData : cachedData[0];};User.blocks.add = async function (targetUid, uid) {