Skip to content

Node.js based forum software built for the modern web

refactor: update lru/ttl cache to use a version map (#14786)

Barış Uşaklı4828d899a3ab
diff --git a/src/cache/lru.js b/src/cache/lru.js
index 208df96de0..f6fb7f04eb 100644
--- a/src/cache/lru.js
+++ b/src/cache/lru.js
@@ -41,7 +41,6 @@ module.exports = function (opts) {
cache.hits = 0;
cache.misses = 0;
cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true;
- const cacheSet = lruCache.set;
// expose properties while keeping backwards compatibility
const propertyMap = new Map([
@@ -63,6 +62,10 @@ module.exports = function (opts) {
});
});
+ const invalidationVersions = new LRUCache({
+ max: Math.max(10000, opts.max ? Math.floor(opts.max / 5) : 10000),
+ });
+
cache.has = function (key) {
if (!cache.enabled) {
return false;
@@ -79,20 +82,74 @@ module.exports = function (opts) {
if (ttl) {
opts.ttl = ttl;
}
- cacheSet.apply(lruCache, [key, value, opts]);
+ lruCache.set(key, value, opts);
};
- cache.get = function (key) {
+ cache.get = function (key, loader) {
if (!cache.enabled) {
- return undefined;
+ return loader ?
+ Promise.resolve().then(loader) :
+ undefined;
}
const data = lruCache.get(key);
- if (data === undefined) {
- cache.misses += 1;
- } else {
+ if (data !== undefined) {
cache.hits += 1;
+ return data;
+ }
+ cache.misses += 1;
+ if (!loader) {
+ return undefined;
}
- return data;
+ const version = getInvalidationVersion(key);
+ return Promise.resolve()
+ .then(() => loader())
+ .then((value) => {
+ if (getInvalidationVersion(key) === version) {
+ cache.set(key, value);
+ }
+ return value;
+ });
+ };
+
+ cache.getMany = function (keys, loader) {
+ if (!cache.enabled) {
+ return loader ?
+ Promise.resolve().then(() => loader(keys)) :
+ keys.map(() => undefined);
+ }
+
+ const data = new Array(keys.length);
+ const uncachedKeys = [];
+ const uncachedIndexes = [];
+ const getManyVersions = new Map();
+
+ keys.forEach((key, index) => {
+ data[index] = cache.get(key);
+
+ if (data[index] === undefined) {
+ uncachedKeys.push(key);
+ uncachedIndexes.push(index);
+ getManyVersions.set(key, getInvalidationVersion(key));
+ }
+ });
+
+ if (!loader || !uncachedKeys.length) {
+ return data;
+ }
+
+ return Promise.resolve()
+ .then(() => loader(uncachedKeys, uncachedIndexes))
+ .then((values) => {
+ uncachedKeys.forEach((key, index) => {
+ const value = values[index];
+ data[uncachedIndexes[index]] = value;
+ if (getInvalidationVersion(key) === getManyVersions.get(key)) {
+ cache.set(key, value);
+ }
+ });
+
+ return data;
+ });
};
cache.del = function (keys) {
@@ -102,8 +159,11 @@ module.exports = function (opts) {
if (!Array.isArray(keys)) {
keys = [keys];
}
- pubsub.publish(`${cache.name}:lruCache:del`, keys);
- keys.forEach(key => lruCache.delete(key));
+ localDel(keys);
+ pubsub.publish(`${cache.name}:lruCache:del`, {
+ id: `${os.hostname()}:${process.pid}`,
+ keys,
+ });
};
cache.delete = cache.del;
@@ -121,40 +181,29 @@ module.exports = function (opts) {
cache.misses = 0;
}
+ function getInvalidationVersion(key) {
+ return invalidationVersions.get(key) || 0;
+ }
+
+ function localDel(keys) {
+ keys.forEach((key) => {
+ invalidationVersions.set(key, getInvalidationVersion(key) + 1);
+ lruCache.delete(key);
+ });
+ }
+
pubsub.on(`${cache.name}:lruCache:reset`, ({ id }) => {
if (id !== `${os.hostname()}:${process.pid}`) {
localReset();
}
});
- pubsub.on(`${cache.name}:lruCache:del`, (keys) => {
- if (Array.isArray(keys)) {
- keys.forEach(key => lruCache.delete(key));
+ pubsub.on(`${cache.name}:lruCache:del`, ({ id, keys }) => {
+ if (id !== `${os.hostname()}:${process.pid}` && Array.isArray(keys)) {
+ localDel(keys);
}
});
- cache.getUnCachedKeys = function (keys, cachedData) {
- if (!cache.enabled) {
- return keys;
- }
- let data;
- let isCached;
- const unCachedKeys = keys.filter((key) => {
- data = cache.get(key);
- isCached = data !== undefined;
- if (isCached) {
- cachedData[key] = data;
- }
- return !isCached;
- });
-
- const hits = keys.length - unCachedKeys.length;
- const misses = keys.length - hits;
- cache.hits += hits;
- cache.misses += misses;
- return unCachedKeys;
- };
-
cache.dump = function () {
return lruCache.dump();
};
diff --git a/src/cache/ttl.js b/src/cache/ttl.js
index d56cfec042..8659e0eefa 100644
--- a/src/cache/ttl.js
+++ b/src/cache/ttl.js
@@ -2,6 +2,7 @@
module.exports = function (opts) {
const { TTLCache } = require('@isaacs/ttlcache');
+ const { LRUCache } = require('lru-cache');
const os = require('os');
const winston = require('winston');
const chalk = require('chalk').default;
@@ -19,7 +20,6 @@ module.exports = function (opts) {
cache.hits = 0;
cache.misses = 0;
cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true;
- const cacheSet = ttlCache.set;
// expose properties
const propertyMap = new Map([
@@ -38,6 +38,10 @@ module.exports = function (opts) {
});
});
+ const invalidationVersions = new LRUCache({
+ max: Math.max(10000, opts.max ? Math.floor(opts.max / 5) : 10000),
+ });
+
cache.has = function (key) {
if (!cache.enabled) {
return false;
@@ -54,20 +58,74 @@ module.exports = function (opts) {
if (ttl) {
opts.ttl = ttl;
}
- cacheSet.apply(ttlCache, [key, value, opts]);
+ ttlCache.set(key, value, opts);
};
- cache.get = function (key) {
+ cache.get = function (key, loader) {
if (!cache.enabled) {
- return undefined;
+ return loader ?
+ Promise.resolve().then(loader) :
+ undefined;
}
const data = ttlCache.get(key);
- if (data === undefined) {
- cache.misses += 1;
- } else {
+ if (data !== undefined) {
cache.hits += 1;
+ return data;
+ }
+ cache.misses += 1;
+ if (!loader) {
+ return undefined;
+ }
+ const version = getInvalidationVersion(key);
+ return Promise.resolve()
+ .then(() => loader())
+ .then((value) => {
+ if (getInvalidationVersion(key) === version) {
+ cache.set(key, value);
+ }
+ return value;
+ });
+ };
+
+ cache.getMany = function (keys, loader) {
+ if (!cache.enabled) {
+ return loader ?
+ Promise.resolve().then(() => loader(keys)) :
+ keys.map(() => undefined);
+ }
+
+ const data = new Array(keys.length);
+ const uncachedKeys = [];
+ const uncachedIndexes = [];
+ const getManyVersions = new Map();
+
+ keys.forEach((key, index) => {
+ data[index] = cache.get(key);
+
+ if (data[index] === undefined) {
+ uncachedKeys.push(key);
+ uncachedIndexes.push(index);
+ getManyVersions.set(key, getInvalidationVersion(key));
+ }
+ });
+
+ if (!loader || !uncachedKeys.length) {
+ return data;
}
- return data;
+
+ return Promise.resolve()
+ .then(() => loader(uncachedKeys, uncachedIndexes))
+ .then((values) => {
+ uncachedKeys.forEach((key, index) => {
+ const value = values[index];
+ data[uncachedIndexes[index]] = value;
+ if (getInvalidationVersion(key) === getManyVersions.get(key)) {
+ cache.set(key, value);
+ }
+ });
+
+ return data;
+ });
};
cache.del = function (keys) {
@@ -77,8 +135,9 @@ module.exports = function (opts) {
if (!Array.isArray(keys)) {
keys = [keys];
}
- pubsub.publish(`${cache.name}:ttlCache:del`, keys);
- keys.forEach(key => ttlCache.delete(key));
+
+ localDel(keys);
+ pubsub.publish(`${cache.name}:ttlCache:del`, { id: `${os.hostname()}:${process.pid}`, keys });
};
cache.delete = cache.del;
@@ -96,40 +155,29 @@ module.exports = function (opts) {
cache.misses = 0;
}
+ function getInvalidationVersion(key) {
+ return invalidationVersions.get(key) || 0;
+ }
+
+ function localDel(keys) {
+ keys.forEach((key) => {
+ invalidationVersions.set(key, (invalidationVersions.get(key) || 0) + 1);
+ ttlCache.delete(key);
+ });
+ }
+
pubsub.on(`${cache.name}:ttlCache:reset`, ({ id }) => {
if (id !== `${os.hostname()}:${process.pid}`) {
localReset();
}
});
- pubsub.on(`${cache.name}:ttlCache:del`, (keys) => {
- if (Array.isArray(keys)) {
- keys.forEach(key => ttlCache.delete(key));
+ pubsub.on(`${cache.name}:ttlCache:del`, ({ id, keys }) => {
+ if (id !== `${os.hostname()}:${process.pid}` && Array.isArray(keys)) {
+ localDel(keys);
}
});
- cache.getUnCachedKeys = function (keys, cachedData) {
- if (!cache.enabled) {
- return keys;
- }
- let data;
- let isCached;
- const unCachedKeys = keys.filter((key) => {
- data = cache.get(key);
- isCached = data !== undefined;
- if (isCached) {
- cachedData[key] = data;
- }
- return !isCached;
- });
-
- const hits = keys.length - unCachedKeys.length;
- const misses = keys.length - hits;
- cache.hits += hits;
- cache.misses += misses;
- return unCachedKeys;
- };
-
cache.dump = function () {
return Array.from(ttlCache.entries());
};
diff --git a/src/categories/index.js b/src/categories/index.js
index 87e37d0b11..6c365536f0 100644
--- a/src/categories/index.js
+++ b/src/categories/index.js
@@ -98,13 +98,7 @@ Categories.getCidByHandle = async function (handle) {
};
Categories.getAllCidsFromSet = async function (key) {
- let cids = cache.get(key);
- if (cids) {
- return cids.slice();
- }
-
- cids = await db.getSortedSetRange(key, 0, -1);
- cache.set(key, cids);
+ const cids = await cache.get(key, () => db.getSortedSetRange(key, 0, -1));
return cids.slice();
};
@@ -185,29 +179,11 @@ Categories.setUnread = async function (tree, cids, uid) {
};
Categories.getTagWhitelist = async function (cids) {
- const cachedData = {};
-
- const nonCachedCids = cids.filter((cid) => {
- const data = cache.get(`cid:${cid}:tag:whitelist`);
- const isInCache = data !== undefined;
- if (isInCache) {
- cachedData[cid] = data;
- }
- return !isInCache;
- });
-
- if (!nonCachedCids.length) {
- return cids.map(cid => cachedData[cid]);
- }
-
- const keys = nonCachedCids.map(cid => `cid:${cid}:tag:whitelist`);
- const data = await db.getSortedSetsMembers(keys);
-
- nonCachedCids.forEach((cid, index) => {
- cachedData[cid] = data[index];
- cache.set(`cid:${cid}:tag:whitelist`, data[index]);
- });
- return cids.map(cid => cachedData[cid]);
+ const cachedData = await cache.getMany(
+ cids.map(cid => `cid:${cid}:tag:whitelist`),
+ uncachedKeys => db.getSortedSetsMembers(uncachedKeys)
+ );
+ return cachedData;
};
// remove system tags from tag whitelist for non privileged user
@@ -286,29 +262,25 @@ Categories.getParentCids = async function (currentCid) {
};
Categories.getChildrenCids = async function (rootCid) {
- let allCids = [];
+ const allCids = new Set();
async function recursive(keys) {
let childrenCids = await db.getSortedSetRange(keys, 0, -1);
- childrenCids = childrenCids.filter(cid => !allCids.includes(cid));
+ childrenCids = childrenCids.filter(cid => !allCids.has(cid));
if (!childrenCids.length) {
return;
}
- allCids.push(...childrenCids);
+ childrenCids.forEach(cid => allCids.add(cid));
keys = childrenCids.map(cid => `cid:${cid}:children`);
await recursive(keys);
}
const key = `cid:${rootCid}:children`;
const cacheKey = `${key}:all`;
- const childrenCids = cache.get(cacheKey);
- if (childrenCids) {
- return childrenCids.slice();
- }
-
- await recursive(key);
- allCids = _.uniq(allCids);
- cache.set(cacheKey, allCids);
- return allCids.slice();
+ const childrenCids = await cache.get(cacheKey, async () => {
+ await recursive(key);
+ return Array.from(allCids);
+ });
+ return childrenCids.slice();
};
Categories.flattenCategories = function (allCategories, categoryData) {
diff --git a/src/controllers/topics.js b/src/controllers/topics.js
index c3df881eab..e549f036d4 100644
--- a/src/controllers/topics.js
+++ b/src/controllers/topics.js
@@ -227,12 +227,10 @@ async function markAsRead(req, tid) {
async function loadCrosspostPrivilege(req, excludeCid) {
excludeCid = String(excludeCid || '');
- let cidsUserCanCrosspost = crosspostCache.get(`uid:${req.uid}`);
- if (cidsUserCanCrosspost === undefined) {
+ const cidsUserCanCrosspost = await crosspostCache.get(`uid:${req.uid}`, async () => {
const cids = await categories.getAllCidsFromSet('categories:cid');
- cidsUserCanCrosspost = await privileges.categories.filterCids('topics:crosspost', cids, req.uid);
- crosspostCache.set(`uid:${req.uid}`, cidsUserCanCrosspost);
- }
+ return await privileges.categories.filterCids('topics:crosspost', cids, req.uid);
+ });
return cidsUserCanCrosspost.some(cid => cid !== excludeCid);
}
diff --git a/src/database/mongo/hash.js b/src/database/mongo/hash.js
index 8dabf11831..bdac3f5bc8 100644
--- a/src/database/mongo/hash.js
+++ b/src/database/mongo/hash.js
@@ -98,11 +98,12 @@ module.exports = function (module) {
if (!key || !field) {
return null;
}
- const cachedData = {};
- cache.getUnCachedKeys([key], cachedData);
- if (cachedData[key]) {
- return cachedData[key].hasOwnProperty(field) ? cachedData[key][field] : null;
+
+ const cachedData = cache.get(key);
+ if (cachedData !== undefined) {
+ return cachedData?.hasOwnProperty(field) ? cachedData[field] : null;
}
+
field = helpers.fieldToString(field);
const item = await module.client.collection('objects').findOne({
_key: key,
@@ -127,28 +128,22 @@ module.exports = function (module) {
if (!Array.isArray(keys) || !keys.length) {
return [];
}
- const cachedData = {};
- const unCachedKeys = cache.getUnCachedKeys(keys, cachedData);
-
- if (unCachedKeys.length >= 1) {
+ const cachedData = await cache.getMany(keys, async (unCachedKeys) => {
let data = await module.client.collection('objects').find(
{ _key: unCachedKeys.length === 1 ? unCachedKeys[0] : { $in: unCachedKeys } },
{ projection: { _id: 0 } }
).toArray();
- data = data.map(helpers.deserializeData);
+ data = data.map(helpers.deserializeData);
const map = helpers.toMap(data);
- unCachedKeys.forEach((key) => {
- cachedData[key] = map[key] || null;
- cache.set(key, cachedData[key]);
- });
- }
+ return unCachedKeys.map(key => map[key] || null);
+ });
if (!Array.isArray(fields) || !fields.length) {
- return keys.map(key => (cachedData[key] ? { ...cachedData[key] } : null));
+ return cachedData.map(data => (data ? { ...data } : null));
}
- return keys.map((key) => {
- const item = cachedData[key] || {};
+ return cachedData.map((data) => {
+ const item = data || {};
const result = {};
fields.forEach((field) => {
result[field] = item[field] !== undefined ? item[field] : null;
diff --git a/src/database/postgres/hash.js b/src/database/postgres/hash.js
index f967e4ae4b..46e792ba17 100644
--- a/src/database/postgres/hash.js
+++ b/src/database/postgres/hash.js
@@ -132,10 +132,9 @@ module.exports = function (module) {
if (!key || !field) {
return null;
}
- const cachedData = {};
- cache.getUnCachedKeys([key], cachedData);
- if (cachedData[key]) {
- return cachedData[key].hasOwnProperty(field) ? cachedData[key][field] : null;
+ const cachedData = cache.get(key);
+ if (cachedData !== undefined) {
+ return cachedData?.hasOwnProperty(field) ? cachedData[field] : null;
}
const res = await module.pool.query({
@@ -167,10 +166,7 @@ SELECT h."data"->>$2::TEXT f
return [];
}
- const cachedData = {};
- const unCachedKeys = cache.getUnCachedKeys(keys, cachedData);
-
- if (unCachedKeys.length) {
+ const cachedData = await cache.getMany(keys, async (unCachedKeys) => {
const res = await module.pool.query({
name: 'getObjectsFields',
text: `
@@ -185,18 +181,14 @@ SELECT h."data"
values: [unCachedKeys],
});
const data = res.rows.map(row => row.data);
-
- unCachedKeys.forEach((key, i) => {
- cachedData[key] = data[i] || null;
- cache.set(key, cachedData[key]);
- });
- }
+ return data;
+ });
if (!Array.isArray(fields) || !fields.length) {
- return keys.map(key => (cachedData[key] ? { ...cachedData[key] } : null));
+ return cachedData.map(data => (data ? { ...data } : null));
}
- return keys.map((key) => {
- const item = cachedData[key] || {};
+ return cachedData.map((data) => {
+ const item = data || {};
const result = {};
fields.forEach((field) => {
result[field] = item[field] !== undefined ? item[field] : null;
diff --git a/src/database/redis/hash.js b/src/database/redis/hash.js
index f046e62180..1f0b1104ea 100644
--- a/src/database/redis/hash.js
+++ b/src/database/redis/hash.js
@@ -99,10 +99,12 @@ module.exports = function (module) {
if (!key || !field) {
return null;
}
- const cachedData = {};
- cache.getUnCachedKeys([key], cachedData);
- if (cachedData[key]) {
- return Object.hasOwn(cachedData[key], field) ? cachedData[key][field] : null;
+ const cachedData = cache.get(key);
+ if (cachedData !== undefined) {
+ if (cachedData !== null && typeof cachedData === 'object') {
+ return Object.hasOwn(cachedData, field) ? cachedData[field] : null;
+ }
+ return null;
}
return await module.client.hGet(key, String(field));
};
@@ -120,36 +122,31 @@ module.exports = function (module) {
return [];
}
- const cachedData = {};
- const unCachedKeys = cache.getUnCachedKeys(keys, cachedData);
-
- let data = [];
- if (unCachedKeys.length > 1) {
- const batch = module.client.batch();
- unCachedKeys.forEach(k => batch.hGetAll(k));
- data = await helpers.execBatch(batch);
- } else if (unCachedKeys.length === 1) {
- data = [await module.client.hGetAll(unCachedKeys[0])];
- }
+ let cachedData = await cache.getMany(keys, async (unCachedKeys) => {
+ let data = [];
+ if (unCachedKeys.length > 1) {
+ const batch = module.client.batch();
+ unCachedKeys.forEach(k => batch.hGetAll(k));
+ data = await helpers.execBatch(batch);
+ } else if (unCachedKeys.length === 1) {
+ data = [await module.client.hGetAll(unCachedKeys[0])];
+ }
+ return data;
+ });
// convert empty objects into null for back-compat with node_redis
- data = data.map((elem) => {
+ cachedData = cachedData.map((elem) => {
if (!Object.keys(elem).length) {
return null;
}
return elem;
});
- unCachedKeys.forEach((key, i) => {
- cachedData[key] = data[i] || null;
- cache.set(key, cachedData[key]);
- });
-
if (!Array.isArray(fields) || !fields.length) {
- return keys.map(key => (cachedData[key] ? { ...cachedData[key] } : null));
+ return cachedData.map(data => (data ? { ...data } : null));
}
- return keys.map((key) => {
- const item = cachedData[key] || {};
+ return cachedData.map((data) => {
+ const item = data || {};
const result = {};
fields.forEach((field) => {
result[field] = item[field] !== undefined ? item[field] : null;
diff --git a/src/groups/index.js b/src/groups/index.js
index 1f3d824839..e0597340b5 100644
--- a/src/groups/index.js
+++ b/src/groups/index.js
@@ -114,13 +114,7 @@ Groups.getNonPrivilegeGroups = async function (set, start, stop, flags) {
};
Groups.getAllGroupNames = async function (set) {
- const cacheKey = `zset:${set}`;
- let names = cache.get(cacheKey);
- if (names !== undefined) {
- return [...names];
- }
- names = await db.getSortedSetRevRange(set, 0, -1);
- cache.set(cacheKey, names);
+ const names = await cache.get(`zset:${set}`, () => db.getSortedSetRevRange(set, 0, -1));
return [...names];
};
diff --git a/src/groups/membership.js b/src/groups/membership.js
index b0bc10697c..70c187c4c6 100644
--- a/src/groups/membership.js
+++ b/src/groups/membership.js
@@ -29,13 +29,9 @@ module.exports = function (Groups) {
return isMemberOfEphemeralGroup(uid, groupName);
}
- const cacheKey = `${uid}:${groupName}`;
- let isMember = Groups.cache.get(cacheKey);
- if (isMember !== undefined) {
- return isMember;
- }
- isMember = await db.isSortedSetMember(`group:${groupName}:members`, uid);
- Groups.cache.set(cacheKey, isMember);
+ const isMember = await Groups.cache.get(
+ `${uid}:${groupName}`, () => db.isSortedSetMember(`group:${groupName}:members`, uid)
+ );
return isMember;
};
@@ -47,40 +43,30 @@ module.exports = function (Groups) {
if (groupName === 'guests' || groupName === 'spiders') {
return uids.map(uid => isMemberOfEphemeralGroup(uid, groupName));
}
-
- const cachedData = {};
- const nonCachedUids = uids.filter(uid => filterNonCached(cachedData, uid, groupName));
-
- if (!nonCachedUids.length) {
- return uids.map(uid => cachedData[`${uid}:${groupName}`]);
- }
-
- const isMembers = await db.isSortedSetMembers(`group:${groupName}:members`, nonCachedUids);
- nonCachedUids.forEach((uid, index) => {
- cachedData[`${uid}:${groupName}`] = isMembers[index];
- Groups.cache.set(`${uid}:${groupName}`, isMembers[index]);
- });
- return uids.map(uid => cachedData[`${uid}:${groupName}`]);
+ const cacheKeys = uids.map(uid => `${uid}:${groupName}`);
+ const isMembers = await Groups.cache.getMany(
+ cacheKeys,
+ async (uncachedKeys, uncachedIndexes) => {
+ const uncachedUids = uncachedIndexes.map(index => uids[index]);
+ return await db.isSortedSetMembers(`group:${groupName}:members`, uncachedUids);
+ }
+ );
+ return isMembers.slice();
};
Groups.isMemberOfGroups = async function (uid, groups) {
if (!uid || (utils.isNumber(uid) && parseInt(uid, 10) <= 0) || !groups.length) {
return groups.map(groupName => isMemberOfEphemeralGroup(uid, groupName));
}
- const cachedData = {};
- const nonCachedGroups = groups.filter(groupName => filterNonCached(cachedData, uid, groupName));
+ const keys = groups.map(groupName => `${uid}:${groupName}`);
- if (!nonCachedGroups.length) {
- return groups.map(groupName => cachedData[`${uid}:${groupName}`]);
- }
- const nonCachedGroupsMemberSets = nonCachedGroups.map(groupName => `group:${groupName}:members`);
- const isMembers = await db.isMemberOfSortedSets(nonCachedGroupsMemberSets, uid);
- nonCachedGroups.forEach((groupName, index) => {
- cachedData[`${uid}:${groupName}`] = isMembers[index];
- Groups.cache.set(`${uid}:${groupName}`, isMembers[index]);
- });
+ return await Groups.cache.getMany(keys, async (uncachedKeys, uncachedIndexes) => {
+ const nonCachedGroupsMemberSets = uncachedIndexes.map(index => (
+ `group:${groups[index]}:members`
+ ));
- return groups.map(groupName => cachedData[`${uid}:${groupName}`]);
+ return await db.isMemberOfSortedSets(nonCachedGroupsMemberSets, uid);
+ });
};
function isMemberOfEphemeralGroup(uid, groupName) {
@@ -90,15 +76,6 @@ module.exports = function (Groups) {
(groupName === 'spiders' && parsedUid === -1);
}
- function filterNonCached(cachedData, uid, groupName) {
- const isMember = Groups.cache.get(`${uid}:${groupName}`);
- const isInCache = isMember !== undefined;
- if (isInCache) {
- cachedData[`${uid}:${groupName}`] = isMember;
- }
- return !isInCache;
- }
-
Groups.isMemberOfAny = async function (uid, groups) {
if (!Array.isArray(groups) || !groups.length) {
return false;
@@ -159,27 +136,11 @@ module.exports = function (Groups) {
const isArray = Array.isArray(keys);
keys = isArray ? keys : [keys];
- const cachedData = {};
- const nonCachedKeys = keys.filter((groupName) => {
- const groupMembers = cache.get(`group:${groupName}:members`);
- const isInCache = groupMembers !== undefined;
- if (isInCache) {
- cachedData[groupName] = groupMembers;
- }
- return !isInCache;
- });
-
- if (!nonCachedKeys.length) {
- return isArray ? keys.map(groupName => cachedData[groupName]) : cachedData[keys[0]];
- }
- const groupMembers = await db.getSortedSetsMembers(
- nonCachedKeys.map(name => `group:${name}:members`)
+ const cacheKeys = keys.map(groupName => `group:${groupName}:members`);
+ const groupMembers = await cache.getMany(
+ cacheKeys, uncachedKeys => db.getSortedSetsMembers(uncachedKeys)
);
- nonCachedKeys.forEach((groupName, index) => {
- cachedData[groupName] = groupMembers[index];
- cache.set(`group:${groupName}:members`, groupMembers[index]);
- });
- return isArray ? keys.map(groupName => cachedData[groupName]) : cachedData[keys[0]];
+ return isArray ? groupMembers : groupMembers[0];
}
};
diff --git a/src/messaging/index.js b/src/messaging/index.js
index 26d4810cc1..d1a970c13b 100644
--- a/src/messaging/index.js
+++ b/src/messaging/index.js
@@ -115,11 +115,7 @@ Messaging.isNewSet = async (uid, roomId, timestamp) => {
Messaging.getPublicRoomIdsFromSet = async function (set) {
const cacheKey = `${set}:all`;
- let allRoomIds = cache.get(cacheKey);
- if (allRoomIds === undefined) {
- allRoomIds = await db.getSortedSetRange(set, 0, -1);
- cache.set(cacheKey, allRoomIds);
- }
+ const allRoomIds = await cache.get(cacheKey, () => db.getSortedSetRange(set, 0, -1));
return allRoomIds.slice();
};
diff --git a/src/messaging/rooms.js b/src/messaging/rooms.js
index ad7b26eab4..dbfa09b08b 100644
--- a/src/messaging/rooms.js
+++ b/src/messaging/rooms.js
@@ -406,14 +406,11 @@ module.exports = function (Messaging) {
}
Messaging.getAllUidsInRoomFromSet = async function (set) {
- const cacheKey = `${set}:all`;
- let uids = roomUidCache.get(cacheKey);
- if (uids !== undefined) {
- return uids;
- }
- uids = await Messaging.getUidsInRoomFromSet(set, 0, -1);
- roomUidCache.set(cacheKey, uids);
- return uids;
+ const uids = await roomUidCache.get(
+ `${set}:all`,
+ () => Messaging.getUidsInRoomFromSet(set, 0, -1)
+ );
+ return uids.slice();
};
Messaging.getUidsInRoomFromSet = async (set, start, stop, reverse = false) => db[
diff --git a/src/meta/settings.js b/src/meta/settings.js
index 213b61be11..89e460a3ff 100644
--- a/src/meta/settings.js
+++ b/src/meta/settings.js
@@ -11,30 +11,28 @@ const cache = require('../cache');
const Settings = module.exports;
Settings.get = async function (hash) {
- const cached = cache.get(`settings:${hash}`);
- if (cached) {
- return _.cloneDeep(cached);
- }
- const [data, sortedLists] = await Promise.all([
- db.getObject(`settings:${hash}`),
- db.getSetMembers(`settings:${hash}:sorted-lists`),
- ]);
- const values = data || {};
- await Promise.all(sortedLists.map(async (list) => {
- const members = await db.getSortedSetRange(`settings:${hash}:sorted-list:${list}`, 0, -1);
- const keys = members.map(order => `settings:${hash}:sorted-list:${list}:${order}`);
-
- values[list] = [];
-
- const objects = await db.getObjects(keys);
- objects.forEach((obj) => {
- values[list].push(obj);
- });
- }));
+ const cached = await cache.get(`settings:${hash}`, async () => {
+ const [data, sortedLists] = await Promise.all([
+ db.getObject(`settings:${hash}`),
+ db.getSetMembers(`settings:${hash}:sorted-lists`),
+ ]);
+ const values = data || {};
+ await Promise.all(sortedLists.map(async (list) => {
+ const members = await db.getSortedSetRange(`settings:${hash}:sorted-list:${list}`, 0, -1);
+ const keys = members.map(order => `settings:${hash}:sorted-list:${list}:${order}`);
+
+ values[list] = [];
- const result = await plugins.hooks.fire('filter:settings.get', { plugin: hash, values: values });
- cache.set(`settings:${hash}`, result.values);
- return _.cloneDeep(result.values);
+ const objects = await db.getObjects(keys);
+ objects.forEach((obj) => {
+ values[list].push(obj);
+ });
+ }));
+
+ const result = await plugins.hooks.fire('filter:settings.get', { plugin: hash, values: values });
+ return result.values;
+ });
+ return _.cloneDeep(cached);
};
Settings.getOne = async function (hash, field) {
diff --git a/src/plugins/install.js b/src/plugins/install.js
index 4c5cd2031f..b8df9a8fe8 100644
--- a/src/plugins/install.js
+++ b/src/plugins/install.js
@@ -222,12 +222,9 @@ module.exports = function (Plugins) {
if (nconf.get('plugins:active')) {
return nconf.get('plugins:active').includes(id);
}
- const cached = cache.get(`plugin:isActive:${id}`);
- if (cached !== undefined) {
- return cached;
- }
- const isActive = await db.isSortedSetMember('plugins:active', id);
- cache.set(`plugin:isActive:${id}`, isActive);
+ const isActive = await cache.get(`plugin:isActive:${id}`, async () => {
+ return await db.isSortedSetMember('plugins:active', id);
+ });
return isActive;
};
diff --git a/src/socket.io/topics.js b/src/socket.io/topics.js
index 3df9cdc1a2..1e30ebf65a 100644
--- a/src/socket.io/topics.js
+++ b/src/socket.io/topics.js
@@ -81,25 +81,19 @@ SocketTopics.getMyNextPostIndex = async function (socket, data) {
const topicSet = data.sort === 'most_votes' ? `tid:${data.tid}:posts:votes` : `tid:${data.tid}:posts`;
const reverse = data.sort === 'newest_to_oldest' || data.sort === 'most_votes';
const cacheKey = `np:s:${topicSet}:r:${String(reverse)}:tid:${data.tid}:pids`;
- const topicPids = cache.get(cacheKey);
- if (topicPids) {
- return topicPids.slice(index - 1);
- }
- const pids = await db[reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'](topicSet, 0, -1);
- cache.set(cacheKey, pids, 30000);
- return pids.slice(index - 1);
+ const topicPids = await cache.get(cacheKey, async () => {
+ return await db[reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'](topicSet, 0, -1);
+ });
+ return topicPids.slice(index - 1);
}
async function getUserPids() {
const cid = await topics.getTopicField(data.tid, 'cid');
const cacheKey = `np:cid:${cid}:uid:${socket.uid}:pids`;
- const userPids = cache.get(cacheKey);
- if (userPids) {
- return userPids;
- }
- const pids = await db.getSortedSetRange(`cid:${cid}:uid:${socket.uid}:pids`, 0, -1);
- cache.set(cacheKey, pids, 30000);
- return pids;
+ const userPids = await cache.get(cacheKey, async () => {
+ return await db.getSortedSetRange(`cid:${cid}:uid:${socket.uid}:pids`, 0, -1);
+ });
+ return userPids;
}
const postCountInTopic = await db.sortedSetScore(`tid:${data.tid}:posters`, socket.uid);
if (postCountInTopic <= 0) {
diff --git a/src/topics/tags.js b/src/topics/tags.js
index e7a903214c..388e2818ed 100644
--- a/src/topics/tags.js
+++ b/src/topics/tags.js
@@ -476,13 +476,11 @@ module.exports = function (Topics) {
};
async function getAllTags() {
- const cached = cache.get('tags:topic:count');
- if (cached !== undefined) {
- return cached;
- }
- const tags = await db.getSortedSetRevRangeWithScores('tags:topic:count', 0, -1);
- cache.set('tags:topic:count', tags);
- return tags;
+ const cached = await cache.get(
+ 'tags:topic:count',
+ () => db.getSortedSetRevRangeWithScores('tags:topic:count', 0, -1)
+ );
+ return cached;
}
async function findMatches(data) {
diff --git a/src/user/blocks.js b/src/user/blocks.js
index 4292e5dfcd..a10c8b47c6 100644
--- a/src/user/blocks.js
+++ b/src/user/blocks.js
@@ -49,17 +49,14 @@ module.exports = function (User) {
User.blocks.list = async function (uids) {
const isArray = Array.isArray(uids);
uids = (isArray ? uids : [uids]).map(uid => String(uid));
- const cachedData = {};
- const unCachedUids = User.blocks._cache.getUnCachedKeys(uids, cachedData);
- if (unCachedUids.length) {
- const unCachedData = await db.getSortedSetsMembers(unCachedUids.map(uid => `uid:${uid}:blocked_uids`));
- unCachedUids.forEach((uid, index) => {
- cachedData[uid] = (unCachedData[index] || []);
- User.blocks._cache.set(String(uid), cachedData[uid]);
- });
- }
- const result = uids.map(uid => cachedData[uid] || []);
- return isArray ? result.slice() : result[0];
+
+ const cachedData = await User.blocks._cache.getMany(uids, async (unCachedUids) => {
+ const unCachedData = await db.getSortedSetsMembers(
+ unCachedUids.map(uid => `uid:${uid}:blocked_uids`)
+ );
+ return unCachedUids.map((uid, index) => unCachedData[index]);
+ });
+ return isArray ? cachedData : cachedData[0];
};
User.blocks.add = async function (targetUid, uid) {