Node.js based forum software built for the modern web
feat: add inflight request handling to lru/ttl cache
Barış Soner Uşaklı3b778b9b8f39
diff --git a/src/cache/lru.js b/src/cache/lru.jsindex 3650e3a2a9..afe198e089 100644--- a/src/cache/lru.js+++ b/src/cache/lru.js@@ -66,6 +66,8 @@ module.exports = function (opts) {max: Math.max(10000, opts.max ? Math.floor(opts.max / 5) : 10000),});+ const inflight = new Map();+cache.has = function (key) {if (!cache.enabled) {return false;@@ -100,15 +102,22 @@ module.exports = function (opts) {if (!loader) {return undefined;}+ if (inflight.has(key)) {+ return inflight.get(key);+ }+const version = getInvalidationVersion(key);- return Promise.resolve()+ const promise = Promise.resolve().then(() => loader()).then((value) => {if (getInvalidationVersion(key) === version) {cache.set(key, value);}return value;- });+ }).finally(() => inflight.delete(key));++ inflight.set(key, promise);+ return promise;};cache.getMany = function (keys, loader) {@@ -121,35 +130,73 @@ module.exports = function (opts) {const data = new Array(keys.length);const uncachedKeys = [];const uncachedIndexes = [];- const getManyVersions = new Map();+ const pendingPromises = [];keys.forEach((key, index) => {- data[index] = cache.get(key);+ const cachedValue = lruCache.get(key);- if (data[index] === undefined) {+ if (cachedValue !== undefined) {+ cache.hits += 1;+ data[index] = cachedValue;+ } else if (inflight.has(key)) {+ // Re-use active in-flight promise for key+ cache.hits += 1;+ const keyPromise = inflight.get(key).then((val) => {+ data[index] = val;+ });+ pendingPromises.push(keyPromise);+ } else {+ cache.misses += 1;uncachedKeys.push(key);uncachedIndexes.push(index);- getManyVersions.set(key, getInvalidationVersion(key));}});- if (!loader || !uncachedKeys.length) {+ // If no loader provided or all keys resolved synchronously from LRU cache+ if (!loader || (uncachedKeys.length === 0 && pendingPromises.length === 0)) {return data;}- return Promise.resolve()+ // Fast-path: no missing keys, but waiting on in-flight promises from other calls+ if (uncachedKeys.length === 0) {+ return Promise.all(pendingPromises).then(() => data);+ }++ // Execute loader for keys not currently cached or in-flight+ const loaderPromise = Promise.resolve().then(() => loader(uncachedKeys, uncachedIndexes)).then((values) => {- uncachedKeys.forEach((key, index) => {- const value = values[index];- data[uncachedIndexes[index]] = value;- if (getInvalidationVersion(key) === getManyVersions.get(key)) {+ const resultMap = new Map();+ uncachedKeys.forEach((key, i) => {+ const value = values[i];+ resultMap.set(key, value);+ });+ return resultMap;+ });++ // Register individual keys into inflight map+ uncachedKeys.forEach((key, i) => {+ const targetIndex = uncachedIndexes[i];+ const version = getInvalidationVersion(key);++ const keyPromise = loaderPromise+ .then((resultMap) => {+ const value = resultMap.get(key);+ if (getInvalidationVersion(key) === version) {cache.set(key, value);}+ data[targetIndex] = value;+ return value;+ })+ .finally(() => {+ inflight.delete(key);});- return data;- });+ inflight.set(key, keyPromise);+ pendingPromises.push(keyPromise);+ });++ return Promise.all(pendingPromises).then(() => data);};cache.del = function (keys) {diff --git a/src/cache/ttl.js b/src/cache/ttl.jsindex 57abc312e7..58622dca20 100644--- a/src/cache/ttl.js+++ b/src/cache/ttl.js@@ -42,6 +42,8 @@ module.exports = function (opts) {max: Math.max(10000, opts.max ? Math.floor(opts.max / 5) : 10000),});+ const inflight = new Map();+cache.has = function (key) {if (!cache.enabled) {return false;@@ -76,15 +78,22 @@ module.exports = function (opts) {if (!loader) {return undefined;}+ if (inflight.has(key)) {+ return inflight.get(key);+ }+const version = getInvalidationVersion(key);- return Promise.resolve()+ const promise = Promise.resolve().then(() => loader()).then((value) => {if (getInvalidationVersion(key) === version) {cache.set(key, value);}return value;- });+ }).finally(() => inflight.delete(key));++ inflight.set(key, promise);+ return promise;};cache.getMany = function (keys, loader) {@@ -97,35 +106,73 @@ module.exports = function (opts) {const data = new Array(keys.length);const uncachedKeys = [];const uncachedIndexes = [];- const getManyVersions = new Map();+ const pendingPromises = [];keys.forEach((key, index) => {- data[index] = cache.get(key);-- if (data[index] === undefined) {+ const cachedValue = ttlCache.get(key);++ if (cachedValue !== undefined) {+ cache.hits += 1;+ data[index] = cachedValue;+ } else if (inflight.has(key)) {+ // Re-use active in-flight promise for key+ cache.hits += 1;+ const keyPromise = inflight.get(key).then((val) => {+ data[index] = val;+ });+ pendingPromises.push(keyPromise);+ } else {+ cache.misses += 1;uncachedKeys.push(key);uncachedIndexes.push(index);- getManyVersions.set(key, getInvalidationVersion(key));}});- if (!loader || !uncachedKeys.length) {+ // If no loader provided or all keys resolved synchronously from LRU cache+ if (!loader || (uncachedKeys.length === 0 && pendingPromises.length === 0)) {return data;}- return Promise.resolve()+ // Fast-path: no missing keys, but waiting on in-flight promises from other calls+ if (uncachedKeys.length === 0) {+ return Promise.all(pendingPromises).then(() => data);+ }++ // Execute loader for keys not currently cached or in-flight+ const loaderPromise = Promise.resolve().then(() => loader(uncachedKeys, uncachedIndexes)).then((values) => {- uncachedKeys.forEach((key, index) => {- const value = values[index];- data[uncachedIndexes[index]] = value;- if (getInvalidationVersion(key) === getManyVersions.get(key)) {+ const resultMap = new Map();+ uncachedKeys.forEach((key, i) => {+ const value = values[i];+ resultMap.set(key, value);+ });+ return resultMap;+ });++ // Register individual keys into inflight map+ uncachedKeys.forEach((key, i) => {+ const targetIndex = uncachedIndexes[i];+ const version = getInvalidationVersion(key);++ const keyPromise = loaderPromise+ .then((resultMap) => {+ const value = resultMap.get(key);+ if (getInvalidationVersion(key) === version) {cache.set(key, value);}+ data[targetIndex] = value;+ return value;+ })+ .finally(() => {+ inflight.delete(key);});- return data;- });+ inflight.set(key, keyPromise);+ pendingPromises.push(keyPromise);+ });++ return Promise.all(pendingPromises).then(() => data);};cache.del = function (keys) {diff --git a/test/cache.js b/test/cache.jsnew file mode 100644index 0000000000..0dd4cd7887--- /dev/null+++ b/test/cache.js@@ -0,0 +1,50 @@+'use strict';++const async = require('async');+const assert = require('assert');++const db = require('./mocks/databasemock');++const cacheCreate = require('../src/cache/lru');++describe('cache', () => {+ it('should only call loader function once if key is same', () => {+ const cache = cacheCreate({ name: 'testCache1', max: 10 });+ let loadCount = 0;+ const loader = async () => {+ loadCount += 1;+ return 'value';+ };++ return Promise.all([+ cache.get('key', loader),+ cache.get('key', loader),+ cache.get('key', loader),+ ]).then(([a, b, c]) => {+ assert.strictEqual(a, 'value');+ assert.strictEqual(b, 'value');+ assert.strictEqual(c, 'value');+ assert.strictEqual(loadCount, 1);+ });+ });++ it('should only call loader function once if keys are same', () => {+ const cache = cacheCreate({ name: 'testCache2', max: 10 });+ let loadCount = 0;+ const loader = async (keys) => {+ loadCount += 1;+ return keys.map(() => 'value');+ };++ return Promise.all([+ cache.getMany(['key1', 'key2'], loader),+ cache.getMany(['key1', 'key2'], loader),+ cache.getMany(['key1', 'key2'], loader),+ ]).then(([a, b, c]) => {+ assert.deepStrictEqual(a, ['value', 'value']);+ assert.deepStrictEqual(b, ['value', 'value']);+ assert.deepStrictEqual(c, ['value', 'value']);+ assert.strictEqual(loadCount, 1);+ });+ });+});