Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(engine): stop stale loads overwriting chat state
Yudhi Armyndharisf4deeae0fbbd
diff --git a/src/engine/adapters/baileys-chat-state-store.overlap.spec.ts b/src/engine/adapters/baileys-chat-state-store.overlap.spec.tsnew file mode 100644index 000000000..17558ca68--- /dev/null+++ b/src/engine/adapters/baileys-chat-state-store.overlap.spec.ts@@ -0,0 +1,436 @@+import { Repository } from 'typeorm';+import { ChatStateField, ChatStateStoreService } from './baileys-chat-state-store.service';+import { ChatState } from './baileys-chat-state.entity';++const KEY = (s: string, c: string): string => `${s}\u0000${c}`;+const settle = () => new Promise<void>(resolve => setImmediate(resolve));++type Waiting = { name: string; apply: () => unknown; answer: (value: unknown) => void };++/**+ * A repository whose calls wait for the test. `start` applies the oldest waiting call of a kind to the+ * table at once and returns the function that hands its answer back, so a read can run before a write+ * and still be answered after it, as calls over pooled connections can.+ */+function scriptedRepo(initial: Partial<ChatState>[] = []) {+ const rows = new Map<string, ChatState>();+ for (const r of initial) {+ const row = { muteEndTime: null, archived: false, pinned: false, observed: null, updatedAt: new Date(1), ...r };+ rows.set(KEY(r.sessionId!, r.chatId!), row as ChatState);+ }+ const waiting: Waiting[] = [];+ const answers: Array<() => void> = [];+ let stamp = 10;+ const issue = (name: string, apply: () => unknown) =>+ new Promise<unknown>(answer => waiting.push({ name, apply, answer }));+ const repo = {+ find: (opts?: { where?: { sessionId: string }; order?: { updatedAt?: 'DESC' }; take?: number }) =>+ issue('find', () => {+ const found = [...rows.values()].filter(r => !opts?.where || r.sessionId === opts.where.sessionId);+ const at = (r: ChatState) => r.updatedAt?.getTime() ?? 0;+ if (opts?.order?.updatedAt === 'DESC') found.sort((a, b) => at(b) - at(a));+ return (opts?.take ? found.slice(0, opts.take) : found).map(r => ({ ...r }));+ }),+ findOne: ({ where }: { where: { sessionId: string; chatId: string } }) =>+ issue('findOne', () => {+ const r = rows.get(KEY(where.sessionId, where.chatId));+ return r ? { ...r } : undefined;+ }),+ upsert: (v: ChatState) =>+ issue('upsert', () => {+ const k = KEY(v.sessionId, v.chatId);+ rows.set(k, { ...rows.get(k), ...v, updatedAt: new Date(stamp++) });+ }),+ delete: ({ sessionId, chatId }: { sessionId: string; chatId?: string }) =>+ issue('delete', () => {+ for (const [k, r] of rows)+ if (r.sessionId === sessionId && (chatId === undefined || r.chatId === chatId)) rows.delete(k);+ }),+ };+ const applyAt = (i: number): (() => void) => {+ const [call] = waiting.splice(i, 1);+ const value = call.apply();+ return () => call.answer(value);+ };+ /** Applies the oldest waiting call of that kind now; the returned function hands its answer back. */+ const start = async (name: string, nth = 0): Promise<() => Promise<void>> => {+ await settle();+ const i = waiting.map((w, ix) => (w.name === name ? ix : -1)).filter(ix => ix >= 0)[nth] ?? -1;+ if (i < 0) throw new Error(`no waiting ${name}; waiting: ${waiting.map(w => w.name).join(',')}`);+ const answer = applyAt(i);+ return async () => {+ answer();+ await settle();+ };+ };+ const run = async (name: string, nth = 0) => (await start(name, nth))();+ /** Runs what is waiting, oldest first, until nothing is. */+ const drain = async () => {+ for (await settle(); waiting.length; await settle()) await run(waiting[0].name);+ };+ return { repo, rows, waiting, answers, applyAt, start, run, drain };+}++const storeOf = (t: ReturnType<typeof scriptedRepo>) =>+ new ChatStateStoreService(t.repo as unknown as Repository<ChatState>);++describe('ChatStateStoreService loads that overlap a write, a forget or an unlink', () => {+ const ENV = 'BAILEYS_CHAT_STATE_CACHE_MAX';+ const orig = process.env[ENV];+ afterEach(() => {+ if (orig === undefined) delete process.env[ENV];+ else process.env[ENV] = orig;+ });++ const pinned = (chatId = 'c'): Partial<ChatState> => ({ sessionId: 's', chatId, pinned: true, observed: 'pinned' });++ it('does not bring a deleted chat back from a read-through begun before forget()', async () => {+ const t = scriptedRepo([pinned()]);+ const svc = storeOf(t);+ svc.get('s', 'c'); // a miss: the read-through starts+ const answer = await t.start('findOne'); // its SELECT runs while the row exists+ const forgotten = svc.forget('s', ['c']);+ await t.run('delete');+ await forgotten;+ await answer(); // and is answered after the delete+ expect(t.rows.size).toBe(0);+ expect(svc.chatIds('s')).toEqual([]);+ expect(svc.get('s', 'c')).toBeUndefined();+ });++ it('still writes the row of a chat pinned again after such a read-through', async () => {+ const t = scriptedRepo([pinned()]);+ const svc = storeOf(t);+ svc.get('s', 'c');+ const answer = await t.start('findOne');+ const forgotten = svc.forget('s', ['c']);+ await t.run('delete');+ await forgotten;+ await answer();+ const pin = svc.remember('s', 'c', { pinned: true }); // a new message re-created the chat, pinned again+ await t.drain();+ await pin;+ expect(t.rows.get(KEY('s', 'c'))?.pinned).toBe(true);+ });++ it('does not bring an unlinked account chat back from a read-through begun before clearSession()', async () => {+ const t = scriptedRepo([pinned()]);+ const svc = storeOf(t);+ svc.get('s', 'c');+ const answer = await t.start('findOne');+ const cleared = svc.clearSession('s');+ await t.run('delete');+ await cleared;+ await answer();+ expect(t.rows.size).toBe(0);+ expect(svc.chatIds('s')).toEqual([]);+ });++ it.each([+ ['reload', (svc: ChatStateStoreService) => svc.reload()],+ ['refreshSession', (svc: ChatStateStoreService) => svc.refreshSession('s')],+ ])('does not bring unlinked account chats back from a %s whose read ran before clearSession()', async (_n, load) => {+ const t = scriptedRepo([pinned()]);+ const svc = storeOf(t);+ const loading = load(svc);+ const answer = await t.start('find');+ const cleared = svc.clearSession('s');+ await t.run('delete');+ await cleared;+ await answer();+ await loading;+ expect(t.rows.size).toBe(0);+ expect(svc.chatIds('s')).toEqual([]);+ });++ it('does not list a deleted chat from a refresh whose read ran before forget()', async () => {+ const t = scriptedRepo([pinned()]);+ const svc = storeOf(t);+ const refresh = svc.refreshSession('s');+ const forgotten = svc.forget('s', ['c']);+ const answer = await t.start('find'); // the snapshot still holds the row+ await t.run('delete');+ await forgotten;+ await answer();+ await refresh;+ expect(t.rows.size).toBe(0);+ expect(svc.chatIds('s')).toEqual([]);+ });++ describe.each([+ ['reload', (svc: ChatStateStoreService) => svc.reload()],+ ['refreshSession', (svc: ChatStateStoreService) => svc.refreshSession('s')],+ ])('a %s whose read ran before a write landed', (_name, load) => {+ // The write reads the row and caches its change, the load reads the table (still pinned), then the+ // write lands and the load's older rows are handed back.+ const unpinWhileLoading = async () => {+ const t = scriptedRepo([{ ...pinned(), muteEndTime: 99, observed: 'muteEndTime,pinned' }]);+ const svc = storeOf(t);+ const unpin = svc.remember('s', 'c', { pinned: false });+ const loading = load(svc);+ await t.run('findOne');+ const answer = await t.start('find');+ await t.run('upsert');+ await unpin;+ await answer();+ await loading;+ return { t, svc };+ };++ it('never serves the older row over the written state', async () => {+ const { t, svc } = await unpinWhileLoading();+ expect(t.rows.get(KEY('s', 'c'))?.pinned).toBe(false);+ expect(svc.get('s', 'c')?.pinned).not.toBe(true);+ await t.drain(); // a cold read warms from the table+ expect(svc.get('s', 'c')).toMatchObject({ pinned: false, muteEndTime: 99 });+ });++ it('lets the next partial write merge onto the written state', async () => {+ const { t, svc } = await unpinWhileLoading();+ const archive = svc.remember('s', 'c', { archived: true });+ await t.drain();+ await archive;+ expect(t.rows.get(KEY('s', 'c'))).toMatchObject({ archived: true, pinned: false, muteEndTime: 99 });+ });+ });++ it('does not mark a stored chat absent from a no-row answer taken before its write', async () => {+ const t = scriptedRepo();+ const svc = storeOf(t);+ svc.get('s', 'c'); // a read-through for a chat with no row yet+ const write = svc.remember('s', 'c', { archived: true });+ const reload = svc.reload();+ const noRow = await t.start('findOne'); // the read-through finds nothing+ await t.run('findOne'); // the write finds nothing either and caches its change+ const snapshot = await t.start('find'); // the reload reads an empty table+ await t.run('upsert');+ await write;+ await snapshot();+ await reload;+ await noRow(); // the old no-row answer arrives last+ svc.get('s', 'c'); // a cold read must warm from the table, not trust an absent mark+ await t.drain();+ expect(svc.get('s', 'c')?.archived).toBe(true);+ });++ it('does not index an older read-through answer once cache pressure evicted the written chat', async () => {+ process.env[ENV] = '2';+ const t = scriptedRepo([{ ...pinned(), muteEndTime: 99, observed: 'muteEndTime,pinned' }]);+ const svc = storeOf(t);+ const unmute = svc.remember('s', 'c', { muteEndTime: null });+ svc.get('s', 'c'); // a chat-list read starts a read-through, queued before the write's own read+ await t.run('findOne', 1); // the write reads the row and caches its change+ const olderAnswer = await t.start('findOne'); // the read-through runs before the write lands: mute still 99+ await t.run('upsert');+ await unmute;+ for (const other of ['x', 'y']) {+ // two other chats fill the cap and evict the written one+ const w = svc.remember('s', other, { pinned: true });+ await t.drain();+ await w;+ }+ await olderAnswer();+ const unpin = svc.remember('s', 'c', { pinned: false }); // a partial patch for the chat+ await t.drain();+ await unpin;+ expect(t.rows.get(KEY('s', 'c'))).toMatchObject({ pinned: false, muteEndTime: null });+ });++ it('stays correct when more chats change during a load than the store keeps a record of', async () => {+ process.env[ENV] = '2';+ const t = scriptedRepo([pinned('a')]);+ const svc = storeOf(t);+ const refresh = svc.refreshSession('s');+ const snapshot = await t.start('find'); // the snapshot holds a, pinned+ for (const [chat, patch] of [+ ['a', { pinned: false }],+ ['x', { pinned: true }],+ ['y', { pinned: true }],+ ] as const) {+ const w = svc.remember('s', chat, patch);+ await t.drain();+ await w;+ }+ await snapshot();+ await refresh;+ expect(t.rows.get(KEY('s', 'a'))?.pinned).toBe(false);+ expect(svc.get('s', 'a')?.pinned).not.toBe(true);+ });+});++// A seeded scheduler: every repository call is run and answered in an order drawn from the seed, and+// the table (and, once everything has settled, the cache) is compared with a model of what the+// operations should have left.+describe('ChatStateStoreService under a seeded interleaving of loads and writes', () => {+ const ENV = 'BAILEYS_CHAT_STATE_CACHE_MAX';+ const orig = process.env[ENV];+ afterEach(() => {+ if (orig === undefined) delete process.env[ENV];+ else process.env[ENV] = orig;+ });++ const FIELDS: ChatStateField[] = ['muteEndTime', 'archived', 'pinned'];+ const SESSIONS = ['s1', 's2'];+ type Patch = { muteEndTime?: number | null; archived?: boolean; pinned?: boolean };+ type Op =+ | { t: 'remember'; s: string; c: string; patch: Patch; create: boolean }+ | { t: 'forget' | 'get'; s: string; c: string }+ | { t: 'refresh'; s: string }+ | { t: 'reload' };+ type Modelled = { muteEndTime: number | null; archived: boolean; pinned: boolean; observed: Set<string> };++ const mulberry32 = (seed: number) => {+ let a = seed;+ return () => {+ a = (a + 0x6d2b79f5) | 0;+ let t = Math.imul(a ^ (a >>> 15), 1 | a);+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;+ };+ };++ const generate = (rand: () => number, kinds: Array<Op['t']>): Op[] => {+ const pick = <T>(xs: T[]): T => xs[Math.floor(rand() * xs.length)];+ return Array.from({ length: 6 + Math.floor(rand() * 10) }, (): Op => {+ const t = pick(kinds);+ const s = pick(SESSIONS);+ const c = pick(CHATS);+ if (t === 'refresh') return { t, s };+ if (t === 'reload') return { t };+ if (t !== 'remember') return { t, s, c };+ const patch: Patch = {};+ if (rand() < 0.5) patch.pinned = rand() < 0.5;+ if (rand() < 0.4) patch.archived = rand() < 0.5;+ if (rand() < 0.3) patch.muteEndTime = rand() < 0.5 ? null : 1000 + Math.floor(rand() * 5);+ if (!Object.keys(patch).length) patch.pinned = rand() < 0.5;+ return { t, s, c, patch, create: rand() < 0.5 };+ });+ };++ /** What `remember` should leave in the table: the patch merged onto the row, unless it changes nothing. */+ const modelRemember = (model: Map<string, Modelled>, s: string, c: string, patch: Patch, create: boolean) => {+ const row = model.get(KEY(s, c));+ const carried = FIELDS.filter(f => Object.hasOwn(patch, f));+ if (!row && !create && carried.every(f => patch[f] === { muteEndTime: null, archived: false, pinned: false }[f]))+ return;+ const base = row ?? { muteEndTime: null, archived: false, pinned: false, observed: new Set<string>() };+ const observed = new Set([...base.observed, ...carried]);+ const next = { ...base, ...patch, observed };+ const same = FIELDS.every(f => next[f] === base[f]) && observed.size === base.observed.size;+ if (!same) model.set(KEY(s, c), next);+ };++ const describeTable = (rows: Iterable<ChatState>): string[] =>+ [...rows]+ .map(+ r =>+ `${r.sessionId}|${r.chatId}|${r.muteEndTime}|${r.archived}|${r.pinned}|${(r.observed ?? '').split(',').filter(Boolean).sort().join(',')}`,+ )+ .sort();++ const runScript = async (kinds: Array<Op['t']>, cap: number, seed: number): Promise<string[]> => {+ process.env[ENV] = String(cap);+ const rand = mulberry32(seed * 104729);+ const preload =+ rand() < 0.5+ ? [+ { s: 's1', c: CHATS[0] },+ { s: 's2', c: CHATS[1] },+ ]+ : [];+ const ops = generate(rand, kinds);+ const model = new Map<string, Modelled>();+ const t = scriptedRepo(+ preload.map(p => ({+ sessionId: p.s,+ chatId: p.c,+ muteEndTime: 99,+ pinned: true,+ observed: 'muteEndTime,pinned',+ })),+ );+ for (const p of preload) {+ model.set(KEY(p.s, p.c), {+ muteEndTime: 99,+ archived: false,+ pinned: true,+ observed: new Set(['muteEndTime', 'pinned']),+ });+ }+ const svc = storeOf(t);+ const issued: Promise<unknown>[] = [];+ const settleAll = async () => {+ for (await settle(); t.waiting.length || t.answers.length; await settle()) {+ if (t.waiting.length) t.answers.push(t.applyAt(Math.floor(rand() * t.waiting.length)));+ else t.answers.splice(Math.floor(rand() * t.answers.length), 1)[0]();+ }+ };+ let next = 0;+ while (next < ops.length || t.waiting.length || t.answers.length) {+ await settle();+ const choices = [+ ...(next < ops.length ? ['issue', 'issue'] : []),+ ...(t.waiting.length ? ['apply'] : []),+ ...(t.answers.length ? ['answer'] : []),+ ];+ const choice = choices[Math.floor(rand() * choices.length)];+ if (choice === 'apply') t.answers.push(t.applyAt(Math.floor(rand() * t.waiting.length)));+ else if (choice === 'answer') t.answers.splice(Math.floor(rand() * t.answers.length), 1)[0]();+ else {+ const op = ops[next++];+ if (op.t === 'remember') {+ modelRemember(model, op.s, op.c, op.patch, op.create);+ issued.push(svc.remember(op.s, op.c, op.patch, op.create));+ } else if (op.t === 'forget') {+ model.delete(KEY(op.s, op.c));+ issued.push(svc.forget(op.s, [op.c]));+ } else if (op.t === 'get') svc.get(op.s, op.c);+ else issued.push(op.t === 'refresh' ? svc.refreshSession(op.s) : svc.reload());+ }+ }+ await settleAll();+ await Promise.all(issued);+ const problems: string[] = [];+ const expected = describeTable(+ [...model].map(([k, v]) => {+ const [sessionId, chatId] = k.split('\u0000');+ return { sessionId, chatId, ...v, observed: [...v.observed].join(',') } as unknown as ChatState;+ }),+ );+ if (JSON.stringify(describeTable(t.rows.values())) !== JSON.stringify(expected)) {+ problems.push(`table ${JSON.stringify(describeTable(t.rows.values()))} but expected ${JSON.stringify(expected)}`);+ }+ if (cap === 0 || cap >= 100) {+ // Once settled, a read warmed from the table answers what the table holds.+ for (const s of SESSIONS) for (const c of CHATS) svc.get(s, c);+ await settleAll();+ for (const s of SESSIONS) {+ for (const c of CHATS) {+ const row = t.rows.get(KEY(s, c));+ const got = svc.get(s, c);+ const want = row && [row.muteEndTime, row.archived, row.pinned];+ if (JSON.stringify(got && [got.muteEndTime, got.archived, got.pinned]) !== JSON.stringify(want)) {+ problems.push(`cache of ${s}/${c} is ${JSON.stringify(got)} but the table holds ${JSON.stringify(row)}`);+ }+ }+ }+ }+ return problems.map(p => `seed ${seed} ${p}\n ops ${JSON.stringify(ops)}\n preload ${JSON.stringify(preload)}`);+ };++ it.each([+ ['writes, forgets and reads', ['remember', 'remember', 'remember', 'forget', 'get'] as Array<Op['t']>, 5000],+ ['and a refresh', ['remember', 'remember', 'remember', 'forget', 'get', 'refresh'] as Array<Op['t']>, 5000],+ ['and a reload', ['remember', 'remember', 'remember', 'forget', 'get', 'reload'] as Array<Op['t']>, 5000],+ ['under cache pressure', ['remember', 'remember', 'forget', 'get', 'refresh', 'reload'] as Array<Op['t']>, 2],+ ])(+ 'leaves the table as the operations imply: %s',+ async (_name, kinds, cap) => {+ const problems: string[] = [];+ for (let seed = 1; seed <= 600; seed++) problems.push(...(await runScript(kinds, cap, seed)));+ expect({ failing: problems.length, first: problems.slice(0, 2) }).toEqual({ failing: 0, first: [] });+ },+ 60_000,+ );+});diff --git a/src/engine/adapters/baileys-chat-state-store.service.ts b/src/engine/adapters/baileys-chat-state-store.service.tsindex b65e5bfba..47b9a5a45 100644--- a/src/engine/adapters/baileys-chat-state-store.service.ts+++ b/src/engine/adapters/baileys-chat-state-store.service.ts@@ -159,6 +159,19 @@ export class ChatStateStoreService implements ChatStateStore, OnModuleInit {private readonly maxEntries: number;/** One write chain per chat, so each remember() merges onto the state the previous one left. */private readonly writes = new KeyedMutationQueue();+ /**+ * A load (reload, refreshSession, a read-through) reads the table while writes go on, so its rows can+ * be older than a change made after the read began. `inFlight` counts a chat's writes and forgets+ * queued or running. Each one, as it settles, and every unlink of a session takes the next `touchSeq`+ * and is recorded in `touched` under the chat's key (the session's prefix, for an unlink). A load+ * notes `touchSeq` before it reads and, when the answer comes, skips any row {@link changedSince}+ * flags. `touched` is bounded by the cache cap: when full it is dropped and `touchedFloor` makes every+ * load begun before then distrust all of its rows, which only costs those chats another read.+ */+ private readonly inFlight = new Map<string, number>();+ private touchSeq = 0;+ private touchedFloor = 0;+ private readonly touched = new Map<string, number>();constructor(@InjectRepository(ChatState, 'data')@@ -174,6 +187,7 @@ export class ChatStateStoreService implements ChatStateStore, OnModuleInit {async reload(): Promise<void> {const seq = this.writeSeq;const idle = this.writes.size === 0;+ const since = this.touchSeq;// Dropped before the read, as refreshSession does: a restore may have replaced the table, so if// the read fails, every chat not cached must read through rather than trust a stale mark.this.completeSessions.clear();@@ -188,10 +202,13 @@ export class ChatStateStoreService implements ChatStateStore, OnModuleInit {this.completeSessions.clear();// Oldest first, so the newest row ends at the most-recent end of the LRU rather than the first// one evicted (the query is DESC only so `take` keeps the newest rows).+ let skipped = false;for (const row of [...rows].reverse()) {- this.index(this.key(row.sessionId, row.chatId), fromRow(row));+ const k = this.key(row.sessionId, row.chatId);+ if (this.changedSince(k, since)) skipped = true;+ else this.index(k, fromRow(row));}- if (this.loadWasWhole(rows.length, idle && this.writeSeq === seq && this.writes.size === 0)) {+ if (this.loadWasWhole(rows.length, idle && this.writeSeq === seq && this.writes.size === 0 && !skipped)) {for (const row of rows) this.completeSessions.add(row.sessionId);}this.logger.log(@@ -246,7 +263,16 @@ export class ChatStateStoreService implements ChatStateStore, OnModuleInit {/** Queues work on the chat's write chain and counts it under the session until it settles. */private enqueue(sessionId: string, k: string, work: () => Promise<void>): Promise<void> {const { pending } = this.writesOf(sessionId);- const done = new Promise<void>((resolve, reject) => this.writes.enqueue(k, () => work().then(resolve, reject)));+ this.inFlight.set(k, (this.inFlight.get(k) ?? 0) + 1);+ // Settled inside the chain, before the work counts as done, so a load never sees the chat idle+ // without also seeing the change.+ const done = new Promise<void>((resolve, reject) =>+ this.writes.enqueue(k, () =>+ work()+ .finally(() => this.settled(k))+ .then(resolve, reject),+ ),+ );pending.add(done);const settle = () => pending.delete(done);done.then(settle, settle);@@ -272,6 +298,33 @@ export class ChatStateStoreService implements ChatStateStore, OnModuleInit {this.writesOf(sessionId).seq++;}+ /** One write or forget of the chat is over: it no longer counts as in flight, and the chat has changed. */+ private settled(k: string): void {+ const left = (this.inFlight.get(k) ?? 1) - 1;+ if (left > 0) this.inFlight.set(k, left);+ else this.inFlight.delete(k);+ this.touch(k);+ }++ /** Records a change to the chat `key`, or to a whole session when it is the session's prefix. */+ private touch(key: string): void {+ if (this.maxEntries && this.touched.size >= this.maxEntries) {+ this.touched.clear();+ this.touchedFloor = this.touchSeq;+ }+ this.touched.set(key, ++this.touchSeq);+ }++ /**+ * Whether the row a load read for chat `k` may be older than the chat's state: a write or forget of+ * it is under way, or it, or its session, changed after the load began at `since`.+ */+ private changedSince(k: string, since: number): boolean {+ if (this.inFlight.has(k)) return true;+ const last = (key: string): number => this.touched.get(key) ?? this.touchedFloor;+ return last(k) > since || last(k.slice(0, k.indexOf(SEP) + 1)) > since;+ }+/*** The rows merge by {@link mergeTwinStates}, and each row is read from the table when the cache does* not hold it: a key the cache lacks may still have a newer row on disk, so a cached twin alone must@@ -455,6 +508,8 @@ export class ChatStateStoreService implements ChatStateStore, OnModuleInit {for (const k of [...this.states.keys()]) {if (k.startsWith(prefix)) this.states.delete(k);}+ // And noted, for a load whose read ran before the delete and is answered after these entries go.+ this.touch(prefix);}/**@@ -496,6 +551,7 @@ export class ChatStateStoreService implements ChatStateStore, OnModuleInit {const writes = this.writesOf(sessionId);const seq = writes.seq;const idle = writes.pending.size === 0;+ const since = this.touchSeq;this.completeSessions.delete(sessionId);let rows: ChatState[] | undefined;try {@@ -513,24 +569,31 @@ export class ChatStateStoreService implements ChatStateStore, OnModuleInit {if (k.startsWith(prefix)) this.absent.delete(k);}if (!rows) return;+ // Dropped again here, as reload does: a refresh begun later may have marked the session complete+ // while this read was out, and dropping the entries below leaves the cache short of its rows.+ this.completeSessions.delete(sessionId);for (const k of [...this.states.keys()]) {if (k.startsWith(prefix)) this.states.delete(k);}+ let skipped = false;for (const row of [...rows].reverse()) {// Oldest first, as in reload.- this.index(this.key(sessionId, row.chatId), fromRow(row));+ const k = this.key(sessionId, row.chatId);+ if (this.changedSince(k, since)) skipped = true;+ else this.index(k, fromRow(row));}// Marked after the loop: indexing these rows can evict other sessions' keys, never this one's.// Only this session's writes can leave its keys out of the load, so another session's do not count.- if (this.loadWasWhole(rows.length, idle && writes.seq === seq && writes.pending.size === 0)) {+ if (this.loadWasWhole(rows.length, idle && writes.seq === seq && writes.pending.size === 0 && !skipped)) {this.completeSessions.add(sessionId);}}/*** True when a load holds every row it asked for and no write was in flight from the start of its read- * to the end (`quiet`). A write in flight during the read may be missing from `rows` while its key was- * just dropped from the cache, even one that finished before the read did, so the session cannot be+ * to the end, and it set none of its rows aside as older than a change (`quiet`). A write in flight+ * during the read may be missing from `rows` while its key was just dropped from the cache, even one+ * that finished before the read did, and a row set aside is not cached, so the session cannot be* vouched for; skipping the mark only keeps the read-through path.*/private loadWasWhole(count: number, quiet: boolean): boolean {@@ -541,10 +604,13 @@ export class ChatStateStoreService implements ChatStateStore, OnModuleInit {private warmFromTable(k: string, sessionId: string, chatId: string): void {if (this.pendingLookups.has(k)) return;this.pendingLookups.add(k);+ const since = this.touchSeq;void this.repo.findOne({ where: { sessionId, chatId } }).then(row => {- if (this.states.has(k)) return;+ // An answer older than a change to the chat is dropped, not cached and not read as "no row": the+ // next read asks again.+ if (this.states.has(k) || this.changedSince(k, since)) return;if (row) this.index(k, fromRow(row));else this.markAbsent(k);})