Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

perf(cache): batch hit_count persistence instead of one UPDATE per hit (#1319)

suantea (asuan_suan)aed4c66a8c22
diff --git a/server/src/__tests__/services/cache.test.ts b/server/src/__tests__/services/cache.test.ts
index 88809756..224b2710 100644
--- a/server/src/__tests__/services/cache.test.ts
+++ b/server/src/__tests__/services/cache.test.ts
@@ -1,4 +1,4 @@
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { initDb, setSetting, getDb } from '../../db/index.js';
import {
computeCacheKey,
@@ -682,6 +682,70 @@ describe('response cache', () => {
expect(rowCount()).toBe(0);
});
+ describe('batched hit-count flush', () => {
+ const hitRow = (key: string) => getDb()
+ .prepare('SELECT hit_count, last_hit_at_ms FROM response_cache WHERE cache_key = ?')
+ .get(key) as { hit_count: number; last_hit_at_ms: number | null } | undefined;
+
+ beforeEach(() => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); });
+ afterEach(() => { vi.useRealTimers(); });
+
+ it('writes many hits in one debounced pass', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'batched')] });
+ const t0 = 5_000_000;
+ store(key, 'answer', t0);
+ __flushPersistenceForTests();
+ getCachedResponse(key, t0 + 1);
+ getCachedResponse(key, t0 + 2);
+ getCachedResponse(key, t0 + 3);
+ __flushPersistenceForTests();
+ // Nothing written per hit: the row still holds the stored value.
+ expect(hitRow(key)!.hit_count).toBe(0);
+
+ vi.advanceTimersByTime(2_000);
+ __flushPersistenceForTests();
+ expect(hitRow(key)).toEqual({ hit_count: 3, last_hit_at_ms: t0 + 3 });
+ });
+
+ it('clearCache() drops pending counts so they never land on a re-stored entry', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'cleared-pending')] });
+ store(key, 'old answer');
+ __flushPersistenceForTests();
+ getCachedResponse(key);
+ getCachedResponse(key);
+
+ clearCache();
+ store(key, 'fresh answer');
+ __flushPersistenceForTests();
+ vi.advanceTimersByTime(2_000);
+ __flushPersistenceForTests();
+
+ expect(hitRow(key)!.hit_count).toBe(0);
+ expect(getCacheStats().totalHits).toBe(0);
+ });
+
+ it('a re-store drops the old entry\'s pending count', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'restored-pending')] });
+ store(key, 'old answer');
+ __flushPersistenceForTests();
+ getCachedResponse(key);
+ getCachedResponse(key);
+
+ // Overwrite before the flush timer fires: the fresh row starts at 0.
+ store(key, 'new answer');
+ __flushPersistenceForTests();
+ vi.advanceTimersByTime(2_000);
+ __flushPersistenceForTests();
+ expect(hitRow(key)!.hit_count).toBe(0);
+
+ // Hits on the new entry still flush normally.
+ getCachedResponse(key);
+ vi.advanceTimersByTime(2_000);
+ __flushPersistenceForTests();
+ expect(hitRow(key)!.hit_count).toBe(1);
+ });
+ });
+
it('clearCache() purges SQLite too, so a restart does not resurrect entries', () => {
const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'flushed')] });
store(key, 'answer');
diff --git a/server/src/services/cache.ts b/server/src/services/cache.ts
index 8868ea4c..09244c06 100644
--- a/server/src/services/cache.ts
+++ b/server/src/services/cache.ts
@@ -382,6 +382,34 @@ function scheduleRowDelete(cacheKey: string): void {
});
}
+// Batched hit-count persistence: hits dirtied since the last flush, written in
+// one debounced pass. The flush rides the same drain as other queued writes.
+// A pending count belongs to the entry that earned it: clearCache() and a
+// re-store both drop it, so a stale count is never flushed onto a fresh row.
+const HIT_FLUSH_DELAY_MS = 2_000;
+const dirtyHitCounts = new Map<string, { hitCount: number; lastHitAtMs: number }>();
+let hitFlushTimer: NodeJS.Timeout | undefined;
+function scheduleHitCountFlush(): void {
+ if (hitFlushTimer) return;
+ hitFlushTimer = setTimeout(() => {
+ hitFlushTimer = undefined;
+ if (dirtyHitCounts.size === 0) return;
+ const batch = new Map(dirtyHitCounts);
+ dirtyHitCounts.clear();
+ schedulePersist(() => {
+ const stmt = getDb().prepare('UPDATE response_cache SET hit_count = ?, last_hit_at_ms = ? WHERE cache_key = ?');
+ for (const [cacheKey, hit] of batch) stmt.run(hit.hitCount, hit.lastHitAtMs, cacheKey);
+ });
+ }, HIT_FLUSH_DELAY_MS);
+ hitFlushTimer.unref();
+}
+
+function dropPendingHitCounts(): void {
+ dirtyHitCounts.clear();
+ if (hitFlushTimer) clearTimeout(hitFlushTimer);
+ hitFlushTimer = undefined;
+}
+
/**
* Test-only: run the queued write-through immediately instead of on the next
* tick, so a synchronous test can assert on what actually reached SQLite.
@@ -433,13 +461,11 @@ export function getCachedResponse(cacheKey: string, now = Date.now()): CachedRes
store.set(cacheKey, entry);
// Keep the persisted counters in step so the savings numbers survive a
- // restart instead of resetting to whatever the last store wrote.
- const hitCount = entry.hitCount;
- schedulePersist(() => {
- getDb()
- .prepare('UPDATE response_cache SET hit_count = ?, last_hit_at_ms = ? WHERE cache_key = ?')
- .run(hitCount, now, cacheKey);
- });
+ // restart instead of resetting to whatever the last store wrote. Hits are
+ // batched: one debounced UPDATE for all keys dirtied in the window, instead
+ // of one queued UPDATE per hit (a hot entry used to write on every hit).
+ dirtyHitCounts.set(cacheKey, { hitCount: entry.hitCount, lastHitAtMs: now });
+ scheduleHitCountFlush();
return {
body: entry.body,
@@ -473,8 +499,10 @@ export function storeCachedResponse(cacheKey: string, input: StoreInput, now = D
return;
}
- // Delete-then-set so an overwrite also refreshes recency order.
+ // Delete-then-set so an overwrite also refreshes recency order. Any hits
+ // still waiting to be flushed counted the old answer, not this one.
store.delete(cacheKey);
+ dirtyHitCounts.delete(cacheKey);
store.set(cacheKey, {
body: input.body,
platform: input.platform,
@@ -756,6 +784,7 @@ export function clearCache(): number {
store.clear();
streamStore.clear();
pendingWrites.length = 0;
+ dropPendingHitCounts();
// The lookup tallies describe the cache that just went away; keeping them
// would show a 90% hit rate next to zero entries right after a flush.
lookupHits = 0;