7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
perf(cache): batch hit_count persistence instead of one UPDATE per hit (#1319)
suantea (asuan_suan)aed4c66a8c22
diff --git a/server/src/__tests__/services/cache.test.ts b/server/src/__tests__/services/cache.test.tsindex 88809756..224b2710 100644--- a/server/src/__tests__/services/cache.test.ts+++ b/server/src/__tests__/services/cache.test.ts@@ -1,4 +1,4 @@-import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';import { initDb, setSetting, getDb } from '../../db/index.js';import {computeCacheKey,@@ -682,6 +682,70 @@ describe('response cache', () => {expect(rowCount()).toBe(0);});+ describe('batched hit-count flush', () => {+ const hitRow = (key: string) => getDb()+ .prepare('SELECT hit_count, last_hit_at_ms FROM response_cache WHERE cache_key = ?')+ .get(key) as { hit_count: number; last_hit_at_ms: number | null } | undefined;++ beforeEach(() => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); });+ afterEach(() => { vi.useRealTimers(); });++ it('writes many hits in one debounced pass', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'batched')] });+ const t0 = 5_000_000;+ store(key, 'answer', t0);+ __flushPersistenceForTests();+ getCachedResponse(key, t0 + 1);+ getCachedResponse(key, t0 + 2);+ getCachedResponse(key, t0 + 3);+ __flushPersistenceForTests();+ // Nothing written per hit: the row still holds the stored value.+ expect(hitRow(key)!.hit_count).toBe(0);++ vi.advanceTimersByTime(2_000);+ __flushPersistenceForTests();+ expect(hitRow(key)).toEqual({ hit_count: 3, last_hit_at_ms: t0 + 3 });+ });++ it('clearCache() drops pending counts so they never land on a re-stored entry', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'cleared-pending')] });+ store(key, 'old answer');+ __flushPersistenceForTests();+ getCachedResponse(key);+ getCachedResponse(key);++ clearCache();+ store(key, 'fresh answer');+ __flushPersistenceForTests();+ vi.advanceTimersByTime(2_000);+ __flushPersistenceForTests();++ expect(hitRow(key)!.hit_count).toBe(0);+ expect(getCacheStats().totalHits).toBe(0);+ });++ it('a re-store drops the old entry\'s pending count', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'restored-pending')] });+ store(key, 'old answer');+ __flushPersistenceForTests();+ getCachedResponse(key);+ getCachedResponse(key);++ // Overwrite before the flush timer fires: the fresh row starts at 0.+ store(key, 'new answer');+ __flushPersistenceForTests();+ vi.advanceTimersByTime(2_000);+ __flushPersistenceForTests();+ expect(hitRow(key)!.hit_count).toBe(0);++ // Hits on the new entry still flush normally.+ getCachedResponse(key);+ vi.advanceTimersByTime(2_000);+ __flushPersistenceForTests();+ expect(hitRow(key)!.hit_count).toBe(1);+ });+ });+it('clearCache() purges SQLite too, so a restart does not resurrect entries', () => {const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'flushed')] });store(key, 'answer');diff --git a/server/src/services/cache.ts b/server/src/services/cache.tsindex 8868ea4c..09244c06 100644--- a/server/src/services/cache.ts+++ b/server/src/services/cache.ts@@ -382,6 +382,34 @@ function scheduleRowDelete(cacheKey: string): void {});}+// Batched hit-count persistence: hits dirtied since the last flush, written in+// one debounced pass. The flush rides the same drain as other queued writes.+// A pending count belongs to the entry that earned it: clearCache() and a+// re-store both drop it, so a stale count is never flushed onto a fresh row.+const HIT_FLUSH_DELAY_MS = 2_000;+const dirtyHitCounts = new Map<string, { hitCount: number; lastHitAtMs: number }>();+let hitFlushTimer: NodeJS.Timeout | undefined;+function scheduleHitCountFlush(): void {+ if (hitFlushTimer) return;+ hitFlushTimer = setTimeout(() => {+ hitFlushTimer = undefined;+ if (dirtyHitCounts.size === 0) return;+ const batch = new Map(dirtyHitCounts);+ dirtyHitCounts.clear();+ schedulePersist(() => {+ const stmt = getDb().prepare('UPDATE response_cache SET hit_count = ?, last_hit_at_ms = ? WHERE cache_key = ?');+ for (const [cacheKey, hit] of batch) stmt.run(hit.hitCount, hit.lastHitAtMs, cacheKey);+ });+ }, HIT_FLUSH_DELAY_MS);+ hitFlushTimer.unref();+}++function dropPendingHitCounts(): void {+ dirtyHitCounts.clear();+ if (hitFlushTimer) clearTimeout(hitFlushTimer);+ hitFlushTimer = undefined;+}+/*** Test-only: run the queued write-through immediately instead of on the next* tick, so a synchronous test can assert on what actually reached SQLite.@@ -433,13 +461,11 @@ export function getCachedResponse(cacheKey: string, now = Date.now()): CachedResstore.set(cacheKey, entry);// Keep the persisted counters in step so the savings numbers survive a- // restart instead of resetting to whatever the last store wrote.- const hitCount = entry.hitCount;- schedulePersist(() => {- getDb()- .prepare('UPDATE response_cache SET hit_count = ?, last_hit_at_ms = ? WHERE cache_key = ?')- .run(hitCount, now, cacheKey);- });+ // restart instead of resetting to whatever the last store wrote. Hits are+ // batched: one debounced UPDATE for all keys dirtied in the window, instead+ // of one queued UPDATE per hit (a hot entry used to write on every hit).+ dirtyHitCounts.set(cacheKey, { hitCount: entry.hitCount, lastHitAtMs: now });+ scheduleHitCountFlush();return {body: entry.body,@@ -473,8 +499,10 @@ export function storeCachedResponse(cacheKey: string, input: StoreInput, now = Dreturn;}- // Delete-then-set so an overwrite also refreshes recency order.+ // Delete-then-set so an overwrite also refreshes recency order. Any hits+ // still waiting to be flushed counted the old answer, not this one.store.delete(cacheKey);+ dirtyHitCounts.delete(cacheKey);store.set(cacheKey, {body: input.body,platform: input.platform,@@ -756,6 +784,7 @@ export function clearCache(): number {store.clear();streamStore.clear();pendingWrites.length = 0;+ dropPendingHitCounts();// The lookup tallies describe the cache that just went away; keeping them// would show a 90% hit rate next to zero entries right after a flush.lookupHits = 0;