Open source CapCut alternative.
TypeScript
fix(inference): key Gemini context caches by model
David Le47b623a01536
diff --git a/site/src/lib/inference/adapters/gemini-context-cache.test.ts b/site/src/lib/inference/adapters/gemini-context-cache.test.tsindex 1f7ba685..030d9a79 100644--- a/site/src/lib/inference/adapters/gemini-context-cache.test.ts+++ b/site/src/lib/inference/adapters/gemini-context-cache.test.ts@@ -75,6 +75,20 @@ describe("gemini context cache", () => {expect(wide.config?.cachedContent).not.toBe(narrow.config?.cachedContent);});+ test("the same head sent by two models gets a cache per model", async () => {+ const { client, created } = stubClient();+ const head = { systemInstruction: longInstruction, tools: toolBlock };+ const simple = { model: "gemini-test-lite", contents: [], config: { ...head } } as GenerateContentParameters;+ const complex = { model: "gemini-test-flash", contents: [], config: { ...head } } as GenerateContentParameters;++ await applyContextCacheToRequest(simple, client);+ await applyContextCacheToRequest(complex, client);++ expect(created).toHaveLength(2);+ expect(simple.config?.cachedContent).toBe("caches/1");+ expect(complex.config?.cachedContent).toBe("caches/2");+ });+test("a tool block alone is large enough to cache", async () => {const { client, created } = stubClient();const params = request({ tools: toolBlock });diff --git a/site/src/lib/inference/adapters/gemini-context-cache.ts b/site/src/lib/inference/adapters/gemini-context-cache.tsindex b27be4f1..e6bf440c 100644--- a/site/src/lib/inference/adapters/gemini-context-cache.ts+++ b/site/src/lib/inference/adapters/gemini-context-cache.ts@@ -47,10 +47,11 @@ interface CacheHead {tools?: Tool[];}-function displayNameFor(head: CacheHead): string {- // Both parts ride the key: a turn routed to different areas is a different head and must not read back- // another turn's cache.- const material = JSON.stringify([head.systemInstruction ?? "", head.tools ?? []]);+function displayNameFor(model: string, head: CacheHead): string {+ // All three parts ride the key. A turn routed to different areas is a different head and reads back only+ // its own cache. A cached content is bound to the model it was created for, so the same head sent by two+ // models gets two caches; a shared one fails the second model's request with INVALID_ARGUMENT.+ const material = JSON.stringify([model, head.systemInstruction ?? "", head.tools ?? []]);const hash = createHash("sha256").update(material).digest("hex").slice(0, 40);return `${DISPLAY_NAME_PREFIX}${hash}`;}@@ -75,7 +76,7 @@ async function resolveCachedHead(args: {nowMs: number;}): Promise<string | null> {const { client, model, head, nowMs } = args;- const displayName = displayNameFor(head);+ const displayName = displayNameFor(model, head);const memoKey = `${model}:${displayName}`;const remembered = memo.get(memoKey);@@ -84,7 +85,7 @@ async function resolveCachedHead(args: {}try {- // Reuse a live cache created by this or any other instance for the identical head.+ // Reuse a live cache created by this or any other instance for the identical model and head.const pager = await client.caches.list({ config: { pageSize: 100 } });let scanned = 0;for await (const cache of pager) {