Skip to content

Open source CapCut alternative.

TypeScript

fix(inference): key Gemini context caches by model

David Le47b623a01536
diff --git a/site/src/lib/inference/adapters/gemini-context-cache.test.ts b/site/src/lib/inference/adapters/gemini-context-cache.test.ts
index 1f7ba685..030d9a79 100644
--- a/site/src/lib/inference/adapters/gemini-context-cache.test.ts
+++ b/site/src/lib/inference/adapters/gemini-context-cache.test.ts
@@ -75,6 +75,20 @@ describe("gemini context cache", () => {
expect(wide.config?.cachedContent).not.toBe(narrow.config?.cachedContent);
});
+ test("the same head sent by two models gets a cache per model", async () => {
+ const { client, created } = stubClient();
+ const head = { systemInstruction: longInstruction, tools: toolBlock };
+ const simple = { model: "gemini-test-lite", contents: [], config: { ...head } } as GenerateContentParameters;
+ const complex = { model: "gemini-test-flash", contents: [], config: { ...head } } as GenerateContentParameters;
+
+ await applyContextCacheToRequest(simple, client);
+ await applyContextCacheToRequest(complex, client);
+
+ expect(created).toHaveLength(2);
+ expect(simple.config?.cachedContent).toBe("caches/1");
+ expect(complex.config?.cachedContent).toBe("caches/2");
+ });
+
test("a tool block alone is large enough to cache", async () => {
const { client, created } = stubClient();
const params = request({ tools: toolBlock });
diff --git a/site/src/lib/inference/adapters/gemini-context-cache.ts b/site/src/lib/inference/adapters/gemini-context-cache.ts
index b27be4f1..e6bf440c 100644
--- a/site/src/lib/inference/adapters/gemini-context-cache.ts
+++ b/site/src/lib/inference/adapters/gemini-context-cache.ts
@@ -47,10 +47,11 @@ interface CacheHead {
tools?: Tool[];
}
-function displayNameFor(head: CacheHead): string {
- // Both parts ride the key: a turn routed to different areas is a different head and must not read back
- // another turn's cache.
- const material = JSON.stringify([head.systemInstruction ?? "", head.tools ?? []]);
+function displayNameFor(model: string, head: CacheHead): string {
+ // All three parts ride the key. A turn routed to different areas is a different head and reads back only
+ // its own cache. A cached content is bound to the model it was created for, so the same head sent by two
+ // models gets two caches; a shared one fails the second model's request with INVALID_ARGUMENT.
+ const material = JSON.stringify([model, head.systemInstruction ?? "", head.tools ?? []]);
const hash = createHash("sha256").update(material).digest("hex").slice(0, 40);
return `${DISPLAY_NAME_PREFIX}${hash}`;
}
@@ -75,7 +76,7 @@ async function resolveCachedHead(args: {
nowMs: number;
}): Promise<string | null> {
const { client, model, head, nowMs } = args;
- const displayName = displayNameFor(head);
+ const displayName = displayNameFor(model, head);
const memoKey = `${model}:${displayName}`;
const remembered = memo.get(memoKey);
@@ -84,7 +85,7 @@ async function resolveCachedHead(args: {
}
try {
- // Reuse a live cache created by this or any other instance for the identical head.
+ // Reuse a live cache created by this or any other instance for the identical model and head.
const pager = await client.caches.list({ config: { pageSize: 100 } });
let scanned = 0;
for await (const cache of pager) {