Skip to content

Open source CapCut alternative.

TypeScript

perf(site): the context cache holds the tools as well as the instruction

David Le89a053ca9c5d
diff --git a/site/src/lib/inference/adapters/gemini-client.ts b/site/src/lib/inference/adapters/gemini-client.ts
index 1492683d..ab0f0a09 100644
--- a/site/src/lib/inference/adapters/gemini-client.ts
+++ b/site/src/lib/inference/adapters/gemini-client.ts
@@ -14,8 +14,9 @@ import {
// adapter routes through the same service-account path instead of re-parsing the
// environment.
export type AdapterEnvironment = Record<string, string | undefined>;
-// `caches` is included for explicit context caching of the planner's stable system instruction; the real
-// GoogleGenAI client provides it, so the default factory satisfies this without change.
+// `caches` is included for explicit context caching of the planner's stable prompt head — the system
+// instruction and the declared tools; the real GoogleGenAI client provides it, so the default factory
+// satisfies this without change.
export type GeminiClient = Pick<GoogleGenAI, "models" | "caches">;
export type GeminiClientFactory = (options: GoogleGenAIOptions) => GeminiClient;
diff --git a/site/src/lib/inference/adapters/gemini-context-cache.test.ts b/site/src/lib/inference/adapters/gemini-context-cache.test.ts
new file mode 100644
index 00000000..1f7ba685
--- /dev/null
+++ b/site/src/lib/inference/adapters/gemini-context-cache.test.ts
@@ -0,0 +1,136 @@
+import { describe, expect, test } from "bun:test";
+
+import type { GenerateContentParameters } from "@google/genai";
+
+import { applyContextCacheToRequest } from "./gemini-context-cache";
+import type { GeminiClient } from "./gemini-client";
+
+const longInstruction = "doctrine ".repeat(600);
+const toolBlock = [
+ {
+ functionDeclarations: Array.from({ length: 20 }, (_, i) => ({
+ name: `tool_${i}`,
+ description: "an editing command ".repeat(20),
+ parameters: { type: "object", properties: {} },
+ })),
+ },
+];
+
+function stubClient(): { client: GeminiClient; created: Record<string, unknown>[] } {
+ const created: Record<string, unknown>[] = [];
+ const live: { displayName?: string; name?: string; expireTime?: string }[] = [];
+ const client = {
+ models: {} as GeminiClient["models"],
+ caches: {
+ list: async () => ({
+ async *[Symbol.asyncIterator]() {
+ for (const c of live) yield c;
+ },
+ }),
+ create: async (params: { model: string; config: Record<string, unknown> }) => {
+ created.push(params.config);
+ const name = `caches/${created.length}`;
+ live.push({ displayName: String(params.config.displayName), name });
+ return { name };
+ },
+ } as unknown as GeminiClient["caches"],
+ };
+ return { client, created };
+}
+
+function request(config: Record<string, unknown>): GenerateContentParameters {
+ // A fresh model id per request keeps the process-wide memo from answering across tests.
+ return { model: `gemini-test-${Math.random()}`, contents: [], config } as GenerateContentParameters;
+}
+
+describe("gemini context cache", () => {
+ test("caches the instruction and the tools together, and sends neither inline", async () => {
+ const { client, created } = stubClient();
+ const params = request({ systemInstruction: longInstruction, tools: toolBlock });
+
+ await applyContextCacheToRequest(params, client);
+
+ expect(created).toHaveLength(1);
+ expect(created[0].systemInstruction).toBe(longInstruction);
+ expect(created[0].tools).toEqual(toolBlock);
+ expect(params.config?.cachedContent).toBe("caches/1");
+ expect(params.config?.systemInstruction).toBeUndefined();
+ expect(params.config?.tools).toBeUndefined();
+ });
+
+ test("a turn routed to different tools gets its own cache", async () => {
+ const { client, created } = stubClient();
+ const model = "gemini-test-routing";
+ const wide = { model, contents: [], config: { systemInstruction: longInstruction, tools: toolBlock } } as GenerateContentParameters;
+ const narrow = {
+ model,
+ contents: [],
+ config: { systemInstruction: longInstruction, tools: [{ functionDeclarations: toolBlock[0].functionDeclarations.slice(0, 5) }] },
+ } as GenerateContentParameters;
+
+ await applyContextCacheToRequest(wide, client);
+ await applyContextCacheToRequest(narrow, client);
+
+ expect(created).toHaveLength(2);
+ expect(wide.config?.cachedContent).not.toBe(narrow.config?.cachedContent);
+ });
+
+ test("a tool block alone is large enough to cache", async () => {
+ const { client, created } = stubClient();
+ const params = request({ tools: toolBlock });
+
+ await applyContextCacheToRequest(params, client);
+
+ expect(created).toHaveLength(1);
+ expect(created[0].systemInstruction).toBeUndefined();
+ expect(params.config?.tools).toBeUndefined();
+ expect(params.config?.cachedContent).toBe("caches/1");
+ });
+
+ test("a short head is left inline", async () => {
+ const { client, created } = stubClient();
+ const params = request({ systemInstruction: "be brief" });
+
+ await applyContextCacheToRequest(params, client);
+
+ expect(created).toHaveLength(0);
+ expect(params.config?.systemInstruction).toBe("be brief");
+ expect(params.config?.cachedContent).toBeUndefined();
+ });
+
+ test("a tool config keeps the call inline", async () => {
+ const { client, created } = stubClient();
+ const params = request({
+ systemInstruction: longInstruction,
+ tools: toolBlock,
+ toolConfig: { functionCallingConfig: { mode: "ANY" } },
+ });
+
+ await applyContextCacheToRequest(params, client);
+
+ expect(created).toHaveLength(0);
+ expect(params.config?.tools).toEqual(toolBlock);
+ expect(params.config?.cachedContent).toBeUndefined();
+ });
+
+ test("a caching failure leaves the request valid", async () => {
+ const client = {
+ models: {} as GeminiClient["models"],
+ caches: {
+ list: async () => {
+ throw new Error("caching disabled");
+ },
+ create: async () => {
+ throw new Error("caching disabled");
+ },
+ } as unknown as GeminiClient["caches"],
+ };
+ const params = request({ systemInstruction: longInstruction, tools: toolBlock });
+
+ await applyContextCacheToRequest(params, client);
+
+ expect(params.config?.systemInstruction).toBe(longInstruction);
+ expect(params.config?.tools).toEqual(toolBlock);
+ expect(params.config?.cachedContent).toBeUndefined();
+ });
+});
diff --git a/site/src/lib/inference/adapters/gemini-context-cache.ts b/site/src/lib/inference/adapters/gemini-context-cache.ts
index db079f48..b27be4f1 100644
--- a/site/src/lib/inference/adapters/gemini-context-cache.ts
+++ b/site/src/lib/inference/adapters/gemini-context-cache.ts
@@ -1,29 +1,35 @@
import { createHash } from "node:crypto";
-import type { GenerateContentParameters } from "@google/genai";
+import type { GenerateContentParameters, Tool } from "@google/genai";
import type { GeminiClient } from "./gemini-client";
-// Explicit Gemini context caching for the planner's stable system instruction.
+// Explicit Gemini context caching for the planner's stable prompt head: the system instruction and the
+// tool declarations.
//
-// The agent loop sends a byte-identical system-instruction block (doctrine + the whole tool catalog +
-// skills, ~11K tokens) on EVERY step of a task — a call is ~99% input. Caching that block once and
-// referencing it bills those tokens at the reduced cached rate instead of full price every step. Vertex
-// does not implicitly cache, so without this the block is re-billed in full on all ~37 steps of a run.
+// The agent loop sends a byte-identical head on EVERY step of a task — doctrine and skills (~11K tokens)
+// plus the declared tool catalog (the whole catalog is ~40K tokens; a turn routed to a few areas is ~20K).
+// A call is ~99% input. Caching that head once and referencing it bills those tokens at the reduced cached
+// rate rather than full price every step. Vertex does not implicitly cache, so without this the head is
+// re-billed in full on all ~37 steps of a run.
+//
+// The turn judge narrows the tool block per turn (src/cut/lib/turnJudge.ts), so the head varies by routed
+// area set. That is what the content-hash key is for: each recurring combination gets its own cache and
+// every step of that turn — and every later turn routed the same way — reads it back.
//
// Serverless-safe by construction: the cache lives in Gemini's own registry, looked up by a content-hash
// displayName, so reuse survives across isolated function invocations with no shared store of our own. A
// best-effort per-process memo skips the lookup on back-to-back calls a warm instance happens to handle.
-// EVERY failure path falls back to the inline system instruction, so caching can only reduce cost — it can
-// never break a call.
+// EVERY failure path falls back to the inline head, so caching can only reduce cost — it can never break a
+// call.
// Cache lifetime on Gemini's side. Long enough to span a whole run, short enough that stale per-task caches
// expire on their own rather than piling up.
const TTL_SECONDS = 1800;
// Refresh our memo a little before the cache itself expires, so we never hand back a name that just lapsed.
const MEMO_TTL_MS = (TTL_SECONDS - 120) * 1000;
-// After a failed resolve, don't retry for a minute — if caching is unavailable (e.g. the instruction is
-// under the provider's minimum cacheable size), this stops every step from re-attempting it.
+// After a failed resolve, don't retry for a minute — if caching is unavailable (e.g. the head is under the
+// provider's minimum cacheable size), this stops every step from re-attempting it.
const NEGATIVE_TTL_MS = 60_000;
// Below ~1K tokens caching isn't worth a round-trip and the provider rejects it; gate on a char proxy.
const MIN_CACHEABLE_CHARS = 4_096;
@@ -35,11 +41,24 @@ const MAX_LIST_SCAN = 300;
// from the list-or-create below — this only avoids a list round-trip when one instance handles a burst.
const memo = new Map<string, { name: string; expiresAt: number }>();
-function displayNameFor(systemInstruction: string): string {
- const hash = createHash("sha256").update(systemInstruction).digest("hex").slice(0, 40);
+/** The cacheable head of a request: whichever of the instruction and the tools it carries. */
+interface CacheHead {
+ systemInstruction?: string;
+ tools?: Tool[];
+}
+
+function displayNameFor(head: CacheHead): string {
+ // Both parts ride the key: a turn routed to different areas is a different head and must not read back
+ // another turn's cache.
+ const material = JSON.stringify([head.systemInstruction ?? "", head.tools ?? []]);
+ const hash = createHash("sha256").update(material).digest("hex").slice(0, 40);
return `${DISPLAY_NAME_PREFIX}${hash}`;
}
+function headChars(head: CacheHead): number {
+ return (head.systemInstruction?.length ?? 0) + (head.tools ? JSON.stringify(head.tools).length : 0);
+}
+
function cacheIsLive(expireTime: string | undefined, nowMs: number): boolean {
if (!expireTime) {
return true;
@@ -49,14 +68,14 @@ function cacheIsLive(expireTime: string | undefined, nowMs: number): boolean {
return Number.isNaN(expiry) ? true : expiry > nowMs + 30_000;
}
-async function resolveCachedSystemInstruction(args: {
+async function resolveCachedHead(args: {
client: GeminiClient;
model: string;
- systemInstruction: string;
+ head: CacheHead;
nowMs: number;
}): Promise<string | null> {
- const { client, model, systemInstruction, nowMs } = args;
- const displayName = displayNameFor(systemInstruction);
+ const { client, model, head, nowMs } = args;
+ const displayName = displayNameFor(head);
const memoKey = `${model}:${displayName}`;
const remembered = memo.get(memoKey);
@@ -65,7 +84,7 @@ async function resolveCachedSystemInstruction(args: {
}
try {
- // Reuse a live cache created by this or any other instance for the identical instruction.
+ // Reuse a live cache created by this or any other instance for the identical head.
const pager = await client.caches.list({ config: { pageSize: 100 } });
let scanned = 0;
for await (const cache of pager) {
@@ -80,7 +99,12 @@ async function resolveCachedSystemInstruction(args: {
const created = await client.caches.create({
model,
- config: { systemInstruction, displayName, ttl: `${TTL_SECONDS}s` },
+ config: {
+ ...(head.systemInstruction ? { systemInstruction: head.systemInstruction } : {}),
+ ...(head.tools ? { tools: head.tools } : {}),
+ displayName,
+ ttl: `${TTL_SECONDS}s`,
+ },
});
if (created.name) {
memo.set(memoKey, { name: created.name, expiresAt: nowMs + MEMO_TTL_MS });
@@ -89,38 +113,41 @@ async function resolveCachedSystemInstruction(args: {
return null;
} catch {
// Under the provider's minimum cacheable size, a transient API error, or caching disabled — remember
- // the miss briefly so we don't re-attempt every step, and let the caller keep the inline instruction.
+ // the miss briefly so we don't re-attempt every step, and let the caller keep the inline head.
memo.set(memoKey, { name: "", expiresAt: nowMs + NEGATIVE_TTL_MS });
return null;
}
}
/**
- * Swap a large, repeated inline system instruction for a cached reference, in place on `requestParameters`.
- * Applies where the same instruction recurs every step; short instructions are left untouched.
- * A no-op on any failure, so the request always stays valid.
+ * Swap a large, repeated inline prompt head — the system instruction and the tool declarations — for a
+ * cached reference, in place on `requestParameters`. Applies where the same head recurs every step; short
+ * heads are left untouched. A no-op on any failure, so the request always stays valid.
*/
export async function applyContextCacheToRequest(
requestParameters: GenerateContentParameters,
client: GeminiClient,
): Promise<void> {
const config = requestParameters.config;
- const systemInstruction = config?.systemInstruction;
- if (!config || typeof systemInstruction !== "string" || systemInstruction.length < MIN_CACHEABLE_CHARS) {
+ if (!config) {
+ return;
+ }
+ const head: CacheHead = {
+ ...(typeof config.systemInstruction === "string" ? { systemInstruction: config.systemInstruction } : {}),
+ ...(Array.isArray(config.tools) && config.tools.length > 0 ? { tools: config.tools as Tool[] } : {}),
+ };
+ // A tool config steers declarations the cache would then own; leave those calls inline.
+ if (config.toolConfig || headChars(head) < MIN_CACHEABLE_CHARS) {
return;
}
- const cacheName = await resolveCachedSystemInstruction({
- client,
- model: requestParameters.model,
- systemInstruction,
- nowMs: Date.now(),
- });
+ const cacheName = await resolveCachedHead({ client, model: requestParameters.model, head, nowMs: Date.now() });
if (!cacheName) {
return;
}
- // A cache holds the system instruction, so the call must reference it WITHOUT also sending one inline.
+ // A cache holds the head, so the call references it and sends neither part inline.
config.cachedContent = cacheName;
delete config.systemInstruction;
+ delete config.tools;
}