Open source CapCut alternative.
TypeScript
perf(site): the context cache holds the tools as well as the instruction
David Le89a053ca9c5d
diff --git a/site/src/lib/inference/adapters/gemini-client.ts b/site/src/lib/inference/adapters/gemini-client.tsindex 1492683d..ab0f0a09 100644--- a/site/src/lib/inference/adapters/gemini-client.ts+++ b/site/src/lib/inference/adapters/gemini-client.ts@@ -14,8 +14,9 @@ import {// adapter routes through the same service-account path instead of re-parsing the// environment.export type AdapterEnvironment = Record<string, string | undefined>;-// `caches` is included for explicit context caching of the planner's stable system instruction; the real-// GoogleGenAI client provides it, so the default factory satisfies this without change.+// `caches` is included for explicit context caching of the planner's stable prompt head — the system+// instruction and the declared tools; the real GoogleGenAI client provides it, so the default factory+// satisfies this without change.export type GeminiClient = Pick<GoogleGenAI, "models" | "caches">;export type GeminiClientFactory = (options: GoogleGenAIOptions) => GeminiClient;diff --git a/site/src/lib/inference/adapters/gemini-context-cache.test.ts b/site/src/lib/inference/adapters/gemini-context-cache.test.tsnew file mode 100644index 00000000..1f7ba685--- /dev/null+++ b/site/src/lib/inference/adapters/gemini-context-cache.test.ts@@ -0,0 +1,136 @@+import { describe, expect, test } from "bun:test";++import type { GenerateContentParameters } from "@google/genai";++import { applyContextCacheToRequest } from "./gemini-context-cache";+import type { GeminiClient } from "./gemini-client";++const longInstruction = "doctrine ".repeat(600);+const toolBlock = [+ {+ functionDeclarations: Array.from({ length: 20 }, (_, i) => ({+ name: `tool_${i}`,+ description: "an editing command ".repeat(20),+ parameters: { type: "object", properties: {} },+ })),+ },+];++function stubClient(): { client: GeminiClient; created: Record<string, unknown>[] } {+ const created: Record<string, unknown>[] = [];+ const live: { displayName?: string; name?: string; expireTime?: string }[] = [];+ const client = {+ models: {} as GeminiClient["models"],+ caches: {+ list: async () => ({+ async *[Symbol.asyncIterator]() {+ for (const c of live) yield c;+ },+ }),+ create: async (params: { model: string; config: Record<string, unknown> }) => {+ created.push(params.config);+ const name = `caches/${created.length}`;+ live.push({ displayName: String(params.config.displayName), name });+ return { name };+ },+ } as unknown as GeminiClient["caches"],+ };+ return { client, created };+}++function request(config: Record<string, unknown>): GenerateContentParameters {+ // A fresh model id per request keeps the process-wide memo from answering across tests.+ return { model: `gemini-test-${Math.random()}`, contents: [], config } as GenerateContentParameters;+}++describe("gemini context cache", () => {+ test("caches the instruction and the tools together, and sends neither inline", async () => {+ const { client, created } = stubClient();+ const params = request({ systemInstruction: longInstruction, tools: toolBlock });++ await applyContextCacheToRequest(params, client);++ expect(created).toHaveLength(1);+ expect(created[0].systemInstruction).toBe(longInstruction);+ expect(created[0].tools).toEqual(toolBlock);+ expect(params.config?.cachedContent).toBe("caches/1");+ expect(params.config?.systemInstruction).toBeUndefined();+ expect(params.config?.tools).toBeUndefined();+ });++ test("a turn routed to different tools gets its own cache", async () => {+ const { client, created } = stubClient();+ const model = "gemini-test-routing";+ const wide = { model, contents: [], config: { systemInstruction: longInstruction, tools: toolBlock } } as GenerateContentParameters;+ const narrow = {+ model,+ contents: [],+ config: { systemInstruction: longInstruction, tools: [{ functionDeclarations: toolBlock[0].functionDeclarations.slice(0, 5) }] },+ } as GenerateContentParameters;++ await applyContextCacheToRequest(wide, client);+ await applyContextCacheToRequest(narrow, client);++ expect(created).toHaveLength(2);+ expect(wide.config?.cachedContent).not.toBe(narrow.config?.cachedContent);+ });++ test("a tool block alone is large enough to cache", async () => {+ const { client, created } = stubClient();+ const params = request({ tools: toolBlock });++ await applyContextCacheToRequest(params, client);++ expect(created).toHaveLength(1);+ expect(created[0].systemInstruction).toBeUndefined();+ expect(params.config?.tools).toBeUndefined();+ expect(params.config?.cachedContent).toBe("caches/1");+ });++ test("a short head is left inline", async () => {+ const { client, created } = stubClient();+ const params = request({ systemInstruction: "be brief" });++ await applyContextCacheToRequest(params, client);++ expect(created).toHaveLength(0);+ expect(params.config?.systemInstruction).toBe("be brief");+ expect(params.config?.cachedContent).toBeUndefined();+ });++ test("a tool config keeps the call inline", async () => {+ const { client, created } = stubClient();+ const params = request({+ systemInstruction: longInstruction,+ tools: toolBlock,+ toolConfig: { functionCallingConfig: { mode: "ANY" } },+ });++ await applyContextCacheToRequest(params, client);++ expect(created).toHaveLength(0);+ expect(params.config?.tools).toEqual(toolBlock);+ expect(params.config?.cachedContent).toBeUndefined();+ });++ test("a caching failure leaves the request valid", async () => {+ const client = {+ models: {} as GeminiClient["models"],+ caches: {+ list: async () => {+ throw new Error("caching disabled");+ },+ create: async () => {+ throw new Error("caching disabled");+ },+ } as unknown as GeminiClient["caches"],+ };+ const params = request({ systemInstruction: longInstruction, tools: toolBlock });++ await applyContextCacheToRequest(params, client);++ expect(params.config?.systemInstruction).toBe(longInstruction);+ expect(params.config?.tools).toEqual(toolBlock);+ expect(params.config?.cachedContent).toBeUndefined();+ });+});diff --git a/site/src/lib/inference/adapters/gemini-context-cache.ts b/site/src/lib/inference/adapters/gemini-context-cache.tsindex db079f48..b27be4f1 100644--- a/site/src/lib/inference/adapters/gemini-context-cache.ts+++ b/site/src/lib/inference/adapters/gemini-context-cache.ts@@ -1,29 +1,35 @@import { createHash } from "node:crypto";-import type { GenerateContentParameters } from "@google/genai";+import type { GenerateContentParameters, Tool } from "@google/genai";import type { GeminiClient } from "./gemini-client";-// Explicit Gemini context caching for the planner's stable system instruction.+// Explicit Gemini context caching for the planner's stable prompt head: the system instruction and the+// tool declarations.//-// The agent loop sends a byte-identical system-instruction block (doctrine + the whole tool catalog +-// skills, ~11K tokens) on EVERY step of a task — a call is ~99% input. Caching that block once and-// referencing it bills those tokens at the reduced cached rate instead of full price every step. Vertex-// does not implicitly cache, so without this the block is re-billed in full on all ~37 steps of a run.+// The agent loop sends a byte-identical head on EVERY step of a task — doctrine and skills (~11K tokens)+// plus the declared tool catalog (the whole catalog is ~40K tokens; a turn routed to a few areas is ~20K).+// A call is ~99% input. Caching that head once and referencing it bills those tokens at the reduced cached+// rate rather than full price every step. Vertex does not implicitly cache, so without this the head is+// re-billed in full on all ~37 steps of a run.+//+// The turn judge narrows the tool block per turn (src/cut/lib/turnJudge.ts), so the head varies by routed+// area set. That is what the content-hash key is for: each recurring combination gets its own cache and+// every step of that turn — and every later turn routed the same way — reads it back.//// Serverless-safe by construction: the cache lives in Gemini's own registry, looked up by a content-hash// displayName, so reuse survives across isolated function invocations with no shared store of our own. A// best-effort per-process memo skips the lookup on back-to-back calls a warm instance happens to handle.-// EVERY failure path falls back to the inline system instruction, so caching can only reduce cost — it can-// never break a call.+// EVERY failure path falls back to the inline head, so caching can only reduce cost — it can never break a+// call.// Cache lifetime on Gemini's side. Long enough to span a whole run, short enough that stale per-task caches// expire on their own rather than piling up.const TTL_SECONDS = 1800;// Refresh our memo a little before the cache itself expires, so we never hand back a name that just lapsed.const MEMO_TTL_MS = (TTL_SECONDS - 120) * 1000;-// After a failed resolve, don't retry for a minute — if caching is unavailable (e.g. the instruction is-// under the provider's minimum cacheable size), this stops every step from re-attempting it.+// After a failed resolve, don't retry for a minute — if caching is unavailable (e.g. the head is under the+// provider's minimum cacheable size), this stops every step from re-attempting it.const NEGATIVE_TTL_MS = 60_000;// Below ~1K tokens caching isn't worth a round-trip and the provider rejects it; gate on a char proxy.const MIN_CACHEABLE_CHARS = 4_096;@@ -35,11 +41,24 @@ const MAX_LIST_SCAN = 300;// from the list-or-create below — this only avoids a list round-trip when one instance handles a burst.const memo = new Map<string, { name: string; expiresAt: number }>();-function displayNameFor(systemInstruction: string): string {- const hash = createHash("sha256").update(systemInstruction).digest("hex").slice(0, 40);+/** The cacheable head of a request: whichever of the instruction and the tools it carries. */+interface CacheHead {+ systemInstruction?: string;+ tools?: Tool[];+}++function displayNameFor(head: CacheHead): string {+ // Both parts ride the key: a turn routed to different areas is a different head and must not read back+ // another turn's cache.+ const material = JSON.stringify([head.systemInstruction ?? "", head.tools ?? []]);+ const hash = createHash("sha256").update(material).digest("hex").slice(0, 40);return `${DISPLAY_NAME_PREFIX}${hash}`;}+function headChars(head: CacheHead): number {+ return (head.systemInstruction?.length ?? 0) + (head.tools ? JSON.stringify(head.tools).length : 0);+}+function cacheIsLive(expireTime: string | undefined, nowMs: number): boolean {if (!expireTime) {return true;@@ -49,14 +68,14 @@ function cacheIsLive(expireTime: string | undefined, nowMs: number): boolean {return Number.isNaN(expiry) ? true : expiry > nowMs + 30_000;}-async function resolveCachedSystemInstruction(args: {+async function resolveCachedHead(args: {client: GeminiClient;model: string;- systemInstruction: string;+ head: CacheHead;nowMs: number;}): Promise<string | null> {- const { client, model, systemInstruction, nowMs } = args;- const displayName = displayNameFor(systemInstruction);+ const { client, model, head, nowMs } = args;+ const displayName = displayNameFor(head);const memoKey = `${model}:${displayName}`;const remembered = memo.get(memoKey);@@ -65,7 +84,7 @@ async function resolveCachedSystemInstruction(args: {}try {- // Reuse a live cache created by this or any other instance for the identical instruction.+ // Reuse a live cache created by this or any other instance for the identical head.const pager = await client.caches.list({ config: { pageSize: 100 } });let scanned = 0;for await (const cache of pager) {@@ -80,7 +99,12 @@ async function resolveCachedSystemInstruction(args: {const created = await client.caches.create({model,- config: { systemInstruction, displayName, ttl: `${TTL_SECONDS}s` },+ config: {+ ...(head.systemInstruction ? { systemInstruction: head.systemInstruction } : {}),+ ...(head.tools ? { tools: head.tools } : {}),+ displayName,+ ttl: `${TTL_SECONDS}s`,+ },});if (created.name) {memo.set(memoKey, { name: created.name, expiresAt: nowMs + MEMO_TTL_MS });@@ -89,38 +113,41 @@ async function resolveCachedSystemInstruction(args: {return null;} catch {// Under the provider's minimum cacheable size, a transient API error, or caching disabled — remember- // the miss briefly so we don't re-attempt every step, and let the caller keep the inline instruction.+ // the miss briefly so we don't re-attempt every step, and let the caller keep the inline head.memo.set(memoKey, { name: "", expiresAt: nowMs + NEGATIVE_TTL_MS });return null;}}/**- * Swap a large, repeated inline system instruction for a cached reference, in place on `requestParameters`.- * Applies where the same instruction recurs every step; short instructions are left untouched.- * A no-op on any failure, so the request always stays valid.+ * Swap a large, repeated inline prompt head — the system instruction and the tool declarations — for a+ * cached reference, in place on `requestParameters`. Applies where the same head recurs every step; short+ * heads are left untouched. A no-op on any failure, so the request always stays valid.*/export async function applyContextCacheToRequest(requestParameters: GenerateContentParameters,client: GeminiClient,): Promise<void> {const config = requestParameters.config;- const systemInstruction = config?.systemInstruction;- if (!config || typeof systemInstruction !== "string" || systemInstruction.length < MIN_CACHEABLE_CHARS) {+ if (!config) {+ return;+ }+ const head: CacheHead = {+ ...(typeof config.systemInstruction === "string" ? { systemInstruction: config.systemInstruction } : {}),+ ...(Array.isArray(config.tools) && config.tools.length > 0 ? { tools: config.tools as Tool[] } : {}),+ };+ // A tool config steers declarations the cache would then own; leave those calls inline.+ if (config.toolConfig || headChars(head) < MIN_CACHEABLE_CHARS) {return;}- const cacheName = await resolveCachedSystemInstruction({- client,- model: requestParameters.model,- systemInstruction,- nowMs: Date.now(),- });+ const cacheName = await resolveCachedHead({ client, model: requestParameters.model, head, nowMs: Date.now() });if (!cacheName) {return;}- // A cache holds the system instruction, so the call must reference it WITHOUT also sending one inline.+ // A cache holds the head, so the call references it and sends neither part inline.config.cachedContent = cacheName;delete config.systemInstruction;+ delete config.tools;}