Open source CapCut alternative.
TypeScript
import { createHash } from "node:crypto";
import type { GenerateContentParameters, Tool } from "@google/genai";
import type { GeminiClient } from "./gemini-client";
// Explicit Gemini context caching for the planner's stable prompt head: the system instruction and the
// tool declarations.
//
// The agent loop sends a byte-identical head on EVERY step of a task — doctrine and skills (~11K tokens)
// plus the declared tool catalog (the whole catalog is ~40K tokens; a turn routed to a few areas is ~20K).
// A call is ~99% input. Caching that head once and referencing it bills those tokens at the reduced cached
// rate rather than full price every step. Vertex does not implicitly cache, so without this the head is
// re-billed in full on all ~37 steps of a run.
//
// The turn judge narrows the tool block per turn (src/cut/lib/turnJudge.ts), so the head varies by routed
// area set. That is what the content-hash key is for: each recurring combination gets its own cache and
// every step of that turn — and every later turn routed the same way — reads it back.
//
// Serverless-safe by construction: the cache lives in Gemini's own registry, looked up by a content-hash
// displayName, so reuse survives across isolated function invocations with no shared store of our own. A
// best-effort per-process memo skips the lookup on back-to-back calls a warm instance happens to handle.
// EVERY failure path falls back to the inline head, so caching can only reduce cost — it can never break a
// call.
// Cache lifetime on Gemini's side. Long enough to span a whole run, short enough that stale per-task caches
// expire on their own rather than piling up.
const TTL_SECONDS = 1800;
// Refresh our memo a little before the cache itself expires, so we never hand back a name that just lapsed.
const MEMO_TTL_MS = (TTL_SECONDS - 120) * 1000;
// After a failed resolve, don't retry for a minute — if caching is unavailable (e.g. the head is under the
// provider's minimum cacheable size), this stops every step from re-attempting it.
const NEGATIVE_TTL_MS = 60_000;
// Below ~1K tokens caching isn't worth a round-trip and the provider rejects it; gate on a char proxy.
const MIN_CACHEABLE_CHARS = 4_096;
const DISPLAY_NAME_PREFIX = "donkey-ctx-";
// Bound the registry scan so a long cache list can't stall a request.
const MAX_LIST_SCAN = 300;
// hash → resolved cache name (or "" as a short-lived negative entry). Lost on cold start; correctness comes
// from the list-or-create below — this only avoids a list round-trip when one instance handles a burst.
const memo = new Map<string, { name: string; expiresAt: number }>();
/** The cacheable head of a request: whichever of the instruction and the tools it carries. */
interface CacheHead {
systemInstruction?: string;
tools?: Tool[];
}
function displayNameFor(model: string, head: CacheHead): string {
// All three parts ride the key. A turn routed to different areas is a different head and reads back only
// its own cache. A cached content is bound to the model it was created for, so the same head sent by two
// models gets two caches; a shared one fails the second model's request with INVALID_ARGUMENT.
const material = JSON.stringify([model, head.systemInstruction ?? "", head.tools ?? []]);
const hash = createHash("sha256").update(material).digest("hex").slice(0, 40);
return `${DISPLAY_NAME_PREFIX}${hash}`;
}
function headChars(head: CacheHead): number {
return (head.systemInstruction?.length ?? 0) + (head.tools ? JSON.stringify(head.tools).length : 0);
}
function cacheIsLive(expireTime: string | undefined, nowMs: number): boolean {
if (!expireTime) {
return true;
}
const expiry = Date.parse(expireTime);
// Treat an unparseable time as live (let the provider be the judge); require a 30s margin otherwise.
return Number.isNaN(expiry) ? true : expiry > nowMs + 30_000;
}
async function resolveCachedHead(args: {
client: GeminiClient;
model: string;
head: CacheHead;
nowMs: number;
}): Promise<string | null> {
const { client, model, head, nowMs } = args;
const displayName = displayNameFor(model, head);
const memoKey = `${model}:${displayName}`;
const remembered = memo.get(memoKey);
if (remembered && remembered.expiresAt > nowMs) {
return remembered.name || null;
}
try {
// Reuse a live cache created by this or any other instance for the identical model and head.
const pager = await client.caches.list({ config: { pageSize: 100 } });
let scanned = 0;
for await (const cache of pager) {
if (++scanned > MAX_LIST_SCAN) {
break;
}
if (cache.displayName === displayName && cache.name && cacheIsLive(cache.expireTime, nowMs)) {
memo.set(memoKey, { name: cache.name, expiresAt: nowMs + MEMO_TTL_MS });
return cache.name;
}
}
const created = await client.caches.create({
model,
config: {
...(head.systemInstruction ? { systemInstruction: head.systemInstruction } : {}),
...(head.tools ? { tools: head.tools } : {}),
displayName,
ttl: `${TTL_SECONDS}s`,
},
});
if (created.name) {
memo.set(memoKey, { name: created.name, expiresAt: nowMs + MEMO_TTL_MS });
return created.name;
}
return null;
} catch {
// Under the provider's minimum cacheable size, a transient API error, or caching disabled — remember
// the miss briefly so we don't re-attempt every step, and let the caller keep the inline head.
memo.set(memoKey, { name: "", expiresAt: nowMs + NEGATIVE_TTL_MS });
return null;
}
}
/**
* Swap a large, repeated inline prompt head — the system instruction and the tool declarations — for a
* cached reference, in place on `requestParameters`. Applies where the same head recurs every step; short
* heads are left untouched. A no-op on any failure, so the request always stays valid.
*/
export async function applyContextCacheToRequest(
requestParameters: GenerateContentParameters,
client: GeminiClient,
): Promise<void> {
const config = requestParameters.config;
if (!config) {
return;
}
const head: CacheHead = {
...(typeof config.systemInstruction === "string" ? { systemInstruction: config.systemInstruction } : {}),
...(Array.isArray(config.tools) && config.tools.length > 0 ? { tools: config.tools as Tool[] } : {}),
};
// A tool config steers declarations the cache would then own; leave those calls inline.
if (config.toolConfig || headChars(head) < MIN_CACHEABLE_CHARS) {
return;
}
const cacheName = await resolveCachedHead({ client, model: requestParameters.model, head, nowMs: Date.now() });
if (!cacheName) {
return;
}
// A cache holds the head, so the call references it and sends neither part inline.
config.cachedContent = cacheName;
delete config.systemInstruction;
delete config.tools;
}