YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Improve Anthropic prompt cache breakpoints (#4977)
Louis Beaumont61fecde34c6a
diff --git a/packages/ai-gateway/src/providers/anthropic-proxy.ts b/packages/ai-gateway/src/providers/anthropic-proxy.tsindex 475b5f464..b7e156289 100644--- a/packages/ai-gateway/src/providers/anthropic-proxy.ts+++ b/packages/ai-gateway/src/providers/anthropic-proxy.ts@@ -53,13 +53,15 @@ export function countCacheBreakpoints(body: any): number {/*** Inject prompt-caching breakpoints for clients that don't manage caching* themselves (the Agent SDK sets its own markers — those pass through- * untouched). Two markers:+ * untouched). Three markers:*- * 1. System prompt → caches tools + system across requests. Unconditional on+ * 1. Last tool definition → caches stable tool schemas even when a request has+ * no system prompt or when the system prompt changes per request.+ * 2. System prompt → caches tools + system across requests. Unconditional on* size: below-minimum prefixes are a free no-op, and the old 4096-char* gate undershot the real model minimums anyway (4096 TOKENS on Opus 4.x* and Haiku 4.5).- * 2. Last cacheable block of the last message → caches the whole conversation+ * 3. Last cacheable block of the last message → caches the whole conversation* prefix, so each turn of an agentic loop re-reads history at ~0.1x input* price instead of full price.*@@ -68,7 +70,24 @@ export function countCacheBreakpoints(body: any): number {export function injectCacheBreakpoints(body: any): void {let breakpoints = countCacheBreakpoints(body);- // 1. System prompt marker+ // 1. Tool schema marker. Anthropic renders tools before system/messages;+ // this gives large stable tool sets their own cache entry and keeps them+ // cacheable even if the system prompt is absent or user-specific.+ if (Array.isArray(body.tools) && breakpoints < 4) {+ const hasToolMarker = body.tools.some((tool: any) => tool?.cache_control);+ if (!hasToolMarker) {+ for (let t = body.tools.length - 1; t >= 0; t--) {+ const tool = body.tools[t];+ if (tool && typeof tool === 'object') {+ body.tools[t] = { ...tool, cache_control: { type: 'ephemeral' } };+ breakpoints++;+ break;+ }+ }+ }+ }++ // 2. System prompt markerif (body.system && breakpoints < 4) {if (typeof body.system === 'string' && body.system.length > 0) {body.system = [{ type: 'text', text: body.system, cache_control: { type: 'ephemeral' } }];@@ -82,7 +101,7 @@ export function injectCacheBreakpoints(body: any): void {}}- // 2. Conversation-history marker — only when the client set none in+ // 3. Conversation-history marker — only when the client set none in// messages (a client that places its own markers knows what it's doing).if (!Array.isArray(body.messages) || breakpoints >= 4) return;const hasMessageMarker = body.messages.some(diff --git a/packages/ai-gateway/src/providers/anthropic.ts b/packages/ai-gateway/src/providers/anthropic.tsindex 877bfefea..28c3228b5 100644--- a/packages/ai-gateway/src/providers/anthropic.ts+++ b/packages/ai-gateway/src/providers/anthropic.ts@@ -55,6 +55,10 @@ function anthropicStopReasonToOpenAI(reason: unknown): string {}}+type AnthropicToolWithCache = AnthropicTool & {+ cache_control?: { type: 'ephemeral'; ttl?: string };+};+export class AnthropicProvider implements AIProvider {supportsTools = true;supportsVision = true;@@ -69,6 +73,8 @@ export class AnthropicProvider implements AIProvider {* Build the system prompt from system messages and response_format.** Caching layout (prefix caching — see Anthropic prompt-caching docs):+ * - The last tool definition gets a cache breakpoint in `formatTools`, so+ * stable tool schemas still cache when there is no system prompt.* - One block per system message, so when a client sends a shared base* prompt followed by per-user additions as separate system messages,* the shared block can cache-hit across users.@@ -348,17 +354,24 @@ export class AnthropicProvider implements AIProvider {// SCREENPIPE-AI-PROXY-K (`Cannot read properties of undefined (reading// 'name')`). Drop tools that don't have a usable name rather than 500.private formatTools(tools: Tool[]): AnthropicTool[] {- const out: AnthropicTool[] = [];+ const out: AnthropicToolWithCache[] = [];for (const tool of tools || []) {if (!tool) continue;const fn: any = (tool as any).function ?? tool;const name = fn?.name;if (!name) continue;- out.push({+ const formatted: AnthropicToolWithCache = {name,description: fn.description,input_schema: fn.parameters ?? fn.input_schema,- });+ };+ out.push(formatted);+ }+ if (out.length > 0 && !out.some((tool) => tool.cache_control)) {+ out[out.length - 1] = {+ ...out[out.length - 1],+ cache_control: { type: 'ephemeral' },+ };}return out;}diff --git a/packages/ai-gateway/src/test/anthropic-caching.unit.test.ts b/packages/ai-gateway/src/test/anthropic-caching.unit.test.tsindex 1b10f8318..9e1cc12cd 100644--- a/packages/ai-gateway/src/test/anthropic-caching.unit.test.ts+++ b/packages/ai-gateway/src/test/anthropic-caching.unit.test.ts@@ -6,14 +6,16 @@* (Pi chat + pipes go through AnthropicProvider).** Pins the invariants that make caching actually save money:- * 1. System prompt always carries cache_control (no size gate — below-minimum+ * 1. Tool schemas carry cache_control, so large stable tool lists cache even+ * when there is no system prompt.+ * 2. System prompt always carries cache_control (no size gate — below-minimum* markers are free no-ops, missing markers lose hits).- * 2. The LAST cacheable block of the LAST message carries cache_control, so+ * 3. The LAST cacheable block of the LAST message carries cache_control, so* agentic loops re-read the whole conversation at ~0.1x instead of 1x.- * 3. ≤ 4 breakpoints per request (hard API limit).- * 4. Conversion is deterministic AND turn N's blocks are a strict prefix of+ * 4. ≤ 4 breakpoints per request (hard API limit).+ * 5. Conversion is deterministic AND turn N's blocks are a strict prefix of* turn N+1's blocks — byte-stable prefixes are what cache hits match on.- * 5. Cache usage flows back out (totals + cached/written subsets) so cost+ * 6. Cache usage flows back out (totals + cached/written subsets) so cost* tracking can verify the savings.** Run with: bun test src/test/anthropic-caching.unit.test.ts@@ -120,6 +122,28 @@ describe('system prompt cache breakpoints', () => {});});+describe('tool schema cache breakpoints', () => {+ it('marks the last tool definition even when there is no system prompt', async () => {+ const { provider, calls } = makeProvider();+ await provider.createCompletion(body(+ [{ role: 'user', content: 'hi' }],+ {+ tools: [+ { type: 'function', function: { name: 'search', description: 'Search screen data', parameters: {} } },+ { type: 'function', function: { name: 'read_file', description: 'Read a file', parameters: {} } },+ ],+ },+ ));++ expect(calls[0].tools).toHaveLength(2);+ expect(calls[0].tools[0].cache_control).toBeUndefined();+ expect(calls[0].tools[1].cache_control).toEqual({ type: 'ephemeral' });+ const last = calls[0].messages[calls[0].messages.length - 1];+ expect(last.content[last.content.length - 1].cache_control).toEqual({ type: 'ephemeral' });+ expect(countBreakpoints(calls[0])).toBe(2);+ });+});+describe('message-history cache breakpoint', () => {it('marks the last text block of the last message', async () => {const { provider, calls } = makeProvider();@@ -170,8 +194,9 @@ describe('message-history cache breakpoint', () => {{ tools: [{ type: 'function', function: { name: 't', parameters: {} } }] },));expect(countBreakpoints(calls[0])).toBeLessThanOrEqual(4);- // exactly: system first + system last + last message block = 3- expect(countBreakpoints(calls[0])).toBe(3);+ // exactly: tool + system first + system last + last message block = 4+ expect(calls[0].tools[0].cache_control).toEqual({ type: 'ephemeral' });+ expect(countBreakpoints(calls[0])).toBe(4);});it('applies the same breakpoints on the streaming path', async () => {diff --git a/packages/ai-gateway/src/test/anthropic-proxy-caching.unit.test.ts b/packages/ai-gateway/src/test/anthropic-proxy-caching.unit.test.tsindex 1364d0eac..d1f275ae7 100644--- a/packages/ai-gateway/src/test/anthropic-proxy-caching.unit.test.ts+++ b/packages/ai-gateway/src/test/anthropic-proxy-caching.unit.test.ts@@ -70,6 +70,26 @@ describe('injectCacheBreakpoints — clients without cache management', () => {expect(countCacheBreakpoints(captured.body)).toBe(2);});+ it('adds a tool-schema marker before system/history markers when tools are present', async () => {+ const captured = captureForwarded();+ await proxyToAnthropic(makeRequest({+ model: 'claude-opus-4-8',+ max_tokens: 100,+ system: 'short system prompt',+ tools: [+ { name: 'search', input_schema: { type: 'object', properties: { q: { type: 'string' } } } },+ { name: 'read_file', input_schema: { type: 'object', properties: { path: { type: 'string' } } } },+ ],+ messages: [{ role: 'user', content: [{ type: 'text', text: 'q1' }] }],+ }), 'sk-test');++ expect(captured.body.tools[0].cache_control).toBeUndefined();+ expect(captured.body.tools[1].cache_control).toEqual({ type: 'ephemeral' });+ expect(captured.body.system[0].cache_control).toEqual({ type: 'ephemeral' });+ expect(captured.body.messages[0].content[0].cache_control).toEqual({ type: 'ephemeral' });+ expect(countCacheBreakpoints(captured.body)).toBe(3);+ });+it('converts string message content to block form to attach the marker', async () => {const captured = captureForwarded();await proxyToAnthropic(makeRequest({@@ -160,6 +180,21 @@ describe('injectCacheBreakpoints — clients that manage their own caching', ()const markers = body.system.filter((b: any) => b.cache_control).length;expect(markers).toBe(1);});++ it('keeps an existing tool marker instead of double-marking tool schemas', () => {+ const body = {+ tools: [+ { name: 'search', input_schema: {}, cache_control: { type: 'ephemeral', ttl: '1h' } },+ { name: 'read_file', input_schema: {} },+ ],+ messages: [{ role: 'user', content: [{ type: 'text', text: 'q' }] }],+ };+ injectCacheBreakpoints(body);+ expect((body.tools[0] as any).cache_control).toEqual({ type: 'ephemeral', ttl: '1h' });+ expect((body.tools[1] as any).cache_control).toBeUndefined();+ expect((body.messages[0].content[0] as any).cache_control).toEqual({ type: 'ephemeral' });+ expect(countCacheBreakpoints(body)).toBe(2);+ });});describe('countCacheBreakpoints', () => {