Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Improve Anthropic prompt cache breakpoints (#4977)

Louis Beaumont61fecde34c6a
diff --git a/packages/ai-gateway/src/providers/anthropic-proxy.ts b/packages/ai-gateway/src/providers/anthropic-proxy.ts
index 475b5f464..b7e156289 100644
--- a/packages/ai-gateway/src/providers/anthropic-proxy.ts
+++ b/packages/ai-gateway/src/providers/anthropic-proxy.ts
@@ -53,13 +53,15 @@ export function countCacheBreakpoints(body: any): number {
/**
* Inject prompt-caching breakpoints for clients that don't manage caching
* themselves (the Agent SDK sets its own markers — those pass through
- * untouched). Two markers:
+ * untouched). Three markers:
*
- * 1. System prompt → caches tools + system across requests. Unconditional on
+ * 1. Last tool definition → caches stable tool schemas even when a request has
+ * no system prompt or when the system prompt changes per request.
+ * 2. System prompt → caches tools + system across requests. Unconditional on
* size: below-minimum prefixes are a free no-op, and the old 4096-char
* gate undershot the real model minimums anyway (4096 TOKENS on Opus 4.x
* and Haiku 4.5).
- * 2. Last cacheable block of the last message → caches the whole conversation
+ * 3. Last cacheable block of the last message → caches the whole conversation
* prefix, so each turn of an agentic loop re-reads history at ~0.1x input
* price instead of full price.
*
@@ -68,7 +70,24 @@ export function countCacheBreakpoints(body: any): number {
export function injectCacheBreakpoints(body: any): void {
let breakpoints = countCacheBreakpoints(body);
- // 1. System prompt marker
+ // 1. Tool schema marker. Anthropic renders tools before system/messages;
+ // this gives large stable tool sets their own cache entry and keeps them
+ // cacheable even if the system prompt is absent or user-specific.
+ if (Array.isArray(body.tools) && breakpoints < 4) {
+ const hasToolMarker = body.tools.some((tool: any) => tool?.cache_control);
+ if (!hasToolMarker) {
+ for (let t = body.tools.length - 1; t >= 0; t--) {
+ const tool = body.tools[t];
+ if (tool && typeof tool === 'object') {
+ body.tools[t] = { ...tool, cache_control: { type: 'ephemeral' } };
+ breakpoints++;
+ break;
+ }
+ }
+ }
+ }
+
+ // 2. System prompt marker
if (body.system && breakpoints < 4) {
if (typeof body.system === 'string' && body.system.length > 0) {
body.system = [{ type: 'text', text: body.system, cache_control: { type: 'ephemeral' } }];
@@ -82,7 +101,7 @@ export function injectCacheBreakpoints(body: any): void {
}
}
- // 2. Conversation-history marker — only when the client set none in
+ // 3. Conversation-history marker — only when the client set none in
// messages (a client that places its own markers knows what it's doing).
if (!Array.isArray(body.messages) || breakpoints >= 4) return;
const hasMessageMarker = body.messages.some(
diff --git a/packages/ai-gateway/src/providers/anthropic.ts b/packages/ai-gateway/src/providers/anthropic.ts
index 877bfefea..28c3228b5 100644
--- a/packages/ai-gateway/src/providers/anthropic.ts
+++ b/packages/ai-gateway/src/providers/anthropic.ts
@@ -55,6 +55,10 @@ function anthropicStopReasonToOpenAI(reason: unknown): string {
}
}
+type AnthropicToolWithCache = AnthropicTool & {
+ cache_control?: { type: 'ephemeral'; ttl?: string };
+};
+
export class AnthropicProvider implements AIProvider {
supportsTools = true;
supportsVision = true;
@@ -69,6 +73,8 @@ export class AnthropicProvider implements AIProvider {
* Build the system prompt from system messages and response_format.
*
* Caching layout (prefix caching — see Anthropic prompt-caching docs):
+ * - The last tool definition gets a cache breakpoint in `formatTools`, so
+ * stable tool schemas still cache when there is no system prompt.
* - One block per system message, so when a client sends a shared base
* prompt followed by per-user additions as separate system messages,
* the shared block can cache-hit across users.
@@ -348,17 +354,24 @@ export class AnthropicProvider implements AIProvider {
// SCREENPIPE-AI-PROXY-K (`Cannot read properties of undefined (reading
// 'name')`). Drop tools that don't have a usable name rather than 500.
private formatTools(tools: Tool[]): AnthropicTool[] {
- const out: AnthropicTool[] = [];
+ const out: AnthropicToolWithCache[] = [];
for (const tool of tools || []) {
if (!tool) continue;
const fn: any = (tool as any).function ?? tool;
const name = fn?.name;
if (!name) continue;
- out.push({
+ const formatted: AnthropicToolWithCache = {
name,
description: fn.description,
input_schema: fn.parameters ?? fn.input_schema,
- });
+ };
+ out.push(formatted);
+ }
+ if (out.length > 0 && !out.some((tool) => tool.cache_control)) {
+ out[out.length - 1] = {
+ ...out[out.length - 1],
+ cache_control: { type: 'ephemeral' },
+ };
}
return out;
}
diff --git a/packages/ai-gateway/src/test/anthropic-caching.unit.test.ts b/packages/ai-gateway/src/test/anthropic-caching.unit.test.ts
index 1b10f8318..9e1cc12cd 100644
--- a/packages/ai-gateway/src/test/anthropic-caching.unit.test.ts
+++ b/packages/ai-gateway/src/test/anthropic-caching.unit.test.ts
@@ -6,14 +6,16 @@
* (Pi chat + pipes go through AnthropicProvider).
*
* Pins the invariants that make caching actually save money:
- * 1. System prompt always carries cache_control (no size gate — below-minimum
+ * 1. Tool schemas carry cache_control, so large stable tool lists cache even
+ * when there is no system prompt.
+ * 2. System prompt always carries cache_control (no size gate — below-minimum
* markers are free no-ops, missing markers lose hits).
- * 2. The LAST cacheable block of the LAST message carries cache_control, so
+ * 3. The LAST cacheable block of the LAST message carries cache_control, so
* agentic loops re-read the whole conversation at ~0.1x instead of 1x.
- * 3. ≤ 4 breakpoints per request (hard API limit).
- * 4. Conversion is deterministic AND turn N's blocks are a strict prefix of
+ * 4. ≤ 4 breakpoints per request (hard API limit).
+ * 5. Conversion is deterministic AND turn N's blocks are a strict prefix of
* turn N+1's blocks — byte-stable prefixes are what cache hits match on.
- * 5. Cache usage flows back out (totals + cached/written subsets) so cost
+ * 6. Cache usage flows back out (totals + cached/written subsets) so cost
* tracking can verify the savings.
*
* Run with: bun test src/test/anthropic-caching.unit.test.ts
@@ -120,6 +122,28 @@ describe('system prompt cache breakpoints', () => {
});
});
+describe('tool schema cache breakpoints', () => {
+ it('marks the last tool definition even when there is no system prompt', async () => {
+ const { provider, calls } = makeProvider();
+ await provider.createCompletion(body(
+ [{ role: 'user', content: 'hi' }],
+ {
+ tools: [
+ { type: 'function', function: { name: 'search', description: 'Search screen data', parameters: {} } },
+ { type: 'function', function: { name: 'read_file', description: 'Read a file', parameters: {} } },
+ ],
+ },
+ ));
+
+ expect(calls[0].tools).toHaveLength(2);
+ expect(calls[0].tools[0].cache_control).toBeUndefined();
+ expect(calls[0].tools[1].cache_control).toEqual({ type: 'ephemeral' });
+ const last = calls[0].messages[calls[0].messages.length - 1];
+ expect(last.content[last.content.length - 1].cache_control).toEqual({ type: 'ephemeral' });
+ expect(countBreakpoints(calls[0])).toBe(2);
+ });
+});
+
describe('message-history cache breakpoint', () => {
it('marks the last text block of the last message', async () => {
const { provider, calls } = makeProvider();
@@ -170,8 +194,9 @@ describe('message-history cache breakpoint', () => {
{ tools: [{ type: 'function', function: { name: 't', parameters: {} } }] },
));
expect(countBreakpoints(calls[0])).toBeLessThanOrEqual(4);
- // exactly: system first + system last + last message block = 3
- expect(countBreakpoints(calls[0])).toBe(3);
+ // exactly: tool + system first + system last + last message block = 4
+ expect(calls[0].tools[0].cache_control).toEqual({ type: 'ephemeral' });
+ expect(countBreakpoints(calls[0])).toBe(4);
});
it('applies the same breakpoints on the streaming path', async () => {
diff --git a/packages/ai-gateway/src/test/anthropic-proxy-caching.unit.test.ts b/packages/ai-gateway/src/test/anthropic-proxy-caching.unit.test.ts
index 1364d0eac..d1f275ae7 100644
--- a/packages/ai-gateway/src/test/anthropic-proxy-caching.unit.test.ts
+++ b/packages/ai-gateway/src/test/anthropic-proxy-caching.unit.test.ts
@@ -70,6 +70,26 @@ describe('injectCacheBreakpoints — clients without cache management', () => {
expect(countCacheBreakpoints(captured.body)).toBe(2);
});
+ it('adds a tool-schema marker before system/history markers when tools are present', async () => {
+ const captured = captureForwarded();
+ await proxyToAnthropic(makeRequest({
+ model: 'claude-opus-4-8',
+ max_tokens: 100,
+ system: 'short system prompt',
+ tools: [
+ { name: 'search', input_schema: { type: 'object', properties: { q: { type: 'string' } } } },
+ { name: 'read_file', input_schema: { type: 'object', properties: { path: { type: 'string' } } } },
+ ],
+ messages: [{ role: 'user', content: [{ type: 'text', text: 'q1' }] }],
+ }), 'sk-test');
+
+ expect(captured.body.tools[0].cache_control).toBeUndefined();
+ expect(captured.body.tools[1].cache_control).toEqual({ type: 'ephemeral' });
+ expect(captured.body.system[0].cache_control).toEqual({ type: 'ephemeral' });
+ expect(captured.body.messages[0].content[0].cache_control).toEqual({ type: 'ephemeral' });
+ expect(countCacheBreakpoints(captured.body)).toBe(3);
+ });
+
it('converts string message content to block form to attach the marker', async () => {
const captured = captureForwarded();
await proxyToAnthropic(makeRequest({
@@ -160,6 +180,21 @@ describe('injectCacheBreakpoints — clients that manage their own caching', ()
const markers = body.system.filter((b: any) => b.cache_control).length;
expect(markers).toBe(1);
});
+
+ it('keeps an existing tool marker instead of double-marking tool schemas', () => {
+ const body = {
+ tools: [
+ { name: 'search', input_schema: {}, cache_control: { type: 'ephemeral', ttl: '1h' } },
+ { name: 'read_file', input_schema: {} },
+ ],
+ messages: [{ role: 'user', content: [{ type: 'text', text: 'q' }] }],
+ };
+ injectCacheBreakpoints(body);
+ expect((body.tools[0] as any).cache_control).toEqual({ type: 'ephemeral', ttl: '1h' });
+ expect((body.tools[1] as any).cache_control).toBeUndefined();
+ expect((body.messages[0].content[0] as any).cache_control).toEqual({ type: 'ephemeral' });
+ expect(countCacheBreakpoints(body)).toBe(2);
+ });
});
describe('countCacheBreakpoints', () => {