The Open Framework for Building Agents
fix(eve): detect Anthropic-on-Bedrock Converse models as cacheable (#1019)
Chad Hietala707de7ff16d3
diff --git a/.changeset/bedrock-anthropic-prompt-cache.md b/.changeset/bedrock-anthropic-prompt-cache.mdnew file mode 100644index 000000000..3c1086bf5--- /dev/null+++ b/.changeset/bedrock-anthropic-prompt-cache.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Anthropic models served through the standard `@ai-sdk/amazon-bedrock` Converse provider are now detected as cacheable. Prompt-cache breakpoints previously only matched on the provider name, so Bedrock (which reports provider `amazon-bedrock` and carries the Anthropic identity in the model id) fell through to no caching. The cache marker now also carries the Bedrock `cachePoint` namespace that the Converse provider reads.diff --git a/packages/eve/src/harness/prompt-cache.test.ts b/packages/eve/src/harness/prompt-cache.test.tsindex a4005bf25..4242a24e0 100644--- a/packages/eve/src/harness/prompt-cache.test.ts+++ b/packages/eve/src/harness/prompt-cache.test.ts@@ -9,10 +9,10 @@ import {mergeGatewayAutoCaching,} from "#harness/prompt-cache.js";-function makeObjectModel(provider: string): LanguageModel {+function makeObjectModel(provider: string, modelId = "test-model"): LanguageModel {return {provider,- modelId: "test-model",+ modelId,specificationVersion: "v3",} as unknown as LanguageModel;}@@ -68,20 +68,45 @@ describe("detectPromptCachePath", () => {it("returns none for generic Bedrock Converse (no anthropic subpath)", () => {expect(detectPromptCachePath(makeObjectModel("bedrock"))).toEqual({ kind: "none" });});++ it("returns anthropic-direct for a Bedrock Converse model whose id is Anthropic", () => {+ // `@ai-sdk/amazon-bedrock` reports provider `amazon-bedrock` and carries+ // the Anthropic identity in the model id.+ expect(+ detectPromptCachePath(+ makeObjectModel("amazon-bedrock", "anthropic.claude-3-5-sonnet-20241022-v2:0"),+ ),+ ).toEqual({ kind: "anthropic-direct" });+ });++ it("matches the Bedrock Anthropic model id regardless of case", () => {+ expect(+ detectPromptCachePath(makeObjectModel("amazon-bedrock", "ANTHROPIC.CLAUDE-3-5-SONNET")),+ ).toEqual({ kind: "anthropic-direct" });+ });++ it("returns none for a non-Anthropic Bedrock Converse model id", () => {+ expect(+ detectPromptCachePath(makeObjectModel("amazon-bedrock", "amazon.nova-pro-v1:0")),+ ).toEqual({ kind: "none" });+ });});describe("getAnthropicCacheMarker", () => {it("returns the Anthropic cache marker shape", () => {expect(getAnthropicCacheMarker()).toEqual({anthropic: { cacheControl: { type: "ephemeral" } },+ bedrock: { cachePoint: { type: "default" } },});});- it("returns a frozen object", () => {+ it("returns a deeply frozen object", () => {const marker = getAnthropicCacheMarker();expect(Object.isFrozen(marker)).toBe(true);expect(Object.isFrozen(marker.anthropic)).toBe(true);expect(Object.isFrozen(marker.anthropic.cacheControl)).toBe(true);+ expect(Object.isFrozen(marker.bedrock)).toBe(true);+ expect(Object.isFrozen(marker.bedrock.cachePoint)).toBe(true);});});@@ -146,7 +171,7 @@ describe("applyLastToolCacheBreakpoint", () => {expect(result.beta).toEqual({ description: "second" });expect(result.gamma).toEqual({description: "third",- providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },+ providerOptions: { ...marker },});});@@ -163,9 +188,10 @@ describe("applyLastToolCacheBreakpoint", () => {{ providerOptions: Record<string, unknown> } | undefined>;- // The marker's anthropic namespace overrides the existing one.+ // The marker's namespaces override any existing ones; foreign namespaces+ // are preserved.expect(result.only?.providerOptions).toEqual({- anthropic: { cacheControl: { type: "ephemeral" } },+ ...marker,openai: { something: 2 },});});@@ -197,7 +223,7 @@ describe("applyConversationCacheControl", () => {expect(out[0]).toEqual({role: "user",content: "hi",- providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },+ providerOptions: { ...marker },});});@@ -214,13 +240,13 @@ describe("applyConversationCacheControl", () => {expect(out[1]).toEqual({role: "assistant",content: "yo",- providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },+ providerOptions: { ...marker },});expect(out[2]).toEqual({ role: "user", content: "hi2" });expect(out[3]).toEqual({role: "assistant",content: "yo2",- providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },+ providerOptions: { ...marker },});});@@ -246,14 +272,10 @@ describe("applyConversationCacheControl", () => {const out = applyConversationCacheControl(messages, marker);// The trailing tool-result message carries the final breakpoint.- expect((out[2] as { providerOptions?: unknown }).providerOptions).toEqual({- anthropic: { cacheControl: { type: "ephemeral" } },- });+ expect((out[2] as { providerOptions?: unknown }).providerOptions).toEqual({ ...marker });// The most recent assistant (index 1) is the advancement anchor.- expect((out[1] as { providerOptions?: unknown }).providerOptions).toEqual({- anthropic: { cacheControl: { type: "ephemeral" } },- });+ expect((out[1] as { providerOptions?: unknown }).providerOptions).toEqual({ ...marker });// The user message is untouched.expect(out[0]).toEqual(messages[0]);@@ -270,7 +292,7 @@ describe("applyConversationCacheControl", () => {const out = applyConversationCacheControl(messages, marker);expect((out[0] as { providerOptions: Record<string, unknown> }).providerOptions).toEqual({openai: { someKey: "someValue" },- anthropic: { cacheControl: { type: "ephemeral" } },+ ...marker,});});diff --git a/packages/eve/src/harness/prompt-cache.ts b/packages/eve/src/harness/prompt-cache.tsindex 8f9ca4472..b94d03898 100644--- a/packages/eve/src/harness/prompt-cache.ts+++ b/packages/eve/src/harness/prompt-cache.ts@@ -11,16 +11,25 @@ export type PromptCachePath =/*** Cache marker injected on the Anthropic-direct path.*- * The AI SDK Anthropic provider reads `providerOptions.anthropic.cacheControl`- * from message, message-part, system-message, and tool objects. The same- * namespace is used by `@ai-sdk/amazon-bedrock/anthropic` and- * `@ai-sdk/google-vertex/anthropic`, which both implement the native- * Anthropic Messages API.+ * The marker carries two provider namespaces because Anthropic models are+ * reachable through providers that read different provider-options keys:+ *+ * - `anthropic.cacheControl` — read by the AI SDK Anthropic provider and by+ * `@ai-sdk/amazon-bedrock/anthropic` and `@ai-sdk/google-vertex/anthropic`,+ * which implement the native Anthropic Messages API.+ * - `bedrock.cachePoint` — read by the standard `@ai-sdk/amazon-bedrock`+ * Converse provider, which does not understand `anthropic.cacheControl`.+ *+ * A provider ignores namespaces it does not own, so carrying both is safe on+ * every Anthropic-direct request regardless of which provider serves it.*/export interface AnthropicCacheMarker {readonly anthropic: {readonly cacheControl: { readonly type: "ephemeral" };};+ readonly bedrock: {+ readonly cachePoint: { readonly type: "default" };+ };}/**@@ -31,6 +40,9 @@ const ANTHROPIC_CACHE_MARKER: AnthropicCacheMarker = Object.freeze({anthropic: Object.freeze({cacheControl: Object.freeze({ type: "ephemeral" as const }),}),+ bedrock: Object.freeze({+ cachePoint: Object.freeze({ type: "default" as const }),+ }),});/**@@ -48,6 +60,15 @@ export function detectPromptCachePath(model: LanguageModel): PromptCachePath {return { kind: "anthropic-direct" };}+ // The standard `@ai-sdk/amazon-bedrock` Converse provider reports its+ // provider as `amazon-bedrock` and carries the Anthropic identity in the+ // model id (e.g. `anthropic.claude-3-5-sonnet-20241022-v2:0`), so it must be+ // matched on the model id rather than the provider name.+ const modelId = typeof model.modelId === "string" ? model.modelId.toLowerCase() : "";+ if (providerName.includes("bedrock") && modelId.includes("anthropic")) {+ return { kind: "anthropic-direct" };+ }+return { kind: "none" };}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex e7d63a217..d4dc2feab 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -8211,6 +8211,7 @@ describe("createToolLoopHarness", () => {const lastTool = toolEntries[toolEntries.length - 1]?.[1];expect(lastTool?.providerOptions).toEqual({anthropic: { cacheControl: { type: "ephemeral" } },+ bedrock: { cachePoint: { type: "default" } },});});@@ -8265,9 +8266,11 @@ describe("createToolLoopHarness", () => {expect(result.messages?.[0]?.providerOptions).toBeUndefined();expect(result.messages?.[1]?.providerOptions).toEqual({anthropic: { cacheControl: { type: "ephemeral" } },+ bedrock: { cachePoint: { type: "default" } },});expect(result.messages?.[2]?.providerOptions).toEqual({anthropic: { cacheControl: { type: "ephemeral" } },+ bedrock: { cachePoint: { type: "default" } },});});@@ -9398,6 +9401,7 @@ describe("createToolLoopHarness", () => {content: "You are a test assistant.\n\ndynamic-system-instruction",providerOptions: {anthropic: { cacheControl: { type: "ephemeral" } },+ bedrock: { cachePoint: { type: "default" } },},});});