Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): detect Anthropic-on-Bedrock Converse models as cacheable (#1019)

Chad Hietala707de7ff16d3
diff --git a/.changeset/bedrock-anthropic-prompt-cache.md b/.changeset/bedrock-anthropic-prompt-cache.md
new file mode 100644
index 000000000..3c1086bf5
--- /dev/null
+++ b/.changeset/bedrock-anthropic-prompt-cache.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Anthropic models served through the standard `@ai-sdk/amazon-bedrock` Converse provider are now detected as cacheable. Prompt-cache breakpoints previously only matched on the provider name, so Bedrock (which reports provider `amazon-bedrock` and carries the Anthropic identity in the model id) fell through to no caching. The cache marker now also carries the Bedrock `cachePoint` namespace that the Converse provider reads.
diff --git a/packages/eve/src/harness/prompt-cache.test.ts b/packages/eve/src/harness/prompt-cache.test.ts
index a4005bf25..4242a24e0 100644
--- a/packages/eve/src/harness/prompt-cache.test.ts
+++ b/packages/eve/src/harness/prompt-cache.test.ts
@@ -9,10 +9,10 @@ import {
mergeGatewayAutoCaching,
} from "#harness/prompt-cache.js";
-function makeObjectModel(provider: string): LanguageModel {
+function makeObjectModel(provider: string, modelId = "test-model"): LanguageModel {
return {
provider,
- modelId: "test-model",
+ modelId,
specificationVersion: "v3",
} as unknown as LanguageModel;
}
@@ -68,20 +68,45 @@ describe("detectPromptCachePath", () => {
it("returns none for generic Bedrock Converse (no anthropic subpath)", () => {
expect(detectPromptCachePath(makeObjectModel("bedrock"))).toEqual({ kind: "none" });
});
+
+ it("returns anthropic-direct for a Bedrock Converse model whose id is Anthropic", () => {
+ // `@ai-sdk/amazon-bedrock` reports provider `amazon-bedrock` and carries
+ // the Anthropic identity in the model id.
+ expect(
+ detectPromptCachePath(
+ makeObjectModel("amazon-bedrock", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
+ ),
+ ).toEqual({ kind: "anthropic-direct" });
+ });
+
+ it("matches the Bedrock Anthropic model id regardless of case", () => {
+ expect(
+ detectPromptCachePath(makeObjectModel("amazon-bedrock", "ANTHROPIC.CLAUDE-3-5-SONNET")),
+ ).toEqual({ kind: "anthropic-direct" });
+ });
+
+ it("returns none for a non-Anthropic Bedrock Converse model id", () => {
+ expect(
+ detectPromptCachePath(makeObjectModel("amazon-bedrock", "amazon.nova-pro-v1:0")),
+ ).toEqual({ kind: "none" });
+ });
});
describe("getAnthropicCacheMarker", () => {
it("returns the Anthropic cache marker shape", () => {
expect(getAnthropicCacheMarker()).toEqual({
anthropic: { cacheControl: { type: "ephemeral" } },
+ bedrock: { cachePoint: { type: "default" } },
});
});
- it("returns a frozen object", () => {
+ it("returns a deeply frozen object", () => {
const marker = getAnthropicCacheMarker();
expect(Object.isFrozen(marker)).toBe(true);
expect(Object.isFrozen(marker.anthropic)).toBe(true);
expect(Object.isFrozen(marker.anthropic.cacheControl)).toBe(true);
+ expect(Object.isFrozen(marker.bedrock)).toBe(true);
+ expect(Object.isFrozen(marker.bedrock.cachePoint)).toBe(true);
});
});
@@ -146,7 +171,7 @@ describe("applyLastToolCacheBreakpoint", () => {
expect(result.beta).toEqual({ description: "second" });
expect(result.gamma).toEqual({
description: "third",
- providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
+ providerOptions: { ...marker },
});
});
@@ -163,9 +188,10 @@ describe("applyLastToolCacheBreakpoint", () => {
{ providerOptions: Record<string, unknown> } | undefined
>;
- // The marker's anthropic namespace overrides the existing one.
+ // The marker's namespaces override any existing ones; foreign namespaces
+ // are preserved.
expect(result.only?.providerOptions).toEqual({
- anthropic: { cacheControl: { type: "ephemeral" } },
+ ...marker,
openai: { something: 2 },
});
});
@@ -197,7 +223,7 @@ describe("applyConversationCacheControl", () => {
expect(out[0]).toEqual({
role: "user",
content: "hi",
- providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
+ providerOptions: { ...marker },
});
});
@@ -214,13 +240,13 @@ describe("applyConversationCacheControl", () => {
expect(out[1]).toEqual({
role: "assistant",
content: "yo",
- providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
+ providerOptions: { ...marker },
});
expect(out[2]).toEqual({ role: "user", content: "hi2" });
expect(out[3]).toEqual({
role: "assistant",
content: "yo2",
- providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
+ providerOptions: { ...marker },
});
});
@@ -246,14 +272,10 @@ describe("applyConversationCacheControl", () => {
const out = applyConversationCacheControl(messages, marker);
// The trailing tool-result message carries the final breakpoint.
- expect((out[2] as { providerOptions?: unknown }).providerOptions).toEqual({
- anthropic: { cacheControl: { type: "ephemeral" } },
- });
+ expect((out[2] as { providerOptions?: unknown }).providerOptions).toEqual({ ...marker });
// The most recent assistant (index 1) is the advancement anchor.
- expect((out[1] as { providerOptions?: unknown }).providerOptions).toEqual({
- anthropic: { cacheControl: { type: "ephemeral" } },
- });
+ expect((out[1] as { providerOptions?: unknown }).providerOptions).toEqual({ ...marker });
// The user message is untouched.
expect(out[0]).toEqual(messages[0]);
@@ -270,7 +292,7 @@ describe("applyConversationCacheControl", () => {
const out = applyConversationCacheControl(messages, marker);
expect((out[0] as { providerOptions: Record<string, unknown> }).providerOptions).toEqual({
openai: { someKey: "someValue" },
- anthropic: { cacheControl: { type: "ephemeral" } },
+ ...marker,
});
});
diff --git a/packages/eve/src/harness/prompt-cache.ts b/packages/eve/src/harness/prompt-cache.ts
index 8f9ca4472..b94d03898 100644
--- a/packages/eve/src/harness/prompt-cache.ts
+++ b/packages/eve/src/harness/prompt-cache.ts
@@ -11,16 +11,25 @@ export type PromptCachePath =
/**
* Cache marker injected on the Anthropic-direct path.
*
- * The AI SDK Anthropic provider reads `providerOptions.anthropic.cacheControl`
- * from message, message-part, system-message, and tool objects. The same
- * namespace is used by `@ai-sdk/amazon-bedrock/anthropic` and
- * `@ai-sdk/google-vertex/anthropic`, which both implement the native
- * Anthropic Messages API.
+ * The marker carries two provider namespaces because Anthropic models are
+ * reachable through providers that read different provider-options keys:
+ *
+ * - `anthropic.cacheControl` — read by the AI SDK Anthropic provider and by
+ * `@ai-sdk/amazon-bedrock/anthropic` and `@ai-sdk/google-vertex/anthropic`,
+ * which implement the native Anthropic Messages API.
+ * - `bedrock.cachePoint` — read by the standard `@ai-sdk/amazon-bedrock`
+ * Converse provider, which does not understand `anthropic.cacheControl`.
+ *
+ * A provider ignores namespaces it does not own, so carrying both is safe on
+ * every Anthropic-direct request regardless of which provider serves it.
*/
export interface AnthropicCacheMarker {
readonly anthropic: {
readonly cacheControl: { readonly type: "ephemeral" };
};
+ readonly bedrock: {
+ readonly cachePoint: { readonly type: "default" };
+ };
}
/**
@@ -31,6 +40,9 @@ const ANTHROPIC_CACHE_MARKER: AnthropicCacheMarker = Object.freeze({
anthropic: Object.freeze({
cacheControl: Object.freeze({ type: "ephemeral" as const }),
}),
+ bedrock: Object.freeze({
+ cachePoint: Object.freeze({ type: "default" as const }),
+ }),
});
/**
@@ -48,6 +60,15 @@ export function detectPromptCachePath(model: LanguageModel): PromptCachePath {
return { kind: "anthropic-direct" };
}
+ // The standard `@ai-sdk/amazon-bedrock` Converse provider reports its
+ // provider as `amazon-bedrock` and carries the Anthropic identity in the
+ // model id (e.g. `anthropic.claude-3-5-sonnet-20241022-v2:0`), so it must be
+ // matched on the model id rather than the provider name.
+ const modelId = typeof model.modelId === "string" ? model.modelId.toLowerCase() : "";
+ if (providerName.includes("bedrock") && modelId.includes("anthropic")) {
+ return { kind: "anthropic-direct" };
+ }
+
return { kind: "none" };
}
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index e7d63a217..d4dc2feab 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -8211,6 +8211,7 @@ describe("createToolLoopHarness", () => {
const lastTool = toolEntries[toolEntries.length - 1]?.[1];
expect(lastTool?.providerOptions).toEqual({
anthropic: { cacheControl: { type: "ephemeral" } },
+ bedrock: { cachePoint: { type: "default" } },
});
});
@@ -8265,9 +8266,11 @@ describe("createToolLoopHarness", () => {
expect(result.messages?.[0]?.providerOptions).toBeUndefined();
expect(result.messages?.[1]?.providerOptions).toEqual({
anthropic: { cacheControl: { type: "ephemeral" } },
+ bedrock: { cachePoint: { type: "default" } },
});
expect(result.messages?.[2]?.providerOptions).toEqual({
anthropic: { cacheControl: { type: "ephemeral" } },
+ bedrock: { cachePoint: { type: "default" } },
});
});
@@ -9398,6 +9401,7 @@ describe("createToolLoopHarness", () => {
content: "You are a test assistant.\n\ndynamic-system-instruction",
providerOptions: {
anthropic: { cacheControl: { type: "ephemeral" } },
+ bedrock: { cachePoint: { type: "default" } },
},
});
});