Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(in-app-agent): prompt cache breakpoints for Claude behind an OpenAI Responses gateway (#16865)

David Traina72e8be9ef4f2
diff --git a/worker/src/features/in-app-agent/runtime/promptCache.test.ts b/worker/src/features/in-app-agent/runtime/promptCache.test.ts
index b173752bca..92ca7b6b55 100644
--- a/worker/src/features/in-app-agent/runtime/promptCache.test.ts
+++ b/worker/src/features/in-app-agent/runtime/promptCache.test.ts
@@ -10,11 +10,47 @@ const anthropicCacheControl = {
anthropic: { cacheControl: { type: "ephemeral" } },
};
+const openaiCacheBreakpoint = {
+ openai: { promptCacheBreakpoint: { mode: "explicit" } },
+};
+
const twoMessagePrompt = [
{ role: "system", content: "You are the Langfuse assistant." },
{ role: "user", content: [{ type: "text", text: "hello" }] },
];
+// One completed tool turn followed by a new user turn.
+const toolTurnPrompt = [
+ { role: "system", content: "You are the Langfuse assistant." },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ },
+ {
+ role: "assistant",
+ content: [{ type: "text", text: "You have 20 prompts." }],
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "and the versions?" }],
+ },
+];
+
+const currentTimeMessage = {
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',
+ },
+ ],
+};
+
describe("applyPromptCachePoints", () => {
it("caches the stable system prefix and the growing conversation prefix", () => {
// Checkpoints are tools → system → messages. Tagging the last leading
@@ -58,28 +94,38 @@ describe("applyPromptCachePoints", () => {
});
it("re-stamps the previous turn's last prefix so a follow-up user message can cache-read it", () => {
+ expect(applyPromptCachePoints(toolTurnPrompt, "bedrock")).toEqual([
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: bedrockCachePoint,
+ },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ providerOptions: bedrockCachePoint,
+ },
+ {
+ role: "assistant",
+ content: [{ type: "text", text: "You have 20 prompts." }],
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "and the versions?" }],
+ providerOptions: bedrockCachePoint,
+ },
+ ]);
+ });
+
+ it("keeps the previous-turn checkpoint when a trailing current-time suffix is present", () => {
expect(
applyPromptCachePoints(
- [
- { role: "system", content: "You are the Langfuse assistant." },
- { role: "user", content: [{ type: "text", text: "hello" }] },
- {
- role: "assistant",
- content: [{ type: "tool-call", toolCallId: "call-1" }],
- },
- {
- role: "tool",
- content: [{ type: "tool-result", toolCallId: "call-1" }],
- },
- {
- role: "assistant",
- content: [{ type: "text", text: "You have 20 prompts." }],
- },
- {
- role: "user",
- content: [{ type: "text", text: "and the versions?" }],
- },
- ],
+ [...toolTurnPrompt, currentTimeMessage],
"bedrock",
),
).toEqual([
@@ -107,15 +153,36 @@ describe("applyPromptCachePoints", () => {
content: [{ type: "text", text: "and the versions?" }],
providerOptions: bedrockCachePoint,
},
+ {
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',
+ },
+ ],
+ },
]);
});
- it("keeps the previous-turn checkpoint when a trailing current-time suffix is present", () => {
+ it("walks a Responses checkpoint back from a tool result to the nearest user message", () => {
+ // The Responses converter emits `prompt_cache_breakpoint` for system
+ // messages, user parts, and content-typed tool outputs, not for assistant
+ // messages or this agent's JSON tool results. A checkpoint chosen for one
+ // of those walks back to the nearest user or system message so the next
+ // in-loop step still has an explicit read point after tools+system.
expect(
applyPromptCachePoints(
[
{ role: "system", content: "You are the Langfuse assistant." },
- { role: "user", content: [{ type: "text", text: "hello" }] },
+ { role: "system", content: "Skill: error analysis." },
+ {
+ role: "user",
+ content: [
+ { type: "file", data: "aGk=", mediaType: "image/png" },
+ { type: "text", text: "hello" },
+ ],
+ },
{
role: "assistant",
content: [{ type: "tool-call", toolCallId: "call-1" }],
@@ -124,33 +191,65 @@ describe("applyPromptCachePoints", () => {
role: "tool",
content: [{ type: "tool-result", toolCallId: "call-1" }],
},
+ currentTimeMessage,
+ ],
+ "openai-responses",
+ ),
+ ).toEqual([
+ { role: "system", content: "You are the Langfuse assistant." },
+ {
+ role: "system",
+ content: "Skill: error analysis.",
+ providerOptions: openaiCacheBreakpoint,
+ },
+ {
+ role: "user",
+ content: [
+ { type: "file", data: "aGk=", mediaType: "image/png" },
{
- role: "assistant",
- content: [{ type: "text", text: "You have 20 prompts." }],
- },
- {
- role: "user",
- content: [{ type: "text", text: "and the versions?" }],
+ type: "text",
+ text: "hello",
+ providerOptions: openaiCacheBreakpoint,
},
+ ],
+ },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ },
+ {
+ role: "user",
+ content: [
{
- role: "user",
- content: [
- {
- type: "text",
- text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',
- },
- ],
+ type: "text",
+ text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',
},
],
- "bedrock",
- ),
- ).toEqual([
+ },
+ ]);
+ });
+
+ it("stamps the previous turn's user message and the new user turn for Responses", () => {
+ expect(applyPromptCachePoints(toolTurnPrompt, "openai-responses")).toEqual([
{
role: "system",
content: "You are the Langfuse assistant.",
- providerOptions: bedrockCachePoint,
+ providerOptions: openaiCacheBreakpoint,
+ },
+ {
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: "hello",
+ providerOptions: openaiCacheBreakpoint,
+ },
+ ],
},
- { role: "user", content: [{ type: "text", text: "hello" }] },
{
role: "assistant",
content: [{ type: "tool-call", toolCallId: "call-1" }],
@@ -158,23 +257,18 @@ describe("applyPromptCachePoints", () => {
{
role: "tool",
content: [{ type: "tool-result", toolCallId: "call-1" }],
- providerOptions: bedrockCachePoint,
},
{
role: "assistant",
content: [{ type: "text", text: "You have 20 prompts." }],
},
- {
- role: "user",
- content: [{ type: "text", text: "and the versions?" }],
- providerOptions: bedrockCachePoint,
- },
{
role: "user",
content: [
{
type: "text",
- text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',
+ text: "and the versions?",
+ providerOptions: openaiCacheBreakpoint,
},
],
},
@@ -302,6 +396,47 @@ describe("applyPromptCacheToCall", () => {
});
});
+ it("stamps Responses prompt_cache_breakpoint for Claude ids and leaves GPT ids unchanged", () => {
+ const options = {
+ prompt: twoMessagePrompt,
+ maxOutputTokens: 1024,
+ };
+
+ expect(
+ applyPromptCacheToCall({
+ provider: "openai.responses",
+ modelId: "gpt-5.6-sol",
+ options,
+ }),
+ ).toBe(options);
+ expect(
+ applyPromptCacheToCall({
+ provider: "openai.responses",
+ modelId: "claude-opus-5",
+ options,
+ }),
+ ).toEqual({
+ maxOutputTokens: 1024,
+ prompt: [
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: openaiCacheBreakpoint,
+ },
+ {
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: "hello",
+ providerOptions: openaiCacheBreakpoint,
+ },
+ ],
+ },
+ ],
+ });
+ });
+
it("stamps Bedrock cache points for Claude models and leaves other models unchanged", () => {
const options = {
prompt: twoMessagePrompt,
diff --git a/worker/src/features/in-app-agent/runtime/promptCache.ts b/worker/src/features/in-app-agent/runtime/promptCache.ts
index 534542c18d..6b179bee8b 100644
--- a/worker/src/features/in-app-agent/runtime/promptCache.ts
+++ b/worker/src/features/in-app-agent/runtime/promptCache.ts
@@ -1,19 +1,27 @@
import { isRecord } from "@langfuse/shared/in-app-agent/server/toolErrors";
-type PromptCacheProvider = "bedrock" | "anthropic" | "openai-compatible";
+type PromptCacheProvider =
+ | "bedrock"
+ | "anthropic"
+ | "openai-compatible"
+ | "openai-responses";
const BEDROCK_CLAUDE_MODEL_ID_PART = "anthropic.claude";
const ANTHROPIC_CLAUDE_MODEL_ID_PART = "claude";
const BEDROCK_PROMPT_CACHE_POINT = { type: "default" as const };
const ANTHROPIC_CACHE_CONTROL = { type: "ephemeral" as const };
+const OPENAI_PROMPT_CACHE_BREAKPOINT = { mode: "explicit" as const };
/**
* Prompt cache for Claude on Bedrock Converse, native Anthropic Messages,
- * and Anthropic slugs behind an OpenAI-compatible gateway (OpenRouter,
+ * and Claude behind an OpenAI-compatible gateway (Ramp Router, OpenRouter,
* LiteLLM). Native Messages get `anthropic.cacheControl`. Compatible
* Chat Completions get `openaiCompatible.cache_control` because that SDK
- * only forwards the `openaiCompatible` namespace.
+ * only forwards the `openaiCompatible` namespace. Responses get
+ * `openai.promptCacheBreakpoint`, which the SDK emits as the
+ * `prompt_cache_breakpoint` marker; a translating gateway maps it to
+ * Anthropic `cache_control`.
*
* A checkpoint writes the prefix `tools → that message`. The next call
* cache-reads only if that prefix is byte-identical. Hits last 5 minutes
@@ -35,6 +43,13 @@ const ANTHROPIC_CACHE_CONTROL = { type: "ephemeral" as const };
* new user turn. The previous write sat on a tool result or prior user,
* not the closing assistant (that was model output). Skip trailing
* assistants, then stamp that predecessor.
+ *
+ * The Responses converter emits the marker only for system messages, user
+ * parts, and content-typed tool outputs. Assistant messages never carry it
+ * and this agent returns JSON tool results, so a checkpoint chosen for an
+ * assistant or tool message moves back to the nearest preceding user or
+ * system message. In-loop steps read through the latest user turn and send
+ * this turn's tool calls and results uncached.
*/
export function applyPromptCacheToCall<T>({
provider,
@@ -78,11 +93,18 @@ export function applyPromptCachePoints(
const lastConversationIndex = getLastConversationIndex(prompt);
const cacheIndices = new Set<number>();
+ const addCheckpoint = (index: number) => {
+ const carrierIndex = findCarrierIndex(prompt, index, cacheProvider);
+ if (carrierIndex >= 0) {
+ cacheIndices.add(carrierIndex);
+ }
+ };
+
if (lastConversationIndex >= 0) {
- cacheIndices.add(lastConversationIndex);
+ addCheckpoint(lastConversationIndex);
}
if (lastLeadingSystemIndex >= 0) {
- cacheIndices.add(lastLeadingSystemIndex);
+ addCheckpoint(lastLeadingSystemIndex);
}
const previousTurnIndex = findPreviousTurnCacheIndex(
@@ -91,7 +113,7 @@ export function applyPromptCachePoints(
lastConversationIndex,
);
if (previousTurnIndex >= 0) {
- cacheIndices.add(previousTurnIndex);
+ addCheckpoint(previousTurnIndex);
}
return prompt.map((message, index) =>
@@ -99,6 +121,37 @@ export function applyPromptCachePoints(
);
}
+function findCarrierIndex(
+ prompt: unknown[],
+ index: number,
+ cacheProvider: PromptCacheProvider,
+) {
+ for (let i = index; i >= 0; i--) {
+ if (canCarryPromptCache(prompt[i], cacheProvider)) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+function canCarryPromptCache(
+ message: unknown,
+ cacheProvider: PromptCacheProvider,
+) {
+ if (cacheProvider !== "openai-responses") {
+ return true;
+ }
+
+ const role = getMessageRole(message);
+ return (
+ role === "system" ||
+ (role === "user" &&
+ isRecord(message) &&
+ Array.isArray(message.content) &&
+ message.content.length > 0)
+ );
+}
+
function resolvePromptCacheProvider(
provider: string,
modelId: string,
@@ -115,6 +168,12 @@ function resolvePromptCacheProvider(
) {
return "anthropic";
}
+ if (
+ provider.includes("openai.responses") &&
+ modelId.includes(ANTHROPIC_CLAUDE_MODEL_ID_PART)
+ ) {
+ return "openai-responses";
+ }
// OpenRouter / LiteLLM Chat Completions: provider is `openai.chat` but
// the upstream model id is still `anthropic/claude-…`.
if (modelId.includes("anthropic")) {
@@ -192,9 +251,74 @@ function withPromptCache(message: unknown, cacheProvider: PromptCacheProvider) {
if (cacheProvider === "openai-compatible") {
return withOpenAICompatibleCacheControl(message);
}
+ if (cacheProvider === "openai-responses") {
+ return withOpenAIResponsesCacheBreakpoint(message);
+ }
return withAnthropicCacheControl(message);
}
+// The Responses converter reads `openai.promptCacheBreakpoint` from the
+// message for system messages and from each part for user messages.
+function withOpenAIResponsesCacheBreakpoint(message: unknown) {
+ if (!isRecord(message)) {
+ return message;
+ }
+
+ if (getMessageRole(message) === "system") {
+ return {
+ ...message,
+ providerOptions: stampOpenAIPromptCacheBreakpoint(
+ message.providerOptions,
+ ),
+ };
+ }
+
+ return withLastPartProviderOptions(message, stampOpenAIPromptCacheBreakpoint);
+}
+
+function stampOpenAIPromptCacheBreakpoint(providerOptions: unknown) {
+ const options = isRecord(providerOptions) ? providerOptions : {};
+ const openai = isRecord(options.openai) ? options.openai : {};
+
+ if (isRecord(openai.promptCacheBreakpoint)) {
+ return options;
+ }
+
+ return {
+ ...options,
+ openai: {
+ ...openai,
+ promptCacheBreakpoint: OPENAI_PROMPT_CACHE_BREAKPOINT,
+ },
+ };
+}
+
+function withLastPartProviderOptions(
+ message: Record<string, unknown>,
+ stamp: (providerOptions: unknown) => unknown,
+) {
+ if (!Array.isArray(message.content) || message.content.length === 0) {
+ return message;
+ }
+
+ const lastIndex = message.content.length - 1;
+ const lastPart = message.content[lastIndex];
+ if (!isRecord(lastPart)) {
+ return message;
+ }
+
+ const nextContent = message.content.slice();
+ nextContent[lastIndex] = {
+ ...lastPart,
+ providerOptions: stamp(lastPart.providerOptions),
+ };
+
+ return {
+ ...message,
+ content: nextContent,
+ };
+}
+
function withBedrockCachePoint(message: unknown) {
if (!isRecord(message)) {
return message;
@@ -267,56 +391,30 @@ function withOpenAICompatibleCacheControl(message: unknown) {
return message;
}
- return withOpenAICompatibleCacheControlOnLastPart({
- ...message,
- providerOptions: {
- ...providerOptions,
- openaiCompatible: {
- ...openaiCompatible,
- cache_control: ANTHROPIC_CACHE_CONTROL,
- },
+ return withLastPartProviderOptions(
+ {
+ ...message,
+ providerOptions: stampOpenAICompatibleCacheControl(providerOptions),
},
- });
+ stampOpenAICompatibleCacheControl,
+ );
}
-function withOpenAICompatibleCacheControlOnLastPart(
- message: Record<string, unknown>,
-) {
- if (!Array.isArray(message.content) || message.content.length === 0) {
- return message;
- }
-
- const lastIndex = message.content.length - 1;
- const lastPart = message.content[lastIndex];
- if (!isRecord(lastPart)) {
- return message;
- }
-
- const partOptions = isRecord(lastPart.providerOptions)
- ? lastPart.providerOptions
- : {};
- const openaiCompatible = isRecord(partOptions.openaiCompatible)
- ? partOptions.openaiCompatible
+function stampOpenAICompatibleCacheControl(providerOptions: unknown) {
+ const options = isRecord(providerOptions) ? providerOptions : {};
+ const openaiCompatible = isRecord(options.openaiCompatible)
+ ? options.openaiCompatible
: {};
if (isRecord(openaiCompatible.cache_control)) {
- return message;
+ return options;
}
- const nextContent = message.content.slice();
- nextContent[lastIndex] = {
- ...lastPart,
- providerOptions: {
- ...partOptions,
- openaiCompatible: {
- ...openaiCompatible,
- cache_control: ANTHROPIC_CACHE_CONTROL,
- },
- },
- };
-
return {
- ...message,
- content: nextContent,
+ ...options,
+ openaiCompatible: {
+ ...openaiCompatible,
+ cache_control: ANTHROPIC_CACHE_CONTROL,
+ },
};
}
diff --git a/worker/src/features/in-app-agent/runtime/reasoning.test.ts b/worker/src/features/in-app-agent/runtime/reasoning.test.ts
index 98ccb72a98..12e070279b 100644
--- a/worker/src/features/in-app-agent/runtime/reasoning.test.ts
+++ b/worker/src/features/in-app-agent/runtime/reasoning.test.ts
@@ -158,6 +158,60 @@ describe("OpenAI Responses request shape", () => {
expect(calls[0]?.body.include).toEqual(["reasoning.encrypted_content"]);
});
+ it("posts prompt_cache_breakpoint on Responses input for Claude ids", async () => {
+ Object.assign(env, { LANGFUSE_AI_USE_RESPONSES_API: "true" });
+
+ const config = {
+ provider: "openai" as const,
+ modelId: "claude-opus-5",
+ titleModelId: "gpt-5.6-luna",
+ apiKey: "sk-test",
+ baseURL: "https://llm-gateway.internal/v1",
+ };
+ const { calls, fetch } = createCaptureFetch(OPENAI_RESPONSES_RESPONSE);
+ vi.stubGlobal("fetch", fetch);
+
+ const model = createInAppAgentLanguageModel({ config });
+ const prompt = [
+ { role: "system" as const, content: "You are the Langfuse assistant." },
+ {
+ role: "user" as const,
+ content: [{ type: "text" as const, text: "hello" }],
+ },
+ ];
+ await model.doGenerate(
+ applyPromptCacheToCall({
+ provider: String(model.provider),
+ modelId: model.modelId,
+ options: { prompt },
+ }),
+ );
+
+ expect(calls[0]?.url).toBe("https://llm-gateway.internal/v1/responses");
+ expect(calls[0]?.body.input).toEqual([
+ {
+ role: "system",
+ content: [
+ {
+ type: "input_text",
+ text: "You are the Langfuse assistant.",
+ prompt_cache_breakpoint: { mode: "explicit" },
+ },
+ ],
+ },
+ {
+ role: "user",
+ content: [
+ {
+ type: "input_text",
+ text: "hello",
+ prompt_cache_breakpoint: { mode: "explicit" },
+ },
+ ],
+ },
+ ]);
+ });
+
it("posts to /v1/responses when a custom URL opts into Responses", async () => {
Object.assign(env, { LANGFUSE_AI_USE_RESPONSES_API: "true" });