🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(in-app-agent): prompt cache breakpoints for Claude behind an OpenAI Responses gateway (#16865)
David Traina72e8be9ef4f2
diff --git a/worker/src/features/in-app-agent/runtime/promptCache.test.ts b/worker/src/features/in-app-agent/runtime/promptCache.test.tsindex b173752bca..92ca7b6b55 100644--- a/worker/src/features/in-app-agent/runtime/promptCache.test.ts+++ b/worker/src/features/in-app-agent/runtime/promptCache.test.ts@@ -10,11 +10,47 @@ const anthropicCacheControl = {anthropic: { cacheControl: { type: "ephemeral" } },};+const openaiCacheBreakpoint = {+ openai: { promptCacheBreakpoint: { mode: "explicit" } },+};+const twoMessagePrompt = [{ role: "system", content: "You are the Langfuse assistant." },{ role: "user", content: [{ type: "text", text: "hello" }] },];+// One completed tool turn followed by a new user turn.+const toolTurnPrompt = [+ { role: "system", content: "You are the Langfuse assistant." },+ { role: "user", content: [{ type: "text", text: "hello" }] },+ {+ role: "assistant",+ content: [{ type: "tool-call", toolCallId: "call-1" }],+ },+ {+ role: "tool",+ content: [{ type: "tool-result", toolCallId: "call-1" }],+ },+ {+ role: "assistant",+ content: [{ type: "text", text: "You have 20 prompts." }],+ },+ {+ role: "user",+ content: [{ type: "text", text: "and the versions?" }],+ },+];++const currentTimeMessage = {+ role: "user",+ content: [+ {+ type: "text",+ text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',+ },+ ],+};+describe("applyPromptCachePoints", () => {it("caches the stable system prefix and the growing conversation prefix", () => {// Checkpoints are tools → system → messages. Tagging the last leading@@ -58,28 +94,38 @@ describe("applyPromptCachePoints", () => {});it("re-stamps the previous turn's last prefix so a follow-up user message can cache-read it", () => {+ expect(applyPromptCachePoints(toolTurnPrompt, "bedrock")).toEqual([+ {+ role: "system",+ content: "You are the Langfuse assistant.",+ providerOptions: bedrockCachePoint,+ },+ { role: "user", content: [{ type: "text", text: "hello" }] },+ {+ role: "assistant",+ content: [{ type: "tool-call", toolCallId: "call-1" }],+ },+ {+ role: "tool",+ content: [{ type: "tool-result", toolCallId: "call-1" }],+ providerOptions: bedrockCachePoint,+ },+ {+ role: "assistant",+ content: [{ type: "text", text: "You have 20 prompts." }],+ },+ {+ role: "user",+ content: [{ type: "text", text: "and the versions?" }],+ providerOptions: bedrockCachePoint,+ },+ ]);+ });++ it("keeps the previous-turn checkpoint when a trailing current-time suffix is present", () => {expect(applyPromptCachePoints(- [- { role: "system", content: "You are the Langfuse assistant." },- { role: "user", content: [{ type: "text", text: "hello" }] },- {- role: "assistant",- content: [{ type: "tool-call", toolCallId: "call-1" }],- },- {- role: "tool",- content: [{ type: "tool-result", toolCallId: "call-1" }],- },- {- role: "assistant",- content: [{ type: "text", text: "You have 20 prompts." }],- },- {- role: "user",- content: [{ type: "text", text: "and the versions?" }],- },- ],+ [...toolTurnPrompt, currentTimeMessage],"bedrock",),).toEqual([@@ -107,15 +153,36 @@ describe("applyPromptCachePoints", () => {content: [{ type: "text", text: "and the versions?" }],providerOptions: bedrockCachePoint,},+ {+ role: "user",+ content: [+ {+ type: "text",+ text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',+ },+ ],+ },]);});- it("keeps the previous-turn checkpoint when a trailing current-time suffix is present", () => {+ it("walks a Responses checkpoint back from a tool result to the nearest user message", () => {+ // The Responses converter emits `prompt_cache_breakpoint` for system+ // messages, user parts, and content-typed tool outputs, not for assistant+ // messages or this agent's JSON tool results. A checkpoint chosen for one+ // of those walks back to the nearest user or system message so the next+ // in-loop step still has an explicit read point after tools+system.expect(applyPromptCachePoints([{ role: "system", content: "You are the Langfuse assistant." },- { role: "user", content: [{ type: "text", text: "hello" }] },+ { role: "system", content: "Skill: error analysis." },+ {+ role: "user",+ content: [+ { type: "file", data: "aGk=", mediaType: "image/png" },+ { type: "text", text: "hello" },+ ],+ },{role: "assistant",content: [{ type: "tool-call", toolCallId: "call-1" }],@@ -124,33 +191,65 @@ describe("applyPromptCachePoints", () => {role: "tool",content: [{ type: "tool-result", toolCallId: "call-1" }],},+ currentTimeMessage,+ ],+ "openai-responses",+ ),+ ).toEqual([+ { role: "system", content: "You are the Langfuse assistant." },+ {+ role: "system",+ content: "Skill: error analysis.",+ providerOptions: openaiCacheBreakpoint,+ },+ {+ role: "user",+ content: [+ { type: "file", data: "aGk=", mediaType: "image/png" },{- role: "assistant",- content: [{ type: "text", text: "You have 20 prompts." }],- },- {- role: "user",- content: [{ type: "text", text: "and the versions?" }],+ type: "text",+ text: "hello",+ providerOptions: openaiCacheBreakpoint,},+ ],+ },+ {+ role: "assistant",+ content: [{ type: "tool-call", toolCallId: "call-1" }],+ },+ {+ role: "tool",+ content: [{ type: "tool-result", toolCallId: "call-1" }],+ },+ {+ role: "user",+ content: [{- role: "user",- content: [- {- type: "text",- text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',- },- ],+ type: "text",+ text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',},],- "bedrock",- ),- ).toEqual([+ },+ ]);+ });++ it("stamps the previous turn's user message and the new user turn for Responses", () => {+ expect(applyPromptCachePoints(toolTurnPrompt, "openai-responses")).toEqual([{role: "system",content: "You are the Langfuse assistant.",- providerOptions: bedrockCachePoint,+ providerOptions: openaiCacheBreakpoint,+ },+ {+ role: "user",+ content: [+ {+ type: "text",+ text: "hello",+ providerOptions: openaiCacheBreakpoint,+ },+ ],},- { role: "user", content: [{ type: "text", text: "hello" }] },{role: "assistant",content: [{ type: "tool-call", toolCallId: "call-1" }],@@ -158,23 +257,18 @@ describe("applyPromptCachePoints", () => {{role: "tool",content: [{ type: "tool-result", toolCallId: "call-1" }],- providerOptions: bedrockCachePoint,},{role: "assistant",content: [{ type: "text", text: "You have 20 prompts." }],},- {- role: "user",- content: [{ type: "text", text: "and the versions?" }],- providerOptions: bedrockCachePoint,- },{role: "user",content: [{type: "text",- text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',+ text: "and the versions?",+ providerOptions: openaiCacheBreakpoint,},],},@@ -302,6 +396,47 @@ describe("applyPromptCacheToCall", () => {});});+ it("stamps Responses prompt_cache_breakpoint for Claude ids and leaves GPT ids unchanged", () => {+ const options = {+ prompt: twoMessagePrompt,+ maxOutputTokens: 1024,+ };++ expect(+ applyPromptCacheToCall({+ provider: "openai.responses",+ modelId: "gpt-5.6-sol",+ options,+ }),+ ).toBe(options);+ expect(+ applyPromptCacheToCall({+ provider: "openai.responses",+ modelId: "claude-opus-5",+ options,+ }),+ ).toEqual({+ maxOutputTokens: 1024,+ prompt: [+ {+ role: "system",+ content: "You are the Langfuse assistant.",+ providerOptions: openaiCacheBreakpoint,+ },+ {+ role: "user",+ content: [+ {+ type: "text",+ text: "hello",+ providerOptions: openaiCacheBreakpoint,+ },+ ],+ },+ ],+ });+ });+it("stamps Bedrock cache points for Claude models and leaves other models unchanged", () => {const options = {prompt: twoMessagePrompt,diff --git a/worker/src/features/in-app-agent/runtime/promptCache.ts b/worker/src/features/in-app-agent/runtime/promptCache.tsindex 534542c18d..6b179bee8b 100644--- a/worker/src/features/in-app-agent/runtime/promptCache.ts+++ b/worker/src/features/in-app-agent/runtime/promptCache.ts@@ -1,19 +1,27 @@import { isRecord } from "@langfuse/shared/in-app-agent/server/toolErrors";-type PromptCacheProvider = "bedrock" | "anthropic" | "openai-compatible";+type PromptCacheProvider =+ | "bedrock"+ | "anthropic"+ | "openai-compatible"+ | "openai-responses";const BEDROCK_CLAUDE_MODEL_ID_PART = "anthropic.claude";const ANTHROPIC_CLAUDE_MODEL_ID_PART = "claude";const BEDROCK_PROMPT_CACHE_POINT = { type: "default" as const };const ANTHROPIC_CACHE_CONTROL = { type: "ephemeral" as const };+const OPENAI_PROMPT_CACHE_BREAKPOINT = { mode: "explicit" as const };/*** Prompt cache for Claude on Bedrock Converse, native Anthropic Messages,- * and Anthropic slugs behind an OpenAI-compatible gateway (OpenRouter,+ * and Claude behind an OpenAI-compatible gateway (Ramp Router, OpenRouter,* LiteLLM). Native Messages get `anthropic.cacheControl`. Compatible* Chat Completions get `openaiCompatible.cache_control` because that SDK- * only forwards the `openaiCompatible` namespace.+ * only forwards the `openaiCompatible` namespace. Responses get+ * `openai.promptCacheBreakpoint`, which the SDK emits as the+ * `prompt_cache_breakpoint` marker; a translating gateway maps it to+ * Anthropic `cache_control`.** A checkpoint writes the prefix `tools → that message`. The next call* cache-reads only if that prefix is byte-identical. Hits last 5 minutes@@ -35,6 +43,13 @@ const ANTHROPIC_CACHE_CONTROL = { type: "ephemeral" as const };* new user turn. The previous write sat on a tool result or prior user,* not the closing assistant (that was model output). Skip trailing* assistants, then stamp that predecessor.+ *+ * The Responses converter emits the marker only for system messages, user+ * parts, and content-typed tool outputs. Assistant messages never carry it+ * and this agent returns JSON tool results, so a checkpoint chosen for an+ * assistant or tool message moves back to the nearest preceding user or+ * system message. In-loop steps read through the latest user turn and send+ * this turn's tool calls and results uncached.*/export function applyPromptCacheToCall<T>({provider,@@ -78,11 +93,18 @@ export function applyPromptCachePoints(const lastConversationIndex = getLastConversationIndex(prompt);const cacheIndices = new Set<number>();+ const addCheckpoint = (index: number) => {+ const carrierIndex = findCarrierIndex(prompt, index, cacheProvider);+ if (carrierIndex >= 0) {+ cacheIndices.add(carrierIndex);+ }+ };+if (lastConversationIndex >= 0) {- cacheIndices.add(lastConversationIndex);+ addCheckpoint(lastConversationIndex);}if (lastLeadingSystemIndex >= 0) {- cacheIndices.add(lastLeadingSystemIndex);+ addCheckpoint(lastLeadingSystemIndex);}const previousTurnIndex = findPreviousTurnCacheIndex(@@ -91,7 +113,7 @@ export function applyPromptCachePoints(lastConversationIndex,);if (previousTurnIndex >= 0) {- cacheIndices.add(previousTurnIndex);+ addCheckpoint(previousTurnIndex);}return prompt.map((message, index) =>@@ -99,6 +121,37 @@ export function applyPromptCachePoints();}+function findCarrierIndex(+ prompt: unknown[],+ index: number,+ cacheProvider: PromptCacheProvider,+) {+ for (let i = index; i >= 0; i--) {+ if (canCarryPromptCache(prompt[i], cacheProvider)) {+ return i;+ }+ }+ return -1;+}++function canCarryPromptCache(+ message: unknown,+ cacheProvider: PromptCacheProvider,+) {+ if (cacheProvider !== "openai-responses") {+ return true;+ }++ const role = getMessageRole(message);+ return (+ role === "system" ||+ (role === "user" &&+ isRecord(message) &&+ Array.isArray(message.content) &&+ message.content.length > 0)+ );+}+function resolvePromptCacheProvider(provider: string,modelId: string,@@ -115,6 +168,12 @@ function resolvePromptCacheProvider() {return "anthropic";}+ if (+ provider.includes("openai.responses") &&+ modelId.includes(ANTHROPIC_CLAUDE_MODEL_ID_PART)+ ) {+ return "openai-responses";+ }// OpenRouter / LiteLLM Chat Completions: provider is `openai.chat` but// the upstream model id is still `anthropic/claude-…`.if (modelId.includes("anthropic")) {@@ -192,9 +251,74 @@ function withPromptCache(message: unknown, cacheProvider: PromptCacheProvider) {if (cacheProvider === "openai-compatible") {return withOpenAICompatibleCacheControl(message);}+ if (cacheProvider === "openai-responses") {+ return withOpenAIResponsesCacheBreakpoint(message);+ }return withAnthropicCacheControl(message);}+// The Responses converter reads `openai.promptCacheBreakpoint` from the+// message for system messages and from each part for user messages.+function withOpenAIResponsesCacheBreakpoint(message: unknown) {+ if (!isRecord(message)) {+ return message;+ }++ if (getMessageRole(message) === "system") {+ return {+ ...message,+ providerOptions: stampOpenAIPromptCacheBreakpoint(+ message.providerOptions,+ ),+ };+ }++ return withLastPartProviderOptions(message, stampOpenAIPromptCacheBreakpoint);+}++function stampOpenAIPromptCacheBreakpoint(providerOptions: unknown) {+ const options = isRecord(providerOptions) ? providerOptions : {};+ const openai = isRecord(options.openai) ? options.openai : {};++ if (isRecord(openai.promptCacheBreakpoint)) {+ return options;+ }++ return {+ ...options,+ openai: {+ ...openai,+ promptCacheBreakpoint: OPENAI_PROMPT_CACHE_BREAKPOINT,+ },+ };+}++function withLastPartProviderOptions(+ message: Record<string, unknown>,+ stamp: (providerOptions: unknown) => unknown,+) {+ if (!Array.isArray(message.content) || message.content.length === 0) {+ return message;+ }++ const lastIndex = message.content.length - 1;+ const lastPart = message.content[lastIndex];+ if (!isRecord(lastPart)) {+ return message;+ }++ const nextContent = message.content.slice();+ nextContent[lastIndex] = {+ ...lastPart,+ providerOptions: stamp(lastPart.providerOptions),+ };++ return {+ ...message,+ content: nextContent,+ };+}+function withBedrockCachePoint(message: unknown) {if (!isRecord(message)) {return message;@@ -267,56 +391,30 @@ function withOpenAICompatibleCacheControl(message: unknown) {return message;}- return withOpenAICompatibleCacheControlOnLastPart({- ...message,- providerOptions: {- ...providerOptions,- openaiCompatible: {- ...openaiCompatible,- cache_control: ANTHROPIC_CACHE_CONTROL,- },+ return withLastPartProviderOptions(+ {+ ...message,+ providerOptions: stampOpenAICompatibleCacheControl(providerOptions),},- });+ stampOpenAICompatibleCacheControl,+ );}-function withOpenAICompatibleCacheControlOnLastPart(- message: Record<string, unknown>,-) {- if (!Array.isArray(message.content) || message.content.length === 0) {- return message;- }-- const lastIndex = message.content.length - 1;- const lastPart = message.content[lastIndex];- if (!isRecord(lastPart)) {- return message;- }-- const partOptions = isRecord(lastPart.providerOptions)- ? lastPart.providerOptions- : {};- const openaiCompatible = isRecord(partOptions.openaiCompatible)- ? partOptions.openaiCompatible+function stampOpenAICompatibleCacheControl(providerOptions: unknown) {+ const options = isRecord(providerOptions) ? providerOptions : {};+ const openaiCompatible = isRecord(options.openaiCompatible)+ ? options.openaiCompatible: {};if (isRecord(openaiCompatible.cache_control)) {- return message;+ return options;}- const nextContent = message.content.slice();- nextContent[lastIndex] = {- ...lastPart,- providerOptions: {- ...partOptions,- openaiCompatible: {- ...openaiCompatible,- cache_control: ANTHROPIC_CACHE_CONTROL,- },- },- };-return {- ...message,- content: nextContent,+ ...options,+ openaiCompatible: {+ ...openaiCompatible,+ cache_control: ANTHROPIC_CACHE_CONTROL,+ },};}diff --git a/worker/src/features/in-app-agent/runtime/reasoning.test.ts b/worker/src/features/in-app-agent/runtime/reasoning.test.tsindex 98ccb72a98..12e070279b 100644--- a/worker/src/features/in-app-agent/runtime/reasoning.test.ts+++ b/worker/src/features/in-app-agent/runtime/reasoning.test.ts@@ -158,6 +158,60 @@ describe("OpenAI Responses request shape", () => {expect(calls[0]?.body.include).toEqual(["reasoning.encrypted_content"]);});+ it("posts prompt_cache_breakpoint on Responses input for Claude ids", async () => {+ Object.assign(env, { LANGFUSE_AI_USE_RESPONSES_API: "true" });++ const config = {+ provider: "openai" as const,+ modelId: "claude-opus-5",+ titleModelId: "gpt-5.6-luna",+ apiKey: "sk-test",+ baseURL: "https://llm-gateway.internal/v1",+ };+ const { calls, fetch } = createCaptureFetch(OPENAI_RESPONSES_RESPONSE);+ vi.stubGlobal("fetch", fetch);++ const model = createInAppAgentLanguageModel({ config });+ const prompt = [+ { role: "system" as const, content: "You are the Langfuse assistant." },+ {+ role: "user" as const,+ content: [{ type: "text" as const, text: "hello" }],+ },+ ];+ await model.doGenerate(+ applyPromptCacheToCall({+ provider: String(model.provider),+ modelId: model.modelId,+ options: { prompt },+ }),+ );++ expect(calls[0]?.url).toBe("https://llm-gateway.internal/v1/responses");+ expect(calls[0]?.body.input).toEqual([+ {+ role: "system",+ content: [+ {+ type: "input_text",+ text: "You are the Langfuse assistant.",+ prompt_cache_breakpoint: { mode: "explicit" },+ },+ ],+ },+ {+ role: "user",+ content: [+ {+ type: "input_text",+ text: "hello",+ prompt_cache_breakpoint: { mode: "explicit" },+ },+ ],+ },+ ]);+ });+it("posts to /v1/responses when a custom URL opts into Responses", async () => {Object.assign(env, { LANGFUSE_AI_USE_RESPONSES_API: "true" });