🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(in-app-agent): support OpenAI-compatible APIs for instance AI features (#16581)
Lysander Kiesel67eec1cbd534
diff --git a/.env.dev.example b/.env.dev.exampleindex 66d7ffca59..f8241ccd62 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -196,7 +196,8 @@ LANGFUSE_AI_FEATURES_PROJECT_ID=7a88fb47-b4e2-43b8-a06c-a5ce950dc53a# Langfuse AI. Unset LANGFUSE_AI_PROVIDER defaults to Bedrock, including when# NEXT_PUBLIC_LANGFUSE_CLOUD_REGION is set.# LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION-# apply to both providers.+# apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /+# LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.# Langfuse AI Bedrock credentialsAWS_ACCESS_KEY_ID="A123456789"@@ -213,6 +214,19 @@ LANGFUSE_AI_SMALL_MODEL="eu.anthropic.claude-haiku-4-5-20251001-v1:0"# LANGFUSE_AI_API_KEY="sk-ant-..."# LANGFUSE_AI_BASE_URL="https://api.anthropic.com"# Do not set ANTHROPIC_* — Claude Code injects those without /v1 and can 404.+# OpenAI or OpenAI-compatible (LiteLLM, internal proxy).+# Omit LANGFUSE_AI_BASE_URL for api.openai.com (Responses API is the default).+# Custom URLs default to Chat Completions. Set LANGFUSE_AI_USE_RESPONSES_API+# to "true" when the gateway implements /v1/responses (tools + thinking on+# gpt-5.6). Authorization: Bearer still comes from LANGFUSE_AI_API_KEY;+# extra headers are additive JSON, not a replacement for Bearer.+# LANGFUSE_AI_PROVIDER="openai"+# LANGFUSE_AI_MODEL="gpt-5.6-sol"+# LANGFUSE_AI_SMALL_MODEL="gpt-5.6-luna"+# LANGFUSE_AI_API_KEY="sk-..."+# LANGFUSE_AI_BASE_URL="https://llm-exec.internal/v1"+# LANGFUSE_AI_USE_RESPONSES_API="true"+# LANGFUSE_AI_EXTRA_HEADERS='{"X-LLM-Exec-Token":"..."}'LANGFUSE_IN_APP_AGENT_AWS_PROFILE="playground"# Self-hosted Assistant instance switch. Unset or "false" hides Assistant.# Cloud is on unless this is "false".diff --git a/.env.prod.example b/.env.prod.exampleindex 16187443b8..1debe46c72 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -295,7 +295,7 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# One instance-wide model for the Assistant and Ask AI, on web and worker.# Unset LANGFUSE_AI_PROVIDER defaults to bedrock. Bedrock authenticates through-# the AWS credential chain; anthropic requires LANGFUSE_AI_API_KEY.+# the AWS credential chain; anthropic and openai require LANGFUSE_AI_API_KEY.# LANGFUSE_AI_PROVIDER=bedrock# LANGFUSE_AI_MODEL=eu.anthropic.claude-opus-5# LANGFUSE_AI_SMALL_MODEL=eu.anthropic.claude-haiku-4-5-20251001-v1:0@@ -303,6 +303,17 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# LANGFUSE_AI_API_KEY=# Origin only, the app appends /v1# LANGFUSE_AI_BASE_URL=https://api.anthropic.com+# OpenAI: omit LANGFUSE_AI_BASE_URL for api.openai.com (Responses API is the+# default). OpenAI-compatible: include /v1 on LANGFUSE_AI_BASE_URL (Chat+# Completions unless LANGFUSE_AI_USE_RESPONSES_API=true and the gateway+# implements /v1/responses). Extra headers are additive JSON;+# Authorization: Bearer is still sent from LANGFUSE_AI_API_KEY.+# LANGFUSE_AI_PROVIDER=openai+# LANGFUSE_AI_MODEL=gpt-5.6-sol+# LANGFUSE_AI_SMALL_MODEL=gpt-5.6-luna+# LANGFUSE_AI_BASE_URL=https://llm-exec.internal/v1+# LANGFUSE_AI_USE_RESPONSES_API=true+# LANGFUSE_AI_EXTRA_HEADERS={"X-LLM-Exec-Token":"..."}# Workers follow LANGFUSE_IN_APP_AGENT_ENABLED for the run queue and integrity# runner. Split-role workers (ingestion-only) can opt out without turning thediff --git a/docker-compose.build.yml b/docker-compose.build.ymlindex 4365d68222..d06dae168c 100644--- a/docker-compose.build.yml+++ b/docker-compose.build.yml@@ -63,6 +63,8 @@ services:LANGFUSE_AI_SMALL_MODEL: ${LANGFUSE_AI_SMALL_MODEL:-}LANGFUSE_AI_API_KEY: ${LANGFUSE_AI_API_KEY:-}LANGFUSE_AI_BASE_URL: ${LANGFUSE_AI_BASE_URL:-}+ LANGFUSE_AI_USE_RESPONSES_API: ${LANGFUSE_AI_USE_RESPONSES_API:-}+ LANGFUSE_AI_EXTRA_HEADERS: ${LANGFUSE_AI_EXTRA_HEADERS:-}LANGFUSE_AI_AWS_BEDROCK_REGION: ${LANGFUSE_AI_AWS_BEDROCK_REGION:-}LANGFUSE_IN_APP_AGENT_ENABLED: ${LANGFUSE_IN_APP_AGENT_ENABLED:-}AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}diff --git a/docker-compose.yml b/docker-compose.ymlindex b7f5f80b60..821a5668f4 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -82,6 +82,8 @@ services:LANGFUSE_AI_SMALL_MODEL: ${LANGFUSE_AI_SMALL_MODEL:-}LANGFUSE_AI_API_KEY: ${LANGFUSE_AI_API_KEY:-}LANGFUSE_AI_BASE_URL: ${LANGFUSE_AI_BASE_URL:-}+ LANGFUSE_AI_USE_RESPONSES_API: ${LANGFUSE_AI_USE_RESPONSES_API:-}+ LANGFUSE_AI_EXTRA_HEADERS: ${LANGFUSE_AI_EXTRA_HEADERS:-}LANGFUSE_AI_AWS_BEDROCK_REGION: ${LANGFUSE_AI_AWS_BEDROCK_REGION:-}AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}diff --git a/packages/shared/package.json b/packages/shared/package.jsonindex 5bd752e4e2..3829811350 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -110,6 +110,10 @@"./in-app-agent/server/modelProvider": {"import": "./dist/src/in-app-agent/server/modelProvider.js","require": "./dist/src/in-app-agent/server/modelProvider.js"+ },+ "./in-app-agent/server/openaiCompatibility": {+ "import": "./dist/src/in-app-agent/server/openaiCompatibility.js",+ "require": "./dist/src/in-app-agent/server/openaiCompatibility.js"}},"scripts": {diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex f9fb833b2b..956ee1336d 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -523,13 +523,36 @@ const EnvSchema = z.object({.default(DEFAULT_LLM_COMPLETION_TIMEOUT_MS), // 2 minutes// LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION- // apply to both providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL are- // Anthropic-only.- LANGFUSE_AI_PROVIDER: z.enum(["bedrock", "anthropic"]).optional(),+ // apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /+ // LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.+ // LANGFUSE_AI_USE_RESPONSES_API applies to openai only.+ LANGFUSE_AI_PROVIDER: z.enum(["bedrock", "anthropic", "openai"]).optional(),LANGFUSE_AI_MODEL: z.string().optional(),LANGFUSE_AI_SMALL_MODEL: z.string().optional(),LANGFUSE_AI_API_KEY: z.string().optional(),LANGFUSE_AI_BASE_URL: z.string().optional(),+ LANGFUSE_AI_USE_RESPONSES_API: z.enum(["true", "false"]).optional(),+ LANGFUSE_AI_EXTRA_HEADERS: z+ .string()+ .optional()+ .refine(+ (value) => {+ if (value == null || value.trim() === "") {+ return true;+ }++ try {+ z.record(z.string(), z.string()).parse(JSON.parse(value));+ return true;+ } catch {+ return false;+ }+ },+ {+ message:+ "LANGFUSE_AI_EXTRA_HEADERS must be a JSON object of string header names and values",+ },+ ),LANGFUSE_AI_AWS_BEDROCK_REGION: z.string().optional(),LANGFUSE_IN_APP_AGENT_ENABLED: z.enum(["true", "false"]).optional(),diff --git a/packages/shared/src/in-app-agent/server/modelProvider.test.ts b/packages/shared/src/in-app-agent/server/modelProvider.test.tsindex 481b6c19b9..bafe263860 100644--- a/packages/shared/src/in-app-agent/server/modelProvider.test.ts+++ b/packages/shared/src/in-app-agent/server/modelProvider.test.ts@@ -13,6 +13,8 @@ const original = {LANGFUSE_AI_SMALL_MODEL: env.LANGFUSE_AI_SMALL_MODEL,LANGFUSE_AI_API_KEY: env.LANGFUSE_AI_API_KEY,LANGFUSE_AI_BASE_URL: env.LANGFUSE_AI_BASE_URL,+ LANGFUSE_AI_USE_RESPONSES_API: env.LANGFUSE_AI_USE_RESPONSES_API,+ LANGFUSE_AI_EXTRA_HEADERS: env.LANGFUSE_AI_EXTRA_HEADERS,LANGFUSE_AI_AWS_BEDROCK_REGION: env.LANGFUSE_AI_AWS_BEDROCK_REGION,LANGFUSE_IN_APP_AGENT_ENABLED: env.LANGFUSE_IN_APP_AGENT_ENABLED,NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,@@ -30,6 +32,7 @@ describe("getInAppAgentModelConfig", () => {LANGFUSE_AI_SMALL_MODEL: "eu.anthropic.claude-haiku-4-5-20251001-v1:0",LANGFUSE_AI_API_KEY: undefined,LANGFUSE_AI_BASE_URL: undefined,+ LANGFUSE_AI_EXTRA_HEADERS: undefined,LANGFUSE_AI_AWS_BEDROCK_REGION: "eu-west-1",NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,});@@ -49,6 +52,7 @@ describe("getInAppAgentModelConfig", () => {LANGFUSE_AI_SMALL_MODEL: "claude-haiku-4-5",LANGFUSE_AI_API_KEY: "sk-ant-test",LANGFUSE_AI_BASE_URL: "https://api.anthropic.com",+ LANGFUSE_AI_EXTRA_HEADERS: '{"X-LLM-Exec-Token":"proxy-token"}',NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,});@@ -58,6 +62,59 @@ describe("getInAppAgentModelConfig", () => {titleModelId: "claude-haiku-4-5",apiKey: "sk-ant-test",baseURL: "https://api.anthropic.com/v1",+ extraHeaders: { "X-LLM-Exec-Token": "proxy-token" },+ });+ });++ it("resolves OpenAI-compatible Chat Completions from LANGFUSE_AI_PROVIDER", () => {+ Object.assign(env, {+ LANGFUSE_AI_PROVIDER: "openai",+ LANGFUSE_AI_MODEL: "gpt-5.6-sol",+ LANGFUSE_AI_SMALL_MODEL: "gpt-5.6-luna",+ LANGFUSE_AI_API_KEY: "sk-test",+ LANGFUSE_AI_BASE_URL: "https://llm-exec.internal/v1",+ LANGFUSE_AI_EXTRA_HEADERS: '{"X-LLM-Exec-Token":"proxy-token"}',+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,+ });++ expect(getInAppAgentModelConfig()).toEqual({+ provider: "openai",+ modelId: "gpt-5.6-sol",+ titleModelId: "gpt-5.6-luna",+ apiKey: "sk-test",+ baseURL: "https://llm-exec.internal/v1",+ extraHeaders: { "X-LLM-Exec-Token": "proxy-token" },+ });+ });++ it("treats incomplete OpenAI env as unconfigured", () => {+ Object.assign(env, {+ LANGFUSE_AI_PROVIDER: "openai",+ LANGFUSE_AI_MODEL: "gpt-5.6-sol",+ LANGFUSE_AI_API_KEY: undefined,+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,+ });++ expect(getInAppAgentModelConfig()).toBeUndefined();+ });++ it("treats whitespace-only extra headers as unset", () => {+ Object.assign(env, {+ LANGFUSE_AI_PROVIDER: "openai",+ LANGFUSE_AI_MODEL: "gpt-5.6-sol",+ LANGFUSE_AI_SMALL_MODEL: "gpt-5.6-luna",+ LANGFUSE_AI_API_KEY: "sk-test",+ LANGFUSE_AI_BASE_URL: "https://llm-exec.internal/v1",+ LANGFUSE_AI_EXTRA_HEADERS: " ",+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,+ });++ expect(getInAppAgentModelConfig()).toEqual({+ provider: "openai",+ modelId: "gpt-5.6-sol",+ titleModelId: "gpt-5.6-luna",+ apiKey: "sk-test",+ baseURL: "https://llm-exec.internal/v1",});});@@ -79,6 +136,7 @@ describe("getInAppAgentModelConfig", () => {LANGFUSE_AI_SMALL_MODEL: "claude-haiku-4-5",LANGFUSE_AI_API_KEY: "sk-ant-test",LANGFUSE_AI_BASE_URL: "https://api.anthropic.com",+ LANGFUSE_AI_EXTRA_HEADERS: undefined,NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",});@@ -91,7 +149,7 @@ describe("getInAppAgentModelConfig", () => {});});- it("warns that Anthropic key and base URL are ignored on Bedrock", () => {+ it("warns that API key, base URL, and extra headers are ignored on Bedrock", () => {const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);Object.assign(env, {@@ -100,6 +158,7 @@ describe("getInAppAgentModelConfig", () => {LANGFUSE_AI_SMALL_MODEL: undefined,LANGFUSE_AI_API_KEY: "sk-ant-test",LANGFUSE_AI_BASE_URL: "https://api.anthropic.com",+ LANGFUSE_AI_EXTRA_HEADERS: '{"X-LLM-Exec-Token":"proxy-token"}',LANGFUSE_AI_AWS_BEDROCK_REGION: "eu-west-1",NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",});@@ -112,7 +171,7 @@ describe("getInAppAgentModelConfig", () => {});expect(warn).toHaveBeenCalledTimes(1);expect(warn.mock.calls[0]?.[0]).toEqual(- "Ignoring LANGFUSE_AI_API_KEY and LANGFUSE_AI_BASE_URL because the Langfuse AI provider is bedrock. Bedrock uses the instance AWS credential chain, not an Anthropic API key or base URL.",+ "Ignoring LANGFUSE_AI_API_KEY and LANGFUSE_AI_BASE_URL and LANGFUSE_AI_EXTRA_HEADERS because the Langfuse AI provider is bedrock. Bedrock uses the instance AWS credential chain, not an API key, base URL, extra headers, or the OpenAI Responses API toggle.",);warn.mockRestore();@@ -125,6 +184,7 @@ describe("getInAppAgentModelConfig", () => {LANGFUSE_AI_SMALL_MODEL: undefined,LANGFUSE_AI_API_KEY: undefined,LANGFUSE_AI_BASE_URL: undefined,+ LANGFUSE_AI_EXTRA_HEADERS: undefined,LANGFUSE_AI_AWS_BEDROCK_REGION: "us-east-1.attacker.test",NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",});@@ -139,6 +199,7 @@ describe("getInAppAgentModelConfig", () => {LANGFUSE_AI_SMALL_MODEL: undefined,LANGFUSE_AI_API_KEY: undefined,LANGFUSE_AI_BASE_URL: undefined,+ LANGFUSE_AI_EXTRA_HEADERS: undefined,LANGFUSE_AI_AWS_BEDROCK_REGION: undefined,NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",});diff --git a/packages/shared/src/in-app-agent/server/modelProvider.ts b/packages/shared/src/in-app-agent/server/modelProvider.tsindex fa98e5f010..a8f25d348b 100644--- a/packages/shared/src/in-app-agent/server/modelProvider.ts+++ b/packages/shared/src/in-app-agent/server/modelProvider.ts@@ -1,3 +1,5 @@+import { z } from "zod";+import { env } from "../../env";import { logger } from "../../server/logger";import {@@ -5,8 +7,17 @@ import {getLangfuseAIBedrockRegion,} from "../../server/llm/ai-sdk/providers/bedrock";import { toAnthropicBaseURL } from "../../server/llm/ai-sdk/providers/anthropic";+import { processOpenAIBaseURL } from "../../server/llm/utils";++export type LangfuseAIProvider = "bedrock" | "anthropic" | "openai";-export type LangfuseAIProvider = "bedrock" | "anthropic";+type LangfuseAIHttpProviderConfig = {+ modelId: string;+ titleModelId: string;+ apiKey: string;+ baseURL?: string;+ extraHeaders?: Record<string, string>;+};export type InAppAgentModelConfig =| {@@ -15,49 +26,42 @@ export type InAppAgentModelConfig =titleModelId: string;region?: string;}- | {+ | ({provider: "anthropic";- modelId: string;- titleModelId: string;- apiKey: string;- baseURL?: string;- };+ } & LangfuseAIHttpProviderConfig)+ | ({+ provider: "openai";+ } & LangfuseAIHttpProviderConfig);export const LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE =- "Langfuse AI model is not configured. Set LANGFUSE_AI_MODEL and LANGFUSE_AI_SMALL_MODEL, plus LANGFUSE_AI_API_KEY when LANGFUSE_AI_PROVIDER=anthropic.";+ "Langfuse AI model is not configured. Set LANGFUSE_AI_MODEL and LANGFUSE_AI_SMALL_MODEL, plus LANGFUSE_AI_API_KEY when LANGFUSE_AI_PROVIDER is anthropic or openai.";++const extraHeadersRecordSchema = z.record(z.string(), z.string());/*** Resolves the instance-wide Langfuse-operated AI model (Assistant + Ask AI).** Callers should not branch on vendor beyond this discriminated config.- * `LANGFUSE_AI_PROVIDER` selects `bedrock` or `anthropic`. Unset provider- * defaults to Bedrock, including when `NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` is- * set. Cloud does not override an explicit Anthropic provider.+ * `LANGFUSE_AI_PROVIDER` selects `bedrock`, `anthropic`, or `openai`. Unset+ * provider defaults to Bedrock, including when+ * `NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` is set. Cloud does not override an+ * explicit Anthropic or OpenAI provider.** Region is optional for Bedrock: Cloud web historically omits it and lets* the AWS SDK use the task region.** LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION- * apply to both providers.+ * apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /+ * LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.+ * LANGFUSE_AI_USE_RESPONSES_API applies to openai only.*/export function getInAppAgentModelConfig(params?: {modelId?: string | null;}): InAppAgentModelConfig | undefined {- if (resolveLangfuseAIProvider() === "anthropic") {- const modelId = params?.modelId ?? env.LANGFUSE_AI_MODEL;- const apiKey = env.LANGFUSE_AI_API_KEY;-- if (!modelId || !apiKey) {- return undefined;- }+ const provider = resolveLangfuseAIProvider();- return {- provider: "anthropic",- modelId,- titleModelId: env.LANGFUSE_AI_SMALL_MODEL ?? modelId,- apiKey,- baseURL: toAnthropicBaseURL(env.LANGFUSE_AI_BASE_URL),- };+ if (provider === "anthropic" || provider === "openai") {+ return resolveHttpProviderConfig({ provider, modelId: params?.modelId });}const modelId = params?.modelId ?? env.LANGFUSE_AI_MODEL;@@ -73,7 +77,7 @@ export function getInAppAgentModelConfig(params?: {return undefined;}- warnIfBedrockIgnoresAnthropicEnv();+ warnIfBedrockIgnoresNonBedrockEnv();return {provider: "bedrock",@@ -83,11 +87,60 @@ export function getInAppAgentModelConfig(params?: {};}+function resolveHttpProviderConfig(params: {+ provider: "anthropic" | "openai";+ modelId?: string | null;+}): InAppAgentModelConfig | undefined {+ const modelId = params.modelId ?? env.LANGFUSE_AI_MODEL;+ const apiKey = env.LANGFUSE_AI_API_KEY;++ if (!modelId || !apiKey) {+ return undefined;+ }++ const extraHeaders = parseLangfuseAIExtraHeaders();+ const titleModelId = env.LANGFUSE_AI_SMALL_MODEL ?? modelId;++ if (params.provider === "anthropic") {+ return {+ provider: "anthropic",+ modelId,+ titleModelId,+ apiKey,+ baseURL: toAnthropicBaseURL(env.LANGFUSE_AI_BASE_URL),+ ...(extraHeaders ? { extraHeaders } : {}),+ };+ }++ return {+ provider: "openai",+ modelId,+ titleModelId,+ apiKey,+ baseURL:+ processOpenAIBaseURL({+ url: env.LANGFUSE_AI_BASE_URL,+ modelName: modelId,+ }) ?? undefined,+ ...(extraHeaders ? { extraHeaders } : {}),+ };+}++function parseLangfuseAIExtraHeaders(): Record<string, string> | undefined {+ const raw = env.LANGFUSE_AI_EXTRA_HEADERS;+ if (!raw || raw.trim() === "") {+ return undefined;+ }++ const headers = extraHeadersRecordSchema.parse(JSON.parse(raw));+ return Object.keys(headers).length > 0 ? headers : undefined;+}+function resolveLangfuseAIProvider(): LangfuseAIProvider {return env.LANGFUSE_AI_PROVIDER ?? "bedrock";}-function warnIfBedrockIgnoresAnthropicEnv() {+function warnIfBedrockIgnoresNonBedrockEnv() {const ignored: string[] = [];if (env.LANGFUSE_AI_API_KEY) {ignored.push("LANGFUSE_AI_API_KEY");@@ -95,12 +148,18 @@ function warnIfBedrockIgnoresAnthropicEnv() {if (env.LANGFUSE_AI_BASE_URL) {ignored.push("LANGFUSE_AI_BASE_URL");}+ if (env.LANGFUSE_AI_EXTRA_HEADERS) {+ ignored.push("LANGFUSE_AI_EXTRA_HEADERS");+ }+ if (env.LANGFUSE_AI_USE_RESPONSES_API) {+ ignored.push("LANGFUSE_AI_USE_RESPONSES_API");+ }if (ignored.length === 0) {return;}logger.warn(- `Ignoring ${ignored.join(" and ")} because the Langfuse AI provider is bedrock. Bedrock uses the instance AWS credential chain, not an Anthropic API key or base URL.`,+ `Ignoring ${ignored.join(" and ")} because the Langfuse AI provider is bedrock. Bedrock uses the instance AWS credential chain, not an API key, base URL, extra headers, or the OpenAI Responses API toggle.`,);}diff --git a/packages/shared/src/in-app-agent/server/openaiCompatibility.ts b/packages/shared/src/in-app-agent/server/openaiCompatibility.tsnew file mode 100644index 0000000000..07bd884d1c--- /dev/null+++ b/packages/shared/src/in-app-agent/server/openaiCompatibility.ts@@ -0,0 +1,38 @@+import { isOpenAICompatibleEndpoint } from "../../server/llm/ai-sdk/providers/openai";++export { isOpenAICompatibleEndpoint };++const RESPONSES_CALL = {+ apiMode: "responses" as const,+ providerOptions: { openai: { reasoningSummary: "auto" as const } },+};++const CHAT_COMPLETIONS_CALL = {+ apiMode: "chat-completions" as const,+ // LiteLLM / OpenAI-compatible Chat Completions knob. Gateways such as+ // LiteLLM and OpenRouter translate `reasoning_effort` into the upstream+ // model's thinking config. The compatible SDK maps response+ // `reasoning_content` to thinking deltas.+ providerOptions: { openai: { reasoningEffort: "medium" as const } },+};++/**+ * Instance-AI equivalent of a project LLM connection's `useResponsesApi`+ * toggle. Unset follows first-party OpenAI (`api.openai.com` / blank URL);+ * any other host stays on Chat Completions until the operator opts in.+ */+export function resolveLangfuseAIOpenAICall(params: {+ baseURL?: string | null;+ useResponsesApi?: boolean | "true" | "false";+}) {+ if (params.useResponsesApi === true || params.useResponsesApi === "true") {+ return RESPONSES_CALL;+ }+ if (params.useResponsesApi === false || params.useResponsesApi === "false") {+ return CHAT_COMPLETIONS_CALL;+ }++ return isOpenAICompatibleEndpoint(params.baseURL)+ ? CHAT_COMPLETIONS_CALL+ : RESPONSES_CALL;+}diff --git a/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts b/packages/shared/src/server/llm/ai-sdk/requestShape.test.tsindex 109ceaee4c..f950dc5ef2 100644--- a/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts+++ b/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts@@ -1,6 +1,7 @@import { afterEach, describe, expect, it, vi } from "vitest";import { encrypt } from "../../../encryption";+import { env } from "../../../env";import {type ChatMessage,ChatMessageRole,@@ -9,6 +10,7 @@ import {type ModelParams,type TraceSinkParams,} from "../types";+import { generateLangfuseAIText } from "../langfuseAiCompletion";import {createLLMOutput,generateLLMText,@@ -261,6 +263,43 @@ describe("AI SDK request shapes", () => {expect(request.body.model).toBe("gpt-4o");});+ it("Langfuse AI first-party OpenAI credentials hit /v1/responses", async () => {+ const original = {+ LANGFUSE_AI_PROVIDER: env.LANGFUSE_AI_PROVIDER,+ LANGFUSE_AI_MODEL: env.LANGFUSE_AI_MODEL,+ LANGFUSE_AI_SMALL_MODEL: env.LANGFUSE_AI_SMALL_MODEL,+ LANGFUSE_AI_API_KEY: env.LANGFUSE_AI_API_KEY,+ LANGFUSE_AI_BASE_URL: env.LANGFUSE_AI_BASE_URL,+ LANGFUSE_AI_USE_RESPONSES_API: env.LANGFUSE_AI_USE_RESPONSES_API,+ };+ Object.assign(env, {+ LANGFUSE_AI_PROVIDER: "openai",+ LANGFUSE_AI_MODEL: "gpt-5.6-sol",+ LANGFUSE_AI_SMALL_MODEL: "gpt-5.6-luna",+ LANGFUSE_AI_API_KEY: "sk-test",+ LANGFUSE_AI_BASE_URL: undefined,+ LANGFUSE_AI_USE_RESPONSES_API: undefined,+ });++ try {+ const { calls, fetch } = createCaptureFetch(OPENAI_RESPONSES_RESPONSE);+ vi.stubGlobal("fetch", fetch);++ const text = await generateLangfuseAIText({+ messages,+ timeout: 10_000,+ });++ expect(text).toBe("ok");+ expect(calls).toHaveLength(1);+ expect(calls[0]?.url).toBe("https://api.openai.com/v1/responses");+ expect(calls[0]?.headers.get("authorization")).toBe("Bearer sk-test");+ expect(calls[0]?.body.model).toBe("gpt-5.6-sol");+ } finally {+ Object.assign(env, original);+ }+ });+it("OpenAI-compatible chat completions: preserves custom provider options on the wire", async () => {const { result, request } = await runCompletion({modelParams: {diff --git a/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts b/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.tsindex 293849a9ca..6a03baaa76 100644--- a/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts+++ b/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts@@ -40,12 +40,13 @@ export function resolveAiSdkModelConfig(params: {if (credentialSource === "langfuse" &&model.adapter !== LLMAdapter.Bedrock &&- model.adapter !== LLMAdapter.Anthropic+ model.adapter !== LLMAdapter.Anthropic &&+ model.adapter !== LLMAdapter.OpenAI) {throw new LLMValidationError({code: "invalid-connection",message:- "Langfuse credentials are only supported for Amazon Bedrock and Anthropic",+ "Langfuse credentials are only supported for Amazon Bedrock, Anthropic, and OpenAI",});}diff --git a/packages/shared/src/server/llm/langfuseAiCompletion.ts b/packages/shared/src/server/llm/langfuseAiCompletion.tsindex 46cbcb7e3b..0007d357f2 100644--- a/packages/shared/src/server/llm/langfuseAiCompletion.ts+++ b/packages/shared/src/server/llm/langfuseAiCompletion.ts@@ -1,7 +1,11 @@import { BEDROCK_USE_DEFAULT_CREDENTIALS } from "../../interfaces/customLLMProviderConfigSchemas";import { encrypt } from "../../encryption";import { env } from "../../env";-import { getInAppAgentModelConfig } from "../../in-app-agent/server/modelProvider";+import {+ getInAppAgentModelConfig,+ type InAppAgentModelConfig,+} from "../../in-app-agent/server/modelProvider";+import { resolveLangfuseAIOpenAICall } from "../../in-app-agent/server/openaiCompatibility";import { type ChatMessage, LLMAdapter, type TraceSinkParams } from "./types";import { generateLLMText, mapLegacyLLMCompletionParams } from "./llmText";import { randomBytes } from "crypto";@@ -52,41 +56,16 @@ export async function generateLangfuseAIText(params: {throw new Error("Langfuse AI completion model is not configured.");}- const maxTokens =+ const maxTokens: { max_tokens?: number } =params.maxTokens !== undefined ? { max_tokens: params.maxTokens } : {};const result = await generateLLMText({...mapLegacyLLMCompletionParams(- modelConfig.provider === "anthropic"- ? {- messages: params.messages,- modelParams: {- provider: "anthropic",- adapter: LLMAdapter.Anthropic,- model: modelConfig.modelId,- ...maxTokens,- },- connection: {- secretKey: encrypt(modelConfig.apiKey),- baseURL: modelConfig.baseURL,- },- credentialSource: "langfuse",- }- : {- messages: params.messages,- modelParams: {- provider: "bedrock",- adapter: LLMAdapter.Bedrock,- model: modelConfig.modelId,- // Intentionally omit temperature/top_p: newer Bedrock models reject these- // inference params, while AI-feature generation works at model defaults.- ...maxTokens,- },- connection: {- secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),- },- credentialSource: "langfuse",- },+ toLangfuseAICompletionParams({+ modelConfig,+ messages: params.messages,+ maxTokens,+ }),),trace: params.traceSinkParams,timeout: params.timeout,@@ -94,3 +73,83 @@ export async function generateLangfuseAIText(params: {return result.text;}++function toLangfuseAICompletionParams(params: {+ modelConfig: InAppAgentModelConfig;+ messages: ChatMessage[];+ maxTokens: { max_tokens?: number };+}) {+ const { modelConfig, messages, maxTokens } = params;++ switch (modelConfig.provider) {+ case "anthropic":+ return {+ messages,+ modelParams: {+ provider: "anthropic",+ adapter: LLMAdapter.Anthropic,+ model: modelConfig.modelId,+ ...maxTokens,+ },+ connection: {+ secretKey: encrypt(modelConfig.apiKey),+ baseURL: modelConfig.baseURL,+ ...encryptedExtraHeaders(modelConfig.extraHeaders),+ },+ credentialSource: "langfuse" as const,+ };+ case "openai": {+ const { apiMode } = resolveLangfuseAIOpenAICall({+ baseURL: modelConfig.baseURL,+ useResponsesApi: env.LANGFUSE_AI_USE_RESPONSES_API,+ });++ return {+ messages,+ modelParams: {+ provider: "openai",+ adapter: LLMAdapter.OpenAI,+ model: modelConfig.modelId,+ ...maxTokens,+ },+ connection: {+ secretKey: encrypt(modelConfig.apiKey),+ baseURL: modelConfig.baseURL,+ config: { useResponsesApi: apiMode === "responses" },+ ...encryptedExtraHeaders(modelConfig.extraHeaders),+ },+ credentialSource: "langfuse" as const,+ };+ }+ case "bedrock":+ return {+ messages,+ modelParams: {+ provider: "bedrock",+ adapter: LLMAdapter.Bedrock,+ model: modelConfig.modelId,+ // Intentionally omit temperature/top_p: newer Bedrock models reject these+ // inference params, while AI-feature generation works at model defaults.+ ...maxTokens,+ },+ connection: {+ secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),+ },+ credentialSource: "langfuse" as const,+ };+ default: {+ const _exhaustive: never = modelConfig;+ throw new Error(+ `Unsupported Langfuse AI provider: ${(_exhaustive as InAppAgentModelConfig).provider}`,+ );+ }+ }+}++function encryptedExtraHeaders(+ extraHeaders: Record<string, string> | undefined,+): { extraHeaders: string } | Record<string, never> {+ return extraHeaders+ ? { extraHeaders: encrypt(JSON.stringify(extraHeaders)) }+ : {};+}diff --git a/packages/shared/src/server/llm/llmText.test.ts b/packages/shared/src/server/llm/llmText.test.tsindex c552fb7a7b..60af5baba0 100644--- a/packages/shared/src/server/llm/llmText.test.ts+++ b/packages/shared/src/server/llm/llmText.test.ts@@ -568,15 +568,37 @@ describe("legacy compatibility boundary", () => {await expect(generateLLMText({- ...openAIOptions(),+ model: { adapter: LLMAdapter.GoogleAIStudio, id: "gemini-2.5-flash" },+ connection: encryptedConnection,+ messages: [...messages],credentialSource: "langfuse",}),).rejects.toMatchObject({name: "LLMValidationError",message:- "Langfuse credentials are only supported for Amazon Bedrock and Anthropic",+ "Langfuse credentials are only supported for Amazon Bedrock, Anthropic, and OpenAI",statusCode: 400,code: "invalid-connection",});});++ it("allows Langfuse credentials for OpenAI", async () => {+ useModel(+ new MockLanguageModelV4({+ doGenerate: {+ content: [{ type: "text", text: "ok" }],+ finishReason,+ usage,+ warnings: [],+ },+ }),+ );++ const result = await generateLLMText({+ ...openAIOptions(),+ credentialSource: "langfuse",+ });++ expect(result.text).toBe("ok");+ });});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 45f9330397..1c6f51e672 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1090,6 +1090,12 @@ importers:ai-sdk-anthropic-v4:specifier: npm:@ai-sdk/[email protected]version: '@ai-sdk/[email protected]([email protected])'+ ai-sdk-openai-compatible-v4:+ specifier: npm:@ai-sdk/[email protected]+ version: '@ai-sdk/[email protected]([email protected])'+ ai-sdk-openai-v4:+ specifier: npm:@ai-sdk/[email protected]+ version: '@ai-sdk/[email protected]([email protected])'bullmq:specifier: ^5.76.3version: 5.76.3([email protected])@@ -1331,6 +1337,12 @@ packages:peerDependencies:zod: 4.3.6+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-0Y6E5bvR+3eugt5rq2iy39EAAFi4nijXf1kdvcfoMm4erEGMe/6egIpvdvv5ePBU2YsnsxFuLRmMJ76Kv/babQ==}+ engines: {node: '>=18'}+ peerDependencies:+ zod: 4.3.6+'@ai-sdk/[email protected]':resolution: {integrity: sha512-Yh+PsXaf9NbN7oA3oKwOuyjTiHMPD75phf3SqGbDNNKQ3Yj3oTntp/WhO3nCHIPA6gr5/1lyridQokmOpPf9oQ==}engines: {node: '>=22'}@@ -11611,6 +11623,12 @@ snapshots:'@ai-sdk/provider-utils': 5.0.3([email protected])zod: 4.3.6+ '@ai-sdk/[email protected]([email protected])':+ dependencies:+ '@ai-sdk/provider': 3.0.15+ '@ai-sdk/provider-utils': 4.0.46([email protected])+ zod: 4.3.6+'@ai-sdk/[email protected]([email protected])':dependencies:'@ai-sdk/provider': 4.0.3diff --git a/web/src/env.mjs b/web/src/env.mjsindex 4c178926d6..f85164cf4a 100644--- a/web/src/env.mjs+++ b/web/src/env.mjs@@ -479,13 +479,36 @@ export const env = createEnv({SLACK_STATE_SECRET: z.string().optional(),// LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION- // apply to both providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL are- // Anthropic-only.- LANGFUSE_AI_PROVIDER: z.enum(["bedrock", "anthropic"]).optional(),+ // apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /+ // LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.+ // LANGFUSE_AI_USE_RESPONSES_API applies to openai only.+ LANGFUSE_AI_PROVIDER: z.enum(["bedrock", "anthropic", "openai"]).optional(),LANGFUSE_AI_MODEL: z.string().optional(),LANGFUSE_AI_SMALL_MODEL: z.string().optional(),LANGFUSE_AI_API_KEY: z.string().optional(),LANGFUSE_AI_BASE_URL: z.string().optional(),+ LANGFUSE_AI_USE_RESPONSES_API: z.enum(["true", "false"]).optional(),+ LANGFUSE_AI_EXTRA_HEADERS: z+ .string()+ .optional()+ .refine(+ (value) => {+ if (value == null || value.trim() === "") {+ return true;+ }++ try {+ z.record(z.string(), z.string()).parse(JSON.parse(value));+ return true;+ } catch {+ return false;+ }+ },+ {+ message:+ "LANGFUSE_AI_EXTRA_HEADERS must be a JSON object of string header names and values",+ },+ ),LANGFUSE_AI_AWS_BEDROCK_REGION: z.string().optional(),LANGFUSE_IN_APP_AGENT_ENABLED: z.enum(["true", "false"]).optional(),@@ -1041,6 +1064,8 @@ export const env = createEnv({LANGFUSE_AI_SMALL_MODEL: process.env.LANGFUSE_AI_SMALL_MODEL,LANGFUSE_AI_API_KEY: process.env.LANGFUSE_AI_API_KEY,LANGFUSE_AI_BASE_URL: process.env.LANGFUSE_AI_BASE_URL,+ LANGFUSE_AI_USE_RESPONSES_API: process.env.LANGFUSE_AI_USE_RESPONSES_API,+ LANGFUSE_AI_EXTRA_HEADERS: process.env.LANGFUSE_AI_EXTRA_HEADERS,LANGFUSE_AI_AWS_BEDROCK_REGION: process.env.LANGFUSE_AI_AWS_BEDROCK_REGION,LANGFUSE_IN_APP_AGENT_ENABLED: process.env.LANGFUSE_IN_APP_AGENT_ENABLED,diff --git a/worker/package.json b/worker/package.jsonindex 95b369ecc0..cc4b4d4005 100644--- a/worker/package.json+++ b/worker/package.json@@ -55,6 +55,8 @@"@prisma/instrumentation": "^6.19.3","ai-sdk-amazon-bedrock-v4": "npm:@ai-sdk/[email protected]","ai-sdk-anthropic-v4": "npm:@ai-sdk/[email protected]",+ "ai-sdk-openai-compatible-v4": "npm:@ai-sdk/[email protected]",+ "ai-sdk-openai-v4": "npm:@ai-sdk/[email protected]","bullmq": "^5.76.3","cors": "^2.8.5","csv-parse": "^5.6.0",diff --git a/worker/src/features/in-app-agent/runtime/model.ts b/worker/src/features/in-app-agent/runtime/model.tsindex 21b3f97223..b8a36b3670 100644--- a/worker/src/features/in-app-agent/runtime/model.ts+++ b/worker/src/features/in-app-agent/runtime/model.ts@@ -1,7 +1,14 @@import { createAmazonBedrock } from "ai-sdk-amazon-bedrock-v4";import { createAnthropic } from "ai-sdk-anthropic-v4";+import { createOpenAICompatible } from "ai-sdk-openai-compatible-v4";+import { createOpenAI } from "ai-sdk-openai-v4";import type { InAppAgentModelConfig } from "@langfuse/shared/in-app-agent/server/modelProvider";+import {+ isOpenAICompatibleEndpoint,+ resolveLangfuseAIOpenAICall,+} from "@langfuse/shared/in-app-agent/server/openaiCompatibility";+import { env } from "@langfuse/shared/src/env";import { createDefaultBedrockProviderAuth } from "@langfuse/shared/src/server";const BEDROCK_CLAUDE_MODEL_ID_PART = "anthropic.claude";@@ -15,25 +22,81 @@ export function createInAppAgentLanguageModel(params: {config: InAppAgentModelConfig;awsProfile?: string;}): InAppAgentLanguageModel {- if (params.config.provider === "anthropic") {- const anthropic = createAnthropic({- apiKey: params.config.apiKey,- baseURL: params.config.baseURL,+ switch (params.config.provider) {+ case "anthropic": {+ const anthropic = createAnthropic({+ apiKey: params.config.apiKey,+ baseURL: params.config.baseURL,+ ...(params.config.extraHeaders+ ? { headers: params.config.extraHeaders }+ : {}),+ });++ return anthropic(+ params.config.modelId as Parameters<typeof anthropic>[0],+ ) as InAppAgentLanguageModel;+ }+ case "openai": {+ return createOpenAIInAppAgentLanguageModel(+ params.config,+ ) as InAppAgentLanguageModel;+ }+ case "bedrock": {+ const bedrock = createAmazonBedrock({+ ...(params.config.region ? { region: params.config.region } : {}),+ ...createDefaultBedrockProviderAuth(+ params.awsProfile ? { profile: params.awsProfile } : undefined,+ ),+ });++ return bedrock(params.config.modelId as Parameters<typeof bedrock>[0]);+ }+ default: {+ const _exhaustive: never = params.config;+ throw new Error(+ `Unsupported Langfuse AI provider: ${(_exhaustive as InAppAgentModelConfig).provider}`,+ );+ }+ }+}++function createOpenAIInAppAgentLanguageModel(config: {+ modelId: string;+ apiKey: string;+ baseURL?: string;+ extraHeaders?: Record<string, string>;+}) {+ const { apiMode } = resolveLangfuseAIOpenAICallFromEnv(config.baseURL);+ const extraHeaders = config.extraHeaders+ ? { headers: config.extraHeaders }+ : {};++ // Match Ask AI / playground: custom OpenAI-compatible URLs use the+ // compatible Chat Completions client so OpenAI chat heuristics do not+ // run on a Claude (or other) model id behind a gateway.+ if (+ apiMode === "chat-completions" &&+ isOpenAICompatibleEndpoint(config.baseURL)+ ) {+ const openaiCompatible = createOpenAICompatible({+ name: "openai",+ apiKey: config.apiKey,+ baseURL: config.baseURL,+ ...extraHeaders,});- return anthropic(- params.config.modelId as Parameters<typeof anthropic>[0],- ) as InAppAgentLanguageModel;+ return openaiCompatible.languageModel(config.modelId);}- const bedrock = createAmazonBedrock({- ...(params.config.region ? { region: params.config.region } : {}),- ...createDefaultBedrockProviderAuth(- params.awsProfile ? { profile: params.awsProfile } : undefined,- ),+ const openai = createOpenAI({+ apiKey: config.apiKey,+ ...(config.baseURL ? { baseURL: config.baseURL } : {}),+ ...extraHeaders,});- return bedrock(params.config.modelId as Parameters<typeof bedrock>[0]);+ return apiMode === "responses"+ ? openai.responses(config.modelId)+ : openai.chat(config.modelId);}// Adaptive thinking is the default for every Claude model so new generations@@ -63,20 +126,35 @@ export function getBedrockReasoningProviderOptions(modelId: string) {export function getInAppAgentReasoningProviderOptions(config: InAppAgentModelConfig,) {- if (config.provider === "anthropic") {- if (!config.modelId.includes(ANTHROPIC_CLAUDE_MODEL_ID_PART)) {- return undefined;- }+ switch (config.provider) {+ case "anthropic": {+ if (!config.modelId.includes(ANTHROPIC_CLAUDE_MODEL_ID_PART)) {+ return undefined;+ }- return {- anthropic: {- thinking: {- type: "adaptive" as const,- display: "summarized" as const,+ return {+ anthropic: {+ thinking: {+ type: "adaptive" as const,+ display: "summarized" as const,+ },},- },- };+ };+ }+ case "openai":+ return resolveLangfuseAIOpenAICallFromEnv(config.baseURL).providerOptions;+ case "bedrock":+ return getBedrockReasoningProviderOptions(config.modelId);+ default: {+ const _exhaustive: never = config;+ return undefined;+ }}+}- return getBedrockReasoningProviderOptions(config.modelId);+function resolveLangfuseAIOpenAICallFromEnv(baseURL: string | undefined) {+ return resolveLangfuseAIOpenAICall({+ baseURL,+ useResponsesApi: env.LANGFUSE_AI_USE_RESPONSES_API,+ });}diff --git a/worker/src/features/in-app-agent/runtime/promptCache.ts b/worker/src/features/in-app-agent/runtime/promptCache.tsindex 157a1532c8..1b25f6419a 100644--- a/worker/src/features/in-app-agent/runtime/promptCache.ts+++ b/worker/src/features/in-app-agent/runtime/promptCache.ts@@ -1,6 +1,6 @@import { isRecord } from "@langfuse/shared/in-app-agent/server/toolErrors";-export type PromptCacheProvider = "bedrock" | "anthropic";+type PromptCacheProvider = "bedrock" | "anthropic" | "openai-compatible";const BEDROCK_CLAUDE_MODEL_ID_PART = "anthropic.claude";const ANTHROPIC_CLAUDE_MODEL_ID_PART = "claude";@@ -9,7 +9,11 @@ const BEDROCK_PROMPT_CACHE_POINT = { type: "default" as const };const ANTHROPIC_CACHE_CONTROL = { type: "ephemeral" as const };/**- * Prompt cache for Claude on Bedrock Converse and native Anthropic Messages.+ * Prompt cache for Claude on Bedrock Converse, native Anthropic Messages,+ * and Anthropic slugs behind an OpenAI-compatible gateway (OpenRouter,+ * LiteLLM). Native Messages get `anthropic.cacheControl`. Compatible+ * Chat Completions get `openaiCompatible.cache_control` because that SDK+ * only forwards the `openaiCompatible` namespace.** A checkpoint writes the prefix `tools → that message`. The next call* cache-reads only if that prefix is byte-identical. Hits last 5 minutes@@ -111,6 +115,11 @@ function resolvePromptCacheProvider() {return "anthropic";}+ // OpenRouter / LiteLLM Chat Completions: provider is `openai.chat` but+ // the upstream model id is still `anthropic/claude-…`.+ if (modelId.includes("anthropic")) {+ return "openai-compatible";+ }return undefined;}@@ -177,9 +186,13 @@ function getMessageRole(message: unknown) {}function withPromptCache(message: unknown, cacheProvider: PromptCacheProvider) {- return cacheProvider === "bedrock"- ? withBedrockCachePoint(message)- : withAnthropicCacheControl(message);+ if (cacheProvider === "bedrock") {+ return withBedrockCachePoint(message);+ }+ if (cacheProvider === "openai-compatible") {+ return withOpenAICompatibleCacheControl(message);+ }+ return withAnthropicCacheControl(message);}function withBedrockCachePoint(message: unknown) {@@ -237,3 +250,73 @@ function withAnthropicCacheControl(message: unknown) {},};}++function withOpenAICompatibleCacheControl(message: unknown) {+ if (!isRecord(message)) {+ return message;+ }++ const providerOptions = isRecord(message.providerOptions)+ ? message.providerOptions+ : {};+ const openaiCompatible = isRecord(providerOptions.openaiCompatible)+ ? providerOptions.openaiCompatible+ : {};++ if (isRecord(openaiCompatible.cache_control)) {+ return message;+ }++ return withOpenAICompatibleCacheControlOnLastPart({+ ...message,+ providerOptions: {+ ...providerOptions,+ openaiCompatible: {+ ...openaiCompatible,+ cache_control: ANTHROPIC_CACHE_CONTROL,+ },+ },+ });+}++function withOpenAICompatibleCacheControlOnLastPart(+ message: Record<string, unknown>,+) {+ if (!Array.isArray(message.content) || message.content.length === 0) {+ return message;+ }++ const lastIndex = message.content.length - 1;+ const lastPart = message.content[lastIndex];+ if (!isRecord(lastPart)) {+ return message;+ }++ const partOptions = isRecord(lastPart.providerOptions)+ ? lastPart.providerOptions+ : {};+ const openaiCompatible = isRecord(partOptions.openaiCompatible)+ ? partOptions.openaiCompatible+ : {};++ if (isRecord(openaiCompatible.cache_control)) {+ return message;+ }++ const nextContent = message.content.slice();+ nextContent[lastIndex] = {+ ...lastPart,+ providerOptions: {+ ...partOptions,+ openaiCompatible: {+ ...openaiCompatible,+ cache_control: ANTHROPIC_CACHE_CONTROL,+ },+ },+ };++ return {+ ...message,+ content: nextContent,+ };+}diff --git a/worker/src/features/in-app-agent/runtime/reasoning.test.ts b/worker/src/features/in-app-agent/runtime/reasoning.test.tsindex b7ca77b5be..c4c63b533e 100644--- a/worker/src/features/in-app-agent/runtime/reasoning.test.ts+++ b/worker/src/features/in-app-agent/runtime/reasoning.test.ts@@ -1,10 +1,24 @@-import { afterEach, describe, expect, it, vi } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { env } from "@langfuse/shared/src/env";import {createInAppAgentLanguageModel,getBedrockReasoningProviderOptions,getInAppAgentReasoningProviderOptions,} from "./model";+import { applyPromptCacheToCall } from "./promptCache";++const originalUseResponsesApi = env.LANGFUSE_AI_USE_RESPONSES_API;++beforeEach(() => {+ Object.assign(env, { LANGFUSE_AI_USE_RESPONSES_API: undefined });+});++afterEach(() => {+ Object.assign(env, {+ LANGFUSE_AI_USE_RESPONSES_API: originalUseResponsesApi,+ });+});describe("getBedrockReasoningProviderOptions", () => {it("sends adaptive thinking with medium effort and summarized display to Claude models", () => {@@ -39,31 +53,130 @@ describe("getBedrockReasoningProviderOptions", () => {});});-describe("getInAppAgentReasoningProviderOptions", () => {- it("sends Anthropic adaptive thinking for Claude model ids", () => {- expect(- getInAppAgentReasoningProviderOptions({- provider: "anthropic",- modelId: "claude-opus-4-8",- titleModelId: "claude-haiku-4-5",- apiKey: "sk-ant-test",+describe("OpenAI Chat Completions request shape", () => {+ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it("sends Bearer plus additive extra headers to Chat Completions", async () => {+ const config = {+ provider: "openai" as const,+ modelId: "gpt-5.6-sol",+ titleModelId: "gpt-5.6-luna",+ apiKey: "sk-test",+ baseURL: "https://llm-exec.internal/v1",+ extraHeaders: { "X-LLM-Exec-Token": "proxy-token" },+ };+ const { calls, fetch } = createCaptureFetch(OPENAI_CHAT_RESPONSE);+ vi.stubGlobal("fetch", fetch);++ const model = createInAppAgentLanguageModel({ config });+ await model.doGenerate({+ prompt: userPrompt(),+ providerOptions: getInAppAgentReasoningProviderOptions(config),+ });++ expect(calls).toHaveLength(1);+ expect(calls[0]?.url).toBe("https://llm-exec.internal/v1/chat/completions");+ expect(calls[0]?.headers.get("authorization")).toBe("Bearer sk-test");+ expect(calls[0]?.headers.get("x-llm-exec-token")).toBe("proxy-token");+ expect(calls[0]?.body).not.toHaveProperty("thinking");+ expect(calls[0]?.body.reasoning_effort).toBe("medium");+ expect(calls[0]?.body).not.toHaveProperty("reasoning");+ });++ it("posts Anthropic cache_control on Chat Completions for anthropic model slugs", async () => {+ const config = {+ provider: "openai" as const,+ modelId: "anthropic/claude-opus-4.6",+ titleModelId: "anthropic/claude-haiku-4.5",+ apiKey: "sk-or-test",+ baseURL: "https://openrouter.ai/api/v1",+ };+ const { calls, fetch } = createCaptureFetch(OPENAI_CHAT_RESPONSE);+ vi.stubGlobal("fetch", fetch);++ const model = createInAppAgentLanguageModel({ config });+ const prompt = [+ { role: "system" as const, content: "You are the Langfuse assistant." },+ {+ role: "user" as const,+ content: [{ type: "text" as const, text: "hello" }],+ },+ ];+ await model.doGenerate(+ applyPromptCacheToCall({+ provider: String(model.provider),+ modelId: model.modelId,+ options: { prompt },}),- ).toEqual({- anthropic: {- thinking: { type: "adaptive", display: "summarized" },+ );++ expect(calls[0]?.url).toBe("https://openrouter.ai/api/v1/chat/completions");+ expect(calls[0]?.body.messages).toEqual([+ {+ role: "system",+ content: "You are the Langfuse assistant.",+ cache_control: { type: "ephemeral" },+ },+ {+ role: "user",+ content: "hello",+ cache_control: { type: "ephemeral" },},+ ]);+ });+});++describe("OpenAI Responses request shape", () => {+ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it("posts first-party OpenAI calls to the Responses API", async () => {+ const config = {+ provider: "openai" as const,+ modelId: "gpt-5.6-sol",+ titleModelId: "gpt-5.6-luna",+ apiKey: "sk-test",+ };+ const { calls, fetch } = createCaptureFetch(OPENAI_RESPONSES_RESPONSE);+ vi.stubGlobal("fetch", fetch);++ const model = createInAppAgentLanguageModel({ config });+ await model.doGenerate({+ prompt: userPrompt(),+ providerOptions: getInAppAgentReasoningProviderOptions(config),});++ expect(calls).toHaveLength(1);+ expect(calls[0]?.url).toBe("https://api.openai.com/v1/responses");+ expect(calls[0]?.headers.get("authorization")).toBe("Bearer sk-test");+ expect(calls[0]?.body.reasoning).toEqual({ summary: "auto" });+ expect(calls[0]?.body).not.toHaveProperty("reasoning_effort");});- it("sends no thinking config for non-Claude Anthropic model ids", () => {- expect(- getInAppAgentReasoningProviderOptions({- provider: "anthropic",- modelId: "some-other-model",- titleModelId: "some-other-model",- apiKey: "sk-ant-test",- }),- ).toBeUndefined();+ it("posts to /v1/responses when a custom URL opts into Responses", async () => {+ Object.assign(env, { LANGFUSE_AI_USE_RESPONSES_API: "true" });++ const config = {+ provider: "openai" as const,+ modelId: "gpt-5.6-sol",+ titleModelId: "gpt-5.6-luna",+ apiKey: "sk-test",+ baseURL: "https://llm-exec.internal/v1",+ };+ const { calls, fetch } = createCaptureFetch(OPENAI_RESPONSES_RESPONSE);+ vi.stubGlobal("fetch", fetch);++ const model = createInAppAgentLanguageModel({ config });+ await model.doGenerate({+ prompt: userPrompt(),+ providerOptions: getInAppAgentReasoningProviderOptions(config),+ });++ expect(calls).toHaveLength(1);+ expect(calls[0]?.url).toBe("https://llm-exec.internal/v1/responses");});});@@ -98,12 +211,7 @@ describe("Anthropic Messages request shape", () => {const model = createInAppAgentLanguageModel({ config });await model.doGenerate({- prompt: [- {- role: "user",- content: [{ type: "text", text: "hi" }],- },- ],+ prompt: userPrompt(),providerOptions: getInAppAgentReasoningProviderOptions(config),});@@ -116,13 +224,66 @@ describe("Anthropic Messages request shape", () => {});});+const OPENAI_CHAT_RESPONSE = {+ id: "chatcmpl-1",+ object: "chat.completion",+ choices: [+ {+ index: 0,+ message: { role: "assistant", content: "ok" },+ finish_reason: "stop",+ },+ ],+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },+};++const OPENAI_RESPONSES_RESPONSE = {+ id: "resp_1",+ object: "response",+ created_at: 1,+ status: "completed",+ error: null,+ incomplete_details: null,+ model: "gpt-5.6-sol",+ output: [+ {+ type: "message",+ id: "msg_1",+ status: "completed",+ role: "assistant",+ content: [{ type: "output_text", text: "ok", annotations: [] }],+ },+ ],+ usage: {+ input_tokens: 1,+ input_tokens_details: { cached_tokens: 0 },+ output_tokens: 1,+ output_tokens_details: { reasoning_tokens: 0 },+ total_tokens: 2,+ },+};++function userPrompt() {+ return [+ {+ role: "user" as const,+ content: [{ type: "text" as const, text: "hi" }],+ },+ ];+}+function createCaptureFetch(response: unknown) {- const calls: Array<{ url: string; body: Record<string, unknown> }> = [];+ const calls: Array<{+ url: string;+ body: Record<string, unknown>;+ headers: Headers;+ }> = [];const fetchImpl: typeof fetch = async (input, init) => {const request = new Request(input, init);calls.push({url: request.url,body: JSON.parse(await request.text()) as Record<string, unknown>,+ headers: request.headers,});return new Response(JSON.stringify(response), {status: 200,