Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(in-app-agent): support OpenAI-compatible APIs for instance AI features (#16581)

Lysander Kiesel67eec1cbd534
diff --git a/.env.dev.example b/.env.dev.example
index 66d7ffca59..f8241ccd62 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -196,7 +196,8 @@ LANGFUSE_AI_FEATURES_PROJECT_ID=7a88fb47-b4e2-43b8-a06c-a5ce950dc53a
# Langfuse AI. Unset LANGFUSE_AI_PROVIDER defaults to Bedrock, including when
# NEXT_PUBLIC_LANGFUSE_CLOUD_REGION is set.
# LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION
-# apply to both providers.
+# apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /
+# LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.
# Langfuse AI Bedrock credentials
AWS_ACCESS_KEY_ID="A123456789"
@@ -213,6 +214,19 @@ LANGFUSE_AI_SMALL_MODEL="eu.anthropic.claude-haiku-4-5-20251001-v1:0"
# LANGFUSE_AI_API_KEY="sk-ant-..."
# LANGFUSE_AI_BASE_URL="https://api.anthropic.com"
# Do not set ANTHROPIC_* — Claude Code injects those without /v1 and can 404.
+# OpenAI or OpenAI-compatible (LiteLLM, internal proxy).
+# Omit LANGFUSE_AI_BASE_URL for api.openai.com (Responses API is the default).
+# Custom URLs default to Chat Completions. Set LANGFUSE_AI_USE_RESPONSES_API
+# to "true" when the gateway implements /v1/responses (tools + thinking on
+# gpt-5.6). Authorization: Bearer still comes from LANGFUSE_AI_API_KEY;
+# extra headers are additive JSON, not a replacement for Bearer.
+# LANGFUSE_AI_PROVIDER="openai"
+# LANGFUSE_AI_MODEL="gpt-5.6-sol"
+# LANGFUSE_AI_SMALL_MODEL="gpt-5.6-luna"
+# LANGFUSE_AI_API_KEY="sk-..."
+# LANGFUSE_AI_BASE_URL="https://llm-exec.internal/v1"
+# LANGFUSE_AI_USE_RESPONSES_API="true"
+# LANGFUSE_AI_EXTRA_HEADERS='{"X-LLM-Exec-Token":"..."}'
LANGFUSE_IN_APP_AGENT_AWS_PROFILE="playground"
# Self-hosted Assistant instance switch. Unset or "false" hides Assistant.
# Cloud is on unless this is "false".
diff --git a/.env.prod.example b/.env.prod.example
index 16187443b8..1debe46c72 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -295,7 +295,7 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# One instance-wide model for the Assistant and Ask AI, on web and worker.
# Unset LANGFUSE_AI_PROVIDER defaults to bedrock. Bedrock authenticates through
-# the AWS credential chain; anthropic requires LANGFUSE_AI_API_KEY.
+# the AWS credential chain; anthropic and openai require LANGFUSE_AI_API_KEY.
# LANGFUSE_AI_PROVIDER=bedrock
# LANGFUSE_AI_MODEL=eu.anthropic.claude-opus-5
# LANGFUSE_AI_SMALL_MODEL=eu.anthropic.claude-haiku-4-5-20251001-v1:0
@@ -303,6 +303,17 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# LANGFUSE_AI_API_KEY=
# Origin only, the app appends /v1
# LANGFUSE_AI_BASE_URL=https://api.anthropic.com
+# OpenAI: omit LANGFUSE_AI_BASE_URL for api.openai.com (Responses API is the
+# default). OpenAI-compatible: include /v1 on LANGFUSE_AI_BASE_URL (Chat
+# Completions unless LANGFUSE_AI_USE_RESPONSES_API=true and the gateway
+# implements /v1/responses). Extra headers are additive JSON;
+# Authorization: Bearer is still sent from LANGFUSE_AI_API_KEY.
+# LANGFUSE_AI_PROVIDER=openai
+# LANGFUSE_AI_MODEL=gpt-5.6-sol
+# LANGFUSE_AI_SMALL_MODEL=gpt-5.6-luna
+# LANGFUSE_AI_BASE_URL=https://llm-exec.internal/v1
+# LANGFUSE_AI_USE_RESPONSES_API=true
+# LANGFUSE_AI_EXTRA_HEADERS={"X-LLM-Exec-Token":"..."}
# Workers follow LANGFUSE_IN_APP_AGENT_ENABLED for the run queue and integrity
# runner. Split-role workers (ingestion-only) can opt out without turning the
diff --git a/docker-compose.build.yml b/docker-compose.build.yml
index 4365d68222..d06dae168c 100644
--- a/docker-compose.build.yml
+++ b/docker-compose.build.yml
@@ -63,6 +63,8 @@ services:
LANGFUSE_AI_SMALL_MODEL: ${LANGFUSE_AI_SMALL_MODEL:-}
LANGFUSE_AI_API_KEY: ${LANGFUSE_AI_API_KEY:-}
LANGFUSE_AI_BASE_URL: ${LANGFUSE_AI_BASE_URL:-}
+ LANGFUSE_AI_USE_RESPONSES_API: ${LANGFUSE_AI_USE_RESPONSES_API:-}
+ LANGFUSE_AI_EXTRA_HEADERS: ${LANGFUSE_AI_EXTRA_HEADERS:-}
LANGFUSE_AI_AWS_BEDROCK_REGION: ${LANGFUSE_AI_AWS_BEDROCK_REGION:-}
LANGFUSE_IN_APP_AGENT_ENABLED: ${LANGFUSE_IN_APP_AGENT_ENABLED:-}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
diff --git a/docker-compose.yml b/docker-compose.yml
index b7f5f80b60..821a5668f4 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -82,6 +82,8 @@ services:
LANGFUSE_AI_SMALL_MODEL: ${LANGFUSE_AI_SMALL_MODEL:-}
LANGFUSE_AI_API_KEY: ${LANGFUSE_AI_API_KEY:-}
LANGFUSE_AI_BASE_URL: ${LANGFUSE_AI_BASE_URL:-}
+ LANGFUSE_AI_USE_RESPONSES_API: ${LANGFUSE_AI_USE_RESPONSES_API:-}
+ LANGFUSE_AI_EXTRA_HEADERS: ${LANGFUSE_AI_EXTRA_HEADERS:-}
LANGFUSE_AI_AWS_BEDROCK_REGION: ${LANGFUSE_AI_AWS_BEDROCK_REGION:-}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 5bd752e4e2..3829811350 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -110,6 +110,10 @@
"./in-app-agent/server/modelProvider": {
"import": "./dist/src/in-app-agent/server/modelProvider.js",
"require": "./dist/src/in-app-agent/server/modelProvider.js"
+ },
+ "./in-app-agent/server/openaiCompatibility": {
+ "import": "./dist/src/in-app-agent/server/openaiCompatibility.js",
+ "require": "./dist/src/in-app-agent/server/openaiCompatibility.js"
}
},
"scripts": {
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index f9fb833b2b..956ee1336d 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -523,13 +523,36 @@ const EnvSchema = z.object({
.default(DEFAULT_LLM_COMPLETION_TIMEOUT_MS), // 2 minutes
// LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION
- // apply to both providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL are
- // Anthropic-only.
- LANGFUSE_AI_PROVIDER: z.enum(["bedrock", "anthropic"]).optional(),
+ // apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /
+ // LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.
+ // LANGFUSE_AI_USE_RESPONSES_API applies to openai only.
+ LANGFUSE_AI_PROVIDER: z.enum(["bedrock", "anthropic", "openai"]).optional(),
LANGFUSE_AI_MODEL: z.string().optional(),
LANGFUSE_AI_SMALL_MODEL: z.string().optional(),
LANGFUSE_AI_API_KEY: z.string().optional(),
LANGFUSE_AI_BASE_URL: z.string().optional(),
+ LANGFUSE_AI_USE_RESPONSES_API: z.enum(["true", "false"]).optional(),
+ LANGFUSE_AI_EXTRA_HEADERS: z
+ .string()
+ .optional()
+ .refine(
+ (value) => {
+ if (value == null || value.trim() === "") {
+ return true;
+ }
+
+ try {
+ z.record(z.string(), z.string()).parse(JSON.parse(value));
+ return true;
+ } catch {
+ return false;
+ }
+ },
+ {
+ message:
+ "LANGFUSE_AI_EXTRA_HEADERS must be a JSON object of string header names and values",
+ },
+ ),
LANGFUSE_AI_AWS_BEDROCK_REGION: z.string().optional(),
LANGFUSE_IN_APP_AGENT_ENABLED: z.enum(["true", "false"]).optional(),
diff --git a/packages/shared/src/in-app-agent/server/modelProvider.test.ts b/packages/shared/src/in-app-agent/server/modelProvider.test.ts
index 481b6c19b9..bafe263860 100644
--- a/packages/shared/src/in-app-agent/server/modelProvider.test.ts
+++ b/packages/shared/src/in-app-agent/server/modelProvider.test.ts
@@ -13,6 +13,8 @@ const original = {
LANGFUSE_AI_SMALL_MODEL: env.LANGFUSE_AI_SMALL_MODEL,
LANGFUSE_AI_API_KEY: env.LANGFUSE_AI_API_KEY,
LANGFUSE_AI_BASE_URL: env.LANGFUSE_AI_BASE_URL,
+ LANGFUSE_AI_USE_RESPONSES_API: env.LANGFUSE_AI_USE_RESPONSES_API,
+ LANGFUSE_AI_EXTRA_HEADERS: env.LANGFUSE_AI_EXTRA_HEADERS,
LANGFUSE_AI_AWS_BEDROCK_REGION: env.LANGFUSE_AI_AWS_BEDROCK_REGION,
LANGFUSE_IN_APP_AGENT_ENABLED: env.LANGFUSE_IN_APP_AGENT_ENABLED,
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
@@ -30,6 +32,7 @@ describe("getInAppAgentModelConfig", () => {
LANGFUSE_AI_SMALL_MODEL: "eu.anthropic.claude-haiku-4-5-20251001-v1:0",
LANGFUSE_AI_API_KEY: undefined,
LANGFUSE_AI_BASE_URL: undefined,
+ LANGFUSE_AI_EXTRA_HEADERS: undefined,
LANGFUSE_AI_AWS_BEDROCK_REGION: "eu-west-1",
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
});
@@ -49,6 +52,7 @@ describe("getInAppAgentModelConfig", () => {
LANGFUSE_AI_SMALL_MODEL: "claude-haiku-4-5",
LANGFUSE_AI_API_KEY: "sk-ant-test",
LANGFUSE_AI_BASE_URL: "https://api.anthropic.com",
+ LANGFUSE_AI_EXTRA_HEADERS: '{"X-LLM-Exec-Token":"proxy-token"}',
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
});
@@ -58,6 +62,59 @@ describe("getInAppAgentModelConfig", () => {
titleModelId: "claude-haiku-4-5",
apiKey: "sk-ant-test",
baseURL: "https://api.anthropic.com/v1",
+ extraHeaders: { "X-LLM-Exec-Token": "proxy-token" },
+ });
+ });
+
+ it("resolves OpenAI-compatible Chat Completions from LANGFUSE_AI_PROVIDER", () => {
+ Object.assign(env, {
+ LANGFUSE_AI_PROVIDER: "openai",
+ LANGFUSE_AI_MODEL: "gpt-5.6-sol",
+ LANGFUSE_AI_SMALL_MODEL: "gpt-5.6-luna",
+ LANGFUSE_AI_API_KEY: "sk-test",
+ LANGFUSE_AI_BASE_URL: "https://llm-exec.internal/v1",
+ LANGFUSE_AI_EXTRA_HEADERS: '{"X-LLM-Exec-Token":"proxy-token"}',
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ });
+
+ expect(getInAppAgentModelConfig()).toEqual({
+ provider: "openai",
+ modelId: "gpt-5.6-sol",
+ titleModelId: "gpt-5.6-luna",
+ apiKey: "sk-test",
+ baseURL: "https://llm-exec.internal/v1",
+ extraHeaders: { "X-LLM-Exec-Token": "proxy-token" },
+ });
+ });
+
+ it("treats incomplete OpenAI env as unconfigured", () => {
+ Object.assign(env, {
+ LANGFUSE_AI_PROVIDER: "openai",
+ LANGFUSE_AI_MODEL: "gpt-5.6-sol",
+ LANGFUSE_AI_API_KEY: undefined,
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ });
+
+ expect(getInAppAgentModelConfig()).toBeUndefined();
+ });
+
+ it("treats whitespace-only extra headers as unset", () => {
+ Object.assign(env, {
+ LANGFUSE_AI_PROVIDER: "openai",
+ LANGFUSE_AI_MODEL: "gpt-5.6-sol",
+ LANGFUSE_AI_SMALL_MODEL: "gpt-5.6-luna",
+ LANGFUSE_AI_API_KEY: "sk-test",
+ LANGFUSE_AI_BASE_URL: "https://llm-exec.internal/v1",
+ LANGFUSE_AI_EXTRA_HEADERS: " ",
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ });
+
+ expect(getInAppAgentModelConfig()).toEqual({
+ provider: "openai",
+ modelId: "gpt-5.6-sol",
+ titleModelId: "gpt-5.6-luna",
+ apiKey: "sk-test",
+ baseURL: "https://llm-exec.internal/v1",
});
});
@@ -79,6 +136,7 @@ describe("getInAppAgentModelConfig", () => {
LANGFUSE_AI_SMALL_MODEL: "claude-haiku-4-5",
LANGFUSE_AI_API_KEY: "sk-ant-test",
LANGFUSE_AI_BASE_URL: "https://api.anthropic.com",
+ LANGFUSE_AI_EXTRA_HEADERS: undefined,
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",
});
@@ -91,7 +149,7 @@ describe("getInAppAgentModelConfig", () => {
});
});
- it("warns that Anthropic key and base URL are ignored on Bedrock", () => {
+ it("warns that API key, base URL, and extra headers are ignored on Bedrock", () => {
const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);
Object.assign(env, {
@@ -100,6 +158,7 @@ describe("getInAppAgentModelConfig", () => {
LANGFUSE_AI_SMALL_MODEL: undefined,
LANGFUSE_AI_API_KEY: "sk-ant-test",
LANGFUSE_AI_BASE_URL: "https://api.anthropic.com",
+ LANGFUSE_AI_EXTRA_HEADERS: '{"X-LLM-Exec-Token":"proxy-token"}',
LANGFUSE_AI_AWS_BEDROCK_REGION: "eu-west-1",
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",
});
@@ -112,7 +171,7 @@ describe("getInAppAgentModelConfig", () => {
});
expect(warn).toHaveBeenCalledTimes(1);
expect(warn.mock.calls[0]?.[0]).toEqual(
- "Ignoring LANGFUSE_AI_API_KEY and LANGFUSE_AI_BASE_URL because the Langfuse AI provider is bedrock. Bedrock uses the instance AWS credential chain, not an Anthropic API key or base URL.",
+ "Ignoring LANGFUSE_AI_API_KEY and LANGFUSE_AI_BASE_URL and LANGFUSE_AI_EXTRA_HEADERS because the Langfuse AI provider is bedrock. Bedrock uses the instance AWS credential chain, not an API key, base URL, extra headers, or the OpenAI Responses API toggle.",
);
warn.mockRestore();
@@ -125,6 +184,7 @@ describe("getInAppAgentModelConfig", () => {
LANGFUSE_AI_SMALL_MODEL: undefined,
LANGFUSE_AI_API_KEY: undefined,
LANGFUSE_AI_BASE_URL: undefined,
+ LANGFUSE_AI_EXTRA_HEADERS: undefined,
LANGFUSE_AI_AWS_BEDROCK_REGION: "us-east-1.attacker.test",
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",
});
@@ -139,6 +199,7 @@ describe("getInAppAgentModelConfig", () => {
LANGFUSE_AI_SMALL_MODEL: undefined,
LANGFUSE_AI_API_KEY: undefined,
LANGFUSE_AI_BASE_URL: undefined,
+ LANGFUSE_AI_EXTRA_HEADERS: undefined,
LANGFUSE_AI_AWS_BEDROCK_REGION: undefined,
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",
});
diff --git a/packages/shared/src/in-app-agent/server/modelProvider.ts b/packages/shared/src/in-app-agent/server/modelProvider.ts
index fa98e5f010..a8f25d348b 100644
--- a/packages/shared/src/in-app-agent/server/modelProvider.ts
+++ b/packages/shared/src/in-app-agent/server/modelProvider.ts
@@ -1,3 +1,5 @@
+import { z } from "zod";
+
import { env } from "../../env";
import { logger } from "../../server/logger";
import {
@@ -5,8 +7,17 @@ import {
getLangfuseAIBedrockRegion,
} from "../../server/llm/ai-sdk/providers/bedrock";
import { toAnthropicBaseURL } from "../../server/llm/ai-sdk/providers/anthropic";
+import { processOpenAIBaseURL } from "../../server/llm/utils";
+
+export type LangfuseAIProvider = "bedrock" | "anthropic" | "openai";
-export type LangfuseAIProvider = "bedrock" | "anthropic";
+type LangfuseAIHttpProviderConfig = {
+ modelId: string;
+ titleModelId: string;
+ apiKey: string;
+ baseURL?: string;
+ extraHeaders?: Record<string, string>;
+};
export type InAppAgentModelConfig =
| {
@@ -15,49 +26,42 @@ export type InAppAgentModelConfig =
titleModelId: string;
region?: string;
}
- | {
+ | ({
provider: "anthropic";
- modelId: string;
- titleModelId: string;
- apiKey: string;
- baseURL?: string;
- };
+ } & LangfuseAIHttpProviderConfig)
+ | ({
+ provider: "openai";
+ } & LangfuseAIHttpProviderConfig);
export const LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE =
- "Langfuse AI model is not configured. Set LANGFUSE_AI_MODEL and LANGFUSE_AI_SMALL_MODEL, plus LANGFUSE_AI_API_KEY when LANGFUSE_AI_PROVIDER=anthropic.";
+ "Langfuse AI model is not configured. Set LANGFUSE_AI_MODEL and LANGFUSE_AI_SMALL_MODEL, plus LANGFUSE_AI_API_KEY when LANGFUSE_AI_PROVIDER is anthropic or openai.";
+
+const extraHeadersRecordSchema = z.record(z.string(), z.string());
/**
* Resolves the instance-wide Langfuse-operated AI model (Assistant + Ask AI).
*
* Callers should not branch on vendor beyond this discriminated config.
- * `LANGFUSE_AI_PROVIDER` selects `bedrock` or `anthropic`. Unset provider
- * defaults to Bedrock, including when `NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` is
- * set. Cloud does not override an explicit Anthropic provider.
+ * `LANGFUSE_AI_PROVIDER` selects `bedrock`, `anthropic`, or `openai`. Unset
+ * provider defaults to Bedrock, including when
+ * `NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` is set. Cloud does not override an
+ * explicit Anthropic or OpenAI provider.
*
* Region is optional for Bedrock: Cloud web historically omits it and lets
* the AWS SDK use the task region.
*
* LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION
- * apply to both providers.
+ * apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /
+ * LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.
+ * LANGFUSE_AI_USE_RESPONSES_API applies to openai only.
*/
export function getInAppAgentModelConfig(params?: {
modelId?: string | null;
}): InAppAgentModelConfig | undefined {
- if (resolveLangfuseAIProvider() === "anthropic") {
- const modelId = params?.modelId ?? env.LANGFUSE_AI_MODEL;
- const apiKey = env.LANGFUSE_AI_API_KEY;
-
- if (!modelId || !apiKey) {
- return undefined;
- }
+ const provider = resolveLangfuseAIProvider();
- return {
- provider: "anthropic",
- modelId,
- titleModelId: env.LANGFUSE_AI_SMALL_MODEL ?? modelId,
- apiKey,
- baseURL: toAnthropicBaseURL(env.LANGFUSE_AI_BASE_URL),
- };
+ if (provider === "anthropic" || provider === "openai") {
+ return resolveHttpProviderConfig({ provider, modelId: params?.modelId });
}
const modelId = params?.modelId ?? env.LANGFUSE_AI_MODEL;
@@ -73,7 +77,7 @@ export function getInAppAgentModelConfig(params?: {
return undefined;
}
- warnIfBedrockIgnoresAnthropicEnv();
+ warnIfBedrockIgnoresNonBedrockEnv();
return {
provider: "bedrock",
@@ -83,11 +87,60 @@ export function getInAppAgentModelConfig(params?: {
};
}
+function resolveHttpProviderConfig(params: {
+ provider: "anthropic" | "openai";
+ modelId?: string | null;
+}): InAppAgentModelConfig | undefined {
+ const modelId = params.modelId ?? env.LANGFUSE_AI_MODEL;
+ const apiKey = env.LANGFUSE_AI_API_KEY;
+
+ if (!modelId || !apiKey) {
+ return undefined;
+ }
+
+ const extraHeaders = parseLangfuseAIExtraHeaders();
+ const titleModelId = env.LANGFUSE_AI_SMALL_MODEL ?? modelId;
+
+ if (params.provider === "anthropic") {
+ return {
+ provider: "anthropic",
+ modelId,
+ titleModelId,
+ apiKey,
+ baseURL: toAnthropicBaseURL(env.LANGFUSE_AI_BASE_URL),
+ ...(extraHeaders ? { extraHeaders } : {}),
+ };
+ }
+
+ return {
+ provider: "openai",
+ modelId,
+ titleModelId,
+ apiKey,
+ baseURL:
+ processOpenAIBaseURL({
+ url: env.LANGFUSE_AI_BASE_URL,
+ modelName: modelId,
+ }) ?? undefined,
+ ...(extraHeaders ? { extraHeaders } : {}),
+ };
+}
+
+function parseLangfuseAIExtraHeaders(): Record<string, string> | undefined {
+ const raw = env.LANGFUSE_AI_EXTRA_HEADERS;
+ if (!raw || raw.trim() === "") {
+ return undefined;
+ }
+
+ const headers = extraHeadersRecordSchema.parse(JSON.parse(raw));
+ return Object.keys(headers).length > 0 ? headers : undefined;
+}
+
function resolveLangfuseAIProvider(): LangfuseAIProvider {
return env.LANGFUSE_AI_PROVIDER ?? "bedrock";
}
-function warnIfBedrockIgnoresAnthropicEnv() {
+function warnIfBedrockIgnoresNonBedrockEnv() {
const ignored: string[] = [];
if (env.LANGFUSE_AI_API_KEY) {
ignored.push("LANGFUSE_AI_API_KEY");
@@ -95,12 +148,18 @@ function warnIfBedrockIgnoresAnthropicEnv() {
if (env.LANGFUSE_AI_BASE_URL) {
ignored.push("LANGFUSE_AI_BASE_URL");
}
+ if (env.LANGFUSE_AI_EXTRA_HEADERS) {
+ ignored.push("LANGFUSE_AI_EXTRA_HEADERS");
+ }
+ if (env.LANGFUSE_AI_USE_RESPONSES_API) {
+ ignored.push("LANGFUSE_AI_USE_RESPONSES_API");
+ }
if (ignored.length === 0) {
return;
}
logger.warn(
- `Ignoring ${ignored.join(" and ")} because the Langfuse AI provider is bedrock. Bedrock uses the instance AWS credential chain, not an Anthropic API key or base URL.`,
+ `Ignoring ${ignored.join(" and ")} because the Langfuse AI provider is bedrock. Bedrock uses the instance AWS credential chain, not an API key, base URL, extra headers, or the OpenAI Responses API toggle.`,
);
}
diff --git a/packages/shared/src/in-app-agent/server/openaiCompatibility.ts b/packages/shared/src/in-app-agent/server/openaiCompatibility.ts
new file mode 100644
index 0000000000..07bd884d1c
--- /dev/null
+++ b/packages/shared/src/in-app-agent/server/openaiCompatibility.ts
@@ -0,0 +1,38 @@
+import { isOpenAICompatibleEndpoint } from "../../server/llm/ai-sdk/providers/openai";
+
+export { isOpenAICompatibleEndpoint };
+
+const RESPONSES_CALL = {
+ apiMode: "responses" as const,
+ providerOptions: { openai: { reasoningSummary: "auto" as const } },
+};
+
+const CHAT_COMPLETIONS_CALL = {
+ apiMode: "chat-completions" as const,
+ // LiteLLM / OpenAI-compatible Chat Completions knob. Gateways such as
+ // LiteLLM and OpenRouter translate `reasoning_effort` into the upstream
+ // model's thinking config. The compatible SDK maps response
+ // `reasoning_content` to thinking deltas.
+ providerOptions: { openai: { reasoningEffort: "medium" as const } },
+};
+
+/**
+ * Instance-AI equivalent of a project LLM connection's `useResponsesApi`
+ * toggle. Unset follows first-party OpenAI (`api.openai.com` / blank URL);
+ * any other host stays on Chat Completions until the operator opts in.
+ */
+export function resolveLangfuseAIOpenAICall(params: {
+ baseURL?: string | null;
+ useResponsesApi?: boolean | "true" | "false";
+}) {
+ if (params.useResponsesApi === true || params.useResponsesApi === "true") {
+ return RESPONSES_CALL;
+ }
+ if (params.useResponsesApi === false || params.useResponsesApi === "false") {
+ return CHAT_COMPLETIONS_CALL;
+ }
+
+ return isOpenAICompatibleEndpoint(params.baseURL)
+ ? CHAT_COMPLETIONS_CALL
+ : RESPONSES_CALL;
+}
diff --git a/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts b/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts
index 109ceaee4c..f950dc5ef2 100644
--- a/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts
+++ b/packages/shared/src/server/llm/ai-sdk/requestShape.test.ts
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { encrypt } from "../../../encryption";
+import { env } from "../../../env";
import {
type ChatMessage,
ChatMessageRole,
@@ -9,6 +10,7 @@ import {
type ModelParams,
type TraceSinkParams,
} from "../types";
+import { generateLangfuseAIText } from "../langfuseAiCompletion";
import {
createLLMOutput,
generateLLMText,
@@ -261,6 +263,43 @@ describe("AI SDK request shapes", () => {
expect(request.body.model).toBe("gpt-4o");
});
+ it("Langfuse AI first-party OpenAI credentials hit /v1/responses", async () => {
+ const original = {
+ LANGFUSE_AI_PROVIDER: env.LANGFUSE_AI_PROVIDER,
+ LANGFUSE_AI_MODEL: env.LANGFUSE_AI_MODEL,
+ LANGFUSE_AI_SMALL_MODEL: env.LANGFUSE_AI_SMALL_MODEL,
+ LANGFUSE_AI_API_KEY: env.LANGFUSE_AI_API_KEY,
+ LANGFUSE_AI_BASE_URL: env.LANGFUSE_AI_BASE_URL,
+ LANGFUSE_AI_USE_RESPONSES_API: env.LANGFUSE_AI_USE_RESPONSES_API,
+ };
+ Object.assign(env, {
+ LANGFUSE_AI_PROVIDER: "openai",
+ LANGFUSE_AI_MODEL: "gpt-5.6-sol",
+ LANGFUSE_AI_SMALL_MODEL: "gpt-5.6-luna",
+ LANGFUSE_AI_API_KEY: "sk-test",
+ LANGFUSE_AI_BASE_URL: undefined,
+ LANGFUSE_AI_USE_RESPONSES_API: undefined,
+ });
+
+ try {
+ const { calls, fetch } = createCaptureFetch(OPENAI_RESPONSES_RESPONSE);
+ vi.stubGlobal("fetch", fetch);
+
+ const text = await generateLangfuseAIText({
+ messages,
+ timeout: 10_000,
+ });
+
+ expect(text).toBe("ok");
+ expect(calls).toHaveLength(1);
+ expect(calls[0]?.url).toBe("https://api.openai.com/v1/responses");
+ expect(calls[0]?.headers.get("authorization")).toBe("Bearer sk-test");
+ expect(calls[0]?.body.model).toBe("gpt-5.6-sol");
+ } finally {
+ Object.assign(env, original);
+ }
+ });
+
it("OpenAI-compatible chat completions: preserves custom provider options on the wire", async () => {
const { result, request } = await runCompletion({
modelParams: {
diff --git a/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts b/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts
index 293849a9ca..6a03baaa76 100644
--- a/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts
+++ b/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts
@@ -40,12 +40,13 @@ export function resolveAiSdkModelConfig(params: {
if (
credentialSource === "langfuse" &&
model.adapter !== LLMAdapter.Bedrock &&
- model.adapter !== LLMAdapter.Anthropic
+ model.adapter !== LLMAdapter.Anthropic &&
+ model.adapter !== LLMAdapter.OpenAI
) {
throw new LLMValidationError({
code: "invalid-connection",
message:
- "Langfuse credentials are only supported for Amazon Bedrock and Anthropic",
+ "Langfuse credentials are only supported for Amazon Bedrock, Anthropic, and OpenAI",
});
}
diff --git a/packages/shared/src/server/llm/langfuseAiCompletion.ts b/packages/shared/src/server/llm/langfuseAiCompletion.ts
index 46cbcb7e3b..0007d357f2 100644
--- a/packages/shared/src/server/llm/langfuseAiCompletion.ts
+++ b/packages/shared/src/server/llm/langfuseAiCompletion.ts
@@ -1,7 +1,11 @@
import { BEDROCK_USE_DEFAULT_CREDENTIALS } from "../../interfaces/customLLMProviderConfigSchemas";
import { encrypt } from "../../encryption";
import { env } from "../../env";
-import { getInAppAgentModelConfig } from "../../in-app-agent/server/modelProvider";
+import {
+ getInAppAgentModelConfig,
+ type InAppAgentModelConfig,
+} from "../../in-app-agent/server/modelProvider";
+import { resolveLangfuseAIOpenAICall } from "../../in-app-agent/server/openaiCompatibility";
import { type ChatMessage, LLMAdapter, type TraceSinkParams } from "./types";
import { generateLLMText, mapLegacyLLMCompletionParams } from "./llmText";
import { randomBytes } from "crypto";
@@ -52,41 +56,16 @@ export async function generateLangfuseAIText(params: {
throw new Error("Langfuse AI completion model is not configured.");
}
- const maxTokens =
+ const maxTokens: { max_tokens?: number } =
params.maxTokens !== undefined ? { max_tokens: params.maxTokens } : {};
const result = await generateLLMText({
...mapLegacyLLMCompletionParams(
- modelConfig.provider === "anthropic"
- ? {
- messages: params.messages,
- modelParams: {
- provider: "anthropic",
- adapter: LLMAdapter.Anthropic,
- model: modelConfig.modelId,
- ...maxTokens,
- },
- connection: {
- secretKey: encrypt(modelConfig.apiKey),
- baseURL: modelConfig.baseURL,
- },
- credentialSource: "langfuse",
- }
- : {
- messages: params.messages,
- modelParams: {
- provider: "bedrock",
- adapter: LLMAdapter.Bedrock,
- model: modelConfig.modelId,
- // Intentionally omit temperature/top_p: newer Bedrock models reject these
- // inference params, while AI-feature generation works at model defaults.
- ...maxTokens,
- },
- connection: {
- secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
- },
- credentialSource: "langfuse",
- },
+ toLangfuseAICompletionParams({
+ modelConfig,
+ messages: params.messages,
+ maxTokens,
+ }),
),
trace: params.traceSinkParams,
timeout: params.timeout,
@@ -94,3 +73,83 @@ export async function generateLangfuseAIText(params: {
return result.text;
}
+
+function toLangfuseAICompletionParams(params: {
+ modelConfig: InAppAgentModelConfig;
+ messages: ChatMessage[];
+ maxTokens: { max_tokens?: number };
+}) {
+ const { modelConfig, messages, maxTokens } = params;
+
+ switch (modelConfig.provider) {
+ case "anthropic":
+ return {
+ messages,
+ modelParams: {
+ provider: "anthropic",
+ adapter: LLMAdapter.Anthropic,
+ model: modelConfig.modelId,
+ ...maxTokens,
+ },
+ connection: {
+ secretKey: encrypt(modelConfig.apiKey),
+ baseURL: modelConfig.baseURL,
+ ...encryptedExtraHeaders(modelConfig.extraHeaders),
+ },
+ credentialSource: "langfuse" as const,
+ };
+ case "openai": {
+ const { apiMode } = resolveLangfuseAIOpenAICall({
+ baseURL: modelConfig.baseURL,
+ useResponsesApi: env.LANGFUSE_AI_USE_RESPONSES_API,
+ });
+
+ return {
+ messages,
+ modelParams: {
+ provider: "openai",
+ adapter: LLMAdapter.OpenAI,
+ model: modelConfig.modelId,
+ ...maxTokens,
+ },
+ connection: {
+ secretKey: encrypt(modelConfig.apiKey),
+ baseURL: modelConfig.baseURL,
+ config: { useResponsesApi: apiMode === "responses" },
+ ...encryptedExtraHeaders(modelConfig.extraHeaders),
+ },
+ credentialSource: "langfuse" as const,
+ };
+ }
+ case "bedrock":
+ return {
+ messages,
+ modelParams: {
+ provider: "bedrock",
+ adapter: LLMAdapter.Bedrock,
+ model: modelConfig.modelId,
+ // Intentionally omit temperature/top_p: newer Bedrock models reject these
+ // inference params, while AI-feature generation works at model defaults.
+ ...maxTokens,
+ },
+ connection: {
+ secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
+ },
+ credentialSource: "langfuse" as const,
+ };
+ default: {
+ const _exhaustive: never = modelConfig;
+ throw new Error(
+ `Unsupported Langfuse AI provider: ${(_exhaustive as InAppAgentModelConfig).provider}`,
+ );
+ }
+ }
+}
+
+function encryptedExtraHeaders(
+ extraHeaders: Record<string, string> | undefined,
+): { extraHeaders: string } | Record<string, never> {
+ return extraHeaders
+ ? { extraHeaders: encrypt(JSON.stringify(extraHeaders)) }
+ : {};
+}
diff --git a/packages/shared/src/server/llm/llmText.test.ts b/packages/shared/src/server/llm/llmText.test.ts
index c552fb7a7b..60af5baba0 100644
--- a/packages/shared/src/server/llm/llmText.test.ts
+++ b/packages/shared/src/server/llm/llmText.test.ts
@@ -568,15 +568,37 @@ describe("legacy compatibility boundary", () => {
await expect(
generateLLMText({
- ...openAIOptions(),
+ model: { adapter: LLMAdapter.GoogleAIStudio, id: "gemini-2.5-flash" },
+ connection: encryptedConnection,
+ messages: [...messages],
credentialSource: "langfuse",
}),
).rejects.toMatchObject({
name: "LLMValidationError",
message:
- "Langfuse credentials are only supported for Amazon Bedrock and Anthropic",
+ "Langfuse credentials are only supported for Amazon Bedrock, Anthropic, and OpenAI",
statusCode: 400,
code: "invalid-connection",
});
});
+
+ it("allows Langfuse credentials for OpenAI", async () => {
+ useModel(
+ new MockLanguageModelV4({
+ doGenerate: {
+ content: [{ type: "text", text: "ok" }],
+ finishReason,
+ usage,
+ warnings: [],
+ },
+ }),
+ );
+
+ const result = await generateLLMText({
+ ...openAIOptions(),
+ credentialSource: "langfuse",
+ });
+
+ expect(result.text).toBe("ok");
+ });
});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 45f9330397..1c6f51e672 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1090,6 +1090,12 @@ importers:
ai-sdk-anthropic-v4:
specifier: npm:@ai-sdk/[email protected]
+ ai-sdk-openai-compatible-v4:
+ specifier: npm:@ai-sdk/[email protected]
+ ai-sdk-openai-v4:
+ specifier: npm:@ai-sdk/[email protected]
bullmq:
specifier: ^5.76.3
version: 5.76.3([email protected])
@@ -1331,6 +1337,12 @@ packages:
peerDependencies:
zod: 4.3.6
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-0Y6E5bvR+3eugt5rq2iy39EAAFi4nijXf1kdvcfoMm4erEGMe/6egIpvdvv5ePBU2YsnsxFuLRmMJ76Kv/babQ==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ zod: 4.3.6
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-Yh+PsXaf9NbN7oA3oKwOuyjTiHMPD75phf3SqGbDNNKQ3Yj3oTntp/WhO3nCHIPA6gr5/1lyridQokmOpPf9oQ==}
engines: {node: '>=22'}
@@ -11611,6 +11623,12 @@ snapshots:
'@ai-sdk/provider-utils': 5.0.3([email protected])
zod: 4.3.6
+ dependencies:
+ '@ai-sdk/provider': 3.0.15
+ '@ai-sdk/provider-utils': 4.0.46([email protected])
+ zod: 4.3.6
+
dependencies:
'@ai-sdk/provider': 4.0.3
diff --git a/web/src/env.mjs b/web/src/env.mjs
index 4c178926d6..f85164cf4a 100644
--- a/web/src/env.mjs
+++ b/web/src/env.mjs
@@ -479,13 +479,36 @@ export const env = createEnv({
SLACK_STATE_SECRET: z.string().optional(),
// LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION
- // apply to both providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL are
- // Anthropic-only.
- LANGFUSE_AI_PROVIDER: z.enum(["bedrock", "anthropic"]).optional(),
+ // apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /
+ // LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.
+ // LANGFUSE_AI_USE_RESPONSES_API applies to openai only.
+ LANGFUSE_AI_PROVIDER: z.enum(["bedrock", "anthropic", "openai"]).optional(),
LANGFUSE_AI_MODEL: z.string().optional(),
LANGFUSE_AI_SMALL_MODEL: z.string().optional(),
LANGFUSE_AI_API_KEY: z.string().optional(),
LANGFUSE_AI_BASE_URL: z.string().optional(),
+ LANGFUSE_AI_USE_RESPONSES_API: z.enum(["true", "false"]).optional(),
+ LANGFUSE_AI_EXTRA_HEADERS: z
+ .string()
+ .optional()
+ .refine(
+ (value) => {
+ if (value == null || value.trim() === "") {
+ return true;
+ }
+
+ try {
+ z.record(z.string(), z.string()).parse(JSON.parse(value));
+ return true;
+ } catch {
+ return false;
+ }
+ },
+ {
+ message:
+ "LANGFUSE_AI_EXTRA_HEADERS must be a JSON object of string header names and values",
+ },
+ ),
LANGFUSE_AI_AWS_BEDROCK_REGION: z.string().optional(),
LANGFUSE_IN_APP_AGENT_ENABLED: z.enum(["true", "false"]).optional(),
@@ -1041,6 +1064,8 @@ export const env = createEnv({
LANGFUSE_AI_SMALL_MODEL: process.env.LANGFUSE_AI_SMALL_MODEL,
LANGFUSE_AI_API_KEY: process.env.LANGFUSE_AI_API_KEY,
LANGFUSE_AI_BASE_URL: process.env.LANGFUSE_AI_BASE_URL,
+ LANGFUSE_AI_USE_RESPONSES_API: process.env.LANGFUSE_AI_USE_RESPONSES_API,
+ LANGFUSE_AI_EXTRA_HEADERS: process.env.LANGFUSE_AI_EXTRA_HEADERS,
LANGFUSE_AI_AWS_BEDROCK_REGION: process.env.LANGFUSE_AI_AWS_BEDROCK_REGION,
LANGFUSE_IN_APP_AGENT_ENABLED: process.env.LANGFUSE_IN_APP_AGENT_ENABLED,
diff --git a/worker/package.json b/worker/package.json
index 95b369ecc0..cc4b4d4005 100644
--- a/worker/package.json
+++ b/worker/package.json
@@ -55,6 +55,8 @@
"@prisma/instrumentation": "^6.19.3",
"ai-sdk-amazon-bedrock-v4": "npm:@ai-sdk/[email protected]",
"ai-sdk-anthropic-v4": "npm:@ai-sdk/[email protected]",
+ "ai-sdk-openai-compatible-v4": "npm:@ai-sdk/[email protected]",
+ "ai-sdk-openai-v4": "npm:@ai-sdk/[email protected]",
"bullmq": "^5.76.3",
"cors": "^2.8.5",
"csv-parse": "^5.6.0",
diff --git a/worker/src/features/in-app-agent/runtime/model.ts b/worker/src/features/in-app-agent/runtime/model.ts
index 21b3f97223..b8a36b3670 100644
--- a/worker/src/features/in-app-agent/runtime/model.ts
+++ b/worker/src/features/in-app-agent/runtime/model.ts
@@ -1,7 +1,14 @@
import { createAmazonBedrock } from "ai-sdk-amazon-bedrock-v4";
import { createAnthropic } from "ai-sdk-anthropic-v4";
+import { createOpenAICompatible } from "ai-sdk-openai-compatible-v4";
+import { createOpenAI } from "ai-sdk-openai-v4";
import type { InAppAgentModelConfig } from "@langfuse/shared/in-app-agent/server/modelProvider";
+import {
+ isOpenAICompatibleEndpoint,
+ resolveLangfuseAIOpenAICall,
+} from "@langfuse/shared/in-app-agent/server/openaiCompatibility";
+import { env } from "@langfuse/shared/src/env";
import { createDefaultBedrockProviderAuth } from "@langfuse/shared/src/server";
const BEDROCK_CLAUDE_MODEL_ID_PART = "anthropic.claude";
@@ -15,25 +22,81 @@ export function createInAppAgentLanguageModel(params: {
config: InAppAgentModelConfig;
awsProfile?: string;
}): InAppAgentLanguageModel {
- if (params.config.provider === "anthropic") {
- const anthropic = createAnthropic({
- apiKey: params.config.apiKey,
- baseURL: params.config.baseURL,
+ switch (params.config.provider) {
+ case "anthropic": {
+ const anthropic = createAnthropic({
+ apiKey: params.config.apiKey,
+ baseURL: params.config.baseURL,
+ ...(params.config.extraHeaders
+ ? { headers: params.config.extraHeaders }
+ : {}),
+ });
+
+ return anthropic(
+ params.config.modelId as Parameters<typeof anthropic>[0],
+ ) as InAppAgentLanguageModel;
+ }
+ case "openai": {
+ return createOpenAIInAppAgentLanguageModel(
+ params.config,
+ ) as InAppAgentLanguageModel;
+ }
+ case "bedrock": {
+ const bedrock = createAmazonBedrock({
+ ...(params.config.region ? { region: params.config.region } : {}),
+ ...createDefaultBedrockProviderAuth(
+ params.awsProfile ? { profile: params.awsProfile } : undefined,
+ ),
+ });
+
+ return bedrock(params.config.modelId as Parameters<typeof bedrock>[0]);
+ }
+ default: {
+ const _exhaustive: never = params.config;
+ throw new Error(
+ `Unsupported Langfuse AI provider: ${(_exhaustive as InAppAgentModelConfig).provider}`,
+ );
+ }
+ }
+}
+
+function createOpenAIInAppAgentLanguageModel(config: {
+ modelId: string;
+ apiKey: string;
+ baseURL?: string;
+ extraHeaders?: Record<string, string>;
+}) {
+ const { apiMode } = resolveLangfuseAIOpenAICallFromEnv(config.baseURL);
+ const extraHeaders = config.extraHeaders
+ ? { headers: config.extraHeaders }
+ : {};
+
+ // Match Ask AI / playground: custom OpenAI-compatible URLs use the
+ // compatible Chat Completions client so OpenAI chat heuristics do not
+ // run on a Claude (or other) model id behind a gateway.
+ if (
+ apiMode === "chat-completions" &&
+ isOpenAICompatibleEndpoint(config.baseURL)
+ ) {
+ const openaiCompatible = createOpenAICompatible({
+ name: "openai",
+ apiKey: config.apiKey,
+ baseURL: config.baseURL,
+ ...extraHeaders,
});
- return anthropic(
- params.config.modelId as Parameters<typeof anthropic>[0],
- ) as InAppAgentLanguageModel;
+ return openaiCompatible.languageModel(config.modelId);
}
- const bedrock = createAmazonBedrock({
- ...(params.config.region ? { region: params.config.region } : {}),
- ...createDefaultBedrockProviderAuth(
- params.awsProfile ? { profile: params.awsProfile } : undefined,
- ),
+ const openai = createOpenAI({
+ apiKey: config.apiKey,
+ ...(config.baseURL ? { baseURL: config.baseURL } : {}),
+ ...extraHeaders,
});
- return bedrock(params.config.modelId as Parameters<typeof bedrock>[0]);
+ return apiMode === "responses"
+ ? openai.responses(config.modelId)
+ : openai.chat(config.modelId);
}
// Adaptive thinking is the default for every Claude model so new generations
@@ -63,20 +126,35 @@ export function getBedrockReasoningProviderOptions(modelId: string) {
export function getInAppAgentReasoningProviderOptions(
config: InAppAgentModelConfig,
) {
- if (config.provider === "anthropic") {
- if (!config.modelId.includes(ANTHROPIC_CLAUDE_MODEL_ID_PART)) {
- return undefined;
- }
+ switch (config.provider) {
+ case "anthropic": {
+ if (!config.modelId.includes(ANTHROPIC_CLAUDE_MODEL_ID_PART)) {
+ return undefined;
+ }
- return {
- anthropic: {
- thinking: {
- type: "adaptive" as const,
- display: "summarized" as const,
+ return {
+ anthropic: {
+ thinking: {
+ type: "adaptive" as const,
+ display: "summarized" as const,
+ },
},
- },
- };
+ };
+ }
+ case "openai":
+ return resolveLangfuseAIOpenAICallFromEnv(config.baseURL).providerOptions;
+ case "bedrock":
+ return getBedrockReasoningProviderOptions(config.modelId);
+ default: {
+ const _exhaustive: never = config;
+ return undefined;
+ }
}
+}
- return getBedrockReasoningProviderOptions(config.modelId);
+function resolveLangfuseAIOpenAICallFromEnv(baseURL: string | undefined) {
+ return resolveLangfuseAIOpenAICall({
+ baseURL,
+ useResponsesApi: env.LANGFUSE_AI_USE_RESPONSES_API,
+ });
}
diff --git a/worker/src/features/in-app-agent/runtime/promptCache.ts b/worker/src/features/in-app-agent/runtime/promptCache.ts
index 157a1532c8..1b25f6419a 100644
--- a/worker/src/features/in-app-agent/runtime/promptCache.ts
+++ b/worker/src/features/in-app-agent/runtime/promptCache.ts
@@ -1,6 +1,6 @@
import { isRecord } from "@langfuse/shared/in-app-agent/server/toolErrors";
-export type PromptCacheProvider = "bedrock" | "anthropic";
+type PromptCacheProvider = "bedrock" | "anthropic" | "openai-compatible";
const BEDROCK_CLAUDE_MODEL_ID_PART = "anthropic.claude";
const ANTHROPIC_CLAUDE_MODEL_ID_PART = "claude";
@@ -9,7 +9,11 @@ const BEDROCK_PROMPT_CACHE_POINT = { type: "default" as const };
const ANTHROPIC_CACHE_CONTROL = { type: "ephemeral" as const };
/**
- * Prompt cache for Claude on Bedrock Converse and native Anthropic Messages.
+ * Prompt cache for Claude on Bedrock Converse, native Anthropic Messages,
+ * and Anthropic slugs behind an OpenAI-compatible gateway (OpenRouter,
+ * LiteLLM). Native Messages get `anthropic.cacheControl`. Compatible
+ * Chat Completions get `openaiCompatible.cache_control` because that SDK
+ * only forwards the `openaiCompatible` namespace.
*
* A checkpoint writes the prefix `tools → that message`. The next call
* cache-reads only if that prefix is byte-identical. Hits last 5 minutes
@@ -111,6 +115,11 @@ function resolvePromptCacheProvider(
) {
return "anthropic";
}
+ // OpenRouter / LiteLLM Chat Completions: provider is `openai.chat` but
+ // the upstream model id is still `anthropic/claude-…`.
+ if (modelId.includes("anthropic")) {
+ return "openai-compatible";
+ }
return undefined;
}
@@ -177,9 +186,13 @@ function getMessageRole(message: unknown) {
}
function withPromptCache(message: unknown, cacheProvider: PromptCacheProvider) {
- return cacheProvider === "bedrock"
- ? withBedrockCachePoint(message)
- : withAnthropicCacheControl(message);
+ if (cacheProvider === "bedrock") {
+ return withBedrockCachePoint(message);
+ }
+ if (cacheProvider === "openai-compatible") {
+ return withOpenAICompatibleCacheControl(message);
+ }
+ return withAnthropicCacheControl(message);
}
function withBedrockCachePoint(message: unknown) {
@@ -237,3 +250,73 @@ function withAnthropicCacheControl(message: unknown) {
},
};
}
+
+function withOpenAICompatibleCacheControl(message: unknown) {
+ if (!isRecord(message)) {
+ return message;
+ }
+
+ const providerOptions = isRecord(message.providerOptions)
+ ? message.providerOptions
+ : {};
+ const openaiCompatible = isRecord(providerOptions.openaiCompatible)
+ ? providerOptions.openaiCompatible
+ : {};
+
+ if (isRecord(openaiCompatible.cache_control)) {
+ return message;
+ }
+
+ return withOpenAICompatibleCacheControlOnLastPart({
+ ...message,
+ providerOptions: {
+ ...providerOptions,
+ openaiCompatible: {
+ ...openaiCompatible,
+ cache_control: ANTHROPIC_CACHE_CONTROL,
+ },
+ },
+ });
+}
+
+function withOpenAICompatibleCacheControlOnLastPart(
+ message: Record<string, unknown>,
+) {
+ if (!Array.isArray(message.content) || message.content.length === 0) {
+ return message;
+ }
+
+ const lastIndex = message.content.length - 1;
+ const lastPart = message.content[lastIndex];
+ if (!isRecord(lastPart)) {
+ return message;
+ }
+
+ const partOptions = isRecord(lastPart.providerOptions)
+ ? lastPart.providerOptions
+ : {};
+ const openaiCompatible = isRecord(partOptions.openaiCompatible)
+ ? partOptions.openaiCompatible
+ : {};
+
+ if (isRecord(openaiCompatible.cache_control)) {
+ return message;
+ }
+
+ const nextContent = message.content.slice();
+ nextContent[lastIndex] = {
+ ...lastPart,
+ providerOptions: {
+ ...partOptions,
+ openaiCompatible: {
+ ...openaiCompatible,
+ cache_control: ANTHROPIC_CACHE_CONTROL,
+ },
+ },
+ };
+
+ return {
+ ...message,
+ content: nextContent,
+ };
+}
diff --git a/worker/src/features/in-app-agent/runtime/reasoning.test.ts b/worker/src/features/in-app-agent/runtime/reasoning.test.ts
index b7ca77b5be..c4c63b533e 100644
--- a/worker/src/features/in-app-agent/runtime/reasoning.test.ts
+++ b/worker/src/features/in-app-agent/runtime/reasoning.test.ts
@@ -1,10 +1,24 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { env } from "@langfuse/shared/src/env";
import {
createInAppAgentLanguageModel,
getBedrockReasoningProviderOptions,
getInAppAgentReasoningProviderOptions,
} from "./model";
+import { applyPromptCacheToCall } from "./promptCache";
+
+const originalUseResponsesApi = env.LANGFUSE_AI_USE_RESPONSES_API;
+
+beforeEach(() => {
+ Object.assign(env, { LANGFUSE_AI_USE_RESPONSES_API: undefined });
+});
+
+afterEach(() => {
+ Object.assign(env, {
+ LANGFUSE_AI_USE_RESPONSES_API: originalUseResponsesApi,
+ });
+});
describe("getBedrockReasoningProviderOptions", () => {
it("sends adaptive thinking with medium effort and summarized display to Claude models", () => {
@@ -39,31 +53,130 @@ describe("getBedrockReasoningProviderOptions", () => {
});
});
-describe("getInAppAgentReasoningProviderOptions", () => {
- it("sends Anthropic adaptive thinking for Claude model ids", () => {
- expect(
- getInAppAgentReasoningProviderOptions({
- provider: "anthropic",
- modelId: "claude-opus-4-8",
- titleModelId: "claude-haiku-4-5",
- apiKey: "sk-ant-test",
+describe("OpenAI Chat Completions request shape", () => {
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it("sends Bearer plus additive extra headers to Chat Completions", async () => {
+ const config = {
+ provider: "openai" as const,
+ modelId: "gpt-5.6-sol",
+ titleModelId: "gpt-5.6-luna",
+ apiKey: "sk-test",
+ baseURL: "https://llm-exec.internal/v1",
+ extraHeaders: { "X-LLM-Exec-Token": "proxy-token" },
+ };
+ const { calls, fetch } = createCaptureFetch(OPENAI_CHAT_RESPONSE);
+ vi.stubGlobal("fetch", fetch);
+
+ const model = createInAppAgentLanguageModel({ config });
+ await model.doGenerate({
+ prompt: userPrompt(),
+ providerOptions: getInAppAgentReasoningProviderOptions(config),
+ });
+
+ expect(calls).toHaveLength(1);
+ expect(calls[0]?.url).toBe("https://llm-exec.internal/v1/chat/completions");
+ expect(calls[0]?.headers.get("authorization")).toBe("Bearer sk-test");
+ expect(calls[0]?.headers.get("x-llm-exec-token")).toBe("proxy-token");
+ expect(calls[0]?.body).not.toHaveProperty("thinking");
+ expect(calls[0]?.body.reasoning_effort).toBe("medium");
+ expect(calls[0]?.body).not.toHaveProperty("reasoning");
+ });
+
+ it("posts Anthropic cache_control on Chat Completions for anthropic model slugs", async () => {
+ const config = {
+ provider: "openai" as const,
+ modelId: "anthropic/claude-opus-4.6",
+ titleModelId: "anthropic/claude-haiku-4.5",
+ apiKey: "sk-or-test",
+ baseURL: "https://openrouter.ai/api/v1",
+ };
+ const { calls, fetch } = createCaptureFetch(OPENAI_CHAT_RESPONSE);
+ vi.stubGlobal("fetch", fetch);
+
+ const model = createInAppAgentLanguageModel({ config });
+ const prompt = [
+ { role: "system" as const, content: "You are the Langfuse assistant." },
+ {
+ role: "user" as const,
+ content: [{ type: "text" as const, text: "hello" }],
+ },
+ ];
+ await model.doGenerate(
+ applyPromptCacheToCall({
+ provider: String(model.provider),
+ modelId: model.modelId,
+ options: { prompt },
}),
- ).toEqual({
- anthropic: {
- thinking: { type: "adaptive", display: "summarized" },
+ );
+
+ expect(calls[0]?.url).toBe("https://openrouter.ai/api/v1/chat/completions");
+ expect(calls[0]?.body.messages).toEqual([
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ cache_control: { type: "ephemeral" },
+ },
+ {
+ role: "user",
+ content: "hello",
+ cache_control: { type: "ephemeral" },
},
+ ]);
+ });
+});
+
+describe("OpenAI Responses request shape", () => {
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it("posts first-party OpenAI calls to the Responses API", async () => {
+ const config = {
+ provider: "openai" as const,
+ modelId: "gpt-5.6-sol",
+ titleModelId: "gpt-5.6-luna",
+ apiKey: "sk-test",
+ };
+ const { calls, fetch } = createCaptureFetch(OPENAI_RESPONSES_RESPONSE);
+ vi.stubGlobal("fetch", fetch);
+
+ const model = createInAppAgentLanguageModel({ config });
+ await model.doGenerate({
+ prompt: userPrompt(),
+ providerOptions: getInAppAgentReasoningProviderOptions(config),
});
+
+ expect(calls).toHaveLength(1);
+ expect(calls[0]?.url).toBe("https://api.openai.com/v1/responses");
+ expect(calls[0]?.headers.get("authorization")).toBe("Bearer sk-test");
+ expect(calls[0]?.body.reasoning).toEqual({ summary: "auto" });
+ expect(calls[0]?.body).not.toHaveProperty("reasoning_effort");
});
- it("sends no thinking config for non-Claude Anthropic model ids", () => {
- expect(
- getInAppAgentReasoningProviderOptions({
- provider: "anthropic",
- modelId: "some-other-model",
- titleModelId: "some-other-model",
- apiKey: "sk-ant-test",
- }),
- ).toBeUndefined();
+ it("posts to /v1/responses when a custom URL opts into Responses", async () => {
+ Object.assign(env, { LANGFUSE_AI_USE_RESPONSES_API: "true" });
+
+ const config = {
+ provider: "openai" as const,
+ modelId: "gpt-5.6-sol",
+ titleModelId: "gpt-5.6-luna",
+ apiKey: "sk-test",
+ baseURL: "https://llm-exec.internal/v1",
+ };
+ const { calls, fetch } = createCaptureFetch(OPENAI_RESPONSES_RESPONSE);
+ vi.stubGlobal("fetch", fetch);
+
+ const model = createInAppAgentLanguageModel({ config });
+ await model.doGenerate({
+ prompt: userPrompt(),
+ providerOptions: getInAppAgentReasoningProviderOptions(config),
+ });
+
+ expect(calls).toHaveLength(1);
+ expect(calls[0]?.url).toBe("https://llm-exec.internal/v1/responses");
});
});
@@ -98,12 +211,7 @@ describe("Anthropic Messages request shape", () => {
const model = createInAppAgentLanguageModel({ config });
await model.doGenerate({
- prompt: [
- {
- role: "user",
- content: [{ type: "text", text: "hi" }],
- },
- ],
+ prompt: userPrompt(),
providerOptions: getInAppAgentReasoningProviderOptions(config),
});
@@ -116,13 +224,66 @@ describe("Anthropic Messages request shape", () => {
});
});
+const OPENAI_CHAT_RESPONSE = {
+ id: "chatcmpl-1",
+ object: "chat.completion",
+ choices: [
+ {
+ index: 0,
+ message: { role: "assistant", content: "ok" },
+ finish_reason: "stop",
+ },
+ ],
+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+};
+
+const OPENAI_RESPONSES_RESPONSE = {
+ id: "resp_1",
+ object: "response",
+ created_at: 1,
+ status: "completed",
+ error: null,
+ incomplete_details: null,
+ model: "gpt-5.6-sol",
+ output: [
+ {
+ type: "message",
+ id: "msg_1",
+ status: "completed",
+ role: "assistant",
+ content: [{ type: "output_text", text: "ok", annotations: [] }],
+ },
+ ],
+ usage: {
+ input_tokens: 1,
+ input_tokens_details: { cached_tokens: 0 },
+ output_tokens: 1,
+ output_tokens_details: { reasoning_tokens: 0 },
+ total_tokens: 2,
+ },
+};
+
+function userPrompt() {
+ return [
+ {
+ role: "user" as const,
+ content: [{ type: "text" as const, text: "hi" }],
+ },
+ ];
+}
+
function createCaptureFetch(response: unknown) {
- const calls: Array<{ url: string; body: Record<string, unknown> }> = [];
+ const calls: Array<{
+ url: string;
+ body: Record<string, unknown>;
+ headers: Headers;
+ }> = [];
const fetchImpl: typeof fetch = async (input, init) => {
const request = new Request(input, init);
calls.push({
url: request.url,
body: JSON.parse(await request.text()) as Record<string, unknown>,
+ headers: request.headers,
});
return new Response(JSON.stringify(response), {
status: 200,