🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ai)!: require LANGFUSE_AI_PROVIDER instead of defaulting to Bedrock (#16766)
Lysander Kiesele89dfbe0ca23
diff --git a/.env.dev.example b/.env.dev.exampleindex f8241ccd62..252e4bb62d 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -193,8 +193,8 @@ LANGFUSE_AI_FEATURES_PROJECT_ID=7a88fb47-b4e2-43b8-a06c-a5ce950dc53a# LANGFUSE_SSO_DISCOVERY_WHITELISTED_IPS=127.0.0.1,::1# LANGFUSE_SSO_DISCOVERY_WHITELISTED_IP_SEGMENTS=127.0.0.0/8-# Langfuse AI. Unset LANGFUSE_AI_PROVIDER defaults to Bedrock, including when-# NEXT_PUBLIC_LANGFUSE_CLOUD_REGION is set.+# Langfuse AI. Unset LANGFUSE_AI_PROVIDER is unconfigured. Bedrock requires+# LANGFUSE_AI_PROVIDER=bedrock.# LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION# apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /# LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.@@ -203,6 +203,7 @@ LANGFUSE_AI_FEATURES_PROJECT_ID=7a88fb47-b4e2-43b8-a06c-a5ce950dc53aAWS_ACCESS_KEY_ID="A123456789"AWS_SECRET_ACCESS_KEY="SAK123456789"LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY="1234567890abcdef"+LANGFUSE_AI_PROVIDER="bedrock"LANGFUSE_AI_AWS_BEDROCK_REGION="eu-west-1"LANGFUSE_AI_MODEL="eu.anthropic.claude-opus-5"LANGFUSE_AI_SMALL_MODEL="eu.anthropic.claude-haiku-4-5-20251001-v1:0"diff --git a/.env.prod.example b/.env.prod.exampleindex 1debe46c72..5777527a4a 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -294,8 +294,9 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# LANGFUSE_IN_APP_AGENT_ENABLED=true# One instance-wide model for the Assistant and Ask AI, on web and worker.-# Unset LANGFUSE_AI_PROVIDER defaults to bedrock. Bedrock authenticates through-# the AWS credential chain; anthropic and openai require LANGFUSE_AI_API_KEY.+# Unset LANGFUSE_AI_PROVIDER is unconfigured. Bedrock requires+# LANGFUSE_AI_PROVIDER=bedrock. Bedrock authenticates through the AWS+# credential chain; anthropic and openai require LANGFUSE_AI_API_KEY.# LANGFUSE_AI_PROVIDER=bedrock# LANGFUSE_AI_MODEL=eu.anthropic.claude-opus-5# LANGFUSE_AI_SMALL_MODEL=eu.anthropic.claude-haiku-4-5-20251001-v1:0diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex 956ee1336d..564c2b49e1 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -522,6 +522,8 @@ const EnvSchema = z.object({.positive().default(DEFAULT_LLM_COMPLETION_TIMEOUT_MS), // 2 minutes+ // LANGFUSE_AI_PROVIDER is optional at boot. Unset means unconfigured;+ // bedrock requires LANGFUSE_AI_PROVIDER=bedrock.// LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION// apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /// LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.diff --git a/packages/shared/src/in-app-agent/server/modelProvider.test.ts b/packages/shared/src/in-app-agent/server/modelProvider.test.tsindex bafe263860..112ad34f56 100644--- a/packages/shared/src/in-app-agent/server/modelProvider.test.ts+++ b/packages/shared/src/in-app-agent/server/modelProvider.test.ts@@ -192,24 +192,19 @@ describe("getInAppAgentModelConfig", () => {expect(getInAppAgentModelConfig()).toBeUndefined();});- it("leaves the Bedrock region undefined when LANGFUSE_AI_AWS_BEDROCK_REGION is unset", () => {+ it("treats an unset LANGFUSE_AI_PROVIDER as unconfigured even when model vars are set", () => {Object.assign(env, {LANGFUSE_AI_PROVIDER: undefined,LANGFUSE_AI_MODEL: "eu.anthropic.claude-opus-5",- LANGFUSE_AI_SMALL_MODEL: undefined,+ LANGFUSE_AI_SMALL_MODEL: "eu.anthropic.claude-haiku-4-5-20251001-v1:0",LANGFUSE_AI_API_KEY: undefined,LANGFUSE_AI_BASE_URL: undefined,LANGFUSE_AI_EXTRA_HEADERS: undefined,- LANGFUSE_AI_AWS_BEDROCK_REGION: undefined,+ LANGFUSE_AI_AWS_BEDROCK_REGION: "eu-west-1",NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",});- expect(getInAppAgentModelConfig()).toEqual({- provider: "bedrock",- modelId: "eu.anthropic.claude-opus-5",- titleModelId: "eu.anthropic.claude-opus-5",- region: undefined,- });+ expect(getInAppAgentModelConfig()).toBeUndefined();});});diff --git a/packages/shared/src/in-app-agent/server/modelProvider.ts b/packages/shared/src/in-app-agent/server/modelProvider.tsindex a8f25d348b..a21746c490 100644--- a/packages/shared/src/in-app-agent/server/modelProvider.ts+++ b/packages/shared/src/in-app-agent/server/modelProvider.ts@@ -34,7 +34,7 @@ export type InAppAgentModelConfig =} & LangfuseAIHttpProviderConfig);export const LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE =- "Langfuse AI model is not configured. Set LANGFUSE_AI_MODEL and LANGFUSE_AI_SMALL_MODEL, plus LANGFUSE_AI_API_KEY when LANGFUSE_AI_PROVIDER is anthropic or openai.";+ "Langfuse AI is not configured. Set LANGFUSE_AI_PROVIDER and LANGFUSE_AI_MODEL, plus LANGFUSE_AI_API_KEY when LANGFUSE_AI_PROVIDER is anthropic or openai.";const extraHeadersRecordSchema = z.record(z.string(), z.string());@@ -43,9 +43,9 @@ const extraHeadersRecordSchema = z.record(z.string(), z.string());** Callers should not branch on vendor beyond this discriminated config.* `LANGFUSE_AI_PROVIDER` selects `bedrock`, `anthropic`, or `openai`. Unset- * provider defaults to Bedrock, including when- * `NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` is set. Cloud does not override an- * explicit Anthropic or OpenAI provider.+ * provider is unconfigured, even when model variables and+ * `NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` are set. Bedrock requires+ * `LANGFUSE_AI_PROVIDER=bedrock`.** Region is optional for Bedrock: Cloud web historically omits it and lets* the AWS SDK use the task region.@@ -58,7 +58,10 @@ const extraHeadersRecordSchema = z.record(z.string(), z.string());export function getInAppAgentModelConfig(params?: {modelId?: string | null;}): InAppAgentModelConfig | undefined {- const provider = resolveLangfuseAIProvider();+ const provider = env.LANGFUSE_AI_PROVIDER;+ if (!provider) {+ return undefined;+ }if (provider === "anthropic" || provider === "openai") {return resolveHttpProviderConfig({ provider, modelId: params?.modelId });@@ -136,10 +139,6 @@ function parseLangfuseAIExtraHeaders(): Record<string, string> | undefined {return Object.keys(headers).length > 0 ? headers : undefined;}-function resolveLangfuseAIProvider(): LangfuseAIProvider {- return env.LANGFUSE_AI_PROVIDER ?? "bedrock";-}-function warnIfBedrockIgnoresNonBedrockEnv() {const ignored: string[] = [];if (env.LANGFUSE_AI_API_KEY) {diff --git a/web/src/__tests__/server/ai-filter-access.servertest.ts b/web/src/__tests__/server/ai-filter-access.servertest.tsindex 2e5d36ca1d..ec2b4d128e 100644--- a/web/src/__tests__/server/ai-filter-access.servertest.ts+++ b/web/src/__tests__/server/ai-filter-access.servertest.ts@@ -145,6 +145,7 @@ describe("Ask AI filter generation access", () => {const originalBedrockModel = env.LANGFUSE_AI_MODEL;const originalBedrockSmallModel = env.LANGFUSE_AI_SMALL_MODEL;const originalSharedBedrockModel = sharedEnv.LANGFUSE_AI_MODEL;+ const originalSharedProvider = sharedEnv.LANGFUSE_AI_PROVIDER;const originalAiFeaturesProjectId =sharedEnv.LANGFUSE_AI_FEATURES_PROJECT_ID;@@ -156,6 +157,8 @@ describe("Ask AI filter generation access", () => {undefined;(sharedEnv as { LANGFUSE_AI_MODEL?: string }).LANGFUSE_AI_MODEL ="test-model";+ (sharedEnv as { LANGFUSE_AI_PROVIDER?: string }).LANGFUSE_AI_PROVIDER =+ "bedrock";(sharedEnv as { LANGFUSE_AI_FEATURES_PROJECT_ID?: string }).LANGFUSE_AI_FEATURES_PROJECT_ID = undefined;@@ -195,6 +198,8 @@ describe("Ask AI filter generation access", () => {originalBedrockSmallModel;(sharedEnv as { LANGFUSE_AI_MODEL?: string }).LANGFUSE_AI_MODEL =originalSharedBedrockModel;+ (sharedEnv as { LANGFUSE_AI_PROVIDER?: string }).LANGFUSE_AI_PROVIDER =+ originalSharedProvider;(sharedEnv as { LANGFUSE_AI_FEATURES_PROJECT_ID?: string }).LANGFUSE_AI_FEATURES_PROJECT_ID = originalAiFeaturesProjectId;diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.tsindex f8d32f5275..5d6e84b21c 100644--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts@@ -139,7 +139,7 @@ const setSharedAiModel = (params: {smallModel: string | undefined;}) => {Object.assign(sharedEnv, {- LANGFUSE_AI_PROVIDER: undefined,+ LANGFUSE_AI_PROVIDER: "bedrock",LANGFUSE_AI_API_KEY: undefined,LANGFUSE_AI_MODEL: params.model,LANGFUSE_AI_SMALL_MODEL: params.smallModel,diff --git a/web/src/__tests__/server/in-app-agent-background-run.servertest.ts b/web/src/__tests__/server/in-app-agent-background-run.servertest.tsindex 08018417d8..63536ac824 100644--- a/web/src/__tests__/server/in-app-agent-background-run.servertest.ts+++ b/web/src/__tests__/server/in-app-agent-background-run.servertest.ts@@ -115,9 +115,9 @@ describe("in-app agent background runs", () => {(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";(env as any).LANGFUSE_AI_MODEL = "test-model";(sharedEnv as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";- // Pin the provider: a local .env with LANGFUSE_AI_PROVIDER=anthropic would- // otherwise route these Bedrock cases through the Anthropic branch.- (sharedEnv as any).LANGFUSE_AI_PROVIDER = undefined;+ // Pin Bedrock: a local .env with LANGFUSE_AI_PROVIDER=anthropic would+ // otherwise route these cases through the Anthropic branch.+ (sharedEnv as any).LANGFUSE_AI_PROVIDER = "bedrock";(sharedEnv as any).LANGFUSE_AI_API_KEY = undefined;(sharedEnv as any).LANGFUSE_AI_MODEL = "test-model";(sharedEnv as any).LANGFUSE_AI_SMALL_MODEL = undefined;diff --git a/web/src/env.mjs b/web/src/env.mjsindex f85164cf4a..f471d8ddde 100644--- a/web/src/env.mjs+++ b/web/src/env.mjs@@ -478,6 +478,8 @@ export const env = createEnv({SLACK_CLIENT_SECRET: z.string().optional(),SLACK_STATE_SECRET: z.string().optional(),+ // LANGFUSE_AI_PROVIDER is optional at boot. Unset means unconfigured;+ // bedrock requires LANGFUSE_AI_PROVIDER=bedrock.// LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL / LANGFUSE_AI_AWS_BEDROCK_REGION// apply to all providers. LANGFUSE_AI_API_KEY / LANGFUSE_AI_BASE_URL /// LANGFUSE_AI_EXTRA_HEADERS apply to anthropic and openai.diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.test.tsindex d56b9830bc..ac44defe69 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts@@ -19,6 +19,7 @@ vi.hoisted(() => {delete process.env.LANGFUSE_IN_APP_AGENT_SANDBOX_PROVIDER;process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ??= "DEV";process.env.NEXTAUTH_URL ??= "http://localhost:3000";+ process.env.LANGFUSE_AI_PROVIDER ??= "bedrock";process.env.LANGFUSE_AI_AWS_BEDROCK_REGION ??= "eu-central-1";process.env.LANGFUSE_AI_MODEL ??= "test-bedrock-model";});