Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(in-app-agent): Add Anthropic Messages as a supported model provider for Langfuse Assistant (OSS) (#16385)

Lysander Kiesel014a2d9ff132
diff --git a/.env.dev.example b/.env.dev.example
index 595d8b1598..f63c89d2d9 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -191,13 +191,25 @@ LANGFUSE_AI_FEATURES_PROJECT_ID=7a88fb47-b4e2-43b8-a06c-a5ce950dc53a
# LANGFUSE_SSO_DISCOVERY_WHITELISTED_IPS=127.0.0.1,::1
# LANGFUSE_SSO_DISCOVERY_WHITELISTED_IP_SEGMENTS=127.0.0.0/8
+# Langfuse AI. Unset LANGFUSE_AI_PROVIDER defaults to Bedrock, including when
+# NEXT_PUBLIC_LANGFUSE_CLOUD_REGION is set. Direct Anthropic Messages:
+# LANGFUSE_AI_PROVIDER="anthropic"
+# LANGFUSE_AI_MODEL="claude-opus-4-8"
+# LANGFUSE_AI_SMALL_MODEL="claude-haiku-4-5"
+# LANGFUSE_AI_API_KEY="sk-ant-..."
+# LANGFUSE_AI_BASE_URL="https://api.anthropic.com"
+# Do not set ANTHROPIC_* — Claude Code injects those without /v1 and can 404.
+
# Langfuse AI Bedrock credentials
AWS_ACCESS_KEY_ID="A123456789"
AWS_SECRET_ACCESS_KEY="SAK123456789"
LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY="1234567890abcdef"
+# LANGFUSE_AWS_BEDROCK_REGION is a deprecated alias of
+# LANGFUSE_AI_AWS_BEDROCK_REGION. Bedrock model IDs stay on
+# LANGFUSE_AWS_BEDROCK_MODEL / LANGFUSE_AWS_BEDROCK_SMALL_MODEL;
+# LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL are Anthropic-only.
+# LANGFUSE_AI_AWS_BEDROCK_REGION="eu-west-1"
LANGFUSE_AWS_BEDROCK_REGION="eu-west-1"
-# Default model for AI features. Search-bar filter generation prefers the small
-# model below and falls back to this model when the small model is not configured.
LANGFUSE_AWS_BEDROCK_MODEL="eu.anthropic.claude-opus-4-8"
LANGFUSE_AWS_BEDROCK_SMALL_MODEL="eu.anthropic.claude-haiku-4-5-20251001-v1:0"
LANGFUSE_IN_APP_AGENT_AWS_PROFILE="playground"
diff --git a/docker-compose.build.yml b/docker-compose.build.yml
index e1391f962d..0b85bd53df 100644
--- a/docker-compose.build.yml
+++ b/docker-compose.build.yml
@@ -57,9 +57,18 @@ services:
REDIS_TLS_KEY: ${REDIS_TLS_KEY:-/certs/redis.key}
# Shared AI features configuration. These apply to AI features that run
# in both web and worker.
+ # LANGFUSE_AWS_BEDROCK_REGION is a deprecated alias of
+ # LANGFUSE_AI_AWS_BEDROCK_REGION. Bedrock model IDs stay on
+ # LANGFUSE_AWS_BEDROCK_MODEL / LANGFUSE_AWS_BEDROCK_SMALL_MODEL.
LANGFUSE_AWS_BEDROCK_MODEL: ${LANGFUSE_AWS_BEDROCK_MODEL:-}
LANGFUSE_AWS_BEDROCK_SMALL_MODEL: ${LANGFUSE_AWS_BEDROCK_SMALL_MODEL:-}
LANGFUSE_AWS_BEDROCK_REGION: ${LANGFUSE_AWS_BEDROCK_REGION:-}
+ LANGFUSE_AI_PROVIDER: ${LANGFUSE_AI_PROVIDER:-}
+ LANGFUSE_AI_MODEL: ${LANGFUSE_AI_MODEL:-}
+ LANGFUSE_AI_SMALL_MODEL: ${LANGFUSE_AI_SMALL_MODEL:-}
+ LANGFUSE_AI_API_KEY: ${LANGFUSE_AI_API_KEY:-}
+ LANGFUSE_AI_BASE_URL: ${LANGFUSE_AI_BASE_URL:-}
+ LANGFUSE_AI_AWS_BEDROCK_REGION: ${LANGFUSE_AI_AWS_BEDROCK_REGION:-}
LANGFUSE_IN_APP_AGENT_ENABLED: ${LANGFUSE_IN_APP_AGENT_ENABLED:-}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
diff --git a/docker-compose.yml b/docker-compose.yml
index 2778f135d4..1aa23a583f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -76,9 +76,18 @@ services:
# Shared AI features configuration. These apply to AI features that run
# in both web and worker.
LANGFUSE_IN_APP_AGENT_ENABLED: ${LANGFUSE_IN_APP_AGENT_ENABLED:-}
+ # LANGFUSE_AWS_BEDROCK_REGION is a deprecated alias of
+ # LANGFUSE_AI_AWS_BEDROCK_REGION. Bedrock model IDs stay on
+ # LANGFUSE_AWS_BEDROCK_MODEL / LANGFUSE_AWS_BEDROCK_SMALL_MODEL.
LANGFUSE_AWS_BEDROCK_MODEL: ${LANGFUSE_AWS_BEDROCK_MODEL:-}
LANGFUSE_AWS_BEDROCK_SMALL_MODEL: ${LANGFUSE_AWS_BEDROCK_SMALL_MODEL:-}
LANGFUSE_AWS_BEDROCK_REGION: ${LANGFUSE_AWS_BEDROCK_REGION:-}
+ LANGFUSE_AI_PROVIDER: ${LANGFUSE_AI_PROVIDER:-}
+ LANGFUSE_AI_MODEL: ${LANGFUSE_AI_MODEL:-}
+ LANGFUSE_AI_SMALL_MODEL: ${LANGFUSE_AI_SMALL_MODEL:-}
+ LANGFUSE_AI_API_KEY: ${LANGFUSE_AI_API_KEY:-}
+ LANGFUSE_AI_BASE_URL: ${LANGFUSE_AI_BASE_URL:-}
+ LANGFUSE_AI_AWS_BEDROCK_REGION: ${LANGFUSE_AI_AWS_BEDROCK_REGION:-}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-}
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index 53ec693f3d..d5b75b8cb5 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -519,9 +519,19 @@ const EnvSchema = z.object({
.positive()
.default(DEFAULT_LLM_COMPLETION_TIMEOUT_MS), // 2 minutes
+ // LANGFUSE_AWS_BEDROCK_REGION is a deprecated alias of
+ // LANGFUSE_AI_AWS_BEDROCK_REGION. Bedrock model IDs stay on
+ // LANGFUSE_AWS_BEDROCK_MODEL / LANGFUSE_AWS_BEDROCK_SMALL_MODEL;
+ // LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL are Anthropic-only.
LANGFUSE_AWS_BEDROCK_REGION: z.string().optional(),
LANGFUSE_AWS_BEDROCK_MODEL: z.string().optional(),
LANGFUSE_AWS_BEDROCK_SMALL_MODEL: z.string().optional(),
+ LANGFUSE_AI_PROVIDER: z.enum(["bedrock", "anthropic"]).optional(),
+ LANGFUSE_AI_MODEL: z.string().optional(),
+ LANGFUSE_AI_SMALL_MODEL: z.string().optional(),
+ LANGFUSE_AI_API_KEY: z.string().optional(),
+ LANGFUSE_AI_BASE_URL: z.string().optional(),
+ LANGFUSE_AI_AWS_BEDROCK_REGION: z.string().optional(),
LANGFUSE_IN_APP_AGENT_ENABLED: z.enum(["true", "false"]).optional(),
// API Performance Flags
diff --git a/packages/shared/src/in-app-agent/server/modelProvider.test.ts b/packages/shared/src/in-app-agent/server/modelProvider.test.ts
new file mode 100644
index 0000000000..2061746b3c
--- /dev/null
+++ b/packages/shared/src/in-app-agent/server/modelProvider.test.ts
@@ -0,0 +1,143 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { env } from "../../env";
+import { logger } from "../../server/logger";
+import { getInAppAgentModelConfig } from "./modelProvider";
+
+const original = {
+ LANGFUSE_AI_PROVIDER: env.LANGFUSE_AI_PROVIDER,
+ LANGFUSE_AI_MODEL: env.LANGFUSE_AI_MODEL,
+ LANGFUSE_AI_SMALL_MODEL: env.LANGFUSE_AI_SMALL_MODEL,
+ LANGFUSE_AI_API_KEY: env.LANGFUSE_AI_API_KEY,
+ LANGFUSE_AI_BASE_URL: env.LANGFUSE_AI_BASE_URL,
+ LANGFUSE_AI_AWS_BEDROCK_REGION: env.LANGFUSE_AI_AWS_BEDROCK_REGION,
+ LANGFUSE_AWS_BEDROCK_MODEL: env.LANGFUSE_AWS_BEDROCK_MODEL,
+ LANGFUSE_AWS_BEDROCK_SMALL_MODEL: env.LANGFUSE_AWS_BEDROCK_SMALL_MODEL,
+ LANGFUSE_AWS_BEDROCK_REGION: env.LANGFUSE_AWS_BEDROCK_REGION,
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
+};
+
+afterEach(() => {
+ Object.assign(env, original);
+});
+
+describe("getInAppAgentModelConfig", () => {
+ it("resolves Bedrock from existing env when provider is unset", () => {
+ Object.assign(env, {
+ LANGFUSE_AI_PROVIDER: undefined,
+ LANGFUSE_AI_API_KEY: undefined,
+ LANGFUSE_AI_BASE_URL: undefined,
+ LANGFUSE_AI_AWS_BEDROCK_REGION: undefined,
+ LANGFUSE_AWS_BEDROCK_MODEL: "eu.anthropic.claude-opus-4-8",
+ LANGFUSE_AWS_BEDROCK_SMALL_MODEL: "eu.anthropic.claude-haiku-4-5",
+ LANGFUSE_AWS_BEDROCK_REGION: "eu-west-1",
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ });
+
+ expect(getInAppAgentModelConfig()).toEqual({
+ provider: "bedrock",
+ modelId: "eu.anthropic.claude-opus-4-8",
+ titleModelId: "eu.anthropic.claude-haiku-4-5",
+ region: "eu-west-1",
+ });
+ });
+
+ it("resolves Anthropic Messages from LANGFUSE_AI_PROVIDER and namespaced key/url", () => {
+ Object.assign(env, {
+ LANGFUSE_AI_PROVIDER: "anthropic",
+ LANGFUSE_AI_MODEL: "claude-opus-4-8",
+ LANGFUSE_AI_SMALL_MODEL: "claude-haiku-4-5",
+ LANGFUSE_AI_API_KEY: "sk-ant-test",
+ LANGFUSE_AI_BASE_URL: "https://api.anthropic.com",
+ LANGFUSE_AWS_BEDROCK_MODEL: "eu.anthropic.claude-opus-4-8",
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ });
+
+ expect(getInAppAgentModelConfig()).toEqual({
+ provider: "anthropic",
+ modelId: "claude-opus-4-8",
+ titleModelId: "claude-haiku-4-5",
+ apiKey: "sk-ant-test",
+ baseURL: "https://api.anthropic.com/v1",
+ });
+ });
+
+ it("treats incomplete Anthropic env as unconfigured", () => {
+ Object.assign(env, {
+ LANGFUSE_AI_PROVIDER: "anthropic",
+ LANGFUSE_AI_MODEL: "claude-opus-4-8",
+ LANGFUSE_AI_API_KEY: undefined,
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ });
+
+ expect(getInAppAgentModelConfig()).toBeUndefined();
+ });
+
+ it("honors LANGFUSE_AI_PROVIDER=anthropic even when Cloud region is set", () => {
+ Object.assign(env, {
+ LANGFUSE_AI_PROVIDER: "anthropic",
+ LANGFUSE_AI_MODEL: "claude-opus-4-8",
+ LANGFUSE_AI_SMALL_MODEL: "claude-haiku-4-5",
+ LANGFUSE_AI_API_KEY: "sk-ant-test",
+ LANGFUSE_AI_BASE_URL: "https://api.anthropic.com",
+ LANGFUSE_AWS_BEDROCK_MODEL: "eu.anthropic.claude-opus-4-8",
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",
+ });
+
+ expect(getInAppAgentModelConfig()).toEqual({
+ provider: "anthropic",
+ modelId: "claude-opus-4-8",
+ titleModelId: "claude-haiku-4-5",
+ apiKey: "sk-ant-test",
+ baseURL: "https://api.anthropic.com/v1",
+ });
+ });
+
+ it("warns that Anthropic key and base URL are ignored on Bedrock", () => {
+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);
+
+ Object.assign(env, {
+ LANGFUSE_AI_PROVIDER: "bedrock",
+ LANGFUSE_AI_API_KEY: "sk-ant-test",
+ LANGFUSE_AI_BASE_URL: "https://api.anthropic.com",
+ LANGFUSE_AWS_BEDROCK_MODEL: "eu.anthropic.claude-opus-4-8",
+ LANGFUSE_AWS_BEDROCK_SMALL_MODEL: undefined,
+ LANGFUSE_AWS_BEDROCK_REGION: "eu-west-1",
+ LANGFUSE_AI_AWS_BEDROCK_REGION: undefined,
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",
+ });
+
+ expect(getInAppAgentModelConfig()).toEqual({
+ provider: "bedrock",
+ modelId: "eu.anthropic.claude-opus-4-8",
+ titleModelId: "eu.anthropic.claude-opus-4-8",
+ region: "eu-west-1",
+ });
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(warn.mock.calls[0]?.[0]).toEqual(
+ "Ignoring LANGFUSE_AI_API_KEY and LANGFUSE_AI_BASE_URL because the Langfuse AI provider is bedrock. Bedrock uses the instance AWS credential chain, not an Anthropic API key or base URL.",
+ );
+
+ warn.mockRestore();
+ });
+
+ it("prefers LANGFUSE_AI_AWS_BEDROCK_REGION over LANGFUSE_AWS_BEDROCK_REGION", () => {
+ Object.assign(env, {
+ LANGFUSE_AI_PROVIDER: undefined,
+ LANGFUSE_AI_API_KEY: undefined,
+ LANGFUSE_AI_BASE_URL: undefined,
+ LANGFUSE_AI_AWS_BEDROCK_REGION: "us-east-1",
+ LANGFUSE_AWS_BEDROCK_MODEL: "eu.anthropic.claude-opus-4-8",
+ LANGFUSE_AWS_BEDROCK_SMALL_MODEL: undefined,
+ LANGFUSE_AWS_BEDROCK_REGION: "eu-west-1",
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",
+ });
+
+ expect(getInAppAgentModelConfig()).toEqual({
+ provider: "bedrock",
+ modelId: "eu.anthropic.claude-opus-4-8",
+ titleModelId: "eu.anthropic.claude-opus-4-8",
+ region: "us-east-1",
+ });
+ });
+});
diff --git a/packages/shared/src/in-app-agent/server/modelProvider.ts b/packages/shared/src/in-app-agent/server/modelProvider.ts
index 649235f24a..f1cd535c69 100644
--- a/packages/shared/src/in-app-agent/server/modelProvider.ts
+++ b/packages/shared/src/in-app-agent/server/modelProvider.ts
@@ -1,36 +1,71 @@
import { env } from "../../env";
-import { assertValidBedrockRegion } from "../../server/llm/ai-sdk/providers/bedrock";
+import { logger } from "../../server/logger";
+import {
+ assertValidBedrockRegion,
+ getLangfuseAIBedrockRegion,
+} from "../../server/llm/ai-sdk/providers/bedrock";
+import { toAnthropicBaseURL } from "../../server/llm/ai-sdk/providers/anthropic";
-export type InAppAgentModelConfig = {
- provider: "bedrock";
- modelId: string;
- titleModelId: string;
- region?: string;
-};
+export type LangfuseAIProvider = "bedrock" | "anthropic";
+
+export type InAppAgentModelConfig =
+ | {
+ provider: "bedrock";
+ modelId: string;
+ titleModelId: string;
+ region?: string;
+ }
+ | {
+ provider: "anthropic";
+ modelId: string;
+ titleModelId: string;
+ apiKey: string;
+ baseURL?: string;
+ };
+
+export const LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE =
+ "Langfuse AI model is not configured. Set LANGFUSE_AI_PROVIDER=anthropic with LANGFUSE_AI_MODEL and LANGFUSE_AI_API_KEY, or set LANGFUSE_AWS_BEDROCK_MODEL.";
/**
- * Resolves the instance-wide in-app agent model configuration.
+ * Resolves the instance-wide Langfuse-operated AI model (Assistant + Ask AI).
*
- * The discriminated provider contract is intentionally narrower than the
- * generic LLM connection model: in-app agent execution currently supports
- * Bedrock only, but callers do not need to know which provider supplies the
- * model. A future provider can add another branch here without changing the
- * worker or title-generation call sites.
+ * Callers should not branch on vendor beyond this discriminated config.
+ * `LANGFUSE_AI_PROVIDER` selects `bedrock` or `anthropic`. Unset provider
+ * defaults to Bedrock, including when `NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` is
+ * set. Cloud does not override an explicit Anthropic provider.
*
- * Region is optional: Cloud web historically omits LANGFUSE_AWS_BEDROCK_REGION
- * and lets the AWS SDK use the task region. Missing region must not treat the
- * model as unconfigured.
+ * Region is optional for Bedrock: Cloud web historically omits it and lets
+ * the AWS SDK use the task region. Prefer LANGFUSE_AI_AWS_BEDROCK_REGION;
+ * LANGFUSE_AWS_BEDROCK_REGION is the fallback during the Cloud cutover.
*/
export function getInAppAgentModelConfig(params?: {
modelId?: string | null;
}): InAppAgentModelConfig | undefined {
+ if (resolveLangfuseAIProvider() === "anthropic") {
+ const modelId = params?.modelId ?? env.LANGFUSE_AI_MODEL;
+ const apiKey = env.LANGFUSE_AI_API_KEY;
+
+ if (!modelId || !apiKey) {
+ return undefined;
+ }
+
+ return {
+ provider: "anthropic",
+ modelId,
+ titleModelId: env.LANGFUSE_AI_SMALL_MODEL ?? modelId,
+ apiKey,
+ baseURL: toAnthropicBaseURL(env.LANGFUSE_AI_BASE_URL),
+ };
+ }
+
const modelId = params?.modelId ?? env.LANGFUSE_AWS_BEDROCK_MODEL;
- const region = env.LANGFUSE_AWS_BEDROCK_REGION;
+ const region = getLangfuseAIBedrockRegion();
if (!modelId) {
return undefined;
}
+ warnIfBedrockIgnoresAnthropicEnv();
assertValidBedrockRegion(region);
return {
@@ -41,6 +76,27 @@ export function getInAppAgentModelConfig(params?: {
};
}
+function resolveLangfuseAIProvider(): LangfuseAIProvider {
+ return env.LANGFUSE_AI_PROVIDER ?? "bedrock";
+}
+
+function warnIfBedrockIgnoresAnthropicEnv() {
+ const ignored: string[] = [];
+ if (env.LANGFUSE_AI_API_KEY) {
+ ignored.push("LANGFUSE_AI_API_KEY");
+ }
+ if (env.LANGFUSE_AI_BASE_URL) {
+ ignored.push("LANGFUSE_AI_BASE_URL");
+ }
+ if (ignored.length === 0) {
+ return;
+ }
+
+ logger.warn(
+ `Ignoring ${ignored.join(" and ")} because the Langfuse AI provider is bedrock. Bedrock uses the instance AWS credential chain, not an Anthropic API key or base URL.`,
+ );
+}
+
/**
* Instance-wide in-app agent switch.
*
diff --git a/packages/shared/src/server/llm/ai-sdk/providers/anthropic.ts b/packages/shared/src/server/llm/ai-sdk/providers/anthropic.ts
index e98ddf1fd6..5d8eee2215 100644
--- a/packages/shared/src/server/llm/ai-sdk/providers/anthropic.ts
+++ b/packages/shared/src/server/llm/ai-sdk/providers/anthropic.ts
@@ -6,8 +6,8 @@ import { ensureBaseURLSuffix, isPlainObject } from "./utils";
/**
* The stored Anthropic base URL is the API origin, without `/v1/messages`.
- * The AI SDK expects the `/v1` prefix to be part of `baseURL` (default
- * `https://api.anthropic.com/v1`) and appends only `/messages`.
+ * The AI SDK expects the `/v1` prefix to be part of `baseURL` and appends
+ * only `/messages`.
*/
export function toAnthropicBaseURL(
baseURL: string | null | undefined,
diff --git a/packages/shared/src/server/llm/ai-sdk/providers/bedrock.ts b/packages/shared/src/server/llm/ai-sdk/providers/bedrock.ts
index 5940830a56..f4b6142d20 100644
--- a/packages/shared/src/server/llm/ai-sdk/providers/bedrock.ts
+++ b/packages/shared/src/server/llm/ai-sdk/providers/bedrock.ts
@@ -28,6 +28,15 @@ export function assertValidBedrockRegion(region: string | undefined): void {
}
}
+/**
+ * Bedrock region for Langfuse-operated AI (Assistant, Ask AI).
+ * `LANGFUSE_AI_AWS_BEDROCK_REGION` is preferred; `LANGFUSE_AWS_BEDROCK_REGION`
+ * remains the fallback until Cloud infra finishes the cutover.
+ */
+export function getLangfuseAIBedrockRegion(): string | undefined {
+ return env.LANGFUSE_AI_AWS_BEDROCK_REGION ?? env.LANGFUSE_AWS_BEDROCK_REGION;
+}
+
/**
* Translation of Langfuse `modelParams.providerOptions` to AI SDK Bedrock
* provider options.
@@ -166,7 +175,7 @@ export function buildBedrockModel(params: {
const shouldUseLangfuseAPIKey = credentialSource === "langfuse";
const { region } = shouldUseLangfuseAPIKey
- ? { region: env.LANGFUSE_AWS_BEDROCK_REGION }
+ ? { region: getLangfuseAIBedrockRegion() }
: BedrockConfigSchema.parse(config);
assertValidBedrockRegion(region);
diff --git a/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts b/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts
index 1d9aef3178..293849a9ca 100644
--- a/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts
+++ b/packages/shared/src/server/llm/ai-sdk/resolveAiSdkModelConfig.ts
@@ -39,11 +39,13 @@ export function resolveAiSdkModelConfig(params: {
try {
if (
credentialSource === "langfuse" &&
- model.adapter !== LLMAdapter.Bedrock
+ model.adapter !== LLMAdapter.Bedrock &&
+ model.adapter !== LLMAdapter.Anthropic
) {
throw new LLMValidationError({
code: "invalid-connection",
- message: "Langfuse credentials are only supported for Amazon Bedrock",
+ message:
+ "Langfuse credentials are only supported for Amazon Bedrock and Anthropic",
});
}
diff --git a/packages/shared/src/server/llm/langfuseAiCompletion.ts b/packages/shared/src/server/llm/langfuseAiCompletion.ts
index 3fa0a96ca0..46cbcb7e3b 100644
--- a/packages/shared/src/server/llm/langfuseAiCompletion.ts
+++ b/packages/shared/src/server/llm/langfuseAiCompletion.ts
@@ -1,6 +1,7 @@
import { BEDROCK_USE_DEFAULT_CREDENTIALS } from "../../interfaces/customLLMProviderConfigSchemas";
import { encrypt } from "../../encryption";
import { env } from "../../env";
+import { getInAppAgentModelConfig } from "../../in-app-agent/server/modelProvider";
import { type ChatMessage, LLMAdapter, type TraceSinkParams } from "./types";
import { generateLLMText, mapLegacyLLMCompletionParams } from "./llmText";
import { randomBytes } from "crypto";
@@ -45,30 +46,48 @@ export async function generateLangfuseAIText(params: {
traceSinkParams?: TraceSinkParams;
timeout?: number;
}): Promise<string> {
- const model = params.model ?? env.LANGFUSE_AWS_BEDROCK_MODEL;
+ const modelConfig = getInAppAgentModelConfig({ modelId: params.model });
- if (!model) {
+ if (!modelConfig) {
throw new Error("Langfuse AI completion model is not configured.");
}
+ const maxTokens =
+ params.maxTokens !== undefined ? { max_tokens: params.maxTokens } : {};
+
const result = await generateLLMText({
- ...mapLegacyLLMCompletionParams({
- messages: params.messages,
- modelParams: {
- provider: "bedrock",
- adapter: LLMAdapter.Bedrock,
- model,
- // Intentionally omit temperature/top_p: newer Bedrock models reject these
- // inference params, while AI-feature generation works at model defaults.
- ...(params.maxTokens !== undefined
- ? { max_tokens: params.maxTokens }
- : {}),
- },
- connection: {
- secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
- },
- credentialSource: "langfuse",
- }),
+ ...mapLegacyLLMCompletionParams(
+ modelConfig.provider === "anthropic"
+ ? {
+ messages: params.messages,
+ modelParams: {
+ provider: "anthropic",
+ adapter: LLMAdapter.Anthropic,
+ model: modelConfig.modelId,
+ ...maxTokens,
+ },
+ connection: {
+ secretKey: encrypt(modelConfig.apiKey),
+ baseURL: modelConfig.baseURL,
+ },
+ credentialSource: "langfuse",
+ }
+ : {
+ messages: params.messages,
+ modelParams: {
+ provider: "bedrock",
+ adapter: LLMAdapter.Bedrock,
+ model: modelConfig.modelId,
+ // Intentionally omit temperature/top_p: newer Bedrock models reject these
+ // inference params, while AI-feature generation works at model defaults.
+ ...maxTokens,
+ },
+ connection: {
+ secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
+ },
+ credentialSource: "langfuse",
+ },
+ ),
trace: params.traceSinkParams,
timeout: params.timeout,
});
diff --git a/packages/shared/src/server/llm/llmText.test.ts b/packages/shared/src/server/llm/llmText.test.ts
index 56ce8b0c61..c552fb7a7b 100644
--- a/packages/shared/src/server/llm/llmText.test.ts
+++ b/packages/shared/src/server/llm/llmText.test.ts
@@ -573,7 +573,8 @@ describe("legacy compatibility boundary", () => {
}),
).rejects.toMatchObject({
name: "LLMValidationError",
- message: "Langfuse credentials are only supported for Amazon Bedrock",
+ message:
+ "Langfuse credentials are only supported for Amazon Bedrock and Anthropic",
statusCode: 400,
code: "invalid-connection",
});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index f7757830f8..a28762b3cd 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1084,6 +1084,9 @@ importers:
ai-sdk-amazon-bedrock-v4:
specifier: npm:@ai-sdk/[email protected]
+ ai-sdk-anthropic-v4:
+ specifier: npm:@ai-sdk/[email protected]
bullmq:
specifier: ^5.76.3
version: 5.76.3([email protected])
@@ -1259,6 +1262,12 @@ packages:
peerDependencies:
zod: 4.3.6
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-atgBW8jZPr/KuaKX5FvDIHuXBI8VCol6kVeoD4P0657+VXR73QsLogXQVN/Zt5FHtq9WzpdIZseCJiXPqkgwwA==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ zod: 4.3.6
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-CkShXR8tmNO7QQnvpKbSMe2Vr1zUUcpqlp69iR+DYrbHm+tDJO9u6zZsjEHjcoRU9/e9z++p0W6NiuLC3aZ4Bg==}
engines: {node: '>=18'}
@@ -1349,6 +1358,12 @@ packages:
peerDependencies:
zod: 4.3.6
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-tEtld97plCFiYevsJuOkGkeuhQndeMWFBVrJS4AjnbD5AqrNSXRCe0p+BZ3Cju/sxDeeZ9ym3q9YUV8fASA7aQ==}
+ engines: {node: '>=18.17'}
+ peerDependencies:
+ zod: 4.3.6
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-7/96wE+ZsKB35iS9ASyllrE4Ym/EolXEB7AkuJ5FI++fmS85BVTAs77890C+1Z2jwHfBKjBQSBmsliOsAh0iFQ==}
engines: {node: '>=22'}
@@ -1387,6 +1402,10 @@ packages:
resolution: {integrity: sha512-5X1k57JBJ4H7H1QjX7CnJYAB1I19r/trVZTMcSms7/kLNZ8RaU4Nt2agcwZzv82Hfx6Q7/TOLU7agAKeFfc8cA==}
engines: {node: '>=18'}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-XeZW1CcDF2GMbH4wejW6xBRI2QCOgnkVYUnxoeDadB1mf85riL2bMUeDoh+6gJ/r4mjNfzUPW8OjLjvwTP0u1Q==}
+ engines: {node: '>=18'}
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==}
engines: {node: '>=18'}
@@ -10595,6 +10614,10 @@ packages:
resolution: {integrity: sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==}
+ resolution: {integrity: sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==}
+ engines: {node: '>=18.17'}
+
resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==}
engines: {node: '>=20.18.1'}
@@ -11218,6 +11241,12 @@ snapshots:
aws4fetch: 1.0.20
zod: 4.3.6
+ dependencies:
+ '@ai-sdk/provider': 3.0.15
+ '@ai-sdk/provider-utils': 4.0.46([email protected])
+ zod: 4.3.6
+
dependencies:
'@ai-sdk/provider': 3.0.8
@@ -11323,6 +11352,14 @@ snapshots:
zod: 4.3.6
optional: true
+ dependencies:
+ '@ai-sdk/provider': 3.0.15
+ '@standard-schema/spec': 1.1.0
+ eventsource-parser: 3.1.0
+ undici: 6.28.0
+ zod: 4.3.6
+
dependencies:
'@ai-sdk/provider': 4.0.3
@@ -11368,6 +11405,10 @@ snapshots:
json-schema: 0.4.0
optional: true
+ '@ai-sdk/[email protected]':
+ dependencies:
+ json-schema: 0.4.0
+
'@ai-sdk/[email protected]':
dependencies:
json-schema: 0.4.0
@@ -21825,6 +21866,8 @@ snapshots:
+
diff --git a/web/src/__tests__/server/ai-filter-access.servertest.ts b/web/src/__tests__/server/ai-filter-access.servertest.ts
index a6f6815366..95cf80ac7f 100644
--- a/web/src/__tests__/server/ai-filter-access.servertest.ts
+++ b/web/src/__tests__/server/ai-filter-access.servertest.ts
@@ -144,6 +144,7 @@ describe("Ask AI filter generation access", () => {
const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
const originalBedrockModel = env.LANGFUSE_AWS_BEDROCK_MODEL;
const originalBedrockSmallModel = env.LANGFUSE_AWS_BEDROCK_SMALL_MODEL;
+ const originalSharedBedrockModel = sharedEnv.LANGFUSE_AWS_BEDROCK_MODEL;
const originalAiFeaturesProjectId =
sharedEnv.LANGFUSE_AI_FEATURES_PROJECT_ID;
@@ -156,6 +157,9 @@ describe("Ask AI filter generation access", () => {
(
env as { LANGFUSE_AWS_BEDROCK_SMALL_MODEL?: string }
).LANGFUSE_AWS_BEDROCK_SMALL_MODEL = undefined;
+ (
+ sharedEnv as { LANGFUSE_AWS_BEDROCK_MODEL?: string }
+ ).LANGFUSE_AWS_BEDROCK_MODEL = "test-model";
(
sharedEnv as { LANGFUSE_AI_FEATURES_PROJECT_ID?: string }
).LANGFUSE_AI_FEATURES_PROJECT_ID = undefined;
@@ -195,6 +199,9 @@ describe("Ask AI filter generation access", () => {
(
env as { LANGFUSE_AWS_BEDROCK_SMALL_MODEL?: string }
).LANGFUSE_AWS_BEDROCK_SMALL_MODEL = originalBedrockSmallModel;
+ (
+ sharedEnv as { LANGFUSE_AWS_BEDROCK_MODEL?: string }
+ ).LANGFUSE_AWS_BEDROCK_MODEL = originalSharedBedrockModel;
(
sharedEnv as { LANGFUSE_AI_FEATURES_PROJECT_ID?: string }
).LANGFUSE_AI_FEATURES_PROJECT_ID = originalAiFeaturesProjectId;
diff --git a/web/src/__tests__/server/in-app-agent-background-run.servertest.ts b/web/src/__tests__/server/in-app-agent-background-run.servertest.ts
index d3dea9ba96..abcb495e22 100644
--- a/web/src/__tests__/server/in-app-agent-background-run.servertest.ts
+++ b/web/src/__tests__/server/in-app-agent-background-run.servertest.ts
@@ -13,6 +13,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { type Plan } from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import { env as sharedEnv } from "@langfuse/shared/src/env";
+import { LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE } from "@langfuse/shared/in-app-agent/server/modelProvider";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import {
IN_APP_AGENT_APPROVAL_DECISION_EVENT_NAME,
@@ -100,6 +101,8 @@ describe("in-app agent background runs", () => {
const originalSharedCloudRegion = sharedEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
const originalSharedBedrockModel = sharedEnv.LANGFUSE_AWS_BEDROCK_MODEL;
const originalSharedBedrockRegion = sharedEnv.LANGFUSE_AWS_BEDROCK_REGION;
+ const originalSharedAiBedrockRegion =
+ sharedEnv.LANGFUSE_AI_AWS_BEDROCK_REGION;
const originalSharedInAppAgentEnabled =
sharedEnv.LANGFUSE_IN_APP_AGENT_ENABLED;
const originalMaxActiveRunsPerUser =
@@ -113,6 +116,7 @@ describe("in-app agent background runs", () => {
(sharedEnv as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
(sharedEnv as any).LANGFUSE_AWS_BEDROCK_MODEL = "test-model";
(sharedEnv as any).LANGFUSE_AWS_BEDROCK_REGION = "eu-central-1";
+ (sharedEnv as any).LANGFUSE_AI_AWS_BEDROCK_REGION = undefined;
(sharedEnv as any).LANGFUSE_IN_APP_AGENT_ENABLED = undefined;
enqueuedJobs.length = 0;
enqueueShouldFail = false;
@@ -132,6 +136,8 @@ describe("in-app agent background runs", () => {
(sharedEnv as any).LANGFUSE_AWS_BEDROCK_MODEL = originalSharedBedrockModel;
(sharedEnv as any).LANGFUSE_AWS_BEDROCK_REGION =
originalSharedBedrockRegion;
+ (sharedEnv as any).LANGFUSE_AI_AWS_BEDROCK_REGION =
+ originalSharedAiBedrockRegion;
(sharedEnv as any).LANGFUSE_IN_APP_AGENT_ENABLED =
originalSharedInAppAgentEnabled;
(env as any).LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_USER =
@@ -292,6 +298,7 @@ describe("in-app agent background runs", () => {
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
(sharedEnv as any).LANGFUSE_AWS_BEDROCK_REGION = undefined;
+ (sharedEnv as any).LANGFUSE_AI_AWS_BEDROCK_REGION = undefined;
await expect(
caller.startRun({
@@ -318,8 +325,7 @@ describe("in-app agent background runs", () => {
}),
).rejects.toMatchObject({
code: "PRECONDITION_FAILED",
- message:
- "In-app agent Bedrock model is not configured. Set LANGFUSE_AWS_BEDROCK_MODEL.",
+ message: LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE,
});
expect(enqueuedJobs).toHaveLength(0);
diff --git a/web/src/env.mjs b/web/src/env.mjs
index 3595dd1f7e..66ca408f8e 100644
--- a/web/src/env.mjs
+++ b/web/src/env.mjs
@@ -460,9 +460,18 @@ export const env = createEnv({
SLACK_CLIENT_SECRET: z.string().optional(),
SLACK_STATE_SECRET: z.string().optional(),
- // AWS Bedrock for langfuse native AI feature such as natural language filters
+ // LANGFUSE_AWS_BEDROCK_REGION is a deprecated alias of
+ // LANGFUSE_AI_AWS_BEDROCK_REGION. Bedrock model IDs stay on
+ // LANGFUSE_AWS_BEDROCK_MODEL / LANGFUSE_AWS_BEDROCK_SMALL_MODEL;
+ // LANGFUSE_AI_MODEL / LANGFUSE_AI_SMALL_MODEL are Anthropic-only.
LANGFUSE_AWS_BEDROCK_MODEL: z.string().optional(),
LANGFUSE_AWS_BEDROCK_SMALL_MODEL: z.string().optional(),
+ LANGFUSE_AI_PROVIDER: z.enum(["bedrock", "anthropic"]).optional(),
+ LANGFUSE_AI_MODEL: z.string().optional(),
+ LANGFUSE_AI_SMALL_MODEL: z.string().optional(),
+ LANGFUSE_AI_API_KEY: z.string().optional(),
+ LANGFUSE_AI_BASE_URL: z.string().optional(),
+ LANGFUSE_AI_AWS_BEDROCK_REGION: z.string().optional(),
LANGFUSE_IN_APP_AGENT_ENABLED: z.enum(["true", "false"]).optional(),
// Tracing for Langfuse AI Features
@@ -562,6 +571,7 @@ export const env = createEnv({
}),
AWS_ACCESS_KEY_ID: z.string().optional(),
AWS_SECRET_ACCESS_KEY: z.string().optional(),
+ // Deprecated. Fallback for LANGFUSE_AI_AWS_BEDROCK_REGION; remove after cutover.
LANGFUSE_AWS_BEDROCK_REGION: z.string().optional(),
LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_USER: z.coerce
.number()
@@ -986,10 +996,18 @@ export const env = createEnv({
SLACK_CLIENT_SECRET: process.env.SLACK_CLIENT_SECRET,
SLACK_STATE_SECRET: process.env.SLACK_STATE_SECRET,
- // AWS Bedrock for langfuse native AI feature such as natural language filters
+ // LANGFUSE_AWS_BEDROCK_REGION is a deprecated alias of
+ // LANGFUSE_AI_AWS_BEDROCK_REGION. Bedrock model IDs stay on
+ // LANGFUSE_AWS_BEDROCK_MODEL / LANGFUSE_AWS_BEDROCK_SMALL_MODEL.
LANGFUSE_AWS_BEDROCK_MODEL: process.env.LANGFUSE_AWS_BEDROCK_MODEL,
LANGFUSE_AWS_BEDROCK_SMALL_MODEL:
process.env.LANGFUSE_AWS_BEDROCK_SMALL_MODEL,
+ LANGFUSE_AI_PROVIDER: process.env.LANGFUSE_AI_PROVIDER,
+ LANGFUSE_AI_MODEL: process.env.LANGFUSE_AI_MODEL,
+ LANGFUSE_AI_SMALL_MODEL: process.env.LANGFUSE_AI_SMALL_MODEL,
+ LANGFUSE_AI_API_KEY: process.env.LANGFUSE_AI_API_KEY,
+ LANGFUSE_AI_BASE_URL: process.env.LANGFUSE_AI_BASE_URL,
+ LANGFUSE_AI_AWS_BEDROCK_REGION: process.env.LANGFUSE_AI_AWS_BEDROCK_REGION,
LANGFUSE_IN_APP_AGENT_ENABLED: process.env.LANGFUSE_IN_APP_AGENT_ENABLED,
// Langfuse Tracing AI Features
diff --git a/web/src/features/in-app-agent/server/availability.ts b/web/src/features/in-app-agent/server/availability.ts
index 589e20b70a..c06880f192 100644
--- a/web/src/features/in-app-agent/server/availability.ts
+++ b/web/src/features/in-app-agent/server/availability.ts
@@ -5,6 +5,7 @@ import type { PrismaClient } from "@langfuse/shared/src/db";
import {
getInAppAgentModelConfig,
isInAppAgentInstanceEnabled,
+ LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE,
} from "@langfuse/shared/in-app-agent/server/modelProvider";
import { hasEntitlement } from "@/src/features/entitlements/server/hasEntitlement";
@@ -64,12 +65,16 @@ export async function assertInAppAgentAvailable({
/** Only startRun and approval continuation dispatch to the model. */
export function assertInAppAgentModelConfigured() {
- if (!getInAppAgentModelConfig()) {
+ const modelConfig = getInAppAgentModelConfig();
+
+ if (!modelConfig) {
throw new BaseError(
"PreconditionFailedError",
412,
- "In-app agent Bedrock model is not configured. Set LANGFUSE_AWS_BEDROCK_MODEL.",
+ LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE,
true,
);
}
+
+ return modelConfig;
}
diff --git a/web/src/features/in-app-agent/server/router.ts b/web/src/features/in-app-agent/server/router.ts
index 92a2a40e05..f1ccc84c79 100644
--- a/web/src/features/in-app-agent/server/router.ts
+++ b/web/src/features/in-app-agent/server/router.ts
@@ -212,7 +212,7 @@ export const inAppAgentRouter = createTRPCRouter({
projectId: input.projectId,
user: ctx.session.user,
});
- assertInAppAgentModelConfigured();
+ const modelConfig = assertInAppAgentModelConfigured();
const rateLimitScope = getInAppAgentApiAccessScope(
ctx.session.user,
@@ -234,7 +234,7 @@ export const inAppAgentRouter = createTRPCRouter({
message: input.message,
context: input.context,
isV4Enabled: ctx.session.user.v4BetaEnabled === true,
- model: env.LANGFUSE_AWS_BEDROCK_MODEL,
+ model: modelConfig.modelId,
aiTelemetryEnabled: projectAvailability.aiTelemetryEnabled,
});
}),
@@ -272,7 +272,7 @@ export const inAppAgentRouter = createTRPCRouter({
projectId: input.projectId,
user: ctx.session.user,
});
- assertInAppAgentModelConfigured();
+ const modelConfig = assertInAppAgentModelConfigured();
const rateLimitScope = getInAppAgentApiAccessScope(
ctx.session.user,
@@ -300,7 +300,7 @@ export const inAppAgentRouter = createTRPCRouter({
approved: input.approved,
approvalScope: input.approvalScope,
userId: ctx.session.user.id,
- model: env.LANGFUSE_AWS_BEDROCK_MODEL,
+ model: modelConfig.modelId,
});
}),
diff --git a/web/src/features/natural-language-filters/server/router.ts b/web/src/features/natural-language-filters/server/router.ts
index 06e93e1c76..648300fd0c 100644
--- a/web/src/features/natural-language-filters/server/router.ts
+++ b/web/src/features/natural-language-filters/server/router.ts
@@ -13,6 +13,10 @@ import {
getLangfuseAITraceSinkParams,
isLangfuseAITracingConfigured,
} from "@langfuse/shared/src/server";
+import {
+ getInAppAgentModelConfig,
+ LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE,
+} from "@langfuse/shared/in-app-agent/server/modelProvider";
import { env } from "@/src/env.mjs";
import { CreateNaturalLanguageFilterCompletion } from "./validation";
import { parseFiltersFromCompletion, getLangfuseClient } from "./utils";
@@ -73,11 +77,10 @@ export const naturalLanguageFilterRouter = createTRPCRouter({
});
}
- if (!env.LANGFUSE_AWS_BEDROCK_MODEL) {
+ if (!getInAppAgentModelConfig()) {
throw new TRPCError({
code: "PRECONDITION_FAILED",
- message:
- "Bedrock environment variables not configured. Please set LANGFUSE_AWS_BEDROCK_* variables.",
+ message: LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE,
});
}
diff --git a/web/src/features/search-bar/server/router.ts b/web/src/features/search-bar/server/router.ts
index ced9f82e11..0459b0bb53 100644
--- a/web/src/features/search-bar/server/router.ts
+++ b/web/src/features/search-bar/server/router.ts
@@ -29,6 +29,10 @@ import {
getLangfuseAITraceSinkParams,
isLangfuseAITracingConfigured,
} from "@langfuse/shared/src/server";
+import {
+ getInAppAgentModelConfig,
+ LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE,
+} from "@langfuse/shared/in-app-agent/server/modelProvider";
import { env } from "@/src/env.mjs";
import { randomBytes } from "crypto";
import { z } from "zod";
@@ -119,18 +123,17 @@ export const searchBarRouter = createTRPCRouter({
});
}
- const model =
- env.LANGFUSE_AWS_BEDROCK_SMALL_MODEL ??
- env.LANGFUSE_AWS_BEDROCK_MODEL;
+ const modelConfig = getInAppAgentModelConfig();
- if (!model) {
+ if (!modelConfig) {
throw new TRPCError({
code: "PRECONDITION_FAILED",
- message:
- "Bedrock environment variables not configured. Please set LANGFUSE_AWS_BEDROCK_* variables.",
+ message: LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE,
});
}
+ const model = modelConfig.titleModelId;
+
// Anchor relative time expressions ("today", "last 24h") to now.
const now = new Date();
const dayOfWeek = now.toLocaleDateString("en-US", { weekday: "long" });
diff --git a/worker/package.json b/worker/package.json
index 88caa031de..cab2af727d 100644
--- a/worker/package.json
+++ b/worker/package.json
@@ -54,6 +54,7 @@
"@prisma/client": "^6.19.3",
"@prisma/instrumentation": "^6.19.3",
"ai-sdk-amazon-bedrock-v4": "npm:@ai-sdk/[email protected]",
+ "ai-sdk-anthropic-v4": "npm:@ai-sdk/[email protected]",
"bullmq": "^5.76.3",
"cors": "^2.8.5",
"csv-parse": "^5.6.0",
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
index 9f887c0c20..09b768ac0c 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
@@ -1017,9 +1017,13 @@ describe("executeInAppAgentRun", () => {
it("claims a run when LANGFUSE_AWS_BEDROCK_REGION is unset", async () => {
const originalRegion = sharedEnv.LANGFUSE_AWS_BEDROCK_REGION;
+ const originalAiRegion = sharedEnv.LANGFUSE_AI_AWS_BEDROCK_REGION;
(
sharedEnv as { LANGFUSE_AWS_BEDROCK_REGION?: string }
).LANGFUSE_AWS_BEDROCK_REGION = undefined;
+ (
+ sharedEnv as { LANGFUSE_AI_AWS_BEDROCK_REGION?: string }
+ ).LANGFUSE_AI_AWS_BEDROCK_REGION = undefined;
try {
const { projectId, run } = await seedBackgroundRun();
@@ -1033,6 +1037,9 @@ describe("executeInAppAgentRun", () => {
(
sharedEnv as { LANGFUSE_AWS_BEDROCK_REGION?: string }
).LANGFUSE_AWS_BEDROCK_REGION = originalRegion;
+ (
+ sharedEnv as { LANGFUSE_AI_AWS_BEDROCK_REGION?: string }
+ ).LANGFUSE_AI_AWS_BEDROCK_REGION = originalAiRegion;
}
});
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
index c10176a732..6e522df55a 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
@@ -33,6 +33,7 @@ import {
import {
getInAppAgentModelConfig,
isInAppAgentInstanceEnabled,
+ LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE,
} from "@langfuse/shared/in-app-agent/server/modelProvider";
import {
claimQueuedRun,
@@ -203,9 +204,7 @@ export async function executeInAppAgentRun(params: {
const modelConfig = getInAppAgentModelConfig({ modelId: run.model });
if (!modelConfig) {
- throw new InAppAgentRunInitError(
- "In-app agent Bedrock model is not configured",
- );
+ throw new InAppAgentRunInitError(LANGFUSE_AI_MODEL_UNCONFIGURED_MESSAGE);
}
const useBundledPrompt =
diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.ts
index c9f32edc3f..b98d863f3e 100644
--- a/worker/src/features/in-app-agent/runtime/agent.test.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts
@@ -33,6 +33,14 @@ const testBedrockModel = (modelId: string) => ({
region: "eu-central-1",
});
+const testAnthropicModel = (modelId: string) => ({
+ provider: "anthropic" as const,
+ modelId,
+ titleModelId: modelId,
+ apiKey: "sk-ant-test",
+ baseURL: "https://api.anthropic.com/v1",
+});
+
// Shape of the tool entries the mocked MCP client feeds into the Agent
// constructor. `Agent`'s own `tools` type is a `DynamicArgument` union that
// does not allow property access, so tests read it through this view.
@@ -55,6 +63,8 @@ const getAgentTools = (
type MockedAgentConfig = {
instructions?: (() => string) | string;
model?: {
+ provider?: string;
+ supportedUrls?: unknown;
doStream: (options: unknown) => Promise<{
stream: ReadableStream<unknown>;
}>;
@@ -271,6 +281,41 @@ vi.mock("ai-sdk-amazon-bedrock-v4", () => ({
})),
}));
+// Match @ai-sdk/anthropic: provider/supportedUrls are prototype getters,
+// not own enumerable properties. A plain object spread drops them.
+vi.mock("ai-sdk-anthropic-v4", () => {
+ class AnthropicMessagesLanguageModel {
+ specificationVersion = "v3";
+ modelId: string;
+
+ constructor(modelId: string) {
+ this.modelId = modelId;
+ }
+
+ get provider() {
+ return "anthropic.messages";
+ }
+
+ get supportedUrls() {
+ return {};
+ }
+
+ doGenerate(options: unknown) {
+ return bedrockMocks.doGenerate(options);
+ }
+
+ doStream(options: unknown) {
+ return bedrockMocks.doStream(options);
+ }
+ }
+
+ return {
+ createAnthropic: vi.fn(
+ () => (modelId: string) => new AnthropicMessagesLanguageModel(modelId),
+ ),
+ };
+});
+
vi.mock("@aws-sdk/credential-providers", () => ({
fromNodeProviderChain: vi.fn(() => vi.fn()),
}));
@@ -600,6 +645,46 @@ describe("createAgUiStream", () => {
});
});
+ it("uses Anthropic Messages with the namespaced API key and thinking options", async () => {
+ const { createAmazonBedrock } = await import("ai-sdk-amazon-bedrock-v4");
+ const { createAnthropic } = await import("ai-sdk-anthropic-v4");
+
+ await initializeBasicTracedAgent(
+ "run-anthropic-messages",
+ testAnthropicModel("claude-opus-4-8"),
+ );
+
+ expect(createAmazonBedrock).not.toHaveBeenCalled();
+ expect(createAnthropic).toHaveBeenCalledWith({
+ apiKey: "sk-ant-test",
+ baseURL: "https://api.anthropic.com/v1",
+ });
+
+ const { Agent } = await import("@mastra/core/agent");
+ const agentConfig = vi.mocked(Agent).mock.calls.at(-1)?.[0] as
+ | MockedAgentConfig
+ | undefined;
+ expect(agentConfig?.defaultOptions).toMatchObject({
+ providerOptions: {
+ anthropic: {
+ thinking: { type: "adaptive", display: "summarized" },
+ },
+ },
+ });
+ });
+
+ it("keeps Anthropic provider getters on the traced model wrapper", async () => {
+ await initializeBasicTracedAgent(
+ "run-anthropic-provider-getters",
+ testAnthropicModel("claude-opus-4-8"),
+ );
+
+ const model = getLastAgentConfig()?.model;
+ expect(model?.provider).toBe("anthropic.messages");
+ expect(model?.supportedUrls).toEqual({});
+ expect(model?.provider?.includes("anthropic")).toBe(true);
+ });
+
it("forwards model stream parts when finish tracing throws", async () => {
instrumentationMocks.instrumentation.recordModelStreamPart
.mockImplementationOnce(() => undefined)
@@ -696,6 +781,63 @@ describe("createAgUiStream", () => {
).toHaveBeenCalledWith({ prompt: cachedPrompt });
});
+ it("adds Anthropic cacheControl to Claude Messages prompts so later steps can reuse prior turns", async () => {
+ await initializeBasicTracedAgent(
+ "run-anthropic-prompt-cache",
+ testAnthropicModel("claude-opus-4-8"),
+ );
+
+ const model = vi.mocked(Agent).mock.calls.at(-1)?.[0]?.model as unknown as {
+ doStream: (options: unknown) => Promise<{
+ stream: ReadableStream<unknown>;
+ }>;
+ };
+
+ const options = {
+ prompt: [
+ { role: "system", content: "You are the Langfuse assistant." },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ },
+ ],
+ };
+ await model.doStream(options);
+
+ const cachedPrompt = [
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: {
+ anthropic: { cacheControl: { type: "ephemeral" } },
+ },
+ },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ providerOptions: {
+ anthropic: { cacheControl: { type: "ephemeral" } },
+ },
+ },
+ ];
+ expect(bedrockMocks.doStream).toHaveBeenCalledWith({
+ prompt: cachedPrompt,
+ });
+ expect(
+ instrumentationMocks.instrumentation.recordModelCallStart,
+ ).toHaveBeenCalledWith({ prompt: cachedPrompt });
+ });
+
it("appends a trailing current-time message on each model request", async () => {
await initializeBasicTracedAgent("run-current-time");
const processor = getLastAgentConfig()?.inputProcessors?.find(
@@ -984,6 +1126,7 @@ describe("createAgUiStream", () => {
bedrock: {
additionalModelRequestFields: {
thinking: { type: "adaptive", display: "summarized" },
+ output_config: { effort: "medium" },
},
},
},
diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.ts
index 43462b4b3b..172ef74668 100644
--- a/worker/src/features/in-app-agent/runtime/agent.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.ts
@@ -1,7 +1,6 @@
import { EventType } from "@ag-ui/core";
import { MastraAgent } from "@ag-ui/mastra";
import { IN_APP_AGENT_SYSTEM_PROMPT_TEMPLATE } from "@langfuse/shared/in-app-agent/server/systemPrompt";
-import { createAmazonBedrock } from "ai-sdk-amazon-bedrock-v4";
import { Agent } from "@mastra/core/agent";
import type {
ProcessInputStepArgs,
@@ -50,20 +49,21 @@ import {
import { LANGFUSE_IN_APP_AGENT_SKILLS } from "./skills";
import type { InAppAgentSandbox } from "./sandbox";
import { DEFAULT_SIDEBAR_HIDDEN_ENVIRONMENTS } from "@langfuse/shared";
-import {
- createDefaultBedrockProviderAuth,
- logger,
-} from "@langfuse/shared/src/server";
+import { logger } from "@langfuse/shared/src/server";
import {
IN_APP_AGENT_MCP_TOOL_OVERRIDE_HEADER,
IN_APP_AGENT_REDIRECT_TOOL_NAME,
} from "@langfuse/shared/in-app-agent";
import type { InAppAgentModelConfig } from "@langfuse/shared/in-app-agent/server/modelProvider";
-import { applyBedrockPromptCacheToCall } from "./bedrockPromptCache";
+import { applyPromptCacheToCall } from "./promptCache";
+import {
+ createInAppAgentLanguageModel,
+ getInAppAgentReasoningProviderOptions,
+ type InAppAgentLanguageModel,
+} from "./model";
const ASSISTANT_TITLE = "Langfuse Assistant";
const IN_APP_AGENT_SYSTEM_PROMPT_NAME = "in-app-agent-system-prompt";
-const BEDROCK_CLAUDE_MODEL_ID_PART = "anthropic.claude";
const LANGFUSE_DOCS_MCP_URL = "https://langfuse.com/api/mcp";
const IN_APP_AGENT_MCP_USER_AGENT = "langfuse-in-app-agent";
@@ -273,30 +273,6 @@ function isTruncatedByStepLimit(state: StepLimitState): boolean {
);
}
-// Adaptive thinking is the default for every Claude model so new generations
-// work without maintaining a model list. Older models that only support
-// thinking.type.enabled (e.g. haiku 4.5) reject adaptive with a 400 — the
-// in-app agent must run on a model generation that supports it.
-export function getBedrockReasoningProviderOptions(modelId: string) {
- if (!modelId.includes(BEDROCK_CLAUDE_MODEL_ID_PART)) {
- return undefined;
- }
-
- return {
- bedrock: {
- // Passed as raw request fields instead of reasoningConfig because
- // @ai-sdk/amazon-bedrock overwrites additionalModelRequestFields
- // .thinking when reasoningConfig is set, and these models default
- // display to "omitted" (empty thinking text) — without "summarized"
- // the reasoning UI would render blank blocks.
- additionalModelRequestFields: {
- thinking: { type: "adaptive" as const, display: "summarized" },
- output_config: { effort: "medium" as const },
- },
- },
- };
-}
-
type CreateAgUiStreamOptions = {
onEvent?: (event: AgUiEvent) => void | Promise<void>;
onMcpToolCallCompleted?: (toolCall: CompletedInAppAgentMcpToolCall) => void;
@@ -983,10 +959,6 @@ type ExecutableInAppAgentTool = {
toModelOutput?: (output: unknown) => unknown | PromiseLike<unknown>;
};
-type InAppAgentLanguageModel = ReturnType<
- ReturnType<typeof createAmazonBedrock>
->;
-
function withModelTracing(
model: InAppAgentLanguageModel,
callbacks: {
@@ -994,15 +966,31 @@ function withModelTracing(
onStreamPart?: (part: unknown) => void;
},
): InAppAgentLanguageModel {
+ // Copy provider/supportedUrls after the spread: @ai-sdk/anthropic defines
+ // them as prototype getters, and object spread only copies own enumerable
+ // properties. Mastra then does `model.provider.includes(...)` on every turn.
+ // Always wrap so prompt-cache checkpoints are applied even without
+ // tracing callbacks.
return {
+ ...model,
specificationVersion: model.specificationVersion,
provider: model.provider,
modelId: model.modelId,
supportedUrls: model.supportedUrls,
doGenerate: (options) =>
- model.doGenerate(applyBedrockPromptCacheToCall(model.modelId, options)),
+ model.doGenerate(
+ applyPromptCacheToCall({
+ provider: String(model.provider ?? ""),
+ modelId: model.modelId,
+ options,
+ }),
+ ),
doStream: async (options) => {
- const nextOptions = applyBedrockPromptCacheToCall(model.modelId, options);
+ const nextOptions = applyPromptCacheToCall({
+ provider: String(model.provider ?? ""),
+ modelId: model.modelId,
+ options,
+ });
callbacks.onStart?.(nextOptions);
const result = await model.doStream(nextOptions);
@@ -1078,13 +1066,9 @@ async function createMastraAdapter(params: {
onToolExecutionEnd?: (toolCallId: string) => void;
stepLimitState: StepLimitState;
}) {
- const bedrock = createAmazonBedrock({
- ...(params.options.model.region
- ? { region: params.options.model.region }
- : {}),
- ...createDefaultBedrockProviderAuth(
- params.awsProfile ? { profile: params.awsProfile } : undefined,
- ),
+ const languageModel = createInAppAgentLanguageModel({
+ config: params.options.model,
+ awsProfile: params.awsProfile,
});
const mcpClient = new MCPClient({
@@ -1198,8 +1182,8 @@ async function createMastraAdapter(params: {
});
params.onToolsAvailable?.(tools);
- const reasoningProviderOptions = getBedrockReasoningProviderOptions(
- params.options.model.modelId,
+ const reasoningProviderOptions = getInAppAgentReasoningProviderOptions(
+ params.options.model,
);
// @ag-ui/mastra currently forwards only assistant, user, and tool
@@ -1208,13 +1192,10 @@ async function createMastraAdapter(params: {
// instruction channel so the model receives the same higher-priority
// guidance on resumed runs.
let developerGuidance: string | undefined;
- const model = withModelTracing(
- bedrock(params.options.model.modelId as Parameters<typeof bedrock>[0]),
- {
- onStart: params.onModelCallStart,
- onStreamPart: params.onModelStreamPart,
- },
- );
+ const model = withModelTracing(languageModel, {
+ onStart: params.onModelCallStart,
+ onStreamPart: params.onModelStreamPart,
+ });
const agent = new Agent({
id: "langfuse-in-app-assistant",
name: ASSISTANT_TITLE,
diff --git a/worker/src/features/in-app-agent/runtime/bedrockPromptCache.test.ts b/worker/src/features/in-app-agent/runtime/bedrockPromptCache.test.ts
deleted file mode 100644
index a314cf2150..0000000000
--- a/worker/src/features/in-app-agent/runtime/bedrockPromptCache.test.ts
+++ /dev/null
@@ -1,249 +0,0 @@
-import { describe, expect, it } from "vitest";
-
-import {
- applyBedrockPromptCachePoints,
- applyBedrockPromptCacheToCall,
-} from "./bedrockPromptCache";
-
-const cachePoint = {
- bedrock: { cachePoint: { type: "default" } },
-};
-
-describe("applyBedrockPromptCachePoints", () => {
- it("caches the stable system prefix and the growing conversation prefix", () => {
- // Bedrock evaluates checkpoints as tools → system → messages. Tagging the
- // last leading system message writes tools+system; tagging the last
- // message writes that plus prior turns so the next agent step can read it.
- expect(
- applyBedrockPromptCachePoints([
- { role: "system", content: "You are the Langfuse assistant." },
- { role: "system", content: "Skill: error analysis." },
- { role: "user", content: [{ type: "text", text: "hello" }] },
- {
- role: "assistant",
- content: [{ type: "tool-call", toolCallId: "call-1" }],
- },
- {
- role: "tool",
- content: [{ type: "tool-result", toolCallId: "call-1" }],
- },
- ]),
- ).toEqual([
- { role: "system", content: "You are the Langfuse assistant." },
- {
- role: "system",
- content: "Skill: error analysis.",
- providerOptions: cachePoint,
- },
- { role: "user", content: [{ type: "text", text: "hello" }] },
- {
- role: "assistant",
- content: [{ type: "tool-call", toolCallId: "call-1" }],
- },
- {
- role: "tool",
- content: [{ type: "tool-result", toolCallId: "call-1" }],
- providerOptions: cachePoint,
- },
- ]);
- });
-
- it("re-stamps the previous turn's last prefix so a follow-up user message can cache-read it", () => {
- expect(
- applyBedrockPromptCachePoints([
- { role: "system", content: "You are the Langfuse assistant." },
- { role: "user", content: [{ type: "text", text: "hello" }] },
- {
- role: "assistant",
- content: [{ type: "tool-call", toolCallId: "call-1" }],
- },
- {
- role: "tool",
- content: [{ type: "tool-result", toolCallId: "call-1" }],
- },
- {
- role: "assistant",
- content: [{ type: "text", text: "You have 20 prompts." }],
- },
- {
- role: "user",
- content: [{ type: "text", text: "and the versions?" }],
- },
- ]),
- ).toEqual([
- {
- role: "system",
- content: "You are the Langfuse assistant.",
- providerOptions: cachePoint,
- },
- { role: "user", content: [{ type: "text", text: "hello" }] },
- {
- role: "assistant",
- content: [{ type: "tool-call", toolCallId: "call-1" }],
- },
- {
- role: "tool",
- content: [{ type: "tool-result", toolCallId: "call-1" }],
- providerOptions: cachePoint,
- },
- {
- role: "assistant",
- content: [{ type: "text", text: "You have 20 prompts." }],
- },
- {
- role: "user",
- content: [{ type: "text", text: "and the versions?" }],
- providerOptions: cachePoint,
- },
- ]);
- });
-
- it("keeps the previous-turn checkpoint when a trailing current-time suffix is present", () => {
- expect(
- applyBedrockPromptCachePoints([
- { role: "system", content: "You are the Langfuse assistant." },
- { role: "user", content: [{ type: "text", text: "hello" }] },
- {
- role: "assistant",
- content: [{ type: "tool-call", toolCallId: "call-1" }],
- },
- {
- role: "tool",
- content: [{ type: "tool-result", toolCallId: "call-1" }],
- },
- {
- role: "assistant",
- content: [{ type: "text", text: "You have 20 prompts." }],
- },
- {
- role: "user",
- content: [{ type: "text", text: "and the versions?" }],
- },
- {
- role: "user",
- content: [
- {
- type: "text",
- text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',
- },
- ],
- },
- ]),
- ).toEqual([
- {
- role: "system",
- content: "You are the Langfuse assistant.",
- providerOptions: cachePoint,
- },
- { role: "user", content: [{ type: "text", text: "hello" }] },
- {
- role: "assistant",
- content: [{ type: "tool-call", toolCallId: "call-1" }],
- },
- {
- role: "tool",
- content: [{ type: "tool-result", toolCallId: "call-1" }],
- providerOptions: cachePoint,
- },
- {
- role: "assistant",
- content: [{ type: "text", text: "You have 20 prompts." }],
- },
- {
- role: "user",
- content: [{ type: "text", text: "and the versions?" }],
- providerOptions: cachePoint,
- },
- {
- role: "user",
- content: [
- {
- type: "text",
- text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',
- },
- ],
- },
- ]);
- });
-
- it("writes a single checkpoint when the prompt is only system messages", () => {
- expect(
- applyBedrockPromptCachePoints([
- { role: "system", content: "You are the Langfuse assistant." },
- ]),
- ).toEqual([
- {
- role: "system",
- content: "You are the Langfuse assistant.",
- providerOptions: cachePoint,
- },
- ]);
- });
-
- it("keeps existing provider options and does not replace an existing cache point", () => {
- expect(
- applyBedrockPromptCachePoints([
- {
- role: "system",
- content: "You are the Langfuse assistant.",
- providerOptions: { bedrock: { somethingElse: true } },
- },
- {
- role: "user",
- content: [{ type: "text", text: "hello" }],
- providerOptions: {
- bedrock: { cachePoint: { type: "default", ttl: "1h" } },
- },
- },
- ]),
- ).toEqual([
- {
- role: "system",
- content: "You are the Langfuse assistant.",
- providerOptions: {
- bedrock: { somethingElse: true, cachePoint: { type: "default" } },
- },
- },
- {
- role: "user",
- content: [{ type: "text", text: "hello" }],
- providerOptions: {
- bedrock: { cachePoint: { type: "default", ttl: "1h" } },
- },
- },
- ]);
- });
-});
-
-describe("applyBedrockPromptCacheToCall", () => {
- it("adds cache points for Claude models and leaves other models unchanged", () => {
- const options = {
- prompt: [
- { role: "system", content: "You are the Langfuse assistant." },
- { role: "user", content: [{ type: "text", text: "hello" }] },
- ],
- maxOutputTokens: 1024,
- };
-
- expect(
- applyBedrockPromptCacheToCall("meta.llama3-70b-instruct-v1:0", options),
- ).toBe(options);
- expect(
- applyBedrockPromptCacheToCall("eu.anthropic.claude-opus-4-8", options),
- ).toEqual({
- maxOutputTokens: 1024,
- prompt: [
- {
- role: "system",
- content: "You are the Langfuse assistant.",
- providerOptions: cachePoint,
- },
- {
- role: "user",
- content: [{ type: "text", text: "hello" }],
- providerOptions: cachePoint,
- },
- ],
- });
- });
-});
diff --git a/worker/src/features/in-app-agent/runtime/model.ts b/worker/src/features/in-app-agent/runtime/model.ts
new file mode 100644
index 0000000000..21b3f97223
--- /dev/null
+++ b/worker/src/features/in-app-agent/runtime/model.ts
@@ -0,0 +1,82 @@
+import { createAmazonBedrock } from "ai-sdk-amazon-bedrock-v4";
+import { createAnthropic } from "ai-sdk-anthropic-v4";
+
+import type { InAppAgentModelConfig } from "@langfuse/shared/in-app-agent/server/modelProvider";
+import { createDefaultBedrockProviderAuth } from "@langfuse/shared/src/server";
+
+const BEDROCK_CLAUDE_MODEL_ID_PART = "anthropic.claude";
+const ANTHROPIC_CLAUDE_MODEL_ID_PART = "claude";
+
+export type InAppAgentLanguageModel = ReturnType<
+ ReturnType<typeof createAmazonBedrock>
+>;
+
+export function createInAppAgentLanguageModel(params: {
+ config: InAppAgentModelConfig;
+ awsProfile?: string;
+}): InAppAgentLanguageModel {
+ if (params.config.provider === "anthropic") {
+ const anthropic = createAnthropic({
+ apiKey: params.config.apiKey,
+ baseURL: params.config.baseURL,
+ });
+
+ return anthropic(
+ params.config.modelId as Parameters<typeof anthropic>[0],
+ ) as InAppAgentLanguageModel;
+ }
+
+ const bedrock = createAmazonBedrock({
+ ...(params.config.region ? { region: params.config.region } : {}),
+ ...createDefaultBedrockProviderAuth(
+ params.awsProfile ? { profile: params.awsProfile } : undefined,
+ ),
+ });
+
+ return bedrock(params.config.modelId as Parameters<typeof bedrock>[0]);
+}
+
+// Adaptive thinking is the default for every Claude model so new generations
+// work without maintaining a model list. Older models that only support
+// thinking.type.enabled (e.g. haiku 4.5) reject adaptive with a 400 — the
+// in-app agent must run on a model generation that supports it.
+export function getBedrockReasoningProviderOptions(modelId: string) {
+ if (!modelId.includes(BEDROCK_CLAUDE_MODEL_ID_PART)) {
+ return undefined;
+ }
+
+ return {
+ bedrock: {
+ // Passed as raw request fields instead of reasoningConfig because
+ // @ai-sdk/amazon-bedrock overwrites additionalModelRequestFields
+ // .thinking when reasoningConfig is set, and these models default
+ // display to "omitted" (empty thinking text) — without "summarized"
+ // the reasoning UI would render blank blocks.
+ additionalModelRequestFields: {
+ thinking: { type: "adaptive" as const, display: "summarized" },
+ output_config: { effort: "medium" as const },
+ },
+ },
+ };
+}
+
+export function getInAppAgentReasoningProviderOptions(
+ config: InAppAgentModelConfig,
+) {
+ if (config.provider === "anthropic") {
+ if (!config.modelId.includes(ANTHROPIC_CLAUDE_MODEL_ID_PART)) {
+ return undefined;
+ }
+
+ return {
+ anthropic: {
+ thinking: {
+ type: "adaptive" as const,
+ display: "summarized" as const,
+ },
+ },
+ };
+ }
+
+ return getBedrockReasoningProviderOptions(config.modelId);
+}
diff --git a/worker/src/features/in-app-agent/runtime/promptCache.test.ts b/worker/src/features/in-app-agent/runtime/promptCache.test.ts
new file mode 100644
index 0000000000..b173752bca
--- /dev/null
+++ b/worker/src/features/in-app-agent/runtime/promptCache.test.ts
@@ -0,0 +1,340 @@
+import { describe, expect, it } from "vitest";
+
+import { applyPromptCachePoints, applyPromptCacheToCall } from "./promptCache";
+
+const bedrockCachePoint = {
+ bedrock: { cachePoint: { type: "default" } },
+};
+
+const anthropicCacheControl = {
+ anthropic: { cacheControl: { type: "ephemeral" } },
+};
+
+const twoMessagePrompt = [
+ { role: "system", content: "You are the Langfuse assistant." },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+];
+
+describe("applyPromptCachePoints", () => {
+ it("caches the stable system prefix and the growing conversation prefix", () => {
+ // Checkpoints are tools → system → messages. Tagging the last leading
+ // system message writes tools+system; tagging the last message writes
+ // that plus prior turns so the next agent step can read it.
+ expect(
+ applyPromptCachePoints(
+ [
+ { role: "system", content: "You are the Langfuse assistant." },
+ { role: "system", content: "Skill: error analysis." },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ },
+ ],
+ "bedrock",
+ ),
+ ).toEqual([
+ { role: "system", content: "You are the Langfuse assistant." },
+ {
+ role: "system",
+ content: "Skill: error analysis.",
+ providerOptions: bedrockCachePoint,
+ },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ providerOptions: bedrockCachePoint,
+ },
+ ]);
+ });
+
+ it("re-stamps the previous turn's last prefix so a follow-up user message can cache-read it", () => {
+ expect(
+ applyPromptCachePoints(
+ [
+ { role: "system", content: "You are the Langfuse assistant." },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ },
+ {
+ role: "assistant",
+ content: [{ type: "text", text: "You have 20 prompts." }],
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "and the versions?" }],
+ },
+ ],
+ "bedrock",
+ ),
+ ).toEqual([
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: bedrockCachePoint,
+ },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ providerOptions: bedrockCachePoint,
+ },
+ {
+ role: "assistant",
+ content: [{ type: "text", text: "You have 20 prompts." }],
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "and the versions?" }],
+ providerOptions: bedrockCachePoint,
+ },
+ ]);
+ });
+
+ it("keeps the previous-turn checkpoint when a trailing current-time suffix is present", () => {
+ expect(
+ applyPromptCachePoints(
+ [
+ { role: "system", content: "You are the Langfuse assistant." },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ },
+ {
+ role: "assistant",
+ content: [{ type: "text", text: "You have 20 prompts." }],
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "and the versions?" }],
+ },
+ {
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',
+ },
+ ],
+ },
+ ],
+ "bedrock",
+ ),
+ ).toEqual([
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: bedrockCachePoint,
+ },
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ {
+ role: "assistant",
+ content: [{ type: "tool-call", toolCallId: "call-1" }],
+ },
+ {
+ role: "tool",
+ content: [{ type: "tool-result", toolCallId: "call-1" }],
+ providerOptions: bedrockCachePoint,
+ },
+ {
+ role: "assistant",
+ content: [{ type: "text", text: "You have 20 prompts." }],
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "and the versions?" }],
+ providerOptions: bedrockCachePoint,
+ },
+ {
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: '<current_time tz="Europe/London">2026-08-24 08:53</current_time>',
+ },
+ ],
+ },
+ ]);
+ });
+
+ it("writes a single checkpoint when the prompt is only system messages", () => {
+ expect(
+ applyPromptCachePoints(
+ [{ role: "system", content: "You are the Langfuse assistant." }],
+ "bedrock",
+ ),
+ ).toEqual([
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: bedrockCachePoint,
+ },
+ ]);
+ });
+
+ it("keeps existing provider options and does not replace an existing cache stamp", () => {
+ expect(
+ applyPromptCachePoints(
+ [
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: { bedrock: { somethingElse: true } },
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "hello" }],
+ providerOptions: {
+ bedrock: { cachePoint: { type: "default", ttl: "1h" } },
+ },
+ },
+ ],
+ "bedrock",
+ ),
+ ).toEqual([
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: {
+ bedrock: { somethingElse: true, cachePoint: { type: "default" } },
+ },
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "hello" }],
+ providerOptions: {
+ bedrock: { cachePoint: { type: "default", ttl: "1h" } },
+ },
+ },
+ ]);
+
+ expect(
+ applyPromptCachePoints(
+ [
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: { anthropic: { somethingElse: true } },
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "hello" }],
+ providerOptions: {
+ anthropic: { cacheControl: { type: "ephemeral", ttl: "1h" } },
+ },
+ },
+ ],
+ "anthropic",
+ ),
+ ).toEqual([
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: {
+ anthropic: {
+ somethingElse: true,
+ cacheControl: { type: "ephemeral" },
+ },
+ },
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "hello" }],
+ providerOptions: {
+ anthropic: { cacheControl: { type: "ephemeral", ttl: "1h" } },
+ },
+ },
+ ]);
+ });
+});
+
+describe("applyPromptCacheToCall", () => {
+ it("stamps Anthropic cacheControl for native Claude Messages", () => {
+ const options = {
+ prompt: twoMessagePrompt,
+ maxOutputTokens: 1024,
+ };
+
+ expect(
+ applyPromptCacheToCall({
+ provider: "anthropic.messages",
+ modelId: "claude-opus-4-8",
+ options,
+ }),
+ ).toEqual({
+ maxOutputTokens: 1024,
+ prompt: [
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: anthropicCacheControl,
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "hello" }],
+ providerOptions: anthropicCacheControl,
+ },
+ ],
+ });
+ });
+
+ it("stamps Bedrock cache points for Claude models and leaves other models unchanged", () => {
+ const options = {
+ prompt: twoMessagePrompt,
+ maxOutputTokens: 1024,
+ };
+
+ expect(
+ applyPromptCacheToCall({
+ provider: "amazon-bedrock",
+ modelId: "meta.llama3-70b-instruct-v1:0",
+ options,
+ }),
+ ).toBe(options);
+ expect(
+ applyPromptCacheToCall({
+ provider: "amazon-bedrock",
+ modelId: "eu.anthropic.claude-opus-4-8",
+ options,
+ }),
+ ).toEqual({
+ maxOutputTokens: 1024,
+ prompt: [
+ {
+ role: "system",
+ content: "You are the Langfuse assistant.",
+ providerOptions: bedrockCachePoint,
+ },
+ {
+ role: "user",
+ content: [{ type: "text", text: "hello" }],
+ providerOptions: bedrockCachePoint,
+ },
+ ],
+ });
+ });
+});
diff --git a/worker/src/features/in-app-agent/runtime/bedrockPromptCache.ts b/worker/src/features/in-app-agent/runtime/promptCache.ts
similarity index 64%
rename from worker/src/features/in-app-agent/runtime/bedrockPromptCache.ts
rename to worker/src/features/in-app-agent/runtime/promptCache.ts
index 919578a651..157a1532c8 100644
--- a/worker/src/features/in-app-agent/runtime/bedrockPromptCache.ts
+++ b/worker/src/features/in-app-agent/runtime/promptCache.ts
@@ -1,16 +1,22 @@
import { isRecord } from "@langfuse/shared/in-app-agent/server/toolErrors";
+export type PromptCacheProvider = "bedrock" | "anthropic";
+
const BEDROCK_CLAUDE_MODEL_ID_PART = "anthropic.claude";
+const ANTHROPIC_CLAUDE_MODEL_ID_PART = "claude";
const BEDROCK_PROMPT_CACHE_POINT = { type: "default" as const };
+const ANTHROPIC_CACHE_CONTROL = { type: "ephemeral" as const };
/**
- * Bedrock Converse prompt cache.
+ * Prompt cache for Claude on Bedrock Converse and native Anthropic Messages.
*
- * A cachePoint writes the prefix `tools → that message`. The next call
+ * A checkpoint writes the prefix `tools → that message`. The next call
* cache-reads only if that prefix is byte-identical. Hits last 5 minutes
- * and refresh on read. Claude-only: Bedrock Converse ignores Anthropic
- * `cacheControl`, so we do not dual-write it.
+ * and refresh on read.
+ *
+ * Stamp only the matching provider field. Bedrock Converse ignores Anthropic
+ * `cacheControl`; Anthropic Messages ignores Bedrock `cachePoint`.
*
* Three checkpoints, each for a different prefix:
*
@@ -26,25 +32,34 @@ const BEDROCK_PROMPT_CACHE_POINT = { type: "default" as const };
* not the closing assistant (that was model output). Skip trailing
* assistants, then stamp that predecessor.
*/
-export function applyBedrockPromptCacheToCall<T>(
- modelId: string,
- options: T,
-): T {
+export function applyPromptCacheToCall<T>({
+ provider,
+ modelId,
+ options,
+}: {
+ provider: string;
+ modelId: string;
+ options: T;
+}): T {
+ const cacheProvider = resolvePromptCacheProvider(provider, modelId);
if (
+ cacheProvider == null ||
!isRecord(options) ||
- !Array.isArray(options.prompt) ||
- !shouldApplyBedrockPromptCache(modelId)
+ !Array.isArray(options.prompt)
) {
return options;
}
return {
...options,
- prompt: applyBedrockPromptCachePoints(options.prompt),
+ prompt: applyPromptCachePoints(options.prompt, cacheProvider),
};
}
-export function applyBedrockPromptCachePoints(prompt: unknown) {
+export function applyPromptCachePoints(
+ prompt: unknown,
+ cacheProvider: PromptCacheProvider,
+) {
if (!Array.isArray(prompt) || prompt.length === 0) {
return prompt;
}
@@ -76,12 +91,27 @@ export function applyBedrockPromptCachePoints(prompt: unknown) {
}
return prompt.map((message, index) =>
- cacheIndices.has(index) ? withBedrockCachePoint(message) : message,
+ cacheIndices.has(index) ? withPromptCache(message, cacheProvider) : message,
);
}
-function shouldApplyBedrockPromptCache(modelId: string) {
- return modelId.includes(BEDROCK_CLAUDE_MODEL_ID_PART);
+function resolvePromptCacheProvider(
+ provider: string,
+ modelId: string,
+): PromptCacheProvider | undefined {
+ if (
+ provider.includes("bedrock") &&
+ modelId.includes(BEDROCK_CLAUDE_MODEL_ID_PART)
+ ) {
+ return "bedrock";
+ }
+ if (
+ provider.includes("anthropic") &&
+ modelId.includes(ANTHROPIC_CLAUDE_MODEL_ID_PART)
+ ) {
+ return "anthropic";
+ }
+ return undefined;
}
function getLastConversationIndex(prompt: unknown[]) {
@@ -146,6 +176,12 @@ function getMessageRole(message: unknown) {
: undefined;
}
+function withPromptCache(message: unknown, cacheProvider: PromptCacheProvider) {
+ return cacheProvider === "bedrock"
+ ? withBedrockCachePoint(message)
+ : withAnthropicCacheControl(message);
+}
+
function withBedrockCachePoint(message: unknown) {
if (!isRecord(message)) {
return message;
@@ -173,3 +209,31 @@ function withBedrockCachePoint(message: unknown) {
},
};
}
+
+function withAnthropicCacheControl(message: unknown) {
+ if (!isRecord(message)) {
+ return message;
+ }
+
+ const providerOptions = isRecord(message.providerOptions)
+ ? message.providerOptions
+ : {};
+ const anthropic = isRecord(providerOptions.anthropic)
+ ? providerOptions.anthropic
+ : {};
+
+ if (isRecord(anthropic.cacheControl)) {
+ return message;
+ }
+
+ return {
+ ...message,
+ providerOptions: {
+ ...providerOptions,
+ anthropic: {
+ ...anthropic,
+ cacheControl: ANTHROPIC_CACHE_CONTROL,
+ },
+ },
+ };
+}
diff --git a/worker/src/features/in-app-agent/runtime/reasoning.test.ts b/worker/src/features/in-app-agent/runtime/reasoning.test.ts
index b55a331647..b7ca77b5be 100644
--- a/worker/src/features/in-app-agent/runtime/reasoning.test.ts
+++ b/worker/src/features/in-app-agent/runtime/reasoning.test.ts
@@ -1,5 +1,10 @@
-import { getBedrockReasoningProviderOptions } from "./agent";
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import {
+ createInAppAgentLanguageModel,
+ getBedrockReasoningProviderOptions,
+ getInAppAgentReasoningProviderOptions,
+} from "./model";
describe("getBedrockReasoningProviderOptions", () => {
it("sends adaptive thinking with medium effort and summarized display to Claude models", () => {
@@ -33,3 +38,96 @@ describe("getBedrockReasoningProviderOptions", () => {
).toBeUndefined();
});
});
+
+describe("getInAppAgentReasoningProviderOptions", () => {
+ it("sends Anthropic adaptive thinking for Claude model ids", () => {
+ expect(
+ getInAppAgentReasoningProviderOptions({
+ provider: "anthropic",
+ modelId: "claude-opus-4-8",
+ titleModelId: "claude-haiku-4-5",
+ apiKey: "sk-ant-test",
+ }),
+ ).toEqual({
+ anthropic: {
+ thinking: { type: "adaptive", display: "summarized" },
+ },
+ });
+ });
+
+ it("sends no thinking config for non-Claude Anthropic model ids", () => {
+ expect(
+ getInAppAgentReasoningProviderOptions({
+ provider: "anthropic",
+ modelId: "some-other-model",
+ titleModelId: "some-other-model",
+ apiKey: "sk-ant-test",
+ }),
+ ).toBeUndefined();
+ });
+});
+
+describe("Anthropic Messages request shape", () => {
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it("puts adaptive summarized thinking on the Messages request body", async () => {
+ // Older @ai-sdk/anthropic pins accepted thinking.type=adaptive but
+ // stripped display from the wire body, so Opus omitted summaries and
+ // the UI dropped empty completed thinking blocks. Capture the real
+ // SDK request instead of asserting the helper object we pass in.
+ const config = {
+ provider: "anthropic" as const,
+ modelId: "claude-opus-4-8",
+ titleModelId: "claude-haiku-4-5",
+ apiKey: "sk-ant-test",
+ baseURL: "https://anthropic.test/v1",
+ };
+ const { calls, fetch } = createCaptureFetch({
+ id: "msg_1",
+ type: "message",
+ role: "assistant",
+ model: config.modelId,
+ content: [{ type: "text", text: "ok" }],
+ stop_reason: "end_turn",
+ stop_sequence: null,
+ usage: { input_tokens: 1, output_tokens: 1 },
+ });
+ vi.stubGlobal("fetch", fetch);
+
+ const model = createInAppAgentLanguageModel({ config });
+ await model.doGenerate({
+ prompt: [
+ {
+ role: "user",
+ content: [{ type: "text", text: "hi" }],
+ },
+ ],
+ providerOptions: getInAppAgentReasoningProviderOptions(config),
+ });
+
+ expect(calls).toHaveLength(1);
+ expect(calls[0]?.url).toBe("https://anthropic.test/v1/messages");
+ expect(calls[0]?.body.thinking).toEqual({
+ type: "adaptive",
+ display: "summarized",
+ });
+ });
+});
+
+function createCaptureFetch(response: unknown) {
+ const calls: Array<{ url: string; body: Record<string, unknown> }> = [];
+ const fetchImpl: typeof fetch = async (input, init) => {
+ const request = new Request(input, init);
+ calls.push({
+ url: request.url,
+ body: JSON.parse(await request.text()) as Record<string, unknown>,
+ });
+ return new Response(JSON.stringify(response), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ });
+ };
+ return { calls, fetch: fetchImpl };
+}