Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(llm-connections): custom base URL and extra headers for TypeSafe connections (#17866)

Hassieb Pakzadd7ae7f0c776a
diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.ts
index 7be561caad..15d5aaab69 100644
--- a/packages/shared/src/server/llm/types.ts
+++ b/packages/shared/src/server/llm/types.ts
@@ -477,7 +477,9 @@ export const typeSafeModels = ["jev-1.13.0", "jev-latest"] as const;
/**
* Providers that serve Jev through TypeSafe's `/v1/systemone` API. A TypeSafe
* connection stores a gateway's `baseURL`, or none for TypeSafe itself, which
- * the AI SDK provider then defaults to.
+ * the AI SDK provider then defaults to. Presets only prefill the base URL; a
+ * `custom` connection stores any base URL the provider appends `/systemone`
+ * to.
*/
export const TYPESAFE_UPSTREAMS = [
{
@@ -498,15 +500,24 @@ export const TYPESAFE_UPSTREAMS = [
baseURL: "https://openrouter.ai/api/v1",
apiKeyLabel: "OpenRouter API key",
},
+ {
+ id: "custom",
+ label: "Custom",
+ baseURL: null,
+ apiKeyLabel: "API key",
+ },
] as const;
export type TypeSafeUpstream = (typeof TYPESAFE_UPSTREAMS)[number];
-export function findTypeSafeUpstream(
+export function resolveTypeSafeUpstream(
baseURL: string | null | undefined,
-): TypeSafeUpstream | undefined {
+): TypeSafeUpstream {
if (!baseURL) return TYPESAFE_UPSTREAMS[0];
- return TYPESAFE_UPSTREAMS.find((upstream) => upstream.baseURL === baseURL);
+ return (
+ TYPESAFE_UPSTREAMS.find((upstream) => upstream.baseURL === baseURL) ??
+ TYPESAFE_UPSTREAMS[TYPESAFE_UPSTREAMS.length - 1]
+ );
}
export type AnthropicModel = (typeof anthropicModels)[number];
diff --git a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts
index dc31e8a8b2..a68682db9f 100644
--- a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts
+++ b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts
@@ -1,8 +1,17 @@
import { describe, expect, it, vi } from "vitest";
import type { DecisionModelRequest } from "../../evals/decisionModelEvaluatorExecution";
+import { createSecureLlmFetch } from "../secureLlmFetch";
import { TYPESAFE_UPSTREAMS } from "../types";
import { createTypeSafeDecisionModelClient } from "./typeSafeDecisionModelClient";
+vi.mock("../secureLlmFetch", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("../secureLlmFetch")>();
+ return {
+ ...actual,
+ createSecureLlmFetch: vi.fn(actual.createSecureLlmFetch),
+ };
+});
+
function jsonResponse(body: unknown, status = 200) {
return new Response(JSON.stringify(body), {
status,
@@ -134,6 +143,63 @@ describe("createTypeSafeDecisionModelClient", () => {
},
);
+ it("posts to a custom base URL with the connection's extra headers", async () => {
+ const fetchImpl = vi.fn().mockResolvedValue(
+ jsonResponse({
+ model: "jev",
+ answers: { refund: { type: "noul", noul: 0.5 } },
+ }),
+ );
+
+ const client = createTypeSafeDecisionModelClient({
+ apiKey: "sk-test",
+ model: "jev-latest",
+ baseURL: "https://llm-proxy.example.com/typesafe/v1/",
+ extraHeaders: { "x-team": "evals" },
+ fetchImpl,
+ });
+ await client.evaluate({
+ state: request.state,
+ questions: { refund: request.questions.refund },
+ });
+
+ const [url, init] = fetchImpl.mock.calls[0] as [string, RequestInit];
+ expect(url).toBe("https://llm-proxy.example.com/typesafe/v1/systemone");
+ const headers = new Headers(init.headers);
+ expect(headers.get("x-team")).toBe("evals");
+ expect(headers.get("authorization")).toBe("Bearer sk-test");
+ });
+
+ it("strips the connection's extra headers on cross-origin redirects", () => {
+ createTypeSafeDecisionModelClient({
+ apiKey: "sk-test",
+ model: "jev-latest",
+ baseURL: "https://llm-proxy.example.com/typesafe/v1",
+ extraHeaders: { "X-Api-Key": "proxy-secret", "x-team": "evals" },
+ });
+
+ expect(vi.mocked(createSecureLlmFetch)).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ additionalSensitiveHeaders: ["X-Api-Key", "x-team"],
+ }),
+ );
+ });
+
+ it("blocks a custom base URL that points at an internal address", async () => {
+ const client = createTypeSafeDecisionModelClient({
+ apiKey: "sk-test",
+ model: "jev-latest",
+ baseURL: "http://169.254.169.254/v1",
+ });
+
+ await expect(
+ client.evaluate({
+ state: request.state,
+ questions: { refund: request.questions.refund },
+ }),
+ ).rejects.toMatchObject({ name: "LLMValidationError" });
+ });
+
it("tolerates the extra routing fields gateways add to the TypeSafe response", async () => {
// OpenRouter returns its own id/provider and a cost inside usage; the model
// is the resolved OpenRouter slug rather than the requested alias.
diff --git a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts
index a31828140a..da80d48d01 100644
--- a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts
+++ b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts
@@ -15,14 +15,19 @@ export function createTypeSafeDecisionModelClient(params: {
apiKey: string;
model: string;
baseURL?: string | null;
+ extraHeaders?: Record<string, string>;
fetchImpl?: typeof fetch;
}): DecisionModelClient {
const provider = createTypeSafeAi({
apiKey: params.apiKey,
baseURL: params.baseURL ?? undefined,
+ headers: params.extraHeaders,
fetch:
params.fetchImpl ??
- createSecureLlmFetch({ logContext: "TypeSafe decision model" }),
+ createSecureLlmFetch({
+ logContext: "TypeSafe decision model",
+ additionalSensitiveHeaders: Object.keys(params.extraHeaders ?? {}),
+ }),
});
const model = provider.evaluationModel(params.model);
diff --git a/web/src/__tests__/server/llm-api-key.servertest.ts b/web/src/__tests__/server/llm-api-key.servertest.ts
index 15546a28f0..2ac2778a23 100644
--- a/web/src/__tests__/server/llm-api-key.servertest.ts
+++ b/web/src/__tests__/server/llm-api-key.servertest.ts
@@ -355,28 +355,53 @@ describe("llmApiKey.all RPC", () => {
expect(llmApiKeys[0].baseURL).toBe("https://openrouter.ai/api/v1");
});
- it.each(["https://example.com/v1", "https://api.typesafe.ai/v1"])(
- "should reject decision-model connections with the non-preset base URL %s",
- async (baseURL) => {
- await expect(
- caller.llmApiKey.create({
- projectId,
- secretKey: "sk-test",
- provider: "jev-custom-url",
- adapter: LLMAdapter.TypeSafe,
- baseURL,
- }),
- ).rejects.toThrow(
- "only support the TypeSafe, Vercel AI Gateway, and OpenRouter base URLs",
- );
+ it("should store a custom base URL and encrypted extra headers on a decision-model connection", async () => {
+ await caller.llmApiKey.create({
+ projectId,
+ secretKey: "sk-proxy",
+ provider: "jev-via-proxy",
+ adapter: LLMAdapter.TypeSafe,
+ baseURL: "https://example.com/typesafe/v1",
+ extraHeaders: { "x-team": "evals" },
+ });
+
+ const storedKey = await prisma.llmApiKeys.findFirstOrThrow({
+ where: { projectId, provider: "jev-via-proxy" },
+ });
+ expect(storedKey.baseURL).toBe("https://example.com/typesafe/v1");
+ expect(storedKey.extraHeaderKeys).toEqual(["x-team"]);
+ expect(JSON.parse(decrypt(storedKey.extraHeaders!))).toEqual({
+ "x-team": "evals",
+ });
+ });
- const { data: llmApiKeys } = await caller.llmApiKey.all({
+ it("should block decision-model connections with an internal custom base URL", async () => {
+ await expect(
+ caller.llmApiKey.create({
projectId,
- includeDecisionModels: true,
- });
- expect(llmApiKeys).toHaveLength(0);
- },
- );
+ secretKey: "sk-test",
+ provider: "jev-internal",
+ adapter: LLMAdapter.TypeSafe,
+ baseURL: "http://localhost:8080/v1",
+ }),
+ ).rejects.toThrow("Invalid base URL: Blocked hostname detected");
+
+ await expect(
+ caller.llmApiKey.test({
+ projectId,
+ secretKey: "sk-test",
+ provider: "jev-internal",
+ adapter: LLMAdapter.TypeSafe,
+ baseURL: "http://localhost:8080/v1",
+ }),
+ ).resolves.toMatchObject({ success: false });
+
+ const { data: llmApiKeys } = await caller.llmApiKey.all({
+ projectId,
+ includeDecisionModels: true,
+ });
+ expect(llmApiKeys).toHaveLength(0);
+ });
it("should require a new secret key when moving a decision-model connection to another upstream", async () => {
await caller.llmApiKey.create({
@@ -656,6 +681,67 @@ describe("llmApiKey.all RPC", () => {
);
});
+ it("should fill masked header values from the stored headers in testUpdate", async () => {
+ await caller.llmApiKey.create({
+ projectId,
+ provider: "openai",
+ adapter: LLMAdapter.OpenAI,
+ secretKey: "sk-original",
+ baseURL: "https://api.openai.com/v1",
+ extraHeaders: { "X-Api-Key": "stored-token", "X-Team": "stored-team" },
+ });
+ const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+ where: { projectId, provider: "openai" },
+ });
+
+ const result = await caller.llmApiKey.testUpdate({
+ id: existingKey.id,
+ projectId,
+ provider: "openai",
+ adapter: LLMAdapter.OpenAI,
+ baseURL: "https://api.openai.com/v1",
+ extraHeaders: { "X-Api-Key": "", "X-Team": "new-team" },
+ });
+
+ expect(result).toEqual({ success: true });
+ const connection = mockGenerateLLMText.mock.calls[0][0].connection;
+ expect(JSON.parse(decrypt(connection.extraHeaders!))).toEqual({
+ "X-Api-Key": "stored-token",
+ "X-Team": "new-team",
+ });
+ });
+
+ it("should not fill masked header values from the stored headers when testUpdate changes the base URL", async () => {
+ await caller.llmApiKey.create({
+ projectId,
+ provider: "openai",
+ adapter: LLMAdapter.OpenAI,
+ secretKey: "sk-original",
+ baseURL: "https://api.openai.com/v1",
+ extraHeaders: { "X-Api-Key": "stored-token" },
+ });
+ const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+ where: { projectId, provider: "openai" },
+ });
+
+ const result = await caller.llmApiKey.testUpdate({
+ id: existingKey.id,
+ projectId,
+ provider: "openai",
+ adapter: LLMAdapter.OpenAI,
+ secretKey: "sk-rotated",
+ baseURL: "https://example.net/v1",
+ extraHeaders: { "X-Api-Key": "", "X-Team": "new-team" },
+ });
+
+ expect(result).toEqual({ success: true });
+ const connection = mockGenerateLLMText.mock.calls[0][0].connection;
+ expect(connection.baseURL).toBe("https://example.net/v1");
+ expect(JSON.parse(decrypt(connection.extraHeaders!))).toEqual({
+ "X-Team": "new-team",
+ });
+ });
+
it("should allow testUpdate when the base URL changes and a new secret key is provided", async () => {
const existingExtraHeaders = {
Authorization: "Bearer stored-token",
diff --git a/web/src/components/design-system/PasswordInput/PasswordInput.stories.tsx b/web/src/components/design-system/PasswordInput/PasswordInput.stories.tsx
index 2329e6724d..2b66a8b591 100644
--- a/web/src/components/design-system/PasswordInput/PasswordInput.stories.tsx
+++ b/web/src/components/design-system/PasswordInput/PasswordInput.stories.tsx
@@ -43,6 +43,26 @@ export const TestTogglesVisibility = meta.story({
},
});
+export const TestPasswordManagerOptIn = meta.story({
+ name: "(Test) Password Manager Opt-In",
+ render: () => (
+ <>
+ <PasswordInput aria-label="API key" />
+ <PasswordInput aria-label="Account password" allowPasswordManager />
+ </>
+ ),
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+
+ await expect(canvas.getByLabelText("API key")).toHaveAttribute(
+ "data-1p-ignore",
+ );
+ await expect(canvas.getByLabelText("Account password")).not.toHaveAttribute(
+ "data-1p-ignore",
+ );
+ },
+});
+
export const TestSkipsVisibilityToggleWhenTabbing = meta.story({
name: "(Test) Skips Visibility Toggle When Tabbing",
args: {
diff --git a/web/src/components/design-system/PasswordInput/PasswordInput.tsx b/web/src/components/design-system/PasswordInput/PasswordInput.tsx
index 6711626161..b80017955c 100644
--- a/web/src/components/design-system/PasswordInput/PasswordInput.tsx
+++ b/web/src/components/design-system/PasswordInput/PasswordInput.tsx
@@ -23,11 +23,13 @@ type PasswordInputProps = Pick<
| "tabIndex"
| "value"
> & {
+ allowPasswordManager?: boolean;
error?: boolean;
ref?: Ref<HTMLInputElement>;
};
export function PasswordInput({
+ allowPasswordManager,
ref,
disabled,
error,
@@ -50,6 +52,7 @@ export function PasswordInput({
>
<input
{...props}
+ {...(!allowPasswordManager && { "data-1p-ignore": true })}
ref={ref}
type={isPasswordVisible ? "text" : "password"}
disabled={disabled}
diff --git a/web/src/features/auth-credentials/components/ResetPasswordPage.tsx b/web/src/features/auth-credentials/components/ResetPasswordPage.tsx
index fea14c21d1..40d1cd9b3c 100644
--- a/web/src/features/auth-credentials/components/ResetPasswordPage.tsx
+++ b/web/src/features/auth-credentials/components/ResetPasswordPage.tsx
@@ -253,6 +253,7 @@ export function ResetPasswordPage({
</FormLabel>
<FormControl>
<PasswordInput
+ allowPasswordManager
autoComplete="new-password"
{...field}
/>
@@ -273,6 +274,7 @@ export function ResetPasswordPage({
</FormLabel>
<FormControl>
<PasswordInput
+ allowPasswordManager
autoComplete="new-password"
{...field}
/>
diff --git a/web/src/features/auth/SignInPage.tsx b/web/src/features/auth/SignInPage.tsx
index 7df804fc44..adbd1e6771 100644
--- a/web/src/features/auth/SignInPage.tsx
+++ b/web/src/features/auth/SignInPage.tsx
@@ -864,7 +864,7 @@ export default function SignInPage({
</Link>
</FormLabel>
<FormControl>
- <PasswordInput {...field} />
+ <PasswordInput allowPasswordManager {...field} />
</FormControl>
<FormMessage />
</FormItem>
diff --git a/web/src/features/auth/SignUpPage.tsx b/web/src/features/auth/SignUpPage.tsx
index 457c9e9fd2..f70b1e0341 100644
--- a/web/src/features/auth/SignUpPage.tsx
+++ b/web/src/features/auth/SignUpPage.tsx
@@ -288,7 +288,7 @@ function StandardSignupFlow({
<FormItem>
<FormLabel>Password</FormLabel>
<FormControl>
- <PasswordInput {...field} />
+ <PasswordInput allowPasswordManager {...field} />
</FormControl>
<FormMessage />
</FormItem>
diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
index 2897828d4c..7c5b4846ff 100644
--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
@@ -8,6 +8,7 @@ import {
compileLangfuseMediaMessages,
createTypeSafeDecisionModelClient,
createW3CTraceId,
+ decryptAndParseExtraHeaders,
DefaultEvalModelService,
createLLMOutput,
executeDecisionModelEvaluator,
@@ -147,6 +148,9 @@ async function testDecisionModelEvaluator(params: {
apiKey: decrypt(modelConfig.config.apiKey.secretKey),
model: modelConfig.config.model,
baseURL: modelConfig.config.apiKey.baseURL,
+ extraHeaders: decryptAndParseExtraHeaders(
+ modelConfig.config.apiKey.extraHeaders,
+ ),
});
const execution = await executeDecisionModelEvaluator({
variables: params.variables,
diff --git a/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.stories.tsx b/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.stories.tsx
index 0b5350ba88..1fd5f7c6c7 100644
--- a/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.stories.tsx
+++ b/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.stories.tsx
@@ -38,6 +38,14 @@ export const OpenRouterSelected = meta.story({
},
});
+export const CustomSelected = meta.story({
+ args: {
+ "aria-label": "Upstream",
+ value: "custom",
+ onValueChange: fn(),
+ },
+});
+
export const Disabled = meta.story({
args: {
"aria-label": "Upstream",
diff --git a/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.tsx b/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.tsx
index 07eb687290..29096c5b42 100644
--- a/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.tsx
+++ b/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.tsx
@@ -11,6 +11,7 @@ const UPSTREAM_DESCRIPTIONS: Record<TypeSafeUpstreamId, string> = {
typesafe: "Direct connection, billed by TypeSafe.",
"vercel-ai-gateway": "Routed and billed through your AI Gateway.",
openrouter: "Routed and billed through OpenRouter.",
+ custom: "Your own TypeSafe-compatible gateway.",
};
type TypeSafeUpstreamCardsProps = {
@@ -35,7 +36,7 @@ export function TypeSafeUpstreamCards({
{...rootProps}
value={value}
onValueChange={(next) => onValueChange(next as TypeSafeUpstreamId)}
- className="grid gap-2 sm:grid-cols-3"
+ className="grid gap-2 sm:grid-cols-4"
>
{TYPESAFE_UPSTREAMS.map(({ id, label }) => {
return (
diff --git a/web/src/features/llm-api-key/server/router.ts b/web/src/features/llm-api-key/server/router.ts
index 1e837acbb7..77dc83aa95 100644
--- a/web/src/features/llm-api-key/server/router.ts
+++ b/web/src/features/llm-api-key/server/router.ts
@@ -26,7 +26,6 @@ import {
BEDROCK_USE_DEFAULT_CREDENTIALS,
VERTEXAI_USE_DEFAULT_CREDENTIALS,
EvaluatorBlockReason,
- findTypeSafeUpstream,
type LLMConnectionConfig,
} from "@langfuse/shared";
@@ -108,37 +107,18 @@ type TestLLMConnectionParams = {
config?: unknown;
};
-function assertDecisionModelConnectionInput(input: {
- adapter: LLMAdapter;
- baseURL?: string | null;
- extraHeaders?: Record<string, string | null | undefined> | null;
-}) {
- if (!isDecisionModelAdapter(input.adapter)) return;
- if (!findTypeSafeUpstream(input.baseURL)) {
- throw new TRPCError({
- code: "BAD_REQUEST",
- message:
- "Decision-model connections only support the TypeSafe, Vercel AI Gateway, and OpenRouter base URLs.",
- });
- }
- if (input.extraHeaders && Object.keys(input.extraHeaders).length > 0) {
- throw new TRPCError({
- code: "BAD_REQUEST",
- message: "Decision-model connections do not support extra headers.",
- });
- }
-}
-
async function testDecisionModelConnection(params: {
secretKey: string;
model: string;
baseURL?: string | null;
+ extraHeaders?: Record<string, string>;
}): Promise<{ success: boolean; error?: string }> {
try {
const client = createTypeSafeDecisionModelClient({
apiKey: params.secretKey,
model: params.model,
baseURL: params.baseURL,
+ extraHeaders: params.extraHeaders,
});
await client.evaluate({
state: { message: "Hello, is anyone there?" },
@@ -175,6 +155,7 @@ async function testLLMConnection(
secretKey: params.secretKey,
model,
baseURL: params.baseURL,
+ extraHeaders: params.extraHeaders,
});
}
@@ -264,6 +245,35 @@ async function validateBaseURLForWrite(params: {
}
}
+function resolveUpdatedExtraHeaders(params: {
+ inputHeaders: Record<string, string | null | undefined> | undefined;
+ storedHeaders: string | null;
+ isBaseURLChanged: boolean;
+}): Record<string, string> | undefined {
+ const existingHeaders: Record<string, string> = params.isBaseURLChanged
+ ? {}
+ : (decryptAndParseExtraHeaders(params.storedHeaders) ?? {});
+
+ if (params.inputHeaders === undefined) {
+ return Object.keys(existingHeaders).length > 0
+ ? existingHeaders
+ : undefined;
+ }
+
+ const extraHeaders: Record<string, string> = {};
+ for (const [key, value] of Object.entries(params.inputHeaders)) {
+ if (value === null || value === undefined || value === "") {
+ if (existingHeaders[key] !== undefined) {
+ extraHeaders[key] = existingHeaders[key];
+ }
+ } else {
+ extraHeaders[key] = value;
+ }
+ }
+
+ return Object.keys(extraHeaders).length > 0 ? extraHeaders : undefined;
+}
+
export const llmApiKeyRouter = createTRPCRouter({
create: protectedProjectProcedureWithoutTracing
.input(CreateLlmApiKey)
@@ -275,8 +285,6 @@ export const llmApiKeyRouter = createTRPCRouter({
scope: "llmApiKeys:create",
});
- assertDecisionModelConnectionInput(input);
-
await validateBaseURLForWrite({
baseURL: input.baseURL,
});
@@ -511,8 +519,6 @@ export const llmApiKeyRouter = createTRPCRouter({
scope: "llmApiKeys:create",
});
- assertDecisionModelConnectionInput(input);
-
if (input.baseURL) {
try {
await validateLlmConnectionBaseURL(input.baseURL);
@@ -560,8 +566,6 @@ export const llmApiKeyRouter = createTRPCRouter({
});
}
- assertDecisionModelConnectionInput(input);
-
const hasNewSecretKey =
typeof input.secretKey === "string" && input.secretKey.length > 0;
const baseURL =
@@ -589,15 +593,11 @@ export const llmApiKeyRouter = createTRPCRouter({
const customModels = input.customModels ?? existingKey.customModels;
const config = input.config ?? existingKey.config;
- // Never reuse stored headers across a destination change.
- const extraHeaders =
- input.extraHeaders !== undefined
- ? input.extraHeaders
- : isBaseURLChanged
- ? undefined
- : existingKey.extraHeaders
- ? decryptAndParseExtraHeaders(existingKey.extraHeaders)
- : undefined;
+ const extraHeaders = resolveUpdatedExtraHeaders({
+ inputHeaders: input.extraHeaders,
+ storedHeaders: existingKey.extraHeaders,
+ isBaseURLChanged,
+ });
return testLLMConnection({
adapter,
@@ -654,8 +654,6 @@ export const llmApiKeyRouter = createTRPCRouter({
});
}
- assertDecisionModelConnectionInput(input);
-
// Validate that default credentials sentinel is only allowed for Bedrock/VertexAI in self-hosted deployments
const isLangfuseCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
const isBaseURLChanged =
@@ -713,44 +711,11 @@ export const llmApiKeyRouter = createTRPCRouter({
input.extraHeaders = {};
}
- // Get existing decrypted headers for comparison
- const decryptedHeaders = existingKey.extraHeaders
- ? decryptAndParseExtraHeaders(existingKey.extraHeaders)
- : null;
- const existingHeaders: Record<string, string> = isBaseURLChanged
- ? {}
- : (decryptedHeaders ?? {});
-
- // Ensure we only update the extraHeaders where the value is not null
- let extraHeaders: Record<string, string> | undefined;
-
- if (input.extraHeaders === undefined) {
- // Keep all existing headers unchanged
- extraHeaders =
- Object.keys(existingHeaders).length > 0
- ? existingHeaders
- : undefined;
- } else {
- // Process input headers, preserving existing values for empty inputs
- extraHeaders = {};
-
- for (const [key, value] of Object.entries(input.extraHeaders)) {
- if (value === null || value === undefined || value === "") {
- // Keep existing value if input value is empty and key exists
- if (existingHeaders[key] !== undefined) {
- extraHeaders[key] = existingHeaders[key];
- }
- } else {
- // Use the new non-empty value
- extraHeaders[key] = value;
- }
- }
-
- // If no headers remain, set to undefined
- if (Object.keys(extraHeaders).length === 0) {
- extraHeaders = undefined;
- }
- }
+ const extraHeaders = resolveUpdatedExtraHeaders({
+ inputHeaders: input.extraHeaders,
+ storedHeaders: existingKey.extraHeaders,
+ isBaseURLChanged,
+ });
const key = await ctx.prisma.llmApiKeys.update({
where: {
diff --git a/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx b/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx
index 8713b46f0c..67914765a3 100644
--- a/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx
+++ b/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx
@@ -12,8 +12,9 @@ import {
BEDROCK_USE_DEFAULT_CREDENTIALS,
TYPESAFE_UPSTREAMS,
VERTEXAI_USE_DEFAULT_CREDENTIALS,
- findTypeSafeUpstream,
isDecisionModelAdapter,
+ resolveTypeSafeUpstream,
+ type TypeSafeUpstream,
} from "@langfuse/shared";
import { ChevronDown, PlusIcon, TrashIcon } from "lucide-react";
import { z } from "zod";
@@ -109,6 +110,11 @@ const createFormSchema = (params: {
),
adapter: z.enum(LLMAdapter),
baseURL: z.union([z.literal(""), z.url()]),
+ typeSafeUpstream: z.enum(
+ TYPESAFE_UPSTREAMS.map(
+ (upstream): TypeSafeUpstream["id"] => upstream.id,
+ ),
+ ),
withDefaultModels: z.boolean(),
customModels: z.array(z.object({ value: z.string().min(1) })),
awsAccessKeyId: z.string().optional(),
@@ -252,6 +258,16 @@ const createFormSchema = (params: {
message: "API Base URL is required for Azure connections.",
path: ["baseURL"],
},
+ )
+ .refine(
+ (data) =>
+ data.adapter !== LLMAdapter.TypeSafe ||
+ data.typeSafeUpstream !== "custom" ||
+ data.baseURL.trim() !== "",
+ {
+ message: "A base URL is required for a custom upstream.",
+ path: ["baseURL"],
+ },
);
interface CreateLLMApiKeyFormProps {
@@ -334,6 +350,7 @@ export function CreateLLMApiKeyForm({
baseURL:
existingKey.baseURL ??
getCustomizedBaseURL(existingKey.adapter as LLMAdapter),
+ typeSafeUpstream: resolveTypeSafeUpstream(existingKey.baseURL).id,
withDefaultModels: existingKey.withDefaultModels,
customModels: existingKey.customModels.map((value) => ({ value })),
extraHeaders:
@@ -365,6 +382,7 @@ export function CreateLLMApiKeyForm({
provider: "",
secretKey: "",
baseURL: getCustomizedBaseURL(defaultAdapter),
+ typeSafeUpstream: TYPESAFE_UPSTREAMS[0].id,
withDefaultModels: true,
customModels: [],
extraHeaders: [],
@@ -382,8 +400,11 @@ export function CreateLLMApiKeyForm({
const currentAdapter = form.watch("adapter");
const currentAuthMethod = form.watch("authMethod");
+ const currentTypeSafeUpstreamId = form.watch("typeSafeUpstream");
const currentTypeSafeUpstream =
- findTypeSafeUpstream(form.watch("baseURL")) ?? TYPESAFE_UPSTREAMS[0];
+ TYPESAFE_UPSTREAMS.find(
+ (upstream) => upstream.id === currentTypeSafeUpstreamId,
+ ) ?? TYPESAFE_UPSTREAMS[0];
const isKeepingCurrentBedrockAuthMethod =
mode === "update" &&
currentAdapter === LLMAdapter.Bedrock &&
@@ -713,6 +734,10 @@ export function CreateLLMApiKeyForm({
"baseURL",
getCustomizedBaseURL(value as LLMAdapter),
);
+ form.setValue(
+ "typeSafeUpstream",
+ TYPESAFE_UPSTREAMS[0].id,
+ );
}
field.onChange(value as LLMAdapter);
}}
@@ -795,26 +820,29 @@ export function CreateLLMApiKeyForm({
{currentAdapter === LLMAdapter.TypeSafe && (
<FormField
control={form.control}
- name="baseURL"
+ name="typeSafeUpstream"
render={({ field }) => (
<FormItem>
<FormLabel>Upstream</FormLabel>
<FormDescription>
- Provider that serves the Jev decision model. OpenRouter
- and Vercel AI Gateway expose TypeSafe&apos;s API, so
- only the API key changes.
+ Provider that serves the Jev decision model. Vercel AI
+ Gateway, OpenRouter, and custom gateways expose
+ TypeSafe&apos;s API, so evaluators behave the same on
+ every upstream.
</FormDescription>
<FormControl>
<TypeSafeUpstreamCards
aria-label="Upstream"
- value={currentTypeSafeUpstream.id}
- onValueChange={(id) =>
- field.onChange(
+ value={field.value}
+ onValueChange={(id) => {
+ field.onChange(id);
+ form.setValue(
+ "baseURL",
TYPESAFE_UPSTREAMS.find(
(upstream) => upstream.id === id,
)?.baseURL ?? "",
- )
- }
+ );
+ }}
/>
</FormControl>
<FormMessage />
@@ -823,6 +851,30 @@ export function CreateLLMApiKeyForm({
/>
)}
+ {currentAdapter === LLMAdapter.TypeSafe &&
+ currentTypeSafeUpstream.id === "custom" && (
+ <FormField
+ control={form.control}
+ name="baseURL"
+ render={({ field }) => (
+ <FormItem>
+ <FormLabel>Custom base URL</FormLabel>
+ <FormDescription>
+ Base URL of a TypeSafe-compatible API, e.g.
+ <code>https://gateway.example.com/typesafe/v1</code>.
+ </FormDescription>
+ <FormControl>
+ <Input
+ {...field}
+ placeholder="https://gateway.example.com/v1"
+ />
+ </FormControl>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+ )}
+
{/* API Key or AWS Credentials or Vertex AI Credentials */}
{currentAdapter === LLMAdapter.Bedrock ? (
<>
@@ -1254,8 +1306,10 @@ export function CreateLLMApiKeyForm({
{isCustomModelsRequired(currentAdapter) &&
renderCustomModelsField()}
- {/* Extra headers - show for Azure in main section (Azure has no advanced settings) */}
- {currentAdapter === LLMAdapter.Azure && renderExtraHeadersField()}
+ {/* Extra headers - main section for adapters without advanced settings */}
+ {[LLMAdapter.Azure, LLMAdapter.TypeSafe].includes(
+ currentAdapter,
+ ) && renderExtraHeadersField()}
{hasAdvancedSettings(currentAdapter) && (
<div className="flex items-center">
diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.ts
index fc026be143..0577f7b944 100644
--- a/worker/src/features/evaluation/evalExecutionDeps.ts
+++ b/worker/src/features/evaluation/evalExecutionDeps.ts
@@ -9,6 +9,7 @@ import {
compileLangfuseMediaMessages,
createLLMOutput,
createTypeSafeDecisionModelClient,
+ decryptAndParseExtraHeaders,
DefaultEvalModelService,
generateLLMText,
IngestionQueue,
@@ -350,11 +351,15 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
}
let decryptedSecretKey: string;
+ let extraHeaders: Record<string, string> | undefined;
try {
decryptedSecretKey = decrypt(secretKey);
+ extraHeaders = decryptAndParseExtraHeaders(
+ typeof apiKey.extraHeaders === "string" ? apiKey.extraHeaders : null,
+ );
} catch {
throw new UnrecoverableError(
- "TypeSafe connection secret could not be decrypted",
+ "TypeSafe connection secrets could not be decrypted",
);
}
@@ -362,6 +367,7 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
apiKey: decryptedSecretKey,
model: params.modelConfig.model,
baseURL: typeof apiKey.baseURL === "string" ? apiKey.baseURL : null,
+ extraHeaders,
});
return client.evaluate(params.request);