🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(llm-connections): custom base URL and extra headers for TypeSafe connections (#17866)
Hassieb Pakzadd7ae7f0c776a
diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.tsindex 7be561caad..15d5aaab69 100644--- a/packages/shared/src/server/llm/types.ts+++ b/packages/shared/src/server/llm/types.ts@@ -477,7 +477,9 @@ export const typeSafeModels = ["jev-1.13.0", "jev-latest"] as const;/*** Providers that serve Jev through TypeSafe's `/v1/systemone` API. A TypeSafe* connection stores a gateway's `baseURL`, or none for TypeSafe itself, which- * the AI SDK provider then defaults to.+ * the AI SDK provider then defaults to. Presets only prefill the base URL; a+ * `custom` connection stores any base URL the provider appends `/systemone`+ * to.*/export const TYPESAFE_UPSTREAMS = [{@@ -498,15 +500,24 @@ export const TYPESAFE_UPSTREAMS = [baseURL: "https://openrouter.ai/api/v1",apiKeyLabel: "OpenRouter API key",},+ {+ id: "custom",+ label: "Custom",+ baseURL: null,+ apiKeyLabel: "API key",+ },] as const;export type TypeSafeUpstream = (typeof TYPESAFE_UPSTREAMS)[number];-export function findTypeSafeUpstream(+export function resolveTypeSafeUpstream(baseURL: string | null | undefined,-): TypeSafeUpstream | undefined {+): TypeSafeUpstream {if (!baseURL) return TYPESAFE_UPSTREAMS[0];- return TYPESAFE_UPSTREAMS.find((upstream) => upstream.baseURL === baseURL);+ return (+ TYPESAFE_UPSTREAMS.find((upstream) => upstream.baseURL === baseURL) ??+ TYPESAFE_UPSTREAMS[TYPESAFE_UPSTREAMS.length - 1]+ );}export type AnthropicModel = (typeof anthropicModels)[number];diff --git a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.tsindex dc31e8a8b2..a68682db9f 100644--- a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts+++ b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts@@ -1,8 +1,17 @@import { describe, expect, it, vi } from "vitest";import type { DecisionModelRequest } from "../../evals/decisionModelEvaluatorExecution";+import { createSecureLlmFetch } from "../secureLlmFetch";import { TYPESAFE_UPSTREAMS } from "../types";import { createTypeSafeDecisionModelClient } from "./typeSafeDecisionModelClient";+vi.mock("../secureLlmFetch", async (importOriginal) => {+ const actual = await importOriginal<typeof import("../secureLlmFetch")>();+ return {+ ...actual,+ createSecureLlmFetch: vi.fn(actual.createSecureLlmFetch),+ };+});+function jsonResponse(body: unknown, status = 200) {return new Response(JSON.stringify(body), {status,@@ -134,6 +143,63 @@ describe("createTypeSafeDecisionModelClient", () => {},);+ it("posts to a custom base URL with the connection's extra headers", async () => {+ const fetchImpl = vi.fn().mockResolvedValue(+ jsonResponse({+ model: "jev",+ answers: { refund: { type: "noul", noul: 0.5 } },+ }),+ );++ const client = createTypeSafeDecisionModelClient({+ apiKey: "sk-test",+ model: "jev-latest",+ baseURL: "https://llm-proxy.example.com/typesafe/v1/",+ extraHeaders: { "x-team": "evals" },+ fetchImpl,+ });+ await client.evaluate({+ state: request.state,+ questions: { refund: request.questions.refund },+ });++ const [url, init] = fetchImpl.mock.calls[0] as [string, RequestInit];+ expect(url).toBe("https://llm-proxy.example.com/typesafe/v1/systemone");+ const headers = new Headers(init.headers);+ expect(headers.get("x-team")).toBe("evals");+ expect(headers.get("authorization")).toBe("Bearer sk-test");+ });++ it("strips the connection's extra headers on cross-origin redirects", () => {+ createTypeSafeDecisionModelClient({+ apiKey: "sk-test",+ model: "jev-latest",+ baseURL: "https://llm-proxy.example.com/typesafe/v1",+ extraHeaders: { "X-Api-Key": "proxy-secret", "x-team": "evals" },+ });++ expect(vi.mocked(createSecureLlmFetch)).toHaveBeenLastCalledWith(+ expect.objectContaining({+ additionalSensitiveHeaders: ["X-Api-Key", "x-team"],+ }),+ );+ });++ it("blocks a custom base URL that points at an internal address", async () => {+ const client = createTypeSafeDecisionModelClient({+ apiKey: "sk-test",+ model: "jev-latest",+ baseURL: "http://169.254.169.254/v1",+ });++ await expect(+ client.evaluate({+ state: request.state,+ questions: { refund: request.questions.refund },+ }),+ ).rejects.toMatchObject({ name: "LLMValidationError" });+ });+it("tolerates the extra routing fields gateways add to the TypeSafe response", async () => {// OpenRouter returns its own id/provider and a cost inside usage; the model// is the resolved OpenRouter slug rather than the requested alias.diff --git a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.tsindex a31828140a..da80d48d01 100644--- a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts+++ b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts@@ -15,14 +15,19 @@ export function createTypeSafeDecisionModelClient(params: {apiKey: string;model: string;baseURL?: string | null;+ extraHeaders?: Record<string, string>;fetchImpl?: typeof fetch;}): DecisionModelClient {const provider = createTypeSafeAi({apiKey: params.apiKey,baseURL: params.baseURL ?? undefined,+ headers: params.extraHeaders,fetch:params.fetchImpl ??- createSecureLlmFetch({ logContext: "TypeSafe decision model" }),+ createSecureLlmFetch({+ logContext: "TypeSafe decision model",+ additionalSensitiveHeaders: Object.keys(params.extraHeaders ?? {}),+ }),});const model = provider.evaluationModel(params.model);diff --git a/web/src/__tests__/server/llm-api-key.servertest.ts b/web/src/__tests__/server/llm-api-key.servertest.tsindex 15546a28f0..2ac2778a23 100644--- a/web/src/__tests__/server/llm-api-key.servertest.ts+++ b/web/src/__tests__/server/llm-api-key.servertest.ts@@ -355,28 +355,53 @@ describe("llmApiKey.all RPC", () => {expect(llmApiKeys[0].baseURL).toBe("https://openrouter.ai/api/v1");});- it.each(["https://example.com/v1", "https://api.typesafe.ai/v1"])(- "should reject decision-model connections with the non-preset base URL %s",- async (baseURL) => {- await expect(- caller.llmApiKey.create({- projectId,- secretKey: "sk-test",- provider: "jev-custom-url",- adapter: LLMAdapter.TypeSafe,- baseURL,- }),- ).rejects.toThrow(- "only support the TypeSafe, Vercel AI Gateway, and OpenRouter base URLs",- );+ it("should store a custom base URL and encrypted extra headers on a decision-model connection", async () => {+ await caller.llmApiKey.create({+ projectId,+ secretKey: "sk-proxy",+ provider: "jev-via-proxy",+ adapter: LLMAdapter.TypeSafe,+ baseURL: "https://example.com/typesafe/v1",+ extraHeaders: { "x-team": "evals" },+ });++ const storedKey = await prisma.llmApiKeys.findFirstOrThrow({+ where: { projectId, provider: "jev-via-proxy" },+ });+ expect(storedKey.baseURL).toBe("https://example.com/typesafe/v1");+ expect(storedKey.extraHeaderKeys).toEqual(["x-team"]);+ expect(JSON.parse(decrypt(storedKey.extraHeaders!))).toEqual({+ "x-team": "evals",+ });+ });- const { data: llmApiKeys } = await caller.llmApiKey.all({+ it("should block decision-model connections with an internal custom base URL", async () => {+ await expect(+ caller.llmApiKey.create({projectId,- includeDecisionModels: true,- });- expect(llmApiKeys).toHaveLength(0);- },- );+ secretKey: "sk-test",+ provider: "jev-internal",+ adapter: LLMAdapter.TypeSafe,+ baseURL: "http://localhost:8080/v1",+ }),+ ).rejects.toThrow("Invalid base URL: Blocked hostname detected");++ await expect(+ caller.llmApiKey.test({+ projectId,+ secretKey: "sk-test",+ provider: "jev-internal",+ adapter: LLMAdapter.TypeSafe,+ baseURL: "http://localhost:8080/v1",+ }),+ ).resolves.toMatchObject({ success: false });++ const { data: llmApiKeys } = await caller.llmApiKey.all({+ projectId,+ includeDecisionModels: true,+ });+ expect(llmApiKeys).toHaveLength(0);+ });it("should require a new secret key when moving a decision-model connection to another upstream", async () => {await caller.llmApiKey.create({@@ -656,6 +681,67 @@ describe("llmApiKey.all RPC", () => {);});+ it("should fill masked header values from the stored headers in testUpdate", async () => {+ await caller.llmApiKey.create({+ projectId,+ provider: "openai",+ adapter: LLMAdapter.OpenAI,+ secretKey: "sk-original",+ baseURL: "https://api.openai.com/v1",+ extraHeaders: { "X-Api-Key": "stored-token", "X-Team": "stored-team" },+ });+ const existingKey = await prisma.llmApiKeys.findFirstOrThrow({+ where: { projectId, provider: "openai" },+ });++ const result = await caller.llmApiKey.testUpdate({+ id: existingKey.id,+ projectId,+ provider: "openai",+ adapter: LLMAdapter.OpenAI,+ baseURL: "https://api.openai.com/v1",+ extraHeaders: { "X-Api-Key": "", "X-Team": "new-team" },+ });++ expect(result).toEqual({ success: true });+ const connection = mockGenerateLLMText.mock.calls[0][0].connection;+ expect(JSON.parse(decrypt(connection.extraHeaders!))).toEqual({+ "X-Api-Key": "stored-token",+ "X-Team": "new-team",+ });+ });++ it("should not fill masked header values from the stored headers when testUpdate changes the base URL", async () => {+ await caller.llmApiKey.create({+ projectId,+ provider: "openai",+ adapter: LLMAdapter.OpenAI,+ secretKey: "sk-original",+ baseURL: "https://api.openai.com/v1",+ extraHeaders: { "X-Api-Key": "stored-token" },+ });+ const existingKey = await prisma.llmApiKeys.findFirstOrThrow({+ where: { projectId, provider: "openai" },+ });++ const result = await caller.llmApiKey.testUpdate({+ id: existingKey.id,+ projectId,+ provider: "openai",+ adapter: LLMAdapter.OpenAI,+ secretKey: "sk-rotated",+ baseURL: "https://example.net/v1",+ extraHeaders: { "X-Api-Key": "", "X-Team": "new-team" },+ });++ expect(result).toEqual({ success: true });+ const connection = mockGenerateLLMText.mock.calls[0][0].connection;+ expect(connection.baseURL).toBe("https://example.net/v1");+ expect(JSON.parse(decrypt(connection.extraHeaders!))).toEqual({+ "X-Team": "new-team",+ });+ });+it("should allow testUpdate when the base URL changes and a new secret key is provided", async () => {const existingExtraHeaders = {Authorization: "Bearer stored-token",diff --git a/web/src/components/design-system/PasswordInput/PasswordInput.stories.tsx b/web/src/components/design-system/PasswordInput/PasswordInput.stories.tsxindex 2329e6724d..2b66a8b591 100644--- a/web/src/components/design-system/PasswordInput/PasswordInput.stories.tsx+++ b/web/src/components/design-system/PasswordInput/PasswordInput.stories.tsx@@ -43,6 +43,26 @@ export const TestTogglesVisibility = meta.story({},});+export const TestPasswordManagerOptIn = meta.story({+ name: "(Test) Password Manager Opt-In",+ render: () => (+ <>+ <PasswordInput aria-label="API key" />+ <PasswordInput aria-label="Account password" allowPasswordManager />+ </>+ ),+ play: async ({ canvasElement }) => {+ const canvas = within(canvasElement);++ await expect(canvas.getByLabelText("API key")).toHaveAttribute(+ "data-1p-ignore",+ );+ await expect(canvas.getByLabelText("Account password")).not.toHaveAttribute(+ "data-1p-ignore",+ );+ },+});+export const TestSkipsVisibilityToggleWhenTabbing = meta.story({name: "(Test) Skips Visibility Toggle When Tabbing",args: {diff --git a/web/src/components/design-system/PasswordInput/PasswordInput.tsx b/web/src/components/design-system/PasswordInput/PasswordInput.tsxindex 6711626161..b80017955c 100644--- a/web/src/components/design-system/PasswordInput/PasswordInput.tsx+++ b/web/src/components/design-system/PasswordInput/PasswordInput.tsx@@ -23,11 +23,13 @@ type PasswordInputProps = Pick<| "tabIndex"| "value"> & {+ allowPasswordManager?: boolean;error?: boolean;ref?: Ref<HTMLInputElement>;};export function PasswordInput({+ allowPasswordManager,ref,disabled,error,@@ -50,6 +52,7 @@ export function PasswordInput({><input{...props}+ {...(!allowPasswordManager && { "data-1p-ignore": true })}ref={ref}type={isPasswordVisible ? "text" : "password"}disabled={disabled}diff --git a/web/src/features/auth-credentials/components/ResetPasswordPage.tsx b/web/src/features/auth-credentials/components/ResetPasswordPage.tsxindex fea14c21d1..40d1cd9b3c 100644--- a/web/src/features/auth-credentials/components/ResetPasswordPage.tsx+++ b/web/src/features/auth-credentials/components/ResetPasswordPage.tsx@@ -253,6 +253,7 @@ export function ResetPasswordPage({</FormLabel><FormControl><PasswordInput+ allowPasswordManagerautoComplete="new-password"{...field}/>@@ -273,6 +274,7 @@ export function ResetPasswordPage({</FormLabel><FormControl><PasswordInput+ allowPasswordManagerautoComplete="new-password"{...field}/>diff --git a/web/src/features/auth/SignInPage.tsx b/web/src/features/auth/SignInPage.tsxindex 7df804fc44..adbd1e6771 100644--- a/web/src/features/auth/SignInPage.tsx+++ b/web/src/features/auth/SignInPage.tsx@@ -864,7 +864,7 @@ export default function SignInPage({</Link></FormLabel><FormControl>- <PasswordInput {...field} />+ <PasswordInput allowPasswordManager {...field} /></FormControl><FormMessage /></FormItem>diff --git a/web/src/features/auth/SignUpPage.tsx b/web/src/features/auth/SignUpPage.tsxindex 457c9e9fd2..f70b1e0341 100644--- a/web/src/features/auth/SignUpPage.tsx+++ b/web/src/features/auth/SignUpPage.tsx@@ -288,7 +288,7 @@ function StandardSignupFlow({<FormItem><FormLabel>Password</FormLabel><FormControl>- <PasswordInput {...field} />+ <PasswordInput allowPasswordManager {...field} /></FormControl><FormMessage /></FormItem>diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.tsindex 2897828d4c..7c5b4846ff 100644--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts@@ -8,6 +8,7 @@ import {compileLangfuseMediaMessages,createTypeSafeDecisionModelClient,createW3CTraceId,+ decryptAndParseExtraHeaders,DefaultEvalModelService,createLLMOutput,executeDecisionModelEvaluator,@@ -147,6 +148,9 @@ async function testDecisionModelEvaluator(params: {apiKey: decrypt(modelConfig.config.apiKey.secretKey),model: modelConfig.config.model,baseURL: modelConfig.config.apiKey.baseURL,+ extraHeaders: decryptAndParseExtraHeaders(+ modelConfig.config.apiKey.extraHeaders,+ ),});const execution = await executeDecisionModelEvaluator({variables: params.variables,diff --git a/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.stories.tsx b/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.stories.tsxindex 0b5350ba88..1fd5f7c6c7 100644--- a/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.stories.tsx+++ b/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.stories.tsx@@ -38,6 +38,14 @@ export const OpenRouterSelected = meta.story({},});+export const CustomSelected = meta.story({+ args: {+ "aria-label": "Upstream",+ value: "custom",+ onValueChange: fn(),+ },+});+export const Disabled = meta.story({args: {"aria-label": "Upstream",diff --git a/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.tsx b/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.tsxindex 07eb687290..29096c5b42 100644--- a/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.tsx+++ b/web/src/features/llm-api-key/components/TypeSafeUpstreamCards/TypeSafeUpstreamCards.tsx@@ -11,6 +11,7 @@ const UPSTREAM_DESCRIPTIONS: Record<TypeSafeUpstreamId, string> = {typesafe: "Direct connection, billed by TypeSafe.","vercel-ai-gateway": "Routed and billed through your AI Gateway.",openrouter: "Routed and billed through OpenRouter.",+ custom: "Your own TypeSafe-compatible gateway.",};type TypeSafeUpstreamCardsProps = {@@ -35,7 +36,7 @@ export function TypeSafeUpstreamCards({{...rootProps}value={value}onValueChange={(next) => onValueChange(next as TypeSafeUpstreamId)}- className="grid gap-2 sm:grid-cols-3"+ className="grid gap-2 sm:grid-cols-4">{TYPESAFE_UPSTREAMS.map(({ id, label }) => {return (diff --git a/web/src/features/llm-api-key/server/router.ts b/web/src/features/llm-api-key/server/router.tsindex 1e837acbb7..77dc83aa95 100644--- a/web/src/features/llm-api-key/server/router.ts+++ b/web/src/features/llm-api-key/server/router.ts@@ -26,7 +26,6 @@ import {BEDROCK_USE_DEFAULT_CREDENTIALS,VERTEXAI_USE_DEFAULT_CREDENTIALS,EvaluatorBlockReason,- findTypeSafeUpstream,type LLMConnectionConfig,} from "@langfuse/shared";@@ -108,37 +107,18 @@ type TestLLMConnectionParams = {config?: unknown;};-function assertDecisionModelConnectionInput(input: {- adapter: LLMAdapter;- baseURL?: string | null;- extraHeaders?: Record<string, string | null | undefined> | null;-}) {- if (!isDecisionModelAdapter(input.adapter)) return;- if (!findTypeSafeUpstream(input.baseURL)) {- throw new TRPCError({- code: "BAD_REQUEST",- message:- "Decision-model connections only support the TypeSafe, Vercel AI Gateway, and OpenRouter base URLs.",- });- }- if (input.extraHeaders && Object.keys(input.extraHeaders).length > 0) {- throw new TRPCError({- code: "BAD_REQUEST",- message: "Decision-model connections do not support extra headers.",- });- }-}-async function testDecisionModelConnection(params: {secretKey: string;model: string;baseURL?: string | null;+ extraHeaders?: Record<string, string>;}): Promise<{ success: boolean; error?: string }> {try {const client = createTypeSafeDecisionModelClient({apiKey: params.secretKey,model: params.model,baseURL: params.baseURL,+ extraHeaders: params.extraHeaders,});await client.evaluate({state: { message: "Hello, is anyone there?" },@@ -175,6 +155,7 @@ async function testLLMConnection(secretKey: params.secretKey,model,baseURL: params.baseURL,+ extraHeaders: params.extraHeaders,});}@@ -264,6 +245,35 @@ async function validateBaseURLForWrite(params: {}}+function resolveUpdatedExtraHeaders(params: {+ inputHeaders: Record<string, string | null | undefined> | undefined;+ storedHeaders: string | null;+ isBaseURLChanged: boolean;+}): Record<string, string> | undefined {+ const existingHeaders: Record<string, string> = params.isBaseURLChanged+ ? {}+ : (decryptAndParseExtraHeaders(params.storedHeaders) ?? {});++ if (params.inputHeaders === undefined) {+ return Object.keys(existingHeaders).length > 0+ ? existingHeaders+ : undefined;+ }++ const extraHeaders: Record<string, string> = {};+ for (const [key, value] of Object.entries(params.inputHeaders)) {+ if (value === null || value === undefined || value === "") {+ if (existingHeaders[key] !== undefined) {+ extraHeaders[key] = existingHeaders[key];+ }+ } else {+ extraHeaders[key] = value;+ }+ }++ return Object.keys(extraHeaders).length > 0 ? extraHeaders : undefined;+}+export const llmApiKeyRouter = createTRPCRouter({create: protectedProjectProcedureWithoutTracing.input(CreateLlmApiKey)@@ -275,8 +285,6 @@ export const llmApiKeyRouter = createTRPCRouter({scope: "llmApiKeys:create",});- assertDecisionModelConnectionInput(input);-await validateBaseURLForWrite({baseURL: input.baseURL,});@@ -511,8 +519,6 @@ export const llmApiKeyRouter = createTRPCRouter({scope: "llmApiKeys:create",});- assertDecisionModelConnectionInput(input);-if (input.baseURL) {try {await validateLlmConnectionBaseURL(input.baseURL);@@ -560,8 +566,6 @@ export const llmApiKeyRouter = createTRPCRouter({});}- assertDecisionModelConnectionInput(input);-const hasNewSecretKey =typeof input.secretKey === "string" && input.secretKey.length > 0;const baseURL =@@ -589,15 +593,11 @@ export const llmApiKeyRouter = createTRPCRouter({const customModels = input.customModels ?? existingKey.customModels;const config = input.config ?? existingKey.config;- // Never reuse stored headers across a destination change.- const extraHeaders =- input.extraHeaders !== undefined- ? input.extraHeaders- : isBaseURLChanged- ? undefined- : existingKey.extraHeaders- ? decryptAndParseExtraHeaders(existingKey.extraHeaders)- : undefined;+ const extraHeaders = resolveUpdatedExtraHeaders({+ inputHeaders: input.extraHeaders,+ storedHeaders: existingKey.extraHeaders,+ isBaseURLChanged,+ });return testLLMConnection({adapter,@@ -654,8 +654,6 @@ export const llmApiKeyRouter = createTRPCRouter({});}- assertDecisionModelConnectionInput(input);-// Validate that default credentials sentinel is only allowed for Bedrock/VertexAI in self-hosted deploymentsconst isLangfuseCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);const isBaseURLChanged =@@ -713,44 +711,11 @@ export const llmApiKeyRouter = createTRPCRouter({input.extraHeaders = {};}- // Get existing decrypted headers for comparison- const decryptedHeaders = existingKey.extraHeaders- ? decryptAndParseExtraHeaders(existingKey.extraHeaders)- : null;- const existingHeaders: Record<string, string> = isBaseURLChanged- ? {}- : (decryptedHeaders ?? {});-- // Ensure we only update the extraHeaders where the value is not null- let extraHeaders: Record<string, string> | undefined;-- if (input.extraHeaders === undefined) {- // Keep all existing headers unchanged- extraHeaders =- Object.keys(existingHeaders).length > 0- ? existingHeaders- : undefined;- } else {- // Process input headers, preserving existing values for empty inputs- extraHeaders = {};-- for (const [key, value] of Object.entries(input.extraHeaders)) {- if (value === null || value === undefined || value === "") {- // Keep existing value if input value is empty and key exists- if (existingHeaders[key] !== undefined) {- extraHeaders[key] = existingHeaders[key];- }- } else {- // Use the new non-empty value- extraHeaders[key] = value;- }- }-- // If no headers remain, set to undefined- if (Object.keys(extraHeaders).length === 0) {- extraHeaders = undefined;- }- }+ const extraHeaders = resolveUpdatedExtraHeaders({+ inputHeaders: input.extraHeaders,+ storedHeaders: existingKey.extraHeaders,+ isBaseURLChanged,+ });const key = await ctx.prisma.llmApiKeys.update({where: {diff --git a/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx b/web/src/features/public-api/components/CreateLLMApiKeyForm.tsxindex 8713b46f0c..67914765a3 100644--- a/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx+++ b/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx@@ -12,8 +12,9 @@ import {BEDROCK_USE_DEFAULT_CREDENTIALS,TYPESAFE_UPSTREAMS,VERTEXAI_USE_DEFAULT_CREDENTIALS,- findTypeSafeUpstream,isDecisionModelAdapter,+ resolveTypeSafeUpstream,+ type TypeSafeUpstream,} from "@langfuse/shared";import { ChevronDown, PlusIcon, TrashIcon } from "lucide-react";import { z } from "zod";@@ -109,6 +110,11 @@ const createFormSchema = (params: {),adapter: z.enum(LLMAdapter),baseURL: z.union([z.literal(""), z.url()]),+ typeSafeUpstream: z.enum(+ TYPESAFE_UPSTREAMS.map(+ (upstream): TypeSafeUpstream["id"] => upstream.id,+ ),+ ),withDefaultModels: z.boolean(),customModels: z.array(z.object({ value: z.string().min(1) })),awsAccessKeyId: z.string().optional(),@@ -252,6 +258,16 @@ const createFormSchema = (params: {message: "API Base URL is required for Azure connections.",path: ["baseURL"],},+ )+ .refine(+ (data) =>+ data.adapter !== LLMAdapter.TypeSafe ||+ data.typeSafeUpstream !== "custom" ||+ data.baseURL.trim() !== "",+ {+ message: "A base URL is required for a custom upstream.",+ path: ["baseURL"],+ },);interface CreateLLMApiKeyFormProps {@@ -334,6 +350,7 @@ export function CreateLLMApiKeyForm({baseURL:existingKey.baseURL ??getCustomizedBaseURL(existingKey.adapter as LLMAdapter),+ typeSafeUpstream: resolveTypeSafeUpstream(existingKey.baseURL).id,withDefaultModels: existingKey.withDefaultModels,customModels: existingKey.customModels.map((value) => ({ value })),extraHeaders:@@ -365,6 +382,7 @@ export function CreateLLMApiKeyForm({provider: "",secretKey: "",baseURL: getCustomizedBaseURL(defaultAdapter),+ typeSafeUpstream: TYPESAFE_UPSTREAMS[0].id,withDefaultModels: true,customModels: [],extraHeaders: [],@@ -382,8 +400,11 @@ export function CreateLLMApiKeyForm({const currentAdapter = form.watch("adapter");const currentAuthMethod = form.watch("authMethod");+ const currentTypeSafeUpstreamId = form.watch("typeSafeUpstream");const currentTypeSafeUpstream =- findTypeSafeUpstream(form.watch("baseURL")) ?? TYPESAFE_UPSTREAMS[0];+ TYPESAFE_UPSTREAMS.find(+ (upstream) => upstream.id === currentTypeSafeUpstreamId,+ ) ?? TYPESAFE_UPSTREAMS[0];const isKeepingCurrentBedrockAuthMethod =mode === "update" &¤tAdapter === LLMAdapter.Bedrock &&@@ -713,6 +734,10 @@ export function CreateLLMApiKeyForm({"baseURL",getCustomizedBaseURL(value as LLMAdapter),);+ form.setValue(+ "typeSafeUpstream",+ TYPESAFE_UPSTREAMS[0].id,+ );}field.onChange(value as LLMAdapter);}}@@ -795,26 +820,29 @@ export function CreateLLMApiKeyForm({{currentAdapter === LLMAdapter.TypeSafe && (<FormFieldcontrol={form.control}- name="baseURL"+ name="typeSafeUpstream"render={({ field }) => (<FormItem><FormLabel>Upstream</FormLabel><FormDescription>- Provider that serves the Jev decision model. OpenRouter- and Vercel AI Gateway expose TypeSafe's API, so- only the API key changes.+ Provider that serves the Jev decision model. Vercel AI+ Gateway, OpenRouter, and custom gateways expose+ TypeSafe's API, so evaluators behave the same on+ every upstream.</FormDescription><FormControl><TypeSafeUpstreamCardsaria-label="Upstream"- value={currentTypeSafeUpstream.id}- onValueChange={(id) =>- field.onChange(+ value={field.value}+ onValueChange={(id) => {+ field.onChange(id);+ form.setValue(+ "baseURL",TYPESAFE_UPSTREAMS.find((upstream) => upstream.id === id,)?.baseURL ?? "",- )- }+ );+ }}/></FormControl><FormMessage />@@ -823,6 +851,30 @@ export function CreateLLMApiKeyForm({/>)}+ {currentAdapter === LLMAdapter.TypeSafe &&+ currentTypeSafeUpstream.id === "custom" && (+ <FormField+ control={form.control}+ name="baseURL"+ render={({ field }) => (+ <FormItem>+ <FormLabel>Custom base URL</FormLabel>+ <FormDescription>+ Base URL of a TypeSafe-compatible API, e.g.+ <code>https://gateway.example.com/typesafe/v1</code>.+ </FormDescription>+ <FormControl>+ <Input+ {...field}+ placeholder="https://gateway.example.com/v1"+ />+ </FormControl>+ <FormMessage />+ </FormItem>+ )}+ />+ )}+{/* API Key or AWS Credentials or Vertex AI Credentials */}{currentAdapter === LLMAdapter.Bedrock ? (<>@@ -1254,8 +1306,10 @@ export function CreateLLMApiKeyForm({{isCustomModelsRequired(currentAdapter) &&renderCustomModelsField()}- {/* Extra headers - show for Azure in main section (Azure has no advanced settings) */}- {currentAdapter === LLMAdapter.Azure && renderExtraHeadersField()}+ {/* Extra headers - main section for adapters without advanced settings */}+ {[LLMAdapter.Azure, LLMAdapter.TypeSafe].includes(+ currentAdapter,+ ) && renderExtraHeadersField()}{hasAdvancedSettings(currentAdapter) && (<div className="flex items-center">diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.tsindex fc026be143..0577f7b944 100644--- a/worker/src/features/evaluation/evalExecutionDeps.ts+++ b/worker/src/features/evaluation/evalExecutionDeps.ts@@ -9,6 +9,7 @@ import {compileLangfuseMediaMessages,createLLMOutput,createTypeSafeDecisionModelClient,+ decryptAndParseExtraHeaders,DefaultEvalModelService,generateLLMText,IngestionQueue,@@ -350,11 +351,15 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {}let decryptedSecretKey: string;+ let extraHeaders: Record<string, string> | undefined;try {decryptedSecretKey = decrypt(secretKey);+ extraHeaders = decryptAndParseExtraHeaders(+ typeof apiKey.extraHeaders === "string" ? apiKey.extraHeaders : null,+ );} catch {throw new UnrecoverableError(- "TypeSafe connection secret could not be decrypted",+ "TypeSafe connection secrets could not be decrypted",);}@@ -362,6 +367,7 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {apiKey: decryptedSecretKey,model: params.modelConfig.model,baseURL: typeof apiKey.baseURL === "string" ? apiKey.baseURL : null,+ extraHeaders,});return client.evaluate(params.request);