🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
const mockFinalizeEvaluatorBlocks = vi.hoisted(() => vi.fn());
vi.mock("@langfuse/shared/src/server", async () => {
const actual = await vi.importActual("@langfuse/shared/src/server");
return {
...actual,
finalizeEvaluatorBlocks: mockFinalizeEvaluatorBlocks,
generateLLMText: vi.fn(),
};
});
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import * as auditLogs from "@/src/features/audit-logs/auditLog";
import type { Session } from "next-auth";
import { BEDROCK_USE_DEFAULT_CREDENTIALS, LLMAdapter } from "@langfuse/shared";
import { env } from "@/src/env.mjs";
import { randomUUID } from "crypto";
import {
EvalTemplateType,
EvaluatorBlockReason,
prisma,
} from "@langfuse/shared/src/db";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import { decrypt, encrypt } from "@langfuse/shared/encryption";
import { AuthMethod } from "@/src/features/llm-api-key/types";
import { LlmConnectionService } from "@/src/features/llm-api-key/server/llmConnectionService";
import {
createOrgProjectAndApiKey,
DefaultEvalModelService,
EvaluatorBlockSource,
generateLLMText,
} from "@langfuse/shared/src/server";
const mockGenerateLLMText = vi.mocked(generateLLMText);
describe("llmApiKey.all RPC", () => {
let projectId: string;
let orgId: string;
let session: Session;
let caller: ReturnType<typeof appRouter.createCaller>;
const createCallerForProjectRole = (
projectRole: "ADMIN" | "MEMBER" | "VIEWER",
) => {
const limitedSession: Session = {
...session,
user: {
...session.user!,
admin: false,
organizations: [
{
...session.user!.organizations[0],
role: "MEMBER",
projects: [
{
...session.user!.organizations[0].projects[0],
role: projectRole,
},
],
},
],
},
};
const limitedCtx = createInnerTRPCContext({
session: limitedSession,
headers: {},
});
return appRouter.createCaller({ ...limitedCtx, prisma });
};
beforeEach(async () => {
const setup = await createOrgProjectAndApiKey();
projectId = setup.projectId;
orgId = setup.orgId;
mockFinalizeEvaluatorBlocks.mockReset().mockResolvedValue(undefined);
mockGenerateLLMText.mockReset().mockResolvedValue({} as never);
session = {
expires: "1",
user: {
id: "user-1",
name: "Demo User",
canCreateOrganizations: true,
organizations: [
{
id: orgId,
role: "OWNER",
plan: "cloud:hobby",
cloudConfig: undefined,
name: "Test Organization",
metadata: {},
aiFeaturesEnabled: false,
aiTelemetryEnabled: false,
projects: [
{
id: projectId,
role: "ADMIN",
name: "Test Project",
deletedAt: null,
retentionDays: null,
hasTraces: false,
metadata: {},
createdAt: new Date().toISOString(),
},
],
},
],
featureFlags: testFeatureFlags(),
admin: true,
},
environment: {} as any,
};
const ctx = createInnerTRPCContext({ session, headers: {} });
caller = appRouter.createCaller({ ...ctx, prisma });
});
it("should create an llm api key", async () => {
const secret = "test-secret";
const provider = "openai";
const adapter = LLMAdapter.OpenAI;
const customModels = ["fancy-gpt-3.5-turbo"];
const baseURL = "https://example.com/v1";
const withDefaultModels = false;
const result = await caller.llmApiKey.create({
projectId,
secretKey: secret,
provider,
adapter,
baseURL,
customModels,
withDefaultModels,
});
expect(result).toBeUndefined();
const llmApiKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
},
});
expect(llmApiKeys.length).toBe(1);
expect(llmApiKeys[0].projectId).toBe(projectId);
expect(llmApiKeys[0].secretKey).not.toBeNull();
expect(llmApiKeys[0].secretKey).not.toEqual(secret);
expect(llmApiKeys[0].provider).toBe(provider);
expect(llmApiKeys[0].adapter).toBe(adapter);
expect(llmApiKeys[0].baseURL).toBe(baseURL);
expect(llmApiKeys[0].customModels).toEqual(customModels);
expect(llmApiKeys[0].withDefaultModels).toBe(withDefaultModels);
// this has to be 3 dots and the last 4 characters of the secret
expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
});
it("resolves an organization connection for the project default model", async () => {
const provider = `organization-default-${randomUUID()}`;
const organizationConnection = await prisma.llmApiKeys.create({
data: {
organizationId: orgId,
provider,
adapter: LLMAdapter.OpenAI,
secretKey: encrypt("organization-secret"),
displaySecretKey: "...cret",
},
});
await prisma.defaultLlmModel.create({
data: {
projectId,
llmApiKeyId: organizationConnection.id,
provider,
adapter: LLMAdapter.OpenAI,
model: "gpt-4o",
},
});
const result =
await DefaultEvalModelService.fetchValidModelConfig(projectId);
expect(result).toMatchObject({
valid: true,
config: {
apiKey: {
id: organizationConnection.id,
projectId,
},
},
});
});
it("returns CONFLICT when creating a key for an existing provider", async () => {
const provider = `duplicate-provider-${randomUUID()}`;
const input = {
projectId,
secretKey: "test-secret",
provider,
adapter: LLMAdapter.OpenAI,
customModels: [],
withDefaultModels: true,
};
await caller.llmApiKey.create(input);
await expect(caller.llmApiKey.create(input)).rejects.toMatchObject({
code: "CONFLICT",
message: "There already exists an API key for this provider.",
});
await expect(
prisma.llmApiKeys.count({ where: { projectId, provider } }),
).resolves.toBe(1);
});
it("rejects an incompatible project override without blocking edits to an existing mismatch", async () => {
const provider = `incompatible-project-override-${randomUUID()}`;
await prisma.llmApiKeys.create({
data: {
organizationId: orgId,
provider,
adapter: LLMAdapter.OpenAI,
secretKey: encrypt("organization-secret"),
displaySecretKey: "...cret",
},
});
await expect(
caller.llmApiKey.create({
projectId,
provider,
adapter: LLMAdapter.Anthropic,
secretKey: "project-secret",
}),
).rejects.toThrow(
"Project and organization connections with the same provider name must use the same adapter.",
);
const incompatibleProjectConnection = await prisma.llmApiKeys.create({
data: {
projectId,
provider,
adapter: LLMAdapter.Anthropic,
secretKey: encrypt("project-secret"),
displaySecretKey: "...cret",
},
});
await expect(
caller.llmApiKey.update({
id: incompatibleProjectConnection.id,
projectId,
provider,
adapter: LLMAdapter.Anthropic,
customModels: ["claude-sonnet-4-5"],
}),
).resolves.toBeUndefined();
await expect(
prisma.llmApiKeys.findUniqueOrThrow({
where: { id: incompatibleProjectConnection.id },
select: { customModels: true },
}),
).resolves.toEqual({ customModels: ["claude-sonnet-4-5"] });
const service = new LlmConnectionService();
await expect(
service.upsert({
owner: {
type: "project",
projectId,
organizationId: orgId,
},
input: {
provider,
adapter: LLMAdapter.Anthropic,
secretKey: "rotated-project-secret",
},
actor: { userId: session.user!.id, orgId, projectId },
}),
).resolves.toMatchObject({ created: false });
await expect(
service.upsert({
owner: {
type: "project",
projectId,
organizationId: orgId,
},
input: {
provider,
adapter: LLMAdapter.Azure,
secretKey: "azure-project-secret",
},
actor: { userId: session.user!.id, orgId, projectId },
}),
).rejects.toThrow(
"Project and organization connections with the same provider name must use the same adapter.",
);
});
it("should rotate a secret without revalidating an unchanged stored base URL", async () => {
const provider = `operator-provisioned-${randomUUID()}`;
const connection = await prisma.llmApiKeys.create({
data: {
projectId,
provider,
adapter: LLMAdapter.OpenAI,
secretKey: encrypt("old-secret"),
displaySecretKey: "...cret",
baseURL: "http://localhost:11434/v1",
},
});
const result = await new LlmConnectionService().upsert({
owner: {
type: "project",
projectId,
organizationId: orgId,
},
input: {
provider,
adapter: LLMAdapter.OpenAI,
secretKey: "rotated-secret",
baseURL: connection.baseURL,
},
actor: { userId: session.user!.id, orgId, projectId },
});
expect(result.created).toBe(false);
const updatedConnection = await prisma.llmApiKeys.findUniqueOrThrow({
where: { id: connection.id },
});
expect(decrypt(updatedConnection.secretKey)).toBe("rotated-secret");
});
it("should create a Bedrock llm api key with a Bedrock API key", async () => {
const secret = "bedrock-api-key-1234";
await caller.llmApiKey.create({
projectId,
secretKey: JSON.stringify({ apiKey: secret }),
provider: "bedrock",
adapter: LLMAdapter.Bedrock,
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "us-east-1" },
});
const llmApiKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider: "bedrock",
},
});
expect(decrypt(llmApiKey.secretKey)).toBe(
JSON.stringify({
apiKey: secret,
}),
);
expect(llmApiKey.displaySecretKey).toBe("...1234");
expect(llmApiKey.config).toEqual({ region: "us-east-1" });
});
it("should reject creating a Bedrock key with invalid secret key JSON", async () => {
await expect(
caller.llmApiKey.create({
projectId,
secretKey: JSON.stringify({ unknownField: "value" }),
provider: "bedrock",
adapter: LLMAdapter.Bedrock,
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "us-east-1" },
}),
).rejects.toThrow("Invalid Bedrock credentials");
});
it("should block creating an llm api key with a localhost base URL", async () => {
await expect(
caller.llmApiKey.create({
projectId,
secretKey: "test-secret",
provider: "openai",
adapter: LLMAdapter.OpenAI,
baseURL: "http://localhost:11434/v1",
}),
).rejects.toThrow("Invalid base URL: Blocked hostname detected");
});
it("should create and get an llm api key", async () => {
const secret = "test-secret";
const provider = "openai";
const adapter = LLMAdapter.OpenAI;
const customModels = ["fancy-gpt-3.5-turbo"];
const baseURL = "https://example.com/v1";
const withDefaultModels = false;
await caller.llmApiKey.create({
projectId,
secretKey: secret,
provider,
adapter,
baseURL,
customModels,
withDefaultModels,
});
const dbLlmApiKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
},
});
expect(dbLlmApiKeys.length).toBe(1);
const { data: llmApiKeys } = await caller.llmApiKey.all({
projectId,
});
expect(llmApiKeys.length).toBe(1);
expect(llmApiKeys[0].projectId).toBe(projectId);
expect(llmApiKeys[0].secretKey).not.toBeNull();
expect(llmApiKeys[0].secretKey).not.toEqual(secret);
expect(llmApiKeys[0].provider).toBe(provider);
expect(llmApiKeys[0].adapter).toBe(adapter);
expect(llmApiKeys[0].baseURL).toBe(baseURL);
expect(llmApiKeys[0].customModels).toEqual(customModels);
expect(llmApiKeys[0].withDefaultModels).toBe(withDefaultModels);
// this has to be 3 dots and the last 4 characters of the secret
expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
// response must not contain the secret key itself
const secretKey = llmApiKeys[0].secretKey;
expect(secretKey).toBeUndefined();
});
it("should create and get an OpenAI llm api key with Responses API config", async () => {
await caller.llmApiKey.create({
projectId,
secretKey: "test-secret",
provider: "openai-responses",
adapter: LLMAdapter.OpenAI,
baseURL: "https://example.com/v1",
customModels: ["openai.gpt-5.4"],
withDefaultModels: false,
config: { useResponsesApi: true },
});
const { data: llmApiKeys } = await caller.llmApiKey.all({
projectId,
});
expect(llmApiKeys).toHaveLength(1);
expect(llmApiKeys[0].config).toEqual({ useResponsesApi: true });
expect(llmApiKeys[0].customModels).toEqual(["openai.gpt-5.4"]);
expect(llmApiKeys[0].secretKey).toBeUndefined();
});
it("should update an OpenAI llm api key to disable Responses API config", async () => {
await caller.llmApiKey.create({
projectId,
secretKey: "test-secret",
provider: "openai-responses",
adapter: LLMAdapter.OpenAI,
baseURL: "https://example.com/v1",
customModels: ["openai.gpt-5.4"],
withDefaultModels: false,
config: { useResponsesApi: true },
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider: "openai-responses",
},
});
const result = await caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider: "openai-responses",
adapter: LLMAdapter.OpenAI,
baseURL: "https://example.com/v1",
customModels: ["openai.gpt-5.4"],
withDefaultModels: false,
config: { useResponsesApi: false },
});
expect(result).toBeUndefined();
const { data: llmApiKeys } = await caller.llmApiKey.all({
projectId,
});
expect(llmApiKeys).toHaveLength(1);
expect(llmApiKeys[0].config).toEqual({ useResponsesApi: false });
});
it("should preserve empty VertexAI config without applying OpenAI defaults", async () => {
await prisma.llmApiKeys.create({
data: {
projectId,
provider: "vertex-empty-config",
adapter: LLMAdapter.VertexAI,
secretKey: encrypt("test-secret"),
displaySecretKey: "...cret",
customModels: ["gemini-2.5-flash"],
withDefaultModels: false,
extraHeaderKeys: [],
config: {},
},
});
const { data: llmApiKeys } = await caller.llmApiKey.all({
projectId,
});
expect(llmApiKeys).toHaveLength(1);
expect(llmApiKeys[0].config).toEqual({});
});
it("should store a preset upstream as the base URL of a decision-model connection", async () => {
await caller.llmApiKey.create({
projectId,
secretKey: "sk-or-test",
provider: "jev-via-openrouter",
adapter: LLMAdapter.TypeSafe,
baseURL: "https://openrouter.ai/api/v1",
});
const { data: llmApiKeys } = await caller.llmApiKey.all({
projectId,
includeDecisionModels: true,
});
expect(llmApiKeys).toHaveLength(1);
expect(llmApiKeys[0].baseURL).toBe("https://openrouter.ai/api/v1");
});
it("should store a custom base URL and encrypted extra headers on a decision-model connection", async () => {
await caller.llmApiKey.create({
projectId,
secretKey: "sk-proxy",
provider: "jev-via-proxy",
adapter: LLMAdapter.TypeSafe,
baseURL: "https://example.com/typesafe/v1",
extraHeaders: { "x-team": "evals" },
});
const storedKey = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider: "jev-via-proxy" },
});
expect(storedKey.baseURL).toBe("https://example.com/typesafe/v1");
expect(storedKey.extraHeaderKeys).toEqual(["x-team"]);
expect(JSON.parse(decrypt(storedKey.extraHeaders!))).toEqual({
"x-team": "evals",
});
});
it("should treat a decision-model connection as working when the probe is answered", async () => {
const fetchMock = vi.fn().mockResolvedValue(
new Response(
JSON.stringify({
model: "jev-1.13.0",
answers: {
kind: {
type: "choice",
choice: "greeting",
confidence: 0.9,
probabilities: { greeting: 0.9, other: 0.1 },
},
},
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
),
);
vi.stubGlobal("fetch", fetchMock);
try {
const result = await caller.llmApiKey.test({
projectId,
secretKey: "sk-typesafe",
provider: "jev",
adapter: LLMAdapter.TypeSafe,
baseURL: "https://example.com/typesafe/v1",
extraHeaders: { "x-team": "evals" },
});
expect(result).toEqual({ success: true });
expect(mockGenerateLLMText).not.toHaveBeenCalled();
expect(fetchMock).toHaveBeenCalledTimes(1);
const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit];
expect(url).toBe("https://example.com/typesafe/v1/systemone");
expect(new Headers(init.headers).get("authorization")).toBe(
"Bearer sk-typesafe",
);
expect(new Headers(init.headers).get("x-team")).toBe("evals");
expect(JSON.parse(init.body as string)).toEqual({
model: "jev-latest",
state: { message: "Hello, is anyone there?" },
questions: {
kind: {
type: "choice",
instructions: "What kind of message is `message`?",
criteria: { greeting: null, other: null },
},
},
});
} finally {
vi.unstubAllGlobals();
}
});
it("should report the upstream error when a decision-model connection test fails", async () => {
const fetchMock = vi.fn().mockResolvedValue(
new Response(JSON.stringify({ error: { message: "invalid api key" } }), {
status: 401,
headers: { "Content-Type": "application/json" },
}),
);
vi.stubGlobal("fetch", fetchMock);
try {
const result = await caller.llmApiKey.test({
projectId,
secretKey: "sk-typesafe",
provider: "jev",
adapter: LLMAdapter.TypeSafe,
baseURL: "https://example.com/typesafe/v1",
});
expect(result.success).toBe(false);
expect(result.error).toEqual(expect.any(String));
expect(result.error!.length).toBeGreaterThan(0);
expect(mockGenerateLLMText).not.toHaveBeenCalled();
} finally {
vi.unstubAllGlobals();
}
});
it("should block decision-model connections with an internal custom base URL", async () => {
await expect(
caller.llmApiKey.create({
projectId,
secretKey: "sk-test",
provider: "jev-internal",
adapter: LLMAdapter.TypeSafe,
baseURL: "http://localhost:8080/v1",
}),
).rejects.toThrow("Invalid base URL: Blocked hostname detected");
await expect(
caller.llmApiKey.test({
projectId,
secretKey: "sk-test",
provider: "jev-internal",
adapter: LLMAdapter.TypeSafe,
baseURL: "http://localhost:8080/v1",
}),
).resolves.toEqual({
success: false,
error: "Blocked hostname detected",
});
const { data: llmApiKeys } = await caller.llmApiKey.all({
projectId,
includeDecisionModels: true,
});
expect(llmApiKeys).toHaveLength(0);
});
it("should reject decision-model base URLs that end in /systemone or have a query string", async () => {
const connection = {
projectId,
secretKey: "sk-proxy",
provider: "jev-via-proxy",
adapter: LLMAdapter.TypeSafe,
};
await expect(
caller.llmApiKey.create({
...connection,
baseURL: "https://example.com/typesafe/v1/systemone",
}),
).rejects.toThrow("Remove /systemone");
await expect(
caller.llmApiKey.test({
...connection,
baseURL: "https://example.com/typesafe/v1/systemone/",
}),
).resolves.toMatchObject({
success: false,
error: expect.stringContaining("Remove /systemone"),
});
await expect(
caller.llmApiKey.create({
...connection,
baseURL: "https://example.com/typesafe/v1/systemone?api-version=1",
}),
).rejects.toThrow("Remove /systemone");
await expect(
caller.llmApiKey.create({
...connection,
baseURL: "https://example.com/typesafe/v1?api-version=1",
}),
).rejects.toThrow("Remove the query string");
await caller.llmApiKey.create({
...connection,
baseURL: "https://example.com/typesafe/v1",
});
const { id } = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider: connection.provider },
});
await expect(
caller.llmApiKey.update({
...connection,
id,
baseURL: "https://example.com/typesafe/v1/systemone",
}),
).rejects.toThrow("Remove /systemone");
await expect(
caller.llmApiKey.testUpdate({
...connection,
id,
baseURL: "https://example.com/typesafe/v1/systemone",
}),
).resolves.toMatchObject({
success: false,
error: expect.stringContaining("Remove /systemone"),
});
});
it("should require a new secret key when moving a decision-model connection to another upstream", async () => {
await caller.llmApiKey.create({
projectId,
secretKey: "sk-typesafe",
provider: "jev",
adapter: LLMAdapter.TypeSafe,
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider: "jev" },
});
await expect(
caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider: "jev",
adapter: LLMAdapter.TypeSafe,
baseURL: "https://ai-gateway.vercel.sh/typesafe/v1",
}),
).rejects.toThrow("Secret key is required when changing the base URL");
await caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider: "jev",
adapter: LLMAdapter.TypeSafe,
secretKey: "vck-gateway",
baseURL: "https://ai-gateway.vercel.sh/typesafe/v1",
});
const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
where: { id: existingKey.id, projectId },
});
expect(updatedKey.baseURL).toBe("https://ai-gateway.vercel.sh/typesafe/v1");
expect(decrypt(updatedKey.secretKey)).toBe("vck-gateway");
await caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider: "jev",
adapter: LLMAdapter.TypeSafe,
secretKey: "sk-typesafe-2",
baseURL: null,
});
const revertedKey = await prisma.llmApiKeys.findUniqueOrThrow({
where: { id: existingKey.id, projectId },
});
expect(revertedKey.baseURL).toBeNull();
});
it("should derive the Bedrock auth method in llmApiKey.all without returning secrets", async () => {
await prisma.llmApiKeys.createMany({
data: [
{
projectId,
provider: "bedrock-access",
adapter: LLMAdapter.Bedrock,
secretKey: encrypt(
JSON.stringify({
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
}),
),
displaySecretKey: "...MPLE",
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
extraHeaderKeys: [],
config: { region: "us-east-1" },
},
{
projectId,
provider: "bedrock-api",
adapter: LLMAdapter.Bedrock,
secretKey: encrypt(
JSON.stringify({
apiKey: "bedrock-api-key-1234",
}),
),
displaySecretKey: "...1234",
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
extraHeaderKeys: [],
config: { region: "us-east-1" },
},
{
projectId,
provider: "bedrock-default",
adapter: LLMAdapter.Bedrock,
secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
displaySecretKey: "Default AWS credentials",
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
extraHeaderKeys: [],
config: { region: "us-east-1" },
},
{
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: encrypt("sk-test"),
displaySecretKey: "...test",
customModels: [],
withDefaultModels: true,
extraHeaderKeys: [],
},
],
});
const { data: llmApiKeys } = await caller.llmApiKey.all({
projectId,
});
expect(
llmApiKeys.find((key) => key.provider === "bedrock-access")?.authMethod,
).toBe(AuthMethod.AccessKeys);
expect(
llmApiKeys.find((key) => key.provider === "bedrock-api")?.authMethod,
).toBe(AuthMethod.ApiKey);
expect(
llmApiKeys.find((key) => key.provider === "bedrock-default")?.authMethod,
).toBe(AuthMethod.DefaultCredentials);
expect(
llmApiKeys.find((key) => key.provider === "openai")?.authMethod,
).toBeUndefined();
expect(
llmApiKeys.every(
(key) => key.secretKey === undefined && key.extraHeaders === undefined,
),
).toBe(true);
});
it("should require llmApiKeys:create access for testing a new llm api key", async () => {
const memberCaller = createCallerForProjectRole("MEMBER");
await expect(
memberCaller.llmApiKey.test({
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-test",
baseURL: "https://attacker.example.com/v1",
}),
).rejects.toThrow("User does not have access to this resource or action");
});
it("should require llmApiKeys:update access for testing an existing llm api key", async () => {
await caller.llmApiKey.create({
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-test",
baseURL: "https://api.openai.com/v1",
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider: "openai",
},
});
const memberCaller = createCallerForProjectRole("MEMBER");
await expect(
memberCaller.llmApiKey.testUpdate({
id: existingKey.id,
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
baseURL: "https://attacker.example.com/v1",
}),
).rejects.toThrow("User does not have access to this resource or action");
});
it("should block testUpdate when the base URL changes without a new secret key", async () => {
await caller.llmApiKey.create({
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-original",
baseURL: "https://api.openai.com/v1",
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider: "openai",
},
});
const result = await caller.llmApiKey.testUpdate({
id: existingKey.id,
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
baseURL: "https://attacker.example.com/v1",
});
expect(result).toEqual({
success: false,
error: "Secret key is required when changing the base URL",
});
expect(mockGenerateLLMText).not.toHaveBeenCalled();
});
it("should allow testing an existing connection with an unchanged localhost base URL", async () => {
const connection = await prisma.llmApiKeys.create({
data: {
projectId,
provider: "local-ollama",
adapter: LLMAdapter.OpenAI,
secretKey: encrypt("sk-existing"),
displaySecretKey: "...ting",
baseURL: "http://localhost:11434/v1",
customModels: ["llama3.1"],
withDefaultModels: true,
},
});
const result = await caller.llmApiKey.testUpdate({
id: connection.id,
projectId,
provider: "local-ollama",
adapter: LLMAdapter.OpenAI,
});
expect(result).toEqual({ success: true });
expect(mockGenerateLLMText).toHaveBeenCalledTimes(1);
expect(mockGenerateLLMText).toHaveBeenCalledWith(
expect.objectContaining({ timeout: 95_000 }),
);
});
it("should allow testUpdate without a new secret key when the base URL is unchanged", async () => {
const existingExtraHeaders = {
Authorization: "Bearer stored-token",
"X-Custom-Header": "stored-value",
};
await caller.llmApiKey.create({
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-original",
baseURL: "https://api.openai.com/v1",
extraHeaders: existingExtraHeaders,
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider: "openai",
},
});
const result = await caller.llmApiKey.testUpdate({
id: existingKey.id,
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
baseURL: "https://api.openai.com/v1",
});
expect(result).toEqual({ success: true });
expect(mockGenerateLLMText).toHaveBeenCalledTimes(1);
const connection = mockGenerateLLMText.mock.calls[0][0].connection;
expect(connection.baseURL).toBe("https://api.openai.com/v1");
expect(decrypt(connection.secretKey)).toBe("sk-original");
expect(connection.projectId).toBe(existingKey.projectId);
const encryptedExtraHeaders = connection.extraHeaders;
expect(encryptedExtraHeaders).toBeTruthy();
if (!encryptedExtraHeaders) {
throw new Error("Expected extra headers to be preserved");
}
expect(JSON.parse(decrypt(encryptedExtraHeaders))).toEqual(
existingExtraHeaders,
);
});
it("should test an operator-provisioned Bedrock connection without revalidating its stored secret", async () => {
const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "eu";
try {
const existingKey = await prisma.llmApiKeys.create({
data: {
projectId,
provider: "bedrock-runtime",
adapter: LLMAdapter.Bedrock,
secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
displaySecretKey: "Default AWS credentials",
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "us-east-1" },
},
});
const result = await caller.llmApiKey.testUpdate({
id: existingKey.id,
projectId,
provider: existingKey.provider,
adapter: LLMAdapter.Bedrock,
customModels: existingKey.customModels,
config: { region: "us-east-1" },
});
expect(result).toEqual({ success: true });
expect(mockGenerateLLMText).toHaveBeenCalledWith(
expect.objectContaining({
connection: expect.objectContaining({ projectId }),
}),
);
} finally {
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
}
});
it("should fill masked header values from the stored headers in testUpdate", async () => {
await caller.llmApiKey.create({
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-original",
baseURL: "https://api.openai.com/v1",
extraHeaders: { "X-Api-Key": "stored-token", "X-Team": "stored-team" },
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider: "openai" },
});
const result = await caller.llmApiKey.testUpdate({
id: existingKey.id,
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
baseURL: "https://api.openai.com/v1",
extraHeaders: { "X-Api-Key": "", "X-Team": "new-team" },
});
expect(result).toEqual({ success: true });
const connection = mockGenerateLLMText.mock.calls[0][0].connection;
expect(JSON.parse(decrypt(connection.extraHeaders!))).toEqual({
"X-Api-Key": "stored-token",
"X-Team": "new-team",
});
});
it("should not fill masked header values from the stored headers when testUpdate changes the base URL", async () => {
await caller.llmApiKey.create({
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-original",
baseURL: "https://api.openai.com/v1",
extraHeaders: { "X-Api-Key": "stored-token" },
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider: "openai" },
});
const result = await caller.llmApiKey.testUpdate({
id: existingKey.id,
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-rotated",
baseURL: "https://example.net/v1",
extraHeaders: { "X-Api-Key": "", "X-Team": "new-team" },
});
expect(result).toEqual({ success: true });
const connection = mockGenerateLLMText.mock.calls[0][0].connection;
expect(connection.baseURL).toBe("https://example.net/v1");
expect(JSON.parse(decrypt(connection.extraHeaders!))).toEqual({
"X-Team": "new-team",
});
});
it("should allow testUpdate when the base URL changes and a new secret key is provided", async () => {
const existingExtraHeaders = {
Authorization: "Bearer stored-token",
"X-Custom-Header": "stored-value",
};
await caller.llmApiKey.create({
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-original",
baseURL: "https://api.openai.com/v1",
extraHeaders: existingExtraHeaders,
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider: "openai",
},
});
const result = await caller.llmApiKey.testUpdate({
id: existingKey.id,
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-rotated",
baseURL: "https://example.net/v1",
});
expect(result).toEqual({ success: true });
expect(mockGenerateLLMText).toHaveBeenCalledTimes(1);
const connection = mockGenerateLLMText.mock.calls[0][0].connection;
expect(connection.baseURL).toBe("https://example.net/v1");
expect(decrypt(connection.secretKey)).toBe("sk-rotated");
expect(connection.projectId).toBeUndefined();
expect(connection.extraHeaders).toBeUndefined();
});
it("should reject updating the base URL without a new secret key", async () => {
const existingExtraHeaders = {
Authorization: "Bearer stored-token",
};
await caller.llmApiKey.create({
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-original",
baseURL: "https://api.openai.com/v1",
extraHeaders: existingExtraHeaders,
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider: "openai",
},
});
await expect(
caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
baseURL: "https://example.net/v1",
}),
).rejects.toThrow("Secret key is required when changing the base URL");
const unchangedKey = await prisma.llmApiKeys.findUniqueOrThrow({
where: {
id: existingKey.id,
projectId,
},
});
expect(unchangedKey.baseURL).toBe("https://api.openai.com/v1");
expect(decrypt(unchangedKey.secretKey)).toBe("sk-original");
expect(JSON.parse(decrypt(unchangedKey.extraHeaders as string))).toEqual(
existingExtraHeaders,
);
});
it("should not reuse stored extra headers when updating the base URL", async () => {
await caller.llmApiKey.create({
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-original",
baseURL: "https://api.openai.com/v1",
extraHeaders: {
Authorization: "Bearer stored-token",
"X-Custom-Header": "stored-value",
},
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider: "openai",
},
});
await caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-rotated",
baseURL: "https://example.net/v1",
extraHeaders: {
Authorization: "",
"X-Custom-Header": "",
},
});
const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
where: {
id: existingKey.id,
projectId,
},
});
expect(updatedKey.baseURL).toBe("https://example.net/v1");
expect(decrypt(updatedKey.secretKey)).toBe("sk-rotated");
expect(updatedKey.extraHeaders).toBeNull();
expect(updatedKey.extraHeaderKeys).toEqual([]);
});
it("should allow new extra headers when updating the base URL", async () => {
await caller.llmApiKey.create({
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-original",
baseURL: "https://api.openai.com/v1",
extraHeaders: {
Authorization: "Bearer stored-token",
"X-Old-Header": "stored-value",
},
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider: "openai",
},
});
await caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider: "openai",
adapter: LLMAdapter.OpenAI,
secretKey: "sk-rotated",
baseURL: "https://example.net/v1",
extraHeaders: {
Authorization: "Bearer rotated-token",
"X-Old-Header": "",
},
});
const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
where: {
id: existingKey.id,
projectId,
},
});
expect(JSON.parse(decrypt(updatedKey.extraHeaders as string))).toEqual({
Authorization: "Bearer rotated-token",
});
expect(updatedKey.extraHeaderKeys).toEqual(["Authorization"]);
});
it("should create and update an llm api key", async () => {
const secret = "test-secret";
const provider = "openai";
const adapter = LLMAdapter.OpenAI;
const customModels = ["fancy-gpt-3.5-turbo"];
const baseURL = "https://example.com/v1";
const withDefaultModels = false;
// Create initial key
await caller.llmApiKey.create({
projectId,
secretKey: secret,
provider,
adapter,
baseURL,
customModels,
withDefaultModels,
});
// Verify initial key
const initialKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
},
});
expect(initialKeys.length).toBe(1);
expect(initialKeys[0].projectId).toBe(projectId);
expect(initialKeys[0].secretKey).not.toBeNull();
expect(initialKeys[0].secretKey).not.toEqual(secret);
expect(initialKeys[0].provider).toBe(provider);
expect(initialKeys[0].adapter).toBe(adapter);
expect(initialKeys[0].baseURL).toBe(baseURL);
expect(initialKeys[0].customModels).toEqual(customModels);
expect(initialKeys[0].withDefaultModels).toBe(withDefaultModels);
// Update the key
const newSecret = "new-test-secret";
const newBaseURL = "https://example.org/v1";
const newCustomModels = ["new-fancy-gpt-3.5-turbo"];
const newWithDefaultModels = true;
await caller.llmApiKey.update({
id: initialKeys[0].id,
projectId,
secretKey: newSecret,
provider,
adapter,
baseURL: newBaseURL,
customModels: newCustomModels,
withDefaultModels: newWithDefaultModels,
});
// Verify updated key
const updatedKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
},
});
expect(updatedKeys.length).toBe(1);
expect(updatedKeys[0].projectId).toBe(projectId);
expect(updatedKeys[0].secretKey).not.toBeNull();
expect(updatedKeys[0].secretKey).not.toEqual(newSecret);
expect(updatedKeys[0].provider).toBe(provider); // Should not change
expect(updatedKeys[0].adapter).toBe(adapter); // Should not change
expect(updatedKeys[0].baseURL).toBe(newBaseURL);
expect(updatedKeys[0].customModels).toEqual(newCustomModels);
expect(updatedKeys[0].withDefaultModels).toBe(newWithDefaultModels);
});
it("should scope the llm api key update write by project id", async () => {
const secret = "test-secret";
const provider = "openai";
const adapter = LLMAdapter.OpenAI;
await caller.llmApiKey.create({
projectId,
secretKey: secret,
provider,
adapter,
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider,
},
});
const updateSpy = vi.spyOn(prisma.llmApiKeys, "update");
try {
await caller.llmApiKey.update({
id: existingKey.id,
projectId,
secretKey: "new-test-secret",
provider,
adapter,
});
expect(updateSpy).toHaveBeenCalledWith(
expect.objectContaining({
where: {
id: existingKey.id,
projectId,
},
}),
);
} finally {
updateSpy.mockRestore();
}
});
it("should update a Bedrock Access key auth to a Bedrock API key", async () => {
const provider = "bedrock";
await caller.llmApiKey.create({
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: JSON.stringify({
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
}),
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "us-east-1" },
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: {
projectId,
provider,
},
});
await caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: JSON.stringify({ apiKey: "bedrock-api-key-5678" }),
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "eu-west-1" },
});
const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
where: { id: existingKey.id },
});
expect(decrypt(updatedKey.secretKey)).toBe(
JSON.stringify({
apiKey: "bedrock-api-key-5678",
}),
);
expect(updatedKey.displaySecretKey).toBe("...5678");
expect(updatedKey.config).toEqual({ region: "eu-west-1" });
});
it("should update a Bedrock API key auth to Access keys", async () => {
const provider = "bedrock";
await caller.llmApiKey.create({
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: JSON.stringify({ apiKey: "bedrock-api-key-1234" }),
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "us-east-1" },
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider },
});
await caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: JSON.stringify({
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
}),
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "eu-west-1" },
});
const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
where: { id: existingKey.id },
});
expect(decrypt(updatedKey.secretKey)).toBe(
JSON.stringify({
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
}),
);
expect(updatedKey.displaySecretKey).toBe("...EKEY");
expect(updatedKey.config).toEqual({ region: "eu-west-1" });
});
it("should update a Bedrock DefaultCredentials key to explicit Access keys", async () => {
const provider = "bedrock";
const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
try {
// Simulate self-hosted to allow default credentials
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
await caller.llmApiKey.create({
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "us-east-1" },
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider },
});
expect(decrypt(existingKey.secretKey)).toBe(
BEDROCK_USE_DEFAULT_CREDENTIALS,
);
expect(existingKey.displaySecretKey).toBe("Default AWS credentials");
await caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: JSON.stringify({
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
}),
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "eu-west-1" },
});
const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
where: { id: existingKey.id },
});
expect(decrypt(updatedKey.secretKey)).toBe(
JSON.stringify({
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
}),
);
expect(updatedKey.displaySecretKey).toBe("...EKEY");
expect(updatedKey.config).toEqual({ region: "eu-west-1" });
} finally {
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
}
});
it("should update a Bedrock DefaultCredentials key to a Bedrock API key", async () => {
const provider = "bedrock";
const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
try {
// Simulate self-hosted to allow default credentials
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
await caller.llmApiKey.create({
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "us-east-1" },
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider },
});
await caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: JSON.stringify({ apiKey: "bedrock-api-key-9999" }),
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "eu-west-1" },
});
const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
where: { id: existingKey.id },
});
expect(decrypt(updatedKey.secretKey)).toBe(
JSON.stringify({ apiKey: "bedrock-api-key-9999" }),
);
expect(updatedKey.displaySecretKey).toBe("...9999");
expect(updatedKey.config).toEqual({ region: "eu-west-1" });
} finally {
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
}
});
it("should reject updating a Bedrock key back to DefaultCredentials on cloud", async () => {
const provider = "bedrock";
await caller.llmApiKey.create({
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: JSON.stringify({
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
}),
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "us-east-1" },
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider },
});
await expect(
caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "eu-west-1" },
}),
).rejects.toThrow(
"Default AWS credentials are only allowed for Bedrock in self-hosted deployments",
);
});
it("should update a Bedrock Access key auth back to DefaultCredentials (self-hosted)", async () => {
const provider = "bedrock";
const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
await caller.llmApiKey.create({
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: JSON.stringify({
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
}),
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "us-east-1" },
});
const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider },
});
try {
// Simulate self-hosted deployment where default credentials are allowed
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
await caller.llmApiKey.update({
id: existingKey.id,
projectId,
provider,
adapter: LLMAdapter.Bedrock,
secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
withDefaultModels: false,
config: { region: "eu-west-1" },
});
} finally {
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
}
const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
where: { id: existingKey.id },
});
expect(decrypt(updatedKey.secretKey)).toBe(BEDROCK_USE_DEFAULT_CREDENTIALS);
expect(updatedKey.displaySecretKey).toBe("Default AWS credentials");
expect(updatedKey.config).toEqual({ region: "eu-west-1" });
});
it("should update only the secret key", async () => {
const secret = "test-secret";
const provider = "openai";
const adapter = LLMAdapter.OpenAI;
const customModels = ["fancy-gpt-3.5-turbo"];
const baseURL = "https://example.com/v1";
const withDefaultModels = false;
// Create initial key
await caller.llmApiKey.create({
projectId,
secretKey: secret,
provider,
adapter,
baseURL,
customModels,
withDefaultModels,
});
const initialKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
provider,
},
});
expect(initialKeys.length).toBe(1);
const initialDisplaySecretKey = initialKeys[0].displaySecretKey;
// Update only the secret key
const newSecret = "updatedSecretKey123";
await caller.llmApiKey.update({
id: initialKeys[0].id,
projectId,
secretKey: newSecret,
provider,
adapter,
});
// Verify updated key
const updatedKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
provider,
},
});
expect(updatedKeys.length).toBe(1);
expect(decrypt(updatedKeys[0].secretKey)).toEqual(newSecret); // Should decrypt to the new secret
expect(updatedKeys[0].displaySecretKey).not.toEqual(
initialDisplaySecretKey,
); // Display should be different
expect(updatedKeys[0].displaySecretKey).toEqual("...y123"); // Should match format with hyphens allowed
// Other fields should remain unchanged
expect(updatedKeys[0].baseURL).toBe(baseURL);
expect(updatedKeys[0].customModels).toEqual(customModels);
expect(updatedKeys[0].withDefaultModels).toBe(withDefaultModels);
expect(updatedKeys[0].provider).toBe(provider);
expect(updatedKeys[0].adapter).toBe(adapter);
});
it("should update only the extra headers", async () => {
const secret = "test-secret";
const provider = "openai";
const adapter = LLMAdapter.OpenAI;
const customModels = ["fancy-gpt-3.5-turbo"];
const baseURL = "https://example.com/v1";
const withDefaultModels = false;
const extraHeaders = {
"X-Custom-Header": "custom-value",
Authorization: "Bearer token123",
};
// Create initial key with extra headers
await caller.llmApiKey.create({
projectId,
secretKey: secret,
provider,
adapter,
baseURL,
customModels,
withDefaultModels,
extraHeaders,
});
const initialKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
},
});
expect(initialKeys.length).toBe(1);
expect(initialKeys[0].extraHeaders).not.toBeNull();
expect(initialKeys[0].extraHeaderKeys).toEqual(Object.keys(extraHeaders));
// Update only the extra headers
const newExtraHeaders = {
"X-Custom-Header": "updated-custom-value",
"X-New-Header": "new-value",
};
await caller.llmApiKey.update({
id: initialKeys[0].id,
projectId,
provider,
adapter,
extraHeaders: newExtraHeaders,
});
// Verify updated key
const updatedKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
},
});
expect(updatedKeys.length).toBe(1);
expect(updatedKeys[0].extraHeaders).not.toBeNull();
expect(updatedKeys[0].extraHeaders).not.toEqual(
initialKeys[0].extraHeaders,
); // Should be different
expect(updatedKeys[0].extraHeaderKeys).toEqual(
Object.keys(newExtraHeaders),
);
// Other fields should remain unchanged
expect(updatedKeys[0].secretKey).toEqual(initialKeys[0].secretKey); // Secret should be same
expect(updatedKeys[0].displaySecretKey).toEqual(
initialKeys[0].displaySecretKey,
); // Display should be same
expect(updatedKeys[0].baseURL).toBe(baseURL);
expect(updatedKeys[0].customModels).toEqual(customModels);
expect(updatedKeys[0].withDefaultModels).toBe(withDefaultModels);
expect(updatedKeys[0].provider).toBe(provider);
expect(updatedKeys[0].adapter).toBe(adapter);
});
it("should remove extra headers when updated with empty object", async () => {
const secret = "test-secret";
const provider = "openai";
const adapter = LLMAdapter.OpenAI;
const extraHeaders = {
"X-Custom-Header": "custom-value",
Authorization: "Bearer token123",
};
// Create initial key with extra headers
await caller.llmApiKey.create({
projectId,
secretKey: secret,
provider,
adapter,
extraHeaders,
});
const initialKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
},
});
expect(initialKeys.length).toBe(1);
expect(initialKeys[0].extraHeaders).not.toBeNull();
expect(initialKeys[0].extraHeaderKeys).toEqual(Object.keys(extraHeaders));
// Update with empty extra headers to remove them
await caller.llmApiKey.update({
id: initialKeys[0].id,
projectId,
provider,
adapter,
extraHeaders: {},
});
// Verify updated key
const updatedKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
},
});
expect(updatedKeys.length).toBe(1);
expect(updatedKeys[0].extraHeaders).toBeNull();
expect(updatedKeys[0].extraHeaderKeys).toEqual([]);
// Other fields should remain unchanged
expect(updatedKeys[0].secretKey).toEqual(initialKeys[0].secretKey);
expect(updatedKeys[0].displaySecretKey).toEqual(
initialKeys[0].displaySecretKey,
);
expect(updatedKeys[0].provider).toBe(provider);
expect(updatedKeys[0].adapter).toBe(adapter);
});
it("should partially update extra headers preserving existing values for empty inputs", async () => {
const secret = "test-secret";
const provider = "openai";
const adapter = LLMAdapter.OpenAI;
const extraHeaders = {
"X-Custom-Header": "custom-value",
Authorization: "Bearer token123",
"X-Another-Header": "another-value",
};
// Create initial key with extra headers
await caller.llmApiKey.create({
projectId,
secretKey: secret,
provider,
adapter,
extraHeaders,
});
const initialKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
},
});
expect(initialKeys.length).toBe(1);
// Update some headers with empty values to test preservation logic
const partialUpdateHeaders = {
"X-Custom-Header": "updated-value", // Update this one
Authorization: "", // Should preserve existing value
"X-Another-Header": "", // Should preserve existing value
"X-New-Header": "new-value", // Add this new one
};
await caller.llmApiKey.update({
id: initialKeys[0].id,
projectId,
provider,
adapter,
extraHeaders: partialUpdateHeaders,
});
// Verify updated key
const updatedKeys = await prisma.llmApiKeys.findMany({
where: {
projectId,
},
});
expect(updatedKeys.length).toBe(1);
expect(updatedKeys[0].extraHeaders).not.toBeNull();
// Should have 4 headers: 3 original + 1 new
expect(updatedKeys[0].extraHeaderKeys).toHaveLength(4);
expect(updatedKeys[0].extraHeaderKeys).toContain("X-Custom-Header");
expect(updatedKeys[0].extraHeaderKeys).toContain("Authorization");
expect(updatedKeys[0].extraHeaderKeys).toContain("X-Another-Header");
expect(updatedKeys[0].extraHeaderKeys).toContain("X-New-Header");
});
it("rolls back connection deletion when audit logging fails", async () => {
const connection = await prisma.llmApiKeys.create({
data: {
projectId,
secretKey: encrypt("project-secret"),
displaySecretKey: "...cret",
provider: `audit-rollback-${randomUUID()}`,
adapter: LLMAdapter.OpenAI,
},
});
const auditLogSpy = vi
.spyOn(auditLogs, "auditLog")
.mockRejectedValueOnce(new Error("audit unavailable"));
try {
await expect(
caller.llmApiKey.delete({ projectId, id: connection.id }),
).rejects.toThrow();
await expect(
prisma.llmApiKeys.findUniqueOrThrow({ where: { id: connection.id } }),
).resolves.toBeDefined();
} finally {
auditLogSpy.mockRestore();
}
});
describe("deleting a connection pauses the evaluators that ran on it", () => {
const PROVIDER = "openai";
/** Versions are given oldest-first; the last one is the evaluator's head. */
const createV2Evaluator = async (
versions: Array<{ provider: string | null; model: string | null }>,
) => {
const evaluator = await prisma.evaluator.create({
data: {
projectId,
name: `evaluator-${randomUUID()}`,
type: EvalTemplateType.LLM_AS_JUDGE,
versions: {
create: versions.map((version, index) => ({
version: index + 1,
prompt: "Evaluate {{output}}",
vars: ["output"],
...version,
})),
},
},
});
return evaluator.id;
};
it("blocks evaluators by their current model and leaves the rest running", async () => {
await caller.llmApiKey.create({
projectId,
secretKey: "test-secret",
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
customModels: [],
withDefaultModels: true,
});
const connection = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider: PROVIDER },
});
const [
v2OnProvider,
v2OnDefaultModel,
v2MovedOffProvider,
v2OnOtherProvider,
] = await Promise.all([
createV2Evaluator([{ provider: PROVIDER, model: "gpt-4o" }]),
createV2Evaluator([{ provider: null, model: null }]),
// Upgraded off the deleted provider: only the head version counts.
createV2Evaluator([
{ provider: PROVIDER, model: "gpt-4o" },
{ provider: "anthropic", model: "claude" },
]),
createV2Evaluator([{ provider: "anthropic", model: "claude" }]),
]);
// Use this provider as the project's default too, so both block reasons
// fire when its last effective connection is deleted.
await prisma.defaultLlmModel.create({
data: {
projectId,
llmApiKeyId: connection.id,
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
model: "gpt-4o",
},
});
await caller.llmApiKey.delete({ projectId, id: connection.id });
expect(mockFinalizeEvaluatorBlocks).toHaveBeenCalledOnce();
expect(mockFinalizeEvaluatorBlocks).toHaveBeenCalledWith({
projectId,
source: EvaluatorBlockSource.LLM_API_KEY_DELETION,
evaluatorIdsByReason: {
[EvaluatorBlockReason.LLM_CONNECTION_MISSING]: [v2OnProvider],
[EvaluatorBlockReason.DEFAULT_EVAL_MODEL_MISSING]: [v2OnDefaultModel],
},
});
const evaluators = await prisma.evaluator.findMany({
where: { projectId },
});
const blockStateById = new Map(
evaluators.map((row) => [
row.id,
{ blocked: row.blockedAt !== null, reason: row.blockReason },
]),
);
expect(blockStateById.get(v2OnProvider)).toEqual({
blocked: true,
reason: EvaluatorBlockReason.LLM_CONNECTION_MISSING,
});
expect(blockStateById.get(v2OnDefaultModel)).toEqual({
blocked: true,
reason: EvaluatorBlockReason.DEFAULT_EVAL_MODEL_MISSING,
});
for (const untouched of [v2MovedOffProvider, v2OnOtherProvider]) {
expect(blockStateById.get(untouched)).toEqual({
blocked: false,
reason: null,
});
}
expect(
await prisma.defaultLlmModel.findUnique({ where: { projectId } }),
).toBeNull();
});
it("keeps the default model when deleting an override with an organization fallback", async () => {
const organizationConnection = await prisma.llmApiKeys.create({
data: {
organizationId: orgId,
secretKey: encrypt("organization-secret"),
displaySecretKey: "...cret",
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
},
});
const projectConnection = await prisma.llmApiKeys.create({
data: {
projectId,
secretKey: encrypt("project-secret"),
displaySecretKey: "...cret",
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
},
});
const providerEvaluatorId = await createV2Evaluator([
{ provider: PROVIDER, model: "gpt-4o" },
]);
const defaultModelEvaluatorId = await createV2Evaluator([
{ provider: null, model: null },
]);
await prisma.defaultLlmModel.create({
data: {
projectId,
llmApiKeyId: projectConnection.id,
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
model: "gpt-4o",
},
});
await prisma.topicConfig.create({
data: {
projectId,
enabled: true,
summaryLlmApiKeyId: projectConnection.id,
summaryModel: "gpt-6-luna",
embeddingLlmApiKeyId: projectConnection.id,
embeddingModel: "text-embedding-3-small",
clusteringLlmApiKeyId: projectConnection.id,
clusteringModel: "gpt-5.6-terra",
},
});
await caller.llmApiKey.delete({
projectId,
id: projectConnection.id,
});
expect(mockFinalizeEvaluatorBlocks).not.toHaveBeenCalled();
expect(
await prisma.evaluator.findUniqueOrThrow({
where: { id: providerEvaluatorId },
select: { blockedAt: true, blockReason: true },
}),
).toEqual({ blockedAt: null, blockReason: null });
expect(
await prisma.evaluator.findUniqueOrThrow({
where: { id: defaultModelEvaluatorId },
select: { blockedAt: true, blockReason: true },
}),
).toEqual({ blockedAt: null, blockReason: null });
expect(
await prisma.defaultLlmModel.findUniqueOrThrow({
where: { projectId },
select: {
llmApiKeyId: true,
provider: true,
adapter: true,
model: true,
},
}),
).toEqual({
llmApiKeyId: organizationConnection.id,
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
model: "gpt-4o",
});
expect(
await prisma.topicConfig.findUniqueOrThrow({
where: { projectId },
select: {
enabled: true,
summaryLlmApiKeyId: true,
embeddingLlmApiKeyId: true,
clusteringLlmApiKeyId: true,
},
}),
).toEqual({
enabled: true,
summaryLlmApiKeyId: organizationConnection.id,
embeddingLlmApiKeyId: organizationConnection.id,
clusteringLlmApiKeyId: organizationConnection.id,
});
});
it("deletes the default model when the organization fallback uses a different adapter", async () => {
await prisma.llmApiKeys.createMany({
data: [
{
organizationId: orgId,
secretKey: encrypt("organization-secret"),
displaySecretKey: "...cret",
provider: PROVIDER,
adapter: LLMAdapter.Anthropic,
},
{
projectId,
secretKey: encrypt("project-secret"),
displaySecretKey: "...cret",
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
},
],
});
const projectConnection = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider: PROVIDER },
});
const defaultModelEvaluatorId = await createV2Evaluator([
{ provider: null, model: null },
]);
await prisma.defaultLlmModel.create({
data: {
projectId,
llmApiKeyId: projectConnection.id,
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
model: "gpt-4o",
},
});
await caller.llmApiKey.delete({
projectId,
id: projectConnection.id,
});
expect(mockFinalizeEvaluatorBlocks).toHaveBeenCalledWith({
projectId,
source: EvaluatorBlockSource.LLM_API_KEY_DELETION,
evaluatorIdsByReason: {
[EvaluatorBlockReason.LLM_CONNECTION_MISSING]: [],
[EvaluatorBlockReason.DEFAULT_EVAL_MODEL_MISSING]: [
defaultModelEvaluatorId,
],
},
});
expect(
await prisma.defaultLlmModel.findUnique({ where: { projectId } }),
).toBeNull();
});
it("does not rebind organization-backed configuration to an incompatible project override", async () => {
const organizationConnection = await prisma.llmApiKeys.create({
data: {
organizationId: orgId,
secretKey: encrypt("organization-secret"),
displaySecretKey: "...cret",
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
},
});
await prisma.llmApiKeys.create({
data: {
projectId,
secretKey: encrypt("project-secret"),
displaySecretKey: "...cret",
provider: PROVIDER,
adapter: LLMAdapter.Anthropic,
},
});
await prisma.defaultLlmModel.create({
data: {
projectId,
llmApiKeyId: organizationConnection.id,
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
model: "gpt-4o",
},
});
await prisma.topicConfig.create({
data: {
projectId,
enabled: true,
summaryLlmApiKeyId: organizationConnection.id,
summaryModel: "gpt-6-luna",
embeddingLlmApiKeyId: organizationConnection.id,
embeddingModel: "text-embedding-3-small",
clusteringLlmApiKeyId: organizationConnection.id,
clusteringModel: "gpt-5.6-terra",
},
});
await caller.organizationLlmApiKey.delete({
orgId,
id: organizationConnection.id,
});
expect(
await prisma.defaultLlmModel.findUnique({
where: { projectId },
}),
).toBeNull();
expect(
await prisma.topicConfig.findUniqueOrThrow({
where: { projectId },
select: {
enabled: true,
summaryLlmApiKeyId: true,
embeddingLlmApiKeyId: true,
clusteringLlmApiKeyId: true,
},
}),
).toEqual({
enabled: false,
summaryLlmApiKeyId: null,
embeddingLlmApiKeyId: null,
clusteringLlmApiKeyId: null,
});
});
it("turns Topics off and records why when its connection is deleted", async () => {
await caller.llmApiKey.create({
projectId,
secretKey: "test-secret",
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
customModels: [],
withDefaultModels: true,
});
const connection = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider: PROVIDER },
});
await prisma.topicConfig.create({
data: {
projectId,
enabled: true,
summaryLlmApiKeyId: connection.id,
summaryModel: "gpt-6-luna",
embeddingLlmApiKeyId: connection.id,
embeddingModel: "text-embedding-3-small",
clusteringLlmApiKeyId: connection.id,
clusteringModel: "gpt-5.6-terra",
},
});
await caller.llmApiKey.delete({ projectId, id: connection.id });
expect(
await prisma.topicConfig.findUniqueOrThrow({
where: { projectId },
}),
).toMatchObject({
enabled: false,
blockReason: "LLM_CONNECTION_MISSING",
blockMessage: expect.stringContaining(`"${PROVIDER}" was deleted`),
blockedAt: expect.any(Date),
summaryLlmApiKeyId: null,
embeddingLlmApiKeyId: null,
clusteringLlmApiKeyId: null,
});
});
it("clears deleted Topics connections without marking an already disabled config as paused", async () => {
await caller.llmApiKey.create({
projectId,
secretKey: "test-secret",
provider: PROVIDER,
adapter: LLMAdapter.OpenAI,
customModels: [],
withDefaultModels: true,
});
const connection = await prisma.llmApiKeys.findFirstOrThrow({
where: { projectId, provider: PROVIDER },
});
await prisma.topicConfig.create({
data: {
projectId,
enabled: false,
summaryLlmApiKeyId: connection.id,
summaryModel: "gpt-6-luna",
embeddingLlmApiKeyId: connection.id,
embeddingModel: "text-embedding-3-small",
clusteringLlmApiKeyId: connection.id,
clusteringModel: "gpt-5.6-terra",
},
});
await caller.llmApiKey.delete({ projectId, id: connection.id });
expect(
await prisma.topicConfig.findUniqueOrThrow({
where: { projectId },
}),
).toMatchObject({
enabled: false,
blockReason: null,
blockMessage: null,
blockedAt: null,
summaryLlmApiKeyId: null,
embeddingLlmApiKeyId: null,
clusteringLlmApiKeyId: null,
});
});
});
});