Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

71.6 KB
Raw Blame
const mockFinalizeEvaluatorBlocks = vi.hoisted(() => vi.fn());

vi.mock("@langfuse/shared/src/server", async () => {
  const actual = await vi.importActual("@langfuse/shared/src/server");
  return {
    ...actual,
    finalizeEvaluatorBlocks: mockFinalizeEvaluatorBlocks,
    generateLLMText: vi.fn(),
  };
});

import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import * as auditLogs from "@/src/features/audit-logs/auditLog";
import type { Session } from "next-auth";
import { BEDROCK_USE_DEFAULT_CREDENTIALS, LLMAdapter } from "@langfuse/shared";
import { env } from "@/src/env.mjs";
import { randomUUID } from "crypto";
import {
  EvalTemplateType,
  EvaluatorBlockReason,
  prisma,
} from "@langfuse/shared/src/db";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import { decrypt, encrypt } from "@langfuse/shared/encryption";
import { AuthMethod } from "@/src/features/llm-api-key/types";
import { LlmConnectionService } from "@/src/features/llm-api-key/server/llmConnectionService";
import {
  createOrgProjectAndApiKey,
  DefaultEvalModelService,
  EvaluatorBlockSource,
  generateLLMText,
} from "@langfuse/shared/src/server";

const mockGenerateLLMText = vi.mocked(generateLLMText);

describe("llmApiKey.all RPC", () => {
  let projectId: string;
  let orgId: string;
  let session: Session;
  let caller: ReturnType<typeof appRouter.createCaller>;

  const createCallerForProjectRole = (
    projectRole: "ADMIN" | "MEMBER" | "VIEWER",
  ) => {
    const limitedSession: Session = {
      ...session,
      user: {
        ...session.user!,
        admin: false,
        organizations: [
          {
            ...session.user!.organizations[0],
            role: "MEMBER",
            projects: [
              {
                ...session.user!.organizations[0].projects[0],
                role: projectRole,
              },
            ],
          },
        ],
      },
    };

    const limitedCtx = createInnerTRPCContext({
      session: limitedSession,
      headers: {},
    });

    return appRouter.createCaller({ ...limitedCtx, prisma });
  };

  beforeEach(async () => {
    const setup = await createOrgProjectAndApiKey();
    projectId = setup.projectId;
    orgId = setup.orgId;
    mockFinalizeEvaluatorBlocks.mockReset().mockResolvedValue(undefined);
    mockGenerateLLMText.mockReset().mockResolvedValue({} as never);

    session = {
      expires: "1",
      user: {
        id: "user-1",
        name: "Demo User",
        canCreateOrganizations: true,
        organizations: [
          {
            id: orgId,
            role: "OWNER",
            plan: "cloud:hobby",
            cloudConfig: undefined,
            name: "Test Organization",
            metadata: {},
            aiFeaturesEnabled: false,
            aiTelemetryEnabled: false,
            projects: [
              {
                id: projectId,
                role: "ADMIN",
                name: "Test Project",
                deletedAt: null,
                retentionDays: null,
                hasTraces: false,
                metadata: {},
                createdAt: new Date().toISOString(),
              },
            ],
          },
        ],
        featureFlags: testFeatureFlags(),
        admin: true,
      },
      environment: {} as any,
    };

    const ctx = createInnerTRPCContext({ session, headers: {} });
    caller = appRouter.createCaller({ ...ctx, prisma });
  });

  it("should create an llm api key", async () => {
    const secret = "test-secret";
    const provider = "openai";
    const adapter = LLMAdapter.OpenAI;
    const customModels = ["fancy-gpt-3.5-turbo"];
    const baseURL = "https://example.com/v1";
    const withDefaultModels = false;

    const result = await caller.llmApiKey.create({
      projectId,
      secretKey: secret,
      provider,
      adapter,
      baseURL,
      customModels,
      withDefaultModels,
    });
    expect(result).toBeUndefined();

    const llmApiKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
      },
    });

    expect(llmApiKeys.length).toBe(1);
    expect(llmApiKeys[0].projectId).toBe(projectId);
    expect(llmApiKeys[0].secretKey).not.toBeNull();
    expect(llmApiKeys[0].secretKey).not.toEqual(secret);
    expect(llmApiKeys[0].provider).toBe(provider);
    expect(llmApiKeys[0].adapter).toBe(adapter);
    expect(llmApiKeys[0].baseURL).toBe(baseURL);
    expect(llmApiKeys[0].customModels).toEqual(customModels);
    expect(llmApiKeys[0].withDefaultModels).toBe(withDefaultModels);
    // this has to be 3 dots and the last 4 characters of the secret
    expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
  });

  it("resolves an organization connection for the project default model", async () => {
    const provider = `organization-default-${randomUUID()}`;
    const organizationConnection = await prisma.llmApiKeys.create({
      data: {
        organizationId: orgId,
        provider,
        adapter: LLMAdapter.OpenAI,
        secretKey: encrypt("organization-secret"),
        displaySecretKey: "...cret",
      },
    });
    await prisma.defaultLlmModel.create({
      data: {
        projectId,
        llmApiKeyId: organizationConnection.id,
        provider,
        adapter: LLMAdapter.OpenAI,
        model: "gpt-4o",
      },
    });

    const result =
      await DefaultEvalModelService.fetchValidModelConfig(projectId);

    expect(result).toMatchObject({
      valid: true,
      config: {
        apiKey: {
          id: organizationConnection.id,
          projectId,
        },
      },
    });
  });

  it("returns CONFLICT when creating a key for an existing provider", async () => {
    const provider = `duplicate-provider-${randomUUID()}`;
    const input = {
      projectId,
      secretKey: "test-secret",
      provider,
      adapter: LLMAdapter.OpenAI,
      customModels: [],
      withDefaultModels: true,
    };

    await caller.llmApiKey.create(input);

    await expect(caller.llmApiKey.create(input)).rejects.toMatchObject({
      code: "CONFLICT",
      message: "There already exists an API key for this provider.",
    });

    await expect(
      prisma.llmApiKeys.count({ where: { projectId, provider } }),
    ).resolves.toBe(1);
  });

  it("rejects an incompatible project override without blocking edits to an existing mismatch", async () => {
    const provider = `incompatible-project-override-${randomUUID()}`;
    await prisma.llmApiKeys.create({
      data: {
        organizationId: orgId,
        provider,
        adapter: LLMAdapter.OpenAI,
        secretKey: encrypt("organization-secret"),
        displaySecretKey: "...cret",
      },
    });

    await expect(
      caller.llmApiKey.create({
        projectId,
        provider,
        adapter: LLMAdapter.Anthropic,
        secretKey: "project-secret",
      }),
    ).rejects.toThrow(
      "Project and organization connections with the same provider name must use the same adapter.",
    );

    const incompatibleProjectConnection = await prisma.llmApiKeys.create({
      data: {
        projectId,
        provider,
        adapter: LLMAdapter.Anthropic,
        secretKey: encrypt("project-secret"),
        displaySecretKey: "...cret",
      },
    });
    await expect(
      caller.llmApiKey.update({
        id: incompatibleProjectConnection.id,
        projectId,
        provider,
        adapter: LLMAdapter.Anthropic,
        customModels: ["claude-sonnet-4-5"],
      }),
    ).resolves.toBeUndefined();
    await expect(
      prisma.llmApiKeys.findUniqueOrThrow({
        where: { id: incompatibleProjectConnection.id },
        select: { customModels: true },
      }),
    ).resolves.toEqual({ customModels: ["claude-sonnet-4-5"] });

    const service = new LlmConnectionService();
    await expect(
      service.upsert({
        owner: {
          type: "project",
          projectId,
          organizationId: orgId,
        },
        input: {
          provider,
          adapter: LLMAdapter.Anthropic,
          secretKey: "rotated-project-secret",
        },
        actor: { userId: session.user!.id, orgId, projectId },
      }),
    ).resolves.toMatchObject({ created: false });
    await expect(
      service.upsert({
        owner: {
          type: "project",
          projectId,
          organizationId: orgId,
        },
        input: {
          provider,
          adapter: LLMAdapter.Azure,
          secretKey: "azure-project-secret",
        },
        actor: { userId: session.user!.id, orgId, projectId },
      }),
    ).rejects.toThrow(
      "Project and organization connections with the same provider name must use the same adapter.",
    );
  });

  it("should rotate a secret without revalidating an unchanged stored base URL", async () => {
    const provider = `operator-provisioned-${randomUUID()}`;
    const connection = await prisma.llmApiKeys.create({
      data: {
        projectId,
        provider,
        adapter: LLMAdapter.OpenAI,
        secretKey: encrypt("old-secret"),
        displaySecretKey: "...cret",
        baseURL: "http://localhost:11434/v1",
      },
    });

    const result = await new LlmConnectionService().upsert({
      owner: {
        type: "project",
        projectId,
        organizationId: orgId,
      },
      input: {
        provider,
        adapter: LLMAdapter.OpenAI,
        secretKey: "rotated-secret",
        baseURL: connection.baseURL,
      },
      actor: { userId: session.user!.id, orgId, projectId },
    });

    expect(result.created).toBe(false);
    const updatedConnection = await prisma.llmApiKeys.findUniqueOrThrow({
      where: { id: connection.id },
    });
    expect(decrypt(updatedConnection.secretKey)).toBe("rotated-secret");
  });

  it("should create a Bedrock llm api key with a Bedrock API key", async () => {
    const secret = "bedrock-api-key-1234";

    await caller.llmApiKey.create({
      projectId,
      secretKey: JSON.stringify({ apiKey: secret }),
      provider: "bedrock",
      adapter: LLMAdapter.Bedrock,
      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
      withDefaultModels: false,
      config: { region: "us-east-1" },
    });

    const llmApiKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider: "bedrock",
      },
    });

    expect(decrypt(llmApiKey.secretKey)).toBe(
      JSON.stringify({
        apiKey: secret,
      }),
    );
    expect(llmApiKey.displaySecretKey).toBe("...1234");
    expect(llmApiKey.config).toEqual({ region: "us-east-1" });
  });

  it("should reject creating a Bedrock key with invalid secret key JSON", async () => {
    await expect(
      caller.llmApiKey.create({
        projectId,
        secretKey: JSON.stringify({ unknownField: "value" }),
        provider: "bedrock",
        adapter: LLMAdapter.Bedrock,
        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
        withDefaultModels: false,
        config: { region: "us-east-1" },
      }),
    ).rejects.toThrow("Invalid Bedrock credentials");
  });

  it("should block creating an llm api key with a localhost base URL", async () => {
    await expect(
      caller.llmApiKey.create({
        projectId,
        secretKey: "test-secret",
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        baseURL: "http://localhost:11434/v1",
      }),
    ).rejects.toThrow("Invalid base URL: Blocked hostname detected");
  });

  it("should create and get an llm api key", async () => {
    const secret = "test-secret";
    const provider = "openai";
    const adapter = LLMAdapter.OpenAI;
    const customModels = ["fancy-gpt-3.5-turbo"];
    const baseURL = "https://example.com/v1";
    const withDefaultModels = false;

    await caller.llmApiKey.create({
      projectId,
      secretKey: secret,
      provider,
      adapter,
      baseURL,
      customModels,
      withDefaultModels,
    });

    const dbLlmApiKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
      },
    });

    expect(dbLlmApiKeys.length).toBe(1);

    const { data: llmApiKeys } = await caller.llmApiKey.all({
      projectId,
    });

    expect(llmApiKeys.length).toBe(1);
    expect(llmApiKeys[0].projectId).toBe(projectId);
    expect(llmApiKeys[0].secretKey).not.toBeNull();
    expect(llmApiKeys[0].secretKey).not.toEqual(secret);
    expect(llmApiKeys[0].provider).toBe(provider);
    expect(llmApiKeys[0].adapter).toBe(adapter);
    expect(llmApiKeys[0].baseURL).toBe(baseURL);
    expect(llmApiKeys[0].customModels).toEqual(customModels);
    expect(llmApiKeys[0].withDefaultModels).toBe(withDefaultModels);
    // this has to be 3 dots and the last 4 characters of the secret
    expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);

    // response must not contain the secret key itself
    const secretKey = llmApiKeys[0].secretKey;
    expect(secretKey).toBeUndefined();
  });

  it("should create and get an OpenAI llm api key with Responses API config", async () => {
    await caller.llmApiKey.create({
      projectId,
      secretKey: "test-secret",
      provider: "openai-responses",
      adapter: LLMAdapter.OpenAI,
      baseURL: "https://example.com/v1",
      customModels: ["openai.gpt-5.4"],
      withDefaultModels: false,
      config: { useResponsesApi: true },
    });

    const { data: llmApiKeys } = await caller.llmApiKey.all({
      projectId,
    });

    expect(llmApiKeys).toHaveLength(1);
    expect(llmApiKeys[0].config).toEqual({ useResponsesApi: true });
    expect(llmApiKeys[0].customModels).toEqual(["openai.gpt-5.4"]);
    expect(llmApiKeys[0].secretKey).toBeUndefined();
  });

  it("should update an OpenAI llm api key to disable Responses API config", async () => {
    await caller.llmApiKey.create({
      projectId,
      secretKey: "test-secret",
      provider: "openai-responses",
      adapter: LLMAdapter.OpenAI,
      baseURL: "https://example.com/v1",
      customModels: ["openai.gpt-5.4"],
      withDefaultModels: false,
      config: { useResponsesApi: true },
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider: "openai-responses",
      },
    });

    const result = await caller.llmApiKey.update({
      id: existingKey.id,
      projectId,
      provider: "openai-responses",
      adapter: LLMAdapter.OpenAI,
      baseURL: "https://example.com/v1",
      customModels: ["openai.gpt-5.4"],
      withDefaultModels: false,
      config: { useResponsesApi: false },
    });
    expect(result).toBeUndefined();

    const { data: llmApiKeys } = await caller.llmApiKey.all({
      projectId,
    });

    expect(llmApiKeys).toHaveLength(1);
    expect(llmApiKeys[0].config).toEqual({ useResponsesApi: false });
  });

  it("should preserve empty VertexAI config without applying OpenAI defaults", async () => {
    await prisma.llmApiKeys.create({
      data: {
        projectId,
        provider: "vertex-empty-config",
        adapter: LLMAdapter.VertexAI,
        secretKey: encrypt("test-secret"),
        displaySecretKey: "...cret",
        customModels: ["gemini-2.5-flash"],
        withDefaultModels: false,
        extraHeaderKeys: [],
        config: {},
      },
    });

    const { data: llmApiKeys } = await caller.llmApiKey.all({
      projectId,
    });

    expect(llmApiKeys).toHaveLength(1);
    expect(llmApiKeys[0].config).toEqual({});
  });

  it("should store a preset upstream as the base URL of a decision-model connection", async () => {
    await caller.llmApiKey.create({
      projectId,
      secretKey: "sk-or-test",
      provider: "jev-via-openrouter",
      adapter: LLMAdapter.TypeSafe,
      baseURL: "https://openrouter.ai/api/v1",
    });

    const { data: llmApiKeys } = await caller.llmApiKey.all({
      projectId,
      includeDecisionModels: true,
    });

    expect(llmApiKeys).toHaveLength(1);
    expect(llmApiKeys[0].baseURL).toBe("https://openrouter.ai/api/v1");
  });

  it("should store a custom base URL and encrypted extra headers on a decision-model connection", async () => {
    await caller.llmApiKey.create({
      projectId,
      secretKey: "sk-proxy",
      provider: "jev-via-proxy",
      adapter: LLMAdapter.TypeSafe,
      baseURL: "https://example.com/typesafe/v1",
      extraHeaders: { "x-team": "evals" },
    });

    const storedKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: { projectId, provider: "jev-via-proxy" },
    });
    expect(storedKey.baseURL).toBe("https://example.com/typesafe/v1");
    expect(storedKey.extraHeaderKeys).toEqual(["x-team"]);
    expect(JSON.parse(decrypt(storedKey.extraHeaders!))).toEqual({
      "x-team": "evals",
    });
  });

  it("should treat a decision-model connection as working when the probe is answered", async () => {
    const fetchMock = vi.fn().mockResolvedValue(
      new Response(
        JSON.stringify({
          model: "jev-1.13.0",
          answers: {
            kind: {
              type: "choice",
              choice: "greeting",
              confidence: 0.9,
              probabilities: { greeting: 0.9, other: 0.1 },
            },
          },
        }),
        { status: 200, headers: { "Content-Type": "application/json" } },
      ),
    );
    vi.stubGlobal("fetch", fetchMock);

    try {
      const result = await caller.llmApiKey.test({
        projectId,
        secretKey: "sk-typesafe",
        provider: "jev",
        adapter: LLMAdapter.TypeSafe,
        baseURL: "https://example.com/typesafe/v1",
        extraHeaders: { "x-team": "evals" },
      });

      expect(result).toEqual({ success: true });
      expect(mockGenerateLLMText).not.toHaveBeenCalled();
      expect(fetchMock).toHaveBeenCalledTimes(1);
      const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit];
      expect(url).toBe("https://example.com/typesafe/v1/systemone");
      expect(new Headers(init.headers).get("authorization")).toBe(
        "Bearer sk-typesafe",
      );
      expect(new Headers(init.headers).get("x-team")).toBe("evals");
      expect(JSON.parse(init.body as string)).toEqual({
        model: "jev-latest",
        state: { message: "Hello, is anyone there?" },
        questions: {
          kind: {
            type: "choice",
            instructions: "What kind of message is `message`?",
            criteria: { greeting: null, other: null },
          },
        },
      });
    } finally {
      vi.unstubAllGlobals();
    }
  });

  it("should report the upstream error when a decision-model connection test fails", async () => {
    const fetchMock = vi.fn().mockResolvedValue(
      new Response(JSON.stringify({ error: { message: "invalid api key" } }), {
        status: 401,
        headers: { "Content-Type": "application/json" },
      }),
    );
    vi.stubGlobal("fetch", fetchMock);

    try {
      const result = await caller.llmApiKey.test({
        projectId,
        secretKey: "sk-typesafe",
        provider: "jev",
        adapter: LLMAdapter.TypeSafe,
        baseURL: "https://example.com/typesafe/v1",
      });

      expect(result.success).toBe(false);
      expect(result.error).toEqual(expect.any(String));
      expect(result.error!.length).toBeGreaterThan(0);
      expect(mockGenerateLLMText).not.toHaveBeenCalled();
    } finally {
      vi.unstubAllGlobals();
    }
  });

  it("should block decision-model connections with an internal custom base URL", async () => {
    await expect(
      caller.llmApiKey.create({
        projectId,
        secretKey: "sk-test",
        provider: "jev-internal",
        adapter: LLMAdapter.TypeSafe,
        baseURL: "http://localhost:8080/v1",
      }),
    ).rejects.toThrow("Invalid base URL: Blocked hostname detected");

    await expect(
      caller.llmApiKey.test({
        projectId,
        secretKey: "sk-test",
        provider: "jev-internal",
        adapter: LLMAdapter.TypeSafe,
        baseURL: "http://localhost:8080/v1",
      }),
    ).resolves.toEqual({
      success: false,
      error: "Blocked hostname detected",
    });

    const { data: llmApiKeys } = await caller.llmApiKey.all({
      projectId,
      includeDecisionModels: true,
    });
    expect(llmApiKeys).toHaveLength(0);
  });

  it("should reject decision-model base URLs that end in /systemone or have a query string", async () => {
    const connection = {
      projectId,
      secretKey: "sk-proxy",
      provider: "jev-via-proxy",
      adapter: LLMAdapter.TypeSafe,
    };

    await expect(
      caller.llmApiKey.create({
        ...connection,
        baseURL: "https://example.com/typesafe/v1/systemone",
      }),
    ).rejects.toThrow("Remove /systemone");
    await expect(
      caller.llmApiKey.test({
        ...connection,
        baseURL: "https://example.com/typesafe/v1/systemone/",
      }),
    ).resolves.toMatchObject({
      success: false,
      error: expect.stringContaining("Remove /systemone"),
    });
    await expect(
      caller.llmApiKey.create({
        ...connection,
        baseURL: "https://example.com/typesafe/v1/systemone?api-version=1",
      }),
    ).rejects.toThrow("Remove /systemone");
    await expect(
      caller.llmApiKey.create({
        ...connection,
        baseURL: "https://example.com/typesafe/v1?api-version=1",
      }),
    ).rejects.toThrow("Remove the query string");

    await caller.llmApiKey.create({
      ...connection,
      baseURL: "https://example.com/typesafe/v1",
    });
    const { id } = await prisma.llmApiKeys.findFirstOrThrow({
      where: { projectId, provider: connection.provider },
    });

    await expect(
      caller.llmApiKey.update({
        ...connection,
        id,
        baseURL: "https://example.com/typesafe/v1/systemone",
      }),
    ).rejects.toThrow("Remove /systemone");
    await expect(
      caller.llmApiKey.testUpdate({
        ...connection,
        id,
        baseURL: "https://example.com/typesafe/v1/systemone",
      }),
    ).resolves.toMatchObject({
      success: false,
      error: expect.stringContaining("Remove /systemone"),
    });
  });

  it("should require a new secret key when moving a decision-model connection to another upstream", async () => {
    await caller.llmApiKey.create({
      projectId,
      secretKey: "sk-typesafe",
      provider: "jev",
      adapter: LLMAdapter.TypeSafe,
    });
    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: { projectId, provider: "jev" },
    });

    await expect(
      caller.llmApiKey.update({
        id: existingKey.id,
        projectId,
        provider: "jev",
        adapter: LLMAdapter.TypeSafe,
        baseURL: "https://ai-gateway.vercel.sh/typesafe/v1",
      }),
    ).rejects.toThrow("Secret key is required when changing the base URL");

    await caller.llmApiKey.update({
      id: existingKey.id,
      projectId,
      provider: "jev",
      adapter: LLMAdapter.TypeSafe,
      secretKey: "vck-gateway",
      baseURL: "https://ai-gateway.vercel.sh/typesafe/v1",
    });

    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
      where: { id: existingKey.id, projectId },
    });
    expect(updatedKey.baseURL).toBe("https://ai-gateway.vercel.sh/typesafe/v1");
    expect(decrypt(updatedKey.secretKey)).toBe("vck-gateway");

    await caller.llmApiKey.update({
      id: existingKey.id,
      projectId,
      provider: "jev",
      adapter: LLMAdapter.TypeSafe,
      secretKey: "sk-typesafe-2",
      baseURL: null,
    });

    const revertedKey = await prisma.llmApiKeys.findUniqueOrThrow({
      where: { id: existingKey.id, projectId },
    });
    expect(revertedKey.baseURL).toBeNull();
  });
  it("should derive the Bedrock auth method in llmApiKey.all without returning secrets", async () => {
    await prisma.llmApiKeys.createMany({
      data: [
        {
          projectId,
          provider: "bedrock-access",
          adapter: LLMAdapter.Bedrock,
          secretKey: encrypt(
            JSON.stringify({
              accessKeyId: "AKIAIOSFODNN7EXAMPLE",
              secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
            }),
          ),
          displaySecretKey: "...MPLE",
          customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
          withDefaultModels: false,
          extraHeaderKeys: [],
          config: { region: "us-east-1" },
        },
        {
          projectId,
          provider: "bedrock-api",
          adapter: LLMAdapter.Bedrock,
          secretKey: encrypt(
            JSON.stringify({
              apiKey: "bedrock-api-key-1234",
            }),
          ),
          displaySecretKey: "...1234",
          customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
          withDefaultModels: false,
          extraHeaderKeys: [],
          config: { region: "us-east-1" },
        },
        {
          projectId,
          provider: "bedrock-default",
          adapter: LLMAdapter.Bedrock,
          secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
          displaySecretKey: "Default AWS credentials",
          customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
          withDefaultModels: false,
          extraHeaderKeys: [],
          config: { region: "us-east-1" },
        },
        {
          projectId,
          provider: "openai",
          adapter: LLMAdapter.OpenAI,
          secretKey: encrypt("sk-test"),
          displaySecretKey: "...test",
          customModels: [],
          withDefaultModels: true,
          extraHeaderKeys: [],
        },
      ],
    });

    const { data: llmApiKeys } = await caller.llmApiKey.all({
      projectId,
    });

    expect(
      llmApiKeys.find((key) => key.provider === "bedrock-access")?.authMethod,
    ).toBe(AuthMethod.AccessKeys);
    expect(
      llmApiKeys.find((key) => key.provider === "bedrock-api")?.authMethod,
    ).toBe(AuthMethod.ApiKey);
    expect(
      llmApiKeys.find((key) => key.provider === "bedrock-default")?.authMethod,
    ).toBe(AuthMethod.DefaultCredentials);
    expect(
      llmApiKeys.find((key) => key.provider === "openai")?.authMethod,
    ).toBeUndefined();
    expect(
      llmApiKeys.every(
        (key) => key.secretKey === undefined && key.extraHeaders === undefined,
      ),
    ).toBe(true);
  });

  it("should require llmApiKeys:create access for testing a new llm api key", async () => {
    const memberCaller = createCallerForProjectRole("MEMBER");

    await expect(
      memberCaller.llmApiKey.test({
        projectId,
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        secretKey: "sk-test",
        baseURL: "https://attacker.example.com/v1",
      }),
    ).rejects.toThrow("User does not have access to this resource or action");
  });

  it("should require llmApiKeys:update access for testing an existing llm api key", async () => {
    await caller.llmApiKey.create({
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-test",
      baseURL: "https://api.openai.com/v1",
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider: "openai",
      },
    });

    const memberCaller = createCallerForProjectRole("MEMBER");

    await expect(
      memberCaller.llmApiKey.testUpdate({
        id: existingKey.id,
        projectId,
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        baseURL: "https://attacker.example.com/v1",
      }),
    ).rejects.toThrow("User does not have access to this resource or action");
  });

  it("should block testUpdate when the base URL changes without a new secret key", async () => {
    await caller.llmApiKey.create({
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-original",
      baseURL: "https://api.openai.com/v1",
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider: "openai",
      },
    });

    const result = await caller.llmApiKey.testUpdate({
      id: existingKey.id,
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      baseURL: "https://attacker.example.com/v1",
    });

    expect(result).toEqual({
      success: false,
      error: "Secret key is required when changing the base URL",
    });
    expect(mockGenerateLLMText).not.toHaveBeenCalled();
  });

  it("should allow testing an existing connection with an unchanged localhost base URL", async () => {
    const connection = await prisma.llmApiKeys.create({
      data: {
        projectId,
        provider: "local-ollama",
        adapter: LLMAdapter.OpenAI,
        secretKey: encrypt("sk-existing"),
        displaySecretKey: "...ting",
        baseURL: "http://localhost:11434/v1",
        customModels: ["llama3.1"],
        withDefaultModels: true,
      },
    });

    const result = await caller.llmApiKey.testUpdate({
      id: connection.id,
      projectId,
      provider: "local-ollama",
      adapter: LLMAdapter.OpenAI,
    });

    expect(result).toEqual({ success: true });
    expect(mockGenerateLLMText).toHaveBeenCalledTimes(1);
    expect(mockGenerateLLMText).toHaveBeenCalledWith(
      expect.objectContaining({ timeout: 95_000 }),
    );
  });

  it("should allow testUpdate without a new secret key when the base URL is unchanged", async () => {
    const existingExtraHeaders = {
      Authorization: "Bearer stored-token",
      "X-Custom-Header": "stored-value",
    };

    await caller.llmApiKey.create({
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-original",
      baseURL: "https://api.openai.com/v1",
      extraHeaders: existingExtraHeaders,
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider: "openai",
      },
    });

    const result = await caller.llmApiKey.testUpdate({
      id: existingKey.id,
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      baseURL: "https://api.openai.com/v1",
    });

    expect(result).toEqual({ success: true });
    expect(mockGenerateLLMText).toHaveBeenCalledTimes(1);
    const connection = mockGenerateLLMText.mock.calls[0][0].connection;
    expect(connection.baseURL).toBe("https://api.openai.com/v1");
    expect(decrypt(connection.secretKey)).toBe("sk-original");
    expect(connection.projectId).toBe(existingKey.projectId);
    const encryptedExtraHeaders = connection.extraHeaders;
    expect(encryptedExtraHeaders).toBeTruthy();
    if (!encryptedExtraHeaders) {
      throw new Error("Expected extra headers to be preserved");
    }
    expect(JSON.parse(decrypt(encryptedExtraHeaders))).toEqual(
      existingExtraHeaders,
    );
  });

  it("should test an operator-provisioned Bedrock connection without revalidating its stored secret", async () => {
    const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
    (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "eu";

    try {
      const existingKey = await prisma.llmApiKeys.create({
        data: {
          projectId,
          provider: "bedrock-runtime",
          adapter: LLMAdapter.Bedrock,
          secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
          displaySecretKey: "Default AWS credentials",
          customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
          withDefaultModels: false,
          config: { region: "us-east-1" },
        },
      });

      const result = await caller.llmApiKey.testUpdate({
        id: existingKey.id,
        projectId,
        provider: existingKey.provider,
        adapter: LLMAdapter.Bedrock,
        customModels: existingKey.customModels,
        config: { region: "us-east-1" },
      });

      expect(result).toEqual({ success: true });
      expect(mockGenerateLLMText).toHaveBeenCalledWith(
        expect.objectContaining({
          connection: expect.objectContaining({ projectId }),
        }),
      );
    } finally {
      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
    }
  });

  it("should fill masked header values from the stored headers in testUpdate", async () => {
    await caller.llmApiKey.create({
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-original",
      baseURL: "https://api.openai.com/v1",
      extraHeaders: { "X-Api-Key": "stored-token", "X-Team": "stored-team" },
    });
    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: { projectId, provider: "openai" },
    });

    const result = await caller.llmApiKey.testUpdate({
      id: existingKey.id,
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      baseURL: "https://api.openai.com/v1",
      extraHeaders: { "X-Api-Key": "", "X-Team": "new-team" },
    });

    expect(result).toEqual({ success: true });
    const connection = mockGenerateLLMText.mock.calls[0][0].connection;
    expect(JSON.parse(decrypt(connection.extraHeaders!))).toEqual({
      "X-Api-Key": "stored-token",
      "X-Team": "new-team",
    });
  });

  it("should not fill masked header values from the stored headers when testUpdate changes the base URL", async () => {
    await caller.llmApiKey.create({
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-original",
      baseURL: "https://api.openai.com/v1",
      extraHeaders: { "X-Api-Key": "stored-token" },
    });
    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: { projectId, provider: "openai" },
    });

    const result = await caller.llmApiKey.testUpdate({
      id: existingKey.id,
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-rotated",
      baseURL: "https://example.net/v1",
      extraHeaders: { "X-Api-Key": "", "X-Team": "new-team" },
    });

    expect(result).toEqual({ success: true });
    const connection = mockGenerateLLMText.mock.calls[0][0].connection;
    expect(connection.baseURL).toBe("https://example.net/v1");
    expect(JSON.parse(decrypt(connection.extraHeaders!))).toEqual({
      "X-Team": "new-team",
    });
  });

  it("should allow testUpdate when the base URL changes and a new secret key is provided", async () => {
    const existingExtraHeaders = {
      Authorization: "Bearer stored-token",
      "X-Custom-Header": "stored-value",
    };

    await caller.llmApiKey.create({
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-original",
      baseURL: "https://api.openai.com/v1",
      extraHeaders: existingExtraHeaders,
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider: "openai",
      },
    });

    const result = await caller.llmApiKey.testUpdate({
      id: existingKey.id,
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-rotated",
      baseURL: "https://example.net/v1",
    });

    expect(result).toEqual({ success: true });
    expect(mockGenerateLLMText).toHaveBeenCalledTimes(1);
    const connection = mockGenerateLLMText.mock.calls[0][0].connection;
    expect(connection.baseURL).toBe("https://example.net/v1");
    expect(decrypt(connection.secretKey)).toBe("sk-rotated");
    expect(connection.projectId).toBeUndefined();
    expect(connection.extraHeaders).toBeUndefined();
  });

  it("should reject updating the base URL without a new secret key", async () => {
    const existingExtraHeaders = {
      Authorization: "Bearer stored-token",
    };

    await caller.llmApiKey.create({
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-original",
      baseURL: "https://api.openai.com/v1",
      extraHeaders: existingExtraHeaders,
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider: "openai",
      },
    });

    await expect(
      caller.llmApiKey.update({
        id: existingKey.id,
        projectId,
        provider: "openai",
        adapter: LLMAdapter.OpenAI,
        baseURL: "https://example.net/v1",
      }),
    ).rejects.toThrow("Secret key is required when changing the base URL");

    const unchangedKey = await prisma.llmApiKeys.findUniqueOrThrow({
      where: {
        id: existingKey.id,
        projectId,
      },
    });

    expect(unchangedKey.baseURL).toBe("https://api.openai.com/v1");
    expect(decrypt(unchangedKey.secretKey)).toBe("sk-original");
    expect(JSON.parse(decrypt(unchangedKey.extraHeaders as string))).toEqual(
      existingExtraHeaders,
    );
  });

  it("should not reuse stored extra headers when updating the base URL", async () => {
    await caller.llmApiKey.create({
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-original",
      baseURL: "https://api.openai.com/v1",
      extraHeaders: {
        Authorization: "Bearer stored-token",
        "X-Custom-Header": "stored-value",
      },
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider: "openai",
      },
    });

    await caller.llmApiKey.update({
      id: existingKey.id,
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-rotated",
      baseURL: "https://example.net/v1",
      extraHeaders: {
        Authorization: "",
        "X-Custom-Header": "",
      },
    });

    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
      where: {
        id: existingKey.id,
        projectId,
      },
    });

    expect(updatedKey.baseURL).toBe("https://example.net/v1");
    expect(decrypt(updatedKey.secretKey)).toBe("sk-rotated");
    expect(updatedKey.extraHeaders).toBeNull();
    expect(updatedKey.extraHeaderKeys).toEqual([]);
  });

  it("should allow new extra headers when updating the base URL", async () => {
    await caller.llmApiKey.create({
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-original",
      baseURL: "https://api.openai.com/v1",
      extraHeaders: {
        Authorization: "Bearer stored-token",
        "X-Old-Header": "stored-value",
      },
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider: "openai",
      },
    });

    await caller.llmApiKey.update({
      id: existingKey.id,
      projectId,
      provider: "openai",
      adapter: LLMAdapter.OpenAI,
      secretKey: "sk-rotated",
      baseURL: "https://example.net/v1",
      extraHeaders: {
        Authorization: "Bearer rotated-token",
        "X-Old-Header": "",
      },
    });

    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
      where: {
        id: existingKey.id,
        projectId,
      },
    });

    expect(JSON.parse(decrypt(updatedKey.extraHeaders as string))).toEqual({
      Authorization: "Bearer rotated-token",
    });
    expect(updatedKey.extraHeaderKeys).toEqual(["Authorization"]);
  });

  it("should create and update an llm api key", async () => {
    const secret = "test-secret";
    const provider = "openai";
    const adapter = LLMAdapter.OpenAI;
    const customModels = ["fancy-gpt-3.5-turbo"];
    const baseURL = "https://example.com/v1";
    const withDefaultModels = false;

    // Create initial key
    await caller.llmApiKey.create({
      projectId,
      secretKey: secret,
      provider,
      adapter,
      baseURL,
      customModels,
      withDefaultModels,
    });

    // Verify initial key
    const initialKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
      },
    });

    expect(initialKeys.length).toBe(1);
    expect(initialKeys[0].projectId).toBe(projectId);
    expect(initialKeys[0].secretKey).not.toBeNull();
    expect(initialKeys[0].secretKey).not.toEqual(secret);
    expect(initialKeys[0].provider).toBe(provider);
    expect(initialKeys[0].adapter).toBe(adapter);
    expect(initialKeys[0].baseURL).toBe(baseURL);
    expect(initialKeys[0].customModels).toEqual(customModels);
    expect(initialKeys[0].withDefaultModels).toBe(withDefaultModels);

    // Update the key
    const newSecret = "new-test-secret";
    const newBaseURL = "https://example.org/v1";
    const newCustomModels = ["new-fancy-gpt-3.5-turbo"];
    const newWithDefaultModels = true;

    await caller.llmApiKey.update({
      id: initialKeys[0].id,
      projectId,
      secretKey: newSecret,
      provider,
      adapter,
      baseURL: newBaseURL,
      customModels: newCustomModels,
      withDefaultModels: newWithDefaultModels,
    });

    // Verify updated key
    const updatedKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
      },
    });

    expect(updatedKeys.length).toBe(1);
    expect(updatedKeys[0].projectId).toBe(projectId);
    expect(updatedKeys[0].secretKey).not.toBeNull();
    expect(updatedKeys[0].secretKey).not.toEqual(newSecret);
    expect(updatedKeys[0].provider).toBe(provider); // Should not change
    expect(updatedKeys[0].adapter).toBe(adapter); // Should not change
    expect(updatedKeys[0].baseURL).toBe(newBaseURL);
    expect(updatedKeys[0].customModels).toEqual(newCustomModels);
    expect(updatedKeys[0].withDefaultModels).toBe(newWithDefaultModels);
  });

  it("should scope the llm api key update write by project id", async () => {
    const secret = "test-secret";
    const provider = "openai";
    const adapter = LLMAdapter.OpenAI;

    await caller.llmApiKey.create({
      projectId,
      secretKey: secret,
      provider,
      adapter,
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider,
      },
    });

    const updateSpy = vi.spyOn(prisma.llmApiKeys, "update");

    try {
      await caller.llmApiKey.update({
        id: existingKey.id,
        projectId,
        secretKey: "new-test-secret",
        provider,
        adapter,
      });

      expect(updateSpy).toHaveBeenCalledWith(
        expect.objectContaining({
          where: {
            id: existingKey.id,
            projectId,
          },
        }),
      );
    } finally {
      updateSpy.mockRestore();
    }
  });

  it("should update a Bedrock Access key auth to a Bedrock API key", async () => {
    const provider = "bedrock";

    await caller.llmApiKey.create({
      projectId,
      provider,
      adapter: LLMAdapter.Bedrock,
      secretKey: JSON.stringify({
        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
      }),
      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
      withDefaultModels: false,
      config: { region: "us-east-1" },
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: {
        projectId,
        provider,
      },
    });

    await caller.llmApiKey.update({
      id: existingKey.id,
      projectId,
      provider,
      adapter: LLMAdapter.Bedrock,
      secretKey: JSON.stringify({ apiKey: "bedrock-api-key-5678" }),
      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
      withDefaultModels: false,
      config: { region: "eu-west-1" },
    });

    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
      where: { id: existingKey.id },
    });

    expect(decrypt(updatedKey.secretKey)).toBe(
      JSON.stringify({
        apiKey: "bedrock-api-key-5678",
      }),
    );
    expect(updatedKey.displaySecretKey).toBe("...5678");
    expect(updatedKey.config).toEqual({ region: "eu-west-1" });
  });

  it("should update a Bedrock API key auth to Access keys", async () => {
    const provider = "bedrock";

    await caller.llmApiKey.create({
      projectId,
      provider,
      adapter: LLMAdapter.Bedrock,
      secretKey: JSON.stringify({ apiKey: "bedrock-api-key-1234" }),
      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
      withDefaultModels: false,
      config: { region: "us-east-1" },
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: { projectId, provider },
    });

    await caller.llmApiKey.update({
      id: existingKey.id,
      projectId,
      provider,
      adapter: LLMAdapter.Bedrock,
      secretKey: JSON.stringify({
        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
      }),
      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
      withDefaultModels: false,
      config: { region: "eu-west-1" },
    });

    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
      where: { id: existingKey.id },
    });

    expect(decrypt(updatedKey.secretKey)).toBe(
      JSON.stringify({
        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
      }),
    );
    expect(updatedKey.displaySecretKey).toBe("...EKEY");
    expect(updatedKey.config).toEqual({ region: "eu-west-1" });
  });

  it("should update a Bedrock DefaultCredentials key to explicit Access keys", async () => {
    const provider = "bedrock";
    const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;

    try {
      // Simulate self-hosted to allow default credentials
      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;

      await caller.llmApiKey.create({
        projectId,
        provider,
        adapter: LLMAdapter.Bedrock,
        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
        withDefaultModels: false,
        config: { region: "us-east-1" },
      });

      const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
        where: { projectId, provider },
      });

      expect(decrypt(existingKey.secretKey)).toBe(
        BEDROCK_USE_DEFAULT_CREDENTIALS,
      );
      expect(existingKey.displaySecretKey).toBe("Default AWS credentials");

      await caller.llmApiKey.update({
        id: existingKey.id,
        projectId,
        provider,
        adapter: LLMAdapter.Bedrock,
        secretKey: JSON.stringify({
          accessKeyId: "AKIAIOSFODNN7EXAMPLE",
          secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
        }),
        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
        withDefaultModels: false,
        config: { region: "eu-west-1" },
      });

      const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
        where: { id: existingKey.id },
      });

      expect(decrypt(updatedKey.secretKey)).toBe(
        JSON.stringify({
          accessKeyId: "AKIAIOSFODNN7EXAMPLE",
          secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
        }),
      );
      expect(updatedKey.displaySecretKey).toBe("...EKEY");
      expect(updatedKey.config).toEqual({ region: "eu-west-1" });
    } finally {
      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
    }
  });

  it("should update a Bedrock DefaultCredentials key to a Bedrock API key", async () => {
    const provider = "bedrock";
    const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;

    try {
      // Simulate self-hosted to allow default credentials
      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;

      await caller.llmApiKey.create({
        projectId,
        provider,
        adapter: LLMAdapter.Bedrock,
        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
        withDefaultModels: false,
        config: { region: "us-east-1" },
      });

      const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
        where: { projectId, provider },
      });

      await caller.llmApiKey.update({
        id: existingKey.id,
        projectId,
        provider,
        adapter: LLMAdapter.Bedrock,
        secretKey: JSON.stringify({ apiKey: "bedrock-api-key-9999" }),
        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
        withDefaultModels: false,
        config: { region: "eu-west-1" },
      });

      const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
        where: { id: existingKey.id },
      });

      expect(decrypt(updatedKey.secretKey)).toBe(
        JSON.stringify({ apiKey: "bedrock-api-key-9999" }),
      );
      expect(updatedKey.displaySecretKey).toBe("...9999");
      expect(updatedKey.config).toEqual({ region: "eu-west-1" });
    } finally {
      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
    }
  });

  it("should reject updating a Bedrock key back to DefaultCredentials on cloud", async () => {
    const provider = "bedrock";

    await caller.llmApiKey.create({
      projectId,
      provider,
      adapter: LLMAdapter.Bedrock,
      secretKey: JSON.stringify({
        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
      }),
      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
      withDefaultModels: false,
      config: { region: "us-east-1" },
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: { projectId, provider },
    });

    await expect(
      caller.llmApiKey.update({
        id: existingKey.id,
        projectId,
        provider,
        adapter: LLMAdapter.Bedrock,
        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
        withDefaultModels: false,
        config: { region: "eu-west-1" },
      }),
    ).rejects.toThrow(
      "Default AWS credentials are only allowed for Bedrock in self-hosted deployments",
    );
  });

  it("should update a Bedrock Access key auth back to DefaultCredentials (self-hosted)", async () => {
    const provider = "bedrock";
    const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;

    await caller.llmApiKey.create({
      projectId,
      provider,
      adapter: LLMAdapter.Bedrock,
      secretKey: JSON.stringify({
        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
      }),
      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
      withDefaultModels: false,
      config: { region: "us-east-1" },
    });

    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
      where: { projectId, provider },
    });

    try {
      // Simulate self-hosted deployment where default credentials are allowed
      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;

      await caller.llmApiKey.update({
        id: existingKey.id,
        projectId,
        provider,
        adapter: LLMAdapter.Bedrock,
        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
        withDefaultModels: false,
        config: { region: "eu-west-1" },
      });
    } finally {
      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
    }

    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
      where: { id: existingKey.id },
    });

    expect(decrypt(updatedKey.secretKey)).toBe(BEDROCK_USE_DEFAULT_CREDENTIALS);
    expect(updatedKey.displaySecretKey).toBe("Default AWS credentials");
    expect(updatedKey.config).toEqual({ region: "eu-west-1" });
  });

  it("should update only the secret key", async () => {
    const secret = "test-secret";
    const provider = "openai";
    const adapter = LLMAdapter.OpenAI;
    const customModels = ["fancy-gpt-3.5-turbo"];
    const baseURL = "https://example.com/v1";
    const withDefaultModels = false;

    // Create initial key
    await caller.llmApiKey.create({
      projectId,
      secretKey: secret,
      provider,
      adapter,
      baseURL,
      customModels,
      withDefaultModels,
    });

    const initialKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
        provider,
      },
    });

    expect(initialKeys.length).toBe(1);
    const initialDisplaySecretKey = initialKeys[0].displaySecretKey;

    // Update only the secret key
    const newSecret = "updatedSecretKey123";

    await caller.llmApiKey.update({
      id: initialKeys[0].id,
      projectId,
      secretKey: newSecret,
      provider,
      adapter,
    });

    // Verify updated key
    const updatedKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
        provider,
      },
    });

    expect(updatedKeys.length).toBe(1);
    expect(decrypt(updatedKeys[0].secretKey)).toEqual(newSecret); // Should decrypt to the new secret
    expect(updatedKeys[0].displaySecretKey).not.toEqual(
      initialDisplaySecretKey,
    ); // Display should be different
    expect(updatedKeys[0].displaySecretKey).toEqual("...y123"); // Should match format with hyphens allowed

    // Other fields should remain unchanged
    expect(updatedKeys[0].baseURL).toBe(baseURL);
    expect(updatedKeys[0].customModels).toEqual(customModels);
    expect(updatedKeys[0].withDefaultModels).toBe(withDefaultModels);
    expect(updatedKeys[0].provider).toBe(provider);
    expect(updatedKeys[0].adapter).toBe(adapter);
  });

  it("should update only the extra headers", async () => {
    const secret = "test-secret";
    const provider = "openai";
    const adapter = LLMAdapter.OpenAI;
    const customModels = ["fancy-gpt-3.5-turbo"];
    const baseURL = "https://example.com/v1";
    const withDefaultModels = false;
    const extraHeaders = {
      "X-Custom-Header": "custom-value",
      Authorization: "Bearer token123",
    };

    // Create initial key with extra headers
    await caller.llmApiKey.create({
      projectId,
      secretKey: secret,
      provider,
      adapter,
      baseURL,
      customModels,
      withDefaultModels,
      extraHeaders,
    });

    const initialKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
      },
    });

    expect(initialKeys.length).toBe(1);
    expect(initialKeys[0].extraHeaders).not.toBeNull();
    expect(initialKeys[0].extraHeaderKeys).toEqual(Object.keys(extraHeaders));

    // Update only the extra headers
    const newExtraHeaders = {
      "X-Custom-Header": "updated-custom-value",
      "X-New-Header": "new-value",
    };

    await caller.llmApiKey.update({
      id: initialKeys[0].id,
      projectId,
      provider,
      adapter,
      extraHeaders: newExtraHeaders,
    });

    // Verify updated key
    const updatedKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
      },
    });

    expect(updatedKeys.length).toBe(1);
    expect(updatedKeys[0].extraHeaders).not.toBeNull();
    expect(updatedKeys[0].extraHeaders).not.toEqual(
      initialKeys[0].extraHeaders,
    ); // Should be different
    expect(updatedKeys[0].extraHeaderKeys).toEqual(
      Object.keys(newExtraHeaders),
    );

    // Other fields should remain unchanged
    expect(updatedKeys[0].secretKey).toEqual(initialKeys[0].secretKey); // Secret should be same
    expect(updatedKeys[0].displaySecretKey).toEqual(
      initialKeys[0].displaySecretKey,
    ); // Display should be same
    expect(updatedKeys[0].baseURL).toBe(baseURL);
    expect(updatedKeys[0].customModels).toEqual(customModels);
    expect(updatedKeys[0].withDefaultModels).toBe(withDefaultModels);
    expect(updatedKeys[0].provider).toBe(provider);
    expect(updatedKeys[0].adapter).toBe(adapter);
  });

  it("should remove extra headers when updated with empty object", async () => {
    const secret = "test-secret";
    const provider = "openai";
    const adapter = LLMAdapter.OpenAI;
    const extraHeaders = {
      "X-Custom-Header": "custom-value",
      Authorization: "Bearer token123",
    };

    // Create initial key with extra headers
    await caller.llmApiKey.create({
      projectId,
      secretKey: secret,
      provider,
      adapter,
      extraHeaders,
    });

    const initialKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
      },
    });

    expect(initialKeys.length).toBe(1);
    expect(initialKeys[0].extraHeaders).not.toBeNull();
    expect(initialKeys[0].extraHeaderKeys).toEqual(Object.keys(extraHeaders));

    // Update with empty extra headers to remove them
    await caller.llmApiKey.update({
      id: initialKeys[0].id,
      projectId,
      provider,
      adapter,
      extraHeaders: {},
    });

    // Verify updated key
    const updatedKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
      },
    });

    expect(updatedKeys.length).toBe(1);
    expect(updatedKeys[0].extraHeaders).toBeNull();
    expect(updatedKeys[0].extraHeaderKeys).toEqual([]);

    // Other fields should remain unchanged
    expect(updatedKeys[0].secretKey).toEqual(initialKeys[0].secretKey);
    expect(updatedKeys[0].displaySecretKey).toEqual(
      initialKeys[0].displaySecretKey,
    );
    expect(updatedKeys[0].provider).toBe(provider);
    expect(updatedKeys[0].adapter).toBe(adapter);
  });

  it("should partially update extra headers preserving existing values for empty inputs", async () => {
    const secret = "test-secret";
    const provider = "openai";
    const adapter = LLMAdapter.OpenAI;
    const extraHeaders = {
      "X-Custom-Header": "custom-value",
      Authorization: "Bearer token123",
      "X-Another-Header": "another-value",
    };

    // Create initial key with extra headers
    await caller.llmApiKey.create({
      projectId,
      secretKey: secret,
      provider,
      adapter,
      extraHeaders,
    });

    const initialKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
      },
    });

    expect(initialKeys.length).toBe(1);

    // Update some headers with empty values to test preservation logic
    const partialUpdateHeaders = {
      "X-Custom-Header": "updated-value", // Update this one
      Authorization: "", // Should preserve existing value
      "X-Another-Header": "", // Should preserve existing value
      "X-New-Header": "new-value", // Add this new one
    };

    await caller.llmApiKey.update({
      id: initialKeys[0].id,
      projectId,
      provider,
      adapter,
      extraHeaders: partialUpdateHeaders,
    });

    // Verify updated key
    const updatedKeys = await prisma.llmApiKeys.findMany({
      where: {
        projectId,
      },
    });

    expect(updatedKeys.length).toBe(1);
    expect(updatedKeys[0].extraHeaders).not.toBeNull();

    // Should have 4 headers: 3 original + 1 new
    expect(updatedKeys[0].extraHeaderKeys).toHaveLength(4);
    expect(updatedKeys[0].extraHeaderKeys).toContain("X-Custom-Header");
    expect(updatedKeys[0].extraHeaderKeys).toContain("Authorization");
    expect(updatedKeys[0].extraHeaderKeys).toContain("X-Another-Header");
    expect(updatedKeys[0].extraHeaderKeys).toContain("X-New-Header");
  });

  it("rolls back connection deletion when audit logging fails", async () => {
    const connection = await prisma.llmApiKeys.create({
      data: {
        projectId,
        secretKey: encrypt("project-secret"),
        displaySecretKey: "...cret",
        provider: `audit-rollback-${randomUUID()}`,
        adapter: LLMAdapter.OpenAI,
      },
    });
    const auditLogSpy = vi
      .spyOn(auditLogs, "auditLog")
      .mockRejectedValueOnce(new Error("audit unavailable"));

    try {
      await expect(
        caller.llmApiKey.delete({ projectId, id: connection.id }),
      ).rejects.toThrow();
      await expect(
        prisma.llmApiKeys.findUniqueOrThrow({ where: { id: connection.id } }),
      ).resolves.toBeDefined();
    } finally {
      auditLogSpy.mockRestore();
    }
  });

  describe("deleting a connection pauses the evaluators that ran on it", () => {
    const PROVIDER = "openai";

    /** Versions are given oldest-first; the last one is the evaluator's head. */
    const createV2Evaluator = async (
      versions: Array<{ provider: string | null; model: string | null }>,
    ) => {
      const evaluator = await prisma.evaluator.create({
        data: {
          projectId,
          name: `evaluator-${randomUUID()}`,
          type: EvalTemplateType.LLM_AS_JUDGE,
          versions: {
            create: versions.map((version, index) => ({
              version: index + 1,
              prompt: "Evaluate {{output}}",
              vars: ["output"],
              ...version,
            })),
          },
        },
      });
      return evaluator.id;
    };

    it("blocks evaluators by their current model and leaves the rest running", async () => {
      await caller.llmApiKey.create({
        projectId,
        secretKey: "test-secret",
        provider: PROVIDER,
        adapter: LLMAdapter.OpenAI,
        customModels: [],
        withDefaultModels: true,
      });
      const connection = await prisma.llmApiKeys.findFirstOrThrow({
        where: { projectId, provider: PROVIDER },
      });

      const [
        v2OnProvider,
        v2OnDefaultModel,
        v2MovedOffProvider,
        v2OnOtherProvider,
      ] = await Promise.all([
        createV2Evaluator([{ provider: PROVIDER, model: "gpt-4o" }]),
        createV2Evaluator([{ provider: null, model: null }]),
        // Upgraded off the deleted provider: only the head version counts.
        createV2Evaluator([
          { provider: PROVIDER, model: "gpt-4o" },
          { provider: "anthropic", model: "claude" },
        ]),
        createV2Evaluator([{ provider: "anthropic", model: "claude" }]),
      ]);

      // Use this provider as the project's default too, so both block reasons
      // fire when its last effective connection is deleted.
      await prisma.defaultLlmModel.create({
        data: {
          projectId,
          llmApiKeyId: connection.id,
          provider: PROVIDER,
          adapter: LLMAdapter.OpenAI,
          model: "gpt-4o",
        },
      });
      await caller.llmApiKey.delete({ projectId, id: connection.id });

      expect(mockFinalizeEvaluatorBlocks).toHaveBeenCalledOnce();
      expect(mockFinalizeEvaluatorBlocks).toHaveBeenCalledWith({
        projectId,
        source: EvaluatorBlockSource.LLM_API_KEY_DELETION,
        evaluatorIdsByReason: {
          [EvaluatorBlockReason.LLM_CONNECTION_MISSING]: [v2OnProvider],
          [EvaluatorBlockReason.DEFAULT_EVAL_MODEL_MISSING]: [v2OnDefaultModel],
        },
      });

      const evaluators = await prisma.evaluator.findMany({
        where: { projectId },
      });
      const blockStateById = new Map(
        evaluators.map((row) => [
          row.id,
          { blocked: row.blockedAt !== null, reason: row.blockReason },
        ]),
      );

      expect(blockStateById.get(v2OnProvider)).toEqual({
        blocked: true,
        reason: EvaluatorBlockReason.LLM_CONNECTION_MISSING,
      });
      expect(blockStateById.get(v2OnDefaultModel)).toEqual({
        blocked: true,
        reason: EvaluatorBlockReason.DEFAULT_EVAL_MODEL_MISSING,
      });

      for (const untouched of [v2MovedOffProvider, v2OnOtherProvider]) {
        expect(blockStateById.get(untouched)).toEqual({
          blocked: false,
          reason: null,
        });
      }
      expect(
        await prisma.defaultLlmModel.findUnique({ where: { projectId } }),
      ).toBeNull();
    });

    it("keeps the default model when deleting an override with an organization fallback", async () => {
      const organizationConnection = await prisma.llmApiKeys.create({
        data: {
          organizationId: orgId,
          secretKey: encrypt("organization-secret"),
          displaySecretKey: "...cret",
          provider: PROVIDER,
          adapter: LLMAdapter.OpenAI,
        },
      });
      const projectConnection = await prisma.llmApiKeys.create({
        data: {
          projectId,
          secretKey: encrypt("project-secret"),
          displaySecretKey: "...cret",
          provider: PROVIDER,
          adapter: LLMAdapter.OpenAI,
        },
      });
      const providerEvaluatorId = await createV2Evaluator([
        { provider: PROVIDER, model: "gpt-4o" },
      ]);
      const defaultModelEvaluatorId = await createV2Evaluator([
        { provider: null, model: null },
      ]);
      await prisma.defaultLlmModel.create({
        data: {
          projectId,
          llmApiKeyId: projectConnection.id,
          provider: PROVIDER,
          adapter: LLMAdapter.OpenAI,
          model: "gpt-4o",
        },
      });
      await prisma.topicConfig.create({
        data: {
          projectId,
          enabled: true,
          summaryLlmApiKeyId: projectConnection.id,
          summaryModel: "gpt-6-luna",
          embeddingLlmApiKeyId: projectConnection.id,
          embeddingModel: "text-embedding-3-small",
          clusteringLlmApiKeyId: projectConnection.id,
          clusteringModel: "gpt-5.6-terra",
        },
      });

      await caller.llmApiKey.delete({
        projectId,
        id: projectConnection.id,
      });

      expect(mockFinalizeEvaluatorBlocks).not.toHaveBeenCalled();
      expect(
        await prisma.evaluator.findUniqueOrThrow({
          where: { id: providerEvaluatorId },
          select: { blockedAt: true, blockReason: true },
        }),
      ).toEqual({ blockedAt: null, blockReason: null });
      expect(
        await prisma.evaluator.findUniqueOrThrow({
          where: { id: defaultModelEvaluatorId },
          select: { blockedAt: true, blockReason: true },
        }),
      ).toEqual({ blockedAt: null, blockReason: null });
      expect(
        await prisma.defaultLlmModel.findUniqueOrThrow({
          where: { projectId },
          select: {
            llmApiKeyId: true,
            provider: true,
            adapter: true,
            model: true,
          },
        }),
      ).toEqual({
        llmApiKeyId: organizationConnection.id,
        provider: PROVIDER,
        adapter: LLMAdapter.OpenAI,
        model: "gpt-4o",
      });
      expect(
        await prisma.topicConfig.findUniqueOrThrow({
          where: { projectId },
          select: {
            enabled: true,
            summaryLlmApiKeyId: true,
            embeddingLlmApiKeyId: true,
            clusteringLlmApiKeyId: true,
          },
        }),
      ).toEqual({
        enabled: true,
        summaryLlmApiKeyId: organizationConnection.id,
        embeddingLlmApiKeyId: organizationConnection.id,
        clusteringLlmApiKeyId: organizationConnection.id,
      });
    });

    it("deletes the default model when the organization fallback uses a different adapter", async () => {
      await prisma.llmApiKeys.createMany({
        data: [
          {
            organizationId: orgId,
            secretKey: encrypt("organization-secret"),
            displaySecretKey: "...cret",
            provider: PROVIDER,
            adapter: LLMAdapter.Anthropic,
          },
          {
            projectId,
            secretKey: encrypt("project-secret"),
            displaySecretKey: "...cret",
            provider: PROVIDER,
            adapter: LLMAdapter.OpenAI,
          },
        ],
      });
      const projectConnection = await prisma.llmApiKeys.findFirstOrThrow({
        where: { projectId, provider: PROVIDER },
      });
      const defaultModelEvaluatorId = await createV2Evaluator([
        { provider: null, model: null },
      ]);
      await prisma.defaultLlmModel.create({
        data: {
          projectId,
          llmApiKeyId: projectConnection.id,
          provider: PROVIDER,
          adapter: LLMAdapter.OpenAI,
          model: "gpt-4o",
        },
      });
      await caller.llmApiKey.delete({
        projectId,
        id: projectConnection.id,
      });

      expect(mockFinalizeEvaluatorBlocks).toHaveBeenCalledWith({
        projectId,
        source: EvaluatorBlockSource.LLM_API_KEY_DELETION,
        evaluatorIdsByReason: {
          [EvaluatorBlockReason.LLM_CONNECTION_MISSING]: [],
          [EvaluatorBlockReason.DEFAULT_EVAL_MODEL_MISSING]: [
            defaultModelEvaluatorId,
          ],
        },
      });
      expect(
        await prisma.defaultLlmModel.findUnique({ where: { projectId } }),
      ).toBeNull();
    });

    it("does not rebind organization-backed configuration to an incompatible project override", async () => {
      const organizationConnection = await prisma.llmApiKeys.create({
        data: {
          organizationId: orgId,
          secretKey: encrypt("organization-secret"),
          displaySecretKey: "...cret",
          provider: PROVIDER,
          adapter: LLMAdapter.OpenAI,
        },
      });
      await prisma.llmApiKeys.create({
        data: {
          projectId,
          secretKey: encrypt("project-secret"),
          displaySecretKey: "...cret",
          provider: PROVIDER,
          adapter: LLMAdapter.Anthropic,
        },
      });
      await prisma.defaultLlmModel.create({
        data: {
          projectId,
          llmApiKeyId: organizationConnection.id,
          provider: PROVIDER,
          adapter: LLMAdapter.OpenAI,
          model: "gpt-4o",
        },
      });
      await prisma.topicConfig.create({
        data: {
          projectId,
          enabled: true,
          summaryLlmApiKeyId: organizationConnection.id,
          summaryModel: "gpt-6-luna",
          embeddingLlmApiKeyId: organizationConnection.id,
          embeddingModel: "text-embedding-3-small",
          clusteringLlmApiKeyId: organizationConnection.id,
          clusteringModel: "gpt-5.6-terra",
        },
      });

      await caller.organizationLlmApiKey.delete({
        orgId,
        id: organizationConnection.id,
      });

      expect(
        await prisma.defaultLlmModel.findUnique({
          where: { projectId },
        }),
      ).toBeNull();
      expect(
        await prisma.topicConfig.findUniqueOrThrow({
          where: { projectId },
          select: {
            enabled: true,
            summaryLlmApiKeyId: true,
            embeddingLlmApiKeyId: true,
            clusteringLlmApiKeyId: true,
          },
        }),
      ).toEqual({
        enabled: false,
        summaryLlmApiKeyId: null,
        embeddingLlmApiKeyId: null,
        clusteringLlmApiKeyId: null,
      });
    });

    it("turns Topics off and records why when its connection is deleted", async () => {
      await caller.llmApiKey.create({
        projectId,
        secretKey: "test-secret",
        provider: PROVIDER,
        adapter: LLMAdapter.OpenAI,
        customModels: [],
        withDefaultModels: true,
      });
      const connection = await prisma.llmApiKeys.findFirstOrThrow({
        where: { projectId, provider: PROVIDER },
      });
      await prisma.topicConfig.create({
        data: {
          projectId,
          enabled: true,
          summaryLlmApiKeyId: connection.id,
          summaryModel: "gpt-6-luna",
          embeddingLlmApiKeyId: connection.id,
          embeddingModel: "text-embedding-3-small",
          clusteringLlmApiKeyId: connection.id,
          clusteringModel: "gpt-5.6-terra",
        },
      });

      await caller.llmApiKey.delete({ projectId, id: connection.id });

      expect(
        await prisma.topicConfig.findUniqueOrThrow({
          where: { projectId },
        }),
      ).toMatchObject({
        enabled: false,
        blockReason: "LLM_CONNECTION_MISSING",
        blockMessage: expect.stringContaining(`"${PROVIDER}" was deleted`),
        blockedAt: expect.any(Date),
        summaryLlmApiKeyId: null,
        embeddingLlmApiKeyId: null,
        clusteringLlmApiKeyId: null,
      });
    });

    it("clears deleted Topics connections without marking an already disabled config as paused", async () => {
      await caller.llmApiKey.create({
        projectId,
        secretKey: "test-secret",
        provider: PROVIDER,
        adapter: LLMAdapter.OpenAI,
        customModels: [],
        withDefaultModels: true,
      });
      const connection = await prisma.llmApiKeys.findFirstOrThrow({
        where: { projectId, provider: PROVIDER },
      });
      await prisma.topicConfig.create({
        data: {
          projectId,
          enabled: false,
          summaryLlmApiKeyId: connection.id,
          summaryModel: "gpt-6-luna",
          embeddingLlmApiKeyId: connection.id,
          embeddingModel: "text-embedding-3-small",
          clusteringLlmApiKeyId: connection.id,
          clusteringModel: "gpt-5.6-terra",
        },
      });

      await caller.llmApiKey.delete({ projectId, id: connection.id });

      expect(
        await prisma.topicConfig.findUniqueOrThrow({
          where: { projectId },
        }),
      ).toMatchObject({
        enabled: false,
        blockReason: null,
        blockMessage: null,
        blockedAt: null,
        summaryLlmApiKeyId: null,
        embeddingLlmApiKeyId: null,
        clusteringLlmApiKeyId: null,
      });
    });
  });
});