🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { beforeEach, describe, expect, it, vi } from "vitest";
import { LLMAdapter } from "../llm/types";
import {
getEnabledTopicsModels,
prepareTopicsModelSettings,
reconcileTopicsConnectionsForProject,
writeTopicsModelSettings,
} from "./model-config";
const mocks = vi.hoisted(() => ({
keys: vi.fn(),
candidates: vi.fn(),
projects: vi.fn(),
configs: vi.fn(),
current: vi.fn(),
update: vi.fn(),
upsert: vi.fn(),
transaction: vi.fn(),
invalidate: vi.fn(),
hasEmbeddings: vi.fn(),
generateText: vi.fn(),
embed: vi.fn(),
}));
vi.mock("./clickhouse", () => ({ hasTopicEmbeddings: mocks.hasEmbeddings }));
vi.mock("./text", () => ({ generateTopicText: mocks.generateText }));
vi.mock("./embeddings", () => ({ generateTopicEmbeddings: mocks.embed }));
vi.mock("./processing-gate", () => ({
invalidateTopicsProcessingActive: mocks.invalidate,
}));
vi.mock("../../db", () => {
const tx = {
llmApiKeys: { findMany: mocks.keys },
project: { findMany: mocks.projects, findUnique: mocks.candidates },
topicConfig: {
findMany: mocks.configs,
findUnique: mocks.current,
update: mocks.update,
upsert: mocks.upsert,
},
};
return {
prisma: {
...tx,
$transaction: (callback: (client: typeof tx) => unknown) => {
mocks.transaction();
return callback(tx);
},
},
};
});
const connection = (id: string, adapter: LLMAdapter) => ({
id,
adapter,
provider: id,
secretKey: "encrypted",
extraHeaders: null,
baseURL: null,
config: null,
});
const slot = (provider: string, model: string) => ({ provider, model });
const settings = {
summary: slot("openai", "gpt-6-luna"),
embedding: slot("openai", "text-embedding-3-small"),
embeddingDimensions: 1024 as const,
clustering: slot("openai", "gpt-5.6-terra"),
};
const saveTopicsModelSettings = async (
projectId: string,
value: Parameters<typeof prepareTopicsModelSettings>[1],
) =>
writeTopicsModelSettings(
projectId,
await prepareTopicsModelSettings(projectId, value),
);
beforeEach(() => {
vi.resetAllMocks();
mocks.candidates.mockResolvedValue({
LlmApiKeys: [
connection("openai", LLMAdapter.OpenAI),
connection("anthropic", LLMAdapter.Anthropic),
],
organization: { organizationLlmApiKeys: [] },
});
mocks.current.mockResolvedValue(null);
mocks.projects.mockResolvedValue([{ id: "project", orgId: "target-org" }]);
mocks.hasEmbeddings.mockResolvedValue(false);
mocks.generateText.mockResolvedValue({
output: {
summary: "Sign in",
status: "applicable",
name: "Access",
description: "Sign-in help",
},
});
mocks.embed.mockImplementation(async ({ summaries, dimensions }) => ({
embeddings: summaries.map(() => Array(dimensions).fill(0.1)),
}));
});
describe("prepareTopicsModelSettings", () => {
it("validates text slots through the Topics transport before saving", async () => {
await saveTopicsModelSettings("project", {
...settings,
clustering: slot("anthropic", "claude-sonnet"),
});
expect(mocks.generateText).toHaveBeenCalledWith(
expect.objectContaining({
model: expect.objectContaining({ model: "claude-sonnet" }),
maxOutputTokens: 512,
timeoutMs: expect.any(Number),
}),
);
expect(mocks.upsert).toHaveBeenCalledOnce();
mocks.generateText.mockResolvedValueOnce({ output: { status: "invalid" } });
await expect(
saveTopicsModelSettings("project", {
...settings,
embedding: null,
clustering: null,
}),
).rejects.toThrow("Facet summaries (gpt-6-luna)");
expect(mocks.upsert).toHaveBeenCalledOnce();
});
it("saves a partial setup only while automatic processing is off", async () => {
await saveTopicsModelSettings("project", {
...settings,
clustering: null,
});
expect(mocks.upsert).toHaveBeenCalledOnce();
expect(mocks.upsert.mock.calls[0][0].create).not.toHaveProperty("enabled");
mocks.current.mockResolvedValue({ enabled: true });
await expect(
saveTopicsModelSettings("project", { ...settings, clustering: null }),
).rejects.toThrow("while automatic processing is on");
expect(mocks.upsert).toHaveBeenCalledOnce();
});
it("skips the test calls for unchanged models while automatic processing is on", async () => {
mocks.current.mockResolvedValue({
enabled: true,
summaryLlmApiKeyId: "openai",
summaryModel: "gpt-6-luna",
embeddingLlmApiKeyId: "openai",
embeddingModel: "text-embedding-3-small",
embeddingDimensions: 1024,
clusteringLlmApiKeyId: "openai",
clusteringModel: "gpt-5.6-terra",
});
await saveTopicsModelSettings("project", {
...settings,
clustering: slot("openai", "gpt-6.1-sol"),
});
expect(mocks.generateText).toHaveBeenCalledOnce();
expect(mocks.generateText).toHaveBeenCalledWith(
expect.objectContaining({
model: expect.objectContaining({ model: "gpt-6.1-sol" }),
}),
);
expect(mocks.embed).not.toHaveBeenCalled();
});
it("tests every changed slot and names the ones that fail", async () => {
mocks.current.mockResolvedValue({ enabled: true });
mocks.generateText.mockImplementation(async ({ model }) => {
if (model.model === "gpt-5.6-terra")
throw new Error("Model not found (HTTP 404)");
return { output: { summary: "Sign in", status: "applicable" } };
});
mocks.embed.mockResolvedValue({
embeddings: [Array(1536).fill(0.1), Array(1536).fill(0.1)],
});
await expect(saveTopicsModelSettings("project", settings)).rejects.toThrow(
"Embeddings (text-embedding-3-small): The model returned 1536 dimensions instead of 1024. Choose a size this model supports.; Topic clustering (gpt-5.6-terra): Model not found (HTTP 404)",
);
expect(mocks.generateText).toHaveBeenCalledTimes(2);
expect(mocks.embed).toHaveBeenCalledWith(
expect.objectContaining({ dimensions: 1024 }),
);
expect(mocks.upsert).not.toHaveBeenCalled();
});
it.each([
["model", { embedding: slot("openai", "text-embedding-3-large") }],
["dimensions", { embeddingDimensions: 512 as const }],
])(
"locks the embedding %s once summaries are embedded",
async (_, change) => {
mocks.current.mockResolvedValue({
embeddingModel: "text-embedding-3-small",
embeddingDimensions: 1024,
});
mocks.hasEmbeddings.mockResolvedValue(true);
await expect(
saveTopicsModelSettings("project", { ...settings, ...change }),
).rejects.toThrow("cannot change after summaries were embedded");
// Other slots stay editable; the embedding connection may change too.
await saveTopicsModelSettings("project", {
...settings,
summary: slot("openai", "gpt-6-luna-mini"),
});
expect(mocks.upsert).toHaveBeenCalledOnce();
},
);
});
describe("reconcileTopicsConnectionsForProject", () => {
it("rebinds inherited connections after a project transfer", async () => {
const sourceConnection = {
...connection("source", LLMAdapter.OpenAI),
projectId: null,
organizationId: "source-org",
};
const targetConnection = {
...connection("target", LLMAdapter.OpenAI),
projectId: null,
organizationId: "target-org",
};
mocks.current.mockResolvedValue({
id: "config",
projectId: "project",
project: { orgId: "target-org" },
summaryLlmApiKey: sourceConnection,
embeddingLlmApiKey: sourceConnection,
clusteringLlmApiKey: sourceConnection,
});
mocks.keys.mockResolvedValue([targetConnection]);
await reconcileTopicsConnectionsForProject("project");
expect(mocks.update).toHaveBeenCalledWith({
where: { id: "config" },
data: {
summaryLlmApiKeyId: "target",
embeddingLlmApiKeyId: "target",
clusteringLlmApiKeyId: "target",
},
});
expect(mocks.invalidate).toHaveBeenCalledWith("project");
});
it("pauses Topics rather than rebinding to an incompatible adapter", async () => {
const sourceConnection = {
...connection("source", LLMAdapter.OpenAI),
provider: "shared-provider",
projectId: null,
organizationId: "source-org",
};
const targetConnection = {
...connection("target", LLMAdapter.Anthropic),
provider: "shared-provider",
projectId: null,
organizationId: "target-org",
};
mocks.current.mockResolvedValue({
id: "config",
projectId: "project",
project: { orgId: "target-org" },
summaryLlmApiKey: sourceConnection,
embeddingLlmApiKey: sourceConnection,
clusteringLlmApiKey: sourceConnection,
});
mocks.keys.mockImplementation(async ({ where }) =>
where.adapter === targetConnection.adapter ? [targetConnection] : [],
);
await reconcileTopicsConnectionsForProject("project");
expect(mocks.update).toHaveBeenCalledWith({
where: { id: "config" },
data: expect.objectContaining({
summaryLlmApiKeyId: null,
embeddingLlmApiKeyId: null,
clusteringLlmApiKeyId: null,
enabled: false,
}),
});
});
});
describe("getEnabledTopicsModels", () => {
it("returns only enabled projects whose three connections are usable", async () => {
const openai = {
...connection("openai", LLMAdapter.OpenAI),
projectId: "complete",
};
const row = {
projectId: "complete",
project: { orgId: "organization" },
enabled: true,
summaryLlmApiKey: { provider: "openai" },
summaryModel: "gpt-6-luna",
embeddingLlmApiKey: { provider: "openai" },
embeddingModel: "text-embedding-3-small",
embeddingDimensions: 512,
clusteringLlmApiKey: { provider: "openai" },
clusteringModel: "gpt-5.6-terra",
};
mocks.keys.mockResolvedValue([openai]);
mocks.configs.mockResolvedValue([
row,
// A deleted connection leaves its slot empty.
{ ...row, projectId: "missing-slot", clusteringLlmApiKey: null },
]);
const models = await getEnabledTopicsModels(["complete", "missing-slot"]);
expect(mocks.configs).toHaveBeenCalledWith(
expect.objectContaining({
where: {
projectId: { in: ["complete", "missing-slot"] },
enabled: true,
},
}),
);
expect([...models.keys()]).toEqual(["complete"]);
expect(models.get("complete")?.embedding).toMatchObject({
adapter: LLMAdapter.OpenAI,
model: "text-embedding-3-small",
dimensions: 512,
connection: { projectId: "complete" },
});
});
});