🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
getPublishedTopicRun,
getTopicRun,
getTopicRuns,
getTopicProcessingMapIds,
saveTopicRun,
} from "./postgres";
const mocks = vi.hoisted(() => ({
lock: vi.fn(),
runFind: vi.fn(),
runFindMany: vi.fn(),
runUpdate: vi.fn(),
topicFind: vi.fn(),
topicWrite: vi.fn(),
}));
vi.mock("./clickhouse", () => ({
getTopicDefinitions: mocks.topicFind,
writeTopicDefinitions: mocks.topicWrite,
}));
vi.mock("../../db", () => {
const db = {
$queryRaw: mocks.lock,
topicClusteringRun: {
findFirst: mocks.runFind,
findMany: mocks.runFindMany,
update: mocks.runUpdate,
},
};
return {
Prisma: {},
prisma: {
...db,
$transaction: (callback: (tx: typeof db) => Promise<unknown>) =>
callback(db),
},
};
});
const runRow = (overrides: Record<string, unknown> = {}) => ({
id: "run-a",
projectId: "project-a",
facetId: "facet-a",
facetVersion: 1,
status: "pending",
config: {
definitionTimeRange: {
from: "2026-09-01T00:00:00.000Z",
to: "2026-10-01T00:00:00.000Z",
},
},
error: null,
topicVersionIds: [] as string[],
createdAt: new Date("2026-09-16T00:00:00Z"),
startedAt: null as Date | null,
finishedAt: null as Date | null,
...overrides,
});
beforeEach(() => {
vi.resetAllMocks();
mocks.topicFind.mockResolvedValue([]);
});
describe("Topics clustering attempts", () => {
it("preserves the first start time through resume and completion of an empty map", async () => {
let stored = runRow();
mocks.runFind.mockImplementation(async () => stored);
mocks.runUpdate.mockImplementation(async ({ data }) => {
stored = { ...stored, ...data };
return stored;
});
const pending = (await getTopicRun("project-a", "run-a"))!;
const firstStart = "2026-09-16T00:01:00.000Z";
const running = await saveTopicRun({
...pending,
status: "running",
startedAt: firstStart,
});
expect(running.startedAt).toBe(firstStart);
const resumed = await saveTopicRun({
...running,
startedAt: "2026-09-16T00:02:00.000Z",
});
expect(resumed.startedAt).toBe(firstStart);
const cleared = await saveTopicRun({ ...resumed, startedAt: null });
expect(cleared.startedAt).toBe(firstStart);
expect(
await saveTopicRun({
...cleared,
status: "completed",
finishedAt: "2026-09-16T00:03:00.000Z",
}),
).toMatchObject({ status: "completed", topics: [], startedAt: firstStart });
});
});
describe("Topics serving maps", () => {
it("pins only the serving map when prompt version and embedding space match", async () => {
mocks.runFindMany.mockResolvedValue([
{
id: "intent-map",
facetId: "intent",
facetVersion: 2,
config: { embeddingModel: "eu.cohere.embed-v4:0", dimensions: 256 },
},
{
id: "outcome-map",
facetId: "outcome",
facetVersion: 1,
config: { embeddingModel: "eu.cohere.embed-v4:0", dimensions: 512 },
},
]);
expect(
await getTopicProcessingMapIds(
"project-a",
[
{ facetId: "intent", version: 1 },
{ facetId: "intent", version: 2 },
{ facetId: "outcome", version: 1 },
],
{ embeddingModel: "eu.cohere.embed-v4:0", embeddingDimensions: 256 },
),
).toEqual([
{ facetId: "intent", facetVersion: 1, runId: null },
{ facetId: "intent", facetVersion: 2, runId: "intent-map" },
{ facetId: "outcome", facetVersion: 1, runId: null },
]);
expect(mocks.runFindMany).toHaveBeenCalledWith({
where: {
projectId: "project-a",
facetId: { in: ["intent", "outcome"] },
status: "completed",
},
orderBy: [
{ facetVersion: "desc" },
{ createdAt: "desc" },
{ id: "desc" },
],
distinct: ["facetId"],
select: { id: true, facetId: true, facetVersion: true, config: true },
});
});
it("selects completed maps by facet version then creation time and ID, excluding skipped runs", async () => {
mocks.runFind.mockResolvedValue(null);
expect(await getPublishedTopicRun("project-a", "facet-a")).toBeNull();
expect(mocks.runFind).toHaveBeenCalledWith({
where: {
projectId: "project-a",
facetId: "facet-a",
status: "completed",
},
orderBy: [
{ facetVersion: "desc" },
{ createdAt: "desc" },
{ id: "desc" },
],
});
});
it.each(["completed", "skipped"])(
"preserves a %s run when stale work saves another state",
async (status) => {
const row = runRow({
status,
finishedAt: new Date("2026-09-16T00:01:00.000Z"),
});
mocks.runFind.mockResolvedValue(row);
const terminal = (await getTopicRun("project-a", "run-a"))!;
expect(
await saveTopicRun({
...terminal,
status: "failed",
error: "Stale attempt",
finishedAt: null,
}),
).toEqual(terminal);
expect(mocks.runUpdate).not.toHaveBeenCalled();
expect(mocks.topicWrite).not.toHaveBeenCalled();
},
);
});
describe("Topics immutable definitions and run membership", () => {
const topic = {
topicVersionId: "topic-a",
topicId: "stable-a",
projectId: "project-a",
createdByRunId: "run-a",
createdAt: "2026-09-16T00:00:00.000Z",
tags: [],
name: "Billing",
description: "Invoice requests",
centroid: [1, 0],
radius: 0.1,
representativeSummaries: [],
metadata: { count: 0 },
};
it("hydrates a run completed by another worker while acquiring its write lock", async () => {
const row = runRow();
mocks.runFind.mockResolvedValueOnce(row);
const pending = (await getTopicRun("project-a", "run-a"))!;
mocks.runFind.mockResolvedValueOnce(row).mockResolvedValue({
...row,
status: "completed",
topicVersionIds: [topic.topicVersionId],
});
mocks.topicFind.mockResolvedValue([topic]);
const saved = await saveTopicRun({
...pending,
status: "failed",
error: "Stale attempt",
});
expect(saved.status).toBe("completed");
expect(saved.error).toBeNull();
expect(saved.topics).toEqual([topic]);
expect(mocks.runUpdate).not.toHaveBeenCalled();
});
it("hydrates historical runs together using their original definition dates", async () => {
const laterTopic = {
...topic,
topicVersionId: "topic-b",
createdAt: "2026-10-01T00:00:00.000Z",
};
const definitions = [topic, laterTopic];
mocks.runFindMany.mockResolvedValue(
definitions.map((definition, index) =>
runRow({
id: `run-${index}`,
topicVersionIds: [definition.topicVersionId],
config: {
definitionTimeRange: {
from: definition.createdAt,
to: new Date(Date.parse(definition.createdAt) + 1).toISOString(),
},
},
}),
),
);
mocks.topicFind.mockResolvedValue(definitions);
const runs = await getTopicRuns("project-a", ["run-0", "run-1"]);
expect(runs.map((run) => run.topics)).toEqual([[topic], [laterTopic]]);
expect(mocks.runFindMany).toHaveBeenCalledExactlyOnceWith({
where: { projectId: "project-a", id: { in: ["run-0", "run-1"] } },
});
expect(mocks.topicFind).toHaveBeenCalledExactlyOnceWith(
"project-a",
["topic-a", "topic-b"],
{
from: new Date(topic.createdAt),
to: new Date(Date.parse(laterTopic.createdAt) + 1),
},
);
});
it("reuses a definition across runs and preserves the historical set when removing current membership", async () => {
const rows = new Map([
[
"run-a",
runRow({ id: "run-a", topicVersionIds: [topic.topicVersionId] }),
],
["run-b", runRow({ id: "run-b" })],
]);
mocks.runFind.mockImplementation(async ({ where }) => {
const row = rows.get(where.id);
return row?.projectId === where.projectId ? row : null;
});
mocks.topicFind.mockImplementation(async (_project, ids: string[]) =>
ids.includes(topic.topicVersionId) ? [topic] : [],
);
mocks.runUpdate.mockImplementation(async ({ where, data }) => {
const updated = { ...rows.get(where.id), ...data };
rows.set(updated.id, updated);
return updated;
});
const run = (await getTopicRun("project-a", "run-b"))!;
const saved = await saveTopicRun({
...run,
topics: [
{
...topic,
centroid: topic.centroid.map((value) => value + Number.EPSILON),
radius: topic.radius + Number.EPSILON,
},
],
});
expect(saved).toMatchObject({
id: "run-b",
projectId: "project-a",
facetId: "facet-a",
facetVersion: 1,
topics: [topic],
});
expect(await getTopicRun("other-project", "run-b")).toBeNull();
expect(saved.topics[0].createdByRunId).toBe("run-a");
expect(saved.config.definitionTimeRange).toEqual({
from: topic.createdAt,
to: "2026-09-16T00:00:00.001Z",
});
expect(mocks.topicFind).toHaveBeenCalledWith(
"project-a",
[topic.topicVersionId],
{
from: new Date(topic.createdAt),
to: new Date("2026-09-16T00:00:00.001Z"),
},
);
expect(mocks.topicWrite).not.toHaveBeenCalled();
expect((await getTopicRun("project-a", "run-a"))!.topics).toEqual([topic]);
await saveTopicRun({ ...saved, topics: [] });
expect((await getTopicRun("project-a", "run-b"))!.topics).toEqual([]);
expect((await getTopicRun("project-a", "run-a"))!.topics).toEqual([topic]);
expect(mocks.topicWrite).not.toHaveBeenCalled();
});
it("verifies a new definition despite numeric roundoff and returns stored geometry before publishing membership", async () => {
const row = runRow({ id: "run-b" });
const created = {
...topic,
createdByRunId: row.id,
centroid: [-0.003662027125082599, 1],
};
const persisted = {
...created,
centroid: [-0.0036620271250825988, 1],
radius: created.radius + Number.EPSILON,
};
mocks.runFind.mockResolvedValue(row);
let definitionInserted = false;
mocks.topicWrite.mockImplementation(async () => {
definitionInserted = true;
});
mocks.topicFind.mockImplementation(async (_project, ids: string[]) =>
definitionInserted && ids.includes(created.topicVersionId)
? [persisted]
: [],
);
mocks.runUpdate.mockImplementation(async ({ data }) => ({
...row,
...data,
}));
const run = (await getTopicRun("project-a", row.id))!;
const saved = await saveTopicRun({
...run,
topics: [created],
status: "completed",
});
expect(saved.topics).toEqual([persisted]);
expect(mocks.topicWrite).toHaveBeenCalledWith([created]);
expect(mocks.runUpdate).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({
topicVersionIds: [created.topicVersionId],
}),
}),
);
expect(mocks.topicWrite.mock.invocationCallOrder[0]).toBeLessThan(
mocks.lock.mock.invocationCallOrder[0],
);
expect(mocks.topicFind.mock.invocationCallOrder[1]).toBeLessThan(
mocks.runUpdate.mock.invocationCallOrder[0],
);
});
it.each([
"missing",
"changed-name",
"changed-metadata",
"changed-centroid",
"changed-radius",
"changed-dimensions",
"nonfinite-centroid",
"nonfinite-radius",
"empty-centroid",
"sparse-centroid",
"wrong-project",
"insert-failed",
"readback-missing",
])("does not publish a run when its definition is %s", async (failure) => {
const row = runRow({ id: "run-b" });
mocks.runFind.mockResolvedValue(row);
const changes: Record<string, Partial<typeof topic>> = {
"changed-name": { name: "Changed" },
"changed-metadata": { metadata: { count: Number.EPSILON } },
"changed-centroid": { centroid: [1, 1e-8] },
"changed-radius": { radius: topic.radius + 1e-8 },
"changed-dimensions": { centroid: [1] },
"nonfinite-centroid": { centroid: [1, Number.NaN] },
"nonfinite-radius": { radius: Number.POSITIVE_INFINITY },
"empty-centroid": { centroid: [] },
"sparse-centroid": { centroid: new Array<number>(2) },
};
const candidate = {
...topic,
...changes[failure],
projectId:
failure === "wrong-project" ? "other-project" : topic.projectId,
createdByRunId:
failure.startsWith("insert") || failure.startsWith("readback")
? row.id
: topic.createdByRunId,
};
if (failure.startsWith("changed")) {
mocks.topicFind.mockResolvedValue([topic]);
} else {
mocks.topicFind.mockResolvedValue(changes[failure] ? [candidate] : []);
}
mocks.topicWrite.mockImplementation(async () => {
if (failure === "insert-failed") throw new Error("Insert failed");
});
const run = (await getTopicRun("project-a", row.id))!;
await expect(
saveTopicRun({
...run,
topics: [candidate],
status: "completed",
}),
).rejects.toThrow();
expect(mocks.runUpdate).not.toHaveBeenCalled();
});
it("names the fields a definition readback lost, never their values", async () => {
const row = runRow({ id: "run-b" });
const created = {
...topic,
createdByRunId: row.id,
name: "Secret topic name",
metadata: { count: 1, namingSampled: undefined },
};
mocks.runFind.mockResolvedValue(row);
let definitionInserted = false;
mocks.topicWrite.mockImplementation(async () => {
definitionInserted = true;
});
mocks.topicFind.mockImplementation(async () =>
definitionInserted
? [{ ...created, name: "Other", metadata: { count: 2 } }]
: [],
);
const run = (await getTopicRun("project-a", row.id))!;
const failure = await saveTopicRun({
...run,
topics: [created],
status: "completed",
}).catch((error: unknown) => error);
expect(failure).toMatchObject({
name: "TopicDefinitionReadbackError",
message:
"Topic definition readback did not match for 1 of 1 new definitions (name, metadata.count, metadata.namingSampled).",
});
expect((failure as Error).message).not.toMatch(/Secret|Other/);
expect(mocks.runUpdate).not.toHaveBeenCalled();
});
it("fails hydration instead of exposing only the definitions that remain", async () => {
mocks.runFind.mockResolvedValue(
runRow({
id: "run-b",
topicVersionIds: [topic.topicVersionId, "missing"],
}),
);
mocks.topicFind.mockResolvedValue([topic]);
await expect(getTopicRun("project-a", "run-b")).rejects.toThrow(
"missing definition",
);
});
});