Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

14.6 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
  getPublishedTopicRun,
  getTopicRun,
  getTopicRuns,
  getTopicProcessingMapIds,
  saveTopicRun,
} from "./postgres";

const mocks = vi.hoisted(() => ({
  lock: vi.fn(),
  runFind: vi.fn(),
  runFindMany: vi.fn(),
  runUpdate: vi.fn(),
  topicFind: vi.fn(),
  topicWrite: vi.fn(),
}));
vi.mock("./clickhouse", () => ({
  getTopicDefinitions: mocks.topicFind,
  writeTopicDefinitions: mocks.topicWrite,
}));
vi.mock("../../db", () => {
  const db = {
    $queryRaw: mocks.lock,
    topicClusteringRun: {
      findFirst: mocks.runFind,
      findMany: mocks.runFindMany,
      update: mocks.runUpdate,
    },
  };
  return {
    Prisma: {},
    prisma: {
      ...db,
      $transaction: (callback: (tx: typeof db) => Promise<unknown>) =>
        callback(db),
    },
  };
});

const runRow = (overrides: Record<string, unknown> = {}) => ({
  id: "run-a",
  projectId: "project-a",
  facetId: "facet-a",
  facetVersion: 1,
  status: "pending",
  config: {
    definitionTimeRange: {
      from: "2026-09-01T00:00:00.000Z",
      to: "2026-10-01T00:00:00.000Z",
    },
  },
  error: null,
  topicVersionIds: [] as string[],
  createdAt: new Date("2026-09-16T00:00:00Z"),
  startedAt: null as Date | null,
  finishedAt: null as Date | null,
  ...overrides,
});

beforeEach(() => {
  vi.resetAllMocks();
  mocks.topicFind.mockResolvedValue([]);
});

describe("Topics clustering attempts", () => {
  it("preserves the first start time through resume and completion of an empty map", async () => {
    let stored = runRow();
    mocks.runFind.mockImplementation(async () => stored);
    mocks.runUpdate.mockImplementation(async ({ data }) => {
      stored = { ...stored, ...data };
      return stored;
    });
    const pending = (await getTopicRun("project-a", "run-a"))!;
    const firstStart = "2026-09-16T00:01:00.000Z";
    const running = await saveTopicRun({
      ...pending,
      status: "running",
      startedAt: firstStart,
    });
    expect(running.startedAt).toBe(firstStart);
    const resumed = await saveTopicRun({
      ...running,
      startedAt: "2026-09-16T00:02:00.000Z",
    });
    expect(resumed.startedAt).toBe(firstStart);
    const cleared = await saveTopicRun({ ...resumed, startedAt: null });
    expect(cleared.startedAt).toBe(firstStart);
    expect(
      await saveTopicRun({
        ...cleared,
        status: "completed",
        finishedAt: "2026-09-16T00:03:00.000Z",
      }),
    ).toMatchObject({ status: "completed", topics: [], startedAt: firstStart });
  });
});

describe("Topics serving maps", () => {
  it("pins only the serving map when prompt version and embedding space match", async () => {
    mocks.runFindMany.mockResolvedValue([
      {
        id: "intent-map",
        facetId: "intent",
        facetVersion: 2,
        config: { embeddingModel: "eu.cohere.embed-v4:0", dimensions: 256 },
      },
      {
        id: "outcome-map",
        facetId: "outcome",
        facetVersion: 1,
        config: { embeddingModel: "eu.cohere.embed-v4:0", dimensions: 512 },
      },
    ]);
    expect(
      await getTopicProcessingMapIds(
        "project-a",
        [
          { facetId: "intent", version: 1 },
          { facetId: "intent", version: 2 },
          { facetId: "outcome", version: 1 },
        ],
        { embeddingModel: "eu.cohere.embed-v4:0", embeddingDimensions: 256 },
      ),
    ).toEqual([
      { facetId: "intent", facetVersion: 1, runId: null },
      { facetId: "intent", facetVersion: 2, runId: "intent-map" },
      { facetId: "outcome", facetVersion: 1, runId: null },
    ]);
    expect(mocks.runFindMany).toHaveBeenCalledWith({
      where: {
        projectId: "project-a",
        facetId: { in: ["intent", "outcome"] },
        status: "completed",
      },
      orderBy: [
        { facetVersion: "desc" },
        { createdAt: "desc" },
        { id: "desc" },
      ],
      distinct: ["facetId"],
      select: { id: true, facetId: true, facetVersion: true, config: true },
    });
  });

  it("selects completed maps by facet version then creation time and ID, excluding skipped runs", async () => {
    mocks.runFind.mockResolvedValue(null);
    expect(await getPublishedTopicRun("project-a", "facet-a")).toBeNull();
    expect(mocks.runFind).toHaveBeenCalledWith({
      where: {
        projectId: "project-a",
        facetId: "facet-a",
        status: "completed",
      },
      orderBy: [
        { facetVersion: "desc" },
        { createdAt: "desc" },
        { id: "desc" },
      ],
    });
  });

  it.each(["completed", "skipped"])(
    "preserves a %s run when stale work saves another state",
    async (status) => {
      const row = runRow({
        status,
        finishedAt: new Date("2026-09-16T00:01:00.000Z"),
      });
      mocks.runFind.mockResolvedValue(row);
      const terminal = (await getTopicRun("project-a", "run-a"))!;
      expect(
        await saveTopicRun({
          ...terminal,
          status: "failed",
          error: "Stale attempt",
          finishedAt: null,
        }),
      ).toEqual(terminal);
      expect(mocks.runUpdate).not.toHaveBeenCalled();
      expect(mocks.topicWrite).not.toHaveBeenCalled();
    },
  );
});

describe("Topics immutable definitions and run membership", () => {
  const topic = {
    topicVersionId: "topic-a",
    topicId: "stable-a",
    projectId: "project-a",
    createdByRunId: "run-a",
    createdAt: "2026-09-16T00:00:00.000Z",
    tags: [],
    name: "Billing",
    description: "Invoice requests",
    centroid: [1, 0],
    radius: 0.1,
    representativeSummaries: [],
    metadata: { count: 0 },
  };

  it("hydrates a run completed by another worker while acquiring its write lock", async () => {
    const row = runRow();
    mocks.runFind.mockResolvedValueOnce(row);
    const pending = (await getTopicRun("project-a", "run-a"))!;
    mocks.runFind.mockResolvedValueOnce(row).mockResolvedValue({
      ...row,
      status: "completed",
      topicVersionIds: [topic.topicVersionId],
    });
    mocks.topicFind.mockResolvedValue([topic]);
    const saved = await saveTopicRun({
      ...pending,
      status: "failed",
      error: "Stale attempt",
    });
    expect(saved.status).toBe("completed");
    expect(saved.error).toBeNull();
    expect(saved.topics).toEqual([topic]);
    expect(mocks.runUpdate).not.toHaveBeenCalled();
  });

  it("hydrates historical runs together using their original definition dates", async () => {
    const laterTopic = {
      ...topic,
      topicVersionId: "topic-b",
      createdAt: "2026-10-01T00:00:00.000Z",
    };
    const definitions = [topic, laterTopic];
    mocks.runFindMany.mockResolvedValue(
      definitions.map((definition, index) =>
        runRow({
          id: `run-${index}`,
          topicVersionIds: [definition.topicVersionId],
          config: {
            definitionTimeRange: {
              from: definition.createdAt,
              to: new Date(Date.parse(definition.createdAt) + 1).toISOString(),
            },
          },
        }),
      ),
    );
    mocks.topicFind.mockResolvedValue(definitions);

    const runs = await getTopicRuns("project-a", ["run-0", "run-1"]);

    expect(runs.map((run) => run.topics)).toEqual([[topic], [laterTopic]]);
    expect(mocks.runFindMany).toHaveBeenCalledExactlyOnceWith({
      where: { projectId: "project-a", id: { in: ["run-0", "run-1"] } },
    });
    expect(mocks.topicFind).toHaveBeenCalledExactlyOnceWith(
      "project-a",
      ["topic-a", "topic-b"],
      {
        from: new Date(topic.createdAt),
        to: new Date(Date.parse(laterTopic.createdAt) + 1),
      },
    );
  });

  it("reuses a definition across runs and preserves the historical set when removing current membership", async () => {
    const rows = new Map([
      [
        "run-a",
        runRow({ id: "run-a", topicVersionIds: [topic.topicVersionId] }),
      ],
      ["run-b", runRow({ id: "run-b" })],
    ]);
    mocks.runFind.mockImplementation(async ({ where }) => {
      const row = rows.get(where.id);
      return row?.projectId === where.projectId ? row : null;
    });
    mocks.topicFind.mockImplementation(async (_project, ids: string[]) =>
      ids.includes(topic.topicVersionId) ? [topic] : [],
    );
    mocks.runUpdate.mockImplementation(async ({ where, data }) => {
      const updated = { ...rows.get(where.id), ...data };
      rows.set(updated.id, updated);
      return updated;
    });
    const run = (await getTopicRun("project-a", "run-b"))!;
    const saved = await saveTopicRun({
      ...run,
      topics: [
        {
          ...topic,
          centroid: topic.centroid.map((value) => value + Number.EPSILON),
          radius: topic.radius + Number.EPSILON,
        },
      ],
    });
    expect(saved).toMatchObject({
      id: "run-b",
      projectId: "project-a",
      facetId: "facet-a",
      facetVersion: 1,
      topics: [topic],
    });
    expect(await getTopicRun("other-project", "run-b")).toBeNull();
    expect(saved.topics[0].createdByRunId).toBe("run-a");
    expect(saved.config.definitionTimeRange).toEqual({
      from: topic.createdAt,
      to: "2026-09-16T00:00:00.001Z",
    });
    expect(mocks.topicFind).toHaveBeenCalledWith(
      "project-a",
      [topic.topicVersionId],
      {
        from: new Date(topic.createdAt),
        to: new Date("2026-09-16T00:00:00.001Z"),
      },
    );
    expect(mocks.topicWrite).not.toHaveBeenCalled();
    expect((await getTopicRun("project-a", "run-a"))!.topics).toEqual([topic]);

    await saveTopicRun({ ...saved, topics: [] });
    expect((await getTopicRun("project-a", "run-b"))!.topics).toEqual([]);
    expect((await getTopicRun("project-a", "run-a"))!.topics).toEqual([topic]);
    expect(mocks.topicWrite).not.toHaveBeenCalled();
  });

  it("verifies a new definition despite numeric roundoff and returns stored geometry before publishing membership", async () => {
    const row = runRow({ id: "run-b" });
    const created = {
      ...topic,
      createdByRunId: row.id,
      centroid: [-0.003662027125082599, 1],
    };
    const persisted = {
      ...created,
      centroid: [-0.0036620271250825988, 1],
      radius: created.radius + Number.EPSILON,
    };
    mocks.runFind.mockResolvedValue(row);
    let definitionInserted = false;
    mocks.topicWrite.mockImplementation(async () => {
      definitionInserted = true;
    });
    mocks.topicFind.mockImplementation(async (_project, ids: string[]) =>
      definitionInserted && ids.includes(created.topicVersionId)
        ? [persisted]
        : [],
    );
    mocks.runUpdate.mockImplementation(async ({ data }) => ({
      ...row,
      ...data,
    }));
    const run = (await getTopicRun("project-a", row.id))!;
    const saved = await saveTopicRun({
      ...run,
      topics: [created],
      status: "completed",
    });
    expect(saved.topics).toEqual([persisted]);
    expect(mocks.topicWrite).toHaveBeenCalledWith([created]);
    expect(mocks.runUpdate).toHaveBeenCalledWith(
      expect.objectContaining({
        data: expect.objectContaining({
          topicVersionIds: [created.topicVersionId],
        }),
      }),
    );
    expect(mocks.topicWrite.mock.invocationCallOrder[0]).toBeLessThan(
      mocks.lock.mock.invocationCallOrder[0],
    );
    expect(mocks.topicFind.mock.invocationCallOrder[1]).toBeLessThan(
      mocks.runUpdate.mock.invocationCallOrder[0],
    );
  });

  it.each([
    "missing",
    "changed-name",
    "changed-metadata",
    "changed-centroid",
    "changed-radius",
    "changed-dimensions",
    "nonfinite-centroid",
    "nonfinite-radius",
    "empty-centroid",
    "sparse-centroid",
    "wrong-project",
    "insert-failed",
    "readback-missing",
  ])("does not publish a run when its definition is %s", async (failure) => {
    const row = runRow({ id: "run-b" });
    mocks.runFind.mockResolvedValue(row);
    const changes: Record<string, Partial<typeof topic>> = {
      "changed-name": { name: "Changed" },
      "changed-metadata": { metadata: { count: Number.EPSILON } },
      "changed-centroid": { centroid: [1, 1e-8] },
      "changed-radius": { radius: topic.radius + 1e-8 },
      "changed-dimensions": { centroid: [1] },
      "nonfinite-centroid": { centroid: [1, Number.NaN] },
      "nonfinite-radius": { radius: Number.POSITIVE_INFINITY },
      "empty-centroid": { centroid: [] },
      "sparse-centroid": { centroid: new Array<number>(2) },
    };
    const candidate = {
      ...topic,
      ...changes[failure],
      projectId:
        failure === "wrong-project" ? "other-project" : topic.projectId,
      createdByRunId:
        failure.startsWith("insert") || failure.startsWith("readback")
          ? row.id
          : topic.createdByRunId,
    };
    if (failure.startsWith("changed")) {
      mocks.topicFind.mockResolvedValue([topic]);
    } else {
      mocks.topicFind.mockResolvedValue(changes[failure] ? [candidate] : []);
    }
    mocks.topicWrite.mockImplementation(async () => {
      if (failure === "insert-failed") throw new Error("Insert failed");
    });
    const run = (await getTopicRun("project-a", row.id))!;
    await expect(
      saveTopicRun({
        ...run,
        topics: [candidate],
        status: "completed",
      }),
    ).rejects.toThrow();
    expect(mocks.runUpdate).not.toHaveBeenCalled();
  });

  it("names the fields a definition readback lost, never their values", async () => {
    const row = runRow({ id: "run-b" });
    const created = {
      ...topic,
      createdByRunId: row.id,
      name: "Secret topic name",
      metadata: { count: 1, namingSampled: undefined },
    };
    mocks.runFind.mockResolvedValue(row);
    let definitionInserted = false;
    mocks.topicWrite.mockImplementation(async () => {
      definitionInserted = true;
    });
    mocks.topicFind.mockImplementation(async () =>
      definitionInserted
        ? [{ ...created, name: "Other", metadata: { count: 2 } }]
        : [],
    );
    const run = (await getTopicRun("project-a", row.id))!;
    const failure = await saveTopicRun({
      ...run,
      topics: [created],
      status: "completed",
    }).catch((error: unknown) => error);
    expect(failure).toMatchObject({
      name: "TopicDefinitionReadbackError",
      message:
        "Topic definition readback did not match for 1 of 1 new definitions (name, metadata.count, metadata.namingSampled).",
    });
    expect((failure as Error).message).not.toMatch(/Secret|Other/);
    expect(mocks.runUpdate).not.toHaveBeenCalled();
  });

  it("fails hydration instead of exposing only the definitions that remain", async () => {
    mocks.runFind.mockResolvedValue(
      runRow({
        id: "run-b",
        topicVersionIds: [topic.topicVersionId, "missing"],
      }),
    );
    mocks.topicFind.mockResolvedValue([topic]);
    await expect(getTopicRun("project-a", "run-b")).rejects.toThrow(
      "missing definition",
    );
  });
});