Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

37.9 KB
Raw Blame
const {
  mockAddScoreDelete,
  mockAddBatchAction,
  mockGetEventsExactFilterOptionsForColumns,
} = vi.hoisted(() => ({
  mockAddScoreDelete: vi.fn(),
  mockAddBatchAction: vi.fn(),
  mockGetEventsExactFilterOptionsForColumns: vi.fn(async () => []),
}));

vi.mock("@langfuse/shared/src/server", async () => {
  const originalModule = await vi.importActual("@langfuse/shared/src/server");
  return {
    ...originalModule,
    ScoreDeleteQueue: {
      getInstance: vi.fn(() => ({
        add: mockAddScoreDelete,
      })),
    },
    BatchActionQueue: {
      getInstance: vi.fn(() => ({
        add: mockAddBatchAction,
      })),
    },
    getEventsExactFilterOptionsForColumns:
      mockGetEventsExactFilterOptionsForColumns,
  };
});

import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import type { Session } from "next-auth";
import { prisma } from "@langfuse/shared/src/db";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import { ScoreConfigDataType } from "@langfuse/shared";
import {
  createEvent,
  createEventsCh,
  createObservation,
  createObservationsCh,
  createTrace,
  createTraceScore,
  createTracesCh,
  createScoresCh,
  ScoreDeleteQueue,
  BatchActionQueue,
  QueueJobs,
  createOrgProjectAndApiKey,
  getScoreById,
} from "@langfuse/shared/src/server";
import { env } from "@/src/env.mjs";
import { observationScopeFilter } from "@/src/features/filters/config/scores-config";
import { SCORES_FIELD_REGISTRY } from "@/src/features/scores/constants/scoresSearchRegistry";
import { planCommit } from "@/src/features/search-bar/lib/commit";
import { randomUUID } from "crypto";

const maybeEvents =
  env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true"
    ? describe
    : describe.skip;

describe("scores trpc", () => {
  let projectId: string;
  let orgId: string;
  let caller: ReturnType<typeof appRouter.createCaller>;

  beforeEach(async () => {
    const setup = await createOrgProjectAndApiKey();
    projectId = setup.projectId;
    orgId = setup.orgId;
    mockAddScoreDelete.mockClear();
    mockAddBatchAction.mockClear();
    mockGetEventsExactFilterOptionsForColumns.mockClear();

    const session: Session = {
      expires: "1",
      user: {
        id: "user-1",
        canCreateOrganizations: true,
        name: "Demo User",
        organizations: [
          {
            id: orgId,
            name: "Test Organization",
            role: "OWNER",
            plan: "cloud:hobby",
            cloudConfig: undefined,
            metadata: {},
            aiFeaturesEnabled: false,
            aiTelemetryEnabled: true,
            projects: [
              {
                id: projectId,
                role: "ADMIN",
                retentionDays: 30,
                deletedAt: null,
                name: "Test Project",
                hasTraces: false,
                metadata: {},
                createdAt: new Date().toISOString(),
              },
            ],
          },
        ],
        featureFlags: testFeatureFlags(),
        admin: true,
      },
      environment: {} as any,
    };

    const ctx = createInnerTRPCContext({ session, headers: {} });
    caller = appRouter.createCaller({ ...ctx, prisma });
  });

  describe("scores.all", () => {
    it("preserves evaluator-test filters for score rows and counts on both read paths", async () => {
      const scores = [
        createTraceScore({
          project_id: projectId,
          name: "missing-test-marker",
          metadata: {},
        }),
        createTraceScore({
          project_id: projectId,
          name: "false-test-marker",
          metadata: { evaluator_test: "false" },
        }),
        createTraceScore({
          project_id: projectId,
          name: "true-test-marker",
          metadata: { evaluator_test: "true" },
        }),
      ];
      await createScoresCh(scores);

      for (const { operator, value, expectedIds } of [
        {
          operator: "=" as const,
          value: false,
          expectedIds: [scores[0].id, scores[1].id],
        },
        {
          operator: "=" as const,
          value: true,
          expectedIds: [scores[2].id],
        },
        {
          operator: "<>" as const,
          value: true,
          expectedIds: [scores[0].id, scores[1].id],
        },
        {
          operator: "<>" as const,
          value: false,
          expectedIds: [scores[2].id],
        },
      ]) {
        const payload = {
          projectId,
          filter: [
            {
              column: "isEvaluatorTest",
              type: "boolean" as const,
              operator,
              value,
            },
          ],
          orderBy: { column: "timestamp", order: "DESC" as const },
          page: 0,
          limit: 50,
        };
        const [rows, eventRows, count, eventCount] = await Promise.all([
          caller.scores.all(payload),
          caller.scores.allFromEvents(payload),
          caller.scores.countAll({ ...payload, orderBy: null }),
          caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
        ]);

        expect(rows.scores.map(({ id }) => id).sort()).toEqual(
          [...expectedIds].sort(),
        );
        expect(eventRows.scores.map(({ id }) => id).sort()).toEqual(
          [...expectedIds].sort(),
        );
        expect(count.totalCount).toBe(expectedIds.length);
        expect(eventCount.totalCount).toBe(expectedIds.length);
      }
    });

    it("pages through scores with identical timestamps without duplicates or gaps on both read paths", async () => {
      const timestamp = new Date();
      const scores = Array.from({ length: 8 }, () =>
        createTraceScore({ project_id: projectId, name: "batch", timestamp }),
      );
      await createScoresCh(scores);

      for (const procedure of [
        caller.scores.all,
        caller.scores.allFromEvents,
      ]) {
        const pages = await Promise.all(
          scores.map((_, page) =>
            procedure({
              projectId,
              filter: [],
              orderBy: { column: "timestamp", order: "DESC" },
              page,
              limit: 1,
            }),
          ),
        );
        const pagedIds = pages.flatMap((p) => p.scores.map(({ id }) => id));

        expect(pagedIds).toHaveLength(scores.length);
        expect(new Set(pagedIds)).toEqual(new Set(scores.map(({ id }) => id)));
      }
    });

    it("applies search-bar name matching and repeated numeric bounds to v4 rows and counts", async () => {
      await createScoresCh(
        [
          { name: "Rouge Score", value: 0.5 },
          { name: "Weighted Rouge Score", value: 0.7 },
          { name: "confidence", value: 0.6 },
          { name: "Rouge Score", value: 0.1 },
          { name: "confidence", value: 0.95 },
        ].map((score) =>
          createTraceScore({
            project_id: projectId,
            data_type: "NUMERIC",
            ...score,
          }),
        ),
      );

      for (const { query, names } of [
        {
          query: "Rouge Score value:>0.2 value:<0.8",
          names: ["Rouge Score", "Weighted Rouge Score"],
        },
        {
          query: 'name:("Rouge Score" OR confidence) value:>0.2 value:<0.8',
          names: ["Rouge Score", "confidence"],
        },
      ]) {
        const committed = planCommit(query, undefined, SCORES_FIELD_REGISTRY);
        expect(committed.status, query).toBe("committed");
        if (committed.status !== "committed") {
          throw new Error(`Invalid score search: ${query}`);
        }

        const payload = { projectId, filter: committed.filters };
        const [rows, count] = await Promise.all([
          caller.scores.allFromEvents({
            ...payload,
            orderBy: { column: "timestamp", order: "DESC" },
            page: 0,
            limit: 50,
          }),
          caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
        ]);

        expect(rows.scores.map((score) => score.name).sort(), query).toEqual(
          names,
        );
        expect(count.totalCount, query).toBe(names.length);
      }
    });

    it("counts v4 scores within a timestamp window, pruning other day-buckets and honoring the exact bound", async () => {
      // Exercises the FINAL-free count path's date handling: the coarse
      // toDate(timestamp) prune must drop other day-buckets pre-dedup, while the
      // exact timestamp bound (re-applied post-dedup) must still exclude an
      // in-bucket-but-out-of-window row. Regression guard for
      // buildScoresDatePrune.
      const from = new Date("2024-06-15T10:00:00.000Z");
      const to = new Date("2024-06-15T14:00:00.000Z");

      await createScoresCh([
        createTraceScore({
          project_id: projectId,
          name: "in-window",
          timestamp: new Date("2024-06-15T12:00:00.000Z"),
        }),
        // Same day (survives the coarse toDate prune) but after `to` — only the
        // exact post-dedup bound excludes it.
        createTraceScore({
          project_id: projectId,
          name: "same-day-after-window",
          timestamp: new Date("2024-06-15T20:00:00.000Z"),
        }),
        createTraceScore({
          project_id: projectId,
          name: "earlier-day",
          timestamp: new Date("2024-06-10T12:00:00.000Z"),
        }),
      ]);

      const payload = {
        projectId,
        filter: [
          {
            column: "timestamp",
            type: "datetime" as const,
            operator: ">=" as const,
            value: from,
          },
          {
            column: "timestamp",
            type: "datetime" as const,
            operator: "<" as const,
            value: to,
          },
        ],
      };
      const [rows, count] = await Promise.all([
        caller.scores.allFromEvents({
          ...payload,
          orderBy: { column: "timestamp", order: "DESC" },
          page: 0,
          limit: 50,
        }),
        caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
      ]);

      expect(rows.scores.map((score) => score.name)).toEqual(["in-window"]);
      expect(count.totalCount).toBe(rows.scores.length);
      expect(count.totalCount).toBe(1);
    });

    it("reconstructs the latest version of a score id without FINAL", async () => {
      // Two versions of one score id share the dedup key
      // (project_id, toDate(timestamp), name, id) and differ only in event_ts.
      // The FINAL-free rows path (ORDER BY event_ts DESC + LIMIT 1 BY) and count
      // path (argMax) must both collapse them to the single latest version and
      // return the whole latest row.
      const scoreId = randomUUID();
      const timestamp = new Date("2024-06-15T12:00:00.000Z");

      await createScoresCh([
        createTraceScore({
          id: scoreId,
          project_id: projectId,
          name: "versioned-score",
          timestamp,
          event_ts: new Date("2024-06-15T12:00:00.000Z"),
          value: 0.1,
          comment: "stale",
        }),
        createTraceScore({
          id: scoreId,
          project_id: projectId,
          name: "versioned-score",
          timestamp,
          event_ts: new Date("2024-06-15T12:05:00.000Z"),
          value: 0.9,
          comment: "latest",
        }),
      ]);

      const [rows, count] = await Promise.all([
        caller.scores.allFromEvents({
          projectId,
          filter: [],
          orderBy: { column: "timestamp", order: "DESC" },
          page: 0,
          limit: 50,
        }),
        caller.scores.countAllFromEvents({
          projectId,
          filter: [],
          orderBy: null,
        }),
      ]);

      expect(rows.scores.map((score) => score.id)).toEqual([scoreId]);
      expect(rows.scores[0].value).toBe(0.9);
      expect(rows.scores[0].comment).toBe("latest");
      expect(count.totalCount).toBe(1);
    });

    it("filters mutable columns after dedup, never against a stale version", async () => {
      // value/comment/timestamp/trace_id are mutable across a score's versions,
      // so a filter must run AFTER dedup. Each id below has a stale version that
      // disagrees with its latest on `value > 0.5`; filtering the raw rows
      // pre-dedup would wrongly keep `latest-low` (its stale 0.9) and could drop
      // `latest-high`. The predicate must be decided on the latest version only.
      const latestLowId = randomUUID(); // stale 0.9, latest 0.1 -> excluded
      const latestHighId = randomUUID(); // stale 0.1, latest 0.9 -> included
      const timestamp = new Date("2024-06-15T12:00:00.000Z");

      await createScoresCh([
        createTraceScore({
          id: latestLowId,
          project_id: projectId,
          name: "latest-low",
          timestamp,
          event_ts: new Date("2024-06-15T12:00:00.000Z"),
          value: 0.9,
        }),
        createTraceScore({
          id: latestLowId,
          project_id: projectId,
          name: "latest-low",
          timestamp,
          event_ts: new Date("2024-06-15T12:05:00.000Z"),
          value: 0.1,
        }),
        createTraceScore({
          id: latestHighId,
          project_id: projectId,
          name: "latest-high",
          timestamp,
          event_ts: new Date("2024-06-15T12:00:00.000Z"),
          value: 0.1,
        }),
        createTraceScore({
          id: latestHighId,
          project_id: projectId,
          name: "latest-high",
          timestamp,
          event_ts: new Date("2024-06-15T12:05:00.000Z"),
          value: 0.9,
        }),
      ]);

      const payload = {
        projectId,
        filter: [
          {
            column: "value",
            type: "number" as const,
            operator: ">" as const,
            value: 0.5,
          },
        ],
      };
      const [rows, count] = await Promise.all([
        caller.scores.allFromEvents({
          ...payload,
          orderBy: { column: "timestamp", order: "DESC" },
          page: 0,
          limit: 50,
        }),
        caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
      ]);

      expect(rows.scores.map((score) => score.id)).toEqual([latestHighId]);
      expect(count.totalCount).toBe(1);
    });

    it("returns the recorded evaluator and resolves legacy scores by rule assignment and score name", async () => {
      const [recordedEvaluator, matchingEvaluator, otherEvaluator] =
        await Promise.all(
          ["Recorded evaluator", "Matching evaluator", "Other evaluator"].map(
            (name) =>
              prisma.evaluator.create({
                data: {
                  projectId,
                  name,
                  type: "LLM_AS_JUDGE",
                },
              }),
          ),
        );
      const rule = await prisma.evaluationRule.create({
        data: {
          projectId,
          name: "Evaluation rule",
          targetObject: "EVENT",
          filter: [],
          sampling: 1,
          delay: 0,
          assignments: {
            create: [matchingEvaluator, otherEvaluator].map((evaluator) => ({
              projectId,
              evaluatorId: evaluator.id,
            })),
          },
        },
      });
      const recordedScore = createTraceScore({
        project_id: projectId,
        name: recordedEvaluator.name,
        metadata: { evaluator_id: recordedEvaluator.id },
      });
      const legacyScore = createTraceScore({
        project_id: projectId,
        name: matchingEvaluator.name,
        metadata: {},
      });

      await Promise.all([
        createScoresCh([recordedScore, legacyScore]),
        prisma.jobExecution.createMany({
          data: [recordedScore, legacyScore].map((score) => ({
            projectId,
            jobConfigurationId: rule.id,
            jobOutputScoreId: score.id,
            status: "COMPLETED",
          })),
        }),
      ]);

      const result = await caller.scores.allFromEvents({
        projectId,
        filter: [],
        orderBy: { column: "timestamp", order: "DESC" },
        page: 0,
        limit: 50,
      });
      const evaluatorIdByScoreId = new Map(
        result.scores.map((score) => [score.id, score.evaluatorId]),
      );

      expect(evaluatorIdByScoreId.get(recordedScore.id)).toBe(
        recordedEvaluator.id,
      );
      expect(evaluatorIdByScoreId.get(legacyScore.id)).toBe(
        matchingEvaluator.id,
      );
    });

    it("filters evaluator scores only by recorded evaluator metadata", async () => {
      const [evaluator, otherEvaluator] = await Promise.all(
        ["Evaluator", "Other evaluator"].map((name) =>
          prisma.evaluator.create({
            data: { projectId, name, type: "CODE" },
          }),
        ),
      );
      const rule = await prisma.evaluationRule.create({
        data: {
          projectId,
          name: "Legacy rule",
          targetObject: "EVENT",
          filter: [],
          sampling: 1,
          delay: 0,
          assignments: {
            create: { projectId, evaluatorId: evaluator.id },
          },
        },
      });
      const directScore = createTraceScore({
        project_id: projectId,
        name: "direct-score",
        metadata: { evaluator_id: evaluator.id },
      });
      const legacyScore = createTraceScore({
        project_id: projectId,
        name: "legacy-score-with-a-different-name",
        metadata: { job_configuration_id: rule.id },
      });
      const otherScore = createTraceScore({
        project_id: projectId,
        name: "other-score",
        metadata: { evaluator_id: otherEvaluator.id },
      });
      await createScoresCh([directScore, legacyScore, otherScore]);

      const payload = {
        projectId,
        filter: [
          {
            column: "evaluatorId",
            type: "stringOptions" as const,
            operator: "any of" as const,
            value: [evaluator.id, rule.id],
          },
        ],
        orderBy: { column: "timestamp", order: "DESC" as const },
        page: 0,
        limit: 50,
      };

      const [scores, eventScores, count, eventCount] = await Promise.all([
        caller.scores.all(payload),
        caller.scores.allFromEvents(payload),
        caller.scores.countAll({ ...payload, orderBy: null }),
        caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
      ]);

      expect(new Set(scores.scores.map(({ id }) => id))).toEqual(
        new Set([directScore.id]),
      );
      expect(new Set(eventScores.scores.map(({ id }) => id))).toEqual(
        new Set([directScore.id]),
      );
      expect(count.totalCount).toBe(1);
      expect(eventCount.totalCount).toBe(1);
    });

    it("does not match empty boolean representations when filtering boolean values", async () => {
      const trueBooleanScore = createTraceScore({
        project_id: projectId,
        name: "boolean-score-true",
        data_type: "BOOLEAN",
        value: 1,
        string_value: "True",
      });
      const falseBooleanScore = createTraceScore({
        project_id: projectId,
        name: "boolean-score-false",
        data_type: "BOOLEAN",
        value: 0,
        string_value: "False",
      });
      const emptyBooleanScore = createTraceScore({
        project_id: projectId,
        name: "boolean-score-empty",
        data_type: "BOOLEAN",
        value: 1,
        string_value: "",
      });
      const numericScore = createTraceScore({
        project_id: projectId,
        name: "numeric-score",
        data_type: "NUMERIC",
        value: 0.7,
        string_value: null,
      });

      await createScoresCh([
        trueBooleanScore,
        falseBooleanScore,
        emptyBooleanScore,
        numericScore,
      ]);

      const payload = {
        projectId,
        filter: [
          {
            column: "booleanValue",
            type: "stringOptions" as const,
            operator: "none of" as const,
            value: ["false"],
          },
        ],
        orderBy: { column: "timestamp", order: "DESC" as const },
        page: 0,
        limit: 50,
      };

      const result = await caller.scores.all(payload);
      const resultFromEvents = await caller.scores.allFromEvents(payload);

      expect(result.scores.map((score) => score.id)).toEqual([
        trueBooleanScore.id,
      ]);
      expect(resultFromEvents.scores.map((score) => score.id)).toEqual([
        trueBooleanScore.id,
      ]);
    });
  });

  describe("observation scope filter", () => {
    it("lists trace-level scores for a trace-level owner span and for no other span", async () => {
      const traceId = randomUUID();
      const rootObservationId = randomUUID();
      const childObservationId = randomUUID();

      const traceLevelScore = createTraceScore({
        project_id: projectId,
        trace_id: traceId,
        name: "trace-level-score",
      });
      const rootObservationScore = createTraceScore({
        project_id: projectId,
        trace_id: traceId,
        observation_id: rootObservationId,
        name: "root-observation-score",
      });
      const childObservationScore = createTraceScore({
        project_id: projectId,
        trace_id: traceId,
        observation_id: childObservationId,
        name: "child-observation-score",
      });

      await createScoresCh([
        traceLevelScore,
        rootObservationScore,
        childObservationScore,
      ]);

      const scoreIdsFor = async (
        observationId: string,
        includeTraceLevelScores: boolean,
      ) => {
        const payload = {
          projectId,
          filter: [
            {
              column: "traceId",
              type: "string" as const,
              operator: "=" as const,
              value: traceId,
            },
            ...observationScopeFilter(observationId, includeTraceLevelScores),
          ],
          orderBy: { column: "timestamp", order: "DESC" as const },
          page: 0,
          limit: 50,
        };
        const [v3, v4] = await Promise.all([
          caller.scores.all(payload),
          caller.scores.allFromEvents(payload),
        ]);
        return {
          v3: v3.scores.map((score) => score.id).sort(),
          v4: v4.scores.map((score) => score.id).sort(),
        };
      };

      // Trace-level owner: its own score plus the trace-level one, each once.
      const owner = await scoreIdsFor(rootObservationId, true);
      const expectedOwnerScoreIds = [
        traceLevelScore.id,
        rootObservationScore.id,
      ].sort();
      expect(owner.v3).toEqual(expectedOwnerScoreIds);
      expect(owner.v4).toEqual(expectedOwnerScoreIds);

      // Any other span stays observation-scoped.
      const child = await scoreIdsFor(childObservationId, false);
      expect(child.v3).toEqual([childObservationScore.id]);
      expect(child.v4).toEqual([childObservationScore.id]);
    });
  });

  maybeEvents("scores.allFromEvents trace-name filters", () => {
    it("keeps scores for semantic roots without a stored trace name", async () => {
      const traceId = randomUUID();
      const score = createTraceScore({
        project_id: projectId,
        trace_id: traceId,
        name: "semantic-root-score",
        value: 0.9,
      });

      await createEventsCh([
        createEvent({
          trace_id: traceId,
          project_id: projectId,
          parent_span_id: "external-parent",
          is_app_root: true,
          name: "semantic-root-trace",
          trace_name: "",
        }),
      ]);
      await createScoresCh([score]);

      const result = await caller.scores.allFromEvents({
        projectId,
        filter: [
          {
            column: "traceName",
            operator: "=",
            value: "semantic-root-trace",
            type: "string",
          },
        ],
        orderBy: { column: "timestamp", order: "DESC" },
        page: 0,
        limit: 50,
      });

      expect(result.scores.map(({ id }) => id)).toContain(score.id);
    });

    it("returns every score of a trace, not one per trace", async () => {
      const traceName = `multi-score-trace-${randomUUID()}`;
      const traceIds = [randomUUID(), randomUUID()];
      const scores = traceIds.flatMap((traceId) =>
        ["a", "b", "c"].map((name) =>
          createTraceScore({ project_id: projectId, trace_id: traceId, name }),
        ),
      );

      await createEventsCh(
        traceIds.map((traceId) =>
          createEvent({
            trace_id: traceId,
            project_id: projectId,
            parent_span_id: "",
            name: traceName,
            trace_name: traceName,
          }),
        ),
      );
      await createScoresCh(scores);

      const payload = {
        projectId,
        filter: [
          {
            column: "traceName",
            operator: "=" as const,
            value: traceName,
            type: "string" as const,
          },
        ],
        page: 0,
        limit: 50,
      };
      const [result, count] = await Promise.all([
        caller.scores.allFromEvents({
          ...payload,
          orderBy: { column: "timestamp", order: "DESC" },
        }),
        caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
      ]);

      expect(result.scores.map(({ id }) => id).sort()).toEqual(
        scores.map(({ id }) => id).sort(),
      );
      expect(count.totalCount).toBe(scores.length);
    });
  });

  describe("scores.createAnnotationScore", () => {
    it("rejects empty stringValue for boolean annotation scores", async () => {
      const configId = randomUUID();
      const scoreName = `boolean-annotation-score-${configId.slice(0, 8)}`;

      await expect(
        caller.scores.createAnnotationScore({
          projectId,
          name: scoreName,
          value: 1,
          stringValue: "",
          dataType: "BOOLEAN",
          scoreTarget: { type: "trace", traceId: randomUUID() },
          configId,
          environment: "default",
        } as any),
      ).rejects.toThrow();
    });

    it("accepts explicit boolean annotation stringValue labels", async () => {
      const traceId = randomUUID();
      const configId = randomUUID();
      const scoreName = `boolean-annotation-score-${configId.slice(0, 8)}`;

      await createTracesCh([
        createTrace({
          id: traceId,
          project_id: projectId,
        }),
      ]);
      await prisma.scoreConfig.create({
        data: {
          id: configId,
          projectId,
          name: scoreName,
          dataType: ScoreConfigDataType.BOOLEAN,
          categories: [
            { label: "True", value: 1 },
            { label: "False", value: 0 },
          ],
        },
      });

      const score = await caller.scores.createAnnotationScore({
        projectId,
        name: scoreName,
        value: 1,
        stringValue: "True",
        dataType: "BOOLEAN",
        scoreTarget: { type: "trace", traceId },
        configId,
        environment: "default",
      });

      expect(score.stringValue).toBe("True");
    });
  });

  describe("scores.updateAnnotationScore", () => {
    it("bumps updated_at and keeps created_at when editing an existing score", async () => {
      const originalTime = new Date(Date.now() - 60 * 60 * 1000);
      const configId = randomUUID();
      const scoreName = `numeric-annotation-score-${configId.slice(0, 8)}`;
      await prisma.scoreConfig.create({
        data: {
          id: configId,
          projectId,
          name: scoreName,
          dataType: ScoreConfigDataType.NUMERIC,
        },
      });
      const existing = createTraceScore({
        project_id: projectId,
        source: "ANNOTATION",
        name: scoreName,
        config_id: configId,
        value: 1,
        timestamp: originalTime.getTime(),
        created_at: originalTime.getTime(),
        updated_at: originalTime.getTime(),
      });
      await createScoresCh([existing]);

      await caller.scores.updateAnnotationScore({
        id: existing.id,
        projectId,
        name: scoreName,
        value: 2,
        stringValue: null,
        dataType: "NUMERIC",
        scoreTarget: { type: "trace", traceId: existing.trace_id! },
        configId,
        environment: "default",
      });

      const updated = await getScoreById({
        projectId,
        scoreId: existing.id,
      });
      expect(updated?.value).toBe(2);
      expect(updated?.createdAt.getTime()).toBe(originalTime.getTime());
      expect(updated?.updatedAt.getTime()).toBeGreaterThan(
        originalTime.getTime(),
      );
    });
  });

  describe("scores.deleteAnnotationScore", () => {
    it("deletes a correction ingested via the API", async () => {
      const correction = createTraceScore({
        project_id: projectId,
        name: "output",
        source: "API",
        data_type: "CORRECTION",
        value: 0,
        long_string_value: "corrected response",
      });
      await createScoresCh([correction]);

      await caller.scores.deleteAnnotationScore({
        projectId,
        id: correction.id,
      });

      expect(
        await getScoreById({ projectId, scoreId: correction.id }),
      ).toBeUndefined();
    });
  });

  describe("scores.deleteMany", () => {
    it("should delete scores by ids", async () => {
      // Setup
      const createdScore = createTraceScore({
        project_id: projectId,
      });
      await createScoresCh([createdScore]);
      const scoreDeleteQueue = ScoreDeleteQueue.getInstance();

      // When
      await caller.scores.deleteMany({
        projectId,
        scoreIds: [createdScore.id],
      });

      expect(scoreDeleteQueue).not.toBeNull();

      // Then
      expect(scoreDeleteQueue!.add).toHaveBeenCalledWith(
        QueueJobs.ScoreDelete,
        expect.objectContaining({
          payload: expect.objectContaining({
            projectId,
            scoreIds: [createdScore.id],
          }),
        }),
      );
    });

    it("should delete scores via batch query", async () => {
      // Setup
      const scoreName = randomUUID();
      const createdScore = createTraceScore({
        project_id: projectId,
        name: scoreName,
      });
      await createScoresCh([createdScore]);
      const batchActionQueue = BatchActionQueue.getInstance();

      // When
      await caller.scores.deleteMany({
        projectId,
        scoreIds: null,
        isBatchAction: true,
        query: {
          orderBy: { column: "timestamp", order: "ASC" },
          filter: [
            {
              column: "name",
              operator: "=",
              value: scoreName,
              type: "string",
            },
          ],
        },
      });

      expect(batchActionQueue).not.toBeNull();

      // Then
      expect(batchActionQueue!.add).toHaveBeenCalledWith(
        QueueJobs.BatchActionProcessingJob,
        expect.objectContaining({
          payload: expect.objectContaining({
            projectId,
            actionId: "score-delete",
          }),
        }),
        expect.objectContaining({}),
      );
    });

    it("should throw an error if batchAction and scoreIds are missing", async () => {
      // When
      const promise = caller.scores.deleteMany({
        projectId,
        scoreIds: null,
        isBatchAction: false,
      });

      // Then
      await expect(promise).rejects.toMatchObject({
        code: "BAD_REQUEST",
        message:
          "Either batchAction or scoreIds must be provided to delete scores.",
      });
    });
  });

  describe("scores.getScoreColumns", () => {
    it("should distinguish trace-level from trace-scoped score discovery", async () => {
      const traceId = randomUUID();
      const observationId = randomUUID();

      await createTracesCh([
        createTrace({
          id: traceId,
          project_id: projectId,
        }),
      ]);
      await createObservationsCh([
        createObservation({
          id: observationId,
          trace_id: traceId,
          project_id: projectId,
        }),
      ]);

      await createScoresCh([
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: "trace_level_score",
          source: "API",
          data_type: "NUMERIC",
          value: 0.9,
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: observationId,
          name: "observation_level_score",
          source: "API",
          data_type: "NUMERIC",
          value: 0.7,
        }),
      ]);

      const traceScopedColumns = await caller.scores.getScoreColumns({
        projectId,
        filter: [
          {
            column: "traceId",
            operator: "is not null",
            value: "",
            type: "null",
          },
        ],
      });

      const traceLevelColumns = await caller.scores.getScoreColumns({
        projectId,
        filter: [
          {
            column: "traceId",
            operator: "is not null",
            value: "",
            type: "null",
          },
          {
            column: "observationId",
            operator: "is null",
            value: "",
            type: "null",
          },
        ],
      });

      expect(
        traceScopedColumns.scoreColumns.map((column) => column.name),
      ).toEqual(
        expect.arrayContaining([
          "trace_level_score",
          "observation_level_score",
        ]),
      );
      expect(
        traceLevelColumns.scoreColumns.map((column) => column.name),
      ).toEqual(["trace_level_score"]);
    });
  });

  describe("scores.filterOptions", () => {
    it("returns static boolean value options for both scores views", async () => {
      await expect(
        caller.scores.filterOptions({ projectId }),
      ).resolves.toMatchObject({
        booleanValue: [{ value: "true" }, { value: "false" }],
      });

      await expect(
        caller.scores.filterOptionsFromEvents({ projectId }),
      ).resolves.toMatchObject({
        booleanValue: [{ value: "true" }, { value: "false" }],
      });
    });
  });

  describe("scoreConfigs.all", () => {
    it("should paginate score configs deterministically when createdAt timestamps tie", async () => {
      const sharedCreatedAt = new Date("2100-05-12T00:00:00.000Z");
      const configIds: string[] = [randomUUID(), randomUUID(), randomUUID()];

      await prisma.scoreConfig.createMany({
        data: configIds.map((id, index) => ({
          id,
          projectId,
          name: `trpc-tie-config-${index}-${id.slice(0, 8)}`,
          description: `trpc tie config ${index}`,
          dataType: ScoreConfigDataType.NUMERIC,
          minValue: index,
          maxValue: index + 1,
          createdAt: sharedCreatedAt,
          updatedAt: sharedCreatedAt,
        })),
      });

      const firstPage = await caller.scoreConfigs.all({
        projectId,
        page: 0,
        limit: 2,
      });
      const secondPage = await caller.scoreConfigs.all({
        projectId,
        page: 1,
        limit: 2,
      });

      const tiedIds = [...firstPage.configs, ...secondPage.configs]
        .filter((config) => configIds.includes(config.id))
        .map((config) => config.id);

      expect(tiedIds).toEqual(configIds.slice().sort());
      expect(new Set(tiedIds).size).toBe(configIds.length);
    });
  });

  describe("scoreConfigs.byId", () => {
    it("returns NOT_FOUND when the config is missing in the project", async () => {
      await expect(
        caller.scoreConfigs.byId({
          projectId,
          id: randomUUID(),
        }),
      ).rejects.toMatchObject({
        code: "NOT_FOUND",
        message: "No score config with this id in this project.",
      });
    });

    it("returns the config when it exists in the project", async () => {
      const config = await prisma.scoreConfig.create({
        data: {
          projectId,
          name: `byid-${randomUUID().slice(0, 8)}`,
          dataType: ScoreConfigDataType.NUMERIC,
          minValue: 0,
          maxValue: 1,
        },
      });

      await expect(
        caller.scoreConfigs.byId({
          projectId,
          id: config.id,
        }),
      ).resolves.toMatchObject({
        id: config.id,
        name: config.name,
        dataType: ScoreConfigDataType.NUMERIC,
      });
    });
  });

  describe("scoreConfigs.appendCategory", () => {
    it("keeps both categories when two appends race", async () => {
      const config = await prisma.scoreConfig.create({
        data: {
          projectId,
          name: `append-race-${randomUUID().slice(0, 8)}`,
          dataType: ScoreConfigDataType.CATEGORICAL,
          categories: [{ label: "internal_user", value: 0 }],
        },
      });

      await Promise.all([
        caller.scoreConfigs.appendCategory({
          projectId,
          id: config.id,
          label: "pen_testing",
        }),
        caller.scoreConfigs.appendCategory({
          projectId,
          id: config.id,
          label: "just_testing",
        }),
      ]);

      const latest = await caller.scoreConfigs.byId({
        projectId,
        id: config.id,
      });

      expect(
        latest.categories?.map((category) => category.label).sort(),
      ).toEqual(["internal_user", "just_testing", "pen_testing"]);
    });

    it("rejects a duplicate label", async () => {
      const config = await prisma.scoreConfig.create({
        data: {
          projectId,
          name: `append-dup-${randomUUID().slice(0, 8)}`,
          dataType: ScoreConfigDataType.CATEGORICAL,
          categories: [{ label: "internal_user", value: 0 }],
        },
      });

      await expect(
        caller.scoreConfigs.appendCategory({
          projectId,
          id: config.id,
          label: "internal_user",
        }),
      ).rejects.toMatchObject({
        code: "BAD_REQUEST",
        message: "A category with this name already exists",
      });
    });

    it("rejects a non-categorical config", async () => {
      const config = await prisma.scoreConfig.create({
        data: {
          projectId,
          name: `append-numeric-${randomUUID().slice(0, 8)}`,
          dataType: ScoreConfigDataType.NUMERIC,
          minValue: 0,
          maxValue: 1,
        },
      });

      await expect(
        caller.scoreConfigs.appendCategory({
          projectId,
          id: config.id,
          label: "pen_testing",
        }),
      ).rejects.toMatchObject({
        code: "BAD_REQUEST",
        message: "Only categorical score configs can append categories.",
      });
    });
  });
});