Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

116 KB
Raw Blame
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import type { Session } from "next-auth";
import { prisma } from "@langfuse/shared/src/db";
import { createScoreAnalyticsRouter } from "@/src/features/score-analytics/server/scoreAnalyticsRouter";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import {
  createTraceScore,
  createScoresCh,
  createSessionScore,
  createDatasetRunScore,
} from "@langfuse/shared/src/server";
import { v4 } from "uuid";

describe("Score Comparison Analytics tRPC", () => {
  const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";

  const session: Session = {
    user: {
      id: "user-1",
      canCreateOrganizations: true,
      name: "Demo User",
      organizations: [
        {
          id: "seed-org-id",
          name: "Test Organization",
          role: "OWNER",
          plan: "cloud:hobby",
          cloudConfig: undefined,
          metadata: {},
          aiFeaturesEnabled: false,
          aiTelemetryEnabled: true,
          projects: [
            {
              id: projectId,
              role: "ADMIN",
              retentionDays: 30,
              deletedAt: null,
              name: "Test Project",
              hasTraces: false,
              metadata: {},
              createdAt: new Date().toISOString(),
            },
          ],
        },
      ],
      featureFlags: testFeatureFlags(),
      admin: true,
    },
    environment: {} as any,
    expires: new Date().toISOString(),
  };

  const ctx = createInnerTRPCContext({ session, headers: {} });
  const caller = appRouter.createCaller({ ...ctx, prisma });
  const samplingCaller = createScoreAnalyticsRouter({
    adaptiveFinalThreshold: 1_000,
    samplingThreshold: 1_000,
    targetSampleSize: 1_000,
  }).createCaller({ ...ctx, prisma });
  type ScoreComparisonAnalyticsInput = Parameters<
    typeof caller.scoreAnalytics.getScoreComparisonAnalytics
  >[0];
  type ScoreComparisonEstimateResults = NonNullable<
    ScoreComparisonAnalyticsInput["estimateResults"]
  >;

  const defaultEstimateResults: ScoreComparisonEstimateResults = {
    score1Count: 1_000,
    score2Count: 1_000,
    estimatedMatchedCount: 1_000,
  };

  const buildEstimateResults = (
    estimatedMatchedCount: number,
  ): ScoreComparisonEstimateResults => ({
    score1Count: estimatedMatchedCount,
    score2Count: estimatedMatchedCount,
    estimatedMatchedCount,
  });

  const rawGetScoreComparisonAnalytics =
    caller.scoreAnalytics.getScoreComparisonAnalytics;

  const getScoreComparisonAnalytics = async (
    input: ScoreComparisonAnalyticsInput,
  ) =>
    rawGetScoreComparisonAnalytics({
      ...input,
      estimateResults: input.estimateResults ?? defaultEstimateResults,
    });

  const getScoreComparisonAnalyticsWithPreflight = async (
    input: ScoreComparisonAnalyticsInput,
  ) => rawGetScoreComparisonAnalytics(input);

  // Most tests validate result shape and aggregation logic, not the preflight query.
  // Patch the caller once so those cases skip the duplicate estimate round-trip.
  Object.assign(caller.scoreAnalytics, {
    getScoreComparisonAnalytics,
  });

  const createOneHourWindow = () => {
    const now = new Date();
    return {
      now,
      fromTimestamp: new Date(now.getTime() - 3600000),
      toTimestamp: new Date(now.getTime() + 3600000),
    };
  };

  const insertLargeTraceLevelScorePairs = async ({
    batchSize = 10_000,
    totalRows,
    scoreName1,
    scoreName2,
  }: {
    batchSize?: number;
    totalRows: number;
    scoreName1: string;
    scoreName2: string;
  }) => {
    const scoreTimestamp = Date.now();

    for (let offset = 0; offset < totalRows; offset += batchSize) {
      const currentBatchSize = Math.min(batchSize, totalRows - offset);
      const scores = [];

      for (let i = 0; i < currentBatchSize; i++) {
        const traceId = v4();

        scores.push(
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName1,
            source: "ANNOTATION",
            value: Math.random() * 100,
            data_type: "NUMERIC",
            timestamp: scoreTimestamp,
          }),
        );

        scores.push(
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName2,
            source: "ANNOTATION",
            value: Math.random() * 100,
            data_type: "NUMERIC",
            timestamp: scoreTimestamp,
          }),
        );
      }

      await createScoresCh(scores);
    }
  };

  const insertLargeIdenticalTraceLevelScores = async ({
    batchSize = 10_000,
    totalRows,
    scoreName,
  }: {
    batchSize?: number;
    totalRows: number;
    scoreName: string;
  }) => {
    const scoreTimestamp = Date.now();

    for (let offset = 0; offset < totalRows; offset += batchSize) {
      const currentBatchSize = Math.min(batchSize, totalRows - offset);
      const scores = [];

      for (let i = 0; i < currentBatchSize; i++) {
        const traceId = v4();

        scores.push(
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName,
            source: "ANNOTATION",
            value: Math.random() * 100,
            data_type: "NUMERIC",
            timestamp: scoreTimestamp,
          }),
        );
      }

      await createScoresCh(scores);
    }
  };

  describe("getScoreComparisonAnalytics", () => {
    // Test 1: Returns all result types with valid data
    it("should return all result types with matching scores", async () => {
      const traceId = v4();

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000); // 1 hour ago
      const toTimestamp = new Date(now.getTime() + 3600000); // 1 hour from now

      // Use unique score names for test isolation
      const scoreName1 = `test1-score1-${v4()}`;
      const scoreName2 = `test1-score2-${v4()}`;

      // Create two numeric scores on the same trace
      // IMPORTANT: Must set observation_id to null for trace-level scores
      const score1 = createTraceScore({
        project_id: projectId,
        trace_id: traceId,
        observation_id: null,
        name: scoreName1,
        source: "ANNOTATION",
        data_type: "NUMERIC",
        value: 0.8,
        timestamp: now.getTime(),
      });

      const score2 = createTraceScore({
        project_id: projectId,
        trace_id: traceId,
        observation_id: null,
        name: scoreName2,
        source: "ANNOTATION",
        data_type: "NUMERIC",
        value: 0.9,
        timestamp: now.getTime(),
      });

      await createScoresCh([score1, score2]);

      const result = await getScoreComparisonAnalyticsWithPreflight({
        projectId,
        score1: {
          name: scoreName1,
          dataType: "NUMERIC",
          source: "ANNOTATION",
        },
        score2: {
          name: scoreName2,
          dataType: "NUMERIC",
          source: "ANNOTATION",
        },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Verify all result types are present
      expect(result.counts).toBeDefined();
      expect(result.counts.score1Total).toBe(1);
      expect(result.counts.score2Total).toBe(1);
      expect(result.counts.matchedCount).toBe(1);

      expect(result.heatmap).toBeDefined();
      expect(Array.isArray(result.heatmap)).toBe(true);

      expect(result.confusionMatrix).toBeDefined();
      expect(Array.isArray(result.confusionMatrix)).toBe(true);

      expect(result.statistics).toBeDefined();
      expect(result.statistics).not.toBeNull();
      if (result.statistics) {
        expect(result.statistics.pearsonCorrelation).toBeDefined();
        expect(result.statistics.spearmanCorrelation).toBeDefined();
      }

      expect(result.timeSeries).toBeDefined();
      expect(Array.isArray(result.timeSeries)).toBe(true);

      expect(result.distribution1).toBeDefined();
      expect(Array.isArray(result.distribution1)).toBe(true);

      expect(result.distribution2).toBeDefined();
      expect(Array.isArray(result.distribution2)).toBe(true);

      // Verify sampling metadata is present
      expect(result.samplingMetadata).toBeDefined();
      expect(result.samplingMetadata.isSampled).toBe(false); // No sampling for small dataset
      expect(result.samplingMetadata.samplingMethod).toBe("none");
      expect(result.samplingMetadata.samplingRate).toBe(1.0);
      expect(
        result.samplingMetadata.estimatedTotalMatches,
      ).toBeGreaterThanOrEqual(0);
      expect(result.samplingMetadata.actualSampleSize).toBe(1); // Matches matchedCount
      expect(result.samplingMetadata.samplingExpression).toBeNull();
    });

    // Test 2: Returns empty results when no scores exist
    it("should return empty results when no scores in time range", async () => {
      const fromTimestamp = new Date("2020-01-01");
      const toTimestamp = new Date("2020-01-02");

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: {
          name: "nonexistent1",
          dataType: "NUMERIC",
          source: "API",
        },
        score2: {
          name: "nonexistent2",
          dataType: "NUMERIC",
          source: "API",
        },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.counts.score1Total).toBe(0);
      expect(result.counts.score2Total).toBe(0);
      expect(result.counts.matchedCount).toBe(0);
      expect(result.heatmap).toEqual([]);
      expect(result.confusionMatrix).toEqual([]);
      expect(result.timeSeries).toEqual([]);
      expect(result.distribution1).toEqual([]);
      expect(result.distribution2).toEqual([]);

      // Verify sampling metadata even when no data
      expect(result.samplingMetadata).toBeDefined();
      expect(result.samplingMetadata.isSampled).toBe(false);
      expect(result.samplingMetadata.samplingMethod).toBe("none");
      expect(result.samplingMetadata.actualSampleSize).toBe(0);
    });

    // Test 3: Validates input schema
    it("should reject invalid nBins values", async () => {
      const now = new Date();

      await expect(
        caller.scoreAnalytics.getScoreComparisonAnalytics({
          projectId,
          score1: {
            name: "score1",
            dataType: "NUMERIC",
            source: "API",
          },
          score2: {
            name: "score2",
            dataType: "NUMERIC",
            source: "API",
          },
          fromTimestamp: now,
          toTimestamp: now,
          interval: { count: 1, unit: "day" },
          nBins: 3, // Too small
        }),
      ).rejects.toThrow();

      await expect(
        caller.scoreAnalytics.getScoreComparisonAnalytics({
          projectId,
          score1: {
            name: "score1",
            dataType: "NUMERIC",
            source: "API",
          },
          score2: {
            name: "score2",
            dataType: "NUMERIC",
            source: "API",
          },
          fromTimestamp: now,
          toTimestamp: now,
          interval: { count: 1, unit: "day" },
          nBins: 100, // Too large
        }),
      ).rejects.toThrow();
    });

    // Test 4: Adaptive FINAL optimization with small dataset
    // For datasets with estimated counts < 100k: uses FINAL for accuracy
    // For datasets with estimated counts >= 100k: skips FINAL for performance
    it("should use FINAL for small datasets (adaptive FINAL)", async () => {
      const traceId = v4();

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test-adaptive-score1-${v4()}`;
      const scoreName2 = `test-adaptive-score2-${v4()}`;

      // Create a small dataset (will use FINAL)
      const score1 = createTraceScore({
        project_id: projectId,
        trace_id: traceId,
        observation_id: null,
        name: scoreName1,
        source: "ANNOTATION",
        data_type: "NUMERIC",
        value: 0.5,
        timestamp: now.getTime(),
      });

      const score2 = createTraceScore({
        project_id: projectId,
        trace_id: traceId,
        observation_id: null,
        name: scoreName2,
        source: "ANNOTATION",
        data_type: "NUMERIC",
        value: 0.6,
        timestamp: now.getTime(),
      });

      await createScoresCh([score1, score2]);

      const result = await getScoreComparisonAnalyticsWithPreflight({
        projectId,
        score1: {
          name: scoreName1,
          dataType: "NUMERIC",
          source: "ANNOTATION",
        },
        score2: {
          name: scoreName2,
          dataType: "NUMERIC",
          source: "ANNOTATION",
        },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Verify query succeeded
      expect(result.counts).toBeDefined();
      expect(result.counts.matchedCount).toBe(1);

      // Verify adaptive FINAL decision via samplingMetadata
      expect(result.samplingMetadata.adaptiveFinal).toBeDefined();
      expect(result.samplingMetadata.adaptiveFinal?.usedFinal).toBe(true);
      expect(result.samplingMetadata.adaptiveFinal?.reason).toContain(
        "Small dataset - using FINAL for accuracy",
      );

      // Verify preflight estimates are included
      expect(result.samplingMetadata.preflightEstimates).toBeDefined();
      expect(
        result.samplingMetadata.preflightEstimates?.score1Count,
      ).toBeLessThan(100_000);
      expect(
        result.samplingMetadata.preflightEstimates?.score2Count,
      ).toBeLessThan(100_000);
    });

    // Test 5: Large matched datasets should skip FINAL and use hash sampling
    it("should skip FINAL and apply hash-based sampling for large matched datasets", async () => {
      const { fromTimestamp, toTimestamp } = createOneHourWindow();
      const scoreName1 = `test-large-score1-${v4()}`;
      const scoreName2 = `test-large-score2-${v4()}`;
      const totalRows = 5_000;

      await insertLargeTraceLevelScorePairs({
        totalRows,
        scoreName1,
        scoreName2,
      });

      const result = await samplingCaller.getScoreComparisonAnalytics({
        projectId,
        score1: {
          name: scoreName1,
          dataType: "NUMERIC",
          source: "ANNOTATION",
        },
        score2: {
          name: scoreName2,
          dataType: "NUMERIC",
          source: "ANNOTATION",
        },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "hour" as const },
        objectType: "all",
      });

      expect(result.counts).toBeDefined();
      expect(
        result.samplingMetadata.preflightEstimates?.score1Count,
      ).toBeGreaterThan(1_000);
      expect(
        result.samplingMetadata.preflightEstimates?.score2Count,
      ).toBeGreaterThan(1_000);
      expect(
        result.samplingMetadata.preflightEstimates?.estimatedMatchedCount,
      ).toBeGreaterThan(1_000);

      expect(result.samplingMetadata.adaptiveFinal?.usedFinal).toBe(false);
      expect(result.samplingMetadata.adaptiveFinal?.reason).toContain(
        "Large dataset - skipping FINAL for performance",
      );

      expect(result.samplingMetadata.isSampled).toBe(true);
      expect(result.samplingMetadata.samplingMethod).toBe("hash");
      expect(result.samplingMetadata.samplingRate).toBeLessThan(1);
      expect(result.samplingMetadata.samplingRate).toBeGreaterThan(0);
      expect(result.samplingMetadata.samplingExpression).toContain(
        "cityHash64",
      );
      expect(result.samplingMetadata.actualSampleSize).toBeGreaterThan(500);
      expect(result.samplingMetadata.actualSampleSize).toBeLessThan(totalRows);

      expect(result.counts.score1Total).toBe(result.counts.score2Total);
      expect(result.counts.matchedCount).toBe(result.counts.score1Total);
      expect(result.counts.matchedCount).toBe(
        result.samplingMetadata.actualSampleSize,
      );
      expect(result.heatmap.length).toBeGreaterThan(0);
      expect(result.timeSeries.length).toBeGreaterThan(0);
    }, 120000);

    // Test 6: Identical scores should stay perfectly aligned under sampling
    it("should return perfect correlation for identical scores with sampling", async () => {
      const { fromTimestamp, toTimestamp } = createOneHourWindow();
      const scoreName = `test-identical-${v4()}`;
      const totalRows = 2_000;
      const forcedEstimateResults = buildEstimateResults(1_200);

      await insertLargeIdenticalTraceLevelScores({
        totalRows,
        scoreName,
      });

      // Compare score to itself
      const result = await samplingCaller.getScoreComparisonAnalytics({
        projectId,
        score1: {
          name: scoreName,
          source: "ANNOTATION",
          dataType: "NUMERIC",
        },
        score2: {
          name: scoreName,
          source: "ANNOTATION",
          dataType: "NUMERIC",
        },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "hour" as const },
        objectType: "all",
        estimateResults: forcedEstimateResults,
      });

      expect(result.samplingMetadata.isSampled).toBe(true);
      expect(result.samplingMetadata.samplingMethod).toBe("hash");

      expect(result.counts.score1Total).toBe(result.counts.score2Total);
      expect(result.counts.matchedCount).toBe(result.counts.score1Total);
      expect(result.counts.matchedCount).toBe(result.counts.score2Total);
      expect(result.counts.matchedCount).toBeGreaterThan(1_000);
      expect(result.counts.matchedCount).toBeLessThan(totalRows);

      const offDiagonalPoints = result.heatmap.filter(
        (point) => point.binX !== point.binY,
      );
      expect(offDiagonalPoints.length).toBe(0);

      expect(result.statistics?.spearmanCorrelation).toBeNull();
    }, 120000);

    // Test 9: Calculates counts correctly with partial matches
    it("should calculate counts correctly with partial matches", async () => {
      const trace1 = v4();
      const trace2 = v4();
      const trace3 = v4();

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test4-s1-${v4()}`;
      const scoreName2 = `test4-s2-${v4()}`;

      // Create scores: trace1 has both, trace2 has only score1, trace3 has only score2
      const scores = [
        createTraceScore({
          project_id: projectId,
          trace_id: trace1,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1.0,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: trace1,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2.0,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: trace2,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 3.0,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: trace3,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 4.0,
          timestamp: now.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.counts.score1Total).toBe(2); // trace1, trace2
      expect(result.counts.score2Total).toBe(2); // trace1, trace3
      expect(result.counts.matchedCount).toBe(1); // only trace1 has both
    });

    // Test 5: Handles unmatched scores correctly
    it("should handle case where no scores match", async () => {
      const trace1 = v4();
      const trace2 = v4();

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test5-isolated1-${v4()}`;
      const scoreName2 = `test5-isolated2-${v4()}`;

      // trace1 has only score1, trace2 has only score2
      const scores = [
        createTraceScore({
          project_id: projectId,
          trace_id: trace1,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1.0,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: trace2,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2.0,
          timestamp: now.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.counts.score1Total).toBe(1);
      expect(result.counts.score2Total).toBe(1);
      expect(result.counts.matchedCount).toBe(0);
    });

    // Test 6: Generates correct bins for heatmap
    it("should generate correct bins for numeric heatmap", async () => {
      const traces = [v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test6-scoreA-${v4()}`;
      const scoreName2 = `test6-scoreB-${v4()}`;

      // Create 4 matched pairs with known values: (0,0), (25,25), (50,50), (100,100)
      const scores = traces.flatMap((traceId, idx) => {
        const value = idx * 25;
        return [
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName1,
            source: "API",
            data_type: "NUMERIC",
            value: value,
            timestamp: now.getTime(),
          }),
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName2,
            source: "API",
            data_type: "NUMERIC",
            value: value,
            timestamp: now.getTime(),
          }),
        ];
      });

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.heatmap.length).toBeGreaterThan(0);

      // Each heatmap cell should have binX, binY, count, and min/max ranges
      result.heatmap.forEach((cell) => {
        expect(cell.binX).toBeGreaterThanOrEqual(0);
        expect(cell.binX).toBeLessThan(10);
        expect(cell.binY).toBeGreaterThanOrEqual(0);
        expect(cell.binY).toBeLessThan(10);
        expect(cell.count).toBeGreaterThan(0);
        expect(cell.min1).toBeDefined();
        expect(cell.max1).toBeDefined();
        expect(cell.min2).toBeDefined();
        expect(cell.max2).toBeDefined();
      });

      // Total count across heatmap should equal matched count
      const totalHeatmapCount = result.heatmap.reduce(
        (sum, cell) => sum + cell.count,
        0,
      );
      expect(totalHeatmapCount).toBe(result.counts.matchedCount);
    });

    // Test 7: Respects custom nBins parameter
    it("should respect a custom nBins value", async () => {
      const traceId = v4();

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test7-binTest1-${v4()}`;
      const scoreName2 = `test7-binTest2-${v4()}`;

      const scores = [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 50,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 50,
          timestamp: now.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 5,
      });

      result.heatmap.forEach((cell) => {
        expect(cell.binX).toBeLessThan(5);
        expect(cell.binY).toBeLessThan(5);
      });
    });

    // Test 8: Includes min/max ranges for heatmap bins
    it("should include accurate min/max ranges for each heatmap bin", async () => {
      const traces = [v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test8-rangeA-${v4()}`;
      const scoreName2 = `test8-rangeB-${v4()}`;

      // Create scores with known values: (10,20), (15,25), (18,28)
      const scores = [
        ...traces.map((traceId, idx) =>
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName1,
            source: "API",
            data_type: "NUMERIC",
            value: 10 + idx * 5,
            timestamp: now.getTime(),
          }),
        ),
        ...traces.map((traceId, idx) =>
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName2,
            source: "API",
            data_type: "NUMERIC",
            value: 20 + idx * 5,
            timestamp: now.getTime(),
          }),
        ),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Verify that min/max ranges make sense
      result.heatmap.forEach((cell) => {
        expect(cell.max1).toBeGreaterThanOrEqual(cell.min1);
        expect(cell.max2).toBeGreaterThanOrEqual(cell.min2);
      });
    });

    // Test 9: Generates confusion matrix for BOOLEAN scores
    it("should generate 2x2 confusion matrix for BOOLEAN scores", async () => {
      const traces = [v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test9-boolA-${v4()}`;
      const scoreName2 = `test9-boolB-${v4()}`;

      // Create all four combinations: (T,T), (T,F), (F,T), (F,F)
      const combinations = [
        { val1: 1, val2: 1 }, // True, True
        { val1: 1, val2: 0 }, // True, False
        { val1: 0, val2: 1 }, // False, True
        { val1: 0, val2: 0 }, // False, False
      ];

      const scores = traces.flatMap((traceId, idx) => [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "BOOLEAN",
          value: combinations[idx].val1,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "BOOLEAN",
          value: combinations[idx].val2,
          timestamp: now.getTime(),
        }),
      ]);

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "BOOLEAN", source: "API" },
        score2: { name: scoreName2, dataType: "BOOLEAN", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Should have confusion matrix entries
      expect(result.confusionMatrix.length).toBeGreaterThan(0);

      // Each entry should have rowCategory, colCategory, and count
      result.confusionMatrix.forEach((entry) => {
        expect(entry.rowCategory).toBeDefined();
        expect(entry.colCategory).toBeDefined();
        expect(entry.count).toBeGreaterThan(0);
      });

      // Total count should equal matched count
      const totalConfusionCount = result.confusionMatrix.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      expect(totalConfusionCount).toBe(result.counts.matchedCount);
    });

    // Test 10: Generates confusion matrix for CATEGORICAL scores
    it("should generate NxN confusion matrix for CATEGORICAL scores", async () => {
      const traces = [v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test10-catA-${v4()}`;
      const scoreName2 = `test10-catB-${v4()}`;

      // Create categorical scores: (A,A), (A,B), (B,B)
      const categories = [
        { cat1: "A", cat2: "A" },
        { cat1: "A", cat2: "B" },
        { cat1: "B", cat2: "B" },
      ];

      const scores = traces.flatMap((traceId, idx) => [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "CATEGORICAL",
          string_value: categories[idx].cat1,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "CATEGORICAL",
          string_value: categories[idx].cat2,
          timestamp: now.getTime(),
        }),
      ]);

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "CATEGORICAL", source: "API" },
        score2: { name: scoreName2, dataType: "CATEGORICAL", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.confusionMatrix.length).toBeGreaterThan(0);

      // Should have entries for (A,A), (A,B), (B,B)
      const aaEntry = result.confusionMatrix.find(
        (e) => e.rowCategory === "A" && e.colCategory === "A",
      );
      const abEntry = result.confusionMatrix.find(
        (e) => e.rowCategory === "A" && e.colCategory === "B",
      );
      const bbEntry = result.confusionMatrix.find(
        (e) => e.rowCategory === "B" && e.colCategory === "B",
      );

      expect(aaEntry).toBeDefined();
      expect(abEntry).toBeDefined();
      expect(bbEntry).toBeDefined();

      if (aaEntry && abEntry && bbEntry) {
        expect(aaEntry.count).toBe(1);
        expect(abEntry.count).toBe(1);
        expect(bbEntry.count).toBe(1);
      }
    });

    // Test 11: Calculates perfect correlation correctly
    it("should calculate perfect correlation for identical scores", async () => {
      const traces = [v4(), v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test11-perfect1-${v4()}`;
      const scoreName2 = `test11-perfect2-${v4()}`;

      // Create identical scores
      const scores = traces.flatMap((traceId, idx) => {
        const value = idx * 10;
        return [
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName1,
            source: "API",
            data_type: "NUMERIC",
            value: value,
            timestamp: now.getTime(),
          }),
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName2,
            source: "API",
            data_type: "NUMERIC",
            value: value,
            timestamp: now.getTime(),
          }),
        ];
      });

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.statistics).toBeDefined();
      expect(result.statistics).not.toBeNull();
      if (result.statistics) {
        expect(result.statistics.pearsonCorrelation).toBeCloseTo(1.0, 2);
        expect(result.statistics.spearmanCorrelation).toBeCloseTo(1.0, 2);
        expect(result.statistics.mae).toBeCloseTo(0, 2);
        expect(result.statistics.rmse).toBeCloseTo(0, 2);
      }
    });

    // Test 12: Calculates statistics with known correlation
    it("should calculate statistics correctly for known dataset", async () => {
      const traces = [v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test12-linear1-${v4()}`;
      const scoreName2 = `test12-linear2-${v4()}`;

      // Create known dataset: (1,2), (2,4), (3,6), (4,8) - perfect linear relationship
      const values = [1, 2, 3, 4];
      const scores = traces.flatMap((traceId, idx) => [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: values[idx],
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: values[idx] * 2,
          timestamp: now.getTime(),
        }),
      ]);

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.statistics).not.toBeNull();
      if (result.statistics) {
        // Perfect linear correlation should be 1.0
        expect(result.statistics.pearsonCorrelation).toBeCloseTo(1.0, 2);
        expect(result.statistics.spearmanCorrelation).toBeCloseTo(1.0, 2);

        // MAE for y=2x should be 0
        expect(result.statistics.mae).toBeGreaterThan(0);

        // RMSE should also be > 0
        expect(result.statistics.rmse).toBeGreaterThan(0);
      }
    });

    // Test 13: Aggregates time series by hour
    it("should aggregate time series correctly by hour", async () => {
      const traceId = v4();

      const baseTime = new Date("2024-01-01T10:00:00Z");
      const fromTimestamp = new Date("2024-01-01T09:00:00Z");
      const toTimestamp = new Date("2024-01-01T13:00:00Z");

      const scoreName1 = `test13-hourly1-${v4()}`;
      const scoreName2 = `test13-hourly2-${v4()}`;

      // Create scores at different hours: 10:00, 10:30, 11:00, 12:00
      const timestamps = [
        baseTime.getTime(),
        baseTime.getTime() + 1800000, // +30 min
        baseTime.getTime() + 3600000, // +1 hour
        baseTime.getTime() + 7200000, // +2 hours
      ];

      const scores = timestamps.flatMap((ts) => [
        createTraceScore({
          project_id: projectId,
          trace_id: v4(),
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1,
          timestamp: ts,
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: v4(),
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2,
          timestamp: ts,
        }),
      ]);

      // Create matching pairs by using same trace IDs
      const matchedScores = [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1,
          timestamp: timestamps[0],
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2,
          timestamp: timestamps[0],
        }),
      ];

      await createScoresCh([...scores, ...matchedScores]);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "hour" },
        nBins: 10,
      });

      expect(result.timeSeries.length).toBeGreaterThan(0);

      // Each time series entry should have timestamp, avg1, avg2, and count
      result.timeSeries.forEach((entry) => {
        expect(entry.timestamp).toBeInstanceOf(Date);
        expect(entry.count).toBeGreaterThan(0);
      });
    });

    // Test 14: Aggregates time series by day
    it("should aggregate time series correctly by day", async () => {
      const traces = [v4(), v4(), v4()];

      const day1 = new Date("2024-01-01T12:00:00Z");
      const day2 = new Date("2024-01-02T12:00:00Z");
      const fromTimestamp = new Date("2024-01-01T00:00:00Z");
      const toTimestamp = new Date("2024-01-04T00:00:00Z");

      const scoreName1 = `test14-daily1-${v4()}`;
      const scoreName2 = `test14-daily2-${v4()}`;

      // Create scores across 3 days
      const scores = [
        ...traces.slice(0, 2).flatMap((traceId) => [
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName1,
            source: "API",
            data_type: "NUMERIC",
            value: 1,
            timestamp: day1.getTime(),
          }),
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName2,
            source: "API",
            data_type: "NUMERIC",
            value: 2,
            timestamp: day1.getTime(),
          }),
        ]),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[2],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1,
          timestamp: day2.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[2],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2,
          timestamp: day2.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.timeSeries.length).toBeGreaterThanOrEqual(2);

      // Total count across MATCHED time series should equal matched count
      // Note: timeSeries (ALL) includes unmatched scores, so we check timeSeriesMatched
      const totalMatchedTimeSeriesCount = result.timeSeriesMatched.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      expect(totalMatchedTimeSeriesCount).toBe(result.counts.matchedCount);
    });

    // Test 15: Aggregates time series by week and month
    it("should aggregate time series correctly by week and month", async () => {
      const traceId = v4();

      const fromTimestamp = new Date("2024-01-01T00:00:00Z");
      const toTimestamp = new Date("2024-03-01T00:00:00Z");

      const scoreName1 = `test15-period1-${v4()}`;
      const scoreName2 = `test15-period2-${v4()}`;

      const scores = [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1,
          timestamp: new Date("2024-01-15T12:00:00Z").getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2,
          timestamp: new Date("2024-01-15T12:00:00Z").getTime(),
        }),
      ];

      await createScoresCh(scores);

      const [weekResult, monthResult] = await Promise.all([
        caller.scoreAnalytics.getScoreComparisonAnalytics({
          projectId,
          score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
          score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
          fromTimestamp,
          toTimestamp,
          interval: { count: 7, unit: "day" },
          nBins: 10,
        }),
        caller.scoreAnalytics.getScoreComparisonAnalytics({
          projectId,
          score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
          score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
          fromTimestamp,
          toTimestamp,
          interval: { count: 1, unit: "month" },
          nBins: 10,
        }),
      ]);

      expect(weekResult.timeSeries.length).toBeGreaterThan(0);
      expect(monthResult.timeSeries.length).toBeGreaterThan(0);
    });

    // Test 16: Calculates distribution1 accurately
    it("should calculate distribution for first score accurately", async () => {
      const traces = [v4(), v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test16-dist1-${v4()}`;
      const scoreName2 = `test16-dist2-${v4()}`;

      // Create 5 scores with known values: 0, 25, 50, 75, 100
      const scores = traces.flatMap((traceId, idx) => [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: idx * 25,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 50, // constant
          timestamp: now.getTime(),
        }),
      ]);

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.distribution1.length).toBeGreaterThan(0);

      // Each distribution entry should have binIndex and count
      result.distribution1.forEach((entry) => {
        expect(entry.binIndex).toBeGreaterThanOrEqual(0);
        expect(entry.binIndex).toBeLessThan(10);
        expect(entry.count).toBeGreaterThan(0);
      });

      // Total count should equal score1Total
      const totalDist1Count = result.distribution1.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      expect(totalDist1Count).toBe(result.counts.score1Total);
    });

    // Test 17: Calculates distribution2 accurately
    it("should calculate distribution for second score accurately", async () => {
      const traces = [v4(), v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test17-distA-${v4()}`;
      const scoreName2 = `test17-distB-${v4()}`;

      // Create 5 scores with known values
      const scores = traces.flatMap((traceId, idx) => [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 50, // constant
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: idx * 20,
          timestamp: now.getTime(),
        }),
      ]);

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.distribution2.length).toBeGreaterThan(0);

      // Total count should equal score2Total
      const totalDist2Count = result.distribution2.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      expect(totalDist2Count).toBe(result.counts.score2Total);
    });

    // Test 18: Matches scores on trace level
    it("should match scores correctly at trace level", async () => {
      const trace1 = v4();
      const trace2 = v4();

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test18-traceLevel1-${v4()}`;
      const scoreName2 = `test18-traceLevel2-${v4()}`;

      const scores = [
        // Trace 1 has both scores
        createTraceScore({
          project_id: projectId,
          trace_id: trace1,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: trace1,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2,
          timestamp: now.getTime(),
        }),
        // Trace 2 has only one score
        createTraceScore({
          project_id: projectId,
          trace_id: trace2,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 3,
          timestamp: now.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Only trace1 should match
      expect(result.counts.matchedCount).toBe(1);
    });

    // Test 19: Matches scores on observation level
    it("should match scores correctly at observation level", async () => {
      const traceId = v4();
      const obs1 = v4();
      const obs2 = v4();

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test19-obsLevel1-${v4()}`;
      const scoreName2 = `test19-obsLevel2-${v4()}`;

      const scores = [
        // obs1 has both scores
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: obs1,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: obs1,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2,
          timestamp: now.getTime(),
        }),
        // obs2 has only one score
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: obs2,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 3,
          timestamp: now.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Only obs1 should match
      expect(result.counts.matchedCount).toBe(1);
    });

    // Test 20: Matches scores on session level
    it("should match scores correctly at session level", async () => {
      const session1 = v4();
      const session2 = v4();

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test20-sessionLevel1-${v4()}`;
      const scoreName2 = `test20-sessionLevel2-${v4()}`;

      const scores = [
        // session1 has both scores
        createSessionScore({
          project_id: projectId,
          session_id: session1,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1,
          timestamp: now.getTime(),
        }),
        createSessionScore({
          project_id: projectId,
          session_id: session1,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2,
          timestamp: now.getTime(),
        }),
        // session2 has only one score
        createSessionScore({
          project_id: projectId,
          session_id: session2,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 3,
          timestamp: now.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Only session1 should match
      expect(result.counts.matchedCount).toBe(1);
    });

    // Test 22: Handles out-of-order timestamps
    it("should handle scores created in random order", async () => {
      const traces = [v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 7200000); // 2 hours ago
      const toTimestamp = new Date(now.getTime() + 3600000); // 1 hour from now

      const scoreName1 = `test22-order1-${v4()}`;
      const scoreName2 = `test22-order2-${v4()}`;

      // Create scores with out-of-order timestamps
      const scores = [
        createTraceScore({
          project_id: projectId,
          trace_id: traces[0],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1,
          timestamp: now.getTime() - 3600000, // 1 hour ago
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[1],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 2,
          timestamp: now.getTime(), // now
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[2],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 3,
          timestamp: now.getTime() - 1800000, // 30 min ago
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[0],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 10,
          timestamp: now.getTime() - 3600000,
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[1],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 20,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[2],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 30,
          timestamp: now.getTime() - 1800000,
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // All 3 pairs should match despite out-of-order insertion
      expect(result.counts.matchedCount).toBe(3);
    });

    // Test 23: Verify 7-day intervals align to Monday (ISO 8601 week)
    it("should align 7-day intervals to Monday (ISO 8601 week)", async () => {
      // Test that 7-day intervals use Monday-aligned weeks, not Thursday-aligned epochs
      const traces = [v4(), v4(), v4()];

      // Use a known Monday and Thursday
      // Nov 3, 2025 is a Monday
      // Oct 30, 2025 is a Thursday (would be start of epoch-aligned week)
      const monday = new Date("2025-11-03T10:00:00.000Z"); // Monday
      const thursday = new Date("2025-10-30T10:00:00.000Z"); // Previous Thursday

      const scoreName1 = `test23-score1-${v4()}`;
      const scoreName2 = `test23-score2-${v4()}`;

      // Create scores on Monday
      const scores = [
        createTraceScore({
          project_id: projectId,
          trace_id: traces[0],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1.0,
          timestamp: monday.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[0],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2.0,
          timestamp: monday.getTime(),
        }),
      ];

      await createScoresCh(scores);

      // Query with 7-day interval starting from Thursday (90 days back)
      const fromTimestamp = new Date(
        thursday.getTime() - 90 * 24 * 60 * 60 * 1000,
      );
      const toTimestamp = new Date(monday.getTime() + 24 * 60 * 60 * 1000); // Day after Monday

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 7, unit: "day" },
        nBins: 10,
      });

      // The Monday data should appear in a bucket with timestamp = Monday
      // (If using epoch alignment, it would appear in Thursday bucket)
      const mondayBuckets = result.timeSeries.filter((ts) => {
        const tsDate = new Date(ts.timestamp);
        return tsDate.getUTCDay() === 1; // Monday = 1
      });

      // Should have at least one Monday bucket with data
      expect(mondayBuckets.length).toBeGreaterThan(0);

      // The Monday bucket should contain our data
      const mondayBucketWithData = mondayBuckets.find(
        (ts) => ts.avg1 !== null && ts.avg2 !== null,
      );
      expect(mondayBucketWithData).toBeDefined();
      expect(mondayBucketWithData?.avg1).toBe(1.0);
      expect(mondayBucketWithData?.avg2).toBe(2.0);
    });

    // Test 24: Verify 1-day intervals align to calendar day boundaries
    it("should align 1-day intervals to calendar day boundaries (midnight)", async () => {
      // Create scores at different times within the same calendar day on different traces
      const day = new Date("2025-11-03T00:00:00.000Z"); // Midnight
      const morning = new Date("2025-11-03T09:00:00.000Z");
      const evening = new Date("2025-11-03T22:45:00.000Z");

      const scoreName1 = `test24-score1-${v4()}`;
      const scoreName2 = `test24-score2-${v4()}`;

      // Create two traces - one for morning, one for evening
      const trace1 = v4();
      const trace2 = v4();

      const scores = [
        // Morning scores on trace1
        createTraceScore({
          project_id: projectId,
          trace_id: trace1,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1.0,
          timestamp: morning.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: trace1,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2.0,
          timestamp: morning.getTime(),
        }),
        // Evening scores on trace2
        createTraceScore({
          project_id: projectId,
          trace_id: trace2,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 3.0,
          timestamp: evening.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: trace2,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 4.0,
          timestamp: evening.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const fromTimestamp = new Date(day.getTime() - 24 * 60 * 60 * 1000); // Day before
      const toTimestamp = new Date(day.getTime() + 2 * 24 * 60 * 60 * 1000); // Day after

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // All scores from the same calendar day should be in ONE bucket
      // The bucket timestamp should be midnight (start of day)
      const dayBucket = result.timeSeries.find((ts) => {
        const tsDate = new Date(ts.timestamp);
        return (
          tsDate.getUTCFullYear() === 2025 &&
          tsDate.getUTCMonth() === 10 && // November = 10
          tsDate.getUTCDate() === 3 &&
          tsDate.getUTCHours() === 0 &&
          tsDate.getUTCMinutes() === 0 &&
          tsDate.getUTCSeconds() === 0
        );
      });

      expect(dayBucket).toBeDefined();
      // Average of [1.0, 3.0] = 2.0
      expect(dayBucket?.avg1).toBe(2.0);
      // Average of [2.0, 4.0] = 3.0
      expect(dayBucket?.avg2).toBe(3.0);
    });

    // Test 25: Verify 1-month intervals align to calendar month boundaries
    it("should align 1-month intervals to calendar month boundaries (1st of month)", async () => {
      // Create scores at different times within November 2025 on different traces
      const startOfMonth = new Date("2025-11-01T00:00:00.000Z");
      const midMonth = new Date("2025-11-15T12:00:00.000Z");
      const endOfMonth = new Date("2025-11-30T23:59:59.000Z");

      const scoreName1 = `test25-score1-${v4()}`;
      const scoreName2 = `test25-score2-${v4()}`;

      // Create three traces - one for start, mid, and end of month
      const trace1 = v4();
      const trace2 = v4();
      const trace3 = v4();

      const scores = [
        // Start of month on trace1
        createTraceScore({
          project_id: projectId,
          trace_id: trace1,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1.0,
          timestamp: startOfMonth.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: trace1,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2.0,
          timestamp: startOfMonth.getTime(),
        }),
        // Mid month on trace2
        createTraceScore({
          project_id: projectId,
          trace_id: trace2,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 3.0,
          timestamp: midMonth.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: trace2,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 4.0,
          timestamp: midMonth.getTime(),
        }),
        // End of month on trace3
        createTraceScore({
          project_id: projectId,
          trace_id: trace3,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 5.0,
          timestamp: endOfMonth.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: trace3,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 6.0,
          timestamp: endOfMonth.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const fromTimestamp = new Date("2025-10-01T00:00:00.000Z"); // Oct 1
      const toTimestamp = new Date("2025-12-01T00:00:00.000Z"); // Dec 1

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "month" },
        nBins: 10,
      });

      // All scores from November should be in ONE bucket
      // The bucket timestamp should be Nov 1st midnight
      const novemberBucket = result.timeSeries.find((ts) => {
        const tsDate = new Date(ts.timestamp);
        return (
          tsDate.getUTCFullYear() === 2025 &&
          tsDate.getUTCMonth() === 10 && // November = 10
          tsDate.getUTCDate() === 1 &&
          tsDate.getUTCHours() === 0 &&
          tsDate.getUTCMinutes() === 0 &&
          tsDate.getUTCSeconds() === 0
        );
      });

      expect(novemberBucket).toBeDefined();
      // Average of [1.0, 3.0, 5.0] = 3.0
      expect(novemberBucket?.avg1).toBe(3.0);
      // Average of [2.0, 4.0, 6.0] = 4.0
      expect(novemberBucket?.avg2).toBe(4.0);
    });

    // Test 26: Matched Distributions - Basic Functionality
    it("should return matched distributions excluding unmatched scores", async () => {
      const traces = [v4(), v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test26-matched1-${v4()}`;
      const scoreName2 = `test26-matched2-${v4()}`;

      // Create matched pairs on first 3 traces and unmatched on last 2
      const scores = [
        // Matched pairs
        ...traces.slice(0, 3).flatMap((traceId, idx) => [
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName1,
            source: "API",
            data_type: "NUMERIC",
            value: 10 + idx * 20,
            timestamp: now.getTime(),
          }),
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName2,
            source: "API",
            data_type: "NUMERIC",
            value: 20 + idx * 20,
            timestamp: now.getTime(),
          }),
        ]),
        // Unmatched score1 on trace 4
        createTraceScore({
          project_id: projectId,
          trace_id: traces[3],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 70,
          timestamp: now.getTime(),
        }),
        // Unmatched score2 on trace 5
        createTraceScore({
          project_id: projectId,
          trace_id: traces[4],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 80,
          timestamp: now.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Matched distributions should only include the 3 matched pairs
      const matchedCount1 = result.distribution1Matched.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      const matchedCount2 = result.distribution2Matched.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );

      expect(matchedCount1).toBe(3);
      expect(matchedCount2).toBe(3);

      // Regular distributions should include all scores (3 matched + 1 unmatched each)
      const totalCount1 = result.distribution1.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      const totalCount2 = result.distribution2.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );

      expect(totalCount1).toBe(4);
      expect(totalCount2).toBe(4);

      // Verify counts match
      expect(result.counts.matchedCount).toBe(3);
      expect(result.counts.score1Total).toBe(4);
      expect(result.counts.score2Total).toBe(4);
    });

    // Test 27: Matched Distributions - Empty When No Matches
    it("should return empty matched distributions when no scores match", async () => {
      const traces = [v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test27-nomatch1-${v4()}`;
      const scoreName2 = `test27-nomatch2-${v4()}`;

      // Create only unmatched scores
      const scores = [
        createTraceScore({
          project_id: projectId,
          trace_id: traces[0],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 50,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[1],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 75,
          timestamp: now.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Matched distributions should be empty
      expect(result.distribution1Matched).toEqual([]);
      expect(result.distribution2Matched).toEqual([]);

      // Regular distributions should have data
      expect(result.distribution1.length).toBeGreaterThan(0);
      expect(result.distribution2.length).toBeGreaterThan(0);

      // matchedCount should be 0
      expect(result.counts.matchedCount).toBe(0);
    });

    // Test 28: Observation-Level Matched Scores
    it("should match scores correctly at observation level in matched distributions", async () => {
      const traceId = v4();
      const obs1 = v4();
      const obs2 = v4();
      const obs3 = v4();

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test28-obs1-${v4()}`;
      const scoreName2 = `test28-obs2-${v4()}`;

      const scores = [
        // Obs1 has both scores (matched)
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: obs1,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 10,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: obs1,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 20,
          timestamp: now.getTime(),
        }),
        // Obs2 has only score1 (unmatched)
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: obs2,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 30,
          timestamp: now.getTime(),
        }),
        // Obs3 has both scores (matched)
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: obs3,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 40,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: obs3,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 50,
          timestamp: now.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Only obs1 and obs3 should match
      expect(result.counts.matchedCount).toBe(2);

      const matchedCount1 = result.distribution1Matched.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      const matchedCount2 = result.distribution2Matched.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );

      expect(matchedCount1).toBe(2);
      expect(matchedCount2).toBe(2);

      // Regular distributions should include all 3 score1s and 2 score2s
      const totalCount1 = result.distribution1.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      const totalCount2 = result.distribution2.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );

      expect(totalCount1).toBe(3);
      expect(totalCount2).toBe(2);
    });

    // Test 29: Individual Distributions - Correct Bounds for Numeric Scores with Different Ranges
    it("should use individual bounds for better visualization when score ranges differ", async () => {
      const traces = [v4(), v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test29-individual1-${v4()}`;
      const scoreName2 = `test29-individual2-${v4()}`;

      // Score1 range: 10-50 (span: 40), Score2 range: 100-500 (span: 400)
      const scores = traces.flatMap((traceId, idx) => [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 10 + idx * 10,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 100 + idx * 100,
          timestamp: now.getTime(),
        }),
      ]);

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Both distributions should have 5 entries total
      const totalCount1 = result.distribution1.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      const totalCount2 = result.distribution2.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      const individualCount1 = result.distribution1Individual.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      const individualCount2 = result.distribution2Individual.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );

      expect(totalCount1).toBe(5);
      expect(totalCount2).toBe(5);
      expect(individualCount1).toBe(5);
      expect(individualCount2).toBe(5);

      // Individual distributions should exist and have entries
      expect(result.distribution1Individual.length).toBeGreaterThan(0);
      expect(result.distribution2Individual.length).toBeGreaterThan(0);

      // Verify heatmap contains individual bounds
      const heatmapRow = result.heatmap[0];
      expect(heatmapRow).toBeDefined();
      if (heatmapRow) {
        expect(heatmapRow.min1).toBe(10);
        expect(heatmapRow.max1).toBe(50);
        expect(heatmapRow.min2).toBe(100);
        expect(heatmapRow.max2).toBe(500);
        // Global bounds should span both ranges
        expect(heatmapRow.globalMin).toBe(10);
        expect(heatmapRow.globalMax).toBe(500);
      }
    });

    // Test 30: Individual Distributions - Match Global When Ranges Similar
    it("should have similar distributions when score ranges are similar", async () => {
      const traces = [v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test30-similar1-${v4()}`;
      const scoreName2 = `test30-similar2-${v4()}`;

      // Similar ranges: 10-40 vs 15-45
      const scores = traces.flatMap((traceId, idx) => [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 10 + idx * 10,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 15 + idx * 10,
          timestamp: now.getTime(),
        }),
      ]);

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Counts should match across regular and individual distributions
      const totalCount1 = result.distribution1.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      const individualCount1 = result.distribution1Individual.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );

      expect(totalCount1).toBe(4);
      expect(individualCount1).toBe(4);

      // Verify ranges are similar
      const heatmapRow = result.heatmap[0];
      if (heatmapRow) {
        const range1 = heatmapRow.max1 - heatmapRow.min1;
        const range2 = heatmapRow.max2 - heatmapRow.min2;
        const globalRange = heatmapRow.globalMax - heatmapRow.globalMin;

        // Ranges should be close to each other
        expect(Math.abs(range1 - range2)).toBeLessThan(10);
        // Global range shouldn't be much larger
        expect(globalRange).toBeLessThan(range1 + range2);
      }
    });

    // Test 31: Individual Distributions - Categorical Scores Reference Original
    it("should have individual distributions match regular distributions for categorical scores", async () => {
      const traces = [v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test31-cat1-${v4()}`;
      const scoreName2 = `test31-cat2-${v4()}`;

      const categories1 = ["A", "B", "A"];
      const categories2 = ["X", "Y", "Z"];

      const scores = traces.flatMap((traceId, idx) => [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "CATEGORICAL",
          string_value: categories1[idx],
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "CATEGORICAL",
          string_value: categories2[idx],
          timestamp: now.getTime(),
        }),
      ]);

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "CATEGORICAL", source: "API" },
        score2: { name: scoreName2, dataType: "CATEGORICAL", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // For categorical data, individual distributions should exactly match regular distributions
      expect(result.distribution1Individual).toEqual(result.distribution1);
      expect(result.distribution2Individual).toEqual(result.distribution2);
    });

    // Test 32: Cross-Data Type Handling
    it("should handle individual distributions correctly for cross-type comparison", async () => {
      const traces = [v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test32-numeric-${v4()}`;
      const scoreName2 = `test32-categorical-${v4()}`;

      const scores = [
        ...traces.map((traceId, idx) =>
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName1,
            source: "API",
            data_type: "NUMERIC",
            value: 10 + idx * 10,
            timestamp: now.getTime(),
          }),
        ),
        ...traces.map((traceId, idx) =>
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName2,
            source: "API",
            data_type: "CATEGORICAL",
            string_value: ["A", "B", "C"][idx],
            timestamp: now.getTime(),
          }),
        ),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "CATEGORICAL", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Numeric score should have individual distribution
      expect(result.distribution1Individual.length).toBeGreaterThan(0);

      // For cross-type, categorical may have empty or same distribution
      // In cross-type comparisons, individual distributions may behave differently
      if (result.distribution2Individual.length > 0) {
        // If present, categorical distribution2 should match its regular distribution
        expect(result.distribution2Individual).toEqual(result.distribution2);
      }

      // Counts should be consistent for numeric score
      const totalCount1 = result.distribution1.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );
      const individualCount1 = result.distribution1Individual.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );

      expect(totalCount1).toBe(3);
      expect(individualCount1).toBe(3);
    });

    // Test 33: Time Series Matched - Two-Score Functionality
    it("should return matched time series excluding unmatched scores", async () => {
      const traces = [v4(), v4(), v4(), v4(), v4(), v4()];

      const day1 = new Date("2024-01-01T12:00:00Z");
      const day2 = new Date("2024-01-02T12:00:00Z");
      const day3 = new Date("2024-01-03T12:00:00Z");
      const fromTimestamp = new Date("2024-01-01T00:00:00Z");
      const toTimestamp = new Date("2024-01-04T00:00:00Z");

      const scoreName1 = `test33-ts1-${v4()}`;
      const scoreName2 = `test33-ts2-${v4()}`;

      const scores = [
        // Day 1: 2 matched pairs
        ...traces.slice(0, 2).flatMap((traceId, idx) => [
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName1,
            source: "API",
            data_type: "NUMERIC",
            value: 10 + idx * 5,
            timestamp: day1.getTime(),
          }),
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName2,
            source: "API",
            data_type: "NUMERIC",
            value: 20 + idx * 5,
            timestamp: day1.getTime(),
          }),
        ]),
        // Day 2: 1 matched pair + 1 unmatched score1
        createTraceScore({
          project_id: projectId,
          trace_id: traces[2],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 30,
          timestamp: day2.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[2],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 40,
          timestamp: day2.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[3],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 50,
          timestamp: day2.getTime(),
        }),
        // Day 3: 1 matched pair + 1 unmatched score2
        createTraceScore({
          project_id: projectId,
          trace_id: traces[4],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 60,
          timestamp: day3.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[4],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 70,
          timestamp: day3.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[5],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 80,
          timestamp: day3.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // timeSeriesMatched should only include matched pairs
      expect(result.timeSeriesMatched.length).toBeGreaterThan(0);

      const totalMatchedCount = result.timeSeriesMatched.reduce(
        (sum, entry) => sum + entry.count,
        0,
      );

      expect(totalMatchedCount).toBe(4); // 2 + 1 + 1 matched pairs

      // Verify each day's data
      const day1Bucket = result.timeSeriesMatched.find(
        (ts) => new Date(ts.timestamp).getUTCDate() === 1,
      );
      const day2Bucket = result.timeSeriesMatched.find(
        (ts) => new Date(ts.timestamp).getUTCDate() === 2,
      );
      const day3Bucket = result.timeSeriesMatched.find(
        (ts) => new Date(ts.timestamp).getUTCDate() === 3,
      );

      expect(day1Bucket?.count).toBe(2);
      expect(day2Bucket?.count).toBe(1); // Unmatched score1 not included
      expect(day3Bucket?.count).toBe(1); // Unmatched score2 not included
    });

    // Test 34: Time Series Matched - Single Score Mode
    it("should handle timeSeriesMatched in single-score mode", async () => {
      const traces = [v4(), v4(), v4(), v4()];

      const day1 = new Date("2024-01-01T12:00:00Z");
      const day2 = new Date("2024-01-02T12:00:00Z");
      const fromTimestamp = new Date("2024-01-01T00:00:00Z");
      const toTimestamp = new Date("2024-01-03T00:00:00Z");

      const scoreName = `test34-single-${v4()}`;

      const scores = [
        // Day 1: 2 scores
        ...traces.slice(0, 2).map((traceId, idx) =>
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName,
            source: "API",
            data_type: "NUMERIC",
            value: 10 + idx * 10,
            timestamp: day1.getTime(),
          }),
        ),
        // Day 2: 2 scores
        ...traces.slice(2, 4).map((traceId, idx) =>
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName,
            source: "API",
            data_type: "NUMERIC",
            value: 30 + idx * 10,
            timestamp: day2.getTime(),
          }),
        ),
      ];

      await createScoresCh(scores);

      // Query with same score for both score1 and score2 (single-score mode)
      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // In single-score mode, avg2 should be null
      result.timeSeriesMatched.forEach((entry) => {
        expect(entry.avg2).toBeNull();
        expect(entry.avg1).not.toBeNull();
      });

      // Should have 2 time buckets (one for each day)
      expect(result.timeSeriesMatched.length).toBeGreaterThanOrEqual(2);
    });

    // Test 36: Time Series Matched - Timestamp Precision (Critical)
    it("should return timestamps in seconds not milliseconds in timeSeriesMatched", async () => {
      const traceId = v4();

      // Use specific timestamp: 2024-01-15 12:30:45.123 UTC
      const specificTime = new Date("2024-01-15T12:30:45.123Z");
      const fromTimestamp = new Date("2024-01-15T00:00:00Z");
      const toTimestamp = new Date("2024-01-16T00:00:00Z");

      const scoreName1 = `test35-timestamp1-${v4()}`;
      const scoreName2 = `test35-timestamp2-${v4()}`;

      const scores = [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 100,
          timestamp: specificTime.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 200,
          timestamp: specificTime.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.timeSeriesMatched.length).toBeGreaterThan(0);

      const entry = result.timeSeriesMatched[0];
      expect(entry).toBeDefined();

      if (entry) {
        // Timestamp should be a Date object
        expect(entry.timestamp).toBeInstanceOf(Date);

        // Timestamp should be bucketed to start of day (midnight)
        const tsDate = new Date(entry.timestamp);
        expect(tsDate.getUTCHours()).toBe(0);
        expect(tsDate.getUTCMinutes()).toBe(0);
        expect(tsDate.getUTCSeconds()).toBe(0);

        // The milliseconds should be 0 (seconds precision)
        expect(entry.timestamp.getTime() % 1000).toBe(0);

        // Verify it's the correct day
        expect(tsDate.getUTCFullYear()).toBe(2024);
        expect(tsDate.getUTCMonth()).toBe(0); // January = 0
        expect(tsDate.getUTCDate()).toBe(15);
      }
    });

    // Test 37: Time Series Matched - Empty When No Matches
    it("should return empty timeSeriesMatched when no scores match", async () => {
      const traces = [v4(), v4(), v4()];

      const day1 = new Date("2024-01-01T12:00:00Z");
      const day2 = new Date("2024-01-02T12:00:00Z");
      const day3 = new Date("2024-01-03T12:00:00Z");
      const fromTimestamp = new Date("2024-01-01T00:00:00Z");
      const toTimestamp = new Date("2024-01-04T00:00:00Z");

      const scoreName1 = `test36-nomatch1-${v4()}`;
      const scoreName2 = `test36-nomatch2-${v4()}`;

      // Create only unmatched scores across different days
      const scores = [
        createTraceScore({
          project_id: projectId,
          trace_id: traces[0],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 10,
          timestamp: day1.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[1],
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 20,
          timestamp: day2.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traces[2],
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 30,
          timestamp: day3.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // timeSeriesMatched should be empty when no matches exist
      expect(result.timeSeriesMatched).toEqual([]);

      // Regular timeSeries may still have data (mixing unmatched scores)
      expect(result.timeSeries.length).toBeGreaterThanOrEqual(0);

      // matchedCount should be 0
      expect(result.counts.matchedCount).toBe(0);
    });

    // Test 38: Heatmap GlobalMin/GlobalMax - Correct Position
    it("should include globalMin and globalMax in heatmap with correct values", async () => {
      const traces = [v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test37-global1-${v4()}`;
      const scoreName2 = `test37-global2-${v4()}`;

      // Score1: 10-40, Score2: 50-80
      const scores = traces.flatMap((traceId, idx) => [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 10 + idx * 10,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 50 + idx * 10,
          timestamp: now.getTime(),
        }),
      ]);

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.heatmap.length).toBeGreaterThan(0);

      // Every heatmap cell should have all bound properties
      result.heatmap.forEach((cell) => {
        expect(cell.globalMin).toBeDefined();
        expect(cell.globalMax).toBeDefined();
        expect(cell.min1).toBeDefined();
        expect(cell.max1).toBeDefined();
        expect(cell.min2).toBeDefined();
        expect(cell.max2).toBeDefined();

        // Verify values
        expect(cell.globalMin).toBe(10);
        expect(cell.globalMax).toBe(80);
        expect(cell.min1).toBe(10);
        expect(cell.max1).toBe(40);
        expect(cell.min2).toBe(50);
        expect(cell.max2).toBe(80);
      });
    });

    // Test 39: Heatmap GlobalMin/GlobalMax - Single Score Scenario
    it("should have identical bounds in single-score mode for heatmap", async () => {
      const traces = [v4(), v4(), v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName = `test38-singlescore-${v4()}`;

      const scores = traces.map((traceId, idx) =>
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName,
          source: "API",
          data_type: "NUMERIC",
          value: 5 + idx * 10,
          timestamp: now.getTime(),
        }),
      );

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.heatmap.length).toBeGreaterThan(0);

      // In single-score mode, all bounds should be the same
      const firstCell = result.heatmap[0];
      expect(firstCell).toBeDefined();
      if (firstCell) {
        expect(firstCell.globalMin).toBe(5);
        expect(firstCell.globalMax).toBe(45);
        expect(firstCell.min1).toBe(5);
        expect(firstCell.max1).toBe(45);
        expect(firstCell.min2).toBe(5);
        expect(firstCell.max2).toBe(45);

        // All bounds should be identical
        expect(firstCell.min1).toBe(firstCell.min2);
        expect(firstCell.max1).toBe(firstCell.max2);
        expect(firstCell.globalMin).toBe(firstCell.min1);
        expect(firstCell.globalMax).toBe(firstCell.max1);
      }
    });

    // Test 40: Heatmap GlobalMin/GlobalMax - Disjoint Ranges
    it("should have global bounds spanning disjoint score ranges", async () => {
      const traces = [v4(), v4(), v4()];

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `test39-disjoint1-${v4()}`;
      const scoreName2 = `test39-disjoint2-${v4()}`;

      // Disjoint ranges: Score1: 1-3, Score2: 100-102
      const scores = traces.flatMap((traceId, idx) => [
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1 + idx,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 100 + idx,
          timestamp: now.getTime(),
        }),
      ]);

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      expect(result.heatmap.length).toBeGreaterThan(0);

      const firstCell = result.heatmap[0];
      if (firstCell) {
        // Individual bounds
        expect(firstCell.min1).toBe(1);
        expect(firstCell.max1).toBe(3);
        expect(firstCell.min2).toBe(100);
        expect(firstCell.max2).toBe(102);

        // Global bounds should span both ranges
        expect(firstCell.globalMin).toBe(1);
        expect(firstCell.globalMax).toBe(102);

        // Verify global range is much larger than individual ranges
        const range1 = firstCell.max1 - firstCell.min1; // 2
        const range2 = firstCell.max2 - firstCell.min2; // 2
        const globalRange = firstCell.globalMax - firstCell.globalMin; // 101

        expect(globalRange).toBeGreaterThan(range1 + range2);
      }
    });
  });

  describe("Categorical Time Series", () => {
    it("should return categorical time series for boolean scores", async () => {
      const scoreName1 = `bool-score-1-${v4()}`;
      const scoreName2 = `bool-score-2-${v4()}`;
      const fromTimestamp = new Date("2024-01-01T00:00:00Z");
      const toTimestamp = new Date("2024-01-03T00:00:00Z");

      // Create boolean scores across multiple days
      const scores: any[] = [];
      for (let day = 0; day < 3; day++) {
        const timestamp = new Date(
          Date.UTC(2024, 0, 1 + day, 12, 0, 0),
        ).getTime();
        const traceId = `trace-bool-${day}`;

        scores.push(
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName1,
            source: "API",
            data_type: "BOOLEAN",
            string_value: day % 2 === 0 ? "true" : "false",
            timestamp,
          }),
          createTraceScore({
            project_id: projectId,
            trace_id: traceId,
            observation_id: null,
            name: scoreName2,
            source: "API",
            data_type: "BOOLEAN",
            string_value: day % 2 === 1 ? "true" : "false",
            timestamp,
          }),
        );
      }

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "BOOLEAN", source: "API" },
        score2: { name: scoreName2, dataType: "BOOLEAN", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Verify categorical time series data is returned
      expect(result.timeSeriesCategorical1.length).toBeGreaterThan(0);
      expect(result.timeSeriesCategorical2.length).toBeGreaterThan(0);

      // Verify data structure
      const firstEntry1 = result.timeSeriesCategorical1[0];
      if (firstEntry1) {
        expect(firstEntry1.timestamp).toBeInstanceOf(Date);
        expect(typeof firstEntry1.category).toBe("string");
        expect(typeof firstEntry1.count).toBe("number");
        expect(["true", "false"]).toContain(firstEntry1.category.toLowerCase());
      }

      // Verify both true and false categories exist across the data
      const categories1 = new Set(
        result.timeSeriesCategorical1.map((e) => e.category.toLowerCase()),
      );
      const categories2 = new Set(
        result.timeSeriesCategorical2.map((e) => e.category.toLowerCase()),
      );

      expect(categories1.size).toBeGreaterThan(0);
      expect(categories2.size).toBeGreaterThan(0);
    });

    it("should return categorical time series for categorical scores", async () => {
      const scoreName1 = `cat-score-1-${v4()}`;
      const scoreName2 = `cat-score-2-${v4()}`;
      const fromTimestamp = new Date("2024-01-01T00:00:00Z");
      const toTimestamp = new Date("2024-01-05T00:00:00Z");

      const categories1 = ["A", "B", "C"];
      const categories2 = ["X", "Y", "Z"];

      const scores: any[] = [];
      for (let day = 0; day < 4; day++) {
        const timestamp = new Date(
          Date.UTC(2024, 0, 1 + day, 12, 0, 0),
        ).getTime();

        for (let i = 0; i < 3; i++) {
          const traceId = `trace-cat-${day}-${i}`;

          scores.push(
            createTraceScore({
              project_id: projectId,
              trace_id: traceId,
              observation_id: null,
              name: scoreName1,
              source: "API",
              data_type: "CATEGORICAL",
              string_value: categories1[i % categories1.length],
              timestamp,
            }),
            createTraceScore({
              project_id: projectId,
              trace_id: traceId,
              observation_id: null,
              name: scoreName2,
              source: "API",
              data_type: "CATEGORICAL",
              string_value: categories2[i % categories2.length],
              timestamp,
            }),
          );
        }
      }

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "CATEGORICAL", source: "API" },
        score2: { name: scoreName2, dataType: "CATEGORICAL", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Verify categorical time series data is returned
      expect(result.timeSeriesCategorical1.length).toBeGreaterThan(0);
      expect(result.timeSeriesCategorical2.length).toBeGreaterThan(0);

      // Verify all categories appear
      const returnedCategories1 = new Set(
        result.timeSeriesCategorical1.map((e) => e.category),
      );
      const returnedCategories2 = new Set(
        result.timeSeriesCategorical2.map((e) => e.category),
      );

      categories1.forEach((cat) => {
        expect(returnedCategories1.has(cat)).toBe(true);
      });
      categories2.forEach((cat) => {
        expect(returnedCategories2.has(cat)).toBe(true);
      });

      // Verify counts are correct (should be 4 days * 1 occurrence per day = 4 for each category)
      const categoryCounts1 = result.timeSeriesCategorical1.reduce(
        (acc, entry) => {
          acc[entry.category] = (acc[entry.category] || 0) + entry.count;
          return acc;
        },
        {} as Record<string, number>,
      );

      Object.values(categoryCounts1).forEach((count) => {
        expect(count).toBe(4); // 4 days
      });
    });

    it("should return matched categorical time series", async () => {
      const scoreName1 = `bool-score-matched-1-${v4()}`;
      const scoreName2 = `bool-score-matched-2-${v4()}`;
      const fromTimestamp = new Date("2024-01-01T00:00:00Z");
      const toTimestamp = new Date("2024-01-04T00:00:00Z");

      const scores: any[] = [];

      // Create 2 matched scores and 1 unmatched for each
      for (let day = 0; day < 3; day++) {
        const timestamp = new Date(
          Date.UTC(2024, 0, 1 + day, 12, 0, 0),
        ).getTime();

        // Matched pair
        const matchedTraceId = `trace-matched-${day}`;
        scores.push(
          createTraceScore({
            project_id: projectId,
            trace_id: matchedTraceId,
            observation_id: null,
            name: scoreName1,
            source: "API",
            data_type: "BOOLEAN",
            string_value: "true",
            timestamp,
          }),
          createTraceScore({
            project_id: projectId,
            trace_id: matchedTraceId,
            observation_id: null,
            name: scoreName2,
            source: "API",
            data_type: "BOOLEAN",
            string_value: "false",
            timestamp,
          }),
        );

        // Unmatched score1
        scores.push(
          createTraceScore({
            project_id: projectId,
            trace_id: `trace-unmatched-1-${day}`,
            observation_id: null,
            name: scoreName1,
            source: "API",
            data_type: "BOOLEAN",
            string_value: "false",
            timestamp,
          }),
        );

        // Unmatched score2
        scores.push(
          createTraceScore({
            project_id: projectId,
            trace_id: `trace-unmatched-2-${day}`,
            observation_id: null,
            name: scoreName2,
            source: "API",
            data_type: "BOOLEAN",
            string_value: "true",
            timestamp,
          }),
        );
      }

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "BOOLEAN", source: "API" },
        score2: { name: scoreName2, dataType: "BOOLEAN", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "day" },
        nBins: 10,
      });

      // Verify matched categorical time series is returned
      expect(result.timeSeriesCategorical1Matched.length).toBeGreaterThan(0);
      expect(result.timeSeriesCategorical2Matched.length).toBeGreaterThan(0);

      // Verify unmatched categorical time series has more data
      expect(result.timeSeriesCategorical1.length).toBeGreaterThan(
        result.timeSeriesCategorical1Matched.length,
      );
      expect(result.timeSeriesCategorical2.length).toBeGreaterThan(
        result.timeSeriesCategorical2Matched.length,
      );

      // Count total entries
      const totalCount1 = result.timeSeriesCategorical1.reduce(
        (sum, e) => sum + e.count,
        0,
      );
      const matchedCount1 = result.timeSeriesCategorical1Matched.reduce(
        (sum, e) => sum + e.count,
        0,
      );
      const totalCount2 = result.timeSeriesCategorical2.reduce(
        (sum, e) => sum + e.count,
        0,
      );
      const matchedCount2 = result.timeSeriesCategorical2Matched.reduce(
        (sum, e) => sum + e.count,
        0,
      );

      // Each score has 3 matched + 3 unmatched = 6 total per score
      expect(totalCount1).toBe(6);
      expect(matchedCount1).toBe(3);
      expect(totalCount2).toBe(6);
      expect(matchedCount2).toBe(3);
    });

    it("should group categorical time series by time intervals", async () => {
      const scoreName1 = `cat-score-intervals-1-${v4()}`;
      const scoreName2 = `cat-score-intervals-2-${v4()}`;
      const fromTimestamp = new Date("2024-01-01T00:00:00Z");
      const toTimestamp = new Date("2024-01-01T03:00:00Z");

      const scores: any[] = [];

      // Create scores across 3 hours, 2 per hour
      for (let hour = 0; hour < 3; hour++) {
        for (let i = 0; i < 2; i++) {
          const timestamp = new Date(
            Date.UTC(2024, 0, 1, hour, 0, 0),
          ).getTime();
          const traceId = `trace-hour-${hour}-${i}`;

          scores.push(
            createTraceScore({
              project_id: projectId,
              trace_id: traceId,
              observation_id: null,
              name: scoreName1,
              source: "API",
              data_type: "CATEGORICAL",
              string_value: "CategoryA",
              timestamp,
            }),
            createTraceScore({
              project_id: projectId,
              trace_id: traceId,
              observation_id: null,
              name: scoreName2,
              source: "API",
              data_type: "CATEGORICAL",
              string_value: "CategoryB",
              timestamp,
            }),
          );
        }
      }

      await createScoresCh(scores);

      const result = await caller.scoreAnalytics.getScoreComparisonAnalytics({
        projectId,
        score1: { name: scoreName1, dataType: "CATEGORICAL", source: "API" },
        score2: { name: scoreName2, dataType: "CATEGORICAL", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "hour" },
        nBins: 10,
      });

      // Verify data is bucketed by hour
      const uniqueTimestamps = new Set(
        result.timeSeriesCategorical1.map((e) =>
          e.timestamp.toISOString().substring(0, 13),
        ),
      );

      // Should have 3 unique hour buckets
      expect(uniqueTimestamps.size).toBe(3);

      // Each hour should have count = 2
      result.timeSeriesCategorical1.forEach((entry) => {
        expect(entry.count).toBe(2);
      });
    });

    // Test: ObjectType filtering works correctly for all types
    it("should filter scores correctly by objectType (trace, observation, session, dataset_run)", async () => {
      const traceId = v4();
      const observationId = v4();
      const sessionId = v4();
      const datasetRunId = v4();

      const now = new Date();
      const fromTimestamp = new Date(now.getTime() - 3600000);
      const toTimestamp = new Date(now.getTime() + 3600000);

      const scoreName1 = `objectType-test-score1-${v4()}`;
      const scoreName2 = `objectType-test-score2-${v4()}`;

      // Create trace for trace-level scores

      const scores = [
        // Trace-level scores (2 pairs)
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          session_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 1.0,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: null,
          session_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 2.0,
          timestamp: now.getTime(),
        }),

        // Observation-level scores (2 pairs)
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: observationId,
          session_id: null,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 3.0,
          timestamp: now.getTime(),
        }),
        createTraceScore({
          project_id: projectId,
          trace_id: traceId,
          observation_id: observationId,
          session_id: null,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 4.0,
          timestamp: now.getTime(),
        }),

        // Session-level scores (2 pairs)
        createSessionScore({
          project_id: projectId,
          session_id: sessionId,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 5.0,
          timestamp: now.getTime(),
        }),
        createSessionScore({
          project_id: projectId,
          session_id: sessionId,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 6.0,
          timestamp: now.getTime(),
        }),

        // Dataset run-level scores (2 pairs)
        createDatasetRunScore({
          project_id: projectId,
          dataset_run_id: datasetRunId,
          name: scoreName1,
          source: "API",
          data_type: "NUMERIC",
          value: 7.0,
          timestamp: now.getTime(),
        }),
        createDatasetRunScore({
          project_id: projectId,
          dataset_run_id: datasetRunId,
          name: scoreName2,
          source: "API",
          data_type: "NUMERIC",
          value: 8.0,
          timestamp: now.getTime(),
        }),
      ];

      await createScoresCh(scores);

      const baseParams = {
        projectId,
        score1: { name: scoreName1, dataType: "NUMERIC", source: "API" },
        score2: { name: scoreName2, dataType: "NUMERIC", source: "API" },
        fromTimestamp,
        toTimestamp,
        interval: { count: 1, unit: "hour" as const },
      };

      const [
        resultAll,
        resultTrace,
        resultObservation,
        resultSession,
        resultDatasetRun,
      ] = await Promise.all([
        caller.scoreAnalytics.getScoreComparisonAnalytics({
          ...baseParams,
          objectType: "all",
        }),
        caller.scoreAnalytics.getScoreComparisonAnalytics({
          ...baseParams,
          objectType: "trace",
        }),
        caller.scoreAnalytics.getScoreComparisonAnalytics({
          ...baseParams,
          objectType: "observation",
        }),
        caller.scoreAnalytics.getScoreComparisonAnalytics({
          ...baseParams,
          objectType: "session",
        }),
        caller.scoreAnalytics.getScoreComparisonAnalytics({
          ...baseParams,
          objectType: "dataset_run",
        }),
      ]);

      expect(resultAll.counts.matchedCount).toBe(4);
      expect(resultAll.counts.score1Total).toBe(4);
      expect(resultAll.counts.score2Total).toBe(4);

      expect(resultTrace.counts.matchedCount).toBe(1);
      expect(resultTrace.counts.score1Total).toBe(1);
      expect(resultTrace.counts.score2Total).toBe(1);

      expect(resultObservation.counts.matchedCount).toBe(1);
      expect(resultObservation.counts.score1Total).toBe(1);
      expect(resultObservation.counts.score2Total).toBe(1);

      expect(resultSession.counts.matchedCount).toBe(1);
      expect(resultSession.counts.score1Total).toBe(1);
      expect(resultSession.counts.score2Total).toBe(1);

      expect(resultDatasetRun.counts.matchedCount).toBe(1);
      expect(resultDatasetRun.counts.score1Total).toBe(1);
      expect(resultDatasetRun.counts.score2Total).toBe(1);
    });
  });
});