Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

94.6 KB
Raw Blame
import { randomUUID } from "node:crypto";
import {
  afterEach,
  beforeEach,
  describe,
  expect,
  it,
  vi,
  type TestContext,
} from "vitest";
import { uuid, z } from "zod";
import { prisma } from "@langfuse/shared/src/db";
import {
  clickhouseClient,
  ObservationEvent,
  observationRecordReadSchema,
  ObservationRecordReadType,
  redis,
  ScoreEventType,
  scoreRecordReadSchema,
  ScoreRecordReadType,
  TraceEventType,
  traceRecordReadSchema,
  TraceRecordReadType,
  createOrgProjectAndApiKey,
  createIngestionEventSchema,
  getObservationsV2FromEventsTableForPublicApi,
  queryClickhouse,
  setNoEvalConfigsCache,
} from "@langfuse/shared/src/server";
import waitForExpect from "wait-for-expect";
import { ClickhouseWriter, TableName } from "../../ClickhouseWriter";
import { IngestionService } from "../../IngestionService";
import { ModelUsageUnit, ScoreSourceEnum } from "@langfuse/shared";
import {
  clickhouseTableExists,
  skipUnlessClickhouseTablesExist,
} from "../../../__tests__/helpers/clickhouseTables";

let projectId = "";
const environment = "default";
const testIngestionAttribution = {
  ingestionApiKey: "pk-lf-ingestion-service-test",
  ingestionSdkName: "langfuse-test",
  ingestionSdkVersion: "0.0.0",
};

async function skipUnlessEventsTablesExist(ctx: TestContext): Promise<void> {
  await skipUnlessClickhouseTablesExist(
    ctx,
    [TableName.EventsFull],
    "events ClickHouse tables are not enabled",
  );
}

describe("Ingestion end-to-end tests", () => {
  let ingestionService: IngestionService;
  let clickhouseWriter: ClickhouseWriter;
  let IngestionEventBatchSchema: z.ZodType<any>;

  beforeEach(async () => {
    if (!redis) throw new Error("Redis not initialized");
    ({ projectId } = await createOrgProjectAndApiKey());
    await setNoEvalConfigsCache(projectId, "traceBased");

    clickhouseWriter = ClickhouseWriter.getInstance();

    ingestionService = new IngestionService(
      redis,
      prisma,
      clickhouseWriter,
      clickhouseClient(),
    );

    IngestionEventBatchSchema = z.array(createIngestionEventSchema());
  });

  afterEach(async () => {
    vi.restoreAllMocks();
    vi.useRealTimers();

    await ClickhouseWriter.shutdownAll();
  });

  it("should correctly ingest a trace", async () => {
    const traceId = randomUUID();
    const traceName = "test-trace";
    const timestamp = new Date().toISOString();

    const eventList: TraceEventType[] = [
      {
        type: "trace-create",
        id: traceId,
        timestamp,
        body: {
          name: traceName,
          timestamp,
          environment,
          input: "foo",
          output: "bar",
        },
      },
    ];

    await ingestionService.processTraceEventList({
      projectId,
      entityId: traceId,
      createdAtTimestamp: new Date(timestamp),
      traceEventList: eventList,
    });
    await clickhouseWriter.flushAll(true);

    const trace = await getClickhouseRecord(TableName.Traces, traceId);

    expect(trace.id).toBe(traceId);
    expect(trace.name).toBe(traceName);
    expect(trace.user_id).toBeNull();
    expect(trace.metadata).toEqual({});
    expect(trace.release).toBeNull();
    expect(trace.version).toBeNull();
    expect(trace.project_id).toBe(projectId);
    expect(trace.public).toBe(false);
    expect(trace.bookmarked).toBe(false);
    expect(trace.tags).toEqual([]);
    expect(trace.input).toBe("foo");
    expect(trace.output).toBe("bar");
    expect(trace.session_id).toBeNull();
    expect(trace.timestamp).toBe(timestamp);
  });

  it("should paginate events with sub-millisecond wire timestamps after ingestion normalization", async (ctx) => {
    await skipUnlessEventsTablesExist(ctx);

    const traceId = randomUUID();
    const firstSpanId = randomUUID();
    const secondSpanId = randomUUID();
    const wireStartTimeA = "2026-01-01T12:00:00.123456Z";
    const wireStartTimeB = "2026-01-01T12:00:00.123789Z";
    const normalizedStartTime = "2026-01-01 12:00:00.123";

    const eventRecords = await Promise.all(
      [
        { spanId: firstSpanId, startTimeISO: wireStartTimeA },
        { spanId: secondSpanId, startTimeISO: wireStartTimeB },
      ].map((event) =>
        ingestionService.createEventRecord(
          {
            projectId,
            traceId,
            spanId: event.spanId,
            parentSpanId: "",
            name: `sub-ms-cursor-${event.spanId}`,
            type: "SPAN",
            environment,
            startTimeISO: event.startTimeISO,
            endTimeISO: "2026-01-01T12:00:00.124000Z",
            metadata: {},
            source: "test",
          },
          `sub-ms-cursor/${event.spanId}.json`,
        ),
      ),
    );

    expect(eventRecords.map((record) => record.start_time)).toEqual([
      normalizedStartTime,
      normalizedStartTime,
    ]);

    await Promise.all(
      eventRecords.map((record) => ingestionService.writeEventRecord(record)),
    );
    await clickhouseWriter.flushAll(true);

    await waitForExpect(async () => {
      const rows = await queryClickhouse<{ count: string }>({
        query: `
          SELECT count() AS count
          FROM events_core
          WHERE project_id = {projectId: String}
            AND trace_id = {traceId: String}
            AND span_id IN ({spanIds: Array(String)})
        `,
        params: {
          projectId,
          traceId,
          spanIds: [firstSpanId, secondSpanId],
        },
      });

      expect(Number(rows[0]?.count ?? 0)).toBe(2);
    }, 5_000);

    const firstPageItems = await getObservationsV2FromEventsTableForPublicApi({
      projectId,
      traceId,
      page: 1,
      limit: 1,
      fields: ["core"],
    });
    const firstPage = firstPageItems.slice(0, 1);
    expect(firstPage).toHaveLength(1);

    const firstItem = firstPage[0]!;
    expect(firstItem.startTime.getTime()).toBe(Date.parse(wireStartTimeA));

    const secondPageItems = await getObservationsV2FromEventsTableForPublicApi({
      projectId,
      traceId,
      page: 1,
      limit: 1,
      fields: ["core"],
      cursor: {
        lastStartTimeTo: firstItem.startTime,
        lastTraceId: firstItem.traceId!,
        lastId: firstItem.id,
      },
    });
    const secondPage = secondPageItems.slice(0, 1);
    expect(secondPage).toHaveLength(1);

    expect(secondPage[0]!.id).not.toBe(firstItem.id);
    expect(new Set([firstItem.id, secondPage[0]!.id])).toEqual(
      new Set([firstSpanId, secondSpanId]),
    );
  });

  it("should write ingestion attribution to observation staging records", async (ctx) => {
    await skipUnlessObservationsBatchStagingEnabled(ctx);

    const traceId = randomUUID();
    const generationId = randomUUID();
    const timestamp = new Date().toISOString();
    const attribution = {
      ingestionApiKey: "pk-lf-ingestion-service-integration",
      ingestionSdkName: "langfuse-js",
      ingestionSdkVersion: "4.2.0",
    };

    const generationEventList: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "generation-create",
        timestamp,
        body: {
          id: generationId,
          traceId,
          startTime: timestamp,
          name: "generation-with-attribution",
          environment,
        },
      },
    ];

    await ingestionService.mergeAndWrite({
      projectId,
      entityId: generationId,
      createdAtTimestamp: new Date(timestamp),
      eventType: "observation",
      events: generationEventList,
      forwardToEventsTable: true,
      attribution,
    });

    await clickhouseWriter.flushAll(true);

    const generation = await getClickhouseRecord(
      TableName.Observations,
      generationId,
    );
    const stagingAttribution =
      await getClickhouseObservationBatchStagingAttribution(generationId);

    expect(generation.id).toBe(generationId);
    expect(stagingAttribution).toEqual({
      ingestion_api_key: attribution.ingestionApiKey,
      ingestion_sdk_name: attribution.ingestionSdkName,
      ingestion_sdk_version: attribution.ingestionSdkVersion,
    });
  });

  [
    {
      usage: {
        input: 100,
        output: 200,
        total: 100,
        unit: ModelUsageUnit.Characters,
        inputCost: 123,
        outputCost: 456,
        totalCost: 789,
      },
      expectedUsageDetails: {
        input: 100,
        output: 200,
        total: 100,
      },
    },
    {
      usage: {
        total: 100,
        unit: ModelUsageUnit.Characters,
      },
      expectedUsageDetails: {
        total: 100,
      },
    },
    {
      usage: {
        total: 100,
        unit: ModelUsageUnit.Milliseconds,
      },
      expectedUsageDetails: {
        total: 100,
      },
    },
    {
      usage: {
        input: 1,
        output: 2,
        unit: ModelUsageUnit.Images,
      },
      expectedUsageDetails: {
        input: 1,
        output: 2,
        total: 3,
      },
    },
    {
      usage: {
        input: 1,
        output: 2,
        unit: ModelUsageUnit.Requests,
      },
      expectedUsageDetails: {
        input: 1,
        output: 2,
        total: 3,
      },
    },
    {
      usage: {
        input: 30,
        output: 10,
        unit: ModelUsageUnit.Seconds,
      },
      expectedUsageDetails: {
        input: 30,
        output: 10,
        total: 40,
      },
    },
    {
      usage: {
        total: 100,
      },
      expectedUsageDetails: {
        total: 100,
      },
    },
    {
      usage: undefined,
      expectedUsageDetails: {},
    },
    {
      usage: null,
      expectedUsageDetails: {},
    },
    {
      usage: {},
      expectedUsageDetails: {},
    },
    {
      usage: {},
      usageDetails: {
        input: 1,
        output: 2,
        total: 3,
        cached: 1,
      },
      expectedUsageDetails: {
        input: 1,
        output: 2,
        total: 3,
        cached: 1,
      },
    },
    {
      usage: {
        input: 1,
        output: 2,
        total: 3,
      },
      usageDetails: {
        cached: 1,
      },
      expectedUsageDetails: {
        input: 1,
        output: 2,
        total: 3,
        cached: 1,
      },
    },
    {
      usage: {
        input: 1,
        output: 2,
        total: 3,
      },
      usageDetails: {
        input: 2,
        output: 3,
        total: 5,
        cached: 1,
      },
      expectedUsageDetails: {
        input: 2,
        output: 3,
        total: 5,
        cached: 1,
      },
    },
    {
      usage: {
        input: 1,
        output: 2,
        total: 3,
      },
      costDetails: {
        input: 123,
        output: 456,
        total: 789,
      },
      expectedUsageDetails: {
        input: 1,
        output: 2,
        total: 3,
      },
      expectedCostDetails: {
        input: 123,
        output: 456,
        total: 789,
      },
    },
    {
      usage: {},
      usageDetails: {},
      costDetails: {},
      expectedUsageDetails: {},
      expectedCostDetails: {},
    },
    {
      usage: null,
      usageDetails: null,
      costDetails: null,
      expectedUsageDetails: {},
      expectedCostDetails: {},
    },
    {
      usage: undefined,
      usageDetails: undefined,
      costDetails: undefined,
      expectedUsageDetails: {},
      expectedCostDetails: {},
    },
    {
      usage: { input: 1 },
      usageDetails: { input: 2 },
      costDetails: { input: 3 },
      expectedUsageDetails: { input: 2, total: 2 },
      expectedCostDetails: { input: 3, total: 3 },
    },
    {
      usage: { input: 1 },
      usageDetails: {
        input: 1,
        cached: 2,
        reasoning: 3,
      },
      expectedUsageDetails: { input: 1, cached: 2, reasoning: 3, total: 6 },
    },
    {
      usage: {},
      usageDetails: {
        input: 1,
        output: null,
        total: undefined,
      },
      expectedUsageDetails: { input: 1, total: 1 },
      costDetails: {
        input: 123,
        output: null,
        cached: undefined,
      },
      expectedCostDetails: { input: 123, total: 123 },
    },
    // OpenAI format
    {
      usage: null,
      usageDetails: {
        prompt_tokens: 5,
        completion_tokens: 11,
        total_tokens: 16,
        prompt_tokens_details: {
          cached_tokens: 2,
          audio_tokens: 3,
        },
        completion_tokens_details: {
          text_tokens: 3,
          audio_tokens: 4,
          reasoning_tokens: 4,
        },
      },
      expectedUsageDetails: {
        input: 0,
        output: 0,
        total: 16,
        input_cached_tokens: 2,
        input_audio_tokens: 3,
        output_text_tokens: 3,
        output_audio_tokens: 4,
        output_reasoning_tokens: 4,
      },
    },
    {
      usage: null,
      usageDetails: {
        prompt_tokens: 5,
        completion_tokens: 11,
        total_tokens: 16,
        prompt_tokens_details: {
          cached_tokens: 2,
          audio_tokens: null,
          image_tokens: undefined,
        },
        completion_tokens_details: {
          text_tokens: 3,
          audio_tokens: undefined,
          reasoning_tokens: 4,
          image_tokens: null,
        },
      },
      expectedUsageDetails: {
        input: 3,
        output: 4,
        total: 16,
        input_cached_tokens: 2,
        output_text_tokens: 3,
        output_reasoning_tokens: 4,
      },
    },
    // OpenAI-compatible provider extensions (e.g. OpenRouter cost fields)
    {
      usage: null,
      usageDetails: {
        prompt_tokens: 194,
        completion_tokens: 2,
        total_tokens: 196,
        prompt_tokens_details: { cached_tokens: 4 },
        completion_tokens_details: { reasoning_tokens: 1 },
        cost: 0.0001,
        cost_details: { upstream_inference_cost: 0.0001 },
      },
      expectedUsageDetails: {
        input: 190,
        output: 1,
        total: 196,
        input_cached_tokens: 4,
        output_reasoning_tokens: 1,
      },
      absentUsageDetailKeys: [
        "prompt_tokens",
        "completion_tokens",
        "total_tokens",
      ],
    },
    // OpenAI Response API format
    {
      usage: null,
      usageDetails: {
        input_tokens: 5,
        output_tokens: 11,
        total_tokens: 16,
        input_tokens_details: {
          cached_tokens: 2,
          audio_tokens: 3,
        },
        output_tokens_details: {
          text_tokens: 3,
          audio_tokens: 4,
          reasoning_tokens: 4,
        },
      },
      expectedUsageDetails: {
        input: 0,
        output: 0,
        total: 16,
        input_cached_tokens: 2,
        input_audio_tokens: 3,
        output_text_tokens: 3,
        output_audio_tokens: 4,
        output_reasoning_tokens: 4,
      },
    },
    {
      usage: null,
      usageDetails: {
        input_tokens: 5,
        output_tokens: 11,
        total_tokens: 16,
        input_tokens_details: {
          cached_tokens: 2,
          audio_tokens: null,
          image_tokens: undefined,
        },
        output_tokens_details: {
          text_tokens: 3,
          audio_tokens: null,
          reasoning_tokens: 4,
          image_tokens: undefined,
        },
      },
      expectedUsageDetails: {
        input: 3,
        output: 4,
        total: 16,
        input_cached_tokens: 2,
        output_text_tokens: 3,
        output_reasoning_tokens: 4,
      },
    },
    {
      usage: null,
      usageDetails: {
        input_tokens: 194,
        output_tokens: 2,
        total_tokens: 196,
        input_tokens_details: { cached_tokens: 4 },
        output_tokens_details: { reasoning_tokens: 1 },
        cost: 0.0001,
      },
      expectedUsageDetails: {
        input: 190,
        output: 1,
        total: 196,
        input_cached_tokens: 4,
        output_reasoning_tokens: 1,
      },
      absentUsageDetailKeys: ["input_tokens", "output_tokens", "total_tokens"],
    },
    {
      usage: null,
      usageDetails: {
        input_tokens: 3,
        output_tokens: 94,
        total_tokens: 16496,
        cache_read_input_tokens: 16399,
        service_tier: "standard",
      },
      expectedUsageDetails: {
        input: 3,
        output: 94,
        total: 16496,
        cache_read_input_tokens: 16399,
      },
      absentUsageDetailKeys: ["input_tokens", "output_tokens"],
    },
    // Native Anthropic Messages usage with TTL-split cache writes
    {
      usage: null,
      usageDetails: {
        input_tokens: 3,
        output_tokens: 94,
        cache_creation_input_tokens: 12256,
        cache_read_input_tokens: 16399,
        cache_creation: {
          ephemeral_5m_input_tokens: 2000,
          ephemeral_1h_input_tokens: 10256,
        },
        server_tool_use: { web_search_requests: 0 },
        service_tier: "standard",
      },
      expectedUsageDetails: {
        input_tokens: 3,
        output_tokens: 94,
        cache_read_input_tokens: 16399,
        input_cache_creation_5m: 2000,
        input_cache_creation_1h: 10256,
      },
      absentUsageDetailKeys: ["cache_creation_input_tokens"],
    },
  ].forEach((testConfig) => {
    it(`should create trace, generation and score without matching models ${JSON.stringify(
      testConfig,
      null,
      2,
    )}`, async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();
      const spanId = randomUUID();
      const scoreId = randomUUID();

      const traceEventList: TraceEventType[] = [
        {
          id: randomUUID(),
          type: "trace-create",
          timestamp: new Date().toISOString(),
          body: {
            id: traceId,
            timestamp: new Date().toISOString(),
            name: "trace-name",
            userId: "user-1",
            metadata: { key: "value" },
            release: "1.0.0",
            version: "2.0.0",
            tags: ["tag-1", "tag-2"],
            environment,
          },
        },
      ];

      const generationEventList: ObservationEvent[] =
        IngestionEventBatchSchema.parse([
          {
            id: randomUUID(),
            type: "generation-create",
            timestamp: new Date().toISOString(),
            body: {
              id: generationId,
              traceId: traceId,
              name: "generation-name",
              startTime: "2021-01-01T00:00:00.000Z",
              endTime: "2021-01-01T00:00:00.000Z",
              modelParameters: { key: "value" },
              input: { key: "value" },
              metadata: { key: "value" },
              version: "2.0.0",
              environment,
            },
          },
          {
            id: randomUUID(),
            type: "generation-update",
            timestamp: new Date().toISOString(),
            body: {
              id: generationId,
              output: { key: "this is a great gpt output" },
              usage: testConfig.usage,
              usageDetails: testConfig.usageDetails,
              costDetails: testConfig.costDetails,
              environment,
            },
          },
        ]);

      const spanEventList: ObservationEvent[] = [
        {
          id: randomUUID(),
          type: "observation-create",
          timestamp: new Date().toISOString(),
          body: {
            id: spanId,
            traceId: traceId,
            type: "SPAN",
            name: "span-name",
            startTime: "2021-01-01T00:00:00.000Z",
            endTime: "2021-01-01T00:00:00.000Z",
            input: { input: "value" },
            metadata: { meta: "value" },
            version: "2.0.0",
            environment,
          },
        },
      ];

      const scoreEventList: ScoreEventType[] = [
        {
          id: randomUUID(),
          type: "score-create",
          timestamp: new Date().toISOString(),
          body: {
            id: scoreId,
            dataType: "NUMERIC",
            name: "score-name",
            value: 100.5,
            source: ScoreSourceEnum.EVAL,
            traceId: traceId,
            environment,
          },
        },
      ];

      await Promise.all([
        ingestionService.processTraceEventList({
          projectId,
          entityId: traceId,
          createdAtTimestamp: new Date(),
          traceEventList,
        }),
        ingestionService.processObservationEventList({
          attribution: testIngestionAttribution,
          projectId,
          entityId: spanId,
          createdAtTimestamp: new Date(),
          observationEventList: spanEventList,
        }),
        ingestionService.processObservationEventList({
          attribution: testIngestionAttribution,
          projectId,
          entityId: generationId,
          createdAtTimestamp: new Date(),
          observationEventList: generationEventList,
        }),
        ingestionService.processScoreEventList({
          projectId,
          entityId: scoreId,
          createdAtTimestamp: new Date(),
          scoreEventList,
          attribution: testIngestionAttribution,
        }),
      ]);

      await clickhouseWriter.flushAll(true);

      const trace = await getClickhouseRecord(TableName.Traces, traceId);

      expect(trace.name).toBe("trace-name");
      expect(trace.release).toBe("1.0.0");
      expect(trace.version).toBe("2.0.0");
      expect(trace.project_id).toBe(projectId);
      expect(trace.tags).toEqual(["tag-1", "tag-2"]);

      const generation = await getClickhouseRecord(
        TableName.Observations,
        generationId,
      );

      expect(generation.id).toBe(generationId);
      expect(generation.trace_id).toBe(traceId);
      expect(generation.name).toBe("generation-name");
      expect(generation.start_time).toEqual("2021-01-01T00:00:00.000Z");
      expect(generation.end_time).toEqual("2021-01-01T00:00:00.000Z");
      expect(generation.provided_model_name).toBeNull();
      expect(generation.model_parameters).toEqual(
        JSON.stringify({
          key: "value",
        }),
      );
      expect(generation.input).toEqual(JSON.stringify({ key: "value" }));
      expect(generation.metadata).toEqual({ key: "value" });
      expect(generation.version).toBe("2.0.0");
      expect(generation.internal_model_id).toBeNull();
      expect(generation.usage_details).toMatchObject(
        testConfig.expectedUsageDetails,
      );
      for (const key of testConfig.absentUsageDetailKeys ?? []) {
        expect(generation.usage_details).not.toHaveProperty(key);
      }
      expect(generation.output).toEqual(
        JSON.stringify({
          key: "this is a great gpt output",
        }),
      );

      const span = await getClickhouseRecord(TableName.Observations, spanId);

      expect(span.id).toBe(spanId);
      expect(span.name).toBe("span-name");
      expect(span.start_time).toEqual("2021-01-01T00:00:00.000Z");
      expect(span.end_time).toEqual("2021-01-01T00:00:00.000Z");
      expect(span.input).toEqual(JSON.stringify({ input: "value" }));
      expect(span.metadata).toEqual({ meta: "value" });
      expect(span.version).toBe("2.0.0");

      const score = await getClickhouseRecord(TableName.Scores, scoreId);

      expect(score.id).toBe(scoreId);
      expect(score.trace_id).toBe(traceId);
      expect(score.name).toBe("score-name");
      expect(score.value).toBe(100.5);
      expect(score.observation_id).toBeNull();
      expect(score.source).toBe(ScoreSourceEnum.EVAL);
      expect(score.project_id).toBe(projectId);
    }, 10_000);
  });

  it("should correctly ingest a TEXT score", async () => {
    const traceId = randomUUID();
    const scoreId = randomUUID();

    const traceEventList: TraceEventType[] = [
      {
        type: "trace-create",
        id: traceId,
        timestamp: new Date().toISOString(),
        body: {
          name: "trace-for-text-score",
          timestamp: new Date().toISOString(),
          environment,
        },
      },
    ];

    const scoreEventList: ScoreEventType[] = [
      {
        id: randomUUID(),
        type: "score-create",
        timestamp: new Date().toISOString(),
        body: {
          id: scoreId,
          dataType: "TEXT",
          name: "text-score",
          value: "Great explanation of the concept",
          source: ScoreSourceEnum.API,
          traceId: traceId,
          environment,
        },
      },
    ];

    await Promise.all([
      ingestionService.processTraceEventList({
        projectId,
        entityId: traceId,
        createdAtTimestamp: new Date(),
        traceEventList,
      }),
      ingestionService.processScoreEventList({
        projectId,
        entityId: scoreId,
        createdAtTimestamp: new Date(),
        scoreEventList,
        attribution: testIngestionAttribution,
      }),
    ]);

    await clickhouseWriter.flushAll(true);

    const score = await getClickhouseRecord(TableName.Scores, scoreId);

    expect(score.id).toBe(scoreId);
    expect(score.trace_id).toBe(traceId);
    expect(score.name).toBe("text-score");
    expect(score.data_type).toBe("TEXT");
    expect(score.string_value).toBe("Great explanation of the concept");
    expect(score.value).toBe(0);
    expect(score.source).toBe(ScoreSourceEnum.API);
    expect(score.project_id).toBe(projectId);
  }, 10_000);

  [
    {
      observationExternalModel: "gpt-3.5",
      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
      modelUnit: ModelUsageUnit.Tokens,
      expectedInternalModelId: "custom-model-id",
      expectedUsageDetails: {
        input: 5,
        output: 7,
      },
      models: [
        {
          id: "custom-model-id",
          modelName: "gpt-3.5-turbo",
          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
        },
      ],
    },
    {
      observationExternalModel: "gpt-3.5",
      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
      modelUnit: ModelUsageUnit.Tokens,
      expectedInternalModelId: "custom-model-id",
      expectedUsageDetails: {
        input: 5,
        output: 7,
      },
      models: [
        {
          id: "custom-model-id",
          modelName: "gpt-3.5-turbo",
          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
          startDate: null,
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
        },
      ],
    },
    {
      observationExternalModel: "GPT-3.5",
      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
      modelUnit: ModelUsageUnit.Tokens,
      expectedInternalModelId: "custom-model-id",
      expectedUsageDetails: {
        input: 5,
        output: 7,
      },
      models: [
        {
          id: "custom-model-id",
          modelName: "gpt-3.5-turbo",
          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
        },
      ],
    },
    {
      observationExternalModel: "GPT-3.5",
      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
      modelUnit: ModelUsageUnit.Tokens,
      expectedInternalModelId: "custom-model-id-2",
      expectedUsageDetails: {
        input: 5,
        output: 7,
      },
      models: [
        {
          id: "custom-model-id-1",
          modelName: "gpt-3.5-turbo",
          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
        },
        {
          id: "custom-model-id-2",
          modelName: "gpt-3.5-turbo-new",
          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
          startDate: new Date("2021-01-01T10:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
          tokenizerModel: "gpt-3.5-turbo",
        },
      ],
    },
    {
      observationExternalModel: "GPT-3.5",
      observationStartTime: new Date("2021-01-02T00:00:00.000Z"),
      modelUnit: ModelUsageUnit.Tokens,
      expectedInternalModelId: "custom-model-id-2",
      expectedUsageDetails: {
        input: 5,
        output: 7,
      },
      models: [
        {
          id: "custom-model-id-1",
          modelName: "gpt-3.5-turbo-new",
          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
        },
        {
          id: "custom-model-id-2",
          modelName: "gpt-3.5-turbo",
          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
          startDate: new Date("2021-01-01T10:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
          tokenizerModel: "gpt-3.5-turbo",
        },
      ],
    },
    {
      observationExternalModel: "ft:gpt-3.5-turbo-1106:my-org:custom_suffix:id",
      observationStartTime: new Date("2022-01-01T10:00:00.000Z"),
      modelUnit: ModelUsageUnit.Tokens,
      expectedInternalModelId: "custom-model-id-1",
      expectedUsageDetails: {},
      models: [
        {
          id: "custom-model-id-1",
          modelName: "ft:gpt-3.5-turbo-1106",
          matchPattern: "(?i)^(ft:)(gpt-3.5-turbo-1106:)(.+)(:)(.*)(:)(.+)$",
          startDate: new Date("2022-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
        },
      ],
    },
    {
      observationExternalModel: "ft:babbage-002:my-org#2:custom_suffix-2:id",
      observationStartTime: new Date("2022-01-01T10:00:00.000Z"),
      modelUnit: ModelUsageUnit.Tokens,
      expectedInternalModelId: "custom-model-id-1",
      expectedUsageDetails: {},
      models: [
        {
          id: "custom-model-id-1",
          modelName: "ft:babbage-002",
          matchPattern: "(?i)^(ft:)(babbage-002:)(.+)(:)(.*)(:)(.+)$",
          startDate: new Date("2022-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
        },
      ],
    },
    {
      observationExternalModel: "lf-unmatched-model-for-token-test",
      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
      modelUnit: ModelUsageUnit.Tokens,
      expectedInternalModelId: null,
      expectedUsageDetails: {},
      models: [
        {
          id: "custom-model-id-1",
          modelName: "gpt-3.5-turbo",
          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
        },
      ],
    },
    {
      observationExternalModel: "GPT-3",
      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
      modelUnit: ModelUsageUnit.Characters,
      expectedInternalModelId: null,
      expectedUsageDetails: {},
      models: [
        {
          id: "custom-model-id-1",
          modelName: "gpt-3.5-turbo",
          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          tokenizerId: "openai",
        },
      ],
    },
  ].forEach((testConfig) => {
    it(`should match observations to internal models ${JSON.stringify(
      testConfig,
      null,
      2,
    )}`, async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();
      const modelIdMap = new Map<string, string>();

      const getModelId = (id: string) => {
        const existing = modelIdMap.get(id);
        if (existing) return existing;
        const generatedId = randomUUID();
        modelIdMap.set(id, generatedId);
        return generatedId;
      };

      await Promise.all(
        testConfig.models.map(async (model) =>
          prisma.model.create({
            data: {
              id: getModelId(model.id),
              projectId,
              modelName: model.modelName,
              matchPattern: model.matchPattern,
              startDate: model.startDate,
              unit: model.unit,
              tokenizerId: model.tokenizerId,
              tokenizerConfig: {
                tokensPerMessage: 3,
                tokensPerName: 1,
                tokenizerModel:
                  "tokenizerModel" in model
                    ? model.tokenizerModel
                    : model.modelName,
              },
            },
          }),
        ),
      );

      const traceEventList: TraceEventType[] = [
        {
          id: randomUUID(),
          type: "trace-create",
          timestamp: new Date().toISOString(),
          body: {
            id: traceId,
            name: "trace-name",
            timestamp: new Date().toISOString(),
            environment,
          },
        },
      ];
      const generationEventList: ObservationEvent[] = [
        {
          id: randomUUID(),
          type: "observation-create",
          timestamp: new Date().toISOString(),
          body: {
            id: generationId,
            traceId: traceId,
            type: "GENERATION",
            name: "generation-name",
            startTime: testConfig.observationStartTime.toISOString(),
            model: testConfig.observationExternalModel,
            usage: {
              unit: testConfig.modelUnit,
            },
            input: "This is a great prompt",
            output: "This is a great gpt output",
            environment,
          },
        },
      ];

      await Promise.all([
        ingestionService.processTraceEventList({
          projectId,
          entityId: traceId,
          createdAtTimestamp: new Date(),
          traceEventList,
        }),
        ingestionService.processObservationEventList({
          attribution: testIngestionAttribution,
          projectId,
          entityId: generationId,
          createdAtTimestamp: new Date(),
          observationEventList: generationEventList,
        }),
      ]);
      await clickhouseWriter.flushAll(true);

      const generation = await getClickhouseRecord(
        TableName.Observations,
        generationId,
      );

      expect(generation.id).toBe(generationId);
      expect(generation.trace_id).toBe(traceId);
      expect(generation.name).toBe("generation-name");
      expect(generation.start_time).toEqual(
        testConfig.observationStartTime.toISOString(),
      );
      expect(generation.provided_model_name).toBe(
        testConfig.observationExternalModel,
      );
      expect(generation.usage_details.input).toBe(
        testConfig.expectedUsageDetails.input,
      );
      expect(generation.usage_details.output).toBe(
        testConfig.expectedUsageDetails.output,
      );
      const expectedModelId =
        testConfig.expectedInternalModelId === null
          ? null
          : getModelId(testConfig.expectedInternalModelId);
      expect(generation.internal_model_id).toBe(expectedModelId);
    }, 15_000);
  });

  it("should create and update all events", async () => {
    const traceId = randomUUID();
    const generationId = randomUUID();
    const spanId = randomUUID();
    const eventId = randomUUID();
    const scoreId = randomUUID();

    const traceEventList: TraceEventType[] = [
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: traceId,
          timestamp: new Date().toISOString(),
          environment,
        },
      },
    ];

    const spanEventList: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "span-create",
        timestamp: new Date().toISOString(),
        body: {
          id: spanId,
          traceId: traceId,
          startTime: new Date().toISOString(),
          environment,
        },
      },
      {
        id: randomUUID(),
        type: "span-update",
        timestamp: new Date().toISOString(),
        body: {
          id: spanId,
          traceId: traceId,
          name: "span-name",
          startTime: new Date().toISOString(),
          environment,
        },
      },
    ];

    const generationEventList: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "generation-create",
        timestamp: new Date().toISOString(),
        body: {
          id: generationId,
          traceId: traceId,
          startTime: new Date().toISOString(),
          parentObservationId: spanId,
          environment: environment,
          modelParameters: { someKey: ["user-1", "user-2"] },
        },
      },
      {
        id: randomUUID(),
        type: "generation-update",
        timestamp: new Date().toISOString(),
        body: {
          id: generationId,
          name: "generation-name",
          startTime: new Date().toISOString(),
          environment,
        },
      },
    ];

    const eventEventList: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "event-create",
        timestamp: new Date().toISOString(),
        body: {
          id: eventId,
          traceId: traceId,
          name: "event-name",
          startTime: new Date().toISOString(),
          parentObservationId: generationId,
          environment,
        },
      },
    ];

    const scoreConfigId = randomUUID();
    await prisma.scoreConfig.create({
      data: {
        id: scoreConfigId,
        dataType: "NUMERIC",
        name: "test-config",
        projectId,
      },
    });

    const queueId = randomUUID();
    const scoreEventList: ScoreEventType[] = [
      {
        id: randomUUID(),
        type: "score-create",
        timestamp: new Date().toISOString(),
        body: {
          id: scoreId,
          dataType: "NUMERIC",
          configId: scoreConfigId,
          name: "score-name",
          traceId: traceId,
          source: ScoreSourceEnum.API,
          value: 100.5,
          observationId: generationId,
          queueId,
          environment,
        },
      },
    ];

    await Promise.all([
      ingestionService.processTraceEventList({
        projectId,
        entityId: traceId,
        createdAtTimestamp: new Date(),
        traceEventList,
      }),
      ingestionService.processObservationEventList({
        attribution: testIngestionAttribution,
        projectId,
        entityId: spanId,
        createdAtTimestamp: new Date(),
        observationEventList: spanEventList,
      }),
      ingestionService.processObservationEventList({
        attribution: testIngestionAttribution,
        projectId,
        entityId: generationId,
        createdAtTimestamp: new Date(),
        observationEventList: generationEventList,
      }),
      ingestionService.processObservationEventList({
        attribution: testIngestionAttribution,
        projectId,
        entityId: eventId,
        createdAtTimestamp: new Date(),
        observationEventList: eventEventList,
      }),
      ingestionService.processScoreEventList({
        projectId,
        entityId: scoreId,
        createdAtTimestamp: new Date(),
        scoreEventList,
        attribution: testIngestionAttribution,
      }),
    ]);

    await clickhouseWriter.flushAll(true);

    const trace = await getClickhouseRecord(TableName.Traces, traceId);

    expect(trace.project_id).toBe(projectId);

    const span = await getClickhouseRecord(TableName.Observations, spanId);

    expect(span.id).toBe(spanId);
    expect(span.name).toBe("span-name");
    expect(span.trace_id).toBe(traceId);

    const generation = await getClickhouseRecord(
      TableName.Observations,
      generationId,
    );

    expect(generation?.id).toBe(generationId);
    expect(generation?.trace_id).toBe(traceId);
    expect(generation?.name).toBe("generation-name");
    expect(generation?.parent_observation_id).toBe(spanId);
    expect(generation?.model_parameters).toEqual(
      JSON.stringify({
        someKey: ["user-1", "user-2"],
      }),
    );

    const event = await getClickhouseRecord(TableName.Observations, eventId);

    expect(event.id).toBe(eventId);
    expect(event.trace_id).toBe(traceId);
    expect(event.name).toBe("event-name");
    expect(event.parent_observation_id).toBe(generationId);

    const score = await getClickhouseRecord(TableName.Scores, scoreId);

    expect(score.id).toBe(scoreId);
    expect(score.trace_id).toBe(traceId);
    expect(score.observation_id).toBe(generationId);
    expect(score.value).toBe(100.5);
    expect(score.config_id).toBe(scoreConfigId);
    expect(score.queue_id).toBe(queueId);
  });

  it("should silently reject invalid scores while processing valid ones", async () => {
    const traceId = randomUUID();
    const validScoreId1 = randomUUID();
    const validScoreId2 = randomUUID();
    const invalidScoreId1 = randomUUID(); // Will have value out of range
    const invalidScoreId2 = randomUUID(); // Will use archived config

    // Create a trace first
    const traceEventList: TraceEventType[] = [
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: traceId,
          name: "test-trace",
          timestamp: new Date().toISOString(),
          environment,
        },
      },
    ];

    await ingestionService.processTraceEventList({
      projectId,
      entityId: traceId,
      createdAtTimestamp: new Date(),
      traceEventList,
    });

    // Create score configs
    const validScoreConfigId = randomUUID();
    const archivedScoreConfigId = randomUUID();

    await Promise.all([
      // Valid numeric config with range 0-100
      prisma.scoreConfig.create({
        data: {
          id: validScoreConfigId,
          dataType: "NUMERIC",
          name: "valid-config",
          minValue: 0,
          maxValue: 100,
          projectId,
        },
      }),
      // Archived config that should be rejected
      prisma.scoreConfig.create({
        data: {
          id: archivedScoreConfigId,
          dataType: "NUMERIC",
          name: "archived-config",
          isArchived: true,
          projectId,
        },
      }),
    ]);

    // Process all scores - invalid ones should be rejected silently
    // and valid ones should be processed
    await Promise.all([
      // Valid score 1
      ingestionService.processScoreEventList({
        projectId,
        entityId: validScoreId1,
        createdAtTimestamp: new Date(),
        attribution: testIngestionAttribution,
        scoreEventList: [
          {
            id: validScoreId1,
            type: "score-create",
            timestamp: new Date().toISOString(),
            body: {
              id: validScoreId1,
              dataType: "NUMERIC",
              name: "valid-config",
              value: 85.5, // Within range 0-100
              source: ScoreSourceEnum.API,
              traceId: traceId,
              environment,
              configId: validScoreConfigId,
            },
          },
        ],
      }),
      // One valid score, two invalid scores
      ingestionService.processScoreEventList({
        projectId,
        entityId: validScoreId2,
        createdAtTimestamp: new Date(),
        attribution: testIngestionAttribution,
        scoreEventList: [
          // invalid score 1
          {
            id: invalidScoreId1,
            type: "score-create",
            timestamp: new Date().toISOString(),
            body: {
              id: invalidScoreId1,
              dataType: "NUMERIC",
              configId: validScoreConfigId,
              name: "valid-config",
              traceId: traceId,
              source: ScoreSourceEnum.API,
              value: 150, // Outside range 0-100, should fail validation
              environment,
            },
          },
          // valid score 2
          {
            id: validScoreId2,
            type: "score-create",
            timestamp: new Date().toISOString(),
            body: {
              id: validScoreId2,
              dataType: "NUMERIC",
              configId: validScoreConfigId,
              name: "archived-config",
              traceId: traceId,
              source: ScoreSourceEnum.API,
              value: 50,
              environment,
            },
          },
          // invalid score 2
          {
            id: invalidScoreId2,
            type: "score-create",
            timestamp: new Date().toISOString(),
            body: {
              id: invalidScoreId2,
              dataType: "NUMERIC",
              configId: archivedScoreConfigId,
              name: "archived-config",
              traceId: traceId,
              source: ScoreSourceEnum.API,
              value: 50, // Valid value but config is archived
              environment,
            },
          },
        ],
      }),
    ]);

    await clickhouseWriter.flushAll(true);

    // Verify that valid scores were inserted
    const validScore1 = await getClickhouseRecord(
      TableName.Scores,
      validScoreId1,
    );
    expect(validScore1).toBeDefined();
    expect(validScore1.trace_id).toBe(traceId);
    expect(validScore1.value).toBe(85.5);
    expect(validScore1.config_id).toBe(validScoreConfigId);

    // Verify that invalid scores were silently rejected (not inserted)
    await expect(
      getClickhouseRecord(TableName.Scores, invalidScoreId1, {
        waitForRecord: false,
      }),
    ).rejects.toThrow();

    await expect(
      getClickhouseRecord(TableName.Scores, invalidScoreId2, {
        waitForRecord: false,
      }),
    ).rejects.toThrow();
  });

  it("should upsert traces", async () => {
    const traceId = randomUUID();

    // First flush
    const traceEventList1: TraceEventType[] = [
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: traceId,
          name: "trace-name",
          timestamp: new Date().toISOString(),
          userId: "user-1",
          metadata: { key: "value" },
          release: "1.0.0",
          version: "2.0.0",
          tags: ["tag-1", "tag-2", "tag-2"],
          environment,
        },
      },
    ];

    await ingestionService.processTraceEventList({
      projectId,
      entityId: traceId,
      createdAtTimestamp: new Date(),
      traceEventList: traceEventList1,
    });

    await clickhouseWriter.flushAll(true);

    // Second flush
    const traceEventList2: TraceEventType[] = [
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: traceId,
          name: "trace-name",
          userId: "user-2",
          tags: ["tag-1", "tag-4", "tag-3"],
          environment,
        },
      },
    ];

    await ingestionService.processTraceEventList({
      projectId,
      entityId: traceId,
      createdAtTimestamp: new Date(),
      traceEventList: traceEventList2,
    });

    await clickhouseWriter.flushAll(true);

    await waitForExpect(async () => {
      const trace = await getClickhouseRecord(TableName.Traces, traceId);

      expect(trace.name).toBe("trace-name");
      expect(trace.user_id).toBe("user-2");
      expect(trace.release).toBe("1.0.0");
      expect(trace.version).toBe("2.0.0");
      expect(trace.project_id).toBe(projectId);
      expect(trace.tags.sort()).toEqual(
        ["tag-1", "tag-2", "tag-3", "tag-4"].sort(),
      );
      expect(trace.tags.length).toBe(4);
    });
  });

  it("should upsert traces in the right order", async () => {
    const traceId = randomUUID();

    const latestEvent = new Date();
    const oldEvent = new Date(latestEvent).setSeconds(
      latestEvent.getSeconds() - 1,
    );

    const traceEventList: TraceEventType[] = [
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: latestEvent.toISOString(),
        body: {
          id: traceId,
          timestamp: latestEvent.toISOString(),
          name: "trace-name",
          userId: "user-1",
          environment,
        },
      },
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date(oldEvent).toISOString(),
        body: {
          id: traceId,
          timestamp: new Date(oldEvent).toISOString(),
          name: "trace-name",
          userId: "user-2",
          environment,
        },
      },
    ];

    await ingestionService.processTraceEventList({
      projectId,
      entityId: traceId,
      createdAtTimestamp: new Date(),
      traceEventList,
    });

    await clickhouseWriter.flushAll(true);

    const trace = await getClickhouseRecord(TableName.Traces, traceId);

    expect(trace.name).toBe("trace-name");
    expect(trace.user_id).toBe("user-1");
    expect(trace.project_id).toBe(projectId);
  }, 10_000);

  it("should merge observations and set negative tokens and cost to null", async () => {
    const modelId = randomUUID();
    const pricingTierId = randomUUID();
    const inputPriceId = randomUUID();
    const outputPriceId = randomUUID();
    const observationId = randomUUID();
    const traceId = randomUUID();

    await prisma.model.create({
      data: {
        id: modelId,
        projectId,
        modelName: "gpt-4o-mini-2024-07-18",
        matchPattern: "(?i)^(gpt-4o-mini-2024-07-18)$",
        startDate: new Date("2021-01-01T00:00:00.000Z"),
        unit: ModelUsageUnit.Tokens,
        tokenizerId: "openai",
        inputPrice: 0.00000015,
        outputPrice: 0.0000006,
        tokenizerConfig: {
          tokensPerName: 1,
          tokenizerModel: "gpt-4o",
          tokensPerMessage: 3,
        },
      },
    });

    await prisma.pricingTier.create({
      data: {
        id: pricingTierId,
        name: "Standard",
        conditions: [],
        isDefault: true,
        priority: 0,
        modelId: modelId,
      },
    });

    await prisma.price.create({
      data: {
        id: inputPriceId,
        pricingTierId,
        modelId,
        projectId: null,
        price: 0.00000015,
        usageType: "input",
      },
    });

    await prisma.price.create({
      data: {
        id: outputPriceId,
        pricingTierId,
        modelId,
        projectId: null,
        price: 0.0000006,
        usageType: "output",
      },
    });

    const observationEventList: ObservationEvent[] = [
      {
        id: randomUUID(),
        timestamp: "2024-11-04T16:13:51.496457Z",
        type: "generation-create",
        body: {
          traceId,
          name: "extract_location",
          startTime: "2024-11-04T16:13:51.495868Z",
          metadata: {
            ls_provider: "openai",
            ls_model_name: "gpt-4o-mini-2024-07-18",
            ls_model_type: "chat",
            ls_temperature: 0.4,
            ls_max_tokens: 1000,
          },
          input: "Sample input",
          id: observationId,
          model: "gpt-4o-mini-2024-07-18",
          modelParameters: {
            temperature: "0.4",
            max_tokens: 1000,
          },
          usage: null,
          environment,
        },
      },
      {
        id: randomUUID(),
        timestamp: "2024-11-04T16:13:52.156691Z",
        type: "generation-update",
        body: {
          traceId,
          output: "Sample output",
          id: observationId,
          endTime: "2024-11-04T16:13:52.156248Z",
          model: "gpt-4o-mini-2024-07-18",
          usage: {
            input: 4,
            output: -7,
            total: -3,
            unit: "TOKENS",
          },
          environment,
        },
      },
    ];

    await ingestionService.processObservationEventList({
      attribution: testIngestionAttribution,
      projectId,
      entityId: observationId,
      createdAtTimestamp: new Date(),
      observationEventList,
    });

    await clickhouseWriter.flushAll(true);

    const observation = await getClickhouseRecord(
      TableName.Observations,
      observationId,
    );

    expect(observation.name).toBe("extract_location");
    expect(observation.provided_usage_details).toStrictEqual({
      input: 4,
    });
    expect(observation.usage_details).toStrictEqual({
      input: 4,
      total: 4,
    });
    expect(observation.provided_cost_details).toStrictEqual({});
    expect(observation.cost_details).toStrictEqual({
      input: 0.0000006,
      total: 0.0000006,
    });
    expect(observation.total_cost).toBe(0.0000006);
  });

  it("should merge observations and calculate cost", async () => {
    const modelId = randomUUID();
    const pricingTierId = randomUUID();
    const inputPriceId = randomUUID();
    const outputPriceId = randomUUID();
    const observationId = randomUUID();
    const traceId = randomUUID();

    await prisma.model.create({
      data: {
        id: modelId,
        projectId,
        modelName: "gpt-4o-mini-2024-07-18",
        matchPattern: "(?i)^(gpt-4o-mini-2024-07-18)$",
        startDate: new Date("2021-01-01T00:00:00.000Z"),
        unit: ModelUsageUnit.Tokens,
        tokenizerId: "openai",
        inputPrice: 0.00000015,
        outputPrice: 0.0000006,
        tokenizerConfig: {
          tokensPerName: 1,
          tokenizerModel: "gpt-4o",
          tokensPerMessage: 3,
        },
      },
    });

    await prisma.pricingTier.create({
      data: {
        id: pricingTierId,
        name: "Standard",
        conditions: [],
        isDefault: true,
        priority: 0,
        modelId: modelId,
      },
    });

    await prisma.price.create({
      data: {
        id: inputPriceId,
        pricingTierId,
        modelId,
        projectId: null,
        price: 0.00000015,
        usageType: "input",
      },
    });

    await prisma.price.create({
      data: {
        id: outputPriceId,
        pricingTierId,
        modelId,
        projectId: null,
        price: 0.0000006,
        usageType: "output",
      },
    });

    const observationEventList: ObservationEvent[] = [
      {
        id: randomUUID(),
        timestamp: "2024-11-04T16:13:51.496457Z",
        type: "generation-create",
        body: {
          traceId,
          name: "extract_location",
          startTime: "2024-11-04T16:13:51.495868Z",
          metadata: {
            ls_provider: "openai",
            ls_model_name: "gpt-4o-mini-2024-07-18",
            ls_model_type: "chat",
            ls_temperature: 0.4,
            ls_max_tokens: 1000,
          },
          input: "Sample input",
          id: observationId,
          model: "gpt-4o-mini-2024-07-18",
          modelParameters: {
            temperature: "0.4",
            max_tokens: 1000,
          },
          usage: null,
          environment,
        },
      },
      {
        id: randomUUID(),
        timestamp: "2024-11-04T16:13:52.156691Z",
        type: "generation-update",
        body: {
          traceId,
          output: "Sample output",
          id: observationId,
          endTime: "2024-11-04T16:13:52.156248Z",
          model: "gpt-4o-mini-2024-07-18",
          usage: {
            input: 1295,
            output: 18,
            total: 1313,
            unit: "TOKENS",
          },
          environment,
        },
      },
    ];

    await ingestionService.processObservationEventList({
      attribution: testIngestionAttribution,
      projectId,
      entityId: observationId,
      createdAtTimestamp: new Date(),
      observationEventList,
    });

    await clickhouseWriter.flushAll(true);

    const observation = await getClickhouseRecord(
      TableName.Observations,
      observationId,
    );

    expect(observation.name).toBe("extract_location");
    expect(observation.provided_usage_details).toStrictEqual({
      input: 1295,
      output: 18,
      total: 1313,
    });
    expect(observation.usage_details).toStrictEqual({
      input: 1295,
      output: 18,
      total: 1313,
    });
    expect(observation.provided_cost_details).toStrictEqual({});
    expect(observation.cost_details).toStrictEqual({
      input: 0.00019425,
      output: 0.0000108,
      total: 0.00020505,
    });
    expect(observation.total_cost).toBe(0.00020505);
  });

  it("should merge observations from clickhouse and event list", async () => {
    const traceId = randomUUID();
    const observationId = randomUUID();

    const latestEvent = new Date();

    const observationEventList1: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "generation-create",
        timestamp: new Date().toISOString(),
        body: {
          id: observationId,
          traceId: traceId,
          startTime: new Date().toISOString(),
          output: "to overwrite",
          usage: undefined,
          environment,
        },
      },
    ];
    await ingestionService.processObservationEventList({
      attribution: testIngestionAttribution,
      projectId,
      entityId: observationId,
      createdAtTimestamp: new Date(),
      observationEventList: observationEventList1,
    });
    await clickhouseWriter.flushAll(true);

    const observationEventList2: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "generation-update",
        timestamp: new Date().toISOString(),
        body: {
          id: observationId,
          name: "generation-name",
          traceId: traceId,
          output: "overwritten",
          usage: undefined,
          environment,
        },
      },
    ];
    await ingestionService.processObservationEventList({
      attribution: testIngestionAttribution,
      projectId,
      entityId: observationId,
      createdAtTimestamp: new Date(),
      observationEventList: observationEventList2,
    });
    await clickhouseWriter.flushAll(true);

    const observation = await getClickhouseRecord(
      TableName.Observations,
      observationId,
    );

    expect(observation.name).toBe("generation-name");
    expect(observation.output).toBe("overwritten");
  });

  it("should put observation updates after creates if timestamp is same", async () => {
    const generationId = randomUUID();

    const timestamp = new Date().toISOString();

    const generationEventList: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "observation-update",
        timestamp,
        body: {
          id: generationId,
          type: "GENERATION",
          startTime: new Date().toISOString(),
          output: { key: "this is a great gpt output" },
          environment,
        },
      },
      {
        id: randomUUID(),
        type: "observation-create",
        timestamp,
        body: {
          id: generationId,
          type: "GENERATION",
          startTime: new Date().toISOString(),
          name: "generation-name",
          input: { key: "value" },
          output: "should be overwritten",
          model: "gpt-3.5",
          environment,
        },
      },
    ];

    await ingestionService.processObservationEventList({
      attribution: testIngestionAttribution,
      projectId,
      entityId: generationId,
      createdAtTimestamp: new Date(),
      observationEventList: generationEventList,
    });

    await clickhouseWriter.flushAll(true);

    const generation = await getClickhouseRecord(
      TableName.Observations,
      generationId,
    );

    expect(generation.output).toEqual(
      JSON.stringify({
        key: "this is a great gpt output",
      }),
    );
    expect(generation.input).toEqual(JSON.stringify({ key: "value" }));
    expect(generation.provided_model_name).toEqual("gpt-3.5");
    expect(generation.output).toEqual(
      JSON.stringify({ key: "this is a great gpt output" }),
    );
  });

  it("should correctly set tokens if usage provided as null", async () => {
    const generationId = randomUUID();
    const traceId = randomUUID();

    const timestamp = new Date().toISOString();
    const traceEventList: TraceEventType[] = [
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp,
        body: {
          id: traceId,
          timestamp,
          name: "trace-name",
          userId: "user-1",
          environment,
        },
      },
    ];

    const generationEventList1: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "observation-create",
        timestamp,
        body: {
          type: "GENERATION",
          id: generationId,
          traceId,
          startTime: new Date().toISOString(),
          name: "LiteLLM.run",
          // usage: null,
          environment,
        },
      },
    ];

    await Promise.all([
      ingestionService.processTraceEventList({
        projectId,
        entityId: traceId,
        createdAtTimestamp: new Date(),
        traceEventList,
      }),
      ingestionService.processObservationEventList({
        attribution: testIngestionAttribution,
        projectId,
        entityId: generationId,
        createdAtTimestamp: new Date(),
        observationEventList: generationEventList1,
      }),
    ]);

    await clickhouseWriter.flushAll(true);

    const generationEventList2: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "observation-update",
        timestamp,
        body: {
          traceId,
          type: "GENERATION",
          id: generationId,
          endTime: new Date().toISOString(),
          model: "azure/gpt35turbo0125",
          modelParameters: {
            model: "azure/gpt35turbo0125",
            max_tokens: 2000,
            temperature: 0,
            n: 1,
            stream: false,
          },
          usage: {
            input: 1285,
            output: 513,
            total: 1798,
            unit: "TOKENS" as any,
            inputCost: 0.0006425,
            outputCost: 0.0007695,
            totalCost: 0.001412,
          },
          environment,
        },
      },
    ];

    await ingestionService.processObservationEventList({
      attribution: testIngestionAttribution,
      projectId,
      entityId: generationId,
      createdAtTimestamp: new Date(),
      observationEventList: generationEventList2,
    });

    await clickhouseWriter.flushAll(true);

    const generation = await getClickhouseRecord(
      TableName.Observations,
      generationId,
    );

    expect(generation.usage_details.input).toEqual(1285);
    expect(generation.usage_details.output).toEqual(513);
    expect(generation.usage_details.total).toEqual(1798);

    expect(generation.provided_usage_details.input).toEqual(1285);
    expect(generation.provided_usage_details.output).toEqual(513);
    expect(generation.provided_usage_details.total).toEqual(1798);

    expect(generation.cost_details.input).toEqual(0.0006425);
    expect(generation.cost_details.output).toEqual(0.0007695);
    expect(generation.cost_details.total).toEqual(0.001412);

    expect(generation.provided_cost_details.input).toEqual(0.0006425);
    expect(generation.provided_cost_details.output).toEqual(0.0007695);
    expect(generation.provided_cost_details.total).toEqual(0.001412);
  });

  it("should update all token counts if update does not contain model name", async () => {
    const traceId = randomUUID();
    const generationId = randomUUID();

    await prisma.model.create({
      data: {
        projectId,
        modelName: "gpt-3.5",
        matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
        startDate: new Date("2021-01-01T00:00:00.000Z"),
        unit: ModelUsageUnit.Tokens,
        tokenizerId: "openai",
        tokenizerConfig: {
          tokensPerMessage: 3,
          tokensPerName: 1,
          tokenizerModel: "gpt-3.5-turbo",
        },
      },
    });

    const traceEventList: TraceEventType[] = [
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: traceId,
          name: "trace-name",
          timestamp: new Date().toISOString(),
          userId: "user-1",
          environment,
        },
      },
    ];

    const generationEventList: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "observation-create",
        timestamp: new Date().toISOString(),
        body: {
          id: generationId,
          traceId: traceId,
          type: "GENERATION",
          startTime: new Date().toISOString(),
          name: "generation-name",
          input: { key: "value" },
          model: "gpt-3.5",
          environment,
        },
      },
      {
        id: randomUUID(),
        type: "observation-update",
        timestamp: new Date().toISOString(),
        body: {
          id: generationId,
          type: "GENERATION",
          output: { key: "this is a great gpt output" },
          environment,
        },
      },
    ];

    await Promise.all([
      ingestionService.processTraceEventList({
        projectId,
        entityId: traceId,
        createdAtTimestamp: new Date(),
        traceEventList,
      }),
      ingestionService.processObservationEventList({
        attribution: testIngestionAttribution,
        projectId,
        entityId: generationId,
        createdAtTimestamp: new Date(),
        observationEventList: generationEventList,
      }),
    ]);

    await clickhouseWriter.flushAll(true);

    const trace = await getClickhouseRecord(TableName.Traces, traceId);

    expect(trace.name).toBe("trace-name");
    expect(trace.project_id).toBe(projectId);
    expect(trace.user_id).toBe("user-1");

    const generation = await getClickhouseRecord(
      TableName.Observations,
      generationId,
    );

    expect(generation?.output).toEqual(
      JSON.stringify({
        key: "this is a great gpt output",
      }),
    );
    expect(generation?.input).toEqual(JSON.stringify({ key: "value" }));
    expect(generation?.provided_model_name).toEqual("gpt-3.5");
    expect(generation?.output).toEqual(
      JSON.stringify({ key: "this is a great gpt output" }),
    );
    expect(generation?.usage_details.input).toEqual(5);
    expect(generation?.usage_details.output).toEqual(11);
  });

  it("should update all token counts if update does not contain model name and events come in wrong order", async () => {
    const traceId = randomUUID();
    const generationId = randomUUID();

    await prisma.model.create({
      data: {
        projectId,
        modelName: "gpt-3.5",
        matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
        startDate: new Date("2021-01-01T00:00:00.000Z"),
        unit: ModelUsageUnit.Tokens,
        tokenizerId: "openai",
        tokenizerConfig: {
          tokensPerMessage: 3,
          tokensPerName: 1,
          tokenizerModel: "gpt-3.5-turbo",
        },
      },
    });

    const traceEventList: TraceEventType[] = [
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: traceId,
          name: "trace-name",
          timestamp: new Date().toISOString(),
          userId: "user-1",
          environment,
        },
      },
    ];

    const generationEventList: ObservationEvent[] = [
      {
        id: randomUUID(),
        type: "observation-update",
        timestamp: new Date().toISOString(),
        body: {
          id: generationId,
          type: "GENERATION",
          output: { key: "this is a great gpt output" },
          environment,
        },
      },
      {
        id: randomUUID(),
        type: "observation-create",
        timestamp: new Date().toISOString(),
        body: {
          id: generationId,
          traceId: traceId,
          type: "GENERATION",
          startTime: new Date().toISOString(),
          name: "generation-name",
          input: { key: "value" },
          model: "gpt-3.5",
          environment,
        },
      },
    ];

    await Promise.all([
      ingestionService.processTraceEventList({
        projectId,
        entityId: traceId,
        createdAtTimestamp: new Date(),
        traceEventList,
      }),
      ingestionService.processObservationEventList({
        attribution: testIngestionAttribution,
        projectId,
        entityId: generationId,
        createdAtTimestamp: new Date(),
        observationEventList: generationEventList,
      }),
    ]);

    await clickhouseWriter.flushAll(true);

    const trace = await getClickhouseRecord(TableName.Traces, traceId);
    const observation = await getClickhouseRecord(
      TableName.Observations,
      generationId,
    );

    expect(observation?.output).toEqual(
      JSON.stringify({
        key: "this is a great gpt output",
      }),
    );
    expect(observation?.input).toEqual(JSON.stringify({ key: "value" }));
    expect(observation?.provided_model_name).toEqual("gpt-3.5");
    expect(observation?.output).toEqual(
      JSON.stringify({ key: "this is a great gpt output" }),
    );
    expect(observation?.usage_details.input).toEqual(5);
    expect(observation?.usage_details.output).toEqual(11);
  });

  [
    {
      inputs: [{ a: "a" }, { b: "b" }],
      output: { a: "a", b: "b" },
    },
    // The following two blocks are nice, but not critical for correct behaviour.
    // Stringifying them produces flaky tests, hence we skip them for now.
    // {
    //   inputs: [[{ a: "a" }], [{ b: "b" }]],
    //   output: { metadata: '[{"a":"a","b":"b"}]' },
    // },
    // {
    //   inputs: [
    //     {
    //       a: { "1": 1 },
    //     },
    //     {
    //       b: "b",
    //       a: { "2": 2 },
    //     },
    //   ],
    //   output: { a: '{ "1": 1, "2": 2 }', b: "b" },
    // },
    {
      inputs: [{ a: "a" }, undefined],
      output: { a: "a" },
    },
    {
      inputs: [undefined, { b: "b" }],
      output: { b: "b" },
    },
    {
      inputs: [{ bar: "baz" }, { foo: "bar" }],
      output: { foo: "bar", bar: "baz" },
    },
    {
      inputs: [{ foo: { bar: "baz" } }, { hello: "world" }],
      output: { foo: '{"bar":"baz"}', hello: "world" },
    },
  ].forEach(({ inputs, output }) => {
    it(`merges metadata ${JSON.stringify(inputs)}, ${JSON.stringify(output)}`, async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();

      const traceEventList: TraceEventType[] = [
        {
          id: randomUUID(),
          type: "trace-create",
          timestamp: new Date().toISOString(),
          body: {
            id: traceId,
            name: "trace-name",
            timestamp: new Date().toISOString(),
            userId: "user-1",
            metadata: inputs[0],
            environment,
          },
        },
        {
          id: randomUUID(),
          type: "trace-create",
          timestamp: new Date().toISOString(),
          body: {
            id: traceId,
            name: "trace-name",
            timestamp: new Date().toISOString(),
            metadata: inputs[1],
            environment,
          },
        },
      ];

      const generationEventList: ObservationEvent[] = [
        {
          id: randomUUID(),
          type: "observation-create",
          timestamp: new Date().toISOString(),
          body: {
            id: generationId,
            traceId: traceId,
            startTime: new Date().toISOString(),
            type: "GENERATION",
            name: "generation-name",
            metadata: inputs[0],
            environment,
          },
        },
        {
          id: randomUUID(),
          type: "observation-update",
          timestamp: new Date().toISOString(),
          body: {
            id: generationId,
            traceId: traceId,
            startTime: new Date().toISOString(),
            type: "GENERATION",
            metadata: inputs[1],
            environment,
          },
        },
      ];

      await Promise.all([
        ingestionService.processTraceEventList({
          projectId,
          entityId: traceId,
          createdAtTimestamp: new Date(),
          traceEventList,
        }),
        ingestionService.processObservationEventList({
          attribution: testIngestionAttribution,
          projectId,
          entityId: generationId,
          createdAtTimestamp: new Date(),
          observationEventList: generationEventList,
        }),
      ]);

      await clickhouseWriter.flushAll(true);

      const trace = await getClickhouseRecord(TableName.Traces, traceId);

      expect(trace.metadata).toEqual(output);

      const generation = await getClickhouseRecord(
        TableName.Observations,
        generationId,
      );

      expect(generation.metadata).toEqual(output);
    });
  });

  describe("Tiered Pricing", () => {
    it("applies an exact model parameter tier to direct event writes", async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();
      const modelId = randomUUID();
      const modelName = `priority-tier-${randomUUID()}`;

      await prisma.model.create({
        data: {
          id: modelId,
          projectId,
          modelName,
          matchPattern: `(?i)^(${modelName})$`,
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          pricingTiers: {
            create: [
              {
                name: "Standard",
                isDefault: true,
                priority: 0,
                conditions: [],
                prices: {
                  create: [
                    { usageType: "input", price: 0.000005, modelId },
                    { usageType: "output", price: 0.00003, modelId },
                  ],
                },
              },
              {
                name: "Priority",
                isDefault: false,
                priority: 1,
                conditions: [
                  {
                    source: "model_parameters",
                    key: "service_tier",
                    operator: "in",
                    values: ["priority"],
                  },
                ],
                prices: {
                  create: [
                    { usageType: "input", price: 0.0000125, modelId },
                    { usageType: "output", price: 0.000075, modelId },
                  ],
                },
              },
            ],
          },
        },
      });

      const eventRecord = await ingestionService.createEventRecord(
        {
          projectId,
          traceId,
          spanId: generationId,
          name: "priority-generation",
          type: "GENERATION",
          environment,
          startTimeISO: new Date().toISOString(),
          modelName,
          modelParameters: { service_tier: "priority" },
          metadata: {},
          providedUsageDetails: { input: 12, output: 21 },
          source: "otel",
        },
        `otel/${projectId}/priority-generation.json`,
      );

      expect(eventRecord.usage_pricing_tier_name).toBe("Priority");
      expect(eventRecord.cost_details.total).toBeCloseTo(0.001725, 9);
    });

    it("applies exact attribute tiers from a legacy usage event", async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();
      const modelId = randomUUID();
      const modelName = `legacy-priority-tier-${randomUUID()}`;

      await prisma.model.create({
        data: {
          id: modelId,
          projectId,
          modelName,
          matchPattern: `(?i)^(${modelName})$`,
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          pricingTiers: {
            create: [
              {
                name: "Standard",
                isDefault: true,
                priority: 0,
                conditions: [],
                prices: {
                  create: [
                    { usageType: "input", price: 0.000005, modelId },
                    { usageType: "output", price: 0.00003, modelId },
                  ],
                },
              },
              {
                name: "Priority US",
                isDefault: false,
                priority: 1,
                conditions: [
                  {
                    source: "model_parameters",
                    key: "service_tier",
                    operator: "in",
                    values: ["priority"],
                  },
                  {
                    source: "metadata",
                    key: "region",
                    operator: "in",
                    values: ["us"],
                  },
                ],
                prices: {
                  create: [
                    { usageType: "input", price: 0.0000125, modelId },
                    { usageType: "output", price: 0.000075, modelId },
                  ],
                },
              },
            ],
          },
        },
      });

      const observationEventList: ObservationEvent[] = [
        {
          id: randomUUID(),
          timestamp: new Date().toISOString(),
          type: "generation-create",
          body: {
            id: generationId,
            traceId,
            name: "legacy-priority-generation",
            startTime: new Date().toISOString(),
            model: modelName,
            environment,
          },
        },
        {
          id: randomUUID(),
          timestamp: new Date().toISOString(),
          type: "generation-update",
          body: {
            id: generationId,
            traceId,
            usage: {
              input: 12,
              output: 21,
              unit: ModelUsageUnit.Tokens,
            },
            modelParameters: { service_tier: "priority" },
            metadata: { region: "us" },
            environment,
          },
        },
      ];

      await ingestionService.processObservationEventList({
        attribution: testIngestionAttribution,
        projectId,
        entityId: generationId,
        createdAtTimestamp: new Date(),
        observationEventList,
      });
      await clickhouseWriter.flushAll(true);

      const generation = await getClickhouseRecord(
        TableName.Observations,
        generationId,
      );

      expect(generation.usage_pricing_tier_name).toBe("Priority US");
      expect(generation.cost_details.total).toBeCloseTo(0.001725, 9);
    });

    it("should apply default tier for usage below threshold (Anthropic Claude example)", async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();
      const modelId = randomUUID();
      const modelName = `claude-sonnet-4.5-${randomUUID()}`;

      // Create model with tiered pricing (Anthropic Claude pattern: $3/M tokens default, $6/M tokens >200K)
      await prisma.model.create({
        data: {
          id: modelId,
          projectId,
          modelName,
          matchPattern: `(?i)^(${modelName})$`,
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          pricingTiers: {
            create: [
              {
                name: "Standard",
                isDefault: true,
                priority: 0,
                conditions: [],
                prices: {
                  create: [
                    {
                      usageType: "input",
                      price: 0.000003, // $3/M tokens
                      modelId,
                    },
                    {
                      usageType: "output",
                      price: 0.000015, // $15/M tokens
                      modelId,
                    },
                  ],
                },
              },
              {
                name: "Large Context (>200K)",
                isDefault: false,
                priority: 1,
                conditions: [
                  {
                    usageDetailPattern: "^input",
                    operator: "gt",
                    value: 200000,
                    caseSensitive: false,
                  },
                ],
                prices: {
                  create: [
                    {
                      usageType: "input",
                      price: 0.000006, // $6/M tokens
                      modelId,
                    },
                    {
                      usageType: "output",
                      price: 0.000015, // $15/M tokens
                      modelId,
                    },
                  ],
                },
              },
            ],
          },
        },
      });

      const generationEventList: ObservationEvent[] = [
        {
          id: randomUUID(),
          type: "observation-create",
          timestamp: new Date().toISOString(),
          body: {
            id: generationId,
            traceId: traceId,
            type: "GENERATION",
            name: "generation-name",
            startTime: new Date().toISOString(),
            model: modelName,
            usage: {
              input: 100000, // Below 200K threshold
              output: 2000,
              unit: ModelUsageUnit.Tokens,
            },
            environment,
          },
        },
      ];

      await Promise.all([
        ingestionService.processTraceEventList({
          projectId,
          entityId: traceId,
          createdAtTimestamp: new Date(),
          traceEventList: [
            {
              id: randomUUID(),
              type: "trace-create",
              timestamp: new Date().toISOString(),
              body: {
                id: traceId,
                name: "trace-name",
                timestamp: new Date().toISOString(),
                environment,
              },
            },
          ],
        }),
        ingestionService.processObservationEventList({
          attribution: testIngestionAttribution,
          projectId,
          entityId: generationId,
          createdAtTimestamp: new Date(),
          observationEventList: generationEventList,
        }),
      ]);

      await clickhouseWriter.flushAll(true);

      const generation = await getClickhouseRecord(
        TableName.Observations,
        generationId,
      );

      expect(generation.internal_model_id).toBe(modelId);
      expect(generation.usage_details.input).toBe(100000);
      expect(generation.usage_details.output).toBe(2000);

      // Verify default tier was used
      expect(generation.usage_pricing_tier_name).toBe("Standard");
      expect(generation.usage_pricing_tier_id).toBeDefined();

      // Verify cost calculation with default tier prices ($3/M input, $15/M output)
      expect(generation.cost_details.input).toBeCloseTo(0.3, 6); // 100K * $3/M
      expect(generation.cost_details.output).toBeCloseTo(0.03, 6); // 2K * $15/M
      expect(generation.cost_details.total).toBeCloseTo(0.33, 6);
      expect(generation.total_cost).toBeCloseTo(0.33, 6);
    });

    it("should apply large context tier for usage above threshold (Anthropic Claude example)", async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();
      const modelId = randomUUID();
      const modelName = `claude-sonnet-4.5-test-${randomUUID()}`;

      // Create model with tiered pricing
      await prisma.model.create({
        data: {
          id: modelId,
          projectId,
          modelName,
          matchPattern: `(?i)^(${modelName})$`,
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          pricingTiers: {
            create: [
              {
                name: "Standard",
                isDefault: true,
                priority: 0,
                conditions: [],
                prices: {
                  create: [
                    {
                      usageType: "input",
                      price: 0.000003, // $3/M tokens
                      modelId,
                    },
                    {
                      usageType: "output",
                      price: 0.000015, // $15/M tokens
                      modelId,
                    },
                  ],
                },
              },
              {
                name: "Large Context (>200K)",
                isDefault: false,
                priority: 1,
                conditions: [
                  {
                    usageDetailPattern: "^input",
                    operator: "gt",
                    value: 200000,
                    caseSensitive: false,
                  },
                ],
                prices: {
                  create: [
                    {
                      usageType: "input",
                      price: 0.000006, // $6/M tokens
                      modelId,
                    },
                    {
                      usageType: "output",
                      price: 0.000015, // $15/M tokens
                      modelId,
                    },
                  ],
                },
              },
            ],
          },
        },
      });

      const generationEventList: ObservationEvent[] = [
        {
          id: randomUUID(),
          type: "observation-create",
          timestamp: new Date().toISOString(),
          body: {
            id: generationId,
            traceId: traceId,
            type: "GENERATION",
            name: "generation-name",
            startTime: new Date().toISOString(),
            model: modelName,
            usage: {
              input: 250000, // Above 200K threshold
              output: 2000,
              unit: ModelUsageUnit.Tokens,
            },
            environment,
          },
        },
      ];

      await Promise.all([
        ingestionService.processTraceEventList({
          projectId,
          entityId: traceId,
          createdAtTimestamp: new Date(),
          traceEventList: [
            {
              id: randomUUID(),
              type: "trace-create",
              timestamp: new Date().toISOString(),
              body: {
                id: traceId,
                name: "trace-name",
                timestamp: new Date().toISOString(),
                environment,
              },
            },
          ],
        }),
        ingestionService.processObservationEventList({
          attribution: testIngestionAttribution,
          projectId,
          entityId: generationId,
          createdAtTimestamp: new Date(),
          observationEventList: generationEventList,
        }),
      ]);

      await clickhouseWriter.flushAll(true);

      const generation = await getClickhouseRecord(
        TableName.Observations,
        generationId,
      );

      expect(generation.internal_model_id).toBe(modelId);
      expect(generation.usage_details.input).toBe(250000);
      expect(generation.usage_details.output).toBe(2000);

      // Verify large context tier was used
      expect(generation.usage_pricing_tier_name).toBe("Large Context (>200K)");
      expect(generation.usage_pricing_tier_id).toBeDefined();

      // Verify cost calculation with large context tier prices ($6/M input, $15/M output)
      expect(generation.cost_details.input).toBeCloseTo(1.5, 6); // 250K * $6/M
      expect(generation.cost_details.output).toBeCloseTo(0.03, 6); // 2K * $15/M
      expect(generation.cost_details.total).toBeCloseTo(1.53, 6);
      expect(generation.total_cost).toBeCloseTo(1.53, 6);
    });

    it("should match pattern with granular usage details (input_cached + input_regular)", async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();
      const modelId = randomUUID();
      const modelName = `claude-test-granular-${randomUUID()}`;

      // Create model with tiered pricing that sums all input* fields
      await prisma.model.create({
        data: {
          id: modelId,
          projectId,
          modelName,
          matchPattern: `(?i)^(${modelName})$`,
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          pricingTiers: {
            create: [
              {
                name: "Standard",
                isDefault: true,
                priority: 0,
                conditions: [],
                prices: {
                  create: [
                    {
                      usageType: "input",
                      price: 0.000003,
                      modelId,
                    },
                  ],
                },
              },
              {
                name: "Large Context (>200K)",
                isDefault: false,
                priority: 1,
                conditions: [
                  {
                    usageDetailPattern: "^input", // Matches input_cached + input_regular
                    operator: "gt",
                    value: 200000,
                    caseSensitive: false,
                  },
                ],
                prices: {
                  create: [
                    {
                      usageType: "input",
                      price: 0.000006,
                      modelId,
                    },
                  ],
                },
              },
            ],
          },
        },
      });

      const generationEventList: ObservationEvent[] = [
        {
          id: randomUUID(),
          type: "observation-create",
          timestamp: new Date().toISOString(),
          body: {
            id: generationId,
            traceId: traceId,
            type: "GENERATION",
            name: "generation-name",
            startTime: new Date().toISOString(),
            model: modelName,
            usageDetails: {
              input_cached: 150000,
              input_regular: 60000, // Total: 210K > 200K
            },
            environment,
          },
        },
      ];

      await Promise.all([
        ingestionService.processTraceEventList({
          projectId,
          entityId: traceId,
          createdAtTimestamp: new Date(),
          traceEventList: [
            {
              id: randomUUID(),
              type: "trace-create",
              timestamp: new Date().toISOString(),
              body: {
                id: traceId,
                name: "trace-name",
                timestamp: new Date().toISOString(),
                environment,
              },
            },
          ],
        }),
        ingestionService.processObservationEventList({
          attribution: testIngestionAttribution,
          projectId,
          entityId: generationId,
          createdAtTimestamp: new Date(),
          observationEventList: generationEventList,
        }),
      ]);

      await clickhouseWriter.flushAll(true);

      const generation = await getClickhouseRecord(
        TableName.Observations,
        generationId,
      );

      expect(generation.internal_model_id).toBe(modelId);

      // Verify large context tier was used (150K + 60K = 210K > 200K)
      expect(generation.usage_pricing_tier_name).toBe("Large Context (>200K)");
      expect(generation.usage_pricing_tier_id).toBeDefined();
    });

    it("should handle exactly at threshold boundary (200K tokens)", async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();
      const modelId = randomUUID();
      const modelName = `claude-boundary-${randomUUID()}`;

      await prisma.model.create({
        data: {
          id: modelId,
          projectId,
          modelName,
          matchPattern: `(?i)^(${modelName})$`,
          startDate: new Date("2021-01-01T00:00:00.000Z"),
          unit: ModelUsageUnit.Tokens,
          pricingTiers: {
            create: [
              {
                name: "Standard",
                isDefault: true,
                priority: 0,
                conditions: [],
                prices: {
                  create: [
                    {
                      usageType: "input",
                      price: 0.000003,
                      modelId,
                    },
                  ],
                },
              },
              {
                name: "Large Context (>200K)",
                isDefault: false,
                priority: 1,
                conditions: [
                  {
                    usageDetailPattern: "^input",
                    operator: "gt", // Strictly greater than
                    value: 200000,
                    caseSensitive: false,
                  },
                ],
                prices: {
                  create: [
                    {
                      usageType: "input",
                      price: 0.000006,
                      modelId,
                    },
                  ],
                },
              },
            ],
          },
        },
      });

      const generationEventList: ObservationEvent[] = [
        {
          id: randomUUID(),
          type: "observation-create",
          timestamp: new Date().toISOString(),
          body: {
            id: generationId,
            traceId: traceId,
            type: "GENERATION",
            name: "generation-name",
            startTime: new Date().toISOString(),
            model: modelName,
            usage: {
              input: 200000, // Exactly at threshold
              output: 100,
              unit: ModelUsageUnit.Tokens,
            },
            environment,
          },
        },
      ];

      await Promise.all([
        ingestionService.processTraceEventList({
          projectId,
          entityId: traceId,
          createdAtTimestamp: new Date(),
          traceEventList: [
            {
              id: randomUUID(),
              type: "trace-create",
              timestamp: new Date().toISOString(),
              body: {
                id: traceId,
                name: "trace-name",
                timestamp: new Date().toISOString(),
                environment,
              },
            },
          ],
        }),
        ingestionService.processObservationEventList({
          attribution: testIngestionAttribution,
          projectId,
          entityId: generationId,
          createdAtTimestamp: new Date(),
          observationEventList: generationEventList,
        }),
      ]);

      await clickhouseWriter.flushAll(true);

      const generation = await getClickhouseRecord(
        TableName.Observations,
        generationId,
      );

      // At exactly 200K, should use default tier (operator is "gt", not "gte")
      expect(generation.usage_pricing_tier_name).toBe("Standard");
      expect(generation.cost_details.input).toBeCloseTo(0.6, 6); // 200K * $3/M
    });
  });
});

async function getClickhouseRecord<T extends TableName>(
  tableName: T,
  entityId: string,
  options: { waitForRecord?: boolean } = {},
): Promise<RecordReadType<T>> {
  let result: unknown;
  const waitForRecord = options.waitForRecord ?? true;

  const loadResult = async () => {
    let query = await clickhouseClient().query({
      query: `SELECT * FROM ${tableName} FINAL WHERE project_id = '${projectId}' AND id = '${entityId}'`,
      format: "JSONEachRow",
    });

    result = (await query.json())[0];
  };

  if (waitForRecord) {
    await waitForExpect(async () => {
      await loadResult();
      expect(result).toBeDefined();
    }, 1_500);
  } else {
    await loadResult();
    expect(result).toBeDefined();
  }

  return (
    tableName === TableName.Traces
      ? traceRecordReadSchema.parse(result)
      : tableName === TableName.TracesNull
        ? traceRecordReadSchema.parse(result)
        : tableName === TableName.Observations
          ? observationRecordReadSchema.parse(result)
          : scoreRecordReadSchema.parse(result)
  ) as RecordReadType<T>;
}

async function getClickhouseObservationBatchStagingAttribution(
  entityId: string,
): Promise<{
  ingestion_api_key: string;
  ingestion_sdk_name: string;
  ingestion_sdk_version: string;
}> {
  let result: unknown;

  const loadResult = async () => {
    const query = await clickhouseClient().query({
      query: `
        SELECT
          ingestion_api_key,
          ingestion_sdk_name,
          ingestion_sdk_version
        FROM ${TableName.ObservationsBatchStaging} FINAL
        WHERE project_id = '${projectId}' AND id = '${entityId}'
      `,
      format: "JSONEachRow",
    });

    result = (await query.json())[0];
  };

  await waitForExpect(async () => {
    await loadResult();
    expect(result).toBeDefined();
  }, 1_500);

  return result as {
    ingestion_api_key: string;
    ingestion_sdk_name: string;
    ingestion_sdk_version: string;
  };
}

type RecordReadType<T extends TableName> = T extends TableName.Scores
  ? ScoreRecordReadType
  : T extends TableName.Observations
    ? ObservationRecordReadType
    : T extends TableName.Traces
      ? TraceRecordReadType
      : never;

async function isObservationsBatchStagingEnabled(): Promise<boolean> {
  return clickhouseTableExists(TableName.ObservationsBatchStaging);
}

async function skipUnlessObservationsBatchStagingEnabled(
  ctx: TestContext,
): Promise<void> {
  if (!(await isObservationsBatchStagingEnabled())) {
    ctx.skip("observations_batch_staging is not enabled in this ClickHouse");
  }
}