Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

75.3 KB
Raw Blame
/* eslint-disable no-nested-ternary */
import { Cluster, Redis } from "ioredis";
import { v4 } from "uuid";
import { Decimal } from "decimal.js";
import {
  InvalidRequestError,
  LangfuseNotFoundError,
  Model,
  MAX_SKILLS_PER_OBSERVATION,
  ObservationLevel,
  PrismaClient,
  Prompt,
  safeJsonParse,
  type JsonNested,
  type SkillsAvailable,
  type SkillsAvailableInput,
  type SkillsResourceLoaded,
} from "@langfuse/shared";
import {
  ClickhouseClientType,
  convertDateToClickhouseDateTime,
  toClickhouseDateTime,
  parseClickhouseUTCDateTimeFormat,
  convertObservationReadToInsert,
  convertScoreReadToInsert,
  convertTraceReadToInsert,
  convertTraceToStagingObservation,
  eventTypes,
  IngestionEntityTypes,
  IngestionEventType,
  instrumentAsync,
  logger,
  ObservationEvent,
  observationRecordInsertSchema,
  ObservationRecordInsertType,
  observationRecordReadSchema,
  PromptService,
  SkillCacheService,
  QueueJobs,
  recordIncrement,
  ScoreEventType,
  DatasetRunItemEventType,
  scoreRecordInsertSchema,
  ScoreRecordInsertType,
  scoreRecordReadSchema,
  TraceEventType,
  traceRecordInsertSchema,
  TraceRecordInsertType,
  traceRecordReadSchema,
  TraceUpsertQueue,
  UsageCostType,
  findModel,
  hasPricingTierUsageDetails,
  matchPricingTier,
  validateAndInflateScore,
  DatasetRunItemRecordInsertType,
  EventRecordInsertType,
  InternalTraceEventInput,
  traceException,
  flattenJsonToPathArrays,
  getDatasetItemById,
  normalizeToolsForObservation,
  hasNoEvalConfigsCache,
  buildClickHouseLogComment,
  sanitizeSdkMetricTagValue,
  type IngestionAttribution,
  type PricingTierMatchAttributes,
  AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME,
} from "@langfuse/shared/src/server";

import { tokenCountAsync } from "../../features/tokenisation/async-usage";
import { ClickhouseWriter, TableName } from "../ClickhouseWriter";
import {
  convertJsonSchemaToRecord,
  convertPostgresJsonToMetadataRecord,
  convertRecordValuesToString,
  hasAiGatewayScope,
  isAiGatewayGenerationEvent,
  overwriteObject,
} from "./utils";
import { resolveAiGatewayAttribution } from "./aiGatewayAttribution";
import { randomUUID } from "crypto";
import { SpanKind } from "@opentelemetry/api";
import { ClickhouseReadSkipCache } from "../../utils/clickhouseReadSkipCache";
import { applyObservationFieldOverflow } from "../../features/observation-field-overflow/processObservationFieldOverflow";

/**
 * Parse a value to a UInt16-compatible number (0–65535).
 * Returns undefined if the value is nullish or not a valid UInt16 integer.
 */
function parseUInt16(value: string | null | undefined): number | undefined {
  if (value == null) return undefined;
  const num = parseInt(value, 10);
  if (!Number.isInteger(num) || num < 0 || num > 65535) return undefined;
  return num;
}

function toPricingAttributeRecord(value: unknown): Record<string, string> {
  const parsedValue = typeof value === "string" ? safeJsonParse(value) : value;

  if (
    !parsedValue ||
    typeof parsedValue !== "object" ||
    Array.isArray(parsedValue)
  ) {
    return {};
  }

  return Object.fromEntries(
    Object.entries(parsedValue).flatMap(([key, attributeValue]) =>
      typeof attributeValue === "string" ||
      typeof attributeValue === "number" ||
      typeof attributeValue === "boolean"
        ? [[key, String(attributeValue)]]
        : [],
    ),
  );
}

export type EventInput = InternalTraceEventInput;

function parseEventModelParameters(
  value: EventInput["modelParameters"],
): string | object | number | boolean | null {
  if (!value) return {};
  if (typeof value !== "string") return value;

  try {
    return JSON.parse(value) as string | object | number | boolean | null;
  } catch {
    return value;
  }
}

type InsertRecord =
  | TraceRecordInsertType
  | ScoreRecordInsertType
  | ObservationRecordInsertType
  | DatasetRunItemRecordInsertType;
type MergeAndWriteParams = {
  eventType: IngestionEntityTypes;
  projectId: string;
  entityId: string;
  createdAtTimestamp: Date;
  events: IngestionEventType[];
  forwardToEventsTable: boolean;
  attribution: IngestionAttribution;
};

type PricingTierMatchAttributeValues = {
  modelParameters?: unknown;
  metadata?: unknown;
};

/**
 * Returns a new event record whose `event_bytes` value is the UTF-8 size of
 * the final JSONEachRow payload, excluding the self-referential accounting
 * field. The supplied record is not mutated.
 */
function withSerializedEventByteLength(
  eventRecord: EventRecordInsertType,
): EventRecordInsertType {
  const { event_bytes: _eventBytes, ...eventWithoutSize } = eventRecord;

  return {
    ...eventWithoutSize,
    event_bytes: Buffer.byteLength(JSON.stringify(eventWithoutSize), "utf8"),
  };
}

// Models priced before TTL-split cache writes existed (e.g. custom models)
// only carry an aggregate cache-write price; TTL-split usage falls back to it.
const TTL_CACHE_CREATION_USAGE_TYPES = new Set([
  "input_cache_creation_5m",
  "input_cache_creation_1h",
]);
const AGGREGATE_CACHE_CREATION_USAGE_TYPES = [
  "cache_creation_input_tokens",
  "input_cache_creation",
];

const immutableEntityKeys: {
  [TableName.Traces]: (keyof TraceRecordInsertType)[];
  [TableName.Scores]: (keyof ScoreRecordInsertType)[];
  [TableName.Observations]: (keyof ObservationRecordInsertType)[];
  [TableName.DatasetRunItems]: (keyof DatasetRunItemRecordInsertType)[];
} = {
  [TableName.Traces]: [
    "id",
    "project_id",
    "timestamp",
    "created_at",
    "environment",
  ],
  [TableName.Scores]: [
    "id",
    "project_id",
    "timestamp",
    "trace_id",
    "created_at",
    "environment",
  ],
  [TableName.Observations]: [
    "id",
    "project_id",
    "trace_id",
    "start_time",
    "created_at",
    "environment",
  ],
  // We do not accept updates, hence this list is currently not used.
  [TableName.DatasetRunItems]: [
    "id",
    "project_id",
    "dataset_run_id",
    "dataset_item_id",
    "dataset_id",
    "trace_id",
    "observation_id",
    "error",
    "created_at",
    "updated_at",
    "dataset_run_name",
    "dataset_run_description",
    "dataset_run_metadata",
    "dataset_run_created_at",
    "dataset_item_input",
    "dataset_item_expected_output",
    "dataset_item_metadata",
  ],
};

export class IngestionService {
  private promptService: PromptService;
  private skillCacheService: SkillCacheService;

  constructor(
    private redis: Redis | Cluster,
    private prisma: PrismaClient,
    private clickHouseWriter: ClickhouseWriter,
    private clickhouseClient: ClickhouseClientType,
  ) {
    this.promptService = new PromptService(prisma, redis);
    this.skillCacheService = new SkillCacheService(prisma, redis);
  }

  public async mergeAndWrite(params: MergeAndWriteParams): Promise<void> {
    const {
      eventType,
      projectId,
      entityId,
      createdAtTimestamp,
      events,
      forwardToEventsTable,
      attribution,
    } = params;

    logger.debug(
      `Merging ingestion ${eventType} event for project ${projectId} and event ${entityId}`,
    );

    switch (eventType) {
      case "trace":
        return await this.processTraceEventList({
          projectId,
          entityId,
          createdAtTimestamp,
          traceEventList: events as TraceEventType[],
          createEventTraceRecord: forwardToEventsTable,
          attribution,
        });
      case "observation":
        return await this.processObservationEventList({
          projectId,
          entityId,
          createdAtTimestamp,
          observationEventList: events as ObservationEvent[],
          writeToStagingTables: forwardToEventsTable,
          attribution,
        });
      case "score": {
        return await this.processScoreEventList({
          projectId,
          entityId,
          createdAtTimestamp,
          scoreEventList: events as ScoreEventType[],
          attribution,
        });
      }
      case "dataset_run_item": {
        return await this.processDatasetRunItemEventList({
          projectId,
          entityId,
          createdAtTimestamp,
          datasetRunItemEventList: events as DatasetRunItemEventType[],
        });
      }
    }
  }

  /**
   * Creates an EventRecordInsertType from EventInput.
   * Performs all necessary enrichments:
   * - Prompt lookup (by name + version)
   * - Model/usage enrichment (tokenization, cost calculation)
   * - Metadata flattening
   * - Timestamp normalization
   *
   * This is the single point of transformation from loose EventInput
   * to strict EventRecordInsertType.
   *
   * @param eventData - The event data from processToEvent()
   * @param fileKey - The file key where the raw event data is stored
   * @returns The enriched event record ready for writing or eval scheduling
   */
  public async createEventRecord(
    eventData: EventInput,
    fileKey: string,
  ): Promise<EventRecordInsertType> {
    logger.debug(
      `Creating event record for project ${eventData.projectId} and span ${eventData.spanId}`,
    );

    // processToEvent can keep normalized input/output as objects so tool data
    // can be extracted before write time. EventRecordInsertType stores both
    // fields as strings, so stringify at this schema boundary.
    const input = this.stringify(eventData.input);
    const output = this.stringify(eventData.output);
    const modelParameters = parseEventModelParameters(
      eventData.modelParameters,
    );
    const metadata = eventData.metadata ?? {};
    const spanAttribution = resolveAiGatewayAttribution({
      ingestionApiKey: eventData.ingestionApiKey,
      metadata,
      serviceName: eventData.serviceName,
      serviceVersion: eventData.serviceVersion,
      scopeVersion: eventData.scopeVersion,
    });

    // Runs outside the modelName gate below so model-less events with provided
    // usage are still checked.
    this.warnOnUsageTotalMismatch(
      eventData.providedUsageDetails ?? {},
      { id: eventData.spanId, project_id: eventData.projectId },
      "events",
    );

    const shouldEnrichUsageAndCost =
      Boolean(eventData.modelName) ||
      Object.keys(eventData.providedUsageDetails ?? {}).length > 0 ||
      Object.keys(eventData.providedCostDetails ?? {}).length > 0;

    // Enrich prompt, usage and skill references concurrently.
    const [prompt, generationUsage, skillReferences] = await Promise.all([
      // Lookup prompt by name and version
      eventData.promptName && eventData.promptVersion
        ? this.promptService.getPrompt({
            projectId: eventData.projectId,
            promptName: eventData.promptName,
            version:
              typeof eventData.promptVersion === "string"
                ? parseInt(eventData.promptVersion, 10)
                : eventData.promptVersion,
            label: undefined,
          })
        : null,
      // Promote provided usage/cost and enrich from the model when available.
      shouldEnrichUsageAndCost
        ? this.getGenerationUsage({
            projectId: eventData.projectId,
            pricingMatchAttributeValues: {
              modelParameters,
              metadata,
            },
            isAiGatewayGeneration: isAiGatewayGenerationEvent({
              ingestionApiKey: eventData.ingestionApiKey,
              hasGatewayScope: () =>
                eventData.scopeName === AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME,
            }),
            observationRecord: {
              id: eventData.spanId,
              project_id: eventData.projectId,
              trace_id: eventData.traceId,
              provided_model_name: eventData.modelName,
              provided_usage_details: eventData.providedUsageDetails ?? {},
              provided_cost_details: eventData.providedCostDetails ?? {},
              level: eventData.level,
              input,
              output,
            },
          })
        : null,
      this.resolveSkillReferences({
        projectId: eventData.projectId,
        type: eventData.type ?? "SPAN",
        available: eventData.skillsAvailable,
        loaded: eventData.skillsResourceLoaded,
      }),
    ]);

    const now = toClickhouseDateTime();

    // Flatten raw metadata first (before stringification destroys nested structure)
    const flattened = flattenJsonToPathArrays(metadata);
    const metadataNames = flattened.names;
    // Defensive: coerce null/undefined to empty string for Array(String) ClickHouse column.
    // Should not be required as convertValueToPlainJavascript() never returns null.
    const metadataValues = flattened.values.map((v) => v ?? "");
    const eventRecord: EventRecordInsertType = {
      // Required identifiers
      id: eventData.spanId,
      project_id: eventData.projectId,
      trace_id: eventData.traceId,
      span_id: eventData.spanId,

      // Optional identifiers
      parent_span_id: eventData.parentSpanId,

      // Core properties with defaults
      name: eventData.name ?? "",
      type: eventData.type ?? "SPAN",
      environment: eventData.environment ?? "default",
      version: eventData.version,
      release: eventData.release,

      tags: eventData.tags ?? [],
      bookmarked: eventData.bookmarked ?? false,
      public: eventData.public ?? false,
      is_app_root: eventData.isAppRoot ?? false,

      // Trace-level attributes: Name/User/session
      trace_name: eventData.traceName,
      user_id: eventData.userId,
      session_id: eventData.sessionId,

      // Status
      level: eventData.level ?? "DEFAULT",
      status_message: eventData.statusMessage,

      // Timestamps
      start_time: toClickhouseDateTime(eventData.startTimeISO),
      end_time: toClickhouseDateTime(eventData.endTimeISO),
      completion_start_time: eventData.completionStartTime
        ? toClickhouseDateTime(eventData.completionStartTime)
        : null,

      // Prompt
      prompt_id: prompt?.id || "",
      prompt_name: eventData.promptName,
      prompt_version: parseUInt16(eventData.promptVersion),
      skills_available: skillReferences.skills_available,
      skills_resource_loaded: skillReferences.skills_resource_loaded,

      // Model
      model_id: generationUsage?.internal_model_id || "",
      provided_model_name: eventData.modelName,
      // Evaluation scheduling consumes this enriched record before it is
      // written and expects parsed model parameters. The established runtime
      // contract is therefore wider than the ClickHouse insert type declares.
      model_parameters:
        modelParameters as EventRecordInsertType["model_parameters"],

      // Usage & Cost
      provided_usage_details: eventData.providedUsageDetails ?? {},
      usage_details:
        generationUsage?.usage_details ?? eventData.usageDetails ?? {},
      provided_cost_details: eventData.providedCostDetails ?? {},
      cost_details:
        generationUsage?.cost_details ?? eventData.costDetails ?? {},

      usage_pricing_tier_id: generationUsage?.usage_pricing_tier_id,
      usage_pricing_tier_name: generationUsage?.usage_pricing_tier_name,

      // Tool Calls
      tool_definitions: eventData.toolDefinitions ?? {},
      tool_calls: eventData.toolCalls ?? [],
      tool_call_names: eventData.toolCallNames ?? [],

      // I/O
      input,
      output,

      // Metadata
      metadata_names: metadataNames,
      metadata_values: metadataValues,
      evaluator_id: eventData.evaluationContext?.evaluatorId,
      evaluation_rule_id: eventData.evaluationContext?.evaluationRuleId,
      evaluator_execution_is_test:
        eventData.evaluationContext?.evaluatorExecutionIsTest,

      // Source/instrumentation metadata
      source: eventData.source,
      ingestion_api_key: spanAttribution.ingestionApiKey,
      ingestion_sdk_name: eventData.ingestionSdkName ?? "",
      ingestion_sdk_version: eventData.ingestionSdkVersion ?? "",
      service_name: spanAttribution.serviceName,
      service_version: spanAttribution.serviceVersion,
      scope_name: eventData.scopeName,
      scope_version: eventData.scopeVersion,
      telemetry_sdk_language: eventData.telemetrySdkLanguage,
      telemetry_sdk_name: eventData.telemetrySdkName,
      telemetry_sdk_version: eventData.telemetrySdkVersion,

      // Storage
      blob_storage_file_path: fileKey,
      event_bytes: eventData.eventBytes ?? 0,

      // Experiment fields
      experiment_id: eventData.experimentId,
      experiment_name: eventData.experimentName,
      experiment_metadata_names: eventData.experimentMetadataNames ?? [],
      experiment_metadata_values: eventData.experimentMetadataValues ?? [],
      experiment_description: eventData.experimentDescription,
      experiment_dataset_id: eventData.experimentDatasetId,
      experiment_item_id: eventData.experimentItemId,
      experiment_item_version: eventData.experimentItemVersion,
      experiment_item_root_span_id: eventData.experimentItemRootSpanId,
      experiment_item_expected_output: eventData.experimentItemExpectedOutput,
      experiment_item_metadata_names:
        eventData.experimentItemMetadataNames ?? [],
      experiment_item_metadata_values:
        eventData.experimentItemMetadataValues ?? [],

      // System timestamps
      created_at: now,
      updated_at: now,
      event_ts: now,
      is_deleted: 0,
    };

    return eventRecord;
  }

  /**
   * Writes an event record directly to the events_full table.
   * A materialized view auto-populates events_core from events_full.
   * Legacy observation writes and staging-based dual writes intentionally do
   * not use field overflow.
   * Use createEventRecord() first to get the record, then call this to write.
   *
   * Enqueues a new record whose `event_bytes` describes the final normalized
   * event rather than the raw OTEL span measured before media processing. The
   * supplied record is not mutated.
   *
   * @param eventRecord - The event record to write
   * @returns The serialized byte size of the accepted event record
   */
  public async writeEventRecord(
    eventRecord: EventRecordInsertType,
  ): Promise<number> {
    const persistedRecord = withSerializedEventByteLength(
      await applyObservationFieldOverflow(eventRecord),
    );

    this.clickHouseWriter.addToQueue(TableName.EventsFull, persistedRecord);
    return persistedRecord.event_bytes;
  }

  private async processDatasetRunItemEventList(params: {
    projectId: string;
    entityId: string;
    createdAtTimestamp: Date;
    datasetRunItemEventList: DatasetRunItemEventType[];
  }) {
    const { projectId, entityId, datasetRunItemEventList } = params;
    if (datasetRunItemEventList.length === 0) return;

    const finalDatasetRunItemRecords: DatasetRunItemRecordInsertType[] = (
      await Promise.all(
        datasetRunItemEventList.map(
          async (
            event: DatasetRunItemEventType,
          ): Promise<DatasetRunItemRecordInsertType[]> => {
            const [runData, itemData] = await Promise.all([
              this.prisma.datasetRuns.findFirst({
                where: {
                  id: event.body.runId,
                  datasetId: event.body.datasetId,
                  projectId,
                },
                select: {
                  name: true,
                  description: true,
                  metadata: true,
                  createdAt: true,
                },
              }),
              await getDatasetItemById({
                projectId,
                datasetItemId: event.body.datasetItemId,
                datasetId: event.body.datasetId,
                version: event.body.datasetVersion
                  ? new Date(event.body.datasetVersion)
                  : undefined,
                status: "ACTIVE",
              }),
            ]);

            if (!runData || !itemData) return [];

            const timestamp = toClickhouseDateTime(event.body.createdAt);

            const datasetItemVersion = itemData.validFrom
              ? toClickhouseDateTime(itemData.validFrom)
              : null;

            return [
              {
                id: entityId,
                project_id: projectId,
                dataset_run_id: event.body.runId,
                dataset_item_id: event.body.datasetItemId,
                dataset_id: event.body.datasetId,
                trace_id: event.body.traceId,
                observation_id: event.body.observationId,
                error: event.body.error,
                created_at: timestamp,
                updated_at: timestamp,
                event_ts: timestamp,
                is_deleted: 0,
                // enriched with run data
                dataset_run_name: runData.name,
                dataset_run_description: runData.description,
                dataset_run_metadata: runData.metadata
                  ? convertPostgresJsonToMetadataRecord(runData.metadata)
                  : {},
                dataset_run_created_at: toClickhouseDateTime(runData.createdAt),
                // enriched with item data
                dataset_item_version: datasetItemVersion,
                dataset_item_input: JSON.stringify(itemData.input),
                dataset_item_expected_output: JSON.stringify(
                  itemData.expectedOutput,
                ),
                dataset_item_metadata: itemData.metadata
                  ? convertPostgresJsonToMetadataRecord(itemData.metadata)
                  : {},
              },
            ];
          },
        ),
      )
    ).flat();

    finalDatasetRunItemRecords.forEach((record) => {
      if (record) {
        this.clickHouseWriter.addToQueue(TableName.DatasetRunItems, record);
      }
    });
  }

  private async processScoreEventList(params: {
    projectId: string;
    entityId: string;
    createdAtTimestamp: Date;
    scoreEventList: ScoreEventType[];
    attribution: IngestionAttribution;
  }) {
    const {
      projectId,
      entityId,
      createdAtTimestamp,
      scoreEventList,
      attribution,
    } = params;
    if (scoreEventList.length === 0) return;

    // Re-sending a score with the same id and timestamp is the documented way
    // to overwrite it, so ties must keep arrival order for the later one to
    // win. Score events are all creates, so no create-first rule is needed.
    const timeSortedEvents = scoreEventList
      .slice()
      .sort(
        (a, b) =>
          new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(),
      );

    const minTimestamp = Math.min(
      ...timeSortedEvents.flatMap((e) =>
        e.timestamp ? [new Date(e.timestamp).getTime()] : [],
      ),
    );
    const timestamp =
      minTimestamp === Infinity
        ? undefined
        : convertDateToClickhouseDateTime(new Date(minTimestamp));
    const unexpectedScoreValidationErrors: unknown[] = [];
    let expectedScoreValidationDrops = 0;
    const [clickhouseScoreRecord, scoreRecords] = await Promise.all([
      this.getClickhouseRecord({
        projectId,
        entityId,
        table: TableName.Scores,
        additionalFilters: {
          whereCondition: timestamp
            ? " AND timestamp >= {timestamp: DateTime64(3)} "
            : "",
          params: { timestamp },
        },
      }),
      Promise.all(
        timeSortedEvents.map(async (scoreEvent) => {
          try {
            const validatedScore = await validateAndInflateScore({
              body: scoreEvent.body,
              scoreId: entityId,
              projectId,
            });
            const rawMetadata = scoreEvent.body.metadata
              ? convertJsonSchemaToRecord(scoreEvent.body.metadata)
              : {};
            const evaluationFields =
              scoreEvent.body as ScoreEventType["body"] & {
                evaluatorId?: string;
                evaluationRuleId?: string;
              };
            return {
              id: entityId,
              project_id: projectId,
              environment: validatedScore.environment,
              timestamp: toClickhouseDateTime(scoreEvent.timestamp),
              name: validatedScore.name,
              value: validatedScore.value,
              source: validatedScore.source,
              trace_id: validatedScore.traceId,
              session_id: validatedScore.sessionId,
              dataset_run_id: validatedScore.datasetRunId,
              data_type: validatedScore.dataType,
              observation_id: validatedScore.observationId,
              config_id: validatedScore.configId,
              comment: validatedScore.comment,
              metadata: convertRecordValuesToString(rawMetadata),
              evaluator_id: evaluationFields.evaluatorId,
              evaluation_rule_id: evaluationFields.evaluationRuleId,
              string_value: validatedScore.stringValue,
              long_string_value: validatedScore.longStringValue,
              execution_trace_id: validatedScore.executionTraceId,
              queue_id: validatedScore.queueId ?? null,
              ingestion_api_key: attribution.ingestionApiKey,
              ingestion_sdk_name: attribution.ingestionSdkName,
              ingestion_sdk_version: attribution.ingestionSdkVersion,
              created_at: toClickhouseDateTime(),
              updated_at: toClickhouseDateTime(),
              event_ts: toClickhouseDateTime(scoreEvent.timestamp),
              is_deleted: 0,
            };
            // Gracefully handle any score schema validation errors, skip the score insert and reject silently.
          } catch (error) {
            if (
              error instanceof InvalidRequestError ||
              error instanceof LangfuseNotFoundError
            ) {
              expectedScoreValidationDrops += 1;
            } else {
              unexpectedScoreValidationErrors.push(error);
            }

            logger.info(
              `Failed to validate and enrich score body for project: ${projectId} and score: ${entityId}`,
              error,
            );
            return null;
          }
        }),
      ).then((results) =>
        results.filter(
          (record): record is NonNullable<typeof record> => record !== null,
        ),
      ),
    ]);

    if (clickhouseScoreRecord) {
      recordIncrement("langfuse.ingestion.lookup.hit", 1, {
        store: "clickhouse",
        object: "score",
      });
    }

    if (unexpectedScoreValidationErrors.length > 0) {
      throw new AggregateError(
        unexpectedScoreValidationErrors,
        `Unexpected error(s) validating score batch for project: ${projectId} and score: ${entityId}`,
      );
    }

    // Count drops only on an attempt that proceeds: a rethrowing batch is
    // redelivered, and counting it would inflate the metric once per retry.
    for (let i = 0; i < expectedScoreValidationDrops; i++) {
      recordIncrement("langfuse.ingestion.metadata_dropped", 1, {
        reason: "score_validation_dropped",
        source: "api",
        domain: "score",
        projectId,
        sdkName: sanitizeSdkMetricTagValue(attribution.ingestionSdkName),
        sdkVersion: sanitizeSdkMetricTagValue(attribution.ingestionSdkVersion),
      });
    }

    if (scoreRecords.length === 0 && !clickhouseScoreRecord) {
      logger.warn(
        `No valid score records found for project: ${projectId} and score: ${entityId}`,
      );
      return;
    }

    // Update = merging new events with a pre-existing record, found either in
    // ClickHouse or as earlier event-log files for the same score id.
    if (
      scoreRecords.length > 0 &&
      (clickhouseScoreRecord || scoreRecords.length > 1)
    ) {
      recordIncrement("langfuse.ingestion.score_update", 1, {
        store: clickhouseScoreRecord ? "clickhouse" : "event_log",
      });
    }

    const finalScoreRecord: ScoreRecordInsertType =
      await this.mergeScoreRecords({
        clickhouseScoreRecord,
        scoreRecords,
      });
    finalScoreRecord.created_at =
      clickhouseScoreRecord?.created_at ??
      toClickhouseDateTime(createdAtTimestamp);

    this.clickHouseWriter.addToQueue(TableName.Scores, finalScoreRecord);
  }

  private async processTraceEventList(params: {
    projectId: string;
    entityId: string;
    createdAtTimestamp: Date;
    traceEventList: TraceEventType[];
    createEventTraceRecord: boolean;
    attribution: IngestionAttribution;
  }) {
    const {
      projectId,
      entityId,
      createdAtTimestamp,
      traceEventList,
      createEventTraceRecord,
      attribution,
    } = params;
    if (traceEventList.length === 0) return;

    const timeSortedEvents =
      IngestionService.toTimeSortedEventList(traceEventList);

    const traceRecords = this.mapTraceEventsToRecords({
      projectId,
      entityId,
      traceEventList: timeSortedEvents,
    });

    // Search for the first non-null input and output in the trace events and set them on the merged result.
    // Fallback to the ClickHouse input/output if none are found within the events list.
    const reversedRawRecords = timeSortedEvents.slice().reverse();
    const finalIO = {
      input: this.stringify(
        reversedRawRecords.find((record) => record?.body?.input)?.body?.input,
      ),
      output: this.stringify(
        reversedRawRecords.find((record) => record?.body?.output)?.body?.output,
      ),
    };

    const minTimestamp = Math.min(
      ...timeSortedEvents.flatMap((e) =>
        e.body?.timestamp ? [new Date(e.body.timestamp).getTime()] : [],
      ),
    );
    const timestamp =
      minTimestamp === Infinity
        ? undefined
        : convertDateToClickhouseDateTime(new Date(minTimestamp));
    const clickhouseTraceRecord = await this.getClickhouseRecord({
      projectId,
      entityId,
      table: TableName.Traces,
      additionalFilters: {
        whereCondition: timestamp
          ? " AND timestamp >= {timestamp: DateTime64(3)} "
          : "",
        params: { timestamp },
      },
    });

    if (clickhouseTraceRecord) {
      recordIncrement("langfuse.ingestion.lookup.hit", 1, {
        store: "clickhouse",
        object: "trace",
      });
    }

    const finalTraceRecord = await this.mergeTraceRecords({
      clickhouseTraceRecord,
      traceRecords,
    });
    finalTraceRecord.created_at =
      clickhouseTraceRecord?.created_at ??
      toClickhouseDateTime(createdAtTimestamp);

    finalTraceRecord.input = finalIO.input ?? clickhouseTraceRecord?.input;
    finalTraceRecord.output = finalIO.output ?? clickhouseTraceRecord?.output;

    this.clickHouseWriter.addToQueue(TableName.Traces, finalTraceRecord);

    // If the trace has a sessionId, we upsert the corresponding session into Postgres.
    const traceRecordWithSession = traceRecords
      .slice()
      .reverse()
      .find((t) => t.session_id);
    if (traceRecordWithSession) {
      try {
        await this.prisma.$executeRaw`
          INSERT INTO trace_sessions (id, project_id, environment, created_at, updated_at)
          VALUES (${traceRecordWithSession.session_id}, ${projectId}, ${traceRecordWithSession.environment}, NOW(), NOW())
          ON CONFLICT (id, project_id)
          DO NOTHING
        `;
      } catch (e) {
        logger.error(
          `Failed to upsert session ${traceRecordWithSession.session_id}`,
          e,
        );
        throw e;
      }
    }

    // Dual-write to staging table for batch propagation to events table
    // We pretend the trace is a "span" where span_id = trace_id
    if (createEventTraceRecord) {
      const traceAsStagingObservation = convertTraceToStagingObservation(
        finalTraceRecord,
        this.getPartitionAwareTimestamp(createdAtTimestamp),
        attribution,
      );
      this.clickHouseWriter.addToQueue(
        TableName.ObservationsBatchStaging,
        traceAsStagingObservation,
      );
    }

    // Add trace into trace upsert queue for eval processing
    // First check if we already know this project has no job configurations
    const hasNoJobConfigs = await hasNoEvalConfigsCache(
      projectId,
      "traceBased",
    );
    if (hasNoJobConfigs) {
      logger.debug(
        `Skipping TraceUpsert queue for project ${projectId} - no job configs cached`,
      );
      return;
    }

    // Job configs present, so we add to the TraceUpsert queue.
    const shardingKey = `${projectId}-${entityId}`;
    const traceUpsertQueue = TraceUpsertQueue.getInstance({ shardingKey });
    if (!traceUpsertQueue) {
      logger.error("TraceUpsertQueue is not initialized");
      return;
    }
    await traceUpsertQueue.add(QueueJobs.TraceUpsert, {
      payload: {
        projectId,
        traceId: entityId,
        exactTimestamp: parseClickhouseUTCDateTimeFormat(
          finalTraceRecord.timestamp,
        ),
        traceEnvironment: finalTraceRecord.environment,
      },
      id: randomUUID(),
      timestamp: new Date(),
      name: QueueJobs.TraceUpsert as const,
    });
  }

  private async processObservationEventList(params: {
    projectId: string;
    entityId: string;
    createdAtTimestamp: Date;
    observationEventList: ObservationEvent[];
    writeToStagingTables: boolean;
    attribution: IngestionAttribution;
  }) {
    const {
      projectId,
      entityId,
      createdAtTimestamp,
      observationEventList,
      writeToStagingTables,
      attribution,
    } = params;
    if (observationEventList.length === 0) return;

    const timeSortedEvents =
      IngestionService.toTimeSortedEventList(observationEventList);

    const type = this.getObservationType(observationEventList[0]);
    const minStartTime = Math.min(
      ...observationEventList.flatMap((e) =>
        e.body?.startTime ? [new Date(e.body.startTime).getTime()] : [],
      ),
    );
    const startTime =
      minStartTime === Infinity
        ? undefined
        : convertDateToClickhouseDateTime(new Date(minStartTime));

    const [clickhouseObservationRecord, prompt] = await Promise.all([
      this.getClickhouseRecord({
        projectId,
        entityId,
        table: TableName.Observations,
        additionalFilters: {
          whereCondition: `AND type = {type: String} ${startTime ? "AND start_time >= {startTime: DateTime64(3)} " : ""}`,
          params: {
            type,
            startTime,
          },
        },
      }),
      this.getPrompt(projectId, observationEventList),
    ]);

    if (clickhouseObservationRecord) {
      recordIncrement("langfuse.ingestion.lookup.hit", 1, {
        store: "clickhouse",
        object: "observation",
      });
    }

    const observationRecords = this.mapObservationEventsToRecords({
      observationEventList: timeSortedEvents,
      projectId,
      entityId,
      prompt,
    });

    const mergedObservationRecord = await this.mergeObservationRecords({
      projectId,
      observationRecords,
      clickhouseObservationRecord,
    });
    mergedObservationRecord.created_at =
      clickhouseObservationRecord?.created_at ??
      toClickhouseDateTime(createdAtTimestamp);
    mergedObservationRecord.level = mergedObservationRecord.level ?? "DEFAULT";

    // Search for the first non-null input and output in the observation events and set them on the merged result.
    // Fallback to the ClickHouse input/output if none are found within the events list.
    const reversedRawRecords = timeSortedEvents.slice().reverse();
    const rawInput =
      reversedRawRecords.find((record) => record?.body?.input)?.body?.input ??
      clickhouseObservationRecord?.input;
    const rawOutput =
      reversedRawRecords.find((record) => record?.body?.output)?.body?.output ??
      clickhouseObservationRecord?.output;
    const pricingEventIndex = observationRecords.findLastIndex(
      (record) => Object.keys(record.provided_usage_details ?? {}).length > 0,
    );
    const pricingEvent = timeSortedEvents[pricingEventIndex];
    const pricingMatchAttributeValues:
      | PricingTierMatchAttributeValues
      | undefined = pricingEvent
      ? {
          modelParameters:
            "modelParameters" in pricingEvent.body
              ? pricingEvent.body.modelParameters
              : undefined,
          metadata: pricingEvent.body.metadata,
        }
      : undefined;
    const normalizedTools = normalizeToolsForObservation(
      rawInput,
      rawOutput,
      mergedObservationRecord.metadata,
    );

    mergedObservationRecord.input = this.stringify(normalizedTools.input);
    mergedObservationRecord.output = this.stringify(normalizedTools.output);
    const normalizedMetadata = normalizedTools.metadata ?? {};
    mergedObservationRecord.metadata =
      normalizedMetadata &&
      typeof normalizedMetadata === "object" &&
      !Array.isArray(normalizedMetadata)
        ? convertRecordValuesToString(
            normalizedMetadata as Record<string, unknown>,
          )
        : convertJsonSchemaToRecord(normalizedMetadata as JsonNested);

    if (Object.keys(normalizedTools.toolDefinitions).length > 0) {
      mergedObservationRecord.tool_definitions =
        normalizedTools.toolDefinitions;
    }

    if (normalizedTools.toolCalls.length > 0) {
      mergedObservationRecord.tool_calls = normalizedTools.toolCalls;
      mergedObservationRecord.tool_call_names = normalizedTools.toolCallNames;
    }

    // Only check when the incoming events carry usage themselves — partial
    // updates merge stale usage back in from ClickHouse and must not re-fire.
    if (
      observationRecords.some(
        (record) => Object.keys(record.provided_usage_details ?? {}).length > 0,
      )
    ) {
      this.warnOnUsageTotalMismatch(
        mergedObservationRecord.provided_usage_details ?? {},
        mergedObservationRecord,
        "legacy",
      );
    }

    const generationUsage = await this.getGenerationUsage({
      projectId,
      pricingMatchAttributeValues,
      isAiGatewayGeneration: isAiGatewayGenerationEvent({
        ingestionApiKey: attribution.ingestionApiKey,
        hasGatewayScope: () =>
          timeSortedEvents.some((event) =>
            hasAiGatewayScope(event.body?.metadata),
          ),
      }),
      observationRecord: mergedObservationRecord,
    });
    const finalObservationRecord = {
      ...mergedObservationRecord,
      ...generationUsage,
    };

    // Backward compat: create wrapper trace for SDK < 2.0.0 events that do not have a traceId
    if (!finalObservationRecord.trace_id) {
      const wrapperTraceRecord: TraceRecordInsertType = {
        id: finalObservationRecord.id,
        timestamp: finalObservationRecord.start_time,
        project_id: projectId,
        environment: finalObservationRecord.environment,
        created_at: toClickhouseDateTime(),
        updated_at: toClickhouseDateTime(),
        metadata: {},
        tags: [],
        bookmarked: false,
        public: false,
        event_ts: toClickhouseDateTime(),
        is_deleted: 0,
      };

      this.clickHouseWriter.addToQueue(TableName.Traces, wrapperTraceRecord);
      finalObservationRecord.trace_id = finalObservationRecord.id;
    }

    this.clickHouseWriter.addToQueue(
      TableName.Observations,
      finalObservationRecord,
    );

    // Dual-write to staging table for batch propagation to events table
    // Here, we add some additional logic around the first seen timestamp.
    // We "lock" partitions 4min after their creation, i.e. the 15:00:00 partition
    // should stop receiving updates at 15:04:00.
    // This means that we keep the createdAtTimestamp as-is if it is within the last
    // 3.5 minutes (incl. a 30s buffer around writes) and otherwise,
    // we set the current timestamp for the event.
    if (writeToStagingTables) {
      const stagingRecord = {
        ...finalObservationRecord,
        ingestion_api_key: attribution.ingestionApiKey,
        ingestion_sdk_name: attribution.ingestionSdkName,
        ingestion_sdk_version: attribution.ingestionSdkVersion,
        s3_first_seen_timestamp:
          this.getPartitionAwareTimestamp(createdAtTimestamp),
      };
      this.clickHouseWriter.addToQueue(
        TableName.ObservationsBatchStaging,
        stagingRecord,
      );
    }
  }

  private async mergeScoreRecords(params: {
    scoreRecords: ScoreRecordInsertType[];
    clickhouseScoreRecord?: ScoreRecordInsertType | null;
  }): Promise<ScoreRecordInsertType> {
    const { scoreRecords, clickhouseScoreRecord } = params;

    // Set clickhouse first as this is the baseline for immutable fields
    const recordsToMerge = [clickhouseScoreRecord, ...scoreRecords].filter(
      Boolean,
    ) as ScoreRecordInsertType[];

    const mergedRecord = this.mergeRecords(
      recordsToMerge,
      immutableEntityKeys[TableName.Scores],
    );

    // If metadata exists, it is an object due to previous parsing
    mergedRecord.metadata = convertRecordValuesToString(
      (mergedRecord.metadata as Record<string, unknown>) ?? {},
    );

    return scoreRecordInsertSchema.parse(mergedRecord);
  }

  private async mergeTraceRecords(params: {
    traceRecords: TraceRecordInsertType[];
    clickhouseTraceRecord?: TraceRecordInsertType | null;
  }): Promise<TraceRecordInsertType> {
    const { traceRecords, clickhouseTraceRecord } = params;

    // Set clickhouse first as this is the baseline for immutable fields
    const recordsToMerge = [clickhouseTraceRecord, ...traceRecords].filter(
      Boolean,
    ) as TraceRecordInsertType[];

    const mergedRecord = this.mergeRecords(
      recordsToMerge,
      immutableEntityKeys[TableName.Traces],
    );

    // If metadata exists, it is an object due to previous parsing
    mergedRecord.metadata = convertRecordValuesToString(
      (mergedRecord.metadata as Record<string, unknown>) ?? {},
    );

    return traceRecordInsertSchema.parse(mergedRecord);
  }

  private async mergeObservationRecords(params: {
    projectId: string;
    observationRecords: ObservationRecordInsertType[];
    clickhouseObservationRecord?: ObservationRecordInsertType | null;
  }): Promise<ObservationRecordInsertType> {
    const { observationRecords, clickhouseObservationRecord } = params;

    // Set clickhouse first as this is the baseline for immutable fields
    const recordsToMerge = [
      clickhouseObservationRecord,
      ...observationRecords,
    ].filter(Boolean) as ObservationRecordInsertType[];

    const mergedRecord = this.mergeRecords(
      recordsToMerge,
      immutableEntityKeys[TableName.Observations],
    );

    // If metadata exists, it is an object due to previous parsing
    mergedRecord.metadata = convertRecordValuesToString(
      (mergedRecord.metadata as Record<string, unknown>) ?? {},
    );

    const parsedObservationRecord =
      observationRecordInsertSchema.parse(mergedRecord);

    // Override endTimes that are before startTimes with the startTime
    if (
      parsedObservationRecord.end_time &&
      parsedObservationRecord.end_time < parsedObservationRecord.start_time
    ) {
      parsedObservationRecord.end_time = parsedObservationRecord.start_time;
    }

    return parsedObservationRecord;
  }

  private async resolveSkillReferences({
    projectId,
    type,
    available = [],
    loaded = [],
  }: {
    projectId: string;
    type: EventRecordInsertType["type"];
    available?: SkillsAvailableInput;
    loaded?: SkillsResourceLoaded;
  }): Promise<{
    skills_available: SkillsAvailable;
    skills_resource_loaded: SkillsResourceLoaded;
  }> {
    // Bound internal callers too, before building the lookup or its cache key.
    if (
      available.length > MAX_SKILLS_PER_OBSERVATION ||
      loaded.length > MAX_SKILLS_PER_OBSERVATION
    ) {
      logger.warn("Skill references truncated: item limit exceeded", {
        projectId,
        availableCount: available.length,
        loadedCount: loaded.length,
        limit: MAX_SKILLS_PER_OBSERVATION,
      });
    }
    available = available.slice(0, MAX_SKILLS_PER_OBSERVATION);
    loaded = loaded.slice(0, MAX_SKILLS_PER_OBSERVATION);
    const versions = available.flatMap(({ name, version }) =>
      version == null ? [] : [{ name, version }],
    );
    const skills = await this.skillCacheService.getReferences({
      projectId,
      versions,
    });

    const availableSkills = available.map(({ name, version }) => {
      const skill = skills.find(
        (skill) => skill.name === name && skill.version === version,
      );
      return {
        name,
        version: skill?.version ?? version,
        langfuseSkillId: skill?.id,
      };
    });

    const loadedResources = loaded.map(
      ({ skillName, filePath, skillVersion }) => {
        const candidates = availableSkills.filter(
          (skill) => skill.name === skillName,
        );
        if (candidates.length !== 1)
          return { skillName, filePath, skillVersion };

        const skill = candidates[0];
        return {
          skillName,
          filePath,
          langfuseSkillId: skill?.langfuseSkillId,
          skillVersion: skill.version ?? skillVersion,
        };
      },
    );

    return {
      skills_available:
        type === "GENERATION"
          ? availableSkills.map(({ name, version, langfuseSkillId }) => ({
              skillName: name,
              skillVersion: version,
              langfuseSkillId,
            }))
          : [],
      skills_resource_loaded: loadedResources,
    };
  }

  private mergeRecords<T extends InsertRecord>(
    records: T[],
    immutableEntityKeys: string[],
  ): Record<string, unknown> {
    if (records.length === 0) {
      throw new Error("No records to merge");
    }

    let result: {
      id: string;
      project_id: string;
      [key: string]: any;
    } = { id: records[0].id, project_id: records[0].project_id };

    for (const record of records) {
      result = overwriteObject(result, record, immutableEntityKeys);
    }

    result.event_ts = toClickhouseDateTime();

    return result;
  }

  private static toTimeSortedEventList<
    T extends TraceEventType | ObservationEvent,
  >(eventList: T[]): T[] {
    // The merge folds this list left to right with a last-wins overwrite, so
    // the winner for any group is whichever event sorts last. Events arrive in
    // upload order (the queue processor lists files by upload time), captured
    // here as the arrival index and used as the tie-break so the ordering is a
    // total order rather than relying on the sort engine's handling of ties.
    return eventList
      .map((event, index) => ({ event, index }))
      .sort((a, b) => {
        const aTimestamp = new Date(a.event.timestamp).getTime();
        const bTimestamp = new Date(b.event.timestamp).getTime();
        if (aTimestamp !== bTimestamp) return aTimestamp - bTimestamp;

        const aIsCreate = a.event.type.includes("create");
        const bIsCreate = b.event.type.includes("create");
        // Creates sort before updates, so an update at the same timestamp wins.
        if (aIsCreate !== bIsCreate) return aIsCreate ? -1 : 1;

        // Tied creates: the first-arriving one wins. OTel emits a root span's
        // trace-create and a child span's trace-create carrying explicit
        // update_current_trace() values at the same span start time; the child
        // arrives first and its explicit fields must beat the root's derived
        // ones. Placing the earlier arrival last makes it win the merge.
        if (aIsCreate) return b.index - a.index;

        // Tied updates: the last-arriving one wins.
        return a.index - b.index;
      })
      .map(({ event }) => event);
  }

  private async getPrompt(
    projectId: string,
    observationEventList: ObservationEvent[],
  ): Promise<ObservationPrompt | null> {
    const lastObservationWithPromptInfo = observationEventList
      .slice()
      .reverse()
      .find(this.hasPromptInformation);

    if (!lastObservationWithPromptInfo) return null;

    const { promptName, promptVersion: version } =
      lastObservationWithPromptInfo.body;

    return this.promptService.getPrompt({
      projectId,
      promptName,
      version,
      label: undefined,
    });
  }

  private hasPromptInformation(
    event: ObservationEvent,
  ): event is ObservationEvent & {
    body: { promptName: string; promptVersion: number };
  } {
    return (
      "promptName" in event.body &&
      typeof event.body.promptName === "string" &&
      "promptVersion" in event.body &&
      typeof event.body.promptVersion === "number"
    );
  }

  private async getGenerationUsage(params: {
    projectId: string;
    pricingMatchAttributeValues?: PricingTierMatchAttributeValues;
    isAiGatewayGeneration?: boolean;
    observationRecord: Pick<
      ObservationRecordInsertType,
      | "project_id"
      | "trace_id"
      | "id"
      | "provided_model_name"
      | "provided_usage_details"
      | "provided_cost_details"
      | "level"
      | "input"
      | "output"
    >;
  }): Promise<
    Pick<
      ObservationRecordInsertType,
      | "usage_details"
      | "cost_details"
      | "total_cost"
      | "internal_model_id"
      | "usage_pricing_tier_id"
      | "usage_pricing_tier_name"
    >
  > {
    const {
      projectId,
      observationRecord,
      pricingMatchAttributeValues,
      isAiGatewayGeneration = false,
    } = params;
    const { model: internalModel, pricingTiers } =
      observationRecord.provided_model_name
        ? await findModel({
            projectId,
            model: observationRecord.provided_model_name,
          })
        : { model: null, pricingTiers: [] };

    const final_usage_details = await this.getUsageUnits(
      observationRecord,
      internalModel,
      isAiGatewayGeneration,
    );

    // Match pricing tier based on usage_details. Skip when usage is empty
    // (e.g. tokenization skipped because costs were provided, or ERROR-level
    // generations): matching {} would stamp a tier chosen against a fabricated
    // all-zero usage vector instead of leaving the tier unset.
    let modelPrices: Array<{ usageType: string; price: Decimal }> = [];
    let usage_pricing_tier_id: string | null = null;
    let usage_pricing_tier_name: string | null = null;
    if (
      pricingTiers.length > 0 &&
      hasPricingTierUsageDetails(final_usage_details.usage_details)
    ) {
      const attributeSources = new Set(
        pricingTiers.flatMap((tier) =>
          tier.isDefault
            ? []
            : tier.conditions.flatMap((condition) =>
                "source" in condition ? [condition.source] : [],
              ),
        ),
      );
      const pricingMatchAttributes: PricingTierMatchAttributes | undefined =
        attributeSources.size > 0
          ? {
              modelParameters: attributeSources.has("model_parameters")
                ? toPricingAttributeRecord(
                    pricingMatchAttributeValues?.modelParameters,
                  )
                : {},
              metadata: attributeSources.has("metadata")
                ? toPricingAttributeRecord(
                    pricingMatchAttributeValues?.metadata,
                  )
                : {},
            }
          : undefined;
      const matchedTier = matchPricingTier(
        pricingTiers,
        final_usage_details.usage_details,
        pricingMatchAttributes,
      );

      if (matchedTier) {
        usage_pricing_tier_id = matchedTier.pricingTierId;
        usage_pricing_tier_name = matchedTier.pricingTierName;

        // Convert matched tier prices to simple format for calculateUsageCosts
        modelPrices = Object.entries(matchedTier.prices).map(
          ([usageType, price]) => ({
            usageType,
            price,
          }),
        );
      }
    }

    const final_cost_details = IngestionService.calculateUsageCosts(
      modelPrices,
      observationRecord,
      final_usage_details.usage_details ?? {},
    );

    logger.debug(
      `Calculated costs and usage for observation ${observationRecord.id} with model ${internalModel?.id}`,
      {
        cost: final_cost_details.cost_details,
        usage: final_usage_details.usage_details,
        pricingTier: usage_pricing_tier_name,
      },
    );

    return {
      ...final_usage_details,
      ...final_cost_details,
      internal_model_id: internalModel?.id,
      usage_pricing_tier_id,
      usage_pricing_tier_name,
    };
  }

  private async getUsageUnits(
    observationRecord: Pick<
      ObservationRecordInsertType,
      | "project_id"
      | "provided_usage_details"
      | "provided_cost_details"
      | "level"
      | "input"
      | "output"
      | "id"
    >,
    model: Model | null | undefined,
    isAiGatewayGeneration: boolean,
  ): Promise<
    Pick<
      ObservationRecordInsertType,
      "usage_details" | "provided_usage_details"
    >
  > {
    const providedUsageDetails = IngestionService.normalizeProvidedUsageDetails(
      observationRecord.provided_usage_details,
    );

    // Provided costs are authoritative: calculateUsageCosts ignores computed
    // usage once any cost point is provided, so tokenised counts could never
    // affect costs — leave usage_details blank instead of paying for
    // tokenisation.
    const hasProvidedCostDetails = Object.values(
      observationRecord.provided_cost_details ?? {},
    ).some((value) => value != null);

    // The AI gateway forwards provider-reported usage whenever the provider
    // returns it. Without it (failed or cancelled upstream calls), the true
    // usage is unknown: the request body holds tool schemas and encrypted
    // reasoning, so tokenizing it fabricates usage and cost.
    if (
      // Manual tokenisation when no user provided usage or cost and generation has not status ERROR
      model &&
      Object.keys(providedUsageDetails).length === 0 &&
      !hasProvidedCostDetails &&
      observationRecord.level !== ObservationLevel.ERROR &&
      !isAiGatewayGeneration
    ) {
      try {
        let newInputCount: number | undefined;
        let newOutputCount: number | undefined;
        await instrumentAsync(
          {
            name: "token-count",
          },
          async (span) => {
            try {
              [newInputCount, newOutputCount] = await Promise.all([
                tokenCountAsync({
                  text: observationRecord.input,
                  model,
                }),
                tokenCountAsync({
                  text: observationRecord.output,
                  model,
                }),
              ]);
            } catch (error) {
              // No synchronous fallback: payloads that make the worker thread
              // exceed its timeout would block the event loop for minutes if
              // re-tokenized on the main thread. Absent counts are
              // detectable by users, unlike a 0 or a bytes-based estimate.
              logger.warn(
                `Async tokenization failed for observation ${observationRecord.id} in project ${observationRecord.project_id}. Skipping token counts.`,
                {
                  projectId: observationRecord.project_id,
                  observationId: observationRecord.id,
                  modelId: model.id,
                  tokenizerId: model.tokenizerId,
                  inputBytes:
                    typeof observationRecord.input === "string"
                      ? observationRecord.input.length
                      : undefined,
                  outputBytes:
                    typeof observationRecord.output === "string"
                      ? observationRecord.output.length
                      : undefined,
                  error,
                },
              );
              span.setAttribute("langfuse.tokenization.skipped", true);
              recordIncrement("langfuse.tokenisation.skipped", 1);
            }

            // Tracing
            newInputCount
              ? span.setAttribute(
                  "langfuse.tokenization.input-count",
                  newInputCount,
                )
              : undefined;
            newOutputCount
              ? span.setAttribute(
                  "langfuse.tokenization.output-count",
                  newOutputCount,
                )
              : undefined;
            newInputCount || newOutputCount
              ? span.setAttribute(
                  "langfuse.tokenization.tokenizer",
                  model.tokenizerId || "unknown",
                )
              : undefined;
            newInputCount
              ? recordIncrement("langfuse.tokenisedTokens", newInputCount)
              : undefined;
            newOutputCount
              ? recordIncrement("langfuse.tokenisedTokens", newOutputCount)
              : undefined;
          },
        );

        logger.debug(
          `Tokenized observation ${observationRecord.id} with model ${model.id}, input: ${newInputCount}, output: ${newOutputCount}`,
        );

        const newTotalCount =
          newInputCount || newOutputCount
            ? (newInputCount ?? 0) + (newOutputCount ?? 0)
            : undefined;

        const usage_details: Record<string, number> = {};

        if (newInputCount != null) usage_details.input = newInputCount;
        if (newOutputCount != null) usage_details.output = newOutputCount;
        if (newTotalCount != null) usage_details.total = newTotalCount;

        return { usage_details, provided_usage_details: providedUsageDetails };
      } catch (error) {
        traceException(error);
        logger.error(
          `Tokenization failed for observation ${observationRecord.id} with model ${model.id}. Continuing without token counts.`,
          error,
        );
        // Continue without token counts - return empty usage_details
        return {
          usage_details: {},
          provided_usage_details: providedUsageDetails,
        };
      }
    }

    const usageDetails = { ...providedUsageDetails };
    if (Object.keys(usageDetails).length > 0 && !("total" in usageDetails)) {
      usageDetails.total = Object.values(providedUsageDetails).reduce(
        (acc, value) => acc + value,
        0,
      );
    }

    return {
      usage_details: usageDetails,
      provided_usage_details: providedUsageDetails,
    };
  }

  // Convert all values to numbers to handle cases where ClickHouse returns UInt64 as strings.
  // This prevents string concatenation bugs like "100" + "200" = "100200" instead of 300.
  private static normalizeProvidedUsageDetails(
    providedUsageDetails: Record<string, unknown>,
  ): Record<string, number> {
    const normalized: Record<string, number> = {};
    for (const [key, value] of Object.entries(providedUsageDetails)) {
      if (value != null) {
        const numValue = Number(value);
        if (!isNaN(numValue) && numValue >= 0) {
          normalized[key] = numValue;
        }
      }
    }
    return normalized;
  }

  private static lastUsageTotalMismatchLogAt = 0;
  private static readonly USAGE_TOTAL_MISMATCH_TOLERANCE = 0.01;
  private static readonly USAGE_TOTAL_MISMATCH_LOG_INTERVAL_MS = 60_000;

  /**
   * Detects the double-count class from
   * https://github.com/langfuse/langfuse/issues/10592: instrumentors that send
   * an inclusive `input` alongside cache buckets while providing a smaller
   * `total`. Warn only — buckets can be genuinely additive (e.g. Bedrock cache
   * writes), so auto-correcting could corrupt valid payloads.
   *
   * Called once per write path (`legacy` merge path, `events` direct path);
   * dual-write mode fires both, so the metric carries a write_path tag to keep
   * the counts reconcilable.
   */
  private warnOnUsageTotalMismatch(
    rawProvidedUsageDetails: Record<string, unknown>,
    observationRecord: Pick<ObservationRecordInsertType, "id" | "project_id">,
    writePath: "legacy" | "events",
  ): void {
    const providedUsageDetails = IngestionService.normalizeProvidedUsageDetails(
      rawProvidedUsageDetails,
    );
    const providedTotal = providedUsageDetails.total;
    if (providedTotal == null) return;

    const bucketSum = Object.entries(providedUsageDetails)
      .filter(([key]) => key !== "total")
      .reduce((sum, [, value]) => sum + value, 0);
    const tolerance = Math.max(
      1,
      providedTotal * IngestionService.USAGE_TOTAL_MISMATCH_TOLERANCE,
    );
    if (bucketSum <= providedTotal + tolerance) return;

    recordIncrement("langfuse.ingestion.usage_details.total_mismatch", 1, {
      write_path: writePath,
    });

    const now = Date.now();
    if (
      now - IngestionService.lastUsageTotalMismatchLogAt <
      IngestionService.USAGE_TOTAL_MISMATCH_LOG_INTERVAL_MS
    ) {
      return;
    }
    IngestionService.lastUsageTotalMismatchLogAt = now;
    // The `langfuse.ingestion.usage_details.total_mismatch` metric above carries
    // the aggregate signal; keep the detailed line at debug to avoid drowning
    // warn-level log volume with a per-event customer-data condition.
    logger.debug(
      "Sum of provided non-total usage_details buckets exceeds provided total; the instrumentor may be sending an inclusive input alongside cache buckets",
      {
        projectId: observationRecord.project_id,
        observationId: observationRecord.id,
        providedTotal,
        bucketSum,
        providedUsageDetails,
        writePath,
      },
    );
  }

  static calculateUsageCosts(
    modelPrices:
      | Array<{ usageType: string; price: Decimal }>
      | null
      | undefined,
    observationRecord: Pick<
      ObservationRecordInsertType,
      "provided_cost_details"
    >,
    usageUnits: UsageCostType,
  ): Pick<ObservationRecordInsertType, "cost_details" | "total_cost"> {
    const { provided_cost_details } = observationRecord;

    const providedCostKeys = Object.entries(provided_cost_details ?? {})
      .filter(([_, value]) => value != null)
      .map(([key]) => key);

    // If user has provided any cost point, do not calculate any other cost points
    if (providedCostKeys.length) {
      const cost_details = { ...provided_cost_details };
      const finalTotalCost =
        (provided_cost_details ?? {})["total"] ??
        // Use provided input and output cost if available, but only if no other cost points are provided
        (providedCostKeys.every((key) => ["input", "output"].includes(key))
          ? ((provided_cost_details ?? {})["input"] ?? 0) +
            ((provided_cost_details ?? {})["output"] ?? 0)
          : undefined);

      if (
        !Object.prototype.hasOwnProperty.call(cost_details, "total") &&
        finalTotalCost != null
      ) {
        cost_details.total = finalTotalCost;
      }

      return {
        cost_details,
        total_cost: finalTotalCost,
      };
    }

    const finalCostEntries: [string, number][] = [];

    for (const [key, units] of Object.entries(usageUnits)) {
      const price =
        modelPrices?.find((price) => price.usageType === key) ??
        (TTL_CACHE_CREATION_USAGE_TYPES.has(key)
          ? AGGREGATE_CACHE_CREATION_USAGE_TYPES.map((usageType) =>
              modelPrices?.find((price) => price.usageType === usageType),
            ).find(Boolean)
          : undefined);

      if (units != null && price) {
        finalCostEntries.push([key, price.price.mul(units).toNumber()]);
      }
    }

    const finalCostDetails = Object.fromEntries(finalCostEntries);

    let finalTotalCost;
    if (
      Object.prototype.hasOwnProperty.call(finalCostDetails, "total") &&
      finalCostDetails.total != null
    ) {
      finalTotalCost = finalCostDetails.total;
    } else if (finalCostEntries.length > 0) {
      finalTotalCost = finalCostEntries.reduce(
        (acc, [_, cost]) => acc + cost,
        0,
      );

      finalCostDetails.total = finalTotalCost;
    }

    return {
      cost_details: finalCostDetails,
      total_cost: finalTotalCost,
    };
  }

  private async getClickhouseRecord(params: {
    projectId: string;
    entityId: string;
    table: TableName.Traces;
    additionalFilters: {
      whereCondition: string;
      params: Record<string, unknown>;
    };
  }): Promise<TraceRecordInsertType | null>;

  private async getClickhouseRecord(params: {
    projectId: string;
    entityId: string;
    table: TableName.Scores;
    additionalFilters: {
      whereCondition: string;
      params: Record<string, unknown>;
    };
  }): Promise<ScoreRecordInsertType | null>;

  private async getClickhouseRecord(params: {
    projectId: string;
    entityId: string;
    table: TableName.Observations;
    additionalFilters: {
      whereCondition: string;
      params: Record<string, unknown>;
    };
  }): Promise<ObservationRecordInsertType | null>;

  private async getClickhouseRecord(params: {
    projectId: string;
    entityId: string;
    table: TableName;
    additionalFilters: {
      whereCondition: string;
      params: Record<string, unknown>;
    };
  }) {
    if (
      await ClickhouseReadSkipCache.getInstance(
        this.prisma,
      ).shouldSkipClickHouseRead(params.projectId)
    ) {
      recordIncrement("langfuse.ingestion.clickhouse_read_for_update", 1, {
        skipped: "true",
        table: params.table,
      });
      return null;
    }
    recordIncrement("langfuse.ingestion.clickhouse_read_for_update", 1, {
      skipped: "false",
      table: params.table,
    });

    const recordParser = {
      traces: traceRecordReadSchema,
      scores: scoreRecordReadSchema,
      observations: observationRecordReadSchema,
    };
    const { projectId, entityId, table, additionalFilters } = params;

    return await instrumentAsync(
      { name: `get-clickhouse-${table}`, spanKind: SpanKind.CLIENT },
      async (span) => {
        span.setAttribute("ch.query.table", table);
        span.setAttribute("db.system", "clickhouse");
        span.setAttribute("db.operation.name", "SELECT");
        span.setAttribute("projectId", projectId);
        const queryResult = await this.clickhouseClient.query({
          query: `
            SELECT *
            FROM ${table}
            WHERE project_id = {projectId: String}
            AND id = {entityId: String}
            ${additionalFilters.whereCondition}
            ORDER BY event_ts DESC
            LIMIT 1 BY id, project_id SETTINGS use_query_cache = false;
          `,
          format: "JSONEachRow",
          query_params: { projectId, entityId, ...additionalFilters.params },
          clickhouse_settings: {
            log_comment: buildClickHouseLogComment({
              projectId,
            }),
          },
        });

        span.setAttribute("ch.queryId", queryResult.query_id);
        const summaryHeader =
          queryResult.response_headers["x-clickhouse-summary"];
        if (summaryHeader) {
          try {
            const summary = Array.isArray(summaryHeader)
              ? JSON.parse(summaryHeader[0])
              : JSON.parse(summaryHeader);
            for (const key in summary) {
              span.setAttribute(`ch.${key}`, summary[key]);
            }
          } catch (error) {
            logger.debug(
              `Failed to parse clickhouse summary header ${summaryHeader}`,
              error,
            );
          }
        }

        const result = await queryResult.json();

        if (result.length === 0) return null;

        switch (table) {
          case TableName.Traces:
            return convertTraceReadToInsert(
              recordParser[table].parse(result[0]),
            );
          case TableName.Scores:
            return convertScoreReadToInsert(
              recordParser[table].parse(result[0]),
            );
          case TableName.Observations:
            return convertObservationReadToInsert(
              recordParser[table].parse(result[0]),
            );
          default:
            throw new Error(`Unsupported table name: ${table}`);
        }
      },
    );
  }

  private mapTraceEventsToRecords(params: {
    traceEventList: TraceEventType[];
    projectId: string;
    entityId: string;
  }) {
    const { traceEventList, projectId, entityId } = params;

    return traceEventList.map((trace) => {
      const traceRecord: TraceRecordInsertType = {
        id: entityId,
        timestamp: toClickhouseDateTime(
          trace.body.timestamp ?? trace.timestamp,
        ),
        // timestamp: ("timestamp" in trace.body && trace.body.timestamp
        //   ? this.getMillisecondTimestamp(trace.body.timestamp)
        //   : undefined) as number, // Casting here is dirty, but our requirement is to have a start_time _after_ the merge
        name: trace.body.name,
        user_id: trace.body.userId,
        metadata: trace.body.metadata
          ? convertJsonSchemaToRecord(trace.body.metadata)
          : {},
        release: trace.body.release,
        version: trace.body.version,
        project_id: projectId,
        environment: trace.body.environment,
        public: trace.body.public ?? false,
        bookmarked: false,
        tags: trace.body.tags ?? [],
        // We skip the processing here as stringifying is an expensive operation on large objects.
        // Instead, we only take the last truthy value and apply it on the merge step.
        // input: this.stringify(trace.body.input),
        // output: this.stringify(trace.body.output), // convert even json to string
        session_id: trace.body.sessionId,
        created_at: toClickhouseDateTime(),
        updated_at: toClickhouseDateTime(),
        event_ts: toClickhouseDateTime(trace.timestamp),
        is_deleted: 0,
      };

      return traceRecord;
    });
  }

  private getObservationType(
    observation: ObservationEvent,
  ):
    | "EVENT"
    | "SPAN"
    | "GENERATION"
    | "AGENT"
    | "TOOL"
    | "CHAIN"
    | "RETRIEVER"
    | "EVALUATOR"
    | "GUARDRAIL"
    | "EMBEDDING" {
    switch (observation.type) {
      case eventTypes.OBSERVATION_CREATE:
      case eventTypes.OBSERVATION_UPDATE:
        return observation.body.type;
      case eventTypes.EVENT_CREATE:
        return "EVENT" as const;
      case eventTypes.SPAN_CREATE:
      case eventTypes.SPAN_UPDATE:
        return "SPAN" as const;
      case eventTypes.GENERATION_CREATE:
      case eventTypes.GENERATION_UPDATE:
        return "GENERATION" as const;
      case eventTypes.AGENT_CREATE:
        return "AGENT" as const;
      case eventTypes.TOOL_CREATE:
        return "TOOL" as const;
      case eventTypes.CHAIN_CREATE:
        return "CHAIN" as const;
      case eventTypes.RETRIEVER_CREATE:
        return "RETRIEVER" as const;
      case eventTypes.EVALUATOR_CREATE:
        return "EVALUATOR" as const;
      case eventTypes.EMBEDDING_CREATE:
        return "EMBEDDING" as const;
      case eventTypes.GUARDRAIL_CREATE:
        return "GUARDRAIL" as const;
    }
  }

  private mapObservationEventsToRecords(params: {
    projectId: string;
    entityId: string;
    observationEventList: ObservationEvent[];
    prompt: ObservationPrompt | null;
  }) {
    const { projectId, entityId, observationEventList, prompt } = params;

    return observationEventList.map((obs) => {
      const observationType = this.getObservationType(obs);

      const newInputCount =
        "usage" in obs.body ? obs.body.usage?.input : undefined;

      const newOutputCount =
        "usage" in obs.body ? obs.body.usage?.output : undefined;

      const newTotalCount =
        ("usage" in obs.body ? obs.body.usage?.total : undefined) ||
        (Object.keys(
          "usageDetails" in obs.body ? (obs.body.usageDetails ?? {}) : {},
        ).length === 0
          ? newInputCount && newOutputCount
            ? newInputCount + newOutputCount
            : (newInputCount ?? newOutputCount)
          : undefined);

      let provided_usage_details: Record<string, number> = {};

      if (newInputCount != null) provided_usage_details.input = newInputCount;
      if (newOutputCount != null)
        provided_usage_details.output = newOutputCount;
      if (newTotalCount != null) provided_usage_details.total = newTotalCount;

      provided_usage_details = {
        ...provided_usage_details,
        ...("usageDetails" in obs.body
          ? (Object.fromEntries(
              Object.entries(obs.body.usageDetails ?? {}).filter(
                ([_, val]) => val != null,
              ),
            ) as Record<string, number>)
          : {}),
      };

      let provided_cost_details: Record<string, number> = {};

      if ("usage" in obs.body) {
        const { inputCost, outputCost, totalCost } = obs.body.usage ?? {};

        if (inputCost != null) provided_cost_details.input = inputCost;
        if (outputCost != null) provided_cost_details.output = outputCost;
        if (totalCost != null) provided_cost_details.total = totalCost;
      }

      provided_cost_details = {
        ...provided_cost_details,
        ...("costDetails" in obs.body
          ? (Object.fromEntries(
              Object.entries(obs.body.costDetails ?? {}).filter(
                ([_, val]) => val != null,
              ),
            ) as Record<string, number>)
          : {}),
      };

      const observationRecord: ObservationRecordInsertType = {
        id: entityId,
        trace_id: obs.body.traceId ?? v4(),
        type: observationType,
        name: obs.body.name,
        environment:
          "environment" in obs.body ? obs.body.environment : "default",
        start_time: toClickhouseDateTime(obs.body.startTime ?? obs.timestamp),
        // start_time: ("startTime" in obs.body && obs.body.startTime
        //   ? this.getMillisecondTimestamp(obs.body.startTime)
        //   : undefined) as number, // Casting here is dirty, but our requirement is to have a start_time _after_ the merge
        end_time:
          "endTime" in obs.body && obs.body.endTime
            ? toClickhouseDateTime(obs.body.endTime)
            : undefined,
        completion_start_time:
          "completionStartTime" in obs.body && obs.body.completionStartTime
            ? toClickhouseDateTime(obs.body.completionStartTime)
            : undefined,
        metadata: obs.body.metadata
          ? convertJsonSchemaToRecord(obs.body.metadata)
          : {},
        provided_model_name: "model" in obs.body ? obs.body.model : undefined,
        model_parameters:
          "modelParameters" in obs.body
            ? obs.body.modelParameters
              ? JSON.stringify(obs.body.modelParameters)
              : undefined
            : undefined,
        // We skip the processing here as stringifying is an expensive operation on large objects.
        // Instead, we only take the last truthy value and apply it on the merge step.
        // input: this.stringify(obs.body.input),
        // output: this.stringify(obs.body.output),
        provided_usage_details,
        provided_cost_details,
        usage_details: provided_usage_details,
        cost_details: provided_cost_details,
        level: obs.body.level,
        status_message: obs.body.statusMessage ?? undefined,
        parent_observation_id: obs.body.parentObservationId ?? undefined,
        version: obs.body.version ?? undefined,
        project_id: projectId,
        prompt_id: prompt?.id,
        prompt_name: prompt?.name,
        prompt_version: prompt?.version,
        created_at: toClickhouseDateTime(),
        updated_at: toClickhouseDateTime(),
        event_ts: toClickhouseDateTime(obs.timestamp),
        is_deleted: 0,
      };

      return observationRecord;
    });
  }

  private stringify(
    obj: string | object | number | boolean | undefined | null,
  ): string | undefined {
    if (obj == null) return; // return undefined on undefined or null

    return typeof obj === "string" ? obj : JSON.stringify(obj);
  }

  /**
   * Returns a partition-aware timestamp for staging table writes.
   * If the createdAtTimestamp is within the last 2 minutes, returns it as-is.
   * Otherwise, returns the current timestamp to prevent updates to old partitions.
   *
   * This implements the partition locking strategy where partitions are "locked"
   * 4 minutes after creation (2 min + 2 min buffer for writes).
   *
   * Going down from 3.5min to 2min here, as we see gaps in the data that may come from deletions.
   * This reduces that chance that updates are handled in the same batch, but should increase the chance
   * that data is processed correctly. Worst case is slightly more duplication in the events table
   * which should resolve automatically using the ReplacingMergeTree.
   */
  private getPartitionAwareTimestamp(createdAtTimestamp: Date): string {
    const now = Date.now();
    const createdAt = createdAtTimestamp.getTime();
    const ageInMs = now - createdAt;
    const twoMinutesInMs = 2 * 60 * 1000;

    // If the createdAtTimestamp is within the last 2 minutes, use it
    // Otherwise, use the current timestamp to avoid updating old partitions
    return toClickhouseDateTime(ageInMs < twoMinutesInMs ? createdAt : now);
  }
}

type ObservationPrompt = Pick<Prompt, "id" | "name" | "version">;