🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable no-nested-ternary */
import { Cluster, Redis } from "ioredis";
import { v4 } from "uuid";
import { Decimal } from "decimal.js";
import {
InvalidRequestError,
LangfuseNotFoundError,
Model,
MAX_SKILLS_PER_OBSERVATION,
ObservationLevel,
PrismaClient,
Prompt,
safeJsonParse,
type JsonNested,
type SkillsAvailable,
type SkillsAvailableInput,
type SkillsResourceLoaded,
} from "@langfuse/shared";
import {
ClickhouseClientType,
convertDateToClickhouseDateTime,
toClickhouseDateTime,
parseClickhouseUTCDateTimeFormat,
convertObservationReadToInsert,
convertScoreReadToInsert,
convertTraceReadToInsert,
convertTraceToStagingObservation,
eventTypes,
IngestionEntityTypes,
IngestionEventType,
instrumentAsync,
logger,
ObservationEvent,
observationRecordInsertSchema,
ObservationRecordInsertType,
observationRecordReadSchema,
PromptService,
SkillCacheService,
QueueJobs,
recordIncrement,
ScoreEventType,
DatasetRunItemEventType,
scoreRecordInsertSchema,
ScoreRecordInsertType,
scoreRecordReadSchema,
TraceEventType,
traceRecordInsertSchema,
TraceRecordInsertType,
traceRecordReadSchema,
TraceUpsertQueue,
UsageCostType,
findModel,
hasPricingTierUsageDetails,
matchPricingTier,
validateAndInflateScore,
DatasetRunItemRecordInsertType,
EventRecordInsertType,
InternalTraceEventInput,
traceException,
flattenJsonToPathArrays,
getDatasetItemById,
normalizeToolsForObservation,
hasNoEvalConfigsCache,
buildClickHouseLogComment,
sanitizeSdkMetricTagValue,
type IngestionAttribution,
type PricingTierMatchAttributes,
AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME,
} from "@langfuse/shared/src/server";
import { tokenCountAsync } from "../../features/tokenisation/async-usage";
import { ClickhouseWriter, TableName } from "../ClickhouseWriter";
import {
convertJsonSchemaToRecord,
convertPostgresJsonToMetadataRecord,
convertRecordValuesToString,
hasAiGatewayScope,
isAiGatewayGenerationEvent,
overwriteObject,
} from "./utils";
import { resolveAiGatewayAttribution } from "./aiGatewayAttribution";
import { randomUUID } from "crypto";
import { SpanKind } from "@opentelemetry/api";
import { ClickhouseReadSkipCache } from "../../utils/clickhouseReadSkipCache";
import { applyObservationFieldOverflow } from "../../features/observation-field-overflow/processObservationFieldOverflow";
/**
* Parse a value to a UInt16-compatible number (0–65535).
* Returns undefined if the value is nullish or not a valid UInt16 integer.
*/
function parseUInt16(value: string | null | undefined): number | undefined {
if (value == null) return undefined;
const num = parseInt(value, 10);
if (!Number.isInteger(num) || num < 0 || num > 65535) return undefined;
return num;
}
function toPricingAttributeRecord(value: unknown): Record<string, string> {
const parsedValue = typeof value === "string" ? safeJsonParse(value) : value;
if (
!parsedValue ||
typeof parsedValue !== "object" ||
Array.isArray(parsedValue)
) {
return {};
}
return Object.fromEntries(
Object.entries(parsedValue).flatMap(([key, attributeValue]) =>
typeof attributeValue === "string" ||
typeof attributeValue === "number" ||
typeof attributeValue === "boolean"
? [[key, String(attributeValue)]]
: [],
),
);
}
export type EventInput = InternalTraceEventInput;
function parseEventModelParameters(
value: EventInput["modelParameters"],
): string | object | number | boolean | null {
if (!value) return {};
if (typeof value !== "string") return value;
try {
return JSON.parse(value) as string | object | number | boolean | null;
} catch {
return value;
}
}
type InsertRecord =
| TraceRecordInsertType
| ScoreRecordInsertType
| ObservationRecordInsertType
| DatasetRunItemRecordInsertType;
type MergeAndWriteParams = {
eventType: IngestionEntityTypes;
projectId: string;
entityId: string;
createdAtTimestamp: Date;
events: IngestionEventType[];
forwardToEventsTable: boolean;
attribution: IngestionAttribution;
};
type PricingTierMatchAttributeValues = {
modelParameters?: unknown;
metadata?: unknown;
};
/**
* Returns a new event record whose `event_bytes` value is the UTF-8 size of
* the final JSONEachRow payload, excluding the self-referential accounting
* field. The supplied record is not mutated.
*/
function withSerializedEventByteLength(
eventRecord: EventRecordInsertType,
): EventRecordInsertType {
const { event_bytes: _eventBytes, ...eventWithoutSize } = eventRecord;
return {
...eventWithoutSize,
event_bytes: Buffer.byteLength(JSON.stringify(eventWithoutSize), "utf8"),
};
}
// Models priced before TTL-split cache writes existed (e.g. custom models)
// only carry an aggregate cache-write price; TTL-split usage falls back to it.
const TTL_CACHE_CREATION_USAGE_TYPES = new Set([
"input_cache_creation_5m",
"input_cache_creation_1h",
]);
const AGGREGATE_CACHE_CREATION_USAGE_TYPES = [
"cache_creation_input_tokens",
"input_cache_creation",
];
const immutableEntityKeys: {
[TableName.Traces]: (keyof TraceRecordInsertType)[];
[TableName.Scores]: (keyof ScoreRecordInsertType)[];
[TableName.Observations]: (keyof ObservationRecordInsertType)[];
[TableName.DatasetRunItems]: (keyof DatasetRunItemRecordInsertType)[];
} = {
[TableName.Traces]: [
"id",
"project_id",
"timestamp",
"created_at",
"environment",
],
[TableName.Scores]: [
"id",
"project_id",
"timestamp",
"trace_id",
"created_at",
"environment",
],
[TableName.Observations]: [
"id",
"project_id",
"trace_id",
"start_time",
"created_at",
"environment",
],
// We do not accept updates, hence this list is currently not used.
[TableName.DatasetRunItems]: [
"id",
"project_id",
"dataset_run_id",
"dataset_item_id",
"dataset_id",
"trace_id",
"observation_id",
"error",
"created_at",
"updated_at",
"dataset_run_name",
"dataset_run_description",
"dataset_run_metadata",
"dataset_run_created_at",
"dataset_item_input",
"dataset_item_expected_output",
"dataset_item_metadata",
],
};
export class IngestionService {
private promptService: PromptService;
private skillCacheService: SkillCacheService;
constructor(
private redis: Redis | Cluster,
private prisma: PrismaClient,
private clickHouseWriter: ClickhouseWriter,
private clickhouseClient: ClickhouseClientType,
) {
this.promptService = new PromptService(prisma, redis);
this.skillCacheService = new SkillCacheService(prisma, redis);
}
public async mergeAndWrite(params: MergeAndWriteParams): Promise<void> {
const {
eventType,
projectId,
entityId,
createdAtTimestamp,
events,
forwardToEventsTable,
attribution,
} = params;
logger.debug(
`Merging ingestion ${eventType} event for project ${projectId} and event ${entityId}`,
);
switch (eventType) {
case "trace":
return await this.processTraceEventList({
projectId,
entityId,
createdAtTimestamp,
traceEventList: events as TraceEventType[],
createEventTraceRecord: forwardToEventsTable,
attribution,
});
case "observation":
return await this.processObservationEventList({
projectId,
entityId,
createdAtTimestamp,
observationEventList: events as ObservationEvent[],
writeToStagingTables: forwardToEventsTable,
attribution,
});
case "score": {
return await this.processScoreEventList({
projectId,
entityId,
createdAtTimestamp,
scoreEventList: events as ScoreEventType[],
attribution,
});
}
case "dataset_run_item": {
return await this.processDatasetRunItemEventList({
projectId,
entityId,
createdAtTimestamp,
datasetRunItemEventList: events as DatasetRunItemEventType[],
});
}
}
}
/**
* Creates an EventRecordInsertType from EventInput.
* Performs all necessary enrichments:
* - Prompt lookup (by name + version)
* - Model/usage enrichment (tokenization, cost calculation)
* - Metadata flattening
* - Timestamp normalization
*
* This is the single point of transformation from loose EventInput
* to strict EventRecordInsertType.
*
* @param eventData - The event data from processToEvent()
* @param fileKey - The file key where the raw event data is stored
* @returns The enriched event record ready for writing or eval scheduling
*/
public async createEventRecord(
eventData: EventInput,
fileKey: string,
): Promise<EventRecordInsertType> {
logger.debug(
`Creating event record for project ${eventData.projectId} and span ${eventData.spanId}`,
);
// processToEvent can keep normalized input/output as objects so tool data
// can be extracted before write time. EventRecordInsertType stores both
// fields as strings, so stringify at this schema boundary.
const input = this.stringify(eventData.input);
const output = this.stringify(eventData.output);
const modelParameters = parseEventModelParameters(
eventData.modelParameters,
);
const metadata = eventData.metadata ?? {};
const spanAttribution = resolveAiGatewayAttribution({
ingestionApiKey: eventData.ingestionApiKey,
metadata,
serviceName: eventData.serviceName,
serviceVersion: eventData.serviceVersion,
scopeVersion: eventData.scopeVersion,
});
// Runs outside the modelName gate below so model-less events with provided
// usage are still checked.
this.warnOnUsageTotalMismatch(
eventData.providedUsageDetails ?? {},
{ id: eventData.spanId, project_id: eventData.projectId },
"events",
);
const shouldEnrichUsageAndCost =
Boolean(eventData.modelName) ||
Object.keys(eventData.providedUsageDetails ?? {}).length > 0 ||
Object.keys(eventData.providedCostDetails ?? {}).length > 0;
// Enrich prompt, usage and skill references concurrently.
const [prompt, generationUsage, skillReferences] = await Promise.all([
// Lookup prompt by name and version
eventData.promptName && eventData.promptVersion
? this.promptService.getPrompt({
projectId: eventData.projectId,
promptName: eventData.promptName,
version:
typeof eventData.promptVersion === "string"
? parseInt(eventData.promptVersion, 10)
: eventData.promptVersion,
label: undefined,
})
: null,
// Promote provided usage/cost and enrich from the model when available.
shouldEnrichUsageAndCost
? this.getGenerationUsage({
projectId: eventData.projectId,
pricingMatchAttributeValues: {
modelParameters,
metadata,
},
isAiGatewayGeneration: isAiGatewayGenerationEvent({
ingestionApiKey: eventData.ingestionApiKey,
hasGatewayScope: () =>
eventData.scopeName === AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME,
}),
observationRecord: {
id: eventData.spanId,
project_id: eventData.projectId,
trace_id: eventData.traceId,
provided_model_name: eventData.modelName,
provided_usage_details: eventData.providedUsageDetails ?? {},
provided_cost_details: eventData.providedCostDetails ?? {},
level: eventData.level,
input,
output,
},
})
: null,
this.resolveSkillReferences({
projectId: eventData.projectId,
type: eventData.type ?? "SPAN",
available: eventData.skillsAvailable,
loaded: eventData.skillsResourceLoaded,
}),
]);
const now = toClickhouseDateTime();
// Flatten raw metadata first (before stringification destroys nested structure)
const flattened = flattenJsonToPathArrays(metadata);
const metadataNames = flattened.names;
// Defensive: coerce null/undefined to empty string for Array(String) ClickHouse column.
// Should not be required as convertValueToPlainJavascript() never returns null.
const metadataValues = flattened.values.map((v) => v ?? "");
const eventRecord: EventRecordInsertType = {
// Required identifiers
id: eventData.spanId,
project_id: eventData.projectId,
trace_id: eventData.traceId,
span_id: eventData.spanId,
// Optional identifiers
parent_span_id: eventData.parentSpanId,
// Core properties with defaults
name: eventData.name ?? "",
type: eventData.type ?? "SPAN",
environment: eventData.environment ?? "default",
version: eventData.version,
release: eventData.release,
tags: eventData.tags ?? [],
bookmarked: eventData.bookmarked ?? false,
public: eventData.public ?? false,
is_app_root: eventData.isAppRoot ?? false,
// Trace-level attributes: Name/User/session
trace_name: eventData.traceName,
user_id: eventData.userId,
session_id: eventData.sessionId,
// Status
level: eventData.level ?? "DEFAULT",
status_message: eventData.statusMessage,
// Timestamps
start_time: toClickhouseDateTime(eventData.startTimeISO),
end_time: toClickhouseDateTime(eventData.endTimeISO),
completion_start_time: eventData.completionStartTime
? toClickhouseDateTime(eventData.completionStartTime)
: null,
// Prompt
prompt_id: prompt?.id || "",
prompt_name: eventData.promptName,
prompt_version: parseUInt16(eventData.promptVersion),
skills_available: skillReferences.skills_available,
skills_resource_loaded: skillReferences.skills_resource_loaded,
// Model
model_id: generationUsage?.internal_model_id || "",
provided_model_name: eventData.modelName,
// Evaluation scheduling consumes this enriched record before it is
// written and expects parsed model parameters. The established runtime
// contract is therefore wider than the ClickHouse insert type declares.
model_parameters:
modelParameters as EventRecordInsertType["model_parameters"],
// Usage & Cost
provided_usage_details: eventData.providedUsageDetails ?? {},
usage_details:
generationUsage?.usage_details ?? eventData.usageDetails ?? {},
provided_cost_details: eventData.providedCostDetails ?? {},
cost_details:
generationUsage?.cost_details ?? eventData.costDetails ?? {},
usage_pricing_tier_id: generationUsage?.usage_pricing_tier_id,
usage_pricing_tier_name: generationUsage?.usage_pricing_tier_name,
// Tool Calls
tool_definitions: eventData.toolDefinitions ?? {},
tool_calls: eventData.toolCalls ?? [],
tool_call_names: eventData.toolCallNames ?? [],
// I/O
input,
output,
// Metadata
metadata_names: metadataNames,
metadata_values: metadataValues,
evaluator_id: eventData.evaluationContext?.evaluatorId,
evaluation_rule_id: eventData.evaluationContext?.evaluationRuleId,
evaluator_execution_is_test:
eventData.evaluationContext?.evaluatorExecutionIsTest,
// Source/instrumentation metadata
source: eventData.source,
ingestion_api_key: spanAttribution.ingestionApiKey,
ingestion_sdk_name: eventData.ingestionSdkName ?? "",
ingestion_sdk_version: eventData.ingestionSdkVersion ?? "",
service_name: spanAttribution.serviceName,
service_version: spanAttribution.serviceVersion,
scope_name: eventData.scopeName,
scope_version: eventData.scopeVersion,
telemetry_sdk_language: eventData.telemetrySdkLanguage,
telemetry_sdk_name: eventData.telemetrySdkName,
telemetry_sdk_version: eventData.telemetrySdkVersion,
// Storage
blob_storage_file_path: fileKey,
event_bytes: eventData.eventBytes ?? 0,
// Experiment fields
experiment_id: eventData.experimentId,
experiment_name: eventData.experimentName,
experiment_metadata_names: eventData.experimentMetadataNames ?? [],
experiment_metadata_values: eventData.experimentMetadataValues ?? [],
experiment_description: eventData.experimentDescription,
experiment_dataset_id: eventData.experimentDatasetId,
experiment_item_id: eventData.experimentItemId,
experiment_item_version: eventData.experimentItemVersion,
experiment_item_root_span_id: eventData.experimentItemRootSpanId,
experiment_item_expected_output: eventData.experimentItemExpectedOutput,
experiment_item_metadata_names:
eventData.experimentItemMetadataNames ?? [],
experiment_item_metadata_values:
eventData.experimentItemMetadataValues ?? [],
// System timestamps
created_at: now,
updated_at: now,
event_ts: now,
is_deleted: 0,
};
return eventRecord;
}
/**
* Writes an event record directly to the events_full table.
* A materialized view auto-populates events_core from events_full.
* Legacy observation writes and staging-based dual writes intentionally do
* not use field overflow.
* Use createEventRecord() first to get the record, then call this to write.
*
* Enqueues a new record whose `event_bytes` describes the final normalized
* event rather than the raw OTEL span measured before media processing. The
* supplied record is not mutated.
*
* @param eventRecord - The event record to write
* @returns The serialized byte size of the accepted event record
*/
public async writeEventRecord(
eventRecord: EventRecordInsertType,
): Promise<number> {
const persistedRecord = withSerializedEventByteLength(
await applyObservationFieldOverflow(eventRecord),
);
this.clickHouseWriter.addToQueue(TableName.EventsFull, persistedRecord);
return persistedRecord.event_bytes;
}
private async processDatasetRunItemEventList(params: {
projectId: string;
entityId: string;
createdAtTimestamp: Date;
datasetRunItemEventList: DatasetRunItemEventType[];
}) {
const { projectId, entityId, datasetRunItemEventList } = params;
if (datasetRunItemEventList.length === 0) return;
const finalDatasetRunItemRecords: DatasetRunItemRecordInsertType[] = (
await Promise.all(
datasetRunItemEventList.map(
async (
event: DatasetRunItemEventType,
): Promise<DatasetRunItemRecordInsertType[]> => {
const [runData, itemData] = await Promise.all([
this.prisma.datasetRuns.findFirst({
where: {
id: event.body.runId,
datasetId: event.body.datasetId,
projectId,
},
select: {
name: true,
description: true,
metadata: true,
createdAt: true,
},
}),
await getDatasetItemById({
projectId,
datasetItemId: event.body.datasetItemId,
datasetId: event.body.datasetId,
version: event.body.datasetVersion
? new Date(event.body.datasetVersion)
: undefined,
status: "ACTIVE",
}),
]);
if (!runData || !itemData) return [];
const timestamp = toClickhouseDateTime(event.body.createdAt);
const datasetItemVersion = itemData.validFrom
? toClickhouseDateTime(itemData.validFrom)
: null;
return [
{
id: entityId,
project_id: projectId,
dataset_run_id: event.body.runId,
dataset_item_id: event.body.datasetItemId,
dataset_id: event.body.datasetId,
trace_id: event.body.traceId,
observation_id: event.body.observationId,
error: event.body.error,
created_at: timestamp,
updated_at: timestamp,
event_ts: timestamp,
is_deleted: 0,
// enriched with run data
dataset_run_name: runData.name,
dataset_run_description: runData.description,
dataset_run_metadata: runData.metadata
? convertPostgresJsonToMetadataRecord(runData.metadata)
: {},
dataset_run_created_at: toClickhouseDateTime(runData.createdAt),
// enriched with item data
dataset_item_version: datasetItemVersion,
dataset_item_input: JSON.stringify(itemData.input),
dataset_item_expected_output: JSON.stringify(
itemData.expectedOutput,
),
dataset_item_metadata: itemData.metadata
? convertPostgresJsonToMetadataRecord(itemData.metadata)
: {},
},
];
},
),
)
).flat();
finalDatasetRunItemRecords.forEach((record) => {
if (record) {
this.clickHouseWriter.addToQueue(TableName.DatasetRunItems, record);
}
});
}
private async processScoreEventList(params: {
projectId: string;
entityId: string;
createdAtTimestamp: Date;
scoreEventList: ScoreEventType[];
attribution: IngestionAttribution;
}) {
const {
projectId,
entityId,
createdAtTimestamp,
scoreEventList,
attribution,
} = params;
if (scoreEventList.length === 0) return;
// Re-sending a score with the same id and timestamp is the documented way
// to overwrite it, so ties must keep arrival order for the later one to
// win. Score events are all creates, so no create-first rule is needed.
const timeSortedEvents = scoreEventList
.slice()
.sort(
(a, b) =>
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(),
);
const minTimestamp = Math.min(
...timeSortedEvents.flatMap((e) =>
e.timestamp ? [new Date(e.timestamp).getTime()] : [],
),
);
const timestamp =
minTimestamp === Infinity
? undefined
: convertDateToClickhouseDateTime(new Date(minTimestamp));
const unexpectedScoreValidationErrors: unknown[] = [];
let expectedScoreValidationDrops = 0;
const [clickhouseScoreRecord, scoreRecords] = await Promise.all([
this.getClickhouseRecord({
projectId,
entityId,
table: TableName.Scores,
additionalFilters: {
whereCondition: timestamp
? " AND timestamp >= {timestamp: DateTime64(3)} "
: "",
params: { timestamp },
},
}),
Promise.all(
timeSortedEvents.map(async (scoreEvent) => {
try {
const validatedScore = await validateAndInflateScore({
body: scoreEvent.body,
scoreId: entityId,
projectId,
});
const rawMetadata = scoreEvent.body.metadata
? convertJsonSchemaToRecord(scoreEvent.body.metadata)
: {};
const evaluationFields =
scoreEvent.body as ScoreEventType["body"] & {
evaluatorId?: string;
evaluationRuleId?: string;
};
return {
id: entityId,
project_id: projectId,
environment: validatedScore.environment,
timestamp: toClickhouseDateTime(scoreEvent.timestamp),
name: validatedScore.name,
value: validatedScore.value,
source: validatedScore.source,
trace_id: validatedScore.traceId,
session_id: validatedScore.sessionId,
dataset_run_id: validatedScore.datasetRunId,
data_type: validatedScore.dataType,
observation_id: validatedScore.observationId,
config_id: validatedScore.configId,
comment: validatedScore.comment,
metadata: convertRecordValuesToString(rawMetadata),
evaluator_id: evaluationFields.evaluatorId,
evaluation_rule_id: evaluationFields.evaluationRuleId,
string_value: validatedScore.stringValue,
long_string_value: validatedScore.longStringValue,
execution_trace_id: validatedScore.executionTraceId,
queue_id: validatedScore.queueId ?? null,
ingestion_api_key: attribution.ingestionApiKey,
ingestion_sdk_name: attribution.ingestionSdkName,
ingestion_sdk_version: attribution.ingestionSdkVersion,
created_at: toClickhouseDateTime(),
updated_at: toClickhouseDateTime(),
event_ts: toClickhouseDateTime(scoreEvent.timestamp),
is_deleted: 0,
};
// Gracefully handle any score schema validation errors, skip the score insert and reject silently.
} catch (error) {
if (
error instanceof InvalidRequestError ||
error instanceof LangfuseNotFoundError
) {
expectedScoreValidationDrops += 1;
} else {
unexpectedScoreValidationErrors.push(error);
}
logger.info(
`Failed to validate and enrich score body for project: ${projectId} and score: ${entityId}`,
error,
);
return null;
}
}),
).then((results) =>
results.filter(
(record): record is NonNullable<typeof record> => record !== null,
),
),
]);
if (clickhouseScoreRecord) {
recordIncrement("langfuse.ingestion.lookup.hit", 1, {
store: "clickhouse",
object: "score",
});
}
if (unexpectedScoreValidationErrors.length > 0) {
throw new AggregateError(
unexpectedScoreValidationErrors,
`Unexpected error(s) validating score batch for project: ${projectId} and score: ${entityId}`,
);
}
// Count drops only on an attempt that proceeds: a rethrowing batch is
// redelivered, and counting it would inflate the metric once per retry.
for (let i = 0; i < expectedScoreValidationDrops; i++) {
recordIncrement("langfuse.ingestion.metadata_dropped", 1, {
reason: "score_validation_dropped",
source: "api",
domain: "score",
projectId,
sdkName: sanitizeSdkMetricTagValue(attribution.ingestionSdkName),
sdkVersion: sanitizeSdkMetricTagValue(attribution.ingestionSdkVersion),
});
}
if (scoreRecords.length === 0 && !clickhouseScoreRecord) {
logger.warn(
`No valid score records found for project: ${projectId} and score: ${entityId}`,
);
return;
}
// Update = merging new events with a pre-existing record, found either in
// ClickHouse or as earlier event-log files for the same score id.
if (
scoreRecords.length > 0 &&
(clickhouseScoreRecord || scoreRecords.length > 1)
) {
recordIncrement("langfuse.ingestion.score_update", 1, {
store: clickhouseScoreRecord ? "clickhouse" : "event_log",
});
}
const finalScoreRecord: ScoreRecordInsertType =
await this.mergeScoreRecords({
clickhouseScoreRecord,
scoreRecords,
});
finalScoreRecord.created_at =
clickhouseScoreRecord?.created_at ??
toClickhouseDateTime(createdAtTimestamp);
this.clickHouseWriter.addToQueue(TableName.Scores, finalScoreRecord);
}
private async processTraceEventList(params: {
projectId: string;
entityId: string;
createdAtTimestamp: Date;
traceEventList: TraceEventType[];
createEventTraceRecord: boolean;
attribution: IngestionAttribution;
}) {
const {
projectId,
entityId,
createdAtTimestamp,
traceEventList,
createEventTraceRecord,
attribution,
} = params;
if (traceEventList.length === 0) return;
const timeSortedEvents =
IngestionService.toTimeSortedEventList(traceEventList);
const traceRecords = this.mapTraceEventsToRecords({
projectId,
entityId,
traceEventList: timeSortedEvents,
});
// Search for the first non-null input and output in the trace events and set them on the merged result.
// Fallback to the ClickHouse input/output if none are found within the events list.
const reversedRawRecords = timeSortedEvents.slice().reverse();
const finalIO = {
input: this.stringify(
reversedRawRecords.find((record) => record?.body?.input)?.body?.input,
),
output: this.stringify(
reversedRawRecords.find((record) => record?.body?.output)?.body?.output,
),
};
const minTimestamp = Math.min(
...timeSortedEvents.flatMap((e) =>
e.body?.timestamp ? [new Date(e.body.timestamp).getTime()] : [],
),
);
const timestamp =
minTimestamp === Infinity
? undefined
: convertDateToClickhouseDateTime(new Date(minTimestamp));
const clickhouseTraceRecord = await this.getClickhouseRecord({
projectId,
entityId,
table: TableName.Traces,
additionalFilters: {
whereCondition: timestamp
? " AND timestamp >= {timestamp: DateTime64(3)} "
: "",
params: { timestamp },
},
});
if (clickhouseTraceRecord) {
recordIncrement("langfuse.ingestion.lookup.hit", 1, {
store: "clickhouse",
object: "trace",
});
}
const finalTraceRecord = await this.mergeTraceRecords({
clickhouseTraceRecord,
traceRecords,
});
finalTraceRecord.created_at =
clickhouseTraceRecord?.created_at ??
toClickhouseDateTime(createdAtTimestamp);
finalTraceRecord.input = finalIO.input ?? clickhouseTraceRecord?.input;
finalTraceRecord.output = finalIO.output ?? clickhouseTraceRecord?.output;
this.clickHouseWriter.addToQueue(TableName.Traces, finalTraceRecord);
// If the trace has a sessionId, we upsert the corresponding session into Postgres.
const traceRecordWithSession = traceRecords
.slice()
.reverse()
.find((t) => t.session_id);
if (traceRecordWithSession) {
try {
await this.prisma.$executeRaw`
INSERT INTO trace_sessions (id, project_id, environment, created_at, updated_at)
VALUES (${traceRecordWithSession.session_id}, ${projectId}, ${traceRecordWithSession.environment}, NOW(), NOW())
ON CONFLICT (id, project_id)
DO NOTHING
`;
} catch (e) {
logger.error(
`Failed to upsert session ${traceRecordWithSession.session_id}`,
e,
);
throw e;
}
}
// Dual-write to staging table for batch propagation to events table
// We pretend the trace is a "span" where span_id = trace_id
if (createEventTraceRecord) {
const traceAsStagingObservation = convertTraceToStagingObservation(
finalTraceRecord,
this.getPartitionAwareTimestamp(createdAtTimestamp),
attribution,
);
this.clickHouseWriter.addToQueue(
TableName.ObservationsBatchStaging,
traceAsStagingObservation,
);
}
// Add trace into trace upsert queue for eval processing
// First check if we already know this project has no job configurations
const hasNoJobConfigs = await hasNoEvalConfigsCache(
projectId,
"traceBased",
);
if (hasNoJobConfigs) {
logger.debug(
`Skipping TraceUpsert queue for project ${projectId} - no job configs cached`,
);
return;
}
// Job configs present, so we add to the TraceUpsert queue.
const shardingKey = `${projectId}-${entityId}`;
const traceUpsertQueue = TraceUpsertQueue.getInstance({ shardingKey });
if (!traceUpsertQueue) {
logger.error("TraceUpsertQueue is not initialized");
return;
}
await traceUpsertQueue.add(QueueJobs.TraceUpsert, {
payload: {
projectId,
traceId: entityId,
exactTimestamp: parseClickhouseUTCDateTimeFormat(
finalTraceRecord.timestamp,
),
traceEnvironment: finalTraceRecord.environment,
},
id: randomUUID(),
timestamp: new Date(),
name: QueueJobs.TraceUpsert as const,
});
}
private async processObservationEventList(params: {
projectId: string;
entityId: string;
createdAtTimestamp: Date;
observationEventList: ObservationEvent[];
writeToStagingTables: boolean;
attribution: IngestionAttribution;
}) {
const {
projectId,
entityId,
createdAtTimestamp,
observationEventList,
writeToStagingTables,
attribution,
} = params;
if (observationEventList.length === 0) return;
const timeSortedEvents =
IngestionService.toTimeSortedEventList(observationEventList);
const type = this.getObservationType(observationEventList[0]);
const minStartTime = Math.min(
...observationEventList.flatMap((e) =>
e.body?.startTime ? [new Date(e.body.startTime).getTime()] : [],
),
);
const startTime =
minStartTime === Infinity
? undefined
: convertDateToClickhouseDateTime(new Date(minStartTime));
const [clickhouseObservationRecord, prompt] = await Promise.all([
this.getClickhouseRecord({
projectId,
entityId,
table: TableName.Observations,
additionalFilters: {
whereCondition: `AND type = {type: String} ${startTime ? "AND start_time >= {startTime: DateTime64(3)} " : ""}`,
params: {
type,
startTime,
},
},
}),
this.getPrompt(projectId, observationEventList),
]);
if (clickhouseObservationRecord) {
recordIncrement("langfuse.ingestion.lookup.hit", 1, {
store: "clickhouse",
object: "observation",
});
}
const observationRecords = this.mapObservationEventsToRecords({
observationEventList: timeSortedEvents,
projectId,
entityId,
prompt,
});
const mergedObservationRecord = await this.mergeObservationRecords({
projectId,
observationRecords,
clickhouseObservationRecord,
});
mergedObservationRecord.created_at =
clickhouseObservationRecord?.created_at ??
toClickhouseDateTime(createdAtTimestamp);
mergedObservationRecord.level = mergedObservationRecord.level ?? "DEFAULT";
// Search for the first non-null input and output in the observation events and set them on the merged result.
// Fallback to the ClickHouse input/output if none are found within the events list.
const reversedRawRecords = timeSortedEvents.slice().reverse();
const rawInput =
reversedRawRecords.find((record) => record?.body?.input)?.body?.input ??
clickhouseObservationRecord?.input;
const rawOutput =
reversedRawRecords.find((record) => record?.body?.output)?.body?.output ??
clickhouseObservationRecord?.output;
const pricingEventIndex = observationRecords.findLastIndex(
(record) => Object.keys(record.provided_usage_details ?? {}).length > 0,
);
const pricingEvent = timeSortedEvents[pricingEventIndex];
const pricingMatchAttributeValues:
| PricingTierMatchAttributeValues
| undefined = pricingEvent
? {
modelParameters:
"modelParameters" in pricingEvent.body
? pricingEvent.body.modelParameters
: undefined,
metadata: pricingEvent.body.metadata,
}
: undefined;
const normalizedTools = normalizeToolsForObservation(
rawInput,
rawOutput,
mergedObservationRecord.metadata,
);
mergedObservationRecord.input = this.stringify(normalizedTools.input);
mergedObservationRecord.output = this.stringify(normalizedTools.output);
const normalizedMetadata = normalizedTools.metadata ?? {};
mergedObservationRecord.metadata =
normalizedMetadata &&
typeof normalizedMetadata === "object" &&
!Array.isArray(normalizedMetadata)
? convertRecordValuesToString(
normalizedMetadata as Record<string, unknown>,
)
: convertJsonSchemaToRecord(normalizedMetadata as JsonNested);
if (Object.keys(normalizedTools.toolDefinitions).length > 0) {
mergedObservationRecord.tool_definitions =
normalizedTools.toolDefinitions;
}
if (normalizedTools.toolCalls.length > 0) {
mergedObservationRecord.tool_calls = normalizedTools.toolCalls;
mergedObservationRecord.tool_call_names = normalizedTools.toolCallNames;
}
// Only check when the incoming events carry usage themselves — partial
// updates merge stale usage back in from ClickHouse and must not re-fire.
if (
observationRecords.some(
(record) => Object.keys(record.provided_usage_details ?? {}).length > 0,
)
) {
this.warnOnUsageTotalMismatch(
mergedObservationRecord.provided_usage_details ?? {},
mergedObservationRecord,
"legacy",
);
}
const generationUsage = await this.getGenerationUsage({
projectId,
pricingMatchAttributeValues,
isAiGatewayGeneration: isAiGatewayGenerationEvent({
ingestionApiKey: attribution.ingestionApiKey,
hasGatewayScope: () =>
timeSortedEvents.some((event) =>
hasAiGatewayScope(event.body?.metadata),
),
}),
observationRecord: mergedObservationRecord,
});
const finalObservationRecord = {
...mergedObservationRecord,
...generationUsage,
};
// Backward compat: create wrapper trace for SDK < 2.0.0 events that do not have a traceId
if (!finalObservationRecord.trace_id) {
const wrapperTraceRecord: TraceRecordInsertType = {
id: finalObservationRecord.id,
timestamp: finalObservationRecord.start_time,
project_id: projectId,
environment: finalObservationRecord.environment,
created_at: toClickhouseDateTime(),
updated_at: toClickhouseDateTime(),
metadata: {},
tags: [],
bookmarked: false,
public: false,
event_ts: toClickhouseDateTime(),
is_deleted: 0,
};
this.clickHouseWriter.addToQueue(TableName.Traces, wrapperTraceRecord);
finalObservationRecord.trace_id = finalObservationRecord.id;
}
this.clickHouseWriter.addToQueue(
TableName.Observations,
finalObservationRecord,
);
// Dual-write to staging table for batch propagation to events table
// Here, we add some additional logic around the first seen timestamp.
// We "lock" partitions 4min after their creation, i.e. the 15:00:00 partition
// should stop receiving updates at 15:04:00.
// This means that we keep the createdAtTimestamp as-is if it is within the last
// 3.5 minutes (incl. a 30s buffer around writes) and otherwise,
// we set the current timestamp for the event.
if (writeToStagingTables) {
const stagingRecord = {
...finalObservationRecord,
ingestion_api_key: attribution.ingestionApiKey,
ingestion_sdk_name: attribution.ingestionSdkName,
ingestion_sdk_version: attribution.ingestionSdkVersion,
s3_first_seen_timestamp:
this.getPartitionAwareTimestamp(createdAtTimestamp),
};
this.clickHouseWriter.addToQueue(
TableName.ObservationsBatchStaging,
stagingRecord,
);
}
}
private async mergeScoreRecords(params: {
scoreRecords: ScoreRecordInsertType[];
clickhouseScoreRecord?: ScoreRecordInsertType | null;
}): Promise<ScoreRecordInsertType> {
const { scoreRecords, clickhouseScoreRecord } = params;
// Set clickhouse first as this is the baseline for immutable fields
const recordsToMerge = [clickhouseScoreRecord, ...scoreRecords].filter(
Boolean,
) as ScoreRecordInsertType[];
const mergedRecord = this.mergeRecords(
recordsToMerge,
immutableEntityKeys[TableName.Scores],
);
// If metadata exists, it is an object due to previous parsing
mergedRecord.metadata = convertRecordValuesToString(
(mergedRecord.metadata as Record<string, unknown>) ?? {},
);
return scoreRecordInsertSchema.parse(mergedRecord);
}
private async mergeTraceRecords(params: {
traceRecords: TraceRecordInsertType[];
clickhouseTraceRecord?: TraceRecordInsertType | null;
}): Promise<TraceRecordInsertType> {
const { traceRecords, clickhouseTraceRecord } = params;
// Set clickhouse first as this is the baseline for immutable fields
const recordsToMerge = [clickhouseTraceRecord, ...traceRecords].filter(
Boolean,
) as TraceRecordInsertType[];
const mergedRecord = this.mergeRecords(
recordsToMerge,
immutableEntityKeys[TableName.Traces],
);
// If metadata exists, it is an object due to previous parsing
mergedRecord.metadata = convertRecordValuesToString(
(mergedRecord.metadata as Record<string, unknown>) ?? {},
);
return traceRecordInsertSchema.parse(mergedRecord);
}
private async mergeObservationRecords(params: {
projectId: string;
observationRecords: ObservationRecordInsertType[];
clickhouseObservationRecord?: ObservationRecordInsertType | null;
}): Promise<ObservationRecordInsertType> {
const { observationRecords, clickhouseObservationRecord } = params;
// Set clickhouse first as this is the baseline for immutable fields
const recordsToMerge = [
clickhouseObservationRecord,
...observationRecords,
].filter(Boolean) as ObservationRecordInsertType[];
const mergedRecord = this.mergeRecords(
recordsToMerge,
immutableEntityKeys[TableName.Observations],
);
// If metadata exists, it is an object due to previous parsing
mergedRecord.metadata = convertRecordValuesToString(
(mergedRecord.metadata as Record<string, unknown>) ?? {},
);
const parsedObservationRecord =
observationRecordInsertSchema.parse(mergedRecord);
// Override endTimes that are before startTimes with the startTime
if (
parsedObservationRecord.end_time &&
parsedObservationRecord.end_time < parsedObservationRecord.start_time
) {
parsedObservationRecord.end_time = parsedObservationRecord.start_time;
}
return parsedObservationRecord;
}
private async resolveSkillReferences({
projectId,
type,
available = [],
loaded = [],
}: {
projectId: string;
type: EventRecordInsertType["type"];
available?: SkillsAvailableInput;
loaded?: SkillsResourceLoaded;
}): Promise<{
skills_available: SkillsAvailable;
skills_resource_loaded: SkillsResourceLoaded;
}> {
// Bound internal callers too, before building the lookup or its cache key.
if (
available.length > MAX_SKILLS_PER_OBSERVATION ||
loaded.length > MAX_SKILLS_PER_OBSERVATION
) {
logger.warn("Skill references truncated: item limit exceeded", {
projectId,
availableCount: available.length,
loadedCount: loaded.length,
limit: MAX_SKILLS_PER_OBSERVATION,
});
}
available = available.slice(0, MAX_SKILLS_PER_OBSERVATION);
loaded = loaded.slice(0, MAX_SKILLS_PER_OBSERVATION);
const versions = available.flatMap(({ name, version }) =>
version == null ? [] : [{ name, version }],
);
const skills = await this.skillCacheService.getReferences({
projectId,
versions,
});
const availableSkills = available.map(({ name, version }) => {
const skill = skills.find(
(skill) => skill.name === name && skill.version === version,
);
return {
name,
version: skill?.version ?? version,
langfuseSkillId: skill?.id,
};
});
const loadedResources = loaded.map(
({ skillName, filePath, skillVersion }) => {
const candidates = availableSkills.filter(
(skill) => skill.name === skillName,
);
if (candidates.length !== 1)
return { skillName, filePath, skillVersion };
const skill = candidates[0];
return {
skillName,
filePath,
langfuseSkillId: skill?.langfuseSkillId,
skillVersion: skill.version ?? skillVersion,
};
},
);
return {
skills_available:
type === "GENERATION"
? availableSkills.map(({ name, version, langfuseSkillId }) => ({
skillName: name,
skillVersion: version,
langfuseSkillId,
}))
: [],
skills_resource_loaded: loadedResources,
};
}
private mergeRecords<T extends InsertRecord>(
records: T[],
immutableEntityKeys: string[],
): Record<string, unknown> {
if (records.length === 0) {
throw new Error("No records to merge");
}
let result: {
id: string;
project_id: string;
[key: string]: any;
} = { id: records[0].id, project_id: records[0].project_id };
for (const record of records) {
result = overwriteObject(result, record, immutableEntityKeys);
}
result.event_ts = toClickhouseDateTime();
return result;
}
private static toTimeSortedEventList<
T extends TraceEventType | ObservationEvent,
>(eventList: T[]): T[] {
// The merge folds this list left to right with a last-wins overwrite, so
// the winner for any group is whichever event sorts last. Events arrive in
// upload order (the queue processor lists files by upload time), captured
// here as the arrival index and used as the tie-break so the ordering is a
// total order rather than relying on the sort engine's handling of ties.
return eventList
.map((event, index) => ({ event, index }))
.sort((a, b) => {
const aTimestamp = new Date(a.event.timestamp).getTime();
const bTimestamp = new Date(b.event.timestamp).getTime();
if (aTimestamp !== bTimestamp) return aTimestamp - bTimestamp;
const aIsCreate = a.event.type.includes("create");
const bIsCreate = b.event.type.includes("create");
// Creates sort before updates, so an update at the same timestamp wins.
if (aIsCreate !== bIsCreate) return aIsCreate ? -1 : 1;
// Tied creates: the first-arriving one wins. OTel emits a root span's
// trace-create and a child span's trace-create carrying explicit
// update_current_trace() values at the same span start time; the child
// arrives first and its explicit fields must beat the root's derived
// ones. Placing the earlier arrival last makes it win the merge.
if (aIsCreate) return b.index - a.index;
// Tied updates: the last-arriving one wins.
return a.index - b.index;
})
.map(({ event }) => event);
}
private async getPrompt(
projectId: string,
observationEventList: ObservationEvent[],
): Promise<ObservationPrompt | null> {
const lastObservationWithPromptInfo = observationEventList
.slice()
.reverse()
.find(this.hasPromptInformation);
if (!lastObservationWithPromptInfo) return null;
const { promptName, promptVersion: version } =
lastObservationWithPromptInfo.body;
return this.promptService.getPrompt({
projectId,
promptName,
version,
label: undefined,
});
}
private hasPromptInformation(
event: ObservationEvent,
): event is ObservationEvent & {
body: { promptName: string; promptVersion: number };
} {
return (
"promptName" in event.body &&
typeof event.body.promptName === "string" &&
"promptVersion" in event.body &&
typeof event.body.promptVersion === "number"
);
}
private async getGenerationUsage(params: {
projectId: string;
pricingMatchAttributeValues?: PricingTierMatchAttributeValues;
isAiGatewayGeneration?: boolean;
observationRecord: Pick<
ObservationRecordInsertType,
| "project_id"
| "trace_id"
| "id"
| "provided_model_name"
| "provided_usage_details"
| "provided_cost_details"
| "level"
| "input"
| "output"
>;
}): Promise<
Pick<
ObservationRecordInsertType,
| "usage_details"
| "cost_details"
| "total_cost"
| "internal_model_id"
| "usage_pricing_tier_id"
| "usage_pricing_tier_name"
>
> {
const {
projectId,
observationRecord,
pricingMatchAttributeValues,
isAiGatewayGeneration = false,
} = params;
const { model: internalModel, pricingTiers } =
observationRecord.provided_model_name
? await findModel({
projectId,
model: observationRecord.provided_model_name,
})
: { model: null, pricingTiers: [] };
const final_usage_details = await this.getUsageUnits(
observationRecord,
internalModel,
isAiGatewayGeneration,
);
// Match pricing tier based on usage_details. Skip when usage is empty
// (e.g. tokenization skipped because costs were provided, or ERROR-level
// generations): matching {} would stamp a tier chosen against a fabricated
// all-zero usage vector instead of leaving the tier unset.
let modelPrices: Array<{ usageType: string; price: Decimal }> = [];
let usage_pricing_tier_id: string | null = null;
let usage_pricing_tier_name: string | null = null;
if (
pricingTiers.length > 0 &&
hasPricingTierUsageDetails(final_usage_details.usage_details)
) {
const attributeSources = new Set(
pricingTiers.flatMap((tier) =>
tier.isDefault
? []
: tier.conditions.flatMap((condition) =>
"source" in condition ? [condition.source] : [],
),
),
);
const pricingMatchAttributes: PricingTierMatchAttributes | undefined =
attributeSources.size > 0
? {
modelParameters: attributeSources.has("model_parameters")
? toPricingAttributeRecord(
pricingMatchAttributeValues?.modelParameters,
)
: {},
metadata: attributeSources.has("metadata")
? toPricingAttributeRecord(
pricingMatchAttributeValues?.metadata,
)
: {},
}
: undefined;
const matchedTier = matchPricingTier(
pricingTiers,
final_usage_details.usage_details,
pricingMatchAttributes,
);
if (matchedTier) {
usage_pricing_tier_id = matchedTier.pricingTierId;
usage_pricing_tier_name = matchedTier.pricingTierName;
// Convert matched tier prices to simple format for calculateUsageCosts
modelPrices = Object.entries(matchedTier.prices).map(
([usageType, price]) => ({
usageType,
price,
}),
);
}
}
const final_cost_details = IngestionService.calculateUsageCosts(
modelPrices,
observationRecord,
final_usage_details.usage_details ?? {},
);
logger.debug(
`Calculated costs and usage for observation ${observationRecord.id} with model ${internalModel?.id}`,
{
cost: final_cost_details.cost_details,
usage: final_usage_details.usage_details,
pricingTier: usage_pricing_tier_name,
},
);
return {
...final_usage_details,
...final_cost_details,
internal_model_id: internalModel?.id,
usage_pricing_tier_id,
usage_pricing_tier_name,
};
}
private async getUsageUnits(
observationRecord: Pick<
ObservationRecordInsertType,
| "project_id"
| "provided_usage_details"
| "provided_cost_details"
| "level"
| "input"
| "output"
| "id"
>,
model: Model | null | undefined,
isAiGatewayGeneration: boolean,
): Promise<
Pick<
ObservationRecordInsertType,
"usage_details" | "provided_usage_details"
>
> {
const providedUsageDetails = IngestionService.normalizeProvidedUsageDetails(
observationRecord.provided_usage_details,
);
// Provided costs are authoritative: calculateUsageCosts ignores computed
// usage once any cost point is provided, so tokenised counts could never
// affect costs — leave usage_details blank instead of paying for
// tokenisation.
const hasProvidedCostDetails = Object.values(
observationRecord.provided_cost_details ?? {},
).some((value) => value != null);
// The AI gateway forwards provider-reported usage whenever the provider
// returns it. Without it (failed or cancelled upstream calls), the true
// usage is unknown: the request body holds tool schemas and encrypted
// reasoning, so tokenizing it fabricates usage and cost.
if (
// Manual tokenisation when no user provided usage or cost and generation has not status ERROR
model &&
Object.keys(providedUsageDetails).length === 0 &&
!hasProvidedCostDetails &&
observationRecord.level !== ObservationLevel.ERROR &&
!isAiGatewayGeneration
) {
try {
let newInputCount: number | undefined;
let newOutputCount: number | undefined;
await instrumentAsync(
{
name: "token-count",
},
async (span) => {
try {
[newInputCount, newOutputCount] = await Promise.all([
tokenCountAsync({
text: observationRecord.input,
model,
}),
tokenCountAsync({
text: observationRecord.output,
model,
}),
]);
} catch (error) {
// No synchronous fallback: payloads that make the worker thread
// exceed its timeout would block the event loop for minutes if
// re-tokenized on the main thread. Absent counts are
// detectable by users, unlike a 0 or a bytes-based estimate.
logger.warn(
`Async tokenization failed for observation ${observationRecord.id} in project ${observationRecord.project_id}. Skipping token counts.`,
{
projectId: observationRecord.project_id,
observationId: observationRecord.id,
modelId: model.id,
tokenizerId: model.tokenizerId,
inputBytes:
typeof observationRecord.input === "string"
? observationRecord.input.length
: undefined,
outputBytes:
typeof observationRecord.output === "string"
? observationRecord.output.length
: undefined,
error,
},
);
span.setAttribute("langfuse.tokenization.skipped", true);
recordIncrement("langfuse.tokenisation.skipped", 1);
}
// Tracing
newInputCount
? span.setAttribute(
"langfuse.tokenization.input-count",
newInputCount,
)
: undefined;
newOutputCount
? span.setAttribute(
"langfuse.tokenization.output-count",
newOutputCount,
)
: undefined;
newInputCount || newOutputCount
? span.setAttribute(
"langfuse.tokenization.tokenizer",
model.tokenizerId || "unknown",
)
: undefined;
newInputCount
? recordIncrement("langfuse.tokenisedTokens", newInputCount)
: undefined;
newOutputCount
? recordIncrement("langfuse.tokenisedTokens", newOutputCount)
: undefined;
},
);
logger.debug(
`Tokenized observation ${observationRecord.id} with model ${model.id}, input: ${newInputCount}, output: ${newOutputCount}`,
);
const newTotalCount =
newInputCount || newOutputCount
? (newInputCount ?? 0) + (newOutputCount ?? 0)
: undefined;
const usage_details: Record<string, number> = {};
if (newInputCount != null) usage_details.input = newInputCount;
if (newOutputCount != null) usage_details.output = newOutputCount;
if (newTotalCount != null) usage_details.total = newTotalCount;
return { usage_details, provided_usage_details: providedUsageDetails };
} catch (error) {
traceException(error);
logger.error(
`Tokenization failed for observation ${observationRecord.id} with model ${model.id}. Continuing without token counts.`,
error,
);
// Continue without token counts - return empty usage_details
return {
usage_details: {},
provided_usage_details: providedUsageDetails,
};
}
}
const usageDetails = { ...providedUsageDetails };
if (Object.keys(usageDetails).length > 0 && !("total" in usageDetails)) {
usageDetails.total = Object.values(providedUsageDetails).reduce(
(acc, value) => acc + value,
0,
);
}
return {
usage_details: usageDetails,
provided_usage_details: providedUsageDetails,
};
}
// Convert all values to numbers to handle cases where ClickHouse returns UInt64 as strings.
// This prevents string concatenation bugs like "100" + "200" = "100200" instead of 300.
private static normalizeProvidedUsageDetails(
providedUsageDetails: Record<string, unknown>,
): Record<string, number> {
const normalized: Record<string, number> = {};
for (const [key, value] of Object.entries(providedUsageDetails)) {
if (value != null) {
const numValue = Number(value);
if (!isNaN(numValue) && numValue >= 0) {
normalized[key] = numValue;
}
}
}
return normalized;
}
private static lastUsageTotalMismatchLogAt = 0;
private static readonly USAGE_TOTAL_MISMATCH_TOLERANCE = 0.01;
private static readonly USAGE_TOTAL_MISMATCH_LOG_INTERVAL_MS = 60_000;
/**
* Detects the double-count class from
* https://github.com/langfuse/langfuse/issues/10592: instrumentors that send
* an inclusive `input` alongside cache buckets while providing a smaller
* `total`. Warn only — buckets can be genuinely additive (e.g. Bedrock cache
* writes), so auto-correcting could corrupt valid payloads.
*
* Called once per write path (`legacy` merge path, `events` direct path);
* dual-write mode fires both, so the metric carries a write_path tag to keep
* the counts reconcilable.
*/
private warnOnUsageTotalMismatch(
rawProvidedUsageDetails: Record<string, unknown>,
observationRecord: Pick<ObservationRecordInsertType, "id" | "project_id">,
writePath: "legacy" | "events",
): void {
const providedUsageDetails = IngestionService.normalizeProvidedUsageDetails(
rawProvidedUsageDetails,
);
const providedTotal = providedUsageDetails.total;
if (providedTotal == null) return;
const bucketSum = Object.entries(providedUsageDetails)
.filter(([key]) => key !== "total")
.reduce((sum, [, value]) => sum + value, 0);
const tolerance = Math.max(
1,
providedTotal * IngestionService.USAGE_TOTAL_MISMATCH_TOLERANCE,
);
if (bucketSum <= providedTotal + tolerance) return;
recordIncrement("langfuse.ingestion.usage_details.total_mismatch", 1, {
write_path: writePath,
});
const now = Date.now();
if (
now - IngestionService.lastUsageTotalMismatchLogAt <
IngestionService.USAGE_TOTAL_MISMATCH_LOG_INTERVAL_MS
) {
return;
}
IngestionService.lastUsageTotalMismatchLogAt = now;
// The `langfuse.ingestion.usage_details.total_mismatch` metric above carries
// the aggregate signal; keep the detailed line at debug to avoid drowning
// warn-level log volume with a per-event customer-data condition.
logger.debug(
"Sum of provided non-total usage_details buckets exceeds provided total; the instrumentor may be sending an inclusive input alongside cache buckets",
{
projectId: observationRecord.project_id,
observationId: observationRecord.id,
providedTotal,
bucketSum,
providedUsageDetails,
writePath,
},
);
}
static calculateUsageCosts(
modelPrices:
| Array<{ usageType: string; price: Decimal }>
| null
| undefined,
observationRecord: Pick<
ObservationRecordInsertType,
"provided_cost_details"
>,
usageUnits: UsageCostType,
): Pick<ObservationRecordInsertType, "cost_details" | "total_cost"> {
const { provided_cost_details } = observationRecord;
const providedCostKeys = Object.entries(provided_cost_details ?? {})
.filter(([_, value]) => value != null)
.map(([key]) => key);
// If user has provided any cost point, do not calculate any other cost points
if (providedCostKeys.length) {
const cost_details = { ...provided_cost_details };
const finalTotalCost =
(provided_cost_details ?? {})["total"] ??
// Use provided input and output cost if available, but only if no other cost points are provided
(providedCostKeys.every((key) => ["input", "output"].includes(key))
? ((provided_cost_details ?? {})["input"] ?? 0) +
((provided_cost_details ?? {})["output"] ?? 0)
: undefined);
if (
!Object.prototype.hasOwnProperty.call(cost_details, "total") &&
finalTotalCost != null
) {
cost_details.total = finalTotalCost;
}
return {
cost_details,
total_cost: finalTotalCost,
};
}
const finalCostEntries: [string, number][] = [];
for (const [key, units] of Object.entries(usageUnits)) {
const price =
modelPrices?.find((price) => price.usageType === key) ??
(TTL_CACHE_CREATION_USAGE_TYPES.has(key)
? AGGREGATE_CACHE_CREATION_USAGE_TYPES.map((usageType) =>
modelPrices?.find((price) => price.usageType === usageType),
).find(Boolean)
: undefined);
if (units != null && price) {
finalCostEntries.push([key, price.price.mul(units).toNumber()]);
}
}
const finalCostDetails = Object.fromEntries(finalCostEntries);
let finalTotalCost;
if (
Object.prototype.hasOwnProperty.call(finalCostDetails, "total") &&
finalCostDetails.total != null
) {
finalTotalCost = finalCostDetails.total;
} else if (finalCostEntries.length > 0) {
finalTotalCost = finalCostEntries.reduce(
(acc, [_, cost]) => acc + cost,
0,
);
finalCostDetails.total = finalTotalCost;
}
return {
cost_details: finalCostDetails,
total_cost: finalTotalCost,
};
}
private async getClickhouseRecord(params: {
projectId: string;
entityId: string;
table: TableName.Traces;
additionalFilters: {
whereCondition: string;
params: Record<string, unknown>;
};
}): Promise<TraceRecordInsertType | null>;
private async getClickhouseRecord(params: {
projectId: string;
entityId: string;
table: TableName.Scores;
additionalFilters: {
whereCondition: string;
params: Record<string, unknown>;
};
}): Promise<ScoreRecordInsertType | null>;
private async getClickhouseRecord(params: {
projectId: string;
entityId: string;
table: TableName.Observations;
additionalFilters: {
whereCondition: string;
params: Record<string, unknown>;
};
}): Promise<ObservationRecordInsertType | null>;
private async getClickhouseRecord(params: {
projectId: string;
entityId: string;
table: TableName;
additionalFilters: {
whereCondition: string;
params: Record<string, unknown>;
};
}) {
if (
await ClickhouseReadSkipCache.getInstance(
this.prisma,
).shouldSkipClickHouseRead(params.projectId)
) {
recordIncrement("langfuse.ingestion.clickhouse_read_for_update", 1, {
skipped: "true",
table: params.table,
});
return null;
}
recordIncrement("langfuse.ingestion.clickhouse_read_for_update", 1, {
skipped: "false",
table: params.table,
});
const recordParser = {
traces: traceRecordReadSchema,
scores: scoreRecordReadSchema,
observations: observationRecordReadSchema,
};
const { projectId, entityId, table, additionalFilters } = params;
return await instrumentAsync(
{ name: `get-clickhouse-${table}`, spanKind: SpanKind.CLIENT },
async (span) => {
span.setAttribute("ch.query.table", table);
span.setAttribute("db.system", "clickhouse");
span.setAttribute("db.operation.name", "SELECT");
span.setAttribute("projectId", projectId);
const queryResult = await this.clickhouseClient.query({
query: `
SELECT *
FROM ${table}
WHERE project_id = {projectId: String}
AND id = {entityId: String}
${additionalFilters.whereCondition}
ORDER BY event_ts DESC
LIMIT 1 BY id, project_id SETTINGS use_query_cache = false;
`,
format: "JSONEachRow",
query_params: { projectId, entityId, ...additionalFilters.params },
clickhouse_settings: {
log_comment: buildClickHouseLogComment({
projectId,
}),
},
});
span.setAttribute("ch.queryId", queryResult.query_id);
const summaryHeader =
queryResult.response_headers["x-clickhouse-summary"];
if (summaryHeader) {
try {
const summary = Array.isArray(summaryHeader)
? JSON.parse(summaryHeader[0])
: JSON.parse(summaryHeader);
for (const key in summary) {
span.setAttribute(`ch.${key}`, summary[key]);
}
} catch (error) {
logger.debug(
`Failed to parse clickhouse summary header ${summaryHeader}`,
error,
);
}
}
const result = await queryResult.json();
if (result.length === 0) return null;
switch (table) {
case TableName.Traces:
return convertTraceReadToInsert(
recordParser[table].parse(result[0]),
);
case TableName.Scores:
return convertScoreReadToInsert(
recordParser[table].parse(result[0]),
);
case TableName.Observations:
return convertObservationReadToInsert(
recordParser[table].parse(result[0]),
);
default:
throw new Error(`Unsupported table name: ${table}`);
}
},
);
}
private mapTraceEventsToRecords(params: {
traceEventList: TraceEventType[];
projectId: string;
entityId: string;
}) {
const { traceEventList, projectId, entityId } = params;
return traceEventList.map((trace) => {
const traceRecord: TraceRecordInsertType = {
id: entityId,
timestamp: toClickhouseDateTime(
trace.body.timestamp ?? trace.timestamp,
),
// timestamp: ("timestamp" in trace.body && trace.body.timestamp
// ? this.getMillisecondTimestamp(trace.body.timestamp)
// : undefined) as number, // Casting here is dirty, but our requirement is to have a start_time _after_ the merge
name: trace.body.name,
user_id: trace.body.userId,
metadata: trace.body.metadata
? convertJsonSchemaToRecord(trace.body.metadata)
: {},
release: trace.body.release,
version: trace.body.version,
project_id: projectId,
environment: trace.body.environment,
public: trace.body.public ?? false,
bookmarked: false,
tags: trace.body.tags ?? [],
// We skip the processing here as stringifying is an expensive operation on large objects.
// Instead, we only take the last truthy value and apply it on the merge step.
// input: this.stringify(trace.body.input),
// output: this.stringify(trace.body.output), // convert even json to string
session_id: trace.body.sessionId,
created_at: toClickhouseDateTime(),
updated_at: toClickhouseDateTime(),
event_ts: toClickhouseDateTime(trace.timestamp),
is_deleted: 0,
};
return traceRecord;
});
}
private getObservationType(
observation: ObservationEvent,
):
| "EVENT"
| "SPAN"
| "GENERATION"
| "AGENT"
| "TOOL"
| "CHAIN"
| "RETRIEVER"
| "EVALUATOR"
| "GUARDRAIL"
| "EMBEDDING" {
switch (observation.type) {
case eventTypes.OBSERVATION_CREATE:
case eventTypes.OBSERVATION_UPDATE:
return observation.body.type;
case eventTypes.EVENT_CREATE:
return "EVENT" as const;
case eventTypes.SPAN_CREATE:
case eventTypes.SPAN_UPDATE:
return "SPAN" as const;
case eventTypes.GENERATION_CREATE:
case eventTypes.GENERATION_UPDATE:
return "GENERATION" as const;
case eventTypes.AGENT_CREATE:
return "AGENT" as const;
case eventTypes.TOOL_CREATE:
return "TOOL" as const;
case eventTypes.CHAIN_CREATE:
return "CHAIN" as const;
case eventTypes.RETRIEVER_CREATE:
return "RETRIEVER" as const;
case eventTypes.EVALUATOR_CREATE:
return "EVALUATOR" as const;
case eventTypes.EMBEDDING_CREATE:
return "EMBEDDING" as const;
case eventTypes.GUARDRAIL_CREATE:
return "GUARDRAIL" as const;
}
}
private mapObservationEventsToRecords(params: {
projectId: string;
entityId: string;
observationEventList: ObservationEvent[];
prompt: ObservationPrompt | null;
}) {
const { projectId, entityId, observationEventList, prompt } = params;
return observationEventList.map((obs) => {
const observationType = this.getObservationType(obs);
const newInputCount =
"usage" in obs.body ? obs.body.usage?.input : undefined;
const newOutputCount =
"usage" in obs.body ? obs.body.usage?.output : undefined;
const newTotalCount =
("usage" in obs.body ? obs.body.usage?.total : undefined) ||
(Object.keys(
"usageDetails" in obs.body ? (obs.body.usageDetails ?? {}) : {},
).length === 0
? newInputCount && newOutputCount
? newInputCount + newOutputCount
: (newInputCount ?? newOutputCount)
: undefined);
let provided_usage_details: Record<string, number> = {};
if (newInputCount != null) provided_usage_details.input = newInputCount;
if (newOutputCount != null)
provided_usage_details.output = newOutputCount;
if (newTotalCount != null) provided_usage_details.total = newTotalCount;
provided_usage_details = {
...provided_usage_details,
...("usageDetails" in obs.body
? (Object.fromEntries(
Object.entries(obs.body.usageDetails ?? {}).filter(
([_, val]) => val != null,
),
) as Record<string, number>)
: {}),
};
let provided_cost_details: Record<string, number> = {};
if ("usage" in obs.body) {
const { inputCost, outputCost, totalCost } = obs.body.usage ?? {};
if (inputCost != null) provided_cost_details.input = inputCost;
if (outputCost != null) provided_cost_details.output = outputCost;
if (totalCost != null) provided_cost_details.total = totalCost;
}
provided_cost_details = {
...provided_cost_details,
...("costDetails" in obs.body
? (Object.fromEntries(
Object.entries(obs.body.costDetails ?? {}).filter(
([_, val]) => val != null,
),
) as Record<string, number>)
: {}),
};
const observationRecord: ObservationRecordInsertType = {
id: entityId,
trace_id: obs.body.traceId ?? v4(),
type: observationType,
name: obs.body.name,
environment:
"environment" in obs.body ? obs.body.environment : "default",
start_time: toClickhouseDateTime(obs.body.startTime ?? obs.timestamp),
// start_time: ("startTime" in obs.body && obs.body.startTime
// ? this.getMillisecondTimestamp(obs.body.startTime)
// : undefined) as number, // Casting here is dirty, but our requirement is to have a start_time _after_ the merge
end_time:
"endTime" in obs.body && obs.body.endTime
? toClickhouseDateTime(obs.body.endTime)
: undefined,
completion_start_time:
"completionStartTime" in obs.body && obs.body.completionStartTime
? toClickhouseDateTime(obs.body.completionStartTime)
: undefined,
metadata: obs.body.metadata
? convertJsonSchemaToRecord(obs.body.metadata)
: {},
provided_model_name: "model" in obs.body ? obs.body.model : undefined,
model_parameters:
"modelParameters" in obs.body
? obs.body.modelParameters
? JSON.stringify(obs.body.modelParameters)
: undefined
: undefined,
// We skip the processing here as stringifying is an expensive operation on large objects.
// Instead, we only take the last truthy value and apply it on the merge step.
// input: this.stringify(obs.body.input),
// output: this.stringify(obs.body.output),
provided_usage_details,
provided_cost_details,
usage_details: provided_usage_details,
cost_details: provided_cost_details,
level: obs.body.level,
status_message: obs.body.statusMessage ?? undefined,
parent_observation_id: obs.body.parentObservationId ?? undefined,
version: obs.body.version ?? undefined,
project_id: projectId,
prompt_id: prompt?.id,
prompt_name: prompt?.name,
prompt_version: prompt?.version,
created_at: toClickhouseDateTime(),
updated_at: toClickhouseDateTime(),
event_ts: toClickhouseDateTime(obs.timestamp),
is_deleted: 0,
};
return observationRecord;
});
}
private stringify(
obj: string | object | number | boolean | undefined | null,
): string | undefined {
if (obj == null) return; // return undefined on undefined or null
return typeof obj === "string" ? obj : JSON.stringify(obj);
}
/**
* Returns a partition-aware timestamp for staging table writes.
* If the createdAtTimestamp is within the last 2 minutes, returns it as-is.
* Otherwise, returns the current timestamp to prevent updates to old partitions.
*
* This implements the partition locking strategy where partitions are "locked"
* 4 minutes after creation (2 min + 2 min buffer for writes).
*
* Going down from 3.5min to 2min here, as we see gaps in the data that may come from deletions.
* This reduces that chance that updates are handled in the same batch, but should increase the chance
* that data is processed correctly. Worst case is slightly more duplication in the events table
* which should resolve automatically using the ReplacingMergeTree.
*/
private getPartitionAwareTimestamp(createdAtTimestamp: Date): string {
const now = Date.now();
const createdAt = createdAtTimestamp.getTime();
const ageInMs = now - createdAt;
const twoMinutesInMs = 2 * 60 * 1000;
// If the createdAtTimestamp is within the last 2 minutes, use it
// Otherwise, use the current timestamp to avoid updating old partitions
return toClickhouseDateTime(ageInMs < twoMinutesInMs ? createdAt : now);
}
}
type ObservationPrompt = Pick<Prompt, "id" | "name" | "version">;