🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable no-nested-ternary */
import { randomUUID } from "crypto";
import { extractSkillsFromObservation } from "../ingestion/extractSkillsBackend";
import {
ForbiddenError,
ObservationLevel,
type ObservationLevelType,
ObservationType,
ObservationTypeDomain,
} from "../../";
import {
type TraceEventType,
type IngestionEventType,
redis,
logger,
instrumentAsync,
recordIncrement,
traceException,
getS3EventStorageClient,
QueueJobs,
instrumentSync,
recordDistribution,
UsageDetails,
normalizeToolsForObservation,
normalizeToolMetadataForObservation,
normalizeEnvironment,
DEFAULT_TRACE_ENVIRONMENT,
LangfuseInternalTraceEnvironment,
sanitizeSdkMetricTagValue,
normalizeIngestionSdkName,
isStructuredMetadataSdkVersion,
type IngestionSdkCanonicalName,
} from "../";
import { LangfuseOtelSpanAttributes } from "./attributes";
import {
isClaudeManagedAgentSessionCreation,
isClaudeManagedAgentSpan,
ObservationTypeMapperRegistry,
} from "./ObservationTypeMapper";
import { env } from "../../env";
import { OtelIngestionQueue } from "../redis/otelIngestionQueue";
import { isValidDateString, flattenJsonToPathArrays } from "./utils";
import { convertDateToClickhouseDateTime } from "../clickhouse/client";
import { isNullOrUndefined } from "../../utils/isNullOrUndefined";
import { defineOwnValue } from "../../utils/objects";
export const AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME = "langfuse-ai-gateway";
const AI_GATEWAY_INGESTION_API_KEY_PREFIX = "gateway:";
/** Metadata key under which the AI gateway records the caller's gateway API key id. */
export const AI_GATEWAY_KEY_ID_METADATA_KEY = "langfuse.gateway.api_key.id";
/**
* The ingestion API key Web records for an upload authorized by a gateway
* ingestion grant. Unlike the instrumentation scope name, an OTLP sender cannot
* choose it, so it is the marker for behavior that trusts the gateway.
*/
export const aiGatewayIngestionApiKey = (grantId: string) =>
`${AI_GATEWAY_INGESTION_API_KEY_PREFIX}${grantId}`;
export const isAiGatewayIngestionApiKey = (
ingestionApiKey: string | null | undefined,
): boolean =>
ingestionApiKey?.startsWith(AI_GATEWAY_INGESTION_API_KEY_PREFIX) ?? false;
// Foreign level vocabularies observed from OTel senders (OTel severity
// names, python logging, loguru, console) mapped onto the Langfuse enum.
// The classic ingestion API keeps its strict enum; only the
// OTel adapter translates. Unknown values return undefined so the caller's
// status-derived fallback applies instead of masking it with DEFAULT.
const OBSERVATION_LEVEL_ALIASES: Record<string, ObservationLevelType> = {
DEBUG: ObservationLevel.DEBUG,
TRACE: ObservationLevel.DEBUG,
VERBOSE: ObservationLevel.DEBUG,
DEFAULT: ObservationLevel.DEFAULT,
INFO: ObservationLevel.DEFAULT,
LOG: ObservationLevel.DEFAULT,
NOTICE: ObservationLevel.DEFAULT,
OK: ObservationLevel.DEFAULT,
SUCCESS: ObservationLevel.DEFAULT,
WARNING: ObservationLevel.WARNING,
WARN: ObservationLevel.WARNING,
ERROR: ObservationLevel.ERROR,
FATAL: ObservationLevel.ERROR,
CRITICAL: ObservationLevel.ERROR,
};
function parseObservationLevel(
value: unknown,
): ObservationLevelType | undefined {
if (typeof value !== "string") return undefined;
return OBSERVATION_LEVEL_ALIASES[value.trim().toUpperCase()];
}
// Type definitions for internal processor state
interface TraceState {
hasFullTrace: boolean;
shallowEventIds: string[];
}
export interface OtelIngestionProcessorConfig {
projectId: string;
publicKey: string;
orgId?: string;
propagatedHeaders?: Record<string, string>;
sdkName: string;
sdkVersion: string;
ingestionVersion?: string;
/**
* Marks Langfuse-internal telemetry (e.g. LLM-as-a-judge /
* prompt-experiment executions). Propagated through the ingestion queue so
* the consumer parses these events with the internal ingestion schema,
* preserving the reserved "langfuse-" environment prefix that the public
* schema strips.
*/
isLangfuseInternal?: boolean;
/**
* S3 key of the raw OTLP payload this batch was replayed from. Only set on
* the queue-consumer side; included in conversion failure logs so a single
* log line points at the replayable payload.
*/
fileKey?: string;
}
interface MetadataDropContext {
domain: string;
attributeKey: string;
dropScope: object;
}
// OTel `telemetry.sdk.language` values emitted by the Langfuse SDK runtimes.
const OTEL_SDK_LANGUAGE_TO_SDK_NAME = new Map<
string,
IngestionSdkCanonicalName
>([
["python", "python"],
["nodejs", "javascript"],
["webjs", "javascript"],
]);
const DANGEROUS_OTEL_PATH_SEGMENTS = new Set([
"__proto__",
"constructor",
"prototype",
]);
const CANONICAL_ARRAY_INDEX = /^(0|[1-9]\d*)$/;
type ReconstructedAttributeDropReason =
| "reconstruction_array_index_exceeded"
| "reconstruction_budget_exceeded"
| "reconstruction_path_depth_exceeded";
const getOtelArrayIndex = (segment: string): number | undefined => {
if (!CANONICAL_ARRAY_INDEX.test(segment)) return undefined;
const index = Number(segment);
return Number.isSafeInteger(index) ? index : undefined;
};
/**
* Tracks the total number of array slots that dotted attribute paths would
* materialize. A path is reserved transactionally: when it would exceed the
* limit, none of its array lengths are committed.
*
* Multiple attributes can point into the same array, so only growth beyond
* that array's previously reserved length consumes budget. For example,
* `0.role` and `0.content` together reserve one slot, while `9999.content`
* reserves 10,000 slots even though it is only one attribute.
*/
class ArraySlotBudget {
private usedSlots = 0;
private readonly lengthsByPath = new Map<string, number>();
constructor(private readonly maxSlots: number) {}
tryReserve(pathParts: readonly string[]): boolean {
const pendingLengths = new Map<string, number>();
let additionalSlots = 0;
for (let index = 0; index < pathParts.length; index++) {
const segment = pathParts[index];
const arrayIndex = getOtelArrayIndex(segment);
if (arrayIndex === undefined) continue;
const arrayPath = pathParts.slice(0, index).join(".");
const previousLength =
pendingLengths.get(arrayPath) ?? this.lengthsByPath.get(arrayPath) ?? 0;
const requiredLength = arrayIndex + 1;
if (requiredLength > previousLength) {
additionalSlots += requiredLength - previousLength;
pendingLengths.set(arrayPath, requiredLength);
}
}
if (this.usedSlots + additionalSlots > this.maxSlots) return false;
this.usedSlots += additionalSlots;
pendingLengths.forEach((length, path) =>
this.lengthsByPath.set(path, length),
);
return true;
}
}
interface CreateTraceEventParams {
traceId: string;
startTimeISO: string;
attributes: Record<string, unknown>;
resourceAttributes: Record<string, unknown>;
resourceAttributeMetadata: Record<string, unknown>;
spanAttributeMetadata: Record<string, unknown>;
scopeSpan: any;
scopeAttributes: Record<string, unknown>;
isLangfuseSDKSpans: boolean;
decodeMetadataValues: boolean;
isRootSpan: boolean;
hasTraceUpdates: boolean;
parentObservationId: string | null;
span: any;
}
interface CreateObservationEventParams {
span: any;
traceId: string;
parentObservationId: string | null;
attributes: Record<string, unknown>;
resourceAttributes: Record<string, unknown>;
resourceAttributeMetadata: Record<string, unknown>;
spanAttributeMetadata: Record<string, unknown>;
scopeSpan: any;
scopeAttributes: Record<string, unknown>;
isLangfuseSDKSpans: boolean;
startTimeISO: string;
endTimeISO: string;
}
type NanoTimestamp =
| number
| string
| {
high: number;
low: number;
}
| null
| undefined;
type TimestampField = "start_time" | "end_time" | "unknown";
export interface ResourceSpan {
resource?: {
attributes?: Array<{ key: string; value: any }>;
};
scopeSpans?: Array<{
scope?: {
name: string;
version?: string;
attributes?: Array<{ key: string; value: any }>;
};
spans?: Array<{
traceId: { data?: Buffer } | Buffer;
spanId: { data?: Buffer } | Buffer;
parentSpanId?: { data?: Buffer } | Buffer;
name: string;
kind: number;
startTimeUnixNano?: NanoTimestamp;
endTimeUnixNano?: NanoTimestamp;
attributes?: Array<{ key: string; value: any }>;
events?: any[];
status?: { code?: number; message?: string };
}>;
}>;
}
const observationTypeMapper = new ObservationTypeMapperRegistry();
/**
* Processor class that encapsulates all logic for converting OpenTelemetry
* resource spans into Langfuse ingestion events.
*
* Manages trace deduplication internally and provides a clean interface
* for converting OTEL spans to Langfuse events.
*/
const LIVEKIT_DEBUG_SPAN_NAMES = new Set([
"user_speaking",
"eou_detection",
"llm_request_run",
"tts_node",
"tts_request",
"tts_request_run",
"tts_stream_adapter",
"agent_speaking",
"drain_agent_activity",
"on_exit",
"on_enter",
"llm_fallback_adapter",
"tts_fallback_adapter",
"start_agent_activity",
"on_enter",
]);
export class OtelIngestionProcessor {
private static readonly OTEL_CONVERSION_FAILURE_METRIC =
"langfuse.ingestion.otel.conversion_failure";
private static readonly OTEL_ARRAY_ATTRIBUTE_DROPPED_METRIC =
"langfuse.ingestion.otel.array_attribute_dropped";
// Flattened OTel message attributes can create deeply nested objects and
// sparse arrays, so bound both forms of structural expansion.
private static readonly MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS = 10_001;
private static readonly MAX_OTEL_RECONSTRUCTED_PATH_DEPTH = 64;
private static readonly ARRAY_ATTRIBUTE_DROP_WARN_CAP = 10;
private seenTraces: Set<string> = new Set();
private reportedMetadataDrops = new WeakMap<object, Set<string>>();
private arrayAttributeDropWarnCounts = new Map<
ReconstructedAttributeDropReason,
number
>();
private isInitialized = false;
private traceEventCounts = {
shallow: 0,
rootSpanClosed: 0,
traceUpdated: 0,
};
private readonly projectId: string;
private readonly publicKey: string;
private readonly orgId?: string;
private readonly propagatedHeaders?: Record<string, string>;
private readonly sdkName: string;
private readonly sdkVersion: string;
private readonly ingestionVersion?: string;
private readonly isLangfuseInternal?: boolean;
private readonly fileKey?: string;
constructor(config: OtelIngestionProcessorConfig) {
this.projectId = config.projectId;
this.publicKey = config.publicKey;
this.orgId = config.orgId;
this.propagatedHeaders = config.propagatedHeaders;
this.sdkName = config.sdkName;
// Langfuse SDK version from x-langfuse-sdk-version. This is persisted as
// ingestionSdkVersion attribution on emitted events.
this.sdkVersion = config.sdkVersion;
// Ingestion protocol version from x-langfuse-ingestion-version. This is
// only used as a write-path hint, not as SDK attribution.
this.ingestionVersion = config.ingestionVersion;
this.isLangfuseInternal = config.isLangfuseInternal;
this.fileKey = config.fileKey;
}
/**
* Returns the current time as yyyy/mm/dd/hh/mm`.
*/
private getCurrentTimePath(): string {
const now = new Date();
return `${now.getFullYear()}/${String(now.getMonth() + 1).padStart(2, "0")}/${String(now.getDate()).padStart(2, "0")}/${String(now.getHours()).padStart(2, "0")}/${String(now.getMinutes()).padStart(2, "0")}`;
}
/**
* Uploads a batch of resourceSpans to blob storage and adds a job to process them
* into the otel-ingestion-queue.
*/
async publishToOtelIngestionQueue(resourceSpans: ResourceSpan[]) {
const fileKey = `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}otel/${this.projectId}/${this.getCurrentTimePath()}/${randomUUID()}.json`;
// Upload to S3
await getS3EventStorageClient(
env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
).uploadJson(fileKey, resourceSpans as Record<string, unknown>[]);
// Add queue job
const queue = OtelIngestionQueue.getInstance({
shardingKey: `${this.projectId}-${fileKey}`,
});
return queue
? queue.add(QueueJobs.OtelIngestionJob, {
id: randomUUID(),
timestamp: new Date(),
name: QueueJobs.OtelIngestionJob as const,
payload: {
data: {
fileKey,
publicKey: this.publicKey,
},
authCheck: {
validKey: true,
scope: {
projectId: this.projectId,
accessLevel: "project" as const,
orgId: this.orgId,
},
},
propagatedHeaders: this.propagatedHeaders,
sdkName: this.sdkName,
sdkVersion: this.sdkVersion,
ingestionVersion: this.ingestionVersion,
...(this.isLangfuseInternal ? { isLangfuseInternal: true } : {}),
},
})
: Promise.reject("Failed to instantiate otel ingestion queue");
}
/**
* Processes incoming resourceSpans and produces an event base record that can be enriched
* using the IngestionService.
* @param resourceSpans
*/
processToEvent(resourceSpans: ResourceSpan[]): any[] {
return instrumentSync({ name: "otel-event-processor" }, (span) => {
try {
span.setAttribute("project_id", this.projectId);
span.setAttribute(
"total_span_count",
this.getTotalSpanCount(resourceSpans),
);
// Input validation
if (!Array.isArray(resourceSpans)) {
return [];
}
if (resourceSpans.length === 0) {
return [];
}
const events = resourceSpans
.filter((r) => Boolean(r))
.flatMap((resourceSpan) => {
const resourceAttributes =
this.extractResourceAttributes(resourceSpan);
const events: any[] = [];
for (const scopeSpan of resourceSpan?.scopeSpans ?? []) {
const scopeAttributes = this.extractScopeAttributes(scopeSpan);
for (const span of scopeSpan?.spans ?? []) {
const spanAttributes = this.extractSpanAttributes(span);
const traceId = this.parseId(span.traceId);
const spanId = this.parseId(span.spanId);
const parentSpanId = span?.parentSpanId
? this.parseId(span.parentSpanId)
: null;
const { startTimeISO, endTimeISO } =
OtelIngestionProcessor.resolveSpanTimestamps({
startTimeUnixNano: span.startTimeUnixNano,
endTimeUnixNano: span.endTimeUnixNano,
});
const isLangfuseSDKSpans =
scopeSpan.scope?.name?.startsWith("langfuse-sdk") ?? false;
const decodeMetadataValues = this.shouldDecodeMetadataValues(
scopeSpan,
resourceAttributes,
);
// Extract metadata from different sources
const spanMetadata = this.extractMetadata(
spanAttributes,
"observation",
span,
decodeMetadataValues,
);
const traceMetadata = this.extractMetadata(
spanAttributes,
"trace",
span,
decodeMetadataValues,
);
const { input, output, filteredAttributes } =
this.extractInputAndOutput({
events: span?.events ?? [],
attributes: spanAttributes,
instrumentationScopeName: scopeSpan?.scope?.name ?? "",
});
// Construct metadata object with the specified structure
// Match v3 path: store full scope object (name, version, attributes)
const metadata = {
...(isLangfuseSDKSpans
? {}
: { attributes: filteredAttributes }),
resourceAttributes: resourceAttributes,
scope: {
...(scopeSpan.scope || {}),
attributes: scopeAttributes,
},
// Note: top-level user metadata can overwrite `scope` here.
// This preserves the v3 behavior/shape, even though it means
// the instrumentation scope object is not guaranteed to win.
...spanMetadata,
...traceMetadata,
};
const normalizedTools = normalizeToolsForObservation(
input,
output,
metadata,
);
// Extract instrumentation metadata
const serviceName = resourceAttributes?.["service.name"] as
| string
| undefined;
const serviceVersion = resourceAttributes?.[
"service.version"
] as string | undefined;
const telemetrySdkLanguage = resourceAttributes?.[
"telemetry.sdk.language"
] as string | undefined;
const telemetrySdkName = resourceAttributes?.[
"telemetry.sdk.name"
] as string | undefined;
const telemetrySdkVersion = resourceAttributes?.[
"telemetry.sdk.version"
] as string | undefined;
const scopeName = scopeSpan?.scope?.name;
const scopeVersion = scopeSpan?.scope?.version;
const stringifiedSpan = JSON.stringify(span);
const eventBytes = Buffer.byteLength(stringifiedSpan, "utf8");
recordDistribution(
"langfuse.ingestion.otel.event.byte_length",
eventBytes,
{
source: "otel",
sdk_language: telemetrySdkLanguage || "",
},
);
this.logOversizedSpan(span, spanAttributes, {
spanId,
traceId,
eventBytes,
});
const experimentFields = this.extractExperimentFields(
spanAttributes,
span,
);
const evaluationFields =
this.extractEvaluationFields(spanAttributes);
const spanContext = {
spanId,
traceId,
source: "event" as const,
};
const observationType =
observationTypeMapper.mapToObservationType(
spanAttributes,
resourceAttributes,
scopeSpan?.scope,
span.name,
);
// AI SDK agent spans carry aggregate usage duplicating their
// child model-call spans — skip model/usage/cost for them.
// Also Claude Managed Agent create_session spans have a model logged
// but no usage
const skipModelUsageAndCost = this.shouldSkipModelUsageAndCost(
spanAttributes,
observationType,
scopeName,
span.name,
);
const usageDetails = UsageDetails.safeParse(
skipModelUsageAndCost
? {}
: this.extractUsageDetails(
spanAttributes,
scopeSpan?.scope?.name ?? "",
spanContext,
),
);
if (!usageDetails.success) {
logger.warn(
`Invalid usage details extracted from OTEL span for traceId ${traceId}: ${JSON.stringify(usageDetails.error)}`,
);
}
const toolDefinitions =
Object.keys(normalizedTools.toolDefinitions).length > 0
? normalizedTools.toolDefinitions
: undefined;
const toolCalls =
normalizedTools.toolCalls.length > 0
? normalizedTools.toolCalls
: undefined;
const toolCallNames =
normalizedTools.toolCallNames.length > 0
? normalizedTools.toolCallNames
: undefined;
// Prompts can only be linked to GENERATION observations
const canLinkPrompt =
observationType === ObservationType.GENERATION;
const name = this.extractName(
span.name,
spanAttributes,
observationType,
scopeSpan.scope?.name,
);
const level =
parseObservationLevel(
spanAttributes[
LangfuseOtelSpanAttributes.OBSERVATION_LEVEL
],
) ??
(span.status?.code === 2
? ObservationLevel.ERROR
: scopeSpan?.scope?.name === "livekit-agents" &&
LIVEKIT_DEBUG_SPAN_NAMES.has(span.name)
? ObservationLevel.DEBUG
: ObservationLevel.DEFAULT);
events.push({
projectId: this.projectId,
traceId,
spanId,
parentSpanId,
name,
type: observationType,
environment: this.extractEnvironment(
spanAttributes,
resourceAttributes,
),
version:
spanAttributes?.[LangfuseOtelSpanAttributes.VERSION] ??
resourceAttributes?.["service.version"] ??
null,
startTimeISO,
endTimeISO,
level,
statusMessage:
spanAttributes[
LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE
] ??
span.status?.message ??
null,
...extractSkillsFromObservation({
skillsAvailable:
spanAttributes[
LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE
],
observationType,
name,
input,
output,
isError: level === ObservationLevel.ERROR,
metadata,
projectId: this.projectId,
}),
promptName: canLinkPrompt
? (spanAttributes?.[
LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME
] ??
spanAttributes["langfuse.prompt.name"] ??
this.parseLangfusePromptFromAISDK(spanAttributes)?.name ??
null)
: null,
promptVersion: canLinkPrompt
? this.extractPromptVersion(spanAttributes)
: null,
modelParameters: this.extractModelParameters(
spanAttributes,
scopeSpan?.scope?.name ?? "",
),
modelName: skipModelUsageAndCost
? undefined
: this.extractModelName(spanAttributes),
completionStartTime: this.extractCompletionStartTime(
spanAttributes,
startTimeISO,
),
// Usage and cost details
providedUsageDetails: usageDetails.success
? usageDetails.data
: undefined,
providedCostDetails: skipModelUsageAndCost
? {}
: this.extractCostDetails(spanAttributes, spanContext),
// Properties
tags: this.extractTags(spanAttributes, decodeMetadataValues),
public: this.extractPublic(spanAttributes),
isAppRoot: this.extractIsAppRoot(spanAttributes),
traceName:
spanAttributes?.[LangfuseOtelSpanAttributes.TRACE_NAME] ??
null,
userId: this.extractUserId(
spanAttributes,
decodeMetadataValues,
resourceAttributes,
),
sessionId: this.extractSessionId(
spanAttributes,
decodeMetadataValues,
scopeSpan.scope?.name,
),
release:
(spanAttributes?.[
LangfuseOtelSpanAttributes.RELEASE
] as string) ??
resourceAttributes?.[LangfuseOtelSpanAttributes.RELEASE] ??
null,
input: normalizedTools.input,
output: normalizedTools.output,
// Metadata
metadata: normalizedTools.metadata,
// Instrumentation metadata
source: "otel",
ingestionApiKey: this.publicKey,
ingestionSdkName: this.sdkName,
ingestionSdkVersion: this.sdkVersion,
serviceName,
serviceVersion,
scopeName,
scopeVersion,
telemetrySdkLanguage,
telemetrySdkName,
telemetrySdkVersion,
// Source data
// eventRaw: stringifiedSpan,
eventBytes,
// Experiment fields
...experimentFields,
// Evaluator execution fields
...evaluationFields,
// Tool calling
toolDefinitions,
toolCalls,
toolCallNames,
});
}
}
return events;
});
return events;
} catch (error) {
logger.error("Error processing OTEL spans to events:", {
error,
...this.getConversionFailureLogContext(resourceSpans),
});
traceException(error, span);
throw error;
}
});
}
/**
* Process resource spans and convert them to Langfuse ingestion events.
* Handles trace deduplication automatically using internal state.
* Initializes seen traces from Redis automatically on first call.
* Filters out shallow trace events if full trace events exist for the same traceId.
*/
async processToIngestionEvents(
resourceSpans: ResourceSpan[],
): Promise<IngestionEventType[]> {
return await instrumentAsync(
{ name: "otel-ingestion-processor" },
async (span) => {
span.setAttribute("project_id", this.projectId);
span.setAttribute(
"total_span_count",
this.getTotalSpanCount(resourceSpans),
);
try {
// Lazy initialization - load seen traces from Redis if not already done
// Seen traces are traces that went through the ingestion pipeline within last 10 minutes
if (!this.isInitialized) {
this.seenTraces = await this.getSeenTracesSet(resourceSpans);
this.isInitialized = true;
}
// Input validation
if (!Array.isArray(resourceSpans)) {
return [];
}
if (resourceSpans.length === 0) {
return [];
}
// Process all events normally first
const allEvents = resourceSpans.flatMap((resourceSpan) => {
if (!resourceSpan) return [];
return this.processResourceSpan(resourceSpan);
});
// Filter out redundant shallow trace events
const finalEvents = this.filterRedundantShallowTraces(allEvents);
span.setAttribute("events_generated", finalEvents.length);
this.traceEventCounts.shallow = Math.max(
this.traceEventCounts.shallow -
(allEvents.length - finalEvents.length),
0,
);
for (const key of Object.keys(
this.traceEventCounts,
) as (keyof typeof this.traceEventCounts)[]) {
recordIncrement(
"langfuse.ingestion.otel.trace_create_event",
this.traceEventCounts[key],
{ reason: key },
);
}
return finalEvents;
} catch (error) {
if (error instanceof ForbiddenError) {
traceException(error, span);
throw error;
}
// Log error but don't throw to avoid breaking the ingestion pipeline
logger.error("Error processing OTEL spans:", {
error,
...this.getConversionFailureLogContext(resourceSpans),
});
traceException(error, span);
return [];
}
},
);
}
/**
* Filter out shallow trace-create events if a full trace-create event exists for the same traceId.
* Maintains optimal trace representation per traceId in the final event list.
*
* Performance: O(n) where n is the number of events
*/
private filterRedundantShallowTraces(
events: IngestionEventType[],
): IngestionEventType[] {
if (events.length === 0) return events;
// Fast path: if no trace-create events, return as-is
const hasTraceEvents = events.some(
(event) => event.type === "trace-create",
);
if (!hasTraceEvents) return events;
// Track trace states by traceId - using simpler structure for better performance
const traceStates = new Map<string, TraceState>();
// Single pass: categorize trace-create events by type
for (const event of events) {
if (event.type === "trace-create") {
const traceId = event.body.id as string;
if (!traceStates.has(traceId)) {
// Initialize entry
traceStates.set(traceId, {
hasFullTrace: false,
shallowEventIds: [],
});
}
const traceState = traceStates.get(traceId)!;
const isShallowTrace = this.isShallowTraceEvent(event.body);
if (isShallowTrace) {
traceState.shallowEventIds.push(event.id);
} else {
traceState.hasFullTrace = true;
}
}
}
// Collect event IDs to exclude (only if we have conflicts)
const eventIdsToExclude = new Set<string>();
for (const traceState of traceStates.values()) {
if (traceState.hasFullTrace && traceState.shallowEventIds.length > 0) {
traceState.shallowEventIds.forEach((id) => eventIdsToExclude.add(id));
}
}
// Early return if no filtering needed
if (eventIdsToExclude.size === 0) return events;
// Filter out redundant shallow traces
return events.filter((event) => !eventIdsToExclude.has(event.id));
}
/**
* Determine if a trace event body represents a shallow trace.
* Shallow traces only contain minimal fields: id, timestamp, environment.
* Full traces have additional meaningful fields like: name, metadata, userId, etc.
*/
private isShallowTraceEvent(traceBody: TraceEventType["body"]): boolean {
// Define minimal fields that shallow traces have
const SHALLOW_TRACE_FIELDS = new Set(["id", "timestamp", "environment"]);
// Check for presence of any full trace indicators
// These fields indicate a full trace with meaningful trace-level data
const FULL_TRACE_INDICATORS = [
"name",
"metadata",
"userId",
"sessionId",
"public",
"tags",
"version",
"release",
"input",
"output",
] as const;
// Fast path: check for any full trace indicators with meaningful values
for (const field of FULL_TRACE_INDICATORS) {
const value = traceBody[field];
if (this.hasMeaningfulValue(value)) {
return false; // Has full trace data
}
}
// If no full trace indicators, verify it has the basic shallow fields
return (
SHALLOW_TRACE_FIELDS.has("id") &&
this.hasMeaningfulValue(traceBody.id) &&
this.hasMeaningfulValue(traceBody.timestamp) &&
this.hasMeaningfulValue(traceBody.environment)
);
}
/**
* Check if a value is meaningful (not null, undefined, empty string, or empty object/array)
*/
private hasMeaningfulValue(value: unknown): boolean {
if (isNullOrUndefined(value) || value === "") {
return false;
}
if (Array.isArray(value)) {
return value.length > 0;
}
if (typeof value === "object" && value !== null) {
return Object.keys(value).length > 0;
}
return true;
}
private processResourceSpan(
resourceSpan: ResourceSpan,
): IngestionEventType[] {
const resourceAttributes = this.extractResourceAttributes(resourceSpan);
const events: IngestionEventType[] = [];
for (const scopeSpan of resourceSpan?.scopeSpans ?? []) {
const isLangfuseSDKSpans =
scopeSpan.scope?.name?.startsWith("langfuse-sdk") ?? false;
const scopeAttributes = this.extractScopeAttributes(scopeSpan);
if (isLangfuseSDKSpans) {
recordIncrement("langfuse.otel.ingestion.langfuse_sdk_batch", 1);
}
for (const span of scopeSpan?.spans ?? []) {
const spanEvents = this.processSpan(
span,
scopeSpan,
resourceAttributes,
scopeAttributes,
isLangfuseSDKSpans,
);
events.push(...spanEvents);
}
}
return events;
}
private processSpan(
span: any,
scopeSpan: any,
resourceAttributes: Record<string, unknown>,
scopeAttributes: Record<string, unknown>,
isLangfuseSDKSpans: boolean,
): IngestionEventType[] {
const events: IngestionEventType[] = [];
const attributes = this.extractSpanAttributes(span);
const traceId = this.parseId(span.traceId?.data ?? span.traceId);
const parentObservationId = span?.parentSpanId
? this.parseId(span.parentSpanId?.data ?? span.parentSpanId)
: null;
const decodeMetadataValues = this.shouldDecodeMetadataValues(
scopeSpan,
resourceAttributes,
);
const spanAttributeMetadata = this.extractMetadata(
attributes,
"observation",
span,
decodeMetadataValues,
);
const resourceAttributeMetadata = this.extractMetadata(
resourceAttributes,
"trace",
resourceAttributes,
);
const { startTimeISO, endTimeISO } =
OtelIngestionProcessor.resolveSpanTimestamps({
startTimeUnixNano: span.startTimeUnixNano,
endTimeUnixNano: span.endTimeUnixNano,
});
const isRootSpan =
!parentObservationId ||
String(attributes[LangfuseOtelSpanAttributes.AS_ROOT]) === "true";
const hasTraceUpdates = this.hasTraceUpdates(attributes);
// Handle trace creation logic with internal seen traces management
if (isRootSpan || hasTraceUpdates || !this.seenTraces.has(traceId)) {
const traceEvent = this.createTraceEvent({
traceId,
startTimeISO,
attributes,
resourceAttributes,
resourceAttributeMetadata,
spanAttributeMetadata,
scopeSpan,
scopeAttributes,
isLangfuseSDKSpans,
decodeMetadataValues,
isRootSpan,
hasTraceUpdates,
parentObservationId,
span,
});
events.push(traceEvent);
// Update internal seen traces cache
this.seenTraces.add(traceId);
}
// Create observation event
const observationEvent = this.createObservationEvent({
span,
traceId,
parentObservationId,
attributes,
resourceAttributes,
resourceAttributeMetadata,
spanAttributeMetadata,
scopeSpan,
scopeAttributes,
isLangfuseSDKSpans,
startTimeISO,
endTimeISO,
});
events.push(observationEvent);
return events;
}
private createTraceEvent(params: CreateTraceEventParams): IngestionEventType {
const {
traceId,
startTimeISO,
attributes,
resourceAttributes,
resourceAttributeMetadata,
spanAttributeMetadata,
scopeSpan,
scopeAttributes,
isLangfuseSDKSpans,
decodeMetadataValues,
isRootSpan,
hasTraceUpdates,
span,
} = params;
// Create shallow trace for new traces without root span or trace updates
let trace: TraceEventType["body"] = {
id: traceId,
timestamp: startTimeISO,
environment: this.extractEnvironment(attributes, resourceAttributes),
};
const instrumentationScopeName = scopeSpan?.scope?.name as string;
// Create full trace for root spans or spans with trace updates
if (isRootSpan) {
// Extract input/output and get filtered attributes
const { input, output, filteredAttributes } = this.extractInputAndOutput({
events: span?.events ?? [],
attributes,
domain: "trace",
instrumentationScopeName,
});
trace = {
...trace,
name:
(attributes[LangfuseOtelSpanAttributes.TRACE_NAME] as string) ??
this.extractName(span.name, attributes),
metadata: {
...resourceAttributeMetadata,
...this.extractMetadata(
attributes,
"trace",
span,
decodeMetadataValues,
),
...spanAttributeMetadata,
...(isLangfuseSDKSpans ? {} : { attributes: filteredAttributes }),
resourceAttributes,
scope: {
...(scopeSpan.scope || {}),
attributes: scopeAttributes,
},
} as Record<string, string | Record<string, string | number>>,
version:
(attributes?.[LangfuseOtelSpanAttributes.VERSION] as string) ??
resourceAttributes?.["service.version"] ??
null,
release:
(attributes?.[LangfuseOtelSpanAttributes.RELEASE] as string) ??
resourceAttributes?.[LangfuseOtelSpanAttributes.RELEASE] ??
null,
userId: this.extractUserId(
attributes,
decodeMetadataValues,
resourceAttributes,
),
sessionId: this.extractSessionId(
attributes,
decodeMetadataValues,
instrumentationScopeName,
),
public: this.extractPublic(attributes),
tags: this.extractTags(attributes, decodeMetadataValues),
environment: this.extractEnvironment(attributes, resourceAttributes),
input,
output,
};
}
if (hasTraceUpdates && !isRootSpan) {
trace = {
...trace,
name: attributes[LangfuseOtelSpanAttributes.TRACE_NAME] as string,
metadata: {
...resourceAttributeMetadata,
...this.extractMetadata(
attributes,
"trace",
span,
decodeMetadataValues,
),
// removed to not remove trace metadata->attributes through subsequent observations
// ...(isLangfuseSDKSpans
// ? {}
// : { attributes: spanAttributesInMetadata }),
resourceAttributes,
scope: {
...(scopeSpan.scope || {}),
attributes: scopeAttributes,
},
} as Record<string, string | Record<string, string | number>>,
version:
(attributes?.[LangfuseOtelSpanAttributes.VERSION] as string) ??
resourceAttributes?.["service.version"] ??
null,
release:
(attributes?.[LangfuseOtelSpanAttributes.RELEASE] as string) ??
resourceAttributes?.[LangfuseOtelSpanAttributes.RELEASE] ??
null,
// Child updates must not replace an explicit trace user with a fallback.
userId: this.extractExplicitUserId(attributes, decodeMetadataValues),
/** CMA session inference belongs to root traces and individual observations. */
sessionId: this.extractSessionId(
attributes,
decodeMetadataValues,
undefined,
),
public: this.extractPublic(attributes),
tags: this.extractTags(attributes, decodeMetadataValues),
environment: this.extractEnvironment(attributes, resourceAttributes),
input: attributes[LangfuseOtelSpanAttributes.TRACE_INPUT],
output: attributes[LangfuseOtelSpanAttributes.TRACE_OUTPUT],
};
}
if (isRootSpan) {
this.traceEventCounts.rootSpanClosed += 1;
} else if (hasTraceUpdates) {
this.traceEventCounts.traceUpdated += 1;
} else {
this.traceEventCounts.shallow += 1;
}
return {
id: randomUUID(),
type: "trace-create",
timestamp: new Date(startTimeISO).toISOString(),
body: trace,
};
}
private extractPublic(
attributes?: Record<string, unknown>,
): boolean | undefined {
const value =
attributes?.[LangfuseOtelSpanAttributes.TRACE_PUBLIC] ??
attributes?.["langfuse.public"];
if (value == null) return;
return value === true || value === "true";
}
private extractIsAppRoot(attributes?: Record<string, unknown>): boolean {
const value = attributes?.[LangfuseOtelSpanAttributes.IS_APP_ROOT];
return value === true || value === "true";
}
private createObservationEvent(
params: CreateObservationEventParams,
): IngestionEventType {
const {
span,
traceId,
parentObservationId,
attributes,
resourceAttributes,
resourceAttributeMetadata,
spanAttributeMetadata,
scopeSpan,
scopeAttributes,
isLangfuseSDKSpans,
startTimeISO,
endTimeISO,
} = params;
const instrumentationScopeName = scopeSpan?.scope?.name;
// Extract input/output and get filtered attributes
const { input, output, filteredAttributes } = this.extractInputAndOutput({
events: span?.events ?? [],
attributes,
instrumentationScopeName,
});
const observationSpanId = this.tryParseSpanId(span);
const observationContext = {
spanId: observationSpanId,
traceId,
source: "ingestion" as const,
};
const metadata = {
...resourceAttributeMetadata,
...spanAttributeMetadata,
...(isLangfuseSDKSpans ? {} : { attributes: filteredAttributes }),
resourceAttributes,
scope: { ...scopeSpan.scope, attributes: scopeAttributes },
};
const normalizedToolMetadata = normalizeToolMetadataForObservation(
input,
metadata,
);
const mappedObservationType = observationTypeMapper.mapToObservationType(
attributes,
resourceAttributes,
scopeSpan?.scope,
span.name,
);
const skipModelUsageAndCost = this.shouldSkipModelUsageAndCost(
attributes,
mappedObservationType,
instrumentationScopeName,
span.name,
);
// Prompts can only be linked to GENERATION observations
const canLinkPrompt = mappedObservationType === ObservationType.GENERATION;
const name = this.extractName(
span.name,
attributes,
mappedObservationType,
instrumentationScopeName,
);
const level =
parseObservationLevel(
attributes[LangfuseOtelSpanAttributes.OBSERVATION_LEVEL],
) ??
(span.status?.code === 2
? ObservationLevel.ERROR
: scopeSpan?.scope?.name === "livekit-agents" &&
LIVEKIT_DEBUG_SPAN_NAMES.has(span.name)
? ObservationLevel.DEBUG
: ObservationLevel.DEFAULT);
const observation = {
id: this.parseId(span.spanId?.data ?? span.spanId),
traceId,
parentObservationId,
name,
startTime: startTimeISO,
endTime: endTimeISO,
environment: this.extractEnvironment(attributes, resourceAttributes),
completionStartTime: this.extractCompletionStartTime(
attributes,
startTimeISO,
),
metadata: normalizedToolMetadata.metadata,
level,
statusMessage:
attributes[LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE] ??
span.status?.message ??
null,
version:
attributes[LangfuseOtelSpanAttributes.VERSION] ??
resourceAttributes?.["service.version"] ??
null,
modelParameters: this.extractModelParameters(
attributes,
instrumentationScopeName,
) as any,
model: skipModelUsageAndCost
? undefined
: this.extractModelName(attributes),
...extractSkillsFromObservation({
skillsAvailable:
attributes[LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE],
observationType: mappedObservationType,
name,
input,
output,
isError: level === ObservationLevel.ERROR,
metadata,
projectId: this.projectId,
}),
promptName: canLinkPrompt
? (attributes?.[LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME] ??
attributes["langfuse.prompt.name"] ??
this.parseLangfusePromptFromAISDK(attributes)?.name ??
null)
: null,
promptVersion: canLinkPrompt
? this.extractPromptVersion(attributes)
: null,
usageDetails: skipModelUsageAndCost
? {}
: this.extractUsageDetails(
attributes,
instrumentationScopeName,
observationContext,
),
costDetails: skipModelUsageAndCost
? {}
: this.extractCostDetails(attributes, observationContext),
input: normalizedToolMetadata.input,
output,
};
const observationType =
mappedObservationType && typeof mappedObservationType === "string"
? mappedObservationType.toLowerCase()
: undefined;
const isKnownObservationType =
observationType &&
ObservationTypeDomain.safeParse(observationType.toUpperCase()).success;
const getIngestionEventType = (): string => {
if (isKnownObservationType) {
return `${observationType}-create`;
}
return "span-create";
};
return {
id: randomUUID(),
type: getIngestionEventType(),
timestamp: new Date().toISOString(),
body: observation,
} as unknown as IngestionEventType;
}
private hasTraceUpdates(attributes: Record<string, unknown>): boolean {
const hasExactMatchingAttributeName = [
LangfuseOtelSpanAttributes.TRACE_NAME,
LangfuseOtelSpanAttributes.TRACE_INPUT,
LangfuseOtelSpanAttributes.TRACE_OUTPUT,
LangfuseOtelSpanAttributes.TRACE_METADATA,
LangfuseOtelSpanAttributes.TRACE_USER_ID,
LangfuseOtelSpanAttributes.TRACE_SESSION_ID,
LangfuseOtelSpanAttributes.TRACE_PUBLIC,
LangfuseOtelSpanAttributes.TRACE_TAGS,
LangfuseOtelSpanAttributes.TRACE_COMPAT_USER_ID,
LangfuseOtelSpanAttributes.TRACE_COMPAT_SESSION_ID,
// OpenAI and Langchain integrations
`${LangfuseOtelSpanAttributes.OBSERVATION_METADATA}.langfuse_user_id`,
`${LangfuseOtelSpanAttributes.OBSERVATION_METADATA}.langfuse_session_id`,
`${LangfuseOtelSpanAttributes.OBSERVATION_METADATA}.langfuse_tags`,
`${LangfuseOtelSpanAttributes.TRACE_METADATA}.langfuse_session_id`,
`${LangfuseOtelSpanAttributes.TRACE_METADATA}.langfuse_user_id`,
`${LangfuseOtelSpanAttributes.TRACE_METADATA}.langfuse_tags`,
// Vercel AI SDK
`ai.telemetry.metadata.sessionId`,
`ai.telemetry.metadata.userId`,
`ai.telemetry.metadata.tags`,
// LlamaIndex
`tag.tags`,
].some((traceAttribute) => Boolean(attributes[traceAttribute]));
const attributeKeys = Object.keys(attributes);
const hasTraceMetadataKey = attributeKeys.some((key) =>
key.startsWith(LangfuseOtelSpanAttributes.TRACE_METADATA),
);
return hasExactMatchingAttributeName || hasTraceMetadataKey;
}
private extractResourceAttributes(
resourceSpan: any,
): Record<string, unknown> {
return (
resourceSpan?.resource?.attributes?.reduce((acc: any, attr: any) => {
const value = this.convertValueToPlainJavascript(attr.value);
if (value !== undefined) acc[attr.key] = value;
return acc;
}, {}) ?? {}
);
}
private extractScopeAttributes(scopeSpan: any): Record<string, unknown> {
return (
scopeSpan?.scope?.attributes?.reduce((acc: any, attr: any) => {
const value = this.convertValueToPlainJavascript(attr.value);
if (value !== undefined) acc[attr.key] = value;
return acc;
}, {}) ?? {}
);
}
private extractSpanAttributes(span: any): Record<string, unknown> {
return (
span?.attributes?.reduce((acc: any, attr: any) => {
const value = this.convertValueToPlainJavascript(attr.value);
if (value !== undefined) acc[attr.key] = value;
return acc;
}, {}) ?? {}
);
}
private static isPlainObject(
value: unknown,
): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
private tryParseSpanId(span: any): string {
try {
if (span?.spanId != null) {
return this.parseId(span.spanId?.data ?? span.spanId);
}
} catch {
// Intentionally swallowed — return "unknown" below
}
return "unknown";
}
/**
* Parses a JSON-encoded attribute expected to be a plain object.
* Returns the parsed object on success, {} if present but wrong type (logged),
* null if absent or unparsable (signals caller to try fallback paths).
*/
private parseJsonObjectAttribute(
attributes: Record<string, unknown>,
key: string,
context: { spanId: string; traceId: string; source: string },
): Record<string, unknown> | null {
const raw = attributes[key];
if (!raw) return null;
const logContext = {
spanId: context.spanId,
traceId: context.traceId,
projectId: this.projectId,
attribute: key,
source: context.source,
};
const metricTags = {
attribute: key,
project_id: this.projectId,
source: context.source,
};
if (typeof raw !== "string") {
logger.warn("OTEL parseJsonObjectAttribute: non-string value", {
...logContext,
reason: "non_string",
valueType: typeof raw,
});
recordIncrement("langfuse.ingestion.otel.bad_json_attribute", 1, {
...metricTags,
reason: "non_string",
});
return {};
}
try {
const parsed = JSON.parse(raw);
if (!OtelIngestionProcessor.isPlainObject(parsed)) {
logger.warn("OTEL parseJsonObjectAttribute: non-object parsed value", {
...logContext,
reason: "not_plain_object",
parsedType: typeof parsed,
});
recordIncrement("langfuse.ingestion.otel.bad_json_attribute", 1, {
...metricTags,
reason: "not_plain_object",
});
return {};
}
return parsed;
} catch {
logger.warn("OTEL parseJsonObjectAttribute: invalid JSON", {
...logContext,
reason: "invalid_json",
});
recordIncrement("langfuse.ingestion.otel.bad_json_attribute", 1, {
...metricTags,
reason: "invalid_json",
});
return null;
}
}
/**
* Logs details about spans that exceed the size threshold.
* Does NOT reject — just logs for observability.
*/
private logOversizedSpan(
span: any,
spanAttributes: Record<string, unknown>,
context: { spanId: string; traceId: string; eventBytes: number },
): void {
const maxBytes = env.LANGFUSE_OTEL_MAX_SPAN_BYTES;
if (context.eventBytes <= maxBytes) return;
const fieldThreshold = 1_000_000; // 1MB
const largeFields: Record<string, number> = {};
// Check extracted span attributes
for (const [key, value] of Object.entries(spanAttributes)) {
if (typeof value === "string") {
const bytes = Buffer.byteLength(value, "utf8");
if (bytes > fieldThreshold) {
largeFields[key] = bytes;
}
}
}
// Check span events (gen_ai.user.message, gen_ai.choice, etc.)
// Prompts and completions following OTel GenAI semantic conventions
// live in span.events[], not in span attributes.
for (const event of span?.events ?? []) {
for (const attr of event?.attributes ?? []) {
const val = attr?.value?.stringValue;
if (typeof val === "string") {
const bytes = Buffer.byteLength(val, "utf8");
if (bytes > fieldThreshold) {
const fieldKey = `events[${event.name ?? "?"}].${attr.key}`;
largeFields[fieldKey] = bytes;
}
}
}
}
// The `langfuse.ingestion.otel.oversized_span` metric below carries the
// aggregate signal; keep the detailed line at debug to avoid drowning
// warn-level log volume with a per-span customer-data condition.
logger.debug("OTEL oversized span detected", {
spanId: context.spanId,
traceId: context.traceId,
projectId: this.projectId,
eventBytes: context.eventBytes,
maxBytes,
largeFields,
});
recordIncrement("langfuse.ingestion.otel.oversized_span", 1, {
project_id: this.projectId,
});
}
// Empty AnyValues are omitted from attribute maps; array callers preserve
// their positions as null instead.
private convertValueToPlainJavascript(
value: Record<string, any> | null | undefined,
): any {
if (
value == null ||
(OtelIngestionProcessor.isPlainObject(value) &&
Object.keys(value).length === 0)
) {
return undefined;
}
if (value.stringValue !== undefined) {
return value.stringValue;
}
if (value.doubleValue !== undefined) {
// OTLP/JSON encodes regular doubles as JSON numbers, but the special
// values "NaN", "Infinity" and "-Infinity" are encoded as strings.
// Coerce numeric strings; keep non-finite values as their string form so
// we never emit NaN/Infinity into metadata (which is not valid JSON and
// fails ingestion validation).
if (typeof value.doubleValue === "string") {
const parsed = Number(value.doubleValue);
return Number.isFinite(parsed) ? parsed : value.doubleValue;
}
return value.doubleValue;
}
if (value.boolValue !== undefined) {
return value.boolValue;
}
if (value.arrayValue) {
return (value.arrayValue.values ?? []).map(
(v: any) => this.convertValueToPlainJavascript(v) ?? null,
);
}
if (value.intValue !== undefined) {
const parsedInt = this.convertOtelIntValue(value.intValue);
if (parsedInt !== undefined) {
return parsedInt;
}
}
return JSON.stringify(value);
}
/**
* Converts an OTLP int64 attribute value into a plain number.
*
* The same logical value reaches us in different shapes depending on the
* transport:
* - protobuf (`application/x-protobuf`) is decoded into a Long-like object
* `{ low, high, unsigned }`.
* - OTLP/JSON (`application/json`) encodes int64 fields as decimal strings
* (e.g. `"7"`) per the spec, since JSON numbers cannot safely represent
* the full int64 range.
* - some encoders send a plain JavaScript number.
*
* Returns `undefined` for values we cannot parse so the caller can fall back
* to a safe representation instead of emitting `NaN`.
*/
private convertOtelIntValue(intValue: any): number | undefined {
if (typeof intValue === "number") {
return intValue;
}
if (typeof intValue === "string") {
const parsed = Number(intValue);
return Number.isNaN(parsed) ? undefined : parsed;
}
if (intValue && typeof intValue === "object") {
if (intValue.high === 0) {
return intValue.low;
}
if (intValue.high === -1 && intValue.low === -1) {
return -1;
}
if (typeof intValue.high === "number") {
return intValue.high * Math.pow(2, 32) + intValue.low;
}
}
return undefined;
}
/**
* Rebuilds observation input/output from flattened adapter attributes.
*
* TraceLoop and OpenInference encode structured messages in attribute names;
* for example, `gen_ai.prompt.0.content` becomes `[{ content: value }]`.
* Paths retain the legacy insertion-order conflict policy. Generated objects
* have no prototype, arrays only accept bounded numeric indices, and the
* writer never descends into an object supplied as an attribute value.
*/
private convertKeyPathToNestedObject(
input: Record<string, unknown>,
prefix: string,
): any {
if (input[prefix]) {
return input[prefix];
}
type ReconstructedContainer = Record<string, unknown> | unknown[];
const droppedAttributeCounts = new Map<
ReconstructedAttributeDropReason,
number
>();
const recordDrop = (reason: ReconstructedAttributeDropReason): void => {
droppedAttributeCounts.set(
reason,
(droppedAttributeCounts.get(reason) ?? 0) + 1,
);
};
const parsePath = (
key: string,
recordLimitDrop: boolean,
): string[] | undefined => {
const segments = key.split(".");
if (
segments.length >
OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_PATH_DEPTH
) {
if (recordLimitDrop) recordDrop("reconstruction_path_depth_exceeded");
return undefined;
}
for (const segment of segments) {
if (segment.length === 0 || DANGEROUS_OTEL_PATH_SEGMENTS.has(segment)) {
return undefined;
}
if (CANONICAL_ARRAY_INDEX.test(segment)) {
const index = getOtelArrayIndex(segment);
if (
index === undefined ||
index >= OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS
) {
if (recordLimitDrop) {
recordDrop("reconstruction_array_index_exceeded");
}
return undefined;
}
}
}
return segments;
};
const prefixWithDot = `${prefix}.`;
const createdContainers = new WeakSet<object>();
const arraySlotBudget = new ArraySlotBudget(
OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS,
);
const rootArraySlotBudget = new ArraySlotBudget(
OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS,
);
const inputKeys = Object.keys(input);
const useArray = inputKeys.some((inputKey) => {
if (!inputKey.startsWith(prefixWithDot)) return false;
const path = parsePath(inputKey.slice(prefixWithDot.length), false);
if (path === undefined || !rootArraySlotBudget.tryReserve(path)) {
return false;
}
return path.length > 1 && getOtelArrayIndex(path[0]) !== undefined;
});
const result: ReconstructedContainer = useArray ? [] : Object.create(null);
createdContainers.add(result);
const isWritableKey = (
container: ReconstructedContainer,
key: string,
): boolean => {
if (!Array.isArray(container)) return true;
const index = getOtelArrayIndex(key);
return (
index !== undefined &&
index < OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS
);
};
const applyPath = (
root: ReconstructedContainer,
path: readonly string[],
value: unknown,
commit: boolean,
): boolean => {
let current = root;
for (let index = 0; index < path.length - 1; index++) {
const key = path[index];
if (!isWritableKey(current, key)) return false;
if (!Object.hasOwn(current, key)) {
if (!commit) return true;
const child: ReconstructedContainer =
getOtelArrayIndex(path[index + 1]) !== undefined
? []
: Object.create(null);
createdContainers.add(child);
defineOwnValue(current, key, child);
current = child;
continue;
}
const next = Reflect.get(current, key);
if (
typeof next !== "object" ||
next === null ||
!createdContainers.has(next)
) {
return false;
}
current = next as ReconstructedContainer;
}
const finalKey = path[path.length - 1];
if (!isWritableKey(current, finalKey)) return false;
if (Object.hasOwn(current, finalKey)) {
const existing = Reflect.get(current, finalKey);
if (
typeof existing === "object" &&
existing !== null &&
createdContainers.has(existing)
) {
return false;
}
}
if (commit) defineOwnValue(current, finalKey, value);
return true;
};
for (const inputKey of inputKeys) {
if (!inputKey.startsWith(prefixWithDot)) continue;
const path = parsePath(inputKey.slice(prefixWithDot.length), true);
if (!path) continue;
const value = input[inputKey];
if (!applyPath(result, path, value, false)) continue;
if (!arraySlotBudget.tryReserve(path)) {
recordDrop("reconstruction_budget_exceeded");
continue;
}
applyPath(result, path, value, true);
}
for (const [reason, droppedAttributeCount] of droppedAttributeCounts) {
this.recordArrayAttributesDropped(prefix, reason, droppedAttributeCount);
}
return result;
}
private recordArrayAttributesDropped(
prefix: string,
reason: ReconstructedAttributeDropReason,
droppedAttributeCount: number,
): void {
if (droppedAttributeCount === 0) return;
recordIncrement(
OtelIngestionProcessor.OTEL_ARRAY_ATTRIBUTE_DROPPED_METRIC,
droppedAttributeCount,
{ reason, prefix },
);
const warningCount = this.arrayAttributeDropWarnCounts.get(reason) ?? 0;
if (warningCount < OtelIngestionProcessor.ARRAY_ATTRIBUTE_DROP_WARN_CAP) {
this.arrayAttributeDropWarnCounts.set(reason, warningCount + 1);
logger.warn("OTEL array attribute dropped", {
projectId: this.projectId,
prefix,
reason,
droppedAttributeCount,
});
}
}
private extractInputAndOutput(params: {
events: any[];
attributes: Record<string, unknown>;
instrumentationScopeName: string;
domain?: "trace" | "observation";
}): { input: any; output: any; filteredAttributes: Record<string, unknown> } {
const { instrumentationScopeName, events, attributes, domain } = params;
let input = null;
let output = null;
// Create a shallow copy of attributes to filter out used keys
const rawFilteredAttributes = { ...attributes };
// Pre-delete all potential input/output attribute keys to avoid duplicates
// This ensures that if multiple frameworks' attributes are present, they're all filtered
const potentialInputOutputKeys = [
// Langfuse SDK
LangfuseOtelSpanAttributes.TRACE_INPUT,
LangfuseOtelSpanAttributes.TRACE_OUTPUT,
LangfuseOtelSpanAttributes.OBSERVATION_INPUT,
LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT,
LangfuseOtelSpanAttributes.OBSERVATION_SKILLS_AVAILABLE,
// Vercel AI SDK
"ai.prompt.messages",
"ai.prompt",
"ai.toolCall.args",
"ai.response.text",
"ai.result.text",
"ai.toolCall.result",
"ai.response.object",
"ai.result.object",
"ai.response.toolCalls",
"ai.result.toolCalls",
// Google Vertex AI
"gcp.vertex.agent.llm_request",
"gcp.vertex.agent.llm_response",
"gcp.vertex.agent.tool_call_args",
"gcp.vertex.agent.tool_response",
// Logfire
"prompt",
"all_messages_events",
"events",
// LiveKit (livekit-agents >= 1.8 marks content attributes with `lk.pii.`)
"lk.input_text",
"lk.user_transcript",
"lk.chat_ctx",
"lk.user_input",
"lk.function_tool.output",
"lk.response.text",
"lk.pii.input_text",
"lk.pii.user_transcript",
"lk.pii.chat_ctx",
"lk.pii.user_input",
"lk.pii.instructions",
"lk.pii.function_tool.arguments",
"lk.pii.function_tool.output",
"lk.pii.response.text",
"lk.pii.response.function_calls",
// MLFlow
"mlflow.spanInputs",
"mlflow.spanOutputs",
// TraceLoop
"traceloop.entity.input",
"traceloop.entity.output",
// SmolAgents
"input.value",
"output.value",
// Pydantic AI agent/root span
"final_result",
"pydantic_ai.all_messages",
// OpenTelemetry system instructions
"gen_ai.system_instructions",
// Pydantic and Pipecat
"input",
"output",
// OpenTelemetry
"gen_ai.input.messages",
"gen_ai.output.messages",
"gen_ai.tool.call.arguments",
"gen_ai.tool.call.result",
// Claude Managed Agents
"anthropic.tool_result.content",
// Genkit
"genkit:input",
"genkit:output",
// Flue (@flue/opentelemetry)
"flue.turn.input",
"flue.turn.output",
"flue.tool.arguments",
"flue.tool.result",
"flue.task.prompt",
"flue.task.result",
"flue.operation.result",
"flue.workflow.payload",
"flue.workflow.result",
];
// Delete simple keys
potentialInputOutputKeys.forEach((key) => {
delete rawFilteredAttributes[key];
});
// Gateway observation attributes are represented by canonical fields.
// Keep unknown attributes available for diagnostics.
if (instrumentationScopeName === AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME) {
for (const key of Object.values(LangfuseOtelSpanAttributes)) {
delete rawFilteredAttributes[key];
}
}
// Delete gen_ai.prompt.*, gen_ai.completion.*, llm.input_messages.*, llm.output_messages.*,
// and metadata blob keys (already extracted into top-level metadata by extractMetadata())
Object.keys(attributes).forEach((key) => {
if (
key.startsWith("gen_ai.prompt") ||
key.startsWith("gen_ai.completion") ||
key.startsWith("llm.input_messages") ||
key.startsWith("llm.output_messages") ||
key.startsWith("langfuse.trace.metadata") ||
key.startsWith("langfuse.observation.metadata") ||
key.startsWith("ai.telemetry.metadata")
) {
delete rawFilteredAttributes[key];
}
});
// Stringify non-string values in filteredAttributes to maintain backward compatibility
// This matches the old spanAttributesInMetadata behavior
const filteredAttributes = Object.fromEntries(
Object.entries(rawFilteredAttributes).map(([key, value]) => [
key,
typeof value === "string" ? value : JSON.stringify(value),
]),
);
// Langfuse
input =
domain === "trace" && attributes[LangfuseOtelSpanAttributes.TRACE_INPUT]
? attributes[LangfuseOtelSpanAttributes.TRACE_INPUT]
: attributes[LangfuseOtelSpanAttributes.OBSERVATION_INPUT];
output =
domain === "trace" && attributes[LangfuseOtelSpanAttributes.TRACE_OUTPUT]
? attributes[LangfuseOtelSpanAttributes.TRACE_OUTPUT]
: attributes[LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT];
if (input != null || output != null) {
return { input, output, filteredAttributes };
}
// Genkit
if (instrumentationScopeName === "genkit-tracer") {
const genkitInput =
this.parseJsonPayload(attributes["genkit:input"]) ??
attributes["genkit:input"];
const genkitOutput =
this.parseJsonPayload(attributes["genkit:output"]) ??
attributes["genkit:output"];
if (genkitInput != null || genkitOutput != null) {
// For model spans prefer messages[] as input and message object as output.
// For flow/util spans there is no messages/message wrapper, so fall back to the full object
input = genkitInput?.messages ?? genkitInput;
output = genkitOutput?.message ?? genkitOutput;
}
return { input, output, filteredAttributes };
}
// Vercel AI SDK
if (instrumentationScopeName === "ai") {
input =
"ai.prompt.messages" in attributes
? attributes["ai.prompt.messages"]
: "ai.prompt" in attributes
? attributes["ai.prompt"]
: "ai.toolCall.args" in attributes
? attributes["ai.toolCall.args"]
: undefined;
if (
"ai.response.text" in attributes &&
"ai.response.toolCalls" in attributes
) {
output = JSON.stringify({
role: "assistant",
content: attributes["ai.response.text"],
tool_calls: attributes["ai.response.toolCalls"],
});
} else {
output =
"ai.response.text" in attributes &&
Boolean(attributes["ai.response.text"])
? attributes["ai.response.text"]
: "ai.result.text" in attributes // Legacy support for ai SDK versions < 4.0.0
? attributes["ai.result.text"]
: "ai.toolCall.result" in attributes
? attributes["ai.toolCall.result"]
: "ai.response.object" in attributes
? attributes["ai.response.object"]
: "ai.result.object" in attributes // Legacy support for ai SDK versions < 4.0.0
? attributes["ai.result.object"]
: "ai.response.toolCalls" in attributes
? attributes["ai.response.toolCalls"]
: "ai.result.toolCalls" in attributes // Legacy support for ai SDK versions < 4.0.0
? attributes["ai.result.toolCalls"]
: undefined;
}
const genAiInputOutput =
this.extractOpenTelemetryGenAiInputAndOutput(attributes);
if (isNullOrUndefined(input)) {
input = genAiInputOutput?.input;
}
if (isNullOrUndefined(output)) {
output = genAiInputOutput?.output;
}
return { input, output, filteredAttributes };
}
// Flue (https://flueframework.com)
// The @flue/opentelemetry adapter emits content under flue.* attributes that
// differ by span type (model turn, tool call, delegated task, workflow,
// operation). Gate on the instrumentation scope (like the Genkit and Vercel
// AI SDK handlers above) so a foreign span that happens to carry a flue.*
// attribute is never affected.
if (instrumentationScopeName === "@flue/opentelemetry") {
const flueInput =
attributes["flue.turn.input"] ??
attributes["flue.tool.arguments"] ??
attributes["flue.task.prompt"] ??
attributes["flue.workflow.payload"];
const flueOutput =
attributes["flue.turn.output"] ??
attributes["flue.tool.result"] ??
attributes["flue.task.result"] ??
attributes["flue.operation.result"] ??
attributes["flue.workflow.result"];
if (flueInput != null || flueOutput != null) {
return {
input: this.parseJsonPayload(flueInput) ?? flueInput ?? null,
output: this.parseJsonPayload(flueOutput) ?? flueOutput ?? null,
filteredAttributes,
};
}
}
// OpenTelemetry GenAI semconv v1.37+ records prompts and completions on a
// gen_ai.client.inference.operation.details span event instead of span
// attributes (https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/)
const operationDetailsEvents = events.filter(
(event: Record<string, unknown>) =>
event.name === "gen_ai.client.inference.operation.details",
);
for (const event of operationDetailsEvents) {
const eventAttributes: Record<string, unknown> =
event.attributes?.reduce(
(acc: Record<string, unknown>, attr: any) => {
const value = this.convertValueToPlainJavascript(attr.value);
if (value !== undefined) acc[attr.key] = value;
return acc;
},
{} as Record<string, unknown>,
) ?? {};
const genAiInputOutput =
this.extractOpenTelemetryGenAiInputAndOutput(eventAttributes);
if (genAiInputOutput) {
return { ...genAiInputOutput, filteredAttributes };
}
}
const inputEvents = events.filter(
(event: Record<string, unknown>) =>
event.name === "gen_ai.system.message" ||
event.name === "gen_ai.user.message" ||
event.name === "gen_ai.assistant.message" ||
event.name === "gen_ai.tool.message",
);
const outputEvents = events.filter(
(event: Record<string, unknown>) => event.name === "gen_ai.choice",
);
if (inputEvents.length > 0 || outputEvents.length > 0) {
const processedInput =
inputEvents.length > 0
? inputEvents.map((event: any) => {
const eventAttributes =
event.attributes?.reduce((acc: any, attr: any) => {
const value = this.convertValueToPlainJavascript(attr.value);
if (value !== undefined) acc[attr.key] = value;
return acc;
}, {}) ?? {};
return {
role: event.name.replace("gen_ai.", "").replace(".message", ""),
...eventAttributes,
};
})
: null;
const processedOutput =
outputEvents.length > 0
? outputEvents.map((event: any) => {
const eventAttributes =
event.attributes?.reduce((acc: any, attr: any) => {
const value = this.convertValueToPlainJavascript(attr.value);
if (value !== undefined) acc[attr.key] = value;
return acc;
}, {}) ?? {};
return eventAttributes;
})
: null;
return {
input: processedInput,
output:
processedOutput && processedOutput.length === 1
? processedOutput[0]
: processedOutput,
filteredAttributes, // No attribute keys used, events are used instead
};
}
// Legacy semantic kernel event definitions
input = events.find(
(event: Record<string, unknown>) =>
event.name === "gen_ai.content.prompt",
)?.attributes;
output = events.find(
(event: Record<string, unknown>) =>
event.name === "gen_ai.content.completion",
)?.attributes;
if (input || output) {
input =
input?.reduce((acc: any, attr: any) => {
const value = this.convertValueToPlainJavascript(attr.value);
if (value !== undefined) acc[attr.key] = value;
return acc;
}, {}) ?? {};
output =
output?.reduce((acc: any, attr: any) => {
const value = this.convertValueToPlainJavascript(attr.value);
if (value !== undefined) acc[attr.key] = value;
return acc;
}, {}) ?? {};
const { input: eventInput } = this.extractInputAndOutput({
events: [],
attributes: input,
instrumentationScopeName,
});
const { output: eventOutput } = this.extractInputAndOutput({
events: [],
attributes: output,
instrumentationScopeName,
});
return {
input: eventInput || input,
output: eventOutput || output,
filteredAttributes, // No attribute keys used, events are used instead
};
}
// Google Vertex AI Agent-Developer-Kit (ADK)
input = attributes["gcp.vertex.agent.llm_request"];
output = attributes["gcp.vertex.agent.llm_response"];
// GCP Vertex Agent Tool call input and output
// see: https://github.com/google/adk-python/blob/9dce06f9b00259ec42241df4f6638955e783a9d1/src/google/adk/telemetry/tracing.py#L142
// Google sets llm_request and llm_response to {} when setting tool_call_args and tool_response
if (input === "{}" || !input) {
input = attributes["gcp.vertex.agent.tool_call_args"];
}
if (output === "{}" || !output) {
output = attributes["gcp.vertex.agent.tool_response"];
}
if (input || output) {
return { input, output, filteredAttributes };
}
// Logfire uses `prompt` and `all_messages_events` property on spans
input = attributes["prompt"];
output = attributes["all_messages_events"];
if (input || output) {
return { input, output, filteredAttributes };
}
// LiveKit. livekit-agents >= 1.8 marks content attributes with a `pii`
// segment (`lk.pii.<name>`); older versions use the bare `lk.<name>`.
const livekitInstructions = attributes["lk.pii.instructions"] || undefined;
// the agent may speak first, in which case user_input is empty
const livekitUserInput =
attributes["lk.user_input"] ||
attributes["lk.pii.user_input"] ||
undefined;
input =
attributes["lk.input_text"] ??
attributes["lk.pii.input_text"] ??
attributes["lk.user_transcript"] ??
attributes["lk.pii.user_transcript"] ??
attributes["lk.chat_ctx"] ??
attributes["lk.pii.chat_ctx"] ??
// agent_turn spans carry the system instructions and the user message
// separately; combine whatever is present into a chat-style input
(livekitInstructions
? [
{ role: "system", content: livekitInstructions },
...(livekitUserInput
? [{ role: "user", content: livekitUserInput }]
: []),
]
: livekitUserInput) ??
attributes["lk.pii.function_tool.arguments"];
const livekitFunctionCalls = attributes["lk.pii.response.function_calls"];
output =
attributes["lk.function_tool.output"] ||
attributes["lk.pii.function_tool.output"] ||
attributes["lk.response.text"] ||
attributes["lk.pii.response.text"] ||
// a turn that only produced tool calls has no response text
(livekitFunctionCalls && livekitFunctionCalls !== "[]"
? livekitFunctionCalls
: undefined);
if (input || output) {
return { input, output, filteredAttributes };
}
// Logfire uses single `events` array for GenAI events
const eventsArray = attributes["events"];
if (typeof eventsArray === "string" || Array.isArray(eventsArray)) {
let events = eventsArray as any[];
if (typeof eventsArray === "string") {
try {
events = JSON.parse(eventsArray);
} catch {
events = [];
}
}
const choiceEvent = events.find(
(event) => event["event.name"] === "gen_ai.choice",
);
const inputEvents = events.filter(
(event) => event["event.name"] !== "gen_ai.choice",
);
if (choiceEvent || inputEvents.length > 0) {
return {
input: inputEvents.length > 0 ? inputEvents : null,
output: choiceEvent || null,
filteredAttributes,
};
}
}
// MLFlow sets mlflow.spanInputs and mlflow.spanOutputs
input = attributes["mlflow.spanInputs"];
output = attributes["mlflow.spanOutputs"];
if (input || output) {
return { input, output, filteredAttributes };
}
// TraceLoop sets traceloop.entity.input and traceloop.entity.output
input = attributes["traceloop.entity.input"];
output = attributes["traceloop.entity.output"];
if (input || output) {
return { input, output, filteredAttributes };
}
// SmolAgents sets input.value and output.value
input = attributes["input.value"];
output = attributes["output.value"];
if (input || output) {
return { input, output, filteredAttributes };
}
// Pydantic and Pipecat uses input and output
input = attributes["input"];
output = attributes["output"];
if (input || output) {
return { input, output, filteredAttributes };
}
// Pydantic AI agent/root span: all_messages → input, final_result → output
if (instrumentationScopeName === "pydantic-ai") {
input = attributes["pydantic_ai.all_messages"] ?? null;
output = attributes["final_result"] ?? null;
if (input && attributes["gen_ai.system_instructions"]) {
input = this.prependSystemInstructions(
input,
attributes["gen_ai.system_instructions"],
);
}
if (input || output) {
return { input, output, filteredAttributes };
}
}
// Pydantic-AI uses tool_arguments and tool_response for tool call input/output
input = attributes["tool_arguments"];
output = attributes["tool_response"];
if (input || output) {
return { input, output, filteredAttributes };
}
// TraceLoop uses attributes property
const inputAttributes = Object.keys(attributes).filter((key) =>
key.startsWith("gen_ai.prompt"),
);
const outputAttributes = Object.keys(attributes).filter((key) =>
key.startsWith("gen_ai.completion"),
);
if (inputAttributes.length > 0 || outputAttributes.length > 0) {
input = inputAttributes.reduce((acc: any, key) => {
acc[key] = attributes[key];
return acc;
}, {});
output = outputAttributes.reduce((acc: any, key) => {
acc[key] = attributes[key];
return acc;
}, {});
return {
input: this.convertKeyPathToNestedObject(input, "gen_ai.prompt"),
output: this.convertKeyPathToNestedObject(output, "gen_ai.completion"),
filteredAttributes,
};
}
// OpenInference llm.input_messages and llm.output_messages (used by Agno, BeeAI, etc.)
const llmInputAttributes = Object.keys(attributes).filter((key) =>
key.startsWith("llm.input_messages"),
);
const llmOutputAttributes = Object.keys(attributes).filter((key) =>
key.startsWith("llm.output_messages"),
);
if (llmInputAttributes.length > 0 || llmOutputAttributes.length > 0) {
const llmInput = llmInputAttributes.reduce((acc: any, key) => {
acc[key] = attributes[key];
return acc;
}, {});
const llmOutput = llmOutputAttributes.reduce((acc: any, key) => {
acc[key] = attributes[key];
return acc;
}, {});
return {
input: this.convertKeyPathToNestedObject(
llmInput,
"llm.input_messages",
),
output: this.convertKeyPathToNestedObject(
llmOutput,
"llm.output_messages",
),
filteredAttributes,
};
}
const genAiInputOutput =
this.extractOpenTelemetryGenAiInputAndOutput(attributes);
if (genAiInputOutput) {
return { ...genAiInputOutput, filteredAttributes };
}
return { input: null, output: null, filteredAttributes };
}
private extractOpenTelemetryGenAiInputAndOutput(
attributes: Record<string, unknown>,
): { input: unknown; output: unknown } | null {
// OpenTelemetry messages (https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans)
let input = attributes["gen_ai.input.messages"];
let output = attributes["gen_ai.output.messages"];
if (input && attributes["gen_ai.system_instructions"]) {
input = this.prependSystemInstructions(
input,
attributes["gen_ai.system_instructions"],
);
}
if (input || output) {
return { input, output };
}
// OpenTelemetry tools (https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans)
input = attributes["gen_ai.tool.call.arguments"];
output =
attributes["gen_ai.tool.call.result"] ??
attributes["anthropic.tool_result.content"];
if (input || output) {
return { input, output };
}
return null;
}
/**
* Prepends gen_ai.system_instructions as a {role: "system", content} message,
* consistent with how gen_ai.system.message events are mapped.
* No-ops if messages already contain a system message.
*/
private prependSystemInstructions(
input: unknown,
systemInstructions: unknown,
): unknown {
try {
const messages = typeof input === "string" ? JSON.parse(input) : input;
if (!Array.isArray(messages)) return input;
const hasSystemMessage = messages.some(
(msg: Record<string, unknown>) => msg?.role === "system",
);
if (hasSystemMessage) return input;
let content: string;
try {
const parsed =
typeof systemInstructions === "string"
? JSON.parse(systemInstructions)
: systemInstructions;
if (Array.isArray(parsed)) {
content = parsed
.map((p: Record<string, unknown>) =>
typeof p === "object" && p?.content
? String(p.content)
: String(p),
)
.join("\n");
} else {
content = String(parsed);
}
} catch {
content = String(systemInstructions);
}
const systemMessage = { role: "system", content };
const merged = [systemMessage, ...messages];
return typeof input === "string" ? JSON.stringify(merged) : merged;
} catch {
return input;
}
}
private extractEnvironment(
attributes: Record<string, unknown>,
resourceAttributes: Record<string, unknown>,
): string {
const environmentAttributeKeys = [
LangfuseOtelSpanAttributes.ENVIRONMENT,
"deployment.environment.name",
"deployment.environment",
];
for (const key of environmentAttributeKeys) {
if (attributes[key]) {
return normalizeEnvironment(attributes[key], {
isLangfuseInternal: this.isLangfuseInternal,
});
}
if (resourceAttributes[key]) {
return normalizeEnvironment(resourceAttributes[key], {
isLangfuseInternal: this.isLangfuseInternal,
});
}
}
const openRouterEnvironment =
this.extractOpenRouterFailedRequestEnvironment(
attributes,
resourceAttributes,
);
if (openRouterEnvironment) {
return normalizeEnvironment(openRouterEnvironment);
}
return DEFAULT_TRACE_ENVIRONMENT;
}
private extractOpenRouterFailedRequestEnvironment(
attributes: Record<string, unknown>,
resourceAttributes: Record<string, unknown>,
): LangfuseInternalTraceEnvironment.LLMJudge | undefined {
if (
resourceAttributes["service.name"] !== "openrouter" ||
attributes["openrouter.source"] !== "openrouter"
) {
return undefined;
}
const completion = this.parseJsonPayload(attributes["gen_ai.completion"]);
if (
!OtelIngestionProcessor.isPlainObject(completion) ||
completion.completion !== null ||
!OtelIngestionProcessor.isPlainObject(completion.rawRequest) ||
!OtelIngestionProcessor.isPlainObject(completion.rawRequest.trace)
) {
return undefined;
}
// OpenRouter Broadcast can omit custom trace metadata when a request fails,
// but echoes the original request here. Recover only our reserved evaluator
// marker so failed judge calls cannot be scheduled as fresh evaluations.
return completion.rawRequest.trace.environment ===
LangfuseInternalTraceEnvironment.LLMJudge
? LangfuseInternalTraceEnvironment.LLMJudge
: undefined;
}
private extractName(
spanName: string,
attributes: Record<string, unknown>,
observationType?: ObservationType,
instrumentationScopeName?: string,
): string {
// GenAI tool name (standard OTel GenAI attribute)
if ("gen_ai.tool.name" in attributes && attributes["gen_ai.tool.name"]) {
return typeof attributes["gen_ai.tool.name"] === "string"
? (attributes["gen_ai.tool.name"] as string)
: JSON.stringify(attributes["gen_ai.tool.name"]);
}
// Genkit
if ("genkit:name" in attributes) {
return attributes["genkit:name"] as string;
}
// Logfire message for pydantic AI
const nameKeys = ["logfire.msg"];
for (const key of nameKeys) {
if (attributes[key]) {
return typeof attributes[key] === "string"
? (attributes[key] as string)
: JSON.stringify(attributes[key]);
}
}
// Vercel AI SDK
if ("ai.toolCall.name" in attributes) {
return attributes["ai.toolCall.name"] as string;
}
const functionIdAttribute = "ai.telemetry.functionId";
const operationIdAttribute = "ai.operationId";
if (operationIdAttribute in attributes) {
const prefix = attributes[functionIdAttribute]
? attributes[functionIdAttribute] + ":"
: "";
return prefix + attributes[operationIdAttribute];
}
const claudeManagedAgentName = this.getClaudeManagedAgentName(
spanName,
attributes,
observationType,
instrumentationScopeName,
);
if (claudeManagedAgentName) {
return claudeManagedAgentName;
}
return spanName;
}
private getClaudeManagedAgentName(
spanName: string,
attributes: Record<string, unknown>,
observationType?: ObservationType,
instrumentationScopeName?: string,
): string | undefined {
// Only replace the generic session-turn name in Claude Managed Agents (CMA) traces.
// The Claude Platform scope alone is also used by regular API calls.
const sessionId = attributes["anthropic.session.id"];
const isClaudeManagedAgent =
spanName === "anthropic.session.turn" &&
observationType === ObservationType.AGENT &&
instrumentationScopeName === "com.anthropic.claude-platform" &&
attributes["gen_ai.operation.name"] === "invoke_agent" &&
typeof sessionId === "string" &&
sessionId.trim().length > 0;
const agentName = attributes["gen_ai.agent.name"];
if (
isClaudeManagedAgent &&
typeof agentName === "string" &&
agentName.trim().length > 0
) {
return agentName;
}
return undefined;
}
private extractMetadata(
attributes: Record<string, unknown>,
domain: "trace" | "observation",
dropScope: object,
decodeValues = false,
): Record<string, unknown> {
const metadataKeyPrefix =
domain === "observation"
? LangfuseOtelSpanAttributes.OBSERVATION_METADATA
: LangfuseOtelSpanAttributes.TRACE_METADATA;
// A falsy-present primary key dies at the `||` fallback below and never
// reaches the parser: count it here, emission-only. Falsy compat values
// survive the fallback and are counted by the parser itself.
const primaryValue = attributes[metadataKeyPrefix];
if (primaryValue !== undefined && primaryValue !== null && !primaryValue) {
const isString = typeof primaryValue === "string";
this.recordMetadataDropped(
isString ? "parse_failure" : "primitive",
{ domain, attributeKey: metadataKeyPrefix, dropScope },
isString ? this.classifyParseFailure(primaryValue) : undefined,
);
}
const topLevelMetadata = this.parseMetadataAttribute(
primaryValue || attributes["langfuse.metadata"],
{
domain,
attributeKey: primaryValue ? metadataKeyPrefix : "langfuse.metadata",
dropScope,
},
);
const langfuseMetadata = this.extractPrefixedMetadataAttributes({
attributes,
prefixes: [
metadataKeyPrefix,
"langfuse.metadata",
"ai.telemetry.metadata",
],
excludedKeys: new Set([
"ai.telemetry.metadata.userId",
"ai.telemetry.metadata.sessionId",
"ai.telemetry.metadata.tags",
"ai.telemetry.metadata.langfusePrompt",
]),
decodedPrefix: decodeValues ? metadataKeyPrefix : undefined,
});
return {
...topLevelMetadata,
...langfuseMetadata,
};
}
private extractUserId(
attributes: Record<string, unknown>,
decodeMetadataValues: boolean,
resourceAttributes: Record<string, unknown>,
): string | undefined {
// Explicit identifiers in either scope take precedence over usernames.
return (
this.extractExplicitUserId(attributes, decodeMetadataValues) ??
this.extractStringAttribute(
resourceAttributes,
["langfuse.user.id", "user.id"],
decodeMetadataValues,
) ??
this.extractStringAttribute(
attributes,
["user.name"],
decodeMetadataValues,
) ??
this.extractStringAttribute(
resourceAttributes,
["user.name"],
decodeMetadataValues,
)
);
}
private extractExplicitUserId(
attributes: Record<string, unknown>,
decodeMetadataValues: boolean,
): string | undefined {
return this.extractStringAttribute(
attributes,
[
"langfuse.user.id",
"user.id",
`${LangfuseOtelSpanAttributes.OBSERVATION_METADATA}.langfuse_user_id`,
`${LangfuseOtelSpanAttributes.TRACE_METADATA}.langfuse_user_id`,
`ai.telemetry.metadata.userId`,
],
decodeMetadataValues,
);
}
private extractSessionId(
attributes: Record<string, unknown>,
decodeMetadataValues: boolean,
instrumentationScopeName: string | undefined,
): string | undefined {
const sessionIdKeys = ["langfuse.session.id", "session.id"];
/** Claude Managed Agents sessions span multiple threads; conversation IDs identify individual threads. */
if (isClaudeManagedAgentSpan(attributes, instrumentationScopeName)) {
sessionIdKeys.push("anthropic.session.id");
}
return this.extractStringAttribute(
attributes,
[
...sessionIdKeys,
"gen_ai.conversation.id",
`${LangfuseOtelSpanAttributes.OBSERVATION_METADATA}.langfuse_session_id`,
`${LangfuseOtelSpanAttributes.TRACE_METADATA}.langfuse_session_id`,
`ai.telemetry.metadata.sessionId`,
],
decodeMetadataValues,
);
}
/**
* readAttribute decodes the langfuse_* compat keys read from metadata, which
* structured-metadata SDK majors JSON-encode like every per-key value.
*/
private readAttribute(
attributes: Record<string, unknown>,
key: string,
decodeMetadataValues: boolean,
): unknown {
const value = attributes[key];
if (
!decodeMetadataValues ||
typeof value !== "string" ||
!(
key.startsWith(`${LangfuseOtelSpanAttributes.OBSERVATION_METADATA}.`) ||
key.startsWith(`${LangfuseOtelSpanAttributes.TRACE_METADATA}.`)
)
) {
return value;
}
try {
return JSON.parse(value);
} catch {
return value;
}
}
/**
* extractStringAttribute keeps a decoded value only when it is a string, so
* an encoded number keeps its exact digits instead of a lossy JS number.
*/
private extractStringAttribute(
attributes: Record<string, unknown>,
keys: string[],
decodeMetadataValues: boolean,
): string | undefined {
for (const key of keys) {
const decoded = this.readAttribute(attributes, key, decodeMetadataValues);
const value = typeof decoded === "string" ? decoded : attributes[key];
if (value) {
return typeof value === "string" ? value : JSON.stringify(value);
}
}
}
private extractModelParameters(
attributes: Record<string, unknown>,
instrumentationScopeName: string,
): Record<string, unknown> {
let explicitModelParameters: Record<string, unknown> | undefined;
if (attributes[LangfuseOtelSpanAttributes.OBSERVATION_MODEL_PARAMETERS]) {
try {
explicitModelParameters = this.sanitizeModelParams(
JSON.parse(
attributes[
LangfuseOtelSpanAttributes.OBSERVATION_MODEL_PARAMETERS
] as string,
),
) as Record<string, unknown>;
} catch {
// Fallthrough
}
}
if (explicitModelParameters && instrumentationScopeName !== "ai") {
return explicitModelParameters;
}
// Genkit
if (instrumentationScopeName === "genkit-tracer") {
const genkitConfig =
this.parseJsonPayload(attributes["genkit:input"])?.config ??
this.parseJsonPayload(attributes["genkit:output"])?.request?.config;
if (genkitConfig) {
return this.sanitizeModelParams(genkitConfig);
}
}
// Vercel AI SDK
if (instrumentationScopeName === "ai") {
const providerMetadata = this.parseJsonPayload(
attributes["ai.response.providerMetadata"],
);
const responseServiceTier = providerMetadata?.openai?.serviceTier;
return {
maxSteps:
"ai.settings.maxSteps" in attributes
? (attributes["ai.settings.maxSteps"]?.toString() ?? null)
: null,
toolChoice:
"ai.prompt.toolChoice" in attributes
? (attributes["ai.prompt.toolChoice"]?.toString() ?? null)
: null,
maxTokens:
"gen_ai.request.max_tokens" in attributes
? (attributes["gen_ai.request.max_tokens"]?.toString() ?? null)
: null,
finishReason:
"gen_ai.response.finish_reasons" in attributes
? (attributes["gen_ai.response.finish_reasons"]?.toString() ?? null)
: "gen_ai.finishReason" in attributes // Legacy support for ai SDK versions < 4.0.0
? (attributes["gen_ai.finishReason"]?.toString() ?? null)
: null,
system:
"gen_ai.system" in attributes
? (attributes["gen_ai.system"]?.toString() ?? null)
: "ai.model.provider" in attributes
? (attributes["ai.model.provider"]?.toString() ?? null)
: null,
maxRetries:
"ai.settings.maxRetries" in attributes
? (attributes["ai.settings.maxRetries"]?.toString() ?? null)
: null,
mode:
"ai.settings.mode" in attributes
? (attributes["ai.settings.mode"]?.toString() ?? null)
: null,
temperature:
"gen_ai.request.temperature" in attributes
? (attributes["gen_ai.request.temperature"]?.toString() ?? null)
: null,
service_tier:
typeof responseServiceTier === "string"
? responseServiceTier
: "gen_ai.request.service_tier" in attributes
? (attributes["gen_ai.request.service_tier"]?.toString() ?? null)
: null,
...explicitModelParameters,
};
}
if (explicitModelParameters) return explicitModelParameters;
if (attributes["llm.invocation_parameters"]) {
try {
return this.sanitizeModelParams(
JSON.parse(attributes["llm.invocation_parameters"] as string),
);
} catch {
// fallthrough
}
}
if (attributes["model_config"]) {
try {
return this.sanitizeModelParams(
JSON.parse(attributes["model_config"] as string),
);
} catch {
// fallthrough
}
}
const modelParameterPrefixes = [
"gen_ai.request.",
"llm.invocation_parameters.",
];
const modelParameters = modelParameterPrefixes.flatMap((prefix) =>
Object.keys(attributes)
.filter((key) => key.startsWith(prefix))
.map((key) => ({ key, prefix })),
);
return this.sanitizeModelParams(
modelParameters.reduce((acc: any, { key, prefix }) => {
const modelParamKey = key.replace(prefix, "");
if (modelParamKey !== "model") {
acc[modelParamKey] = attributes[key];
}
return acc;
}, {}),
);
}
private sanitizeModelParams<T>(params: T): Record<string, string> | T {
// Model params in Langfuse must be key value pairs where value is string
if (typeof params === "object" && params != null)
return Object.fromEntries(
Object.entries(params).map((e) => [
e[0],
["string", "number"].includes(typeof e[1])
? e[1]
: JSON.stringify(e[1]),
]),
);
return params;
}
/**
* The Vercel AI SDK OTel integration (@ai-sdk/otel) emits `invoke_agent`
* and `agent_step` spans carrying aggregate usage from child model calls.
* Skip their model/usage/cost to avoid double counting. Other AGENT spans
* are unaffected unless they have one of these operation names.
*
* CMA session creation only configures a model and must not trigger enrichment.
*/
private shouldSkipModelUsageAndCost(
attributes: Record<string, unknown>,
observationType: ObservationType,
scopeName: string | undefined,
spanName: string,
): boolean {
const operationName = attributes["gen_ai.operation.name"];
if (operationName === "invoke_agent" || operationName === "agent_step") {
return true;
}
return (
observationType === ObservationType.SPAN &&
isClaudeManagedAgentSessionCreation(attributes, scopeName, spanName)
);
}
private extractModelName(
attributes: Record<string, unknown>,
): string | undefined {
const modelNameKeys = [
LangfuseOtelSpanAttributes.OBSERVATION_MODEL,
"litellm.provider.model",
"gen_ai.response.model",
"ai.model.id",
"gen_ai.request.model",
"llm.response.model",
"llm.model_name",
"model",
];
// Genkit: genkit:name is the model identifier only on model-subtype spans
if (attributes["genkit:metadata:subtype"] === "model") {
modelNameKeys.push("genkit:name");
}
for (const key of modelNameKeys) {
if (attributes[key]) {
return typeof attributes[key] === "string"
? (attributes[key] as string)
: JSON.stringify(attributes[key]);
}
}
}
private extractUsageDetails(
attributes: Record<string, unknown>,
instrumentationScopeName: string,
context: { spanId: string; traceId: string; source: string },
): Record<string, unknown> {
const fromAttribute = this.parseJsonObjectAttribute(
attributes,
LangfuseOtelSpanAttributes.OBSERVATION_USAGE_DETAILS,
context,
);
if (fromAttribute !== null) return fromAttribute;
// Genkit
if (instrumentationScopeName === "genkit-tracer") {
const genkitOutput = this.parseJsonPayload(attributes["genkit:output"]);
const usage = genkitOutput?.usage as Record<string, number>;
if (usage) {
const {
inputTokens: input,
thoughtsTokens: thoughts,
outputTokens: output,
totalTokens: total,
} = usage;
const usageDetails: Record<string, number> = {};
if (input) usageDetails.input = input;
if (output) usageDetails.output = Math.max(output - (thoughts ?? 0), 0);
if (total) usageDetails.total = total;
if (thoughts) usageDetails.output_reasoning = thoughts;
return usageDetails;
}
}
if (instrumentationScopeName === "ai") {
try {
const usageDetails: Record<string, number | undefined> = {
input:
"gen_ai.usage.prompt_tokens" in attributes // Backward compat, input_tokens used in latest ai SDK versions
? parseInt(
attributes["gen_ai.usage.prompt_tokens"]?.toString() ?? "0",
)
: "gen_ai.usage.input_tokens" in attributes
? parseInt(
attributes["gen_ai.usage.input_tokens"]?.toString() ?? "0",
)
: undefined,
output:
"gen_ai.usage.completion_tokens" in attributes // Backward compat, output_tokens used in latest ai SDK versions
? parseInt(
attributes["gen_ai.usage.completion_tokens"]?.toString() ??
"0",
)
: "gen_ai.usage.output_tokens" in attributes
? parseInt(
attributes["gen_ai.usage.output_tokens"]?.toString() ?? "0",
)
: undefined,
total:
"ai.usage.tokens" in attributes
? parseInt(attributes["ai.usage.tokens"]?.toString() ?? "0")
: undefined,
};
const providerMetadata = attributes["ai.response.providerMetadata"];
// Try reading token details from ai.usage
if (
["ai.usage.cachedInputTokens", "ai.usage.reasoningTokens"].some((k) =>
Object.keys(attributes).includes(k),
)
) {
if ("ai.usage.cachedInputTokens" in attributes) {
const value = attributes["ai.usage.cachedInputTokens"] as string;
const parsed = JSON.parse(value);
usageDetails["input_cached_tokens"] =
typeof parsed === "number" ? parsed : JSON.parse(value).intValue;
}
if ("ai.usage.reasoningTokens" in attributes) {
const value = attributes["ai.usage.reasoningTokens"] as string;
const parsed = JSON.parse(value);
usageDetails["output_reasoning_tokens"] =
typeof parsed === "number" ? parsed : JSON.parse(value).intValue;
}
}
// Add additional usage details from provider metadata
if (providerMetadata) {
const parsed = JSON.parse(providerMetadata as string);
if ("openai" in parsed) {
const openaiMetadata = parsed["openai"] as Record<string, number>;
if (isNullOrUndefined(usageDetails["input_cached_tokens"])) {
usageDetails["input_cached_tokens"] =
openaiMetadata["cachedPromptTokens"];
}
if (isNullOrUndefined(usageDetails["accepted_prediction_tokens"])) {
usageDetails["accepted_prediction_tokens"] =
openaiMetadata["acceptedPredictionTokens"];
}
if (isNullOrUndefined(usageDetails["rejected_prediction_tokens"])) {
usageDetails["rejected_prediction_tokens"] =
openaiMetadata["rejectedPredictionTokens"];
}
if (isNullOrUndefined(usageDetails["output_reasoning_tokens"])) {
usageDetails["output_reasoning_tokens"] =
openaiMetadata["reasoningTokens"];
}
}
// "ai.response.providerMetadata": {"anthropic":{"usage":{"input_tokens":7,"cache_creation_input_tokens":2089,"cache_read_input_tokens":16399,"cache_creation":{"ephemeral_5m_input_tokens":2089,"ephemeral_1h_input_tokens":0},"output_tokens":445,"service_tier":"standard"},"cacheCreationInputTokens":2089,"stopSequence":null,"container":null,"contextManagement":null}}
if ("anthropic" in parsed && "usage" in parsed["anthropic"]) {
const anthropicMetadata = parsed["anthropic"]["usage"] as Record<
string,
unknown
>;
if (isNullOrUndefined(usageDetails["input_cache_creation"])) {
usageDetails["input_cache_creation"] = (
anthropicMetadata as Record<string, number>
)["cache_creation_input_tokens"];
}
if (isNullOrUndefined(usageDetails["input_cached_tokens"])) {
usageDetails["input_cached_tokens"] = (
anthropicMetadata as Record<string, number>
)["cache_read_input_tokens"];
}
// Extract cache creation duration breakdown (5m and 1h TTLs)
if (
typeof anthropicMetadata["cache_creation"] === "object" &&
anthropicMetadata["cache_creation"] !== null
) {
const cacheCreation = anthropicMetadata[
"cache_creation"
] as Record<string, number>;
if (
typeof cacheCreation["ephemeral_5m_input_tokens"] === "number"
) {
if (
isNullOrUndefined(usageDetails["input_cache_creation_5m"])
) {
usageDetails["input_cache_creation_5m"] =
cacheCreation["ephemeral_5m_input_tokens"];
}
}
if (
typeof cacheCreation["ephemeral_1h_input_tokens"] === "number"
) {
if (
isNullOrUndefined(usageDetails["input_cache_creation_1h"])
) {
usageDetails["input_cache_creation_1h"] =
cacheCreation["ephemeral_1h_input_tokens"];
}
}
// Subtract duration-specific counts from total to avoid double counting
usageDetails["input_cache_creation"] = Math.max(
(usageDetails["input_cache_creation"] ?? 0) -
(usageDetails["input_cache_creation_5m"] ?? 0) -
(usageDetails["input_cache_creation_1h"] ?? 0),
0,
);
}
}
// Bedrock provider metadata extraction
// "ai.response.providerMetadata": "{\"bedrock\":{\"usage\":{\"cacheReadInputTokens\":4482,\"cacheWriteInputTokens\":0,\"cacheCreationInputTokens\":0}}}"
if ("bedrock" in parsed) {
const bedrockMetadata = parsed["bedrock"] as Record<string, any>;
if (bedrockMetadata["usage"]) {
const usage = bedrockMetadata["usage"] as Record<string, number>;
if (usage["cacheReadInputTokens"] !== undefined) {
if (isNullOrUndefined(usageDetails["input_cache_read"])) {
usageDetails["input_cache_read"] =
usage["cacheReadInputTokens"];
}
}
if (usage["cacheWriteInputTokens"] !== undefined) {
if (isNullOrUndefined(usageDetails["input_cache_write"])) {
usageDetails["input_cache_write"] =
usage["cacheWriteInputTokens"];
}
}
if (usage["cacheCreationInputTokens"] !== undefined) {
if (isNullOrUndefined(usageDetails["input_cache_creation"])) {
usageDetails["input_cache_creation"] =
usage["cacheCreationInputTokens"];
}
}
}
}
}
// Subtract cached token count from total input and output
usageDetails["input"] = Math.max(
(usageDetails["input"] ?? 0) -
(usageDetails["input_cached_tokens"] ?? 0) -
(usageDetails["input_cache_creation"] ?? 0) -
(usageDetails["input_cache_creation_5m"] ?? 0) -
(usageDetails["input_cache_creation_1h"] ?? 0) -
(usageDetails["input_cache_read"] ?? 0),
0,
);
usageDetails["output"] = Math.max(
(usageDetails["output"] ?? 0) -
(usageDetails["output_reasoning_tokens"] ?? 0),
0,
);
return usageDetails;
} catch {
// Fallthrough
}
}
if (instrumentationScopeName === "pydantic-ai") {
const usageDetails = this.extractGenericGenAiUsageDetails(attributes);
if (Object.keys(usageDetails).length > 0) return usageDetails;
}
if (
instrumentationScopeName === "gcp.vertex.agent" &&
"gcp.vertex.agent.llm_response" in attributes
) {
const usageDetails: Record<string, number | undefined> =
this.extractGenericGenAiUsageDetails(attributes);
// prompt tokens include cached tokens, so the blob cache-read count is subtracted from input
if (usageDetails["input_cached_tokens"] === undefined) {
const llmResponse = this.parseJsonPayload(
attributes["gcp.vertex.agent.llm_response"],
);
const cachedTokens =
llmResponse?.usage_metadata?.cached_content_token_count;
if (
typeof cachedTokens === "number" &&
!Number.isNaN(cachedTokens) &&
cachedTokens > 0
) {
usageDetails["input_cached_tokens"] = cachedTokens;
if (usageDetails["input"] !== undefined) {
usageDetails["input"] = Math.max(
usageDetails["input"] - cachedTokens,
0,
);
}
}
}
return usageDetails;
}
return this.extractGenericGenAiUsageDetails(attributes);
}
private extractGenericGenAiUsageDetails(
attributes: Record<string, unknown>,
): Record<string, number> {
const usageDetails = Object.keys(attributes).filter(
(key) =>
(key.startsWith("gen_ai.usage.") && key !== "gen_ai.usage.cost") ||
key.startsWith("llm.token_count."),
);
if (usageDetails.length === 0) return {};
const rawUsageDetails = usageDetails.reduce(
(acc: Record<string, number>, key) => {
const usageDetailKey = key
.replace("gen_ai.usage.", "")
.replace("llm.token_count.", "");
const value = Number(attributes[key]);
if (!Number.isNaN(value)) {
acc[usageDetailKey] = value;
}
return acc;
},
{},
);
const inputTokens =
rawUsageDetails["prompt_tokens"] ??
rawUsageDetails["input_tokens"] ??
rawUsageDetails["prompt"];
const outputTokens =
rawUsageDetails["completion_tokens"] ??
rawUsageDetails["output_tokens"] ??
rawUsageDetails["completion"];
const totalTokens =
rawUsageDetails["total_tokens"] ?? rawUsageDetails["total"];
const cacheReadTokens =
rawUsageDetails["cache_read.input_tokens"] ??
rawUsageDetails["cache_read_input_tokens"] ??
rawUsageDetails["cache_read_tokens"] ??
rawUsageDetails["details.cache_read_tokens"] ??
rawUsageDetails["details.cache_read_input_tokens"] ??
rawUsageDetails["prompt_details.cache_read"] ??
rawUsageDetails["input_cached_tokens"];
const cacheCreationTokens =
rawUsageDetails["cache_creation.input_tokens"] ??
rawUsageDetails["cache_write.input_tokens"] ??
rawUsageDetails["cache_creation_input_tokens"] ??
rawUsageDetails["cache_write_tokens"] ??
rawUsageDetails["details.cache_write_tokens"] ??
rawUsageDetails["details.cache_creation_input_tokens"] ??
rawUsageDetails["prompt_details.cache_write"] ??
rawUsageDetails["input_cache_creation"];
// TTL breakdown of the cache-write total; only the unattributed remainder
// stays in input_cache_creation so each write is counted and priced once.
const cacheCreation5mTokens =
rawUsageDetails["cache_creation.5m_input_tokens"];
const cacheCreation1hTokens =
rawUsageDetails["cache_creation.1h_input_tokens"];
const hasCacheCreationBreakdown =
cacheCreation5mTokens !== undefined ||
cacheCreation1hTokens !== undefined;
const cacheCreationBreakdownTokens =
(cacheCreation5mTokens ?? 0) + (cacheCreation1hTokens ?? 0);
const cacheCreationRemainderTokens =
cacheCreationTokens !== undefined && hasCacheCreationBreakdown
? Math.max(cacheCreationTokens - cacheCreationBreakdownTokens, 0)
: cacheCreationTokens;
// Reasoning/audio details are included in the emitted output token count
// and are therefore subtracted from output below to avoid double counting.
const outputReasoningTokens =
rawUsageDetails["reasoning.output_tokens"] ??
rawUsageDetails["completion_details.reasoning"];
const outputAudioTokens = rawUsageDetails["completion_details.audio"];
const normalizedUsageDetails = Object.entries(rawUsageDetails).reduce(
(acc: Record<string, number>, [key, value]) => {
if (
[
"prompt_tokens",
"input_tokens",
"prompt",
"completion_tokens",
"output_tokens",
"completion",
"total_tokens",
"total",
"cache_read.input_tokens",
"cache_read_input_tokens",
"cache_read_tokens",
"details.cache_read_tokens",
"details.cache_read_input_tokens",
"prompt_details.cache_read",
"input_cached_tokens",
"cache_creation.input_tokens",
"cache_write.input_tokens",
"cache_creation_input_tokens",
"cache_write_tokens",
"details.cache_write_tokens",
"details.cache_creation_input_tokens",
"prompt_details.cache_write",
"input_cache_creation",
"cache_creation.5m_input_tokens",
"cache_creation.1h_input_tokens",
"reasoning.output_tokens",
"completion_details.reasoning",
"completion_details.audio",
].includes(key)
) {
return acc;
}
const normalizedKey = key.startsWith("details.")
? key.replace("details.", "")
: key;
acc[normalizedKey] = value;
return acc;
},
{},
);
if (inputTokens !== undefined) {
normalizedUsageDetails.input = Math.max(
inputTokens -
(cacheReadTokens ?? 0) -
(cacheCreationRemainderTokens ?? 0) -
cacheCreationBreakdownTokens,
0,
);
}
if (outputTokens !== undefined) {
normalizedUsageDetails.output = Math.max(
outputTokens - (outputReasoningTokens ?? 0) - (outputAudioTokens ?? 0),
0,
);
}
if (totalTokens !== undefined) {
normalizedUsageDetails.total = totalTokens;
}
if (cacheReadTokens !== undefined) {
normalizedUsageDetails.input_cached_tokens = cacheReadTokens;
}
if (cacheCreationRemainderTokens !== undefined) {
normalizedUsageDetails.input_cache_creation =
cacheCreationRemainderTokens;
}
if (cacheCreation5mTokens !== undefined) {
normalizedUsageDetails.input_cache_creation_5m = cacheCreation5mTokens;
}
if (cacheCreation1hTokens !== undefined) {
normalizedUsageDetails.input_cache_creation_1h = cacheCreation1hTokens;
}
if (outputReasoningTokens !== undefined) {
normalizedUsageDetails.output_reasoning_tokens = outputReasoningTokens;
}
if (outputAudioTokens !== undefined) {
normalizedUsageDetails.output_audio_tokens = outputAudioTokens;
}
return normalizedUsageDetails;
}
private extractCostDetails(
attributes: Record<string, unknown>,
context: { spanId: string; traceId: string; source: string },
): Record<string, number> {
const fromAttribute = this.parseJsonObjectAttribute(
attributes,
LangfuseOtelSpanAttributes.OBSERVATION_COST_DETAILS,
context,
);
if (fromAttribute !== null) return this.toNumericEntries(fromAttribute);
const genAiUsageCost = attributes["gen_ai.usage.cost"];
if (genAiUsageCost != null && genAiUsageCost !== "") {
return this.toNumericEntries({ total: genAiUsageCost });
}
const openInferenceTotalCost = attributes["llm.cost.total"];
if (openInferenceTotalCost != null && openInferenceTotalCost !== "") {
return this.toNumericEntries({ total: openInferenceTotalCost });
}
return {};
}
/** Keeps finite numbers and non-blank numeric strings (as numbers); drops other entries such as currency. */
private toNumericEntries(
details: Record<string, unknown>,
): Record<string, number> {
const result: Record<string, number> = {};
for (const [key, value] of Object.entries(details)) {
const numeric = this.toFiniteNumber(value);
if (Number.isFinite(numeric)) result[key] = numeric;
}
return result;
}
private toFiniteNumber(value: unknown): number {
if (typeof value === "number") return value;
if (typeof value === "string" && value.trim() !== "") return Number(value);
return NaN;
}
private extractCompletionStartTime(
attributes: Record<string, unknown>,
startTimeISO?: string,
): string | null {
try {
const value = attributes[
LangfuseOtelSpanAttributes.OBSERVATION_COMPLETION_START_TIME
] as any;
if (isValidDateString(value)) return value;
// Older SDKs have double stringified timestamps that need JSON parsing
// "\"2025-10-01T08:45:26.112648Z\""
const parsed = JSON.parse(value);
if (isValidDateString(parsed)) return parsed;
} catch {
// Fallthrough
}
// Vercel AI SDK: <= v6 reports milliseconds, v7 reports the GenAI
// semconv metric name in seconds.
try {
const secondsToFirstChunk =
attributes["gen_ai.client.operation.time_to_first_chunk"];
const msToFirstChunk =
attributes["ai.response.msToFirstChunk"] ??
attributes["ai.stream.msToFirstChunk"] ??
(secondsToFirstChunk != null && secondsToFirstChunk !== ""
? Number(secondsToFirstChunk) * 1000
: undefined);
if (msToFirstChunk && startTimeISO) {
const msToFirstChunkNumber = Math.ceil(Number(msToFirstChunk));
const startTimeUnix = new Date(startTimeISO).getTime();
return new Date(startTimeUnix + msToFirstChunkNumber).toISOString();
}
} catch {
// Fallthrough
}
return null;
}
private extractTags(
attributes: Record<string, unknown>,
decodeMetadataValues: boolean,
): string[] {
const tagsValue =
attributes[LangfuseOtelSpanAttributes.TRACE_TAGS] ||
attributes["langfuse.tags"] ||
this.readAttribute(
attributes,
`${LangfuseOtelSpanAttributes.OBSERVATION_METADATA}.langfuse_tags`,
decodeMetadataValues,
) ||
this.readAttribute(
attributes,
`${LangfuseOtelSpanAttributes.TRACE_METADATA}.langfuse_tags`,
decodeMetadataValues,
) ||
attributes["ai.telemetry.metadata.tags"] ||
attributes["tag.tags"];
if (isNullOrUndefined(tagsValue)) {
return [];
}
if (Array.isArray(tagsValue)) {
return tagsValue.map((tag) => String(tag));
}
if (typeof tagsValue === "string" && tagsValue.trim().startsWith("[")) {
try {
const parsedTags = JSON.parse(tagsValue);
if (Array.isArray(parsedTags)) {
return parsedTags.map((tag) => String(tag));
}
} catch {
// Continue with other methods
}
}
if (typeof tagsValue === "string" && tagsValue.includes(",")) {
return tagsValue.split(",").map((tag) => tag.trim());
}
if (typeof tagsValue === "string") {
return [tagsValue];
}
return [];
}
// One increment per dropped attribute VALUE: deduped per drop scope —
// the span object for span attributes (shared across both worker
// pipelines) or the per-resourceSpan attributes object for resource
// attributes — on (attribute key, reason). Distinct spans/resourceSpans
// in one job count separately; the first-seen domain wins the tag.
private recordMetadataDropped(
reason: string,
context: MetadataDropContext,
kind?: string,
): void {
const { domain, attributeKey, dropScope } = context;
let seen = this.reportedMetadataDrops.get(dropScope);
if (!seen) {
seen = new Set();
this.reportedMetadataDrops.set(dropScope, seen);
}
const dedupeKey = `${attributeKey}|${reason}`;
if (seen.has(dedupeKey)) {
return;
}
seen.add(dedupeKey);
// attributeKey is a closed set of Langfuse-defined attribute names
// (never a user-supplied key); sdkName/sdkVersion attribute the emitting
// client and are sanitized because they originate from raw request
// headers; kind sub-classifies parse_failure by the value's shape. None
// of these carry the dropped value itself.
const tags: Record<string, string> = {
reason,
source: "otel",
domain,
projectId: this.projectId,
attributeKey,
sdkName: sanitizeSdkMetricTagValue(this.sdkName),
sdkVersion: sanitizeSdkMetricTagValue(this.sdkVersion),
};
if (kind) {
tags.kind = kind;
}
recordIncrement("langfuse.ingestion.metadata_dropped", 1, tags);
}
// Sub-classifies a JSON.parse failure by the failing string's shape,
// reading only bounded head/tail windows — never re-parses or copies the
// (possibly multi-MB) value, and never logs its content. Slicing first
// keeps the trim/regex work off the full string.
private classifyParseFailure(value: string): string {
const head = value.slice(0, 64).trimStart();
if (head.length === 0) {
return "empty";
}
// Python `str(dict)` / `str(list)` — single-quoted, or bare True/False/None.
if (/^[{[]\s*'/.test(head) || /^(True|False|None)\b/.test(head)) {
return "python_repr";
}
const first = head[0];
if (first === "{" || first === "[") {
const expectedClose = first === "{" ? "}" : "]";
const last = value.slice(-64).trimEnd().slice(-1);
return last === expectedClose ? "loose_json" : "truncated_json";
}
return "unquoted_string";
}
private parseMetadataAttribute(
value: unknown,
context: MetadataDropContext,
): Record<string, unknown> {
if (isNullOrUndefined(value)) {
return {};
}
if (typeof value === "string") {
try {
const parsed = JSON.parse(value);
if (parsed && typeof parsed === "object") {
return parsed as Record<string, unknown>;
}
this.recordMetadataDropped("non_object_top_level", context);
return {};
} catch {
this.recordMetadataDropped(
"parse_failure",
context,
this.classifyParseFailure(value),
);
return {};
}
}
if (typeof value === "object") {
return value as Record<string, unknown>;
}
this.recordMetadataDropped("primitive", context);
return {};
}
private extractPrefixedMetadataAttributes(params: {
attributes: Record<string, unknown>;
prefixes: string[];
excludedKeys?: Set<string>;
/** decodedPrefix holds JSON-encoded values for structured-metadata SDKs */
decodedPrefix?: string;
}): Record<string, unknown> {
const {
attributes,
prefixes,
excludedKeys = new Set<string>(),
decodedPrefix,
} = params;
const metadata: Record<string, unknown> = {};
for (const [key, value] of Object.entries(attributes)) {
for (const prefix of prefixes) {
if (!key.startsWith(`${prefix}.`) || excludedKeys.has(key)) {
continue;
}
const metadataKey = key.slice(prefix.length + 1);
if (!metadataKey) {
continue;
}
if (prefix !== decodedPrefix || typeof value !== "string") {
metadata[metadataKey] = value;
continue;
}
// Decoded values match what a metadata blob holds for the key. An
// undecodable value (e.g. cut by an attribute length limit) is kept as
// sent so its content is not lost.
try {
metadata[metadataKey] = JSON.parse(value);
} catch {
metadata[metadataKey] = value;
recordIncrement("langfuse.ingestion.metadata_undecoded", 1, {
kind: this.classifyParseFailure(value),
});
}
}
}
return metadata;
}
/** shouldDecodeMetadataValues is true for spans of any scope exported by a structured-metadata Langfuse SDK major, versioned by the scope on langfuse-sdk spans and by the request header otherwise */
private shouldDecodeMetadataValues(
scopeSpan: any,
resourceAttributes: Record<string, unknown>,
): boolean {
const sdkName =
normalizeIngestionSdkName(this.sdkName) ??
OTEL_SDK_LANGUAGE_TO_SDK_NAME.get(
String(resourceAttributes["telemetry.sdk.language"]),
) ??
null;
return isStructuredMetadataSdkVersion({
sdkName,
sdkVersion: scopeSpan?.scope?.name?.startsWith("langfuse-sdk")
? scopeSpan.scope.version
: this.sdkVersion,
});
}
private extractMetadataFromPrefix(params: {
attributes: Record<string, unknown>;
prefix: string;
domain: string;
dropScope: object;
}): Record<string, unknown> {
const { attributes, prefix, domain, dropScope } = params;
return {
...this.parseMetadataAttribute(attributes[prefix], {
domain,
attributeKey: prefix,
dropScope,
}),
...this.extractPrefixedMetadataAttributes({
attributes,
prefixes: [prefix],
}),
};
}
/**
* Extracts experiment-related fields from span attributes.
* Returns undefined for fields that are not present.
*/
private extractExperimentFields(
attributes: Record<string, unknown>,
dropScope: object,
): {
experimentId?: string;
experimentName?: string;
experimentDescription?: string;
experimentDatasetId?: string;
experimentItemId?: string;
experimentItemVersion?: string;
experimentItemRootSpanId?: string;
experimentItemExpectedOutput?: string;
experimentMetadataNames?: string[];
experimentMetadataValues?: Array<string | null | undefined>;
experimentItemMetadataNames?: string[];
experimentItemMetadataValues?: Array<string | null | undefined>;
} {
const experimentId = attributes[LangfuseOtelSpanAttributes.EXPERIMENT_ID];
const experimentName =
attributes[LangfuseOtelSpanAttributes.EXPERIMENT_NAME];
const experimentDescription =
attributes[LangfuseOtelSpanAttributes.EXPERIMENT_DESCRIPTION];
const experimentDatasetId =
attributes[LangfuseOtelSpanAttributes.EXPERIMENT_DATASET_ID];
const experimentItemId =
attributes[LangfuseOtelSpanAttributes.EXPERIMENT_ITEM_ID];
const experimentItemRootSpanId =
attributes[
LangfuseOtelSpanAttributes.EXPERIMENT_ITEM_ROOT_OBSERVATION_ID
];
const experimentItemExpectedOutput =
attributes[LangfuseOtelSpanAttributes.EXPERIMENT_ITEM_EXPECTED_OUTPUT];
const experimentItemVersion =
attributes[LangfuseOtelSpanAttributes.EXPERIMENT_ITEM_VERSION];
// Extract experiment metadata
const experimentMetadataFlattened = flattenJsonToPathArrays(
this.extractMetadataFromPrefix({
attributes,
prefix: LangfuseOtelSpanAttributes.EXPERIMENT_METADATA,
domain: "experiment",
dropScope,
}),
);
// Extract experiment item metadata
const experimentItemMetadataFlattened = flattenJsonToPathArrays(
this.extractMetadataFromPrefix({
attributes,
prefix: LangfuseOtelSpanAttributes.EXPERIMENT_ITEM_METADATA,
domain: "experiment_item",
dropScope,
}),
);
return {
experimentId: experimentId ? String(experimentId) : undefined,
experimentName: experimentName ? String(experimentName) : undefined,
experimentDescription: experimentDescription
? String(experimentDescription)
: undefined,
experimentDatasetId: experimentDatasetId
? String(experimentDatasetId)
: undefined,
experimentItemId: experimentItemId ? String(experimentItemId) : undefined,
experimentItemVersion: this.parseExperimentItemVersion(
experimentItemVersion,
),
experimentItemRootSpanId: experimentItemRootSpanId
? String(experimentItemRootSpanId)
: undefined,
experimentItemExpectedOutput: experimentItemExpectedOutput
? String(experimentItemExpectedOutput)
: undefined,
experimentMetadataNames:
experimentMetadataFlattened.names.length > 0
? experimentMetadataFlattened.names
: undefined,
experimentMetadataValues:
experimentMetadataFlattened.values.length > 0
? experimentMetadataFlattened.values
: undefined,
experimentItemMetadataNames:
experimentItemMetadataFlattened.names.length > 0
? experimentItemMetadataFlattened.names
: undefined,
experimentItemMetadataValues:
experimentItemMetadataFlattened.values.length > 0
? experimentItemMetadataFlattened.values
: undefined,
};
}
private extractEvaluationFields(attributes: Record<string, unknown>) {
const evaluatorId = attributes[LangfuseOtelSpanAttributes.EVALUATOR_ID];
const evaluationRuleId =
attributes[LangfuseOtelSpanAttributes.EVALUATION_RULE_ID];
const evaluatorExecutionIsTest =
attributes[LangfuseOtelSpanAttributes.EVALUATOR_EXECUTION_IS_TEST];
if (
evaluatorId == null &&
evaluationRuleId == null &&
evaluatorExecutionIsTest == null
) {
return { evaluationContext: undefined };
}
return {
evaluationContext: {
evaluatorId: evaluatorId ? String(evaluatorId) : undefined,
evaluationRuleId: evaluationRuleId
? String(evaluationRuleId)
: undefined,
evaluatorExecutionIsTest:
evaluatorExecutionIsTest === true ||
evaluatorExecutionIsTest === "true",
},
};
}
/**
* The item version is a pointer to a dataset item version (`valid_from` timestamp),
* not a free-form label; "v1" or "latest" cannot resolve to one, so we drop them.
*/
private parseExperimentItemVersion(value: unknown): string | undefined {
if (value == null || value === "") return undefined;
const stringValue = String(value);
if (isValidDateString(stringValue)) {
return convertDateToClickhouseDateTime(new Date(stringValue));
}
logger.warn(
"OTEL invalid experiment item version, dropping. Expected timestamp.",
);
return undefined;
}
/**
* OTLP exporters may carry the prompt version as a stringValue; downstream
* schemas require an integer. Non-integer values become null.
*/
private extractPromptVersion(
attributes: Record<string, unknown>,
): number | null {
const raw =
attributes[LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_VERSION] ??
attributes["langfuse.prompt.version"] ??
this.parseLangfusePromptFromAISDK(attributes)?.version;
if (typeof raw === "number") return Number.isInteger(raw) ? raw : null;
if (typeof raw === "string" && /^\d+$/.test(raw.trim())) {
return Number(raw);
}
return null;
}
private parseLangfusePromptFromAISDK(
attributes: Record<string, unknown>,
): { name: string; version: number } | undefined {
const aiSDKPrompt = attributes["ai.telemetry.metadata.langfusePrompt"];
if (!aiSDKPrompt) return;
try {
const parsed = JSON.parse(aiSDKPrompt as string);
return typeof parsed === "object" ? parsed : undefined;
} catch {
// Fallthrough
}
}
/**
* Parses a payload (object or JSON string) and returns the object
* or `undefined` if invalid.
*/
private parseJsonPayload = (payload: unknown): any => {
if (payload && typeof payload === "object") return payload;
if (typeof payload === "string") {
try {
return JSON.parse(payload);
} catch {
return undefined;
}
}
return undefined;
};
/**
* Get a set of trace IDs that have been seen recently (from Redis cache).
* Returns a Set of trace IDs that should not trigger new trace creation.
*/
private async getSeenTracesSet(resourceSpans: unknown): Promise<Set<string>> {
if (!redis) {
logger.warn("Redis client not available");
return new Set();
}
const traceIds: Set<string> = new Set();
if (Array.isArray(resourceSpans)) {
resourceSpans.forEach((resourceSpan) => {
for (const scopeSpan of resourceSpan?.scopeSpans ?? []) {
for (const span of scopeSpan?.spans ?? []) {
traceIds.add(this.parseId(span.traceId?.data ?? span.traceId));
}
}
});
}
try {
const results = await Promise.all(
[...traceIds].map(async (traceId) => {
const key = `langfuse:project:${this.projectId}:trace:${traceId}:seen`;
const TTLSeconds = 600; // 10 minutes
const result = await redis?.set(key, "1", "EX", TTLSeconds, "NX");
return {
traceId: traceId,
wasSeen: result !== "OK", // Redis returns "OK" if key did not exist, i.e. trace was NOT seen in last TTL seconds
};
}),
);
const seenTraceIds: Set<string> = new Set();
results.forEach((r) => {
if (r.wasSeen) {
seenTraceIds.add(r.traceId);
}
});
return seenTraceIds;
} catch (error) {
// Redis error will be captured by parent span, just log and continue
logger.error("Redis operation failed in getSeenTracesSet:", error);
// Return empty set to continue processing (fail-safe behavior)
return new Set();
}
}
private parseId(data: any): string {
// JS SDK sends IDs already in hex strings
// Python SDK sends Int array
return typeof data === "string" ? data : Buffer.from(data).toString("hex");
}
/**
* Convert OpenTelemetry nano timestamp to ISO string.
* Handles various timestamp formats: string, number, or object with high/low bits.
*/
public static convertNanoTimestampToISO(
timestamp: NanoTimestamp,
field: TimestampField = "unknown",
): string | undefined {
try {
if (timestamp == null) {
OtelIngestionProcessor.recordConversionFailure(
"timestamp_missing",
field,
);
return undefined;
}
if (typeof timestamp === "string") {
if (timestamp.trim() === "") {
OtelIngestionProcessor.recordConversionFailure(
"timestamp_invalid_empty_string",
field,
);
return undefined;
}
const millisBigInt = BigInt(timestamp) / BigInt(1_000_000);
return new Date(Number(millisBigInt)).toISOString();
}
if (typeof timestamp === "number") {
if (!Number.isFinite(timestamp)) {
OtelIngestionProcessor.recordConversionFailure(
"timestamp_invalid_number",
field,
);
return undefined;
}
return new Date(timestamp / 1e6).toISOString();
}
if (
typeof timestamp.high !== "number" ||
typeof timestamp.low !== "number"
) {
OtelIngestionProcessor.recordConversionFailure(
"timestamp_invalid_object",
field,
);
return undefined;
}
// Preserve protobuf Long's signed high word and unsigned low word.
// eslint-disable-next-line @repo/no-exotic-operators
const highBits = BigInt(timestamp.high) << BigInt(32);
// eslint-disable-next-line @repo/no-exotic-operators
const lowBits = BigInt(timestamp.low >>> 0);
// Combine high and low bits
// eslint-disable-next-line @repo/no-exotic-operators
const nanosBigInt = highBits | lowBits;
// Convert nanoseconds to milliseconds for JavaScript Date
const millisBigInt = nanosBigInt / BigInt(1000000);
return new Date(Number(millisBigInt)).toISOString();
} catch (e) {
logger.warn(`Failed to convert nanotimestamp to ISO`, {
timestamp,
error: e,
});
OtelIngestionProcessor.recordConversionFailure(
typeof timestamp === "string"
? "timestamp_invalid_string"
: "timestamp_conversion_exception",
field,
);
return undefined;
}
}
/**
* Ensure a stable time range for spans even if a timestamp is missing.
* If one edge is missing, use the other edge. If both are missing, use current time.
*/
private static resolveSpanTimestamps(params: {
startTimeUnixNano?: NanoTimestamp;
endTimeUnixNano?: NanoTimestamp;
}): { startTimeISO: string; endTimeISO: string } {
const startTimeISO = OtelIngestionProcessor.convertNanoTimestampToISO(
params.startTimeUnixNano,
"start_time",
);
const endTimeISO = OtelIngestionProcessor.convertNanoTimestampToISO(
params.endTimeUnixNano,
"end_time",
);
const fallbackISO = new Date().toISOString();
if (!startTimeISO && endTimeISO) {
OtelIngestionProcessor.recordConversionFailure(
"timestamp_inferred_start_from_end",
"start_time",
);
} else if (startTimeISO && !endTimeISO) {
OtelIngestionProcessor.recordConversionFailure(
"timestamp_inferred_end_from_start",
"end_time",
);
} else if (!startTimeISO && !endTimeISO) {
OtelIngestionProcessor.recordConversionFailure(
"timestamp_inferred_both_missing",
);
}
return {
startTimeISO: startTimeISO ?? endTimeISO ?? fallbackISO,
endTimeISO: endTimeISO ?? startTimeISO ?? fallbackISO,
};
}
private static recordConversionFailure(
failureType: string,
field: TimestampField = "unknown",
): void {
recordIncrement(OtelIngestionProcessor.OTEL_CONVERSION_FAILURE_METRIC, 1, {
failure_type: failureType,
timestamp_field: field,
});
}
/**
* Attribution context for conversion failure logs so a Datadog log line
* answers which SDK/version/instrumentation produced a malformed batch and
* how many spans were lost. Must never throw.
*/
private getConversionFailureLogContext(
resourceSpans: ResourceSpan[],
): Record<string, unknown> {
return {
projectId: this.projectId,
sdkName: this.sdkName,
sdkVersion: this.sdkVersion,
fileKey: this.fileKey,
spanCount: this.getTotalSpanCount(resourceSpans),
instrumentationScopes: this.getInstrumentationScopes(resourceSpans),
};
}
private getInstrumentationScopes(
resourceSpans: ResourceSpan[],
limit = 10,
): string[] {
try {
if (!Array.isArray(resourceSpans)) {
return [];
}
const scopes = new Set<string>();
for (const resourceSpan of resourceSpans) {
for (const scopeSpan of resourceSpan?.scopeSpans ?? []) {
const name = scopeSpan?.scope?.name;
if (name) {
scopes.add(name);
if (scopes.size >= limit) {
return [...scopes];
}
}
}
}
return [...scopes];
} catch (error) {
logger.warn("Failed to collect instrumentation scopes:", error);
return [];
}
}
/**
* Count the total number of spans across all resource spans.
* Returns -1 if an error occurs during counting to avoid throwing exceptions.
*/
private getTotalSpanCount(resourceSpans: ResourceSpan[]): number {
try {
if (!Array.isArray(resourceSpans)) {
return 0;
}
return resourceSpans.reduce((total, resourceSpan) => {
if (!resourceSpan?.scopeSpans) {
return total;
}
return (
total +
resourceSpan.scopeSpans.reduce((count, scopeSpan) => {
return count + (scopeSpan?.spans?.length ?? 0);
}, 0)
);
}, 0);
} catch (error) {
// Log error but never throw - return -1 to indicate counting failed
logger.warn("Failed to count total spans:", error);
return -1;
}
}
}