🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(otel): extract media before TypeScript parsing (#18239)
Valery Meleshkin525f3914d6ff
diff --git a/.env.dev.example b/.env.dev.exampleindex 68f379d2c0..4f45290cbf 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -159,6 +159,10 @@ LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=trueLANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/# Opt in to extracting and uploading media embedded in OTEL span attributes.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED=false+# Early extraction requires both the toggle and a project selection.+LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED=false+# Comma-separated project IDs; empty selects nobody, * selects everyone.+LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS=# Maximum encoded and decompressed OTLP request body size in bytes.# LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES=536870912diff --git a/.env.prod.example b/.env.prod.exampleindex 1bd617af05..601cad8072 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -300,6 +300,10 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES=536870912# Simulate OTLP worker admission while keeping inline ingestion authoritative (default: false)# LANGFUSE_OTEL_INGESTION_WORKER_SHADOW_ENABLED=false+# Early extraction requires both the toggle and a project selection.+# LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED=false+# Comma-separated project IDs; empty selects nobody, * selects everyone.+# LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS=# Evaluation worker concurrency# LANGFUSE_EVAL_EXECUTION_WORKER_CONCURRENCY=5diff --git a/packages/native/index.d.ts b/packages/native/index.d.tsindex 15434691dc..428e52cb18 100644--- a/packages/native/index.d.ts+++ b/packages/native/index.d.ts@@ -24,6 +24,8 @@ export declare class EarlyOtelBatch {* Omitting the count returns the exact spelling after parsing the outer document.*/originalMedia(index: number, jsonLayers?: number | undefined | null): Promise<string>+ /** Read media text for several entries in one libuv task, preserving request order. */+ originalMediaBatch(requests: Array<MediaRestoreRequest>): Promise<string[]>}/** A snapshot of one finalized event row, retained until its JS handle and any encoder tasks drop it. */@@ -93,6 +95,11 @@ export declare function hello(source: string): void*/export declare function initTelemetry(options?: TelemetryOptions | undefined | null): void+export interface MediaRestoreRequest {+ index: number+ jsonLayers: number+}+/*** A complete ClickHouse Native request body and its row count.*diff --git a/packages/native/src/otel_input.rs b/packages/native/src/otel_input.rsindex 9606f56380..9107b2811e 100644--- a/packages/native/src/otel_input.rs+++ b/packages/native/src/otel_input.rs@@ -152,6 +152,24 @@ pub struct ExtractedOtelMedia {pub original_has_json_escapes: bool,}+#[napi(object)]+pub struct MediaRestoreRequest {+ pub index: u32,+ pub json_layers: u32,+}++fn restore_media(+ data: &ExtractedBatch,+ index: u32,+ json_layers: Option<u32>,+) -> std::result::Result<String, Error<&'static str>> {+ data.media+ .get(index as usize)+ .ok_or_else(|| Error::new("ERR_OTEL_MEDIA_INDEX", "unknown media index"))?+ .original_value_for_layers(json_layers)+ .map_err(|error| Error::new("ERR_OTEL_MEDIA_DECODE", error.to_string()))+}+#[napi]impl EarlyOtelBatch {/// Release this handle; pending reads retain ownership until they complete.@@ -273,12 +291,23 @@ impl EarlyOtelBatch {let inner = self.data();OtelMediaTask::run("restore", move || {let inner = inner.map_err(|error| Error::new("ERR_OTEL_CLOSED", error.reason))?;- inner- .media- .get(index as usize)- .ok_or_else(|| Error::new("ERR_OTEL_MEDIA_INDEX", "unknown media index"))?- .original_value_for_layers(json_layers)- .map_err(|error| Error::new("ERR_OTEL_MEDIA_DECODE", error.to_string()))+ restore_media(&inner, index, json_layers)+ })+ }++ /// Read media text for several entries in one libuv task, preserving request order.+ #[napi(js_name = "originalMediaBatch", ts_return_type = "Promise<string[]>")]+ pub fn original_media_batch(+ &self,+ requests: Vec<MediaRestoreRequest>,+ ) -> AsyncTask<OtelMediaTask<Vec<String>>> {+ let inner = self.data();+ OtelMediaTask::run("restore", move || {+ let inner = inner.map_err(|error| Error::new("ERR_OTEL_CLOSED", error.reason))?;+ requests+ .into_iter()+ .map(|request| restore_media(&inner, request.index, Some(request.json_layers)))+ .collect()})}}diff --git a/packages/native/src/otel_media/rules.rs b/packages/native/src/otel_media/rules.rsindex acdf10c305..b36c7e33d5 100644--- a/packages/native/src/otel_media/rules.rs+++ b/packages/native/src/otel_media/rules.rs@@ -4,8 +4,8 @@//! must stay intact; the attribute's value can contain media. For example://!//! ```text-//! span Envelope-//! `-- attributes[] Attributes+//! span Span+//! `-- attributes[] SpanAttributes//! |-- key: "attachment" Disabled (attribute name)//! `-- value AnyValue (OTLP value wrapper)//! `-- stringValue: "data:..." Payload (inspect for media)@@ -57,10 +57,14 @@ pub(super) enum MediaScanMode {Payload,/// An OTLP KeyValue list; keys are metadata names and values contain data.Attributes,+ /// Span attributes can also supply tags, whose values must stay inline.+ SpanAttributes,/// An OTLP AnyValue wrapper; map keys remain structural until a value is data.AnyValue,/// The OTLP envelope; only known payload-bearing descendants enter `Payload`.Envelope,+ /// An OTLP span; distinguishes its attributes from resource/scope metadata.+ Span,/// A field whose strings are not eligible for extraction.Disabled,}@@ -84,7 +88,7 @@ pub(super) fn media_scan_mode_for_field(parent: MediaScanMode, field: &str) -> Mmatch parent {MediaScanMode::Disabled => MediaScanMode::Disabled,MediaScanMode::Payload => MediaScanMode::Payload,- MediaScanMode::Attributes => match field {+ MediaScanMode::Attributes | MediaScanMode::SpanAttributes => match field {// OTLP KeyValue keys are metadata names, never media payloads."key" => MediaScanMode::Disabled,"value" => MediaScanMode::AnyValue,@@ -97,12 +101,14 @@ pub(super) fn media_scan_mode_for_field(parent: MediaScanMode, field: &str) -> M"stringValue" | "bytesValue" => MediaScanMode::Payload,_ => MediaScanMode::Disabled,},- MediaScanMode::Envelope => match field {+ MediaScanMode::Envelope | MediaScanMode::Span => match field {// These fields contain envelope objects whose data-bearing children// are classified by the branches below.- "resourceSpans" | "scopeSpans" | "spans" | "resource" | "scope" | "events"- | "links" | "resourceLogs" | "scopeLogs" | "logRecords" => MediaScanMode::Envelope,+ "resourceSpans" | "scopeSpans" | "resource" | "scope" | "events" | "links"+ | "resourceLogs" | "scopeLogs" | "logRecords" => MediaScanMode::Envelope,+ "spans" => MediaScanMode::Span,// Attribute values and bodies contain user data.+ "attributes" if parent == MediaScanMode::Span => MediaScanMode::SpanAttributes,"attributes" => MediaScanMode::Attributes,"body" | "value" => MediaScanMode::AnyValue,_ => MediaScanMode::Disabled,@@ -110,6 +116,19 @@ pub(super) fn media_scan_mode_for_field(parent: MediaScanMode, field: &str) -> M}}+// Match OtelIngestionProcessor.extractTags: scalar tag values can be split on commas.+pub(super) fn is_tag_attribute(key: &str) -> bool {+ matches!(+ key,+ "langfuse.trace.tags"+ | "langfuse.tags"+ | "langfuse.observation.metadata.langfuse_tags"+ | "langfuse.trace.metadata.langfuse_tags"+ | "ai.telemetry.metadata.tags"+ | "tag.tags"+ )+}+pub(super) fn is_supported_content_type(value: &str) -> bool {matches!(value,diff --git a/packages/native/src/otel_media/structural_walk.rs b/packages/native/src/otel_media/structural_walk.rsindex 31a3f0eced..2222f714da 100644--- a/packages/native/src/otel_media/structural_walk.rs+++ b/packages/native/src/otel_media/structural_walk.rs@@ -34,8 +34,8 @@ use super::json_cursor::{ContainerState, ContainerStep, JsonCursor};use super::payload::{EarlyMediaError, MediaPayloadKind};use super::provider_shapes::{structured_shape, ProviderFields, ShapeKey, ValueShape};use super::rules::{- is_otel_envelope_field, may_contain_media_candidate, media_scan_mode_for_field, MediaScanMode,- MIN_EARLY_MEDIA_BYTES,+ is_otel_envelope_field, is_tag_attribute, may_contain_media_candidate,+ media_scan_mode_for_field, MediaScanMode, MIN_EARLY_MEDIA_BYTES,};use super::WalkStats;@@ -79,6 +79,7 @@ struct WalkFrame {#[derive(Default)]struct ObjectFrame<'a> {has_envelope_field: bool,+ is_tag_attribute: bool,current_shape_key: Option<ShapeKey>,current_key: Option<Cow<'a, str>>,current_candidate_start: usize,@@ -198,7 +199,7 @@ impl<'a, 'stats> StructuralWalk<'a, 'stats> {};if parent.state.is_object() {let object = object_states.last_mut().expect("open object state");- self.finish_object_field(object, shape);+ self.finish_object_field(object, shape, parent.modes);}self.update_peak(&stack, &object_states);}@@ -248,8 +249,20 @@ impl<'a, 'stats> StructuralWalk<'a, 'stats> {Ok(None)}- fn finish_object_field(&mut self, object: &mut ObjectFrame<'a>, shape: ValueShape) {+ fn finish_object_field(+ &mut self,+ object: &mut ObjectFrame<'a>,+ shape: ValueShape,+ modes: [MediaScanMode; 2],+ ) {if let Some(key) = object.current_key.take() {+ if key == "key" && modes.contains(&MediaScanMode::SpanAttributes) {+ object.is_tag_attribute = match &shape {+ ValueShape::String(range) => decode_json_string(self.input, range.clone())+ .is_some_and(|value| is_tag_attribute(&value)),+ _ => false,+ };+ }if let Some(previous) = object.last_key_candidates.remove(key.as_ref()) {self.add_mask_operation(previous.start, previous.end, 0b11);}@@ -355,6 +368,14 @@ impl<'a, 'stats> StructuralWalk<'a, 'stats> {.pop().expect("every closed object frame has object state");let candidate_end = self.candidates.len();+ // The attribute's key may follow its value; decide before media decoding/hashing.+ if object.is_tag_attribute {+ for (index, mode) in modes.iter().enumerate() {+ if *mode == MediaScanMode::SpanAttributes {+ self.add_mask_operation(candidate_start, candidate_end, 1 << index);+ }+ }+ }let provider_fields = object.provider_fields.as_deref();let active = if is_envelope_decider {let selected_bit = if object.has_envelope_field {diff --git a/packages/native/src/otel_media/tests.rs b/packages/native/src/otel_media/tests.rsindex 04727ad510..2f8b1f199f 100644--- a/packages/native/src/otel_media/tests.rs+++ b/packages/native/src/otel_media/tests.rs@@ -137,6 +137,48 @@ fn nested_otlp_map_keys_stay_structural() {assert_eq!(entry["value"]["stringValue"], result.media[0].reference());}+#[test]+fn tag_attribute_values_stay_inline_only_on_spans() {+ let (uri, _) = large_data_uri(b"tag");+ let value = format!(r#""value":{{"stringValue":"{uri}"}}"#);+ for key in [+ "langfuse.trace.tags",+ "langfuse.tags",+ "langfuse.observation.metadata.langfuse_tags",+ "langfuse.trace.metadata.langfuse_tags",+ "ai.telemetry.metadata.tags",+ "tag.tags",+ ] {+ let tag_key = format!(r#""key":"{key}""#);+ for (fields, is_tag) in [+ (format!("{tag_key},{value}"), true),+ (format!("{value},{tag_key}"), true),+ (format!(r#""key":"input",{value},{tag_key}"#), true),+ (format!(r#"{tag_key},{value},"key":"input""#), false),+ (format!(r#"{tag_key},{value},"key":null"#), false),+ ] {+ let attribute = format!("{{{fields}}}");+ let input = format!(+ r#"[{{+ "resource":{{"attributes":[{attribute}]}},+ "scopeSpans":[{{+ "scope":{{"attributes":[{attribute}]}},+ "spans":[{{"attributes":[{attribute},+ {{"key":"input","value":{{"kvlistValue":{{"values":[{attribute}]}}}}}}+ ]}}]+ }}]+ }},{attribute}]"#+ );+ let result = extract_media(input.as_bytes()).unwrap();+ assert_eq!(result.media.len(), if is_tag { 4 } else { 5 }, "{key}");+ let compact: Value = serde_json::from_str(&result.compact_json).unwrap();+ let value =+ &compact[0]["scopeSpans"][0]["spans"][0]["attributes"][0]["value"]["stringValue"];+ assert_eq!(value == &uri, is_tag);+ }+ }+}+#[test]fn accepts_deep_otlp_structure_before_embedded_json_limit() {let mut value = r#""leaf""#.to_owned();diff --git a/packages/shared/src/server/media/MediaPayloadProcessor.ts b/packages/shared/src/server/media/MediaPayloadProcessor.tsindex 1bb80edc51..d926547402 100644--- a/packages/shared/src/server/media/MediaPayloadProcessor.ts+++ b/packages/shared/src/server/media/MediaPayloadProcessor.ts@@ -1,4 +1,5 @@import { isMediaContentType, type MediaContentType } from "../../domain/media";+import { defineOwnValue } from "../../utils/objects";const MAX_RECURSION_DEPTH = 10;const DATA_URI_PREFIX = "data:";@@ -543,21 +544,6 @@ function setTraversalValue(node: TraversalNode, value: unknown): void {defineOwnValue(node.parent, node.key, value);}-function defineOwnValue(- target: object,- property: PropertyKey,- value: unknown,-): void {- // Avoid invoking user-controlled setters such as an own `__proto__` setter- // when replacing values inside parsed or SDK-provided payloads.- Object.defineProperty(target, property, {- configurable: true,- enumerable: true,- value,- writable: true,- });-}-function matchStructuredMedia(value: Record<string, unknown>,): StructuredMedia | undefined {diff --git a/packages/shared/src/server/media/mediaService.ts b/packages/shared/src/server/media/mediaService.tsindex 1490b0e291..2b4fee60b3 100644--- a/packages/shared/src/server/media/mediaService.ts+++ b/packages/shared/src/server/media/mediaService.ts@@ -187,6 +187,11 @@ export async function uploadMediaForTrace(params: {field: MediaField;contentType: MediaContentType;contentBytes: Buffer;+ /**+ * Base64 SHA-256 of contentBytes, precomputed during native extraction to avoid+ * hashing twice. TypeScript uploads omit it; this function computes their hash.+ */+ sha256Hash?: string;mediaBucket: string;mediaPrefix: string;origin: MediaAssociationOrigin;@@ -202,7 +207,9 @@ export async function uploadMediaForTrace(params: {mediaPrefix,origin,} = params;- const sha256Hash = createHash("sha256").update(contentBytes).digest("base64");+ const sha256Hash =+ params.sha256Hash ??+ createHash("sha256").update(contentBytes).digest("base64");const mediaId = getMediaId(sha256Hash);const existingMedia = await prisma.media.findUnique({where: {diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex 6b6ac015a3..7d4879c79d 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -42,6 +42,7 @@ import { OtelIngestionQueue } from "../redis/otelIngestionQueue";import { isValidDateString, flattenJsonToPathArrays } from "./utils";import { convertDateToClickhouseDateTime } from "../clickhouse/client";import { isNullOrUndefined } from "../../utils/isNullOrUndefined";+import { defineOwnValue } from "../../utils/objects";export const AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME = "langfuse-ai-gateway";@@ -1742,19 +1743,6 @@ export class OtelIngestionProcessor {);};- const defineOwn = (- container: ReconstructedContainer,- key: string,- value: unknown,- ): void => {- Object.defineProperty(container, key, {- configurable: true,- enumerable: true,- value,- writable: true,- });- };-const applyPath = (root: ReconstructedContainer,path: readonly string[],@@ -1773,7 +1761,7 @@ export class OtelIngestionProcessor {? []: Object.create(null);createdContainers.add(child);- defineOwn(current, key, child);+ defineOwnValue(current, key, child);current = child;continue;}@@ -1800,7 +1788,7 @@ export class OtelIngestionProcessor {return false;}}- if (commit) defineOwn(current, finalKey, value);+ if (commit) defineOwnValue(current, finalKey, value);return true;};diff --git a/packages/shared/src/server/otel/OtelMediaProcessor.test.ts b/packages/shared/src/server/otel/OtelMediaProcessor.test.tsindex 1015dac9be..4f2ceb6761 100644--- a/packages/shared/src/server/otel/OtelMediaProcessor.test.ts+++ b/packages/shared/src/server/otel/OtelMediaProcessor.test.ts@@ -90,12 +90,12 @@ describe("processOtelMedia", () => {expect(recordIncrement).toHaveBeenCalledWith("langfuse.ingestion.otel.media.detection_check",1,- { path: "data_uri", write_path: "direct" },+ { path: "data_uri", write_path: "direct", media_path: "reference" },);expect(recordDistribution).toHaveBeenCalledWith("langfuse.ingestion.otel.media.detection_check_byte_length",Buffer.byteLength(dataUri, "utf8"),- { path: "data_uri", write_path: "direct" },+ { path: "data_uri", write_path: "direct", media_path: "reference" },);expect(recordDistribution).toHaveBeenCalledWith("langfuse.ingestion.otel.media.byte_length",@@ -104,6 +104,7 @@ describe("processOtelMedia", () => {outcome: "uploaded",media_kind: "data_uri",write_path: "direct",+ media_path: "reference",},);expect(result).toMatchObject({@@ -135,6 +136,7 @@ describe("processOtelMedia", () => {outcome: "reused",media_kind: "data_uri",write_path: "direct",+ media_path: "reference",},);});@@ -305,6 +307,7 @@ describe("processOtelMedia", () => {media_kind: "data_uri",reason: "unsupported_content_type",write_path: "direct",+ media_path: "reference",},);});@@ -328,6 +331,7 @@ describe("processOtelMedia", () => {media_kind: "data_uri",reason: "invalid_base64",write_path: "direct",+ media_path: "reference",},);});diff --git a/packages/shared/src/server/otel/OtelMediaProcessor.ts b/packages/shared/src/server/otel/OtelMediaProcessor.tsindex 9a90c1b5ec..ad206ae94b 100644--- a/packages/shared/src/server/otel/OtelMediaProcessor.ts+++ b/packages/shared/src/server/otel/OtelMediaProcessor.ts@@ -81,6 +81,7 @@ type ProcessContext = {observationId?: string;field: MediaField;writePath: OtelMediaWritePath;+ mediaPath: "early" | "reference";mediaBucket: string;mediaPrefix: string;uploadMedia: UploadOtelMedia;@@ -100,6 +101,8 @@ export async function processOtelMedia(params: {targets: OtelMediaTarget[];projectId: string;writePath: OtelMediaWritePath;+ /** Pipeline label for metrics; does not enable or disable TypeScript media detection. */+ mediaPath?: "early" | "reference";mediaBucket: string;mediaPrefix: string;uploadMedia: UploadOtelMedia;@@ -108,6 +111,7 @@ export async function processOtelMedia(params: {targets,projectId,writePath,+ mediaPath = "reference",mediaBucket,mediaPrefix,uploadMedia,@@ -143,6 +147,7 @@ export async function processOtelMedia(params: {observationId: target.observationId,field,writePath,+ mediaPath,mediaBucket,mediaPrefix,uploadMedia,@@ -168,7 +173,7 @@ export async function processOtelMedia(params: {recordDistribution("langfuse.ingestion.otel.media.bytes_removed",transformed.bytesRemoved,- { write_path: context.writePath },+ { write_path: context.writePath, media_path: context.mediaPath },);}}@@ -222,6 +227,7 @@ async function processCandidate(outcome: uploadResult.outcome,media_kind: candidate.kind,write_path: context.writePath,+ media_path: context.mediaPath,});recordDistribution("langfuse.ingestion.otel.media.byte_length",@@ -230,6 +236,7 @@ async function processCandidate(outcome: uploadResult.outcome,media_kind: candidate.kind,write_path: context.writePath,+ media_path: context.mediaPath,},);@@ -240,6 +247,7 @@ async function processCandidate(outcome: "failed",media_kind: candidate.kind,write_path: context.writePath,+ media_path: context.mediaPath,});logger.warn("OTEL media upload failed; leaving normalized value unchanged",@@ -344,11 +352,12 @@ function recordDetectionCheck(recordIncrement("langfuse.ingestion.otel.media.detection_check", 1, {path,write_path: context.writePath,+ media_path: context.mediaPath,});recordDistribution("langfuse.ingestion.otel.media.detection_check_byte_length",checkedBytes,- { path, write_path: context.writePath },+ { path, write_path: context.writePath, media_path: context.mediaPath },);}@@ -363,6 +372,7 @@ function recordInvalidCandidate(media_kind: kind,reason,write_path: context.writePath,+ media_path: context.mediaPath,});}@@ -377,5 +387,6 @@ function recordIgnoredCandidate(media_kind: kind,reason,write_path: context.writePath,+ media_path: context.mediaPath,});}diff --git a/packages/shared/src/utils/objects.ts b/packages/shared/src/utils/objects.tsindex aebfb6baed..c1026f874e 100644--- a/packages/shared/src/utils/objects.ts+++ b/packages/shared/src/utils/objects.ts@@ -14,8 +14,22 @@ export function removeObjectKeys<T, K extends keyof T>(return result;}+/** Define an own data property without invoking setters, including `__proto__`. */+export function defineOwnValue(+ target: object,+ property: PropertyKey,+ value: unknown,+): void {+ Object.defineProperty(target, property, {+ configurable: true,+ enumerable: true,+ value,+ writable: true,+ });+}+/**- * Safely coerces a value to a Record if it's a plain object.+ * Safely coerces a value to a Record if it's a non-array object.* Returns undefined for null, undefined, arrays, and non-objects.*/export function asRecord(value: unknown): Record<string, unknown> | undefined {diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex dfcf856de3..625810de06 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -73,10 +73,11 @@records its own metrics and logs; see `../packages/native/AGENTS.md`. Topics runsits synchronous numerical fit in a killable Node child via `src/features/topics/numeric.ts`.- Tests: `src/__tests__/*`, `src/queues/__tests__/*`-- Direct-event replay: `pnpm --filter worker run test:otel-replay` feeds raw S3- JSON through the production OTEL queue and JSON writer into isolated- ClickHouse tables, including retained legacy writes in dual-write mode. Setup- and scope: `src/features/otel-ingestion/README.md`.+- Direct-event replay: `pnpm --filter worker run test:otel-replay` compares the+ original and early-media TypeScript paths from identical raw S3 JSON through+ the production OTEL queue and JSON writer into isolated ClickHouse tables,+ including retained legacy writes in dual-write mode. Setup and scope:+ `src/features/otel-ingestion/README.md`.- Native codec checks: `pnpm --filter worker run test:native-codec` selects`nativeCodec` suites. Build the addon first; the command requires ClickHousewhenever the live parity suite is present.diff --git a/worker/src/__tests__/helpers/otelReplayHarness.ts b/worker/src/__tests__/helpers/otelReplayHarness.tsindex 873f334716..bb62a9b8b6 100644--- a/worker/src/__tests__/helpers/otelReplayHarness.ts+++ b/worker/src/__tests__/helpers/otelReplayHarness.ts@@ -22,8 +22,15 @@ import { configureOtelReplayEnvironment } from "./otelReplaySetup";type OtelReplayStoredRow = Record<string, unknown>;export type OtelReplayResult = {+ mode: OtelRawReplayMode;storedRows: OtelReplayStoredRow[];legacyRows?: Partial<Record<LegacyReplayTable, OtelReplayStoredRow[]>>;+ sideEffects?: unknown;+};++export type OtelRawReplayComparison = {+ originalTs: OtelReplayResult;+ earlyTs: OtelReplayResult;};type LegacyReplayTable =@@ -49,9 +56,14 @@ type RunOtelReplayParams = {overflowEnabled?: boolean;overflowSizeLimitBytes?: number;writeMode?: "events_only" | "dual";+ sdkName?: string;+ sdkVersion?: string;failLegacyQueueProcessing?: boolean;+ captureSideEffects?: () => unknown;};+type OtelRawReplayMode = "original-ts" | "early-ts";+// The legacy processor caches its storage client. Keep this fake's identity stable// across sequential replays, but release all file contents after each run.const eventStorage = {@@ -90,12 +102,63 @@ const eventStorage = {},};+const differentialClockFields = new Set([+ "created_at",+ "updated_at",+ "event_ts",+]);++export function comparableOtelReplayRows(+ rows: OtelReplayStoredRow[] = [],+): OtelReplayStoredRow[] {+ return rows+ .map((row) =>+ Object.fromEntries(+ Object.entries(row).filter(+ ([key]) => !differentialClockFields.has(key),+ ),+ ),+ )+ .sort((left, right) =>+ JSON.stringify(left).localeCompare(JSON.stringify(right)),+ );+}++export function expectRawOtelReplayParity(+ comparison: OtelRawReplayComparison,+): void {+ expect(comparableOtelReplayRows(comparison.earlyTs.storedRows)).toEqual(+ comparableOtelReplayRows(comparison.originalTs.storedRows),+ );+}++/** Replay identical unparsed bytes through the original and early-media TS paths. */+export async function runOtelReplayComparison(+ params: RunOtelReplayParams,+): Promise<OtelRawReplayComparison> {+ const { captureSideEffects, ...replayParams } = params;+ const originalTs = await runOneOtelReplay({+ ...replayParams,+ captureSideEffects,+ mode: "original-ts",+ bytes: Buffer.from(params.bytes),+ });+ const earlyTs = await runOneOtelReplay({+ ...replayParams,+ captureSideEffects,+ mode: "early-ts",+ bytes: Buffer.from(params.bytes),+ });++ return { originalTs, earlyTs };+}+/*** Replay an S3 OTEL document through the production queue and JSON writer, then* read its events_full rows from an isolated ClickHouse Memory table.*/-export async function runOtelReplay(- params: RunOtelReplayParams,+export async function runOneOtelReplay(+ params: RunOtelReplayParams & { mode: OtelRawReplayMode },): Promise<OtelReplayResult> {const projectId = params.projectId ?? "otel-replay-test";const fileKey = params.fileKey ?? "otel-replay/test.json";@@ -107,6 +170,7 @@ export async function runOtelReplay(mediaUploadEnabled: params.mediaUploadEnabled,overflowEnabled: params.overflowEnabled,overflowSizeLimitBytes: params.overflowSizeLimitBytes,+ earlyMediaExtractionEnabled: params.mode === "early-ts",writeMode,});@@ -194,8 +258,8 @@ export async function runOtelReplay(},},ingestionVersion: "4",- sdkName: "otel-replay",- sdkVersion: "test",+ sdkName: params.sdkName ?? "otel-replay",+ sdkVersion: params.sdkVersion ?? "test",},},} as Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>;@@ -227,8 +291,15 @@ export async function runOtelReplay(}await processor(job, undefined);- expect(eventStorage.download).toHaveBeenCalledExactlyOnceWith(fileKey);- expect(eventStorage.downloadBytes).not.toHaveBeenCalled();+ if (params.mode === "original-ts") {+ expect(eventStorage.download).toHaveBeenCalledExactlyOnceWith(fileKey);+ expect(eventStorage.downloadBytes).not.toHaveBeenCalled();+ } else {+ expect(eventStorage.downloadBytes).toHaveBeenCalledExactlyOnceWith(+ fileKey,+ );+ expect(eventStorage.download).not.toHaveBeenCalled();+ }if (writeMode === "dual") {const legacyProcessor = ingestionQueueProcessorBuilder(false);@@ -276,8 +347,10 @@ export async function runOtelReplay(: undefined;replayResult = {+ mode: params.mode,storedRows: (await result.json()) as OtelReplayStoredRow[],legacyRows,+ sideEffects: params.captureSideEffects?.(),};} catch (error) {primaryError = new Error(diff --git a/worker/src/__tests__/helpers/otelReplaySetup.ts b/worker/src/__tests__/helpers/otelReplaySetup.tsindex 1ed70249fb..4433be8fbb 100644--- a/worker/src/__tests__/helpers/otelReplaySetup.ts+++ b/worker/src/__tests__/helpers/otelReplaySetup.ts@@ -24,6 +24,8 @@ vi.mock("../../env", async (importOriginal) => {NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,LANGFUSE_TRACE_BATCH_INGESTION_ENABLED: "false",LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: "false",+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED: "false",+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS: [],LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: "otel-replay-test",LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: "otel-replay/",LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED: "false",@@ -82,6 +84,7 @@ export function configureDefaultOtelReplayMocks(): void {type ReplayEnvironmentOptions = {mediaUploadEnabled?: boolean;+ earlyMediaExtractionEnabled?: boolean;overflowEnabled?: boolean;overflowSizeLimitBytes?: number;writeMode?: "events_only" | "dual";@@ -101,6 +104,10 @@ export function configureOtelReplayEnvironment(LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS:env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS,LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED,+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED:+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED,+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS:+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS,LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: env.LANGFUSE_S3_MEDIA_UPLOAD_PREFIX,LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG:@@ -121,6 +128,10 @@ export function configureOtelReplayEnvironment(LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: options.mediaUploadEnabled? "true": "false",+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED:+ options.earlyMediaExtractionEnabled ? "true" : "false",+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS:+ options.earlyMediaExtractionEnabled ? ["*"] : [],LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: "otel-replay-test",LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: "otel-replay/",LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG: "false",diff --git a/worker/src/__tests__/ingestionMasking.test.ts b/worker/src/__tests__/ingestionMasking.test.tsindex 25631fbc0b..0b2ee706da 100644--- a/worker/src/__tests__/ingestionMasking.test.ts+++ b/worker/src/__tests__/ingestionMasking.test.ts@@ -6,15 +6,29 @@ import {beforeAll,afterAll,afterEach,+ vi,} from "vitest";import { setupServer } from "msw/node";import { http, HttpResponse, delay } from "msw";import type { SharedEnv } from "@langfuse/shared/src/env";+import { prepareOtelBatch } from "../features/otel-ingestion/prepareOtelBatch";+import * as masking from "@langfuse/shared/src/server/ee/ingestionMasking";import {applyIngestionMasking,isIngestionMaskingEnabled,} from "@langfuse/shared/src/server/ee/ingestionMasking";+vi.mock(+ "@langfuse/shared/src/server/ee/ingestionMasking",+ async (importOriginal) => {+ const actual = await importOriginal<typeof masking>();+ return {+ ...actual,+ applyIngestionMasking: vi.fn(actual.applyIngestionMasking),+ };+ },+);+// Sample OTEL span data for testingconst sampleSpanData = [{@@ -232,6 +246,147 @@ describe("Ingestion Masking", () => {});describe("applyIngestionMasking", () => {+ it.each([+ "success",+ "bom-success",+ "double-bom-success",+ "triple-bom-fail-open",+ "fail-open",+ "fail-closed",+ ] as const)(+ "compacts only the accepted raw masking input (%s)",+ async (outcome) => {+ const originalMediaBody = Buffer.alloc(4096, 0x6f);+ const maskedMediaBody = Buffer.alloc(4096, 0x6d);+ const original = {+ input: `data:image/png;base64,${originalMediaBody.toString("base64")}`,+ };+ const originalBytes = Buffer.from(+ JSON.stringify({ ...original, note: "🔥" }, null, 2),+ );+ const masked = {+ input: `data:image/png;base64,${maskedMediaBody.toString("base64")}`,+ };+ const accepted =+ outcome === "success" ||+ outcome === "bom-success" ||+ outcome === "double-bom-success";+ const responseBody = accepted+ ? (outcome === "bom-success"+ ? "\uFEFF"+ : outcome === "double-bom-success"+ ? "\uFEFF\uFEFF"+ : "") + JSON.stringify(masked)+ : outcome === "triple-bom-fail-open"+ ? "\uFEFF\uFEFF\uFEFF" + JSON.stringify(masked)+ : "{";+ const overrides = createTestEnv({+ LANGFUSE_INGESTION_MASKING_CALLBACK_URL:+ "https://masking.example.com/raw",+ LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,+ LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,+ LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED:+ outcome === "fail-closed" ? "true" : "false",+ });+ const configuredMasking = vi.mocked(masking.applyIngestionMasking);+ const applyMasking = configuredMasking.getMockImplementation()!;+ configuredMasking.mockImplementation((params, _env, transport) =>+ applyMasking(params, overrides, transport),+ );+ const fetch = vi+ .spyOn(globalThis, "fetch")+ .mockResolvedValueOnce(new Response(responseBody));+ if (accepted) {+ await expect(new Response(responseBody).json()).resolves.toEqual(+ masked,+ );+ } else if (outcome === "triple-bom-fail-open") {+ await expect(new Response(responseBody).json()).rejects.toThrow();+ }+ let prepared: Awaited<ReturnType<typeof prepareOtelBatch>> | undefined;+ try {+ prepared = await prepareOtelBatch({+ bytes: originalBytes,+ projectId: "test-project",+ extractMedia: true,+ });+ // Masking sees the original inline content, before reference substitution.+ const body = fetch.mock.calls[0]?.[1]?.body;+ expect(body).toBeInstanceOf(Uint8Array);+ expect(Buffer.from(body as Uint8Array)).toEqual(originalBytes);+ if (outcome === "fail-closed") {+ expect(prepared).toEqual({ error: expect.any(String) });+ return;+ }+ const batch = prepared?.batch;+ expect(batch).toBeDefined();+ expect(batch!.media).toHaveLength(1);+ expect(await batch!.mediaBody(0)).toEqual(+ accepted ? maskedMediaBody : originalMediaBody,+ );+ expect(+ JSON.parse(batch!.takeJsonBuffer().toString("utf8")).input,+ ).toBe(batch!.media[0].reference);+ } finally {+ await prepared?.batch?.dispose();+ fetch.mockRestore();+ configuredMasking.mockImplementation(applyMasking);+ }+ },+ );++ it.each(["request", "response"] as const)(+ "sanitizes invalid UTF-8 in the masking %s before media discovery",+ async (side) => {+ const mediaBody = Buffer.alloc(4096, 0x68);+ const mediaUri = `data:image/png;base64,${mediaBody.toString("base64")}`;+ const malformed = Buffer.concat([+ Buffer.from('[{"note":"'),+ Buffer.from([0xff, 0xe2, 0x82]),+ Buffer.from(`","input":"${mediaUri}"}]`),+ ]);+ const sanitized = Buffer.from(malformed.toString("utf8"));+ const originalBytes = side === "request" ? malformed : sanitized;+ const responseBytes = side === "response" ? malformed : sanitized;+ const overrides = createTestEnv({+ LANGFUSE_INGESTION_MASKING_CALLBACK_URL:+ "https://masking.example.com/raw",+ LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,+ LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,+ });+ const configuredMasking = vi.mocked(masking.applyIngestionMasking);+ const applyMasking = configuredMasking.getMockImplementation()!;+ configuredMasking.mockImplementation((params, _env, transport) =>+ applyMasking(params, overrides, transport),+ );+ const fetch = vi+ .spyOn(globalThis, "fetch")+ .mockResolvedValueOnce(new Response(new Uint8Array(responseBytes)));+ let prepared: Awaited<ReturnType<typeof prepareOtelBatch>> | undefined;+ try {+ prepared = await prepareOtelBatch({+ bytes: originalBytes,+ projectId: "test-project",+ extractMedia: true,+ });+ const body = fetch.mock.calls[0]?.[1]?.body;+ expect(body).toBeInstanceOf(Uint8Array);+ expect(Buffer.from(body as Uint8Array)).toEqual(sanitized);+ expect(fetch).toHaveBeenCalledTimes(1);+ const batch = prepared.batch!;+ expect(+ JSON.parse(batch.takeJsonBuffer().toString("utf8"))[0].note,+ ).toBe("��");+ expect(batch.media).toHaveLength(1);+ expect(await batch.mediaBody(0)).toEqual(mediaBody);+ } finally {+ await prepared?.batch?.dispose();+ fetch.mockRestore();+ configuredMasking.mockImplementation(applyMasking);+ }+ },+ );+it.each(["retry-success", "fail-open", "fail-closed"] as const)("validates raw masking responses inside retries (%s)",async (outcome) => {diff --git a/worker/src/__tests__/nativeOtelMedia.unit.test.ts b/worker/src/__tests__/nativeOtelMedia.unit.test.tsindex 1c51ba99c6..1d69c9a643 100644--- a/worker/src/__tests__/nativeOtelMedia.unit.test.ts+++ b/worker/src/__tests__/nativeOtelMedia.unit.test.ts@@ -233,14 +233,18 @@ describe(JSON.parse(JSON.parse(attributes[2]!.value.stringValue)),).toMatchObject({ data: batch.media[2]!.reference });await expect(- batch.originalMedia(0, batch.media[0]!.originalJsonDepth),- ).resolves.toBe(MEDIA_URI);- await expect(- batch.originalMedia(1, batch.media[1]!.originalJsonDepth),- ).resolves.toBe(MEDIA_URI);- await expect(- batch.originalMedia(2, batch.media[2]!.originalJsonDepth),- ).resolves.toBe(providerBody.toString("base64"));+ batch.originalMediaBatch(+ [2, 0, 2, 1].map((index) => ({+ index,+ jsonLayers: batch.media[index]!.originalJsonDepth,+ })),+ ),+ ).resolves.toEqual([+ providerBody.toString("base64"),+ MEDIA_URI,+ providerBody.toString("base64"),+ MEDIA_URI,+ ]);} finally {await batch.dispose();await validated.dispose();@@ -512,13 +516,20 @@ it(createHash("sha256").update(MEDIA_BODY).digest("base64"),);await expect(batch.mediaBody(1)).rejects.toThrow("unknown media index");+ await expect(+ batch.originalMediaBatch([{ index: 1, jsonLayers: 0 }]),+ ).rejects.toMatchObject({ code: "ERR_OTEL_MEDIA_INDEX" });const body = batch.mediaBody(media.index);const original = batch.originalMedia(media.index);+ const originalBatch = batch.originalMediaBatch([+ { index: media.index, jsonLayers: media.originalJsonDepth },+ ]);await batch.dispose();expect(compact.toString("utf8")).toContain(media.reference);await expect(body).resolves.toEqual(MEDIA_BODY);await expect(original).resolves.toBe(uri);+ await expect(originalBatch).resolves.toEqual([uri]);await expect(batch.copyJsonBuffer()).rejects.toMatchObject({code: "ERR_OTEL_CLOSED",});@@ -534,6 +545,9 @@ it(await expect(batch.originalMedia(0)).rejects.toMatchObject({code: "ERR_OTEL_CLOSED",});+ await expect(+ batch.originalMediaBatch([{ index: 0, jsonLayers: 0 }]),+ ).rejects.toMatchObject({ code: "ERR_OTEL_CLOSED" });const copyValidated = await validateOtelJson(Buffer.from(JSON.stringify({ input: uri })),diff --git a/worker/src/__tests__/otelReplay.corpus.test.ts b/worker/src/__tests__/otelReplay.corpus.test.tsindex e4c54ed8ad..d630536747 100644--- a/worker/src/__tests__/otelReplay.corpus.test.ts+++ b/worker/src/__tests__/otelReplay.corpus.test.ts@@ -5,7 +5,10 @@ import { vercelAiSdkMixedToolMessagesFixture } from "../../../packages/shared/srimport { langgraphProductionShapeFixture } from "../../../packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures";import { pydanticAiProductionShapeFixture } from "../../../packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures";import { microsoftAgentProductionShapeFixture } from "../../../packages/shared/src/utils/normalized-io/conventions/providers/otel-genai/fixtures";-import { runOtelReplay } from "./helpers/otelReplayHarness";+import {+ expectRawOtelReplayParity,+ runOtelReplayComparison,+} from "./helpers/otelReplayHarness";type ScopeSpan = NonNullable<ResourceSpan["scopeSpans"]>[number];type Span = NonNullable<ScopeSpan["spans"]>[number];@@ -361,67 +364,71 @@ describe("OTEL replay provider corpus", { retry: 0, timeout: 120_000 }, () => {const sourceSpan = resourceSpans[0].scopeSpans?.[0].spans?.[0];expect(sourceSpan).toBeDefined();- const { storedRows } = await runOtelReplay({+ const comparison = await runOtelReplayComparison({bytes: Buffer.from(JSON.stringify(resourceSpans)),projectId: `otel-replay-${name.toLowerCase().replaceAll(" ", "-")}`,mediaUploadEnabled: true,});- expect(storedRows).toHaveLength(1);- const row = storedRows[0];+ for (const result of [comparison.originalTs, comparison.earlyTs]) {+ expect(result.storedRows, result.mode).toHaveLength(1);+ const row = result.storedRows[0];- expect(row.trace_id).toBe(idToHex(sourceSpan!.traceId));- expect(row.span_id).toBe(idToHex(sourceSpan!.spanId));- if (expectedObservationType) {- expect(row.type).toBe(expectedObservationType);- }- expect(row.provided_model_name).toBe(expectedModelName);-- const spanIOInput = parseFixtureValue(fixture.spanIO.input);- const expectedOutput = parseFixtureValue(fixture.spanIO.output);- const storedInput = persistedJsonColumn(row, "input");- expect(persistedJsonColumn(row, "output")).toEqual(expectedOutput);+ expect(row.trace_id).toBe(idToHex(sourceSpan!.traceId));+ expect(row.span_id).toBe(idToHex(sourceSpan!.spanId));+ if (expectedObservationType) {+ expect(row.type).toBe(expectedObservationType);+ }+ expect(row.provided_model_name).toBe(expectedModelName);- if (toolDefinitionsAttribute) {- const toolDefinitions = fixtureJsonAttribute(- fixture,- toolDefinitionsAttribute,- ) as unknown[];- const expectedInput = addToolDefinitionsToInput- ? {- ...(spanIOInput as Record<string, unknown>),- tools: toolDefinitions,- }- : spanIOInput;- expect(storedInput).toEqual(expectedInput);- expect(persistedJsonColumn(row, "tool_definitions")).toEqual(- expectedToolDefinitions(toolDefinitions),- );- } else {- expect(storedInput).toEqual(spanIOInput);- }+ const spanIOInput = parseFixtureValue(fixture.spanIO.input);+ const expectedOutput = parseFixtureValue(fixture.spanIO.output);+ const storedInput = persistedJsonColumn(row, "input");+ expect(persistedJsonColumn(row, "output")).toEqual(expectedOutput);- expect(persistedJsonColumn(row, "provided_usage_details")).toEqual(- expectedUsageDetails,- );- expect(persistedJsonColumn(row, "usage_details")).toEqual(- expectedFinalUsageDetails,- );+ if (toolDefinitionsAttribute) {+ const toolDefinitions = fixtureJsonAttribute(+ fixture,+ toolDefinitionsAttribute,+ ) as unknown[];+ const expectedInput = addToolDefinitionsToInput+ ? {+ ...(spanIOInput as Record<string, unknown>),+ tools: toolDefinitions,+ }+ : spanIOInput;+ expect(storedInput).toEqual(expectedInput);+ expect(persistedJsonColumn(row, "tool_definitions")).toEqual(+ expectedToolDefinitions(toolDefinitions),+ );+ } else {+ expect(storedInput).toEqual(spanIOInput);+ }- if (expectedToolCalls) {- expect(row.tool_call_names).toEqual(- expectedToolCalls.map((call) => call.name),+ expect(persistedJsonColumn(row, "provided_usage_details")).toEqual(+ expectedUsageDetails,);- expect(persistedToolCalls(row)).toEqual(- expectedToolCalls.map(({ name: _name, ...call }) => call),- );- }- if (expectedMetadata) assertMetadataValues(row, expectedMetadata);- if (expectedModelParameters) {- expect(persistedJsonColumn(row, "model_parameters")).toMatchObject(- expectedModelParameters,+ expect(persistedJsonColumn(row, "usage_details")).toEqual(+ expectedFinalUsageDetails,);++ if (expectedToolCalls) {+ expect(row.tool_call_names).toEqual(+ expectedToolCalls.map((call) => call.name),+ );+ expect(persistedToolCalls(row)).toEqual(+ expectedToolCalls.map(({ name: _name, ...call }) => call),+ );+ }+ if (expectedMetadata) assertMetadataValues(row, expectedMetadata);+ if (expectedModelParameters) {+ expect(persistedJsonColumn(row, "model_parameters")).toMatchObject(+ expectedModelParameters,+ );+ }}++ expectRawOtelReplayParity(comparison);},);});diff --git a/worker/src/__tests__/otelReplay.integration.test.ts b/worker/src/__tests__/otelReplay.integration.test.tsindex 1c756ee3f1..e168785e48 100644--- a/worker/src/__tests__/otelReplay.integration.test.ts+++ b/worker/src/__tests__/otelReplay.integration.test.ts@@ -1,24 +1,47 @@import "./helpers/otelReplaySetup";+import { createHash, randomUUID } from "node:crypto";import { Decimal } from "decimal.js";-import { randomUUID } from "node:crypto";-import { afterEach, beforeEach, describe, expect, it } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { env } from "../env";+import { env as sharedEnv, type SharedEnv } from "@langfuse/shared/src/env";+import { MediaAssociationOrigin } from "@langfuse/shared";import {createOrgProjectAndApiKey,IngestionQueue,QueueJobs,TraceUpsertQueue,+ linkMediaToTraceOrObservation,type ResourceSpan,+ uploadMediaForTrace,} from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";-import { runOtelReplay } from "./helpers/otelReplayHarness";+import * as masking from "@langfuse/shared/src/server/ee/ingestionMasking";+import * as otelPreparation from "../features/otel-ingestion/prepareOtelBatch";+import {+ comparableOtelReplayRows,+ expectRawOtelReplayParity,+ runOtelReplayComparison,+ runOneOtelReplay,+ type OtelRawReplayComparison,+} from "./helpers/otelReplayHarness";import {configureDefaultOtelReplayMocks,- configureOtelReplayEnvironment,otelReplayMocks,} from "./helpers/otelReplaySetup";+vi.mock(+ "@langfuse/shared/src/server/ee/ingestionMasking",+ async (importOriginal) => {+ const actual = await importOriginal<typeof masking>();+ return {+ ...actual,+ applyIngestionMasking: vi.fn(actual.applyIngestionMasking),+ isIngestionMaskingEnabled: vi.fn(actual.isIngestionMaskingEnabled),+ };+ },+);+const PROJECT_ID = "otel-replay-integration";const FILE_KEY = "otel-replay/integration.json";const TRACE_ID = "11111111111111111111111111111111";@@ -56,6 +79,159 @@ async function getIngestionJobsForProject(projectId: string) {.filter((job) => job.data.payload.authCheck.scope.projectId === projectId);}+type MediaUploadCall = Parameters<typeof uploadMediaForTrace>[0];+type MediaLinkCall = Parameters<typeof linkMediaToTraceOrObservation>[0];++type MediaAssociation = {+ mediaId: string;+ traceId: string;+ observationId?: string | null;+ field: string;+ contentType: string;+ contentLength: number;+ contentBase64: string;+ contentDigest: string;+ origin: string;+};++function mediaIdForField(field: string): string {+ return field === "input" ? "image-media-id" : "overflow-media-id";+}++function mediaAssociationFromUpload(params: MediaUploadCall): MediaAssociation {+ const contentBytes = Buffer.from(params.contentBytes);+ const contentDigest = createHash("sha256")+ .update(contentBytes)+ .digest("base64");+ if (params.sha256Hash !== undefined) {+ expect(params.sha256Hash).toBe(contentDigest);+ }+ return {+ mediaId: mediaIdForField(params.field),+ traceId: params.traceId,+ observationId: params.observationId,+ field: params.field,+ contentType: params.contentType,+ contentLength: contentBytes.length,+ contentBase64: contentBytes.toString("base64"),+ contentDigest,+ origin: params.origin,+ };+}++function mediaAssociationKey(association: MediaAssociation): string {+ return [+ association.mediaId,+ association.traceId,+ association.observationId,+ association.field,+ association.origin,+ association.contentDigest,+ ]+ .map((value) => value ?? "")+ .join("\u0000");+}++function sortedMediaAssociations(+ associations: MediaAssociation[],+): MediaAssociation[] {+ return associations.sort((left, right) =>+ mediaAssociationKey(left).localeCompare(mediaAssociationKey(right)),+ );+}++function createMediaAssociationCapture(): () => MediaAssociation[] {+ let uploadCursor = 0;+ let linkCursor = 0;++ return () => {+ const uploads = otelReplayMocks.uploadMediaForTrace.mock.calls+ .slice(uploadCursor)+ .map(([rawParams]) =>+ mediaAssociationFromUpload(rawParams as MediaUploadCall),+ );+ const links = otelReplayMocks.linkMediaToTraceOrObservation.mock.calls+ .slice(linkCursor)+ .map(([rawParams]) => rawParams as MediaLinkCall);+ uploadCursor = otelReplayMocks.uploadMediaForTrace.mock.calls.length;+ linkCursor =+ otelReplayMocks.linkMediaToTraceOrObservation.mock.calls.length;++ const assetsById = new Map(+ uploads.map((upload) => [upload.mediaId, upload]),+ );+ const associations = new Map(+ uploads.map((upload) => [mediaAssociationKey(upload), upload]),+ );+ for (const link of links) {+ const asset = assetsById.get(link.mediaId);+ expect(asset, `media link ${link.mediaId} has no upload`).toBeDefined();+ if (!asset) continue;+ const association = {+ ...asset,+ traceId: link.traceId,+ observationId: link.observationId,+ field: link.field,+ origin: link.origin,+ };+ associations.set(mediaAssociationKey(association), association);+ }+ return sortedMediaAssociations([...associations.values()]);+ };+}++type PreparedBatchSnapshot = {+ json: string;+ references: string[];+ media?: Array<{ reference: string; bodyBase64: string }>;+};++function capturePreparedBatches(options: { includeBodies?: boolean } = {}) {+ const compactedBatches: PreparedBatchSnapshot[] = [];+ const prepareOtelBatch = otelPreparation.prepareOtelBatch;+ const preparation = vi+ .spyOn(otelPreparation, "prepareOtelBatch")+ .mockImplementation(async (params) => {+ const result = await prepareOtelBatch(params);+ const batch = result?.batch;+ if (batch) {+ // Snapshot native output before queue parsing and the late TS detector.+ const media = batch.media;+ const snapshot: PreparedBatchSnapshot = {+ json: (await batch.copyJsonBuffer()).toString("utf8"),+ references: media.map(({ reference }) => reference),+ };+ if (options.includeBodies) {+ snapshot.media = await Promise.all(+ media.map(async ({ index, reference }) => ({+ reference,+ bodyBase64: (await batch.mediaBody(index)).toString("base64"),+ })),+ );+ }+ compactedBatches.push(snapshot);+ }+ return result;+ });++ return { compactedBatches, preparation };+}++function expectLegacyReplayParity(comparison: OtelRawReplayComparison) {+ for (const table of [+ "traces",+ "observations",+ "observations_batch_staging",+ ] as const) {+ expect(+ comparableOtelReplayRows(comparison.earlyTs.legacyRows?.[table]),+ `early-ts ${table}`,+ ).toEqual(+ comparableOtelReplayRows(comparison.originalTs.legacyRows?.[table]),+ );+ }+}+function createBufferId(hex: string): Buffer {return Buffer.from(hex, "hex");}@@ -158,20 +334,146 @@ function buildResourceSpans(];}+type RawOtelAttribute = {+ key: string;+ value: Record<string, unknown>;+};++type FocusedReplayParams = {+ attributes?: RawOtelAttribute[];+ resourceAttributes?: RawOtelAttribute[];+ scopeName?: string;+ scopeVersion?: string;+ spanName?: string;+};++function buildFocusedResourceSpans(+ params: FocusedReplayParams,+): ResourceSpan[] {+ return [+ {+ resource: { attributes: params.resourceAttributes ?? [] },+ scopeSpans: [+ {+ scope: {+ name: params.scopeName ?? "otel-replay",+ ...(params.scopeVersion ? { version: params.scopeVersion } : {}),+ },+ spans: [+ {+ traceId: createBufferId(TRACE_ID),+ spanId: createBufferId(FIRST_SPAN_ID),+ name: params.spanName ?? "focused-replay",+ kind: 1,+ startTimeUnixNano: "1714488530686000000",+ endTimeUnixNano: "1714488530687000000",+ attributes: params.attributes ?? [],+ status: { code: 0 },+ },+ ],+ },+ ],+ },+ ];+}++function focusedReplayBytes(params: FocusedReplayParams): Buffer {+ return Buffer.from(JSON.stringify(buildFocusedResourceSpans(params)));+}++function structuredMetadataReplayBytes(params: {+ scopeVersion: string;+ telemetrySdkLanguage: "python" | "nodejs";+ dataUri: string;+}): {+ bytes: Buffer;+ jsonString: string;+ escapedJsonString: string;+ providerJson: string;+} {+ // A string-valued SDK metadata field can itself contain serialized JSON. The+ // SDK then JSON-encodes that value again for the per-key OTLP attribute.+ const jsonString = JSON.stringify({ image: params.dataUri });+ // Native handles escaped MIME separators that the TS text scan leaves inline.+ // Failed uploads must still restore this exact spelling.+ const escapedJsonString = JSON.stringify({ image: params.dataUri }).replace(+ "data:image/",+ "data:image\\/",+ );+ const providerJson = JSON.stringify({+ type: "file",+ mediaType: "image/png",+ data: params.dataUri.slice(params.dataUri.indexOf(",") + 1),+ });+ const metadata = {+ "": JSON.stringify({ collision: { key: "blob wins" } }),+ ".quoted": JSON.stringify("quoted value"),+ ".asset": JSON.stringify({ attachment: { image: params.dataUri } }),+ ".json_string": JSON.stringify(jsonString),+ ".escaped_json_string": JSON.stringify(escapedJsonString),+ ".quoted_provider": JSON.stringify(providerJson),+ ".truncated": '{"value":"cut',+ ".collision.key": JSON.stringify("dotted loses"),+ };++ return {+ bytes: focusedReplayBytes({+ scopeName: "langfuse-sdk",+ scopeVersion: params.scopeVersion,+ resourceAttributes: [+ stringAttribute("telemetry.sdk.language", params.telemetrySdkLanguage),+ ],+ attributes: Object.entries(metadata).map(([path, value]) =>+ stringAttribute(`langfuse.observation.metadata${path}`, value),+ ),+ }),+ jsonString,+ escapedJsonString,+ providerJson,+ };+}++function eventMetadataByPath(+ row: Record<string, unknown>,+): Map<string, string> {+ const names = row.metadata_names as string[];+ const values = row.metadata_values as string[];+ return new Map(names.map((name, index) => [name, values[index]]));+}++// Compare every persisted column, changing only the explicitly expected media+// values in this SDK metadata fixture. The general replay comparator stays exact.+function expectSdkMetadataExpansion(+ comparison: OtelRawReplayComparison,+ expandedValues: Record<string, string>,+) {+ const expected = structuredClone(comparison.originalTs);+ for (const row of expected.storedRows) {+ const metadata = eventMetadataByPath(row);+ const values = [...metadata].map(+ ([key, value]) => expandedValues[key] ?? value,+ );+ row.event_bytes =+ Number(row.event_bytes) ++ Buffer.byteLength(JSON.stringify(values)) -+ Buffer.byteLength(JSON.stringify(row.metadata_values));+ row.metadata_values = values;+ }+ for (const row of Object.values(expected.legacyRows ?? {}).flat()) {+ row.metadata = { ...(row.metadata as object), ...expandedValues };+ }+ const adjusted = { originalTs: expected, earlyTs: comparison.earlyTs };+ expectRawOtelReplayParity(adjusted);+ expectLegacyReplayParity(adjusted);+}+// Vitest retries can overlap unfinished ClickHouse I/O and shared replay mocks.describe("OTEL replay persists enriched observations after media and overflow handling",{ retry: 0, timeout: 120_000 },() => {- let restoreEnvironment: (() => void) | undefined;-beforeEach(() => {configureDefaultOtelReplayMocks();- restoreEnvironment = configureOtelReplayEnvironment({- mediaUploadEnabled: true,- overflowEnabled: true,- overflowSizeLimitBytes: 256,- });const model = {id: "otel-replay-model",@@ -209,15 +511,13 @@ describe(});afterEach(() => {- restoreEnvironment?.();- restoreEnvironment = undefined;// The test owns this mutable mock setup; restore its deterministic// defaults so another worker suite in the same process is unaffected.configureDefaultOtelReplayMocks();});it("writes both enriched observations and preserves accounting through readback", async () => {- const { storedRows } = await runOtelReplay({+ const comparison = await runOtelReplayComparison({bytes: Buffer.from(JSON.stringify(buildResourceSpans())),projectId: PROJECT_ID,fileKey: FILE_KEY,@@ -226,11 +526,13 @@ describe(overflowSizeLimitBytes: 256,});- expect(storedRows).toHaveLength(2);- expect(otelReplayMocks.scheduleObservationEvals).toHaveBeenCalledTimes(2);+ expectRawOtelReplayParity(comparison);+ expect(otelReplayMocks.scheduleObservationEvals).toHaveBeenCalledTimes(4);+ const result = comparison.originalTs;+ expect(result.storedRows).toHaveLength(2);const rowsBySpanId = new Map(- storedRows.map((row) => [String(row.span_id), row]),+ result.storedRows.map((row) => [String(row.span_id), row]),);const firstRow = rowsBySpanId.get(FIRST_SPAN_ID);const secondRow = rowsBySpanId.get(SECOND_SPAN_ID);@@ -303,7 +605,7 @@ describe("@@@langfuseMedia:type=text/plain|id=overflow-media-id|source=field_size_limit@@@",);- for (const row of storedRows) {+ for (const row of result.storedRows) {expect(Number(row.event_bytes)).toBeGreaterThan(0);}});@@ -314,42 +616,654 @@ describe(const bytes = Buffer.from(`[{"resource":{"attributes":[]},"scopeSpans":[{"scope":{"name":"otel-replay"},"spans":[{"traceId":"${TRACE_ID}","spanId":"${FIRST_SPAN_ID}","name":"unsafe-number","startTimeUnixNano":"1714488530686000000","endTimeUnixNano":"1714488530687000000","attributes":[{"key":"gen_ai.request.temperature","value":{"doubleValue":9007199254740993}}],"status":{"code":0}}]}]}]`,);- const { storedRows } = await runOtelReplay({+ const comparison = await runOtelReplayComparison({bytes,projectId: `${PROJECT_ID}-unsafe-number`,fileKey: `${FILE_KEY}.unsafe-number`,mediaUploadEnabled: true,});- expect(storedRows).toHaveLength(1);- const metadataIndex = (storedRows[0].metadata_names as string[]).indexOf(- "attributes.gen_ai.request.temperature",- );+ expectRawOtelReplayParity(comparison);+ const result = comparison.originalTs;+ expect(result.storedRows).toHaveLength(1);+ const metadataIndex = (+ result.storedRows[0].metadata_names as string[]+ ).indexOf("attributes.gen_ai.request.temperature");expect(metadataIndex).toBeGreaterThanOrEqual(0);- expect(storedRows[0].metadata_values?.[metadataIndex]).toBe(+ expect(result.storedRows[0].metadata_values?.[metadataIndex]).toBe("9007199254740992",);});- it("persists legacy and events_full rows from raw OTEL bytes in dual mode", async () => {+ it.each([+ ...[+ {+ sdkLabel: "Python 5.0",+ sdkName: "python",+ sdkVersion: "5.0.0",+ telemetrySdkLanguage: "python",+ scopeVersion: "5.0.0",+ },+ {+ sdkLabel: "JavaScript 6.0 fallback",+ sdkName: "unknown",+ sdkVersion: "unknown",+ telemetrySdkLanguage: "nodejs",+ scopeVersion: "6.0.0",+ },+ ].flatMap((sdk) =>+ (["events_only", "dual"] as const).map((writeMode) => ({+ ...sdk,+ writeMode,+ uploadOutcome: "success" as const,+ })),+ ),+ {+ sdkLabel: "Python 5.0 failure",+ sdkName: "python",+ sdkVersion: "5.0.0",+ telemetrySdkLanguage: "python",+ scopeVersion: "5.0.0",+ writeMode: "events_only",+ uploadOutcome: "failure",+ },+ {+ sdkLabel: "JavaScript 6.0 fallback failure",+ sdkName: "unknown",+ sdkVersion: "unknown",+ telemetrySdkLanguage: "nodejs",+ scopeVersion: "6.0.0",+ writeMode: "dual",+ uploadOutcome: "failure",+ },+ ] as const)(+ "processes structured metadata for $sdkLabel",+ async ({+ sdkName,+ sdkVersion,+ telemetrySdkLanguage,+ scopeVersion,+ writeMode,+ uploadOutcome,+ }) => {+ const imageBytes = Buffer.alloc(1024, 42);+ const dataUri = `data:image/png;base64,${imageBytes.toString("base64")}`;+ const structuredMetadata = structuredMetadataReplayBytes({+ scopeVersion,+ telemetrySdkLanguage,+ dataUri,+ });+ const createdProject =+ writeMode === "dual" ? await createOrgProjectAndApiKey() : undefined;+ const projectId =+ createdProject?.projectId ??+ `${PROJECT_ID}-${sdkName}-${uploadOutcome}`;+ const fileKey = `${FILE_KEY}.structured-${sdkName}-${writeMode}-${uploadOutcome}`;+ const { compactedBatches, preparation } = capturePreparedBatches();+ if (uploadOutcome === "failure") {+ otelReplayMocks.uploadMediaForTrace.mockRejectedValue(+ new Error("simulated structured media upload failure"),+ );+ }++ try {+ const replayParams = {+ bytes: structuredMetadata.bytes,+ projectId,+ orgId: createdProject?.orgId,+ fileKey,+ mediaUploadEnabled: true,+ writeMode,+ sdkName,+ sdkVersion,+ captureSideEffects: createMediaAssociationCapture(),+ } as const;+ const comparison = await runOtelReplayComparison(replayParams);+ const expandedImage =+ "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=base64_data_uri@@@";+ const expandedProvider = structuredMetadata.providerJson.replace(+ imageBytes.toString("base64"),+ "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=bytes@@@",+ );+ expectSdkMetadataExpansion(+ comparison,+ uploadOutcome === "success"+ ? {+ escaped_json_string: JSON.stringify({ image: expandedImage }),+ quoted_provider: expandedProvider,+ }+ : {},+ );+ expect(comparison.earlyTs.sideEffects).toEqual(+ comparison.originalTs.sideEffects,+ );++ expect(compactedBatches).toHaveLength(1);+ expect(compactedBatches[0].references).toHaveLength(4);+ for (const reference of compactedBatches[0].references) {+ expect(compactedBatches[0].json).toContain(reference);+ }+ expect(compactedBatches[0].json).not.toContain("data:image");+ const uploadedBodies =+ otelReplayMocks.uploadMediaForTrace.mock.calls.map(([params]) =>+ Buffer.from(params.contentBytes),+ );+ expect(uploadedBodies.length).toBeGreaterThan(0);+ expect(uploadedBodies.every((body) => body.equals(imageBytes))).toBe(+ true,+ );++ const expectedImage =+ uploadOutcome === "success"+ ? "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=base64_data_uri@@@"+ : dataUri;+ const expectedJsonString =+ uploadOutcome === "success"+ ? JSON.stringify({ image: expectedImage })+ : structuredMetadata.jsonString;+ expect(comparison.originalTs.storedRows).toHaveLength(1);+ const originalMetadata = Object.fromEntries(+ eventMetadataByPath(comparison.originalTs.storedRows[0]),+ );+ expect(originalMetadata).toMatchObject({+ quoted: "quoted value",+ "asset.attachment.image": expectedImage,+ json_string: expectedJsonString,+ escaped_json_string: structuredMetadata.escapedJsonString,+ quoted_provider: structuredMetadata.providerJson,+ truncated: '{"value":"cut',+ "collision.key": "blob wins",+ });+ if (writeMode === "dual") {+ expect(comparison.originalTs.legacyRows?.observations).toHaveLength(+ 1,+ );+ expect(+ comparison.originalTs.legacyRows!.observations![0].metadata,+ ).toMatchObject({+ quoted: "quoted value",+ asset: JSON.stringify({+ attachment: { image: expectedImage },+ }),+ json_string: expectedJsonString,+ escaped_json_string: structuredMetadata.escapedJsonString,+ quoted_provider: structuredMetadata.providerJson,+ truncated: '{"value":"cut',+ collision: JSON.stringify({ key: "blob wins" }),+ "collision.key": "dotted loses",+ });+ }+ if (uploadOutcome === "success") {+ const earlyMetadata = Object.fromEntries(+ eventMetadataByPath(comparison.earlyTs.storedRows[0]),+ );+ expect(earlyMetadata).toMatchObject({+ escaped_json_string: JSON.stringify({ image: expandedImage }),+ quoted_provider: expandedProvider,+ });+ }+ } finally {+ preparation.mockRestore();+ if (createdProject) {+ await prisma.project.delete({+ where: { id: createdProject.projectId },+ });+ await prisma.organization.delete({+ where: { id: createdProject.orgId },+ });+ }+ }+ },+ );++ it.each([+ {+ sdkLabel: "Python 4.15",+ sdkName: "python",+ sdkVersion: "4.15.6",+ telemetrySdkLanguage: "python",+ scopeVersion: "4.15.6",+ },+ {+ sdkLabel: "JavaScript 5.11",+ sdkName: "javascript",+ sdkVersion: "5.11.1",+ telemetrySdkLanguage: "nodejs",+ scopeVersion: "5.11.1",+ },+ ] as const)(+ "keeps per-key values encoded for older $sdkLabel scope versions",+ async ({ sdkName, sdkVersion, telemetrySdkLanguage, scopeVersion }) => {+ const dataUri = `data:image/png;base64,${Buffer.alloc(1024, 42).toString("base64")}`;+ const structuredMetadata = structuredMetadataReplayBytes({+ scopeVersion,+ telemetrySdkLanguage,+ dataUri,+ });+ const comparison = await runOtelReplayComparison({+ bytes: structuredMetadata.bytes,+ projectId: `${PROJECT_ID}-old-${sdkName}`,+ fileKey: `${FILE_KEY}.old-${sdkName}`,+ mediaUploadEnabled: true,+ sdkName,+ sdkVersion,+ });++ const expandedImage =+ "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=base64_data_uri@@@";+ const expandedJson = JSON.stringify({ image: expandedImage });+ const expandedProvider = structuredMetadata.providerJson.replace(+ dataUri.slice(dataUri.indexOf(",") + 1),+ "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=bytes@@@",+ );+ expectSdkMetadataExpansion(comparison, {+ json_string: JSON.stringify(expandedJson),+ escaped_json_string: JSON.stringify(expandedJson),+ quoted_provider: JSON.stringify(expandedProvider),+ });+ expect(comparison.originalTs.storedRows).toHaveLength(1);+ const originalMetadata = Object.fromEntries(+ eventMetadataByPath(comparison.originalTs.storedRows[0]),+ );+ expect(originalMetadata).toMatchObject({+ quoted: JSON.stringify("quoted value"),+ asset: JSON.stringify({+ attachment: {+ image:+ "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=base64_data_uri@@@",+ },+ }),+ json_string: JSON.stringify(structuredMetadata.jsonString),+ escaped_json_string: JSON.stringify(+ structuredMetadata.escapedJsonString,+ ),+ quoted_provider: JSON.stringify(structuredMetadata.providerJson),+ truncated: '{"value":"cut',+ "collision.key": "blob wins",+ });+ const earlyMetadata = Object.fromEntries(+ eventMetadataByPath(comparison.earlyTs.storedRows[0]),+ );+ expect(earlyMetadata).toMatchObject({+ json_string: JSON.stringify(expandedJson),+ escaped_json_string: JSON.stringify(expandedJson),+ quoted_provider: JSON.stringify(expandedProvider),+ });+ },+ );++ it("extracts only media accepted by masking", async () => {+ const originalMedia = Buffer.from("original-media".repeat(512));+ const maskedMedia = Buffer.from("masked-media".repeat(512));+ const rawBytes = focusedReplayBytes({+ attributes: [+ stringAttribute(+ "langfuse.observation.input",+ `data:image/png;base64,${originalMedia.toString("base64")}`,+ ),+ ],+ });+ const maskedBytes = focusedReplayBytes({+ attributes: [+ stringAttribute(+ "langfuse.observation.input",+ `data:image/png;base64,${maskedMedia.toString("base64")}`,+ ),+ ],+ });+ const maskingEnv: SharedEnv = {+ ...sharedEnv,+ LANGFUSE_INGESTION_MASKING_CALLBACK_URL:+ "https://masking.example.com/mask",+ LANGFUSE_INGESTION_MASKING_CALLBACK_TIMEOUT_MS: 1_000,+ LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED: "true",+ LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,+ LANGFUSE_INGESTION_MASKING_PROPAGATED_HEADERS: [],+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,+ LANGFUSE_EE_LICENSE_KEY: "langfuse_ee_test-license-key",+ };+ const configuredMasking = vi.mocked(masking.applyIngestionMasking);+ const applyMasking = configuredMasking.getMockImplementation()!;+ const configuredMaskingEnabled = vi.mocked(+ masking.isIngestionMaskingEnabled,+ );+ const isMaskingEnabled =+ configuredMaskingEnabled.getMockImplementation()!;+ configuredMasking.mockImplementation((params, _env, transport) =>+ applyMasking(params, maskingEnv, transport),+ );+ configuredMaskingEnabled.mockImplementation(() =>+ isMaskingEnabled(maskingEnv),+ );++ const maskingRequests: Buffer[] = [];+ const fetch = vi+ .spyOn(globalThis, "fetch")+ .mockImplementation(async (_input, init) => {+ const body = init?.body;+ if (typeof body === "string") {+ maskingRequests.push(Buffer.from(body));+ } else if (body instanceof Uint8Array) {+ maskingRequests.push(Buffer.from(body));+ } else {+ throw new Error("masking callback received an unexpected body");+ }+ return new Response(maskedBytes.toString("utf8"), {+ status: 200,+ headers: { "content-type": "application/json" },+ });+ });++ const { compactedBatches, preparation } = capturePreparedBatches({+ includeBodies: true,+ });++ try {+ const comparison = await runOtelReplayComparison({+ bytes: rawBytes,+ projectId: `${PROJECT_ID}-masking-media`,+ fileKey: `${FILE_KEY}.masking-media`,+ mediaUploadEnabled: true,+ captureSideEffects: createMediaAssociationCapture(),+ });++ expect(fetch).toHaveBeenCalledTimes(2);+ expect(maskingRequests).toHaveLength(2);+ for (const request of maskingRequests) {+ expect(request).toEqual(rawBytes);+ }+ expect(compactedBatches).toHaveLength(1);+ const earlyBatch = compactedBatches[0]!;+ const earlyMedia = earlyBatch.media!;+ expect(earlyMedia).toEqual([+ {+ reference: expect.any(String),+ bodyBase64: maskedMedia.toString("base64"),+ },+ ]);+ expect(earlyBatch.json).toContain(earlyMedia[0]!.reference);+ expect(earlyBatch.json).not.toContain(+ `data:image/png;base64,${maskedMedia.toString("base64")}`,+ );+ const associations = comparison.originalTs+ .sideEffects as MediaAssociation[];+ expect(associations).toHaveLength(1);+ expect(associations[0]?.contentBase64).toBe(+ maskedMedia.toString("base64"),+ );+ expect(comparison.earlyTs.sideEffects).toEqual(associations);+ expect(comparison.originalTs.storedRows).toHaveLength(1);+ expect(String(comparison.originalTs.storedRows[0]?.input)).toContain(+ "id=image-media-id",+ );+ expect(+ String(comparison.originalTs.storedRows[0]?.input),+ ).not.toContain(originalMedia.toString("base64"));+ expectRawOtelReplayParity(comparison);+ } finally {+ preparation.mockRestore();+ fetch.mockRestore();+ configuredMasking.mockImplementation(applyMasking);+ configuredMaskingEnabled.mockImplementation(isMaskingEnabled);+ }+ });++ it("keeps tag media inline before scalar tag values are split", async () => {+ const body = Buffer.alloc(1_024, "tag-media").toString("base64");+ const dataUri = `data:image/png;base64,${body}`;+ const { compactedBatches, preparation } = capturePreparedBatches();+ try {+ const comparison = await runOtelReplayComparison({+ bytes: focusedReplayBytes({+ attributes: [stringAttribute("langfuse.trace.tags", dataUri)],+ }),+ projectId: `${PROJECT_ID}-tag-media`,+ fileKey: `${FILE_KEY}.tag-media`,+ mediaUploadEnabled: true,+ });++ expect(compactedBatches).toHaveLength(1);+ expect(compactedBatches[0].references).toEqual([]);+ expectRawOtelReplayParity(comparison);+ expect(comparison.originalTs.storedRows).toHaveLength(1);+ expect(comparison.originalTs.storedRows[0]?.tags).toEqual([+ "data:image/png;base64",+ body,+ ]);+ } finally {+ preparation.mockRestore();+ }+ });++ it.each([+ ["scope name", { scopeName: "data:text/plain;base64,aGk=" }],+ [+ "attribute key",+ {+ attributes: [+ stringAttribute("data:text/plain;base64,aGk=", "attribute-value"),+ ],+ },+ ],+ ] as const)(+ "keeps structural media in the %s aligned",+ async (name, options) => {+ const comparison = await runOtelReplayComparison({+ bytes: focusedReplayBytes(options),+ projectId: `${PROJECT_ID}-structural-media-${name}`,+ fileKey: `${FILE_KEY}.structural-media-${name}`,+ mediaUploadEnabled: true,+ });++ expectRawOtelReplayParity(comparison);+ expect(comparison.originalTs.storedRows).toHaveLength(1);+ },+ );++ const deepValue = `${"[".repeat(200)}null${"]".repeat(200)}`;+ const shallowDocument = focusedReplayBytes({+ spanName: "deep-document",+ })+ .toString("utf8")+ .replace("[{", `[{"ignored":${deepValue},`);+ const overflowingNumber = focusedReplayBytes({+ attributes: [+ { key: "gen_ai.request.temperature", value: { doubleValue: 0.25 } },+ ],+ })+ .toString("utf8")+ .replace('"doubleValue":0.25', '"doubleValue":1e309');++ let nestedInput: unknown = "leaf";+ for (let depth = 0; depth < 120; depth++) nestedInput = { x: nestedInput };+ const repeatedNestedInput = focusedReplayBytes({+ attributes: [+ stringAttribute(+ "langfuse.observation.input",+ JSON.stringify({+ marker: "data: ",+ items: Array.from({ length: 32 }, () => nestedInput),+ }),+ ),+ ],+ });++ it.each([+ ["repeated nested objects", repeatedNestedInput],+ ["deep JSON nesting", Buffer.from(shallowDocument)],+ [+ "embedded lone surrogate",+ focusedReplayBytes({+ attributes: [+ stringAttribute(+ "langfuse.observation.input",+ JSON.stringify({ text: "\ud800" }),+ ),+ ],+ }),+ ],+ ["out-of-range number", Buffer.from(overflowingNumber)],+ ] as const)(+ "preserves persisted values for %s input",+ async (name, bytes) => {+ const comparison = await runOtelReplayComparison({+ bytes,+ projectId: `${PROJECT_ID}-edge-${name}`,+ fileKey: `${FILE_KEY}.edge-${name}`,+ mediaUploadEnabled: true,+ });++ expectRawOtelReplayParity(comparison);+ expect(comparison.originalTs.storedRows).toHaveLength(1);+ },+ );++ it("persists sanitized UTF-8 consistently across both paths", async () => {+ const bytes = focusedReplayBytes({+ attributes: [+ stringAttribute("langfuse.observation.input", "utf8-probe"),+ ],+ });+ bytes[bytes.indexOf("utf8-probe")] = 0xff;++ const comparison = await runOtelReplayComparison({+ bytes,+ projectId: `${PROJECT_ID}-invalid-utf8`,+ fileKey: `${FILE_KEY}.invalid-utf8`,+ mediaUploadEnabled: true,+ });+ expectRawOtelReplayParity(comparison);+ expect(comparison.originalTs.storedRows).toHaveLength(1);+ expect(comparison.originalTs.storedRows[0].input).toContain("�tf8-probe");+ });++ it("uploads a native provider candidate beyond the TS discovery limits", async () => {+ const contentBytes = Buffer.from("replay-deep-media".repeat(512));+ const encoded = contentBytes.toString("base64");+ // TS stops provider discovery after ten object levels. Keep the unsafe+ // integer as source text to also catch parse-and-reserialize corruption.+ const provider = `{"type":"file","mediaType":"image/png","data":"${encoded}","large":9007199254740993}`;+ const input = `${'{"nested":'.repeat(12)}${provider}${"}".repeat(12)}`;+ const { compactedBatches, preparation } = capturePreparedBatches({+ includeBodies: true,+ });++ try {+ const comparison = await runOtelReplayComparison({+ bytes: focusedReplayBytes({+ attributes: [stringAttribute("langfuse.observation.input", input)],+ }),+ projectId: `${PROJECT_ID}-deep-expansion`,+ fileKey: `${FILE_KEY}.deep-expansion`,+ mediaUploadEnabled: true,+ captureSideEffects: createMediaAssociationCapture(),+ });+ // Assert extraction before normalization or the late TS detector runs.+ expect(compactedBatches).toHaveLength(1);+ const compacted = compactedBatches[0]!;+ const compactedMedia = compacted.media!;+ expect(compactedMedia).toHaveLength(1);+ expect(compactedMedia[0]?.bodyBase64).toBe(encoded);+ expect(compacted.json).not.toContain(encoded);+ const pendingReference = compactedMedia[0]!.reference;+ expect(compacted.json).toContain(pendingReference);+ const publicReference = pendingReference.replace(+ /\|id=[^|@]+/,+ "|id=image-media-id",+ );+ expect(comparison.originalTs.storedRows).toHaveLength(1);+ expect(comparison.originalTs.storedRows[0]?.input).toBe(input);+ expect(comparison.originalTs.sideEffects).toEqual([]);+ expect(comparison.earlyTs.storedRows).toHaveLength(1);+ expect(comparison.earlyTs.storedRows[0]?.input).toBe(+ input.replace(encoded, publicReference),+ );+ expect(comparison.earlyTs.sideEffects).toEqual([+ expect.objectContaining({+ mediaId: "image-media-id",+ traceId: TRACE_ID,+ observationId: FIRST_SPAN_ID,+ field: "input",+ contentType: "image/png",+ contentLength: contentBytes.length,+ contentBase64: encoded,+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,+ }),+ ]);+ } finally {+ preparation.mockRestore();+ }+ });++ it("persists matching legacy rows, media associations, and events_full rows in dual mode", async () => {const { projectId, orgId } = await createOrgProjectAndApiKey();const originalBlobStorageFileLogFlag =env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG;env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG = "true";try {- const { storedRows, legacyRows } = await runOtelReplay({+ const captureMediaAssociations = createMediaAssociationCapture();+ const comparison = await runOtelReplayComparison({bytes: Buffer.from(JSON.stringify(buildResourceSpans())),projectId,orgId,fileKey: `${FILE_KEY}.dual-write`,+ mediaUploadEnabled: true,+ overflowEnabled: true,+ overflowSizeLimitBytes: 256,writeMode: "dual",+ captureSideEffects: captureMediaAssociations,});- expect(storedRows).toHaveLength(2);- expect(legacyRows?.traces?.length).toBeGreaterThan(0);- expect(legacyRows?.observations).toHaveLength(2);- expect(legacyRows?.observations_batch_staging).toHaveLength(0);expect(await getTraceUpsertJobsForProject(projectId)).toHaveLength(0);+ expectRawOtelReplayParity(comparison);+ expect(comparison.originalTs.sideEffects).toEqual(+ comparison.earlyTs.sideEffects,+ );+ const associations = comparison.originalTs+ .sideEffects as MediaAssociation[];+ expect(associations).toHaveLength(3);+ expect(associations).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ mediaId: "image-media-id",+ traceId: TRACE_ID,+ observationId: FIRST_SPAN_ID,+ field: "input",+ contentType: "image/png",+ contentBase64: Buffer.from("small-image").toString("base64"),+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,+ }),+ expect.objectContaining({+ mediaId: "image-media-id",+ traceId: TRACE_ID,+ observationId: undefined,+ field: "input",+ contentType: "image/png",+ contentBase64: Buffer.from("small-image").toString("base64"),+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,+ }),+ expect.objectContaining({+ mediaId: "overflow-media-id",+ traceId: TRACE_ID,+ observationId: SECOND_SPAN_ID,+ field: "output",+ contentType: "text/plain",+ contentBase64: Buffer.from("x".repeat(1024)).toString("base64"),+ origin: MediaAssociationOrigin.INGESTION_FIELD_OVERFLOW,+ }),+ ]),+ );++ expect(comparison.originalTs.storedRows).toHaveLength(2);+ expect(+ comparison.originalTs.legacyRows?.traces?.length,+ ).toBeGreaterThan(0);+ expect(comparison.originalTs.legacyRows?.observations).toHaveLength(2);+ expect(+ comparison.originalTs.legacyRows?.observations_batch_staging,+ ).toHaveLength(0);++ expectLegacyReplayParity(comparison);} finally {const traceJobs = await getTraceUpsertJobsForProject(projectId);await Promise.allSettled(traceJobs.map((job) => job.remove()));@@ -360,64 +1274,182 @@ describe(}});- it("removes replay-owned ingestion jobs when a legacy replay job fails", async () => {- const { projectId, orgId } = await createOrgProjectAndApiKey();- const originalBlobStorageFileLogFlag =- env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG;- env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG = "false";-- const unrelatedQueue = IngestionQueue.getInstance({- shardingKey: `${projectId}-unrelated`,- });- if (!unrelatedQueue)- throw new Error("Ingestion queue is not initialized");- const unrelatedJob = await unrelatedQueue.add(QueueJobs.IngestionJob, {- id: `otel-replay-unrelated-${randomUUID()}`,- timestamp: new Date(),- name: QueueJobs.IngestionJob,- payload: {- data: {- type: "span-create",- eventBodyId: `otel-replay-unrelated-${randomUUID()}`,- fileKey: "unrelated.json",- bucketPrefix: "otel-replay/unrelated/",- ingestionApiKey: "",- ingestionSdkName: "otel-replay-test",- ingestionSdkVersion: "test",- },- authCheck: {- validKey: true,- scope: { projectId, orgId, accessLevel: "project" },- },- },- });-- try {- await expect(- runOtelReplay({- bytes: Buffer.from(- JSON.stringify(buildResourceSpans({ separateTraces: true })),- ),+ it.each(["succeed", "fail"] as const)(+ "preserves mixed existing and inline media in dual writes when uploads %s",+ async (outcome) => {+ const { projectId, orgId } = await createOrgProjectAndApiKey();+ const existingReference =+ "@@@langfuseMedia:type=image/png|id=preexisting-media-id|source=bytes@@@";+ const content = Buffer.from("dual-inline-media".repeat(256));+ const dataUri = `data:image/png;base64,${content.toString("base64")}`;+ const input = JSON.stringify([existingReference, dataUri]);+ const uploadFails = outcome === "fail";+ if (uploadFails) {+ otelReplayMocks.uploadMediaForTrace.mockRejectedValue(+ new Error("simulated media upload failure"),+ );+ }+ // Final rows can match even if the late detector does all the work. Capture the+ // real compact document and registry before the queue parses and disposes them.+ const { compactedBatches, preparation } = capturePreparedBatches();+ const captureMediaAssociations = createMediaAssociationCapture();+ let uploadCursor = 0;+ try {+ const comparison = await runOtelReplayComparison({+ bytes: focusedReplayBytes({+ attributes: [+ stringAttribute("langfuse.trace.input", input),+ stringAttribute("langfuse.observation.input", input),+ ],+ }),projectId,orgId,- fileKey: `${FILE_KEY}.legacy-failure`,+ fileKey: `${FILE_KEY}.dual-media-${outcome}`,+ mediaUploadEnabled: true,writeMode: "dual",- failLegacyQueueProcessing: true,- }),- ).rejects.toThrow("simulated legacy ingestion read failure");+ captureSideEffects: () => {+ const uploads =+ otelReplayMocks.uploadMediaForTrace.mock.calls.slice(+ uploadCursor,+ );+ uploadCursor =+ otelReplayMocks.uploadMediaForTrace.mock.calls.length;+ // Each mode must attempt the upload, including when it fails and the+ // original inline content is therefore the correct persisted result.+ expect(uploads.length).toBeGreaterThan(0);+ for (const [params] of uploads) {+ expect(params.contentBytes).toEqual(content);+ }+ return uploadFails ? [] : captureMediaAssociations();+ },+ });+ expect(compactedBatches).toHaveLength(1);+ const compacted = compactedBatches[0]!;+ expect(compacted.json).toContain(existingReference);+ expect(compacted.json).not.toContain(dataUri);+ expect(compacted.references.length).toBeGreaterThan(0);+ for (const reference of compacted.references) {+ expect(reference).not.toBe(existingReference);+ expect(compacted.json).toContain(reference);+ }+ expectRawOtelReplayParity(comparison);+ expectLegacyReplayParity(comparison);+ const expectedInput = JSON.stringify([+ existingReference,+ uploadFails+ ? dataUri+ : "@@@langfuseMedia:type=image/png|id=image-media-id|source=base64_data_uri@@@",+ ]);+ expect(comparison.originalTs.storedRows).toHaveLength(1);+ expect(comparison.originalTs.legacyRows?.traces).toHaveLength(1);+ expect(comparison.originalTs.legacyRows?.observations).toHaveLength(+ 1,+ );+ for (const row of [+ comparison.originalTs.storedRows[0],+ comparison.originalTs.legacyRows?.traces?.[0],+ comparison.originalTs.legacyRows?.observations?.[0],+ ]) {+ expect(row?.input).toBe(expectedInput);+ }+ if (uploadFails) {+ expect(+ otelReplayMocks.linkMediaToTraceOrObservation,+ ).not.toHaveBeenCalled();+ } else {+ expect(comparison.earlyTs.sideEffects).toEqual(+ comparison.originalTs.sideEffects,+ );+ expect(comparison.originalTs.sideEffects).toHaveLength(2);+ expect(comparison.originalTs.sideEffects).toEqual(+ expect.arrayContaining(+ [undefined, FIRST_SPAN_ID].map((observationId) =>+ expect.objectContaining({+ traceId: TRACE_ID,+ observationId,+ field: "input",+ contentBase64: content.toString("base64"),+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,+ }),+ ),+ ),+ );+ }+ } finally {+ preparation.mockRestore();+ await prisma.project.delete({ where: { id: projectId } });+ await prisma.organization.delete({ where: { id: orgId } });+ }+ },+ );- const remainingJobs = await getIngestionJobsForProject(projectId);- expect(remainingJobs.map((job) => job.id)).toEqual([unrelatedJob.id]);- } finally {- const remainingJobs = await getIngestionJobsForProject(projectId);- await Promise.allSettled(remainingJobs.map((job) => job.remove()));- const traceJobs = await getTraceUpsertJobsForProject(projectId);- await Promise.allSettled(traceJobs.map((job) => job.remove()));- env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG =- originalBlobStorageFileLogFlag;- await prisma.project.delete({ where: { id: projectId } });- await prisma.organization.delete({ where: { id: orgId } });- }- });+ it.each([+ ["original-ts", false],+ ["original-ts", true],+ ["early-ts", false],+ ["early-ts", true],+ ] as const)(+ "removes replay-owned jobs after failure (%s, separate traces: %s)",+ async (mode, separateTraces) => {+ const { projectId, orgId } = await createOrgProjectAndApiKey();+ const originalBlobStorageFileLogFlag =+ env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG;+ env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG = "false";++ const unrelatedQueue = IngestionQueue.getInstance({+ shardingKey: `${projectId}-unrelated`,+ });+ if (!unrelatedQueue)+ throw new Error("Ingestion queue is not initialized");+ const unrelatedJob = await unrelatedQueue.add(QueueJobs.IngestionJob, {+ id: `otel-replay-unrelated-${randomUUID()}`,+ timestamp: new Date(),+ name: QueueJobs.IngestionJob,+ payload: {+ data: {+ type: "span-create",+ eventBodyId: `otel-replay-unrelated-${randomUUID()}`,+ fileKey: "unrelated.json",+ bucketPrefix: "otel-replay/unrelated/",+ ingestionApiKey: "",+ ingestionSdkName: "otel-replay-test",+ ingestionSdkVersion: "test",+ },+ authCheck: {+ validKey: true,+ scope: { projectId, orgId, accessLevel: "project" },+ },+ },+ });++ try {+ await expect(+ runOneOtelReplay({+ mode,+ bytes: Buffer.from(+ JSON.stringify(buildResourceSpans({ separateTraces })),+ ),+ projectId,+ orgId,+ fileKey: `${FILE_KEY}.legacy-failure`,+ writeMode: "dual",+ failLegacyQueueProcessing: true,+ }),+ ).rejects.toThrow("simulated legacy ingestion read failure");++ const remainingJobs = await getIngestionJobsForProject(projectId);+ expect(remainingJobs.map((job) => job.id)).toEqual([unrelatedJob.id]);+ } finally {+ const remainingJobs = await getIngestionJobsForProject(projectId);+ await Promise.allSettled(remainingJobs.map((job) => job.remove()));+ const traceJobs = await getTraceUpsertJobsForProject(projectId);+ await Promise.allSettled(traceJobs.map((job) => job.remove()));+ env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG =+ originalBlobStorageFileLogFlag;+ await prisma.project.delete({ where: { id: projectId } });+ await prisma.organization.delete({ where: { id: orgId } });+ }+ },+ );},);diff --git a/worker/src/__tests__/otelReplay.properties.test.ts b/worker/src/__tests__/otelReplay.properties.test.tsindex 7dfd802695..f2272116a9 100644--- a/worker/src/__tests__/otelReplay.properties.test.ts+++ b/worker/src/__tests__/otelReplay.properties.test.ts@@ -4,7 +4,10 @@ import { randomInt } from "node:crypto";import fc, { type Arbitrary } from "fast-check";import { describe, expect, it } from "vitest";import type { ResourceSpan } from "@langfuse/shared/src/server";-import { runOtelReplay } from "./helpers/otelReplayHarness";+import {+ expectRawOtelReplayParity,+ runOtelReplayComparison,+} from "./helpers/otelReplayHarness";import {type Json,jsonKeyArbitrary,@@ -652,31 +655,37 @@ const directedCases: ReplayCase[] = [async function assertReplayPersists(params: ReplayCase): Promise<void> {const replay = buildReplayCase(params);- const { storedRows } = await runOtelReplay({+ const comparison = await runOtelReplayComparison({bytes: Buffer.from(JSON.stringify(replay.resourceSpans)),projectId: replay.projectId,fileKey: FILE_KEY,mediaUploadEnabled: true,});- expect(storedRows).toHaveLength(replay.expectedRows.length);- const rowsBySpanId = new Map(- storedRows.map((row) => [String(row.span_id), row]),- );- expect([...rowsBySpanId.keys()].sort()).toEqual(- replay.expectedRows.map((row) => row.span_id).sort(),- );+ for (const result of [comparison.originalTs, comparison.earlyTs]) {+ expect(result.storedRows, result.mode).toHaveLength(+ replay.expectedRows.length,+ );+ const rowsBySpanId = new Map(+ result.storedRows.map((row) => [String(row.span_id), row]),+ );+ expect([...rowsBySpanId.keys()].sort(), result.mode).toEqual(+ replay.expectedRows.map((row) => row.span_id).sort(),+ );- for (const expected of replay.expectedRows) {- const row = rowsBySpanId.get(expected.span_id)!;- // Compare every contracted field exactly, including both metadata arrays:- // objectContaining permits unrelated stored columns, not extra array elements.- expect({- ...row,- start_time: persistedMillis(row.start_time),- end_time: persistedMillis(row.end_time),- }).toEqual(expect.objectContaining(expected));+ for (const expected of replay.expectedRows) {+ const row = rowsBySpanId.get(expected.span_id)!;+ // Compare every contracted field exactly, including both metadata arrays:+ // objectContaining permits unrelated stored columns, not extra array elements.+ expect({+ ...row,+ start_time: persistedMillis(row.start_time),+ end_time: persistedMillis(row.end_time),+ }).toEqual(expect.objectContaining(expected));+ }}++ expectRawOtelReplayParity(comparison);}// The replay writer retries internally; Vitest retries could overlap andiff --git a/worker/src/env.ts b/worker/src/env.tsindex db3887217c..8d7aa3dc5a 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -116,6 +116,22 @@ const EnvSchema = z.object({LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: z.enum(["true", "false"]).default("false"),+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED: z+ .enum(["true", "false"])+ .default("false"),+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS: z+ .string()+ .default("")+ .transform((value) =>+ value+ .split(",")+ .map((id) => id.trim())+ .filter(Boolean),+ )+ .refine((ids) => !ids.includes("*") || ids.length === 1, {+ message:+ 'Use "*" alone to select all projects, or a comma-separated list of project IDs',+ }),LANGFUSE_TRACE_BATCH_INGESTION_ENABLED: z.enum(["true", "false"]).default("false"),diff --git a/worker/src/features/otel-ingestion/README.md b/worker/src/features/otel-ingestion/README.mdindex 374bbb1628..36f8b1adcf 100644--- a/worker/src/features/otel-ingestion/README.md+++ b/worker/src/features/otel-ingestion/README.md@@ -14,12 +14,11 @@ environment variables. The configured user needs to read the `events_full`schema and create, insert into, select from, and drop the test tables. Anunavailable ClickHouse server fails the run.-The corpus, integration, and property suites serialize their fixtures to S3-document bytes and pass those bytes to the production OTEL queue processor. The-queue performs the normal download and JSON decoding; normalization, enrichment,-tokenization, media and overflow handling, and ClickHouse persistence use the-TypeScript implementations. Model and prompt lookups, media uploads, and-evaluation scheduling use controlled test doubles.+The corpus, integration, and property suites replay identical stored JSON bytes+through the production OTEL queue with early Rust media extraction off and on.+Both runs retain TypeScript normalization, enrichment, tokenization, remaining+media detection, overflow handling, and the production JSON writer. Model and+prompt lookups, media uploads, and evaluation scheduling use test doubles.Each replay creates isolated Memory tables from the live schema and uses theproduction JSON writer singleton. A thin client adapter substitutes thediff --git a/worker/src/features/otel-ingestion/prepareOtelBatch.ts b/worker/src/features/otel-ingestion/prepareOtelBatch.tsnew file mode 100644index 0000000000..3de26b0988--- /dev/null+++ b/worker/src/features/otel-ingestion/prepareOtelBatch.ts@@ -0,0 +1,167 @@+import {+ validateOtelJson,+ type EarlyOtelBatch,+ type ValidatedOtelJson,+} from "@langfuse/native";+import {+ recordDistribution,+ recordIncrement,+ type ResourceSpan,+} from "@langfuse/shared/src/server";+import { applyIngestionMasking } from "@langfuse/shared/src/server/ee/ingestionMasking";++type PreparedInput =+ | {+ batch: EarlyOtelBatch;+ /** Neither masking nor UTF-8 normalization replaced the caller's bytes. */+ sourceUnchanged: boolean;+ error?: never;+ }+ | { batch?: never; sourceUnchanged?: never; error: string | undefined };++/**+ * Validate raw OTLP bytes and optionally extract media before the TypeScript+ * ingestion graph is built. Masking receives the normalized byte view and may+ * return a replacement body.+ */+export async function prepareOtelBatch(params: {+ bytes: Buffer;+ projectId: string;+ orgId?: string;+ propagatedHeaders?: Record<string, string>;+ extractMedia: boolean;+}): Promise<PreparedInput> {+ const { bytes, extractMedia, ...context } = params;+ const startedAt = performance.now();+ let outcome: "native" | "masking_drop" | "error" = "error";+ let originalInput: ValidatedOtelJson | undefined;+ let maskedInput: ValidatedOtelJson | undefined;+ try {+ originalInput = await validateOtelJson(bytes);+ const masking = await applyIngestionMasking(+ {+ ...context,+ data: {+ bytes: originalInput.normalizedBytes() ?? bytes,+ validated: originalInput,+ },+ },+ undefined,+ {+ body: (input) => {+ if (!(input.bytes.buffer instanceof ArrayBuffer)) {+ throw new Error(+ "masking body must have an ArrayBuffer backing store",+ );+ }+ return new Uint8Array(+ input.bytes.buffer,+ input.bytes.byteOffset,+ input.bytes.byteLength,+ );+ },+ read: async (response) => {+ const disposal = Promise.allSettled([maskedInput?.dispose()]);+ try {+ const json = stripResponseJsonBoms(+ Buffer.from(await response.arrayBuffer()),+ );+ maskedInput = await validateOtelJson(json);+ return {+ bytes: maskedInput.normalizedBytes() ?? json,+ validated: maskedInput,+ };+ } finally {+ await disposal.then(([result]) => {+ if (result.status === "rejected") throw result.reason;+ });+ }+ },+ },+ );+ if (!masking.success) {+ outcome = "masking_drop";+ return { error: masking.error };+ }+ // Release the unused input before extraction allocates its result.+ if (masking.data.validated === originalInput) {+ await maskedInput?.dispose();+ } else {+ await originalInput.dispose();+ }+ const batch = await masking.data.validated.extract(extractMedia);+ outcome = "native";+ return {+ batch,+ sourceUnchanged: masking.data.bytes === bytes,+ };+ } catch (error) {+ outcome = "error";+ throw error;+ } finally {+ await Promise.all([originalInput?.dispose(), maskedInput?.dispose()])+ .catch((error) => {+ outcome = "error";+ return Promise.reject(error);+ })+ .finally(() => {+ const tags = {+ outcome,+ extract_media: extractMedia.toString(),+ };+ recordIncrement(+ "langfuse.ingestion.otel.early_media.preparation",+ 1,+ tags,+ );+ recordDistribution(+ "langfuse.ingestion.otel.early_media.preparation_duration_ms",+ performance.now() - startedAt,+ tags,+ );+ });+ }+}++/** Parse for TypeScript consumers. The caller owns the returned batch; failures release it. */+export async function parsePreparedOtelBatch(+ prepared: { batch: EarlyOtelBatch; sourceUnchanged: boolean },+ downloadedBytes: Uint8Array,+): Promise<{ resourceSpans: ResourceSpan[]; batch?: EarlyOtelBatch }> {+ let batch: EarlyOtelBatch | undefined = prepared.batch;+ try {+ let json: Buffer;+ if (prepared.sourceUnchanged && batch.mediaCount() === 0) {+ // Free Rust's duplicate before parsing the unchanged download. An external+ // Buffer would retain that allocation until Node collects it.+ await batch.dispose();+ batch = undefined;+ json = Buffer.from(+ downloadedBytes.buffer,+ downloadedBytes.byteOffset,+ downloadedBytes.byteLength,+ );+ } else {+ json = batch.takeJsonBuffer();+ }+ return { resourceSpans: JSON.parse(json.toString("utf8")), batch };+ } catch (error) {+ await batch?.dispose();+ throw error;+ }+}++// Match Node's Response.json(): accept up to two leading UTF-8 BOMs and leave+// a third for the JSON parser to reject.+function stripResponseJsonBoms(bytes: Buffer): Buffer {+ let offset = 0;+ while (+ offset < 6 &&+ bytes[offset] === 0xef &&+ bytes[offset + 1] === 0xbb &&+ bytes[offset + 2] === 0xbf+ ) {+ offset += 3;+ }+ return offset === 0 ? bytes : bytes.subarray(offset);+}diff --git a/worker/src/features/otel-ingestion/prepareOtelBatch.unit.test.ts b/worker/src/features/otel-ingestion/prepareOtelBatch.unit.test.tsnew file mode 100644index 0000000000..a728d9d5ed--- /dev/null+++ b/worker/src/features/otel-ingestion/prepareOtelBatch.unit.test.ts@@ -0,0 +1,107 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const nativeMocks = vi.hoisted(() => ({+ validateOtelJson: vi.fn(),+}));+const maskingMocks = vi.hoisted(() => ({+ applyIngestionMasking: vi.fn(),+}));++vi.mock("@langfuse/native", async (importOriginal) => {+ const actual = await importOriginal<typeof import("@langfuse/native")>();+ nativeMocks.validateOtelJson.mockImplementation(actual.validateOtelJson);+ return { ...actual, validateOtelJson: nativeMocks.validateOtelJson };+});++vi.mock("@langfuse/shared/src/server", () => ({+ logger: { warn: vi.fn() },+ recordDistribution: vi.fn(),+ recordIncrement: vi.fn(),+}));++vi.mock("@langfuse/shared/src/server/ee/ingestionMasking", () => ({+ applyIngestionMasking: maskingMocks.applyIngestionMasking,+}));++import { parsePreparedOtelBatch, prepareOtelBatch } from "./prepareOtelBatch";++describe("prepareOtelBatch native input boundary", { retry: 0 }, () => {+ beforeEach(() => {+ nativeMocks.validateOtelJson.mockClear();+ maskingMocks.applyIngestionMasking.mockImplementation(+ async (params: { data: unknown }) => ({+ success: true,+ data: params.data,+ masked: false,+ }),+ );+ });++ it.each(["unchanged", "masked", "sanitized"] as const)(+ "allows download reuse only for unchanged bytes (%s)",+ async (inputKind) => {+ const native =+ await vi.importActual<typeof import("@langfuse/native")>(+ "@langfuse/native",+ );+ nativeMocks.validateOtelJson.mockImplementation(native.validateOtelJson);+ const bytes =+ inputKind === "sanitized"+ ? Buffer.concat([+ Buffer.from('{"note":"'),+ Buffer.from([0xff]),+ Buffer.from('"}'),+ ])+ : Buffer.from('{"note":"original"}');+ if (inputKind === "masked") {+ maskingMocks.applyIngestionMasking.mockImplementation(+ async (_params, _env, transport) => ({+ success: true,+ data: await transport.read(new Response('{"note":"masked"}')),+ masked: true,+ }),+ );+ }+ const prepared = await prepareOtelBatch({+ bytes,+ projectId: "project-id",+ extractMedia: true,+ });+ try {+ if (!prepared.batch) throw new Error("expected native preparation");+ expect(prepared.batch.mediaCount()).toBe(0);+ expect(prepared.sourceUnchanged).toBe(inputKind === "unchanged");+ const parsed = await parsePreparedOtelBatch(prepared, bytes);+ expect(parsed.batch).toBe(+ inputKind === "unchanged" ? undefined : prepared.batch,+ );+ expect(parsed.resourceSpans).toEqual({+ note:+ inputKind === "sanitized"+ ? "�"+ : inputKind === "masked"+ ? "masked"+ : "original",+ });+ } finally {+ await prepared.batch?.dispose();+ }+ },+ );++ it("does not fall back to a second TypeScript processing path", async () => {+ const error = Object.assign(+ new Error("native implementation changed its explanation"),+ { code: "ERR_OTEL_INVALID_JSON" },+ );+ nativeMocks.validateOtelJson.mockRejectedValueOnce(error);++ await expect(+ prepareOtelBatch({+ bytes: Buffer.from("{}"),+ projectId: "project-id",+ extractMedia: false,+ }),+ ).rejects.toBe(error);+ });+});diff --git a/worker/src/features/otel-ingestion/processOtelEvents.ts b/worker/src/features/otel-ingestion/processOtelEvents.tsindex fb652b12f0..7ea9fb7365 100644--- a/worker/src/features/otel-ingestion/processOtelEvents.ts+++ b/worker/src/features/otel-ingestion/processOtelEvents.ts@@ -1,3 +1,5 @@+import { restoreInlineMedia } from "../otel-media/resolveExtractedMedia";+import type { EarlyOtelBatch } from "@langfuse/native";import { convertEventRecordToObservationForEval } from "@langfuse/shared";import {logger,@@ -25,6 +27,8 @@ type ProcessOtelEventsParams = {projectId: string;fileKey: string;shouldWriteToEventsTable: boolean;+ earlyBatch?: EarlyOtelBatch;+ mediaPath?: "early" | "reference";};/**@@ -43,6 +47,8 @@ export async function processOtelEvents({projectId,fileKey,shouldWriteToEventsTable,+ earlyBatch,+ mediaPath,}: ProcessOtelEventsParams): Promise<void> {// Process events for observation evals and direct event writes// This phase handles two independent concerns:@@ -52,7 +58,6 @@ export async function processOtelEvents({// Both require enriched event records with trace-level attributes// (userId, sessionId, tags, release) that processToEvent provides.const eventInputs = processor.processToEvent(resourceSpans);-if (eventInputs.length === 0) {return;}@@ -75,6 +80,14 @@ export async function processOtelEvents({return;}+ if (earlyBatch) {+ // Evaluation-only records skip direct media resolution, so restore their+ // extracted references before evaluation scheduling reads their payloads.+ await restoreInlineMedia(earlyBatch, eventInputs, {+ includePayloads: !shouldWriteToEventsTable,+ });+ }+if (env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED === "true" &&shouldWriteToEventsTable@@ -82,6 +95,8 @@ export async function processOtelEvents({await processOtelEventMedia({targets: createDirectOtelMediaTargets(eventInputs),writePath: "direct",+ earlyBatch,+ mediaPath,projectId,fileKey,mediaBucket: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,diff --git a/worker/src/features/otel-media/mediaReferenceText.ts b/worker/src/features/otel-media/mediaReferenceText.tsnew file mode 100644index 0000000000..1d4b267fd2--- /dev/null+++ b/worker/src/features/otel-media/mediaReferenceText.ts@@ -0,0 +1,281 @@+import type {+ EarlyOtelBatch,+ ExtractedOtelMedia,+ MediaRestoreRequest,+} from "@langfuse/native";+import { asRecord, defineOwnValue } from "@langfuse/shared";++export type MediaRegistry = ReadonlyMap<string, ExtractedOtelMedia>;++const MEDIA_REFERENCE_PREFIX = "@@@langfuseMedia:";+const RESTORATION_BATCH_SIZE = 64;+const RESTORATION_BATCH_SOURCE_BYTES = 1024 * 1024;+// `originalMedia` accepts target serializations through sixteen JSON layers.+// This restoration bound is separate from native's two-layer discovery limit.+const MAX_SERIALIZED_REFERENCE_LAYERS = 16;++type KnownMediaReferenceMatch = {+ reference: string;+ entry: ExtractedOtelMedia;+ index: number;+};++/** Find only references owned by this batch; malformed markers cannot consume a later one. */+function* knownMediaReferenceMatches(+ value: string,+ media: MediaRegistry,+): Generator<KnownMediaReferenceMatch> {+ let searchFrom = 0;+ while (searchFrom < value.length) {+ const index = value.indexOf(MEDIA_REFERENCE_PREFIX, searchFrom);+ if (index === -1) return;+ const end = value.indexOf("@@@", index + MEDIA_REFERENCE_PREFIX.length);+ if (end === -1) return;+ const reference = value.slice(index, end + 3);+ const entry = media.get(reference);+ if (entry) {+ yield { reference, entry, index };+ searchFrom = end + 3;+ } else {+ // An unterminated marker can see the opening delimiter of a later known+ // marker as its closing delimiter. Search inside it before giving up.+ searchFrom = index + MEDIA_REFERENCE_PREFIX.length;+ }+ }+}++function parseSerializedJson(value: string): unknown {+ const trimmed = value.trimStart();+ if (+ !trimmed.startsWith("{") &&+ !trimmed.startsWith("[") &&+ !trimmed.startsWith('"')+ )+ return undefined;+ try {+ const parsed: unknown = JSON.parse(value);+ return Array.isArray(parsed) ||+ asRecord(parsed) ||+ typeof parsed === "string"+ ? parsed+ : undefined;+ } catch {+ return undefined;+ }+}++/**+ * Substitute only this batch's temporary references. Successful uploads need no+ * JSON parsing: everything between reference ranges keeps its exact spelling.+ * On restoration, inspect string layers without reserializing surrounding JSON.+ * Restored text is cached for this call, not retained for the batch's lifetime.+ * A missing resolver, or an undefined result from it, requests restoration.+ */+export function createMediaReferenceRewriter(+ batch: EarlyOtelBatch,+ media: MediaRegistry,+) {+ const originals = new Map<string, Promise<string>>();+ return async (+ value: string,+ resolve?: (entry: ExtractedOtelMedia) => Promise<string | undefined>,+ ): Promise<string> => {+ let layers: Map<number, number> | undefined;+ let output = "";+ let end = 0;+ const matches = knownMediaReferenceMatches(value, media);+ let next = matches.next();+ while (!next.done) {+ const group: KnownMediaReferenceMatch[] = [];+ let sourceBytes = 0;+ while (!next.done && group.length < RESTORATION_BATCH_SIZE) {+ const size = next.value.entry.originalByteLength;+ // Source size estimates the return size; JSON escaping can enlarge it. Large items run alone.+ if (+ group.length > 0 &&+ sourceBytes + size > RESTORATION_BATCH_SOURCE_BYTES+ )+ break;+ group.push(next.value);+ sourceBytes += size;+ next = matches.next();+ }+ const requests = new Map<string, MediaRestoreRequest>();+ const replacements: Array<string | { key: string }> = [];+ for (const match of group) {+ const replacement = await resolve?.(match.entry);+ if (replacement !== undefined) {+ replacements.push(replacement);+ continue;+ }+ if (layers === undefined)+ layers = serializedReferenceLayers(value, media);+ const depth = layers.get(match.index)!;+ const key = `${match.reference}\u0000${depth}`;+ requests.set(key, { index: match.entry.index, jsonLayers: depth });+ replacements.push({ key });+ }+ // Another string may have cached a read while the uploads above were awaited.+ const missing = [...requests].filter(([key]) => !originals.has(key));+ if (missing.length > 0) {+ const reading = batch.originalMediaBatch(+ missing.map(([, request]) => request),+ );+ missing.forEach(([key], index) => {+ originals.set(+ key,+ reading.then((values) => values[index]!),+ );+ });+ }+ const values = await Promise.all(+ replacements.map((replacement) =>+ typeof replacement === "string"+ ? replacement+ : originals.get(replacement.key)!,+ ),+ );+ for (const [index, match] of group.entries()) {+ output += value.slice(end, match.index) + values[index]!;+ end = match.index + match.reference.length;+ }+ }+ return end === 0 ? value : output + value.slice(end);+ };+}++/**+ * Rewrite actual object values in place. Actual object keys are not visited;+ * opaque string values are rewritten as source text, including JSON keys+ * contained inside those strings.+ */+export async function rewriteMediaValues(+ value: unknown,+ rewrite: (text: string) => Promise<string>,+): Promise<unknown> {+ if (typeof value === "string") return rewrite(value);+ if (typeof value !== "object" || value === null) return value;++ // Iterate rather than recursing: normalization can leave deeply nested objects.+ const stack: Array<{+ object: Record<string, unknown>;+ keys?: string[];+ length: number;+ next: number;+ }> = [];+ const push = (object: object) => {+ const keys = Array.isArray(object) ? undefined : Object.keys(object);+ stack.push({+ object: object as Record<string, unknown>,+ keys,+ length: keys?.length ?? (object as unknown[]).length,+ next: 0,+ });+ };+ push(value);+ while (stack.length > 0) {+ const frame = stack[stack.length - 1]!;+ if (frame.next === frame.length) {+ stack.pop();+ continue;+ }+ const index = frame.next++;+ const key = frame.keys ? frame.keys[index]! : index;+ const current = frame.object[key];+ if (typeof current === "string") {+ const next = await rewrite(current);+ if (current !== next) defineOwnValue(frame.object, key, next);+ } else if (typeof current === "object" && current !== null) {+ push(current);+ }+ }+ return value;+}++/**+ * Find the stringified-document depth for each reference in one source string.+ *+ * This is used only on a restoration path. Successful replacement remains+ * a source-text pass and does not parse the surrounding+ * document. Parsing here is for depth inspection only; the original source is+ * still rebuilt by replacing the known reference ranges in the current text.+ */+function serializedReferenceLayers(+ value: string,+ media: MediaRegistry,+): Map<number, number> {+ const matches = [...knownMediaReferenceMatches(value, media)];+ const layers = new Map<number, number>();+ if (matches.length === 0) return layers;++ const root = parseSerializedJson(value);+ if (root === undefined) {+ for (const match of matches) layers.set(match.index, 0);+ return layers;+ }++ const pendingByReference = new Map<string, number[]>();+ const pendingCursor = new Map<string, number>();+ for (const [matchIndex, match] of matches.entries()) {+ const pending = pendingByReference.get(match.reference) ?? [];+ pending.push(matchIndex);+ pendingByReference.set(match.reference, pending);+ }++ const assign = (text: string, depth: number): void => {+ for (const match of knownMediaReferenceMatches(text, media)) {+ const pending = pendingByReference.get(match.reference);+ const cursor = pendingCursor.get(match.reference) ?? 0;+ const matchIndex = pending?.[cursor];+ if (matchIndex === undefined) continue;+ pendingCursor.set(match.reference, cursor + 1);+ layers.set(matches[matchIndex]!.index, depth);+ }+ };++ const stack: Array<{ value: unknown; depth: number }> = [+ { value: root, depth: 1 },+ ];+ while (stack.length > 0) {+ const current = stack.pop()!;+ if (typeof current.value === "string") {+ if (knownMediaReferenceMatches(current.value, media).next().done)+ continue;+ if (current.depth >= MAX_SERIALIZED_REFERENCE_LAYERS) {+ assign(current.value, current.depth);+ continue;+ }+ // A nested JSON string owns the references represented inside it. Do not+ // assign the same textual markers once at the outer string layer and+ // again after decoding the nested document.+ const nested = parseSerializedJson(current.value);+ if (nested !== undefined) {+ stack.push({ value: nested, depth: current.depth + 1 });+ } else {+ assign(current.value, current.depth);+ }+ continue;+ }+ if (Array.isArray(current.value)) {+ for (let index = current.value.length - 1; index >= 0; index--)+ stack.push({ value: current.value[index], depth: current.depth });+ continue;+ }+ const record = asRecord(current.value);+ if (record) {+ const entries = Object.entries(record);+ for (let index = entries.length - 1; index >= 0; index--)+ stack.push({ value: entries[index]![1], depth: current.depth });+ }+ }++ // Duplicate JSON keys can be discarded by JSON.parse. Preserve the depth+ // recorded by native for a source occurrence that the inspection walk cannot+ // associate with a surviving parsed value.+ for (const match of matches) {+ if (!layers.has(match.index)) {+ layers.set(match.index, Math.max(1, match.entry.originalJsonDepth));+ }+ }+ return layers;+}diff --git a/worker/src/features/otel-media/processOtelMedia.test.ts b/worker/src/features/otel-media/processOtelMedia.test.tsindex fcf5809227..9da2f31230 100644--- a/worker/src/features/otel-media/processOtelMedia.test.ts+++ b/worker/src/features/otel-media/processOtelMedia.test.ts@@ -1,4 +1,10 @@import { beforeEach, describe, expect, it, vi } from "vitest";+import {+ validateOtelJson,+ type EarlyOtelBatch,+ type ExtractedOtelMedia,+ type MediaRestoreRequest,+} from "@langfuse/native";const mocks = vi.hoisted(() => {const span = { setAttributes: vi.fn() };@@ -12,6 +18,7 @@ const mocks = vi.hoisted(() => {logger: { warn: vi.fn() },recordDistribution: vi.fn(),span,+ linkMediaToTraceOrObservation: vi.fn(),uploadMediaForTrace: vi.fn(),};});@@ -23,6 +30,8 @@ vi.mock("@langfuse/shared/src/server", () => ({logger: mocks.logger,processOtelMedia: vi.fn(),recordDistribution: mocks.recordDistribution,+ recordIncrement: vi.fn(),+ linkMediaToTraceOrObservation: mocks.linkMediaToTraceOrObservation,uploadMediaForTrace: mocks.uploadMediaForTrace,}));@@ -33,6 +42,11 @@ import {createLegacyOtelMediaTargets,processOtelEventMedia,} from "./processOtelMedia";+import {+ resolveExtractedMedia,+ restoreInlineMedia,+ uploadExtractedMediaOnce,+} from "./resolveExtractedMedia";const processResult = {uploaded: 1,@@ -55,8 +69,135 @@ const processResult = {},};+const DEFAULT_MEDIA_REFERENCE =+ "@@@langfuseMedia:type=image/png|id=provider|source=bytes@@@";++const MEDIA_DESTINATION = {+ projectId: "project-id",+ mediaBucket: "media-bucket",+ mediaPrefix: "media/",+} as const;++function createTestDescriptors(+ count: number,+ prefix: string,+ originalByteLength: number,+): ExtractedOtelMedia[] {+ return Array.from({ length: count }, (_, index) => ({+ index,+ reference: DEFAULT_MEDIA_REFERENCE.replace(+ "provider",+ `${prefix}-${index}`,+ ),+ contentType: "image/png",+ sha256Hash: `${prefix}-hash-${index}`,+ kind: "anthropic",+ originalByteLength,+ originalJsonDepth: 0,+ }));+}++function replaceReference(reference: string, mediaId: string) {+ return reference.replace(/\|id=[^|@]+/, `|id=${mediaId}`);+}++function createFakeMediaBatch({+ originalValue = "aGk=",+ entries,+}: {+ originalValue?: string;+ entries?: ExtractedOtelMedia[];+} = {}) {+ const mediaBody = vi.fn().mockResolvedValue(Buffer.from("media"));+ const descriptors = entries ?? [+ {+ index: 0,+ reference: DEFAULT_MEDIA_REFERENCE,+ contentType: "image/png",+ sha256Hash: "hash",+ kind: "anthropic",+ originalByteLength: 4,+ originalJsonDepth: 0,+ },+ ];+ const readMedia = vi.fn(() => descriptors);+ const mediaCount = vi.fn(() => descriptors.length);+ const mediaPage = vi.fn((offset: number, limit: number) =>+ descriptors.slice(offset, offset + limit),+ );+ const batch = {+ get media() {+ return readMedia();+ },+ mediaCount,+ mediaPage,+ mediaBody,+ originalMedia: vi.fn().mockResolvedValue(originalValue),+ originalMediaBatch: vi.fn(async (requests: MediaRestoreRequest[]) =>+ Promise.all(+ requests.map(({ index, jsonLayers }) =>+ batch.originalMedia(index, jsonLayers),+ ),+ ),+ ),+ } as EarlyOtelBatch;+ return { batch, mediaBody, mediaCount, mediaPage, readMedia };+}++function createProviderPayload(reference: string, spanId = "observation-id") {+ return {+ traceId: "trace-id",+ spanId,+ input: {+ type: "base64",+ media_type: "image/png",+ data: reference,+ },+ };+}++function resolveTarget(+ batch: EarlyOtelBatch,+ payload: unknown,+ writePath: "direct" | "legacy" = "direct",+) {+ return resolveExtractedMedia({+ batch,+ targets: createDirectOtelMediaTargets([payload]),+ ...MEDIA_DESTINATION,+ writePath,+ });+}++async function withExtractedBatch<T>(+ source: T,+ callback: (batch: EarlyOtelBatch, compacted: T) => Promise<void>,+): Promise<void> {+ const validated = await validateOtelJson(Buffer.from(JSON.stringify(source)));+ let batch: EarlyOtelBatch | undefined;+ try {+ batch = await validated.extract(true);+ await callback(batch, JSON.parse(batch.takeJsonBuffer().toString("utf8")));+ } finally {+ try {+ await batch?.dispose();+ } finally {+ await validated.dispose();+ }+ }+}+describe("processOtelEventMedia", () => {- beforeEach(() => vi.clearAllMocks());+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.uploadMediaForTrace.mockReset().mockResolvedValue({+ outcome: "uploaded",+ mediaId: "uploaded",+ });+ mocks.linkMediaToTraceOrObservation+ .mockReset()+ .mockResolvedValue(undefined);+ });it("processes only normalized direct events with media fields", async () => {const eventInput = {@@ -75,10 +216,8 @@ describe("processOtelEventMedia", () => {null,]),writePath: "direct",- projectId: "project-id",+ ...MEDIA_DESTINATION,fileKey: "file-key",- mediaBucket: "media-bucket",- mediaPrefix: "media/",processMedia,});@@ -92,6 +231,7 @@ describe("processOtelEventMedia", () => {},],writePath: "direct",+ mediaPath: "reference",}),);const uploadMedia = processMedia.mock.calls[0]?.[0].uploadMedia;@@ -127,17 +267,17 @@ describe("processOtelEventMedia", () => {expect(mocks.recordDistribution).toHaveBeenCalledWith("langfuse.ingestion.otel.media.batch_byte_length",8,- { write_path: "direct" },+ { write_path: "direct", media_path: "reference" },);expect(mocks.recordDistribution).toHaveBeenCalledWith("langfuse.ingestion.otel.media.batch_checked_byte_length",33,- { write_path: "direct" },+ { write_path: "direct", media_path: "reference" },);expect(mocks.recordDistribution).toHaveBeenCalledWith("langfuse.ingestion.otel.media.processing_duration_ms",expect.any(Number),- { write_path: "direct" },+ { write_path: "direct", media_path: "reference" },);});@@ -149,10 +289,8 @@ describe("processOtelEventMedia", () => {{ traceId: "trace-id", spanId: "observation-id" },]),writePath: "direct",- projectId: "project-id",+ ...MEDIA_DESTINATION,fileKey: "file-key",- mediaBucket: "media-bucket",- mediaPrefix: "media/",processMedia,});@@ -173,10 +311,8 @@ describe("processOtelEventMedia", () => {},]),writePath: "direct",- projectId: "project-id",+ ...MEDIA_DESTINATION,fileKey: "file-key",- mediaBucket: "media-bucket",- mediaPrefix: "media/",processMedia,}),).resolves.toBeUndefined();@@ -184,18 +320,44 @@ describe("processOtelEventMedia", () => {expect(mocks.recordDistribution).toHaveBeenCalledWith("langfuse.ingestion.otel.media.processing_duration_ms",expect.any(Number),- { write_path: "direct" },+ { write_path: "direct", media_path: "reference" },);expect(mocks.recordDistribution).not.toHaveBeenCalledWith("langfuse.ingestion.otel.media.batch_byte_length",expect.any(Number),);+ expect(mocks.span.setAttributes).toHaveBeenCalledWith({+ "langfuse.ingestion.otel.media_path": "reference",+ });expect(mocks.logger.warn).toHaveBeenCalledWith("OTEL media processing failed; continuing ingestion with original span values",expect.objectContaining({ projectId: "project-id", fileKey: "file-key" }),);});+ it("restores extracted media if the residual detector fails before reference resolution", async () => {+ const original = {+ traceId: "trace-id",+ spanId: "observation-id",+ input: `data:image/png;base64,${"aGVsbG8h".repeat(1024)}`,+ };+ await withExtractedBatch(original, async (batch, payload) => {+ expect(payload.input).not.toBe(original.input);+ await processOtelEventMedia({+ earlyBatch: batch,+ targets: createDirectOtelMediaTargets([payload]),+ writePath: "direct",+ ...MEDIA_DESTINATION,+ fileKey: "file-key",+ processMedia: vi+ .fn()+ .mockRejectedValue(new Error("residual detector failed")),+ });+ expect(payload).toEqual(original);+ expect(mocks.uploadMediaForTrace).not.toHaveBeenCalled();+ });+ });+it("creates legacy targets that retain trace and observation body references", () => {const traceBody = {id: "trace-id",@@ -232,4 +394,505 @@ describe("processOtelEventMedia", () => {},]);});++ it("uploads every known early reference in payload values", async () => {+ const reference = DEFAULT_MEDIA_REFERENCE;+ const { batch } = createFakeMediaBatch();+ const deep = { value: reference } as Record<string, unknown>;+ let cursor = deep;+ for (let index = 0; index < 12; index++) {+ const next = { value: reference } as Record<string, unknown>;+ cursor.value = next;+ cursor = next;+ }+ const payload = {+ traceId: "trace-id",+ spanId: "observation-id",+ input: {+ provider: {+ type: "base64",+ media_type: "image/png",+ data: reference,+ unrelated: reference,+ },+ nested: JSON.stringify({+ type: "base64",+ media_type: "image/png",+ data: reference,+ unrelated: reference,+ [reference]: "opaque string key",+ }),+ deep,+ [reference]: "object key stays structural",+ },+ };++ const result = await resolveTarget(batch, payload);++ expect(result).toMatchObject({+ uploaded: 1,+ reused: 5,+ failed: 0,+ candidates: 6,+ bytesProcessed: 30,+ });+ expect(payload.input.provider.data).toBe(+ "@@@langfuseMedia:type=image/png|id=uploaded|source=bytes@@@",+ );+ expect(payload.input.provider.unrelated).toBe(+ "@@@langfuseMedia:type=image/png|id=uploaded|source=bytes@@@",+ );+ const uploaded = reference.replace("id=provider", "id=uploaded");+ expect(JSON.parse(payload.input.nested as string)).toEqual({+ type: "base64",+ media_type: "image/png",+ data: uploaded,+ unrelated: uploaded,+ [uploaded]: "opaque string key",+ });+ expect(JSON.stringify(payload.input.deep)).not.toContain(reference);+ expect(payload.input[reference]).toBe("object key stays structural");+ expect(batch.originalMedia).not.toHaveBeenCalled();+ });++ it("uploads native refs in deeply nested serialized input, output, and metadata", async () => {+ const contentBytes = Buffer.from("native-deep-media".repeat(512));+ const encoded = contentBytes.toString("base64");+ const provider = `{+ "type": "file",+ "mediaType": "image/png",+ "data": "${encoded}",+ "large": 9007199254740993+}`;+ const createDeepPayload = () => {+ let value: unknown = JSON.stringify(provider);+ for (let index = 0; index < 12; index++) value = { nested: value };+ return value;+ };+ const original = {+ traceId: "trace-id",+ spanId: "observation-id",+ input: createDeepPayload(),+ output: createDeepPayload(),+ metadata: createDeepPayload(),+ };+ await withExtractedBatch(original, async (batch, payload) => {+ const descriptors = batch.media;+ expect(descriptors).toHaveLength(3);+ const result = await resolveTarget(batch, payload);++ expect(result).toMatchObject({+ uploaded: 1,+ reused: 2,+ failed: 0,+ candidates: 3,+ bytesProcessed: contentBytes.length * 3,+ });+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledWith(+ expect.objectContaining({+ field: "input",+ traceId: "trace-id",+ observationId: "observation-id",+ contentBytes,+ sha256Hash: descriptors[0]!.sha256Hash,+ }),+ );+ expect(contentBytes.length).toBeLessThanOrEqual(+ descriptors[0]!.originalByteLength,+ );+ expect(+ mocks.linkMediaToTraceOrObservation.mock.calls.map(+ ([params]) => params,+ ),+ ).toEqual(+ ["output", "metadata"].map((field) => ({+ projectId: "project-id",+ traceId: "trace-id",+ observationId: "observation-id",+ mediaId: "uploaded",+ field,+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,+ })),+ );+ const uploadedReference = replaceReference(+ descriptors[0]!.reference,+ "uploaded",+ );+ for (const field of ["input", "output", "metadata"] as const) {+ expect(JSON.stringify(payload[field])).toBe(+ JSON.stringify(original[field]).replace(encoded, uploadedReference),+ );+ }+ });+ });++ it("uploads a descriptor without reading the batch media registry", async () => {+ const { batch, mediaBody: loadBody, readMedia } = createFakeMediaBatch();+ readMedia.mockImplementation(() => {+ throw new Error("media registry should not be read");+ });+ const result = await uploadExtractedMediaOnce({+ batch,+ projectId: "project-id",+ traceId: "trace-id",+ observationId: "observation-id",+ field: "input",+ mediaBucket: "media-bucket",+ mediaPrefix: "media/",+ sha256Hash: "hash",+ contentType: "image/png",+ loadBody,+ });++ expect(result).toEqual({+ outcome: "uploaded",+ mediaId: "uploaded",+ byteLength: 5,+ });+ expect(loadBody).toHaveBeenCalledOnce();+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();+ });++ it.each([4, 2 * 1024 * 1024])(+ "caches registry pages and batches restoration reads (%s source bytes)",+ async (sourceBytes) => {+ const entries = createTestDescriptors(4_097, "page", sourceBytes);+ const { batch, mediaCount, mediaPage } = createFakeMediaBatch({+ entries,+ });+ const reference = [...entries.slice(-65), entries.at(-1)!]+ .map((entry) => entry.reference)+ .join(" ");+ const first = { name: reference };+ const second = { name: reference };++ await restoreInlineMedia(batch, [first]);+ await restoreInlineMedia(batch, [second]);++ expect(first.name).toBe(Array(66).fill("aGk=").join(" "));+ expect(second.name).toBe(first.name);+ expect(+ vi+ .mocked(batch.originalMediaBatch)+ .mock.calls.map(([requests]) => requests.length),+ ).toEqual(sourceBytes === 4 ? [64, 1, 64, 1] : Array(130).fill(1));+ expect(mediaCount).toHaveBeenCalledOnce();+ expect(mediaPage).toHaveBeenCalledTimes(2);+ expect(mediaPage).toHaveBeenNthCalledWith(1, 0, 4_096);+ expect(mediaPage).toHaveBeenNthCalledWith(2, 4_096, 4_096);+ },+ );++ it("reuses uploads across destinations even after a failed link", async () => {+ const { batch, mediaPage, mediaBody } = createFakeMediaBatch({+ originalValue: "original-value",+ });+ const firstTarget = createProviderPayload(+ DEFAULT_MEDIA_REFERENCE,+ "legacy",+ );+ const failedTarget = createProviderPayload(+ DEFAULT_MEDIA_REFERENCE,+ "failed",+ );+ const directTarget = createProviderPayload(+ DEFAULT_MEDIA_REFERENCE,+ "direct",+ );++ await restoreInlineMedia(batch, [firstTarget, directTarget]);+ const first = await resolveTarget(batch, firstTarget, "legacy");+ mocks.linkMediaToTraceOrObservation.mockRejectedValueOnce(+ new Error("link failed"),+ );+ const failed = await resolveTarget(batch, failedTarget);+ const direct = await resolveTarget(batch, directTarget);++ expect(first).toMatchObject({ uploaded: 1, reused: 0, bytesProcessed: 5 });+ expect(failed).toMatchObject({ failed: 1, uploaded: 0, reused: 0 });+ expect(failedTarget.input.data).toBe("original-value");+ expect(direct).toMatchObject({+ failed: 0,+ uploaded: 0,+ reused: 1,+ bytesProcessed: 5,+ });+ expect(directTarget.input.data).toBe(+ replaceReference(DEFAULT_MEDIA_REFERENCE, "uploaded"),+ );+ expect(mediaPage).toHaveBeenCalledOnce();+ expect(mediaBody).toHaveBeenCalledOnce();+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();+ expect(mocks.linkMediaToTraceOrObservation).toHaveBeenLastCalledWith({+ projectId: "project-id",+ traceId: "trace-id",+ observationId: "direct",+ mediaId: "uploaded",+ field: "input",+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,+ });+ });++ it("evicts a failed upload so a later occurrence can retry", async () => {+ const reference = DEFAULT_MEDIA_REFERENCE;+ mocks.uploadMediaForTrace+ .mockRejectedValueOnce(new Error("upload failed"))+ .mockResolvedValueOnce({ outcome: "uploaded", mediaId: "retry" });+ const { batch, mediaBody } = createFakeMediaBatch({+ originalValue: "original-value",+ });+ const target = () => createProviderPayload(reference);++ const failed = await resolveTarget(batch, target());+ const retriedTarget = target();+ const retried = await resolveTarget(batch, retriedTarget);++ expect(failed).toMatchObject({ failed: 1, uploaded: 0 });+ expect(retried).toMatchObject({ failed: 0, uploaded: 1 });+ expect(mediaBody).toHaveBeenCalledTimes(2);+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledTimes(2);+ expect(retriedTarget.input).toMatchObject({+ data: "@@@langfuseMedia:type=image/png|id=retry|source=bytes@@@",+ });+ });++ it("restores each source spelling after identical-content uploads fail", async () => {+ const content = "abc".repeat(2048);+ const input = [+ {+ type: "file",+ mediaType: "image/png",+ data: Buffer.from(content).toString("base64"),+ },+ { type: "file", mediaType: "image/png", data: `b'${content}'` },+ ];+ mocks.uploadMediaForTrace.mockRejectedValue(new Error("upload failed"));+ await withExtractedBatch(input, async (batch, compacted) => {+ const media = batch.media;+ expect(media).toHaveLength(2);+ expect(media[0].reference).not.toBe(media[1].reference);+ expect(media[0].sha256Hash).toBe(media[1].sha256Hash);+ const target = {+ traceId: "trace-id",+ spanId: "observation-id",+ input: compacted,+ };+ const result = await resolveTarget(batch, target);+ expect(result).toMatchObject({ failed: 2, uploaded: 0, reused: 0 });+ expect(target.input).toEqual(input);+ // A failed upload is evicted, so the next occurrence can retry independently.+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledTimes(2);+ });+ });++ const pythonLiteral = String.raw`b'${"a".repeat(4096)}\xff\xd8\xff\xe0test\nquote\'slash\\\x00'`;+ // The escaped marker keeps discovery active through the second JSON layer.+ const pythonProvider = String.raw`{+ "type": "file", "mediaType": "image/jpeg", "data": ${JSON.stringify(pythonLiteral)},+ "marker": "\u0061", "large": 9007199254740993+}`;+ const quoteProvider = String.raw`{"type":"file","mediaType":"text/plain","data":"b'${"a".repeat(4096)}\u0022'"}`;++ it.each([+ ...[1, 2].flatMap((layers) =>+ ["root", "object"].map((placement) => {+ const text =+ layers === 1+ ? pythonProvider+ : JSON.stringify({ child: pythonProvider });+ return {+ name: `Python bytes: ${placement}, ${layers} JSON layers`,+ input: placement === "root" ? text : { nested: text },+ addNormalizedLayer: false,+ };+ }),+ ),+ {+ name: "Unicode-escaped quotes with an extra normalization layer",+ input: `{"child":${JSON.stringify(quoteProvider).replace(/\\"/g, "\\u0022")}}`,+ addNormalizedLayer: true,+ },+ ])(+ "restores exact native text after upload failure: $name",+ async ({ input, addNormalizedLayer }) => {+ mocks.uploadMediaForTrace.mockRejectedValue(new Error("upload failed"));+ const original = { traceId: "trace-id", spanId: "observation-id", input };+ await withExtractedBatch(original, async (batch, target) => {+ expect(batch.media).toHaveLength(1);+ if (addNormalizedLayer)+ target.input = JSON.stringify({ content: target.input });+ expect(JSON.stringify(target.input)).toContain(+ batch.media[0]!.reference,+ );++ const result = await resolveTarget(batch, target);+ expect(result).toMatchObject({ failed: 1, uploaded: 0 });+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();++ expect(target.input).toEqual(+ addNormalizedLayer ? JSON.stringify({ content: input }) : input,+ );+ });+ },+ );++ const escapedUri = String.raw`data:image\/png;base64,${String.raw`\/\/\/\/`.repeat(1024)}`;+ const dataUri = `data:image/png;base64,${"aGkh".repeat(1024)}`;+ it.each([+ {+ name: "escaped Data URI and unrelated slash spelling",+ input: String.raw`{+ "type": "file", "mediaType": "image\/png", "data": "${escapedUri}",+ "unrelated": "keep this \/ spelling"+}`,+ mediaText: escapedUri,+ failUpload: false,+ decodedBytes: Buffer.from("/".repeat(4096), "base64"),+ },+ ...[false, true].map((failUpload) => ({+ name: `unsafe integer spelling, upload ${failUpload ? "failure" : "success"}`,+ input: `{+ "type": "base64", "media_type": "image/png", "data": "${dataUri}",+ "large": 9007199254740993+}`,+ mediaText: dataUri,+ failUpload,+ decodedBytes: undefined,+ })),+ ])(+ "preserves native stringified text: $name",+ async ({ input, mediaText, failUpload, decodedBytes }) => {+ if (failUpload)+ mocks.uploadMediaForTrace.mockRejectedValue(new Error("upload failed"));+ await withExtractedBatch(+ { traceId: "trace-id", spanId: "observation-id", input },+ async (batch, target) => {+ expect(batch.media).toHaveLength(1);+ const reference = batch.media[0]!.reference;+ expect(target.input).toContain(reference);++ const result = await resolveTarget(batch, target);+ expect(result).toMatchObject({+ failed: Number(failUpload),+ uploaded: Number(!failUpload),+ });+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();++ expect(target.input).toBe(+ input.replace(+ mediaText,+ failUpload ? mediaText : replaceReference(reference, "uploaded"),+ ),+ );+ if (decodedBytes)+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledWith(+ expect.objectContaining({+ contentType: "image/png",+ contentBytes: decodedBytes,+ }),+ );+ },+ );+ },+ );++ it("matches native references by source occurrence when JSON reorders numeric keys", async () => {+ mocks.uploadMediaForTrace.mockImplementation(+ async ({ contentBytes }: { contentBytes: Buffer }) => ({+ outcome: "uploaded",+ mediaId: contentBytes.toString("utf8", 0, 3),+ }),+ );+ const first = `{"type":"file","mediaType":"image/png","data":"b'${"abc".repeat(2048)}'"}`;+ const second = `{"type":"file","mediaType":"image/png","data":"b'${"def".repeat(2048)}'"}`;+ // JSON.parse enumerates integer-like keys in ascending order even though+ // the source document visits key "2" before key "1".+ const nested = `{"2":${first},"1":${second}}`;+ await withExtractedBatch(+ { traceId: "trace-id", spanId: "observation-id", input: nested },+ async (batch, target) => {+ const references = batch.media.map((entry) => entry.reference);+ expect(references).toHaveLength(2);++ const result = await resolveTarget(batch, target);+ expect(result).toMatchObject({ uploaded: 2, failed: 0 });+ expect(target.input).toBe(+ nested+ .replace(+ `b'${"abc".repeat(2048)}'`,+ replaceReference(references[0]!, "abc"),+ )+ .replace(+ `b'${"def".repeat(2048)}'`,+ replaceReference(references[1]!, "def"),+ ),+ );+ },+ );+ });++ it("does not let an unterminated marker consume a later known reference", async () => {+ const { batch } = createFakeMediaBatch();+ const prefix = "prefix @@@langfuseMedia:unterminated ";+ const payload = {+ traceId: "trace-id",+ spanId: "span-id",+ input: `${prefix}${DEFAULT_MEDIA_REFERENCE}`,+ };++ await resolveTarget(batch, payload);++ expect(payload.input).toBe(+ `${prefix}${DEFAULT_MEDIA_REFERENCE.replace("provider", "uploaded")}`,+ );+ });++ it("rewrites known references without altering unrelated text in either walker", async () => {+ const { batch, mediaBody } = createFakeMediaBatch();+ const known = String.raw`{+ "large": 9007199254740993, "escaped": "\u0061",+ "data": "${DEFAULT_MEDIA_REFERENCE}"+}`;+ const unknown = String.raw`{ "large": 9007199254740993, "escaped": "\u0061", "ref": "@@@langfuseMedia:existing@@@" }`;+ const untouched = `{+ "type": "file", "mediaType": "image/png", "data": "https://example.com/image.png",+ "large": 9007199254740993+}`;+ const fields = { known, unknown, untouched };+ const record = {+ traceId: "trace-id",+ spanId: "observation-id",+ name: { ...fields },+ input: {+ type: "base64",+ media_type: "image/png",+ data: "https://example.com/image.png",+ ...fields,+ },+ };+ const original = structuredClone(record);++ await restoreInlineMedia(batch, [record]);+ expect(record).toEqual({+ ...original,+ name: {+ ...fields,+ known: known.replace(DEFAULT_MEDIA_REFERENCE, "aGk="),+ },+ });+ expect(mediaBody).not.toHaveBeenCalled();+ expect(mocks.uploadMediaForTrace).not.toHaveBeenCalled();++ await resolveTarget(batch, record);+ expect(record.input).toEqual({+ ...original.input,+ known: known.replace(+ DEFAULT_MEDIA_REFERENCE,+ replaceReference(DEFAULT_MEDIA_REFERENCE, "uploaded"),+ ),+ });+ expect(mediaBody).toHaveBeenCalledOnce();+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();+ });});diff --git a/worker/src/features/otel-media/processOtelMedia.ts b/worker/src/features/otel-media/processOtelMedia.tsindex 1e0a86ad9f..5f12e9aa75 100644--- a/worker/src/features/otel-media/processOtelMedia.ts+++ b/worker/src/features/otel-media/processOtelMedia.ts@@ -1,3 +1,8 @@+import type { EarlyOtelBatch } from "@langfuse/native";+import {+ resolveExtractedMedia,+ restoreInlineMedia,+} from "./resolveExtractedMedia";import {getClickhouseEntityType,instrumentAsync,@@ -18,20 +23,27 @@ const MEDIA_FIELDS = ["input", "output", "metadata"] as const;* Worker integration point for media in normalized OTEL payloads selected for* persistence by the ingestion queue.*- * Responsibilities are split across three layers:+ * Responsibilities:* - This adapter filters event shapes and owns storage configuration,* instrumentation, and fail-open behavior.* - `processOtelMedia` iterates input/output/metadata, supplies OTEL upload- * context, and aggregates processing results.+ * context, and aggregates processing results for media still inline.* - `transformMediaPayload`, called by `processOtelMedia`, contains the generic* Data URI/provider-shape detection and replacement algorithm. It has no* knowledge of OTEL, storage, projects, or tracing.+ * - `resolveExtractedMedia` is the TypeScript resolver for references created+ * by Rust before normalization. It uploads media whose references survive in+ * input/output/metadata, replaces them on success, and restores failed uploads+ * inline. TypeScript detection still runs for media Rust left inline.** Successful replacements mutate each target payload in place. Missing storage- * configuration or unexpected processing errors are logged and swallowed so- * media extraction cannot reject the enclosing OTEL ingestion job.+ * configuration is logged and skips processing. Unexpected processing errors+ * are logged and swallowed after restoring any pending references. If restoration+ * itself fails, that error propagates to avoid persisting unresolved references.*/export async function processOtelEventMedia(params: {+ earlyBatch?: EarlyOtelBatch;+ mediaPath?: "early" | "reference";targets: OtelMediaTarget[];writePath: OtelMediaWritePath;projectId: string;@@ -49,6 +61,8 @@ export async function processOtelEventMedia(params: {mediaPrefix,processMedia = processOtelMedia,} = params;+ const mediaPath =+ params.mediaPath ?? (params.earlyBatch ? "early" : "reference");if (!mediaBucket) {logger.warn(@@ -65,11 +79,17 @@ export async function processOtelEventMedia(params: {{ name: "langfuse.ingestion.otel.media.process" },async (span) => {const startedAt = Date.now();+ span.setAttributes({+ "langfuse.ingestion.otel.media_path": mediaPath,+ });try {+ // Scan media still inline, including values below native's 1 KiB cutoff.+ // References from native extraction are resolved separately below.const result = await processMedia({targets,projectId,writePath,+ mediaPath,mediaBucket,mediaPrefix,uploadMedia: (uploadParams) =>@@ -79,6 +99,23 @@ export async function processOtelEventMedia(params: {}),});+ if (params.earlyBatch) {+ const early = await resolveExtractedMedia({+ batch: params.earlyBatch,+ targets,+ projectId,+ mediaBucket,+ mediaPrefix,+ writePath,+ });+ result.uploaded += early.uploaded;+ result.reused += early.reused;+ result.failed += early.failed;+ result.candidates += early.candidates;+ result.bytesProcessed += early.bytesProcessed;+ result.bytesRemoved += early.bytesRemoved;+ }+span.setAttributes({"langfuse.ingestion.otel.media.uploaded": result.uploaded,"langfuse.ingestion.otel.media.reused": result.reused,@@ -107,7 +144,7 @@ export async function processOtelEventMedia(params: {recordDistribution("langfuse.ingestion.otel.media.batch_byte_length",result.bytesProcessed,- { write_path: writePath },+ { write_path: writePath, media_path: mediaPath },);recordDistribution("langfuse.ingestion.otel.media.batch_checked_byte_length",@@ -115,18 +152,28 @@ export async function processOtelEventMedia(params: {(total, bytes) => total + bytes,0,),- { write_path: writePath },+ { write_path: writePath, media_path: mediaPath },);} finally {recordDistribution("langfuse.ingestion.otel.media.processing_duration_ms",Date.now() - startedAt,- { write_path: writePath },+ { write_path: writePath, media_path: mediaPath },);}},);} catch (error) {+ if (params.earlyBatch) {+ // Unexpected detector failures must not leave pending references in persisted payloads.+ await restoreInlineMedia(+ params.earlyBatch,+ targets.map(({ payload }) => payload),+ {+ includePayloads: true,+ },+ );+ }logger.warn("OTEL media processing failed; continuing ingestion with original span values",{ projectId, fileKey, error },diff --git a/worker/src/features/otel-media/resolveExtractedMedia.ts b/worker/src/features/otel-media/resolveExtractedMedia.tsnew file mode 100644index 0000000000..0db671ede4--- /dev/null+++ b/worker/src/features/otel-media/resolveExtractedMedia.ts@@ -0,0 +1,303 @@+import type { EarlyOtelBatch, ExtractedOtelMedia } from "@langfuse/native";+import {+ defineOwnValue,+ isMediaContentType,+ MediaAssociationOrigin,+ type MediaContentType,+ type MediaField,+} from "@langfuse/shared";+import {+ logger,+ recordDistribution,+ recordIncrement,+ type OtelMediaTarget,+ type OtelMediaWritePath,+ linkMediaToTraceOrObservation,+ uploadMediaForTrace,+} from "@langfuse/shared/src/server";++import {+ createMediaReferenceRewriter,+ rewriteMediaValues,+ type MediaRegistry,+} from "./mediaReferenceText";++const MEDIA_PAGE_SIZE = 4_096;++type CachedMediaUpload = {+ mediaId: string;+ outcome: "uploaded" | "reused";+ byteLength: number;+};++// Retain descriptor metadata and completed upload results for the batch's lifetime.+// Decoded bodies and restored source strings stay local to each processing call.+// Upload-only consumers do not need to materialize the descriptor registry in JS;+// media bodies remain native-owned and are read lazily when needed.+const mediaRegistries = new WeakMap<EarlyOtelBatch, Promise<MediaRegistry>>();+const mediaUploadCache = new WeakMap<+ EarlyOtelBatch,+ Map<string, Promise<CachedMediaUpload>>+>();++async function batchMedia(batch: EarlyOtelBatch): Promise<MediaRegistry> {+ const cached = mediaRegistries.get(batch);+ if (cached) return cached;++ const load = (async () => {+ const count = batch.mediaCount();+ const media = new Map<string, ExtractedOtelMedia>();+ for (let offset = 0; offset < count; ) {+ const page = batch.mediaPage(offset, MEDIA_PAGE_SIZE);+ for (const entry of page) media.set(entry.reference, entry);+ const nextOffset = offset + page.length;+ if (nextOffset <= offset)+ throw new Error("native media page returned no progress");+ offset = nextOffset;+ if (offset < count)+ await new Promise<void>((resolve) => setImmediate(resolve));+ }+ return media;+ })();+ mediaRegistries.set(batch, load);+ try {+ return await load;+ } catch (error) {+ if (mediaRegistries.get(batch) === load) mediaRegistries.delete(batch);+ throw error;+ }+}++/**+ * Upload one extracted asset once for an EarlyOtelBatch and retain only its+ * storage result. Legacy and direct writers can call this with their own lazy+ * body reader and destination link; cache hits never retain or decode the body.+ */+export async function uploadExtractedMediaOnce(params: {+ batch: EarlyOtelBatch;+ projectId: string;+ traceId: string;+ observationId?: string;+ field: MediaField;+ mediaBucket: string;+ mediaPrefix: string;+ sha256Hash: string;+ contentType: MediaContentType;+ loadBody: () => Promise<Buffer>;+}): Promise<CachedMediaUpload> {+ const {+ batch,+ projectId,+ traceId,+ observationId,+ field,+ mediaBucket,+ mediaPrefix,+ sha256Hash,+ contentType,+ loadBody,+ } = params;+ const key = [+ projectId,+ mediaBucket,+ mediaPrefix,+ sha256Hash,+ contentType,+ ].join("\u0000");+ const cache =+ mediaUploadCache.get(batch) ??+ new Map<string, Promise<CachedMediaUpload>>();+ mediaUploadCache.set(batch, cache);++ const cached = cache.get(key);+ if (cached) {+ const result = await cached;+ await linkMediaToTraceOrObservation({+ projectId,+ traceId,+ observationId,+ mediaId: result.mediaId,+ field,+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,+ });+ return { ...result, outcome: "reused" };+ }++ const uploadPromise = (async (): Promise<CachedMediaUpload> => {+ const contentBytes = await loadBody();+ return {+ ...(await uploadMediaForTrace({+ projectId,+ traceId,+ observationId,+ field,+ contentType,+ contentBytes,+ sha256Hash,+ mediaBucket,+ mediaPrefix,+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,+ })),+ byteLength: contentBytes.length,+ };+ })();+ cache.set(key, uploadPromise);+ try {+ return await uploadPromise;+ } catch (error) {+ // A failed upload must not poison another occurrence or a later path that+ // can retry the same asset with a fresh decoded body.+ if (cache.get(key) === uploadPromise) cache.delete(key);+ throw error;+ }+}++/** Restore extracted references in fields that do not pass through media resolution. */+export async function restoreInlineMedia(+ batch: EarlyOtelBatch,+ records: Record<string, unknown>[],+ { includePayloads = false }: { includePayloads?: boolean } = {},+): Promise<void> {+ const media = await batchMedia(batch);+ if (media.size === 0) return;+ const rewrite = createMediaReferenceRewriter(batch, media);+ const restore = (value: unknown) => rewriteMediaValues(value, rewrite);+ for (const record of records) {+ for (const key of Object.keys(record)) {+ if (+ includePayloads ||+ (key !== "input" && key !== "output" && key !== "metadata")+ ) {+ const previous = record[key];+ const next = await restore(previous);+ if (previous !== next) defineOwnValue(record, key, next);+ }+ }+ }+}++/**+ * Upload native candidates surviving in input/output/metadata. Discovery has+ * already checked media syntax and provider shapes; TS scan-cost limits do not+ * invalidate that work. Only failed uploads are restored here.+ */+export async function resolveExtractedMedia(params: {+ batch: EarlyOtelBatch;+ targets: OtelMediaTarget[];+ projectId: string;+ mediaBucket: string;+ mediaPrefix: string;+ writePath: OtelMediaWritePath;+}) {+ const { batch, targets, projectId, mediaBucket, mediaPrefix, writePath } =+ params;+ const stats = {+ uploaded: 0,+ reused: 0,+ failed: 0,+ candidates: 0,+ bytesProcessed: 0,+ bytesRemoved: 0,+ };+ const media = await batchMedia(batch);+ if (media.size === 0) return stats;+ const rewrite = createMediaReferenceRewriter(batch, media);++ async function resolveReference(+ entry: ExtractedOtelMedia,+ target: OtelMediaTarget,+ field: MediaField,+ ): Promise<string | undefined> {+ try {+ const contentType = entry.contentType;+ if (!isMediaContentType(contentType))+ throw new Error("unsupported extracted media type");+ const result = await uploadExtractedMediaOnce({+ batch,+ projectId,+ mediaBucket,+ mediaPrefix,+ sha256Hash: entry.sha256Hash,+ contentType,+ traceId: target.traceId,+ observationId: target.observationId,+ field,+ loadBody: () => batch.mediaBody(entry.index),+ });+ const replacement = entry.reference.replace(+ /\|id=[^|@]+/,+ `|id=${result.mediaId}`,+ );+ stats.candidates++;+ stats.bytesProcessed += result.byteLength;+ stats[result.outcome]++;+ const bytesRemoved = Math.max(+ 0,+ entry.originalByteLength - Buffer.byteLength(replacement),+ );+ stats.bytesRemoved += bytesRemoved;+ recordExtractedMediaUpload(+ result.outcome,+ entry.kind,+ writePath,+ result.byteLength,+ bytesRemoved,+ );+ return replacement;+ } catch (error) {+ stats.failed++;+ recordExtractedMediaUpload("failed", entry.kind, writePath);+ logger.warn(+ "Extracted OTEL media upload failed; keeping the original inline value",+ {+ error,+ projectId,+ traceId: target.traceId,+ observationId: target.observationId,+ field,+ },+ );+ return undefined;+ }+ }++ for (const target of targets) {+ for (const field of ["input", "output", "metadata"] as const) {+ target.payload[field] = await rewriteMediaValues(+ target.payload[field],+ (text) =>+ rewrite(text, (entry) => resolveReference(entry, target, field)),+ );+ }+ }+ return stats;+}++/** Record outcomes for media discovered by the native extractor. */+function recordExtractedMediaUpload(+ outcome: "uploaded" | "reused" | "failed",+ kind: string,+ writePath: OtelMediaWritePath,+ byteLength?: number,+ bytesRemoved = 0,+): void {+ const tags = {+ outcome,+ media_kind: kind,+ write_path: writePath,+ media_path: "early",+ };+ recordIncrement("langfuse.ingestion.otel.media", 1, tags);+ if (byteLength !== undefined)+ recordDistribution(+ "langfuse.ingestion.otel.media.byte_length",+ byteLength,+ tags,+ );+ if (bytesRemoved > 0)+ recordDistribution(+ "langfuse.ingestion.otel.media.bytes_removed",+ bytesRemoved,+ { write_path: writePath, media_path: "early" },+ );+}diff --git a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts b/worker/src/queues/__tests__/otelDirectEventWrite.test.tsindex d35f700f92..1f5eef2a5b 100644--- a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts+++ b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts@@ -1,13 +1,17 @@import { describe, it, expect, vi } from "vitest";import { type Job } from "bullmq";+import type { EarlyOtelBatch } from "@langfuse/native";import {getS3EventStorageClient,+ markProjectIngestFailure,OtelIngestionProcessor,QueueJobs,type QueueName,type TQueueJobTypes,} from "@langfuse/shared/src/server";import { env } from "../../env";+import * as otelPreparation from "../../features/otel-ingestion/prepareOtelBatch";+import * as mediaRestoration from "../../features/otel-media/resolveExtractedMedia";import { IngestionService } from "../../services/IngestionService";import { ClickhouseWriter } from "../../services/ClickhouseWriter";import { fetchObservationEvalRules } from "../../features/evaluation/observationEval";@@ -28,6 +32,7 @@ import {vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),getS3EventStorageClient: vi.fn(),+ markProjectIngestFailure: vi.fn(),}));vi.mock("../../features/evaluation/observationEval",@@ -51,16 +56,32 @@ describe("direct-v4 trace batch tracking", () => {LANGFUSE_MIGRATION_V4_WRITE_MODE: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED:env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED,+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED:+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED,+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS:+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS,};env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED = "false";+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED = "true";+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS = ["*"];try {vi.mocked(getS3EventStorageClient).mockReturnValue({- download: vi.fn().mockResolvedValue("[]"),+ downloadBytes: vi.fn().mockResolvedValue(Buffer.from("[]")),} as unknown as ReturnType<typeof getS3EventStorageClient>);+ vi.spyOn(otelPreparation, "prepareOtelBatch").mockResolvedValue({+ sourceUnchanged: false,+ batch: {+ takeJsonBuffer: () => Buffer.from("[]"),+ dispose: vi.fn(),+ } as unknown as EarlyOtelBatch,+ });+ const restore = vi+ .spyOn(mediaRestoration, "restoreInlineMedia")+ .mockResolvedValue(undefined);vi.mocked(fetchObservationEvalRules).mockResolvedValue([]);vi.spyOn(ClickhouseWriter, "getInstance").mockReturnValue({} as ClickhouseWriter,@@ -122,6 +143,9 @@ describe("direct-v4 trace batch tracking", () => {} as Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>;await processor(job, undefined);+ expect(restore.mock.calls.map(([, records]) => records)).toEqual([+ inputs,+ ]);expect(write).toHaveBeenCalledTimes(3);expect(trackTraceBatchActivity).toHaveBeenCalledExactlyOnceWith("project",@@ -148,7 +172,9 @@ describe("direct-v4 trace batch tracking", () => {env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "legacy";write.mockClear();+ restore.mockClear();await processor(job, undefined);+ expect(restore.mock.calls.map(([, records]) => records)).toEqual([[]]);expect(write).not.toHaveBeenCalled();expect(trackTraceBatchActivity).not.toHaveBeenCalled();} finally {@@ -159,6 +185,145 @@ describe("direct-v4 trace batch tracking", () => {});});+describe("early media extraction rollout", () => {+ const createJob = (projectId: string) =>+ ({+ data: {+ id: "early-media-rollout",+ timestamp: new Date(),+ name: QueueJobs.OtelIngestionJob,+ payload: {+ data: { fileKey: "early-media-rollout.json" },+ authCheck: {+ validKey: true,+ scope: {+ projectId,+ orgId: "org",+ accessLevel: "project",+ },+ },+ ingestionVersion: "4",+ },+ },+ }) as Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>;++ it("releases an unchanged media-free native document before JSON.parse", async () => {+ const original = {+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED:+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED,+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS:+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS,+ };+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED = "true";+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS = ["*"];+ const json = '[{"resource":{"attributes":[]},"scopeSpans":[]}]';+ const bytes = Buffer.from(json);+ vi.mocked(getS3EventStorageClient).mockReturnValue({+ downloadBytes: vi.fn().mockResolvedValue(bytes),+ } as unknown as ReturnType<typeof getS3EventStorageClient>);+ const prepare = otelPreparation.prepareOtelBatch;+ const parse = JSON.parse;+ let checkedBeforeParse = false;+ const stop = new Error("stop after parsing");+ try {+ vi.spyOn(otelPreparation, "prepareOtelBatch").mockImplementation(+ async (params) => {+ const prepared = await prepare(params);+ const batch = prepared.batch!;+ expect(batch.mediaCount()).toBe(0);+ vi.spyOn(batch, "takeJsonBuffer").mockImplementation(() => {+ throw new Error("must not transfer a second document to Node");+ });+ vi.spyOn(JSON, "parse").mockImplementation((text, reviver) => {+ if (text === json) {+ expect(() => batch.mediaCount()).toThrow(/disposed/);+ checkedBeforeParse = true;+ }+ return parse(text, reviver);+ });+ return prepared;+ },+ );+ vi.spyOn(+ OtelIngestionProcessor.prototype,+ "processToIngestionEvents",+ ).mockRejectedValue(stop);+ await expect(+ otelIngestionQueueProcessorBuilder(false)(+ createJob("project"),+ undefined,+ ),+ ).rejects.toBe(stop);+ expect(checkedBeforeParse).toBe(true);+ } finally {+ Object.assign(env, original);+ vi.restoreAllMocks();+ vi.clearAllMocks();+ }+ });++ it.each([+ ["reference", "disabled with wildcard", false, ["*"]],+ ["reference", "enabled with no projects", true, []],+ [+ "reference",+ "enabled with a non-matching project",+ true,+ ["other-project"],+ ],+ ["early", "enabled with a matching project", true, ["project-match"]],+ ["early", "enabled for all projects", true, ["*"]],+ ] as const)(+ "selects the %s path when %s",+ async (expectedPath, label, enabled, projectIds) => {+ const original = {+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED:+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED,+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS:+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS,+ };+ const download = vi+ .fn()+ .mockRejectedValue(new Error("stop after reference download"));+ const downloadBytes = vi+ .fn()+ .mockRejectedValue(new Error("stop after early download"));++ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED = enabled+ ? "true"+ : "false";+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS = [...projectIds];+ vi.mocked(getS3EventStorageClient).mockReturnValue({+ download,+ downloadBytes,+ } as unknown as ReturnType<typeof getS3EventStorageClient>);++ try {+ const processor = otelIngestionQueueProcessorBuilder(false);+ await expect(+ processor(createJob("project-match"), undefined),+ ).rejects.toThrow(`stop after ${expectedPath} download`);++ if (expectedPath === "early") {+ expect(downloadBytes).toHaveBeenCalledExactlyOnceWith(+ "early-media-rollout.json",+ );+ expect(download).not.toHaveBeenCalled();+ } else {+ expect(download).toHaveBeenCalledExactlyOnceWith(+ "early-media-rollout.json",+ );+ expect(downloadBytes).not.toHaveBeenCalled();+ }+ } finally {+ Object.assign(env, original);+ vi.mocked(getS3EventStorageClient).mockReset();+ vi.mocked(markProjectIngestFailure).mockClear();+ }+ },+ );+});+describe("isOrgPastOtelDirectWriteCutoff", () => {const cutoff = "2026-08-06";const inScope = {diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.tsindex 1c2fe8b1b6..9b6f1e4e07 100644--- a/worker/src/queues/otelIngestionQueue.ts+++ b/worker/src/queues/otelIngestionQueue.ts@@ -1,4 +1,5 @@import { Job, Processor } from "bullmq";+import type { EarlyOtelBatch } from "@langfuse/native";import { z } from "zod";import {clickhouseClient,@@ -44,7 +45,12 @@ import {createLegacyOtelMediaTargets,processOtelEventMedia,} from "../features/otel-media/processOtelMedia";+import { restoreInlineMedia } from "../features/otel-media/resolveExtractedMedia";import { processOtelEvents } from "../features/otel-ingestion/processOtelEvents";+import {+ parsePreparedOtelBatch,+ prepareOtelBatch,+} from "../features/otel-ingestion/prepareOtelBatch";/*** Legacy media processing follows legacy persistence, independently of@@ -436,6 +442,7 @@ export const otelIngestionQueueProcessorBuilder = (return async (job: Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>,): Promise<void> => {+ let earlyBatch: EarlyOtelBatch | undefined;try {const projectId = job.data.payload.authCheck.scope.projectId;const publicKey = job.data.payload.data.publicKey ?? "";@@ -491,47 +498,92 @@ export const otelIngestionQueueProcessorBuilder = (// Otherwise, we'd probably have to upsert one row per generated event further below.// Easy change, but needs alignment.- // Download file from blob storage- const resourceSpans = await getS3EventStorageClient(+ const earlyMediaEnabled =+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED === "true" &&+ (env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS.includes("*") ||+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS.includes(+ projectId,+ ));+ const mediaPath = earlyMediaEnabled ? "early" : "reference";+ span?.setAttribute("langfuse.ingestion.otel.media_path", mediaPath);+ // Count selection once per queue attempt, including jobs that later fail.+ recordIncrement("langfuse.ingestion.otel.media_path", 1, {+ path: mediaPath,+ });++ const eventStorage = getS3EventStorageClient(env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,- ).download(fileKey);-- recordHistogram(- "langfuse.ingestion.s3_file_size_bytes",- resourceSpans.length, // At this point it's still a string.- {- skippedS3List: "true",- otel: "true",- },);-- // Parse spans from S3 download- let parsedSpans = JSON.parse(resourceSpans);-- // Apply ingestion masking if enabled (EE feature)- if (isIngestionMaskingEnabled()) {- const maskingResult = await applyIngestionMasking({- data: parsedSpans,+ let parsedSpans: ResourceSpan[];+ if (earlyMediaEnabled) {+ const bytes = await eventStorage.downloadBytes(fileKey);+ recordHistogram(+ "langfuse.ingestion.s3_file_size_bytes",+ bytes.byteLength,+ { skippedS3List: "true", otel: "true" },+ );+ const prepared = await prepareOtelBatch({+ bytes: Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength),projectId,orgId: job.data.payload.authCheck.scope.orgId,propagatedHeaders: job.data.payload.propagatedHeaders,+ extractMedia:+ env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED === "true" &&+ Boolean(env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET),});-- if (!maskingResult.success) {- // Fail-closed: drop event. Emit the S3 location so operators can- // scan logs to identify which raw payloads need to be replayed via- // worker/src/scripts/replayIngestionEventsV2 once the upstream- // masking callback is healthy again.- logger.warn(`Dropping OTEL event due to masking failure`, {+ if (!prepared.batch) {+ logger.warn("Dropping OTEL event due to masking failure", {projectId,orgId: job.data.payload.authCheck.scope.orgId,fileKey,- error: maskingResult.error,+ error: prepared.error,propagatedHeaders: job.data.payload.propagatedHeaders,});return;}- parsedSpans = maskingResult.data;+ ({ batch: earlyBatch, resourceSpans: parsedSpans } =+ await parsePreparedOtelBatch(prepared, bytes));+ } else {+ // Download file from blob storage+ const resourceSpans = await eventStorage.download(fileKey);++ recordHistogram(+ "langfuse.ingestion.s3_file_size_bytes",+ resourceSpans.length, // At this point it's still a string.+ {+ skippedS3List: "true",+ otel: "true",+ },+ );++ // Parse spans from S3 download+ parsedSpans = JSON.parse(resourceSpans);++ // Apply ingestion masking if enabled (EE feature)+ if (isIngestionMaskingEnabled()) {+ const maskingResult = await applyIngestionMasking({+ data: parsedSpans,+ projectId,+ orgId: job.data.payload.authCheck.scope.orgId,+ propagatedHeaders: job.data.payload.propagatedHeaders,+ });++ if (!maskingResult.success) {+ // Fail-closed: drop event. Emit the S3 location so operators can+ // scan logs to identify which raw payloads need to be replayed via+ // worker/src/scripts/replayIngestionEventsV2 once the upstream+ // masking callback is healthy again.+ logger.warn(`Dropping OTEL event due to masking failure`, {+ projectId,+ orgId: job.data.payload.authCheck.scope.orgId,+ fileKey,+ error: maskingResult.error,+ propagatedHeaders: job.data.payload.propagatedHeaders,+ });+ return;+ }+ parsedSpans = maskingResult.data;+ }}// Generate events via OtelIngestionProcessor@@ -545,6 +597,13 @@ export const otelIngestionQueueProcessorBuilder = (});const events: IngestionEventType[] =await processor.processToIngestionEvents(parsedSpans);+ const writesToLegacyTables = v4WritesToLegacyTables(env);+ if (earlyBatch && writesToLegacyTables) {+ await restoreInlineMedia(+ earlyBatch,+ events.map((event) => event.body),+ );+ }// Here, we split the events into observations and non-observations.// Observations go into the IngestionService directly whereas the non-observations make another run through the processEventBatch method.@@ -707,7 +766,6 @@ export const otelIngestionQueueProcessorBuilder = (// validation already guarantees useDirectEventWrite is true here, so// observations and traces don't need the mergeAndWrite / IngestionQueue// detour that would otherwise populate the legacy tables.- const writesToLegacyTables = v4WritesToLegacyTables(env);const skipLegacyWrites = !writesToLegacyTables;const shouldProcessLegacyMedia = shouldProcessLegacyOtelMedia({mediaUploadEnabled: env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED === "true",@@ -731,6 +789,8 @@ export const otelIngestionQueueProcessorBuilder = (await processOtelEventMedia({targets: createLegacyOtelMediaTargets(traces.concat(observations)),writePath: "legacy",+ earlyBatch,+ mediaPath,projectId,fileKey,mediaBucket: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,@@ -780,6 +840,8 @@ export const otelIngestionQueueProcessorBuilder = (projectId,fileKey,shouldWriteToEventsTable,+ earlyBatch,+ mediaPath,});} catch (e) {const fileKey = job.data.payload.data.fileKey;@@ -804,6 +866,8 @@ export const otelIngestionQueueProcessorBuilder = (fields: { fileKey },});throw e;+ } finally {+ await earlyBatch?.dispose();}};};