Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(otel): extract media before TypeScript parsing (#18239)

Valery Meleshkin525f3914d6ff
diff --git a/.env.dev.example b/.env.dev.example
index 68f379d2c0..4f45290cbf 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -159,6 +159,10 @@ LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
# Opt in to extracting and uploading media embedded in OTEL span attributes.
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED=false
+# Early extraction requires both the toggle and a project selection.
+LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED=false
+# Comma-separated project IDs; empty selects nobody, * selects everyone.
+LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS=
# Maximum encoded and decompressed OTLP request body size in bytes.
# LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES=536870912
diff --git a/.env.prod.example b/.env.prod.example
index 1bd617af05..601cad8072 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -300,6 +300,10 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES=536870912
# Simulate OTLP worker admission while keeping inline ingestion authoritative (default: false)
# LANGFUSE_OTEL_INGESTION_WORKER_SHADOW_ENABLED=false
+# Early extraction requires both the toggle and a project selection.
+# LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED=false
+# Comma-separated project IDs; empty selects nobody, * selects everyone.
+# LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS=
# Evaluation worker concurrency
# LANGFUSE_EVAL_EXECUTION_WORKER_CONCURRENCY=5
diff --git a/packages/native/index.d.ts b/packages/native/index.d.ts
index 15434691dc..428e52cb18 100644
--- a/packages/native/index.d.ts
+++ b/packages/native/index.d.ts
@@ -24,6 +24,8 @@ export declare class EarlyOtelBatch {
* Omitting the count returns the exact spelling after parsing the outer document.
*/
originalMedia(index: number, jsonLayers?: number | undefined | null): Promise<string>
+ /** Read media text for several entries in one libuv task, preserving request order. */
+ originalMediaBatch(requests: Array<MediaRestoreRequest>): Promise<string[]>
}
/** A snapshot of one finalized event row, retained until its JS handle and any encoder tasks drop it. */
@@ -93,6 +95,11 @@ export declare function hello(source: string): void
*/
export declare function initTelemetry(options?: TelemetryOptions | undefined | null): void
+export interface MediaRestoreRequest {
+ index: number
+ jsonLayers: number
+}
+
/**
* A complete ClickHouse Native request body and its row count.
*
diff --git a/packages/native/src/otel_input.rs b/packages/native/src/otel_input.rs
index 9606f56380..9107b2811e 100644
--- a/packages/native/src/otel_input.rs
+++ b/packages/native/src/otel_input.rs
@@ -152,6 +152,24 @@ pub struct ExtractedOtelMedia {
pub original_has_json_escapes: bool,
}
+#[napi(object)]
+pub struct MediaRestoreRequest {
+ pub index: u32,
+ pub json_layers: u32,
+}
+
+fn restore_media(
+ data: &ExtractedBatch,
+ index: u32,
+ json_layers: Option<u32>,
+) -> std::result::Result<String, Error<&'static str>> {
+ data.media
+ .get(index as usize)
+ .ok_or_else(|| Error::new("ERR_OTEL_MEDIA_INDEX", "unknown media index"))?
+ .original_value_for_layers(json_layers)
+ .map_err(|error| Error::new("ERR_OTEL_MEDIA_DECODE", error.to_string()))
+}
+
#[napi]
impl EarlyOtelBatch {
/// Release this handle; pending reads retain ownership until they complete.
@@ -273,12 +291,23 @@ impl EarlyOtelBatch {
let inner = self.data();
OtelMediaTask::run("restore", move || {
let inner = inner.map_err(|error| Error::new("ERR_OTEL_CLOSED", error.reason))?;
- inner
- .media
- .get(index as usize)
- .ok_or_else(|| Error::new("ERR_OTEL_MEDIA_INDEX", "unknown media index"))?
- .original_value_for_layers(json_layers)
- .map_err(|error| Error::new("ERR_OTEL_MEDIA_DECODE", error.to_string()))
+ restore_media(&inner, index, json_layers)
+ })
+ }
+
+ /// Read media text for several entries in one libuv task, preserving request order.
+ #[napi(js_name = "originalMediaBatch", ts_return_type = "Promise<string[]>")]
+ pub fn original_media_batch(
+ &self,
+ requests: Vec<MediaRestoreRequest>,
+ ) -> AsyncTask<OtelMediaTask<Vec<String>>> {
+ let inner = self.data();
+ OtelMediaTask::run("restore", move || {
+ let inner = inner.map_err(|error| Error::new("ERR_OTEL_CLOSED", error.reason))?;
+ requests
+ .into_iter()
+ .map(|request| restore_media(&inner, request.index, Some(request.json_layers)))
+ .collect()
})
}
}
diff --git a/packages/native/src/otel_media/rules.rs b/packages/native/src/otel_media/rules.rs
index acdf10c305..b36c7e33d5 100644
--- a/packages/native/src/otel_media/rules.rs
+++ b/packages/native/src/otel_media/rules.rs
@@ -4,8 +4,8 @@
//! must stay intact; the attribute's value can contain media. For example:
//!
//! ```text
-//! span Envelope
-//! `-- attributes[] Attributes
+//! span Span
+//! `-- attributes[] SpanAttributes
//! |-- key: "attachment" Disabled (attribute name)
//! `-- value AnyValue (OTLP value wrapper)
//! `-- stringValue: "data:..." Payload (inspect for media)
@@ -57,10 +57,14 @@ pub(super) enum MediaScanMode {
Payload,
/// An OTLP KeyValue list; keys are metadata names and values contain data.
Attributes,
+ /// Span attributes can also supply tags, whose values must stay inline.
+ SpanAttributes,
/// An OTLP AnyValue wrapper; map keys remain structural until a value is data.
AnyValue,
/// The OTLP envelope; only known payload-bearing descendants enter `Payload`.
Envelope,
+ /// An OTLP span; distinguishes its attributes from resource/scope metadata.
+ Span,
/// A field whose strings are not eligible for extraction.
Disabled,
}
@@ -84,7 +88,7 @@ pub(super) fn media_scan_mode_for_field(parent: MediaScanMode, field: &str) -> M
match parent {
MediaScanMode::Disabled => MediaScanMode::Disabled,
MediaScanMode::Payload => MediaScanMode::Payload,
- MediaScanMode::Attributes => match field {
+ MediaScanMode::Attributes | MediaScanMode::SpanAttributes => match field {
// OTLP KeyValue keys are metadata names, never media payloads.
"key" => MediaScanMode::Disabled,
"value" => MediaScanMode::AnyValue,
@@ -97,12 +101,14 @@ pub(super) fn media_scan_mode_for_field(parent: MediaScanMode, field: &str) -> M
"stringValue" | "bytesValue" => MediaScanMode::Payload,
_ => MediaScanMode::Disabled,
},
- MediaScanMode::Envelope => match field {
+ MediaScanMode::Envelope | MediaScanMode::Span => match field {
// These fields contain envelope objects whose data-bearing children
// are classified by the branches below.
- "resourceSpans" | "scopeSpans" | "spans" | "resource" | "scope" | "events"
- | "links" | "resourceLogs" | "scopeLogs" | "logRecords" => MediaScanMode::Envelope,
+ "resourceSpans" | "scopeSpans" | "resource" | "scope" | "events" | "links"
+ | "resourceLogs" | "scopeLogs" | "logRecords" => MediaScanMode::Envelope,
+ "spans" => MediaScanMode::Span,
// Attribute values and bodies contain user data.
+ "attributes" if parent == MediaScanMode::Span => MediaScanMode::SpanAttributes,
"attributes" => MediaScanMode::Attributes,
"body" | "value" => MediaScanMode::AnyValue,
_ => MediaScanMode::Disabled,
@@ -110,6 +116,19 @@ pub(super) fn media_scan_mode_for_field(parent: MediaScanMode, field: &str) -> M
}
}
+// Match OtelIngestionProcessor.extractTags: scalar tag values can be split on commas.
+pub(super) fn is_tag_attribute(key: &str) -> bool {
+ matches!(
+ key,
+ "langfuse.trace.tags"
+ | "langfuse.tags"
+ | "langfuse.observation.metadata.langfuse_tags"
+ | "langfuse.trace.metadata.langfuse_tags"
+ | "ai.telemetry.metadata.tags"
+ | "tag.tags"
+ )
+}
+
pub(super) fn is_supported_content_type(value: &str) -> bool {
matches!(
value,
diff --git a/packages/native/src/otel_media/structural_walk.rs b/packages/native/src/otel_media/structural_walk.rs
index 31a3f0eced..2222f714da 100644
--- a/packages/native/src/otel_media/structural_walk.rs
+++ b/packages/native/src/otel_media/structural_walk.rs
@@ -34,8 +34,8 @@ use super::json_cursor::{ContainerState, ContainerStep, JsonCursor};
use super::payload::{EarlyMediaError, MediaPayloadKind};
use super::provider_shapes::{structured_shape, ProviderFields, ShapeKey, ValueShape};
use super::rules::{
- is_otel_envelope_field, may_contain_media_candidate, media_scan_mode_for_field, MediaScanMode,
- MIN_EARLY_MEDIA_BYTES,
+ is_otel_envelope_field, is_tag_attribute, may_contain_media_candidate,
+ media_scan_mode_for_field, MediaScanMode, MIN_EARLY_MEDIA_BYTES,
};
use super::WalkStats;
@@ -79,6 +79,7 @@ struct WalkFrame {
#[derive(Default)]
struct ObjectFrame<'a> {
has_envelope_field: bool,
+ is_tag_attribute: bool,
current_shape_key: Option<ShapeKey>,
current_key: Option<Cow<'a, str>>,
current_candidate_start: usize,
@@ -198,7 +199,7 @@ impl<'a, 'stats> StructuralWalk<'a, 'stats> {
};
if parent.state.is_object() {
let object = object_states.last_mut().expect("open object state");
- self.finish_object_field(object, shape);
+ self.finish_object_field(object, shape, parent.modes);
}
self.update_peak(&stack, &object_states);
}
@@ -248,8 +249,20 @@ impl<'a, 'stats> StructuralWalk<'a, 'stats> {
Ok(None)
}
- fn finish_object_field(&mut self, object: &mut ObjectFrame<'a>, shape: ValueShape) {
+ fn finish_object_field(
+ &mut self,
+ object: &mut ObjectFrame<'a>,
+ shape: ValueShape,
+ modes: [MediaScanMode; 2],
+ ) {
if let Some(key) = object.current_key.take() {
+ if key == "key" && modes.contains(&MediaScanMode::SpanAttributes) {
+ object.is_tag_attribute = match &shape {
+ ValueShape::String(range) => decode_json_string(self.input, range.clone())
+ .is_some_and(|value| is_tag_attribute(&value)),
+ _ => false,
+ };
+ }
if let Some(previous) = object.last_key_candidates.remove(key.as_ref()) {
self.add_mask_operation(previous.start, previous.end, 0b11);
}
@@ -355,6 +368,14 @@ impl<'a, 'stats> StructuralWalk<'a, 'stats> {
.pop()
.expect("every closed object frame has object state");
let candidate_end = self.candidates.len();
+ // The attribute's key may follow its value; decide before media decoding/hashing.
+ if object.is_tag_attribute {
+ for (index, mode) in modes.iter().enumerate() {
+ if *mode == MediaScanMode::SpanAttributes {
+ self.add_mask_operation(candidate_start, candidate_end, 1 << index);
+ }
+ }
+ }
let provider_fields = object.provider_fields.as_deref();
let active = if is_envelope_decider {
let selected_bit = if object.has_envelope_field {
diff --git a/packages/native/src/otel_media/tests.rs b/packages/native/src/otel_media/tests.rs
index 04727ad510..2f8b1f199f 100644
--- a/packages/native/src/otel_media/tests.rs
+++ b/packages/native/src/otel_media/tests.rs
@@ -137,6 +137,48 @@ fn nested_otlp_map_keys_stay_structural() {
assert_eq!(entry["value"]["stringValue"], result.media[0].reference());
}
+#[test]
+fn tag_attribute_values_stay_inline_only_on_spans() {
+ let (uri, _) = large_data_uri(b"tag");
+ let value = format!(r#""value":{{"stringValue":"{uri}"}}"#);
+ for key in [
+ "langfuse.trace.tags",
+ "langfuse.tags",
+ "langfuse.observation.metadata.langfuse_tags",
+ "langfuse.trace.metadata.langfuse_tags",
+ "ai.telemetry.metadata.tags",
+ "tag.tags",
+ ] {
+ let tag_key = format!(r#""key":"{key}""#);
+ for (fields, is_tag) in [
+ (format!("{tag_key},{value}"), true),
+ (format!("{value},{tag_key}"), true),
+ (format!(r#""key":"input",{value},{tag_key}"#), true),
+ (format!(r#"{tag_key},{value},"key":"input""#), false),
+ (format!(r#"{tag_key},{value},"key":null"#), false),
+ ] {
+ let attribute = format!("{{{fields}}}");
+ let input = format!(
+ r#"[{{
+ "resource":{{"attributes":[{attribute}]}},
+ "scopeSpans":[{{
+ "scope":{{"attributes":[{attribute}]}},
+ "spans":[{{"attributes":[{attribute},
+ {{"key":"input","value":{{"kvlistValue":{{"values":[{attribute}]}}}}}}
+ ]}}]
+ }}]
+ }},{attribute}]"#
+ );
+ let result = extract_media(input.as_bytes()).unwrap();
+ assert_eq!(result.media.len(), if is_tag { 4 } else { 5 }, "{key}");
+ let compact: Value = serde_json::from_str(&result.compact_json).unwrap();
+ let value =
+ &compact[0]["scopeSpans"][0]["spans"][0]["attributes"][0]["value"]["stringValue"];
+ assert_eq!(value == &uri, is_tag);
+ }
+ }
+}
+
#[test]
fn accepts_deep_otlp_structure_before_embedded_json_limit() {
let mut value = r#""leaf""#.to_owned();
diff --git a/packages/shared/src/server/media/MediaPayloadProcessor.ts b/packages/shared/src/server/media/MediaPayloadProcessor.ts
index 1bb80edc51..d926547402 100644
--- a/packages/shared/src/server/media/MediaPayloadProcessor.ts
+++ b/packages/shared/src/server/media/MediaPayloadProcessor.ts
@@ -1,4 +1,5 @@
import { isMediaContentType, type MediaContentType } from "../../domain/media";
+import { defineOwnValue } from "../../utils/objects";
const MAX_RECURSION_DEPTH = 10;
const DATA_URI_PREFIX = "data:";
@@ -543,21 +544,6 @@ function setTraversalValue(node: TraversalNode, value: unknown): void {
defineOwnValue(node.parent, node.key, value);
}
-function defineOwnValue(
- target: object,
- property: PropertyKey,
- value: unknown,
-): void {
- // Avoid invoking user-controlled setters such as an own `__proto__` setter
- // when replacing values inside parsed or SDK-provided payloads.
- Object.defineProperty(target, property, {
- configurable: true,
- enumerable: true,
- value,
- writable: true,
- });
-}
-
function matchStructuredMedia(
value: Record<string, unknown>,
): StructuredMedia | undefined {
diff --git a/packages/shared/src/server/media/mediaService.ts b/packages/shared/src/server/media/mediaService.ts
index 1490b0e291..2b4fee60b3 100644
--- a/packages/shared/src/server/media/mediaService.ts
+++ b/packages/shared/src/server/media/mediaService.ts
@@ -187,6 +187,11 @@ export async function uploadMediaForTrace(params: {
field: MediaField;
contentType: MediaContentType;
contentBytes: Buffer;
+ /**
+ * Base64 SHA-256 of contentBytes, precomputed during native extraction to avoid
+ * hashing twice. TypeScript uploads omit it; this function computes their hash.
+ */
+ sha256Hash?: string;
mediaBucket: string;
mediaPrefix: string;
origin: MediaAssociationOrigin;
@@ -202,7 +207,9 @@ export async function uploadMediaForTrace(params: {
mediaPrefix,
origin,
} = params;
- const sha256Hash = createHash("sha256").update(contentBytes).digest("base64");
+ const sha256Hash =
+ params.sha256Hash ??
+ createHash("sha256").update(contentBytes).digest("base64");
const mediaId = getMediaId(sha256Hash);
const existingMedia = await prisma.media.findUnique({
where: {
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index 6b6ac015a3..7d4879c79d 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -42,6 +42,7 @@ import { OtelIngestionQueue } from "../redis/otelIngestionQueue";
import { isValidDateString, flattenJsonToPathArrays } from "./utils";
import { convertDateToClickhouseDateTime } from "../clickhouse/client";
import { isNullOrUndefined } from "../../utils/isNullOrUndefined";
+import { defineOwnValue } from "../../utils/objects";
export const AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME = "langfuse-ai-gateway";
@@ -1742,19 +1743,6 @@ export class OtelIngestionProcessor {
);
};
- const defineOwn = (
- container: ReconstructedContainer,
- key: string,
- value: unknown,
- ): void => {
- Object.defineProperty(container, key, {
- configurable: true,
- enumerable: true,
- value,
- writable: true,
- });
- };
-
const applyPath = (
root: ReconstructedContainer,
path: readonly string[],
@@ -1773,7 +1761,7 @@ export class OtelIngestionProcessor {
? []
: Object.create(null);
createdContainers.add(child);
- defineOwn(current, key, child);
+ defineOwnValue(current, key, child);
current = child;
continue;
}
@@ -1800,7 +1788,7 @@ export class OtelIngestionProcessor {
return false;
}
}
- if (commit) defineOwn(current, finalKey, value);
+ if (commit) defineOwnValue(current, finalKey, value);
return true;
};
diff --git a/packages/shared/src/server/otel/OtelMediaProcessor.test.ts b/packages/shared/src/server/otel/OtelMediaProcessor.test.ts
index 1015dac9be..4f2ceb6761 100644
--- a/packages/shared/src/server/otel/OtelMediaProcessor.test.ts
+++ b/packages/shared/src/server/otel/OtelMediaProcessor.test.ts
@@ -90,12 +90,12 @@ describe("processOtelMedia", () => {
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.detection_check",
1,
- { path: "data_uri", write_path: "direct" },
+ { path: "data_uri", write_path: "direct", media_path: "reference" },
);
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.detection_check_byte_length",
Buffer.byteLength(dataUri, "utf8"),
- { path: "data_uri", write_path: "direct" },
+ { path: "data_uri", write_path: "direct", media_path: "reference" },
);
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.byte_length",
@@ -104,6 +104,7 @@ describe("processOtelMedia", () => {
outcome: "uploaded",
media_kind: "data_uri",
write_path: "direct",
+ media_path: "reference",
},
);
expect(result).toMatchObject({
@@ -135,6 +136,7 @@ describe("processOtelMedia", () => {
outcome: "reused",
media_kind: "data_uri",
write_path: "direct",
+ media_path: "reference",
},
);
});
@@ -305,6 +307,7 @@ describe("processOtelMedia", () => {
media_kind: "data_uri",
reason: "unsupported_content_type",
write_path: "direct",
+ media_path: "reference",
},
);
});
@@ -328,6 +331,7 @@ describe("processOtelMedia", () => {
media_kind: "data_uri",
reason: "invalid_base64",
write_path: "direct",
+ media_path: "reference",
},
);
});
diff --git a/packages/shared/src/server/otel/OtelMediaProcessor.ts b/packages/shared/src/server/otel/OtelMediaProcessor.ts
index 9a90c1b5ec..ad206ae94b 100644
--- a/packages/shared/src/server/otel/OtelMediaProcessor.ts
+++ b/packages/shared/src/server/otel/OtelMediaProcessor.ts
@@ -81,6 +81,7 @@ type ProcessContext = {
observationId?: string;
field: MediaField;
writePath: OtelMediaWritePath;
+ mediaPath: "early" | "reference";
mediaBucket: string;
mediaPrefix: string;
uploadMedia: UploadOtelMedia;
@@ -100,6 +101,8 @@ export async function processOtelMedia(params: {
targets: OtelMediaTarget[];
projectId: string;
writePath: OtelMediaWritePath;
+ /** Pipeline label for metrics; does not enable or disable TypeScript media detection. */
+ mediaPath?: "early" | "reference";
mediaBucket: string;
mediaPrefix: string;
uploadMedia: UploadOtelMedia;
@@ -108,6 +111,7 @@ export async function processOtelMedia(params: {
targets,
projectId,
writePath,
+ mediaPath = "reference",
mediaBucket,
mediaPrefix,
uploadMedia,
@@ -143,6 +147,7 @@ export async function processOtelMedia(params: {
observationId: target.observationId,
field,
writePath,
+ mediaPath,
mediaBucket,
mediaPrefix,
uploadMedia,
@@ -168,7 +173,7 @@ export async function processOtelMedia(params: {
recordDistribution(
"langfuse.ingestion.otel.media.bytes_removed",
transformed.bytesRemoved,
- { write_path: context.writePath },
+ { write_path: context.writePath, media_path: context.mediaPath },
);
}
}
@@ -222,6 +227,7 @@ async function processCandidate(
outcome: uploadResult.outcome,
media_kind: candidate.kind,
write_path: context.writePath,
+ media_path: context.mediaPath,
});
recordDistribution(
"langfuse.ingestion.otel.media.byte_length",
@@ -230,6 +236,7 @@ async function processCandidate(
outcome: uploadResult.outcome,
media_kind: candidate.kind,
write_path: context.writePath,
+ media_path: context.mediaPath,
},
);
@@ -240,6 +247,7 @@ async function processCandidate(
outcome: "failed",
media_kind: candidate.kind,
write_path: context.writePath,
+ media_path: context.mediaPath,
});
logger.warn(
"OTEL media upload failed; leaving normalized value unchanged",
@@ -344,11 +352,12 @@ function recordDetectionCheck(
recordIncrement("langfuse.ingestion.otel.media.detection_check", 1, {
path,
write_path: context.writePath,
+ media_path: context.mediaPath,
});
recordDistribution(
"langfuse.ingestion.otel.media.detection_check_byte_length",
checkedBytes,
- { path, write_path: context.writePath },
+ { path, write_path: context.writePath, media_path: context.mediaPath },
);
}
@@ -363,6 +372,7 @@ function recordInvalidCandidate(
media_kind: kind,
reason,
write_path: context.writePath,
+ media_path: context.mediaPath,
});
}
@@ -377,5 +387,6 @@ function recordIgnoredCandidate(
media_kind: kind,
reason,
write_path: context.writePath,
+ media_path: context.mediaPath,
});
}
diff --git a/packages/shared/src/utils/objects.ts b/packages/shared/src/utils/objects.ts
index aebfb6baed..c1026f874e 100644
--- a/packages/shared/src/utils/objects.ts
+++ b/packages/shared/src/utils/objects.ts
@@ -14,8 +14,22 @@ export function removeObjectKeys<T, K extends keyof T>(
return result;
}
+/** Define an own data property without invoking setters, including `__proto__`. */
+export function defineOwnValue(
+ target: object,
+ property: PropertyKey,
+ value: unknown,
+): void {
+ Object.defineProperty(target, property, {
+ configurable: true,
+ enumerable: true,
+ value,
+ writable: true,
+ });
+}
+
/**
- * Safely coerces a value to a Record if it's a plain object.
+ * Safely coerces a value to a Record if it's a non-array object.
* Returns undefined for null, undefined, arrays, and non-objects.
*/
export function asRecord(value: unknown): Record<string, unknown> | undefined {
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index dfcf856de3..625810de06 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -73,10 +73,11 @@
records its own metrics and logs; see `../packages/native/AGENTS.md`. Topics runs
its synchronous numerical fit in a killable Node child via `src/features/topics/numeric.ts`.
- Tests: `src/__tests__/*`, `src/queues/__tests__/*`
-- Direct-event replay: `pnpm --filter worker run test:otel-replay` feeds raw S3
- JSON through the production OTEL queue and JSON writer into isolated
- ClickHouse tables, including retained legacy writes in dual-write mode. Setup
- and scope: `src/features/otel-ingestion/README.md`.
+- Direct-event replay: `pnpm --filter worker run test:otel-replay` compares the
+ original and early-media TypeScript paths from identical raw S3 JSON through
+ the production OTEL queue and JSON writer into isolated ClickHouse tables,
+ including retained legacy writes in dual-write mode. Setup and scope:
+ `src/features/otel-ingestion/README.md`.
- Native codec checks: `pnpm --filter worker run test:native-codec` selects
`nativeCodec` suites. Build the addon first; the command requires ClickHouse
whenever the live parity suite is present.
diff --git a/worker/src/__tests__/helpers/otelReplayHarness.ts b/worker/src/__tests__/helpers/otelReplayHarness.ts
index 873f334716..bb62a9b8b6 100644
--- a/worker/src/__tests__/helpers/otelReplayHarness.ts
+++ b/worker/src/__tests__/helpers/otelReplayHarness.ts
@@ -22,8 +22,15 @@ import { configureOtelReplayEnvironment } from "./otelReplaySetup";
type OtelReplayStoredRow = Record<string, unknown>;
export type OtelReplayResult = {
+ mode: OtelRawReplayMode;
storedRows: OtelReplayStoredRow[];
legacyRows?: Partial<Record<LegacyReplayTable, OtelReplayStoredRow[]>>;
+ sideEffects?: unknown;
+};
+
+export type OtelRawReplayComparison = {
+ originalTs: OtelReplayResult;
+ earlyTs: OtelReplayResult;
};
type LegacyReplayTable =
@@ -49,9 +56,14 @@ type RunOtelReplayParams = {
overflowEnabled?: boolean;
overflowSizeLimitBytes?: number;
writeMode?: "events_only" | "dual";
+ sdkName?: string;
+ sdkVersion?: string;
failLegacyQueueProcessing?: boolean;
+ captureSideEffects?: () => unknown;
};
+type OtelRawReplayMode = "original-ts" | "early-ts";
+
// The legacy processor caches its storage client. Keep this fake's identity stable
// across sequential replays, but release all file contents after each run.
const eventStorage = {
@@ -90,12 +102,63 @@ const eventStorage = {
},
};
+const differentialClockFields = new Set([
+ "created_at",
+ "updated_at",
+ "event_ts",
+]);
+
+export function comparableOtelReplayRows(
+ rows: OtelReplayStoredRow[] = [],
+): OtelReplayStoredRow[] {
+ return rows
+ .map((row) =>
+ Object.fromEntries(
+ Object.entries(row).filter(
+ ([key]) => !differentialClockFields.has(key),
+ ),
+ ),
+ )
+ .sort((left, right) =>
+ JSON.stringify(left).localeCompare(JSON.stringify(right)),
+ );
+}
+
+export function expectRawOtelReplayParity(
+ comparison: OtelRawReplayComparison,
+): void {
+ expect(comparableOtelReplayRows(comparison.earlyTs.storedRows)).toEqual(
+ comparableOtelReplayRows(comparison.originalTs.storedRows),
+ );
+}
+
+/** Replay identical unparsed bytes through the original and early-media TS paths. */
+export async function runOtelReplayComparison(
+ params: RunOtelReplayParams,
+): Promise<OtelRawReplayComparison> {
+ const { captureSideEffects, ...replayParams } = params;
+ const originalTs = await runOneOtelReplay({
+ ...replayParams,
+ captureSideEffects,
+ mode: "original-ts",
+ bytes: Buffer.from(params.bytes),
+ });
+ const earlyTs = await runOneOtelReplay({
+ ...replayParams,
+ captureSideEffects,
+ mode: "early-ts",
+ bytes: Buffer.from(params.bytes),
+ });
+
+ return { originalTs, earlyTs };
+}
+
/**
* Replay an S3 OTEL document through the production queue and JSON writer, then
* read its events_full rows from an isolated ClickHouse Memory table.
*/
-export async function runOtelReplay(
- params: RunOtelReplayParams,
+export async function runOneOtelReplay(
+ params: RunOtelReplayParams & { mode: OtelRawReplayMode },
): Promise<OtelReplayResult> {
const projectId = params.projectId ?? "otel-replay-test";
const fileKey = params.fileKey ?? "otel-replay/test.json";
@@ -107,6 +170,7 @@ export async function runOtelReplay(
mediaUploadEnabled: params.mediaUploadEnabled,
overflowEnabled: params.overflowEnabled,
overflowSizeLimitBytes: params.overflowSizeLimitBytes,
+ earlyMediaExtractionEnabled: params.mode === "early-ts",
writeMode,
});
@@ -194,8 +258,8 @@ export async function runOtelReplay(
},
},
ingestionVersion: "4",
- sdkName: "otel-replay",
- sdkVersion: "test",
+ sdkName: params.sdkName ?? "otel-replay",
+ sdkVersion: params.sdkVersion ?? "test",
},
},
} as Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>;
@@ -227,8 +291,15 @@ export async function runOtelReplay(
}
await processor(job, undefined);
- expect(eventStorage.download).toHaveBeenCalledExactlyOnceWith(fileKey);
- expect(eventStorage.downloadBytes).not.toHaveBeenCalled();
+ if (params.mode === "original-ts") {
+ expect(eventStorage.download).toHaveBeenCalledExactlyOnceWith(fileKey);
+ expect(eventStorage.downloadBytes).not.toHaveBeenCalled();
+ } else {
+ expect(eventStorage.downloadBytes).toHaveBeenCalledExactlyOnceWith(
+ fileKey,
+ );
+ expect(eventStorage.download).not.toHaveBeenCalled();
+ }
if (writeMode === "dual") {
const legacyProcessor = ingestionQueueProcessorBuilder(false);
@@ -276,8 +347,10 @@ export async function runOtelReplay(
: undefined;
replayResult = {
+ mode: params.mode,
storedRows: (await result.json()) as OtelReplayStoredRow[],
legacyRows,
+ sideEffects: params.captureSideEffects?.(),
};
} catch (error) {
primaryError = new Error(
diff --git a/worker/src/__tests__/helpers/otelReplaySetup.ts b/worker/src/__tests__/helpers/otelReplaySetup.ts
index 1ed70249fb..4433be8fbb 100644
--- a/worker/src/__tests__/helpers/otelReplaySetup.ts
+++ b/worker/src/__tests__/helpers/otelReplaySetup.ts
@@ -24,6 +24,8 @@ vi.mock("../../env", async (importOriginal) => {
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
LANGFUSE_TRACE_BATCH_INGESTION_ENABLED: "false",
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: "false",
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED: "false",
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS: [],
LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: "otel-replay-test",
LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: "otel-replay/",
LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED: "false",
@@ -82,6 +84,7 @@ export function configureDefaultOtelReplayMocks(): void {
type ReplayEnvironmentOptions = {
mediaUploadEnabled?: boolean;
+ earlyMediaExtractionEnabled?: boolean;
overflowEnabled?: boolean;
overflowSizeLimitBytes?: number;
writeMode?: "events_only" | "dual";
@@ -101,6 +104,10 @@ export function configureOtelReplayEnvironment(
LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS:
env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS,
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED,
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED:
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED,
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS:
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS,
LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,
LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: env.LANGFUSE_S3_MEDIA_UPLOAD_PREFIX,
LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG:
@@ -121,6 +128,10 @@ export function configureOtelReplayEnvironment(
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: options.mediaUploadEnabled
? "true"
: "false",
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED:
+ options.earlyMediaExtractionEnabled ? "true" : "false",
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS:
+ options.earlyMediaExtractionEnabled ? ["*"] : [],
LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: "otel-replay-test",
LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: "otel-replay/",
LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG: "false",
diff --git a/worker/src/__tests__/ingestionMasking.test.ts b/worker/src/__tests__/ingestionMasking.test.ts
index 25631fbc0b..0b2ee706da 100644
--- a/worker/src/__tests__/ingestionMasking.test.ts
+++ b/worker/src/__tests__/ingestionMasking.test.ts
@@ -6,15 +6,29 @@ import {
beforeAll,
afterAll,
afterEach,
+ vi,
} from "vitest";
import { setupServer } from "msw/node";
import { http, HttpResponse, delay } from "msw";
import type { SharedEnv } from "@langfuse/shared/src/env";
+import { prepareOtelBatch } from "../features/otel-ingestion/prepareOtelBatch";
+import * as masking from "@langfuse/shared/src/server/ee/ingestionMasking";
import {
applyIngestionMasking,
isIngestionMaskingEnabled,
} from "@langfuse/shared/src/server/ee/ingestionMasking";
+vi.mock(
+ "@langfuse/shared/src/server/ee/ingestionMasking",
+ async (importOriginal) => {
+ const actual = await importOriginal<typeof masking>();
+ return {
+ ...actual,
+ applyIngestionMasking: vi.fn(actual.applyIngestionMasking),
+ };
+ },
+);
+
// Sample OTEL span data for testing
const sampleSpanData = [
{
@@ -232,6 +246,147 @@ describe("Ingestion Masking", () => {
});
describe("applyIngestionMasking", () => {
+ it.each([
+ "success",
+ "bom-success",
+ "double-bom-success",
+ "triple-bom-fail-open",
+ "fail-open",
+ "fail-closed",
+ ] as const)(
+ "compacts only the accepted raw masking input (%s)",
+ async (outcome) => {
+ const originalMediaBody = Buffer.alloc(4096, 0x6f);
+ const maskedMediaBody = Buffer.alloc(4096, 0x6d);
+ const original = {
+ input: `data:image/png;base64,${originalMediaBody.toString("base64")}`,
+ };
+ const originalBytes = Buffer.from(
+ JSON.stringify({ ...original, note: "🔥" }, null, 2),
+ );
+ const masked = {
+ input: `data:image/png;base64,${maskedMediaBody.toString("base64")}`,
+ };
+ const accepted =
+ outcome === "success" ||
+ outcome === "bom-success" ||
+ outcome === "double-bom-success";
+ const responseBody = accepted
+ ? (outcome === "bom-success"
+ ? "\uFEFF"
+ : outcome === "double-bom-success"
+ ? "\uFEFF\uFEFF"
+ : "") + JSON.stringify(masked)
+ : outcome === "triple-bom-fail-open"
+ ? "\uFEFF\uFEFF\uFEFF" + JSON.stringify(masked)
+ : "{";
+ const overrides = createTestEnv({
+ LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
+ "https://masking.example.com/raw",
+ LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
+ LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,
+ LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED:
+ outcome === "fail-closed" ? "true" : "false",
+ });
+ const configuredMasking = vi.mocked(masking.applyIngestionMasking);
+ const applyMasking = configuredMasking.getMockImplementation()!;
+ configuredMasking.mockImplementation((params, _env, transport) =>
+ applyMasking(params, overrides, transport),
+ );
+ const fetch = vi
+ .spyOn(globalThis, "fetch")
+ .mockResolvedValueOnce(new Response(responseBody));
+ if (accepted) {
+ await expect(new Response(responseBody).json()).resolves.toEqual(
+ masked,
+ );
+ } else if (outcome === "triple-bom-fail-open") {
+ await expect(new Response(responseBody).json()).rejects.toThrow();
+ }
+ let prepared: Awaited<ReturnType<typeof prepareOtelBatch>> | undefined;
+ try {
+ prepared = await prepareOtelBatch({
+ bytes: originalBytes,
+ projectId: "test-project",
+ extractMedia: true,
+ });
+ // Masking sees the original inline content, before reference substitution.
+ const body = fetch.mock.calls[0]?.[1]?.body;
+ expect(body).toBeInstanceOf(Uint8Array);
+ expect(Buffer.from(body as Uint8Array)).toEqual(originalBytes);
+ if (outcome === "fail-closed") {
+ expect(prepared).toEqual({ error: expect.any(String) });
+ return;
+ }
+ const batch = prepared?.batch;
+ expect(batch).toBeDefined();
+ expect(batch!.media).toHaveLength(1);
+ expect(await batch!.mediaBody(0)).toEqual(
+ accepted ? maskedMediaBody : originalMediaBody,
+ );
+ expect(
+ JSON.parse(batch!.takeJsonBuffer().toString("utf8")).input,
+ ).toBe(batch!.media[0].reference);
+ } finally {
+ await prepared?.batch?.dispose();
+ fetch.mockRestore();
+ configuredMasking.mockImplementation(applyMasking);
+ }
+ },
+ );
+
+ it.each(["request", "response"] as const)(
+ "sanitizes invalid UTF-8 in the masking %s before media discovery",
+ async (side) => {
+ const mediaBody = Buffer.alloc(4096, 0x68);
+ const mediaUri = `data:image/png;base64,${mediaBody.toString("base64")}`;
+ const malformed = Buffer.concat([
+ Buffer.from('[{"note":"'),
+ Buffer.from([0xff, 0xe2, 0x82]),
+ Buffer.from(`","input":"${mediaUri}"}]`),
+ ]);
+ const sanitized = Buffer.from(malformed.toString("utf8"));
+ const originalBytes = side === "request" ? malformed : sanitized;
+ const responseBytes = side === "response" ? malformed : sanitized;
+ const overrides = createTestEnv({
+ LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
+ "https://masking.example.com/raw",
+ LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
+ LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,
+ });
+ const configuredMasking = vi.mocked(masking.applyIngestionMasking);
+ const applyMasking = configuredMasking.getMockImplementation()!;
+ configuredMasking.mockImplementation((params, _env, transport) =>
+ applyMasking(params, overrides, transport),
+ );
+ const fetch = vi
+ .spyOn(globalThis, "fetch")
+ .mockResolvedValueOnce(new Response(new Uint8Array(responseBytes)));
+ let prepared: Awaited<ReturnType<typeof prepareOtelBatch>> | undefined;
+ try {
+ prepared = await prepareOtelBatch({
+ bytes: originalBytes,
+ projectId: "test-project",
+ extractMedia: true,
+ });
+ const body = fetch.mock.calls[0]?.[1]?.body;
+ expect(body).toBeInstanceOf(Uint8Array);
+ expect(Buffer.from(body as Uint8Array)).toEqual(sanitized);
+ expect(fetch).toHaveBeenCalledTimes(1);
+ const batch = prepared.batch!;
+ expect(
+ JSON.parse(batch.takeJsonBuffer().toString("utf8"))[0].note,
+ ).toBe("��");
+ expect(batch.media).toHaveLength(1);
+ expect(await batch.mediaBody(0)).toEqual(mediaBody);
+ } finally {
+ await prepared?.batch?.dispose();
+ fetch.mockRestore();
+ configuredMasking.mockImplementation(applyMasking);
+ }
+ },
+ );
+
it.each(["retry-success", "fail-open", "fail-closed"] as const)(
"validates raw masking responses inside retries (%s)",
async (outcome) => {
diff --git a/worker/src/__tests__/nativeOtelMedia.unit.test.ts b/worker/src/__tests__/nativeOtelMedia.unit.test.ts
index 1c51ba99c6..1d69c9a643 100644
--- a/worker/src/__tests__/nativeOtelMedia.unit.test.ts
+++ b/worker/src/__tests__/nativeOtelMedia.unit.test.ts
@@ -233,14 +233,18 @@ describe(
JSON.parse(JSON.parse(attributes[2]!.value.stringValue)),
).toMatchObject({ data: batch.media[2]!.reference });
await expect(
- batch.originalMedia(0, batch.media[0]!.originalJsonDepth),
- ).resolves.toBe(MEDIA_URI);
- await expect(
- batch.originalMedia(1, batch.media[1]!.originalJsonDepth),
- ).resolves.toBe(MEDIA_URI);
- await expect(
- batch.originalMedia(2, batch.media[2]!.originalJsonDepth),
- ).resolves.toBe(providerBody.toString("base64"));
+ batch.originalMediaBatch(
+ [2, 0, 2, 1].map((index) => ({
+ index,
+ jsonLayers: batch.media[index]!.originalJsonDepth,
+ })),
+ ),
+ ).resolves.toEqual([
+ providerBody.toString("base64"),
+ MEDIA_URI,
+ providerBody.toString("base64"),
+ MEDIA_URI,
+ ]);
} finally {
await batch.dispose();
await validated.dispose();
@@ -512,13 +516,20 @@ it(
createHash("sha256").update(MEDIA_BODY).digest("base64"),
);
await expect(batch.mediaBody(1)).rejects.toThrow("unknown media index");
+ await expect(
+ batch.originalMediaBatch([{ index: 1, jsonLayers: 0 }]),
+ ).rejects.toMatchObject({ code: "ERR_OTEL_MEDIA_INDEX" });
const body = batch.mediaBody(media.index);
const original = batch.originalMedia(media.index);
+ const originalBatch = batch.originalMediaBatch([
+ { index: media.index, jsonLayers: media.originalJsonDepth },
+ ]);
await batch.dispose();
expect(compact.toString("utf8")).toContain(media.reference);
await expect(body).resolves.toEqual(MEDIA_BODY);
await expect(original).resolves.toBe(uri);
+ await expect(originalBatch).resolves.toEqual([uri]);
await expect(batch.copyJsonBuffer()).rejects.toMatchObject({
code: "ERR_OTEL_CLOSED",
});
@@ -534,6 +545,9 @@ it(
await expect(batch.originalMedia(0)).rejects.toMatchObject({
code: "ERR_OTEL_CLOSED",
});
+ await expect(
+ batch.originalMediaBatch([{ index: 0, jsonLayers: 0 }]),
+ ).rejects.toMatchObject({ code: "ERR_OTEL_CLOSED" });
const copyValidated = await validateOtelJson(
Buffer.from(JSON.stringify({ input: uri })),
diff --git a/worker/src/__tests__/otelReplay.corpus.test.ts b/worker/src/__tests__/otelReplay.corpus.test.ts
index e4c54ed8ad..d630536747 100644
--- a/worker/src/__tests__/otelReplay.corpus.test.ts
+++ b/worker/src/__tests__/otelReplay.corpus.test.ts
@@ -5,7 +5,10 @@ import { vercelAiSdkMixedToolMessagesFixture } from "../../../packages/shared/sr
import { langgraphProductionShapeFixture } from "../../../packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures";
import { pydanticAiProductionShapeFixture } from "../../../packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures";
import { microsoftAgentProductionShapeFixture } from "../../../packages/shared/src/utils/normalized-io/conventions/providers/otel-genai/fixtures";
-import { runOtelReplay } from "./helpers/otelReplayHarness";
+import {
+ expectRawOtelReplayParity,
+ runOtelReplayComparison,
+} from "./helpers/otelReplayHarness";
type ScopeSpan = NonNullable<ResourceSpan["scopeSpans"]>[number];
type Span = NonNullable<ScopeSpan["spans"]>[number];
@@ -361,67 +364,71 @@ describe("OTEL replay provider corpus", { retry: 0, timeout: 120_000 }, () => {
const sourceSpan = resourceSpans[0].scopeSpans?.[0].spans?.[0];
expect(sourceSpan).toBeDefined();
- const { storedRows } = await runOtelReplay({
+ const comparison = await runOtelReplayComparison({
bytes: Buffer.from(JSON.stringify(resourceSpans)),
projectId: `otel-replay-${name.toLowerCase().replaceAll(" ", "-")}`,
mediaUploadEnabled: true,
});
- expect(storedRows).toHaveLength(1);
- const row = storedRows[0];
+ for (const result of [comparison.originalTs, comparison.earlyTs]) {
+ expect(result.storedRows, result.mode).toHaveLength(1);
+ const row = result.storedRows[0];
- expect(row.trace_id).toBe(idToHex(sourceSpan!.traceId));
- expect(row.span_id).toBe(idToHex(sourceSpan!.spanId));
- if (expectedObservationType) {
- expect(row.type).toBe(expectedObservationType);
- }
- expect(row.provided_model_name).toBe(expectedModelName);
-
- const spanIOInput = parseFixtureValue(fixture.spanIO.input);
- const expectedOutput = parseFixtureValue(fixture.spanIO.output);
- const storedInput = persistedJsonColumn(row, "input");
- expect(persistedJsonColumn(row, "output")).toEqual(expectedOutput);
+ expect(row.trace_id).toBe(idToHex(sourceSpan!.traceId));
+ expect(row.span_id).toBe(idToHex(sourceSpan!.spanId));
+ if (expectedObservationType) {
+ expect(row.type).toBe(expectedObservationType);
+ }
+ expect(row.provided_model_name).toBe(expectedModelName);
- if (toolDefinitionsAttribute) {
- const toolDefinitions = fixtureJsonAttribute(
- fixture,
- toolDefinitionsAttribute,
- ) as unknown[];
- const expectedInput = addToolDefinitionsToInput
- ? {
- ...(spanIOInput as Record<string, unknown>),
- tools: toolDefinitions,
- }
- : spanIOInput;
- expect(storedInput).toEqual(expectedInput);
- expect(persistedJsonColumn(row, "tool_definitions")).toEqual(
- expectedToolDefinitions(toolDefinitions),
- );
- } else {
- expect(storedInput).toEqual(spanIOInput);
- }
+ const spanIOInput = parseFixtureValue(fixture.spanIO.input);
+ const expectedOutput = parseFixtureValue(fixture.spanIO.output);
+ const storedInput = persistedJsonColumn(row, "input");
+ expect(persistedJsonColumn(row, "output")).toEqual(expectedOutput);
- expect(persistedJsonColumn(row, "provided_usage_details")).toEqual(
- expectedUsageDetails,
- );
- expect(persistedJsonColumn(row, "usage_details")).toEqual(
- expectedFinalUsageDetails,
- );
+ if (toolDefinitionsAttribute) {
+ const toolDefinitions = fixtureJsonAttribute(
+ fixture,
+ toolDefinitionsAttribute,
+ ) as unknown[];
+ const expectedInput = addToolDefinitionsToInput
+ ? {
+ ...(spanIOInput as Record<string, unknown>),
+ tools: toolDefinitions,
+ }
+ : spanIOInput;
+ expect(storedInput).toEqual(expectedInput);
+ expect(persistedJsonColumn(row, "tool_definitions")).toEqual(
+ expectedToolDefinitions(toolDefinitions),
+ );
+ } else {
+ expect(storedInput).toEqual(spanIOInput);
+ }
- if (expectedToolCalls) {
- expect(row.tool_call_names).toEqual(
- expectedToolCalls.map((call) => call.name),
+ expect(persistedJsonColumn(row, "provided_usage_details")).toEqual(
+ expectedUsageDetails,
);
- expect(persistedToolCalls(row)).toEqual(
- expectedToolCalls.map(({ name: _name, ...call }) => call),
- );
- }
- if (expectedMetadata) assertMetadataValues(row, expectedMetadata);
- if (expectedModelParameters) {
- expect(persistedJsonColumn(row, "model_parameters")).toMatchObject(
- expectedModelParameters,
+ expect(persistedJsonColumn(row, "usage_details")).toEqual(
+ expectedFinalUsageDetails,
);
+
+ if (expectedToolCalls) {
+ expect(row.tool_call_names).toEqual(
+ expectedToolCalls.map((call) => call.name),
+ );
+ expect(persistedToolCalls(row)).toEqual(
+ expectedToolCalls.map(({ name: _name, ...call }) => call),
+ );
+ }
+ if (expectedMetadata) assertMetadataValues(row, expectedMetadata);
+ if (expectedModelParameters) {
+ expect(persistedJsonColumn(row, "model_parameters")).toMatchObject(
+ expectedModelParameters,
+ );
+ }
}
+
+ expectRawOtelReplayParity(comparison);
},
);
});
diff --git a/worker/src/__tests__/otelReplay.integration.test.ts b/worker/src/__tests__/otelReplay.integration.test.ts
index 1c756ee3f1..e168785e48 100644
--- a/worker/src/__tests__/otelReplay.integration.test.ts
+++ b/worker/src/__tests__/otelReplay.integration.test.ts
@@ -1,24 +1,47 @@
import "./helpers/otelReplaySetup";
+import { createHash, randomUUID } from "node:crypto";
import { Decimal } from "decimal.js";
-import { randomUUID } from "node:crypto";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { env } from "../env";
+import { env as sharedEnv, type SharedEnv } from "@langfuse/shared/src/env";
+import { MediaAssociationOrigin } from "@langfuse/shared";
import {
createOrgProjectAndApiKey,
IngestionQueue,
QueueJobs,
TraceUpsertQueue,
+ linkMediaToTraceOrObservation,
type ResourceSpan,
+ uploadMediaForTrace,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
-import { runOtelReplay } from "./helpers/otelReplayHarness";
+import * as masking from "@langfuse/shared/src/server/ee/ingestionMasking";
+import * as otelPreparation from "../features/otel-ingestion/prepareOtelBatch";
+import {
+ comparableOtelReplayRows,
+ expectRawOtelReplayParity,
+ runOtelReplayComparison,
+ runOneOtelReplay,
+ type OtelRawReplayComparison,
+} from "./helpers/otelReplayHarness";
import {
configureDefaultOtelReplayMocks,
- configureOtelReplayEnvironment,
otelReplayMocks,
} from "./helpers/otelReplaySetup";
+vi.mock(
+ "@langfuse/shared/src/server/ee/ingestionMasking",
+ async (importOriginal) => {
+ const actual = await importOriginal<typeof masking>();
+ return {
+ ...actual,
+ applyIngestionMasking: vi.fn(actual.applyIngestionMasking),
+ isIngestionMaskingEnabled: vi.fn(actual.isIngestionMaskingEnabled),
+ };
+ },
+);
+
const PROJECT_ID = "otel-replay-integration";
const FILE_KEY = "otel-replay/integration.json";
const TRACE_ID = "11111111111111111111111111111111";
@@ -56,6 +79,159 @@ async function getIngestionJobsForProject(projectId: string) {
.filter((job) => job.data.payload.authCheck.scope.projectId === projectId);
}
+type MediaUploadCall = Parameters<typeof uploadMediaForTrace>[0];
+type MediaLinkCall = Parameters<typeof linkMediaToTraceOrObservation>[0];
+
+type MediaAssociation = {
+ mediaId: string;
+ traceId: string;
+ observationId?: string | null;
+ field: string;
+ contentType: string;
+ contentLength: number;
+ contentBase64: string;
+ contentDigest: string;
+ origin: string;
+};
+
+function mediaIdForField(field: string): string {
+ return field === "input" ? "image-media-id" : "overflow-media-id";
+}
+
+function mediaAssociationFromUpload(params: MediaUploadCall): MediaAssociation {
+ const contentBytes = Buffer.from(params.contentBytes);
+ const contentDigest = createHash("sha256")
+ .update(contentBytes)
+ .digest("base64");
+ if (params.sha256Hash !== undefined) {
+ expect(params.sha256Hash).toBe(contentDigest);
+ }
+ return {
+ mediaId: mediaIdForField(params.field),
+ traceId: params.traceId,
+ observationId: params.observationId,
+ field: params.field,
+ contentType: params.contentType,
+ contentLength: contentBytes.length,
+ contentBase64: contentBytes.toString("base64"),
+ contentDigest,
+ origin: params.origin,
+ };
+}
+
+function mediaAssociationKey(association: MediaAssociation): string {
+ return [
+ association.mediaId,
+ association.traceId,
+ association.observationId,
+ association.field,
+ association.origin,
+ association.contentDigest,
+ ]
+ .map((value) => value ?? "")
+ .join("\u0000");
+}
+
+function sortedMediaAssociations(
+ associations: MediaAssociation[],
+): MediaAssociation[] {
+ return associations.sort((left, right) =>
+ mediaAssociationKey(left).localeCompare(mediaAssociationKey(right)),
+ );
+}
+
+function createMediaAssociationCapture(): () => MediaAssociation[] {
+ let uploadCursor = 0;
+ let linkCursor = 0;
+
+ return () => {
+ const uploads = otelReplayMocks.uploadMediaForTrace.mock.calls
+ .slice(uploadCursor)
+ .map(([rawParams]) =>
+ mediaAssociationFromUpload(rawParams as MediaUploadCall),
+ );
+ const links = otelReplayMocks.linkMediaToTraceOrObservation.mock.calls
+ .slice(linkCursor)
+ .map(([rawParams]) => rawParams as MediaLinkCall);
+ uploadCursor = otelReplayMocks.uploadMediaForTrace.mock.calls.length;
+ linkCursor =
+ otelReplayMocks.linkMediaToTraceOrObservation.mock.calls.length;
+
+ const assetsById = new Map(
+ uploads.map((upload) => [upload.mediaId, upload]),
+ );
+ const associations = new Map(
+ uploads.map((upload) => [mediaAssociationKey(upload), upload]),
+ );
+ for (const link of links) {
+ const asset = assetsById.get(link.mediaId);
+ expect(asset, `media link ${link.mediaId} has no upload`).toBeDefined();
+ if (!asset) continue;
+ const association = {
+ ...asset,
+ traceId: link.traceId,
+ observationId: link.observationId,
+ field: link.field,
+ origin: link.origin,
+ };
+ associations.set(mediaAssociationKey(association), association);
+ }
+ return sortedMediaAssociations([...associations.values()]);
+ };
+}
+
+type PreparedBatchSnapshot = {
+ json: string;
+ references: string[];
+ media?: Array<{ reference: string; bodyBase64: string }>;
+};
+
+function capturePreparedBatches(options: { includeBodies?: boolean } = {}) {
+ const compactedBatches: PreparedBatchSnapshot[] = [];
+ const prepareOtelBatch = otelPreparation.prepareOtelBatch;
+ const preparation = vi
+ .spyOn(otelPreparation, "prepareOtelBatch")
+ .mockImplementation(async (params) => {
+ const result = await prepareOtelBatch(params);
+ const batch = result?.batch;
+ if (batch) {
+ // Snapshot native output before queue parsing and the late TS detector.
+ const media = batch.media;
+ const snapshot: PreparedBatchSnapshot = {
+ json: (await batch.copyJsonBuffer()).toString("utf8"),
+ references: media.map(({ reference }) => reference),
+ };
+ if (options.includeBodies) {
+ snapshot.media = await Promise.all(
+ media.map(async ({ index, reference }) => ({
+ reference,
+ bodyBase64: (await batch.mediaBody(index)).toString("base64"),
+ })),
+ );
+ }
+ compactedBatches.push(snapshot);
+ }
+ return result;
+ });
+
+ return { compactedBatches, preparation };
+}
+
+function expectLegacyReplayParity(comparison: OtelRawReplayComparison) {
+ for (const table of [
+ "traces",
+ "observations",
+ "observations_batch_staging",
+ ] as const) {
+ expect(
+ comparableOtelReplayRows(comparison.earlyTs.legacyRows?.[table]),
+ `early-ts ${table}`,
+ ).toEqual(
+ comparableOtelReplayRows(comparison.originalTs.legacyRows?.[table]),
+ );
+ }
+}
+
function createBufferId(hex: string): Buffer {
return Buffer.from(hex, "hex");
}
@@ -158,20 +334,146 @@ function buildResourceSpans(
];
}
+type RawOtelAttribute = {
+ key: string;
+ value: Record<string, unknown>;
+};
+
+type FocusedReplayParams = {
+ attributes?: RawOtelAttribute[];
+ resourceAttributes?: RawOtelAttribute[];
+ scopeName?: string;
+ scopeVersion?: string;
+ spanName?: string;
+};
+
+function buildFocusedResourceSpans(
+ params: FocusedReplayParams,
+): ResourceSpan[] {
+ return [
+ {
+ resource: { attributes: params.resourceAttributes ?? [] },
+ scopeSpans: [
+ {
+ scope: {
+ name: params.scopeName ?? "otel-replay",
+ ...(params.scopeVersion ? { version: params.scopeVersion } : {}),
+ },
+ spans: [
+ {
+ traceId: createBufferId(TRACE_ID),
+ spanId: createBufferId(FIRST_SPAN_ID),
+ name: params.spanName ?? "focused-replay",
+ kind: 1,
+ startTimeUnixNano: "1714488530686000000",
+ endTimeUnixNano: "1714488530687000000",
+ attributes: params.attributes ?? [],
+ status: { code: 0 },
+ },
+ ],
+ },
+ ],
+ },
+ ];
+}
+
+function focusedReplayBytes(params: FocusedReplayParams): Buffer {
+ return Buffer.from(JSON.stringify(buildFocusedResourceSpans(params)));
+}
+
+function structuredMetadataReplayBytes(params: {
+ scopeVersion: string;
+ telemetrySdkLanguage: "python" | "nodejs";
+ dataUri: string;
+}): {
+ bytes: Buffer;
+ jsonString: string;
+ escapedJsonString: string;
+ providerJson: string;
+} {
+ // A string-valued SDK metadata field can itself contain serialized JSON. The
+ // SDK then JSON-encodes that value again for the per-key OTLP attribute.
+ const jsonString = JSON.stringify({ image: params.dataUri });
+ // Native handles escaped MIME separators that the TS text scan leaves inline.
+ // Failed uploads must still restore this exact spelling.
+ const escapedJsonString = JSON.stringify({ image: params.dataUri }).replace(
+ "data:image/",
+ "data:image\\/",
+ );
+ const providerJson = JSON.stringify({
+ type: "file",
+ mediaType: "image/png",
+ data: params.dataUri.slice(params.dataUri.indexOf(",") + 1),
+ });
+ const metadata = {
+ "": JSON.stringify({ collision: { key: "blob wins" } }),
+ ".quoted": JSON.stringify("quoted value"),
+ ".asset": JSON.stringify({ attachment: { image: params.dataUri } }),
+ ".json_string": JSON.stringify(jsonString),
+ ".escaped_json_string": JSON.stringify(escapedJsonString),
+ ".quoted_provider": JSON.stringify(providerJson),
+ ".truncated": '{"value":"cut',
+ ".collision.key": JSON.stringify("dotted loses"),
+ };
+
+ return {
+ bytes: focusedReplayBytes({
+ scopeName: "langfuse-sdk",
+ scopeVersion: params.scopeVersion,
+ resourceAttributes: [
+ stringAttribute("telemetry.sdk.language", params.telemetrySdkLanguage),
+ ],
+ attributes: Object.entries(metadata).map(([path, value]) =>
+ stringAttribute(`langfuse.observation.metadata${path}`, value),
+ ),
+ }),
+ jsonString,
+ escapedJsonString,
+ providerJson,
+ };
+}
+
+function eventMetadataByPath(
+ row: Record<string, unknown>,
+): Map<string, string> {
+ const names = row.metadata_names as string[];
+ const values = row.metadata_values as string[];
+ return new Map(names.map((name, index) => [name, values[index]]));
+}
+
+// Compare every persisted column, changing only the explicitly expected media
+// values in this SDK metadata fixture. The general replay comparator stays exact.
+function expectSdkMetadataExpansion(
+ comparison: OtelRawReplayComparison,
+ expandedValues: Record<string, string>,
+) {
+ const expected = structuredClone(comparison.originalTs);
+ for (const row of expected.storedRows) {
+ const metadata = eventMetadataByPath(row);
+ const values = [...metadata].map(
+ ([key, value]) => expandedValues[key] ?? value,
+ );
+ row.event_bytes =
+ Number(row.event_bytes) +
+ Buffer.byteLength(JSON.stringify(values)) -
+ Buffer.byteLength(JSON.stringify(row.metadata_values));
+ row.metadata_values = values;
+ }
+ for (const row of Object.values(expected.legacyRows ?? {}).flat()) {
+ row.metadata = { ...(row.metadata as object), ...expandedValues };
+ }
+ const adjusted = { originalTs: expected, earlyTs: comparison.earlyTs };
+ expectRawOtelReplayParity(adjusted);
+ expectLegacyReplayParity(adjusted);
+}
+
// Vitest retries can overlap unfinished ClickHouse I/O and shared replay mocks.
describe(
"OTEL replay persists enriched observations after media and overflow handling",
{ retry: 0, timeout: 120_000 },
() => {
- let restoreEnvironment: (() => void) | undefined;
-
beforeEach(() => {
configureDefaultOtelReplayMocks();
- restoreEnvironment = configureOtelReplayEnvironment({
- mediaUploadEnabled: true,
- overflowEnabled: true,
- overflowSizeLimitBytes: 256,
- });
const model = {
id: "otel-replay-model",
@@ -209,15 +511,13 @@ describe(
});
afterEach(() => {
- restoreEnvironment?.();
- restoreEnvironment = undefined;
// The test owns this mutable mock setup; restore its deterministic
// defaults so another worker suite in the same process is unaffected.
configureDefaultOtelReplayMocks();
});
it("writes both enriched observations and preserves accounting through readback", async () => {
- const { storedRows } = await runOtelReplay({
+ const comparison = await runOtelReplayComparison({
bytes: Buffer.from(JSON.stringify(buildResourceSpans())),
projectId: PROJECT_ID,
fileKey: FILE_KEY,
@@ -226,11 +526,13 @@ describe(
overflowSizeLimitBytes: 256,
});
- expect(storedRows).toHaveLength(2);
- expect(otelReplayMocks.scheduleObservationEvals).toHaveBeenCalledTimes(2);
+ expectRawOtelReplayParity(comparison);
+ expect(otelReplayMocks.scheduleObservationEvals).toHaveBeenCalledTimes(4);
+ const result = comparison.originalTs;
+ expect(result.storedRows).toHaveLength(2);
const rowsBySpanId = new Map(
- storedRows.map((row) => [String(row.span_id), row]),
+ result.storedRows.map((row) => [String(row.span_id), row]),
);
const firstRow = rowsBySpanId.get(FIRST_SPAN_ID);
const secondRow = rowsBySpanId.get(SECOND_SPAN_ID);
@@ -303,7 +605,7 @@ describe(
"@@@langfuseMedia:type=text/plain|id=overflow-media-id|source=field_size_limit@@@",
);
- for (const row of storedRows) {
+ for (const row of result.storedRows) {
expect(Number(row.event_bytes)).toBeGreaterThan(0);
}
});
@@ -314,42 +616,654 @@ describe(
const bytes = Buffer.from(
`[{"resource":{"attributes":[]},"scopeSpans":[{"scope":{"name":"otel-replay"},"spans":[{"traceId":"${TRACE_ID}","spanId":"${FIRST_SPAN_ID}","name":"unsafe-number","startTimeUnixNano":"1714488530686000000","endTimeUnixNano":"1714488530687000000","attributes":[{"key":"gen_ai.request.temperature","value":{"doubleValue":9007199254740993}}],"status":{"code":0}}]}]}]`,
);
- const { storedRows } = await runOtelReplay({
+ const comparison = await runOtelReplayComparison({
bytes,
projectId: `${PROJECT_ID}-unsafe-number`,
fileKey: `${FILE_KEY}.unsafe-number`,
mediaUploadEnabled: true,
});
- expect(storedRows).toHaveLength(1);
- const metadataIndex = (storedRows[0].metadata_names as string[]).indexOf(
- "attributes.gen_ai.request.temperature",
- );
+ expectRawOtelReplayParity(comparison);
+ const result = comparison.originalTs;
+ expect(result.storedRows).toHaveLength(1);
+ const metadataIndex = (
+ result.storedRows[0].metadata_names as string[]
+ ).indexOf("attributes.gen_ai.request.temperature");
expect(metadataIndex).toBeGreaterThanOrEqual(0);
- expect(storedRows[0].metadata_values?.[metadataIndex]).toBe(
+ expect(result.storedRows[0].metadata_values?.[metadataIndex]).toBe(
"9007199254740992",
);
});
- it("persists legacy and events_full rows from raw OTEL bytes in dual mode", async () => {
+ it.each([
+ ...[
+ {
+ sdkLabel: "Python 5.0",
+ sdkName: "python",
+ sdkVersion: "5.0.0",
+ telemetrySdkLanguage: "python",
+ scopeVersion: "5.0.0",
+ },
+ {
+ sdkLabel: "JavaScript 6.0 fallback",
+ sdkName: "unknown",
+ sdkVersion: "unknown",
+ telemetrySdkLanguage: "nodejs",
+ scopeVersion: "6.0.0",
+ },
+ ].flatMap((sdk) =>
+ (["events_only", "dual"] as const).map((writeMode) => ({
+ ...sdk,
+ writeMode,
+ uploadOutcome: "success" as const,
+ })),
+ ),
+ {
+ sdkLabel: "Python 5.0 failure",
+ sdkName: "python",
+ sdkVersion: "5.0.0",
+ telemetrySdkLanguage: "python",
+ scopeVersion: "5.0.0",
+ writeMode: "events_only",
+ uploadOutcome: "failure",
+ },
+ {
+ sdkLabel: "JavaScript 6.0 fallback failure",
+ sdkName: "unknown",
+ sdkVersion: "unknown",
+ telemetrySdkLanguage: "nodejs",
+ scopeVersion: "6.0.0",
+ writeMode: "dual",
+ uploadOutcome: "failure",
+ },
+ ] as const)(
+ "processes structured metadata for $sdkLabel",
+ async ({
+ sdkName,
+ sdkVersion,
+ telemetrySdkLanguage,
+ scopeVersion,
+ writeMode,
+ uploadOutcome,
+ }) => {
+ const imageBytes = Buffer.alloc(1024, 42);
+ const dataUri = `data:image/png;base64,${imageBytes.toString("base64")}`;
+ const structuredMetadata = structuredMetadataReplayBytes({
+ scopeVersion,
+ telemetrySdkLanguage,
+ dataUri,
+ });
+ const createdProject =
+ writeMode === "dual" ? await createOrgProjectAndApiKey() : undefined;
+ const projectId =
+ createdProject?.projectId ??
+ `${PROJECT_ID}-${sdkName}-${uploadOutcome}`;
+ const fileKey = `${FILE_KEY}.structured-${sdkName}-${writeMode}-${uploadOutcome}`;
+ const { compactedBatches, preparation } = capturePreparedBatches();
+ if (uploadOutcome === "failure") {
+ otelReplayMocks.uploadMediaForTrace.mockRejectedValue(
+ new Error("simulated structured media upload failure"),
+ );
+ }
+
+ try {
+ const replayParams = {
+ bytes: structuredMetadata.bytes,
+ projectId,
+ orgId: createdProject?.orgId,
+ fileKey,
+ mediaUploadEnabled: true,
+ writeMode,
+ sdkName,
+ sdkVersion,
+ captureSideEffects: createMediaAssociationCapture(),
+ } as const;
+ const comparison = await runOtelReplayComparison(replayParams);
+ const expandedImage =
+ "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=base64_data_uri@@@";
+ const expandedProvider = structuredMetadata.providerJson.replace(
+ imageBytes.toString("base64"),
+ "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=bytes@@@",
+ );
+ expectSdkMetadataExpansion(
+ comparison,
+ uploadOutcome === "success"
+ ? {
+ escaped_json_string: JSON.stringify({ image: expandedImage }),
+ quoted_provider: expandedProvider,
+ }
+ : {},
+ );
+ expect(comparison.earlyTs.sideEffects).toEqual(
+ comparison.originalTs.sideEffects,
+ );
+
+ expect(compactedBatches).toHaveLength(1);
+ expect(compactedBatches[0].references).toHaveLength(4);
+ for (const reference of compactedBatches[0].references) {
+ expect(compactedBatches[0].json).toContain(reference);
+ }
+ expect(compactedBatches[0].json).not.toContain("data:image");
+ const uploadedBodies =
+ otelReplayMocks.uploadMediaForTrace.mock.calls.map(([params]) =>
+ Buffer.from(params.contentBytes),
+ );
+ expect(uploadedBodies.length).toBeGreaterThan(0);
+ expect(uploadedBodies.every((body) => body.equals(imageBytes))).toBe(
+ true,
+ );
+
+ const expectedImage =
+ uploadOutcome === "success"
+ ? "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=base64_data_uri@@@"
+ : dataUri;
+ const expectedJsonString =
+ uploadOutcome === "success"
+ ? JSON.stringify({ image: expectedImage })
+ : structuredMetadata.jsonString;
+ expect(comparison.originalTs.storedRows).toHaveLength(1);
+ const originalMetadata = Object.fromEntries(
+ eventMetadataByPath(comparison.originalTs.storedRows[0]),
+ );
+ expect(originalMetadata).toMatchObject({
+ quoted: "quoted value",
+ "asset.attachment.image": expectedImage,
+ json_string: expectedJsonString,
+ escaped_json_string: structuredMetadata.escapedJsonString,
+ quoted_provider: structuredMetadata.providerJson,
+ truncated: '{"value":"cut',
+ "collision.key": "blob wins",
+ });
+ if (writeMode === "dual") {
+ expect(comparison.originalTs.legacyRows?.observations).toHaveLength(
+ 1,
+ );
+ expect(
+ comparison.originalTs.legacyRows!.observations![0].metadata,
+ ).toMatchObject({
+ quoted: "quoted value",
+ asset: JSON.stringify({
+ attachment: { image: expectedImage },
+ }),
+ json_string: expectedJsonString,
+ escaped_json_string: structuredMetadata.escapedJsonString,
+ quoted_provider: structuredMetadata.providerJson,
+ truncated: '{"value":"cut',
+ collision: JSON.stringify({ key: "blob wins" }),
+ "collision.key": "dotted loses",
+ });
+ }
+ if (uploadOutcome === "success") {
+ const earlyMetadata = Object.fromEntries(
+ eventMetadataByPath(comparison.earlyTs.storedRows[0]),
+ );
+ expect(earlyMetadata).toMatchObject({
+ escaped_json_string: JSON.stringify({ image: expandedImage }),
+ quoted_provider: expandedProvider,
+ });
+ }
+ } finally {
+ preparation.mockRestore();
+ if (createdProject) {
+ await prisma.project.delete({
+ where: { id: createdProject.projectId },
+ });
+ await prisma.organization.delete({
+ where: { id: createdProject.orgId },
+ });
+ }
+ }
+ },
+ );
+
+ it.each([
+ {
+ sdkLabel: "Python 4.15",
+ sdkName: "python",
+ sdkVersion: "4.15.6",
+ telemetrySdkLanguage: "python",
+ scopeVersion: "4.15.6",
+ },
+ {
+ sdkLabel: "JavaScript 5.11",
+ sdkName: "javascript",
+ sdkVersion: "5.11.1",
+ telemetrySdkLanguage: "nodejs",
+ scopeVersion: "5.11.1",
+ },
+ ] as const)(
+ "keeps per-key values encoded for older $sdkLabel scope versions",
+ async ({ sdkName, sdkVersion, telemetrySdkLanguage, scopeVersion }) => {
+ const dataUri = `data:image/png;base64,${Buffer.alloc(1024, 42).toString("base64")}`;
+ const structuredMetadata = structuredMetadataReplayBytes({
+ scopeVersion,
+ telemetrySdkLanguage,
+ dataUri,
+ });
+ const comparison = await runOtelReplayComparison({
+ bytes: structuredMetadata.bytes,
+ projectId: `${PROJECT_ID}-old-${sdkName}`,
+ fileKey: `${FILE_KEY}.old-${sdkName}`,
+ mediaUploadEnabled: true,
+ sdkName,
+ sdkVersion,
+ });
+
+ const expandedImage =
+ "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=base64_data_uri@@@";
+ const expandedJson = JSON.stringify({ image: expandedImage });
+ const expandedProvider = structuredMetadata.providerJson.replace(
+ dataUri.slice(dataUri.indexOf(",") + 1),
+ "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=bytes@@@",
+ );
+ expectSdkMetadataExpansion(comparison, {
+ json_string: JSON.stringify(expandedJson),
+ escaped_json_string: JSON.stringify(expandedJson),
+ quoted_provider: JSON.stringify(expandedProvider),
+ });
+ expect(comparison.originalTs.storedRows).toHaveLength(1);
+ const originalMetadata = Object.fromEntries(
+ eventMetadataByPath(comparison.originalTs.storedRows[0]),
+ );
+ expect(originalMetadata).toMatchObject({
+ quoted: JSON.stringify("quoted value"),
+ asset: JSON.stringify({
+ attachment: {
+ image:
+ "@@@langfuseMedia:type=image/png|id=overflow-media-id|source=base64_data_uri@@@",
+ },
+ }),
+ json_string: JSON.stringify(structuredMetadata.jsonString),
+ escaped_json_string: JSON.stringify(
+ structuredMetadata.escapedJsonString,
+ ),
+ quoted_provider: JSON.stringify(structuredMetadata.providerJson),
+ truncated: '{"value":"cut',
+ "collision.key": "blob wins",
+ });
+ const earlyMetadata = Object.fromEntries(
+ eventMetadataByPath(comparison.earlyTs.storedRows[0]),
+ );
+ expect(earlyMetadata).toMatchObject({
+ json_string: JSON.stringify(expandedJson),
+ escaped_json_string: JSON.stringify(expandedJson),
+ quoted_provider: JSON.stringify(expandedProvider),
+ });
+ },
+ );
+
+ it("extracts only media accepted by masking", async () => {
+ const originalMedia = Buffer.from("original-media".repeat(512));
+ const maskedMedia = Buffer.from("masked-media".repeat(512));
+ const rawBytes = focusedReplayBytes({
+ attributes: [
+ stringAttribute(
+ "langfuse.observation.input",
+ `data:image/png;base64,${originalMedia.toString("base64")}`,
+ ),
+ ],
+ });
+ const maskedBytes = focusedReplayBytes({
+ attributes: [
+ stringAttribute(
+ "langfuse.observation.input",
+ `data:image/png;base64,${maskedMedia.toString("base64")}`,
+ ),
+ ],
+ });
+ const maskingEnv: SharedEnv = {
+ ...sharedEnv,
+ LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
+ "https://masking.example.com/mask",
+ LANGFUSE_INGESTION_MASKING_CALLBACK_TIMEOUT_MS: 1_000,
+ LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED: "true",
+ LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,
+ LANGFUSE_INGESTION_MASKING_PROPAGATED_HEADERS: [],
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ LANGFUSE_EE_LICENSE_KEY: "langfuse_ee_test-license-key",
+ };
+ const configuredMasking = vi.mocked(masking.applyIngestionMasking);
+ const applyMasking = configuredMasking.getMockImplementation()!;
+ const configuredMaskingEnabled = vi.mocked(
+ masking.isIngestionMaskingEnabled,
+ );
+ const isMaskingEnabled =
+ configuredMaskingEnabled.getMockImplementation()!;
+ configuredMasking.mockImplementation((params, _env, transport) =>
+ applyMasking(params, maskingEnv, transport),
+ );
+ configuredMaskingEnabled.mockImplementation(() =>
+ isMaskingEnabled(maskingEnv),
+ );
+
+ const maskingRequests: Buffer[] = [];
+ const fetch = vi
+ .spyOn(globalThis, "fetch")
+ .mockImplementation(async (_input, init) => {
+ const body = init?.body;
+ if (typeof body === "string") {
+ maskingRequests.push(Buffer.from(body));
+ } else if (body instanceof Uint8Array) {
+ maskingRequests.push(Buffer.from(body));
+ } else {
+ throw new Error("masking callback received an unexpected body");
+ }
+ return new Response(maskedBytes.toString("utf8"), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ });
+ });
+
+ const { compactedBatches, preparation } = capturePreparedBatches({
+ includeBodies: true,
+ });
+
+ try {
+ const comparison = await runOtelReplayComparison({
+ bytes: rawBytes,
+ projectId: `${PROJECT_ID}-masking-media`,
+ fileKey: `${FILE_KEY}.masking-media`,
+ mediaUploadEnabled: true,
+ captureSideEffects: createMediaAssociationCapture(),
+ });
+
+ expect(fetch).toHaveBeenCalledTimes(2);
+ expect(maskingRequests).toHaveLength(2);
+ for (const request of maskingRequests) {
+ expect(request).toEqual(rawBytes);
+ }
+ expect(compactedBatches).toHaveLength(1);
+ const earlyBatch = compactedBatches[0]!;
+ const earlyMedia = earlyBatch.media!;
+ expect(earlyMedia).toEqual([
+ {
+ reference: expect.any(String),
+ bodyBase64: maskedMedia.toString("base64"),
+ },
+ ]);
+ expect(earlyBatch.json).toContain(earlyMedia[0]!.reference);
+ expect(earlyBatch.json).not.toContain(
+ `data:image/png;base64,${maskedMedia.toString("base64")}`,
+ );
+ const associations = comparison.originalTs
+ .sideEffects as MediaAssociation[];
+ expect(associations).toHaveLength(1);
+ expect(associations[0]?.contentBase64).toBe(
+ maskedMedia.toString("base64"),
+ );
+ expect(comparison.earlyTs.sideEffects).toEqual(associations);
+ expect(comparison.originalTs.storedRows).toHaveLength(1);
+ expect(String(comparison.originalTs.storedRows[0]?.input)).toContain(
+ "id=image-media-id",
+ );
+ expect(
+ String(comparison.originalTs.storedRows[0]?.input),
+ ).not.toContain(originalMedia.toString("base64"));
+ expectRawOtelReplayParity(comparison);
+ } finally {
+ preparation.mockRestore();
+ fetch.mockRestore();
+ configuredMasking.mockImplementation(applyMasking);
+ configuredMaskingEnabled.mockImplementation(isMaskingEnabled);
+ }
+ });
+
+ it("keeps tag media inline before scalar tag values are split", async () => {
+ const body = Buffer.alloc(1_024, "tag-media").toString("base64");
+ const dataUri = `data:image/png;base64,${body}`;
+ const { compactedBatches, preparation } = capturePreparedBatches();
+ try {
+ const comparison = await runOtelReplayComparison({
+ bytes: focusedReplayBytes({
+ attributes: [stringAttribute("langfuse.trace.tags", dataUri)],
+ }),
+ projectId: `${PROJECT_ID}-tag-media`,
+ fileKey: `${FILE_KEY}.tag-media`,
+ mediaUploadEnabled: true,
+ });
+
+ expect(compactedBatches).toHaveLength(1);
+ expect(compactedBatches[0].references).toEqual([]);
+ expectRawOtelReplayParity(comparison);
+ expect(comparison.originalTs.storedRows).toHaveLength(1);
+ expect(comparison.originalTs.storedRows[0]?.tags).toEqual([
+ "data:image/png;base64",
+ body,
+ ]);
+ } finally {
+ preparation.mockRestore();
+ }
+ });
+
+ it.each([
+ ["scope name", { scopeName: "data:text/plain;base64,aGk=" }],
+ [
+ "attribute key",
+ {
+ attributes: [
+ stringAttribute("data:text/plain;base64,aGk=", "attribute-value"),
+ ],
+ },
+ ],
+ ] as const)(
+ "keeps structural media in the %s aligned",
+ async (name, options) => {
+ const comparison = await runOtelReplayComparison({
+ bytes: focusedReplayBytes(options),
+ projectId: `${PROJECT_ID}-structural-media-${name}`,
+ fileKey: `${FILE_KEY}.structural-media-${name}`,
+ mediaUploadEnabled: true,
+ });
+
+ expectRawOtelReplayParity(comparison);
+ expect(comparison.originalTs.storedRows).toHaveLength(1);
+ },
+ );
+
+ const deepValue = `${"[".repeat(200)}null${"]".repeat(200)}`;
+ const shallowDocument = focusedReplayBytes({
+ spanName: "deep-document",
+ })
+ .toString("utf8")
+ .replace("[{", `[{"ignored":${deepValue},`);
+ const overflowingNumber = focusedReplayBytes({
+ attributes: [
+ { key: "gen_ai.request.temperature", value: { doubleValue: 0.25 } },
+ ],
+ })
+ .toString("utf8")
+ .replace('"doubleValue":0.25', '"doubleValue":1e309');
+
+ let nestedInput: unknown = "leaf";
+ for (let depth = 0; depth < 120; depth++) nestedInput = { x: nestedInput };
+ const repeatedNestedInput = focusedReplayBytes({
+ attributes: [
+ stringAttribute(
+ "langfuse.observation.input",
+ JSON.stringify({
+ marker: "data: ",
+ items: Array.from({ length: 32 }, () => nestedInput),
+ }),
+ ),
+ ],
+ });
+
+ it.each([
+ ["repeated nested objects", repeatedNestedInput],
+ ["deep JSON nesting", Buffer.from(shallowDocument)],
+ [
+ "embedded lone surrogate",
+ focusedReplayBytes({
+ attributes: [
+ stringAttribute(
+ "langfuse.observation.input",
+ JSON.stringify({ text: "\ud800" }),
+ ),
+ ],
+ }),
+ ],
+ ["out-of-range number", Buffer.from(overflowingNumber)],
+ ] as const)(
+ "preserves persisted values for %s input",
+ async (name, bytes) => {
+ const comparison = await runOtelReplayComparison({
+ bytes,
+ projectId: `${PROJECT_ID}-edge-${name}`,
+ fileKey: `${FILE_KEY}.edge-${name}`,
+ mediaUploadEnabled: true,
+ });
+
+ expectRawOtelReplayParity(comparison);
+ expect(comparison.originalTs.storedRows).toHaveLength(1);
+ },
+ );
+
+ it("persists sanitized UTF-8 consistently across both paths", async () => {
+ const bytes = focusedReplayBytes({
+ attributes: [
+ stringAttribute("langfuse.observation.input", "utf8-probe"),
+ ],
+ });
+ bytes[bytes.indexOf("utf8-probe")] = 0xff;
+
+ const comparison = await runOtelReplayComparison({
+ bytes,
+ projectId: `${PROJECT_ID}-invalid-utf8`,
+ fileKey: `${FILE_KEY}.invalid-utf8`,
+ mediaUploadEnabled: true,
+ });
+ expectRawOtelReplayParity(comparison);
+ expect(comparison.originalTs.storedRows).toHaveLength(1);
+ expect(comparison.originalTs.storedRows[0].input).toContain("�tf8-probe");
+ });
+
+ it("uploads a native provider candidate beyond the TS discovery limits", async () => {
+ const contentBytes = Buffer.from("replay-deep-media".repeat(512));
+ const encoded = contentBytes.toString("base64");
+ // TS stops provider discovery after ten object levels. Keep the unsafe
+ // integer as source text to also catch parse-and-reserialize corruption.
+ const provider = `{"type":"file","mediaType":"image/png","data":"${encoded}","large":9007199254740993}`;
+ const input = `${'{"nested":'.repeat(12)}${provider}${"}".repeat(12)}`;
+ const { compactedBatches, preparation } = capturePreparedBatches({
+ includeBodies: true,
+ });
+
+ try {
+ const comparison = await runOtelReplayComparison({
+ bytes: focusedReplayBytes({
+ attributes: [stringAttribute("langfuse.observation.input", input)],
+ }),
+ projectId: `${PROJECT_ID}-deep-expansion`,
+ fileKey: `${FILE_KEY}.deep-expansion`,
+ mediaUploadEnabled: true,
+ captureSideEffects: createMediaAssociationCapture(),
+ });
+ // Assert extraction before normalization or the late TS detector runs.
+ expect(compactedBatches).toHaveLength(1);
+ const compacted = compactedBatches[0]!;
+ const compactedMedia = compacted.media!;
+ expect(compactedMedia).toHaveLength(1);
+ expect(compactedMedia[0]?.bodyBase64).toBe(encoded);
+ expect(compacted.json).not.toContain(encoded);
+ const pendingReference = compactedMedia[0]!.reference;
+ expect(compacted.json).toContain(pendingReference);
+ const publicReference = pendingReference.replace(
+ /\|id=[^|@]+/,
+ "|id=image-media-id",
+ );
+ expect(comparison.originalTs.storedRows).toHaveLength(1);
+ expect(comparison.originalTs.storedRows[0]?.input).toBe(input);
+ expect(comparison.originalTs.sideEffects).toEqual([]);
+ expect(comparison.earlyTs.storedRows).toHaveLength(1);
+ expect(comparison.earlyTs.storedRows[0]?.input).toBe(
+ input.replace(encoded, publicReference),
+ );
+ expect(comparison.earlyTs.sideEffects).toEqual([
+ expect.objectContaining({
+ mediaId: "image-media-id",
+ traceId: TRACE_ID,
+ observationId: FIRST_SPAN_ID,
+ field: "input",
+ contentType: "image/png",
+ contentLength: contentBytes.length,
+ contentBase64: encoded,
+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,
+ }),
+ ]);
+ } finally {
+ preparation.mockRestore();
+ }
+ });
+
+ it("persists matching legacy rows, media associations, and events_full rows in dual mode", async () => {
const { projectId, orgId } = await createOrgProjectAndApiKey();
const originalBlobStorageFileLogFlag =
env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG;
env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG = "true";
try {
- const { storedRows, legacyRows } = await runOtelReplay({
+ const captureMediaAssociations = createMediaAssociationCapture();
+ const comparison = await runOtelReplayComparison({
bytes: Buffer.from(JSON.stringify(buildResourceSpans())),
projectId,
orgId,
fileKey: `${FILE_KEY}.dual-write`,
+ mediaUploadEnabled: true,
+ overflowEnabled: true,
+ overflowSizeLimitBytes: 256,
writeMode: "dual",
+ captureSideEffects: captureMediaAssociations,
});
- expect(storedRows).toHaveLength(2);
- expect(legacyRows?.traces?.length).toBeGreaterThan(0);
- expect(legacyRows?.observations).toHaveLength(2);
- expect(legacyRows?.observations_batch_staging).toHaveLength(0);
expect(await getTraceUpsertJobsForProject(projectId)).toHaveLength(0);
+ expectRawOtelReplayParity(comparison);
+ expect(comparison.originalTs.sideEffects).toEqual(
+ comparison.earlyTs.sideEffects,
+ );
+ const associations = comparison.originalTs
+ .sideEffects as MediaAssociation[];
+ expect(associations).toHaveLength(3);
+ expect(associations).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ mediaId: "image-media-id",
+ traceId: TRACE_ID,
+ observationId: FIRST_SPAN_ID,
+ field: "input",
+ contentType: "image/png",
+ contentBase64: Buffer.from("small-image").toString("base64"),
+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,
+ }),
+ expect.objectContaining({
+ mediaId: "image-media-id",
+ traceId: TRACE_ID,
+ observationId: undefined,
+ field: "input",
+ contentType: "image/png",
+ contentBase64: Buffer.from("small-image").toString("base64"),
+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,
+ }),
+ expect.objectContaining({
+ mediaId: "overflow-media-id",
+ traceId: TRACE_ID,
+ observationId: SECOND_SPAN_ID,
+ field: "output",
+ contentType: "text/plain",
+ contentBase64: Buffer.from("x".repeat(1024)).toString("base64"),
+ origin: MediaAssociationOrigin.INGESTION_FIELD_OVERFLOW,
+ }),
+ ]),
+ );
+
+ expect(comparison.originalTs.storedRows).toHaveLength(2);
+ expect(
+ comparison.originalTs.legacyRows?.traces?.length,
+ ).toBeGreaterThan(0);
+ expect(comparison.originalTs.legacyRows?.observations).toHaveLength(2);
+ expect(
+ comparison.originalTs.legacyRows?.observations_batch_staging,
+ ).toHaveLength(0);
+
+ expectLegacyReplayParity(comparison);
} finally {
const traceJobs = await getTraceUpsertJobsForProject(projectId);
await Promise.allSettled(traceJobs.map((job) => job.remove()));
@@ -360,64 +1274,182 @@ describe(
}
});
- it("removes replay-owned ingestion jobs when a legacy replay job fails", async () => {
- const { projectId, orgId } = await createOrgProjectAndApiKey();
- const originalBlobStorageFileLogFlag =
- env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG;
- env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG = "false";
-
- const unrelatedQueue = IngestionQueue.getInstance({
- shardingKey: `${projectId}-unrelated`,
- });
- if (!unrelatedQueue)
- throw new Error("Ingestion queue is not initialized");
- const unrelatedJob = await unrelatedQueue.add(QueueJobs.IngestionJob, {
- id: `otel-replay-unrelated-${randomUUID()}`,
- timestamp: new Date(),
- name: QueueJobs.IngestionJob,
- payload: {
- data: {
- type: "span-create",
- eventBodyId: `otel-replay-unrelated-${randomUUID()}`,
- fileKey: "unrelated.json",
- bucketPrefix: "otel-replay/unrelated/",
- ingestionApiKey: "",
- ingestionSdkName: "otel-replay-test",
- ingestionSdkVersion: "test",
- },
- authCheck: {
- validKey: true,
- scope: { projectId, orgId, accessLevel: "project" },
- },
- },
- });
-
- try {
- await expect(
- runOtelReplay({
- bytes: Buffer.from(
- JSON.stringify(buildResourceSpans({ separateTraces: true })),
- ),
+ it.each(["succeed", "fail"] as const)(
+ "preserves mixed existing and inline media in dual writes when uploads %s",
+ async (outcome) => {
+ const { projectId, orgId } = await createOrgProjectAndApiKey();
+ const existingReference =
+ "@@@langfuseMedia:type=image/png|id=preexisting-media-id|source=bytes@@@";
+ const content = Buffer.from("dual-inline-media".repeat(256));
+ const dataUri = `data:image/png;base64,${content.toString("base64")}`;
+ const input = JSON.stringify([existingReference, dataUri]);
+ const uploadFails = outcome === "fail";
+ if (uploadFails) {
+ otelReplayMocks.uploadMediaForTrace.mockRejectedValue(
+ new Error("simulated media upload failure"),
+ );
+ }
+ // Final rows can match even if the late detector does all the work. Capture the
+ // real compact document and registry before the queue parses and disposes them.
+ const { compactedBatches, preparation } = capturePreparedBatches();
+ const captureMediaAssociations = createMediaAssociationCapture();
+ let uploadCursor = 0;
+ try {
+ const comparison = await runOtelReplayComparison({
+ bytes: focusedReplayBytes({
+ attributes: [
+ stringAttribute("langfuse.trace.input", input),
+ stringAttribute("langfuse.observation.input", input),
+ ],
+ }),
projectId,
orgId,
- fileKey: `${FILE_KEY}.legacy-failure`,
+ fileKey: `${FILE_KEY}.dual-media-${outcome}`,
+ mediaUploadEnabled: true,
writeMode: "dual",
- failLegacyQueueProcessing: true,
- }),
- ).rejects.toThrow("simulated legacy ingestion read failure");
+ captureSideEffects: () => {
+ const uploads =
+ otelReplayMocks.uploadMediaForTrace.mock.calls.slice(
+ uploadCursor,
+ );
+ uploadCursor =
+ otelReplayMocks.uploadMediaForTrace.mock.calls.length;
+ // Each mode must attempt the upload, including when it fails and the
+ // original inline content is therefore the correct persisted result.
+ expect(uploads.length).toBeGreaterThan(0);
+ for (const [params] of uploads) {
+ expect(params.contentBytes).toEqual(content);
+ }
+ return uploadFails ? [] : captureMediaAssociations();
+ },
+ });
+ expect(compactedBatches).toHaveLength(1);
+ const compacted = compactedBatches[0]!;
+ expect(compacted.json).toContain(existingReference);
+ expect(compacted.json).not.toContain(dataUri);
+ expect(compacted.references.length).toBeGreaterThan(0);
+ for (const reference of compacted.references) {
+ expect(reference).not.toBe(existingReference);
+ expect(compacted.json).toContain(reference);
+ }
+ expectRawOtelReplayParity(comparison);
+ expectLegacyReplayParity(comparison);
+ const expectedInput = JSON.stringify([
+ existingReference,
+ uploadFails
+ ? dataUri
+ : "@@@langfuseMedia:type=image/png|id=image-media-id|source=base64_data_uri@@@",
+ ]);
+ expect(comparison.originalTs.storedRows).toHaveLength(1);
+ expect(comparison.originalTs.legacyRows?.traces).toHaveLength(1);
+ expect(comparison.originalTs.legacyRows?.observations).toHaveLength(
+ 1,
+ );
+ for (const row of [
+ comparison.originalTs.storedRows[0],
+ comparison.originalTs.legacyRows?.traces?.[0],
+ comparison.originalTs.legacyRows?.observations?.[0],
+ ]) {
+ expect(row?.input).toBe(expectedInput);
+ }
+ if (uploadFails) {
+ expect(
+ otelReplayMocks.linkMediaToTraceOrObservation,
+ ).not.toHaveBeenCalled();
+ } else {
+ expect(comparison.earlyTs.sideEffects).toEqual(
+ comparison.originalTs.sideEffects,
+ );
+ expect(comparison.originalTs.sideEffects).toHaveLength(2);
+ expect(comparison.originalTs.sideEffects).toEqual(
+ expect.arrayContaining(
+ [undefined, FIRST_SPAN_ID].map((observationId) =>
+ expect.objectContaining({
+ traceId: TRACE_ID,
+ observationId,
+ field: "input",
+ contentBase64: content.toString("base64"),
+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,
+ }),
+ ),
+ ),
+ );
+ }
+ } finally {
+ preparation.mockRestore();
+ await prisma.project.delete({ where: { id: projectId } });
+ await prisma.organization.delete({ where: { id: orgId } });
+ }
+ },
+ );
- const remainingJobs = await getIngestionJobsForProject(projectId);
- expect(remainingJobs.map((job) => job.id)).toEqual([unrelatedJob.id]);
- } finally {
- const remainingJobs = await getIngestionJobsForProject(projectId);
- await Promise.allSettled(remainingJobs.map((job) => job.remove()));
- const traceJobs = await getTraceUpsertJobsForProject(projectId);
- await Promise.allSettled(traceJobs.map((job) => job.remove()));
- env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG =
- originalBlobStorageFileLogFlag;
- await prisma.project.delete({ where: { id: projectId } });
- await prisma.organization.delete({ where: { id: orgId } });
- }
- });
+ it.each([
+ ["original-ts", false],
+ ["original-ts", true],
+ ["early-ts", false],
+ ["early-ts", true],
+ ] as const)(
+ "removes replay-owned jobs after failure (%s, separate traces: %s)",
+ async (mode, separateTraces) => {
+ const { projectId, orgId } = await createOrgProjectAndApiKey();
+ const originalBlobStorageFileLogFlag =
+ env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG;
+ env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG = "false";
+
+ const unrelatedQueue = IngestionQueue.getInstance({
+ shardingKey: `${projectId}-unrelated`,
+ });
+ if (!unrelatedQueue)
+ throw new Error("Ingestion queue is not initialized");
+ const unrelatedJob = await unrelatedQueue.add(QueueJobs.IngestionJob, {
+ id: `otel-replay-unrelated-${randomUUID()}`,
+ timestamp: new Date(),
+ name: QueueJobs.IngestionJob,
+ payload: {
+ data: {
+ type: "span-create",
+ eventBodyId: `otel-replay-unrelated-${randomUUID()}`,
+ fileKey: "unrelated.json",
+ bucketPrefix: "otel-replay/unrelated/",
+ ingestionApiKey: "",
+ ingestionSdkName: "otel-replay-test",
+ ingestionSdkVersion: "test",
+ },
+ authCheck: {
+ validKey: true,
+ scope: { projectId, orgId, accessLevel: "project" },
+ },
+ },
+ });
+
+ try {
+ await expect(
+ runOneOtelReplay({
+ mode,
+ bytes: Buffer.from(
+ JSON.stringify(buildResourceSpans({ separateTraces })),
+ ),
+ projectId,
+ orgId,
+ fileKey: `${FILE_KEY}.legacy-failure`,
+ writeMode: "dual",
+ failLegacyQueueProcessing: true,
+ }),
+ ).rejects.toThrow("simulated legacy ingestion read failure");
+
+ const remainingJobs = await getIngestionJobsForProject(projectId);
+ expect(remainingJobs.map((job) => job.id)).toEqual([unrelatedJob.id]);
+ } finally {
+ const remainingJobs = await getIngestionJobsForProject(projectId);
+ await Promise.allSettled(remainingJobs.map((job) => job.remove()));
+ const traceJobs = await getTraceUpsertJobsForProject(projectId);
+ await Promise.allSettled(traceJobs.map((job) => job.remove()));
+ env.LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG =
+ originalBlobStorageFileLogFlag;
+ await prisma.project.delete({ where: { id: projectId } });
+ await prisma.organization.delete({ where: { id: orgId } });
+ }
+ },
+ );
},
);
diff --git a/worker/src/__tests__/otelReplay.properties.test.ts b/worker/src/__tests__/otelReplay.properties.test.ts
index 7dfd802695..f2272116a9 100644
--- a/worker/src/__tests__/otelReplay.properties.test.ts
+++ b/worker/src/__tests__/otelReplay.properties.test.ts
@@ -4,7 +4,10 @@ import { randomInt } from "node:crypto";
import fc, { type Arbitrary } from "fast-check";
import { describe, expect, it } from "vitest";
import type { ResourceSpan } from "@langfuse/shared/src/server";
-import { runOtelReplay } from "./helpers/otelReplayHarness";
+import {
+ expectRawOtelReplayParity,
+ runOtelReplayComparison,
+} from "./helpers/otelReplayHarness";
import {
type Json,
jsonKeyArbitrary,
@@ -652,31 +655,37 @@ const directedCases: ReplayCase[] = [
async function assertReplayPersists(params: ReplayCase): Promise<void> {
const replay = buildReplayCase(params);
- const { storedRows } = await runOtelReplay({
+ const comparison = await runOtelReplayComparison({
bytes: Buffer.from(JSON.stringify(replay.resourceSpans)),
projectId: replay.projectId,
fileKey: FILE_KEY,
mediaUploadEnabled: true,
});
- expect(storedRows).toHaveLength(replay.expectedRows.length);
- const rowsBySpanId = new Map(
- storedRows.map((row) => [String(row.span_id), row]),
- );
- expect([...rowsBySpanId.keys()].sort()).toEqual(
- replay.expectedRows.map((row) => row.span_id).sort(),
- );
+ for (const result of [comparison.originalTs, comparison.earlyTs]) {
+ expect(result.storedRows, result.mode).toHaveLength(
+ replay.expectedRows.length,
+ );
+ const rowsBySpanId = new Map(
+ result.storedRows.map((row) => [String(row.span_id), row]),
+ );
+ expect([...rowsBySpanId.keys()].sort(), result.mode).toEqual(
+ replay.expectedRows.map((row) => row.span_id).sort(),
+ );
- for (const expected of replay.expectedRows) {
- const row = rowsBySpanId.get(expected.span_id)!;
- // Compare every contracted field exactly, including both metadata arrays:
- // objectContaining permits unrelated stored columns, not extra array elements.
- expect({
- ...row,
- start_time: persistedMillis(row.start_time),
- end_time: persistedMillis(row.end_time),
- }).toEqual(expect.objectContaining(expected));
+ for (const expected of replay.expectedRows) {
+ const row = rowsBySpanId.get(expected.span_id)!;
+ // Compare every contracted field exactly, including both metadata arrays:
+ // objectContaining permits unrelated stored columns, not extra array elements.
+ expect({
+ ...row,
+ start_time: persistedMillis(row.start_time),
+ end_time: persistedMillis(row.end_time),
+ }).toEqual(expect.objectContaining(expected));
+ }
}
+
+ expectRawOtelReplayParity(comparison);
}
// The replay writer retries internally; Vitest retries could overlap an
diff --git a/worker/src/env.ts b/worker/src/env.ts
index db3887217c..8d7aa3dc5a 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -116,6 +116,22 @@ const EnvSchema = z.object({
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: z
.enum(["true", "false"])
.default("false"),
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED: z
+ .enum(["true", "false"])
+ .default("false"),
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS: z
+ .string()
+ .default("")
+ .transform((value) =>
+ value
+ .split(",")
+ .map((id) => id.trim())
+ .filter(Boolean),
+ )
+ .refine((ids) => !ids.includes("*") || ids.length === 1, {
+ message:
+ 'Use "*" alone to select all projects, or a comma-separated list of project IDs',
+ }),
LANGFUSE_TRACE_BATCH_INGESTION_ENABLED: z
.enum(["true", "false"])
.default("false"),
diff --git a/worker/src/features/otel-ingestion/README.md b/worker/src/features/otel-ingestion/README.md
index 374bbb1628..36f8b1adcf 100644
--- a/worker/src/features/otel-ingestion/README.md
+++ b/worker/src/features/otel-ingestion/README.md
@@ -14,12 +14,11 @@ environment variables. The configured user needs to read the `events_full`
schema and create, insert into, select from, and drop the test tables. An
unavailable ClickHouse server fails the run.
-The corpus, integration, and property suites serialize their fixtures to S3
-document bytes and pass those bytes to the production OTEL queue processor. The
-queue performs the normal download and JSON decoding; normalization, enrichment,
-tokenization, media and overflow handling, and ClickHouse persistence use the
-TypeScript implementations. Model and prompt lookups, media uploads, and
-evaluation scheduling use controlled test doubles.
+The corpus, integration, and property suites replay identical stored JSON bytes
+through the production OTEL queue with early Rust media extraction off and on.
+Both runs retain TypeScript normalization, enrichment, tokenization, remaining
+media detection, overflow handling, and the production JSON writer. Model and
+prompt lookups, media uploads, and evaluation scheduling use test doubles.
Each replay creates isolated Memory tables from the live schema and uses the
production JSON writer singleton. A thin client adapter substitutes the
diff --git a/worker/src/features/otel-ingestion/prepareOtelBatch.ts b/worker/src/features/otel-ingestion/prepareOtelBatch.ts
new file mode 100644
index 0000000000..3de26b0988
--- /dev/null
+++ b/worker/src/features/otel-ingestion/prepareOtelBatch.ts
@@ -0,0 +1,167 @@
+import {
+ validateOtelJson,
+ type EarlyOtelBatch,
+ type ValidatedOtelJson,
+} from "@langfuse/native";
+import {
+ recordDistribution,
+ recordIncrement,
+ type ResourceSpan,
+} from "@langfuse/shared/src/server";
+import { applyIngestionMasking } from "@langfuse/shared/src/server/ee/ingestionMasking";
+
+type PreparedInput =
+ | {
+ batch: EarlyOtelBatch;
+ /** Neither masking nor UTF-8 normalization replaced the caller's bytes. */
+ sourceUnchanged: boolean;
+ error?: never;
+ }
+ | { batch?: never; sourceUnchanged?: never; error: string | undefined };
+
+/**
+ * Validate raw OTLP bytes and optionally extract media before the TypeScript
+ * ingestion graph is built. Masking receives the normalized byte view and may
+ * return a replacement body.
+ */
+export async function prepareOtelBatch(params: {
+ bytes: Buffer;
+ projectId: string;
+ orgId?: string;
+ propagatedHeaders?: Record<string, string>;
+ extractMedia: boolean;
+}): Promise<PreparedInput> {
+ const { bytes, extractMedia, ...context } = params;
+ const startedAt = performance.now();
+ let outcome: "native" | "masking_drop" | "error" = "error";
+ let originalInput: ValidatedOtelJson | undefined;
+ let maskedInput: ValidatedOtelJson | undefined;
+ try {
+ originalInput = await validateOtelJson(bytes);
+ const masking = await applyIngestionMasking(
+ {
+ ...context,
+ data: {
+ bytes: originalInput.normalizedBytes() ?? bytes,
+ validated: originalInput,
+ },
+ },
+ undefined,
+ {
+ body: (input) => {
+ if (!(input.bytes.buffer instanceof ArrayBuffer)) {
+ throw new Error(
+ "masking body must have an ArrayBuffer backing store",
+ );
+ }
+ return new Uint8Array(
+ input.bytes.buffer,
+ input.bytes.byteOffset,
+ input.bytes.byteLength,
+ );
+ },
+ read: async (response) => {
+ const disposal = Promise.allSettled([maskedInput?.dispose()]);
+ try {
+ const json = stripResponseJsonBoms(
+ Buffer.from(await response.arrayBuffer()),
+ );
+ maskedInput = await validateOtelJson(json);
+ return {
+ bytes: maskedInput.normalizedBytes() ?? json,
+ validated: maskedInput,
+ };
+ } finally {
+ await disposal.then(([result]) => {
+ if (result.status === "rejected") throw result.reason;
+ });
+ }
+ },
+ },
+ );
+ if (!masking.success) {
+ outcome = "masking_drop";
+ return { error: masking.error };
+ }
+ // Release the unused input before extraction allocates its result.
+ if (masking.data.validated === originalInput) {
+ await maskedInput?.dispose();
+ } else {
+ await originalInput.dispose();
+ }
+ const batch = await masking.data.validated.extract(extractMedia);
+ outcome = "native";
+ return {
+ batch,
+ sourceUnchanged: masking.data.bytes === bytes,
+ };
+ } catch (error) {
+ outcome = "error";
+ throw error;
+ } finally {
+ await Promise.all([originalInput?.dispose(), maskedInput?.dispose()])
+ .catch((error) => {
+ outcome = "error";
+ return Promise.reject(error);
+ })
+ .finally(() => {
+ const tags = {
+ outcome,
+ extract_media: extractMedia.toString(),
+ };
+ recordIncrement(
+ "langfuse.ingestion.otel.early_media.preparation",
+ 1,
+ tags,
+ );
+ recordDistribution(
+ "langfuse.ingestion.otel.early_media.preparation_duration_ms",
+ performance.now() - startedAt,
+ tags,
+ );
+ });
+ }
+}
+
+/** Parse for TypeScript consumers. The caller owns the returned batch; failures release it. */
+export async function parsePreparedOtelBatch(
+ prepared: { batch: EarlyOtelBatch; sourceUnchanged: boolean },
+ downloadedBytes: Uint8Array,
+): Promise<{ resourceSpans: ResourceSpan[]; batch?: EarlyOtelBatch }> {
+ let batch: EarlyOtelBatch | undefined = prepared.batch;
+ try {
+ let json: Buffer;
+ if (prepared.sourceUnchanged && batch.mediaCount() === 0) {
+ // Free Rust's duplicate before parsing the unchanged download. An external
+ // Buffer would retain that allocation until Node collects it.
+ await batch.dispose();
+ batch = undefined;
+ json = Buffer.from(
+ downloadedBytes.buffer,
+ downloadedBytes.byteOffset,
+ downloadedBytes.byteLength,
+ );
+ } else {
+ json = batch.takeJsonBuffer();
+ }
+ return { resourceSpans: JSON.parse(json.toString("utf8")), batch };
+ } catch (error) {
+ await batch?.dispose();
+ throw error;
+ }
+}
+
+// Match Node's Response.json(): accept up to two leading UTF-8 BOMs and leave
+// a third for the JSON parser to reject.
+function stripResponseJsonBoms(bytes: Buffer): Buffer {
+ let offset = 0;
+ while (
+ offset < 6 &&
+ bytes[offset] === 0xef &&
+ bytes[offset + 1] === 0xbb &&
+ bytes[offset + 2] === 0xbf
+ ) {
+ offset += 3;
+ }
+ return offset === 0 ? bytes : bytes.subarray(offset);
+}
diff --git a/worker/src/features/otel-ingestion/prepareOtelBatch.unit.test.ts b/worker/src/features/otel-ingestion/prepareOtelBatch.unit.test.ts
new file mode 100644
index 0000000000..a728d9d5ed
--- /dev/null
+++ b/worker/src/features/otel-ingestion/prepareOtelBatch.unit.test.ts
@@ -0,0 +1,107 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const nativeMocks = vi.hoisted(() => ({
+ validateOtelJson: vi.fn(),
+}));
+const maskingMocks = vi.hoisted(() => ({
+ applyIngestionMasking: vi.fn(),
+}));
+
+vi.mock("@langfuse/native", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("@langfuse/native")>();
+ nativeMocks.validateOtelJson.mockImplementation(actual.validateOtelJson);
+ return { ...actual, validateOtelJson: nativeMocks.validateOtelJson };
+});
+
+vi.mock("@langfuse/shared/src/server", () => ({
+ logger: { warn: vi.fn() },
+ recordDistribution: vi.fn(),
+ recordIncrement: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared/src/server/ee/ingestionMasking", () => ({
+ applyIngestionMasking: maskingMocks.applyIngestionMasking,
+}));
+
+import { parsePreparedOtelBatch, prepareOtelBatch } from "./prepareOtelBatch";
+
+describe("prepareOtelBatch native input boundary", { retry: 0 }, () => {
+ beforeEach(() => {
+ nativeMocks.validateOtelJson.mockClear();
+ maskingMocks.applyIngestionMasking.mockImplementation(
+ async (params: { data: unknown }) => ({
+ success: true,
+ data: params.data,
+ masked: false,
+ }),
+ );
+ });
+
+ it.each(["unchanged", "masked", "sanitized"] as const)(
+ "allows download reuse only for unchanged bytes (%s)",
+ async (inputKind) => {
+ const native =
+ await vi.importActual<typeof import("@langfuse/native")>(
+ "@langfuse/native",
+ );
+ nativeMocks.validateOtelJson.mockImplementation(native.validateOtelJson);
+ const bytes =
+ inputKind === "sanitized"
+ ? Buffer.concat([
+ Buffer.from('{"note":"'),
+ Buffer.from([0xff]),
+ Buffer.from('"}'),
+ ])
+ : Buffer.from('{"note":"original"}');
+ if (inputKind === "masked") {
+ maskingMocks.applyIngestionMasking.mockImplementation(
+ async (_params, _env, transport) => ({
+ success: true,
+ data: await transport.read(new Response('{"note":"masked"}')),
+ masked: true,
+ }),
+ );
+ }
+ const prepared = await prepareOtelBatch({
+ bytes,
+ projectId: "project-id",
+ extractMedia: true,
+ });
+ try {
+ if (!prepared.batch) throw new Error("expected native preparation");
+ expect(prepared.batch.mediaCount()).toBe(0);
+ expect(prepared.sourceUnchanged).toBe(inputKind === "unchanged");
+ const parsed = await parsePreparedOtelBatch(prepared, bytes);
+ expect(parsed.batch).toBe(
+ inputKind === "unchanged" ? undefined : prepared.batch,
+ );
+ expect(parsed.resourceSpans).toEqual({
+ note:
+ inputKind === "sanitized"
+ ? "�"
+ : inputKind === "masked"
+ ? "masked"
+ : "original",
+ });
+ } finally {
+ await prepared.batch?.dispose();
+ }
+ },
+ );
+
+ it("does not fall back to a second TypeScript processing path", async () => {
+ const error = Object.assign(
+ new Error("native implementation changed its explanation"),
+ { code: "ERR_OTEL_INVALID_JSON" },
+ );
+ nativeMocks.validateOtelJson.mockRejectedValueOnce(error);
+
+ await expect(
+ prepareOtelBatch({
+ bytes: Buffer.from("{}"),
+ projectId: "project-id",
+ extractMedia: false,
+ }),
+ ).rejects.toBe(error);
+ });
+});
diff --git a/worker/src/features/otel-ingestion/processOtelEvents.ts b/worker/src/features/otel-ingestion/processOtelEvents.ts
index fb652b12f0..7ea9fb7365 100644
--- a/worker/src/features/otel-ingestion/processOtelEvents.ts
+++ b/worker/src/features/otel-ingestion/processOtelEvents.ts
@@ -1,3 +1,5 @@
+import { restoreInlineMedia } from "../otel-media/resolveExtractedMedia";
+import type { EarlyOtelBatch } from "@langfuse/native";
import { convertEventRecordToObservationForEval } from "@langfuse/shared";
import {
logger,
@@ -25,6 +27,8 @@ type ProcessOtelEventsParams = {
projectId: string;
fileKey: string;
shouldWriteToEventsTable: boolean;
+ earlyBatch?: EarlyOtelBatch;
+ mediaPath?: "early" | "reference";
};
/**
@@ -43,6 +47,8 @@ export async function processOtelEvents({
projectId,
fileKey,
shouldWriteToEventsTable,
+ earlyBatch,
+ mediaPath,
}: ProcessOtelEventsParams): Promise<void> {
// Process events for observation evals and direct event writes
// This phase handles two independent concerns:
@@ -52,7 +58,6 @@ export async function processOtelEvents({
// Both require enriched event records with trace-level attributes
// (userId, sessionId, tags, release) that processToEvent provides.
const eventInputs = processor.processToEvent(resourceSpans);
-
if (eventInputs.length === 0) {
return;
}
@@ -75,6 +80,14 @@ export async function processOtelEvents({
return;
}
+ if (earlyBatch) {
+ // Evaluation-only records skip direct media resolution, so restore their
+ // extracted references before evaluation scheduling reads their payloads.
+ await restoreInlineMedia(earlyBatch, eventInputs, {
+ includePayloads: !shouldWriteToEventsTable,
+ });
+ }
+
if (
env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED === "true" &&
shouldWriteToEventsTable
@@ -82,6 +95,8 @@ export async function processOtelEvents({
await processOtelEventMedia({
targets: createDirectOtelMediaTargets(eventInputs),
writePath: "direct",
+ earlyBatch,
+ mediaPath,
projectId,
fileKey,
mediaBucket: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,
diff --git a/worker/src/features/otel-media/mediaReferenceText.ts b/worker/src/features/otel-media/mediaReferenceText.ts
new file mode 100644
index 0000000000..1d4b267fd2
--- /dev/null
+++ b/worker/src/features/otel-media/mediaReferenceText.ts
@@ -0,0 +1,281 @@
+import type {
+ EarlyOtelBatch,
+ ExtractedOtelMedia,
+ MediaRestoreRequest,
+} from "@langfuse/native";
+import { asRecord, defineOwnValue } from "@langfuse/shared";
+
+export type MediaRegistry = ReadonlyMap<string, ExtractedOtelMedia>;
+
+const MEDIA_REFERENCE_PREFIX = "@@@langfuseMedia:";
+const RESTORATION_BATCH_SIZE = 64;
+const RESTORATION_BATCH_SOURCE_BYTES = 1024 * 1024;
+// `originalMedia` accepts target serializations through sixteen JSON layers.
+// This restoration bound is separate from native's two-layer discovery limit.
+const MAX_SERIALIZED_REFERENCE_LAYERS = 16;
+
+type KnownMediaReferenceMatch = {
+ reference: string;
+ entry: ExtractedOtelMedia;
+ index: number;
+};
+
+/** Find only references owned by this batch; malformed markers cannot consume a later one. */
+function* knownMediaReferenceMatches(
+ value: string,
+ media: MediaRegistry,
+): Generator<KnownMediaReferenceMatch> {
+ let searchFrom = 0;
+ while (searchFrom < value.length) {
+ const index = value.indexOf(MEDIA_REFERENCE_PREFIX, searchFrom);
+ if (index === -1) return;
+ const end = value.indexOf("@@@", index + MEDIA_REFERENCE_PREFIX.length);
+ if (end === -1) return;
+ const reference = value.slice(index, end + 3);
+ const entry = media.get(reference);
+ if (entry) {
+ yield { reference, entry, index };
+ searchFrom = end + 3;
+ } else {
+ // An unterminated marker can see the opening delimiter of a later known
+ // marker as its closing delimiter. Search inside it before giving up.
+ searchFrom = index + MEDIA_REFERENCE_PREFIX.length;
+ }
+ }
+}
+
+function parseSerializedJson(value: string): unknown {
+ const trimmed = value.trimStart();
+ if (
+ !trimmed.startsWith("{") &&
+ !trimmed.startsWith("[") &&
+ !trimmed.startsWith('"')
+ )
+ return undefined;
+ try {
+ const parsed: unknown = JSON.parse(value);
+ return Array.isArray(parsed) ||
+ asRecord(parsed) ||
+ typeof parsed === "string"
+ ? parsed
+ : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
+/**
+ * Substitute only this batch's temporary references. Successful uploads need no
+ * JSON parsing: everything between reference ranges keeps its exact spelling.
+ * On restoration, inspect string layers without reserializing surrounding JSON.
+ * Restored text is cached for this call, not retained for the batch's lifetime.
+ * A missing resolver, or an undefined result from it, requests restoration.
+ */
+export function createMediaReferenceRewriter(
+ batch: EarlyOtelBatch,
+ media: MediaRegistry,
+) {
+ const originals = new Map<string, Promise<string>>();
+ return async (
+ value: string,
+ resolve?: (entry: ExtractedOtelMedia) => Promise<string | undefined>,
+ ): Promise<string> => {
+ let layers: Map<number, number> | undefined;
+ let output = "";
+ let end = 0;
+ const matches = knownMediaReferenceMatches(value, media);
+ let next = matches.next();
+ while (!next.done) {
+ const group: KnownMediaReferenceMatch[] = [];
+ let sourceBytes = 0;
+ while (!next.done && group.length < RESTORATION_BATCH_SIZE) {
+ const size = next.value.entry.originalByteLength;
+ // Source size estimates the return size; JSON escaping can enlarge it. Large items run alone.
+ if (
+ group.length > 0 &&
+ sourceBytes + size > RESTORATION_BATCH_SOURCE_BYTES
+ )
+ break;
+ group.push(next.value);
+ sourceBytes += size;
+ next = matches.next();
+ }
+ const requests = new Map<string, MediaRestoreRequest>();
+ const replacements: Array<string | { key: string }> = [];
+ for (const match of group) {
+ const replacement = await resolve?.(match.entry);
+ if (replacement !== undefined) {
+ replacements.push(replacement);
+ continue;
+ }
+ if (layers === undefined)
+ layers = serializedReferenceLayers(value, media);
+ const depth = layers.get(match.index)!;
+ const key = `${match.reference}\u0000${depth}`;
+ requests.set(key, { index: match.entry.index, jsonLayers: depth });
+ replacements.push({ key });
+ }
+ // Another string may have cached a read while the uploads above were awaited.
+ const missing = [...requests].filter(([key]) => !originals.has(key));
+ if (missing.length > 0) {
+ const reading = batch.originalMediaBatch(
+ missing.map(([, request]) => request),
+ );
+ missing.forEach(([key], index) => {
+ originals.set(
+ key,
+ reading.then((values) => values[index]!),
+ );
+ });
+ }
+ const values = await Promise.all(
+ replacements.map((replacement) =>
+ typeof replacement === "string"
+ ? replacement
+ : originals.get(replacement.key)!,
+ ),
+ );
+ for (const [index, match] of group.entries()) {
+ output += value.slice(end, match.index) + values[index]!;
+ end = match.index + match.reference.length;
+ }
+ }
+ return end === 0 ? value : output + value.slice(end);
+ };
+}
+
+/**
+ * Rewrite actual object values in place. Actual object keys are not visited;
+ * opaque string values are rewritten as source text, including JSON keys
+ * contained inside those strings.
+ */
+export async function rewriteMediaValues(
+ value: unknown,
+ rewrite: (text: string) => Promise<string>,
+): Promise<unknown> {
+ if (typeof value === "string") return rewrite(value);
+ if (typeof value !== "object" || value === null) return value;
+
+ // Iterate rather than recursing: normalization can leave deeply nested objects.
+ const stack: Array<{
+ object: Record<string, unknown>;
+ keys?: string[];
+ length: number;
+ next: number;
+ }> = [];
+ const push = (object: object) => {
+ const keys = Array.isArray(object) ? undefined : Object.keys(object);
+ stack.push({
+ object: object as Record<string, unknown>,
+ keys,
+ length: keys?.length ?? (object as unknown[]).length,
+ next: 0,
+ });
+ };
+ push(value);
+ while (stack.length > 0) {
+ const frame = stack[stack.length - 1]!;
+ if (frame.next === frame.length) {
+ stack.pop();
+ continue;
+ }
+ const index = frame.next++;
+ const key = frame.keys ? frame.keys[index]! : index;
+ const current = frame.object[key];
+ if (typeof current === "string") {
+ const next = await rewrite(current);
+ if (current !== next) defineOwnValue(frame.object, key, next);
+ } else if (typeof current === "object" && current !== null) {
+ push(current);
+ }
+ }
+ return value;
+}
+
+/**
+ * Find the stringified-document depth for each reference in one source string.
+ *
+ * This is used only on a restoration path. Successful replacement remains
+ * a source-text pass and does not parse the surrounding
+ * document. Parsing here is for depth inspection only; the original source is
+ * still rebuilt by replacing the known reference ranges in the current text.
+ */
+function serializedReferenceLayers(
+ value: string,
+ media: MediaRegistry,
+): Map<number, number> {
+ const matches = [...knownMediaReferenceMatches(value, media)];
+ const layers = new Map<number, number>();
+ if (matches.length === 0) return layers;
+
+ const root = parseSerializedJson(value);
+ if (root === undefined) {
+ for (const match of matches) layers.set(match.index, 0);
+ return layers;
+ }
+
+ const pendingByReference = new Map<string, number[]>();
+ const pendingCursor = new Map<string, number>();
+ for (const [matchIndex, match] of matches.entries()) {
+ const pending = pendingByReference.get(match.reference) ?? [];
+ pending.push(matchIndex);
+ pendingByReference.set(match.reference, pending);
+ }
+
+ const assign = (text: string, depth: number): void => {
+ for (const match of knownMediaReferenceMatches(text, media)) {
+ const pending = pendingByReference.get(match.reference);
+ const cursor = pendingCursor.get(match.reference) ?? 0;
+ const matchIndex = pending?.[cursor];
+ if (matchIndex === undefined) continue;
+ pendingCursor.set(match.reference, cursor + 1);
+ layers.set(matches[matchIndex]!.index, depth);
+ }
+ };
+
+ const stack: Array<{ value: unknown; depth: number }> = [
+ { value: root, depth: 1 },
+ ];
+ while (stack.length > 0) {
+ const current = stack.pop()!;
+ if (typeof current.value === "string") {
+ if (knownMediaReferenceMatches(current.value, media).next().done)
+ continue;
+ if (current.depth >= MAX_SERIALIZED_REFERENCE_LAYERS) {
+ assign(current.value, current.depth);
+ continue;
+ }
+ // A nested JSON string owns the references represented inside it. Do not
+ // assign the same textual markers once at the outer string layer and
+ // again after decoding the nested document.
+ const nested = parseSerializedJson(current.value);
+ if (nested !== undefined) {
+ stack.push({ value: nested, depth: current.depth + 1 });
+ } else {
+ assign(current.value, current.depth);
+ }
+ continue;
+ }
+ if (Array.isArray(current.value)) {
+ for (let index = current.value.length - 1; index >= 0; index--)
+ stack.push({ value: current.value[index], depth: current.depth });
+ continue;
+ }
+ const record = asRecord(current.value);
+ if (record) {
+ const entries = Object.entries(record);
+ for (let index = entries.length - 1; index >= 0; index--)
+ stack.push({ value: entries[index]![1], depth: current.depth });
+ }
+ }
+
+ // Duplicate JSON keys can be discarded by JSON.parse. Preserve the depth
+ // recorded by native for a source occurrence that the inspection walk cannot
+ // associate with a surviving parsed value.
+ for (const match of matches) {
+ if (!layers.has(match.index)) {
+ layers.set(match.index, Math.max(1, match.entry.originalJsonDepth));
+ }
+ }
+ return layers;
+}
diff --git a/worker/src/features/otel-media/processOtelMedia.test.ts b/worker/src/features/otel-media/processOtelMedia.test.ts
index fcf5809227..9da2f31230 100644
--- a/worker/src/features/otel-media/processOtelMedia.test.ts
+++ b/worker/src/features/otel-media/processOtelMedia.test.ts
@@ -1,4 +1,10 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ validateOtelJson,
+ type EarlyOtelBatch,
+ type ExtractedOtelMedia,
+ type MediaRestoreRequest,
+} from "@langfuse/native";
const mocks = vi.hoisted(() => {
const span = { setAttributes: vi.fn() };
@@ -12,6 +18,7 @@ const mocks = vi.hoisted(() => {
logger: { warn: vi.fn() },
recordDistribution: vi.fn(),
span,
+ linkMediaToTraceOrObservation: vi.fn(),
uploadMediaForTrace: vi.fn(),
};
});
@@ -23,6 +30,8 @@ vi.mock("@langfuse/shared/src/server", () => ({
logger: mocks.logger,
processOtelMedia: vi.fn(),
recordDistribution: mocks.recordDistribution,
+ recordIncrement: vi.fn(),
+ linkMediaToTraceOrObservation: mocks.linkMediaToTraceOrObservation,
uploadMediaForTrace: mocks.uploadMediaForTrace,
}));
@@ -33,6 +42,11 @@ import {
createLegacyOtelMediaTargets,
processOtelEventMedia,
} from "./processOtelMedia";
+import {
+ resolveExtractedMedia,
+ restoreInlineMedia,
+ uploadExtractedMediaOnce,
+} from "./resolveExtractedMedia";
const processResult = {
uploaded: 1,
@@ -55,8 +69,135 @@ const processResult = {
},
};
+const DEFAULT_MEDIA_REFERENCE =
+ "@@@langfuseMedia:type=image/png|id=provider|source=bytes@@@";
+
+const MEDIA_DESTINATION = {
+ projectId: "project-id",
+ mediaBucket: "media-bucket",
+ mediaPrefix: "media/",
+} as const;
+
+function createTestDescriptors(
+ count: number,
+ prefix: string,
+ originalByteLength: number,
+): ExtractedOtelMedia[] {
+ return Array.from({ length: count }, (_, index) => ({
+ index,
+ reference: DEFAULT_MEDIA_REFERENCE.replace(
+ "provider",
+ `${prefix}-${index}`,
+ ),
+ contentType: "image/png",
+ sha256Hash: `${prefix}-hash-${index}`,
+ kind: "anthropic",
+ originalByteLength,
+ originalJsonDepth: 0,
+ }));
+}
+
+function replaceReference(reference: string, mediaId: string) {
+ return reference.replace(/\|id=[^|@]+/, `|id=${mediaId}`);
+}
+
+function createFakeMediaBatch({
+ originalValue = "aGk=",
+ entries,
+}: {
+ originalValue?: string;
+ entries?: ExtractedOtelMedia[];
+} = {}) {
+ const mediaBody = vi.fn().mockResolvedValue(Buffer.from("media"));
+ const descriptors = entries ?? [
+ {
+ index: 0,
+ reference: DEFAULT_MEDIA_REFERENCE,
+ contentType: "image/png",
+ sha256Hash: "hash",
+ kind: "anthropic",
+ originalByteLength: 4,
+ originalJsonDepth: 0,
+ },
+ ];
+ const readMedia = vi.fn(() => descriptors);
+ const mediaCount = vi.fn(() => descriptors.length);
+ const mediaPage = vi.fn((offset: number, limit: number) =>
+ descriptors.slice(offset, offset + limit),
+ );
+ const batch = {
+ get media() {
+ return readMedia();
+ },
+ mediaCount,
+ mediaPage,
+ mediaBody,
+ originalMedia: vi.fn().mockResolvedValue(originalValue),
+ originalMediaBatch: vi.fn(async (requests: MediaRestoreRequest[]) =>
+ Promise.all(
+ requests.map(({ index, jsonLayers }) =>
+ batch.originalMedia(index, jsonLayers),
+ ),
+ ),
+ ),
+ } as EarlyOtelBatch;
+ return { batch, mediaBody, mediaCount, mediaPage, readMedia };
+}
+
+function createProviderPayload(reference: string, spanId = "observation-id") {
+ return {
+ traceId: "trace-id",
+ spanId,
+ input: {
+ type: "base64",
+ media_type: "image/png",
+ data: reference,
+ },
+ };
+}
+
+function resolveTarget(
+ batch: EarlyOtelBatch,
+ payload: unknown,
+ writePath: "direct" | "legacy" = "direct",
+) {
+ return resolveExtractedMedia({
+ batch,
+ targets: createDirectOtelMediaTargets([payload]),
+ ...MEDIA_DESTINATION,
+ writePath,
+ });
+}
+
+async function withExtractedBatch<T>(
+ source: T,
+ callback: (batch: EarlyOtelBatch, compacted: T) => Promise<void>,
+): Promise<void> {
+ const validated = await validateOtelJson(Buffer.from(JSON.stringify(source)));
+ let batch: EarlyOtelBatch | undefined;
+ try {
+ batch = await validated.extract(true);
+ await callback(batch, JSON.parse(batch.takeJsonBuffer().toString("utf8")));
+ } finally {
+ try {
+ await batch?.dispose();
+ } finally {
+ await validated.dispose();
+ }
+ }
+}
+
describe("processOtelEventMedia", () => {
- beforeEach(() => vi.clearAllMocks());
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.uploadMediaForTrace.mockReset().mockResolvedValue({
+ outcome: "uploaded",
+ mediaId: "uploaded",
+ });
+ mocks.linkMediaToTraceOrObservation
+ .mockReset()
+ .mockResolvedValue(undefined);
+ });
it("processes only normalized direct events with media fields", async () => {
const eventInput = {
@@ -75,10 +216,8 @@ describe("processOtelEventMedia", () => {
null,
]),
writePath: "direct",
- projectId: "project-id",
+ ...MEDIA_DESTINATION,
fileKey: "file-key",
- mediaBucket: "media-bucket",
- mediaPrefix: "media/",
processMedia,
});
@@ -92,6 +231,7 @@ describe("processOtelEventMedia", () => {
},
],
writePath: "direct",
+ mediaPath: "reference",
}),
);
const uploadMedia = processMedia.mock.calls[0]?.[0].uploadMedia;
@@ -127,17 +267,17 @@ describe("processOtelEventMedia", () => {
expect(mocks.recordDistribution).toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.batch_byte_length",
8,
- { write_path: "direct" },
+ { write_path: "direct", media_path: "reference" },
);
expect(mocks.recordDistribution).toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.batch_checked_byte_length",
33,
- { write_path: "direct" },
+ { write_path: "direct", media_path: "reference" },
);
expect(mocks.recordDistribution).toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.processing_duration_ms",
expect.any(Number),
- { write_path: "direct" },
+ { write_path: "direct", media_path: "reference" },
);
});
@@ -149,10 +289,8 @@ describe("processOtelEventMedia", () => {
{ traceId: "trace-id", spanId: "observation-id" },
]),
writePath: "direct",
- projectId: "project-id",
+ ...MEDIA_DESTINATION,
fileKey: "file-key",
- mediaBucket: "media-bucket",
- mediaPrefix: "media/",
processMedia,
});
@@ -173,10 +311,8 @@ describe("processOtelEventMedia", () => {
},
]),
writePath: "direct",
- projectId: "project-id",
+ ...MEDIA_DESTINATION,
fileKey: "file-key",
- mediaBucket: "media-bucket",
- mediaPrefix: "media/",
processMedia,
}),
).resolves.toBeUndefined();
@@ -184,18 +320,44 @@ describe("processOtelEventMedia", () => {
expect(mocks.recordDistribution).toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.processing_duration_ms",
expect.any(Number),
- { write_path: "direct" },
+ { write_path: "direct", media_path: "reference" },
);
expect(mocks.recordDistribution).not.toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.batch_byte_length",
expect.any(Number),
);
+ expect(mocks.span.setAttributes).toHaveBeenCalledWith({
+ "langfuse.ingestion.otel.media_path": "reference",
+ });
expect(mocks.logger.warn).toHaveBeenCalledWith(
"OTEL media processing failed; continuing ingestion with original span values",
expect.objectContaining({ projectId: "project-id", fileKey: "file-key" }),
);
});
+ it("restores extracted media if the residual detector fails before reference resolution", async () => {
+ const original = {
+ traceId: "trace-id",
+ spanId: "observation-id",
+ input: `data:image/png;base64,${"aGVsbG8h".repeat(1024)}`,
+ };
+ await withExtractedBatch(original, async (batch, payload) => {
+ expect(payload.input).not.toBe(original.input);
+ await processOtelEventMedia({
+ earlyBatch: batch,
+ targets: createDirectOtelMediaTargets([payload]),
+ writePath: "direct",
+ ...MEDIA_DESTINATION,
+ fileKey: "file-key",
+ processMedia: vi
+ .fn()
+ .mockRejectedValue(new Error("residual detector failed")),
+ });
+ expect(payload).toEqual(original);
+ expect(mocks.uploadMediaForTrace).not.toHaveBeenCalled();
+ });
+ });
+
it("creates legacy targets that retain trace and observation body references", () => {
const traceBody = {
id: "trace-id",
@@ -232,4 +394,505 @@ describe("processOtelEventMedia", () => {
},
]);
});
+
+ it("uploads every known early reference in payload values", async () => {
+ const reference = DEFAULT_MEDIA_REFERENCE;
+ const { batch } = createFakeMediaBatch();
+ const deep = { value: reference } as Record<string, unknown>;
+ let cursor = deep;
+ for (let index = 0; index < 12; index++) {
+ const next = { value: reference } as Record<string, unknown>;
+ cursor.value = next;
+ cursor = next;
+ }
+ const payload = {
+ traceId: "trace-id",
+ spanId: "observation-id",
+ input: {
+ provider: {
+ type: "base64",
+ media_type: "image/png",
+ data: reference,
+ unrelated: reference,
+ },
+ nested: JSON.stringify({
+ type: "base64",
+ media_type: "image/png",
+ data: reference,
+ unrelated: reference,
+ [reference]: "opaque string key",
+ }),
+ deep,
+ [reference]: "object key stays structural",
+ },
+ };
+
+ const result = await resolveTarget(batch, payload);
+
+ expect(result).toMatchObject({
+ uploaded: 1,
+ reused: 5,
+ failed: 0,
+ candidates: 6,
+ bytesProcessed: 30,
+ });
+ expect(payload.input.provider.data).toBe(
+ "@@@langfuseMedia:type=image/png|id=uploaded|source=bytes@@@",
+ );
+ expect(payload.input.provider.unrelated).toBe(
+ "@@@langfuseMedia:type=image/png|id=uploaded|source=bytes@@@",
+ );
+ const uploaded = reference.replace("id=provider", "id=uploaded");
+ expect(JSON.parse(payload.input.nested as string)).toEqual({
+ type: "base64",
+ media_type: "image/png",
+ data: uploaded,
+ unrelated: uploaded,
+ [uploaded]: "opaque string key",
+ });
+ expect(JSON.stringify(payload.input.deep)).not.toContain(reference);
+ expect(payload.input[reference]).toBe("object key stays structural");
+ expect(batch.originalMedia).not.toHaveBeenCalled();
+ });
+
+ it("uploads native refs in deeply nested serialized input, output, and metadata", async () => {
+ const contentBytes = Buffer.from("native-deep-media".repeat(512));
+ const encoded = contentBytes.toString("base64");
+ const provider = `{
+ "type": "file",
+ "mediaType": "image/png",
+ "data": "${encoded}",
+ "large": 9007199254740993
+}`;
+ const createDeepPayload = () => {
+ let value: unknown = JSON.stringify(provider);
+ for (let index = 0; index < 12; index++) value = { nested: value };
+ return value;
+ };
+ const original = {
+ traceId: "trace-id",
+ spanId: "observation-id",
+ input: createDeepPayload(),
+ output: createDeepPayload(),
+ metadata: createDeepPayload(),
+ };
+ await withExtractedBatch(original, async (batch, payload) => {
+ const descriptors = batch.media;
+ expect(descriptors).toHaveLength(3);
+ const result = await resolveTarget(batch, payload);
+
+ expect(result).toMatchObject({
+ uploaded: 1,
+ reused: 2,
+ failed: 0,
+ candidates: 3,
+ bytesProcessed: contentBytes.length * 3,
+ });
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledWith(
+ expect.objectContaining({
+ field: "input",
+ traceId: "trace-id",
+ observationId: "observation-id",
+ contentBytes,
+ sha256Hash: descriptors[0]!.sha256Hash,
+ }),
+ );
+ expect(contentBytes.length).toBeLessThanOrEqual(
+ descriptors[0]!.originalByteLength,
+ );
+ expect(
+ mocks.linkMediaToTraceOrObservation.mock.calls.map(
+ ([params]) => params,
+ ),
+ ).toEqual(
+ ["output", "metadata"].map((field) => ({
+ projectId: "project-id",
+ traceId: "trace-id",
+ observationId: "observation-id",
+ mediaId: "uploaded",
+ field,
+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,
+ })),
+ );
+ const uploadedReference = replaceReference(
+ descriptors[0]!.reference,
+ "uploaded",
+ );
+ for (const field of ["input", "output", "metadata"] as const) {
+ expect(JSON.stringify(payload[field])).toBe(
+ JSON.stringify(original[field]).replace(encoded, uploadedReference),
+ );
+ }
+ });
+ });
+
+ it("uploads a descriptor without reading the batch media registry", async () => {
+ const { batch, mediaBody: loadBody, readMedia } = createFakeMediaBatch();
+ readMedia.mockImplementation(() => {
+ throw new Error("media registry should not be read");
+ });
+ const result = await uploadExtractedMediaOnce({
+ batch,
+ projectId: "project-id",
+ traceId: "trace-id",
+ observationId: "observation-id",
+ field: "input",
+ mediaBucket: "media-bucket",
+ mediaPrefix: "media/",
+ sha256Hash: "hash",
+ contentType: "image/png",
+ loadBody,
+ });
+
+ expect(result).toEqual({
+ outcome: "uploaded",
+ mediaId: "uploaded",
+ byteLength: 5,
+ });
+ expect(loadBody).toHaveBeenCalledOnce();
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();
+ });
+
+ it.each([4, 2 * 1024 * 1024])(
+ "caches registry pages and batches restoration reads (%s source bytes)",
+ async (sourceBytes) => {
+ const entries = createTestDescriptors(4_097, "page", sourceBytes);
+ const { batch, mediaCount, mediaPage } = createFakeMediaBatch({
+ entries,
+ });
+ const reference = [...entries.slice(-65), entries.at(-1)!]
+ .map((entry) => entry.reference)
+ .join(" ");
+ const first = { name: reference };
+ const second = { name: reference };
+
+ await restoreInlineMedia(batch, [first]);
+ await restoreInlineMedia(batch, [second]);
+
+ expect(first.name).toBe(Array(66).fill("aGk=").join(" "));
+ expect(second.name).toBe(first.name);
+ expect(
+ vi
+ .mocked(batch.originalMediaBatch)
+ .mock.calls.map(([requests]) => requests.length),
+ ).toEqual(sourceBytes === 4 ? [64, 1, 64, 1] : Array(130).fill(1));
+ expect(mediaCount).toHaveBeenCalledOnce();
+ expect(mediaPage).toHaveBeenCalledTimes(2);
+ expect(mediaPage).toHaveBeenNthCalledWith(1, 0, 4_096);
+ expect(mediaPage).toHaveBeenNthCalledWith(2, 4_096, 4_096);
+ },
+ );
+
+ it("reuses uploads across destinations even after a failed link", async () => {
+ const { batch, mediaPage, mediaBody } = createFakeMediaBatch({
+ originalValue: "original-value",
+ });
+ const firstTarget = createProviderPayload(
+ DEFAULT_MEDIA_REFERENCE,
+ "legacy",
+ );
+ const failedTarget = createProviderPayload(
+ DEFAULT_MEDIA_REFERENCE,
+ "failed",
+ );
+ const directTarget = createProviderPayload(
+ DEFAULT_MEDIA_REFERENCE,
+ "direct",
+ );
+
+ await restoreInlineMedia(batch, [firstTarget, directTarget]);
+ const first = await resolveTarget(batch, firstTarget, "legacy");
+ mocks.linkMediaToTraceOrObservation.mockRejectedValueOnce(
+ new Error("link failed"),
+ );
+ const failed = await resolveTarget(batch, failedTarget);
+ const direct = await resolveTarget(batch, directTarget);
+
+ expect(first).toMatchObject({ uploaded: 1, reused: 0, bytesProcessed: 5 });
+ expect(failed).toMatchObject({ failed: 1, uploaded: 0, reused: 0 });
+ expect(failedTarget.input.data).toBe("original-value");
+ expect(direct).toMatchObject({
+ failed: 0,
+ uploaded: 0,
+ reused: 1,
+ bytesProcessed: 5,
+ });
+ expect(directTarget.input.data).toBe(
+ replaceReference(DEFAULT_MEDIA_REFERENCE, "uploaded"),
+ );
+ expect(mediaPage).toHaveBeenCalledOnce();
+ expect(mediaBody).toHaveBeenCalledOnce();
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();
+ expect(mocks.linkMediaToTraceOrObservation).toHaveBeenLastCalledWith({
+ projectId: "project-id",
+ traceId: "trace-id",
+ observationId: "direct",
+ mediaId: "uploaded",
+ field: "input",
+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,
+ });
+ });
+
+ it("evicts a failed upload so a later occurrence can retry", async () => {
+ const reference = DEFAULT_MEDIA_REFERENCE;
+ mocks.uploadMediaForTrace
+ .mockRejectedValueOnce(new Error("upload failed"))
+ .mockResolvedValueOnce({ outcome: "uploaded", mediaId: "retry" });
+ const { batch, mediaBody } = createFakeMediaBatch({
+ originalValue: "original-value",
+ });
+ const target = () => createProviderPayload(reference);
+
+ const failed = await resolveTarget(batch, target());
+ const retriedTarget = target();
+ const retried = await resolveTarget(batch, retriedTarget);
+
+ expect(failed).toMatchObject({ failed: 1, uploaded: 0 });
+ expect(retried).toMatchObject({ failed: 0, uploaded: 1 });
+ expect(mediaBody).toHaveBeenCalledTimes(2);
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledTimes(2);
+ expect(retriedTarget.input).toMatchObject({
+ data: "@@@langfuseMedia:type=image/png|id=retry|source=bytes@@@",
+ });
+ });
+
+ it("restores each source spelling after identical-content uploads fail", async () => {
+ const content = "abc".repeat(2048);
+ const input = [
+ {
+ type: "file",
+ mediaType: "image/png",
+ data: Buffer.from(content).toString("base64"),
+ },
+ { type: "file", mediaType: "image/png", data: `b'${content}'` },
+ ];
+ mocks.uploadMediaForTrace.mockRejectedValue(new Error("upload failed"));
+ await withExtractedBatch(input, async (batch, compacted) => {
+ const media = batch.media;
+ expect(media).toHaveLength(2);
+ expect(media[0].reference).not.toBe(media[1].reference);
+ expect(media[0].sha256Hash).toBe(media[1].sha256Hash);
+ const target = {
+ traceId: "trace-id",
+ spanId: "observation-id",
+ input: compacted,
+ };
+ const result = await resolveTarget(batch, target);
+ expect(result).toMatchObject({ failed: 2, uploaded: 0, reused: 0 });
+ expect(target.input).toEqual(input);
+ // A failed upload is evicted, so the next occurrence can retry independently.
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledTimes(2);
+ });
+ });
+
+ const pythonLiteral = String.raw`b'${"a".repeat(4096)}\xff\xd8\xff\xe0test\nquote\'slash\\\x00'`;
+ // The escaped marker keeps discovery active through the second JSON layer.
+ const pythonProvider = String.raw`{
+ "type": "file", "mediaType": "image/jpeg", "data": ${JSON.stringify(pythonLiteral)},
+ "marker": "\u0061", "large": 9007199254740993
+}`;
+ const quoteProvider = String.raw`{"type":"file","mediaType":"text/plain","data":"b'${"a".repeat(4096)}\u0022'"}`;
+
+ it.each([
+ ...[1, 2].flatMap((layers) =>
+ ["root", "object"].map((placement) => {
+ const text =
+ layers === 1
+ ? pythonProvider
+ : JSON.stringify({ child: pythonProvider });
+ return {
+ name: `Python bytes: ${placement}, ${layers} JSON layers`,
+ input: placement === "root" ? text : { nested: text },
+ addNormalizedLayer: false,
+ };
+ }),
+ ),
+ {
+ name: "Unicode-escaped quotes with an extra normalization layer",
+ input: `{"child":${JSON.stringify(quoteProvider).replace(/\\"/g, "\\u0022")}}`,
+ addNormalizedLayer: true,
+ },
+ ])(
+ "restores exact native text after upload failure: $name",
+ async ({ input, addNormalizedLayer }) => {
+ mocks.uploadMediaForTrace.mockRejectedValue(new Error("upload failed"));
+ const original = { traceId: "trace-id", spanId: "observation-id", input };
+ await withExtractedBatch(original, async (batch, target) => {
+ expect(batch.media).toHaveLength(1);
+ if (addNormalizedLayer)
+ target.input = JSON.stringify({ content: target.input });
+ expect(JSON.stringify(target.input)).toContain(
+ batch.media[0]!.reference,
+ );
+
+ const result = await resolveTarget(batch, target);
+ expect(result).toMatchObject({ failed: 1, uploaded: 0 });
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();
+
+ expect(target.input).toEqual(
+ addNormalizedLayer ? JSON.stringify({ content: input }) : input,
+ );
+ });
+ },
+ );
+
+ const escapedUri = String.raw`data:image\/png;base64,${String.raw`\/\/\/\/`.repeat(1024)}`;
+ const dataUri = `data:image/png;base64,${"aGkh".repeat(1024)}`;
+ it.each([
+ {
+ name: "escaped Data URI and unrelated slash spelling",
+ input: String.raw`{
+ "type": "file", "mediaType": "image\/png", "data": "${escapedUri}",
+ "unrelated": "keep this \/ spelling"
+}`,
+ mediaText: escapedUri,
+ failUpload: false,
+ decodedBytes: Buffer.from("/".repeat(4096), "base64"),
+ },
+ ...[false, true].map((failUpload) => ({
+ name: `unsafe integer spelling, upload ${failUpload ? "failure" : "success"}`,
+ input: `{
+ "type": "base64", "media_type": "image/png", "data": "${dataUri}",
+ "large": 9007199254740993
+}`,
+ mediaText: dataUri,
+ failUpload,
+ decodedBytes: undefined,
+ })),
+ ])(
+ "preserves native stringified text: $name",
+ async ({ input, mediaText, failUpload, decodedBytes }) => {
+ if (failUpload)
+ mocks.uploadMediaForTrace.mockRejectedValue(new Error("upload failed"));
+ await withExtractedBatch(
+ { traceId: "trace-id", spanId: "observation-id", input },
+ async (batch, target) => {
+ expect(batch.media).toHaveLength(1);
+ const reference = batch.media[0]!.reference;
+ expect(target.input).toContain(reference);
+
+ const result = await resolveTarget(batch, target);
+ expect(result).toMatchObject({
+ failed: Number(failUpload),
+ uploaded: Number(!failUpload),
+ });
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();
+
+ expect(target.input).toBe(
+ input.replace(
+ mediaText,
+ failUpload ? mediaText : replaceReference(reference, "uploaded"),
+ ),
+ );
+ if (decodedBytes)
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledWith(
+ expect.objectContaining({
+ contentType: "image/png",
+ contentBytes: decodedBytes,
+ }),
+ );
+ },
+ );
+ },
+ );
+
+ it("matches native references by source occurrence when JSON reorders numeric keys", async () => {
+ mocks.uploadMediaForTrace.mockImplementation(
+ async ({ contentBytes }: { contentBytes: Buffer }) => ({
+ outcome: "uploaded",
+ mediaId: contentBytes.toString("utf8", 0, 3),
+ }),
+ );
+ const first = `{"type":"file","mediaType":"image/png","data":"b'${"abc".repeat(2048)}'"}`;
+ const second = `{"type":"file","mediaType":"image/png","data":"b'${"def".repeat(2048)}'"}`;
+ // JSON.parse enumerates integer-like keys in ascending order even though
+ // the source document visits key "2" before key "1".
+ const nested = `{"2":${first},"1":${second}}`;
+ await withExtractedBatch(
+ { traceId: "trace-id", spanId: "observation-id", input: nested },
+ async (batch, target) => {
+ const references = batch.media.map((entry) => entry.reference);
+ expect(references).toHaveLength(2);
+
+ const result = await resolveTarget(batch, target);
+ expect(result).toMatchObject({ uploaded: 2, failed: 0 });
+ expect(target.input).toBe(
+ nested
+ .replace(
+ `b'${"abc".repeat(2048)}'`,
+ replaceReference(references[0]!, "abc"),
+ )
+ .replace(
+ `b'${"def".repeat(2048)}'`,
+ replaceReference(references[1]!, "def"),
+ ),
+ );
+ },
+ );
+ });
+
+ it("does not let an unterminated marker consume a later known reference", async () => {
+ const { batch } = createFakeMediaBatch();
+ const prefix = "prefix @@@langfuseMedia:unterminated ";
+ const payload = {
+ traceId: "trace-id",
+ spanId: "span-id",
+ input: `${prefix}${DEFAULT_MEDIA_REFERENCE}`,
+ };
+
+ await resolveTarget(batch, payload);
+
+ expect(payload.input).toBe(
+ `${prefix}${DEFAULT_MEDIA_REFERENCE.replace("provider", "uploaded")}`,
+ );
+ });
+
+ it("rewrites known references without altering unrelated text in either walker", async () => {
+ const { batch, mediaBody } = createFakeMediaBatch();
+ const known = String.raw`{
+ "large": 9007199254740993, "escaped": "\u0061",
+ "data": "${DEFAULT_MEDIA_REFERENCE}"
+}`;
+ const unknown = String.raw`{ "large": 9007199254740993, "escaped": "\u0061", "ref": "@@@langfuseMedia:existing@@@" }`;
+ const untouched = `{
+ "type": "file", "mediaType": "image/png", "data": "https://example.com/image.png",
+ "large": 9007199254740993
+}`;
+ const fields = { known, unknown, untouched };
+ const record = {
+ traceId: "trace-id",
+ spanId: "observation-id",
+ name: { ...fields },
+ input: {
+ type: "base64",
+ media_type: "image/png",
+ data: "https://example.com/image.png",
+ ...fields,
+ },
+ };
+ const original = structuredClone(record);
+
+ await restoreInlineMedia(batch, [record]);
+ expect(record).toEqual({
+ ...original,
+ name: {
+ ...fields,
+ known: known.replace(DEFAULT_MEDIA_REFERENCE, "aGk="),
+ },
+ });
+ expect(mediaBody).not.toHaveBeenCalled();
+ expect(mocks.uploadMediaForTrace).not.toHaveBeenCalled();
+
+ await resolveTarget(batch, record);
+ expect(record.input).toEqual({
+ ...original.input,
+ known: known.replace(
+ DEFAULT_MEDIA_REFERENCE,
+ replaceReference(DEFAULT_MEDIA_REFERENCE, "uploaded"),
+ ),
+ });
+ expect(mediaBody).toHaveBeenCalledOnce();
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();
+ });
});
diff --git a/worker/src/features/otel-media/processOtelMedia.ts b/worker/src/features/otel-media/processOtelMedia.ts
index 1e0a86ad9f..5f12e9aa75 100644
--- a/worker/src/features/otel-media/processOtelMedia.ts
+++ b/worker/src/features/otel-media/processOtelMedia.ts
@@ -1,3 +1,8 @@
+import type { EarlyOtelBatch } from "@langfuse/native";
+import {
+ resolveExtractedMedia,
+ restoreInlineMedia,
+} from "./resolveExtractedMedia";
import {
getClickhouseEntityType,
instrumentAsync,
@@ -18,20 +23,27 @@ const MEDIA_FIELDS = ["input", "output", "metadata"] as const;
* Worker integration point for media in normalized OTEL payloads selected for
* persistence by the ingestion queue.
*
- * Responsibilities are split across three layers:
+ * Responsibilities:
* - This adapter filters event shapes and owns storage configuration,
* instrumentation, and fail-open behavior.
* - `processOtelMedia` iterates input/output/metadata, supplies OTEL upload
- * context, and aggregates processing results.
+ * context, and aggregates processing results for media still inline.
* - `transformMediaPayload`, called by `processOtelMedia`, contains the generic
* Data URI/provider-shape detection and replacement algorithm. It has no
* knowledge of OTEL, storage, projects, or tracing.
+ * - `resolveExtractedMedia` is the TypeScript resolver for references created
+ * by Rust before normalization. It uploads media whose references survive in
+ * input/output/metadata, replaces them on success, and restores failed uploads
+ * inline. TypeScript detection still runs for media Rust left inline.
*
* Successful replacements mutate each target payload in place. Missing storage
- * configuration or unexpected processing errors are logged and swallowed so
- * media extraction cannot reject the enclosing OTEL ingestion job.
+ * configuration is logged and skips processing. Unexpected processing errors
+ * are logged and swallowed after restoring any pending references. If restoration
+ * itself fails, that error propagates to avoid persisting unresolved references.
*/
export async function processOtelEventMedia(params: {
+ earlyBatch?: EarlyOtelBatch;
+ mediaPath?: "early" | "reference";
targets: OtelMediaTarget[];
writePath: OtelMediaWritePath;
projectId: string;
@@ -49,6 +61,8 @@ export async function processOtelEventMedia(params: {
mediaPrefix,
processMedia = processOtelMedia,
} = params;
+ const mediaPath =
+ params.mediaPath ?? (params.earlyBatch ? "early" : "reference");
if (!mediaBucket) {
logger.warn(
@@ -65,11 +79,17 @@ export async function processOtelEventMedia(params: {
{ name: "langfuse.ingestion.otel.media.process" },
async (span) => {
const startedAt = Date.now();
+ span.setAttributes({
+ "langfuse.ingestion.otel.media_path": mediaPath,
+ });
try {
+ // Scan media still inline, including values below native's 1 KiB cutoff.
+ // References from native extraction are resolved separately below.
const result = await processMedia({
targets,
projectId,
writePath,
+ mediaPath,
mediaBucket,
mediaPrefix,
uploadMedia: (uploadParams) =>
@@ -79,6 +99,23 @@ export async function processOtelEventMedia(params: {
}),
});
+ if (params.earlyBatch) {
+ const early = await resolveExtractedMedia({
+ batch: params.earlyBatch,
+ targets,
+ projectId,
+ mediaBucket,
+ mediaPrefix,
+ writePath,
+ });
+ result.uploaded += early.uploaded;
+ result.reused += early.reused;
+ result.failed += early.failed;
+ result.candidates += early.candidates;
+ result.bytesProcessed += early.bytesProcessed;
+ result.bytesRemoved += early.bytesRemoved;
+ }
+
span.setAttributes({
"langfuse.ingestion.otel.media.uploaded": result.uploaded,
"langfuse.ingestion.otel.media.reused": result.reused,
@@ -107,7 +144,7 @@ export async function processOtelEventMedia(params: {
recordDistribution(
"langfuse.ingestion.otel.media.batch_byte_length",
result.bytesProcessed,
- { write_path: writePath },
+ { write_path: writePath, media_path: mediaPath },
);
recordDistribution(
"langfuse.ingestion.otel.media.batch_checked_byte_length",
@@ -115,18 +152,28 @@ export async function processOtelEventMedia(params: {
(total, bytes) => total + bytes,
0,
),
- { write_path: writePath },
+ { write_path: writePath, media_path: mediaPath },
);
} finally {
recordDistribution(
"langfuse.ingestion.otel.media.processing_duration_ms",
Date.now() - startedAt,
- { write_path: writePath },
+ { write_path: writePath, media_path: mediaPath },
);
}
},
);
} catch (error) {
+ if (params.earlyBatch) {
+ // Unexpected detector failures must not leave pending references in persisted payloads.
+ await restoreInlineMedia(
+ params.earlyBatch,
+ targets.map(({ payload }) => payload),
+ {
+ includePayloads: true,
+ },
+ );
+ }
logger.warn(
"OTEL media processing failed; continuing ingestion with original span values",
{ projectId, fileKey, error },
diff --git a/worker/src/features/otel-media/resolveExtractedMedia.ts b/worker/src/features/otel-media/resolveExtractedMedia.ts
new file mode 100644
index 0000000000..0db671ede4
--- /dev/null
+++ b/worker/src/features/otel-media/resolveExtractedMedia.ts
@@ -0,0 +1,303 @@
+import type { EarlyOtelBatch, ExtractedOtelMedia } from "@langfuse/native";
+import {
+ defineOwnValue,
+ isMediaContentType,
+ MediaAssociationOrigin,
+ type MediaContentType,
+ type MediaField,
+} from "@langfuse/shared";
+import {
+ logger,
+ recordDistribution,
+ recordIncrement,
+ type OtelMediaTarget,
+ type OtelMediaWritePath,
+ linkMediaToTraceOrObservation,
+ uploadMediaForTrace,
+} from "@langfuse/shared/src/server";
+
+import {
+ createMediaReferenceRewriter,
+ rewriteMediaValues,
+ type MediaRegistry,
+} from "./mediaReferenceText";
+
+const MEDIA_PAGE_SIZE = 4_096;
+
+type CachedMediaUpload = {
+ mediaId: string;
+ outcome: "uploaded" | "reused";
+ byteLength: number;
+};
+
+// Retain descriptor metadata and completed upload results for the batch's lifetime.
+// Decoded bodies and restored source strings stay local to each processing call.
+// Upload-only consumers do not need to materialize the descriptor registry in JS;
+// media bodies remain native-owned and are read lazily when needed.
+const mediaRegistries = new WeakMap<EarlyOtelBatch, Promise<MediaRegistry>>();
+const mediaUploadCache = new WeakMap<
+ EarlyOtelBatch,
+ Map<string, Promise<CachedMediaUpload>>
+>();
+
+async function batchMedia(batch: EarlyOtelBatch): Promise<MediaRegistry> {
+ const cached = mediaRegistries.get(batch);
+ if (cached) return cached;
+
+ const load = (async () => {
+ const count = batch.mediaCount();
+ const media = new Map<string, ExtractedOtelMedia>();
+ for (let offset = 0; offset < count; ) {
+ const page = batch.mediaPage(offset, MEDIA_PAGE_SIZE);
+ for (const entry of page) media.set(entry.reference, entry);
+ const nextOffset = offset + page.length;
+ if (nextOffset <= offset)
+ throw new Error("native media page returned no progress");
+ offset = nextOffset;
+ if (offset < count)
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ }
+ return media;
+ })();
+ mediaRegistries.set(batch, load);
+ try {
+ return await load;
+ } catch (error) {
+ if (mediaRegistries.get(batch) === load) mediaRegistries.delete(batch);
+ throw error;
+ }
+}
+
+/**
+ * Upload one extracted asset once for an EarlyOtelBatch and retain only its
+ * storage result. Legacy and direct writers can call this with their own lazy
+ * body reader and destination link; cache hits never retain or decode the body.
+ */
+export async function uploadExtractedMediaOnce(params: {
+ batch: EarlyOtelBatch;
+ projectId: string;
+ traceId: string;
+ observationId?: string;
+ field: MediaField;
+ mediaBucket: string;
+ mediaPrefix: string;
+ sha256Hash: string;
+ contentType: MediaContentType;
+ loadBody: () => Promise<Buffer>;
+}): Promise<CachedMediaUpload> {
+ const {
+ batch,
+ projectId,
+ traceId,
+ observationId,
+ field,
+ mediaBucket,
+ mediaPrefix,
+ sha256Hash,
+ contentType,
+ loadBody,
+ } = params;
+ const key = [
+ projectId,
+ mediaBucket,
+ mediaPrefix,
+ sha256Hash,
+ contentType,
+ ].join("\u0000");
+ const cache =
+ mediaUploadCache.get(batch) ??
+ new Map<string, Promise<CachedMediaUpload>>();
+ mediaUploadCache.set(batch, cache);
+
+ const cached = cache.get(key);
+ if (cached) {
+ const result = await cached;
+ await linkMediaToTraceOrObservation({
+ projectId,
+ traceId,
+ observationId,
+ mediaId: result.mediaId,
+ field,
+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,
+ });
+ return { ...result, outcome: "reused" };
+ }
+
+ const uploadPromise = (async (): Promise<CachedMediaUpload> => {
+ const contentBytes = await loadBody();
+ return {
+ ...(await uploadMediaForTrace({
+ projectId,
+ traceId,
+ observationId,
+ field,
+ contentType,
+ contentBytes,
+ sha256Hash,
+ mediaBucket,
+ mediaPrefix,
+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,
+ })),
+ byteLength: contentBytes.length,
+ };
+ })();
+ cache.set(key, uploadPromise);
+ try {
+ return await uploadPromise;
+ } catch (error) {
+ // A failed upload must not poison another occurrence or a later path that
+ // can retry the same asset with a fresh decoded body.
+ if (cache.get(key) === uploadPromise) cache.delete(key);
+ throw error;
+ }
+}
+
+/** Restore extracted references in fields that do not pass through media resolution. */
+export async function restoreInlineMedia(
+ batch: EarlyOtelBatch,
+ records: Record<string, unknown>[],
+ { includePayloads = false }: { includePayloads?: boolean } = {},
+): Promise<void> {
+ const media = await batchMedia(batch);
+ if (media.size === 0) return;
+ const rewrite = createMediaReferenceRewriter(batch, media);
+ const restore = (value: unknown) => rewriteMediaValues(value, rewrite);
+ for (const record of records) {
+ for (const key of Object.keys(record)) {
+ if (
+ includePayloads ||
+ (key !== "input" && key !== "output" && key !== "metadata")
+ ) {
+ const previous = record[key];
+ const next = await restore(previous);
+ if (previous !== next) defineOwnValue(record, key, next);
+ }
+ }
+ }
+}
+
+/**
+ * Upload native candidates surviving in input/output/metadata. Discovery has
+ * already checked media syntax and provider shapes; TS scan-cost limits do not
+ * invalidate that work. Only failed uploads are restored here.
+ */
+export async function resolveExtractedMedia(params: {
+ batch: EarlyOtelBatch;
+ targets: OtelMediaTarget[];
+ projectId: string;
+ mediaBucket: string;
+ mediaPrefix: string;
+ writePath: OtelMediaWritePath;
+}) {
+ const { batch, targets, projectId, mediaBucket, mediaPrefix, writePath } =
+ params;
+ const stats = {
+ uploaded: 0,
+ reused: 0,
+ failed: 0,
+ candidates: 0,
+ bytesProcessed: 0,
+ bytesRemoved: 0,
+ };
+ const media = await batchMedia(batch);
+ if (media.size === 0) return stats;
+ const rewrite = createMediaReferenceRewriter(batch, media);
+
+ async function resolveReference(
+ entry: ExtractedOtelMedia,
+ target: OtelMediaTarget,
+ field: MediaField,
+ ): Promise<string | undefined> {
+ try {
+ const contentType = entry.contentType;
+ if (!isMediaContentType(contentType))
+ throw new Error("unsupported extracted media type");
+ const result = await uploadExtractedMediaOnce({
+ batch,
+ projectId,
+ mediaBucket,
+ mediaPrefix,
+ sha256Hash: entry.sha256Hash,
+ contentType,
+ traceId: target.traceId,
+ observationId: target.observationId,
+ field,
+ loadBody: () => batch.mediaBody(entry.index),
+ });
+ const replacement = entry.reference.replace(
+ /\|id=[^|@]+/,
+ `|id=${result.mediaId}`,
+ );
+ stats.candidates++;
+ stats.bytesProcessed += result.byteLength;
+ stats[result.outcome]++;
+ const bytesRemoved = Math.max(
+ 0,
+ entry.originalByteLength - Buffer.byteLength(replacement),
+ );
+ stats.bytesRemoved += bytesRemoved;
+ recordExtractedMediaUpload(
+ result.outcome,
+ entry.kind,
+ writePath,
+ result.byteLength,
+ bytesRemoved,
+ );
+ return replacement;
+ } catch (error) {
+ stats.failed++;
+ recordExtractedMediaUpload("failed", entry.kind, writePath);
+ logger.warn(
+ "Extracted OTEL media upload failed; keeping the original inline value",
+ {
+ error,
+ projectId,
+ traceId: target.traceId,
+ observationId: target.observationId,
+ field,
+ },
+ );
+ return undefined;
+ }
+ }
+
+ for (const target of targets) {
+ for (const field of ["input", "output", "metadata"] as const) {
+ target.payload[field] = await rewriteMediaValues(
+ target.payload[field],
+ (text) =>
+ rewrite(text, (entry) => resolveReference(entry, target, field)),
+ );
+ }
+ }
+ return stats;
+}
+
+/** Record outcomes for media discovered by the native extractor. */
+function recordExtractedMediaUpload(
+ outcome: "uploaded" | "reused" | "failed",
+ kind: string,
+ writePath: OtelMediaWritePath,
+ byteLength?: number,
+ bytesRemoved = 0,
+): void {
+ const tags = {
+ outcome,
+ media_kind: kind,
+ write_path: writePath,
+ media_path: "early",
+ };
+ recordIncrement("langfuse.ingestion.otel.media", 1, tags);
+ if (byteLength !== undefined)
+ recordDistribution(
+ "langfuse.ingestion.otel.media.byte_length",
+ byteLength,
+ tags,
+ );
+ if (bytesRemoved > 0)
+ recordDistribution(
+ "langfuse.ingestion.otel.media.bytes_removed",
+ bytesRemoved,
+ { write_path: writePath, media_path: "early" },
+ );
+}
diff --git a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
index d35f700f92..1f5eef2a5b 100644
--- a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
+++ b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
@@ -1,13 +1,17 @@
import { describe, it, expect, vi } from "vitest";
import { type Job } from "bullmq";
+import type { EarlyOtelBatch } from "@langfuse/native";
import {
getS3EventStorageClient,
+ markProjectIngestFailure,
OtelIngestionProcessor,
QueueJobs,
type QueueName,
type TQueueJobTypes,
} from "@langfuse/shared/src/server";
import { env } from "../../env";
+import * as otelPreparation from "../../features/otel-ingestion/prepareOtelBatch";
+import * as mediaRestoration from "../../features/otel-media/resolveExtractedMedia";
import { IngestionService } from "../../services/IngestionService";
import { ClickhouseWriter } from "../../services/ClickhouseWriter";
import { fetchObservationEvalRules } from "../../features/evaluation/observationEval";
@@ -28,6 +32,7 @@ import {
vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
getS3EventStorageClient: vi.fn(),
+ markProjectIngestFailure: vi.fn(),
}));
vi.mock(
"../../features/evaluation/observationEval",
@@ -51,16 +56,32 @@ describe("direct-v4 trace batch tracking", () => {
LANGFUSE_MIGRATION_V4_WRITE_MODE: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED:
env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED,
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED:
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED,
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS:
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS,
};
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";
env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";
env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED = "false";
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED = "true";
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS = ["*"];
try {
vi.mocked(getS3EventStorageClient).mockReturnValue({
- download: vi.fn().mockResolvedValue("[]"),
+ downloadBytes: vi.fn().mockResolvedValue(Buffer.from("[]")),
} as unknown as ReturnType<typeof getS3EventStorageClient>);
+ vi.spyOn(otelPreparation, "prepareOtelBatch").mockResolvedValue({
+ sourceUnchanged: false,
+ batch: {
+ takeJsonBuffer: () => Buffer.from("[]"),
+ dispose: vi.fn(),
+ } as unknown as EarlyOtelBatch,
+ });
+ const restore = vi
+ .spyOn(mediaRestoration, "restoreInlineMedia")
+ .mockResolvedValue(undefined);
vi.mocked(fetchObservationEvalRules).mockResolvedValue([]);
vi.spyOn(ClickhouseWriter, "getInstance").mockReturnValue(
{} as ClickhouseWriter,
@@ -122,6 +143,9 @@ describe("direct-v4 trace batch tracking", () => {
} as Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>;
await processor(job, undefined);
+ expect(restore.mock.calls.map(([, records]) => records)).toEqual([
+ inputs,
+ ]);
expect(write).toHaveBeenCalledTimes(3);
expect(trackTraceBatchActivity).toHaveBeenCalledExactlyOnceWith(
"project",
@@ -148,7 +172,9 @@ describe("direct-v4 trace batch tracking", () => {
env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "legacy";
write.mockClear();
+ restore.mockClear();
await processor(job, undefined);
+ expect(restore.mock.calls.map(([, records]) => records)).toEqual([[]]);
expect(write).not.toHaveBeenCalled();
expect(trackTraceBatchActivity).not.toHaveBeenCalled();
} finally {
@@ -159,6 +185,145 @@ describe("direct-v4 trace batch tracking", () => {
});
});
+describe("early media extraction rollout", () => {
+ const createJob = (projectId: string) =>
+ ({
+ data: {
+ id: "early-media-rollout",
+ timestamp: new Date(),
+ name: QueueJobs.OtelIngestionJob,
+ payload: {
+ data: { fileKey: "early-media-rollout.json" },
+ authCheck: {
+ validKey: true,
+ scope: {
+ projectId,
+ orgId: "org",
+ accessLevel: "project",
+ },
+ },
+ ingestionVersion: "4",
+ },
+ },
+ }) as Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>;
+
+ it("releases an unchanged media-free native document before JSON.parse", async () => {
+ const original = {
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED:
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED,
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS:
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS,
+ };
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED = "true";
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS = ["*"];
+ const json = '[{"resource":{"attributes":[]},"scopeSpans":[]}]';
+ const bytes = Buffer.from(json);
+ vi.mocked(getS3EventStorageClient).mockReturnValue({
+ downloadBytes: vi.fn().mockResolvedValue(bytes),
+ } as unknown as ReturnType<typeof getS3EventStorageClient>);
+ const prepare = otelPreparation.prepareOtelBatch;
+ const parse = JSON.parse;
+ let checkedBeforeParse = false;
+ const stop = new Error("stop after parsing");
+ try {
+ vi.spyOn(otelPreparation, "prepareOtelBatch").mockImplementation(
+ async (params) => {
+ const prepared = await prepare(params);
+ const batch = prepared.batch!;
+ expect(batch.mediaCount()).toBe(0);
+ vi.spyOn(batch, "takeJsonBuffer").mockImplementation(() => {
+ throw new Error("must not transfer a second document to Node");
+ });
+ vi.spyOn(JSON, "parse").mockImplementation((text, reviver) => {
+ if (text === json) {
+ expect(() => batch.mediaCount()).toThrow(/disposed/);
+ checkedBeforeParse = true;
+ }
+ return parse(text, reviver);
+ });
+ return prepared;
+ },
+ );
+ vi.spyOn(
+ OtelIngestionProcessor.prototype,
+ "processToIngestionEvents",
+ ).mockRejectedValue(stop);
+ await expect(
+ otelIngestionQueueProcessorBuilder(false)(
+ createJob("project"),
+ undefined,
+ ),
+ ).rejects.toBe(stop);
+ expect(checkedBeforeParse).toBe(true);
+ } finally {
+ Object.assign(env, original);
+ vi.restoreAllMocks();
+ vi.clearAllMocks();
+ }
+ });
+
+ it.each([
+ ["reference", "disabled with wildcard", false, ["*"]],
+ ["reference", "enabled with no projects", true, []],
+ [
+ "reference",
+ "enabled with a non-matching project",
+ true,
+ ["other-project"],
+ ],
+ ["early", "enabled with a matching project", true, ["project-match"]],
+ ["early", "enabled for all projects", true, ["*"]],
+ ] as const)(
+ "selects the %s path when %s",
+ async (expectedPath, label, enabled, projectIds) => {
+ const original = {
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED:
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED,
+ LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS:
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS,
+ };
+ const download = vi
+ .fn()
+ .mockRejectedValue(new Error("stop after reference download"));
+ const downloadBytes = vi
+ .fn()
+ .mockRejectedValue(new Error("stop after early download"));
+
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED = enabled
+ ? "true"
+ : "false";
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS = [...projectIds];
+ vi.mocked(getS3EventStorageClient).mockReturnValue({
+ download,
+ downloadBytes,
+ } as unknown as ReturnType<typeof getS3EventStorageClient>);
+
+ try {
+ const processor = otelIngestionQueueProcessorBuilder(false);
+ await expect(
+ processor(createJob("project-match"), undefined),
+ ).rejects.toThrow(`stop after ${expectedPath} download`);
+
+ if (expectedPath === "early") {
+ expect(downloadBytes).toHaveBeenCalledExactlyOnceWith(
+ "early-media-rollout.json",
+ );
+ expect(download).not.toHaveBeenCalled();
+ } else {
+ expect(download).toHaveBeenCalledExactlyOnceWith(
+ "early-media-rollout.json",
+ );
+ expect(downloadBytes).not.toHaveBeenCalled();
+ }
+ } finally {
+ Object.assign(env, original);
+ vi.mocked(getS3EventStorageClient).mockReset();
+ vi.mocked(markProjectIngestFailure).mockClear();
+ }
+ },
+ );
+});
+
describe("isOrgPastOtelDirectWriteCutoff", () => {
const cutoff = "2026-08-06";
const inScope = {
diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.ts
index 1c2fe8b1b6..9b6f1e4e07 100644
--- a/worker/src/queues/otelIngestionQueue.ts
+++ b/worker/src/queues/otelIngestionQueue.ts
@@ -1,4 +1,5 @@
import { Job, Processor } from "bullmq";
+import type { EarlyOtelBatch } from "@langfuse/native";
import { z } from "zod";
import {
clickhouseClient,
@@ -44,7 +45,12 @@ import {
createLegacyOtelMediaTargets,
processOtelEventMedia,
} from "../features/otel-media/processOtelMedia";
+import { restoreInlineMedia } from "../features/otel-media/resolveExtractedMedia";
import { processOtelEvents } from "../features/otel-ingestion/processOtelEvents";
+import {
+ parsePreparedOtelBatch,
+ prepareOtelBatch,
+} from "../features/otel-ingestion/prepareOtelBatch";
/**
* Legacy media processing follows legacy persistence, independently of
@@ -436,6 +442,7 @@ export const otelIngestionQueueProcessorBuilder = (
return async (
job: Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>,
): Promise<void> => {
+ let earlyBatch: EarlyOtelBatch | undefined;
try {
const projectId = job.data.payload.authCheck.scope.projectId;
const publicKey = job.data.payload.data.publicKey ?? "";
@@ -491,47 +498,92 @@ export const otelIngestionQueueProcessorBuilder = (
// Otherwise, we'd probably have to upsert one row per generated event further below.
// Easy change, but needs alignment.
- // Download file from blob storage
- const resourceSpans = await getS3EventStorageClient(
+ const earlyMediaEnabled =
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED === "true" &&
+ (env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS.includes("*") ||
+ env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS.includes(
+ projectId,
+ ));
+ const mediaPath = earlyMediaEnabled ? "early" : "reference";
+ span?.setAttribute("langfuse.ingestion.otel.media_path", mediaPath);
+ // Count selection once per queue attempt, including jobs that later fail.
+ recordIncrement("langfuse.ingestion.otel.media_path", 1, {
+ path: mediaPath,
+ });
+
+ const eventStorage = getS3EventStorageClient(
env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
- ).download(fileKey);
-
- recordHistogram(
- "langfuse.ingestion.s3_file_size_bytes",
- resourceSpans.length, // At this point it's still a string.
- {
- skippedS3List: "true",
- otel: "true",
- },
);
-
- // Parse spans from S3 download
- let parsedSpans = JSON.parse(resourceSpans);
-
- // Apply ingestion masking if enabled (EE feature)
- if (isIngestionMaskingEnabled()) {
- const maskingResult = await applyIngestionMasking({
- data: parsedSpans,
+ let parsedSpans: ResourceSpan[];
+ if (earlyMediaEnabled) {
+ const bytes = await eventStorage.downloadBytes(fileKey);
+ recordHistogram(
+ "langfuse.ingestion.s3_file_size_bytes",
+ bytes.byteLength,
+ { skippedS3List: "true", otel: "true" },
+ );
+ const prepared = await prepareOtelBatch({
+ bytes: Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength),
projectId,
orgId: job.data.payload.authCheck.scope.orgId,
propagatedHeaders: job.data.payload.propagatedHeaders,
+ extractMedia:
+ env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED === "true" &&
+ Boolean(env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET),
});
-
- if (!maskingResult.success) {
- // Fail-closed: drop event. Emit the S3 location so operators can
- // scan logs to identify which raw payloads need to be replayed via
- // worker/src/scripts/replayIngestionEventsV2 once the upstream
- // masking callback is healthy again.
- logger.warn(`Dropping OTEL event due to masking failure`, {
+ if (!prepared.batch) {
+ logger.warn("Dropping OTEL event due to masking failure", {
projectId,
orgId: job.data.payload.authCheck.scope.orgId,
fileKey,
- error: maskingResult.error,
+ error: prepared.error,
propagatedHeaders: job.data.payload.propagatedHeaders,
});
return;
}
- parsedSpans = maskingResult.data;
+ ({ batch: earlyBatch, resourceSpans: parsedSpans } =
+ await parsePreparedOtelBatch(prepared, bytes));
+ } else {
+ // Download file from blob storage
+ const resourceSpans = await eventStorage.download(fileKey);
+
+ recordHistogram(
+ "langfuse.ingestion.s3_file_size_bytes",
+ resourceSpans.length, // At this point it's still a string.
+ {
+ skippedS3List: "true",
+ otel: "true",
+ },
+ );
+
+ // Parse spans from S3 download
+ parsedSpans = JSON.parse(resourceSpans);
+
+ // Apply ingestion masking if enabled (EE feature)
+ if (isIngestionMaskingEnabled()) {
+ const maskingResult = await applyIngestionMasking({
+ data: parsedSpans,
+ projectId,
+ orgId: job.data.payload.authCheck.scope.orgId,
+ propagatedHeaders: job.data.payload.propagatedHeaders,
+ });
+
+ if (!maskingResult.success) {
+ // Fail-closed: drop event. Emit the S3 location so operators can
+ // scan logs to identify which raw payloads need to be replayed via
+ // worker/src/scripts/replayIngestionEventsV2 once the upstream
+ // masking callback is healthy again.
+ logger.warn(`Dropping OTEL event due to masking failure`, {
+ projectId,
+ orgId: job.data.payload.authCheck.scope.orgId,
+ fileKey,
+ error: maskingResult.error,
+ propagatedHeaders: job.data.payload.propagatedHeaders,
+ });
+ return;
+ }
+ parsedSpans = maskingResult.data;
+ }
}
// Generate events via OtelIngestionProcessor
@@ -545,6 +597,13 @@ export const otelIngestionQueueProcessorBuilder = (
});
const events: IngestionEventType[] =
await processor.processToIngestionEvents(parsedSpans);
+ const writesToLegacyTables = v4WritesToLegacyTables(env);
+ if (earlyBatch && writesToLegacyTables) {
+ await restoreInlineMedia(
+ earlyBatch,
+ events.map((event) => event.body),
+ );
+ }
// Here, we split the events into observations and non-observations.
// Observations go into the IngestionService directly whereas the non-observations make another run through the processEventBatch method.
@@ -707,7 +766,6 @@ export const otelIngestionQueueProcessorBuilder = (
// validation already guarantees useDirectEventWrite is true here, so
// observations and traces don't need the mergeAndWrite / IngestionQueue
// detour that would otherwise populate the legacy tables.
- const writesToLegacyTables = v4WritesToLegacyTables(env);
const skipLegacyWrites = !writesToLegacyTables;
const shouldProcessLegacyMedia = shouldProcessLegacyOtelMedia({
mediaUploadEnabled: env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED === "true",
@@ -731,6 +789,8 @@ export const otelIngestionQueueProcessorBuilder = (
await processOtelEventMedia({
targets: createLegacyOtelMediaTargets(traces.concat(observations)),
writePath: "legacy",
+ earlyBatch,
+ mediaPath,
projectId,
fileKey,
mediaBucket: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,
@@ -780,6 +840,8 @@ export const otelIngestionQueueProcessorBuilder = (
projectId,
fileKey,
shouldWriteToEventsTable,
+ earlyBatch,
+ mediaPath,
});
} catch (e) {
const fileKey = job.data.payload.data.fileKey;
@@ -804,6 +866,8 @@ export const otelIngestionQueueProcessorBuilder = (
fields: { fileKey },
});
throw e;
+ } finally {
+ await earlyBatch?.dispose();
}
};
};