🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(otel): skip native prevalidation for marker-free input (#18663)
Valery Meleshkin627b5b94f334
diff --git a/packages/native/index.d.ts b/packages/native/index.d.tsindex 428e52cb18..c577b405f4 100644--- a/packages/native/index.d.ts+++ b/packages/native/index.d.ts@@ -80,6 +80,12 @@ export interface ExtractedOtelMedia {originalHasJsonEscapes: boolean}+/**+ * Extract media from unmasked input. Null means the caller should parse its original bytes;+ * marker-free input has NOT been validated. Masking callers must use validateOtelJson first.+ */+export declare function extractOtelMedia(bytes: Buffer): Promise<EarlyOtelBatch | null>+/*** Hello-world entry point. It proves that the worker can load and call the* addon and that native code reports through the same telemetry as the restdiff --git a/packages/native/index.js b/packages/native/index.jsindex 9d1bf248db..546a99d340 100644--- a/packages/native/index.js+++ b/packages/native/index.js@@ -705,6 +705,7 @@ module.exports.PreparedEvent = nativeBinding.PreparedEventmodule.exports.ValidatedOtelJson = nativeBinding.ValidatedOtelJsonmodule.exports.clusterTopicEmbeddings = nativeBinding.clusterTopicEmbeddingsmodule.exports.encodeClickhouseEvents = nativeBinding.encodeClickhouseEvents+module.exports.extractOtelMedia = nativeBinding.extractOtelMediamodule.exports.hello = nativeBinding.hellomodule.exports.initTelemetry = nativeBinding.initTelemetrymodule.exports.validateOtelJson = nativeBinding.validateOtelJsondiff --git a/packages/native/src/lib.rs b/packages/native/src/lib.rsindex 94824e0868..9115d42267 100644--- a/packages/native/src/lib.rs+++ b/packages/native/src/lib.rs@@ -19,7 +19,7 @@ mod topics;pub use topics::{cluster_topic_embeddings, TopicClusteringResult, TopicClusteringSettings};-pub use otel_input::{validate_otel_json, EarlyOtelBatch, ValidatedOtelJson};+pub use otel_input::{extract_otel_media, validate_otel_json, EarlyOtelBatch, ValidatedOtelJson};use std::sync::Arc;use std::time::Duration;diff --git a/packages/native/src/otel_input.rs b/packages/native/src/otel_input.rsindex 9107b2811e..5b1e4fc45d 100644--- a/packages/native/src/otel_input.rs+++ b/packages/native/src/otel_input.rs@@ -71,15 +71,7 @@ impl ValidatedOtelJson {media: Vec::new(),}};- let batch = ExtractedBatch {- json: Mutex::new(Some(result.compact_json)),- media: result.media,- _memory: memory,- };- batch._memory.resize(batch.retained_bytes());- Ok(EarlyOtelBatch {- inner: Some(Arc::new(batch)),- })+ Ok(EarlyOtelBatch::new(result, memory))})}}@@ -119,6 +111,18 @@ pub struct EarlyOtelBatch {}impl EarlyOtelBatch {+ fn new(result: EarlyMediaResult, memory: NativeMemory) -> Self {+ let batch = ExtractedBatch {+ json: Mutex::new(Some(result.compact_json)),+ media: result.media,+ _memory: memory,+ };+ batch._memory.resize(batch.retained_bytes());+ Self {+ inner: Some(Arc::new(batch)),+ }+ }+pub(crate) fn data(&self) -> std::result::Result<Arc<ExtractedBatch>, Error<&'static str>> {self.inner.as_ref()@@ -330,3 +334,31 @@ pub fn validate_otel_json(})}))}++/// Extract media from unmasked input. Null means the caller should parse its original bytes;+/// marker-free input has NOT been validated. Masking callers must use validateOtelJson first.+#[napi(ts_return_type = "Promise<EarlyOtelBatch | null>")]+pub fn extract_otel_media(+ env: Env,+ bytes: Buffer,+) -> Result<AsyncTask<OtelMediaTask<Option<EarlyOtelBatch>>>> {+ let bytes = bytes.to_vec();+ let memory = NativeMemory::new(&env, bytes.capacity())?;+ Ok(OtelMediaTask::run("extract", move || {+ // Match raw ASCII markers before UTF-8/JSON validation. Unicode escapes+ // are themselves markers; UTF-8 replacement cannot introduce an ASCII marker.+ if !otel_media::may_contain_media_candidate(&bytes) {+ return Ok(None);+ }+ let validated = otel_media::validate(bytes)+ .map_err(|error| Error::new(error.code(), error.to_string()))?;+ let normalized = validated.normalized_bytes().is_some();+ let result = validated+ .compact()+ .map_err(|error| Error::new(error.code(), error.to_string()))?;+ if result.media.is_empty() && !normalized {+ return Ok(None);+ }+ Ok(Some(EarlyOtelBatch::new(result, memory)))+ }))+}diff --git a/packages/native/src/otel_media/mod.rs b/packages/native/src/otel_media/mod.rsindex 1ae02756b5..5f30d82830 100644--- a/packages/native/src/otel_media/mod.rs+++ b/packages/native/src/otel_media/mod.rs@@ -89,6 +89,7 @@ impl WalkStats {}pub use payload::{validate, EarlyMediaResult, ExtractedMedia, ValidatedPayload};+pub(crate) use rules::may_contain_media_candidate;#[cfg(test)]pub fn extract_media(input: &[u8]) -> Result<EarlyMediaResult, payload::EarlyMediaError> {diff --git a/packages/native/src/otel_media/rules.rs b/packages/native/src/otel_media/rules.rsindex b36c7e33d5..a600f89b4c 100644--- a/packages/native/src/otel_media/rules.rs+++ b/packages/native/src/otel_media/rules.rs@@ -209,12 +209,12 @@ pub(super) fn may_contain_serialized_media(value: &str) -> bool {has_provider_shape || has_quoted_root_candidate}-/// Return true when a validated document might contain a media candidate.+/// Return true when raw or validated JSON might contain a media candidate./// Unicode escapes are included because they can spell a marker after decoding.-pub(super) fn may_contain_media_candidate(value: &str) -> bool {+pub(crate) fn may_contain_media_candidate(value: impl AsRef<[u8]>) -> bool {// Broad fragments are intentional: false positives only select the full// walk, while escaped nesting remains discoverable.- MEDIA_CANDIDATE_MARKERS.is_match(value)+ MEDIA_CANDIDATE_MARKERS.is_match(value.as_ref())}pub(super) fn may_be_serialized_json(value: &str) -> bool {diff --git a/worker/src/__tests__/ingestionMasking.test.ts b/worker/src/__tests__/ingestionMasking.test.tsindex 0b2ee706da..f8291bef28 100644--- a/worker/src/__tests__/ingestionMasking.test.ts+++ b/worker/src/__tests__/ingestionMasking.test.ts@@ -25,6 +25,7 @@ vi.mock(return {...actual,applyIngestionMasking: vi.fn(actual.applyIngestionMasking),+ isIngestionMaskingEnabled: vi.fn(actual.isIngestionMaskingEnabled),};},);@@ -305,6 +306,9 @@ describe("Ingestion Masking", () => {}let prepared: Awaited<ReturnType<typeof prepareOtelBatch>> | undefined;try {+ vi.mocked(masking.isIngestionMaskingEnabled).mockReturnValueOnce(+ true,+ );prepared = await prepareOtelBatch({bytes: originalBytes,projectId: "test-project",@@ -364,6 +368,9 @@ describe("Ingestion Masking", () => {.mockResolvedValueOnce(new Response(new Uint8Array(responseBytes)));let prepared: Awaited<ReturnType<typeof prepareOtelBatch>> | undefined;try {+ vi.mocked(masking.isIngestionMaskingEnabled).mockReturnValueOnce(+ true,+ );prepared = await prepareOtelBatch({bytes: originalBytes,projectId: "test-project",diff --git a/worker/src/__tests__/nativeOtelMedia.unit.test.ts b/worker/src/__tests__/nativeOtelMedia.unit.test.tsindex 1d69c9a643..d9ca6a7275 100644--- a/worker/src/__tests__/nativeOtelMedia.unit.test.ts+++ b/worker/src/__tests__/nativeOtelMedia.unit.test.ts@@ -1,7 +1,7 @@import { createHash } from "node:crypto";import { describe, expect, it, vi } from "vitest";import fc from "fast-check";-import { validateOtelJson } from "@langfuse/native";+import { extractOtelMedia, validateOtelJson } from "@langfuse/native";import { MediaContentType } from "@langfuse/shared";import {mediaPayloadCases,@@ -103,30 +103,33 @@ async function expectMediaParity(json: string): Promise<void> {});const validated = await validateOtelJson(Buffer.from(json));+ const batches: Array<Awaited<ReturnType<typeof extractOtelMedia>>> = [];try {- const batch = await validated.extract(true);- try {- // Assert at the NAPI boundary. Running the late TS detector here could- // repair a missed extraction and let a broken native implementation pass.- const compact = (await batch.takeJsonBuffer()).toString("utf8");- expect(- canonicalizeNativeReferences(JSON.parse(compact), batch.media),- ).toEqual(payload.input);- if (batch.media.length === 0) expect(compact).toBe(json);+ batches.push(await validated.extract(true));+ batches.push(await extractOtelMedia(Buffer.from(json)));+ for (const batch of batches) {+ // Assert at the NAPI boundary, before late TS detection can repair a miss.+ const media = batch?.media ?? [];+ const compact = batch?.takeJsonBuffer().toString("utf8") ?? json;+ expect(canonicalizeNativeReferences(JSON.parse(compact), media)).toEqual(+ payload.input,+ );+ if (media.length === 0) expect(compact).toBe(json);const actualBodies: MediaBody[] = [];- for (const media of batch.media) {- const bytes = await batch.mediaBody(media.index);- expect(media.sha256Hash).toBe(+ for (const entry of media) {+ const bytes = await batch!.mediaBody(entry.index);+ expect(entry.sha256Hash).toBe(createHash("sha256").update(bytes).digest("base64"),);- actualBodies.push({ contentType: media.contentType, bytes });+ actualBodies.push({ contentType: entry.contentType, bytes });}expect(sortedBodies(actualBodies)).toEqual(sortedBodies(expectedBodies));- } finally {- await batch.dispose();}} finally {- await validated.dispose();+ await Promise.all([+ validated.dispose(),+ ...batches.map((batch) => batch?.dispose()),+ ]);}}@@ -137,6 +140,8 @@ describe(it.each(mediaPayloadCases)("$name", ({ json }) => expectMediaParity(json));it.each([+ '{"input":"\\u0064ata:image/png;base64,YWJj"}',+ '{"type":"file","media\\u0054ype":"image/png","data":"YWJj"}','{"type":"file","mediaType":"image/png","data":"YWJj","data":"ZGVm"}','{"type":"other","type":"file","mediaType":"text/plain","mediaType":"image/png","data":"ZGVm"}','{"inline_data":{"mime_type":"image/png","data":"YWJj"},"inline_data":{"mime_type":"image/png","data":"YWJj","data":"ZGVm"}}',@@ -195,8 +200,7 @@ describe(},],});- const validated = await validateOtelJson(Buffer.from(input));- const batch = await validated.extract(true);+ const batch = (await extractOtelMedia(Buffer.from(input)))!;try {expect(batch.media).toHaveLength(3);expect(batch.media.map((media) => media.contentType)).toEqual([@@ -247,7 +251,6 @@ describe(]);} finally {await batch.dispose();- await validated.dispose();}});diff --git a/worker/src/__tests__/otelReplay.integration.test.ts b/worker/src/__tests__/otelReplay.integration.test.tsindex e168785e48..7a51b08a1d 100644--- a/worker/src/__tests__/otelReplay.integration.test.ts+++ b/worker/src/__tests__/otelReplay.integration.test.ts@@ -1022,8 +1022,7 @@ describe(mediaUploadEnabled: true,});- expect(compactedBatches).toHaveLength(1);- expect(compactedBatches[0].references).toEqual([]);+ expect(compactedBatches).toEqual([]);expectRawOtelReplayParity(comparison);expect(comparison.originalTs.storedRows).toHaveLength(1);expect(comparison.originalTs.storedRows[0]?.tags).toEqual([diff --git a/worker/src/features/otel-ingestion/prepareOtelBatch.ts b/worker/src/features/otel-ingestion/prepareOtelBatch.tsindex 3de26b0988..fdc6cb957c 100644--- a/worker/src/features/otel-ingestion/prepareOtelBatch.ts+++ b/worker/src/features/otel-ingestion/prepareOtelBatch.ts@@ -1,4 +1,5 @@import {+ extractOtelMedia,validateOtelJson,type EarlyOtelBatch,type ValidatedOtelJson,@@ -8,11 +9,15 @@ import {recordIncrement,type ResourceSpan,} from "@langfuse/shared/src/server";-import { applyIngestionMasking } from "@langfuse/shared/src/server/ee/ingestionMasking";+import {+ applyIngestionMasking,+ isIngestionMaskingEnabled,+} from "@langfuse/shared/src/server/ee/ingestionMasking";type PreparedInput =| {- batch: EarlyOtelBatch;+ /** Absent when preprocessing is unnecessary; the caller must still parse JSON. */+ batch?: EarlyOtelBatch;/** Neither masking nor UTF-8 normalization replaced the caller's bytes. */sourceUnchanged: boolean;error?: never;@@ -20,9 +25,9 @@ type PreparedInput =| { batch?: never; sourceUnchanged?: never; error: string | undefined };/**- * Validate raw OTLP bytes and optionally extract media before the TypeScript- * ingestion graph is built. Masking receives the normalized byte view and may- * return a replacement body.+ * Extract media before the ingestion graph is built. Without masking, marker-free+ * input skips native validation; the consumer's JSON parser still validates it.+ * Masking always receives validated, normalized bytes.*/export async function prepareOtelBatch(params: {bytes: Buffer;@@ -33,10 +38,15 @@ export async function prepareOtelBatch(params: {}): Promise<PreparedInput> {const { bytes, extractMedia, ...context } = params;const startedAt = performance.now();- let outcome: "native" | "masking_drop" | "error" = "error";+ let outcome: "native" | "unchanged" | "masking_drop" | "error" = "error";let originalInput: ValidatedOtelJson | undefined;let maskedInput: ValidatedOtelJson | undefined;try {+ if (!isIngestionMaskingEnabled()) {+ const batch = extractMedia ? await extractOtelMedia(bytes) : null;+ outcome = batch ? "native" : "unchanged";+ return { batch: batch ?? undefined, sourceUnchanged: !batch };+ }originalInput = await validateOtelJson(bytes);const masking = await applyIngestionMasking({@@ -125,16 +135,16 @@ export async function prepareOtelBatch(params: {/** Parse for TypeScript consumers. The caller owns the returned batch; failures release it. */export async function parsePreparedOtelBatch(- prepared: { batch: EarlyOtelBatch; sourceUnchanged: boolean },+ prepared: { batch?: EarlyOtelBatch; sourceUnchanged: boolean },downloadedBytes: Uint8Array,): Promise<{ resourceSpans: ResourceSpan[]; batch?: EarlyOtelBatch }> {let batch: EarlyOtelBatch | undefined = prepared.batch;try {let json: Buffer;- if (prepared.sourceUnchanged && batch.mediaCount() === 0) {+ if (!batch || (prepared.sourceUnchanged && batch.mediaCount() === 0)) {// Free Rust's duplicate before parsing the unchanged download. An external// Buffer would retain that allocation until Node collects it.- await batch.dispose();+ await batch?.dispose();batch = undefined;json = Buffer.from(downloadedBytes.buffer,diff --git a/worker/src/features/otel-ingestion/prepareOtelBatch.unit.test.ts b/worker/src/features/otel-ingestion/prepareOtelBatch.unit.test.tsindex a728d9d5ed..09c7f34802 100644--- a/worker/src/features/otel-ingestion/prepareOtelBatch.unit.test.ts+++ b/worker/src/features/otel-ingestion/prepareOtelBatch.unit.test.ts@@ -2,15 +2,22 @@ import { beforeEach, describe, expect, it, vi } from "vitest";const nativeMocks = vi.hoisted(() => ({validateOtelJson: vi.fn(),+ extractOtelMedia: vi.fn(),}));const maskingMocks = vi.hoisted(() => ({applyIngestionMasking: vi.fn(),+ isIngestionMaskingEnabled: vi.fn(),}));vi.mock("@langfuse/native", async (importOriginal) => {const actual = await importOriginal<typeof import("@langfuse/native")>();nativeMocks.validateOtelJson.mockImplementation(actual.validateOtelJson);- return { ...actual, validateOtelJson: nativeMocks.validateOtelJson };+ nativeMocks.extractOtelMedia.mockImplementation(actual.extractOtelMedia);+ return {+ ...actual,+ validateOtelJson: nativeMocks.validateOtelJson,+ extractOtelMedia: nativeMocks.extractOtelMedia,+ };});vi.mock("@langfuse/shared/src/server", () => ({@@ -21,6 +28,7 @@ vi.mock("@langfuse/shared/src/server", () => ({vi.mock("@langfuse/shared/src/server/ee/ingestionMasking", () => ({applyIngestionMasking: maskingMocks.applyIngestionMasking,+ isIngestionMaskingEnabled: maskingMocks.isIngestionMaskingEnabled,}));import { parsePreparedOtelBatch, prepareOtelBatch } from "./prepareOtelBatch";@@ -28,6 +36,8 @@ import { parsePreparedOtelBatch, prepareOtelBatch } from "./prepareOtelBatch";describe("prepareOtelBatch native input boundary", { retry: 0 }, () => {beforeEach(() => {nativeMocks.validateOtelJson.mockClear();+ nativeMocks.extractOtelMedia.mockClear();+ maskingMocks.isIngestionMaskingEnabled.mockReturnValue(true);maskingMocks.applyIngestionMasking.mockImplementation(async (params: { data: unknown }) => ({success: true,@@ -37,9 +47,12 @@ describe("prepareOtelBatch native input boundary", { retry: 0 }, () => {);});- it.each(["unchanged", "masked", "sanitized"] as const)(+ it.each(["unchanged", "masked", "sanitized", "disabled"] as const)("allows download reuse only for unchanged bytes (%s)",async (inputKind) => {+ const disabled = inputKind === "disabled";+ const sourceUnchanged = inputKind === "unchanged" || disabled;+ maskingMocks.isIngestionMaskingEnabled.mockReturnValue(!disabled);const native =await vi.importActual<typeof import("@langfuse/native")>("@langfuse/native",@@ -65,16 +78,15 @@ describe("prepareOtelBatch native input boundary", { retry: 0 }, () => {const prepared = await prepareOtelBatch({bytes,projectId: "project-id",- extractMedia: true,+ extractMedia: !disabled,});try {- if (!prepared.batch) throw new Error("expected native preparation");- expect(prepared.batch.mediaCount()).toBe(0);- expect(prepared.sourceUnchanged).toBe(inputKind === "unchanged");+ if ("error" in prepared) throw new Error("unexpected masking failure");+ expect(prepared.batch?.mediaCount()).toBe(disabled ? undefined : 0);+ expect(nativeMocks.extractOtelMedia).not.toHaveBeenCalled();+ expect(prepared.sourceUnchanged).toBe(sourceUnchanged);const parsed = await parsePreparedOtelBatch(prepared, bytes);- expect(parsed.batch).toBe(- inputKind === "unchanged" ? undefined : prepared.batch,- );+ expect(parsed.batch).toBe(sourceUnchanged ? undefined : prepared.batch);expect(parsed.resourceSpans).toEqual({note:inputKind === "sanitized"@@ -89,6 +101,21 @@ describe("prepareOtelBatch native input boundary", { retry: 0 }, () => {},);+ it("rejects malformed JSON after marker-free preparation", async () => {+ maskingMocks.isIngestionMaskingEnabled.mockReturnValue(false);+ const bytes = Buffer.from('{"note":"ordinary"} trailing');+ const prepared = await prepareOtelBatch({+ bytes,+ projectId: "project-id",+ extractMedia: true,+ });+ if ("error" in prepared) throw new Error("unexpected masking failure");+ expect(prepared.batch).toBeUndefined();+ await expect(parsePreparedOtelBatch(prepared, bytes)).rejects.toThrow(+ SyntaxError,+ );+ });+it("does not fall back to a second TypeScript processing path", async () => {const error = Object.assign(new Error("native implementation changed its explanation"),diff --git a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts b/worker/src/queues/__tests__/otelDirectEventWrite.test.tsindex 1f5eef2a5b..4197b84787 100644--- a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts+++ b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts@@ -221,27 +221,30 @@ describe("early media extraction rollout", () => {vi.mocked(getS3EventStorageClient).mockReturnValue({downloadBytes: vi.fn().mockResolvedValue(bytes),} as unknown as ReturnType<typeof getS3EventStorageClient>);- const prepare = otelPreparation.prepareOtelBatch;+ const native =+ await vi.importActual<typeof import("@langfuse/native")>(+ "@langfuse/native",+ );+ const validated = await native.validateOtelJson(bytes);+ const nativeBatch = await validated.extract(false);const parse = JSON.parse;let checkedBeforeParse = false;const stop = new Error("stop after parsing");try {vi.spyOn(otelPreparation, "prepareOtelBatch").mockImplementation(- async (params) => {- const prepared = await prepare(params);- const batch = prepared.batch!;- expect(batch.mediaCount()).toBe(0);- vi.spyOn(batch, "takeJsonBuffer").mockImplementation(() => {+ async () => {+ expect(nativeBatch.mediaCount()).toBe(0);+ vi.spyOn(nativeBatch, "takeJsonBuffer").mockImplementation(() => {throw new Error("must not transfer a second document to Node");});vi.spyOn(JSON, "parse").mockImplementation((text, reviver) => {if (text === json) {- expect(() => batch.mediaCount()).toThrow(/disposed/);+ expect(() => nativeBatch.mediaCount()).toThrow(/disposed/);checkedBeforeParse = true;}return parse(text, reviver);});- return prepared;+ return { batch: nativeBatch, sourceUnchanged: true };},);vi.spyOn(diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.tsindex 9b6f1e4e07..5b610ca48e 100644--- a/worker/src/queues/otelIngestionQueue.ts+++ b/worker/src/queues/otelIngestionQueue.ts@@ -531,7 +531,7 @@ export const otelIngestionQueueProcessorBuilder = (env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED === "true" &&Boolean(env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET),});- if (!prepared.batch) {+ if ("error" in prepared) {logger.warn("Dropping OTEL event due to masking failure", {projectId,orgId: job.data.payload.authCheck.scope.orgId,