🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat: expose OTEL media workload mix and throughput metrics (#18750)
Valery Meleshkin42bd7285b59a
diff --git a/worker/src/features/otel-ingestion/processOtelEvents.ts b/worker/src/features/otel-ingestion/processOtelEvents.tsindex 7ea9fb7365..af4e6acaea 100644--- a/worker/src/features/otel-ingestion/processOtelEvents.ts+++ b/worker/src/features/otel-ingestion/processOtelEvents.ts@@ -29,6 +29,7 @@ type ProcessOtelEventsParams = {shouldWriteToEventsTable: boolean;earlyBatch?: EarlyOtelBatch;mediaPath?: "early" | "reference";+ inputByteLength?: number;};/**@@ -49,6 +50,7 @@ export async function processOtelEvents({shouldWriteToEventsTable,earlyBatch,mediaPath,+ inputByteLength,}: ProcessOtelEventsParams): Promise<void> {// Process events for observation evals and direct event writes// This phase handles two independent concerns:@@ -97,6 +99,7 @@ export async function processOtelEvents({writePath: "direct",earlyBatch,mediaPath,+ inputByteLength,projectId,fileKey,mediaBucket: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,diff --git a/worker/src/features/otel-media/processOtelMedia.test.ts b/worker/src/features/otel-media/processOtelMedia.test.tsindex e397816ecc..b6e14fc7b3 100644--- a/worker/src/features/otel-media/processOtelMedia.test.ts+++ b/worker/src/features/otel-media/processOtelMedia.test.ts@@ -287,7 +287,11 @@ describe("processOtelEventMedia", () => {expect(mocks.recordDistribution).toHaveBeenCalledWith("langfuse.ingestion.otel.media.processing_duration_ms",expect.any(Number),- { write_path: "direct", media_path: "reference" },+ {+ write_path: "direct",+ media_path: "reference",+ media_presence: "present",+ },);});@@ -330,7 +334,11 @@ describe("processOtelEventMedia", () => {expect(mocks.recordDistribution).toHaveBeenCalledWith("langfuse.ingestion.otel.media.processing_duration_ms",expect.any(Number),- { write_path: "direct", media_path: "reference" },+ {+ write_path: "direct",+ media_path: "reference",+ media_presence: "unknown",+ },);expect(mocks.recordDistribution).not.toHaveBeenCalledWith("langfuse.ingestion.otel.media.batch_byte_length",diff --git a/worker/src/features/otel-media/processOtelMedia.ts b/worker/src/features/otel-media/processOtelMedia.tsindex 5f12e9aa75..95de0c69c2 100644--- a/worker/src/features/otel-media/processOtelMedia.ts+++ b/worker/src/features/otel-media/processOtelMedia.ts@@ -9,6 +9,7 @@ import {logger,processOtelMedia,recordDistribution,+ recordIncrement,type IngestionEventType,type OtelMediaPayload,type OtelMediaTarget,@@ -50,6 +51,7 @@ export async function processOtelEventMedia(params: {fileKey: string;mediaBucket?: string;mediaPrefix: string;+ inputByteLength?: number;processMedia?: typeof processOtelMedia;}): Promise<void> {const {@@ -79,6 +81,7 @@ export async function processOtelEventMedia(params: {{ name: "langfuse.ingestion.otel.media.process" },async (span) => {const startedAt = Date.now();+ let mediaPresence: "present" | "absent" | "unknown" = "unknown";span.setAttributes({"langfuse.ingestion.otel.media_path": mediaPath,});@@ -115,6 +118,9 @@ export async function processOtelEventMedia(params: {result.bytesProcessed += early.bytesProcessed;result.bytesRemoved += early.bytesRemoved;}+ // Classify the same completed stage that contributes a duration sample.+ mediaPresence =+ result.candidates > 0 || result.failed > 0 ? "present" : "absent";span.setAttributes({"langfuse.ingestion.otel.media.uploaded": result.uploaded,@@ -155,11 +161,13 @@ export async function processOtelEventMedia(params: {{ write_path: writePath, media_path: mediaPath },);} finally {- recordDistribution(- "langfuse.ingestion.otel.media.processing_duration_ms",- Date.now() - startedAt,- { write_path: writePath, media_path: mediaPath },- );+ recordMediaStageMetrics({+ durationMs: Date.now() - startedAt,+ writePath,+ mediaPath,+ mediaPresence,+ inputByteLength: params.inputByteLength,+ });}},);@@ -181,6 +189,39 @@ export async function processOtelEventMedia(params: {}}+function recordMediaStageMetrics(params: {+ durationMs: number;+ writePath: OtelMediaWritePath;+ mediaPath: "early" | "reference";+ mediaPresence: "present" | "absent" | "unknown";+ inputByteLength?: number;+}): void {+ const tags = {+ write_path: params.writePath,+ media_path: params.mediaPath,+ media_presence: params.mediaPresence,+ };+ recordDistribution(+ "langfuse.ingestion.otel.media.processing_duration_ms",+ params.durationMs,+ tags,+ );+ recordIncrement("langfuse.ingestion.otel.media.processing_sample", 1, tags);+ recordIncrement(+ "langfuse.ingestion.otel.media.processing_duration_total_ms",+ params.durationMs,+ tags,+ );+ if (params.inputByteLength && params.inputByteLength > 0) {+ // Dual writes contribute both their input bytes and stage time.+ recordIncrement(+ "langfuse.ingestion.otel.media.input_bytes_total",+ params.inputByteLength,+ tags,+ );+ }+}+/*** Creates media targets for direct events-table inputs without cloning their* potentially large input, output, or metadata values.diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.tsindex 5b610ca48e..c82ebb9959 100644--- a/worker/src/queues/otelIngestionQueue.ts+++ b/worker/src/queues/otelIngestionQueue.ts@@ -515,11 +515,13 @@ export const otelIngestionQueueProcessorBuilder = (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,);let parsedSpans: ResourceSpan[];+ let inputByteLength: number;if (earlyMediaEnabled) {const bytes = await eventStorage.downloadBytes(fileKey);+ inputByteLength = bytes.byteLength;recordHistogram("langfuse.ingestion.s3_file_size_bytes",- bytes.byteLength,+ inputByteLength,{ skippedS3List: "true", otel: "true" },);const prepared = await prepareOtelBatch({@@ -546,10 +548,11 @@ export const otelIngestionQueueProcessorBuilder = (} else {// Download file from blob storageconst resourceSpans = await eventStorage.download(fileKey);+ inputByteLength = Buffer.byteLength(resourceSpans, "utf8");recordHistogram("langfuse.ingestion.s3_file_size_bytes",- resourceSpans.length, // At this point it's still a string.+ inputByteLength,{skippedS3List: "true",otel: "true",@@ -791,6 +794,7 @@ export const otelIngestionQueueProcessorBuilder = (writePath: "legacy",earlyBatch,mediaPath,+ inputByteLength,projectId,fileKey,mediaBucket: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,@@ -842,6 +846,7 @@ export const otelIngestionQueueProcessorBuilder = (shouldWriteToEventsTable,earlyBatch,mediaPath,+ inputByteLength,});} catch (e) {const fileKey = job.data.payload.data.fileKey;