Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat: expose OTEL media workload mix and throughput metrics (#18750)

Valery Meleshkin42bd7285b59a
diff --git a/worker/src/features/otel-ingestion/processOtelEvents.ts b/worker/src/features/otel-ingestion/processOtelEvents.ts
index 7ea9fb7365..af4e6acaea 100644
--- a/worker/src/features/otel-ingestion/processOtelEvents.ts
+++ b/worker/src/features/otel-ingestion/processOtelEvents.ts
@@ -29,6 +29,7 @@ type ProcessOtelEventsParams = {
shouldWriteToEventsTable: boolean;
earlyBatch?: EarlyOtelBatch;
mediaPath?: "early" | "reference";
+ inputByteLength?: number;
};
/**
@@ -49,6 +50,7 @@ export async function processOtelEvents({
shouldWriteToEventsTable,
earlyBatch,
mediaPath,
+ inputByteLength,
}: ProcessOtelEventsParams): Promise<void> {
// Process events for observation evals and direct event writes
// This phase handles two independent concerns:
@@ -97,6 +99,7 @@ export async function processOtelEvents({
writePath: "direct",
earlyBatch,
mediaPath,
+ inputByteLength,
projectId,
fileKey,
mediaBucket: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,
diff --git a/worker/src/features/otel-media/processOtelMedia.test.ts b/worker/src/features/otel-media/processOtelMedia.test.ts
index e397816ecc..b6e14fc7b3 100644
--- a/worker/src/features/otel-media/processOtelMedia.test.ts
+++ b/worker/src/features/otel-media/processOtelMedia.test.ts
@@ -287,7 +287,11 @@ describe("processOtelEventMedia", () => {
expect(mocks.recordDistribution).toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.processing_duration_ms",
expect.any(Number),
- { write_path: "direct", media_path: "reference" },
+ {
+ write_path: "direct",
+ media_path: "reference",
+ media_presence: "present",
+ },
);
});
@@ -330,7 +334,11 @@ describe("processOtelEventMedia", () => {
expect(mocks.recordDistribution).toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.processing_duration_ms",
expect.any(Number),
- { write_path: "direct", media_path: "reference" },
+ {
+ write_path: "direct",
+ media_path: "reference",
+ media_presence: "unknown",
+ },
);
expect(mocks.recordDistribution).not.toHaveBeenCalledWith(
"langfuse.ingestion.otel.media.batch_byte_length",
diff --git a/worker/src/features/otel-media/processOtelMedia.ts b/worker/src/features/otel-media/processOtelMedia.ts
index 5f12e9aa75..95de0c69c2 100644
--- a/worker/src/features/otel-media/processOtelMedia.ts
+++ b/worker/src/features/otel-media/processOtelMedia.ts
@@ -9,6 +9,7 @@ import {
logger,
processOtelMedia,
recordDistribution,
+ recordIncrement,
type IngestionEventType,
type OtelMediaPayload,
type OtelMediaTarget,
@@ -50,6 +51,7 @@ export async function processOtelEventMedia(params: {
fileKey: string;
mediaBucket?: string;
mediaPrefix: string;
+ inputByteLength?: number;
processMedia?: typeof processOtelMedia;
}): Promise<void> {
const {
@@ -79,6 +81,7 @@ export async function processOtelEventMedia(params: {
{ name: "langfuse.ingestion.otel.media.process" },
async (span) => {
const startedAt = Date.now();
+ let mediaPresence: "present" | "absent" | "unknown" = "unknown";
span.setAttributes({
"langfuse.ingestion.otel.media_path": mediaPath,
});
@@ -115,6 +118,9 @@ export async function processOtelEventMedia(params: {
result.bytesProcessed += early.bytesProcessed;
result.bytesRemoved += early.bytesRemoved;
}
+ // Classify the same completed stage that contributes a duration sample.
+ mediaPresence =
+ result.candidates > 0 || result.failed > 0 ? "present" : "absent";
span.setAttributes({
"langfuse.ingestion.otel.media.uploaded": result.uploaded,
@@ -155,11 +161,13 @@ export async function processOtelEventMedia(params: {
{ write_path: writePath, media_path: mediaPath },
);
} finally {
- recordDistribution(
- "langfuse.ingestion.otel.media.processing_duration_ms",
- Date.now() - startedAt,
- { write_path: writePath, media_path: mediaPath },
- );
+ recordMediaStageMetrics({
+ durationMs: Date.now() - startedAt,
+ writePath,
+ mediaPath,
+ mediaPresence,
+ inputByteLength: params.inputByteLength,
+ });
}
},
);
@@ -181,6 +189,39 @@ export async function processOtelEventMedia(params: {
}
}
+function recordMediaStageMetrics(params: {
+ durationMs: number;
+ writePath: OtelMediaWritePath;
+ mediaPath: "early" | "reference";
+ mediaPresence: "present" | "absent" | "unknown";
+ inputByteLength?: number;
+}): void {
+ const tags = {
+ write_path: params.writePath,
+ media_path: params.mediaPath,
+ media_presence: params.mediaPresence,
+ };
+ recordDistribution(
+ "langfuse.ingestion.otel.media.processing_duration_ms",
+ params.durationMs,
+ tags,
+ );
+ recordIncrement("langfuse.ingestion.otel.media.processing_sample", 1, tags);
+ recordIncrement(
+ "langfuse.ingestion.otel.media.processing_duration_total_ms",
+ params.durationMs,
+ tags,
+ );
+ if (params.inputByteLength && params.inputByteLength > 0) {
+ // Dual writes contribute both their input bytes and stage time.
+ recordIncrement(
+ "langfuse.ingestion.otel.media.input_bytes_total",
+ params.inputByteLength,
+ tags,
+ );
+ }
+}
+
/**
* Creates media targets for direct events-table inputs without cloning their
* potentially large input, output, or metadata values.
diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.ts
index 5b610ca48e..c82ebb9959 100644
--- a/worker/src/queues/otelIngestionQueue.ts
+++ b/worker/src/queues/otelIngestionQueue.ts
@@ -515,11 +515,13 @@ export const otelIngestionQueueProcessorBuilder = (
env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
);
let parsedSpans: ResourceSpan[];
+ let inputByteLength: number;
if (earlyMediaEnabled) {
const bytes = await eventStorage.downloadBytes(fileKey);
+ inputByteLength = bytes.byteLength;
recordHistogram(
"langfuse.ingestion.s3_file_size_bytes",
- bytes.byteLength,
+ inputByteLength,
{ skippedS3List: "true", otel: "true" },
);
const prepared = await prepareOtelBatch({
@@ -546,10 +548,11 @@ export const otelIngestionQueueProcessorBuilder = (
} else {
// Download file from blob storage
const resourceSpans = await eventStorage.download(fileKey);
+ inputByteLength = Buffer.byteLength(resourceSpans, "utf8");
recordHistogram(
"langfuse.ingestion.s3_file_size_bytes",
- resourceSpans.length, // At this point it's still a string.
+ inputByteLength,
{
skippedS3List: "true",
otel: "true",
@@ -791,6 +794,7 @@ export const otelIngestionQueueProcessorBuilder = (
writePath: "legacy",
earlyBatch,
mediaPath,
+ inputByteLength,
projectId,
fileKey,
mediaBucket: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,
@@ -842,6 +846,7 @@ export const otelIngestionQueueProcessorBuilder = (
shouldWriteToEventsTable,
earlyBatch,
mediaPath,
+ inputByteLength,
});
} catch (e) {
const fileKey = job.data.payload.data.fileKey;