πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(otel): extract ingestion processing (#16954)
Valery Meleshkin5ca735f29627
diff --git a/web/src/__tests__/server/unit/process-otel-ingestion.servertest.ts b/web/src/__tests__/server/unit/process-otel-ingestion.servertest.tsnew file mode 100644index 0000000000..6ee7141a1d--- /dev/null+++ b/web/src/__tests__/server/unit/process-otel-ingestion.servertest.ts@@ -0,0 +1,91 @@+import { describe, expect, it } from "vitest";++import {+ processOtelIngestion,+ type OtelIngestionRequest,+} from "@/src/server/otel/processOtelIngestion";++const config = {+ projectId: "project-id",+ publicKey: "public-key",+ sdkName: "test-sdk",+ sdkVersion: "1.0.0",+};++function processRequest(+ request: Omit<OtelIngestionRequest, "encodedBodyBytes">,+) {+ return processOtelIngestion({+ ...request,+ encodedBodyBytes: request.body.byteLength,+ });+}++describe("processOtelIngestion", () => {+ it("preserves the invalid content type response", async () => {+ await expect(+ processRequest({+ body: Buffer.from("{}"),+ contentType: "text/plain",+ config,+ }),+ ).resolves.toEqual({+ kind: "http",+ status: 400,+ body: { error: "Invalid content type" },+ });+ });++ it("preserves the malformed JSON response", async () => {+ await expect(+ processRequest({+ body: Buffer.from("{"),+ contentType: "application/json",+ config,+ }),+ ).resolves.toEqual({+ kind: "http",+ status: 400,+ body: { error: "Failed to parse OTel JSON Trace" },+ });+ });++ it("rejects spans that cannot be processed", async () => {+ await expect(+ processRequest({+ body: Buffer.from(+ JSON.stringify({+ resourceSpans: [+ {+ scopeSpans: [+ {+ scope: { name: "test-scope" },+ spans: [{ name: "missing-ids" }],+ },+ ],+ },+ ],+ }),+ ),+ contentType: "application/json",+ config,+ }),+ ).resolves.toMatchObject({+ kind: "http",+ status: 400,+ body: {+ error: expect.stringContaining("Invalid OTLP trace export"),+ },+ });+ });++ it("skips an empty export", async () => {+ await expect(+ processRequest({+ body: Buffer.from('{"resourceSpans":[]}'),+ contentType: "application/json",+ config,+ }),+ ).resolves.toEqual({ kind: "ok" });+ });+});diff --git a/web/src/pages/api/public/otel/v1/traces/index.ts b/web/src/pages/api/public/otel/v1/traces/index.tsindex 055d9bc419..be98cc8b55 100644--- a/web/src/pages/api/public/otel/v1/traces/index.ts+++ b/web/src/pages/api/public/otel/v1/traces/index.ts@@ -1,18 +1,12 @@import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";import { createAuthedProjectAPIRoute } from "@/src/features/public-api/server/createAuthedProjectAPIRoute";import {- getCurrentSpan,logger,- markProjectIngestFailure,- OtelIngestionProcessor,markProjectAsOtelUser,createIngestionAttribution,getLangfuseHeaderValue,- recordIncrement,- validateOtelSpanIds,} from "@langfuse/shared/src/server";import { z } from "zod";-import { $root } from "@/src/pages/api/public/otel/otlp-proto/generated/root";import { ForbiddenError } from "@langfuse/shared";import { env } from "@/src/env.mjs";import {@@ -21,6 +15,7 @@ import {OtelRequestBodyTooLargeError,readOtelRequestBody,} from "@/src/server/otel/otelRequestBody";+import { processOtelIngestion } from "@/src/server/otel/processOtelIngestion";export const config = {api: {@@ -28,8 +23,6 @@ export const config = {},};-const OTEL_REQUEST_BODY_WARNING_BYTES = 16 * 1024 * 1024;-export default withMiddlewares({POST: createAuthedProjectAPIRoute({name: "OTel Traces",@@ -75,199 +68,7 @@ export default withMiddlewares({return { error: bodyFailureMessage };}- let resourceSpans: any;const contentType = req.headers["content-type"]?.toLowerCase();- // Strict content-type matching does not work if something like `content-type: text/javascript; charset=utf-8` is sent.- if (- !contentType ||- (!contentType.includes("application/json") &&- !contentType.includes("application/x-protobuf"))- ) {- logger.error(`Invalid content type: ${contentType}`);- res.status(400);- return { error: "Invalid content type" };- }- if (contentType.includes("application/x-protobuf")) {- try {- const parsed =- $root.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest.decode(- body,- );- resourceSpans =- $root.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest.toObject(- parsed,- // OTLP JSON encodes int64 fields as decimal strings.- { longs: String },- ).resourceSpans;- } catch (e) {- logger.error(`Failed to parse OTel Protobuf`, e);- res.status(400);- return { error: "Failed to parse OTel Protobuf Trace" };- }- }- if (contentType.includes("application/json")) {- try {- resourceSpans = JSON.parse(body.toString()).resourceSpans;- } catch (e) {- logger.error(`Failed to parse OTel JSON`, e);- res.status(400);- return { error: "Failed to parse OTel JSON Trace" };- }- }-- if (!resourceSpans || resourceSpans.length === 0) {- return {};- }-- // Reject payloads the worker cannot convert: spans whose traceId/spanId is- // missing or malformed, and scopeSpans/spans/attributes/events fields that- // are present but not arrays. All fail in the worker β parseId throws- // ERR_INVALID_ARG_TYPE or yields a truncated id, and a non-array- // collection throws on for...of, reduce or filter β so the job burns all- // six attempts before being dropped anyway. Failing here returns a- // non-retryable status instead.- //- // The common source is an OTLP *logs* export pointed at this endpoint.- // ResourceLogs and ResourceSpans share protobuf field numbers, so log- // records decode into spans with a valid instrumentation scope but no ids.- // The other observed source is a hand-rolled OTLP/JSON exporter writing- // attributes as a {key: value} object rather than a KeyValue list.- //- // One invalid span means the whole export is almost certainly malformed,- // so the batch is rejected as a unit and the response says so explicitly- // rather than dropping spans silently.- const idValidation = validateOtelSpanIds(resourceSpans);- if (- idValidation.invalidSpanCount > 0 ||- idValidation.malformedCollectionCount > 0- ) {- // One increment per reason, so a batch that mixes reasons splits across- // them instead of attributing every rejected span to whichever reason- // happened to come first. The tag set is bounded by construction.- for (const [reason, count] of Object.entries(- idValidation.reasonCounts,- )) {- recordIncrement(- "langfuse.ingestion.otel.rejected_invalid_span_ids",- count,- { reason },- );- }- const rejectionContext = {- projectId: auth.scope.projectId,- invalidSpanCount: idValidation.invalidSpanCount,- malformedCollectionCount: idValidation.malformedCollectionCount,- totalSpanCount: idValidation.totalSpanCount,- reasonCounts: idValidation.reasonCounts,- instrumentationScopes: idValidation.scopeNames,- sdkName: req.headers["x-langfuse-sdk-name"],- };- logger.warn(- "Rejecting unprocessable OTEL trace export",- rejectionContext,- );-- // Rejecting the batch as a unit rests on the assumption that a bad- // export is bad throughout: an OTLP logs payload decoded as spans- // yields uniformly id-less spans, so invalidSpanCount should equal- // totalSpanCount. Any gap means this rejection just discarded spans- // that would have ingested fine, which is data loss and invalidates- // the assumption β escalate so it cannot pass unnoticed among warns.- //- // Only meaningful when every defect was span-attributable. A malformed- // resource- or scope-level collection fails its whole subtree in the- // worker no matter how sound the individual spans underneath are, so- // counting those spans as discarded-but-valid would be a false alarm.- const discardedValidSpans =- idValidation.totalSpanCount - idValidation.invalidSpanCount;- if (- discardedValidSpans > 0 &&- idValidation.malformedCollectionCount === 0- ) {- logger.error(- "Rejected OTEL trace export contained valid spans β batch rejection discarded them",- { ...rejectionContext, discardedValidSpans },- );- }-- const problems: string[] = [];- if (idValidation.invalidSpanCount > 0) {- problems.push(- `${idValidation.invalidSpanCount} of ${idValidation.totalSpanCount} ` +- `span(s) cannot be processed`,- );- }- if (idValidation.malformedCollectionCount > 0) {- problems.push(- `${idValidation.malformedCollectionCount} resource- or scope-level ` +- `field(s) are present but not arrays`,- );- }-- // Name the contract that was broken, so a hand-rolled exporter can be- // corrected from the response alone rather than by guessing at the- // reason keys.- const hints: string[] = [];- if (- idValidation.reasons.some(- (reason) =>- reason.endsWith(":absent") || reason.endsWith(":not_an_id"),- )- ) {- hints.push(- `traceId and spanId are required on every span, and each must be a ` +- `string or a byte array.`,- );- }- if (- idValidation.reasons.some((reason) =>- reason.endsWith(":not_an_array"),- )- ) {- hints.push(- `scopeSpans, spans, attributes and events must all be arrays β OTLP ` +- `attributes are a list of {key, value} entries, not a JSON object.`,- );- }-- res.status(400);- return {- error:- `Invalid OTLP trace export: ${problems.join("; ")} ` +- `(${idValidation.reasons.join(", ")}). ${hints.join(" ")} ` +- `The entire export was rejected and no spans were ingested. ` +- `Instrumentation scopes: ${idValidation.scopeNames.join(", ") || "unknown"}. ` +- `This endpoint accepts OpenTelemetry traces only β if you are exporting ` +- `OpenTelemetry logs, point your log exporter elsewhere.`,- };- }-- // Warn on oversized OTEL request bodies (16MB threshold). Keep one- // warning per accepted request, even when both encoded and decoded sizes- // cross the threshold.- // Keep the encoded size available after optional decompression so the- // warning can identify requests that would cross an encoded-only- // threshold as well as requests that are large after decompression.- const decodedBodyBytes = body.byteLength;- if (- encodedBodyBytes > OTEL_REQUEST_BODY_WARNING_BYTES ||- decodedBodyBytes > OTEL_REQUEST_BODY_WARNING_BYTES- ) {- let spanCount = 0;- for (const rs of resourceSpans) {- for (const ss of rs?.scopeSpans ?? []) {- spanCount += ss?.spans?.length ?? 0;- }- }- logger.warn("OTEL request body exceeds 16MB", {- projectId: auth.scope.projectId,- // Keep bodyBytes as the decoded-size field for existing queries.- bodyBytes: decodedBodyBytes,- decodedBodyBytes,- encodedBodyBytes,- spanCount,- });- }// Extract SDK headers for write path decision (supports both hyphen and underscore formats)const attribution = createIngestionAttribution({@@ -278,27 +79,6 @@ export default withMiddlewares({req.headers,"x-langfuse-ingestion-version",);- if (ingestionVersion) {- getCurrentSpan()?.setAttribute(- "langfuse.ingestion.version",- ingestionVersion,- );- }-- // Reject unsupported future ingestion versions (> 4)- // Lower versions are valid but use dual write (path A)- const parsedIngestionVersion = ingestionVersion- ? parseInt(ingestionVersion, 10)- : undefined;- if (- parsedIngestionVersion !== undefined &&- (isNaN(parsedIngestionVersion) || parsedIngestionVersion > 4)- ) {- res.status(400);- return {- error: `Unsupported x-langfuse-ingestion-version: "${ingestionVersion}". Maximum supported: "4".`,- };- }// Extract headers to propagate for ingestion maskingconst propagatedHeaderNames =@@ -311,30 +91,30 @@ export default withMiddlewares({}}- const processor = new OtelIngestionProcessor({- projectId: auth.scope.projectId,- publicKey: auth.scope.publicKey,- orgId: auth.scope.orgId,- propagatedHeaders:- Object.keys(propagatedHeaders).length > 0- ? propagatedHeaders- : undefined,- sdkName: attribution.ingestionSdkName,- sdkVersion: attribution.ingestionSdkVersion,- ingestionVersion,+ const result = await processOtelIngestion({+ body,+ contentType,+ encodedBodyBytes,+ config: {+ projectId: auth.scope.projectId,+ publicKey: auth.scope.publicKey,+ orgId: auth.scope.orgId,+ propagatedHeaders:+ Object.keys(propagatedHeaders).length > 0+ ? propagatedHeaders+ : undefined,+ sdkName: attribution.ingestionSdkName,+ sdkVersion: attribution.ingestionSdkVersion,+ rejectionSdkName: req.headers["x-langfuse-sdk-name"],+ ingestionVersion,+ },});-- // At this point, we have the raw OpenTelemetry Span body. We upload the full batch to S3- // and the OtelIngestionProcessor logic will handle processing in the worker container.- try {- return await processor.publishToOtelIngestionQueue(resourceSpans);- } catch (error) {- markProjectIngestFailure(auth.scope.projectId, {- source: "public_otel_api",- reason: "publish_failed",- });- throw error;+ if (result.kind === "http") {+ res.status(result.status);+ return result.body;}++ return result.body ?? {};},}),});diff --git a/web/src/server/otel/processOtelIngestion.ts b/web/src/server/otel/processOtelIngestion.tsnew file mode 100644index 0000000000..b97dccad45--- /dev/null+++ b/web/src/server/otel/processOtelIngestion.ts@@ -0,0 +1,269 @@+import { $root } from "@/src/pages/api/public/otel/otlp-proto/generated/root";+import {+ getCurrentSpan,+ logger,+ markProjectIngestFailure,+ OtelIngestionProcessor,+ recordIncrement,+ validateOtelSpanIds,+} from "@langfuse/shared/src/server";++type OtelIngestionConfig = {+ projectId: string;+ publicKey: string;+ orgId?: string;+ propagatedHeaders?: Record<string, string>;+ sdkName: string;+ sdkVersion: string;+ rejectionSdkName?: string | string[];+ ingestionVersion?: string;+};++export type OtelIngestionRequest = {+ body: Buffer;+ contentType?: string;+ encodedBodyBytes: number;+ config: OtelIngestionConfig;+};++export type OtelIngestionResult =+ | {+ kind: "ok";+ body?: unknown;+ }+ | {+ kind: "http";+ status: number;+ body: unknown;+ };++const OTEL_REQUEST_BODY_WARNING_BYTES = 16 * 1024 * 1024;++function httpResult(status: number, body: unknown): OtelIngestionResult {+ return { kind: "http", status, body };+}++export async function processOtelIngestion(+ request: OtelIngestionRequest,+): Promise<OtelIngestionResult> {+ const { body, config, encodedBodyBytes } = request;+ let resourceSpans: any;+ const contentType = request.contentType?.toLowerCase();++ // Strict content-type matching does not work if something like `content-type: text/javascript; charset=utf-8` is sent.+ if (+ !contentType ||+ (!contentType.includes("application/json") &&+ !contentType.includes("application/x-protobuf"))+ ) {+ logger.error(`Invalid content type: ${contentType}`);+ return httpResult(400, { error: "Invalid content type" });+ }+ if (contentType.includes("application/x-protobuf")) {+ try {+ const parsed =+ $root.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest.decode(+ body,+ );+ resourceSpans =+ $root.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest.toObject(+ parsed,+ // OTLP JSON encodes int64 fields as decimal strings.+ { longs: String },+ ).resourceSpans;+ } catch (e) {+ logger.error(`Failed to parse OTel Protobuf`, e);+ return httpResult(400, { error: "Failed to parse OTel Protobuf Trace" });+ }+ }+ if (contentType.includes("application/json")) {+ try {+ resourceSpans = JSON.parse(body.toString()).resourceSpans;+ } catch (e) {+ logger.error(`Failed to parse OTel JSON`, e);+ return httpResult(400, { error: "Failed to parse OTel JSON Trace" });+ }+ }++ if (!resourceSpans || resourceSpans.length === 0) {+ return { kind: "ok" };+ }++ // Reject payloads the worker cannot convert: spans whose traceId/spanId is+ // missing or malformed, and scopeSpans/spans/attributes/events fields that+ // are present but not arrays. All fail in the worker β parseId throws+ // ERR_INVALID_ARG_TYPE or yields a truncated id, and a non-array+ // collection throws on for...of, reduce or filter β so the job burns all+ // six attempts before being dropped anyway. Failing here returns a+ // non-retryable status instead.+ //+ // The common source is an OTLP *logs* export pointed at this endpoint.+ // ResourceLogs and ResourceSpans share protobuf field numbers, so log+ // records decode into spans with a valid instrumentation scope but no ids.+ // The other observed source is a hand-rolled OTLP/JSON exporter writing+ // attributes as a {key: value} object rather than a KeyValue list.+ //+ // One invalid span means the whole export is almost certainly malformed,+ // so the batch is rejected as a unit and the response says so explicitly+ // rather than dropping spans silently.+ const idValidation = validateOtelSpanIds(resourceSpans);+ if (+ idValidation.invalidSpanCount > 0 ||+ idValidation.malformedCollectionCount > 0+ ) {+ // One increment per reason, so a batch that mixes reasons splits across+ // them instead of attributing every rejected span to whichever reason+ // happened to come first. The tag set is bounded by construction.+ for (const [reason, count] of Object.entries(idValidation.reasonCounts)) {+ recordIncrement(+ "langfuse.ingestion.otel.rejected_invalid_span_ids",+ count,+ { reason },+ );+ }+ const rejectionContext = {+ projectId: config.projectId,+ invalidSpanCount: idValidation.invalidSpanCount,+ malformedCollectionCount: idValidation.malformedCollectionCount,+ totalSpanCount: idValidation.totalSpanCount,+ reasonCounts: idValidation.reasonCounts,+ instrumentationScopes: idValidation.scopeNames,+ sdkName: config.rejectionSdkName,+ };+ logger.warn("Rejecting unprocessable OTEL trace export", rejectionContext);++ // Rejecting the batch as a unit rests on the assumption that a bad+ // export is bad throughout: an OTLP logs payload decoded as spans+ // yields uniformly id-less spans, so invalidSpanCount should equal+ // totalSpanCount. Any gap means this rejection just discarded spans+ // that would have ingested fine, which is data loss and invalidates+ // the assumption β escalate so it cannot pass unnoticed among warns.+ //+ // Only meaningful when every defect was span-attributable. A malformed+ // resource- or scope-level collection fails its whole subtree in the+ // worker no matter how sound the individual spans underneath are, so+ // counting those spans as discarded-but-valid would be a false alarm.+ const discardedValidSpans =+ idValidation.totalSpanCount - idValidation.invalidSpanCount;+ if (+ discardedValidSpans > 0 &&+ idValidation.malformedCollectionCount === 0+ ) {+ logger.error(+ "Rejected OTEL trace export contained valid spans β batch rejection discarded them",+ { ...rejectionContext, discardedValidSpans },+ );+ }++ const problems: string[] = [];+ if (idValidation.invalidSpanCount > 0) {+ problems.push(+ `${idValidation.invalidSpanCount} of ${idValidation.totalSpanCount} ` ++ `span(s) cannot be processed`,+ );+ }+ if (idValidation.malformedCollectionCount > 0) {+ problems.push(+ `${idValidation.malformedCollectionCount} resource- or scope-level ` ++ `field(s) are present but not arrays`,+ );+ }++ // Name the contract that was broken, so a hand-rolled exporter can be+ // corrected from the response alone rather than by guessing at the+ // reason keys.+ const hints: string[] = [];+ if (+ idValidation.reasons.some(+ (reason) => reason.endsWith(":absent") || reason.endsWith(":not_an_id"),+ )+ ) {+ hints.push(+ `traceId and spanId are required on every span, and each must be a ` ++ `string or a byte array.`,+ );+ }+ if (+ idValidation.reasons.some((reason) => reason.endsWith(":not_an_array"))+ ) {+ hints.push(+ `scopeSpans, spans, attributes and events must all be arrays β OTLP ` ++ `attributes are a list of {key, value} entries, not a JSON object.`,+ );+ }++ return httpResult(400, {+ error:+ `Invalid OTLP trace export: ${problems.join("; ")} ` ++ `(${idValidation.reasons.join(", ")}). ${hints.join(" ")} ` ++ `The entire export was rejected and no spans were ingested. ` ++ `Instrumentation scopes: ${idValidation.scopeNames.join(", ") || "unknown"}. ` ++ `This endpoint accepts OpenTelemetry traces only β if you are exporting ` ++ `OpenTelemetry logs, point your log exporter elsewhere.`,+ });+ }++ // Warn on oversized OTEL request bodies (16MB threshold). Keep one+ // warning per accepted request, even when both encoded and decoded sizes+ // cross the threshold.+ // Keep the encoded size available after optional decompression so the+ // warning can identify requests that would cross an encoded-only+ // threshold as well as requests that are large after decompression.+ const decodedBodyBytes = body.byteLength;+ if (+ encodedBodyBytes > OTEL_REQUEST_BODY_WARNING_BYTES ||+ decodedBodyBytes > OTEL_REQUEST_BODY_WARNING_BYTES+ ) {+ let spanCount = 0;+ for (const rs of resourceSpans) {+ for (const ss of rs?.scopeSpans ?? []) {+ spanCount += ss?.spans?.length ?? 0;+ }+ }+ logger.warn("OTEL request body exceeds 16MB", {+ projectId: config.projectId,+ // Keep bodyBytes as the decoded-size field for existing queries.+ bodyBytes: decodedBodyBytes,+ decodedBodyBytes,+ encodedBodyBytes,+ spanCount,+ });+ }++ if (config.ingestionVersion) {+ getCurrentSpan()?.setAttribute(+ "langfuse.ingestion.version",+ config.ingestionVersion,+ );+ }++ // Reject unsupported future ingestion versions (> 4)+ // Lower versions are valid but use dual write (path A)+ const { ingestionVersion } = config;+ const parsedIngestionVersion = ingestionVersion+ ? parseInt(ingestionVersion, 10)+ : undefined;+ if (+ parsedIngestionVersion !== undefined &&+ (isNaN(parsedIngestionVersion) || parsedIngestionVersion > 4)+ ) {+ return httpResult(400, {+ error: `Unsupported x-langfuse-ingestion-version: "${ingestionVersion}". Maximum supported: "4".`,+ });+ }++ const processor = new OtelIngestionProcessor(config);++ // At this point, we have the raw OpenTelemetry Span body. We upload the full batch to S3+ // and the OtelIngestionProcessor logic will handle processing in the worker container.+ try {+ const job = await processor.publishToOtelIngestionQueue(resourceSpans);+ return { kind: "ok", body: job };+ } catch (error) {+ markProjectIngestFailure(config.projectId, {+ source: "public_otel_api",+ reason: "publish_failed",+ });+ throw error;+ }+}