🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix: enforce OTLP ingestion request body limits (#16487)
Valery Meleshkinea3c905cd535
diff --git a/.env.dev.example b/.env.dev.exampleindex f63c89d2d9..893fb189dc 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -106,6 +106,8 @@ LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=trueLANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/# Opt in to extracting and uploading media embedded in OTEL span attributes.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED=false+# Maximum encoded and decompressed OTLP request body size in bytes.+# LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES=536870912# S3 Event Bucket Upload## Set to true to test uploading all events to S3diff --git a/.env.prod.example b/.env.prod.exampleindex 59b685b0d8..df9e0e4588 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -256,6 +256,8 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE=# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=# LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=+# Maximum encoded and decompressed OTLP request body size in bytes (default: 536870912 = 512 MiB)+# LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES=536870912# Evaluation worker concurrency# LANGFUSE_EVAL_EXECUTION_WORKER_CONCURRENCY=5diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 6798aaabce..0696f67f40 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -789,6 +789,9 @@ importers:rate-limiter-flexible:specifier: ^5.0.4version: 5.0.5+ raw-body:+ specifier: 3.0.2+ version: 3.0.2react:specifier: 19.2.4version: 19.2.4diff --git a/web/package.json b/web/package.jsonindex 5624dfb02b..ceb2d71ecb 100644--- a/web/package.json+++ b/web/package.json@@ -142,6 +142,7 @@"prism-react-renderer": "^2.4.1","protobufjs": "7.6.5","rate-limiter-flexible": "^5.0.4",+ "raw-body": "3.0.2","react": "19.2.4","react-day-picker": "^9.14.0","react-dom": "19.2.4",diff --git a/web/src/__tests__/server/unit/otel-request-body.servertest.ts b/web/src/__tests__/server/unit/otel-request-body.servertest.tsnew file mode 100644index 0000000000..714da3c285--- /dev/null+++ b/web/src/__tests__/server/unit/otel-request-body.servertest.ts@@ -0,0 +1,175 @@+import type { IncomingMessage } from "node:http";+import { PassThrough } from "node:stream";+import { gzipSync } from "node:zlib";++import { describe, expect, it, vi } from "vitest";++const routeMocks = vi.hoisted(() => ({+ logger: { error: vi.fn(), warn: vi.fn() },+ markProjectAsOtelUser: vi.fn(),+}));+const routeEnv = vi.hoisted(() => ({+ LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES: 4,+}));++vi.mock("@/src/env.mjs", () => ({+ env: routeEnv,+}));+vi.mock("@/src/features/public-api/server/withMiddlewares", () => ({+ withMiddlewares: (handlers: unknown) => handlers,+}));+vi.mock("@/src/features/public-api/server/createAuthedProjectAPIRoute", () => ({+ createAuthedProjectAPIRoute: ({ fn }: { fn: unknown }) => fn,+}));+vi.mock("@langfuse/shared", () => ({+ ForbiddenError: class ForbiddenError extends Error {},+}));+vi.mock("@langfuse/shared/src/server", () => ({+ ClickHouseClientManager: {+ getInstance: () => ({ closeAllConnections: vi.fn() }),+ },+ createIngestionAttribution: vi.fn(),+ getCurrentSpan: vi.fn(),+ getLangfuseHeaderValue: vi.fn(),+ logger: { ...routeMocks.logger, debug: vi.fn() },+ markProjectAsOtelUser: routeMocks.markProjectAsOtelUser,+ markProjectIngestFailure: vi.fn(),+ OtelIngestionProcessor: class OtelIngestionProcessor {},+ recordIncrement: vi.fn(),+ redis: { disconnect: vi.fn(), status: "end" },+ validateOtelSpanIds: vi.fn(),+}));++import {+ gunzipOtelRequestBody,+ OtelRequestBodyTooLargeError,+ readOtelRequestBody,+} from "@/src/server/otel/otelRequestBody";+import otelRoute from "@/src/pages/api/public/otel/v1/traces";++function request(headers: Record<string, string> = {}) {+ const stream = new PassThrough() as PassThrough &+ IncomingMessage & {+ headers: Record<string, string>;+ };+ stream.headers = headers;+ return stream;+}++function response() {+ return {+ status: vi.fn(),+ setHeader: vi.fn(),+ };+}++describe("OTel request body limits", () => {+ it("reads a body within the configured limit", async () => {+ const req = request({ "content-length": "5" });+ const bodyPromise = readOtelRequestBody(req, 5);+ req.end("hello");++ await expect(bodyPromise).resolves.toEqual(Buffer.from("hello"));+ });++ it("rejects a declared body above the limit before reading", async () => {+ const req = request({ "content-length": "5" });++ await expect(readOtelRequestBody(req, 4)).rejects.toMatchObject({+ name: "OtelRequestBodyTooLargeError",+ maxBytes: 4,+ afterDecompression: false,+ });+ expect(req.listenerCount("data")).toBe(0);+ req.destroy();+ });++ it("rejects a chunked body when it crosses the limit", async () => {+ const req = request();+ const bodyPromise = readOtelRequestBody(req, 4);+ req.end("12345");++ await expect(bodyPromise).rejects.toBeInstanceOf(+ OtelRequestBodyTooLargeError,+ );+ req.resume();+ });++ it("decompresses a gzip body within the configured limit", async () => {+ const body = Buffer.from('{"resourceSpans":[]}');++ await expect(+ gunzipOtelRequestBody(gzipSync(body), body.byteLength),+ ).resolves.toEqual(body);+ });++ it("stops decompressing a gzip body when it crosses the limit", async () => {+ const compressedBody = gzipSync(Buffer.alloc(8 * 1024, "x"));++ await expect(+ gunzipOtelRequestBody(compressedBody, 1024),+ ).rejects.toMatchObject({+ name: "OtelRequestBodyTooLargeError",+ message:+ "OTel request body exceeds the 1024 bytes limit after decompression",+ maxBytes: 1024,+ afterDecompression: true,+ });+ });++ it("maps an encoded overflow to the route's 413 response", async () => {+ routeEnv.LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES = 4;+ const req = request({+ "content-length": "5",+ "content-type": "application/json",+ });+ const res = response();+ const post = (+ otelRoute as unknown as {+ POST: (params: unknown) => Promise<unknown>;+ }+ ).POST;++ await expect(+ post({+ req,+ res,+ auth: { scope: { isIngestionSuspended: false, projectId: "project" } },+ }),+ ).resolves.toEqual({+ error: "OTel request body exceeds the 4 bytes limit",+ });+ expect(res.status).toHaveBeenCalledWith(413);+ expect(res.setHeader).toHaveBeenCalledWith("Connection", "close");+ req.destroy();+ });++ it("maps a decompressed overflow to the route's 413 response", async () => {+ routeEnv.LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES = 64;+ const body = gzipSync(Buffer.alloc(8 * 1024, "x"));+ const req = request({+ "content-length": String(body.byteLength),+ "content-encoding": "gzip",+ "content-type": "application/json",+ });+ const res = response();+ const post = (+ otelRoute as unknown as {+ POST: (params: unknown) => Promise<unknown>;+ }+ ).POST;+ const resultPromise = post({+ req,+ res,+ auth: { scope: { isIngestionSuspended: false, projectId: "project" } },+ });+ req.end(body);++ await expect(resultPromise).resolves.toEqual({+ error: "OTel request body exceeds the 64 bytes limit after decompression",+ });+ expect(res.status).toHaveBeenCalledWith(413);+ expect(res.setHeader).not.toHaveBeenCalled();+ req.destroy();+ });+});diff --git a/web/src/env.mjs b/web/src/env.mjsindex 66ca408f8e..8fde9ea4de 100644--- a/web/src/env.mjs+++ b/web/src/env.mjs@@ -488,6 +488,12 @@ export const env = createEnv({LANGFUSE_SKIP_FINAL_FOR_OTEL_PROJECTS: z.enum(["true", "false"]).default("false"),+ // Maximum encoded and decompressed OTLP request body size in bytes.+ LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES: z.coerce+ .number()+ .positive()+ .int()+ .default(512 * 1024 * 1024),// API Traces endpoint controls (may induce breaking changes on API when changed!)LANGFUSE_API_TRACES_DEFAULT_DATE_RANGE_DAYS: z.coerce.number()@@ -1016,6 +1022,8 @@ export const env = createEnv({// Api Performance FlagsLANGFUSE_SKIP_FINAL_FOR_OTEL_PROJECTS:process.env.LANGFUSE_SKIP_FINAL_FOR_OTEL_PROJECTS,+ LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES:+ process.env.LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES,// Natural Language FiltersLANGFUSE_AI_FEATURES_PUBLIC_KEY:diff --git a/web/src/pages/api/public/otel/v1/traces/index.ts b/web/src/pages/api/public/otel/v1/traces/index.tsindex ecd8093528..7ef3257057 100644--- a/web/src/pages/api/public/otel/v1/traces/index.ts+++ b/web/src/pages/api/public/otel/v1/traces/index.ts@@ -13,9 +13,14 @@ import {} from "@langfuse/shared/src/server";import { z } from "zod";import { $root } from "@/src/pages/api/public/otel/otlp-proto/generated/root";-import { gunzip } from "node:zlib";import { ForbiddenError } from "@langfuse/shared";import { env } from "@/src/env.mjs";+import {+ gunzipOtelRequestBody,+ handleOtelRequestBodyTooLarge,+ OtelRequestBodyTooLargeError,+ readOtelRequestBody,+} from "@/src/server/otel/otelRequestBody";export const config = {api: {@@ -40,32 +45,30 @@ export default withMiddlewares({// Mark project as using OTEL APIawait markProjectAsOtelUser(auth.scope.projectId);+ const maxBodyBytes = env.LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES;+let body: Buffer;+ let bodyFailureMessage = "Failed to read request body";try {- body = await new Promise((resolve, reject) => {- let data: any[] = [];- req.on("data", (chunk) => data.push(chunk));- req.on("end", () => resolve(Buffer.concat(data)));- req.on("error", reject);- });- } catch (e) {- logger.error(`Failed to read request body`, e);- res.status(400);- return { error: "Failed to read request body" };- }+ body = await readOtelRequestBody(req, maxBodyBytes);- if (req.headers["content-encoding"]?.includes("gzip")) {- try {- body = await new Promise((resolve, reject) => {- gunzip(new Uint8Array(body), (err, result) =>- err ? reject(err) : resolve(result),- );- });- } catch (e) {- logger.error(`Failed to decompress request body`, e);- res.status(400);- return { error: "Failed to decompress request body" };+ if (req.headers["content-encoding"]?.includes("gzip")) {+ bodyFailureMessage = "Failed to decompress request body";+ body = await gunzipOtelRequestBody(body, maxBodyBytes);}+ } catch (error) {+ if (error instanceof OtelRequestBodyTooLargeError) {+ return handleOtelRequestBodyTooLarge(+ error,+ req,+ res,+ auth.scope.projectId,+ );+ }++ logger.error(bodyFailureMessage, error);+ res.status(400);+ return { error: bodyFailureMessage };}let resourceSpans: any;diff --git a/web/src/server/otel/otelRequestBody.ts b/web/src/server/otel/otelRequestBody.tsnew file mode 100644index 0000000000..c028df0229--- /dev/null+++ b/web/src/server/otel/otelRequestBody.ts@@ -0,0 +1,114 @@+import type { IncomingMessage } from "node:http";+import { gunzip } from "node:zlib";++import {+ getCurrentSpan,+ logger,+ recordIncrement,+} from "@langfuse/shared/src/server";+import type { NextApiResponse } from "next";+import getRawBody from "raw-body";++export class OtelRequestBodyTooLargeError extends Error {+ constructor(+ public readonly maxBytes: number,+ public readonly afterDecompression = false,+ ) {+ const maxMiB = maxBytes / (1024 * 1024);+ const limit = Number.isInteger(maxMiB)+ ? `${maxMiB} MiB`+ : `${maxBytes} bytes`;+ const suffix = afterDecompression ? " after decompression" : "";++ super(`OTel request body exceeds the ${limit} limit${suffix}`);+ this.name = "OtelRequestBodyTooLargeError";+ }+}++export function handleOtelRequestBodyTooLarge(+ error: OtelRequestBodyTooLargeError,+ req: IncomingMessage,+ res: NextApiResponse,+ projectId: string,+) {+ const stage = error.afterDecompression ? "decompressed" : "encoded";++ if (!error.afterDecompression) {+ // raw-body intentionally pauses on overflow. Drain what remains while+ // closing this connection so subsequent requests are not queued here.+ req.resume();+ res.setHeader("Connection", "close");+ }++ logger.warn("Rejecting oversized OTEL request body", {+ projectId,+ maxBodyBytes: error.maxBytes,+ afterDecompression: error.afterDecompression,+ });+ recordIncrement("langfuse.ingestion.otel.request_body_limit_exceeded", 1, {+ stage,+ });+ getCurrentSpan()?.setAttributes({+ "langfuse.ingestion.otel.request_body_limit_exceeded": true,+ "langfuse.ingestion.otel.request_body_limit_bytes": error.maxBytes,+ "langfuse.ingestion.otel.request_body_limit_stage": stage,+ });+ res.status(413);+ return { error: error.message };+}++type RawBodyError = Error & {+ status?: number;+ statusCode?: number;+ type?: string;+};++function isRawBodyTooLargeError(error: unknown): error is RawBodyError {+ if (!(error instanceof Error)) return false;++ const rawBodyError = error as RawBodyError;+ return (+ rawBodyError.status === 413 ||+ rawBodyError.statusCode === 413 ||+ rawBodyError.type === "entity.too.large"+ );+}++export async function readOtelRequestBody(+ req: IncomingMessage,+ maxBytes: number,+): Promise<Buffer> {+ try {+ return await getRawBody(req, {+ length: req.headers["content-length"],+ limit: maxBytes,+ });+ } catch (error) {+ if (isRawBodyTooLargeError(error)) {+ throw new OtelRequestBodyTooLargeError(maxBytes);+ }++ throw error;+ }+}++export function gunzipOtelRequestBody(+ body: Buffer,+ maxBytes: number,+): Promise<Buffer> {+ return new Promise((resolve, reject) => {+ gunzip(body, { maxOutputLength: maxBytes }, (error, result) => {+ if (!error) {+ resolve(result);+ return;+ }++ if ((error as NodeJS.ErrnoException).code === "ERR_BUFFER_TOO_LARGE") {+ reject(new OtelRequestBodyTooLargeError(maxBytes, true));+ return;+ }++ reject(error);+ });+ });+}