Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix: enforce OTLP ingestion request body limits (#16487)

Valery Meleshkinea3c905cd535
diff --git a/.env.dev.example b/.env.dev.example
index f63c89d2d9..893fb189dc 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -106,6 +106,8 @@ LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
# Opt in to extracting and uploading media embedded in OTEL span attributes.
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED=false
+# Maximum encoded and decompressed OTLP request body size in bytes.
+# LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES=536870912
# S3 Event Bucket Upload
## Set to true to test uploading all events to S3
diff --git a/.env.prod.example b/.env.prod.example
index 59b685b0d8..df9e0e4588 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -256,6 +256,8 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE=
# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=
# LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
+# Maximum encoded and decompressed OTLP request body size in bytes (default: 536870912 = 512 MiB)
+# LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES=536870912
# Evaluation worker concurrency
# LANGFUSE_EVAL_EXECUTION_WORKER_CONCURRENCY=5
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 6798aaabce..0696f67f40 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -789,6 +789,9 @@ importers:
rate-limiter-flexible:
specifier: ^5.0.4
version: 5.0.5
+ raw-body:
+ specifier: 3.0.2
+ version: 3.0.2
react:
specifier: 19.2.4
version: 19.2.4
diff --git a/web/package.json b/web/package.json
index 5624dfb02b..ceb2d71ecb 100644
--- a/web/package.json
+++ b/web/package.json
@@ -142,6 +142,7 @@
"prism-react-renderer": "^2.4.1",
"protobufjs": "7.6.5",
"rate-limiter-flexible": "^5.0.4",
+ "raw-body": "3.0.2",
"react": "19.2.4",
"react-day-picker": "^9.14.0",
"react-dom": "19.2.4",
diff --git a/web/src/__tests__/server/unit/otel-request-body.servertest.ts b/web/src/__tests__/server/unit/otel-request-body.servertest.ts
new file mode 100644
index 0000000000..714da3c285
--- /dev/null
+++ b/web/src/__tests__/server/unit/otel-request-body.servertest.ts
@@ -0,0 +1,175 @@
+import type { IncomingMessage } from "node:http";
+import { PassThrough } from "node:stream";
+import { gzipSync } from "node:zlib";
+
+import { describe, expect, it, vi } from "vitest";
+
+const routeMocks = vi.hoisted(() => ({
+ logger: { error: vi.fn(), warn: vi.fn() },
+ markProjectAsOtelUser: vi.fn(),
+}));
+const routeEnv = vi.hoisted(() => ({
+ LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES: 4,
+}));
+
+vi.mock("@/src/env.mjs", () => ({
+ env: routeEnv,
+}));
+vi.mock("@/src/features/public-api/server/withMiddlewares", () => ({
+ withMiddlewares: (handlers: unknown) => handlers,
+}));
+vi.mock("@/src/features/public-api/server/createAuthedProjectAPIRoute", () => ({
+ createAuthedProjectAPIRoute: ({ fn }: { fn: unknown }) => fn,
+}));
+vi.mock("@langfuse/shared", () => ({
+ ForbiddenError: class ForbiddenError extends Error {},
+}));
+vi.mock("@langfuse/shared/src/server", () => ({
+ ClickHouseClientManager: {
+ getInstance: () => ({ closeAllConnections: vi.fn() }),
+ },
+ createIngestionAttribution: vi.fn(),
+ getCurrentSpan: vi.fn(),
+ getLangfuseHeaderValue: vi.fn(),
+ logger: { ...routeMocks.logger, debug: vi.fn() },
+ markProjectAsOtelUser: routeMocks.markProjectAsOtelUser,
+ markProjectIngestFailure: vi.fn(),
+ OtelIngestionProcessor: class OtelIngestionProcessor {},
+ recordIncrement: vi.fn(),
+ redis: { disconnect: vi.fn(), status: "end" },
+ validateOtelSpanIds: vi.fn(),
+}));
+
+import {
+ gunzipOtelRequestBody,
+ OtelRequestBodyTooLargeError,
+ readOtelRequestBody,
+} from "@/src/server/otel/otelRequestBody";
+import otelRoute from "@/src/pages/api/public/otel/v1/traces";
+
+function request(headers: Record<string, string> = {}) {
+ const stream = new PassThrough() as PassThrough &
+ IncomingMessage & {
+ headers: Record<string, string>;
+ };
+ stream.headers = headers;
+ return stream;
+}
+
+function response() {
+ return {
+ status: vi.fn(),
+ setHeader: vi.fn(),
+ };
+}
+
+describe("OTel request body limits", () => {
+ it("reads a body within the configured limit", async () => {
+ const req = request({ "content-length": "5" });
+ const bodyPromise = readOtelRequestBody(req, 5);
+ req.end("hello");
+
+ await expect(bodyPromise).resolves.toEqual(Buffer.from("hello"));
+ });
+
+ it("rejects a declared body above the limit before reading", async () => {
+ const req = request({ "content-length": "5" });
+
+ await expect(readOtelRequestBody(req, 4)).rejects.toMatchObject({
+ name: "OtelRequestBodyTooLargeError",
+ maxBytes: 4,
+ afterDecompression: false,
+ });
+ expect(req.listenerCount("data")).toBe(0);
+ req.destroy();
+ });
+
+ it("rejects a chunked body when it crosses the limit", async () => {
+ const req = request();
+ const bodyPromise = readOtelRequestBody(req, 4);
+ req.end("12345");
+
+ await expect(bodyPromise).rejects.toBeInstanceOf(
+ OtelRequestBodyTooLargeError,
+ );
+ req.resume();
+ });
+
+ it("decompresses a gzip body within the configured limit", async () => {
+ const body = Buffer.from('{"resourceSpans":[]}');
+
+ await expect(
+ gunzipOtelRequestBody(gzipSync(body), body.byteLength),
+ ).resolves.toEqual(body);
+ });
+
+ it("stops decompressing a gzip body when it crosses the limit", async () => {
+ const compressedBody = gzipSync(Buffer.alloc(8 * 1024, "x"));
+
+ await expect(
+ gunzipOtelRequestBody(compressedBody, 1024),
+ ).rejects.toMatchObject({
+ name: "OtelRequestBodyTooLargeError",
+ message:
+ "OTel request body exceeds the 1024 bytes limit after decompression",
+ maxBytes: 1024,
+ afterDecompression: true,
+ });
+ });
+
+ it("maps an encoded overflow to the route's 413 response", async () => {
+ routeEnv.LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES = 4;
+ const req = request({
+ "content-length": "5",
+ "content-type": "application/json",
+ });
+ const res = response();
+ const post = (
+ otelRoute as unknown as {
+ POST: (params: unknown) => Promise<unknown>;
+ }
+ ).POST;
+
+ await expect(
+ post({
+ req,
+ res,
+ auth: { scope: { isIngestionSuspended: false, projectId: "project" } },
+ }),
+ ).resolves.toEqual({
+ error: "OTel request body exceeds the 4 bytes limit",
+ });
+ expect(res.status).toHaveBeenCalledWith(413);
+ expect(res.setHeader).toHaveBeenCalledWith("Connection", "close");
+ req.destroy();
+ });
+
+ it("maps a decompressed overflow to the route's 413 response", async () => {
+ routeEnv.LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES = 64;
+ const body = gzipSync(Buffer.alloc(8 * 1024, "x"));
+ const req = request({
+ "content-length": String(body.byteLength),
+ "content-encoding": "gzip",
+ "content-type": "application/json",
+ });
+ const res = response();
+ const post = (
+ otelRoute as unknown as {
+ POST: (params: unknown) => Promise<unknown>;
+ }
+ ).POST;
+ const resultPromise = post({
+ req,
+ res,
+ auth: { scope: { isIngestionSuspended: false, projectId: "project" } },
+ });
+ req.end(body);
+
+ await expect(resultPromise).resolves.toEqual({
+ error: "OTel request body exceeds the 64 bytes limit after decompression",
+ });
+ expect(res.status).toHaveBeenCalledWith(413);
+ expect(res.setHeader).not.toHaveBeenCalled();
+ req.destroy();
+ });
+});
diff --git a/web/src/env.mjs b/web/src/env.mjs
index 66ca408f8e..8fde9ea4de 100644
--- a/web/src/env.mjs
+++ b/web/src/env.mjs
@@ -488,6 +488,12 @@ export const env = createEnv({
LANGFUSE_SKIP_FINAL_FOR_OTEL_PROJECTS: z
.enum(["true", "false"])
.default("false"),
+ // Maximum encoded and decompressed OTLP request body size in bytes.
+ LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES: z.coerce
+ .number()
+ .positive()
+ .int()
+ .default(512 * 1024 * 1024),
// API Traces endpoint controls (may induce breaking changes on API when changed!)
LANGFUSE_API_TRACES_DEFAULT_DATE_RANGE_DAYS: z.coerce
.number()
@@ -1016,6 +1022,8 @@ export const env = createEnv({
// Api Performance Flags
LANGFUSE_SKIP_FINAL_FOR_OTEL_PROJECTS:
process.env.LANGFUSE_SKIP_FINAL_FOR_OTEL_PROJECTS,
+ LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES:
+ process.env.LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES,
// Natural Language Filters
LANGFUSE_AI_FEATURES_PUBLIC_KEY:
diff --git a/web/src/pages/api/public/otel/v1/traces/index.ts b/web/src/pages/api/public/otel/v1/traces/index.ts
index ecd8093528..7ef3257057 100644
--- a/web/src/pages/api/public/otel/v1/traces/index.ts
+++ b/web/src/pages/api/public/otel/v1/traces/index.ts
@@ -13,9 +13,14 @@ import {
} from "@langfuse/shared/src/server";
import { z } from "zod";
import { $root } from "@/src/pages/api/public/otel/otlp-proto/generated/root";
-import { gunzip } from "node:zlib";
import { ForbiddenError } from "@langfuse/shared";
import { env } from "@/src/env.mjs";
+import {
+ gunzipOtelRequestBody,
+ handleOtelRequestBodyTooLarge,
+ OtelRequestBodyTooLargeError,
+ readOtelRequestBody,
+} from "@/src/server/otel/otelRequestBody";
export const config = {
api: {
@@ -40,32 +45,30 @@ export default withMiddlewares({
// Mark project as using OTEL API
await markProjectAsOtelUser(auth.scope.projectId);
+ const maxBodyBytes = env.LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES;
+
let body: Buffer;
+ let bodyFailureMessage = "Failed to read request body";
try {
- body = await new Promise((resolve, reject) => {
- let data: any[] = [];
- req.on("data", (chunk) => data.push(chunk));
- req.on("end", () => resolve(Buffer.concat(data)));
- req.on("error", reject);
- });
- } catch (e) {
- logger.error(`Failed to read request body`, e);
- res.status(400);
- return { error: "Failed to read request body" };
- }
+ body = await readOtelRequestBody(req, maxBodyBytes);
- if (req.headers["content-encoding"]?.includes("gzip")) {
- try {
- body = await new Promise((resolve, reject) => {
- gunzip(new Uint8Array(body), (err, result) =>
- err ? reject(err) : resolve(result),
- );
- });
- } catch (e) {
- logger.error(`Failed to decompress request body`, e);
- res.status(400);
- return { error: "Failed to decompress request body" };
+ if (req.headers["content-encoding"]?.includes("gzip")) {
+ bodyFailureMessage = "Failed to decompress request body";
+ body = await gunzipOtelRequestBody(body, maxBodyBytes);
}
+ } catch (error) {
+ if (error instanceof OtelRequestBodyTooLargeError) {
+ return handleOtelRequestBodyTooLarge(
+ error,
+ req,
+ res,
+ auth.scope.projectId,
+ );
+ }
+
+ logger.error(bodyFailureMessage, error);
+ res.status(400);
+ return { error: bodyFailureMessage };
}
let resourceSpans: any;
diff --git a/web/src/server/otel/otelRequestBody.ts b/web/src/server/otel/otelRequestBody.ts
new file mode 100644
index 0000000000..c028df0229
--- /dev/null
+++ b/web/src/server/otel/otelRequestBody.ts
@@ -0,0 +1,114 @@
+import type { IncomingMessage } from "node:http";
+import { gunzip } from "node:zlib";
+
+import {
+ getCurrentSpan,
+ logger,
+ recordIncrement,
+} from "@langfuse/shared/src/server";
+import type { NextApiResponse } from "next";
+import getRawBody from "raw-body";
+
+export class OtelRequestBodyTooLargeError extends Error {
+ constructor(
+ public readonly maxBytes: number,
+ public readonly afterDecompression = false,
+ ) {
+ const maxMiB = maxBytes / (1024 * 1024);
+ const limit = Number.isInteger(maxMiB)
+ ? `${maxMiB} MiB`
+ : `${maxBytes} bytes`;
+ const suffix = afterDecompression ? " after decompression" : "";
+
+ super(`OTel request body exceeds the ${limit} limit${suffix}`);
+ this.name = "OtelRequestBodyTooLargeError";
+ }
+}
+
+export function handleOtelRequestBodyTooLarge(
+ error: OtelRequestBodyTooLargeError,
+ req: IncomingMessage,
+ res: NextApiResponse,
+ projectId: string,
+) {
+ const stage = error.afterDecompression ? "decompressed" : "encoded";
+
+ if (!error.afterDecompression) {
+ // raw-body intentionally pauses on overflow. Drain what remains while
+ // closing this connection so subsequent requests are not queued here.
+ req.resume();
+ res.setHeader("Connection", "close");
+ }
+
+ logger.warn("Rejecting oversized OTEL request body", {
+ projectId,
+ maxBodyBytes: error.maxBytes,
+ afterDecompression: error.afterDecompression,
+ });
+ recordIncrement("langfuse.ingestion.otel.request_body_limit_exceeded", 1, {
+ stage,
+ });
+ getCurrentSpan()?.setAttributes({
+ "langfuse.ingestion.otel.request_body_limit_exceeded": true,
+ "langfuse.ingestion.otel.request_body_limit_bytes": error.maxBytes,
+ "langfuse.ingestion.otel.request_body_limit_stage": stage,
+ });
+ res.status(413);
+ return { error: error.message };
+}
+
+type RawBodyError = Error & {
+ status?: number;
+ statusCode?: number;
+ type?: string;
+};
+
+function isRawBodyTooLargeError(error: unknown): error is RawBodyError {
+ if (!(error instanceof Error)) return false;
+
+ const rawBodyError = error as RawBodyError;
+ return (
+ rawBodyError.status === 413 ||
+ rawBodyError.statusCode === 413 ||
+ rawBodyError.type === "entity.too.large"
+ );
+}
+
+export async function readOtelRequestBody(
+ req: IncomingMessage,
+ maxBytes: number,
+): Promise<Buffer> {
+ try {
+ return await getRawBody(req, {
+ length: req.headers["content-length"],
+ limit: maxBytes,
+ });
+ } catch (error) {
+ if (isRawBodyTooLargeError(error)) {
+ throw new OtelRequestBodyTooLargeError(maxBytes);
+ }
+
+ throw error;
+ }
+}
+
+export function gunzipOtelRequestBody(
+ body: Buffer,
+ maxBytes: number,
+): Promise<Buffer> {
+ return new Promise((resolve, reject) => {
+ gunzip(body, { maxOutputLength: maxBytes }, (error, result) => {
+ if (!error) {
+ resolve(result);
+ return;
+ }
+
+ if ((error as NodeJS.ErrnoException).code === "ERR_BUFFER_TOO_LARGE") {
+ reject(new OtelRequestBodyTooLargeError(maxBytes, true));
+ return;
+ }
+
+ reject(error);
+ });
+ });
+}