Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(otel): log large request bodies before they are read and parsed (#18451)

Niklas Semmler520f4d6a07c3
diff --git a/web/src/pages/api/public/otel/v1/traces/index.ts b/web/src/pages/api/public/otel/v1/traces/index.ts
index 74516f6739..3b7121b27a 100644
--- a/web/src/pages/api/public/otel/v1/traces/index.ts
+++ b/web/src/pages/api/public/otel/v1/traces/index.ts
@@ -15,7 +15,10 @@ import {
OtelRequestBodyTooLargeError,
readOtelRequestBody,
} from "@/src/server/otel/otelRequestBody";
-import { processOtelIngestion } from "@/src/server/otel/processOtelIngestion";
+import {
+ OTEL_REQUEST_BODY_WARNING_BYTES,
+ processOtelIngestion,
+} from "@/src/server/otel/processOtelIngestion";
export const config = {
api: {
@@ -43,6 +46,14 @@ export default withMiddlewares({
await markProjectAsOtelUser(auth.scope.projectId);
const maxBodyBytes = env.LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES;
+ const declaredBodyBytes = Number(req.headers["content-length"]);
+ if (declaredBodyBytes > OTEL_REQUEST_BODY_WARNING_BYTES) {
+ logger.warn("Receiving large OTEL request body", {
+ projectId: auth.scope.projectId,
+ declaredBodyBytes,
+ contentEncoding: req.headers["content-encoding"],
+ });
+ }
// Start reading before shadow work so a fast request cannot finish while
// the stream is still unobserved.
const bodyResultPromise = readOtelRequestBody(req, maxBodyBytes).then(
diff --git a/web/src/server/otel/processOtelIngestion.ts b/web/src/server/otel/processOtelIngestion.ts
index ea94ea95d4..4a2b62880d 100644
--- a/web/src/server/otel/processOtelIngestion.ts
+++ b/web/src/server/otel/processOtelIngestion.ts
@@ -39,7 +39,7 @@ export type OtelIngestionResult =
body: unknown;
};
-const OTEL_REQUEST_BODY_WARNING_BYTES = 16 * 1024 * 1024;
+export const OTEL_REQUEST_BODY_WARNING_BYTES = 16 * 1024 * 1024;
function httpResult(status: number, body: unknown): OtelIngestionResult {
return { kind: "http", status, body };
@@ -61,6 +61,21 @@ export async function processOtelIngestion(
logger.error(`Invalid content type: ${contentType}`);
return httpResult(400, { error: "Invalid content type" });
}
+
+ // Parsing multiplies a body's size in memory, so a large body can get the
+ // container OOM-killed before any later log line is written.
+ if (
+ encodedBodyBytes > OTEL_REQUEST_BODY_WARNING_BYTES ||
+ body.byteLength > OTEL_REQUEST_BODY_WARNING_BYTES
+ ) {
+ logger.warn("Parsing large OTEL request body", {
+ projectId: config.projectId,
+ encodedBodyBytes,
+ decodedBodyBytes: body.byteLength,
+ contentType,
+ });
+ }
+
if (contentType.includes("application/x-protobuf")) {
try {
const parsed =