Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(http): read the body nesting depth off the raw bytes

Yudhi Armyndharisf83dbcbc7c35
diff --git a/src/common/validation/body-nesting.spec.ts b/src/common/validation/body-nesting.spec.ts
index d9a94be30..1cccd507a 100644
--- a/src/common/validation/body-nesting.spec.ts
+++ b/src/common/validation/body-nesting.spec.ts
@@ -1,4 +1,4 @@
-import { exceedsBodyNesting, MAX_BODY_NESTING } from './body-nesting';
+import { exceedsBodyNesting, exceedsJsonNesting, MAX_BODY_NESTING } from './body-nesting';
// One value holding `levels` nested containers: {} is one level, {a:{}} is two.
const objects = (levels: number): unknown => {
@@ -11,6 +11,26 @@ const arrays = (levels: number): unknown => {
for (let i = 1; i < levels; i++) value = [value];
return value;
};
+// The same documents as JSON text: '{"a":{}}' and '[[]]' are two levels.
+const objectText = (levels: number): string => '{"a":'.repeat(levels - 1) + '{}' + '}'.repeat(levels - 1);
+const arrayText = (levels: number): string => '['.repeat(levels) + ']'.repeat(levels);
+const mixedText = (levels: number): string => {
+ let text = '0';
+ for (let i = 0; i < levels; i++) text = i % 2 === 0 ? `[${text}]` : `{"a":${text}}`;
+ return text;
+};
+const scan = (text: string): boolean => exceedsJsonNesting(Buffer.from(text));
+
+describe('the nesting limit on the raw bytes', () => {
+ it.each([
+ ['objects', objectText],
+ ['arrays', arrayText],
+ ['alternating objects and arrays', mixedText],
+ ])('admits 100 levels of %s and refuses 101', (_kind, build) => {
+ expect(scan(build(100))).toBe(false);
+ expect(scan(build(101))).toBe(true);
+ });
+});
describe('exceedsBodyNesting', () => {
it.each([
@@ -41,3 +61,114 @@ describe('exceedsBodyNesting', () => {
expect(exceedsBodyNesting(value)).toBe(false);
});
});
+
+describe('exceedsJsonNesting', () => {
+ it('counts the deepest branch, not the breadth', () => {
+ const wide = `{"items":[${Array.from({ length: 5000 }, () => '{"tags":["a","b"],"meta":{"k":[1]}}').join(',')}]}`;
+ expect(scan(wide)).toBe(false);
+ // The wrapper adds two levels around the deeper branch: 99 + 2 is 101, 98 + 2 is 100.
+ expect(scan(`{"items":[${objectText(97)},${objectText(99)}]}`)).toBe(true);
+ expect(scan(`{"items":[${objectText(97)},${objectText(98)}]}`)).toBe(false);
+ });
+
+ it('does not count brackets and braces inside strings', () => {
+ expect(scan(JSON.stringify({ text: '['.repeat(5000) + '{'.repeat(5000) }))).toBe(false);
+ expect(scan(JSON.stringify({ ['['.repeat(500)]: 1 }))).toBe(false);
+ });
+
+ it('reads an escaped quote as part of the string, so what follows it is still inside', () => {
+ expect(scan('{"a":"say \\"' + '['.repeat(500) + '\\" now"}')).toBe(false);
+ });
+
+ it('reads an escaped backslash as complete, so a quote after it ends the string', () => {
+ // "\\" is one backslash; the quote closes the string and the brackets that follow are structure.
+ expect(scan('["\\\\",' + '['.repeat(101) + ']'.repeat(101) + ']')).toBe(true);
+ expect(scan('["\\\\",' + '['.repeat(98) + ']'.repeat(98) + ']')).toBe(false);
+ });
+
+ it('is not fooled by non-ASCII text around the structure', () => {
+ const text = JSON.stringify({ 'ключ[': ['日本語[[[', '\u{1F600}"', 'é\\'], nested: objects(98) });
+ expect(scan(text)).toBe(false);
+ expect(scan(`["日本語",${arrayText(100)}]`)).toBe(true);
+ });
+
+ it('decides on the first level past the cap, before reading the rest of the body', () => {
+ // Never closed and not valid JSON: the answer must not depend on reading it to the end.
+ expect(scan('['.repeat(101))).toBe(true);
+ expect(scan('['.repeat(100))).toBe(false);
+ });
+
+ it('answers for a body nested 100000 levels, and for an array of 8 million empty arrays', () => {
+ expect(scan(arrayText(100_000))).toBe(true);
+ expect(exceedsJsonNesting(Buffer.from('[' + '[],'.repeat(8_000_000) + '[]]'))).toBe(false);
+ });
+
+ // The scan has to agree with the walk over the parsed value, for any text. Random documents built around the
+ // cap, with keys and strings made of the characters that matter to the scan.
+ it('agrees with the walk over the parsed value on random documents around the cap', () => {
+ let seed = 20261022;
+ const next = (): number => {
+ seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;
+ return seed / 2 ** 32;
+ };
+ const pieces = [
+ '',
+ 'a',
+ '[',
+ ']',
+ '{',
+ '}',
+ '"',
+ '\\',
+ '\\"',
+ '[[[[',
+ '"}]',
+ 'é',
+ '日本',
+ '\u{1F600}',
+ '\u0000',
+ ':,',
+ ];
+ const text = (): string => pieces[Math.floor(next() * pieces.length)] + pieces[Math.floor(next() * pieces.length)];
+ const leaf = (): unknown => [text(), 1, true, null, text()][Math.floor(next() * 5)];
+ for (let levels = 95; levels <= 105; levels++) {
+ for (let n = 0; n < 40; n++) {
+ let value: unknown = leaf();
+ for (let i = 0; i < levels; i++) {
+ value = next() < 0.5 ? [leaf(), value, leaf()] : { [text()]: leaf(), [`k${text()}`]: value };
+ }
+ const json = JSON.stringify(value);
+ expect(scan(json)).toBe(levels > MAX_BODY_NESTING);
+ expect(exceedsBodyNesting(JSON.parse(json))).toBe(levels > MAX_BODY_NESTING);
+ }
+ }
+ });
+
+ // The scan runs on every JSON body before authentication, so it has to cost a small fraction of the parse
+ // that follows it. A walk over the parsed value cost 0.4 to 1.8 times the parse on these shapes.
+ it.each([
+ ['many keys', (bytes: number) => `{${Array.from({ length: bytes / 10 }, (_, i) => `"k${i}":0`).join(',')}}`],
+ ['many small containers', (bytes: number) => `[${'[0],'.repeat(bytes / 4 - 1)}[0]]`],
+ ['many empty arrays', (bytes: number) => `[${'[],'.repeat(bytes / 3 - 1)}[]]`],
+ ])(
+ 'reads a body of %s in under half the time JSON.parse takes',
+ (_shape, make) => {
+ const body = make(12_000_000);
+ const bytes = Buffer.from(body);
+ const since = (start: bigint): number => Number(process.hrtime.bigint() - start) / 1e6;
+
+ const parseStart = process.hrtime.bigint();
+ JSON.parse(body);
+ const parseMs = since(parseStart);
+
+ let scanMs = Infinity;
+ for (let run = 0; run < 3; run++) {
+ const start = process.hrtime.bigint();
+ expect(exceedsJsonNesting(bytes)).toBe(false);
+ scanMs = Math.min(scanMs, since(start));
+ }
+ expect(scanMs).toBeLessThan(parseMs / 2);
+ },
+ 60_000,
+ );
+});
diff --git a/src/common/validation/body-nesting.ts b/src/common/validation/body-nesting.ts
index a15468f2f..bf978d064 100644
--- a/src/common/validation/body-nesting.ts
+++ b/src/common/validation/body-nesting.ts
@@ -6,7 +6,48 @@
*/
export const MAX_BODY_NESTING = 100;
-/** Whether a parsed request body nests deeper than the cap. It stops descending at the cap, so the walk cannot overflow. */
+/** The 400 text, the same whether the depth was read from the raw bytes or from the parsed body. */
+export const BODY_NESTING_MESSAGE = `Request body must not nest objects or arrays more than ${MAX_BODY_NESTING} levels deep`;
+
+const QUOTE = 0x22; // "
+const BACKSLASH = 0x5c;
+const OPEN_BRACKET = 0x5b; // [
+const CLOSE_BRACKET = 0x5d; // ]
+const OPEN_BRACE = 0x7b; // {
+const CLOSE_BRACE = 0x7d; // }
+
+/**
+ * Whether UTF-8 JSON text nests deeper than the cap, read from the raw bytes BEFORE it is parsed. This runs on
+ * every JSON body ahead of authentication, so it has to cost a single cheap pass: allocating a value for every
+ * container of a parsed copy, as the walk below does, cost 0.4 to 1.8 times the JSON.parse it followed on a
+ * body of many small containers or many keys.
+ *
+ * It skips each string, and a backslash skips the byte it escapes, which is all valid JSON needs; UTF-8 never
+ * encodes one of these ASCII bytes inside a multi-byte character. Text that is not valid JSON may be judged
+ * either way, and the parse refuses it anyway. It stops at the first level past the cap.
+ */
+export function exceedsJsonNesting(text: Uint8Array): boolean {
+ let depth = 0;
+ for (let i = 0; i < text.length; i++) {
+ const byte = text[i];
+ if (byte === QUOTE) {
+ // Skip the string. A backslash escapes the byte after it, so an escaped quote does not end the string.
+ for (i++; i < text.length && text[i] !== QUOTE; i++) {
+ if (text[i] === BACKSLASH) i++;
+ }
+ } else if (byte === OPEN_BRACKET || byte === OPEN_BRACE) {
+ if (++depth > MAX_BODY_NESTING) return true;
+ } else if (byte === CLOSE_BRACKET || byte === CLOSE_BRACE) {
+ depth--;
+ }
+ }
+ return false;
+}
+
+/**
+ * Whether a parsed request body nests deeper than the cap. It stops descending at the cap, so the walk cannot
+ * overflow. Used for a body the raw scan above cannot read, which is JSON in a charset other than UTF-8.
+ */
export const exceedsBodyNesting = (value: unknown, depth = 1): boolean =>
typeof value === 'object' &&
value !== null &&
diff --git a/src/configure-app.ts b/src/configure-app.ts
index cfa5483d1..a8b7738e7 100644
--- a/src/configure-app.ts
+++ b/src/configure-app.ts
@@ -1,4 +1,4 @@
-import { INestApplication, Type } from '@nestjs/common';
+import { BadRequestException, INestApplication, Type } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import helmet from 'helmet';
import { Request, Response, NextFunction, json, urlencoded } from 'express';
@@ -19,7 +19,7 @@ import { injectDashboardCspNonce } from './config/dashboard-csp';
import { resolveCorsPolicy, isUpgradeInsecureRequestsEnabled, resolveBodyLimit } from './config/bootstrap-security';
import { resolveRequestTimeoutMs } from './config/http-timeouts';
import { parseTrustedProxies } from './common/utils/ip';
-import { exceedsBodyNesting, MAX_BODY_NESTING } from './common/validation/body-nesting';
+import { BODY_NESTING_MESSAGE, exceedsBodyNesting, exceedsJsonNesting } from './common/validation/body-nesting';
/** Where the bundled dashboard documents come from, and whether to serve them at all. */
export interface DashboardSource {
@@ -276,12 +276,23 @@ export function configureApp(app: INestApplication, options: ConfigureAppOptions
// charged to the budget at its compressed size and bounded by nothing. Every other parser in the
// process must carry the same flag for that argument to hold; the MCP route-level fallback
// (src/modules/mcp/mcp.server.ts) does.
+ //
+ // The same callback reads the nesting depth off the raw UTF-8 bytes, before JSON.parse: the recursive
+ // readers of a parsed body (class-transformer in the validation pipe, the send idempotency fingerprint, the
+ // cluster proxy's JSON.stringify) overflow the stack a couple of thousand levels down and answered 500 with a
+ // stack trace in the log. A body is refused here for the price of one pass over its bytes, ahead of the
+ // parse, authentication, and every guard, interceptor and pipe. body-parser reports an error thrown from
+ // `verify` as 403 unless it carries a status; this one carries 400, and Nest answers it in the usual shape.
app.use(
json({
limit: bodyLimit,
inflate: false,
- verify: (req: Request & { rawBody?: Buffer }, _res, buf) => {
+ verify: (req: Request & { rawBody?: Buffer; nestingScanned?: boolean }, _res, buf, encoding) => {
req.rawBody = buf;
+ if (encoding.toLowerCase() === 'utf-8') {
+ if (exceedsJsonNesting(buf)) throw new BadRequestException(BODY_NESTING_MESSAGE);
+ req.nestingScanned = true;
+ }
},
}),
);
@@ -298,17 +309,13 @@ export function configureApp(app: INestApplication, options: ConfigureAppOptions
}),
);
- // The recursive readers of a parsed body (class-transformer in the validation pipe, the send idempotency
- // fingerprint, the cluster proxy's JSON.stringify) overflow the stack a couple of thousand levels down
- // and answered 500 with a stack trace in the log. Refused here, once the parsers have run, so every
- // route and /mcp share one bound and one answer, ahead of the guards, interceptors and pipes.
- app.use((req: Request, res: Response, next: NextFunction) => {
- if (!exceedsBodyNesting(req.body)) return next();
- res.status(400).json({
- statusCode: 400,
- message: `Request body must not nest objects or arrays more than ${MAX_BODY_NESTING} levels deep`,
- error: 'Bad Request',
- });
+ // The raw scan above reads UTF-8 only. JSON in another UTF charset (UTF-16, even UTF-7) is decoded by the
+ // parser, so its depth is walked on the parsed body here, with the same bound and the same answer. A
+ // form-encoded body is walked too and costs nothing: the parser itself stops it at 32 levels and 1000 keys.
+ // Every route and /mcp share the one bound, ahead of the guards, interceptors and pipes.
+ app.use((req: Request & { nestingScanned?: boolean }, _res: Response, next: NextFunction) => {
+ if (req.nestingScanned || !exceedsBodyNesting(req.body)) return next();
+ next(new BadRequestException(BODY_NESTING_MESSAGE));
});
// A DELETE path ending in '/' names no resource. Non-strict routing would still match it to the
diff --git a/test/configure-app.e2e-spec.ts b/test/configure-app.e2e-spec.ts
index 683752857..8aed557ac 100644
--- a/test/configure-app.e2e-spec.ts
+++ b/test/configure-app.e2e-spec.ts
@@ -18,7 +18,8 @@ import { NestFactory } from '@nestjs/core';
import request from 'supertest';
import { App } from 'supertest/types';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs';
-import { request as httpRequest } from 'http';
+import { request as httpRequest, type Server } from 'http';
+import type { AddressInfo } from 'net';
import { tmpdir } from 'os';
import { join } from 'path';
import { configureApp } from '../src/configure-app';
@@ -240,6 +241,54 @@ describe('production HTTP surface (configureApp)', () => {
expect(res.headers['access-control-allow-origin']).toBe('https://allowed.example');
});
+ // The depth is read from the raw bytes, so the refusal does not wait for the parse: a body that is deeper than
+ // the cap and never closes is refused for its depth, not for being invalid JSON.
+ it('refuses a body for its depth before it is parsed', async () => {
+ const res = await request(app.getHttpServer()).post('/api/echo').type('json').send('['.repeat(101));
+
+ expect(res.status).toBe(400);
+ expect(res.body).toMatchObject({
+ message: 'Request body must not nest objects or arrays more than 100 levels deep',
+ });
+ });
+
+ it('does not count brackets inside a string as nesting', async () => {
+ const text = `${'['.repeat(5000)}\\"${'{'.repeat(5000)}`;
+ const res = await request(app.getHttpServer()).post('/api/echo').type('json').send(`{"text":"${text}"}`);
+
+ expect(res.status).toBe(201);
+ });
+
+ // The raw scan reads UTF-8. A body in another UTF charset is decoded by the parser first, so it is walked
+ // after the parse instead and meets the same limit. Sent with node's http client: supertest would turn a
+ // Buffer into JSON.
+ it.each([
+ [100, 201],
+ [101, 400],
+ ])('holds a UTF-16 body nested %i levels to the same limit (%i)', async (levels, status) => {
+ const bytes = Buffer.from(nestedObjects(levels), 'utf16le');
+ const { port } = (app.getHttpServer() as Server).address() as AddressInfo;
+ const answered = await new Promise<number>((resolve, reject) => {
+ const req = httpRequest(
+ {
+ host: '127.0.0.1',
+ port,
+ path: '/api/echo',
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json; charset=utf-16le', 'Content-Length': String(bytes.length) },
+ },
+ res => {
+ res.resume();
+ resolve(res.statusCode ?? 0);
+ },
+ );
+ req.on('error', reject);
+ req.end(bytes);
+ });
+
+ expect(answered).toBe(status);
+ });
+
it('answers a DTO route 400, not 500, for a body nested thousands of levels deep', async () => {
const mentions = '['.repeat(5000) + ']'.repeat(5000);
const res = await request(app.getHttpServer())