Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(http): read the body nesting depth off the raw bytes
Yudhi Armyndharisf83dbcbc7c35
diff --git a/src/common/validation/body-nesting.spec.ts b/src/common/validation/body-nesting.spec.tsindex d9a94be30..1cccd507a 100644--- a/src/common/validation/body-nesting.spec.ts+++ b/src/common/validation/body-nesting.spec.ts@@ -1,4 +1,4 @@-import { exceedsBodyNesting, MAX_BODY_NESTING } from './body-nesting';+import { exceedsBodyNesting, exceedsJsonNesting, MAX_BODY_NESTING } from './body-nesting';// One value holding `levels` nested containers: {} is one level, {a:{}} is two.const objects = (levels: number): unknown => {@@ -11,6 +11,26 @@ const arrays = (levels: number): unknown => {for (let i = 1; i < levels; i++) value = [value];return value;};+// The same documents as JSON text: '{"a":{}}' and '[[]]' are two levels.+const objectText = (levels: number): string => '{"a":'.repeat(levels - 1) + '{}' + '}'.repeat(levels - 1);+const arrayText = (levels: number): string => '['.repeat(levels) + ']'.repeat(levels);+const mixedText = (levels: number): string => {+ let text = '0';+ for (let i = 0; i < levels; i++) text = i % 2 === 0 ? `[${text}]` : `{"a":${text}}`;+ return text;+};+const scan = (text: string): boolean => exceedsJsonNesting(Buffer.from(text));++describe('the nesting limit on the raw bytes', () => {+ it.each([+ ['objects', objectText],+ ['arrays', arrayText],+ ['alternating objects and arrays', mixedText],+ ])('admits 100 levels of %s and refuses 101', (_kind, build) => {+ expect(scan(build(100))).toBe(false);+ expect(scan(build(101))).toBe(true);+ });+});describe('exceedsBodyNesting', () => {it.each([@@ -41,3 +61,114 @@ describe('exceedsBodyNesting', () => {expect(exceedsBodyNesting(value)).toBe(false);});});++describe('exceedsJsonNesting', () => {+ it('counts the deepest branch, not the breadth', () => {+ const wide = `{"items":[${Array.from({ length: 5000 }, () => '{"tags":["a","b"],"meta":{"k":[1]}}').join(',')}]}`;+ expect(scan(wide)).toBe(false);+ // The wrapper adds two levels around the deeper branch: 99 + 2 is 101, 98 + 2 is 100.+ expect(scan(`{"items":[${objectText(97)},${objectText(99)}]}`)).toBe(true);+ expect(scan(`{"items":[${objectText(97)},${objectText(98)}]}`)).toBe(false);+ });++ it('does not count brackets and braces inside strings', () => {+ expect(scan(JSON.stringify({ text: '['.repeat(5000) + '{'.repeat(5000) }))).toBe(false);+ expect(scan(JSON.stringify({ ['['.repeat(500)]: 1 }))).toBe(false);+ });++ it('reads an escaped quote as part of the string, so what follows it is still inside', () => {+ expect(scan('{"a":"say \\"' + '['.repeat(500) + '\\" now"}')).toBe(false);+ });++ it('reads an escaped backslash as complete, so a quote after it ends the string', () => {+ // "\\" is one backslash; the quote closes the string and the brackets that follow are structure.+ expect(scan('["\\\\",' + '['.repeat(101) + ']'.repeat(101) + ']')).toBe(true);+ expect(scan('["\\\\",' + '['.repeat(98) + ']'.repeat(98) + ']')).toBe(false);+ });++ it('is not fooled by non-ASCII text around the structure', () => {+ const text = JSON.stringify({ 'ключ[': ['日本語[[[', '\u{1F600}"', 'é\\'], nested: objects(98) });+ expect(scan(text)).toBe(false);+ expect(scan(`["日本語",${arrayText(100)}]`)).toBe(true);+ });++ it('decides on the first level past the cap, before reading the rest of the body', () => {+ // Never closed and not valid JSON: the answer must not depend on reading it to the end.+ expect(scan('['.repeat(101))).toBe(true);+ expect(scan('['.repeat(100))).toBe(false);+ });++ it('answers for a body nested 100000 levels, and for an array of 8 million empty arrays', () => {+ expect(scan(arrayText(100_000))).toBe(true);+ expect(exceedsJsonNesting(Buffer.from('[' + '[],'.repeat(8_000_000) + '[]]'))).toBe(false);+ });++ // The scan has to agree with the walk over the parsed value, for any text. Random documents built around the+ // cap, with keys and strings made of the characters that matter to the scan.+ it('agrees with the walk over the parsed value on random documents around the cap', () => {+ let seed = 20261022;+ const next = (): number => {+ seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;+ return seed / 2 ** 32;+ };+ const pieces = [+ '',+ 'a',+ '[',+ ']',+ '{',+ '}',+ '"',+ '\\',+ '\\"',+ '[[[[',+ '"}]',+ 'é',+ '日本',+ '\u{1F600}',+ '\u0000',+ ':,',+ ];+ const text = (): string => pieces[Math.floor(next() * pieces.length)] + pieces[Math.floor(next() * pieces.length)];+ const leaf = (): unknown => [text(), 1, true, null, text()][Math.floor(next() * 5)];+ for (let levels = 95; levels <= 105; levels++) {+ for (let n = 0; n < 40; n++) {+ let value: unknown = leaf();+ for (let i = 0; i < levels; i++) {+ value = next() < 0.5 ? [leaf(), value, leaf()] : { [text()]: leaf(), [`k${text()}`]: value };+ }+ const json = JSON.stringify(value);+ expect(scan(json)).toBe(levels > MAX_BODY_NESTING);+ expect(exceedsBodyNesting(JSON.parse(json))).toBe(levels > MAX_BODY_NESTING);+ }+ }+ });++ // The scan runs on every JSON body before authentication, so it has to cost a small fraction of the parse+ // that follows it. A walk over the parsed value cost 0.4 to 1.8 times the parse on these shapes.+ it.each([+ ['many keys', (bytes: number) => `{${Array.from({ length: bytes / 10 }, (_, i) => `"k${i}":0`).join(',')}}`],+ ['many small containers', (bytes: number) => `[${'[0],'.repeat(bytes / 4 - 1)}[0]]`],+ ['many empty arrays', (bytes: number) => `[${'[],'.repeat(bytes / 3 - 1)}[]]`],+ ])(+ 'reads a body of %s in under half the time JSON.parse takes',+ (_shape, make) => {+ const body = make(12_000_000);+ const bytes = Buffer.from(body);+ const since = (start: bigint): number => Number(process.hrtime.bigint() - start) / 1e6;++ const parseStart = process.hrtime.bigint();+ JSON.parse(body);+ const parseMs = since(parseStart);++ let scanMs = Infinity;+ for (let run = 0; run < 3; run++) {+ const start = process.hrtime.bigint();+ expect(exceedsJsonNesting(bytes)).toBe(false);+ scanMs = Math.min(scanMs, since(start));+ }+ expect(scanMs).toBeLessThan(parseMs / 2);+ },+ 60_000,+ );+});diff --git a/src/common/validation/body-nesting.ts b/src/common/validation/body-nesting.tsindex a15468f2f..bf978d064 100644--- a/src/common/validation/body-nesting.ts+++ b/src/common/validation/body-nesting.ts@@ -6,7 +6,48 @@*/export const MAX_BODY_NESTING = 100;-/** Whether a parsed request body nests deeper than the cap. It stops descending at the cap, so the walk cannot overflow. */+/** The 400 text, the same whether the depth was read from the raw bytes or from the parsed body. */+export const BODY_NESTING_MESSAGE = `Request body must not nest objects or arrays more than ${MAX_BODY_NESTING} levels deep`;++const QUOTE = 0x22; // "+const BACKSLASH = 0x5c;+const OPEN_BRACKET = 0x5b; // [+const CLOSE_BRACKET = 0x5d; // ]+const OPEN_BRACE = 0x7b; // {+const CLOSE_BRACE = 0x7d; // }++/**+ * Whether UTF-8 JSON text nests deeper than the cap, read from the raw bytes BEFORE it is parsed. This runs on+ * every JSON body ahead of authentication, so it has to cost a single cheap pass: allocating a value for every+ * container of a parsed copy, as the walk below does, cost 0.4 to 1.8 times the JSON.parse it followed on a+ * body of many small containers or many keys.+ *+ * It skips each string, and a backslash skips the byte it escapes, which is all valid JSON needs; UTF-8 never+ * encodes one of these ASCII bytes inside a multi-byte character. Text that is not valid JSON may be judged+ * either way, and the parse refuses it anyway. It stops at the first level past the cap.+ */+export function exceedsJsonNesting(text: Uint8Array): boolean {+ let depth = 0;+ for (let i = 0; i < text.length; i++) {+ const byte = text[i];+ if (byte === QUOTE) {+ // Skip the string. A backslash escapes the byte after it, so an escaped quote does not end the string.+ for (i++; i < text.length && text[i] !== QUOTE; i++) {+ if (text[i] === BACKSLASH) i++;+ }+ } else if (byte === OPEN_BRACKET || byte === OPEN_BRACE) {+ if (++depth > MAX_BODY_NESTING) return true;+ } else if (byte === CLOSE_BRACKET || byte === CLOSE_BRACE) {+ depth--;+ }+ }+ return false;+}++/**+ * Whether a parsed request body nests deeper than the cap. It stops descending at the cap, so the walk cannot+ * overflow. Used for a body the raw scan above cannot read, which is JSON in a charset other than UTF-8.+ */export const exceedsBodyNesting = (value: unknown, depth = 1): boolean =>typeof value === 'object' &&value !== null &&diff --git a/src/configure-app.ts b/src/configure-app.tsindex cfa5483d1..a8b7738e7 100644--- a/src/configure-app.ts+++ b/src/configure-app.ts@@ -1,4 +1,4 @@-import { INestApplication, Type } from '@nestjs/common';+import { BadRequestException, INestApplication, Type } from '@nestjs/common';import { ModuleRef } from '@nestjs/core';import helmet from 'helmet';import { Request, Response, NextFunction, json, urlencoded } from 'express';@@ -19,7 +19,7 @@ import { injectDashboardCspNonce } from './config/dashboard-csp';import { resolveCorsPolicy, isUpgradeInsecureRequestsEnabled, resolveBodyLimit } from './config/bootstrap-security';import { resolveRequestTimeoutMs } from './config/http-timeouts';import { parseTrustedProxies } from './common/utils/ip';-import { exceedsBodyNesting, MAX_BODY_NESTING } from './common/validation/body-nesting';+import { BODY_NESTING_MESSAGE, exceedsBodyNesting, exceedsJsonNesting } from './common/validation/body-nesting';/** Where the bundled dashboard documents come from, and whether to serve them at all. */export interface DashboardSource {@@ -276,12 +276,23 @@ export function configureApp(app: INestApplication, options: ConfigureAppOptions// charged to the budget at its compressed size and bounded by nothing. Every other parser in the// process must carry the same flag for that argument to hold; the MCP route-level fallback// (src/modules/mcp/mcp.server.ts) does.+ //+ // The same callback reads the nesting depth off the raw UTF-8 bytes, before JSON.parse: the recursive+ // readers of a parsed body (class-transformer in the validation pipe, the send idempotency fingerprint, the+ // cluster proxy's JSON.stringify) overflow the stack a couple of thousand levels down and answered 500 with a+ // stack trace in the log. A body is refused here for the price of one pass over its bytes, ahead of the+ // parse, authentication, and every guard, interceptor and pipe. body-parser reports an error thrown from+ // `verify` as 403 unless it carries a status; this one carries 400, and Nest answers it in the usual shape.app.use(json({limit: bodyLimit,inflate: false,- verify: (req: Request & { rawBody?: Buffer }, _res, buf) => {+ verify: (req: Request & { rawBody?: Buffer; nestingScanned?: boolean }, _res, buf, encoding) => {req.rawBody = buf;+ if (encoding.toLowerCase() === 'utf-8') {+ if (exceedsJsonNesting(buf)) throw new BadRequestException(BODY_NESTING_MESSAGE);+ req.nestingScanned = true;+ }},}),);@@ -298,17 +309,13 @@ export function configureApp(app: INestApplication, options: ConfigureAppOptions}),);- // The recursive readers of a parsed body (class-transformer in the validation pipe, the send idempotency- // fingerprint, the cluster proxy's JSON.stringify) overflow the stack a couple of thousand levels down- // and answered 500 with a stack trace in the log. Refused here, once the parsers have run, so every- // route and /mcp share one bound and one answer, ahead of the guards, interceptors and pipes.- app.use((req: Request, res: Response, next: NextFunction) => {- if (!exceedsBodyNesting(req.body)) return next();- res.status(400).json({- statusCode: 400,- message: `Request body must not nest objects or arrays more than ${MAX_BODY_NESTING} levels deep`,- error: 'Bad Request',- });+ // The raw scan above reads UTF-8 only. JSON in another UTF charset (UTF-16, even UTF-7) is decoded by the+ // parser, so its depth is walked on the parsed body here, with the same bound and the same answer. A+ // form-encoded body is walked too and costs nothing: the parser itself stops it at 32 levels and 1000 keys.+ // Every route and /mcp share the one bound, ahead of the guards, interceptors and pipes.+ app.use((req: Request & { nestingScanned?: boolean }, _res: Response, next: NextFunction) => {+ if (req.nestingScanned || !exceedsBodyNesting(req.body)) return next();+ next(new BadRequestException(BODY_NESTING_MESSAGE));});// A DELETE path ending in '/' names no resource. Non-strict routing would still match it to thediff --git a/test/configure-app.e2e-spec.ts b/test/configure-app.e2e-spec.tsindex 683752857..8aed557ac 100644--- a/test/configure-app.e2e-spec.ts+++ b/test/configure-app.e2e-spec.ts@@ -18,7 +18,8 @@ import { NestFactory } from '@nestjs/core';import request from 'supertest';import { App } from 'supertest/types';import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs';-import { request as httpRequest } from 'http';+import { request as httpRequest, type Server } from 'http';+import type { AddressInfo } from 'net';import { tmpdir } from 'os';import { join } from 'path';import { configureApp } from '../src/configure-app';@@ -240,6 +241,54 @@ describe('production HTTP surface (configureApp)', () => {expect(res.headers['access-control-allow-origin']).toBe('https://allowed.example');});+ // The depth is read from the raw bytes, so the refusal does not wait for the parse: a body that is deeper than+ // the cap and never closes is refused for its depth, not for being invalid JSON.+ it('refuses a body for its depth before it is parsed', async () => {+ const res = await request(app.getHttpServer()).post('/api/echo').type('json').send('['.repeat(101));++ expect(res.status).toBe(400);+ expect(res.body).toMatchObject({+ message: 'Request body must not nest objects or arrays more than 100 levels deep',+ });+ });++ it('does not count brackets inside a string as nesting', async () => {+ const text = `${'['.repeat(5000)}\\"${'{'.repeat(5000)}`;+ const res = await request(app.getHttpServer()).post('/api/echo').type('json').send(`{"text":"${text}"}`);++ expect(res.status).toBe(201);+ });++ // The raw scan reads UTF-8. A body in another UTF charset is decoded by the parser first, so it is walked+ // after the parse instead and meets the same limit. Sent with node's http client: supertest would turn a+ // Buffer into JSON.+ it.each([+ [100, 201],+ [101, 400],+ ])('holds a UTF-16 body nested %i levels to the same limit (%i)', async (levels, status) => {+ const bytes = Buffer.from(nestedObjects(levels), 'utf16le');+ const { port } = (app.getHttpServer() as Server).address() as AddressInfo;+ const answered = await new Promise<number>((resolve, reject) => {+ const req = httpRequest(+ {+ host: '127.0.0.1',+ port,+ path: '/api/echo',+ method: 'POST',+ headers: { 'Content-Type': 'application/json; charset=utf-16le', 'Content-Length': String(bytes.length) },+ },+ res => {+ res.resume();+ resolve(res.statusCode ?? 0);+ },+ );+ req.on('error', reject);+ req.end(bytes);+ });++ expect(answered).toBe(status);+ });+it('answers a DTO route 400, not 500, for a body nested thousands of levels deep', async () => {const mentions = '['.repeat(5000) + ']'.repeat(5000);const res = await request(app.getHttpServer())