Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(http): scan every spelling of UTF-8 before the parse
Yudhi Armyndharis562ab5e201b9
diff --git a/src/common/validation/body-nesting.spec.ts b/src/common/validation/body-nesting.spec.tsindex 3b2e56822..9235bc67f 100644--- a/src/common/validation/body-nesting.spec.ts+++ b/src/common/validation/body-nesting.spec.ts@@ -1,4 +1,4 @@-import { exceedsBodyNesting, exceedsJsonNesting, MAX_BODY_NESTING } from './body-nesting';+import { exceedsBodyNesting, exceedsJsonNesting, isUtf8Charset, MAX_BODY_NESTING } from './body-nesting';// One value holding `levels` nested containers: {} is one level, {a:{}} is two.const objects = (levels: number): unknown => {@@ -178,3 +178,21 @@ describe('exceedsJsonNesting', () => {60_000,);});++describe('isUtf8Charset', () => {+ // The labels iconv-lite, which decodes the body, reads as UTF-8: lower-cased, with every character that is+ // not a letter or digit dropped, and a trailing ':NNNN' year.+ it.each(['utf-8', 'UTF-8', 'Utf-8', 'utf8', 'utf-8.', 'utf-_8', 'utf--8', 'utf-8:1993', 'utf-8!'])(+ 'reads %j as UTF-8',+ charset => {+ expect(isUtf8Charset(charset)).toBe(true);+ },+ );++ it.each(['utf-16', 'utf-16le', 'utf-16be', 'utf-32', 'utf-7', 'utf-80', 'utf-8:19930', 'iso-8859-1', 'ascii', ''])(+ 'does not read %j as UTF-8',+ charset => {+ expect(isUtf8Charset(charset)).toBe(false);+ },+ );+});diff --git a/src/common/validation/body-nesting.ts b/src/common/validation/body-nesting.tsindex bf978d064..4e46d081f 100644--- a/src/common/validation/body-nesting.ts+++ b/src/common/validation/body-nesting.ts@@ -9,6 +9,16 @@ export const MAX_BODY_NESTING = 100;/** The 400 text, the same whether the depth was read from the raw bytes or from the parsed body. */export const BODY_NESTING_MESSAGE = `Request body must not nest objects or arrays more than ${MAX_BODY_NESTING} levels deep`;+/**+ * Whether a charset label names UTF-8 the way the body decoder reads it. body-parser passes the label to+ * iconv-lite, which lower-cases it and drops every character that is not a letter or digit, and a trailing+ * ':NNNN' year, so 'utf-8.', 'utf-_8', 'utf--8' and 'utf-8:1993' all decode as UTF-8. Comparing the label+ * with 'utf-8' would let a client pick the slower path by spelling it differently. The only other labels the+ * parser lets through (it requires the prefix 'utf-') are UTF-16, UTF-32 and UTF-7.+ */+export const isUtf8Charset = (charset: string): boolean =>+ charset.toLowerCase().replace(/:\d{4}$|[^0-9a-z]/g, '') === 'utf8';+const QUOTE = 0x22; // "const BACKSLASH = 0x5c;const OPEN_BRACKET = 0x5b; // [diff --git a/src/configure-app.ts b/src/configure-app.tsindex a8b7738e7..ecd827118 100644--- a/src/configure-app.ts+++ b/src/configure-app.ts@@ -19,7 +19,12 @@ import { injectDashboardCspNonce } from './config/dashboard-csp';import { resolveCorsPolicy, isUpgradeInsecureRequestsEnabled, resolveBodyLimit } from './config/bootstrap-security';import { resolveRequestTimeoutMs } from './config/http-timeouts';import { parseTrustedProxies } from './common/utils/ip';-import { BODY_NESTING_MESSAGE, exceedsBodyNesting, exceedsJsonNesting } from './common/validation/body-nesting';+import {+ BODY_NESTING_MESSAGE,+ exceedsBodyNesting,+ exceedsJsonNesting,+ isUtf8Charset,+} from './common/validation/body-nesting';/** Where the bundled dashboard documents come from, and whether to serve them at all. */export interface DashboardSource {@@ -289,7 +294,7 @@ export function configureApp(app: INestApplication, options: ConfigureAppOptionsinflate: false,verify: (req: Request & { rawBody?: Buffer; nestingScanned?: boolean }, _res, buf, encoding) => {req.rawBody = buf;- if (encoding.toLowerCase() === 'utf-8') {+ if (isUtf8Charset(encoding)) {if (exceedsJsonNesting(buf)) throw new BadRequestException(BODY_NESTING_MESSAGE);req.nestingScanned = true;}diff --git a/test/configure-app.e2e-spec.ts b/test/configure-app.e2e-spec.tsindex 3eef993ae..b58e5f3e7 100644--- a/test/configure-app.e2e-spec.ts+++ b/test/configure-app.e2e-spec.ts@@ -263,34 +263,53 @@ describe('production HTTP surface (configureApp)', () => {expect(res.status).toBe(201);});- // The raw scan reads UTF-8. A body in another UTF charset is decoded by the parser first, so it is walked- // after the parse instead and meets the same limit. Sent with node's http client: supertest would turn a- // Buffer into JSON.- it.each([- [100, 201],- [101, 400],- ])('holds a UTF-16 body nested %i levels to the same limit (%i)', async (levels, status) => {- const bytes = Buffer.from(nestedObjects(levels), 'utf16le');+ // Posts raw bytes with node's http client: supertest would turn a Buffer into JSON.+ const postBytes = async (contentType: string, bytes: Buffer): Promise<{ status: number; body: string }> => {const { port } = (app.getHttpServer() as Server).address() as AddressInfo;- const answered = await new Promise<number>((resolve, reject) => {+ return new Promise((resolve, reject) => {const req = httpRequest({host: '127.0.0.1',port,path: '/api/echo',method: 'POST',- headers: { 'Content-Type': 'application/json; charset=utf-16le', 'Content-Length': String(bytes.length) },+ headers: { 'Content-Type': contentType, 'Content-Length': String(bytes.length) },},res => {- res.resume();- resolve(res.statusCode ?? 0);+ const chunks: Buffer[] = [];+ res.on('data', (chunk: Buffer) => chunks.push(chunk));+ res.on('end', () => resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString('utf8') }));},);req.on('error', reject);req.end(bytes);});+ };+ const NESTING_MESSAGE = 'Request body must not nest objects or arrays more than 100 levels deep';++ // body-parser hands the charset to iconv-lite, which reads every one of these as UTF-8: it lower-cases the+ // name and drops every character that is not a letter or digit, and a trailing ':NNNN' year. A client must+ // not be able to pick the slow path by spelling UTF-8 differently, so each is refused for its depth before+ // the parse, like the plain spelling.+ it.each(['utf-8', 'UTF-8', 'utf-8.', 'utf-_8', 'utf--8', 'utf-8:1993', 'utf-8!'])(+ 'scans a body declared as charset "%s" before it is parsed',+ async charset => {+ const res = await postBytes(`application/json; charset="${charset}"`, Buffer.from('['.repeat(101)));++ expect(res.status).toBe(400);+ expect(JSON.parse(res.body)).toMatchObject({ message: NESTING_MESSAGE });+ },+ );++ // The raw scan reads UTF-8. A body in another UTF charset is decoded by the parser first, so it is walked+ // after the parse instead and meets the same limit.+ it.each([+ [100, 201],+ [101, 400],+ ])('holds a UTF-16 body nested %i levels to the same limit (%i)', async (levels, status) => {+ const res = await postBytes('application/json; charset=utf-16le', Buffer.from(nestedObjects(levels), 'utf16le'));- expect(answered).toBe(status);+ expect(res.status).toBe(status);});it('answers a DTO route 400, not 500, for a body nested thousands of levels deep', async () => {