Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(http): scan every spelling of UTF-8 before the parse

Yudhi Armyndharis562ab5e201b9
diff --git a/src/common/validation/body-nesting.spec.ts b/src/common/validation/body-nesting.spec.ts
index 3b2e56822..9235bc67f 100644
--- a/src/common/validation/body-nesting.spec.ts
+++ b/src/common/validation/body-nesting.spec.ts
@@ -1,4 +1,4 @@
-import { exceedsBodyNesting, exceedsJsonNesting, MAX_BODY_NESTING } from './body-nesting';
+import { exceedsBodyNesting, exceedsJsonNesting, isUtf8Charset, MAX_BODY_NESTING } from './body-nesting';
// One value holding `levels` nested containers: {} is one level, {a:{}} is two.
const objects = (levels: number): unknown => {
@@ -178,3 +178,21 @@ describe('exceedsJsonNesting', () => {
60_000,
);
});
+
+describe('isUtf8Charset', () => {
+ // The labels iconv-lite, which decodes the body, reads as UTF-8: lower-cased, with every character that is
+ // not a letter or digit dropped, and a trailing ':NNNN' year.
+ it.each(['utf-8', 'UTF-8', 'Utf-8', 'utf8', 'utf-8.', 'utf-_8', 'utf--8', 'utf-8:1993', 'utf-8!'])(
+ 'reads %j as UTF-8',
+ charset => {
+ expect(isUtf8Charset(charset)).toBe(true);
+ },
+ );
+
+ it.each(['utf-16', 'utf-16le', 'utf-16be', 'utf-32', 'utf-7', 'utf-80', 'utf-8:19930', 'iso-8859-1', 'ascii', ''])(
+ 'does not read %j as UTF-8',
+ charset => {
+ expect(isUtf8Charset(charset)).toBe(false);
+ },
+ );
+});
diff --git a/src/common/validation/body-nesting.ts b/src/common/validation/body-nesting.ts
index bf978d064..4e46d081f 100644
--- a/src/common/validation/body-nesting.ts
+++ b/src/common/validation/body-nesting.ts
@@ -9,6 +9,16 @@ export const MAX_BODY_NESTING = 100;
/** The 400 text, the same whether the depth was read from the raw bytes or from the parsed body. */
export const BODY_NESTING_MESSAGE = `Request body must not nest objects or arrays more than ${MAX_BODY_NESTING} levels deep`;
+/**
+ * Whether a charset label names UTF-8 the way the body decoder reads it. body-parser passes the label to
+ * iconv-lite, which lower-cases it and drops every character that is not a letter or digit, and a trailing
+ * ':NNNN' year, so 'utf-8.', 'utf-_8', 'utf--8' and 'utf-8:1993' all decode as UTF-8. Comparing the label
+ * with 'utf-8' would let a client pick the slower path by spelling it differently. The only other labels the
+ * parser lets through (it requires the prefix 'utf-') are UTF-16, UTF-32 and UTF-7.
+ */
+export const isUtf8Charset = (charset: string): boolean =>
+ charset.toLowerCase().replace(/:\d{4}$|[^0-9a-z]/g, '') === 'utf8';
+
const QUOTE = 0x22; // "
const BACKSLASH = 0x5c;
const OPEN_BRACKET = 0x5b; // [
diff --git a/src/configure-app.ts b/src/configure-app.ts
index a8b7738e7..ecd827118 100644
--- a/src/configure-app.ts
+++ b/src/configure-app.ts
@@ -19,7 +19,12 @@ import { injectDashboardCspNonce } from './config/dashboard-csp';
import { resolveCorsPolicy, isUpgradeInsecureRequestsEnabled, resolveBodyLimit } from './config/bootstrap-security';
import { resolveRequestTimeoutMs } from './config/http-timeouts';
import { parseTrustedProxies } from './common/utils/ip';
-import { BODY_NESTING_MESSAGE, exceedsBodyNesting, exceedsJsonNesting } from './common/validation/body-nesting';
+import {
+ BODY_NESTING_MESSAGE,
+ exceedsBodyNesting,
+ exceedsJsonNesting,
+ isUtf8Charset,
+} from './common/validation/body-nesting';
/** Where the bundled dashboard documents come from, and whether to serve them at all. */
export interface DashboardSource {
@@ -289,7 +294,7 @@ export function configureApp(app: INestApplication, options: ConfigureAppOptions
inflate: false,
verify: (req: Request & { rawBody?: Buffer; nestingScanned?: boolean }, _res, buf, encoding) => {
req.rawBody = buf;
- if (encoding.toLowerCase() === 'utf-8') {
+ if (isUtf8Charset(encoding)) {
if (exceedsJsonNesting(buf)) throw new BadRequestException(BODY_NESTING_MESSAGE);
req.nestingScanned = true;
}
diff --git a/test/configure-app.e2e-spec.ts b/test/configure-app.e2e-spec.ts
index 3eef993ae..b58e5f3e7 100644
--- a/test/configure-app.e2e-spec.ts
+++ b/test/configure-app.e2e-spec.ts
@@ -263,34 +263,53 @@ describe('production HTTP surface (configureApp)', () => {
expect(res.status).toBe(201);
});
- // The raw scan reads UTF-8. A body in another UTF charset is decoded by the parser first, so it is walked
- // after the parse instead and meets the same limit. Sent with node's http client: supertest would turn a
- // Buffer into JSON.
- it.each([
- [100, 201],
- [101, 400],
- ])('holds a UTF-16 body nested %i levels to the same limit (%i)', async (levels, status) => {
- const bytes = Buffer.from(nestedObjects(levels), 'utf16le');
+ // Posts raw bytes with node's http client: supertest would turn a Buffer into JSON.
+ const postBytes = async (contentType: string, bytes: Buffer): Promise<{ status: number; body: string }> => {
const { port } = (app.getHttpServer() as Server).address() as AddressInfo;
- const answered = await new Promise<number>((resolve, reject) => {
+ return new Promise((resolve, reject) => {
const req = httpRequest(
{
host: '127.0.0.1',
port,
path: '/api/echo',
method: 'POST',
- headers: { 'Content-Type': 'application/json; charset=utf-16le', 'Content-Length': String(bytes.length) },
+ headers: { 'Content-Type': contentType, 'Content-Length': String(bytes.length) },
},
res => {
- res.resume();
- resolve(res.statusCode ?? 0);
+ const chunks: Buffer[] = [];
+ res.on('data', (chunk: Buffer) => chunks.push(chunk));
+ res.on('end', () => resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString('utf8') }));
},
);
req.on('error', reject);
req.end(bytes);
});
+ };
+ const NESTING_MESSAGE = 'Request body must not nest objects or arrays more than 100 levels deep';
+
+ // body-parser hands the charset to iconv-lite, which reads every one of these as UTF-8: it lower-cases the
+ // name and drops every character that is not a letter or digit, and a trailing ':NNNN' year. A client must
+ // not be able to pick the slow path by spelling UTF-8 differently, so each is refused for its depth before
+ // the parse, like the plain spelling.
+ it.each(['utf-8', 'UTF-8', 'utf-8.', 'utf-_8', 'utf--8', 'utf-8:1993', 'utf-8!'])(
+ 'scans a body declared as charset "%s" before it is parsed',
+ async charset => {
+ const res = await postBytes(`application/json; charset="${charset}"`, Buffer.from('['.repeat(101)));
+
+ expect(res.status).toBe(400);
+ expect(JSON.parse(res.body)).toMatchObject({ message: NESTING_MESSAGE });
+ },
+ );
+
+ // The raw scan reads UTF-8. A body in another UTF charset is decoded by the parser first, so it is walked
+ // after the parse instead and meets the same limit.
+ it.each([
+ [100, 201],
+ [101, 400],
+ ])('holds a UTF-16 body nested %i levels to the same limit (%i)', async (levels, status) => {
+ const res = await postBytes('application/json; charset=utf-16le', Buffer.from(nestedObjects(levels), 'utf16le'));
- expect(answered).toBe(status);
+ expect(res.status).toBe(status);
});
it('answers a DTO route 400, not 500, for a body nested thousands of levels deep', async () => {