Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(plugins): measure a sandboxed log meta as the logger writes it
Yudhi Armyndharis22bf7978ef39
diff --git a/src/common/services/logger.service.ts b/src/common/services/logger.service.tsindex 50c90997d..83d4a443b 100644--- a/src/common/services/logger.service.ts+++ b/src/common/services/logger.service.ts@@ -60,7 +60,9 @@ export interface RedactOptions {marker?: string;}-const isSecretName = (key: string): boolean => SECRET_KEY_PATTERN.test(key);+/** Whether a key's name marks its value as a credential, and what is written in its place, unless told otherwise. */+export const isSecretName = (key: string): boolean => SECRET_KEY_PATTERN.test(key);+export const REDACTED = '[REDACTED]';const TYPED_ARRAY_PROTOTYPE: object = Object.getPrototypeOf(Int8Array.prototype) as object;@@ -103,6 +105,40 @@ function formOf(node: object): 'date' | 'map' | 'set' | 'boxed' | 'error' | unde}}+/**+ * What one copy writes for an object before it copies its parts: a Date as its text, a boxed value as its value, a Map+ * as the array of its [key, value] pairs (a credential key's value replaced), a Set as the array of its values, a+ * typed array as its type and length and an Error as its name, message and stack before its own properties. Any other+ * object is returned as it is, to be copied by its own properties. The sandbox log relay measures a plugin's meta+ * through it, so the two write every form alike.+ */+export function writtenForm(node: object, form = formOf(node), isSecretKey = isSecretName, marker = REDACTED): unknown {+ if (form === 'date') return dateText(node as Date);+ if (form === 'boxed') return boxedValue(node);+ if (form === 'map' || form === 'set') {+ const items: unknown[] = [];+ if (form === 'map')+ Map.prototype.forEach.call(node, (item: unknown, key: unknown) =>+ items.push([key, typeof key === 'string' && isSecretKey(key) ? marker : item]),+ );+ else Set.prototype.forEach.call(node, (item: unknown) => items.push(item));+ return items;+ }+ // A typed array, a Buffer included, is written as its type and the elements it holds, never one entry per element.+ // A DataView has no type of that kind and is copied as an object.+ const type: unknown = ArrayBuffer.isView(node) ? Reflect.get(TYPED_ARRAY_PROTOTYPE, Symbol.toStringTag, node) : 0;+ const props =+ typeof type === 'string'+ ? {+ type: Object.getPrototypeOf(node) === Buffer.prototype ? 'Buffer' : type,+ length: elementCount(node as ArrayBufferView),+ }+ : node;+ if (form !== 'error') return props;+ const error = node as Error;+ return { name: error.name, message: error.message, stack: error.stack, ...props };+}+/*** Return a copy of `value` with credential keys' values replaced by `[REDACTED]` (recursive). The copy always* serializes: an object met again inside itself becomes `[Circular]`, a nested BigInt its decimal text, and@@ -112,7 +148,7 @@ function formOf(node: object): 'date' | 'map' | 'set' | 'boxed' | 'error' | unde* array of its [key, value] pairs and a Set as the array of its values. Shared with the Baileys wire logger.*/export function redactSecrets(value: unknown, options: RedactOptions = {}): unknown {- const { isSecretKey = isSecretName, marker = '[REDACTED]' } = options;+ const { isSecretKey = isSecretName, marker = REDACTED } = options;const budget = { nodes: REDACT_DEEP_NODES, entries: REDACT_MAX_ENTRIES, chars: REDACT_MAX_CHARS };const walk = (node: unknown, path: object[]): unknown => {// A function is not data: it is left out, as JSON.stringify leaves it out of an object and writes null in an array.@@ -120,8 +156,8 @@ export function redactSecrets(value: unknown, options: RedactOptions = {}): unknif (typeof node === 'string') return (budget.chars -= node.length) < 0 ? '[Truncated]' : node;if (node === null || typeof node !== 'object') return node;const form = formOf(node);- if (form === 'date') return walk(dateText(node as Date), path);- if (form === 'boxed') return walk(boxedValue(node), path);+ // A Date and a boxed value are written as a primitive, charged like one.+ if (form === 'date' || form === 'boxed') return walk(writtenForm(node, form, isSecretKey, marker), path);if (path.includes(node)) return '[Circular]';const deep = path.length >= REDACT_FULL_DEPTH;// A negative budget means something that did not fit was met: nothing else is copied or listed after it.@@ -137,19 +173,12 @@ export function redactSecrets(value: unknown, options: RedactOptions = {}): unkn};const left = Math.min(budget.entries, deep ? budget.nodes : Infinity);const room = Math.min(left, REDACT_MAX_ARRAY_LENGTH);- // A Map is copied as the array of its pairs, a credential key's value replaced, and a Set as the array of its- // values; its size is held against the room before any entry is read.+ // A Map or a Set is copied as the array its form is; its size is held against the room before any entry is read.let list: unknown = node;const map = form === 'map';if (map || form === 'set') {if (Reflect.get(map ? Map.prototype : Set.prototype, 'size', node) > room) return '[Truncated]';- const items: unknown[] = [];- if (map)- Map.prototype.forEach.call(node, (item: unknown, key: unknown) =>- items.push([key, typeof key === 'string' && isSecretKey(key) ? marker : item]),- );- else Set.prototype.forEach.call(node, (item: unknown) => items.push(item));- list = items;+ list = writtenForm(node, form, isSecretKey, marker);}if (Array.isArray(list)) {// Read once and taken as a count (whole, from zero), as map takes it: one number gates the copy and sizes it.@@ -161,19 +190,7 @@ export function redactSecrets(value: unknown, options: RedactOptions = {}): unknfor (let index = 0; index < length; index++) if (index in list) plain[index] = copy(list[index]);return plain;}- // A typed array, a Buffer included, is written as its type and the elements it holds, never one entry per element.- // A DataView has no type of that kind and is copied as an object.- const type: unknown = ArrayBuffer.isView(node) ? Reflect.get(TYPED_ARRAY_PROTOTYPE, Symbol.toStringTag, node) : 0;- const props =- typeof type === 'string'- ? {- type: Object.getPrototypeOf(node) === Buffer.prototype ? 'Buffer' : type,- length: elementCount(node as ArrayBufferView),- }- : (node as Record<string, unknown>);- const error = node as Error;- const source =- form === 'error' ? { name: error.name, message: error.message, stack: error.stack, ...props } : props;+ const source = writtenForm(node, form, isSecretKey, marker) as Record<string, unknown>;const keys = Object.keys(source);if (keys.length > left) {// Listed once, then charged: another reference to this object, or any other node, is refused without listing.diff --git a/src/core/plugins/plugin-loader-sandbox-routing.spec.ts b/src/core/plugins/plugin-loader-sandbox-routing.spec.tsindex 4af129e35..bf2698623 100644--- a/src/core/plugins/plugin-loader-sandbox-routing.spec.ts+++ b/src/core/plugins/plugin-loader-sandbox-routing.spec.ts@@ -948,7 +948,8 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {});// Few values, but a long text once written: numbers in a Map, or a BigInt in a Set, which JSON.stringify alone- // would write as {}.+ // would write as {}. A worker's meta arrives in the host's realm; structuredClone here would make it in another,+ // which the logger copies by its own properties.it('measures the text of a Map or a Set as the logger writes it', async () => {const loader = makeLoader();seed(loader, { builtIn: false, instance: null });@@ -958,8 +959,8 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {const pairs = Array.from({ length: 2000 }, (_, key): [number, number] => [float * (key + 1), float]);const started = Date.now();- loader.capturedOnLog!('log', 'floats', structuredClone({ m: new Map(pairs) }));- loader.capturedOnLog!('log', 'bigints', structuredClone({ s: new Set([10n ** 100_000n, 10n ** 100_000n + 1n]) }));+ loader.capturedOnLog!('log', 'floats', { m: new Map(pairs) });+ loader.capturedOnLog!('log', 'bigints', { s: new Set([10n ** 100_000n, 10n ** 100_000n + 1n]) });expect(Date.now() - started).toBeLessThan(1000);const [floats, bigints] = logSpy.mock.calls.map(call => call[1] as Record<string, unknown>);@@ -968,6 +969,29 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {expect(bigints).toEqual({});});+ // Few values, but a longer text once written: an invalid Date is written as text where JSON.stringify writes null,+ // and the value of a credential key as the marker however short it is. A typed array is written as its type and+ // length, so one that fits the count is relayed.+ it('measures an invalid Date, a credential value and a typed array as the logger writes them', async () => {+ const loader = makeLoader();+ seed(loader, { builtIn: false, instance: null });+ await loader.enablePlugin('p1');+ const logSpy = jest.spyOn(loggerOf(loader), 'log').mockImplementation(() => undefined);+ const invalid = new Date(Number.NaN);++ loader.capturedOnLog!('log', 'dates', { d: Array.from({ length: 1600 }, () => new Date(Number.NaN)) });+ loader.capturedOnLog!('log', 'shared', { d: Array.from({ length: 1600 }, () => invalid) });+ loader.capturedOnLog!('log', 'tokens', { l: Array.from({ length: 600 }, () => ({ token: 0 })) });+ loader.capturedOnLog!('log', 'bytes', { b: new Uint8Array(5000) });++ const [dates, shared, tokens, bytes] = logSpy.mock.calls.map(call => call[1] as Record<string, unknown>);+ for (const marked of [dates, shared, tokens]) {+ expect(marked).toMatchObject({ metaTruncated: true });+ expect(marked.metaLength).toBeGreaterThan(8192);+ }+ expect(JSON.stringify(redactSecrets(bytes))).toBe('{"b":{"type":"Uint8Array","length":5000}}');+ });+// A boxed string is written as its text, so one that fits is relayed and written in that many characters. A worker's// meta arrives in the host's realm; structuredClone here would make it in another, which the logger copies by its// own properties.diff --git a/src/core/plugins/sandbox-log-meta.spec.ts b/src/core/plugins/sandbox-log-meta.spec.tsindex f7af86ecf..fc6dc7970 100644--- a/src/core/plugins/sandbox-log-meta.spec.ts+++ b/src/core/plugins/sandbox-log-meta.spec.ts@@ -1,3 +1,4 @@+import { redactSecrets } from '../../common/services/logger.service';import { countMetaValues, metaText } from './sandbox-log-meta';describe('countMetaValues', () => {@@ -198,5 +199,36 @@ describe('metaText', () => {expect(metaText({ a: shared, b: shared })).toBe('{"a":[1],"b":[1]}');expect(() => metaText({ self })).toThrow(TypeError);expect(() => metaText({ s: new Set([10n]) })).toThrow(TypeError);+ expect(() => metaText({ boxed: Object(10n) as object })).toThrow(TypeError);+ });++ // What the relay measures is the text the logger writes for the same meta, form by form.+ it.each([+ ['an invalid Date', { d: new Date(Number.NaN), list: [new Date(Number.NaN)] }],+ ['a valid Date', { d: new Date(0) }],+ ['a boxed string, number and boolean', { s: new String('ab'), n: new Number(5), b: new Boolean(false) }],+ ['an Error with own properties', { e: Object.assign(new Error('boom'), { token: 't', code: 7 }) }],+ [+ 'a Map with a credential key',+ {+ m: new Map<unknown, unknown>([+ ['password', 'p'],+ [1, { apiKey: 'k' }],+ ]),+ },+ ],+ ['a Set holding a credential key', { s: new Set([1, { secret: 's' }]) }],+ [+ 'a typed array, a Buffer and a DataView',+ { u: new Uint8Array(3), b: Buffer.from('ab'), v: new DataView(new ArrayBuffer(4)) },+ ],+ [+ 'credential keys at every depth',+ { token: 0, list: [{ password: 'p', ok: 1 }], nested: { a: { authorization: null } } },+ ],+ ['a RegExp and an ArrayBuffer', { r: /x/g, a: new ArrayBuffer(2) }],+ ['a meta that is an array', [new Date(Number.NaN), new String('x'), { token: 1 }]],+ ])('measures %s as the logger writes it', (_name, meta) => {+ expect(metaText(meta)).toBe(JSON.stringify(redactSecrets(meta)));});});diff --git a/src/core/plugins/sandbox-log-meta.ts b/src/core/plugins/sandbox-log-meta.tsindex 87fa442c1..8797d1dc3 100644--- a/src/core/plugins/sandbox-log-meta.ts+++ b/src/core/plugins/sandbox-log-meta.ts@@ -1,4 +1,5 @@import { types } from 'node:util';+import { isSecretName, REDACTED, writtenForm } from '../../common/services/logger.service';/*** Counts what JSON.stringify would write for a worker's log metadata, and stops once the count passes `limit`: one@@ -12,9 +13,10 @@ import { types } from 'node:util';* written at every use of its object.** The count can be higher than the length of the text. A property that JSON.stringify skips (undefined, a function,- * a symbol) is counted as if it were written, and so is the value under a credential key of a Map. A metadata whose- * count passes `limit` is replaced by the size marker whatever its text would measure; for data that is plain JSON- * the count never exceeds the length of the text, so a count past `limit` means a longer text.+ * a symbol) is counted as if it were written, and so is the value under a credential key, which is written as the+ * marker. A metadata whose count passes `limit` is replaced by the size marker whatever its text would measure; for+ * data that is plain JSON without a credential key the count never exceeds the length of the text, so a count past+ * `limit` means a longer text.*/export function countMetaValues(meta: unknown, limit: number): number {let count = 0;@@ -74,23 +76,20 @@ export function countMetaValues(meta: unknown, limit: number): number {}/**- * The JSON text of a worker's log metadata, with what JSON.stringify alone writes as {} written as the logger writes it:- * an Error with its name, message and stack, a Map as the array of its [key, value] pairs and a Set as the array of its- * values. Each is rendered once, so one that holds itself is a cycle JSON.stringify refuses like any other, and a BigInt- * in one is refused as it is anywhere else in the metadata. Run only on a meta whose count fits the limit.+ * The JSON text the logger writes for a worker's log metadata: every object in the form the logger's copy writes it+ * (writtenForm), and the value of a credential key as the marker. Each object is rendered once, so one that holds+ * itself is a cycle JSON.stringify refuses like any other, and a BigInt, boxed or not, is refused wherever it is: its+ * text is never built on the host. Run only on a meta whose count fits the limit.*/export function metaText(meta: unknown): string | undefined {const rendered = new Map<object, unknown>();- return JSON.stringify(meta, (_key, value: unknown) => {- if (typeof value !== 'object' || value === null) return value;- if (!types.isMap(value) && !types.isSet(value) && !types.isNativeError(value)) return value;- if (!rendered.has(value))- rendered.set(- value,- types.isNativeError(value)- ? { name: value.name, message: value.message, stack: value.stack, ...(value as object) }- : Array.from(value as Iterable<unknown>),- );- return rendered.get(value);+ return JSON.stringify(meta, function (this: Record<string, unknown>, key: string, value: unknown): unknown {+ // Read from the holder: JSON.stringify has already put what a Date's toJSON returns (null for an invalid one) in+ // its place.+ const raw: unknown = this[key];+ if (!Array.isArray(this) && isSecretName(key)) return REDACTED;+ if (typeof raw !== 'object' || raw === null) return value;+ if (!rendered.has(raw)) rendered.set(raw, writtenForm(raw));+ return rendered.get(raw);});}