Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(logger): write errors, dates, maps, sets and buffers readably
Yudhi Armyndharis675990677fac
diff --git a/docs/10-devops-infrastructure.md b/docs/10-devops-infrastructure.mdindex b60d7bbcb..2551bf2ac 100644--- a/docs/10-devops-infrastructure.md+++ b/docs/10-devops-infrastructure.md@@ -1106,11 +1106,15 @@ looks like a secret (password, token, api-key, authorization, …) keeps the keyreplaced with `[REDACTED]` before the line is written, at any depth. A log call never throws on itsmetadata: an object that contains itself is written with `[Circular]` in its place, and below the fifthlevel at most 1000 more objects and arrays are copied, the rest written as `[Truncated]`. So is an array,-typed array or object with more entries than that allowance has left (a list of 1000 numbers at the sixth-level), and, at any level, an array or typed array of more than a million entries. At every level one call+Map, Set or object with more entries than that allowance has left (a list of 1000 numbers at the sixth+level), and, at any level, an array, Map or Set of more than a million entries. At every level one callcopies at most two million entries in all (the keys of its objects plus the lengths of its arrays) and writes atmost 2^26 (about 67 million) characters of string values, key names and BigInt text (sixteen to the digit); avalue referenced again is counted again at each use, and what no longer fits is written as `[Truncated]`.+A value that keeps its data outside its own properties is written readably, at any depth: an Error as its+`name`, `message` and `stack` before its own properties, a Date as its ISO text (`Invalid Date` when it holds+none), a Buffer or typed array as its `type` and `length`, never its bytes, a Map as an array of+`[key, value]` pairs (the value of a secret-named string key redacted) and a Set as an array of its values.The same logger is installed as Nest's framework logger, so framework lines (route mapping, unhandledexception stacks from the exception handler) follow `LOG_LEVEL` and `LOG_FORMAT` and carry thediff --git a/src/common/services/logger.service.spec.ts b/src/common/services/logger.service.spec.tsindex e4aa87ac7..8ea04c2ad 100644--- a/src/common/services/logger.service.spec.ts+++ b/src/common/services/logger.service.spec.ts@@ -416,28 +416,29 @@ describe('LoggerService', () => {});});- // Object.entries lists one entry per byte of a Buffer, so a few megabytes cost seconds and hundreds of megabytes.+ // Object.entries lists one entry per byte of a Buffer, so copying one that way costs seconds per few megabytes.describe('a Buffer of 5 million bytes', () => {const parsed = (): Record<string, unknown> => getLogOutput(consoleSpy) as unknown as Record<string, unknown>;+ const written = { type: 'Buffer', length: 5_000_000 };- it('is written as [Truncated] at once in the first levels, with the rest of the line kept', () => {+ it('is written as its type and length at once in the first levels, with the rest of the line kept', () => {const file = Buffer.alloc(5_000_000);const started = Date.now();logger.log('saved', { sessionId: 's1', file });expect(Date.now() - started).toBeLessThan(1000);- expect(parsed()).toMatchObject({ message: 'saved', sessionId: 's1', file: '[Truncated]' });+ expect(parsed()).toMatchObject({ message: 'saved', sessionId: 's1', file: written });});- it('is written as [Truncated] at once below the fifth level, with the rest of the line kept', () => {+ it('is written as its type and length at once below the fifth level, with the rest of the line kept', () => {const file = Buffer.alloc(5_000_000);const started = Date.now();logger.log('saved', { sessionId: 's1', a: { b: { c: { d: { e: { file } } } } } });expect(Date.now() - started).toBeLessThan(1000);- expect(parsed()).toMatchObject({ sessionId: 's1', a: { b: { c: { d: { e: { file: '[Truncated]' } } } } } });+ expect(parsed()).toMatchObject({ sessionId: 's1', a: { b: { c: { d: { e: { file: written } } } } } });});});@@ -923,13 +924,183 @@ describe('redactSecrets', () => {});});- it('copies an Error with its name, message and stack, and its own props through the same checks, when asked to', () => {- const error = Object.assign(new Error('boom'), { token: 't', attempts: 5n });+ // What these values hold is not in their own enumerable properties, so a copy of those alone would write {} or one+ // entry per byte. Each is written in a readable form instead, at every depth.+ describe('a value that keeps its data outside its own properties', () => {+ const deep = (value: unknown): unknown => ({ a: { b: { c: { d: { e: { f: { value } } } } } } });+ const deepValue = (copy: unknown): unknown =>+ (copy as { a: { b: { c: { d: { e: { f: { value: unknown } } } } } } }).a.b.c.d.e.f.value;+ const shallowAndDeep = (value: unknown): unknown[] => [+ (redactSecrets({ value }) as { value: unknown }).value,+ deepValue(redactSecrets(deep(value))),+ ];++ it('copies an Error with its name, message and stack before its own props, which go through the same checks', () => {+ const error = Object.assign(new Error('boom'), { token: 't', attempts: 5n });++ for (const copy of shallowAndDeep(error) as Record<string, unknown>[]) {+ expect(Object.keys(copy)).toEqual(['name', 'message', 'stack', 'token', 'attempts']);+ expect(copy).toMatchObject({ name: 'Error', message: 'boom', token: '[REDACTED]', attempts: '5' });+ expect(copy.stack).toContain('Error: boom');+ }+ });++ it('charges the message and stack of an Error to the characters one copy may write', () => {+ const error = new Error('x'.repeat(2 ** 26));++ expect(redactSecrets({ error, after: 'kept?' })).toEqual({+ error: { name: 'Error', message: '[Truncated]', stack: '[Truncated]' },+ after: '[Truncated]',+ });+ });++ it('writes a Date as its ISO text through the built-in method, and an invalid one as Invalid Date', () => {+ const forged = Object.assign(new Date(0), { toISOString: () => 'forged' });++ expect(shallowAndDeep(new Date(0))).toEqual(['1970-01-01T00:00:00.000Z', '1970-01-01T00:00:00.000Z']);+ expect(shallowAndDeep(new Date(Number.NaN))).toEqual(['Invalid Date', 'Invalid Date']);+ expect(shallowAndDeep(forged)).toEqual(['1970-01-01T00:00:00.000Z', '1970-01-01T00:00:00.000Z']);+ });++ it('writes a Buffer or a typed array as its type and the elements it has, never its bytes', () => {+ const claiming = Object.defineProperty(new Uint8Array(5_000_001), 'length', { value: 0 });++ expect(shallowAndDeep(Buffer.from('ab'))).toEqual([+ { type: 'Buffer', length: 2 },+ { type: 'Buffer', length: 2 },+ ]);+ expect(shallowAndDeep(new Float64Array(3))[1]).toEqual({ type: 'Float64Array', length: 3 });+ expect(shallowAndDeep(claiming)).toEqual([+ { type: 'Uint8Array', length: 5_000_001 },+ { type: 'Uint8Array', length: 5_000_001 },+ ]);+ expect(shallowAndDeep(new DataView(new ArrayBuffer(8)))).toEqual([{}, {}]);+ });++ it('writes a Map as the array of its pairs, the value of a credential key replaced, and a Set as its values', () => {+ const map = new Map<unknown, unknown>([+ ['password', 'value-map'],+ [1, { token: 't', id: 1 }],+ [{ id: 2 }, 'v'],+ ]);++ for (const copy of shallowAndDeep(map)) {+ expect(copy).toEqual([+ ['password', '[REDACTED]'],+ [1, { token: '[REDACTED]', id: 1 }],+ [{ id: 2 }, 'v'],+ ]);+ }+ expect(shallowAndDeep(new Set([1, 'a', { password: 'p' }]))[1]).toEqual([1, 'a', { password: '[REDACTED]' }]);+ expect(redactSecrets(map, { isSecretKey: key => key === 'id', marker: 'M' })).toEqual([+ ['password', 'value-map'],+ [1, { token: 't', id: 'M' }],+ [{ id: 'M' }, 'v'],+ ]);+ });++ it('writes a Map or a Set that holds itself with [Circular], and one held twice in full each time', () => {+ const map = new Map<string, unknown>();+ map.set('self', map);+ const set = new Set([1]);++ expect(redactSecrets({ map, twice: [set, set] })).toEqual({ map: [['self', '[Circular]']], twice: [[1], [1]] });+ });++ it('reads a Map through the built-in size and entries, not what a subclass defines', () => {+ class Lying<K, V> extends Map<K, V> {+ override get size(): number {+ return 0;+ }+ override forEach(): void {}+ }+ const many = new Lying(Array.from({ length: 1_000_001 }, (_, key) => [key, 1]));++ expect(redactSecrets(new Lying([[1, 2]]))).toEqual([[1, 2]]);+ const read = jest.spyOn(Map.prototype, 'forEach');+ try {+ expect(redactSecrets({ many })).toEqual({ many: '[Truncated]' });+ expect(read).not.toHaveBeenCalled();+ } finally {+ read.mockRestore();+ }+ });++ it('tests only a key that is a string against the credential names, and copies any other key', () => {+ const unprintable = {+ toString(): never {+ throw new Error('toString');+ },+ id: 1,+ };++ expect(JSON.stringify(redactSecrets(new Map<unknown, unknown>([[unprintable, 'v']])))).toBe('[[{"id":1},"v"]]');+ });++ // A Proxy is copied by its own properties, whatever its traps answer; a value made from one of these prototypes+ // alone lacks the data the built-in methods read.+ it('copies a Proxy, or a value that only has one of these prototypes, by its own properties', () => {+ const prototypeTrap = new Proxy(+ { a: 1 },+ {+ getPrototypeOf: (): never => {+ throw new Error('getPrototypeOf');+ },+ },+ );+ const fakes = [Map, Set, Date].map(Kind => Object.assign(Object.create(Kind.prototype) as object, { id: 1 }));++ expect(+ redactSecrets({ prototypeTrap, map: new Proxy(new Map([[1, 2]]), {}), error: new Proxy(new Error('x'), {}) }),+ ).toEqual({ prototypeTrap: { a: 1 }, map: {}, error: {} });+ expect(redactSecrets(fakes)).toEqual([{ id: 1 }, { id: 1 }, { id: 1 }]);+ });++ it('writes a Map or a Set with more entries than are left as [Truncated] before reading one', () => {+ const entries = (size: number): Map<number, number> =>+ new Map(Array.from({ length: size }, (_, key) => [key, 1]));+ const fourth = (list: unknown): unknown => ({ a: { b: { c: { d: { list } } } } });+ const big = entries(1_000_001);+ const read = jest.spyOn(Map.prototype, 'forEach');+ try {+ expect(redactSecrets({ big })).toEqual({ big: '[Truncated]' });+ expect(redactSecrets(fourth(entries(1000)))).toEqual(fourth('[Truncated]'));+ expect(read).not.toHaveBeenCalled();+ } finally {+ read.mockRestore();+ }+ expect(+ (redactSecrets(fourth(entries(999))) as { a: { b: { c: { d: { list: unknown[] } } } } }).a.b.c.d.list,+ ).toHaveLength(999);+ expect(redactSecrets(fourth(new Set(Array.from({ length: 1000 }, (_, key) => key))))).toEqual(+ fourth('[Truncated]'),+ );+ });- const copy = redactSecrets({ error }, { errors: true }) as { error: Record<string, unknown> };+ it.each([LogFormat.JSON, LogFormat.PRETTY])('is written readably in %s output', format => {+ const log = jest.spyOn(console, 'log').mockImplementation();+ LoggerService.setLogFormat(format);+ try {+ createLogger('Test').log('m', {+ at: new Date(0),+ bytes: Buffer.from('ab'),+ map: new Map([[1, 2]]),+ set: new Set([1]),+ error: new Error('boom'),+ });- expect(copy.error).toMatchObject({ name: 'Error', message: 'boom', token: '[REDACTED]', attempts: '5' });- expect(copy.error.stack).toContain('Error: boom');+ const line = (log.mock.calls as string[][])[0][0];+ const fields =+ format === LogFormat.JSON+ ? ['"at":"1970-01-01T00:00:00.000Z"', '"bytes":{"type":"Buffer","length":2}', '"map":[[1,2]]', '"set":[1]']+ : ['at=1970-01-01T00:00:00.000Z', 'bytes={"type":"Buffer","length":2}', 'map=[[1,2]]', 'set=[1]'];+ for (const field of fields) expect(line).toContain(field);+ expect(line).toContain('"name":"Error","message":"boom","stack":"Error: boom');+ } finally {+ log.mockRestore();+ LoggerService.setLogFormat(null);+ }+ });});// JSON.parse makes a property named __proto__ an ordinary own property; the copy must define it as one too.@@ -1086,15 +1257,15 @@ describe('redactSecrets', () => {it.each([['an own length property', redefined],['a length getter', (): Uint8Array => new Claiming(1_000_001)],- ])('is written as [Truncated] by the elements it has, with %s', (_name, make) => {- expect(redactSecrets({ file: make() })).toEqual({ file: '[Truncated]' });- expect(redactSecrets(make())).toBe('[Truncated]');+ ])('is written with the elements it has, with %s', (_name, make) => {+ expect(redactSecrets({ file: make() })).toEqual({ file: { type: 'Uint8Array', length: 1_000_001 } });+ expect(redactSecrets(make())).toEqual({ type: 'Uint8Array', length: 1_000_001 });});it('counts the elements of a typed array, not its bytes', () => {- const copy = redactSecrets({ list: new Float64Array(150_000) }) as { list: object };-- expect(Object.keys(copy.list)).toHaveLength(150_000);+ expect(redactSecrets({ list: new Float64Array(150_000) })).toEqual({+ list: { type: 'Float64Array', length: 150_000 },+ });});it('copies a DataView, which has no elements, as an empty object', () => {@@ -1102,7 +1273,7 @@ describe('redactSecrets', () => {});});- // A typed array lists one entry per element, as an array does, and an object one per key.+ // An object lists one entry per key; a typed array is written as its type and length, whatever its length.describe('a typed array or a wide object', () => {const deep = (list: unknown): unknown => ({ a: { b: { c: { d: { list } } } } });const keysOf = (copy: unknown): number =>@@ -1110,10 +1281,10 @@ describe('redactSecrets', () => {const wide = (keys: number): Record<string, number> =>Object.fromEntries(Array.from({ length: keys }, (_, key) => [`k${key}`, key]));- it('copies a deep typed array that fits in what is left of the node budget and writes a longer one as [Truncated]', () => {- expect(keysOf(redactSecrets(deep(new Uint8Array(999))))).toBe(999);- expect(redactSecrets(deep(new Uint8Array(1000)))).toEqual(deep('[Truncated]'));- expect(redactSecrets(deep(Buffer.alloc(1000)))).toEqual(deep('[Truncated]'));+ it('writes a deep typed array as its type and length, also one longer than what is left of the node budget', () => {+ expect(redactSecrets(deep(new Uint8Array(999)))).toEqual(deep({ type: 'Uint8Array', length: 999 }));+ expect(redactSecrets(deep(new Uint8Array(1000)))).toEqual(deep({ type: 'Uint8Array', length: 1000 }));+ expect(redactSecrets(deep(Buffer.alloc(1000)))).toEqual(deep({ type: 'Buffer', length: 1000 }));});it('copies a deep object whose keys fit in what is left of the node budget and writes a wider one as [Truncated]', () => {@@ -1165,8 +1336,8 @@ describe('redactSecrets', () => {expect(Object.keys(copy.list)).toHaveLength(5000);});- it('writes a small typed array in the first levels as one entry per element, as before', () => {- expect(redactSecrets({ list: Buffer.from([1, 2, 3]) })).toEqual({ list: { 0: 1, 1: 2, 2: 3 } });+ it('writes a small typed array in the first levels as its type and length', () => {+ expect(redactSecrets({ list: Buffer.from([1, 2, 3]) })).toEqual({ list: { type: 'Buffer', length: 3 } });});});});@@ -1246,7 +1417,7 @@ describe('redactSecrets', () => {expect(shown(list[3])).toBe('[Truncated]');});- it('charges the elements of a typed array for each view of one buffer, and does not list one that does not fit', () => {+ it('writes every view of one buffer as its type and length, without listing its elements', () => {const buffer = new ArrayBuffer(600_000);const keys = jest.spyOn(Object, 'keys');try {@@ -1255,9 +1426,8 @@ describe('redactSecrets', () => {views: [new Uint8Array(buffer), new Uint8Array(buffer), new Uint8Array(buffer)],}) as { views: unknown[] };- expect(Object.keys(views[0] as object)).toHaveLength(600_000);- expect(views.slice(1).map(shown)).toEqual(['[Truncated]', '[Truncated]']);- expect(keys.mock.calls.filter(([node]) => node instanceof Uint8Array)).toHaveLength(1);+ expect(views).toEqual(Array.from({ length: 3 }, () => ({ type: 'Uint8Array', length: 600_000 })));+ expect(keys.mock.calls.filter(([node]) => node instanceof Uint8Array)).toHaveLength(0);} finally {keys.mockRestore();}diff --git a/src/common/services/logger.service.ts b/src/common/services/logger.service.tsindex 45799690d..6215e44f8 100644--- a/src/common/services/logger.service.ts+++ b/src/common/services/logger.service.ts@@ -1,3 +1,4 @@+import { types } from 'node:util';import { Injectable, LoggerService as NestLoggerService, Scope } from '@nestjs/common';import { getRequestId } from './request-context';@@ -35,9 +36,9 @@ const REDACT_FULL_DEPTH = 5;const REDACT_DEEP_NODES = 1000;// The copy of an array tests every index for a hole, so a sparse array costs its length however little it holds// (2**32 - 1 takes tens of seconds). An array longer than this, or below the whole levels longer than what is left of-// the node budget, is written as [Truncated] before any index is visited. The same goes for a typed array (a-// Buffer included), which lists one entry per element. Below the whole levels an object is listed, then truncated if-// its keys do not fit in what is left of the budget, which they are then charged to.+// the node budget, is written as [Truncated] before any index is visited. The same goes for a Map or a Set, by its+// size. Below the whole levels an object is listed, then truncated if its keys do not fit in what is left of the+// budget, which they are then charged to.const REDACT_MAX_ARRAY_LENGTH = 1_000_000;// One copy takes at most this many entries at every level: the keys of the objects and the length of the arrays it// copies, a shared one charged again at every use, because every use is copied and written out again. It is twice the@@ -57,8 +58,6 @@ export interface RedactOptions {isSecretKey?: (key: string) => boolean;/** Written in place of a credential's value. */marker?: string;- /** Copy an Error with its name, message and stack too, which a copy of its own properties alone would lose. */- errors?: boolean;}const isSecretName = (key: string): boolean => SECRET_KEY_PATTERN.test(key);@@ -74,19 +73,45 @@ function elementCount(view: ArrayBufferView): number {}}+/** The moment a Date holds, read through the built-in methods rather than any the value defines itself. */+function dateText(date: Date): string {+ return Number.isNaN(Date.prototype.getTime.call(date)) ? 'Invalid Date' : Date.prototype.toISOString.call(date);+}++/**+ * The readable form a value takes, if any. It is told apart by its prototype, which costs little, and a Date, Map or Set+ * then by its brand, which a value made from that prototype alone lacks. A Proxy is copied by its own properties, also+ * when its getPrototypeOf trap throws, and so is a value made in another realm.+ */+function formOf(node: object): 'date' | 'map' | 'set' | 'error' | undefined {+ try {+ if (node instanceof Date) return types.isDate(node) ? 'date' : undefined;+ if (node instanceof Map) return types.isMap(node) ? 'map' : undefined;+ if (node instanceof Set) return types.isSet(node) ? 'set' : undefined;+ return node instanceof Error && !types.isProxy(node) ? 'error' : undefined;+ } catch {+ return undefined;+ }+}+/*** Return a copy of `value` with credential keys' values replaced by `[REDACTED]` (recursive). The copy always* serializes: an object met again inside itself becomes `[Circular]`, a nested BigInt its decimal text, and- * whatever lies beyond the deep node, entry or character budget `[Truncated]`. Shared with the Baileys wire logger.+ * whatever lies beyond the deep node, entry or character budget `[Truncated]`. A value whose data is not in its own+ * properties is written readably: an Error as its name, message and stack before its own properties, a Date as its ISO+ * text, a Buffer or typed array as its type and length, a Map as the array of its [key, value] pairs and a Set as the+ * array of its values. Shared with the Baileys wire logger.*/export function redactSecrets(value: unknown, options: RedactOptions = {}): unknown {- const { isSecretKey = isSecretName, marker = '[REDACTED]', errors = false } = options;+ const { isSecretKey = isSecretName, marker = '[REDACTED]' } = options;const budget = { nodes: REDACT_DEEP_NODES, entries: REDACT_MAX_ENTRIES, chars: REDACT_MAX_CHARS };const walk = (node: unknown, path: object[]): unknown => {// A function is not data: it is left out, as JSON.stringify leaves it out of an object and writes null in an array.if (typeof node === 'function') return undefined;if (typeof node === 'string') return (budget.chars -= node.length) < 0 ? '[Truncated]' : node;if (node === null || typeof node !== 'object') return node;+ const form = formOf(node);+ if (form === 'date') return walk(dateText(node as Date), path);if (path.includes(node)) return '[Circular]';const deep = path.length >= REDACT_FULL_DEPTH;// A negative budget means something that did not fit was met: nothing else is copied or listed after it.@@ -102,20 +127,43 @@ export function redactSecrets(value: unknown, options: RedactOptions = {}): unkn};const left = Math.min(budget.entries, deep ? budget.nodes : Infinity);const room = Math.min(left, REDACT_MAX_ARRAY_LENGTH);- if (Array.isArray(node)) {+ // A Map is copied as the array of its pairs, a credential key's value replaced, and a Set as the array of its+ // values; its size is held against the room before any entry is read.+ let list: unknown = node;+ const map = form === 'map';+ if (map || form === 'set') {+ if (Reflect.get(map ? Map.prototype : Set.prototype, 'size', node) > room) return '[Truncated]';+ const items: unknown[] = [];+ if (map)+ Map.prototype.forEach.call(node, (item: unknown, key: unknown) =>+ items.push([key, typeof key === 'string' && isSecretKey(key) ? marker : item]),+ );+ else Set.prototype.forEach.call(node, (item: unknown) => items.push(item));+ list = items;+ }+ if (Array.isArray(list)) {// Read once and taken as a count (whole, from zero), as map takes it: one number gates the copy and sizes it.- const length = Math.max(0, Math.trunc(node.length) || 0);+ const length = Math.max(0, Math.trunc(list.length) || 0);if (length > room) return '[Truncated]';budget.entries -= length;// A plain Array filled by index, holes skipped: map would build it through the constructor of an Array subclass.const plain = new Array<unknown>(length);- for (let index = 0; index < length; index++) if (index in node) plain[index] = copy(node[index]);+ for (let index = 0; index < length; index++) if (index in list) plain[index] = copy(list[index]);return plain;}- if (ArrayBuffer.isView(node) && elementCount(node) > room) return '[Truncated]';- const props = node as Record<string, unknown>;+ // A typed array, a Buffer included, is written as its type and the elements it holds, never one entry per element.+ // A DataView has no type of that kind and is copied as an object.+ const type: unknown = ArrayBuffer.isView(node) ? Reflect.get(TYPED_ARRAY_PROTOTYPE, Symbol.toStringTag, node) : 0;+ const props =+ typeof type === 'string'+ ? {+ type: Object.getPrototypeOf(node) === Buffer.prototype ? 'Buffer' : type,+ length: elementCount(node as ArrayBufferView),+ }+ : (node as Record<string, unknown>);+ const error = node as Error;const source =- errors && node instanceof Error ? { name: node.name, message: node.message, stack: node.stack, ...props } : props;+ form === 'error' ? { name: error.name, message: error.message, stack: error.stack, ...props } : props;const keys = Object.keys(source);if (keys.length > left) {// Listed once, then charged: another reference to this object, or any other node, is refused without listing.diff --git a/src/core/plugins/plugin-loader-sandbox-routing.spec.ts b/src/core/plugins/plugin-loader-sandbox-routing.spec.tsindex c90ad890c..37da6fa59 100644--- a/src/core/plugins/plugin-loader-sandbox-routing.spec.ts+++ b/src/core/plugins/plugin-loader-sandbox-routing.spec.ts@@ -923,6 +923,50 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {expect(logSpy).toHaveBeenNthCalledWith(2, '[p1] set', { metaTruncated: true }, { pluginId: 'p1' });});+ // JSON.stringify writes an Error, a Map and a Set as {}, but the logger writes what they hold.+ it('counts the message and stack of an Error and the entries of a Map or a Set that the logger writes', async () => {+ const loader = makeLoader();+ seed(loader, { builtIn: false, instance: null });+ await loader.enablePlugin('p1');+ const logSpy = jest.spyOn(loggerOf(loader), 'log').mockImplementation(() => undefined);+ const text = 'x'.repeat(9000);++ loader.capturedOnLog!('log', 'error', structuredClone({ e: new Error(text) }));+ loader.capturedOnLog!('log', 'map', structuredClone({ m: new Map([[1, text]]) }));+ loader.capturedOnLog!('log', 'set', structuredClone({ s: new Set([text]) }));+ loader.capturedOnLog!('log', 'small', structuredClone({ e: new Error('boom'), m: new Map([[1, 2]]) }));++ for (const [call, name] of [+ [1, 'error'],+ [2, 'map'],+ [3, 'set'],+ ] as const) {+ expect(logSpy).toHaveBeenNthCalledWith(call, `[p1] ${name}`, { metaTruncated: true }, { pluginId: 'p1' });+ }+ expect((logSpy.mock.calls[3][1] as { e: Error }).e.message).toBe('boom');+ });++ // Few values, but a long text once written: numbers in a Map, or a BigInt in a Set, which JSON.stringify alone+ // would write as {}.+ it('measures the text of a Map or a Set as the logger writes it', async () => {+ const loader = makeLoader();+ seed(loader, { builtIn: false, instance: null });+ await loader.enablePlugin('p1');+ const logSpy = jest.spyOn(loggerOf(loader), 'log').mockImplementation(() => undefined);+ const float = -Math.PI * 1e-300;+ const pairs = Array.from({ length: 2000 }, (_, key): [number, number] => [float * (key + 1), float]);++ const started = Date.now();+ loader.capturedOnLog!('log', 'floats', structuredClone({ m: new Map(pairs) }));+ loader.capturedOnLog!('log', 'bigints', structuredClone({ s: new Set([10n ** 100_000n, 10n ** 100_000n + 1n]) }));++ expect(Date.now() - started).toBeLessThan(1000);+ const [floats, bigints] = logSpy.mock.calls.map(call => call[1] as Record<string, unknown>);+ expect(floats).toMatchObject({ metaTruncated: true });+ expect(floats.metaLength).toBeGreaterThan(8192);+ expect(bigints).toEqual({});+ });+it('still drops a circular meta without a marker, as it cannot be written as JSON', async () => {const loader = makeLoader();seed(loader, { builtIn: false, instance: null });diff --git a/src/core/plugins/plugin-sandbox-bridge.ts b/src/core/plugins/plugin-sandbox-bridge.tsindex 6216d53d0..b062370d9 100644--- a/src/core/plugins/plugin-sandbox-bridge.ts+++ b/src/core/plugins/plugin-sandbox-bridge.ts@@ -11,7 +11,7 @@ import { PluginWorkerHost } from './sandbox/plugin-worker-host';import { dispatchCapabilityVerb } from './sandbox/capability-router';import { PluginLogLevel } from './sandbox/protocol';import { shouldDispatchToPlugin } from './handover-gate';-import { countMetaValues } from './sandbox-log-meta';+import { countMetaValues, metaText } from './sandbox-log-meta';import { makeOnWebhookSubscribe } from './webhook-subscribe.util';import { registerPluginSearchProvider, unregisterPluginSearchProvider } from './search-provider-registration.util';import { INGRESS_DISPATCH_TIMEOUT_MS } from '../../modules/integration/integration.constants';@@ -598,7 +598,7 @@ export class PluginSandboxBridge {boundedMeta = { ...reason, metaTruncated: true };} else {try {- const metaLength = JSON.stringify(meta).length;+ const metaLength = (metaText(meta) ?? '').length;if (metaLength > SANDBOX_LOG_MAX_MESSAGE_LENGTH)boundedMeta = { ...reason, metaTruncated: true, metaLength };} catch {diff --git a/src/core/plugins/sandbox-log-meta.spec.ts b/src/core/plugins/sandbox-log-meta.spec.tsindex c04556d75..f7af86ecf 100644--- a/src/core/plugins/sandbox-log-meta.spec.ts+++ b/src/core/plugins/sandbox-log-meta.spec.ts@@ -1,4 +1,4 @@-import { countMetaValues } from './sandbox-log-meta';+import { countMetaValues, metaText } from './sandbox-log-meta';describe('countMetaValues', () => {const NO_LIMIT = Number.MAX_SAFE_INTEGER;@@ -42,11 +42,41 @@ describe('countMetaValues', () => {expect(countMetaValues({ a: shared, b: shared }, NO_LIMIT)).toBe(1 + 2 * (1 + 1 + 1 + 1));});- it('counts a typed array by its length, and a Map or a Set by its size', () => {+ it('counts a typed array by its length', () => {expect(countMetaValues(new Uint8Array(100), NO_LIMIT)).toBe(1 + 100);expect(countMetaValues(Buffer.alloc(7), NO_LIMIT)).toBe(1 + 7);- expect(countMetaValues(new Map([[1, 2]]), NO_LIMIT)).toBe(1 + 1);- expect(countMetaValues(new Set([1, 2, 3]), NO_LIMIT)).toBe(1 + 3);+ });++ // The logger writes a Map as the array of its [key, value] pairs, a Set as the array of its values and an Error with+ // its name, message and stack, so what those hold is counted as well.+ it('counts a Map or a Set by its size and its entries, one held twice at each use and one that holds itself once', () => {+ expect(countMetaValues(new Map([[1, 'ab']]), NO_LIMIT)).toBe(1 + 1 + (1 + 1 + (1 + 2)));+ expect(countMetaValues(new Set([1, 'ab']), NO_LIMIT)).toBe(1 + 2 + 1 + (1 + 2));+ const shared = new Set(['abc']);+ expect(countMetaValues([shared, shared], NO_LIMIT)).toBe(1 + 2 * (1 + 1 + (1 + 3)));+ const self = new Map<string, unknown>();+ self.set('s', self);+ expect(countMetaValues(self, NO_LIMIT)).toBe(1 + 1 + (1 + (1 + 1) + 1));+ });++ it('counts the entries of a Map or a Set only once their number fits', () => {+ const big = new Set(Array.from({ length: 9000 }, (_, key) => key));+ const from = jest.spyOn(Array, 'from');+ try {+ expect(countMetaValues({ big }, 8192)).toBe(1 + 3 + 1 + 9000);+ expect(from).not.toHaveBeenCalled();+ } finally {+ from.mockRestore();+ }+ expect(countMetaValues(new Map([[1, 'x'.repeat(9000)]]), 8192)).toBeGreaterThan(8192);+ });++ it('counts the name, message and stack of an Error before its own properties', () => {+ const error = Object.assign(new Error('boom'), { code: 'E1' });+ const stack = error.stack ?? '';++ expect(countMetaValues(error, NO_LIMIT)).toBe(1 + (4 + 1 + 5) + (7 + 1 + 4) + (5 + 1 + stack.length) + (4 + 1 + 2));+ expect(countMetaValues({ e: new Error('x'.repeat(9000)) }, 8192)).toBeGreaterThan(8192);});// JSON.stringify writes a boxed primitive as the primitive it holds; its own properties are the characters of a@@ -131,3 +161,42 @@ describe('countMetaValues', () => {expect(countMetaValues({ a: { b: { c: 1 } } }, 8192)).toBe(1 + 2 + 2 + 2);});});++// What the logger writes for an Error, a Map and a Set is invisible to JSON.stringify ({}), so the text of a meta is+// measured with them rendered as the logger renders them.+describe('metaText', () => {+ it('is the JSON text of a meta that holds none of them', () => {+ for (const meta of [{ a: 1, b: [1, 'two', { c: null }] }, [1, 2], 'text', 42, null, { at: new Date(0) }]) {+ expect(metaText(meta)).toBe(JSON.stringify(meta));+ }+ });++ it('writes a Map as its pairs, a Set as its values and an Error with its name, message and stack', () => {+ const error = new Error('boom');++ expect(+ metaText({+ m: new Map<unknown, unknown>([+ [1, 2.5],+ ['k', 'v'],+ ]),+ s: new Set([1, 'a']),+ }),+ ).toBe('{"m":[[1,2.5],["k","v"]],"s":[1,"a"]}');+ expect(metaText({ e: error })).toBe(JSON.stringify({ e: { name: 'Error', message: 'boom', stack: error.stack } }));+ const float = -Math.PI * 1e-300;+ expect(metaText({ list: new Set([float, float / 2, float / 3]) })).toBe(+ `{"list":[${[float, float / 2, float / 3].join(',')}]}`,+ );+ });++ it('writes a Map or a Set held twice at each use, and refuses one that holds itself or a BigInt', () => {+ const shared = new Set([1]);+ const self = new Map<string, unknown>();+ self.set('self', self);++ expect(metaText({ a: shared, b: shared })).toBe('{"a":[1],"b":[1]}');+ expect(() => metaText({ self })).toThrow(TypeError);+ expect(() => metaText({ s: new Set([10n]) })).toThrow(TypeError);+ });+});diff --git a/src/core/plugins/sandbox-log-meta.ts b/src/core/plugins/sandbox-log-meta.tsindex 344917435..87fa442c1 100644--- a/src/core/plugins/sandbox-log-meta.ts+++ b/src/core/plugins/sandbox-log-meta.ts@@ -1,7 +1,10 @@+import { types } from 'node:util';+/*** Counts what JSON.stringify would write for a worker's log metadata, and stops once the count passes `limit`: one* for every value, one per character of a string, a String wrapper or a typed array element, and one per character- * of a property name.+ * of a property name. What the logger writes where JSON.stringify would write {} is counted too: the name, message and+ * stack of an Error, and one per entry of a Map or a Set plus the entries themselves, as the arrays it writes for them.** JSON.stringify cannot be stopped part way: it lists every key of an object or typed array before it calls a* replacer for the first one, and it writes a shared reference out again at every use. So the size of the text has@@ -9,26 +12,33 @@* written at every use of its object.** The count can be higher than the length of the text. A property that JSON.stringify skips (undefined, a function,- * a symbol) and the entries of a Map or Set (written as {}) are counted as if they were written. A metadata whose+ * a symbol) is counted as if it were written, and so is the value under a credential key of a Map. A metadata whose* count passes `limit` is replaced by the size marker whatever its text would measure; for data that is plain JSON* the count never exceeds the length of the text, so a count past `limit` means a longer text.*/export function countMetaValues(meta: unknown, limit: number): number {let count = 0;const open = new Set<object>();- // An object is walked by its own enumerable keys, an array by its elements.- const frames: Array<{ node: object; keys: string[] | null; length: number; next: number }> = [];+ // An object is walked by its own enumerable keys, an array by its elements, a Map or a Set by the list of its entries.+ const frames: Array<{ node: object; keys: string[] | null; list: unknown[] | null; length: number; next: number }> =+ [];const visit = (value: unknown): void => {count++;if (typeof value === 'string') count += value.length;if (typeof value !== 'object' || value === null) return;+ // A cycle: JSON.stringify refuses it, so there is nothing more to count.+ if (open.has(value)) return;if (ArrayBuffer.isView(value)) {count += (value as { length?: number }).length ?? 0;return;}- if (value instanceof Map || value instanceof Set) {+ let list = Array.isArray(value) ? (value as unknown[]) : null;+ // Brand checks, like the String wrapper's below, so a value made in another realm counts too.+ if (types.isMap(value) || types.isSet(value)) {count += value.size;- return;+ // Listed only once the number of its entries fits.+ if (count > limit) return;+ list = Array.from(value as Iterable<unknown>);}// A String wrapper is written as its text, and its own properties are its characters, one per index. Its brand// is checked rather than instanceof, so a wrapper made in another realm counts too.@@ -36,11 +46,13 @@ export function countMetaValues(meta: unknown, limit: number): number {count += (value as { length: number }).length;return;}- // A cycle: JSON.stringify refuses it, so there is nothing more to count.- if (open.has(value)) return;open.add(value);- const keys = Array.isArray(value) ? null : Object.keys(value);- frames.push({ node: value, keys, length: keys ? keys.length : (value as unknown[]).length, next: 0 });+ const keys = list+ ? null+ : types.isNativeError(value)+ ? [...new Set(['name', 'message', 'stack', ...Object.keys(value)])]+ : Object.keys(value);+ frames.push({ node: value, keys, list, length: (keys ?? list ?? []).length, next: 0 });};visit(meta);while (frames.length > 0 && count <= limit) {@@ -51,7 +63,7 @@ export function countMetaValues(meta: unknown, limit: number): number {count += frame.keys[index].length;visit((frame.node as Record<string, unknown>)[frame.keys[index]]);} else {- visit((frame.node as unknown[])[index]);+ visit((frame.list as unknown[])[index]);}} else {open.delete(frame.node);@@ -60,3 +72,25 @@ export function countMetaValues(meta: unknown, limit: number): number {}return count;}++/**+ * The JSON text of a worker's log metadata, with what JSON.stringify alone writes as {} written as the logger writes it:+ * an Error with its name, message and stack, a Map as the array of its [key, value] pairs and a Set as the array of its+ * values. Each is rendered once, so one that holds itself is a cycle JSON.stringify refuses like any other, and a BigInt+ * in one is refused as it is anywhere else in the metadata. Run only on a meta whose count fits the limit.+ */+export function metaText(meta: unknown): string | undefined {+ const rendered = new Map<object, unknown>();+ return JSON.stringify(meta, (_key, value: unknown) => {+ if (typeof value !== 'object' || value === null) return value;+ if (!types.isMap(value) && !types.isSet(value) && !types.isNativeError(value)) return value;+ if (!rendered.has(value))+ rendered.set(+ value,+ types.isNativeError(value)+ ? { name: value.name, message: value.message, stack: value.stack, ...(value as object) }+ : Array.from(value as Iterable<unknown>),+ );+ return rendered.get(value);+ });+}diff --git a/src/engine/adapters/baileys-logger.spec.ts b/src/engine/adapters/baileys-logger.spec.tsindex c055742a3..9dd1f363b 100644--- a/src/engine/adapters/baileys-logger.spec.ts+++ b/src/engine/adapters/baileys-logger.spec.ts@@ -183,7 +183,7 @@ describe('createBaileysLogger writes any record the library hands it', () => {expect(line).toContain('"password":"[redacted]","visible":"v"');});- it('writes a Buffer of 5 million bytes as [Truncated] at once, in the first levels and below the fifth', () => {+ it('writes a Buffer of 5 million bytes as its type and length at once, in the first levels and below the fifth', () => {const file = Buffer.alloc(5_000_000);const started = Date.now();@@ -191,8 +191,8 @@ describe('createBaileysLogger writes any record the library hands it', () => {const deep = write({ a: { b: { c: { d: { e: { file } } } } } }, 'x');expect(Date.now() - started).toBeLessThan(1000);- expect(shallow).toMatchObject({ file: '[Truncated]', jid: 'j', msg: 'x' });- expect(deep).toMatchObject({ a: { b: { c: { d: { e: { file: '[Truncated]' } } } } }, msg: 'x' });+ expect(shallow).toMatchObject({ file: { type: 'Buffer', length: 5_000_000 }, jid: 'j', msg: 'x' });+ expect(deep).toMatchObject({ a: { b: { c: { d: { e: { file: { type: 'Buffer', length: 5_000_000 } } } } } } });});it('writes a property named __proto__ as a field, with the credentials under it redacted', () => {@@ -206,7 +206,23 @@ describe('createBaileysLogger writes any record the library hands it', () => {it('counts a typed array by the elements it has, not by the length it claims', () => {const file = Object.defineProperty(new Uint8Array(1_000_001), 'length', { value: 0 });- expect(write({ file, jid: 'j' }, 'x')).toMatchObject({ file: '[Truncated]', jid: 'j', msg: 'x' });+ expect(write({ file, jid: 'j' }, 'x')).toMatchObject({ file: { type: 'Uint8Array', length: 1_000_001 }, jid: 'j' });+ });++ it('writes a Map as its pairs with the value of a credential key redacted, and a Date as its ISO text', () => {+ const line = JSON.stringify(+ write({+ keys: new Map([+ ['Password', 'value-map'],+ ['jid', 'j'],+ ]),+ at: new Date(0),+ }),+ );++ expect(line).toContain('"keys":[["Password","[redacted]"],["jid","j"]]');+ expect(line).toContain('"at":"1970-01-01T00:00:00.000Z"');+ expect(line).not.toContain('value-map');});it('copies an array that reports a length that is not a count as empty, and keeps the rest of the record', () => {diff --git a/src/engine/adapters/baileys-logger.ts b/src/engine/adapters/baileys-logger.tsindex dd04076b0..c16aace49 100644--- a/src/engine/adapters/baileys-logger.ts+++ b/src/engine/adapters/baileys-logger.ts@@ -33,7 +33,6 @@ const SECRET_KEYS: ReadonlySet<string> = new Set(['password', 'secret', 'token',const REDACTION: RedactOptions = {isSecretKey: key => SECRET_KEYS.has(key.toLowerCase()),marker: '[redacted]',- errors: true,};const BAILEYS_LOG_LEVELS = ['trace', 'debug', 'info', 'warn', 'error'];