Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(logger): bound the entries one metadata copy takes
Yudhi Armyndharis62a34d5097cd
diff --git a/docs/10-devops-infrastructure.md b/docs/10-devops-infrastructure.mdindex db56003e9..02c2c4627 100644--- a/docs/10-devops-infrastructure.md+++ b/docs/10-devops-infrastructure.md@@ -1107,7 +1107,9 @@ replaced with `[REDACTED]` before the line is written, at any depth. A log callmetadata: an object that contains itself is written with `[Circular]` in its place, and below the fifthlevel at most 1000 more objects and arrays are copied, the rest written as `[Truncated]`. So is an array,typed array or object with more entries than that allowance has left (a list of 1000 numbers at the sixth-level), and, at any level, an array or typed array of more than a million entries.+level), and, at any level, an array or typed array of more than a million entries. At every level the metadata+is copied in at most two million entries in all (the keys of its objects plus the lengths of its arrays, a value+referenced again counted again at each use); an object or array that no longer fits is written as `[Truncated]`.The same logger is installed as Nest's framework logger, so framework lines (route mapping, unhandledexception stacks from the exception handler) follow `LOG_LEVEL` and `LOG_FORMAT` and carry thediff --git a/src/common/services/logger.service.spec.ts b/src/common/services/logger.service.spec.tsindex f824cf9a0..3bddd0a36 100644--- a/src/common/services/logger.service.spec.ts+++ b/src/common/services/logger.service.spec.ts@@ -141,7 +141,7 @@ describe('LoggerService', () => {expect(output.sessionId).toBe('s1');});- // The first five levels were always copied whole, however wide, and still are.+ // The first five levels were always copied whole, however wide, and still are, up to what one copy may take.it('writes a wide structure in full at the levels metadata normally has', () => {logger.log('cfg', { items: Array.from({ length: 1500 }, (_, id) => ({ id, tags: [id] })) });@@ -151,6 +151,16 @@ describe('LoggerService', () => {expect(items[1499]).toEqual({ id: 1499, tags: [1499] });});+ it('writes the second of two arrays of a million entries as [Truncated] and keeps the rest of the line', () => {+ const zeros = (): number[] => new Array<number>(1_000_000).fill(0);+ logger.log('cfg', { sessionId: 's1', first: zeros(), second: zeros(), after: { id: 1 }, last: 'l' });++ const output = getLogOutput(consoleSpy) as unknown as Record<string, unknown>;+ expect(output.first).toHaveLength(1_000_000);+ expect(String(output.second).slice(0, 11)).toBe('[Truncated]');+ expect({ ...output, first: 0 }).toMatchObject({ message: 'cfg', sessionId: 's1', after: { id: 1 }, last: 'l' });+ });+it('truncates the rows of a wide array below the last whole level once the budget is spent', () => {logger.log('cfg', { a: { b: { c: { items: Array.from({ length: 1500 }, (_, id) => ({ id })) } } } });@@ -1143,6 +1153,136 @@ describe('redactSecrets', () => {});});+ // One copy takes at most 2,000,000 entries: the keys of the objects and the length of the arrays it copies, a shared+ // one charged again at every use. What a caller built is written as it is; a graph that expands past that is cut.+ describe('the entries one copy may take', () => {+ const zeros = (length: number): number[] => new Array<number>(length).fill(0);+ const sparse = (length: number): unknown[] => {+ const list: unknown[] = [];+ list.length = length;+ return list;+ };+ const wide = (keys: number): Record<string, number> =>+ Object.fromEntries(Array.from({ length: keys }, (_, key) => [`k${key}`, key]));+ // What a failing comparison prints for a copy of a million entries.+ const shown = (value: unknown): unknown =>+ value !== null && typeof value === 'object'+ ? `${Array.isArray(value) ? 'list' : 'object'} of ${Object.keys(value).length}`+ : value;+ // The values the copy holds as they are written: a shared object counts at every use.+ const leaves = (node: unknown): number =>+ node !== null && typeof node === 'object'+ ? Object.values(node).reduce<number>((sum, child) => sum + leaves(child), 0)+ : 1;++ it('copies exactly 2,000,000 entries and writes the array that would take one more as [Truncated]', () => {+ // The top object has 2 keys, which come first.+ const whole = redactSecrets({ a: zeros(1_000_000), b: zeros(999_998) }) as { b: unknown[] };+ const over = redactSecrets({ a: zeros(1_000_000), b: zeros(999_999) }) as { a: unknown[]; b: unknown };++ expect(whole.b).toHaveLength(999_998);+ expect(over.a).toHaveLength(1_000_000);+ expect(shown(over.b)).toBe('[Truncated]');+ });++ it('still copies an empty object and an empty array once the entries are spent, and not an object with a key', () => {+ const spent = redactSecrets({ a: zeros(1_000_000), b: zeros(999_995), c: {}, d: [], e: { id: 1 } }) as Record<+ string,+ unknown+ >;++ expect(spent.b).toHaveLength(999_995);+ expect([spent.c, spent.d, spent.e]).toEqual([{}, [], '[Truncated]']);+ });++ it('redacts a secret-named key beside what is cut, and never writes what is cut', () => {+ const copy = redactSecrets({+ a: zeros(1_000_000),+ b: zeros(999_990),+ token: 'value-token',+ c: { password: 'value-password' },+ d: { ...wide(20), password: 'value-far' },+ });++ expect(JSON.stringify({ ...(copy as object), a: 0, b: 0 })).toBe(+ '{"a":0,"b":0,"token":"[REDACTED]","c":{"password":"[REDACTED]"},"d":"[Truncated]"}',+ );+ });++ it('charges every use of a shared array', () => {+ const shared = zeros(600_000);+ const { list } = redactSecrets({ list: [shared, shared, shared, shared] }) as { list: unknown[] };++ expect(list.slice(0, 3).map(copy => (copy as unknown[]).length)).toEqual([600_000, 600_000, 600_000]);+ expect(shown(list[3])).toBe('[Truncated]');+ });++ it('charges the length of a sparse array, holes included, for each one', () => {+ const { list } = redactSecrets({+ list: [sparse(600_000), sparse(600_000), sparse(600_000), sparse(600_000)],+ }) as {+ list: unknown[];+ };++ expect(list.slice(0, 3).map(copy => (copy as unknown[]).length)).toEqual([600_000, 600_000, 600_000]);+ expect(shown(list[3])).toBe('[Truncated]');+ });++ it('charges the elements of a typed array for each view of one buffer, and does not list one that does not fit', () => {+ const buffer = new ArrayBuffer(600_000);+ const keys = jest.spyOn(Object, 'keys');+ try {+ const { views } = redactSecrets({+ list: zeros(1_000_000),+ views: [new Uint8Array(buffer), new Uint8Array(buffer), new Uint8Array(buffer)],+ }) as { views: unknown[] };++ expect(Object.keys(views[0] as object)).toHaveLength(600_000);+ expect(views.slice(1).map(shown)).toEqual(['[Truncated]', '[Truncated]']);+ expect(keys.mock.calls.filter(([node]) => node instanceof Uint8Array)).toHaveLength(1);+ } finally {+ keys.mockRestore();+ }+ });++ // An object is listed (Object.keys) before its key count can be compared with what is left, so one that does not+ // fit is charged once it is listed: the next reference to it, or any other object, is refused without listing it.+ it('lists a wide object only until it no longer fits, however many references to it follow', () => {+ const shared = wide(60_000);+ const keys = jest.spyOn(Object, 'keys');+ try {+ // The two arrays leave 99,997 entries, and the list takes 6 of them: one reference fits, the second does not.+ const { list } = redactSecrets({+ first: zeros(1_000_000),+ second: zeros(900_000),+ list: Array.from({ length: 6 }, () => shared),+ }) as { list: unknown[] };++ expect(keys.mock.calls.filter(([node]) => node === shared)).toHaveLength(2);+ expect(list.map(copy => (typeof copy === 'string' ? copy : Object.keys(copy as object).length))).toEqual([+ 60_000,+ ...new Array<string>(5).fill('[Truncated]'),+ ]);+ } finally {+ keys.mockRestore();+ }+ });++ // Every level is within the first five, so the 1000 nodes of the deeper levels are not what stops it.+ it('cuts a graph of shared levels that expands past it, and keeps the part it copied', () => {+ let level: Record<string, unknown> = wide(40);+ for (let i = 1; i < 4; i++)+ level = Object.fromEntries(Array.from({ length: 40 }, (_, key) => [`k${key}`, level]));++ const copy = redactSecrets({ id: 'graph', graph: level }) as { id: string; graph: Record<string, unknown> };++ expect(copy.id).toBe('graph');+ expect(leaves(copy)).toBeGreaterThan(1_000_000);+ expect(leaves(copy)).toBeLessThanOrEqual(2_000_000);+ expect(JSON.stringify(copy.graph).includes('[Truncated]')).toBe(true);+ });+ });+describe('the bounds of the copy', () => {// `rows` ends up `above` objects below the top, so its entries sit one level lower: with 3 above they are in// the fifth level, the last one copied whole, and with 4 above they are in the sixth, the first one that costs.diff --git a/src/common/services/logger.service.ts b/src/common/services/logger.service.tsindex 4e05b5180..00d397e36 100644--- a/src/common/services/logger.service.ts+++ b/src/common/services/logger.service.ts@@ -28,8 +28,8 @@ export interface LogContext {// (case-insensitive), never the value's content, to avoid false positives on innocuous fields.const SECRET_KEY_PATTERN = /password|passwd|secret|token|api[-_]?key|authorization|credential|pepper|private[-_]?key/i;-// Metadata is copied whole down to this depth, however wide or dense, which is where it normally lives; that part is-// not bounded. Below it one log call copies at most REDACT_DEEP_NODES more objects and arrays and writes the rest as+// Metadata is copied whole down to this depth, however wide, which is where it normally lives; only REDACT_MAX_ENTRIES+// bounds that part. Below it one copy takes at most REDACT_DEEP_NODES more objects and arrays and writes the rest as// [Truncated], never raw, so a densely linked object graph cannot make it walk that part for long.const REDACT_FULL_DEPTH = 5;const REDACT_DEEP_NODES = 1000;@@ -39,6 +39,11 @@ const REDACT_DEEP_NODES = 1000;// Buffer included), which lists one entry per element. Below the whole levels an object is listed, then truncated if// its keys do not fit in what is left of the budget, which they are then charged to.const REDACT_MAX_ARRAY_LENGTH = 1_000_000;+// One copy takes at most this many entries at every level: the keys of the objects and the length of the arrays it+// copies, a shared one charged again at every use, because every use is copied and written out again. It is twice the+// array limit, so a structure of that size is written whole; an object or array that does not fit in what is left of+// it is written as [Truncated], which is what keeps a few shared references from expanding into an unbounded copy.+const REDACT_MAX_ENTRIES = 2_000_000;export interface RedactOptions {/** Whether the value under this key is a credential. Defaults to the secret-name pattern above. */@@ -65,25 +70,28 @@ function elementCount(view: ArrayBufferView): number {/*** Return a copy of `value` with credential keys' values replaced by `[REDACTED]` (recursive). The copy always* serializes: an object met again inside itself becomes `[Circular]`, a nested BigInt its decimal text, and- * whatever lies beyond the deep node budget `[Truncated]`. Shared with the Baileys wire logger.+ * whatever lies beyond the deep node budget or the entry budget `[Truncated]`. Shared with the Baileys wire logger.*/export function redactSecrets(value: unknown, options: RedactOptions = {}): unknown {const { isSecretKey = isSecretName, marker = '[REDACTED]', errors = false } = options;- const budget = { nodes: REDACT_DEEP_NODES };+ const budget = { nodes: REDACT_DEEP_NODES, entries: REDACT_MAX_ENTRIES };const walk = (node: unknown, path: object[]): unknown => {// A function is not data: it is left out, as JSON.stringify leaves it out of an object and writes null in an array.if (typeof node === 'function') return undefined;if (node === null || typeof node !== 'object') return node;if (path.includes(node)) return '[Circular]';const deep = path.length >= REDACT_FULL_DEPTH;- if (deep && --budget.nodes < 0) return '[Truncated]';+ // A negative entry budget means an object that did not fit was listed: nothing else is copied or listed after it.+ if ((deep && --budget.nodes < 0) || budget.entries < 0) return '[Truncated]';const parents = [...path, node];const copy = (child: unknown): unknown => (typeof child === 'bigint' ? child.toString() : walk(child, parents));- const room = deep ? budget.nodes : REDACT_MAX_ARRAY_LENGTH;+ const left = Math.min(budget.entries, deep ? budget.nodes : Infinity);+ const room = Math.min(left, REDACT_MAX_ARRAY_LENGTH);if (Array.isArray(node)) {// Read once and taken as a count (whole, from zero), as map takes it: one number gates the copy and sizes it.const length = Math.max(0, Math.trunc(node.length) || 0);if (length > room) return '[Truncated]';+ budget.entries -= length;// A plain Array filled by index, holes skipped: map would build it through the constructor of an Array subclass.const plain = new Array<unknown>(length);for (let index = 0; index < length; index++) if (index in node) plain[index] = copy(node[index]);@@ -94,11 +102,13 @@ export function redactSecrets(value: unknown, options: RedactOptions = {}): unknconst source =errors && node instanceof Error ? { name: node.name, message: node.message, stack: node.stack, ...props } : props;const keys = Object.keys(source);- if (deep && keys.length > budget.nodes) {+ if (keys.length > left) {// Listed once, then charged: another reference to this object, or any other node, is refused without listing.budget.nodes -= keys.length;+ budget.entries -= keys.length;return '[Truncated]';}+ budget.entries -= keys.length;const out: Record<string, unknown> = {};for (const key of keys) {const val = source[key];diff --git a/src/engine/adapters/baileys-logger.ts b/src/engine/adapters/baileys-logger.tsindex 6379e885e..ca54ea803 100644--- a/src/engine/adapters/baileys-logger.ts+++ b/src/engine/adapters/baileys-logger.ts@@ -26,8 +26,8 @@ const SECRET_KEYS: ReadonlySet<string> = new Set(['password', 'secret', 'token',* the whole proxy config, password included, as its only enumerable property. At debug level that* went to stdout verbatim, which on a shipped deployment means the proxy password in the container* log. An Error's message and stack are not enumerable, so they are copied explicitly: they are what- * an operator raised the log level to see. Below the fifth level the copy is bounded (see- * redactSecrets), because a logger must not walk an arbitrary graph on the wire path.+ * an operator raised the log level to see. The copy is bounded in the entries it takes, and below the fifth+ * level in the objects too (see redactSecrets), because a logger must not walk an arbitrary graph on the wire path.*/const REDACTION: RedactOptions = {isSecretKey: key => SECRET_KEYS.has(key.toLowerCase()),