Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(logger): bound the text one metadata copy writes
Yudhi Armyndharis02f076c62131
diff --git a/docs/10-devops-infrastructure.md b/docs/10-devops-infrastructure.mdindex 02c2c4627..b60d7bbcb 100644--- a/docs/10-devops-infrastructure.md+++ b/docs/10-devops-infrastructure.md@@ -1107,9 +1107,10 @@ replaced with `[REDACTED]` before the line is written, at any depth. A log callmetadata: an object that contains itself is written with `[Circular]` in its place, and below the fifthlevel at most 1000 more objects and arrays are copied, the rest written as `[Truncated]`. So is an array,typed array or object with more entries than that allowance has left (a list of 1000 numbers at the sixth-level), and, at any level, an array or typed array of more than a million entries. At every level the metadata-is copied in at most two million entries in all (the keys of its objects plus the lengths of its arrays, a value-referenced again counted again at each use); an object or array that no longer fits is written as `[Truncated]`.+level), and, at any level, an array or typed array of more than a million entries. At every level one call+copies at most two million entries in all (the keys of its objects plus the lengths of its arrays) and writes at+most 2^26 (about 67 million) characters of string values, key names and BigInt text (sixteen to the digit); a+value referenced again is counted again at each use, and what no longer fits is written as `[Truncated]`.The same logger is installed as Nest's framework logger, so framework lines (route mapping, unhandledexception stacks from the exception handler) follow `LOG_LEVEL` and `LOG_FORMAT` and carry thediff --git a/src/common/services/logger.service.spec.ts b/src/common/services/logger.service.spec.tsindex 3bddd0a36..1eb833893 100644--- a/src/common/services/logger.service.spec.ts+++ b/src/common/services/logger.service.spec.ts@@ -1283,6 +1283,116 @@ describe('redactSecrets', () => {});});+ // One copy writes at most 2**26 characters of string values, BigInt text and key names, a shared one charged again at+ // every use. A string is not copied, so the entries above cannot see a long one referenced many times.+ describe('the characters one copy may write', () => {+ const LIMIT = 2 ** 26;+ const text = (length: number): string => 'x'.repeat(length);+ // What a failing comparison prints: the length of a long string or key name, never the text.+ const shown = (value: unknown): unknown => {+ if (typeof value === 'string') return value.length > 20 ? value.length : value;+ if (value === null || typeof value !== 'object') return value;+ return Object.fromEntries(Object.entries(value).map(([key, child]) => [shown(key), shown(child)]));+ };++ // The key name counts with the value, so the key `a` leaves room for 2**26 - 1 characters.+ it('keeps a copy of exactly 2**26 characters and writes one character more as [Truncated]', () => {+ expect(shown(redactSecrets({ a: text(LIMIT - 1) }))).toEqual({ a: LIMIT - 1 });+ expect(shown(redactSecrets({ a: text(LIMIT) }))).toEqual({ a: '[Truncated]' });+ });++ it('charges every use of a shared string', () => {+ const shared = text(2 ** 24 - 1);+ const { list } = redactSecrets({ list: [shared, shared, shared, shared, shared] }) as { list: unknown[] };++ expect(list.map(shown)).toEqual([2 ** 24 - 1, 2 ** 24 - 1, 2 ** 24 - 1, 2 ** 24 - 1, '[Truncated]']);+ });++ it('charges the key names of an object at every use', () => {+ const shared = { [text(2 ** 24 - 1)]: 1 };+ const { list } = redactSecrets({ list: [shared, shared, shared, shared, shared] }) as { list: unknown[] };++ expect(list.map(shown)).toEqual([+ { [2 ** 24 - 1]: 1 },+ { [2 ** 24 - 1]: 1 },+ { [2 ** 24 - 1]: 1 },+ { [2 ** 24 - 1]: 1 },+ '[Truncated]',+ ]);+ });++ it('does not write an object whose key name does not fit, and cuts what follows it, not the object around it', () => {+ const copy = redactSecrets({ a: { [text(LIMIT + 1)]: 1 }, b: 'after', c: {}, d: [] });++ expect(shown(copy)).toEqual({ a: '[Truncated]', b: '[Truncated]', c: '[Truncated]', d: '[Truncated]' });+ });++ it('charges the text of a BigInt sixteen characters to the digit', () => {+ const big = 10n ** 999n;+ // The two key names take 2 characters, and `left` is what the BigInt finds when its turn comes.+ const withRoom = (left: number): unknown => shown(redactSecrets({ a: text(LIMIT - 2 - left), b: big }));++ expect(withRoom(16_000)).toEqual({ a: LIMIT - 2 - 16_000, b: 1000 });+ expect(withRoom(15_999)).toEqual({ a: LIMIT - 2 - 15_999, b: '[Truncated]' });+ });++ it('builds no more BigInt text once the characters are spent', () => {+ const big = 10n ** 999n;+ const toString = jest.spyOn(BigInt.prototype, 'toString');+ try {+ // Three uses of 1000 digits fit, 16,000 characters each; the fourth is built and found not to fit.+ const { list } = redactSecrets({+ a: text(LIMIT - 3 * 16_000 - 100),+ list: Array.from({ length: 10 }, () => big),+ }) as {+ list: unknown[];+ };++ expect(list.map(shown)).toEqual([1000, 1000, 1000, ...new Array<string>(7).fill('[Truncated]')]);+ expect(toString).toHaveBeenCalledTimes(4);+ } finally {+ toString.mockRestore();+ }+ });++ it('does not charge the value of a secret-named key, which is not written', () => {+ expect(shown(redactSecrets({ password: text(LIMIT), after: 'ok' }))).toEqual({+ password: '[REDACTED]',+ after: 'ok',+ });+ });++ it('still redacts a secret-named key beside a string that is cut, and does not charge its value', () => {+ const copy = redactSecrets({ big: text(LIMIT + 1), password: text(LIMIT) });++ expect(shown(copy)).toEqual({ big: '[Truncated]', password: '[REDACTED]' });+ });++ it('keeps the rest of a line whose second string is cut', () => {+ const logger = new LoggerService();+ const log = jest.spyOn(console, 'log').mockImplementation();+ LoggerService.setLogFormat(LogFormat.JSON);+ try {+ logger.log('saved', { sessionId: 's1', first: text(LIMIT / 2 + 1), second: text(LIMIT / 2), after: 3 });++ const line = JSON.parse((log.mock.calls as string[][])[0][0]) as Record<string, unknown>;+ expect(shown({ ...line, timestamp: 0 })).toEqual({+ timestamp: 0,+ level: 'info',+ context: 'Application',+ message: 'saved',+ sessionId: 's1',+ first: LIMIT / 2 + 1,+ second: '[Truncated]',+ after: 3,+ });+ } finally {+ log.mockRestore();+ LoggerService.setLogFormat(null);+ }+ });+ });+describe('the bounds of the copy', () => {// `rows` ends up `above` objects below the top, so its entries sit one level lower: with 3 above they are in// the fifth level, the last one copied whole, and with 4 above they are in the sixth, the first one that costs.diff --git a/src/common/services/logger.service.ts b/src/common/services/logger.service.tsindex 00d397e36..d3be15dbb 100644--- a/src/common/services/logger.service.ts+++ b/src/common/services/logger.service.ts@@ -44,6 +44,13 @@ const REDACT_MAX_ARRAY_LENGTH = 1_000_000;// array limit, so a structure of that size is written whole; an object or array that does not fit in what is left of// it is written as [Truncated], which is what keeps a few shared references from expanding into an unbounded copy.const REDACT_MAX_ENTRIES = 2_000_000;+// A string is written whole and is not copied, so a long one referenced many times, or a long key name of a shared+// object, expands past what the entries count. One copy writes at most this many characters of string values, BigInt+// text (each digit charged REDACT_BIGINT_WEIGHT times) and key names, each charged again at every use; one that does+// not fit is written as [Truncated]. It is far above any line a caller writes on purpose (a million entries of 64+// characters fit), and below what V8 can build.+const REDACT_MAX_CHARS = 2 ** 26;+const REDACT_BIGINT_WEIGHT = 16;export interface RedactOptions {/** Whether the value under this key is a credential. Defaults to the secret-name pattern above. */@@ -70,21 +77,29 @@ function elementCount(view: ArrayBufferView): number {/*** Return a copy of `value` with credential keys' values replaced by `[REDACTED]` (recursive). The copy always* serializes: an object met again inside itself becomes `[Circular]`, a nested BigInt its decimal text, and- * whatever lies beyond the deep node budget or the entry budget `[Truncated]`. Shared with the Baileys wire logger.+ * whatever lies beyond the deep node, entry or character budget `[Truncated]`. Shared with the Baileys wire logger.*/export function redactSecrets(value: unknown, options: RedactOptions = {}): unknown {const { isSecretKey = isSecretName, marker = '[REDACTED]', errors = false } = options;- const budget = { nodes: REDACT_DEEP_NODES, entries: REDACT_MAX_ENTRIES };+ const budget = { nodes: REDACT_DEEP_NODES, entries: REDACT_MAX_ENTRIES, chars: REDACT_MAX_CHARS };const walk = (node: unknown, path: object[]): unknown => {// A function is not data: it is left out, as JSON.stringify leaves it out of an object and writes null in an array.if (typeof node === 'function') return undefined;+ if (typeof node === 'string') return (budget.chars -= node.length) < 0 ? '[Truncated]' : node;if (node === null || typeof node !== 'object') return node;if (path.includes(node)) return '[Circular]';const deep = path.length >= REDACT_FULL_DEPTH;- // A negative entry budget means an object that did not fit was listed: nothing else is copied or listed after it.- if ((deep && --budget.nodes < 0) || budget.entries < 0) return '[Truncated]';+ // A negative budget means something that did not fit was met: nothing else is copied or listed after it.+ if ((deep && --budget.nodes < 0) || budget.entries < 0 || budget.chars < 0) return '[Truncated]';const parents = [...path, node];- const copy = (child: unknown): unknown => (typeof child === 'bigint' ? child.toString() : walk(child, parents));+ const copy = (child: unknown): unknown => {+ if (typeof child !== 'bigint') return walk(child, parents);+ // Its text costs far more to build than a string of that length costs to write (ten times or more), and+ // none is built once the budget is spent.+ if (budget.chars < 0) return '[Truncated]';+ const text = child.toString();+ return (budget.chars -= text.length * REDACT_BIGINT_WEIGHT) < 0 ? '[Truncated]' : text;+ };const left = Math.min(budget.entries, deep ? budget.nodes : Infinity);const room = Math.min(left, REDACT_MAX_ARRAY_LENGTH);if (Array.isArray(node)) {@@ -111,6 +126,8 @@ export function redactSecrets(value: unknown, options: RedactOptions = {}): unknbudget.entries -= keys.length;const out: Record<string, unknown> = {};for (const key of keys) {+ // The key that does not fit turns its object into the marker; keys after the budget is spent are only listed.+ if (budget.chars >= 0 && (budget.chars -= key.length) < 0) return '[Truncated]';const val = source[key];// JSON.stringify would call an own toJSON on the copy and write whatever it returns, redacted or not.if (key === 'toJSON' && typeof val === 'function') continue;diff --git a/src/engine/adapters/baileys-logger.ts b/src/engine/adapters/baileys-logger.tsindex ca54ea803..dd04076b0 100644--- a/src/engine/adapters/baileys-logger.ts+++ b/src/engine/adapters/baileys-logger.ts@@ -26,8 +26,9 @@ const SECRET_KEYS: ReadonlySet<string> = new Set(['password', 'secret', 'token',* the whole proxy config, password included, as its only enumerable property. At debug level that* went to stdout verbatim, which on a shipped deployment means the proxy password in the container* log. An Error's message and stack are not enumerable, so they are copied explicitly: they are what- * an operator raised the log level to see. The copy is bounded in the entries it takes, and below the fifth- * level in the objects too (see redactSecrets), because a logger must not walk an arbitrary graph on the wire path.+ * an operator raised the log level to see. The copy is bounded in the entries and characters it takes, and below+ * the fifth level in the objects too (see redactSecrets), because a logger must not walk an arbitrary graph on the+ * wire path.*/const REDACTION: RedactOptions = {isSecretKey: key => SECRET_KEYS.has(key.toLowerCase()),