Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(plugins): keep the plugin id on a log line whose meta is cut
Yudhi Armyndharis7a9e5633100e
diff --git a/src/common/services/logger.service.spec.ts b/src/common/services/logger.service.spec.tsindex 1eb833893..e4aa87ac7 100644--- a/src/common/services/logger.service.spec.ts+++ b/src/common/services/logger.service.spec.ts@@ -676,6 +676,13 @@ describe('LoggerService', () => {expect(output.context).toBe('TestContext');expect(output.sessionId).toBe('s1');});++ it("writes the caller's own fields after the metadata, redacted and below the structural fields", () => {+ logger.log('m', { a: 1, pluginId: 'meta' }, { pluginId: 'own', token: 't', level: 'error' });++ expect(getRawOutput(consoleSpy)).toContain('"message":"m","a":1,"pluginId":"own","token":"[REDACTED]"}');+ expect(getLogOutput(consoleSpy).level).toBe('info');+ });});describe('warn', () => {@@ -871,6 +878,17 @@ describe('LoggerService', () => {new Logger('X').log('m');expect(getLogOutput(consoleSpy).context).toBe('X');});++ // Nest appends the context name of its Logger after the arguments it was given.+ it('writes no field for the context name Nest appends after the metadata', () => {+ const errorSpy = jest.spyOn(console, 'error');++ new Logger('X').log('m', { a: 1 });+ new Logger('X').error('m', 'stack', { a: 1 });++ expect(Object.keys(getLogOutput(consoleSpy))).toEqual(['timestamp', 'level', 'context', 'message', 'a']);+ expect(Object.keys(getLogOutput(errorSpy))).toEqual(['timestamp', 'level', 'context', 'message', 'a', 'trace']);+ });});// Without the option Nest keeps its ConsoleLogger, which ignores LOG_LEVEL and LOG_FORMAT.diff --git a/src/common/services/logger.service.ts b/src/common/services/logger.service.tsindex d3be15dbb..45799690d 100644--- a/src/common/services/logger.service.ts+++ b/src/common/services/logger.service.ts@@ -217,31 +217,38 @@ export class LoggerService implements NestLoggerService {// Messages and traces are `unknown` because this is also Nest's framework logger (main.ts), and Nest// passes the thrown Error itself as the message of an unhandled-exception line; writeLog normalizes.- log(message: unknown, context?: string | LogContext): void {- this.writeLog(LogLevel.INFO, message, context);+ // `fields` are the caller's own, written beside metadata that came from elsewhere (a plugin's id beside its meta).+ log(message: unknown, context?: string | LogContext, fields?: LogContext): void {+ this.writeLog(LogLevel.INFO, message, context, undefined, fields);}- error(message: unknown, trace?: unknown, context?: string | LogContext): void {- this.writeLog(LogLevel.ERROR, message, context, trace);+ error(message: unknown, trace?: unknown, context?: string | LogContext, fields?: LogContext): void {+ this.writeLog(LogLevel.ERROR, message, context, trace, fields);}fatal(message: unknown, context?: string | LogContext): void {this.writeLog(LogLevel.ERROR, message, context);}- warn(message: unknown, context?: string | LogContext): void {- this.writeLog(LogLevel.WARN, message, context);+ warn(message: unknown, context?: string | LogContext, fields?: LogContext): void {+ this.writeLog(LogLevel.WARN, message, context, undefined, fields);}- debug(message: unknown, context?: string | LogContext): void {- this.writeLog(LogLevel.DEBUG, message, context);+ debug(message: unknown, context?: string | LogContext, fields?: LogContext): void {+ this.writeLog(LogLevel.DEBUG, message, context, undefined, fields);}verbose(message: unknown, context?: string | LogContext): void {this.writeLog(LogLevel.VERBOSE, message, context);}- private writeLog(level: LogLevel, rawMessage: unknown, context?: string | LogContext, rawTrace?: unknown): void {+ private writeLog(+ level: LogLevel,+ rawMessage: unknown,+ context?: string | LogContext,+ rawTrace?: unknown,+ fields?: LogContext,+ ): void {if (!this.shouldLog(level)) return;// The message and trace are whatever the caller (or Nest) handed over, so reading them is guarded like the metadata.@@ -289,14 +296,20 @@ export class LoggerService implements NestLoggerService {? this.formatPretty(level, logEntry): JSON.stringify(logEntry);};+ // The caller's own fields are copied apart, so they stay on the line when the metadata is cut or cannot be read. Nest+ // appends its context name to the arguments of a Logger it made, which is not a record.+ const own = typeof fields === 'object' && fields !== null;+ let added: Record<string, unknown> = {};let output: string;try {+ if (own) added = redactSecrets(fields) as Record<string, unknown>;const copy = typeof context === 'object' && context !== null ? redactSecrets(context) : {};// An array over the length limit comes back as the marker text: one field, not a record to spread.- output = render(typeof copy === 'string' ? { metadata: copy } : (copy as Record<string, unknown>));+ const metadata = typeof copy === 'string' ? { metadata: copy } : (copy as Record<string, unknown>);+ output = render(own ? { ...metadata, ...added } : metadata);} catch {// A getter or proxy trap in the caller's metadata threw. The line is still worth writing, without that metadata.- output = render({ metadata: '[Unreadable]' });+ output = render({ metadata: '[Unreadable]', ...added });}switch (level) {diff --git a/src/core/plugins/plugin-capability-context.ts b/src/core/plugins/plugin-capability-context.tsindex 3d9fe6b86..6d3ef646a 100644--- a/src/core/plugins/plugin-capability-context.ts+++ b/src/core/plugins/plugin-capability-context.ts@@ -235,19 +235,22 @@ export class PluginCapabilityContext {};}+ // The plugin id is the host's field, passed apart from the plugin's meta so that a meta the logger cuts keeps it.private buildPluginLogger(plugin: PluginInstance): PluginLogger {return {log: (message, meta) =>- this.logger.log(`[${plugin.manifest.id}] ${message}`, { ...metaFields(meta), pluginId: plugin.manifest.id }),+ this.logger.log(`[${plugin.manifest.id}] ${message}`, metaFields(meta), { pluginId: plugin.manifest.id }),debug: (message, meta) =>- this.logger.debug(`[${plugin.manifest.id}] ${message}`, { ...metaFields(meta), pluginId: plugin.manifest.id }),+ this.logger.debug(`[${plugin.manifest.id}] ${message}`, metaFields(meta), { pluginId: plugin.manifest.id }),warn: (message, meta) =>- this.logger.warn(`[${plugin.manifest.id}] ${message}`, { ...metaFields(meta), pluginId: plugin.manifest.id }),+ this.logger.warn(`[${plugin.manifest.id}] ${message}`, metaFields(meta), { pluginId: plugin.manifest.id }),error: (message, error, meta) =>- this.logger.error(`[${plugin.manifest.id}] ${message}`, error === undefined ? undefined : errorText(error), {- ...metaFields(meta),- pluginId: plugin.manifest.id,- }),+ this.logger.error(+ `[${plugin.manifest.id}] ${message}`,+ error === undefined ? undefined : errorText(error),+ metaFields(meta),+ { pluginId: plugin.manifest.id },+ ),};}diff --git a/src/core/plugins/plugin-capability-logger.spec.ts b/src/core/plugins/plugin-capability-logger.spec.tsindex 6da47fefd..5abb6a9fe 100644--- a/src/core/plugins/plugin-capability-logger.spec.ts+++ b/src/core/plugins/plugin-capability-logger.spec.ts@@ -37,15 +37,25 @@ describe('the logger a plugin gets', () => {logger.error('upload failed');logger.error('upload failed', undefined, { sessionId: 's1' });- expect(host.error).toHaveBeenNthCalledWith(1, '[p1] upload failed', undefined, { pluginId: 'p1' });- expect(host.error).toHaveBeenNthCalledWith(2, '[p1] upload failed', undefined, { sessionId: 's1', pluginId: 'p1' });+ expect(host.error).toHaveBeenNthCalledWith(1, '[p1] upload failed', undefined, {}, { pluginId: 'p1' });+ expect(host.error).toHaveBeenNthCalledWith(+ 2,+ '[p1] upload failed',+ undefined,+ { sessionId: 's1' },+ { pluginId: 'p1' },+ );});// The parameter is a record, but plugin code is not type checked: a string or an array given as the meta must not be// spread into one field per character or per index.describe('a meta that is not a record', () => {- // The meta a host logger call carried: the second argument, or the third of error().- const metaOf = (spy: jest.Mock, argument: 1 | 2): unknown => (spy.mock.calls as unknown[][])[0][argument];+ // The fields a host logger call writes: the meta (the second argument, or the third of error()) and the host's own+ // fields, which follow it.+ const metaOf = (spy: jest.Mock, argument: 1 | 2): unknown => {+ const [meta, own] = (spy.mock.calls as unknown[][])[0].slice(argument);+ return { ...(meta as object), ...(own as object) };+ };const asRecord = (meta: unknown): Record<string, unknown> => meta as Record<string, unknown>;it.each([@@ -73,7 +83,7 @@ describe('the logger a plugin gets', () => {logger.warn('m', asRecord(meta));- expect(host.warn).toHaveBeenCalledWith('[p1] m', { pluginId: 'p1' });+ expect(host.warn).toHaveBeenCalledWith('[p1] m', {}, { pluginId: 'p1' });expect(Object.keys(metaOf(host.warn, 1) as object)).toEqual(['pluginId']);});@@ -108,7 +118,7 @@ describe('the logger a plugin gets', () => {logger.warn('m', asRecord(meta));- expect(host.warn).toHaveBeenCalledWith('[p1] m', { meta, pluginId: 'p1' });+ expect(host.warn).toHaveBeenCalledWith('[p1] m', { meta }, { pluginId: 'p1' });expect(Object.keys(metaOf(host.warn, 1) as object)).toEqual(['meta', 'pluginId']);});@@ -132,7 +142,75 @@ describe('the logger a plugin gets', () => {logger.warn('m', { sessionId: 's1', nested: { a: 1 } });- expect(host.warn).toHaveBeenCalledWith('[p1] m', { sessionId: 's1', nested: { a: 1 }, pluginId: 'p1' });+ expect(host.warn).toHaveBeenCalledWith('[p1] m', { sessionId: 's1', nested: { a: 1 } }, { pluginId: 'p1' });+ });+ });++ // The plugin id is the host's field, not part of the plugin's meta: a meta the logger cuts or cannot read keeps it.+ describe('a meta the logger cannot write whole', () => {+ const wide = (): object => {+ const meta: Record<string, number> = {};+ for (let key = 0; key <= 2_000_000; key++) meta[`k${key}`] = 1;+ return meta;+ };+ const throwing = (): object => ({+ a: {+ get b(): never {+ throw new Error('getter');+ },+ },+ });++ it.each([+ ['a value below it throws when read', throwing, 'metadata', '[Unreadable]'],+ ['it has more keys than one line may copy', wide, 'metadata', '[Truncated]'],+ [+ 'a key name passes the characters one line may write',+ () => ({ a: 'x'.repeat(2 ** 26 - 5), bbbbbbbbbb: 1 }),+ 'metadata',+ '[Truncated]',+ ],+ [+ 'its strings take every character one line may write',+ () => ({ big: 'x'.repeat(2 ** 26) }),+ 'big',+ '[Truncated]',+ ],+ ])('keeps the plugin id when %s', (_name, make, field, marker) => {+ LoggerService.setLogFormat(LogFormat.JSON);+ const warn = jest.spyOn(console, 'warn').mockImplementation();+ const error = jest.spyOn(console, 'error').mockImplementation();+ try {+ const { logger } = setup(createLogger('Plugins'));++ logger.warn('failed', make() as Record<string, unknown>);+ logger.error('failed', new Error('boom'), make() as Record<string, unknown>);++ for (const spy of [warn, error]) {+ const line = JSON.parse((spy.mock.calls as string[][])[0][0]) as Record<string, unknown>;+ expect(line).toMatchObject({ message: '[p1] failed', [field]: marker, pluginId: 'p1' });+ }+ expect(JSON.parse((error.mock.calls as string[][])[0][0])).toMatchObject({ trace: 'boom' });+ } finally {+ warn.mockRestore();+ error.mockRestore();+ LoggerService.setLogFormat(null);+ }+ });++ it('writes the host plugin id where a pluginId of the meta stood, never the meta one', () => {+ LoggerService.setLogFormat(LogFormat.JSON);+ const warn = jest.spyOn(console, 'warn').mockImplementation();+ try {+ const { logger } = setup(createLogger('Plugins'));++ logger.warn('m', { pluginId: 'other', a: 1 });++ expect((warn.mock.calls as string[][])[0][0]).toContain('"message":"[p1] m","pluginId":"p1","a":1}');+ } finally {+ warn.mockRestore();+ LoggerService.setLogFormat(null);+ }});});diff --git a/src/core/plugins/plugin-loader-sandbox-routing.spec.ts b/src/core/plugins/plugin-loader-sandbox-routing.spec.tsindex fd14aa2b2..c90ad890c 100644--- a/src/core/plugins/plugin-loader-sandbox-routing.spec.ts+++ b/src/core/plugins/plugin-loader-sandbox-routing.spec.ts@@ -669,7 +669,7 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {const logSpy = jest.spyOn(loggerOf(loader), 'log').mockImplementation(() => undefined);expect(() => loader.capturedOnLog!('info' as PluginLogLevel, 'hello')).not.toThrow();- expect(logSpy).toHaveBeenCalledWith('[p1] hello', { pluginId: 'p1' });+ expect(logSpy).toHaveBeenCalledWith('[p1] hello', {}, { pluginId: 'p1' });});it('truncates an oversized worker log line before relaying it', async () => {@@ -708,7 +708,7 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', message as unknown as string);- expect(logSpy).toHaveBeenCalledWith('[p1] [non-string log message]', { pluginId: 'p1' });+ expect(logSpy).toHaveBeenCalledWith('[p1] [non-string log message]', {}, { pluginId: 'p1' });});it('is relayed at once when it is an array whose levels share the same child', async () => {@@ -721,7 +721,7 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', sharedArray(7) as unknown as string);expect(Date.now() - started).toBeLessThan(1000);- expect(logSpy).toHaveBeenCalledWith('[p1] [non-string log message]', { pluginId: 'p1' });+ expect(logSpy).toHaveBeenCalledWith('[p1] [non-string log message]', {}, { pluginId: 'p1' });});});@@ -734,8 +734,8 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', 'big', { data: 'x'.repeat(20000) });loader.capturedOnLog!('log', 'small', { n: 1 });- expect(logSpy).toHaveBeenNthCalledWith(1, '[p1] big', { metaTruncated: true, pluginId: 'p1' });- expect(logSpy).toHaveBeenNthCalledWith(2, '[p1] small', { n: 1, pluginId: 'p1' });+ expect(logSpy).toHaveBeenNthCalledWith(1, '[p1] big', { metaTruncated: true }, { pluginId: 'p1' });+ expect(logSpy).toHaveBeenNthCalledWith(2, '[p1] small', { n: 1 }, { pluginId: 'p1' });});it('relays a worker log meta that is a string or an array as one field', async () => {@@ -754,12 +754,12 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', 'boxed', structuredClone(new String('ab')) as unknown as Record<string, unknown>);loader.capturedOnLog!('log', 'record', structuredClone({ id: 1 }));- expect(logSpy).toHaveBeenNthCalledWith(1, '[p1] string', { meta: 'abc', pluginId: 'p1' });- expect(logSpy).toHaveBeenNthCalledWith(2, '[p1] array', { meta: [1, 2], pluginId: 'p1' });- expect(logSpy).toHaveBeenNthCalledWith(3, '[p1] bytes', { meta: new Uint8Array([1, 2]), pluginId: 'p1' });- expect(Object.keys(logSpy.mock.calls[3][1] as object)).toEqual(['meta', 'pluginId']);+ expect(logSpy).toHaveBeenNthCalledWith(1, '[p1] string', { meta: 'abc' }, { pluginId: 'p1' });+ expect(logSpy).toHaveBeenNthCalledWith(2, '[p1] array', { meta: [1, 2] }, { pluginId: 'p1' });+ expect(logSpy).toHaveBeenNthCalledWith(3, '[p1] bytes', { meta: new Uint8Array([1, 2]) }, { pluginId: 'p1' });+ expect(Object.keys(logSpy.mock.calls[3][1] as object)).toEqual(['meta']);expect(String((logSpy.mock.calls[3][1] as { meta: unknown }).meta)).toBe('ab');- expect(logSpy).toHaveBeenNthCalledWith(5, '[p1] record', { id: 1, pluginId: 'p1' });+ expect(logSpy).toHaveBeenNthCalledWith(5, '[p1] record', { id: 1 }, { pluginId: 'p1' });});it('keeps the error text of a worker error log whose meta is oversized or not serializable', async () => {@@ -776,16 +776,18 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('error', 'long', { error: 'e'.repeat(10000) });loader.capturedOnLog!('error', 'nullmeta', null as unknown as Record<string, unknown>);- expect(errorSpy).toHaveBeenNthCalledWith(1, '[p1] big', undefined, {- error: 'boom',- metaTruncated: true,- pluginId: 'p1',- });- expect(errorSpy).toHaveBeenNthCalledWith(2, '[p1] bigint', undefined, { error: 'boom', pluginId: 'p1' });+ expect(errorSpy).toHaveBeenNthCalledWith(+ 1,+ '[p1] big',+ undefined,+ { error: 'boom', metaTruncated: true },+ { pluginId: 'p1' },+ );+ expect(errorSpy).toHaveBeenNthCalledWith(2, '[p1] bigint', undefined, { error: 'boom' }, { pluginId: 'p1' });const longMeta = errorSpy.mock.calls[2][2] as { error: string; metaTruncated: boolean };expect(longMeta.metaTruncated).toBe(true);expect(longMeta.error).toBe(`${'e'.repeat(8192)}…[truncated]`);- expect(errorSpy).toHaveBeenNthCalledWith(4, '[p1] nullmeta', undefined, { pluginId: 'p1' });+ expect(errorSpy).toHaveBeenNthCalledWith(4, '[p1] nullmeta', undefined, {}, { pluginId: 'p1' });});// postMessage keeps shared references shared, so a meta that is small to hold writes out, at every use, as JSON@@ -810,7 +812,7 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', 'fan-out', fanOut(8));expect(Date.now() - started).toBeLessThan(1000);- expect(logSpy).toHaveBeenCalledWith('[p1] fan-out', { metaTruncated: true, pluginId: 'p1' });+ expect(logSpy).toHaveBeenCalledWith('[p1] fan-out', { metaTruncated: true }, { pluginId: 'p1' });});it('keeps the error text of a worker error log', async () => {@@ -825,11 +827,12 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('error', 'fan-out', { error: 'boom', fan: fanOut(8) });expect(Date.now() - started).toBeLessThan(1000);- expect(errorSpy).toHaveBeenCalledWith('[p1] fan-out', undefined, {- error: 'boom',- metaTruncated: true,- pluginId: 'p1',- });+ expect(errorSpy).toHaveBeenCalledWith(+ '[p1] fan-out',+ undefined,+ { error: 'boom', metaTruncated: true },+ { pluginId: 'p1' },+ );});it('still reports the exact length of a meta that has no more values than the size limit', async () => {@@ -843,12 +846,13 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', 'walked', ones(8191) as unknown as Record<string, unknown>);loader.capturedOnLog!('log', 'stopped', ones(8192) as unknown as Record<string, unknown>);- expect(logSpy).toHaveBeenNthCalledWith(1, '[p1] walked', {- metaTruncated: true,- metaLength: 8191 + 8190 + 2,- pluginId: 'p1',- });- expect(logSpy).toHaveBeenNthCalledWith(2, '[p1] stopped', { metaTruncated: true, pluginId: 'p1' });+ expect(logSpy).toHaveBeenNthCalledWith(+ 1,+ '[p1] walked',+ { metaTruncated: true, metaLength: 8191 + 8190 + 2 },+ { pluginId: 'p1' },+ );+ expect(logSpy).toHaveBeenNthCalledWith(2, '[p1] stopped', { metaTruncated: true }, { pluginId: 'p1' });});it('replaces a typed array of 20 million elements by the size marker at once', async () => {@@ -861,7 +865,7 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', 'buffer', { b: new Uint8Array(20_000_000) });expect(Date.now() - started).toBeLessThan(1000);- expect(logSpy).toHaveBeenCalledWith('[p1] buffer', { metaTruncated: true, pluginId: 'p1' });+ expect(logSpy).toHaveBeenCalledWith('[p1] buffer', { metaTruncated: true }, { pluginId: 'p1' });});it('replaces a String wrapper of 10 million characters by the size marker at once', async () => {@@ -875,7 +879,7 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', 'boxed', boxed as unknown as Record<string, unknown>);expect(Date.now() - started).toBeLessThan(1000);- expect(logSpy).toHaveBeenCalledWith('[p1] boxed', { metaTruncated: true, pluginId: 'p1' });+ expect(logSpy).toHaveBeenCalledWith('[p1] boxed', { metaTruncated: true }, { pluginId: 'p1' });});it('replaces one long string shared by 2000 entries by the size marker at once', async () => {@@ -889,7 +893,7 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', 'shared-string', { list: Array.from({ length: 2000 }, () => holder) });expect(Date.now() - started).toBeLessThan(1000);- expect(logSpy).toHaveBeenCalledWith('[p1] shared-string', { metaTruncated: true, pluginId: 'p1' });+ expect(logSpy).toHaveBeenCalledWith('[p1] shared-string', { metaTruncated: true }, { pluginId: 'p1' });});it('replaces one object with a long property name shared by 4000 entries by the size marker at once', async () => {@@ -903,7 +907,7 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', 'shared-name', { list: Array.from({ length: 4000 }, () => holder) });expect(Date.now() - started).toBeLessThan(1000);- expect(logSpy).toHaveBeenCalledWith('[p1] shared-name', { metaTruncated: true, pluginId: 'p1' });+ expect(logSpy).toHaveBeenCalledWith('[p1] shared-name', { metaTruncated: true }, { pluginId: 'p1' });});it('counts a Map or a Set by its size', async () => {@@ -915,8 +919,8 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', 'map', { m: new Map(ones(9000).map((one, key) => [key, one])) });loader.capturedOnLog!('log', 'set', { s: new Set(ones(9000).map((_, key) => key)) });- expect(logSpy).toHaveBeenNthCalledWith(1, '[p1] map', { metaTruncated: true, pluginId: 'p1' });- expect(logSpy).toHaveBeenNthCalledWith(2, '[p1] set', { metaTruncated: true, pluginId: 'p1' });+ expect(logSpy).toHaveBeenNthCalledWith(1, '[p1] map', { metaTruncated: true }, { pluginId: 'p1' });+ expect(logSpy).toHaveBeenNthCalledWith(2, '[p1] set', { metaTruncated: true }, { pluginId: 'p1' });});it('still drops a circular meta without a marker, as it cannot be written as JSON', async () => {@@ -929,7 +933,7 @@ describe('PluginLoaderService — sandbox log relay bounds', () => {loader.capturedOnLog!('log', 'circular', circular);- expect(logSpy).toHaveBeenCalledWith('[p1] circular', { pluginId: 'p1' });+ expect(logSpy).toHaveBeenCalledWith('[p1] circular', {}, { pluginId: 'p1' });});});});