Skip to content

Chrome DevTools for coding agents

27.2 KB
Raw Blame
/**
 * @license
 * Copyright 2025 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

import assert from 'node:assert';
import {describe, it} from 'node:test';

import sinon from 'sinon';

import {SymbolizedError} from '../../src/devtools/DevtoolsUtils.js';
import {ConsoleFormatter} from '../../src/formatters/ConsoleFormatter.js';
import {UncaughtError} from '../../src/collectors/PageCollector.js';
import type {
  ConsoleMessage,
  Protocol,
  DevTools,
} from '../../src/third_party/index.js';
import {createMockCDPSession} from '../mocks.js';

interface MockConsoleMessage {
  type: () => string;
  text: () => string;
  args: () => Array<{
    jsonValue: () => Promise<unknown>;
    remoteObject: () => Protocol.Runtime.RemoteObject;
  }>;
  stackTrace?: DevTools.StackTrace.StackTrace.StackTrace;
}

const createMockMessage = (
  data: Partial<MockConsoleMessage> = {},
): ConsoleMessage => {
  return {
    type: () => data.type?.() ?? 'log',
    text: () => data.text?.() ?? '',
    args: () => data.args?.() ?? [],
    ...data,
  } as unknown as ConsoleMessage;
};

function formatterTestConcise(
  label: string,
  setup: (t: it.TestContext) => Promise<ConsoleFormatter>,
) {
  it(label + ' toString', async t => {
    const formatter = await setup(t);
    t.assert.snapshot(formatter.toString());
  });
  it(label + ' toJSON', async t => {
    const formatter = await setup(t);
    t.assert.snapshot(JSON.stringify(formatter.toJSON(), null, 2));
  });
}

function formatterTestDetailed(
  label: string,
  setup: (t: it.TestContext) => Promise<ConsoleFormatter>,
) {
  it(label + ' toStringDetailed', async t => {
    const formatter = await setup(t);
    t.assert.snapshot(formatter.toStringDetailed());
  });
  it(label + ' toJSONDetailed', async t => {
    const formatter = await setup(t);
    t.assert.snapshot(JSON.stringify(formatter.toJSONDetailed(), null, 2));
  });
}

describe('ConsoleFormatter', () => {
  describe('toString/toJSON', () => {
    formatterTestConcise('formats a console.log message', async () => {
      const message = createMockMessage({
        type: () => 'log',
        text: () => 'Hello, world!',
      });
      return await ConsoleFormatter.from(message, {id: 1});
    });

    formatterTestConcise(
      'formats a console.log message with one argument',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Processing file:',
          args: () => [
            {
              jsonValue: async () => 'file.txt',
              remoteObject: () => ({type: 'string'}),
            },
          ],
        });
        return await ConsoleFormatter.from(message, {id: 1});
      },
    );

    formatterTestConcise(
      'formats a console.log message with multiple arguments',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Processing file:',
          args: () => [
            {
              jsonValue: async () => 'file.txt',
              remoteObject: () => ({type: 'string'}),
            },
            {
              jsonValue: async () => 'another file',
              remoteObject: () => ({type: 'string'}),
            },
          ],
        });
        return await ConsoleFormatter.from(message, {id: 1});
      },
    );

    formatterTestConcise('formats an UncaughtError', async () => {
      const error = new UncaughtError(
        {
          exceptionId: 1,
          lineNumber: 0,
          columnNumber: 5,
          exception: {
            type: 'object',
            description: 'TypeError: Cannot read properties of undefined',
          },
          text: 'Uncaught',
        },
        '<mock target ID>',
      );
      return await ConsoleFormatter.from(error, {id: 1});
    });

    formatterTestConcise(
      'omits the stack trace when fetchStackTrace is not set',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Hello stack trace!',
        });
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'foo.ts',
                name: 'foo',
              },
            ],
          },
          asyncFragments: [],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;

        return await ConsoleFormatter.from(message, {
          id: 1,
          resolvedStackTraceForTesting: stackTrace,
        });
      },
    );

    formatterTestConcise(
      'formats a console message with a stack trace when fetchStackTrace is set',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Hello stack trace!',
        });
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'foo.ts',
                name: 'foo',
              },
              {
                line: 20,
                column: 2,
                url: 'foo.ts',
                name: 'bar',
              },
            ],
          },
          asyncFragments: [
            {
              description: 'setTimeout',
              frames: [
                {
                  line: 5,
                  column: 2,
                  url: 'util.ts',
                  name: 'schedule',
                },
              ],
            },
          ],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;

        return await ConsoleFormatter.from(message, {
          id: 1,
          fetchStackTrace: true,
          resolvedStackTraceForTesting: stackTrace,
        });
      },
    );

    formatterTestConcise(
      'formats an UncaughtError with a stack trace when fetchStackTrace is set',
      async () => {
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'foo.ts',
                name: 'foo',
              },
            ],
          },
          asyncFragments: [],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;
        const error = new UncaughtError(
          {
            exceptionId: 1,
            lineNumber: 0,
            columnNumber: 5,
            exception: {
              type: 'object',
              description: 'TypeError: Cannot read properties of undefined',
            },
            text: 'Uncaught',
          },
          '<mock target ID>',
        );
        return await ConsoleFormatter.from(error, {
          id: 2,
          fetchStackTrace: true,
          resolvedStackTraceForTesting: stackTrace,
        });
      },
    );
  });

  describe('toStringDetailed/toJSONDetailed', () => {
    formatterTestDetailed('formats a console.log message', async () => {
      const message = createMockMessage({
        type: () => 'log',
        text: () => 'Hello, world!',
      });
      return await ConsoleFormatter.from(message, {
        id: 1,
        fetchDetailedData: true,
      });
    });

    formatterTestDetailed(
      'formats a console.log message with one argument',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Processing file:',
          args: () => [
            {
              jsonValue: async () => 'file.txt',
              remoteObject: () => ({type: 'string'}),
            },
          ],
        });
        return await ConsoleFormatter.from(message, {
          id: 1,
          fetchDetailedData: true,
        });
      },
    );

    formatterTestDetailed(
      'formats a console.log message with multiple arguments',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Processing file:',
          args: () => [
            {
              jsonValue: async () => 'file.txt',
              remoteObject: () => ({type: 'string'}),
            },
            {
              jsonValue: async () => 'another file',
              remoteObject: () => ({type: 'string'}),
            },
          ],
        });
        return await ConsoleFormatter.from(message, {
          id: 1,
          fetchDetailedData: true,
        });
      },
    );

    formatterTestDetailed('formats a console.error message', async () => {
      const message = createMockMessage({
        type: () => 'error',
        text: () => 'Something went wrong',
      });
      return await ConsoleFormatter.from(message, {
        id: 1,
        fetchDetailedData: true,
      });
    });

    formatterTestDetailed(
      'formats a console message with a stack trace',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Hello stack trace!',
        });
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'foo.ts',
                name: 'foo',
              },
              {
                line: 20,
                column: 2,
                url: 'foo.ts',
                name: 'bar',
              },
            ],
          },
          asyncFragments: [
            {
              description: 'setTimeout',
              frames: [
                {
                  line: 5,
                  column: 2,
                  url: 'util.ts',
                  name: 'schedule',
                },
              ],
            },
          ],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;

        return await ConsoleFormatter.from(message, {
          id: 1,
          resolvedStackTraceForTesting: stackTrace,
        });
      },
    );

    formatterTestDetailed(
      'handles "Execution context is not available" error in args',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Processing file:',
          args: () => [
            {
              jsonValue: async () => {
                throw new Error('Execution context is not available');
              },
              remoteObject: () => ({type: 'string'}),
            },
          ],
        });
        return await ConsoleFormatter.from(message, {
          id: 6,
          fetchDetailedData: true,
        });
      },
    );

    formatterTestDetailed(
      'formats an UncaughtError with a stack trace',
      async () => {
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'foo.ts',
                name: 'foo',
              },
              {
                line: 20,
                column: 2,
                url: 'foo.ts',
                name: 'bar',
              },
            ],
          },
          asyncFragments: [
            {
              description: 'setTimeout',
              frames: [
                {
                  line: 5,
                  column: 2,
                  url: 'util.ts',
                  name: 'schedule',
                },
              ],
            },
          ],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;
        const error = new UncaughtError(
          {
            exceptionId: 1,
            lineNumber: 0,
            columnNumber: 5,
            exception: {
              type: 'object',
              description: 'TypeError: Cannot read properties of undefined',
            },
            text: 'Uncaught',
          },
          '<mock target ID>',
        );

        return await ConsoleFormatter.from(error, {
          id: 7,
          resolvedStackTraceForTesting: stackTrace,
        });
      },
    );

    formatterTestDetailed(
      'formats a console message with an Error object argument',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'JSHandle@error',
        });
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'foo.ts',
                name: 'foo',
              },
              {
                line: 20,
                column: 2,
                url: 'foo.ts',
                name: 'bar',
              },
            ],
          },
          asyncFragments: [],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;
        const error = SymbolizedError.createForTesting(
          'TypeError: Cannot read properties of undefined',
          stackTrace,
        );

        return await ConsoleFormatter.from(message, {
          id: 8,
          resolvedArgsForTesting: [error],
        });
      },
    );

    formatterTestDetailed(
      'formats a console message with an Error object with cause',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'JSHandle@error',
        });
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'foo.ts',
                name: 'foo',
              },
              {
                line: 20,
                column: 2,
                url: 'foo.ts',
                name: 'bar',
              },
            ],
          },
          asyncFragments: [],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;
        const error = SymbolizedError.createForTesting(
          'AppError: Compute failed',
          stackTrace,
          SymbolizedError.createForTesting(
            'TypeError: Cannot read properties of undefined',
            {
              syncFragment: {
                frames: [
                  {
                    line: 5,
                    column: 10,
                    url: 'library.js',
                    name: 'compute',
                  },
                ],
              },
              asyncFragments: [],
            } as unknown as DevTools.StackTrace.StackTrace.StackTrace,
          ),
        );

        return await ConsoleFormatter.from(message, {
          id: 9,
          resolvedArgsForTesting: [error],
        });
      },
    );

    formatterTestDetailed(
      'formats an UncaughtError with a stack trace and a cause',
      async () => {
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'foo.ts',
                name: 'foo',
              },
              {
                line: 20,
                column: 2,
                url: 'foo.ts',
                name: 'bar',
              },
            ],
          },
          asyncFragments: [
            {
              description: 'setTimeout',
              frames: [
                {
                  line: 5,
                  column: 2,
                  url: 'util.ts',
                  name: 'schedule',
                },
              ],
            },
          ],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;
        const error = new UncaughtError(
          {
            exceptionId: 1,
            lineNumber: 0,
            columnNumber: 5,
            exception: {
              type: 'object',
              description: 'TypeError: Cannot read properties of undefined',
            },
            text: 'Uncaught',
          },
          '<mock target ID>',
        );
        const cause = SymbolizedError.createForTesting(
          'TypeError: Cannot read properties of undefined',
          {
            syncFragment: {
              frames: [
                {
                  line: 5,
                  column: 8,
                  url: 'library.js',
                  name: 'compute',
                },
              ],
            },
            asyncFragments: [],
          } as unknown as DevTools.StackTrace.StackTrace.StackTrace,
        );

        return await ConsoleFormatter.from(error, {
          id: 10,
          resolvedStackTraceForTesting: stackTrace,
          resolvedCauseForTesting: cause,
        });
      },
    );

    formatterTestDetailed(
      'limits the number lines for a stack trace',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Hello stack trace!',
        });
        const frames: DevTools.StackTrace.StackTrace.Frame[] = [];
        for (let i = 0; i < 100; ++i) {
          frames.push({
            line: i,
            column: i,
            url: 'main.js',
            name: `fn${i}`,
            toUiLocation: () => null,
          });
        }
        const stackTrace = {
          syncFragment: {frames},
          asyncFragments: [],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;

        return await ConsoleFormatter.from(message, {
          id: 11,
          resolvedStackTraceForTesting: stackTrace,
        });
      },
    );

    formatterTestDetailed(
      'does not show call frames with ignore listed scripts',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Hello stack trace!',
        });
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'foo.ts',
                name: 'foo',
              },
              {
                line: 200,
                column: 46,
                url: './node_modules/some-third-party-package/lib/index.js',
                name: 'doThings',
              },
              {
                line: 250,
                column: 12,
                url: './node_modules/some-third-party-package/lib/index.js',
                name: 'doThings2',
              },
              {
                line: 20,
                column: 2,
                url: 'foo.ts',
                name: 'bar',
              },
            ],
          },
          asyncFragments: [],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;

        return await ConsoleFormatter.from(message, {
          id: 12,
          resolvedStackTraceForTesting: stackTrace,
          isIgnoredForTesting: frame =>
            Boolean(frame.url?.includes('node_modules')),
        });
      },
    );

    formatterTestDetailed(
      'does not show fragments where all frames are ignore listed',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Hello stack trace!',
        });
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'foo.ts',
                name: 'foo',
              },
            ],
          },
          asyncFragments: [
            {
              description: 'setTimeout',
              frames: [
                {
                  line: 200,
                  column: 46,
                  url: './node_modules/some-third-party-package/lib/index.js',
                  name: 'doThings',
                },
                {
                  line: 250,
                  column: 12,
                  url: './node_modules/some-third-party-package/lib/index.js',
                  name: 'doThings2',
                },
              ],
            },
            {
              description: 'await',
              frames: [
                {
                  line: 20,
                  column: 2,
                  url: 'foo.ts',
                  name: 'bar',
                },
              ],
            },
          ],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;

        return await ConsoleFormatter.from(message, {
          id: 13,
          resolvedStackTraceForTesting: stackTrace,
          isIgnoredForTesting: frame =>
            Boolean(frame.url?.includes('node_modules')),
        });
      },
    );

    formatterTestDetailed(
      'uses the first argument as text when the message text is empty',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => '',
        });
        return await ConsoleFormatter.from(message, {
          id: 14,
          resolvedArgsForTesting: ['Actual message', 'extra-arg'],
        });
      },
    );

    formatterTestDetailed(
      'formats object and primitive arguments',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Mixed args:',
        });
        return await ConsoleFormatter.from(message, {
          id: 15,
          resolvedArgsForTesting: [
            {user: 'alice', roles: ['admin']},
            42,
            null,
            true,
          ],
        });
      },
    );

    formatterTestDetailed(
      'formats frames without a name or url and unnamed async fragments',
      async () => {
        const message = createMockMessage({
          type: () => 'log',
          text: () => 'Hello stack trace!',
        });
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
              },
              {
                line: 20,
                column: 2,
                url: 'foo.ts',
                name: 'bar',
              },
            ],
          },
          asyncFragments: [
            {
              frames: [
                {
                  line: 5,
                  column: 2,
                  url: 'util.ts',
                  name: 'schedule',
                },
              ],
            },
          ],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;

        return await ConsoleFormatter.from(message, {
          id: 16,
          resolvedStackTraceForTesting: stackTrace,
        });
      },
    );

    formatterTestDetailed(
      'formats an async fragment with a description longer than the separator line',
      async () => {
        const message = createMockMessage({
          type: () => 'error',
          text: () => 'TypeError: Cannot read properties of undefined',
        });
        const stackTrace = {
          syncFragment: {
            frames: [
              {
                line: 10,
                column: 2,
                url: 'Component.tsx',
                name: 'MyVeryLongReactComponentNameThatExceedsFortyCharacters',
              },
            ],
          },
          asyncFragments: [
            {
              description:
                '<MyVeryLongReactComponentNameThatExceedsFortyCharacters>',
              frames: [
                {
                  line: 5,
                  column: 2,
                  url: 'App.tsx',
                  name: 'App',
                },
              ],
            },
          ],
        } as unknown as DevTools.StackTrace.StackTrace.StackTrace;

        return await ConsoleFormatter.from(message, {
          id: 17,
          resolvedStackTraceForTesting: stackTrace,
        });
      },
    );
  });

  it('resolves the cause of an UncaughtError on its session', async t => {
    const session = createMockCDPSession();
    const send: sinon.SinonStub = session.send;
    send.onFirstCall().resolves({
      result: [
        {
          name: 'cause',
          configurable: true,
          enumerable: false,
          value: {
            type: 'object',
            subtype: 'error',
            objectId: 'cause-id',
            description: 'Error: root cause',
          },
        },
      ],
    });
    send.onSecondCall().resolves({
      exceptionDetails: {
        exceptionId: 2,
        lineNumber: 0,
        columnNumber: 0,
        text: 'root cause',
      },
    });
    const error = new UncaughtError(
      {
        exceptionId: 1,
        lineNumber: 0,
        columnNumber: 5,
        exception: {
          type: 'object',
          subtype: 'error',
          objectId: 'error-id',
          description: 'Error: wrapper',
        },
        text: 'Uncaught',
      },
      '<mock target ID>',
      session,
    );

    const formatter = await ConsoleFormatter.from(error, {
      id: 18,
      fetchDetailedData: true,
      resolvedStackTraceForTesting: {
        syncFragment: {
          frames: [{line: 1, column: 1, url: 'main.js', name: 'main'}],
        },
        asyncFragments: [],
      } as unknown as DevTools.StackTrace.StackTrace.StackTrace,
    });

    sinon.assert.calledWithExactly(send.firstCall, 'Runtime.getProperties', {
      objectId: 'error-id',
    });
    sinon.assert.calledWithExactly(
      send.secondCall,
      'Runtime.getExceptionDetails',
      {errorObjectId: 'cause-id'},
    );
    t.assert.match(formatter.toStringDetailed(), /Caused by: root cause/);
  });
});

describe('ConsoleFormatter arguments', () => {
  it('formats non-JSON arguments from their preview and keeps JSON for plain values', async () => {
    const message = createMockMessage({
      type: () => 'log',
      text: () => 'values',
      args: () => [
        {
          jsonValue: async () => ({}),
          remoteObject: () => ({
            type: 'object',
            subtype: 'map',
            description: 'Map(1)',
            preview: {
              type: 'object',
              subtype: 'map',
              description: 'Map(1)',
              overflow: false,
              properties: [{name: 'size', type: 'number', value: '1'}],
              entries: [
                {
                  key: {
                    type: 'number',
                    description: '1',
                    overflow: false,
                    properties: [],
                  },
                  value: {
                    type: 'number',
                    description: '2',
                    overflow: false,
                    properties: [],
                  },
                },
              ],
            },
          }),
        },
        {
          jsonValue: async () => {
            throw new Error('Object reference chain is too long');
          },
          remoteObject: () => ({
            type: 'object',
            description: 'Object',
            preview: {
              type: 'object',
              description: 'Object',
              overflow: false,
              properties: [
                {name: 'name', type: 'string', value: 'cyc'},
                {name: 'self', type: 'object', value: 'Object'},
              ],
            },
          }),
        },
        {
          jsonValue: async () => ({a: 1}),
          remoteObject: () => ({type: 'object'}),
        },
      ],
    });
    const formatter = await ConsoleFormatter.from(message, {
      id: 1,
      fetchDetailedData: true,
    });
    assert.deepStrictEqual(formatter.toJSONDetailed().args, [
      'Map(1) {1 => 2}',
      "{name: 'cyc', self: {…}}",
      '{"a":1}',
    ]);
  });

  it('reports an argument without a preview whose jsonValue() fails as unavailable', async () => {
    const message = createMockMessage({
      type: () => 'log',
      text: () => 'gone',
      args: () => [
        {
          jsonValue: async () => {
            throw new Error('Execution context was destroyed');
          },
          remoteObject: () => ({type: 'object', description: 'Object'}),
        },
      ],
    });
    const formatter = await ConsoleFormatter.from(message, {
      id: 1,
      fetchDetailedData: true,
    });
    assert.deepStrictEqual(formatter.toJSONDetailed().args, [
      '<error: Argument 0 is no longer available>',
    ]);
  });
});