Skip to content

Chrome DevTools for coding agents

25.5 KB
Raw Blame
/**
 * @license
 * Copyright 2025 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

import assert from 'node:assert';
import {readFile} from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import {describe, it} from 'node:test';
import {pathToFileURL} from 'node:url';

import sinon from 'sinon';

import {TextSnapshot} from '../../src/TextSnapshot.js';
import {zod} from '../../src/third_party/index.js';
import {installExtension} from '../../src/tools/extensions.js';
import {evaluateScript} from '../../src/tools/script.js';
import {WaitForHelper} from '../../src/utils/WaitForHelper.js';
import {createHandlerMocks, createMockParsedArguments} from '../mocks.js';
import {serverHooks} from '../server.js';
import {
  assertNoServiceWorkerReported,
  createTempDir,
  extractExtensionId,
  getTextContent,
  html,
  withMcpContext,
} from '../utils.js';

const EXTENSION_PATH = path.resolve('tests/tools/fixtures/extension-sw');

describe('script', () => {
  const server = serverHooks();

  describe('browser_evaluate_script', () => {
    it('evaluates', async () => {
      await withMcpContext(async (response, context, args) => {
        await evaluateScript(args).handler(
          {
            params: {function: String(() => 2 * 5)},
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2)!;
        assert.strictEqual(JSON.parse(lineEvaluation), 10);
      });
    });
    it('evaluates an inline classic script', async () => {
      await withMcpContext(async (response, context, args) => {
        await evaluateScript(args).handler(
          {
            params: {
              function: 'document.title = "Script title"; document.title',
              format: 'script',
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2);
        assert.ok(lineEvaluation);
        assert.strictEqual(JSON.parse(lineEvaluation), 'Script title');
      });
    });
    it('evaluates a function ending with a single-line comment', async () => {
      await withMcpContext(async (response, context, args) => {
        await evaluateScript(args).handler(
          {
            params: {
              function: '() => document.title // get title',
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2);
        assert.ok(lineEvaluation);
        assert.strictEqual(JSON.parse(lineEvaluation), '');
      });
    });
    it('evaluates a function ending with a semicolon', async () => {
      await withMcpContext(async (response, context, args) => {
        await evaluateScript(args).handler(
          {
            params: {
              function: '() => document.title;',
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2);
        assert.ok(lineEvaluation);
        assert.strictEqual(JSON.parse(lineEvaluation), '');
      });
    });
    it('does not execute multi-statement source in function mode', async () => {
      await withMcpContext(async (response, context, args) => {
        await assert.rejects(
          evaluateScript(args).handler(
            {
              params: {
                function: '() => document.title; document.title = "unexpected"',
              },
            },
            response,
            context,
          ),
        );
        assert.strictEqual(
          await context.getSelectedMcpPage().pptrPage.title(),
          '',
        );
      });
    });
    it('surfaces non-Error values when script throws or rejects', async () => {
      await withMcpContext(async (response, context, args) => {
        const cases = [
          {fn: "() => { throw 'str-thrown'; }", expected: 'str-thrown'},
          {fn: "async () => { throw 'async-str'; }", expected: 'async-str'},
          {fn: '() => Promise.reject(42)', expected: '42'},
          {fn: '() => { throw {code: 42}; }', expected: '{"code":42}'},
          {
            fn: "() => { throw new Error('err-thrown'); }",
            expected: 'err-thrown',
          },
        ];
        for (const testCase of cases) {
          let caught: unknown;
          try {
            await evaluateScript(args).handler(
              {params: {function: testCase.fn}},
              response,
              context,
            );
          } catch (err) {
            caught = err;
          }
          assert.ok(
            caught instanceof Error,
            `Expected Error for ${testCase.fn}`,
          );
          assert.ok(
            caught.message.includes(testCase.expected),
            `Expected "${caught.message}" to include "${testCase.expected}"`,
          );
        }
      });
    });
    it('serializes script results inside the browser', async () => {
      await withMcpContext(async (response, context, args) => {
        await evaluateScript(args).handler(
          {
            params: {
              function: 'new URL("https://example.com/path")',
              format: 'script',
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2);
        assert.ok(lineEvaluation);
        assert.strictEqual(
          JSON.parse(lineEvaluation),
          'https://example.com/path',
        );
      });
    });
    it('lists errors the page raised while the function ran', async () => {
      await withMcpContext(async (response, context, args) => {
        await evaluateScript(args).handler(
          {
            params: {
              function: String(
                () =>
                  new Promise(resolve => {
                    console.warn('warned in the function');
                    setTimeout(() => {
                      throw new Error('thrown in a timer');
                    }, 0);
                    setTimeout(() => resolve('done'), 50);
                  }),
              ),
            },
          },
          response,
          context,
        );
        const {content} = await response.handle(context);
        const text = getTextContent(content[0]);
        assert.ok(text.includes('"done"'));
        assert.ok(text.includes('## Console errors during this call'));
        assert.ok(text.includes('thrown in a timer'));
        assert.ok(!text.includes('warned in the function'));
      });
    });
    it('hands its page to the response before the function runs', async () => {
      const {page, response, context, args} = createHandlerMocks();
      page.waitForEventsAfterAction.callsFake(async () => {
        sinon.assert.calledOnceWithExactly(
          response.includeConsoleErrorsFrom,
          page,
        );
        return {};
      });

      await evaluateScript(args).handler(
        {params: {function: String(() => 1)}},
        response,
        context,
      );

      sinon.assert.calledOnce(page.waitForEventsAfterAction);
    });
    it('loads function source from a local file', async () => {
      const {page, response, context, args} = createHandlerMocks();
      const sourcePath = path.join(os.tmpdir(), 'function.js');
      context.loadResource.resolves('() => document.title;');

      await evaluateScript(args).handler(
        {params: {sourcePath}},
        response,
        context,
      );

      sinon.assert.calledOnceWithExactly(
        context.loadResource,
        pathToFileURL(sourcePath).href,
      );
      sinon.assert.calledOnce(page.waitForEventsAfterAction);
    });
    it('loads classic script source from a file URL', async () => {
      const {page, response, context, args} = createHandlerMocks();
      const sourcePath = pathToFileURL(
        path.join(os.tmpdir(), 'script.js'),
      ).href;
      context.loadResource.resolves(
        'document.body.dataset.source = "file"; document.body.dataset.source',
      );

      await evaluateScript(args).handler(
        {params: {sourcePath, format: 'script'}},
        response,
        context,
      );

      sinon.assert.calledOnceWithExactly(context.loadResource, sourcePath);
      sinon.assert.calledOnce(page.waitForEventsAfterAction);
    });
    it('requires exactly one script source', async () => {
      const {response, context, args} = createHandlerMocks();
      await assert.rejects(
        evaluateScript(args).handler({params: {}}, response, context),
        /Specify exactly one of function or sourcePath/,
      );
      await assert.rejects(
        evaluateScript(args).handler(
          {
            params: {
              function: '() => true',
              sourcePath: 'script.js',
            },
          },
          response,
          context,
        ),
        /Specify exactly one of function or sourcePath/,
      );
    });
    it('rejects args for classic scripts', async () => {
      const {response, context, args} = createHandlerMocks();
      await assert.rejects(
        evaluateScript(args).handler(
          {
            params: {
              function: 'document.title',
              format: 'script',
              args: ['1_1'],
            },
          },
          response,
          context,
        ),
        /args cannot be used when format is "script"/,
      );
    });
    it('reports unreadable source files', async () => {
      const {response, context, args} = createHandlerMocks();
      const sourcePath = path.join(
        os.tmpdir(),
        'missing-evaluate-script-source.js',
      );
      context.loadResource.rejects(new Error('File not found'));
      await assert.rejects(
        evaluateScript(args).handler({params: {sourcePath}}, response, context),
        /Unable to read script source.*File not found/,
      );
      sinon.assert.calledOnceWithExactly(
        context.loadResource,
        pathToFileURL(sourcePath).href,
      );
    });
    it('throws without evaluating if a dialog is already open', async () => {
      const {page, response, context, args} = createHandlerMocks();
      page.throwIfDialogOpen.throws(
        new Error('A dialog is open (alert: hello).'),
      );
      await assert.rejects(
        evaluateScript(args).handler(
          {params: {function: '() => 1'}},
          response,
          context,
        ),
        {message: 'A dialog is open (alert: hello).'},
      );
      sinon.assert.calledOnceWithExactly(page.init, {emulateFocus: true});
      sinon.assert.calledOnceWithExactly(page.throwIfDialogOpen);
      sinon.assert.callOrder(page.init, page.throwIfDialogOpen);
      sinon.assert.notCalled(page.waitForEventsAfterAction);
    });
    it('skips the stable DOM wait when waitForStableDom is false', async () => {
      await withMcpContext(async (response, context, args) => {
        const spy = sinon.spy(WaitForHelper.prototype, 'waitForStableDom');
        try {
          await evaluateScript(args).handler(
            {
              params: {function: String(() => 1), waitForStableDom: false},
            },
            response,
            context,
          );
          sinon.assert.notCalled(spy);

          await evaluateScript(args).handler(
            {
              params: {function: String(() => 1)},
            },
            response,
            context,
          );
          sinon.assert.calledOnce(spy);
        } finally {
          spy.restore();
        }
      });
    });
    it('runs in selected page', async () => {
      await withMcpContext(async (response, context, args) => {
        await evaluateScript(args).handler(
          {
            params: {function: String(() => document.title)},
          },
          response,
          context,
        );

        let lineEvaluation = response.responseLines.at(2)!;
        assert.strictEqual(JSON.parse(lineEvaluation), '');

        const page = await context.newPage();
        await page.pptrPage.setContent(`
          <head>
            <title>New Page</title>
          </head>
        `);

        response.resetResponseLineForTesting();
        await evaluateScript(args).handler(
          {
            params: {function: String(() => document.title)},
          },
          response,
          context,
        );

        lineEvaluation = response.responseLines.at(2)!;
        assert.strictEqual(JSON.parse(lineEvaluation), 'New Page');
      });
    });

    it('work for complex objects', async () => {
      await withMcpContext(async (response, context, args) => {
        const page = context.getSelectedMcpPage().pptrPage;

        await page.setContent(html`<script src="./scripts.js"></script> `);

        await evaluateScript(args).handler(
          {
            params: {
              function: String(() => {
                const scripts = Array.from(
                  document.head.querySelectorAll('script'),
                ).map(s => ({src: s.src, async: s.async, defer: s.defer}));

                return {scripts};
              }),
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2)!;
        assert.deepEqual(JSON.parse(lineEvaluation), {
          scripts: [],
        });
      });
    });

    it('work for scripts that trigger dialogs', async () => {
      await withMcpContext(async (response, context, args) => {
        const page = context.getSelectedMcpPage().pptrPage;

        await page.setContent(html`<button id="test">test</button>`);

        await evaluateScript(args).handler(
          {
            params: {
              function: String(() => {
                alert('hello');
                return 'Works';
              }),
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2)!;
        assert.strictEqual(JSON.parse(lineEvaluation), 'Works');
      });
    });

    it('work for scripts that trigger dialogs and dismiss them', async () => {
      await withMcpContext(async (response, context, args) => {
        const page = context.getSelectedMcpPage().pptrPage;

        await page.setContent(html`<button id="test">test</button>`);

        await evaluateScript(args).handler(
          {
            params: {
              function: String(() => {
                return confirm('hello');
              }),
              dialogAction: 'dismiss',
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2)!;
        assert.strictEqual(JSON.parse(lineEvaluation), false);
      });
    });

    it('work for scripts that trigger prompts and fill them', async () => {
      await withMcpContext(async (response, context, args) => {
        const page = context.getSelectedMcpPage().pptrPage;

        await page.setContent(html`<button id="test">test</button>`);

        await evaluateScript(args).handler(
          {
            params: {
              function: String(() => {
                return prompt('Enter your name:');
              }),
              dialogAction: 'John Doe',
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2)!;
        assert.strictEqual(JSON.parse(lineEvaluation), 'John Doe');
      });
    });

    it('work for async functions', async () => {
      await withMcpContext(async (response, context, args) => {
        const page = context.getSelectedMcpPage().pptrPage;

        await page.setContent(html`<script src="./scripts.js"></script> `);

        await evaluateScript(args).handler(
          {
            params: {
              function: String(async () => {
                await new Promise(res => setTimeout(res, 0));
                return 'Works';
              }),
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2)!;
        assert.strictEqual(JSON.parse(lineEvaluation), 'Works');
      });
    });

    it('work with one argument', async () => {
      await withMcpContext(async (response, context, args) => {
        const page = context.getSelectedMcpPage().pptrPage;

        await page.setContent(html`<button id="test">test</button>`);

        context.getSelectedMcpPage().textSnapshot = await TextSnapshot.create(
          context.getSelectedMcpPage(),
        );

        await evaluateScript(args).handler(
          {
            params: {
              function: String(async (el: Element) => {
                return el.id;
              }),
              args: ['1_1'],
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2)!;
        assert.strictEqual(JSON.parse(lineEvaluation), 'test');
      });
    });

    it('work with multiple args', async () => {
      await withMcpContext(async (response, context, args) => {
        const page = context.getSelectedMcpPage().pptrPage;

        await page.setContent(html`<button id="test">test</button>`);

        context.getSelectedMcpPage().textSnapshot = await TextSnapshot.create(
          context.getSelectedMcpPage(),
        );

        await evaluateScript(args).handler(
          {
            params: {
              function: String((container: Element, child: Element) => {
                return container.contains(child);
              }),
              args: ['1_0', '1_1'],
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2)!;
        assert.strictEqual(JSON.parse(lineEvaluation), true);
      });
    });

    it('work for elements inside iframes', async () => {
      server.addHtmlRoute(
        '/iframe',
        html`<main><button>I am iframe button</button></main>`,
      );
      server.addHtmlRoute('/main', html`<iframe src="/iframe"></iframe>`);

      await withMcpContext(async (response, context, args) => {
        const page = context.getSelectedMcpPage().pptrPage;
        await page.goto(server.getRoute('/main'));
        context.getSelectedMcpPage().textSnapshot = await TextSnapshot.create(
          context.getSelectedMcpPage(),
        );
        await evaluateScript(args).handler(
          {
            params: {
              function: String((element: Element) => {
                return element.textContent;
              }),
              args: ['1_3'],
            },
          },
          response,
          context,
        );
        const lineEvaluation = response.responseLines.at(2)!;
        assert.strictEqual(JSON.parse(lineEvaluation), 'I am iframe button');
      });
    });
    it('saves output to file when filePath is provided', async () => {
      using tmpDir = createTempDir();
      const filePath = path.join(
        tmpDir.path,
        'test-evaluate-script-output.json',
      );
      await withMcpContext(async (response, context, args) => {
        await evaluateScript(args).handler(
          {
            params: {
              function: String(() => ({hello: 'world'})),
              filePath,
            },
          },
          response,
          context,
        );
        assert.strictEqual(response.responseLines.length, 1);
        assert.ok(
          response.responseLines[0]?.includes('Output saved to'),
          `Expected "Output saved to" but got: ${response.responseLines[0]}`,
        );
      });
      const content = await readFile(filePath, 'utf-8');
      assert.deepStrictEqual(JSON.parse(content), {hello: 'world'});
    });
    it('evaluates inside extension service worker', async () => {
      await withMcpContext(
        async (response, context, args) => {
          await installExtension(args).handler(
            {params: {path: EXTENSION_PATH}},
            response,
            context,
          );

          const extensionId = extractExtensionId(response);
          const swTarget = await context.browser.waitForTarget(
            t => t.type() === 'service_worker' && t.url().includes(extensionId),
          );

          context.createWorkersSnapshot();
          const swList = context.getWorkers();
          const sw = swList.find(s => s.target === swTarget);

          if (!sw) {
            assert.fail('Service worker not found in context list');
          }

          const swId = sw.id;

          await context.triggerExtensionAction(extensionId);

          response.resetResponseLineForTesting();
          const extensionEvaluateScript = evaluateScript(args);
          await extensionEvaluateScript.handler(
            {
              params: {
                function: String(() => {
                  return 'chrome' in globalThis ? 'has-chrome' : 'no-chrome';
                }),
                serviceWorkerId: swId,
              },
            },
            response,
            context,
          );

          const lineEvaluation = response.responseLines.at(2)!;
          assert.strictEqual(JSON.parse(lineEvaluation), 'has-chrome');

          response.resetResponseLineForTesting();
          await extensionEvaluateScript.handler(
            {
              params: {
                function:
                  '"chrome" in globalThis ? "has-chrome-script" : "no-chrome"',
                format: 'script',
                serviceWorkerId: swId,
              },
            },
            response,
            context,
          );
          const scriptEvaluation = response.responseLines.at(2);
          assert.ok(scriptEvaluation);
          assert.strictEqual(JSON.parse(scriptEvaluation), 'has-chrome-script');

          await context.uninstallExtension(extensionId);
          const targets = context.browser.targets();
          assertNoServiceWorkerReported(targets, extensionId);
        },
        {},
        {categoryExtensions: true},
      );
    });

    it('throws error when both pageId and serviceWorkerId are provided', async () => {
      const {page, context, response} = createHandlerMocks();
      await assert.rejects(
        evaluateScript(
          createMockParsedArguments({
            categoryExtensions: true,
          }),
        ).handler(
          {
            params: {
              function: String(() => 'test'),
              serviceWorkerId: 'example_service_worker',
              pageId: 1,
            },
          },
          response,
          context,
        ),
        {
          message: 'specify either a pageId or a serviceWorkerId.',
        },
      );
      sinon.assert.notCalled(context.getWorkers);
      sinon.assert.notCalled(context.getSelectedMcpPage);
      sinon.assert.notCalled(context.getPageById);
      sinon.assert.notCalled(page.waitForEventsAfterAction);
      sinon.assert.notCalled(response.appendResponseLine);
    });

    it('throws error when args are provided with serviceWorkerId', async () => {
      const {page, context, response} = createHandlerMocks();
      await assert.rejects(
        evaluateScript(
          createMockParsedArguments({
            categoryExtensions: true,
          }),
        ).handler(
          {
            params: {
              function: String(() => 'test'),
              serviceWorkerId: 'example_service_worker',
              args: ['1_1'],
            },
          },
          response,
          context,
        ),
        {
          message:
            'args (element uids) cannot be used when evaluating in a service worker.',
        },
      );
      sinon.assert.notCalled(context.getWorkers);
      sinon.assert.notCalled(context.getSelectedMcpPage);
      sinon.assert.notCalled(context.getPageById);
      sinon.assert.notCalled(page.getElementByUid);
      sinon.assert.notCalled(page.waitForEventsAfterAction);
      sinon.assert.notCalled(response.appendResponseLine);
    });

    it('throws error when pageId and serviceWorkerId are omitted with pageIdRouting', async () => {
      const {page, context, response} = createHandlerMocks();
      await assert.rejects(
        evaluateScript(
          createMockParsedArguments({
            categoryExtensions: true,
            pageIdRouting: true,
          }),
        ).handler(
          {
            params: {
              function: String(() => 'test'),
            },
          },
          response,
          context,
        ),
        {
          message: 'specify either a pageId or a serviceWorkerId.',
        },
      );
      sinon.assert.notCalled(context.getWorkers);
      sinon.assert.notCalled(context.getSelectedMcpPage);
      sinon.assert.notCalled(context.getPageById);
      sinon.assert.notCalled(page.waitForEventsAfterAction);
      sinon.assert.notCalled(response.appendResponseLine);
    });

    it('makes pageId optional in schema when categoryExtensions is true and pageIdRouting is true', () => {
      const args = createMockParsedArguments({
        categoryExtensions: true,
        pageIdRouting: true,
      });
      const tool = evaluateScript(args);
      const schema = zod.object(tool.schema);
      const validSw = schema.safeParse({
        function: '() => 1',
        serviceWorkerId: 'sw_1',
      });
      assert.strictEqual(validSw.success, true);

      const validPage = schema.safeParse({
        function: '() => 1',
        pageId: 1,
      });
      assert.strictEqual(validPage.success, true);
    });

    it('makes pageId required in schema when categoryExtensions is false and pageIdRouting is true', () => {
      const args = createMockParsedArguments({
        pageIdRouting: true,
      });
      const tool = evaluateScript(args);
      const schema = zod.object(tool.schema);
      const resultWithoutPageId = schema.safeParse({
        function: '() => 1',
      });
      assert.strictEqual(resultWithoutPageId.success, false);

      const resultWithPageId = schema.safeParse({
        function: '() => 1',
        pageId: 1,
      });
      assert.strictEqual(resultWithPageId.success, true);
    });
  });
});