Skip to content

Chrome DevTools for coding agents

16.5 KB
Raw Blame
/**
 * @license
 * Copyright 2025 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

import assert from 'node:assert/strict';
import fs from 'node:fs/promises';
import path from 'node:path';
import {describe, it} from 'node:test';

import sinon from 'sinon';

import {lighthouseAudit} from '../../src/tools/lighthouse.js';
import {createHandlerMocks, createMockRunnerResult} from '../mocks.js';
import {serverHooks} from '../server.js';
import {html, withMcpContext} from '../utils.js';

describe('lighthouse', () => {
  const server = serverHooks();
  describe('lighthouse_audit', () => {
    it('runs Lighthouse audit by default (navigation, desktop)', async () => {
      server.addHtmlRoute('/test', html`<div>Test</div>`);

      await withMcpContext(async (response, context, args) => {
        const mcpPage = context.getSelectedMcpPage();
        const page = mcpPage.pptrPage;
        await page.goto(server.getRoute('/test'));
        await mcpPage.emulate({
          viewport: {
            width: 390,
            height: 844,
            deviceScaleFactor: 3,
            isMobile: true,
            hasTouch: true,
            isLandscape: false,
          },
        });
        const readViewport = () =>
          page.evaluate(() => ({
            width: window.innerWidth,
            height: window.innerHeight,
            deviceScaleFactor: window.devicePixelRatio,
            screenWidth: screen.width,
            orientation: screen.orientation.type,
            maxTouchPoints: navigator.maxTouchPoints,
          }));
        const emulated = await readViewport();
        assert.strictEqual(emulated.width, 390);
        assert.strictEqual(emulated.deviceScaleFactor, 3);
        assert.ok(emulated.maxTouchPoints > 0);

        await lighthouseAudit(args).handler(
          {
            params: {
              mode: 'navigation',
              device: 'desktop',
            },
            page: mcpPage,
          },
          response,
          context,
        );

        // Lighthouse replaces and then clears the device metrics override, the
        // emulated viewport must be back afterwards.
        assert.deepEqual(await readViewport(), emulated);

        const data = response.attachedLighthouseResult;
        assert.ok(data);

        assert.ok(data.summary);
        assert.strictEqual(data.summary.mode, 'navigation');
        assert.strictEqual(data.summary.device, 'desktop');
        assert.ok(data.reports.length === 2); // json, html
        assert.strictEqual(
          data.failedAudits.length,
          data.summary.audits.failed,
        );

        // Verify files exist
        for (const reportPath of data.reports) {
          const stats = await fs.stat(reportPath);
          assert.ok(stats.isFile());
        }
      });
    });

    it('explains which URLs navigation mode can audit', async () => {
      const {page, context, response, args} = createHandlerMocks();
      page.pptrPage.url.returns('file:///tmp/page.html');
      const invalidUrl = Object.assign(new Error('INVALID_URL'), {
        code: 'INVALID_URL',
      });
      page.lighthouseNavigation.rejects(invalidUrl);

      await assert.rejects(
        () =>
          lighthouseAudit(args).handler(
            {
              params: {
                mode: 'navigation',
                device: 'desktop',
              },
              page,
            },
            response,
            context,
          ),
        {
          message:
            'Lighthouse "navigation" mode cannot audit file:///tmp/page.html: it only supports http:, https:, chrome: and chrome-extension: URLs. Use mode "snapshot" to audit the page in its current state.',
          cause: invalidUrl,
        },
      );
    });

    it('passes other navigation errors through unchanged', async () => {
      const {page, context, response, args} = createHandlerMocks();
      page.lighthouseNavigation.rejects(new Error('PROTOCOL_TIMEOUT'));

      await assert.rejects(
        () =>
          lighthouseAudit(args).handler(
            {
              params: {
                mode: 'navigation',
                device: 'desktop',
              },
              page,
            },
            response,
            context,
          ),
        {message: 'PROTOCOL_TIMEOUT'},
      );
    });

    it('restores emulation', async () => {
      const {page, context, response, args} = createHandlerMocks();
      context.saveTemporaryFile.resolves({filepath: 'report.json'});
      page.lighthouseSnapshot.resolves(createMockRunnerResult());

      await lighthouseAudit(args).handler(
        {
          params: {
            mode: 'snapshot',
            device: 'mobile',
          },
          page,
        },
        response,
        context,
      );

      sinon.assert.calledOnceWithExactly(page.restoreEmulation);
    });

    it('restores emulation even when audit fails', async () => {
      const {page, context, response, args} = createHandlerMocks();
      page.lighthouseSnapshot.rejects(new Error('Audit failed'));

      await assert.rejects(
        () =>
          lighthouseAudit(args).handler(
            {
              params: {
                mode: 'snapshot',
                device: 'mobile',
              },
              page,
            },
            response,
            context,
          ),
        {message: 'Audit failed'},
      );

      sinon.assert.calledOnceWithExactly(page.restoreEmulation);
    });

    it('fails when Lighthouse reports a runtime error', async () => {
      const {page, context, response, args} = createHandlerMocks();
      context.saveTemporaryFile.resolves({filepath: 'report.json'});
      page.lighthouseNavigation.resolves(
        createMockRunnerResult({
          runtimeError: {
            code: 'ERRORED_DOCUMENT_REQUEST',
            message: 'Lighthouse was unable to reliably load the page.',
          },
        }),
      );

      await assert.rejects(
        () =>
          lighthouseAudit(args).handler(
            {
              params: {
                mode: 'navigation',
                device: 'desktop',
              },
              page,
            },
            response,
            context,
          ),
        {
          message:
            'Lighthouse could not audit the page (ERRORED_DOCUMENT_REQUEST): Lighthouse was unable to reliably load the page. Reports: report.json, report.json',
        },
      );

      sinon.assert.calledTwice(context.saveTemporaryFile);
      sinon.assert.notCalled(response.attachLighthouseResult);
      sinon.assert.calledOnceWithExactly(page.restoreEmulation);
    });

    it('emulates a desktop user agent for desktop audits', async () => {
      const {page, context, response, args} = createHandlerMocks();
      context.saveTemporaryFile.resolves({filepath: 'report.json'});
      page.lighthouseNavigation.resolves(createMockRunnerResult());

      await lighthouseAudit(args).handler(
        {
          params: {
            mode: 'navigation',
            device: 'desktop',
          },
          page,
        },
        response,
        context,
      );

      const [flags] = page.lighthouseNavigation.firstCall.args;
      assert.strictEqual(flags?.formFactor, 'desktop');
      assert.match(String(flags?.emulatedUserAgent), /Macintosh/);
      assert.doesNotMatch(String(flags?.emulatedUserAgent), /Mobile/);
    });

    it('emulates a mobile user agent for mobile audits', async () => {
      const {page, context, response, args} = createHandlerMocks();
      context.saveTemporaryFile.resolves({filepath: 'report.json'});
      page.lighthouseSnapshot.resolves(createMockRunnerResult());

      await lighthouseAudit(args).handler(
        {
          params: {
            mode: 'snapshot',
            device: 'mobile',
          },
          page,
        },
        response,
        context,
      );

      const [flags] = page.lighthouseSnapshot.firstCall.args;
      assert.strictEqual(flags?.formFactor, 'mobile');
      assert.match(String(flags?.emulatedUserAgent), /Mobile Safari/);
    });

    it('reports the URL in snapshot mode, where mainDocumentUrl is unset', async () => {
      const {page, context, response, args} = createHandlerMocks();
      context.saveTemporaryFile.resolves({filepath: 'report.json'});
      page.lighthouseSnapshot.resolves(
        createMockRunnerResult({
          mainDocumentUrl: undefined,
          finalDisplayedUrl: 'https://example.com/page',
        }),
      );

      await lighthouseAudit(args).handler(
        {
          params: {
            mode: 'snapshot',
            device: 'mobile',
          },
          page,
        },
        response,
        context,
      );

      sinon.assert.calledOnce(response.attachLighthouseResult);
      assert.strictEqual(
        response.attachLighthouseResult.firstCall.args[0].summary.url,
        'https://example.com/page',
      );
    });

    it('runs Lighthouse in snapshot mode with mobile device', async () => {
      const {page, context, response, args} = createHandlerMocks();
      context.saveTemporaryFile
        .withArgs(sinon.match.instanceOf(Uint8Array), 'report.json')
        .resolves({filepath: '/tmp/report.json'});
      context.saveTemporaryFile
        .withArgs(sinon.match.instanceOf(Uint8Array), 'report.html')
        .resolves({filepath: '/tmp/report.html'});
      page.lighthouseSnapshot.resolves(createMockRunnerResult());

      await lighthouseAudit(args).handler(
        {
          params: {
            mode: 'snapshot',
            device: 'mobile',
          },
          page,
        },
        response,
        context,
      );

      sinon.assert.calledOnce(page.lighthouseSnapshot);
      sinon.assert.notCalled(page.lighthouseNavigation);
      sinon.assert.calledTwice(context.saveTemporaryFile);
      sinon.assert.notCalled(context.saveFile);
      sinon.assert.calledOnce(response.attachLighthouseResult);
      const {summary, reports} =
        response.attachLighthouseResult.firstCall.args[0];
      assert.strictEqual(summary.mode, 'snapshot');
      assert.strictEqual(summary.device, 'mobile');
      assert.deepEqual(reports, ['/tmp/report.json', '/tmp/report.html']);
    });

    it('runs Lighthouse with custom output dir', async () => {
      const {page, context, response, args} = createHandlerMocks();
      const outputDirPath = path.join('custom', 'reports');
      const reportPath = path.join(outputDirPath, 'report');
      context.saveFile
        .withArgs(sinon.match.instanceOf(Uint8Array), reportPath, '.json')
        .resolves({filename: `${reportPath}.json`});
      context.saveFile
        .withArgs(sinon.match.instanceOf(Uint8Array), reportPath, '.html')
        .resolves({filename: `${reportPath}.html`});
      page.lighthouseSnapshot.resolves(createMockRunnerResult());

      await lighthouseAudit(args).handler(
        {
          params: {
            mode: 'snapshot',
            device: 'mobile',
            outputDirPath,
          },
          page,
        },
        response,
        context,
      );

      sinon.assert.calledTwice(context.saveFile);
      sinon.assert.notCalled(context.saveTemporaryFile);
      sinon.assert.calledOnce(response.attachLighthouseResult);
      const {summary, reports} =
        response.attachLighthouseResult.firstCall.args[0];
      assert.strictEqual(summary.mode, 'snapshot');
      assert.strictEqual(summary.device, 'mobile');
      assert.deepEqual(reports, [`${reportPath}.json`, `${reportPath}.html`]);
    });

    it('returns failing audits with their DOM nodes inline', async () => {
      const {page, context, response, args} = createHandlerMocks();
      context.saveTemporaryFile.resolves({filepath: 'report.json'});
      const node = (
        id: number,
      ): {
        type: 'node';
        lhId: string;
        selector: string;
        snippet: string;
        nodeLabel: string;
      } => ({
        type: 'node',
        lhId: `node-${id}`,
        selector: `div.item-${id}`,
        snippet: `<div class="item-${id}">`,
        nodeLabel: `Item ${id}`,
      });
      page.lighthouseSnapshot.resolves(
        createMockRunnerResult({
          categories: {
            accessibility: {
              id: 'accessibility',
              title: 'Accessibility',
              score: 0.5,
              auditRefs: [
                {id: 'color-contrast', weight: 7},
                {id: 'document-title', weight: 7},
                {id: 'html-has-lang', weight: 7},
              ],
            },
            seo: {
              id: 'seo',
              title: 'SEO',
              score: 0.5,
              auditRefs: [{id: 'document-title', weight: 1}],
            },
          },
          audits: {
            'color-contrast': {
              id: 'color-contrast',
              title: 'Insufficient contrast',
              description: 'Low-contrast text is difficult to read.',
              score: 0,
              scoreDisplayMode: 'binary',
              details: {
                type: 'table',
                headings: [],
                items: [
                  // Duplicate node without an explanation, which must be
                  // merged with the copy below.
                  {node: node(1)},
                  {
                    node: {
                      ...node(1),
                      explanation: 'Fix any of the following: contrast',
                    },
                    subItems: {
                      type: 'subitems',
                      items: [{relatedNode: node(2)}],
                    },
                  },
                  ...Array.from({length: 12}, (_, i) => ({node: node(i + 3)})),
                ],
                debugData: {type: 'debugdata', node: node(99)},
              },
            },
            'document-title': {
              id: 'document-title',
              title: 'Missing title',
              description: 'Titles matter.',
              score: 0,
              scoreDisplayMode: 'binary',
              displayValue: '1 issue',
              details: {
                type: 'list',
                items: [
                  {
                    type: 'table',
                    headings: [],
                    items: [
                      {node: node(50)},
                      // Nodes without identifying fields must not be merged.
                      {node: {type: 'node', nodeLabel: 'Unnamed A'}},
                      {node: {type: 'node', nodeLabel: 'Unnamed B'}},
                    ],
                  },
                ],
              },
            },
            'html-has-lang': {
              id: 'html-has-lang',
              title: 'Has lang',
              description: '',
              score: 1,
              scoreDisplayMode: 'binary',
            },
            'manual-audit': {
              id: 'manual-audit',
              title: 'Manual',
              description: '',
              score: null,
              scoreDisplayMode: 'manual',
            },
          },
        }),
      );

      await lighthouseAudit(args).handler(
        {params: {mode: 'snapshot', device: 'desktop'}, page},
        response,
        context,
      );

      sinon.assert.calledOnce(response.attachLighthouseResult);
      const {summary, failedAudits} =
        response.attachLighthouseResult.firstCall.args[0];
      assert.deepEqual(summary.audits, {failed: 2, passed: 1});
      assert.strictEqual(failedAudits.length, 2);

      const [contrast, title] = failedAudits;
      assert.strictEqual(contrast.id, 'color-contrast');
      assert.deepEqual(contrast.categories, ['accessibility']);
      assert.strictEqual(
        contrast.description,
        'Low-contrast text is difficult to read.',
      );
      assert.strictEqual(contrast.totalNodes, 13);
      assert.strictEqual(contrast.nodes.length, 10);
      assert.deepEqual(contrast.nodes[0], {
        selector: 'div.item-1',
        snippet: '<div class="item-1">',
        nodeLabel: 'Item 1',
        explanation: 'Fix any of the following: contrast',
      });
      assert.strictEqual(contrast.nodes[1].selector, 'div.item-3');
      assert.ok(!contrast.nodes.some(n => n.selector === 'div.item-2'));
      assert.ok(!contrast.nodes.some(n => n.selector === 'div.item-99'));

      assert.deepEqual(title, {
        id: 'document-title',
        title: 'Missing title',
        description: 'Titles matter.',
        score: 0,
        displayValue: '1 issue',
        categories: ['accessibility', 'seo'],
        nodes: [
          {
            selector: 'div.item-50',
            snippet: '<div class="item-50">',
            nodeLabel: 'Item 50',
          },
          {nodeLabel: 'Unnamed A'},
          {nodeLabel: 'Unnamed B'},
        ],
        totalNodes: 3,
      });
    });
  });
});