Chrome DevTools for coding agents
feat: fetch DOM node selected in the DevTools Elements panel (#486)
Alex Rudenko4a83574961d8
diff --git a/docs/tool-reference.md b/docs/tool-reference.mdindex b5106dcf..72a41da7 100644--- a/docs/tool-reference.md+++ b/docs/tool-reference.md@@ -329,7 +329,8 @@ so returned values have to JSON-serializable.### `take_snapshot`**Description:** Take a text snapshot of the currently selected page based on the a11y tree. The snapshot lists page elements along with a unique-identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot.+identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot. The snapshot indicates the element selected+in the DevTools Elements panel (if any).**Parameters:**diff --git a/src/McpContext.ts b/src/McpContext.tsindex 47c5030a..59fc72b1 100644--- a/src/McpContext.ts+++ b/src/McpContext.ts@@ -25,12 +25,13 @@ import type {import {listPages} from './tools/pages.js';import {takeSnapshot} from './tools/snapshot.js';import {CLOSE_PAGE_ERROR} from './tools/ToolDefinition.js';-import type {Context} from './tools/ToolDefinition.js';+import type {Context, DevToolsData} from './tools/ToolDefinition.js';import type {TraceResult} from './trace-processing/parse.js';import {WaitForHelper} from './WaitForHelper.js';export interface TextSnapshotNode extends SerializedAXNode {id: string;+ backendNodeId?: number;children: TextSnapshotNode[];}@@ -38,6 +39,7 @@ export interface TextSnapshot {root: TextSnapshotNode;idToNode: Map<string, TextSnapshotNode>;snapshotId: string;+ selectedElementUid?: string;}interface McpContextOptions {@@ -151,6 +153,42 @@ export class McpContext implements Context {return context;}+ resolveCdpRequestId(cdpRequestId: string): number | undefined {+ const selectedPage = this.getSelectedPage();+ if (!cdpRequestId) {+ this.logger('no network request');+ return;+ }+ const request = this.#networkCollector.find(selectedPage, request => {+ // @ts-expect-error id is internal.+ return request.id === cdpRequestId;+ });+ if (!request) {+ this.logger('no network request for ' + cdpRequestId);+ return;+ }+ return this.#networkCollector.getIdForResource(request);+ }++ resolveCdpElementId(cdpBackendNodeId: number): string | undefined {+ if (!cdpBackendNodeId) {+ this.logger('no cdpBackendNodeId');+ return;+ }+ // TODO: index by backendNodeId instead.+ const queue = [this.#textSnapshot?.root];+ while (queue.length) {+ const current = queue.pop()!;+ if (current.backendNodeId === cdpBackendNodeId) {+ return current.id;+ }+ for (const child of current.children) {+ queue.push(child);+ }+ }+ return;+ }+getNetworkRequests(includePreservedRequests?: boolean): HTTPRequest[] {const page = this.getSelectedPage();return this.#networkCollector.getData(page, includePreservedRequests);@@ -380,12 +418,17 @@ export class McpContext implements Context {return this.#pageToDevToolsPage.get(page);}- async getDevToolsData(): Promise<undefined | {requestId?: number}> {+ async getDevToolsData(): Promise<DevToolsData> {try {+ this.logger('Getting DevTools UI data');const selectedPage = this.getSelectedPage();const devtoolsPage = this.getDevToolsPage(selectedPage);- if (devtoolsPage) {- const cdpRequestId = await devtoolsPage.evaluate(async () => {+ if (!devtoolsPage) {+ this.logger('No DevTools page detected');+ return {};+ }+ const {cdpRequestId, cdpBackendNodeId} = await devtoolsPage.evaluate(+ async () => {// @ts-expect-error no typesconst UI = await import('/bundled/ui/legacy/legacy.js');// @ts-expect-error no types@@ -393,36 +436,29 @@ export class McpContext implements Context {const request = UI.Context.Context.instance().flavor(SDK.NetworkRequest.NetworkRequest,);- return request?.requestId();- });- if (!cdpRequestId) {- this.logger('no context request');- return;- }- const request = this.#networkCollector.find(selectedPage, request => {- // @ts-expect-error id is internal.- return request.id === cdpRequestId;- });- if (!request) {- this.logger('no collected request for ' + cdpRequestId);- return;- }- return {- requestId: this.#networkCollector.getIdForResource(request),- };- } else {- this.logger('no devtools page deteched');- }+ const node = UI.Context.Context.instance().flavor(+ SDK.DOMModel.DOMNode,+ );+ return {+ cdpRequestId: request?.requestId(),+ cdpBackendNodeId: node?.backendNodeId(),+ };+ },+ );+ return {cdpBackendNodeId, cdpRequestId};} catch (err) {this.logger('error getting devtools data', err);}- return;+ return {};}/*** Creates a text snapshot of a page.*/- async createTextSnapshot(verbose = false): Promise<void> {+ async createTextSnapshot(+ verbose = false,+ devtoolsData: DevToolsData | undefined = undefined,+ ): Promise<void> {const page = this.getSelectedPage();const rootNode = await page.accessibility.snapshot({includeIframes: true,@@ -465,6 +501,12 @@ export class McpContext implements Context {snapshotId: String(snapshotId),idToNode,};+ const data = devtoolsData ?? (await this.getDevToolsData());+ if (data?.cdpBackendNodeId) {+ this.#textSnapshot.selectedElementUid = this.resolveCdpElementId(+ data?.cdpBackendNodeId,+ );+ }}getTextSnapshot(): TextSnapshot | null {diff --git a/src/McpResponse.ts b/src/McpResponse.tsindex 6798c90b..b5ea74aa 100644--- a/src/McpResponse.ts+++ b/src/McpResponse.ts@@ -15,7 +15,7 @@ import {getShortDescriptionForRequest,getStatusFromRequest,} from './formatters/networkFormatter.js';-import {formatA11ySnapshot} from './formatters/snapshotFormatter.js';+import {formatSnapshotNode} from './formatters/snapshotFormatter.js';import type {McpContext} from './McpContext.js';import type {ConsoleMessage,@@ -25,6 +25,7 @@ import type {} from './third_party/index.js';import {handleDialog} from './tools/pages.js';import type {+ DevToolsData,ImageContentData,Response,SnapshotParams,@@ -52,6 +53,11 @@ export class McpResponse implements Response {types?: string[];includePreservedMessages?: boolean;};+ #devToolsData?: DevToolsData;++ attachDevToolsData(data: DevToolsData): void {+ this.#devToolsData = data;+ }setIncludePages(value: boolean): void {this.#includePages = value;@@ -179,17 +185,22 @@ export class McpResponse implements Response {let formattedSnapshot: string | undefined;if (this.#snapshotParams) {- await context.createTextSnapshot(this.#snapshotParams.verbose);+ await context.createTextSnapshot(+ this.#snapshotParams.verbose,+ this.#devToolsData,+ );const snapshot = context.getTextSnapshot();if (snapshot) {if (this.#snapshotParams.filePath) {await context.saveFile(- new TextEncoder().encode(formatA11ySnapshot(snapshot.root)),+ new TextEncoder().encode(+ formatSnapshotNode(snapshot.root, snapshot),+ ),this.#snapshotParams.filePath,);formattedSnapshot = `Saved snapshot to ${this.#snapshotParams.filePath}.`;} else {- formattedSnapshot = formatA11ySnapshot(snapshot.root);+ formattedSnapshot = formatSnapshotNode(snapshot.root, snapshot);}}}diff --git a/src/formatters/snapshotFormatter.ts b/src/formatters/snapshotFormatter.tsindex e77d7e29..42d567e5 100644--- a/src/formatters/snapshotFormatter.ts+++ b/src/formatters/snapshotFormatter.ts@@ -3,19 +3,26 @@* Copyright 2025 Google LLC* SPDX-License-Identifier: Apache-2.0*/-import type {TextSnapshotNode} from '../McpContext.js';+import type {TextSnapshot, TextSnapshotNode} from '../McpContext.js';-export function formatA11ySnapshot(- serializedAXNodeRoot: TextSnapshotNode,+export function formatSnapshotNode(+ root: TextSnapshotNode,+ snapshot?: TextSnapshot,depth = 0,): string {let result = '';- const attributes = getAttributes(serializedAXNodeRoot);- const line = ' '.repeat(depth * 2) + attributes.join(' ') + '\n';+ const attributes = getAttributes(root);+ const line =+ ' '.repeat(depth * 2) ++ attributes.join(' ') ++ (root.id === snapshot?.selectedElementUid+ ? ' [selected in the DevTools Elements panel]'+ : '') ++ '\n';result += line;- for (const child of serializedAXNodeRoot.children) {- result += formatA11ySnapshot(child, depth + 1);+ for (const child of root.children) {+ result += formatSnapshotNode(child, snapshot, depth + 1);}return result;diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.tsindex 8df0f720..9651718b 100644--- a/src/tools/ToolDefinition.ts+++ b/src/tools/ToolDefinition.ts@@ -47,6 +47,11 @@ export interface SnapshotParams {filePath?: string;}+export interface DevToolsData {+ cdpRequestId?: string;+ cdpBackendNodeId?: number;+}+export interface Response {appendResponseLine(value: string): void;setIncludePages(value: boolean): void;@@ -69,6 +74,8 @@ export interface Response {attachImage(value: ImageContentData): void;attachNetworkRequest(reqid: number): void;attachConsoleMessage(msgid: number): void;+ // Allows re-using DevTools data queried by some tools.+ attachDevToolsData(data: DevToolsData): void;}/**@@ -103,7 +110,15 @@ export type Context = Readonly<{text: string;timeout?: number | undefined;}): Promise<Element>;- getDevToolsData(): Promise<undefined | {requestId?: number}>;+ getDevToolsData(): Promise<DevToolsData>;+ /**+ * Returns a reqid for a cdpRequestId.+ */+ resolveCdpRequestId(cdpRequestId: string): number | undefined;+ /**+ * Returns a reqid for a cdpRequestId.+ */+ resolveCdpElementId(cdpBackendNodeId: number): string | undefined;}>;export function defineTool<Schema extends zod.ZodRawShape>(diff --git a/src/tools/network.ts b/src/tools/network.tsindex e2923c42..d3b6c1fe 100644--- a/src/tools/network.ts+++ b/src/tools/network.ts@@ -72,12 +72,16 @@ export const listNetworkRequests = defineTool({},handler: async (request, response, context) => {const data = await context.getDevToolsData();+ response.attachDevToolsData(data);+ const reqid = data?.cdpRequestId+ ? context.resolveCdpRequestId(data.cdpRequestId)+ : undefined;response.setIncludeNetworkRequests(true, {pageSize: request.params.pageSize,pageIdx: request.params.pageIdx,resourceTypes: request.params.resourceTypes,includePreservedRequests: request.params.includePreservedRequests,- networkRequestIdInDevToolsUI: data?.requestId,+ networkRequestIdInDevToolsUI: reqid,});},});@@ -102,8 +106,12 @@ export const getNetworkRequest = defineTool({response.attachNetworkRequest(request.params.reqid);} else {const data = await context.getDevToolsData();- if (data?.requestId) {- response.attachNetworkRequest(data?.requestId);+ response.attachDevToolsData(data);+ const reqid = data?.cdpRequestId+ ? context.resolveCdpRequestId(data.cdpRequestId)+ : undefined;+ if (reqid) {+ response.attachNetworkRequest(reqid);} else {response.appendResponseLine(`Nothing is currently selected in the DevTools Network panel.`,diff --git a/src/tools/snapshot.ts b/src/tools/snapshot.tsindex 0187f7fc..21099f14 100644--- a/src/tools/snapshot.ts+++ b/src/tools/snapshot.ts@@ -12,7 +12,8 @@ import {defineTool, timeoutSchema} from './ToolDefinition.js';export const takeSnapshot = defineTool({name: 'take_snapshot',description: `Take a text snapshot of the currently selected page based on the a11y tree. The snapshot lists page elements along with a unique-identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot.`,+identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot. The snapshot indicates the element selected+in the DevTools Elements panel (if any).`,annotations: {category: ToolCategory.DEBUGGING,// Not read-only due to filePath param.diff --git a/tests/formatters/snapshotFormatter.test.ts b/tests/formatters/snapshotFormatter.test.tsindex 47b7d288..1232929f 100644--- a/tests/formatters/snapshotFormatter.test.ts+++ b/tests/formatters/snapshotFormatter.test.ts@@ -9,12 +9,12 @@ import {describe, it} from 'node:test';import type {ElementHandle} from 'puppeteer-core';-import {formatA11ySnapshot} from '../../src/formatters/snapshotFormatter.js';+import {formatSnapshotNode} from '../../src/formatters/snapshotFormatter.js';import type {TextSnapshotNode} from '../../src/McpContext.js';describe('snapshotFormatter', () => {it('formats a snapshot with value properties', () => {- const snapshot: TextSnapshotNode = {+ const node: TextSnapshotNode = {id: '1_1',role: 'textbox',name: 'textbox',@@ -35,7 +35,7 @@ describe('snapshotFormatter', () => {},};- const formatted = formatA11ySnapshot(snapshot);+ const formatted = formatSnapshotNode(node);assert.strictEqual(formatted,`uid=1_1 textbox "textbox" value="value"@@ -45,7 +45,7 @@ describe('snapshotFormatter', () => {});it('formats a snapshot with boolean properties', () => {- const snapshot: TextSnapshotNode = {+ const node: TextSnapshotNode = {id: '1_1',role: 'button',name: 'button',@@ -66,7 +66,7 @@ describe('snapshotFormatter', () => {},};- const formatted = formatA11ySnapshot(snapshot);+ const formatted = formatSnapshotNode(node);assert.strictEqual(formatted,`uid=1_1 button "button" disableable disabled@@ -76,7 +76,7 @@ describe('snapshotFormatter', () => {});it('formats a snapshot with checked properties', () => {- const snapshot: TextSnapshotNode = {+ const node: TextSnapshotNode = {id: '1_1',role: 'checkbox',name: 'checkbox',@@ -97,7 +97,7 @@ describe('snapshotFormatter', () => {},};- const formatted = formatA11ySnapshot(snapshot);+ const formatted = formatSnapshotNode(node);assert.strictEqual(formatted,`uid=1_1 checkbox "checkbox" checked@@ -107,7 +107,7 @@ describe('snapshotFormatter', () => {});it('formats a snapshot with multiple different type attributes', () => {- const snapshot: TextSnapshotNode = {+ const node: TextSnapshotNode = {id: '1_1',role: 'root',name: 'root',@@ -139,7 +139,7 @@ describe('snapshotFormatter', () => {},};- const formatted = formatA11ySnapshot(snapshot);+ const formatted = formatSnapshotNode(node);assert.strictEqual(formatted,`uid=1_1 root "root"