Skip to content

Chrome DevTools for coding agents

feat: fetch DOM node selected in the DevTools Elements panel (#486)

Alex Rudenko4a83574961d8
diff --git a/docs/tool-reference.md b/docs/tool-reference.md
index b5106dcf..72a41da7 100644
--- a/docs/tool-reference.md
+++ b/docs/tool-reference.md
@@ -329,7 +329,8 @@ so returned values have to JSON-serializable.
### `take_snapshot`
**Description:** Take a text snapshot of the currently selected page based on the a11y tree. The snapshot lists page elements along with a unique
-identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot.
+identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot. The snapshot indicates the element selected
+in the DevTools Elements panel (if any).
**Parameters:**
diff --git a/src/McpContext.ts b/src/McpContext.ts
index 47c5030a..59fc72b1 100644
--- a/src/McpContext.ts
+++ b/src/McpContext.ts
@@ -25,12 +25,13 @@ import type {
import {listPages} from './tools/pages.js';
import {takeSnapshot} from './tools/snapshot.js';
import {CLOSE_PAGE_ERROR} from './tools/ToolDefinition.js';
-import type {Context} from './tools/ToolDefinition.js';
+import type {Context, DevToolsData} from './tools/ToolDefinition.js';
import type {TraceResult} from './trace-processing/parse.js';
import {WaitForHelper} from './WaitForHelper.js';
export interface TextSnapshotNode extends SerializedAXNode {
id: string;
+ backendNodeId?: number;
children: TextSnapshotNode[];
}
@@ -38,6 +39,7 @@ export interface TextSnapshot {
root: TextSnapshotNode;
idToNode: Map<string, TextSnapshotNode>;
snapshotId: string;
+ selectedElementUid?: string;
}
interface McpContextOptions {
@@ -151,6 +153,42 @@ export class McpContext implements Context {
return context;
}
+ resolveCdpRequestId(cdpRequestId: string): number | undefined {
+ const selectedPage = this.getSelectedPage();
+ if (!cdpRequestId) {
+ this.logger('no network request');
+ return;
+ }
+ const request = this.#networkCollector.find(selectedPage, request => {
+ // @ts-expect-error id is internal.
+ return request.id === cdpRequestId;
+ });
+ if (!request) {
+ this.logger('no network request for ' + cdpRequestId);
+ return;
+ }
+ return this.#networkCollector.getIdForResource(request);
+ }
+
+ resolveCdpElementId(cdpBackendNodeId: number): string | undefined {
+ if (!cdpBackendNodeId) {
+ this.logger('no cdpBackendNodeId');
+ return;
+ }
+ // TODO: index by backendNodeId instead.
+ const queue = [this.#textSnapshot?.root];
+ while (queue.length) {
+ const current = queue.pop()!;
+ if (current.backendNodeId === cdpBackendNodeId) {
+ return current.id;
+ }
+ for (const child of current.children) {
+ queue.push(child);
+ }
+ }
+ return;
+ }
+
getNetworkRequests(includePreservedRequests?: boolean): HTTPRequest[] {
const page = this.getSelectedPage();
return this.#networkCollector.getData(page, includePreservedRequests);
@@ -380,12 +418,17 @@ export class McpContext implements Context {
return this.#pageToDevToolsPage.get(page);
}
- async getDevToolsData(): Promise<undefined | {requestId?: number}> {
+ async getDevToolsData(): Promise<DevToolsData> {
try {
+ this.logger('Getting DevTools UI data');
const selectedPage = this.getSelectedPage();
const devtoolsPage = this.getDevToolsPage(selectedPage);
- if (devtoolsPage) {
- const cdpRequestId = await devtoolsPage.evaluate(async () => {
+ if (!devtoolsPage) {
+ this.logger('No DevTools page detected');
+ return {};
+ }
+ const {cdpRequestId, cdpBackendNodeId} = await devtoolsPage.evaluate(
+ async () => {
// @ts-expect-error no types
const UI = await import('/bundled/ui/legacy/legacy.js');
// @ts-expect-error no types
@@ -393,36 +436,29 @@ export class McpContext implements Context {
const request = UI.Context.Context.instance().flavor(
SDK.NetworkRequest.NetworkRequest,
);
- return request?.requestId();
- });
- if (!cdpRequestId) {
- this.logger('no context request');
- return;
- }
- const request = this.#networkCollector.find(selectedPage, request => {
- // @ts-expect-error id is internal.
- return request.id === cdpRequestId;
- });
- if (!request) {
- this.logger('no collected request for ' + cdpRequestId);
- return;
- }
- return {
- requestId: this.#networkCollector.getIdForResource(request),
- };
- } else {
- this.logger('no devtools page deteched');
- }
+ const node = UI.Context.Context.instance().flavor(
+ SDK.DOMModel.DOMNode,
+ );
+ return {
+ cdpRequestId: request?.requestId(),
+ cdpBackendNodeId: node?.backendNodeId(),
+ };
+ },
+ );
+ return {cdpBackendNodeId, cdpRequestId};
} catch (err) {
this.logger('error getting devtools data', err);
}
- return;
+ return {};
}
/**
* Creates a text snapshot of a page.
*/
- async createTextSnapshot(verbose = false): Promise<void> {
+ async createTextSnapshot(
+ verbose = false,
+ devtoolsData: DevToolsData | undefined = undefined,
+ ): Promise<void> {
const page = this.getSelectedPage();
const rootNode = await page.accessibility.snapshot({
includeIframes: true,
@@ -465,6 +501,12 @@ export class McpContext implements Context {
snapshotId: String(snapshotId),
idToNode,
};
+ const data = devtoolsData ?? (await this.getDevToolsData());
+ if (data?.cdpBackendNodeId) {
+ this.#textSnapshot.selectedElementUid = this.resolveCdpElementId(
+ data?.cdpBackendNodeId,
+ );
+ }
}
getTextSnapshot(): TextSnapshot | null {
diff --git a/src/McpResponse.ts b/src/McpResponse.ts
index 6798c90b..b5ea74aa 100644
--- a/src/McpResponse.ts
+++ b/src/McpResponse.ts
@@ -15,7 +15,7 @@ import {
getShortDescriptionForRequest,
getStatusFromRequest,
} from './formatters/networkFormatter.js';
-import {formatA11ySnapshot} from './formatters/snapshotFormatter.js';
+import {formatSnapshotNode} from './formatters/snapshotFormatter.js';
import type {McpContext} from './McpContext.js';
import type {
ConsoleMessage,
@@ -25,6 +25,7 @@ import type {
} from './third_party/index.js';
import {handleDialog} from './tools/pages.js';
import type {
+ DevToolsData,
ImageContentData,
Response,
SnapshotParams,
@@ -52,6 +53,11 @@ export class McpResponse implements Response {
types?: string[];
includePreservedMessages?: boolean;
};
+ #devToolsData?: DevToolsData;
+
+ attachDevToolsData(data: DevToolsData): void {
+ this.#devToolsData = data;
+ }
setIncludePages(value: boolean): void {
this.#includePages = value;
@@ -179,17 +185,22 @@ export class McpResponse implements Response {
let formattedSnapshot: string | undefined;
if (this.#snapshotParams) {
- await context.createTextSnapshot(this.#snapshotParams.verbose);
+ await context.createTextSnapshot(
+ this.#snapshotParams.verbose,
+ this.#devToolsData,
+ );
const snapshot = context.getTextSnapshot();
if (snapshot) {
if (this.#snapshotParams.filePath) {
await context.saveFile(
- new TextEncoder().encode(formatA11ySnapshot(snapshot.root)),
+ new TextEncoder().encode(
+ formatSnapshotNode(snapshot.root, snapshot),
+ ),
this.#snapshotParams.filePath,
);
formattedSnapshot = `Saved snapshot to ${this.#snapshotParams.filePath}.`;
} else {
- formattedSnapshot = formatA11ySnapshot(snapshot.root);
+ formattedSnapshot = formatSnapshotNode(snapshot.root, snapshot);
}
}
}
diff --git a/src/formatters/snapshotFormatter.ts b/src/formatters/snapshotFormatter.ts
index e77d7e29..42d567e5 100644
--- a/src/formatters/snapshotFormatter.ts
+++ b/src/formatters/snapshotFormatter.ts
@@ -3,19 +3,26 @@
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
-import type {TextSnapshotNode} from '../McpContext.js';
+import type {TextSnapshot, TextSnapshotNode} from '../McpContext.js';
-export function formatA11ySnapshot(
- serializedAXNodeRoot: TextSnapshotNode,
+export function formatSnapshotNode(
+ root: TextSnapshotNode,
+ snapshot?: TextSnapshot,
depth = 0,
): string {
let result = '';
- const attributes = getAttributes(serializedAXNodeRoot);
- const line = ' '.repeat(depth * 2) + attributes.join(' ') + '\n';
+ const attributes = getAttributes(root);
+ const line =
+ ' '.repeat(depth * 2) +
+ attributes.join(' ') +
+ (root.id === snapshot?.selectedElementUid
+ ? ' [selected in the DevTools Elements panel]'
+ : '') +
+ '\n';
result += line;
- for (const child of serializedAXNodeRoot.children) {
- result += formatA11ySnapshot(child, depth + 1);
+ for (const child of root.children) {
+ result += formatSnapshotNode(child, snapshot, depth + 1);
}
return result;
diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.ts
index 8df0f720..9651718b 100644
--- a/src/tools/ToolDefinition.ts
+++ b/src/tools/ToolDefinition.ts
@@ -47,6 +47,11 @@ export interface SnapshotParams {
filePath?: string;
}
+export interface DevToolsData {
+ cdpRequestId?: string;
+ cdpBackendNodeId?: number;
+}
+
export interface Response {
appendResponseLine(value: string): void;
setIncludePages(value: boolean): void;
@@ -69,6 +74,8 @@ export interface Response {
attachImage(value: ImageContentData): void;
attachNetworkRequest(reqid: number): void;
attachConsoleMessage(msgid: number): void;
+ // Allows re-using DevTools data queried by some tools.
+ attachDevToolsData(data: DevToolsData): void;
}
/**
@@ -103,7 +110,15 @@ export type Context = Readonly<{
text: string;
timeout?: number | undefined;
}): Promise<Element>;
- getDevToolsData(): Promise<undefined | {requestId?: number}>;
+ getDevToolsData(): Promise<DevToolsData>;
+ /**
+ * Returns a reqid for a cdpRequestId.
+ */
+ resolveCdpRequestId(cdpRequestId: string): number | undefined;
+ /**
+ * Returns a reqid for a cdpRequestId.
+ */
+ resolveCdpElementId(cdpBackendNodeId: number): string | undefined;
}>;
export function defineTool<Schema extends zod.ZodRawShape>(
diff --git a/src/tools/network.ts b/src/tools/network.ts
index e2923c42..d3b6c1fe 100644
--- a/src/tools/network.ts
+++ b/src/tools/network.ts
@@ -72,12 +72,16 @@ export const listNetworkRequests = defineTool({
},
handler: async (request, response, context) => {
const data = await context.getDevToolsData();
+ response.attachDevToolsData(data);
+ const reqid = data?.cdpRequestId
+ ? context.resolveCdpRequestId(data.cdpRequestId)
+ : undefined;
response.setIncludeNetworkRequests(true, {
pageSize: request.params.pageSize,
pageIdx: request.params.pageIdx,
resourceTypes: request.params.resourceTypes,
includePreservedRequests: request.params.includePreservedRequests,
- networkRequestIdInDevToolsUI: data?.requestId,
+ networkRequestIdInDevToolsUI: reqid,
});
},
});
@@ -102,8 +106,12 @@ export const getNetworkRequest = defineTool({
response.attachNetworkRequest(request.params.reqid);
} else {
const data = await context.getDevToolsData();
- if (data?.requestId) {
- response.attachNetworkRequest(data?.requestId);
+ response.attachDevToolsData(data);
+ const reqid = data?.cdpRequestId
+ ? context.resolveCdpRequestId(data.cdpRequestId)
+ : undefined;
+ if (reqid) {
+ response.attachNetworkRequest(reqid);
} else {
response.appendResponseLine(
`Nothing is currently selected in the DevTools Network panel.`,
diff --git a/src/tools/snapshot.ts b/src/tools/snapshot.ts
index 0187f7fc..21099f14 100644
--- a/src/tools/snapshot.ts
+++ b/src/tools/snapshot.ts
@@ -12,7 +12,8 @@ import {defineTool, timeoutSchema} from './ToolDefinition.js';
export const takeSnapshot = defineTool({
name: 'take_snapshot',
description: `Take a text snapshot of the currently selected page based on the a11y tree. The snapshot lists page elements along with a unique
-identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot.`,
+identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot. The snapshot indicates the element selected
+in the DevTools Elements panel (if any).`,
annotations: {
category: ToolCategory.DEBUGGING,
// Not read-only due to filePath param.
diff --git a/tests/formatters/snapshotFormatter.test.ts b/tests/formatters/snapshotFormatter.test.ts
index 47b7d288..1232929f 100644
--- a/tests/formatters/snapshotFormatter.test.ts
+++ b/tests/formatters/snapshotFormatter.test.ts
@@ -9,12 +9,12 @@ import {describe, it} from 'node:test';
import type {ElementHandle} from 'puppeteer-core';
-import {formatA11ySnapshot} from '../../src/formatters/snapshotFormatter.js';
+import {formatSnapshotNode} from '../../src/formatters/snapshotFormatter.js';
import type {TextSnapshotNode} from '../../src/McpContext.js';
describe('snapshotFormatter', () => {
it('formats a snapshot with value properties', () => {
- const snapshot: TextSnapshotNode = {
+ const node: TextSnapshotNode = {
id: '1_1',
role: 'textbox',
name: 'textbox',
@@ -35,7 +35,7 @@ describe('snapshotFormatter', () => {
},
};
- const formatted = formatA11ySnapshot(snapshot);
+ const formatted = formatSnapshotNode(node);
assert.strictEqual(
formatted,
`uid=1_1 textbox "textbox" value="value"
@@ -45,7 +45,7 @@ describe('snapshotFormatter', () => {
});
it('formats a snapshot with boolean properties', () => {
- const snapshot: TextSnapshotNode = {
+ const node: TextSnapshotNode = {
id: '1_1',
role: 'button',
name: 'button',
@@ -66,7 +66,7 @@ describe('snapshotFormatter', () => {
},
};
- const formatted = formatA11ySnapshot(snapshot);
+ const formatted = formatSnapshotNode(node);
assert.strictEqual(
formatted,
`uid=1_1 button "button" disableable disabled
@@ -76,7 +76,7 @@ describe('snapshotFormatter', () => {
});
it('formats a snapshot with checked properties', () => {
- const snapshot: TextSnapshotNode = {
+ const node: TextSnapshotNode = {
id: '1_1',
role: 'checkbox',
name: 'checkbox',
@@ -97,7 +97,7 @@ describe('snapshotFormatter', () => {
},
};
- const formatted = formatA11ySnapshot(snapshot);
+ const formatted = formatSnapshotNode(node);
assert.strictEqual(
formatted,
`uid=1_1 checkbox "checkbox" checked
@@ -107,7 +107,7 @@ describe('snapshotFormatter', () => {
});
it('formats a snapshot with multiple different type attributes', () => {
- const snapshot: TextSnapshotNode = {
+ const node: TextSnapshotNode = {
id: '1_1',
role: 'root',
name: 'root',
@@ -139,7 +139,7 @@ describe('snapshotFormatter', () => {
},
};
- const formatted = formatA11ySnapshot(snapshot);
+ const formatted = formatSnapshotNode(node);
assert.strictEqual(
formatted,
`uid=1_1 root "root"