Skip to content

Chrome DevTools for coding agents

chore: DOM elements as inputs for in-page tools (#1791)

Wolfgang Beyer627ed68a28de
diff --git a/src/McpResponse.ts b/src/McpResponse.ts
index ed20e4c4..77898b5d 100644
--- a/src/McpResponse.ts
+++ b/src/McpResponse.ts
@@ -19,6 +19,7 @@ import type {
Page,
ResourceType,
TextContent,
+ JSONSchema7Definition,
} from './third_party/index.js';
import type {ToolGroup, ToolDefinition} from './tools/inPage.js';
import {handleDialog} from './tools/pages.js';
@@ -41,6 +42,57 @@ interface TraceInsightData {
insightName: InsightName;
}
+export function replaceHtmlElementsWithUids(schema: JSONSchema7Definition) {
+ if (typeof schema === 'boolean') {
+ return;
+ }
+
+ let isHtmlElement = false;
+ for (const [key, value] of Object.entries(schema)) {
+ if (key === 'x-mcp-type' && value === 'HTMLElement') {
+ isHtmlElement = true;
+ break;
+ }
+ }
+
+ if (isHtmlElement) {
+ schema.properties = {uid: {type: 'string'}};
+ schema.required = ['uid'];
+ }
+
+ if (schema.properties) {
+ for (const key of Object.keys(schema.properties)) {
+ replaceHtmlElementsWithUids(schema.properties[key]);
+ }
+ }
+
+ if (schema.items) {
+ if (Array.isArray(schema.items)) {
+ for (const item of schema.items) {
+ replaceHtmlElementsWithUids(item);
+ }
+ } else {
+ replaceHtmlElementsWithUids(schema.items);
+ }
+ }
+
+ if (schema.anyOf) {
+ for (const s of schema.anyOf) {
+ replaceHtmlElementsWithUids(s);
+ }
+ }
+ if (schema.allOf) {
+ for (const s of schema.allOf) {
+ replaceHtmlElementsWithUids(s);
+ }
+ }
+ if (schema.oneOf) {
+ for (const s of schema.oneOf) {
+ replaceHtmlElementsWithUids(s);
+ }
+ }
+}
+
async function getToolGroup(
page: McpPage,
): Promise<ToolGroup<ToolDefinition> | undefined> {
@@ -91,6 +143,10 @@ async function getToolGroup(
}, 0);
});
});
+
+ for (const tool of toolGroup?.tools ?? []) {
+ replaceHtmlElementsWithUids(tool.inputSchema);
+ }
return toolGroup;
}
diff --git a/src/third_party/index.ts b/src/third_party/index.ts
index 96f5295c..09b0d02c 100644
--- a/src/third_party/index.ts
+++ b/src/third_party/index.ts
@@ -41,7 +41,7 @@ export {default as puppeteer} from 'puppeteer-core';
export type * from 'puppeteer-core';
export {PipeTransport} from 'puppeteer-core/internal/node/PipeTransport.js';
export type {CdpPage} from 'puppeteer-core/internal/cdp/Page.js';
-export type {JSONSchema7} from 'json-schema';
+export type {JSONSchema7, JSONSchema7Definition} from 'json-schema';
export {
resolveDefaultUserDataDir,
detectBrowserPlatform,
diff --git a/src/tools/inPage.ts b/src/tools/inPage.ts
index 5b0031f6..0cb1b6ba 100644
--- a/src/tools/inPage.ts
+++ b/src/tools/inPage.ts
@@ -4,7 +4,12 @@
* SPDX-License-Identifier: Apache-2.0
*/
-import {zod, ajv, type JSONSchema7} from '../third_party/index.js';
+import {
+ zod,
+ ajv,
+ type JSONSchema7,
+ type ElementHandle,
+} from '../third_party/index.js';
import {ToolCategory} from './categories.js';
import {definePageTool} from './ToolDefinition.js';
@@ -87,6 +92,22 @@ export const executeInPageTool = definePageTool({
}
}
+ // Creates array of ElementHandles from the UIDs in the params.
+ // We do not replace the uids with the ElementsHandles yet, because
+ // the `evaluate` function only turns them into DOM elements if they
+ // are passed as non-nested arguments.
+ const handles: ElementHandle[] = [];
+ for (const value of Object.values(params)) {
+ if (
+ value instanceof Object &&
+ 'uid' in value &&
+ typeof value.uid === 'string' &&
+ Object.keys(value).length === 1
+ ) {
+ handles.push(await request.page.getElementByUid(value.uid));
+ }
+ }
+
const toolGroup = request.page.getInPageTools();
const tool = toolGroup?.tools.find(t => t.name === toolName);
if (!tool) {
@@ -102,7 +123,19 @@ export const executeInPageTool = definePageTool({
}
const result = await request.page.pptrPage.evaluate(
- async (name, args) => {
+ async (name, args, ...elements) => {
+ // Replace the UIDs with DOM elements.
+ for (const [key, value] of Object.entries(args)) {
+ if (
+ value instanceof Object &&
+ 'uid' in value &&
+ typeof value.uid === 'string' &&
+ Object.keys(value).length === 1
+ ) {
+ args[key] = elements.shift();
+ }
+ }
+
if (!window.__dtmcp?.executeTool) {
throw new Error('No tools found on the page');
}
@@ -114,6 +147,7 @@ export const executeInPageTool = definePageTool({
},
toolName,
params,
+ ...handles,
);
response.appendResponseLine(JSON.stringify(result, null, 2));
},
diff --git a/tests/McpResponse.test.ts b/tests/McpResponse.test.ts
index 66be61e8..91537719 100644
--- a/tests/McpResponse.test.ts
+++ b/tests/McpResponse.test.ts
@@ -15,6 +15,8 @@ import sinon from 'sinon';
import type {ParsedArguments} from '../src/bin/chrome-devtools-mcp-cli-options.js';
import type {McpContext} from '../src/McpContext.js';
import type {McpResponse} from '../src/McpResponse.js';
+import {replaceHtmlElementsWithUids} from '../src/McpResponse.js';
+import type {JSONSchema7Definition} from '../src/third_party/index.js';
import {
closePage,
listPages,
@@ -1205,3 +1207,251 @@ describe('inPage tools', () => {
}, 'new_page');
});
});
+
+describe('replaceHtmlElementsWithUids', () => {
+ it('does nothing for boolean schemas', () => {
+ const schemaTrue: JSONSchema7Definition = true;
+ const schemaFalse: JSONSchema7Definition = false;
+
+ replaceHtmlElementsWithUids(schemaTrue);
+ replaceHtmlElementsWithUids(schemaFalse);
+
+ assert.strictEqual(schemaTrue, true);
+ assert.strictEqual(schemaFalse, false);
+ });
+
+ it('replaces HTMLElement type with uid string', () => {
+ const schema: JSONSchema7Definition = {
+ type: 'object',
+ properties: {
+ foo: {type: 'string'},
+ bar: {type: 'number'},
+ },
+ required: ['foo'],
+ };
+ Object.assign(schema, {'x-mcp-type': 'HTMLElement'});
+
+ replaceHtmlElementsWithUids(schema);
+
+ if (typeof schema === 'object') {
+ assert.deepStrictEqual(schema.properties, {
+ uid: {type: 'string'},
+ });
+ assert.deepStrictEqual(schema.required, ['uid']);
+ } else {
+ assert.fail('Schema should be an object');
+ }
+ });
+
+ it('does not replace if x-mcp-type is not HTMLElement', () => {
+ const schema: JSONSchema7Definition = {
+ type: 'object',
+ properties: {
+ foo: {type: 'string'},
+ },
+ };
+ Object.assign(schema, {'x-mcp-type': 'OtherType'});
+
+ replaceHtmlElementsWithUids(schema);
+
+ if (typeof schema === 'object') {
+ assert.deepStrictEqual(schema.properties, {
+ foo: {type: 'string'},
+ });
+ assert.strictEqual(schema.required, undefined);
+ } else {
+ assert.fail('Schema should be an object');
+ }
+ });
+
+ it('recurses into nested properties', () => {
+ const schema: JSONSchema7Definition = {
+ type: 'object',
+ properties: {
+ element: {
+ type: 'object',
+ properties: {
+ foo: {type: 'string'},
+ },
+ },
+ other: {
+ type: 'string',
+ },
+ },
+ };
+ if (typeof schema === 'object' && schema.properties) {
+ Object.assign(schema.properties.element, {'x-mcp-type': 'HTMLElement'});
+ }
+
+ replaceHtmlElementsWithUids(schema);
+
+ if (
+ typeof schema === 'object' &&
+ schema.properties &&
+ typeof schema.properties.element === 'object'
+ ) {
+ const elementSchema = schema.properties.element;
+ assert.deepStrictEqual(elementSchema.properties, {
+ uid: {type: 'string'},
+ });
+ assert.deepStrictEqual(elementSchema.required, ['uid']);
+ } else {
+ assert.fail('Unexpected schema structure');
+ }
+ });
+
+ it('recurses into array items (single schema object)', () => {
+ const schema: JSONSchema7Definition = {
+ type: 'array',
+ items: {
+ type: 'object',
+ },
+ };
+ if (typeof schema === 'object' && typeof schema.items === 'object') {
+ Object.assign(schema.items, {'x-mcp-type': 'HTMLElement'});
+ }
+
+ replaceHtmlElementsWithUids(schema);
+
+ if (typeof schema === 'object' && typeof schema.items === 'object') {
+ const itemsSchema = schema.items;
+ if (!Array.isArray(itemsSchema)) {
+ assert.deepStrictEqual(itemsSchema.properties, {
+ uid: {type: 'string'},
+ });
+ assert.deepStrictEqual(itemsSchema.required, ['uid']);
+ } else {
+ assert.fail('items should not be an array in this test case');
+ }
+ } else {
+ assert.fail('Unexpected schema structure');
+ }
+ });
+
+ it('recurses into array items (array of schemas)', () => {
+ const schema: JSONSchema7Definition = {
+ type: 'array',
+ items: [
+ {
+ type: 'object',
+ },
+ {
+ type: 'string',
+ },
+ ],
+ };
+ if (typeof schema === 'object' && Array.isArray(schema.items)) {
+ Object.assign(schema.items[0], {'x-mcp-type': 'HTMLElement'});
+ }
+
+ replaceHtmlElementsWithUids(schema);
+
+ if (typeof schema === 'object' && Array.isArray(schema.items)) {
+ const firstItem = schema.items[0];
+ if (typeof firstItem === 'object') {
+ assert.deepStrictEqual(firstItem.properties, {
+ uid: {type: 'string'},
+ });
+ assert.deepStrictEqual(firstItem.required, ['uid']);
+ } else {
+ assert.fail('First item should be an object');
+ }
+
+ const secondItem = schema.items[1];
+ if (typeof secondItem === 'object') {
+ assert.strictEqual(secondItem.properties, undefined);
+ } else {
+ assert.fail('Second item should be an object');
+ }
+ } else {
+ assert.fail('Unexpected schema structure');
+ }
+ });
+
+ it('recurses into anyOf', () => {
+ const schema: JSONSchema7Definition = {
+ anyOf: [
+ {
+ type: 'object',
+ },
+ {
+ type: 'string',
+ },
+ ],
+ };
+ if (typeof schema === 'object' && Array.isArray(schema.anyOf)) {
+ Object.assign(schema.anyOf[0], {'x-mcp-type': 'HTMLElement'});
+ }
+
+ replaceHtmlElementsWithUids(schema);
+
+ if (typeof schema === 'object' && Array.isArray(schema.anyOf)) {
+ const firstItem = schema.anyOf[0];
+ if (typeof firstItem === 'object') {
+ assert.deepStrictEqual(firstItem.properties, {
+ uid: {type: 'string'},
+ });
+ } else {
+ assert.fail('First item should be an object');
+ }
+ } else {
+ assert.fail('Unexpected schema structure');
+ }
+ });
+
+ it('recurses into allOf', () => {
+ const schema: JSONSchema7Definition = {
+ allOf: [
+ {
+ type: 'object',
+ },
+ ],
+ };
+ if (typeof schema === 'object' && Array.isArray(schema.allOf)) {
+ Object.assign(schema.allOf[0], {'x-mcp-type': 'HTMLElement'});
+ }
+
+ replaceHtmlElementsWithUids(schema);
+
+ if (typeof schema === 'object' && Array.isArray(schema.allOf)) {
+ const firstItem = schema.allOf[0];
+ if (typeof firstItem === 'object') {
+ assert.deepStrictEqual(firstItem.properties, {
+ uid: {type: 'string'},
+ });
+ } else {
+ assert.fail('First item should be an object');
+ }
+ } else {
+ assert.fail('Unexpected schema structure');
+ }
+ });
+
+ it('recurses into oneOf', () => {
+ const schema: JSONSchema7Definition = {
+ oneOf: [
+ {
+ type: 'object',
+ },
+ ],
+ };
+ if (typeof schema === 'object' && Array.isArray(schema.oneOf)) {
+ Object.assign(schema.oneOf[0], {'x-mcp-type': 'HTMLElement'});
+ }
+
+ replaceHtmlElementsWithUids(schema);
+
+ if (typeof schema === 'object' && Array.isArray(schema.oneOf)) {
+ const firstItem = schema.oneOf[0];
+ if (typeof firstItem === 'object') {
+ assert.deepStrictEqual(firstItem.properties, {
+ uid: {type: 'string'},
+ });
+ } else {
+ assert.fail('First item should be an object');
+ }
+ } else {
+ assert.fail('Unexpected schema structure');
+ }
+ });
+});
diff --git a/tests/tools/inPage.test.ts b/tests/tools/inPage.test.ts
index 622da903..d87bbea8 100644
--- a/tests/tools/inPage.test.ts
+++ b/tests/tools/inPage.test.ts
@@ -347,5 +347,98 @@ describe('inPage', () => {
{categoryInPageTools: true} as ParsedArguments,
);
});
+
+ it('replaces uid with element handle in params', async () => {
+ await withMcpContext(async (response, context) => {
+ const page = await context.newPage();
+ response.setPage(page);
+
+ page.inPageTools = {
+ name: 'test-group',
+ description: 'test description',
+ tools: [
+ {
+ name: 'test-tool',
+ description: 'test tool description',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ element: {type: 'object'},
+ },
+ required: ['element'],
+ },
+ },
+ ],
+ };
+
+ await page.pptrPage.evaluate(() => {
+ window.__dtmcp = {
+ executeTool: async (
+ _name: string,
+ args: Record<string, unknown>,
+ ) => {
+ const el = args.element;
+ if (el instanceof HTMLElement) {
+ return {
+ isElement: true,
+ tagName: el.tagName,
+ id: el.id,
+ };
+ }
+ return {
+ isElement: false,
+ tagName: '',
+ id: '',
+ };
+ },
+ };
+ });
+
+ await page.pptrPage.evaluate(() => {
+ const div = document.createElement('div');
+ div.id = 'test-id';
+ document.body.appendChild(div);
+ });
+
+ const handle = await page.pptrPage.$('#test-id');
+ if (!handle) {
+ throw new Error('Handle not found');
+ }
+
+ page.getElementByUid = async (uid: string) => {
+ if (uid === 'some-uid') {
+ return handle;
+ }
+ throw new Error('Not found');
+ };
+
+ await executeInPageTool.handler(
+ {
+ params: {
+ toolName: 'test-tool',
+ params: JSON.stringify({element: {uid: 'some-uid'}}),
+ },
+ page: page,
+ },
+ response,
+ context,
+ );
+
+ assert.strictEqual(
+ response.responseLines[0],
+ JSON.stringify(
+ {
+ result: {
+ isElement: true,
+ tagName: 'DIV',
+ id: 'test-id',
+ },
+ },
+ null,
+ 2,
+ ),
+ );
+ });
+ });
});
});