Chrome DevTools for coding agents
chore: DOM elements as inputs for in-page tools (#1791)
Wolfgang Beyer627ed68a28de
diff --git a/src/McpResponse.ts b/src/McpResponse.tsindex ed20e4c4..77898b5d 100644--- a/src/McpResponse.ts+++ b/src/McpResponse.ts@@ -19,6 +19,7 @@ import type {Page,ResourceType,TextContent,+ JSONSchema7Definition,} from './third_party/index.js';import type {ToolGroup, ToolDefinition} from './tools/inPage.js';import {handleDialog} from './tools/pages.js';@@ -41,6 +42,57 @@ interface TraceInsightData {insightName: InsightName;}+export function replaceHtmlElementsWithUids(schema: JSONSchema7Definition) {+ if (typeof schema === 'boolean') {+ return;+ }++ let isHtmlElement = false;+ for (const [key, value] of Object.entries(schema)) {+ if (key === 'x-mcp-type' && value === 'HTMLElement') {+ isHtmlElement = true;+ break;+ }+ }++ if (isHtmlElement) {+ schema.properties = {uid: {type: 'string'}};+ schema.required = ['uid'];+ }++ if (schema.properties) {+ for (const key of Object.keys(schema.properties)) {+ replaceHtmlElementsWithUids(schema.properties[key]);+ }+ }++ if (schema.items) {+ if (Array.isArray(schema.items)) {+ for (const item of schema.items) {+ replaceHtmlElementsWithUids(item);+ }+ } else {+ replaceHtmlElementsWithUids(schema.items);+ }+ }++ if (schema.anyOf) {+ for (const s of schema.anyOf) {+ replaceHtmlElementsWithUids(s);+ }+ }+ if (schema.allOf) {+ for (const s of schema.allOf) {+ replaceHtmlElementsWithUids(s);+ }+ }+ if (schema.oneOf) {+ for (const s of schema.oneOf) {+ replaceHtmlElementsWithUids(s);+ }+ }+}+async function getToolGroup(page: McpPage,): Promise<ToolGroup<ToolDefinition> | undefined> {@@ -91,6 +143,10 @@ async function getToolGroup(}, 0);});});++ for (const tool of toolGroup?.tools ?? []) {+ replaceHtmlElementsWithUids(tool.inputSchema);+ }return toolGroup;}diff --git a/src/third_party/index.ts b/src/third_party/index.tsindex 96f5295c..09b0d02c 100644--- a/src/third_party/index.ts+++ b/src/third_party/index.ts@@ -41,7 +41,7 @@ export {default as puppeteer} from 'puppeteer-core';export type * from 'puppeteer-core';export {PipeTransport} from 'puppeteer-core/internal/node/PipeTransport.js';export type {CdpPage} from 'puppeteer-core/internal/cdp/Page.js';-export type {JSONSchema7} from 'json-schema';+export type {JSONSchema7, JSONSchema7Definition} from 'json-schema';export {resolveDefaultUserDataDir,detectBrowserPlatform,diff --git a/src/tools/inPage.ts b/src/tools/inPage.tsindex 5b0031f6..0cb1b6ba 100644--- a/src/tools/inPage.ts+++ b/src/tools/inPage.ts@@ -4,7 +4,12 @@* SPDX-License-Identifier: Apache-2.0*/-import {zod, ajv, type JSONSchema7} from '../third_party/index.js';+import {+ zod,+ ajv,+ type JSONSchema7,+ type ElementHandle,+} from '../third_party/index.js';import {ToolCategory} from './categories.js';import {definePageTool} from './ToolDefinition.js';@@ -87,6 +92,22 @@ export const executeInPageTool = definePageTool({}}+ // Creates array of ElementHandles from the UIDs in the params.+ // We do not replace the uids with the ElementsHandles yet, because+ // the `evaluate` function only turns them into DOM elements if they+ // are passed as non-nested arguments.+ const handles: ElementHandle[] = [];+ for (const value of Object.values(params)) {+ if (+ value instanceof Object &&+ 'uid' in value &&+ typeof value.uid === 'string' &&+ Object.keys(value).length === 1+ ) {+ handles.push(await request.page.getElementByUid(value.uid));+ }+ }+const toolGroup = request.page.getInPageTools();const tool = toolGroup?.tools.find(t => t.name === toolName);if (!tool) {@@ -102,7 +123,19 @@ export const executeInPageTool = definePageTool({}const result = await request.page.pptrPage.evaluate(- async (name, args) => {+ async (name, args, ...elements) => {+ // Replace the UIDs with DOM elements.+ for (const [key, value] of Object.entries(args)) {+ if (+ value instanceof Object &&+ 'uid' in value &&+ typeof value.uid === 'string' &&+ Object.keys(value).length === 1+ ) {+ args[key] = elements.shift();+ }+ }+if (!window.__dtmcp?.executeTool) {throw new Error('No tools found on the page');}@@ -114,6 +147,7 @@ export const executeInPageTool = definePageTool({},toolName,params,+ ...handles,);response.appendResponseLine(JSON.stringify(result, null, 2));},diff --git a/tests/McpResponse.test.ts b/tests/McpResponse.test.tsindex 66be61e8..91537719 100644--- a/tests/McpResponse.test.ts+++ b/tests/McpResponse.test.ts@@ -15,6 +15,8 @@ import sinon from 'sinon';import type {ParsedArguments} from '../src/bin/chrome-devtools-mcp-cli-options.js';import type {McpContext} from '../src/McpContext.js';import type {McpResponse} from '../src/McpResponse.js';+import {replaceHtmlElementsWithUids} from '../src/McpResponse.js';+import type {JSONSchema7Definition} from '../src/third_party/index.js';import {closePage,listPages,@@ -1205,3 +1207,251 @@ describe('inPage tools', () => {}, 'new_page');});});++describe('replaceHtmlElementsWithUids', () => {+ it('does nothing for boolean schemas', () => {+ const schemaTrue: JSONSchema7Definition = true;+ const schemaFalse: JSONSchema7Definition = false;++ replaceHtmlElementsWithUids(schemaTrue);+ replaceHtmlElementsWithUids(schemaFalse);++ assert.strictEqual(schemaTrue, true);+ assert.strictEqual(schemaFalse, false);+ });++ it('replaces HTMLElement type with uid string', () => {+ const schema: JSONSchema7Definition = {+ type: 'object',+ properties: {+ foo: {type: 'string'},+ bar: {type: 'number'},+ },+ required: ['foo'],+ };+ Object.assign(schema, {'x-mcp-type': 'HTMLElement'});++ replaceHtmlElementsWithUids(schema);++ if (typeof schema === 'object') {+ assert.deepStrictEqual(schema.properties, {+ uid: {type: 'string'},+ });+ assert.deepStrictEqual(schema.required, ['uid']);+ } else {+ assert.fail('Schema should be an object');+ }+ });++ it('does not replace if x-mcp-type is not HTMLElement', () => {+ const schema: JSONSchema7Definition = {+ type: 'object',+ properties: {+ foo: {type: 'string'},+ },+ };+ Object.assign(schema, {'x-mcp-type': 'OtherType'});++ replaceHtmlElementsWithUids(schema);++ if (typeof schema === 'object') {+ assert.deepStrictEqual(schema.properties, {+ foo: {type: 'string'},+ });+ assert.strictEqual(schema.required, undefined);+ } else {+ assert.fail('Schema should be an object');+ }+ });++ it('recurses into nested properties', () => {+ const schema: JSONSchema7Definition = {+ type: 'object',+ properties: {+ element: {+ type: 'object',+ properties: {+ foo: {type: 'string'},+ },+ },+ other: {+ type: 'string',+ },+ },+ };+ if (typeof schema === 'object' && schema.properties) {+ Object.assign(schema.properties.element, {'x-mcp-type': 'HTMLElement'});+ }++ replaceHtmlElementsWithUids(schema);++ if (+ typeof schema === 'object' &&+ schema.properties &&+ typeof schema.properties.element === 'object'+ ) {+ const elementSchema = schema.properties.element;+ assert.deepStrictEqual(elementSchema.properties, {+ uid: {type: 'string'},+ });+ assert.deepStrictEqual(elementSchema.required, ['uid']);+ } else {+ assert.fail('Unexpected schema structure');+ }+ });++ it('recurses into array items (single schema object)', () => {+ const schema: JSONSchema7Definition = {+ type: 'array',+ items: {+ type: 'object',+ },+ };+ if (typeof schema === 'object' && typeof schema.items === 'object') {+ Object.assign(schema.items, {'x-mcp-type': 'HTMLElement'});+ }++ replaceHtmlElementsWithUids(schema);++ if (typeof schema === 'object' && typeof schema.items === 'object') {+ const itemsSchema = schema.items;+ if (!Array.isArray(itemsSchema)) {+ assert.deepStrictEqual(itemsSchema.properties, {+ uid: {type: 'string'},+ });+ assert.deepStrictEqual(itemsSchema.required, ['uid']);+ } else {+ assert.fail('items should not be an array in this test case');+ }+ } else {+ assert.fail('Unexpected schema structure');+ }+ });++ it('recurses into array items (array of schemas)', () => {+ const schema: JSONSchema7Definition = {+ type: 'array',+ items: [+ {+ type: 'object',+ },+ {+ type: 'string',+ },+ ],+ };+ if (typeof schema === 'object' && Array.isArray(schema.items)) {+ Object.assign(schema.items[0], {'x-mcp-type': 'HTMLElement'});+ }++ replaceHtmlElementsWithUids(schema);++ if (typeof schema === 'object' && Array.isArray(schema.items)) {+ const firstItem = schema.items[0];+ if (typeof firstItem === 'object') {+ assert.deepStrictEqual(firstItem.properties, {+ uid: {type: 'string'},+ });+ assert.deepStrictEqual(firstItem.required, ['uid']);+ } else {+ assert.fail('First item should be an object');+ }++ const secondItem = schema.items[1];+ if (typeof secondItem === 'object') {+ assert.strictEqual(secondItem.properties, undefined);+ } else {+ assert.fail('Second item should be an object');+ }+ } else {+ assert.fail('Unexpected schema structure');+ }+ });++ it('recurses into anyOf', () => {+ const schema: JSONSchema7Definition = {+ anyOf: [+ {+ type: 'object',+ },+ {+ type: 'string',+ },+ ],+ };+ if (typeof schema === 'object' && Array.isArray(schema.anyOf)) {+ Object.assign(schema.anyOf[0], {'x-mcp-type': 'HTMLElement'});+ }++ replaceHtmlElementsWithUids(schema);++ if (typeof schema === 'object' && Array.isArray(schema.anyOf)) {+ const firstItem = schema.anyOf[0];+ if (typeof firstItem === 'object') {+ assert.deepStrictEqual(firstItem.properties, {+ uid: {type: 'string'},+ });+ } else {+ assert.fail('First item should be an object');+ }+ } else {+ assert.fail('Unexpected schema structure');+ }+ });++ it('recurses into allOf', () => {+ const schema: JSONSchema7Definition = {+ allOf: [+ {+ type: 'object',+ },+ ],+ };+ if (typeof schema === 'object' && Array.isArray(schema.allOf)) {+ Object.assign(schema.allOf[0], {'x-mcp-type': 'HTMLElement'});+ }++ replaceHtmlElementsWithUids(schema);++ if (typeof schema === 'object' && Array.isArray(schema.allOf)) {+ const firstItem = schema.allOf[0];+ if (typeof firstItem === 'object') {+ assert.deepStrictEqual(firstItem.properties, {+ uid: {type: 'string'},+ });+ } else {+ assert.fail('First item should be an object');+ }+ } else {+ assert.fail('Unexpected schema structure');+ }+ });++ it('recurses into oneOf', () => {+ const schema: JSONSchema7Definition = {+ oneOf: [+ {+ type: 'object',+ },+ ],+ };+ if (typeof schema === 'object' && Array.isArray(schema.oneOf)) {+ Object.assign(schema.oneOf[0], {'x-mcp-type': 'HTMLElement'});+ }++ replaceHtmlElementsWithUids(schema);++ if (typeof schema === 'object' && Array.isArray(schema.oneOf)) {+ const firstItem = schema.oneOf[0];+ if (typeof firstItem === 'object') {+ assert.deepStrictEqual(firstItem.properties, {+ uid: {type: 'string'},+ });+ } else {+ assert.fail('First item should be an object');+ }+ } else {+ assert.fail('Unexpected schema structure');+ }+ });+});diff --git a/tests/tools/inPage.test.ts b/tests/tools/inPage.test.tsindex 622da903..d87bbea8 100644--- a/tests/tools/inPage.test.ts+++ b/tests/tools/inPage.test.ts@@ -347,5 +347,98 @@ describe('inPage', () => {{categoryInPageTools: true} as ParsedArguments,);});++ it('replaces uid with element handle in params', async () => {+ await withMcpContext(async (response, context) => {+ const page = await context.newPage();+ response.setPage(page);++ page.inPageTools = {+ name: 'test-group',+ description: 'test description',+ tools: [+ {+ name: 'test-tool',+ description: 'test tool description',+ inputSchema: {+ type: 'object',+ properties: {+ element: {type: 'object'},+ },+ required: ['element'],+ },+ },+ ],+ };++ await page.pptrPage.evaluate(() => {+ window.__dtmcp = {+ executeTool: async (+ _name: string,+ args: Record<string, unknown>,+ ) => {+ const el = args.element;+ if (el instanceof HTMLElement) {+ return {+ isElement: true,+ tagName: el.tagName,+ id: el.id,+ };+ }+ return {+ isElement: false,+ tagName: '',+ id: '',+ };+ },+ };+ });++ await page.pptrPage.evaluate(() => {+ const div = document.createElement('div');+ div.id = 'test-id';+ document.body.appendChild(div);+ });++ const handle = await page.pptrPage.$('#test-id');+ if (!handle) {+ throw new Error('Handle not found');+ }++ page.getElementByUid = async (uid: string) => {+ if (uid === 'some-uid') {+ return handle;+ }+ throw new Error('Not found');+ };++ await executeInPageTool.handler(+ {+ params: {+ toolName: 'test-tool',+ params: JSON.stringify({element: {uid: 'some-uid'}}),+ },+ page: page,+ },+ response,+ context,+ );++ assert.strictEqual(+ response.responseLines[0],+ JSON.stringify(+ {+ result: {+ isElement: true,+ tagName: 'DIV',+ id: 'test-id',+ },+ },+ null,+ 2,+ ),+ );+ });+ });});});