Chrome DevTools for coding agents
chore: experimental vision mode (#745)
Alex Rudenkoa94dc0e614ce
diff --git a/scripts/generate-docs.ts b/scripts/generate-docs.tsindex 47740b5f..9e59bdaf 100644--- a/scripts/generate-docs.ts+++ b/scripts/generate-docs.ts@@ -20,6 +20,7 @@ interface ToolWithAnnotations extends Tool {annotations?: {title?: string;category?: typeof ToolCategory;+ conditions?: string[];};}@@ -265,31 +266,39 @@ async function generateToolDocumentation(): Promise<void> {console.log('Generating tool documentation from definitions...');// Convert ToolDefinitions to ToolWithAnnotations- const toolsWithAnnotations: ToolWithAnnotations[] = tools.map(tool => {- const properties: Record<string, TypeInfo> = {};- const required: string[] = [];-- for (const [key, schema] of Object.entries(- tool.schema as unknown as Record<string, ZodSchema>,- )) {- const info = getZodTypeInfo(schema);- properties[key] = info;- if (isRequired(schema)) {- required.push(key);+ const toolsWithAnnotations: ToolWithAnnotations[] = tools+ .filter(tool => {+ if (!tool.annotations.conditions) {+ return true;+ }+ // Only include unconditional tools.+ return tool.annotations.conditions.length === 0;+ })+ .map(tool => {+ const properties: Record<string, TypeInfo> = {};+ const required: string[] = [];++ for (const [key, schema] of Object.entries(+ tool.schema as unknown as Record<string, ZodSchema>,+ )) {+ const info = getZodTypeInfo(schema);+ properties[key] = info;+ if (isRequired(schema)) {+ required.push(key);+ }}- }- return {- name: tool.name,- description: tool.description,- inputSchema: {- type: 'object',- properties,- required,- },- annotations: tool.annotations,- };- });+ return {+ name: tool.name,+ description: tool.description,+ inputSchema: {+ type: 'object',+ properties,+ required,+ },+ annotations: tool.annotations,+ };+ });console.log(`Found ${toolsWithAnnotations.length} tools`);diff --git a/src/cli.ts b/src/cli.tsindex db268058..e1a623e7 100644--- a/src/cli.ts+++ b/src/cli.ts@@ -152,6 +152,11 @@ export const cliOptions = {describe: 'Whether to enable automation over DevTools targets',hidden: true,},+ experimentalVision: {+ type: 'boolean',+ describe: 'Whether to enable vision tools',+ hidden: true,+ },experimentalIncludeAllPages: {type: 'boolean',describe:diff --git a/src/main.ts b/src/main.tsindex 84bb6d9b..ab6ee301 100644--- a/src/main.ts+++ b/src/main.ts@@ -121,6 +121,12 @@ function registerTool(tool: ToolDefinition): void {) {return;}+ if (+ tool.annotations.conditions?.includes('computerVision') &&+ !args.experimentalVision+ ) {+ return;+ }server.registerTool(tool.name,{diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.tsindex f3eaaa7e..909feec1 100644--- a/src/tools/ToolDefinition.ts+++ b/src/tools/ToolDefinition.ts@@ -24,6 +24,7 @@ export interface ToolDefinition<* If true, the tool does not modify its environment.*/readOnlyHint: boolean;+ conditions?: string[];};schema: Schema;handler: (diff --git a/src/tools/input.ts b/src/tools/input.tsindex 50556892..bfbcf6b6 100644--- a/src/tools/input.ts+++ b/src/tools/input.ts@@ -12,6 +12,11 @@ import {parseKey} from '../utils/keyboard.js';import {ToolCategory} from './categories.js';import {defineTool} from './ToolDefinition.js';+const dblClickSchema = zod+ .boolean()+ .optional()+ .describe('Set to true for double clicks. Default is false.');+export const click = defineTool({name: 'click',description: `Clicks on the provided element`,@@ -25,10 +30,7 @@ export const click = defineTool({.describe('The uid of an element on the page from the page content snapshot',),- dblClick: zod- .boolean()- .optional()- .describe('Set to true for double clicks. Default is false.'),+ dblClick: dblClickSchema,},handler: async (request, response, context) => {const uid = request.params.uid;@@ -51,6 +53,35 @@ export const click = defineTool({},});+export const clickAt = defineTool({+ name: 'click_at',+ description: `Clicks at the provided coordinates`,+ annotations: {+ category: ToolCategory.INPUT,+ readOnlyHint: false,+ conditions: ['computerVision'],+ },+ schema: {+ x: zod.number().describe('The x coordinate'),+ y: zod.number().describe('The y coordinate'),+ dblClick: dblClickSchema,+ },+ handler: async (request, response, context) => {+ const page = context.getSelectedPage();+ await context.waitForEventsAfterAction(async () => {+ await page.mouse.click(request.params.x, request.params.y, {+ clickCount: request.params.dblClick ? 2 : 1,+ });+ });+ response.appendResponseLine(+ request.params.dblClick+ ? `Successfully double clicked at the coordinates`+ : `Successfully clicked at the coordinates`,+ );+ response.includeSnapshot();+ },+});+export const hover = defineTool({name: 'hover',description: `Hover over the provided element`,diff --git a/tests/index.test.ts b/tests/index.test.tsindex 5970e890..7b864e45 100644--- a/tests/index.test.ts+++ b/tests/index.test.ts@@ -12,8 +12,13 @@ import {Client} from '@modelcontextprotocol/sdk/client/index.js';import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js';import {executablePath} from 'puppeteer';+import type {ToolDefinition} from '../src/tools/ToolDefinition';+describe('e2e', () => {- async function withClient(cb: (client: Client) => Promise<void>) {+ async function withClient(+ cb: (client: Client) => Promise<void>,+ extraArgs: string[] = [],+ ) {const transport = new StdioClientTransport({command: 'node',args: [@@ -22,6 +27,7 @@ describe('e2e', () => {'--isolated','--executable-path',executablePath(),+ ...extraArgs,],});const client = new Client(@@ -90,8 +96,11 @@ describe('e2e', () => {continue;}const fileTools = await import(`../src/tools/${file}`);- for (const maybeTool of Object.values<object>(fileTools)) {+ for (const maybeTool of Object.values<ToolDefinition>(fileTools)) {if ('name' in maybeTool) {+ if (maybeTool.annotations?.conditions?.includes('computerVision')) {+ continue;+ }definedNames.push(maybeTool.name);}}@@ -100,4 +109,15 @@ describe('e2e', () => {assert.deepStrictEqual(exposedNames, definedNames);});});++ it('has experimental vision tools', async () => {+ await withClient(+ async client => {+ const {tools} = await client.listTools();+ const clickAt = tools.find(t => t.name === 'click_at');+ assert.ok(clickAt);+ },+ ['--experimental-vision'],+ );+ });});diff --git a/tests/tools/input.test.ts b/tests/tools/input.test.tsindex f1f7fc59..cb8ae711 100644--- a/tests/tools/input.test.ts+++ b/tests/tools/input.test.ts@@ -17,6 +17,7 @@ import {fillForm,uploadFile,pressKey,+ clickAt,} from '../../src/tools/input.js';import {parseKey} from '../../src/utils/keyboard.js';import {serverHooks} from '../server.js';@@ -183,6 +184,67 @@ describe('input', () => {});});+ describe('click_at', () => {+ it('clicks at coordinates', async () => {+ await withMcpContext(async (response, context) => {+ const page = context.getSelectedPage();+ await page.setContent(+ html`<div+ style="width: 100px; height: 100px; background: red;"+ onclick="this.innerText = 'clicked'"+ ></div>`,+ );+ await context.createTextSnapshot();+ await clickAt.handler(+ {+ params: {+ x: 50,+ y: 50,+ },+ },+ response,+ context,+ );+ assert.strictEqual(+ response.responseLines[0],+ 'Successfully clicked at the coordinates',+ );+ assert.ok(response.includeSnapshot);+ assert.ok(await page.$('text/clicked'));+ });+ });++ it('double clicks at coordinates', async () => {+ await withMcpContext(async (response, context) => {+ const page = context.getSelectedPage();+ await page.setContent(+ html`<div+ style="width: 100px; height: 100px; background: red;"+ ondblclick="this.innerText = 'dblclicked'"+ ></div>`,+ );+ await context.createTextSnapshot();+ await clickAt.handler(+ {+ params: {+ x: 50,+ y: 50,+ dblClick: true,+ },+ },+ response,+ context,+ );+ assert.strictEqual(+ response.responseLines[0],+ 'Successfully double clicked at the coordinates',+ );+ assert.ok(response.includeSnapshot);+ assert.ok(await page.$('text/dblclicked'));+ });+ });+ });+describe('fill', () => {it('fills out an input', async () => {await withMcpContext(async (response, context) => {