Chrome DevTools for coding agents
fix: add an option value to the snapshot (#362)
Alina Varkki207137edd6d8
diff --git a/src/McpContext.ts b/src/McpContext.tsindex d1037935..cd829e16 100644--- a/src/McpContext.ts+++ b/src/McpContext.ts@@ -272,6 +272,10 @@ export class McpContext implements Context {return page.getDefaultNavigationTimeout();}+ getAXNodeByUid(uid: string) {+ return this.#textSnapshot?.idToNode.get(uid);+ }+async getElementByUid(uid: string): Promise<ElementHandle<Element>> {if (!this.#textSnapshot?.idToNode.size) {throw new Error(@@ -334,6 +338,16 @@ export class McpContext implements Context {? node.children.map(child => assignIds(child)): [],};++ // The AXNode for an option doesn't contain its `value`.+ // Therefore, set text content of the option as value.+ if (node.role === 'option') {+ const optionText = node.name;+ if (optionText) {+ nodeWithId.value = optionText.toString();+ }+ }+idToNode.set(nodeWithId.id, nodeWithId);return nodeWithId;};diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.tsindex fe2fae7b..56fdb53a 100644--- a/src/tools/ToolDefinition.ts+++ b/src/tools/ToolDefinition.ts@@ -7,6 +7,7 @@import type {Dialog, ElementHandle, Page} from 'puppeteer-core';import z from 'zod';+import type {TextSnapshotNode} from '../McpContext.js';import type {TraceResult} from '../trace-processing/parse.js';import type {ToolCategories} from './categories.js';@@ -68,6 +69,7 @@ export type Context = Readonly<{closePage(pageIdx: number): Promise<void>;setSelectedPageIdx(idx: number): void;getElementByUid(uid: string): Promise<ElementHandle<Element>>;+ getAXNodeByUid(uid: string): TextSnapshotNode | undefined;setNetworkConditions(conditions: string | null): void;setCpuThrottlingRate(rate: number): void;saveTemporaryFile(diff --git a/src/tools/input.ts b/src/tools/input.tsindex eda04e80..02bb8a0f 100644--- a/src/tools/input.ts+++ b/src/tools/input.ts@@ -7,6 +7,8 @@import type {ElementHandle} from 'puppeteer-core';import z from 'zod';+import type {McpContext, TextSnapshotNode} from '../McpContext.js';+import {ToolCategories} from './categories.js';import {defineTool} from './ToolDefinition.js';@@ -78,6 +80,61 @@ export const hover = defineTool({},});+// The AXNode for an option doesn't contain its `value`. We set text content of the option as value.+// If the form is a combobox, we need to find the correct option by its text value.+// To do that, loop through the children while checking which child's text matches the requested value (requested value is actually the text content).+// When the correct option is found, use the element handle to get the real value.+async function selectOption(+ handle: ElementHandle,+ aXNode: TextSnapshotNode,+ value: string,+) {+ let optionFound = false;+ for (const child of aXNode.children) {+ if (child.role === 'option' && child.name === value && child.value) {+ optionFound = true;+ const childHandle = await child.elementHandle();+ if (childHandle) {+ try {+ const childValueHandle = await childHandle.getProperty('value');+ try {+ const childValue = await childValueHandle.jsonValue();+ if (childValue) {+ await handle.asLocator().fill(childValue.toString());+ }+ } finally {+ void childValueHandle.dispose();+ }+ break;+ } finally {+ void childHandle.dispose();+ }+ }+ }+ }+ if (!optionFound) {+ throw new Error(`Could not find option with text "${value}"`);+ }+}++async function fillFormElement(+ uid: string,+ value: string,+ context: McpContext,+) {+ const handle = await context.getElementByUid(uid);+ try {+ const aXNode = context.getAXNodeByUid(uid);+ if (aXNode && aXNode.role === 'combobox') {+ await selectOption(handle, aXNode, value);+ } else {+ await handle.asLocator().fill(value);+ }+ } finally {+ void handle.dispose();+ }+}+export const fill = defineTool({name: 'fill',description: `Type text into a input, text area or select an option from a <select> element.`,@@ -94,16 +151,15 @@ export const fill = defineTool({value: z.string().describe('The value to fill in'),},handler: async (request, response, context) => {- const handle = await context.getElementByUid(request.params.uid);- try {- await context.waitForEventsAfterAction(async () => {- await handle.asLocator().fill(request.params.value);- });- response.appendResponseLine(`Successfully filled out the element`);- response.setIncludeSnapshot(true);- } finally {- void handle.dispose();- }+ await context.waitForEventsAfterAction(async () => {+ await fillFormElement(+ request.params.uid,+ request.params.value,+ context as McpContext,+ );+ });+ response.appendResponseLine(`Successfully filled out the element`);+ response.setIncludeSnapshot(true);},});@@ -155,14 +211,13 @@ export const fillForm = defineTool({},handler: async (request, response, context) => {for (const element of request.params.elements) {- const handle = await context.getElementByUid(element.uid);- try {- await context.waitForEventsAfterAction(async () => {- await handle.asLocator().fill(element.value);- });- } finally {- void handle.dispose();- }+ await context.waitForEventsAfterAction(async () => {+ await fillFormElement(+ element.uid,+ element.value,+ context as McpContext,+ );+ });}response.appendResponseLine(`Successfully filled out the form`);response.setIncludeSnapshot(true);diff --git a/tests/tools/input.test.ts b/tests/tools/input.test.tsindex 8329192f..b788c7b5 100644--- a/tests/tools/input.test.ts+++ b/tests/tools/input.test.ts@@ -202,6 +202,35 @@ describe('input', () => {assert.ok(await page.$('text/test'));});});++ it('fills out a select by text', async () => {+ await withBrowser(async (response, context) => {+ const page = context.getSelectedPage();+ await page.setContent(+ `<!DOCTYPE html><select><option value="v1">one</option><option value="v2">two</option></select>`,+ );+ await context.createTextSnapshot();+ await fill.handler(+ {+ params: {+ uid: '1_1',+ value: 'two',+ },+ },+ response,+ context,+ );+ assert.strictEqual(+ response.responseLines[0],+ 'Successfully filled out the element',+ );+ assert.ok(response.includeSnapshot);+ const selectedValue = await page.evaluate(+ () => document.querySelector('select')!.value,+ );+ assert.strictEqual(selectedValue, 'v2');+ });+ });});describe('drags', () => {