Skip to content

Chrome DevTools for coding agents

fix: add an option value to the snapshot (#362)

Alina Varkki207137edd6d8
diff --git a/src/McpContext.ts b/src/McpContext.ts
index d1037935..cd829e16 100644
--- a/src/McpContext.ts
+++ b/src/McpContext.ts
@@ -272,6 +272,10 @@ export class McpContext implements Context {
return page.getDefaultNavigationTimeout();
}
+ getAXNodeByUid(uid: string) {
+ return this.#textSnapshot?.idToNode.get(uid);
+ }
+
async getElementByUid(uid: string): Promise<ElementHandle<Element>> {
if (!this.#textSnapshot?.idToNode.size) {
throw new Error(
@@ -334,6 +338,16 @@ export class McpContext implements Context {
? node.children.map(child => assignIds(child))
: [],
};
+
+ // The AXNode for an option doesn't contain its `value`.
+ // Therefore, set text content of the option as value.
+ if (node.role === 'option') {
+ const optionText = node.name;
+ if (optionText) {
+ nodeWithId.value = optionText.toString();
+ }
+ }
+
idToNode.set(nodeWithId.id, nodeWithId);
return nodeWithId;
};
diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.ts
index fe2fae7b..56fdb53a 100644
--- a/src/tools/ToolDefinition.ts
+++ b/src/tools/ToolDefinition.ts
@@ -7,6 +7,7 @@
import type {Dialog, ElementHandle, Page} from 'puppeteer-core';
import z from 'zod';
+import type {TextSnapshotNode} from '../McpContext.js';
import type {TraceResult} from '../trace-processing/parse.js';
import type {ToolCategories} from './categories.js';
@@ -68,6 +69,7 @@ export type Context = Readonly<{
closePage(pageIdx: number): Promise<void>;
setSelectedPageIdx(idx: number): void;
getElementByUid(uid: string): Promise<ElementHandle<Element>>;
+ getAXNodeByUid(uid: string): TextSnapshotNode | undefined;
setNetworkConditions(conditions: string | null): void;
setCpuThrottlingRate(rate: number): void;
saveTemporaryFile(
diff --git a/src/tools/input.ts b/src/tools/input.ts
index eda04e80..02bb8a0f 100644
--- a/src/tools/input.ts
+++ b/src/tools/input.ts
@@ -7,6 +7,8 @@
import type {ElementHandle} from 'puppeteer-core';
import z from 'zod';
+import type {McpContext, TextSnapshotNode} from '../McpContext.js';
+
import {ToolCategories} from './categories.js';
import {defineTool} from './ToolDefinition.js';
@@ -78,6 +80,61 @@ export const hover = defineTool({
},
});
+// The AXNode for an option doesn't contain its `value`. We set text content of the option as value.
+// If the form is a combobox, we need to find the correct option by its text value.
+// To do that, loop through the children while checking which child's text matches the requested value (requested value is actually the text content).
+// When the correct option is found, use the element handle to get the real value.
+async function selectOption(
+ handle: ElementHandle,
+ aXNode: TextSnapshotNode,
+ value: string,
+) {
+ let optionFound = false;
+ for (const child of aXNode.children) {
+ if (child.role === 'option' && child.name === value && child.value) {
+ optionFound = true;
+ const childHandle = await child.elementHandle();
+ if (childHandle) {
+ try {
+ const childValueHandle = await childHandle.getProperty('value');
+ try {
+ const childValue = await childValueHandle.jsonValue();
+ if (childValue) {
+ await handle.asLocator().fill(childValue.toString());
+ }
+ } finally {
+ void childValueHandle.dispose();
+ }
+ break;
+ } finally {
+ void childHandle.dispose();
+ }
+ }
+ }
+ }
+ if (!optionFound) {
+ throw new Error(`Could not find option with text "${value}"`);
+ }
+}
+
+async function fillFormElement(
+ uid: string,
+ value: string,
+ context: McpContext,
+) {
+ const handle = await context.getElementByUid(uid);
+ try {
+ const aXNode = context.getAXNodeByUid(uid);
+ if (aXNode && aXNode.role === 'combobox') {
+ await selectOption(handle, aXNode, value);
+ } else {
+ await handle.asLocator().fill(value);
+ }
+ } finally {
+ void handle.dispose();
+ }
+}
+
export const fill = defineTool({
name: 'fill',
description: `Type text into a input, text area or select an option from a <select> element.`,
@@ -94,16 +151,15 @@ export const fill = defineTool({
value: z.string().describe('The value to fill in'),
},
handler: async (request, response, context) => {
- const handle = await context.getElementByUid(request.params.uid);
- try {
- await context.waitForEventsAfterAction(async () => {
- await handle.asLocator().fill(request.params.value);
- });
- response.appendResponseLine(`Successfully filled out the element`);
- response.setIncludeSnapshot(true);
- } finally {
- void handle.dispose();
- }
+ await context.waitForEventsAfterAction(async () => {
+ await fillFormElement(
+ request.params.uid,
+ request.params.value,
+ context as McpContext,
+ );
+ });
+ response.appendResponseLine(`Successfully filled out the element`);
+ response.setIncludeSnapshot(true);
},
});
@@ -155,14 +211,13 @@ export const fillForm = defineTool({
},
handler: async (request, response, context) => {
for (const element of request.params.elements) {
- const handle = await context.getElementByUid(element.uid);
- try {
- await context.waitForEventsAfterAction(async () => {
- await handle.asLocator().fill(element.value);
- });
- } finally {
- void handle.dispose();
- }
+ await context.waitForEventsAfterAction(async () => {
+ await fillFormElement(
+ element.uid,
+ element.value,
+ context as McpContext,
+ );
+ });
}
response.appendResponseLine(`Successfully filled out the form`);
response.setIncludeSnapshot(true);
diff --git a/tests/tools/input.test.ts b/tests/tools/input.test.ts
index 8329192f..b788c7b5 100644
--- a/tests/tools/input.test.ts
+++ b/tests/tools/input.test.ts
@@ -202,6 +202,35 @@ describe('input', () => {
assert.ok(await page.$('text/test'));
});
});
+
+ it('fills out a select by text', async () => {
+ await withBrowser(async (response, context) => {
+ const page = context.getSelectedPage();
+ await page.setContent(
+ `<!DOCTYPE html><select><option value="v1">one</option><option value="v2">two</option></select>`,
+ );
+ await context.createTextSnapshot();
+ await fill.handler(
+ {
+ params: {
+ uid: '1_1',
+ value: 'two',
+ },
+ },
+ response,
+ context,
+ );
+ assert.strictEqual(
+ response.responseLines[0],
+ 'Successfully filled out the element',
+ );
+ assert.ok(response.includeSnapshot);
+ const selectedValue = await page.evaluate(
+ () => document.querySelector('select')!.value,
+ );
+ assert.strictEqual(selectedValue, 'v2');
+ });
+ });
});
describe('drags', () => {