Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(mcp): allow passing --console-level (#38439)

Pavel Feldman408ee0fcfe6c
diff --git a/docs/src/api/class-consolemessage.md b/docs/src/api/class-consolemessage.md
index b712bca267..044d070342 100644
--- a/docs/src/api/class-consolemessage.md
+++ b/docs/src/api/class-consolemessage.md
@@ -140,7 +140,7 @@ The text of the console message.
## method: ConsoleMessage.type
* since: v1.8
* langs: js, python
-- returns: <[ConsoleMessageType]<"log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"timeEnd">>
+- returns: <[ConsoleMessageType]<"log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"time"|"timeEnd">>
## method: ConsoleMessage.type
* since: v1.8
diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts
index d6ff2f0ace..861d7a0f66 100644
--- a/packages/playwright-client/types/types.d.ts
+++ b/packages/playwright-client/types/types.d.ts
@@ -18861,7 +18861,7 @@ export interface ConsoleMessage {
*/
text(): string;
- type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"timeEnd";
+ type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"time"|"timeEnd";
/**
* The web worker or service worker that produced this console message, if any. Note that console messages from web
diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts
index d6ff2f0ace..861d7a0f66 100644
--- a/packages/playwright-core/types/types.d.ts
+++ b/packages/playwright-core/types/types.d.ts
@@ -18861,7 +18861,7 @@ export interface ConsoleMessage {
*/
text(): string;
- type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"timeEnd";
+ type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"time"|"timeEnd";
/**
* The web worker or service worker that produced this console message, if any. Note that console messages from web
diff --git a/packages/playwright/src/mcp/browser/config.ts b/packages/playwright/src/mcp/browser/config.ts
index dc83efa9f0..1099f3d281 100644
--- a/packages/playwright/src/mcp/browser/config.ts
+++ b/packages/playwright/src/mcp/browser/config.ts
@@ -39,6 +39,7 @@ export type CLIOptions = {
cdpEndpoint?: string;
cdpHeader?: Record<string, string>;
config?: string;
+ consoleLevel?: 'error' | 'warning' | 'info' | 'debug';
device?: string;
executablePath?: string;
grantPermissions?: string[];
@@ -81,6 +82,9 @@ export const defaultConfig: FullConfig = {
viewport: null,
},
},
+ console: {
+ level: 'info',
+ },
network: {
allowedOrigins: undefined,
blockedOrigins: undefined,
@@ -104,6 +108,9 @@ export type FullConfig = Config & {
launchOptions: NonNullable<BrowserUserConfig['launchOptions']>;
contextOptions: NonNullable<BrowserUserConfig['contextOptions']>;
},
+ console: {
+ level: 'error' | 'warning' | 'info' | 'debug';
+ },
network: NonNullable<Config['network']>,
saveTrace: boolean;
server: NonNullable<Config['server']>,
@@ -241,6 +248,9 @@ export function configFromCLIOptions(cliOptions: CLIOptions): Config {
allowedHosts: cliOptions.allowedHosts,
},
capabilities: cliOptions.caps as ToolCapability[],
+ console: {
+ level: cliOptions.consoleLevel,
+ },
network: {
allowedOrigins: cliOptions.allowedOrigins,
blockedOrigins: cliOptions.blockedOrigins,
@@ -274,6 +284,8 @@ function configFromEnv(): Config {
options.cdpEndpoint = envToString(process.env.PLAYWRIGHT_MCP_CDP_ENDPOINT);
options.cdpHeader = headerParser(process.env.PLAYWRIGHT_MCP_CDP_HEADERS, {});
options.config = envToString(process.env.PLAYWRIGHT_MCP_CONFIG);
+ if (process.env.PLAYWRIGHT_MCP_CONSOLE_LEVEL)
+ options.consoleLevel = enumParser<'error' | 'warning' | 'info' | 'debug'>('--console-level', ['error', 'warning', 'info', 'debug'], process.env.PLAYWRIGHT_MCP_CONSOLE_LEVEL);
options.device = envToString(process.env.PLAYWRIGHT_MCP_DEVICE);
options.executablePath = envToString(process.env.PLAYWRIGHT_MCP_EXECUTABLE_PATH);
options.grantPermissions = commaSeparatedList(process.env.PLAYWRIGHT_MCP_GRANT_PERMISSIONS);
@@ -287,8 +299,8 @@ function configFromEnv(): Config {
if (initScript)
options.initScript = [initScript];
options.isolated = envToBoolean(process.env.PLAYWRIGHT_MCP_ISOLATED);
- if (process.env.PLAYWRIGHT_MCP_IMAGE_RESPONSES === 'omit')
- options.imageResponses = 'omit';
+ if (process.env.PLAYWRIGHT_MCP_IMAGE_RESPONSES)
+ options.imageResponses = enumParser<'allow' | 'omit'>('--image-responses', ['allow', 'omit'], process.env.PLAYWRIGHT_MCP_IMAGE_RESPONSES);
options.sandbox = envToBoolean(process.env.PLAYWRIGHT_MCP_SANDBOX);
options.outputDir = envToString(process.env.PLAYWRIGHT_MCP_OUTPUT_DIR);
options.port = numberParser(process.env.PLAYWRIGHT_MCP_PORT);
@@ -385,6 +397,10 @@ function mergeConfig(base: FullConfig, overrides: Config): FullConfig {
...pickDefined(base),
...pickDefined(overrides),
browser,
+ console: {
+ ...pickDefined(base.console),
+ ...pickDefined(overrides.console),
+ },
network: {
...pickDefined(base.network),
...pickDefined(overrides.network),
@@ -458,6 +474,12 @@ export function headerParser(arg: string | undefined, previous?: Record<string,
return result;
}
+export function enumParser<T extends string>(name: string, options: T[], value: string): T {
+ if (!options.includes(value as T))
+ throw new Error(`Invalid ${name}: ${value}. Valid values are: ${options.join(', ')}`);
+ return value as T;
+}
+
function envToBoolean(value: string | undefined): boolean | undefined {
if (value === 'true' || value === '1')
return true;
diff --git a/packages/playwright/src/mcp/browser/tab.ts b/packages/playwright/src/mcp/browser/tab.ts
index d254b115bf..9563bf7d29 100644
--- a/packages/playwright/src/mcp/browser/tab.ts
+++ b/packages/playwright/src/mcp/browser/tab.ts
@@ -217,9 +217,9 @@ export class Tab extends EventEmitter<TabEventsInterface> {
await this.waitForLoadState('load', { timeout: 5000 });
}
- async consoleMessages(type?: 'error'): Promise<ConsoleMessage[]> {
+ async consoleMessages(level: ConsoleMessageLevel): Promise<ConsoleMessage[]> {
await this._initializedPromise;
- return this._consoleMessages.filter(message => type ? message.type === type : true);
+ return this._consoleMessages.filter(message => shouldIncludeMessage(level, message.type));
}
async requests(): Promise<Set<playwright.Request>> {
@@ -244,7 +244,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {
});
if (tabSnapshot) {
// Assign console message late so that we did not lose any to modal state.
- tabSnapshot.consoleMessages = this._recentConsoleMessages;
+ tabSnapshot.consoleMessages = this._recentConsoleMessages.filter(message => shouldIncludeMessage(this.context.config.console.level, message.type));
this._recentConsoleMessages = [];
}
// If we failed to capture a snapshot this time, make sure we do a full one next time,
@@ -317,7 +317,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {
}
export type ConsoleMessage = {
- type: ReturnType<playwright.ConsoleMessage['type']> | undefined;
+ type: ReturnType<playwright.ConsoleMessage['type']>;
text: string;
toString(): string;
};
@@ -354,4 +354,43 @@ export function renderModalStates(modalStates: ModalState[]): string[] {
return result;
}
+type ConsoleMessageType = ReturnType<playwright.ConsoleMessage['type']>;
+type ConsoleMessageLevel = 'error' | 'warning' | 'info' | 'debug';
+const consoleMessageLevels: ConsoleMessageLevel[] = ['error', 'warning', 'info', 'debug'];
+
+function shouldIncludeMessage(thresholdLevel: ConsoleMessageLevel, type: ConsoleMessageType): boolean {
+ const messageLevel = consoleLevelForMessageType(type);
+ return consoleMessageLevels.indexOf(messageLevel) <= consoleMessageLevels.indexOf(thresholdLevel);
+}
+
+function consoleLevelForMessageType(type: ConsoleMessageType): ConsoleMessageLevel {
+ switch (type) {
+ case 'assert':
+ case 'error':
+ return 'error';
+ case 'warning':
+ return 'warning';
+ case 'count':
+ case 'dir':
+ case 'dirxml':
+ case 'info':
+ case 'log':
+ case 'table':
+ case 'time':
+ case 'timeEnd':
+ return 'info';
+ case 'clear':
+ case 'debug':
+ case 'endGroup':
+ case 'profile':
+ case 'profileEnd':
+ case 'startGroup':
+ case 'startGroupCollapsed':
+ case 'trace':
+ return 'debug';
+ default:
+ return 'info';
+ }
+}
+
const tabSymbol = Symbol('tabSymbol');
diff --git a/packages/playwright/src/mcp/browser/tools/console.ts b/packages/playwright/src/mcp/browser/tools/console.ts
index b9bda093ac..6643da3177 100644
--- a/packages/playwright/src/mcp/browser/tools/console.ts
+++ b/packages/playwright/src/mcp/browser/tools/console.ts
@@ -24,12 +24,12 @@ const console = defineTabTool({
title: 'Get console messages',
description: 'Returns all console messages',
inputSchema: z.object({
- onlyErrors: z.boolean().optional().describe('Only return error messages'),
+ level: z.enum(['error', 'warning', 'info', 'debug']).default('info').describe('Level of the console messages to return. Each level includes the messages of more severe levels. Defaults to "info".'),
}),
type: 'readOnly',
},
handle: async (tab, params, response) => {
- const messages = await tab.consoleMessages(params.onlyErrors ? 'error' : undefined);
+ const messages = await tab.consoleMessages(params.level);
messages.map(message => response.addResult(message.toString()));
},
});
diff --git a/packages/playwright/src/mcp/config.d.ts b/packages/playwright/src/mcp/config.d.ts
index 9a04795ccf..1f10ecdf34 100644
--- a/packages/playwright/src/mcp/config.d.ts
+++ b/packages/playwright/src/mcp/config.d.ts
@@ -142,6 +142,13 @@ export type Config = {
*/
outputDir?: string;
+ console?: {
+ /**
+ * The level of console messages to return. Each level includes the messages of more severe levels. Defaults to "info".
+ */
+ level?: 'error' | 'warning' | 'info' | 'debug';
+ },
+
network?: {
/**
* List of origins to allow the browser to request. Default is to allow all. Origins matching both `allowedOrigins` and `blockedOrigins` will be blocked.
diff --git a/packages/playwright/src/mcp/program.ts b/packages/playwright/src/mcp/program.ts
index 4635b7e1b3..1c0389dcfb 100644
--- a/packages/playwright/src/mcp/program.ts
+++ b/packages/playwright/src/mcp/program.ts
@@ -22,7 +22,7 @@ import { colors, ProgramOption } from 'playwright-core/lib/utilsBundle';
import { registry } from 'playwright-core/lib/server';
import * as mcpServer from './sdk/server';
-import { commaSeparatedList, dotenvFileLoader, headerParser, numberParser, resolutionParser, resolveCLIConfig, semicolonSeparatedList } from './browser/config';
+import { commaSeparatedList, dotenvFileLoader, enumParser, headerParser, numberParser, resolutionParser, resolveCLIConfig, semicolonSeparatedList } from './browser/config';
import { setupExitWatchdog } from './browser/watchdog';
import { contextFactory } from './browser/browserContextFactory';
import { ProxyBackend } from './sdk/proxyBackend';
@@ -43,6 +43,7 @@ export function decorateCommand(command: Command, version: string) {
.option('--cdp-endpoint <endpoint>', 'CDP endpoint to connect to.')
.option('--cdp-header <headers...>', 'CDP headers to send with the connect request, multiple can be specified.', headerParser)
.option('--config <path>', 'path to the configuration file.')
+ .option('--console-level <level>', 'level of console messages to return: "error", "warning", "info", "debug". Each level includes the messages of more severe levels.', enumParser.bind(null, '--console-level', ['error', 'warning', 'info', 'debug']))
.option('--device <device>', 'device to emulate, for example: "iPhone 15"')
.option('--executable-path <path>', 'path to the browser executable.')
.option('--extension', 'Connect to a running browser instance (Edge/Chrome only). Requires the "Playwright MCP Bridge" browser extension to be installed.')
@@ -53,7 +54,7 @@ export function decorateCommand(command: Command, version: string) {
.option('--init-page <path...>', 'path to TypeScript file to evaluate on Playwright page object')
.option('--init-script <path...>', 'path to JavaScript file to add as an initialization script. The script will be evaluated in every page before any of the page\'s scripts. Can be specified multiple times.')
.option('--isolated', 'keep the browser profile in memory, do not save it to disk.')
- .option('--image-responses <mode>', 'whether to send image responses to the client. Can be "allow" or "omit", Defaults to "allow".')
+ .option('--image-responses <mode>', 'whether to send image responses to the client. Can be "allow" or "omit", Defaults to "allow".', enumParser.bind(null, '--image-responses', ['allow', 'omit']))
.option('--no-sandbox', 'disable the sandbox for all process types that are normally sandboxed.')
.option('--output-dir <path>', 'path to the directory for output files.')
.option('--port <port>', 'port to listen on for SSE transport.')
diff --git a/packages/playwright/src/mcp/test/browserBackend.ts b/packages/playwright/src/mcp/test/browserBackend.ts
index 12e8927ecd..7bed39f6bb 100644
--- a/packages/playwright/src/mcp/test/browserBackend.ts
+++ b/packages/playwright/src/mcp/test/browserBackend.ts
@@ -95,7 +95,7 @@ async function generatePausedMessage(testInfo: TestInfo, context: playwright.Bro
);
// Only print console errors when pausing on error, not when everything works as expected.
let console = testInfo.errors.length ? await Tab.collectConsoleMessages(page) : [];
- console = console.filter(msg => !msg.type || msg.type === 'error');
+ console = console.filter(msg => msg.type === 'error');
if (console.length) {
lines.push('- Console Messages:');
for (const message of console)
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
index 16c1d734a6..69e8c7ef74 100644
--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
@@ -1,27 +1,27 @@
{
- "4c37062e059030f9c15a515aba7dea63145b1691": {
+ "1f67d34de9dca04e9ec4d9d7b590b48b6e67fd08": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll help you navigate to the website, fill out the form to order a blue table and a green desk, and report back the total price and delivery time."
+ "text": "I'll help you navigate to the form, fill it out with the specified details, and report back the pricing and delivery information."
}
]
},
"usage": {
- "input": 4390,
- "output": 127
+ "input": 4429,
+ "output": 120
}
},
- "b09d261846c63220d3abdcee433a5987bce04d5f": {
+ "4498c58367500860e392c029b7f90d23ca898b50": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Navigate to the form page to order items",
- "copilotToolCallId": "toolu_vrtx_01Nrr85ePXvh2wqRS6S47Lej"
+ "text": "Navigate to the form page to start the ordering process",
+ "copilotToolCallId": "toolu_vrtx_0129qd3zaawZfjyTyiNQ8dbY"
},
{
"type": "tool_call",
@@ -29,23 +29,45 @@
"arguments": {
"url": "http://localhost:<PORT>/evals/fill-form.html"
},
- "id": "toolu_vrtx_01Nrr85ePXvh2wqRS6S47Lej"
+ "id": "toolu_vrtx_0129qd3zaawZfjyTyiNQ8dbY"
+ }
+ ]
+ },
+ "usage": {
+ "input": 4486,
+ "output": 93
+ }
+ },
+ "1f4cb362bd2e2d63916d74077a8cd478a0f0b1a5": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Get current page state to identify form fields for the first item (blue table)",
+ "copilotToolCallId": "toolu_vrtx_01Sqo3h7zZGVZYkVDWBrmhfg"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_snapshot",
+ "arguments": {},
+ "id": "toolu_vrtx_01Sqo3h7zZGVZYkVDWBrmhfg"
}
]
},
"usage": {
- "input": 4454,
- "output": 91
+ "input": 5200,
+ "output": 67
}
},
- "83da5a4a79d93a494d7f0f76d4758bc493d411f5": {
+ "54801f22b272e5a430fd56cc07ab4b11d87c6514": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Table' as the product type for the first item",
- "copilotToolCallId": "toolu_vrtx_011Vy4WSELJYN4U2tv9NgCLi"
+ "text": "Select Table as the product type for the first order",
+ "copilotToolCallId": "toolu_vrtx_01KZtfPrpLh9Pg7D8hzZwfw5"
},
{
"type": "tool_call",
@@ -57,23 +79,23 @@
"Table"
]
},
- "id": "toolu_vrtx_011Vy4WSELJYN4U2tv9NgCLi"
+ "id": "toolu_vrtx_01KZtfPrpLh9Pg7D8hzZwfw5"
}
]
},
"usage": {
- "input": 5166,
- "output": 123
+ "input": 5858,
+ "output": 120
}
},
- "0a13ca0491d6745366d7b71a32b864412e848d78": {
+ "8585f97b99c7165ca864db490da286f768b8b883": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Blue' as the color for the table",
- "copilotToolCallId": "toolu_vrtx_01AyhQJov7bZQ7dY9CmcZMeY"
+ "text": "Select Blue as the color for the table",
+ "copilotToolCallId": "toolu_vrtx_01XGcSSgUwBCcgqLu11QqxRQ"
},
{
"type": "tool_call",
@@ -85,23 +107,23 @@
"Blue"
]
},
- "id": "toolu_vrtx_01AyhQJov7bZQ7dY9CmcZMeY"
+ "id": "toolu_vrtx_01XGcSSgUwBCcgqLu11QqxRQ"
}
]
},
"usage": {
- "input": 5436,
- "output": 120
+ "input": 6125,
+ "output": 117
}
},
- "0e1345afd5cefb63d40c0d9750aa6b73dfe36f38": {
+ "d78aa88a096ad38aea37e1b0b73aefd529845ab8": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in the customer details for the order",
- "copilotToolCallId": "toolu_vrtx_011xPGae2W6mMbh3TXpbqjaH"
+ "text": "Fill in customer information for the order",
+ "copilotToolCallId": "toolu_vrtx_01Vk16c9EpV35aGtSjn2J6Pk"
},
{
"type": "tool_call",
@@ -124,7 +146,7 @@
"name": "Street Address",
"type": "textbox",
"ref": "e19",
- "value": "123 Main St, Anytown"
+ "value": "123 Main St"
},
{
"name": "City",
@@ -146,23 +168,23 @@
}
]
},
- "id": "toolu_vrtx_011xPGae2W6mMbh3TXpbqjaH"
+ "id": "toolu_vrtx_01Vk16c9EpV35aGtSjn2J6Pk"
}
]
},
"usage": {
- "input": 5705,
- "output": 348
+ "input": 6391,
+ "output": 343
}
},
- "90b95b16881abda623068841009641e51f32e7ae": {
+ "1380ec99c1a8290d52dd9d4bb008bc5add1efb54": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click the Order button to submit the first order (blue table)",
- "copilotToolCallId": "toolu_vrtx_01FwpyQSshn5YP7JyJbptXEh"
+ "text": "Submit the order for the blue table",
+ "copilotToolCallId": "toolu_vrtx_011CLYRgGn3zonGBT97Wxr4r"
},
{
"type": "tool_call",
@@ -171,23 +193,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_01FwpyQSshn5YP7JyJbptXEh"
+ "id": "toolu_vrtx_011CLYRgGn3zonGBT97Wxr4r"
}
]
},
"usage": {
- "input": 6163,
- "output": 100
+ "input": 6840,
+ "output": 95
}
},
- "7160f78eeba943a8bf0daac205c6b183839401ca": {
+ "6ae203915486aa37b263ad3ad8af6af4a0c9e9f6": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Accept the confirmation dialog for the blue table order",
- "copilotToolCallId": "toolu_vrtx_01LW5t6fGbyzj4ARj6wATf6y"
+ "copilotToolCallId": "toolu_vrtx_01BVGW6UrHvCKQ7v5xNPv1xv"
},
{
"type": "tool_call",
@@ -195,23 +217,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01LW5t6fGbyzj4ARj6wATf6y"
+ "id": "toolu_vrtx_01BVGW6UrHvCKQ7v5xNPv1xv"
}
]
},
"usage": {
- "input": 6338,
+ "input": 7010,
"output": 80
}
},
- "80f4d8e16d86fb67f34dae4668d2ac84c71ac3ee": {
+ "9ec58f784b989e7cbbbde0656d22e6be1c5075dd": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click 'Place Another Order' button to order the second item (green desk)",
- "copilotToolCallId": "toolu_vrtx_01Y6z1hSQqTdsRs6VhoAkNrJ"
+ "text": "Click the button to place another order for the green desk",
+ "copilotToolCallId": "toolu_vrtx_016rQY2JhLzVu3TpokRVqdor"
},
{
"type": "tool_call",
@@ -220,23 +242,23 @@
"element": "Place Another Order button",
"ref": "e58"
},
- "id": "toolu_vrtx_01Y6z1hSQqTdsRs6VhoAkNrJ"
+ "id": "toolu_vrtx_016rQY2JhLzVu3TpokRVqdor"
}
]
},
"usage": {
- "input": 7236,
- "output": 106
+ "input": 7897,
+ "output": 101
}
},
- "8a7a2b461872a308c3c471069553fbb110b233e2": {
+ "83a5695faf3e865ff1654007db08a088bc2b9412": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Desk' as the product type for the second item",
- "copilotToolCallId": "toolu_vrtx_01XFkp8Sb3zfxfSX56VB5Ukp"
+ "text": "Select Desk as the product type for the second order",
+ "copilotToolCallId": "toolu_vrtx_01WnDXifmCJWtEMduSaUjHU8"
},
{
"type": "tool_call",
@@ -248,23 +270,23 @@
"Desk"
]
},
- "id": "toolu_vrtx_01XFkp8Sb3zfxfSX56VB5Ukp"
+ "id": "toolu_vrtx_01WnDXifmCJWtEMduSaUjHU8"
}
]
},
"usage": {
- "input": 7962,
- "output": 125
+ "input": 8618,
+ "output": 122
}
},
- "aa6c7116bcda3f6079c1d67f86e781f745b9511f": {
+ "117e2c52b10e5288047ef65ac10a59c9e2275674": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Green' as the color for the desk",
- "copilotToolCallId": "toolu_vrtx_01BBnzzEU2bFJdGS8XeWbKbU"
+ "text": "Select Green as the color for the desk",
+ "copilotToolCallId": "toolu_vrtx_01462K6LBMFTWaVWPmegByzb"
},
{
"type": "tool_call",
@@ -276,23 +298,23 @@
"Green"
]
},
- "id": "toolu_vrtx_01BBnzzEU2bFJdGS8XeWbKbU"
+ "id": "toolu_vrtx_01462K6LBMFTWaVWPmegByzb"
}
]
},
"usage": {
- "input": 8235,
- "output": 120
+ "input": 8888,
+ "output": 117
}
},
- "a30b487338609e8a5d8417c1f90419956bb4b727": {
+ "ad76fc7dda0725834872ca9b8f4395722197026a": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in the customer details for the second order",
- "copilotToolCallId": "toolu_vrtx_01W1QKYyFZvMCWhC3DwYQv2X"
+ "text": "Fill in customer information for the second order",
+ "copilotToolCallId": "toolu_vrtx_01Bonk7ZwhuGS7yc67tAqJMd"
},
{
"type": "tool_call",
@@ -315,7 +337,7 @@
"name": "Street Address",
"ref": "e19",
"type": "textbox",
- "value": "123 Main St, Anytown"
+ "value": "123 Main St"
},
{
"name": "City",
@@ -337,23 +359,23 @@
}
]
},
- "id": "toolu_vrtx_01W1QKYyFZvMCWhC3DwYQv2X"
+ "id": "toolu_vrtx_01Bonk7ZwhuGS7yc67tAqJMd"
}
]
},
"usage": {
- "input": 8504,
- "output": 269
+ "input": 9154,
+ "output": 264
}
},
- "7269d27383720010c65e7e530cf66cff736cb587": {
+ "2dd639bc23dfba249c89bacb6f09a6219538a99d": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click the Order button to submit the second order (green desk)",
- "copilotToolCallId": "toolu_vrtx_019EvisGfKGAHWu4Ehk6orM9"
+ "text": "Submit the order for the green desk",
+ "copilotToolCallId": "toolu_vrtx_01B3Hfa3jbKn1278dRq9izUB"
},
{
"type": "tool_call",
@@ -362,23 +384,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_019EvisGfKGAHWu4Ehk6orM9"
+ "id": "toolu_vrtx_01B3Hfa3jbKn1278dRq9izUB"
}
]
},
"usage": {
- "input": 8963,
- "output": 100
+ "input": 9604,
+ "output": 95
}
},
- "28e8e2a0869e245c707a879c41bf7811648233ff": {
+ "df594f27d4f66356f38861d693b8f14a66ef0af4": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Accept the confirmation dialog for the green desk order",
- "copilotToolCallId": "toolu_vrtx_01MawbHWCqo6WBwPh8YcEBm4"
+ "copilotToolCallId": "toolu_vrtx_01HPLerVExZZZ15M7UeJsh3X"
},
{
"type": "tool_call",
@@ -386,23 +408,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01MawbHWCqo6WBwPh8YcEBm4"
+ "id": "toolu_vrtx_01HPLerVExZZZ15M7UeJsh3X"
}
]
},
"usage": {
- "input": 9138,
+ "input": 9774,
"output": 80
}
},
- "cc4e5987be7c007db351125c487c40e5413aebf1": {
+ "a6b8bcc69f1ddb715921d1814326fea5e5a96f22": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Report the total price and maximum delivery time for both orders",
- "copilotToolCallId": "toolu_vrtx_01DahshwKgzha389UJ8rq29Z"
+ "text": "Report the total price and maximum delivery time for both orders - blue table ($353.98) and green desk ($299.98)",
+ "copilotToolCallId": "toolu_vrtx_019b25HJgtNdXAz3QjmCnHYw"
},
{
"type": "tool_call",
@@ -411,13 +433,13 @@
"price": 653.96,
"deliveryDays": 3
},
- "id": "toolu_vrtx_01DahshwKgzha389UJ8rq29Z"
+ "id": "toolu_vrtx_019b25HJgtNdXAz3QjmCnHYw"
}
]
},
"usage": {
- "input": 10037,
- "output": 102
+ "input": 10662,
+ "output": 117
}
}
}
\ No newline at end of file
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
index 68b9b1d7cf..67864d8c09 100644
--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
@@ -1,27 +1,27 @@
{
- "4c37062e059030f9c15a515aba7dea63145b1691": {
+ "1f67d34de9dca04e9ec4d9d7b590b48b6e67fd08": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll help you complete this task. Let me start by navigating to the page and then filling out the order form."
+ "text": "I'll help you navigate to the form, fill it out, and order the items. Let me start by navigating to the URL and taking a snapshot to see the page structure."
}
]
},
"usage": {
- "input": 4390,
- "output": 114
+ "input": 4429,
+ "output": 131
}
},
- "e1e0bd5a8606f1711cc30f461f0c274f47c58792": {
+ "eb602996f263d0f29d50e40769506f6f47394046": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Navigate to the form page",
- "copilotToolCallId": "toolu_vrtx_01CyPTAyQwbp3tJQ8HRHcZpU"
+ "copilotToolCallId": "toolu_vrtx_016zERgz28Qc3mWdBLi5V2Xp"
},
{
"type": "tool_call",
@@ -29,23 +29,23 @@
"arguments": {
"url": "http://localhost:<PORT>/evals/fill-form.html"
},
- "id": "toolu_vrtx_01CyPTAyQwbp3tJQ8HRHcZpU"
+ "id": "toolu_vrtx_016zERgz28Qc3mWdBLi5V2Xp"
}
]
},
"usage": {
- "input": 4446,
+ "input": 4497,
"output": 88
}
},
- "7570cf20826150ea1678925d3c56cdaf6cc0ce0d": {
+ "ae111fb9371c47d04762f81458d5557be0ca760c": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Table' as the product type for the first order",
- "copilotToolCallId": "toolu_vrtx_01R9atbVxke4YwYTPtDeMwuK"
+ "text": "Select Table as the first product",
+ "copilotToolCallId": "toolu_vrtx_017WjVw9bpcfDw5892VkuT76"
},
{
"type": "tool_call",
@@ -57,23 +57,23 @@
"Table"
]
},
- "id": "toolu_vrtx_01R9atbVxke4YwYTPtDeMwuK"
+ "id": "toolu_vrtx_017WjVw9bpcfDw5892VkuT76"
}
]
},
"usage": {
- "input": 5155,
- "output": 123
+ "input": 5206,
+ "output": 116
}
},
- "1e9bce6dd706be6d7272ab962111b4e7eb5730d7": {
+ "08407b7115ec34eea2a164355fe32b570e965d41": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Blue' as the color for the table",
- "copilotToolCallId": "toolu_vrtx_012uA5vSCDJPndFMhFP9z6AT"
+ "text": "Select Blue as the color for the table",
+ "copilotToolCallId": "toolu_vrtx_01S7UqYcgtqNzZS4zrHwWHpu"
},
{
"type": "tool_call",
@@ -85,23 +85,23 @@
"Blue"
]
},
- "id": "toolu_vrtx_012uA5vSCDJPndFMhFP9z6AT"
+ "id": "toolu_vrtx_01S7UqYcgtqNzZS4zrHwWHpu"
}
]
},
"usage": {
- "input": 5425,
- "output": 120
+ "input": 5469,
+ "output": 117
}
},
- "40f9be0c8022103c35d156209c8f35be0861e84e": {
+ "ebcd1168f56b45b7402e8af39ab3532456911205": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in the customer details for the order",
- "copilotToolCallId": "toolu_vrtx_01WGqTc3Hcmd1FTasnBTmwMv"
+ "text": "Fill in all the required customer information fields",
+ "copilotToolCallId": "toolu_vrtx_011f5gY6a6DjTX4w5Ddzdeeh"
},
{
"type": "tool_call",
@@ -136,7 +136,7 @@
"name": "State/Province",
"type": "textbox",
"ref": "e25",
- "value": "XYZ"
+ "value": "XYZ state"
},
{
"name": "ZIP/Postal Code",
@@ -146,23 +146,23 @@
}
]
},
- "id": "toolu_vrtx_01WGqTc3Hcmd1FTasnBTmwMv"
+ "id": "toolu_vrtx_011f5gY6a6DjTX4w5Ddzdeeh"
}
]
},
"usage": {
- "input": 5694,
- "output": 344
+ "input": 5735,
+ "output": 345
}
},
- "3f143bb307a820583389dcdc308cc4507dd72618": {
+ "a56bfd7a019d224004464dbe721e3cc0038886d6": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the order for the blue table",
- "copilotToolCallId": "toolu_vrtx_01SLsAkw2y3eNVrRTPtAnfYx"
+ "text": "Click the Order button to submit the first item (blue table)",
+ "copilotToolCallId": "toolu_vrtx_013SEMdk1Hq4L5Co41uVazFk"
},
{
"type": "tool_call",
@@ -171,23 +171,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_01SLsAkw2y3eNVrRTPtAnfYx"
+ "id": "toolu_vrtx_013SEMdk1Hq4L5Co41uVazFk"
}
]
},
"usage": {
- "input": 6144,
- "output": 95
+ "input": 6187,
+ "output": 100
}
},
- "6e78e923857b521b4255a872588f577dee2d6795": {
+ "a238d85438b602deb2f96c474f8da1ba4829aba8": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Confirm the order for the blue table",
- "copilotToolCallId": "toolu_vrtx_01MESzajZqXLjdsQiKvVgUhF"
+ "text": "Accept the confirmation dialog for the blue table order",
+ "copilotToolCallId": "toolu_vrtx_018RwPsNMBXzSZ57LVQqCxrQ"
},
{
"type": "tool_call",
@@ -195,23 +195,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01MESzajZqXLjdsQiKvVgUhF"
+ "id": "toolu_vrtx_018RwPsNMBXzSZ57LVQqCxrQ"
}
]
},
"usage": {
- "input": 6314,
- "output": 79
+ "input": 6362,
+ "output": 80
}
},
- "1b5ccd39ae43f194ba622a7bb4bdf639e69ecec0": {
+ "e0b33dfe71582534d7ce4b55c37d7c649f42584a": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click the button to place the second order (green desk)",
- "copilotToolCallId": "toolu_vrtx_01UmUQhxxbPHM7UUXudex9Zg"
+ "text": "Click to place another order for the green desk",
+ "copilotToolCallId": "toolu_vrtx_018NyzMBhTPq4LisBkd4sQ4H"
},
{
"type": "tool_call",
@@ -220,23 +220,23 @@
"element": "Place Another Order button",
"ref": "e58"
},
- "id": "toolu_vrtx_01UmUQhxxbPHM7UUXudex9Zg"
+ "id": "toolu_vrtx_018NyzMBhTPq4LisBkd4sQ4H"
}
]
},
"usage": {
- "input": 7200,
- "output": 101
+ "input": 7252,
+ "output": 99
}
},
- "4b74f6fa7afaa89403a0d6a70838971b04447ce6": {
+ "cc3f1e930399f534ab8b20c7eb2835c64fba1734": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Desk' as the product type for the second order",
- "copilotToolCallId": "toolu_vrtx_01FRxMd3DbZstMP7cdMg3qNE"
+ "text": "Select Desk as the second product",
+ "copilotToolCallId": "toolu_vrtx_01Ltk3N21eQvg1KbBhmPRpFp"
},
{
"type": "tool_call",
@@ -248,23 +248,23 @@
"Desk"
]
},
- "id": "toolu_vrtx_01FRxMd3DbZstMP7cdMg3qNE"
+ "id": "toolu_vrtx_01Ltk3N21eQvg1KbBhmPRpFp"
}
]
},
"usage": {
- "input": 7921,
- "output": 125
+ "input": 7971,
+ "output": 118
}
},
- "0f35bef49760b5c9340ab795b04c5f4ccfb693af": {
+ "ab6e810564774b9fe4f7c834a4929ec7b7083836": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Green' as the color for the desk",
- "copilotToolCallId": "toolu_vrtx_01Ja8AwUYnJxFKNQbwvdTiQF"
+ "text": "Select Green as the color for the desk",
+ "copilotToolCallId": "toolu_vrtx_0176CAudgGEug4v6LMeDiGvm"
},
{
"type": "tool_call",
@@ -276,23 +276,23 @@
"Green"
]
},
- "id": "toolu_vrtx_01Ja8AwUYnJxFKNQbwvdTiQF"
+ "id": "toolu_vrtx_0176CAudgGEug4v6LMeDiGvm"
}
]
},
"usage": {
- "input": 8194,
- "output": 120
+ "input": 8237,
+ "output": 117
}
},
- "86eb224ddafacf54ef2fc05cf0db3893ef76e41e": {
+ "b8132fab3f5a79ff4d0b8c7167b0fc8bd1599a5c": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in the customer details for the second order",
- "copilotToolCallId": "toolu_vrtx_012mxpfp6HJGg7pcLMxFt5fv"
+ "text": "Fill in all the required customer information fields for the second order",
+ "copilotToolCallId": "toolu_vrtx_01ESimVVXfk6MxSe1yHmaywy"
},
{
"type": "tool_call",
@@ -327,7 +327,7 @@
"name": "State/Province",
"ref": "e25",
"type": "textbox",
- "value": "XYZ"
+ "value": "XYZ state"
},
{
"name": "ZIP/Postal Code",
@@ -337,23 +337,23 @@
}
]
},
- "id": "toolu_vrtx_012mxpfp6HJGg7pcLMxFt5fv"
+ "id": "toolu_vrtx_01ESimVVXfk6MxSe1yHmaywy"
}
]
},
"usage": {
- "input": 8463,
- "output": 265
+ "input": 8503,
+ "output": 269
}
},
- "d3c67880cd8b86dcc1311717192dbccd75f6f970": {
+ "b089efdaa78963cf4111a62cdcce00f6b0e37da4": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the order for the green desk",
- "copilotToolCallId": "toolu_vrtx_01E69zhhreSssUMqZWhKaaPa"
+ "text": "Click the Order button to submit the second item (green desk)",
+ "copilotToolCallId": "toolu_vrtx_01CCUVfDLfRxZQnmRHhoSRMw"
},
{
"type": "tool_call",
@@ -362,23 +362,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_01E69zhhreSssUMqZWhKaaPa"
+ "id": "toolu_vrtx_01CCUVfDLfRxZQnmRHhoSRMw"
}
]
},
"usage": {
- "input": 8914,
- "output": 95
+ "input": 8959,
+ "output": 100
}
},
- "c9719d6d50a2845be611b84f4882c960a639850b": {
+ "4ddf2575fb5623b5c9ccc345d065b0d6dd071681": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Confirm the order for the green desk",
- "copilotToolCallId": "toolu_vrtx_01XTAjsdAD5dNfef84Dq8pSS"
+ "text": "Accept the confirmation dialog for the green desk order",
+ "copilotToolCallId": "toolu_vrtx_01Wgbaa8QXuE76APMMV3f9nE"
},
{
"type": "tool_call",
@@ -386,23 +386,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01XTAjsdAD5dNfef84Dq8pSS"
+ "id": "toolu_vrtx_01Wgbaa8QXuE76APMMV3f9nE"
}
]
},
"usage": {
- "input": 9084,
- "output": 79
+ "input": 9134,
+ "output": 80
}
},
- "beae0a9e066cec2c0e6c14b1bab32ea9a85c0448": {
+ "a6c801faaf41a1b323f5d01ac5b7a95c54a4f29c": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Report the total price and maximum delivery days for both orders. Blue table: $353.98, 3 days. Green desk: $299.98, 3 days. Total: $653.96, max delivery: 3 days.",
- "copilotToolCallId": "toolu_vrtx_015FrLR8BJNkcxYXwgxkXkHk"
+ "text": "Report the combined results for both orders: blue table ($353.98) and green desk ($299.98), with a maximum delivery time of 3 days",
+ "copilotToolCallId": "toolu_vrtx_01Wn2WotjzPSUTWaYVu4Ymg8"
},
{
"type": "tool_call",
@@ -411,13 +411,13 @@
"price": 653.96,
"deliveryDays": 3
},
- "id": "toolu_vrtx_015FrLR8BJNkcxYXwgxkXkHk"
+ "id": "toolu_vrtx_01Wn2WotjzPSUTWaYVu4Ymg8"
}
]
},
"usage": {
- "input": 9971,
- "output": 143
+ "input": 10025,
+ "output": 124
}
}
}
\ No newline at end of file
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
index 2ffc395780..5f95f099ac 100644
--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
@@ -1,27 +1,27 @@
{
- "4c37062e059030f9c15a515aba7dea63145b1691": {
+ "1f67d34de9dca04e9ec4d9d7b590b48b6e67fd08": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll help you complete this task. Let me start by navigating to the page and then filling out the form to order the items."
+ "text": "I'll help you navigate to the form, fill in the order details, and report back the price and delivery information."
}
]
},
"usage": {
- "input": 4390,
- "output": 117
+ "input": 4429,
+ "output": 114
}
},
- "fad5bf0fc534ecb1e403acfc02b0040f502dde65": {
+ "498aeb8d624d9373c7e5b5b1dff78828566dc900": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Navigate to the form page to start ordering items",
- "copilotToolCallId": "toolu_vrtx_01LUFwA4XTZMZ1y1NHnRsMNx"
+ "text": "Navigate to the order form page",
+ "copilotToolCallId": "toolu_vrtx_01XPsGbYiofMbWmfgtdWBK7N"
},
{
"type": "tool_call",
@@ -29,107 +29,41 @@
"arguments": {
"url": "http://localhost:<PORT>/evals/fill-form.html"
},
- "id": "toolu_vrtx_01LUFwA4XTZMZ1y1NHnRsMNx"
+ "id": "toolu_vrtx_01XPsGbYiofMbWmfgtdWBK7N"
}
]
},
"usage": {
- "input": 4449,
- "output": 92
+ "input": 4484,
+ "output": 89
}
},
- "7fcc4660d74d7989887daa17f960a092cfa6d75d": {
+ "95fe071871583b1a5ddca2286d61680674dd9fd6": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Get current page state to see the form",
- "copilotToolCallId": "toolu_vrtx_01S1TPuFGdY3PpCwQ1MTs7iA"
- },
- {
- "type": "tool_call",
- "name": "browser_snapshot",
- "arguments": {},
- "id": "toolu_vrtx_01S1TPuFGdY3PpCwQ1MTs7iA"
- }
- ]
- },
- "usage": {
- "input": 5162,
- "output": 60
- }
- },
- "f7864b65b2ac8c7b1371d54672bc5cea4ac0ee3a": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "Select Table as the first product",
- "copilotToolCallId": "toolu_vrtx_01U7qs1Y5D27RMjy5vigdaXR"
- },
- {
- "type": "tool_call",
- "name": "browser_select_option",
- "arguments": {
- "element": "Product Type dropdown",
- "ref": "e7",
- "values": [
- "Table"
- ]
- },
- "id": "toolu_vrtx_01U7qs1Y5D27RMjy5vigdaXR"
- }
- ]
- },
- "usage": {
- "input": 5813,
- "output": 116
- }
- },
- "7c78931db5b30156a7dc1e6612d870e8187a20d6": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "Select Blue color for the table",
- "copilotToolCallId": "toolu_vrtx_01LXzd6tHpmw9uCZGrNc3TSi"
- },
- {
- "type": "tool_call",
- "name": "browser_select_option",
- "arguments": {
- "element": "Color dropdown",
- "ref": "e10",
- "values": [
- "Blue"
- ]
- },
- "id": "toolu_vrtx_01LXzd6tHpmw9uCZGrNc3TSi"
- }
- ]
- },
- "usage": {
- "input": 6076,
- "output": 115
- }
- },
- "cd4bd5106418a526955afe6d3f7c7f2b53007346": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "Fill in the customer information fields",
- "copilotToolCallId": "toolu_vrtx_01YWjwfRNj3KaBujienmqBhV"
+ "text": "Fill in the order form for a blue table with customer details",
+ "copilotToolCallId": "toolu_vrtx_01E9xJ27rpZVDGuRiEKuYgFe"
},
{
"type": "tool_call",
"name": "browser_fill_form",
"arguments": {
"fields": [
+ {
+ "name": "Product Type",
+ "type": "combobox",
+ "ref": "e7",
+ "value": "Table"
+ },
+ {
+ "name": "Color",
+ "type": "combobox",
+ "ref": "e10",
+ "value": "Blue"
+ },
{
"name": "Full Name",
"type": "textbox",
@@ -146,19 +80,7 @@
"name": "Street Address",
"type": "textbox",
"ref": "e19",
- "value": "123 Main St"
- },
- {
- "name": "City",
- "type": "textbox",
- "ref": "e22",
- "value": "Anytown"
- },
- {
- "name": "State/Province",
- "type": "textbox",
- "ref": "e25",
- "value": "XYZ state"
+ "value": "123 Main St, Anytown, XYZ state"
},
{
"name": "ZIP/Postal Code",
@@ -168,159 +90,174 @@
}
]
},
- "id": "toolu_vrtx_01YWjwfRNj3KaBujienmqBhV"
+ "id": "toolu_vrtx_01E9xJ27rpZVDGuRiEKuYgFe"
}
]
},
"usage": {
- "input": 6340,
- "output": 343
+ "input": 5194,
+ "output": 354
}
},
- "41b9db56c70fcd4f84f8d3de9de0c799dce8db88": {
+ "b33fa200303dc6e1c5e5d06e5566d11af9ee1100": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the first order for blue table",
- "copilotToolCallId": "toolu_vrtx_01D6FAeFSJeXh67RCcBh5JMB"
+ "text": "Check if City field needs to be filled before submitting",
+ "copilotToolCallId": "toolu_vrtx_01EyB4XTMFwAVCkJTVZegHBg"
},
{
"type": "tool_call",
- "name": "browser_click",
- "arguments": {
- "element": "Order button",
- "ref": "e29"
- },
- "id": "toolu_vrtx_01D6FAeFSJeXh67RCcBh5JMB"
+ "name": "browser_snapshot",
+ "arguments": {},
+ "id": "toolu_vrtx_01EyB4XTMFwAVCkJTVZegHBg"
}
]
},
"usage": {
- "input": 6790,
- "output": 95
+ "input": 5642,
+ "output": 63
}
},
- "e7da24b30926519b920dfff6d09d1f321ccb3ae7": {
+ "ddfd3471dba960dd013273a8ff6aa41c21a73ba0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Confirm the order for blue table",
- "copilotToolCallId": "toolu_vrtx_01TDuwkwXfeotUgRAu8fLUFD"
+ "text": "Fill in the missing City and State fields that are required",
+ "copilotToolCallId": "toolu_vrtx_01WQQgr577X5DKLNM66yaJvc"
},
{
"type": "tool_call",
- "name": "browser_handle_dialog",
+ "name": "browser_fill_form",
"arguments": {
- "accept": true
+ "fields": [
+ {
+ "name": "City",
+ "ref": "e22",
+ "type": "textbox",
+ "value": "Anytown"
+ },
+ {
+ "name": "State/Province",
+ "ref": "e25",
+ "type": "textbox",
+ "value": "XYZ"
+ }
+ ]
},
- "id": "toolu_vrtx_01TDuwkwXfeotUgRAu8fLUFD"
+ "id": "toolu_vrtx_01WQQgr577X5DKLNM66yaJvc"
}
]
},
"usage": {
- "input": 6960,
- "output": 78
+ "input": 6345,
+ "output": 142
}
},
- "b2200231c17c003417ced022a372a4b11edbf238": {
+ "6e1d42a5ece3458887bb317028560156de53d25d": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click to place another order for the green desk",
- "copilotToolCallId": "toolu_vrtx_014capLBfLNKY3rkD46m8bQ1"
+ "text": "Submit the order for the blue table",
+ "copilotToolCallId": "toolu_vrtx_01YQTMoAzSpRRw9J7kzMeAno"
},
{
"type": "tool_call",
"name": "browser_click",
"arguments": {
- "element": "Place Another Order button",
- "ref": "e58"
+ "element": "Order button",
+ "ref": "e29"
},
- "id": "toolu_vrtx_014capLBfLNKY3rkD46m8bQ1"
+ "id": "toolu_vrtx_01YQTMoAzSpRRw9J7kzMeAno"
}
]
},
"usage": {
- "input": 7848,
- "output": 99
+ "input": 6560,
+ "output": 95
}
},
- "3eace161b63651422689b72dbf4e606a489163e6": {
+ "3090a7d97bc75644ff00d8b32d7fdbcf26c55f6a": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Desk as the second product",
- "copilotToolCallId": "toolu_vrtx_01CY7bczyF9uwULtuDbsv5Bi"
+ "text": "Confirm the order for the blue table",
+ "copilotToolCallId": "toolu_vrtx_01LvLQ372U35LRjMhjzg6Dme"
},
{
"type": "tool_call",
- "name": "browser_select_option",
+ "name": "browser_handle_dialog",
"arguments": {
- "element": "Product Type dropdown",
- "ref": "e7",
- "values": [
- "Desk"
- ]
+ "accept": true
},
- "id": "toolu_vrtx_01CY7bczyF9uwULtuDbsv5Bi"
+ "id": "toolu_vrtx_01LvLQ372U35LRjMhjzg6Dme"
}
]
},
"usage": {
- "input": 8567,
- "output": 118
+ "input": 6730,
+ "output": 79
}
},
- "c59b631891d47364177fda353cfccb126dfbf570": {
+ "326f425b7b11f30fb2caef078ed93f7932923edc": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Green color for the desk",
- "copilotToolCallId": "toolu_vrtx_019fxgUwj5JiAJFrCzxGXPSK"
+ "text": "Click on Place Another Order button to order the second item (green desk)",
+ "copilotToolCallId": "toolu_vrtx_01LEuTE7DgrAUMint4khguYr"
},
{
"type": "tool_call",
- "name": "browser_select_option",
+ "name": "browser_click",
"arguments": {
- "element": "Color dropdown",
- "ref": "e10",
- "values": [
- "Green"
- ]
+ "element": "Place Another Order button",
+ "ref": "e58"
},
- "id": "toolu_vrtx_019fxgUwj5JiAJFrCzxGXPSK"
+ "id": "toolu_vrtx_01LEuTE7DgrAUMint4khguYr"
}
]
},
"usage": {
- "input": 8833,
- "output": 115
+ "input": 7641,
+ "output": 104
}
},
- "56e5cffa84ed757969f9abbbbe898dd6d0b2dbb6": {
+ "9128e6e3f63a0d1eaf2972336082bde3db8449ce": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in the customer information for the second order",
- "copilotToolCallId": "toolu_vrtx_016MMu8vjsBSdpwemMbNoXjV"
+ "text": "Fill in the order form for a green desk with the same customer details",
+ "copilotToolCallId": "toolu_vrtx_01SboLoRCoYELQtRzgagA1EC"
},
{
"type": "tool_call",
"name": "browser_fill_form",
"arguments": {
"fields": [
+ {
+ "name": "Product Type",
+ "ref": "e7",
+ "type": "combobox",
+ "value": "Desk"
+ },
+ {
+ "name": "Color",
+ "ref": "e10",
+ "type": "combobox",
+ "value": "Green"
+ },
{
"name": "Full Name",
"ref": "e13",
@@ -337,7 +274,7 @@
"name": "Street Address",
"ref": "e19",
"type": "textbox",
- "value": "123 Main St"
+ "value": "123 Main St, Anytown, XYZ state"
},
{
"name": "City",
@@ -349,7 +286,7 @@
"name": "State/Province",
"ref": "e25",
"type": "textbox",
- "value": "XYZ state"
+ "value": "XYZ"
},
{
"name": "ZIP/Postal Code",
@@ -359,23 +296,23 @@
}
]
},
- "id": "toolu_vrtx_016MMu8vjsBSdpwemMbNoXjV"
+ "id": "toolu_vrtx_01SboLoRCoYELQtRzgagA1EC"
}
]
},
"usage": {
- "input": 9097,
- "output": 266
+ "input": 8365,
+ "output": 337
}
},
- "d134b61bfaaea4e36415838a62db5cb13f888d20": {
+ "ccd12e74f5216b77104f1e2e6199519ef6bce9c0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the second order for green desk",
- "copilotToolCallId": "toolu_vrtx_01TgMV6W9eKdLpPjDDDewJJf"
+ "text": "Submit the order for the green desk",
+ "copilotToolCallId": "toolu_vrtx_01HsUFw6GuXR6J4YYiNnEHMe"
},
{
"type": "tool_call",
@@ -384,23 +321,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_01TgMV6W9eKdLpPjDDDewJJf"
+ "id": "toolu_vrtx_01HsUFw6GuXR6J4YYiNnEHMe"
}
]
},
"usage": {
- "input": 9550,
+ "input": 8931,
"output": 95
}
},
- "de09913d7981ad51cc18c9ad9ecdf7d1982bb858": {
+ "f3d9f4f90ffe9d6f68f0de098d16e50a7012bf6a": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Confirm the order for green desk",
- "copilotToolCallId": "toolu_vrtx_018ksE83x6ekovoZuomxmrsv"
+ "text": "Confirm the order for the green desk",
+ "copilotToolCallId": "toolu_vrtx_01EhMR6qGdFTfFJuC4GgTrzw"
},
{
"type": "tool_call",
@@ -408,23 +345,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_018ksE83x6ekovoZuomxmrsv"
+ "id": "toolu_vrtx_01EhMR6qGdFTfFJuC4GgTrzw"
}
]
},
"usage": {
- "input": 9720,
- "output": 78
+ "input": 9101,
+ "output": 79
}
},
- "987bfeb4500a519fde51250611923095075f6dd9": {
+ "53eabaad4c0c8ed9499bada78403a56fa3ab6307": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Report the total price for both items and maximum estimated delivery time",
- "copilotToolCallId": "toolu_vrtx_01JAZZ8bo7oMgM9FVCpBEvyf"
+ "text": "Report the total price for both items and maximum estimated delivery. Blue table: $353.98, Green desk: $299.98. Total: $653.96. Both orders have 3 days estimated delivery.",
+ "copilotToolCallId": "toolu_vrtx_01VsdeN1JruLTtnoA2hNUu7u"
},
{
"type": "tool_call",
@@ -433,13 +370,13 @@
"price": 653.96,
"deliveryDays": 3
},
- "id": "toolu_vrtx_01JAZZ8bo7oMgM9FVCpBEvyf"
+ "id": "toolu_vrtx_01VsdeN1JruLTtnoA2hNUu7u"
}
]
},
"usage": {
- "input": 10609,
- "output": 103
+ "input": 10013,
+ "output": 135
}
}
}
\ No newline at end of file
diff --git a/tests/mcp/console.spec.ts b/tests/mcp/console.spec.ts
index 4deec26097..c627c08a54 100644
--- a/tests/mcp/console.spec.ts
+++ b/tests/mcp/console.spec.ts
@@ -99,6 +99,65 @@ test('recent console messages', async ({ client, server }) => {
});
});
+test('recent console messages filter', async ({ startClient, server }) => {
+ server.setContent('/', `
+ <!DOCTYPE html>
+ <html>
+ <script>
+ console.log("console.log");
+ console.error("console.error");
+ </script>
+ </html>
+ `, 'text/html');
+
+ const { client } = await startClient({
+ args: ['--console-level', 'error'],
+ });
+
+ const response = parseResponse(await client.callTool({
+ name: 'browser_navigate',
+ arguments: {
+ url: server.PREFIX,
+ },
+ }));
+
+ expect(response.consoleMessages).toContain('console.error');
+ expect(response.consoleMessages).not.toContain('console.log');
+});
+
+test('browser_console_messages default level', async ({ client, server }) => {
+ await client.callTool({
+ name: 'browser_navigate',
+ arguments: {
+ url: server.HELLO_WORLD,
+ },
+ });
+
+ await client.callTool({
+ name: 'browser_evaluate',
+ arguments: {
+ function: `async () => {
+ console.debug("console.debug");
+ console.log("console.log");
+ console.warn("console.warn");
+ console.error("console.error");
+ setTimeout(() => { throw new Error("unhandled"); }, 0);
+ await fetch('/missing');
+ }`,
+ },
+ });
+
+ const response = parseResponse(await client.callTool({
+ name: 'browser_console_messages',
+ }));
+ expect.soft(response.result).toContain('console.log');
+ expect.soft(response.result).toContain('console.warn');
+ expect.soft(response.result).toContain('console.error');
+ expect.soft(response.result).toContain('Error: unhandled');
+ expect.soft(response.result).toContain('404');
+ expect.soft(response.result).not.toContain('console.debug');
+});
+
test('browser_console_messages errors only', async ({ client, server }) => {
await client.callTool({
name: 'browser_navigate',
@@ -111,6 +170,7 @@ test('browser_console_messages errors only', async ({ client, server }) => {
name: 'browser_evaluate',
arguments: {
function: `async () => {
+ console.debug("console.debug");
console.log("console.log");
console.warn("console.warn");
console.error("console.error");
@@ -123,7 +183,7 @@ test('browser_console_messages errors only', async ({ client, server }) => {
const response = parseResponse(await client.callTool({
name: 'browser_console_messages',
arguments: {
- onlyErrors: true,
+ level: 'error',
},
}));
expect.soft(response.result).toContain('console.error');