Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(mcp): allow passing --console-level (#38439)
Pavel Feldman408ee0fcfe6c
diff --git a/docs/src/api/class-consolemessage.md b/docs/src/api/class-consolemessage.mdindex b712bca267..044d070342 100644--- a/docs/src/api/class-consolemessage.md+++ b/docs/src/api/class-consolemessage.md@@ -140,7 +140,7 @@ The text of the console message.## method: ConsoleMessage.type* since: v1.8* langs: js, python-- returns: <[ConsoleMessageType]<"log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"timeEnd">>+- returns: <[ConsoleMessageType]<"log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"time"|"timeEnd">>## method: ConsoleMessage.type* since: v1.8diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.tsindex d6ff2f0ace..861d7a0f66 100644--- a/packages/playwright-client/types/types.d.ts+++ b/packages/playwright-client/types/types.d.ts@@ -18861,7 +18861,7 @@ export interface ConsoleMessage {*/text(): string;- type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"timeEnd";+ type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"time"|"timeEnd";/*** The web worker or service worker that produced this console message, if any. Note that console messages from webdiff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.tsindex d6ff2f0ace..861d7a0f66 100644--- a/packages/playwright-core/types/types.d.ts+++ b/packages/playwright-core/types/types.d.ts@@ -18861,7 +18861,7 @@ export interface ConsoleMessage {*/text(): string;- type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"timeEnd";+ type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"time"|"timeEnd";/*** The web worker or service worker that produced this console message, if any. Note that console messages from webdiff --git a/packages/playwright/src/mcp/browser/config.ts b/packages/playwright/src/mcp/browser/config.tsindex dc83efa9f0..1099f3d281 100644--- a/packages/playwright/src/mcp/browser/config.ts+++ b/packages/playwright/src/mcp/browser/config.ts@@ -39,6 +39,7 @@ export type CLIOptions = {cdpEndpoint?: string;cdpHeader?: Record<string, string>;config?: string;+ consoleLevel?: 'error' | 'warning' | 'info' | 'debug';device?: string;executablePath?: string;grantPermissions?: string[];@@ -81,6 +82,9 @@ export const defaultConfig: FullConfig = {viewport: null,},},+ console: {+ level: 'info',+ },network: {allowedOrigins: undefined,blockedOrigins: undefined,@@ -104,6 +108,9 @@ export type FullConfig = Config & {launchOptions: NonNullable<BrowserUserConfig['launchOptions']>;contextOptions: NonNullable<BrowserUserConfig['contextOptions']>;},+ console: {+ level: 'error' | 'warning' | 'info' | 'debug';+ },network: NonNullable<Config['network']>,saveTrace: boolean;server: NonNullable<Config['server']>,@@ -241,6 +248,9 @@ export function configFromCLIOptions(cliOptions: CLIOptions): Config {allowedHosts: cliOptions.allowedHosts,},capabilities: cliOptions.caps as ToolCapability[],+ console: {+ level: cliOptions.consoleLevel,+ },network: {allowedOrigins: cliOptions.allowedOrigins,blockedOrigins: cliOptions.blockedOrigins,@@ -274,6 +284,8 @@ function configFromEnv(): Config {options.cdpEndpoint = envToString(process.env.PLAYWRIGHT_MCP_CDP_ENDPOINT);options.cdpHeader = headerParser(process.env.PLAYWRIGHT_MCP_CDP_HEADERS, {});options.config = envToString(process.env.PLAYWRIGHT_MCP_CONFIG);+ if (process.env.PLAYWRIGHT_MCP_CONSOLE_LEVEL)+ options.consoleLevel = enumParser<'error' | 'warning' | 'info' | 'debug'>('--console-level', ['error', 'warning', 'info', 'debug'], process.env.PLAYWRIGHT_MCP_CONSOLE_LEVEL);options.device = envToString(process.env.PLAYWRIGHT_MCP_DEVICE);options.executablePath = envToString(process.env.PLAYWRIGHT_MCP_EXECUTABLE_PATH);options.grantPermissions = commaSeparatedList(process.env.PLAYWRIGHT_MCP_GRANT_PERMISSIONS);@@ -287,8 +299,8 @@ function configFromEnv(): Config {if (initScript)options.initScript = [initScript];options.isolated = envToBoolean(process.env.PLAYWRIGHT_MCP_ISOLATED);- if (process.env.PLAYWRIGHT_MCP_IMAGE_RESPONSES === 'omit')- options.imageResponses = 'omit';+ if (process.env.PLAYWRIGHT_MCP_IMAGE_RESPONSES)+ options.imageResponses = enumParser<'allow' | 'omit'>('--image-responses', ['allow', 'omit'], process.env.PLAYWRIGHT_MCP_IMAGE_RESPONSES);options.sandbox = envToBoolean(process.env.PLAYWRIGHT_MCP_SANDBOX);options.outputDir = envToString(process.env.PLAYWRIGHT_MCP_OUTPUT_DIR);options.port = numberParser(process.env.PLAYWRIGHT_MCP_PORT);@@ -385,6 +397,10 @@ function mergeConfig(base: FullConfig, overrides: Config): FullConfig {...pickDefined(base),...pickDefined(overrides),browser,+ console: {+ ...pickDefined(base.console),+ ...pickDefined(overrides.console),+ },network: {...pickDefined(base.network),...pickDefined(overrides.network),@@ -458,6 +474,12 @@ export function headerParser(arg: string | undefined, previous?: Record<string,return result;}+export function enumParser<T extends string>(name: string, options: T[], value: string): T {+ if (!options.includes(value as T))+ throw new Error(`Invalid ${name}: ${value}. Valid values are: ${options.join(', ')}`);+ return value as T;+}+function envToBoolean(value: string | undefined): boolean | undefined {if (value === 'true' || value === '1')return true;diff --git a/packages/playwright/src/mcp/browser/tab.ts b/packages/playwright/src/mcp/browser/tab.tsindex d254b115bf..9563bf7d29 100644--- a/packages/playwright/src/mcp/browser/tab.ts+++ b/packages/playwright/src/mcp/browser/tab.ts@@ -217,9 +217,9 @@ export class Tab extends EventEmitter<TabEventsInterface> {await this.waitForLoadState('load', { timeout: 5000 });}- async consoleMessages(type?: 'error'): Promise<ConsoleMessage[]> {+ async consoleMessages(level: ConsoleMessageLevel): Promise<ConsoleMessage[]> {await this._initializedPromise;- return this._consoleMessages.filter(message => type ? message.type === type : true);+ return this._consoleMessages.filter(message => shouldIncludeMessage(level, message.type));}async requests(): Promise<Set<playwright.Request>> {@@ -244,7 +244,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {});if (tabSnapshot) {// Assign console message late so that we did not lose any to modal state.- tabSnapshot.consoleMessages = this._recentConsoleMessages;+ tabSnapshot.consoleMessages = this._recentConsoleMessages.filter(message => shouldIncludeMessage(this.context.config.console.level, message.type));this._recentConsoleMessages = [];}// If we failed to capture a snapshot this time, make sure we do a full one next time,@@ -317,7 +317,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {}export type ConsoleMessage = {- type: ReturnType<playwright.ConsoleMessage['type']> | undefined;+ type: ReturnType<playwright.ConsoleMessage['type']>;text: string;toString(): string;};@@ -354,4 +354,43 @@ export function renderModalStates(modalStates: ModalState[]): string[] {return result;}+type ConsoleMessageType = ReturnType<playwright.ConsoleMessage['type']>;+type ConsoleMessageLevel = 'error' | 'warning' | 'info' | 'debug';+const consoleMessageLevels: ConsoleMessageLevel[] = ['error', 'warning', 'info', 'debug'];++function shouldIncludeMessage(thresholdLevel: ConsoleMessageLevel, type: ConsoleMessageType): boolean {+ const messageLevel = consoleLevelForMessageType(type);+ return consoleMessageLevels.indexOf(messageLevel) <= consoleMessageLevels.indexOf(thresholdLevel);+}++function consoleLevelForMessageType(type: ConsoleMessageType): ConsoleMessageLevel {+ switch (type) {+ case 'assert':+ case 'error':+ return 'error';+ case 'warning':+ return 'warning';+ case 'count':+ case 'dir':+ case 'dirxml':+ case 'info':+ case 'log':+ case 'table':+ case 'time':+ case 'timeEnd':+ return 'info';+ case 'clear':+ case 'debug':+ case 'endGroup':+ case 'profile':+ case 'profileEnd':+ case 'startGroup':+ case 'startGroupCollapsed':+ case 'trace':+ return 'debug';+ default:+ return 'info';+ }+}+const tabSymbol = Symbol('tabSymbol');diff --git a/packages/playwright/src/mcp/browser/tools/console.ts b/packages/playwright/src/mcp/browser/tools/console.tsindex b9bda093ac..6643da3177 100644--- a/packages/playwright/src/mcp/browser/tools/console.ts+++ b/packages/playwright/src/mcp/browser/tools/console.ts@@ -24,12 +24,12 @@ const console = defineTabTool({title: 'Get console messages',description: 'Returns all console messages',inputSchema: z.object({- onlyErrors: z.boolean().optional().describe('Only return error messages'),+ level: z.enum(['error', 'warning', 'info', 'debug']).default('info').describe('Level of the console messages to return. Each level includes the messages of more severe levels. Defaults to "info".'),}),type: 'readOnly',},handle: async (tab, params, response) => {- const messages = await tab.consoleMessages(params.onlyErrors ? 'error' : undefined);+ const messages = await tab.consoleMessages(params.level);messages.map(message => response.addResult(message.toString()));},});diff --git a/packages/playwright/src/mcp/config.d.ts b/packages/playwright/src/mcp/config.d.tsindex 9a04795ccf..1f10ecdf34 100644--- a/packages/playwright/src/mcp/config.d.ts+++ b/packages/playwright/src/mcp/config.d.ts@@ -142,6 +142,13 @@ export type Config = {*/outputDir?: string;+ console?: {+ /**+ * The level of console messages to return. Each level includes the messages of more severe levels. Defaults to "info".+ */+ level?: 'error' | 'warning' | 'info' | 'debug';+ },+network?: {/*** List of origins to allow the browser to request. Default is to allow all. Origins matching both `allowedOrigins` and `blockedOrigins` will be blocked.diff --git a/packages/playwright/src/mcp/program.ts b/packages/playwright/src/mcp/program.tsindex 4635b7e1b3..1c0389dcfb 100644--- a/packages/playwright/src/mcp/program.ts+++ b/packages/playwright/src/mcp/program.ts@@ -22,7 +22,7 @@ import { colors, ProgramOption } from 'playwright-core/lib/utilsBundle';import { registry } from 'playwright-core/lib/server';import * as mcpServer from './sdk/server';-import { commaSeparatedList, dotenvFileLoader, headerParser, numberParser, resolutionParser, resolveCLIConfig, semicolonSeparatedList } from './browser/config';+import { commaSeparatedList, dotenvFileLoader, enumParser, headerParser, numberParser, resolutionParser, resolveCLIConfig, semicolonSeparatedList } from './browser/config';import { setupExitWatchdog } from './browser/watchdog';import { contextFactory } from './browser/browserContextFactory';import { ProxyBackend } from './sdk/proxyBackend';@@ -43,6 +43,7 @@ export function decorateCommand(command: Command, version: string) {.option('--cdp-endpoint <endpoint>', 'CDP endpoint to connect to.').option('--cdp-header <headers...>', 'CDP headers to send with the connect request, multiple can be specified.', headerParser).option('--config <path>', 'path to the configuration file.')+ .option('--console-level <level>', 'level of console messages to return: "error", "warning", "info", "debug". Each level includes the messages of more severe levels.', enumParser.bind(null, '--console-level', ['error', 'warning', 'info', 'debug'])).option('--device <device>', 'device to emulate, for example: "iPhone 15"').option('--executable-path <path>', 'path to the browser executable.').option('--extension', 'Connect to a running browser instance (Edge/Chrome only). Requires the "Playwright MCP Bridge" browser extension to be installed.')@@ -53,7 +54,7 @@ export function decorateCommand(command: Command, version: string) {.option('--init-page <path...>', 'path to TypeScript file to evaluate on Playwright page object').option('--init-script <path...>', 'path to JavaScript file to add as an initialization script. The script will be evaluated in every page before any of the page\'s scripts. Can be specified multiple times.').option('--isolated', 'keep the browser profile in memory, do not save it to disk.')- .option('--image-responses <mode>', 'whether to send image responses to the client. Can be "allow" or "omit", Defaults to "allow".')+ .option('--image-responses <mode>', 'whether to send image responses to the client. Can be "allow" or "omit", Defaults to "allow".', enumParser.bind(null, '--image-responses', ['allow', 'omit'])).option('--no-sandbox', 'disable the sandbox for all process types that are normally sandboxed.').option('--output-dir <path>', 'path to the directory for output files.').option('--port <port>', 'port to listen on for SSE transport.')diff --git a/packages/playwright/src/mcp/test/browserBackend.ts b/packages/playwright/src/mcp/test/browserBackend.tsindex 12e8927ecd..7bed39f6bb 100644--- a/packages/playwright/src/mcp/test/browserBackend.ts+++ b/packages/playwright/src/mcp/test/browserBackend.ts@@ -95,7 +95,7 @@ async function generatePausedMessage(testInfo: TestInfo, context: playwright.Bro);// Only print console errors when pausing on error, not when everything works as expected.let console = testInfo.errors.length ? await Tab.collectConsoleMessages(page) : [];- console = console.filter(msg => !msg.type || msg.type === 'error');+ console = console.filter(msg => msg.type === 'error');if (console.length) {lines.push('- Console Messages:');for (const message of console)diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.jsonindex 16c1d734a6..69e8c7ef74 100644--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json@@ -1,27 +1,27 @@{- "4c37062e059030f9c15a515aba7dea63145b1691": {+ "1f67d34de9dca04e9ec4d9d7b590b48b6e67fd08": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I'll help you navigate to the website, fill out the form to order a blue table and a green desk, and report back the total price and delivery time."+ "text": "I'll help you navigate to the form, fill it out with the specified details, and report back the pricing and delivery information."}]},"usage": {- "input": 4390,- "output": 127+ "input": 4429,+ "output": 120}},- "b09d261846c63220d3abdcee433a5987bce04d5f": {+ "4498c58367500860e392c029b7f90d23ca898b50": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Navigate to the form page to order items",- "copilotToolCallId": "toolu_vrtx_01Nrr85ePXvh2wqRS6S47Lej"+ "text": "Navigate to the form page to start the ordering process",+ "copilotToolCallId": "toolu_vrtx_0129qd3zaawZfjyTyiNQ8dbY"},{"type": "tool_call",@@ -29,23 +29,45 @@"arguments": {"url": "http://localhost:<PORT>/evals/fill-form.html"},- "id": "toolu_vrtx_01Nrr85ePXvh2wqRS6S47Lej"+ "id": "toolu_vrtx_0129qd3zaawZfjyTyiNQ8dbY"+ }+ ]+ },+ "usage": {+ "input": 4486,+ "output": 93+ }+ },+ "1f4cb362bd2e2d63916d74077a8cd478a0f0b1a5": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Get current page state to identify form fields for the first item (blue table)",+ "copilotToolCallId": "toolu_vrtx_01Sqo3h7zZGVZYkVDWBrmhfg"+ },+ {+ "type": "tool_call",+ "name": "browser_snapshot",+ "arguments": {},+ "id": "toolu_vrtx_01Sqo3h7zZGVZYkVDWBrmhfg"}]},"usage": {- "input": 4454,- "output": 91+ "input": 5200,+ "output": 67}},- "83da5a4a79d93a494d7f0f76d4758bc493d411f5": {+ "54801f22b272e5a430fd56cc07ab4b11d87c6514": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select 'Table' as the product type for the first item",- "copilotToolCallId": "toolu_vrtx_011Vy4WSELJYN4U2tv9NgCLi"+ "text": "Select Table as the product type for the first order",+ "copilotToolCallId": "toolu_vrtx_01KZtfPrpLh9Pg7D8hzZwfw5"},{"type": "tool_call",@@ -57,23 +79,23 @@"Table"]},- "id": "toolu_vrtx_011Vy4WSELJYN4U2tv9NgCLi"+ "id": "toolu_vrtx_01KZtfPrpLh9Pg7D8hzZwfw5"}]},"usage": {- "input": 5166,- "output": 123+ "input": 5858,+ "output": 120}},- "0a13ca0491d6745366d7b71a32b864412e848d78": {+ "8585f97b99c7165ca864db490da286f768b8b883": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select 'Blue' as the color for the table",- "copilotToolCallId": "toolu_vrtx_01AyhQJov7bZQ7dY9CmcZMeY"+ "text": "Select Blue as the color for the table",+ "copilotToolCallId": "toolu_vrtx_01XGcSSgUwBCcgqLu11QqxRQ"},{"type": "tool_call",@@ -85,23 +107,23 @@"Blue"]},- "id": "toolu_vrtx_01AyhQJov7bZQ7dY9CmcZMeY"+ "id": "toolu_vrtx_01XGcSSgUwBCcgqLu11QqxRQ"}]},"usage": {- "input": 5436,- "output": 120+ "input": 6125,+ "output": 117}},- "0e1345afd5cefb63d40c0d9750aa6b73dfe36f38": {+ "d78aa88a096ad38aea37e1b0b73aefd529845ab8": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Fill in the customer details for the order",- "copilotToolCallId": "toolu_vrtx_011xPGae2W6mMbh3TXpbqjaH"+ "text": "Fill in customer information for the order",+ "copilotToolCallId": "toolu_vrtx_01Vk16c9EpV35aGtSjn2J6Pk"},{"type": "tool_call",@@ -124,7 +146,7 @@"name": "Street Address","type": "textbox","ref": "e19",- "value": "123 Main St, Anytown"+ "value": "123 Main St"},{"name": "City",@@ -146,23 +168,23 @@}]},- "id": "toolu_vrtx_011xPGae2W6mMbh3TXpbqjaH"+ "id": "toolu_vrtx_01Vk16c9EpV35aGtSjn2J6Pk"}]},"usage": {- "input": 5705,- "output": 348+ "input": 6391,+ "output": 343}},- "90b95b16881abda623068841009641e51f32e7ae": {+ "1380ec99c1a8290d52dd9d4bb008bc5add1efb54": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Click the Order button to submit the first order (blue table)",- "copilotToolCallId": "toolu_vrtx_01FwpyQSshn5YP7JyJbptXEh"+ "text": "Submit the order for the blue table",+ "copilotToolCallId": "toolu_vrtx_011CLYRgGn3zonGBT97Wxr4r"},{"type": "tool_call",@@ -171,23 +193,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_01FwpyQSshn5YP7JyJbptXEh"+ "id": "toolu_vrtx_011CLYRgGn3zonGBT97Wxr4r"}]},"usage": {- "input": 6163,- "output": 100+ "input": 6840,+ "output": 95}},- "7160f78eeba943a8bf0daac205c6b183839401ca": {+ "6ae203915486aa37b263ad3ad8af6af4a0c9e9f6": {"result": {"role": "assistant","content": [{"type": "text","text": "Accept the confirmation dialog for the blue table order",- "copilotToolCallId": "toolu_vrtx_01LW5t6fGbyzj4ARj6wATf6y"+ "copilotToolCallId": "toolu_vrtx_01BVGW6UrHvCKQ7v5xNPv1xv"},{"type": "tool_call",@@ -195,23 +217,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_01LW5t6fGbyzj4ARj6wATf6y"+ "id": "toolu_vrtx_01BVGW6UrHvCKQ7v5xNPv1xv"}]},"usage": {- "input": 6338,+ "input": 7010,"output": 80}},- "80f4d8e16d86fb67f34dae4668d2ac84c71ac3ee": {+ "9ec58f784b989e7cbbbde0656d22e6be1c5075dd": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Click 'Place Another Order' button to order the second item (green desk)",- "copilotToolCallId": "toolu_vrtx_01Y6z1hSQqTdsRs6VhoAkNrJ"+ "text": "Click the button to place another order for the green desk",+ "copilotToolCallId": "toolu_vrtx_016rQY2JhLzVu3TpokRVqdor"},{"type": "tool_call",@@ -220,23 +242,23 @@"element": "Place Another Order button","ref": "e58"},- "id": "toolu_vrtx_01Y6z1hSQqTdsRs6VhoAkNrJ"+ "id": "toolu_vrtx_016rQY2JhLzVu3TpokRVqdor"}]},"usage": {- "input": 7236,- "output": 106+ "input": 7897,+ "output": 101}},- "8a7a2b461872a308c3c471069553fbb110b233e2": {+ "83a5695faf3e865ff1654007db08a088bc2b9412": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select 'Desk' as the product type for the second item",- "copilotToolCallId": "toolu_vrtx_01XFkp8Sb3zfxfSX56VB5Ukp"+ "text": "Select Desk as the product type for the second order",+ "copilotToolCallId": "toolu_vrtx_01WnDXifmCJWtEMduSaUjHU8"},{"type": "tool_call",@@ -248,23 +270,23 @@"Desk"]},- "id": "toolu_vrtx_01XFkp8Sb3zfxfSX56VB5Ukp"+ "id": "toolu_vrtx_01WnDXifmCJWtEMduSaUjHU8"}]},"usage": {- "input": 7962,- "output": 125+ "input": 8618,+ "output": 122}},- "aa6c7116bcda3f6079c1d67f86e781f745b9511f": {+ "117e2c52b10e5288047ef65ac10a59c9e2275674": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select 'Green' as the color for the desk",- "copilotToolCallId": "toolu_vrtx_01BBnzzEU2bFJdGS8XeWbKbU"+ "text": "Select Green as the color for the desk",+ "copilotToolCallId": "toolu_vrtx_01462K6LBMFTWaVWPmegByzb"},{"type": "tool_call",@@ -276,23 +298,23 @@"Green"]},- "id": "toolu_vrtx_01BBnzzEU2bFJdGS8XeWbKbU"+ "id": "toolu_vrtx_01462K6LBMFTWaVWPmegByzb"}]},"usage": {- "input": 8235,- "output": 120+ "input": 8888,+ "output": 117}},- "a30b487338609e8a5d8417c1f90419956bb4b727": {+ "ad76fc7dda0725834872ca9b8f4395722197026a": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Fill in the customer details for the second order",- "copilotToolCallId": "toolu_vrtx_01W1QKYyFZvMCWhC3DwYQv2X"+ "text": "Fill in customer information for the second order",+ "copilotToolCallId": "toolu_vrtx_01Bonk7ZwhuGS7yc67tAqJMd"},{"type": "tool_call",@@ -315,7 +337,7 @@"name": "Street Address","ref": "e19","type": "textbox",- "value": "123 Main St, Anytown"+ "value": "123 Main St"},{"name": "City",@@ -337,23 +359,23 @@}]},- "id": "toolu_vrtx_01W1QKYyFZvMCWhC3DwYQv2X"+ "id": "toolu_vrtx_01Bonk7ZwhuGS7yc67tAqJMd"}]},"usage": {- "input": 8504,- "output": 269+ "input": 9154,+ "output": 264}},- "7269d27383720010c65e7e530cf66cff736cb587": {+ "2dd639bc23dfba249c89bacb6f09a6219538a99d": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Click the Order button to submit the second order (green desk)",- "copilotToolCallId": "toolu_vrtx_019EvisGfKGAHWu4Ehk6orM9"+ "text": "Submit the order for the green desk",+ "copilotToolCallId": "toolu_vrtx_01B3Hfa3jbKn1278dRq9izUB"},{"type": "tool_call",@@ -362,23 +384,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_019EvisGfKGAHWu4Ehk6orM9"+ "id": "toolu_vrtx_01B3Hfa3jbKn1278dRq9izUB"}]},"usage": {- "input": 8963,- "output": 100+ "input": 9604,+ "output": 95}},- "28e8e2a0869e245c707a879c41bf7811648233ff": {+ "df594f27d4f66356f38861d693b8f14a66ef0af4": {"result": {"role": "assistant","content": [{"type": "text","text": "Accept the confirmation dialog for the green desk order",- "copilotToolCallId": "toolu_vrtx_01MawbHWCqo6WBwPh8YcEBm4"+ "copilotToolCallId": "toolu_vrtx_01HPLerVExZZZ15M7UeJsh3X"},{"type": "tool_call",@@ -386,23 +408,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_01MawbHWCqo6WBwPh8YcEBm4"+ "id": "toolu_vrtx_01HPLerVExZZZ15M7UeJsh3X"}]},"usage": {- "input": 9138,+ "input": 9774,"output": 80}},- "cc4e5987be7c007db351125c487c40e5413aebf1": {+ "a6b8bcc69f1ddb715921d1814326fea5e5a96f22": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Report the total price and maximum delivery time for both orders",- "copilotToolCallId": "toolu_vrtx_01DahshwKgzha389UJ8rq29Z"+ "text": "Report the total price and maximum delivery time for both orders - blue table ($353.98) and green desk ($299.98)",+ "copilotToolCallId": "toolu_vrtx_019b25HJgtNdXAz3QjmCnHYw"},{"type": "tool_call",@@ -411,13 +433,13 @@"price": 653.96,"deliveryDays": 3},- "id": "toolu_vrtx_01DahshwKgzha389UJ8rq29Z"+ "id": "toolu_vrtx_019b25HJgtNdXAz3QjmCnHYw"}]},"usage": {- "input": 10037,- "output": 102+ "input": 10662,+ "output": 117}}}\ No newline at end of filediff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.jsonindex 68b9b1d7cf..67864d8c09 100644--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json@@ -1,27 +1,27 @@{- "4c37062e059030f9c15a515aba7dea63145b1691": {+ "1f67d34de9dca04e9ec4d9d7b590b48b6e67fd08": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I'll help you complete this task. Let me start by navigating to the page and then filling out the order form."+ "text": "I'll help you navigate to the form, fill it out, and order the items. Let me start by navigating to the URL and taking a snapshot to see the page structure."}]},"usage": {- "input": 4390,- "output": 114+ "input": 4429,+ "output": 131}},- "e1e0bd5a8606f1711cc30f461f0c274f47c58792": {+ "eb602996f263d0f29d50e40769506f6f47394046": {"result": {"role": "assistant","content": [{"type": "text","text": "Navigate to the form page",- "copilotToolCallId": "toolu_vrtx_01CyPTAyQwbp3tJQ8HRHcZpU"+ "copilotToolCallId": "toolu_vrtx_016zERgz28Qc3mWdBLi5V2Xp"},{"type": "tool_call",@@ -29,23 +29,23 @@"arguments": {"url": "http://localhost:<PORT>/evals/fill-form.html"},- "id": "toolu_vrtx_01CyPTAyQwbp3tJQ8HRHcZpU"+ "id": "toolu_vrtx_016zERgz28Qc3mWdBLi5V2Xp"}]},"usage": {- "input": 4446,+ "input": 4497,"output": 88}},- "7570cf20826150ea1678925d3c56cdaf6cc0ce0d": {+ "ae111fb9371c47d04762f81458d5557be0ca760c": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select 'Table' as the product type for the first order",- "copilotToolCallId": "toolu_vrtx_01R9atbVxke4YwYTPtDeMwuK"+ "text": "Select Table as the first product",+ "copilotToolCallId": "toolu_vrtx_017WjVw9bpcfDw5892VkuT76"},{"type": "tool_call",@@ -57,23 +57,23 @@"Table"]},- "id": "toolu_vrtx_01R9atbVxke4YwYTPtDeMwuK"+ "id": "toolu_vrtx_017WjVw9bpcfDw5892VkuT76"}]},"usage": {- "input": 5155,- "output": 123+ "input": 5206,+ "output": 116}},- "1e9bce6dd706be6d7272ab962111b4e7eb5730d7": {+ "08407b7115ec34eea2a164355fe32b570e965d41": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select 'Blue' as the color for the table",- "copilotToolCallId": "toolu_vrtx_012uA5vSCDJPndFMhFP9z6AT"+ "text": "Select Blue as the color for the table",+ "copilotToolCallId": "toolu_vrtx_01S7UqYcgtqNzZS4zrHwWHpu"},{"type": "tool_call",@@ -85,23 +85,23 @@"Blue"]},- "id": "toolu_vrtx_012uA5vSCDJPndFMhFP9z6AT"+ "id": "toolu_vrtx_01S7UqYcgtqNzZS4zrHwWHpu"}]},"usage": {- "input": 5425,- "output": 120+ "input": 5469,+ "output": 117}},- "40f9be0c8022103c35d156209c8f35be0861e84e": {+ "ebcd1168f56b45b7402e8af39ab3532456911205": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Fill in the customer details for the order",- "copilotToolCallId": "toolu_vrtx_01WGqTc3Hcmd1FTasnBTmwMv"+ "text": "Fill in all the required customer information fields",+ "copilotToolCallId": "toolu_vrtx_011f5gY6a6DjTX4w5Ddzdeeh"},{"type": "tool_call",@@ -136,7 +136,7 @@"name": "State/Province","type": "textbox","ref": "e25",- "value": "XYZ"+ "value": "XYZ state"},{"name": "ZIP/Postal Code",@@ -146,23 +146,23 @@}]},- "id": "toolu_vrtx_01WGqTc3Hcmd1FTasnBTmwMv"+ "id": "toolu_vrtx_011f5gY6a6DjTX4w5Ddzdeeh"}]},"usage": {- "input": 5694,- "output": 344+ "input": 5735,+ "output": 345}},- "3f143bb307a820583389dcdc308cc4507dd72618": {+ "a56bfd7a019d224004464dbe721e3cc0038886d6": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Submit the order for the blue table",- "copilotToolCallId": "toolu_vrtx_01SLsAkw2y3eNVrRTPtAnfYx"+ "text": "Click the Order button to submit the first item (blue table)",+ "copilotToolCallId": "toolu_vrtx_013SEMdk1Hq4L5Co41uVazFk"},{"type": "tool_call",@@ -171,23 +171,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_01SLsAkw2y3eNVrRTPtAnfYx"+ "id": "toolu_vrtx_013SEMdk1Hq4L5Co41uVazFk"}]},"usage": {- "input": 6144,- "output": 95+ "input": 6187,+ "output": 100}},- "6e78e923857b521b4255a872588f577dee2d6795": {+ "a238d85438b602deb2f96c474f8da1ba4829aba8": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Confirm the order for the blue table",- "copilotToolCallId": "toolu_vrtx_01MESzajZqXLjdsQiKvVgUhF"+ "text": "Accept the confirmation dialog for the blue table order",+ "copilotToolCallId": "toolu_vrtx_018RwPsNMBXzSZ57LVQqCxrQ"},{"type": "tool_call",@@ -195,23 +195,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_01MESzajZqXLjdsQiKvVgUhF"+ "id": "toolu_vrtx_018RwPsNMBXzSZ57LVQqCxrQ"}]},"usage": {- "input": 6314,- "output": 79+ "input": 6362,+ "output": 80}},- "1b5ccd39ae43f194ba622a7bb4bdf639e69ecec0": {+ "e0b33dfe71582534d7ce4b55c37d7c649f42584a": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Click the button to place the second order (green desk)",- "copilotToolCallId": "toolu_vrtx_01UmUQhxxbPHM7UUXudex9Zg"+ "text": "Click to place another order for the green desk",+ "copilotToolCallId": "toolu_vrtx_018NyzMBhTPq4LisBkd4sQ4H"},{"type": "tool_call",@@ -220,23 +220,23 @@"element": "Place Another Order button","ref": "e58"},- "id": "toolu_vrtx_01UmUQhxxbPHM7UUXudex9Zg"+ "id": "toolu_vrtx_018NyzMBhTPq4LisBkd4sQ4H"}]},"usage": {- "input": 7200,- "output": 101+ "input": 7252,+ "output": 99}},- "4b74f6fa7afaa89403a0d6a70838971b04447ce6": {+ "cc3f1e930399f534ab8b20c7eb2835c64fba1734": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select 'Desk' as the product type for the second order",- "copilotToolCallId": "toolu_vrtx_01FRxMd3DbZstMP7cdMg3qNE"+ "text": "Select Desk as the second product",+ "copilotToolCallId": "toolu_vrtx_01Ltk3N21eQvg1KbBhmPRpFp"},{"type": "tool_call",@@ -248,23 +248,23 @@"Desk"]},- "id": "toolu_vrtx_01FRxMd3DbZstMP7cdMg3qNE"+ "id": "toolu_vrtx_01Ltk3N21eQvg1KbBhmPRpFp"}]},"usage": {- "input": 7921,- "output": 125+ "input": 7971,+ "output": 118}},- "0f35bef49760b5c9340ab795b04c5f4ccfb693af": {+ "ab6e810564774b9fe4f7c834a4929ec7b7083836": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select 'Green' as the color for the desk",- "copilotToolCallId": "toolu_vrtx_01Ja8AwUYnJxFKNQbwvdTiQF"+ "text": "Select Green as the color for the desk",+ "copilotToolCallId": "toolu_vrtx_0176CAudgGEug4v6LMeDiGvm"},{"type": "tool_call",@@ -276,23 +276,23 @@"Green"]},- "id": "toolu_vrtx_01Ja8AwUYnJxFKNQbwvdTiQF"+ "id": "toolu_vrtx_0176CAudgGEug4v6LMeDiGvm"}]},"usage": {- "input": 8194,- "output": 120+ "input": 8237,+ "output": 117}},- "86eb224ddafacf54ef2fc05cf0db3893ef76e41e": {+ "b8132fab3f5a79ff4d0b8c7167b0fc8bd1599a5c": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Fill in the customer details for the second order",- "copilotToolCallId": "toolu_vrtx_012mxpfp6HJGg7pcLMxFt5fv"+ "text": "Fill in all the required customer information fields for the second order",+ "copilotToolCallId": "toolu_vrtx_01ESimVVXfk6MxSe1yHmaywy"},{"type": "tool_call",@@ -327,7 +327,7 @@"name": "State/Province","ref": "e25","type": "textbox",- "value": "XYZ"+ "value": "XYZ state"},{"name": "ZIP/Postal Code",@@ -337,23 +337,23 @@}]},- "id": "toolu_vrtx_012mxpfp6HJGg7pcLMxFt5fv"+ "id": "toolu_vrtx_01ESimVVXfk6MxSe1yHmaywy"}]},"usage": {- "input": 8463,- "output": 265+ "input": 8503,+ "output": 269}},- "d3c67880cd8b86dcc1311717192dbccd75f6f970": {+ "b089efdaa78963cf4111a62cdcce00f6b0e37da4": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Submit the order for the green desk",- "copilotToolCallId": "toolu_vrtx_01E69zhhreSssUMqZWhKaaPa"+ "text": "Click the Order button to submit the second item (green desk)",+ "copilotToolCallId": "toolu_vrtx_01CCUVfDLfRxZQnmRHhoSRMw"},{"type": "tool_call",@@ -362,23 +362,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_01E69zhhreSssUMqZWhKaaPa"+ "id": "toolu_vrtx_01CCUVfDLfRxZQnmRHhoSRMw"}]},"usage": {- "input": 8914,- "output": 95+ "input": 8959,+ "output": 100}},- "c9719d6d50a2845be611b84f4882c960a639850b": {+ "4ddf2575fb5623b5c9ccc345d065b0d6dd071681": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Confirm the order for the green desk",- "copilotToolCallId": "toolu_vrtx_01XTAjsdAD5dNfef84Dq8pSS"+ "text": "Accept the confirmation dialog for the green desk order",+ "copilotToolCallId": "toolu_vrtx_01Wgbaa8QXuE76APMMV3f9nE"},{"type": "tool_call",@@ -386,23 +386,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_01XTAjsdAD5dNfef84Dq8pSS"+ "id": "toolu_vrtx_01Wgbaa8QXuE76APMMV3f9nE"}]},"usage": {- "input": 9084,- "output": 79+ "input": 9134,+ "output": 80}},- "beae0a9e066cec2c0e6c14b1bab32ea9a85c0448": {+ "a6c801faaf41a1b323f5d01ac5b7a95c54a4f29c": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Report the total price and maximum delivery days for both orders. Blue table: $353.98, 3 days. Green desk: $299.98, 3 days. Total: $653.96, max delivery: 3 days.",- "copilotToolCallId": "toolu_vrtx_015FrLR8BJNkcxYXwgxkXkHk"+ "text": "Report the combined results for both orders: blue table ($353.98) and green desk ($299.98), with a maximum delivery time of 3 days",+ "copilotToolCallId": "toolu_vrtx_01Wn2WotjzPSUTWaYVu4Ymg8"},{"type": "tool_call",@@ -411,13 +411,13 @@"price": 653.96,"deliveryDays": 3},- "id": "toolu_vrtx_015FrLR8BJNkcxYXwgxkXkHk"+ "id": "toolu_vrtx_01Wn2WotjzPSUTWaYVu4Ymg8"}]},"usage": {- "input": 9971,- "output": 143+ "input": 10025,+ "output": 124}}}\ No newline at end of filediff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.jsonindex 2ffc395780..5f95f099ac 100644--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json@@ -1,27 +1,27 @@{- "4c37062e059030f9c15a515aba7dea63145b1691": {+ "1f67d34de9dca04e9ec4d9d7b590b48b6e67fd08": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I'll help you complete this task. Let me start by navigating to the page and then filling out the form to order the items."+ "text": "I'll help you navigate to the form, fill in the order details, and report back the price and delivery information."}]},"usage": {- "input": 4390,- "output": 117+ "input": 4429,+ "output": 114}},- "fad5bf0fc534ecb1e403acfc02b0040f502dde65": {+ "498aeb8d624d9373c7e5b5b1dff78828566dc900": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Navigate to the form page to start ordering items",- "copilotToolCallId": "toolu_vrtx_01LUFwA4XTZMZ1y1NHnRsMNx"+ "text": "Navigate to the order form page",+ "copilotToolCallId": "toolu_vrtx_01XPsGbYiofMbWmfgtdWBK7N"},{"type": "tool_call",@@ -29,107 +29,41 @@"arguments": {"url": "http://localhost:<PORT>/evals/fill-form.html"},- "id": "toolu_vrtx_01LUFwA4XTZMZ1y1NHnRsMNx"+ "id": "toolu_vrtx_01XPsGbYiofMbWmfgtdWBK7N"}]},"usage": {- "input": 4449,- "output": 92+ "input": 4484,+ "output": 89}},- "7fcc4660d74d7989887daa17f960a092cfa6d75d": {+ "95fe071871583b1a5ddca2286d61680674dd9fd6": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Get current page state to see the form",- "copilotToolCallId": "toolu_vrtx_01S1TPuFGdY3PpCwQ1MTs7iA"- },- {- "type": "tool_call",- "name": "browser_snapshot",- "arguments": {},- "id": "toolu_vrtx_01S1TPuFGdY3PpCwQ1MTs7iA"- }- ]- },- "usage": {- "input": 5162,- "output": 60- }- },- "f7864b65b2ac8c7b1371d54672bc5cea4ac0ee3a": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "Select Table as the first product",- "copilotToolCallId": "toolu_vrtx_01U7qs1Y5D27RMjy5vigdaXR"- },- {- "type": "tool_call",- "name": "browser_select_option",- "arguments": {- "element": "Product Type dropdown",- "ref": "e7",- "values": [- "Table"- ]- },- "id": "toolu_vrtx_01U7qs1Y5D27RMjy5vigdaXR"- }- ]- },- "usage": {- "input": 5813,- "output": 116- }- },- "7c78931db5b30156a7dc1e6612d870e8187a20d6": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "Select Blue color for the table",- "copilotToolCallId": "toolu_vrtx_01LXzd6tHpmw9uCZGrNc3TSi"- },- {- "type": "tool_call",- "name": "browser_select_option",- "arguments": {- "element": "Color dropdown",- "ref": "e10",- "values": [- "Blue"- ]- },- "id": "toolu_vrtx_01LXzd6tHpmw9uCZGrNc3TSi"- }- ]- },- "usage": {- "input": 6076,- "output": 115- }- },- "cd4bd5106418a526955afe6d3f7c7f2b53007346": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "Fill in the customer information fields",- "copilotToolCallId": "toolu_vrtx_01YWjwfRNj3KaBujienmqBhV"+ "text": "Fill in the order form for a blue table with customer details",+ "copilotToolCallId": "toolu_vrtx_01E9xJ27rpZVDGuRiEKuYgFe"},{"type": "tool_call","name": "browser_fill_form","arguments": {"fields": [+ {+ "name": "Product Type",+ "type": "combobox",+ "ref": "e7",+ "value": "Table"+ },+ {+ "name": "Color",+ "type": "combobox",+ "ref": "e10",+ "value": "Blue"+ },{"name": "Full Name","type": "textbox",@@ -146,19 +80,7 @@"name": "Street Address","type": "textbox","ref": "e19",- "value": "123 Main St"- },- {- "name": "City",- "type": "textbox",- "ref": "e22",- "value": "Anytown"- },- {- "name": "State/Province",- "type": "textbox",- "ref": "e25",- "value": "XYZ state"+ "value": "123 Main St, Anytown, XYZ state"},{"name": "ZIP/Postal Code",@@ -168,159 +90,174 @@}]},- "id": "toolu_vrtx_01YWjwfRNj3KaBujienmqBhV"+ "id": "toolu_vrtx_01E9xJ27rpZVDGuRiEKuYgFe"}]},"usage": {- "input": 6340,- "output": 343+ "input": 5194,+ "output": 354}},- "41b9db56c70fcd4f84f8d3de9de0c799dce8db88": {+ "b33fa200303dc6e1c5e5d06e5566d11af9ee1100": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Submit the first order for blue table",- "copilotToolCallId": "toolu_vrtx_01D6FAeFSJeXh67RCcBh5JMB"+ "text": "Check if City field needs to be filled before submitting",+ "copilotToolCallId": "toolu_vrtx_01EyB4XTMFwAVCkJTVZegHBg"},{"type": "tool_call",- "name": "browser_click",- "arguments": {- "element": "Order button",- "ref": "e29"- },- "id": "toolu_vrtx_01D6FAeFSJeXh67RCcBh5JMB"+ "name": "browser_snapshot",+ "arguments": {},+ "id": "toolu_vrtx_01EyB4XTMFwAVCkJTVZegHBg"}]},"usage": {- "input": 6790,- "output": 95+ "input": 5642,+ "output": 63}},- "e7da24b30926519b920dfff6d09d1f321ccb3ae7": {+ "ddfd3471dba960dd013273a8ff6aa41c21a73ba0": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Confirm the order for blue table",- "copilotToolCallId": "toolu_vrtx_01TDuwkwXfeotUgRAu8fLUFD"+ "text": "Fill in the missing City and State fields that are required",+ "copilotToolCallId": "toolu_vrtx_01WQQgr577X5DKLNM66yaJvc"},{"type": "tool_call",- "name": "browser_handle_dialog",+ "name": "browser_fill_form","arguments": {- "accept": true+ "fields": [+ {+ "name": "City",+ "ref": "e22",+ "type": "textbox",+ "value": "Anytown"+ },+ {+ "name": "State/Province",+ "ref": "e25",+ "type": "textbox",+ "value": "XYZ"+ }+ ]},- "id": "toolu_vrtx_01TDuwkwXfeotUgRAu8fLUFD"+ "id": "toolu_vrtx_01WQQgr577X5DKLNM66yaJvc"}]},"usage": {- "input": 6960,- "output": 78+ "input": 6345,+ "output": 142}},- "b2200231c17c003417ced022a372a4b11edbf238": {+ "6e1d42a5ece3458887bb317028560156de53d25d": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Click to place another order for the green desk",- "copilotToolCallId": "toolu_vrtx_014capLBfLNKY3rkD46m8bQ1"+ "text": "Submit the order for the blue table",+ "copilotToolCallId": "toolu_vrtx_01YQTMoAzSpRRw9J7kzMeAno"},{"type": "tool_call","name": "browser_click","arguments": {- "element": "Place Another Order button",- "ref": "e58"+ "element": "Order button",+ "ref": "e29"},- "id": "toolu_vrtx_014capLBfLNKY3rkD46m8bQ1"+ "id": "toolu_vrtx_01YQTMoAzSpRRw9J7kzMeAno"}]},"usage": {- "input": 7848,- "output": 99+ "input": 6560,+ "output": 95}},- "3eace161b63651422689b72dbf4e606a489163e6": {+ "3090a7d97bc75644ff00d8b32d7fdbcf26c55f6a": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Desk as the second product",- "copilotToolCallId": "toolu_vrtx_01CY7bczyF9uwULtuDbsv5Bi"+ "text": "Confirm the order for the blue table",+ "copilotToolCallId": "toolu_vrtx_01LvLQ372U35LRjMhjzg6Dme"},{"type": "tool_call",- "name": "browser_select_option",+ "name": "browser_handle_dialog","arguments": {- "element": "Product Type dropdown",- "ref": "e7",- "values": [- "Desk"- ]+ "accept": true},- "id": "toolu_vrtx_01CY7bczyF9uwULtuDbsv5Bi"+ "id": "toolu_vrtx_01LvLQ372U35LRjMhjzg6Dme"}]},"usage": {- "input": 8567,- "output": 118+ "input": 6730,+ "output": 79}},- "c59b631891d47364177fda353cfccb126dfbf570": {+ "326f425b7b11f30fb2caef078ed93f7932923edc": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Green color for the desk",- "copilotToolCallId": "toolu_vrtx_019fxgUwj5JiAJFrCzxGXPSK"+ "text": "Click on Place Another Order button to order the second item (green desk)",+ "copilotToolCallId": "toolu_vrtx_01LEuTE7DgrAUMint4khguYr"},{"type": "tool_call",- "name": "browser_select_option",+ "name": "browser_click","arguments": {- "element": "Color dropdown",- "ref": "e10",- "values": [- "Green"- ]+ "element": "Place Another Order button",+ "ref": "e58"},- "id": "toolu_vrtx_019fxgUwj5JiAJFrCzxGXPSK"+ "id": "toolu_vrtx_01LEuTE7DgrAUMint4khguYr"}]},"usage": {- "input": 8833,- "output": 115+ "input": 7641,+ "output": 104}},- "56e5cffa84ed757969f9abbbbe898dd6d0b2dbb6": {+ "9128e6e3f63a0d1eaf2972336082bde3db8449ce": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Fill in the customer information for the second order",- "copilotToolCallId": "toolu_vrtx_016MMu8vjsBSdpwemMbNoXjV"+ "text": "Fill in the order form for a green desk with the same customer details",+ "copilotToolCallId": "toolu_vrtx_01SboLoRCoYELQtRzgagA1EC"},{"type": "tool_call","name": "browser_fill_form","arguments": {"fields": [+ {+ "name": "Product Type",+ "ref": "e7",+ "type": "combobox",+ "value": "Desk"+ },+ {+ "name": "Color",+ "ref": "e10",+ "type": "combobox",+ "value": "Green"+ },{"name": "Full Name","ref": "e13",@@ -337,7 +274,7 @@"name": "Street Address","ref": "e19","type": "textbox",- "value": "123 Main St"+ "value": "123 Main St, Anytown, XYZ state"},{"name": "City",@@ -349,7 +286,7 @@"name": "State/Province","ref": "e25","type": "textbox",- "value": "XYZ state"+ "value": "XYZ"},{"name": "ZIP/Postal Code",@@ -359,23 +296,23 @@}]},- "id": "toolu_vrtx_016MMu8vjsBSdpwemMbNoXjV"+ "id": "toolu_vrtx_01SboLoRCoYELQtRzgagA1EC"}]},"usage": {- "input": 9097,- "output": 266+ "input": 8365,+ "output": 337}},- "d134b61bfaaea4e36415838a62db5cb13f888d20": {+ "ccd12e74f5216b77104f1e2e6199519ef6bce9c0": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Submit the second order for green desk",- "copilotToolCallId": "toolu_vrtx_01TgMV6W9eKdLpPjDDDewJJf"+ "text": "Submit the order for the green desk",+ "copilotToolCallId": "toolu_vrtx_01HsUFw6GuXR6J4YYiNnEHMe"},{"type": "tool_call",@@ -384,23 +321,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_01TgMV6W9eKdLpPjDDDewJJf"+ "id": "toolu_vrtx_01HsUFw6GuXR6J4YYiNnEHMe"}]},"usage": {- "input": 9550,+ "input": 8931,"output": 95}},- "de09913d7981ad51cc18c9ad9ecdf7d1982bb858": {+ "f3d9f4f90ffe9d6f68f0de098d16e50a7012bf6a": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Confirm the order for green desk",- "copilotToolCallId": "toolu_vrtx_018ksE83x6ekovoZuomxmrsv"+ "text": "Confirm the order for the green desk",+ "copilotToolCallId": "toolu_vrtx_01EhMR6qGdFTfFJuC4GgTrzw"},{"type": "tool_call",@@ -408,23 +345,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_018ksE83x6ekovoZuomxmrsv"+ "id": "toolu_vrtx_01EhMR6qGdFTfFJuC4GgTrzw"}]},"usage": {- "input": 9720,- "output": 78+ "input": 9101,+ "output": 79}},- "987bfeb4500a519fde51250611923095075f6dd9": {+ "53eabaad4c0c8ed9499bada78403a56fa3ab6307": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Report the total price for both items and maximum estimated delivery time",- "copilotToolCallId": "toolu_vrtx_01JAZZ8bo7oMgM9FVCpBEvyf"+ "text": "Report the total price for both items and maximum estimated delivery. Blue table: $353.98, Green desk: $299.98. Total: $653.96. Both orders have 3 days estimated delivery.",+ "copilotToolCallId": "toolu_vrtx_01VsdeN1JruLTtnoA2hNUu7u"},{"type": "tool_call",@@ -433,13 +370,13 @@"price": 653.96,"deliveryDays": 3},- "id": "toolu_vrtx_01JAZZ8bo7oMgM9FVCpBEvyf"+ "id": "toolu_vrtx_01VsdeN1JruLTtnoA2hNUu7u"}]},"usage": {- "input": 10609,- "output": 103+ "input": 10013,+ "output": 135}}}\ No newline at end of filediff --git a/tests/mcp/console.spec.ts b/tests/mcp/console.spec.tsindex 4deec26097..c627c08a54 100644--- a/tests/mcp/console.spec.ts+++ b/tests/mcp/console.spec.ts@@ -99,6 +99,65 @@ test('recent console messages', async ({ client, server }) => {});});+test('recent console messages filter', async ({ startClient, server }) => {+ server.setContent('/', `+ <!DOCTYPE html>+ <html>+ <script>+ console.log("console.log");+ console.error("console.error");+ </script>+ </html>+ `, 'text/html');++ const { client } = await startClient({+ args: ['--console-level', 'error'],+ });++ const response = parseResponse(await client.callTool({+ name: 'browser_navigate',+ arguments: {+ url: server.PREFIX,+ },+ }));++ expect(response.consoleMessages).toContain('console.error');+ expect(response.consoleMessages).not.toContain('console.log');+});++test('browser_console_messages default level', async ({ client, server }) => {+ await client.callTool({+ name: 'browser_navigate',+ arguments: {+ url: server.HELLO_WORLD,+ },+ });++ await client.callTool({+ name: 'browser_evaluate',+ arguments: {+ function: `async () => {+ console.debug("console.debug");+ console.log("console.log");+ console.warn("console.warn");+ console.error("console.error");+ setTimeout(() => { throw new Error("unhandled"); }, 0);+ await fetch('/missing');+ }`,+ },+ });++ const response = parseResponse(await client.callTool({+ name: 'browser_console_messages',+ }));+ expect.soft(response.result).toContain('console.log');+ expect.soft(response.result).toContain('console.warn');+ expect.soft(response.result).toContain('console.error');+ expect.soft(response.result).toContain('Error: unhandled');+ expect.soft(response.result).toContain('404');+ expect.soft(response.result).not.toContain('console.debug');+});+test('browser_console_messages errors only', async ({ client, server }) => {await client.callTool({name: 'browser_navigate',@@ -111,6 +170,7 @@ test('browser_console_messages errors only', async ({ client, server }) => {name: 'browser_evaluate',arguments: {function: `async () => {+ console.debug("console.debug");console.log("console.log");console.warn("console.warn");console.error("console.error");@@ -123,7 +183,7 @@ test('browser_console_messages errors only', async ({ client, server }) => {const response = parseResponse(await client.callTool({name: 'browser_console_messages',arguments: {- onlyErrors: true,+ level: 'error',},}));expect.soft(response.result).toContain('console.error');