Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: allow saving snapshot instead of returning (#38499)
Pavel Feldman5216e334769a
diff --git a/packages/playwright/src/mcp/browser/config.ts b/packages/playwright/src/mcp/browser/config.tsindex 1099f3d281..73a69f5ad0 100644--- a/packages/playwright/src/mcp/browser/config.ts+++ b/packages/playwright/src/mcp/browser/config.ts@@ -343,6 +343,7 @@ export function outputDir(config: FullConfig, clientInfo: ClientInfo): string {export async function outputFile(config: FullConfig, clientInfo: ClientInfo, fileName: string, options: { origin: 'code' | 'llm' | 'web', reason: string }): Promise<string> {const file = await resolveFile(config, clientInfo, fileName, options);+ await fs.promises.mkdir(path.dirname(file), { recursive: true });debug('pw:mcp:file')(options.reason, file);return file;}diff --git a/packages/playwright/src/mcp/browser/context.ts b/packages/playwright/src/mcp/browser/context.tsindex 657194f8e9..9d98dd03b6 100644--- a/packages/playwright/src/mcp/browser/context.ts+++ b/packages/playwright/src/mcp/browser/context.ts@@ -15,7 +15,6 @@*/import fs from 'fs';-import path from 'path';import { debug } from 'playwright-core/lib/utilsBundle';import { selectors } from 'playwright-core';@@ -172,7 +171,6 @@ export class Context {await close(async () => {for (const video of videos) {const name = await this.outputFile(dateAsFileName('webm'), { origin: 'code', reason: 'Saving video' });- await fs.promises.mkdir(path.dirname(name), { recursive: true });const p = await video.path();// video.saveAs() does not work for persistent contexts.if (fs.existsSync(p)) {diff --git a/packages/playwright/src/mcp/browser/response.ts b/packages/playwright/src/mcp/browser/response.tsindex 36aeed91b5..570805da5f 100644--- a/packages/playwright/src/mcp/browser/response.ts+++ b/packages/playwright/src/mcp/browser/response.ts@@ -28,10 +28,12 @@ export class Response {private _result: string[] = [];private _code: string[] = [];private _images: { contentType: string, data: Buffer }[] = [];+ private _files: { fileName: string, title: string }[] = [];private _context: Context;private _includeSnapshot: 'none' | 'full' | 'incremental' = 'none';private _includeTabs = false;private _includeModalStates: ModalState[] | undefined;+ private _includeMetaOnly: boolean = false;private _tabSnapshot: TabSnapshot | undefined;readonly toolName: string;@@ -77,6 +79,12 @@ export class Response {return this._images;}+ async addFile(fileName: string, options: { origin: 'code' | 'llm' | 'web', reason: string }) {+ const resolvedFile = await this._context.outputFile(fileName, options);+ this._files.push({ fileName: resolvedFile, title: options.reason });+ return resolvedFile;+ }+setIncludeSnapshot() {this._includeSnapshot = this._context.config.snapshot.mode;}@@ -93,6 +101,10 @@ export class Response {this._includeModalStates = modalStates;}+ setIncludeMetaOnly() {+ this._includeMetaOnly = true;+ }+async finish() {// All the async snapshotting post-action is happening here.// Everything below should race against modal states.@@ -113,10 +125,10 @@ export class Response {logEnd() {if (requestDebug.enabled)- requestDebug(this.serialize({ omitSnapshot: true, omitBlobs: true }));+ requestDebug(this.serialize());}- serialize(options: { omitSnapshot?: boolean, omitBlobs?: boolean, _meta?: Record<string, any> } = {}): { content: (TextContent | ImageContent)[], isError?: boolean, _meta?: Record<string, any> } {+ render(): RenderedResponse{const renderedResponse = new RenderedResponse();if (this._result.length)@@ -138,28 +150,41 @@ export class Response {const modalStatesMarkdown = renderModalStates(this._tabSnapshot.modalStates);renderedResponse.states.modal = modalStatesMarkdown.join('\n');} else if (this._tabSnapshot) {- const includeSnapshot = options.omitSnapshot ? 'none' : this._includeSnapshot;- renderTabSnapshot(this._tabSnapshot, includeSnapshot, renderedResponse);+ renderTabSnapshot(this._tabSnapshot, this._includeSnapshot, renderedResponse);} else if (this._includeModalStates) {const modalStatesMarkdown = renderModalStates(this._includeModalStates);renderedResponse.states.modal = modalStatesMarkdown.join('\n');}- const redactedResponse = this._context.config.secrets ? renderedResponse.redact(this._context.config.secrets) : renderedResponse;+ if (this._files.length) {+ const lines: string[] = [];+ for (const file of this._files)+ lines.push(`- [${file.title}](${file.fileName})`);+ renderedResponse.updates.push({ category: 'files', content: lines.join('\n') });+ }- // Structured response.+ return this._context.config.secrets ? renderedResponse.redact(this._context.config.secrets) : renderedResponse;+ }++ serialize(options: { _meta?: Record<string, any> } = {}): { content: (TextContent | ImageContent)[], isError?: boolean, _meta?: Record<string, any> } {+ const renderedResponse = this.render();const includeMeta = options._meta && 'dev.lowire/history' in options._meta && 'dev.lowire/state' in options._meta;- const _meta: any = includeMeta ? redactedResponse.asMeta() : undefined;+ const _meta: any = includeMeta ? renderedResponse.asMeta() : undefined;- // Main response partconst content: (TextContent | ImageContent)[] = [- { type: 'text', text: redactedResponse.asText() },+ {+ type: 'text',+ text: renderedResponse.asText(this._includeMetaOnly ? { categories: ['files'] } : undefined)+ },];+ if (this._includeMetaOnly)+ return { _meta, content, isError: this._isError };+// Image attachments.if (this._context.config.imageResponses !== 'omit') {for (const image of this._images)- content.push({ type: 'image', data: options.omitBlobs ? '<blob>' : image.data.toString('base64'), mimeType: image.contentType });+ content.push({ type: 'image', data: image.data.toString('base64'), mimeType: image.contentType });}return {@@ -234,11 +259,11 @@ function trim(text: string, maxLength: number) {export class RenderedResponse {readonly states: Partial<Record<'page' | 'tabs' | 'modal', string>> = {};- readonly updates: { category: 'console' | 'downloads', content: string }[] = [];+ readonly updates: { category: 'console' | 'downloads' | 'files', content: string }[] = [];readonly results: string[] = [];readonly code: string[] = [];- constructor(copy?: { states: Partial<Record<'page' | 'tabs' | 'modal', string>>, updates: { category: 'console' | 'downloads', content: string }[], results: string[], code: string[] }) {+ constructor(copy?: { states: Partial<Record<'page' | 'tabs' | 'modal', string>>, updates: { category: 'console' | 'downloads' | 'files', content: string }[], results: string[], code: string[] }) {if (copy) {this.states = copy.states;this.updates = copy.updates;@@ -247,7 +272,7 @@ export class RenderedResponse {}}- asText(): string {+ asText(filter?: { categories: string[] }): string {const text: string[] = [];if (this.results.length)text.push(`### Result\n${this.results.join('\n')}\n`);@@ -255,6 +280,8 @@ export class RenderedResponse {text.push(`### Ran Playwright code\n${this.code.join('\n')}\n`);for (const { category, content } of this.updates) {+ if (filter && !filter.categories.includes(category))+ continue;if (!content.trim())continue;@@ -265,10 +292,15 @@ export class RenderedResponse {case 'downloads':text.push(`### Downloads\n${content}\n`);break;+ case 'files':+ text.push(`### Files\n${content}\n`);+ break;}}for (const [category, value] of Object.entries(this.states)) {+ if (filter && !filter.categories.includes(category))+ continue;if (!value.trim())continue;@@ -342,6 +374,7 @@ export function parseResponse(response: CallToolResult) {const consoleMessages = sections.get('New console messages');const modalState = sections.get('Modal state');const downloads = sections.get('Downloads');+ const files = sections.get('Files');const codeNoFrame = code?.replace(/^```js\n/, '').replace(/\n```$/, '');const isError = response.isError;const attachments = response.content.slice(1);@@ -354,6 +387,7 @@ export function parseResponse(response: CallToolResult) {consoleMessages,modalState,downloads,+ files,isError,attachments,_meta: response._meta,diff --git a/packages/playwright/src/mcp/browser/tools/pdf.ts b/packages/playwright/src/mcp/browser/tools/pdf.tsindex b6ca34a13b..18ae22627f 100644--- a/packages/playwright/src/mcp/browser/tools/pdf.ts+++ b/packages/playwright/src/mcp/browser/tools/pdf.ts@@ -35,9 +35,8 @@ const pdf = defineTabTool({},handle: async (tab, params, response) => {- const fileName = await tab.context.outputFile(params.filename ?? dateAsFileName('pdf'), { origin: 'llm', reason: 'Saving PDF' });+ const fileName = await response.addFile(params.filename ?? dateAsFileName('pdf'), { origin: 'llm', reason: 'Page saved as PDF' });response.addCode(`await page.pdf(${javascript.formatObject({ path: fileName })});`);- response.addResult(`Saved page as ${fileName}`);await tab.page.pdf({ path: fileName });},});diff --git a/packages/playwright/src/mcp/browser/tools/screenshot.ts b/packages/playwright/src/mcp/browser/tools/screenshot.tsindex 8f07f2222b..ba069e1ecb 100644--- a/packages/playwright/src/mcp/browser/tools/screenshot.ts+++ b/packages/playwright/src/mcp/browser/tools/screenshot.ts@@ -51,7 +51,6 @@ const screenshot = defineTabTool({throw new Error('fullPage cannot be used with element screenshots.');const fileType = params.type || 'png';- const fileName = await tab.context.outputFile(params.filename || dateAsFileName(fileType), { origin: 'llm', reason: 'Saving screenshot' });const options: playwright.PageScreenshotOptions = {type: fileType,quality: fileType === 'png' ? undefined : 90,@@ -61,6 +60,7 @@ const screenshot = defineTabTool({const isElementScreenshot = params.element && params.ref;const screenshotTarget = isElementScreenshot ? params.element : (params.fullPage ? 'full page' : 'viewport');+ const fileName = await response.addFile(params.filename || dateAsFileName(fileType), { origin: 'llm', reason: `Screenshot of ${screenshotTarget}` });response.addCode(`// Screenshot ${screenshotTarget} and save it as ${fileName}`);// Only get snapshot when element screenshot is needed@@ -76,8 +76,6 @@ const screenshot = defineTabTool({await mkdirIfNeeded(fileName);await fs.promises.writeFile(fileName, buffer);- response.addResult(`Took the ${screenshotTarget} screenshot and saved it as ${fileName}`);-response.addImage({contentType: fileType === 'png' ? 'image/png' : 'image/jpeg',data: scaleImageToFitMessage(buffer, fileType)diff --git a/packages/playwright/src/mcp/browser/tools/snapshot.ts b/packages/playwright/src/mcp/browser/tools/snapshot.tsindex 06de18da58..58942eaf18 100644--- a/packages/playwright/src/mcp/browser/tools/snapshot.ts+++ b/packages/playwright/src/mcp/browser/tools/snapshot.ts@@ -14,6 +14,8 @@* limitations under the License.*/+import fs from 'fs';+import { z } from 'playwright-core/lib/mcpBundle';import { defineTabTool, defineTool } from './tool';import * as javascript from '../codegen';@@ -24,13 +26,21 @@ const snapshot = defineTool({name: 'browser_snapshot',title: 'Page snapshot',description: 'Capture accessibility snapshot of the current page, this is better than screenshot',- inputSchema: z.object({}),+ inputSchema: z.object({+ filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'),+ }),type: 'readOnly',},handle: async (context, params, response) => {await context.ensureTab();response.setIncludeFullSnapshot();+ if (params.filename) {+ const renderedResponse = response.render();+ const fileName = await response.addFile(params.filename, { origin: 'llm', reason: 'Saved snapshot' });+ await fs.promises.writeFile(fileName, renderedResponse.asText());+ response.setIncludeMetaOnly();+ }},});diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.jsonindex 4634c34ee1..660806af61 100644--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json@@ -1,42 +1,27 @@{- "3ead929e37fa1cb83a37e09ee827c20293652543": {+ "9618bcb461bf695d7a9f9dff483361df53be8815": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I'll help you navigate to the form, fill it out to order a blue table and green desk, and report back the results."+ "text": "I'll help you navigate to the form, fill it out with the specified details, and report back the price and delivery information."}]},"usage": {- "input": 4471,- "output": 116+ "input": 4498,+ "output": 115}},- "3219899a2bc56b57af31754f9ca6180f988cf2fe": {+ "cc75c9676a9e2450d7da588c814a28f5b464fafa": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I'll start by navigating to the form and then proceed with filling it out."- }- ]- },- "usage": {- "input": 4529,- "output": 107- }- },- "c331f7d028fdc2c008445aeccf1381511958c1ec": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "Navigate to the form page",- "copilotToolCallId": "toolu_vrtx_01F7ZmAjLNmvezxFfGNeXacs"+ "text": "Navigate to the form page to start the ordering process",+ "copilotToolCallId": "toolu_vrtx_019A43H2Z6Vk52FtTtmdgUkv"},{"type": "tool_call",@@ -44,23 +29,23 @@"arguments": {"url": "http://localhost:<PORT>/evals/fill-form.html"},- "id": "toolu_vrtx_01F7ZmAjLNmvezxFfGNeXacs"+ "id": "toolu_vrtx_019A43H2Z6Vk52FtTtmdgUkv"}]},"usage": {- "input": 4577,- "output": 88+ "input": 4555,+ "output": 93}},- "bc220b4e13c6dc33750280791d7e129da176ae08": {+ "980f66d3b436de385561bc71b6f708714d55a480": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Table as the first product",- "copilotToolCallId": "toolu_vrtx_018Nhst3Sa7BQMD21R1W5eAv"+ "text": "Select 'Table' as the first product type",+ "copilotToolCallId": "toolu_vrtx_01Q78j2chYdXjoS8zWxgbMsM"},{"type": "tool_call",@@ -72,23 +57,23 @@"Table"]},- "id": "toolu_vrtx_018Nhst3Sa7BQMD21R1W5eAv"+ "id": "toolu_vrtx_01Q78j2chYdXjoS8zWxgbMsM"}]},"usage": {- "input": 5286,- "output": 116+ "input": 5269,+ "output": 120}},- "aa16e56b7dbe11331771f4b202462a761adcb0d6": {+ "e1ca8894598ec04fc844f0f661dfb7588695e652": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Blue as the color for the table",- "copilotToolCallId": "toolu_vrtx_01Gk4qurt6wv2p5TXw2rfyk8"+ "text": "Select 'Blue' as the color for the table",+ "copilotToolCallId": "toolu_vrtx_01XL8rxxtEXgN9ykGXpj1cWu"},{"type": "tool_call",@@ -100,23 +85,23 @@"Blue"]},- "id": "toolu_vrtx_01Gk4qurt6wv2p5TXw2rfyk8"+ "id": "toolu_vrtx_01XL8rxxtEXgN9ykGXpj1cWu"}]},"usage": {- "input": 5549,- "output": 117+ "input": 5536,+ "output": 120}},- "19053de781776232ada37a39b113987cdfacdf41": {+ "dd6dbb24cce4fe3129f0a830faf6048802a55c4b": {"result": {"role": "assistant","content": [{"type": "text","text": "Fill in the customer details for the first order",- "copilotToolCallId": "toolu_vrtx_01N527bhb1DzBQAs532f4WQJ"+ "copilotToolCallId": "toolu_vrtx_013confcdST6e9sheNQFrmV9"},{"type": "tool_call",@@ -161,23 +146,23 @@}]},- "id": "toolu_vrtx_01N527bhb1DzBQAs532f4WQJ"+ "id": "toolu_vrtx_013confcdST6e9sheNQFrmV9"}]},"usage": {- "input": 5815,+ "input": 5805,"output": 274}},- "d5ac4e87e2af7f0f6e80eabefb485dbe45c9921d": {+ "e985be3f0fdacd8dcdb86bc7540bc0d4b102f4ab": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Submit the first order for blue table",- "copilotToolCallId": "toolu_vrtx_01NQqfZy2VZXGhaFASZqBcaF"+ "text": "Submit the first order for a blue table",+ "copilotToolCallId": "toolu_vrtx_01CaraEChMZBTSy5yBU2i5Tt"},{"type": "tool_call",@@ -186,23 +171,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_01NQqfZy2VZXGhaFASZqBcaF"+ "id": "toolu_vrtx_01CaraEChMZBTSy5yBU2i5Tt"}]},"usage": {- "input": 6268,- "output": 95+ "input": 6258,+ "output": 96}},- "4e2825b7b8d7ec20c52b14540dae9e12dcdfd743": {+ "0fc3f5e5cde041ffb80ba10114d4d5eb35a5dfcd": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Accept the confirmation dialog for blue table",- "copilotToolCallId": "toolu_vrtx_013gcm6ax8g2JWKssZfL3Sog"+ "text": "Accept the confirmation dialog for the blue table order",+ "copilotToolCallId": "toolu_vrtx_017SiTrBCbZmhLci7SSm123d"},{"type": "tool_call",@@ -210,23 +195,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_013gcm6ax8g2JWKssZfL3Sog"+ "id": "toolu_vrtx_017SiTrBCbZmhLci7SSm123d"}]},"usage": {- "input": 6438,- "output": 78+ "input": 6429,+ "output": 80}},- "8bfe6aa2bdaaedcef7a1c73f4d420d55e6c292d0": {+ "55e321443a3b6141b36538f285e21731b68c337c": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Click to place another order for the green desk",- "copilotToolCallId": "toolu_vrtx_01HRHawjDCbqqVhW7PcxmY3r"+ "text": "Click the button to place another order for the green desk",+ "copilotToolCallId": "toolu_vrtx_01CxZFPgcehAJyZgFN8neKUk"},{"type": "tool_call",@@ -235,23 +220,23 @@"element": "Place Another Order button","ref": "e58"},- "id": "toolu_vrtx_01HRHawjDCbqqVhW7PcxmY3r"+ "id": "toolu_vrtx_01CxZFPgcehAJyZgFN8neKUk"}]},"usage": {- "input": 7326,- "output": 99+ "input": 7319,+ "output": 101}},- "012fc6fa5f4a194ce969dbdece4f32835bbd5f3f": {+ "4d1794ac41beff04dc0fa4145315a5bcc6ddf57f": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Desk for the second product",- "copilotToolCallId": "toolu_vrtx_01YHy8qDmTAnrffRhxo8uneD"+ "text": "Select 'Desk' as the second product type",+ "copilotToolCallId": "toolu_vrtx_01SW8tZ18V51sx8UYX9SkVEH"},{"type": "tool_call",@@ -263,23 +248,23 @@"Desk"]},- "id": "toolu_vrtx_01YHy8qDmTAnrffRhxo8uneD"+ "id": "toolu_vrtx_01SW8tZ18V51sx8UYX9SkVEH"}]},"usage": {- "input": 8045,- "output": 118+ "input": 8040,+ "output": 122}},- "d5ee9fb0a787ecd5ee180fd181e4a3d5fb6a68f2": {+ "db83906845c209b0970abe762d515c3eff3e157d": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Green as the color for the desk",- "copilotToolCallId": "toolu_vrtx_016tGB5BYqmwKAxhynpyMmdm"+ "text": "Select 'Green' as the color for the desk",+ "copilotToolCallId": "toolu_vrtx_01FzNfhLHb8P6etWvaZfVoPd"},{"type": "tool_call",@@ -291,23 +276,23 @@"Green"]},- "id": "toolu_vrtx_016tGB5BYqmwKAxhynpyMmdm"+ "id": "toolu_vrtx_01FzNfhLHb8P6etWvaZfVoPd"}]},"usage": {- "input": 8311,- "output": 117+ "input": 8310,+ "output": 120}},- "440ee34411e291b4dbe9617db95c196eace37f1e": {+ "e5766217902078a0f31d7d61a66d431fb8330064": {"result": {"role": "assistant","content": [{"type": "text","text": "Fill in the customer details for the second order",- "copilotToolCallId": "toolu_vrtx_01Af37pxR5WzUMaws88V3sa2"+ "copilotToolCallId": "toolu_vrtx_01DTZk3sgWdkwEcTnQvNWk8u"},{"type": "tool_call",@@ -352,23 +337,23 @@}]},- "id": "toolu_vrtx_01Af37pxR5WzUMaws88V3sa2"+ "id": "toolu_vrtx_01DTZk3sgWdkwEcTnQvNWk8u"}]},"usage": {- "input": 8577,+ "input": 8579,"output": 266}},- "81089209173d7b018430ef6887a408fcb2e24e84": {+ "eb966490e95b464a6ab8834e3bd7cb2909ac17ea": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Submit the second order for green desk",- "copilotToolCallId": "toolu_vrtx_01JqdAnXaitbPWsyUAP4KuzW"+ "text": "Submit the second order for a green desk",+ "copilotToolCallId": "toolu_vrtx_01MhFtYhTdNE99Zzzzc7q2pF"},{"type": "tool_call",@@ -377,23 +362,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_01JqdAnXaitbPWsyUAP4KuzW"+ "id": "toolu_vrtx_01MhFtYhTdNE99Zzzzc7q2pF"}]},"usage": {- "input": 9030,- "output": 95+ "input": 9032,+ "output": 96}},- "74c9b7b48c20ea2d1385fba336bb20d392637d6c": {+ "053fba57de7fbb68c913562b623324a34788df24": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Accept the confirmation dialog for green desk",- "copilotToolCallId": "toolu_vrtx_01UHoxYquxpfX4meEBSstWZh"+ "text": "Accept the confirmation dialog for the green desk order",+ "copilotToolCallId": "toolu_vrtx_01CB3dJULpX59rXX3S9Spkbb"},{"type": "tool_call",@@ -401,23 +386,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_01UHoxYquxpfX4meEBSstWZh"+ "id": "toolu_vrtx_01CB3dJULpX59rXX3S9Spkbb"}]},"usage": {- "input": 9200,- "output": 78+ "input": 9203,+ "output": 80}},- "90563878bf810e2606705227663679163b7359b0": {+ "e62145520af148b1bb0e735f14cddab91f4bf639": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Report the combined total price and maximum delivery time for both orders",- "copilotToolCallId": "toolu_vrtx_01La4TqsDPCAXcofLq5LApTh"+ "text": "Report the total price for both items and maximum delivery time. Blue table: $353.98, Green desk: $299.98. Total: $653.96. Both orders have 3 days delivery.",+ "copilotToolCallId": "toolu_vrtx_01J347sjiTcJ4P3WUq3ehW9R"},{"type": "tool_call",@@ -426,13 +411,13 @@"price": 653.96,"deliveryDays": 3},- "id": "toolu_vrtx_01La4TqsDPCAXcofLq5LApTh"+ "id": "toolu_vrtx_01J347sjiTcJ4P3WUq3ehW9R"}]},"usage": {- "input": 10089,- "output": 103+ "input": 10094,+ "output": 134}}}\ No newline at end of filediff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.jsonindex cf0083d92d..13df8bbaed 100644--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json@@ -1,27 +1,27 @@{- "3ead929e37fa1cb83a37e09ee827c20293652543": {+ "9618bcb461bf695d7a9f9dff483361df53be8815": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I'll help you navigate to the form, fill it out with the specified details, and report back the price and delivery information."+ "text": "I'll help you navigate to the form, fill in the order details, and report back the results. Let me start by navigating to the page."}]},"usage": {- "input": 4471,- "output": 115+ "input": 4498,+ "output": 120}},- "b863793e2b65f06307dd0e7f06d79c720ddf6518": {+ "6dcc2c52acd06283151dfbd5e29f59186a89e2d5": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Navigate to the form page to start the task",- "copilotToolCallId": "toolu_vrtx_01EtHjyUbs5JU6AqdzHh3k44"+ "text": "Navigate to the form page to start the order process",+ "copilotToolCallId": "toolu_vrtx_01Jekq7CtSogEHaP6U7MYCuZ"},{"type": "tool_call",@@ -29,23 +29,23 @@"arguments": {"url": "http://localhost:<PORT>/evals/fill-form.html"},- "id": "toolu_vrtx_01EtHjyUbs5JU6AqdzHh3k44"+ "id": "toolu_vrtx_01Jekq7CtSogEHaP6U7MYCuZ"}]},"usage": {- "input": 4528,- "output": 92+ "input": 4560,+ "output": 93}},- "874e465379d3c65b5c2be0a270730c1242c1dbde": {+ "e0aa88b89e1a8fe978fbed56774a0f57621ac72d": {"result": {"role": "assistant","content": [{"type": "text","text": "Select Table as the first product",- "copilotToolCallId": "toolu_vrtx_012DUPssGjyvzP2i8Ga7e4LV"+ "copilotToolCallId": "toolu_vrtx_01BaHkDosZGEZXs6YjuvLdFa"},{"type": "tool_call",@@ -57,23 +57,23 @@"Table"]},- "id": "toolu_vrtx_012DUPssGjyvzP2i8Ga7e4LV"+ "id": "toolu_vrtx_01BaHkDosZGEZXs6YjuvLdFa"}]},"usage": {- "input": 5241,+ "input": 5274,"output": 116}},- "2b5eec7b6cf93f318562e07f971ac93aaf177fed": {+ "61aba6d5fc9c3ac919b5336e19b1cd52fdd02121": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Blue color for the table",- "copilotToolCallId": "toolu_vrtx_017NBjQL34v9FM1xUPp8mxYw"+ "text": "Select Blue as the color for the table",+ "copilotToolCallId": "toolu_vrtx_01WN9vgEzcxWmUgDADdX4b3T"},{"type": "tool_call",@@ -85,84 +85,179 @@"Blue"]},- "id": "toolu_vrtx_017NBjQL34v9FM1xUPp8mxYw"+ "id": "toolu_vrtx_01WN9vgEzcxWmUgDADdX4b3T"}]},"usage": {- "input": 5504,- "output": 115+ "input": 5537,+ "output": 117}},- "495aa09d1f5a15bd7d5ea2a13fc8e536a8553d8e": {+ "33d14001ce5af7847b134df5adf2229124538057": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Fill in all the customer information fields",- "copilotToolCallId": "toolu_vrtx_01E2kVwYUZy4tYqUYc7CBku2"+ "text": "Fill in the name field",+ "copilotToolCallId": "toolu_vrtx_019UBYFGPaM1r1s7p2WJut1W"},{"type": "tool_call",- "name": "browser_fill_form",+ "name": "browser_type","arguments": {- "fields": [- {- "name": "Full Name",- "type": "textbox",- "ref": "e13",- "value": "John Doe"- },- {- "name": "Email Address",- "type": "textbox",- "ref": "e16",- "value": "[email protected]"- },- {- "name": "Street Address",- "type": "textbox",- "ref": "e19",- "value": "123 Main St"- },- {- "name": "City",- "type": "textbox",- "ref": "e22",- "value": "Anytown"- },- {- "name": "State/Province",- "type": "textbox",- "ref": "e25",- "value": "XYZ state"- },- {- "name": "ZIP/Postal Code",- "type": "textbox",- "ref": "e28",- "value": "12345"- }- ]+ "element": "Full Name textbox",+ "ref": "e13",+ "text": "John Doe"+ },+ "id": "toolu_vrtx_019UBYFGPaM1r1s7p2WJut1W"+ }+ ]+ },+ "usage": {+ "input": 5803,+ "output": 114+ }+ },+ "74038cf37965dc7824a01a036e9a04e4bae6a721": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the email field",+ "copilotToolCallId": "toolu_vrtx_01MEoP2sPB6DqaGGv3ERgcGs"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "Email Address textbox",+ "ref": "e16",+ "text": "[email protected]"+ },+ "id": "toolu_vrtx_01MEoP2sPB6DqaGGv3ERgcGs"+ }+ ]+ },+ "usage": {+ "input": 5963,+ "output": 116+ }+ },+ "2136b8e6418e90e20ee465f406579da6bf065bd5": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the street address field",+ "copilotToolCallId": "toolu_vrtx_011AAMYvdaj9v6Lj6MWRZRdh"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "Street Address textbox",+ "ref": "e19",+ "text": "123 Main St"+ },+ "id": "toolu_vrtx_011AAMYvdaj9v6Lj6MWRZRdh"+ }+ ]+ },+ "usage": {+ "input": 6127,+ "output": 116+ }+ },+ "29800f02b0f51a774d4e7f69f99b1c9ac4f26511": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the city field with Anytown",+ "copilotToolCallId": "toolu_vrtx_01X3nicst7BVm8LWbSoHM59s"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "City textbox",+ "ref": "e22",+ "text": "Anytown"+ },+ "id": "toolu_vrtx_01X3nicst7BVm8LWbSoHM59s"+ }+ ]+ },+ "usage": {+ "input": 6290,+ "output": 117+ }+ },+ "1de325e1e70e801b92a1ad6e1c628fa084d862b8": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the state field with XYZ",+ "copilotToolCallId": "toolu_vrtx_0154bhMfXbG9vcfEKYHKL8AJ"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "State/Province textbox",+ "ref": "e25",+ "text": "XYZ"+ },+ "id": "toolu_vrtx_0154bhMfXbG9vcfEKYHKL8AJ"+ }+ ]+ },+ "usage": {+ "input": 6452,+ "output": 119+ }+ },+ "f80886b488400b4a7840b02abf25cc7b8d334394": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the ZIP code field",+ "copilotToolCallId": "toolu_vrtx_017KCKyqrm6Nr1RJ8MNVjYrU"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "ZIP/Postal Code textbox",+ "ref": "e28",+ "text": "12345"},- "id": "toolu_vrtx_01E2kVwYUZy4tYqUYc7CBku2"+ "id": "toolu_vrtx_017KCKyqrm6Nr1RJ8MNVjYrU"}]},"usage": {- "input": 5768,- "output": 344+ "input": 6618,+ "output": 117}},- "dcd381e74fa1687308551cdad91683e7b1b498ee": {+ "6a4d45c6995b3070d336621bc3f3587d9bf23b1d": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Click the Order button to submit the first item (blue table)",- "copilotToolCallId": "toolu_vrtx_014nFz1pRcwFfs5vv4SjJhYL"+ "text": "Submit the first order (blue table)",+ "copilotToolCallId": "toolu_vrtx_0195LEteYBG6fxNVGxPuyb3k"},{"type": "tool_call",@@ -171,23 +266,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_014nFz1pRcwFfs5vv4SjJhYL"+ "id": "toolu_vrtx_0195LEteYBG6fxNVGxPuyb3k"}]},"usage": {- "input": 6219,- "output": 100+ "input": 6783,+ "output": 95}},- "19664a3e596c10157b109c88f2d66e4cb13cc5db": {+ "7744f2df7189cd0edd36fd53671660cced79f716": {"result": {"role": "assistant","content": [{"type": "text","text": "Accept the confirmation dialog for the blue table order",- "copilotToolCallId": "toolu_vrtx_01NvPAjDPFV9kCUb4e34YMPV"+ "copilotToolCallId": "toolu_vrtx_01SAAcwWFPqroJzW7My84G4z"},{"type": "tool_call",@@ -195,23 +290,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_01NvPAjDPFV9kCUb4e34YMPV"+ "id": "toolu_vrtx_01SAAcwWFPqroJzW7My84G4z"}]},"usage": {- "input": 6394,+ "input": 6953,"output": 80}},- "91a65f3c9adf4ecc0f2541db1da36e61844a3200": {+ "e8ec627c81c07f9566337a80ee39323d3210abc0": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Click Place Another Order to order the second item (green desk)",- "copilotToolCallId": "toolu_vrtx_01Ko8Nf61uLZmXBLU7usYUup"+ "text": "Click Place Another Order button to order the second item (green desk)",+ "copilotToolCallId": "toolu_vrtx_01A6XjxYpScgiKggZPmdyyBL"},{"type": "tool_call",@@ -220,23 +315,23 @@"element": "Place Another Order button","ref": "e58"},- "id": "toolu_vrtx_01Ko8Nf61uLZmXBLU7usYUup"+ "id": "toolu_vrtx_01A6XjxYpScgiKggZPmdyyBL"}]},"usage": {- "input": 7284,- "output": 102+ "input": 7840,+ "output": 103}},- "5f0e607d8750e68007f731bdd9c82d1e73c3b8ff": {+ "acbaa036380782cb995285df7bc2346ef2b6d756": {"result": {"role": "assistant","content": [{"type": "text","text": "Select Desk as the second product",- "copilotToolCallId": "toolu_vrtx_01Mci1LUTixN6P6X7jM5AXqi"+ "copilotToolCallId": "toolu_vrtx_01Ko4WVozNnZTuvY9AP3YmeC"},{"type": "tool_call",@@ -248,23 +343,23 @@"Desk"]},- "id": "toolu_vrtx_01Mci1LUTixN6P6X7jM5AXqi"+ "id": "toolu_vrtx_01Ko4WVozNnZTuvY9AP3YmeC"}]},"usage": {- "input": 8006,+ "input": 8563,"output": 118}},- "d4e8a88c5cd3e8c102920f0fd9b17f9fd932bf4d": {+ "462192a6362e907a2b9ea0e6e545b521a3e3c09d": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Green color for the desk",- "copilotToolCallId": "toolu_vrtx_018NKZFMWDtMrud3cxqcx7nu"+ "text": "Select Green as the color for the desk",+ "copilotToolCallId": "toolu_vrtx_01CDBDKtPA7ZwbTJJRHCUFoM"},{"type": "tool_call",@@ -276,84 +371,179 @@"Green"]},- "id": "toolu_vrtx_018NKZFMWDtMrud3cxqcx7nu"+ "id": "toolu_vrtx_01CDBDKtPA7ZwbTJJRHCUFoM"}]},"usage": {- "input": 8272,- "output": 115+ "input": 8829,+ "output": 117}},- "f02a2ac089582728acf943c2871a0791dfb6bcd4": {+ "13567665222bacf36f5d7157ef02c0d22612676c": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Fill in all the customer information fields for the second order",- "copilotToolCallId": "toolu_vrtx_014FYeLbYMadQ89LzaQXFdoy"+ "text": "Fill in the name field for second order",+ "copilotToolCallId": "toolu_vrtx_01ARssnd9eTw1uHRbWLFQaAA"},{"type": "tool_call",- "name": "browser_fill_form",+ "name": "browser_type","arguments": {- "fields": [- {- "name": "Full Name",- "ref": "e13",- "type": "textbox",- "value": "John Doe"- },- {- "name": "Email Address",- "ref": "e16",- "type": "textbox",- "value": "[email protected]"- },- {- "name": "Street Address",- "ref": "e19",- "type": "textbox",- "value": "123 Main St"- },- {- "name": "City",- "ref": "e22",- "type": "textbox",- "value": "Anytown"- },- {- "name": "State/Province",- "ref": "e25",- "type": "textbox",- "value": "XYZ state"- },- {- "name": "ZIP/Postal Code",- "ref": "e28",- "type": "textbox",- "value": "12345"- }- ]+ "element": "Full Name textbox",+ "ref": "e13",+ "text": "John Doe"+ },+ "id": "toolu_vrtx_01ARssnd9eTw1uHRbWLFQaAA"+ }+ ]+ },+ "usage": {+ "input": 9095,+ "output": 117+ }+ },+ "7b8bd077cd1541bfca55303c45fd96a4fdd6337e": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the email field for second order",+ "copilotToolCallId": "toolu_vrtx_01QwWGsWv9EKzghtCawineA5"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "Email Address textbox",+ "ref": "e16",+ "text": "[email protected]"+ },+ "id": "toolu_vrtx_01QwWGsWv9EKzghtCawineA5"+ }+ ]+ },+ "usage": {+ "input": 9258,+ "output": 119+ }+ },+ "aec3d303241453b82b622337aea0a774f1490908": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the street address field for second order",+ "copilotToolCallId": "toolu_vrtx_01KXY1BeK5D62GkRwrKuTzoD"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "Street Address textbox",+ "ref": "e19",+ "text": "123 Main St"+ },+ "id": "toolu_vrtx_01KXY1BeK5D62GkRwrKuTzoD"+ }+ ]+ },+ "usage": {+ "input": 9425,+ "output": 119+ }+ },+ "ab3c87f72a039a7bd16601c4a77ef6c6f6c8d36f": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the city field for second order",+ "copilotToolCallId": "toolu_vrtx_01AQ8ymjcSsF9inQD4Vdi9qx"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "City textbox",+ "ref": "e22",+ "text": "Anytown"+ },+ "id": "toolu_vrtx_01AQ8ymjcSsF9inQD4Vdi9qx"+ }+ ]+ },+ "usage": {+ "input": 9591,+ "output": 116+ }+ },+ "3444e227160908670bb9267f25ab95112550a5b6": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the state field for second order",+ "copilotToolCallId": "toolu_vrtx_013sgmYiQHkztUVLRY8FUEKq"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "State/Province textbox",+ "ref": "e25",+ "text": "XYZ"+ },+ "id": "toolu_vrtx_013sgmYiQHkztUVLRY8FUEKq"+ }+ ]+ },+ "usage": {+ "input": 9752,+ "output": 118+ }+ },+ "a284cc1b4f7f15a31cea4f5b6301058cf0935ea4": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the ZIP code field for second order",+ "copilotToolCallId": "toolu_vrtx_01Ep3dXycgf6gLsB45tQgisL"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "ZIP/Postal Code textbox",+ "ref": "e28",+ "text": "12345"},- "id": "toolu_vrtx_014FYeLbYMadQ89LzaQXFdoy"+ "id": "toolu_vrtx_01Ep3dXycgf6gLsB45tQgisL"}]},"usage": {- "input": 8536,- "output": 268+ "input": 9917,+ "output": 120}},- "50baf0fb89e0d4b60244120121fb12d9ec326131": {+ "fd7b954cb041fcf5fa0fb998761dd9b398cc1cee": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Click the Order button to submit the second item (green desk)",- "copilotToolCallId": "toolu_vrtx_01WCTR9W4hh6CrYNvgm25Yjo"+ "text": "Submit the second order (green desk)",+ "copilotToolCallId": "toolu_vrtx_01Ugc3Z7yYLZtiTuMifEfhAZ"},{"type": "tool_call",@@ -362,23 +552,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_01WCTR9W4hh6CrYNvgm25Yjo"+ "id": "toolu_vrtx_01Ugc3Z7yYLZtiTuMifEfhAZ"}]},"usage": {- "input": 8991,- "output": 100+ "input": 10085,+ "output": 95}},- "f58af62434ae328de38eee3842082050991524de": {+ "6c7a1d4b5bb21c55df6eaaeee1cc3fd93a20978c": {"result": {"role": "assistant","content": [{"type": "text","text": "Accept the confirmation dialog for the green desk order",- "copilotToolCallId": "toolu_vrtx_01D8bGvuBZnBaaHg9vEPhkto"+ "copilotToolCallId": "toolu_vrtx_01Lpxhy6JBxhSTkJHFemHrEc"},{"type": "tool_call",@@ -386,23 +576,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_01D8bGvuBZnBaaHg9vEPhkto"+ "id": "toolu_vrtx_01Lpxhy6JBxhSTkJHFemHrEc"}]},"usage": {- "input": 9166,+ "input": 10255,"output": 80}},- "5c0839470e9ae67f3beeaa8113b7d39e52c616c9": {+ "7a450713f8ee68d4c1e806ae3144c9c167335ba5": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Report the total price and maximum estimated delivery for both orders. Blue table: $353.98, 3 days. Green desk: $299.98, 3 days. Total: $653.96, max delivery: 3 days.",- "copilotToolCallId": "toolu_vrtx_01KyKWN5mj68Pe9d7DUbiEpi"+ "text": "Report the total price and maximum delivery time for both orders: Blue Table ($353.98, 3 days) and Green Desk ($299.98, 3 days). Total: $653.96, Max delivery: 3 days",+ "copilotToolCallId": "toolu_vrtx_01UvTtaaPPEWqLR1mtJjLB3k"},{"type": "tool_call",@@ -411,12 +601,12 @@"price": 653.96,"deliveryDays": 3},- "id": "toolu_vrtx_01KyKWN5mj68Pe9d7DUbiEpi"+ "id": "toolu_vrtx_01UvTtaaPPEWqLR1mtJjLB3k"}]},"usage": {- "input": 10057,+ "input": 11143,"output": 143}}diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.jsonindex 56a5bf4b2f..2f7073a0bc 100644--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json@@ -1,42 +1,27 @@{- "3ead929e37fa1cb83a37e09ee827c20293652543": {+ "9618bcb461bf695d7a9f9dff483361df53be8815": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I'll help you navigate to the form, fill it out with the details provided, and report back the price and delivery information."+ "text": "I'll help you navigate to the form, fill it out with the specified items and details, and report back the results."}]},"usage": {- "input": 4471,- "output": 115- }- },- "f2ac9e2c1fb83e54daedc3a1607ee770dd6cd05c": {- "result": {- "role": "assistant",- "content": [- {- "type": "text",- "text": "I'll navigate to the page and complete the form as requested."- }- ]- },- "usage": {- "input": 4528,- "output": 107+ "input": 4498,+ "output": 118}},- "82c674fde5a1942a704bedf38590a35c19551537": {+ "651ad41a348d6db6488f4746b0b4905094330b85": {"result": {"role": "assistant","content": [{"type": "text","text": "Navigate to the form page",- "copilotToolCallId": "toolu_vrtx_01PUGZnaWbXrBgkeYjhWXPui"+ "copilotToolCallId": "toolu_vrtx_015A4Qxa4UTA1YLM31KPKBk5"},{"type": "tool_call",@@ -44,23 +29,23 @@"arguments": {"url": "http://localhost:<PORT>/evals/fill-form.html"},- "id": "toolu_vrtx_01PUGZnaWbXrBgkeYjhWXPui"+ "id": "toolu_vrtx_015A4Qxa4UTA1YLM31KPKBk5"}]},"usage": {- "input": 4572,+ "input": 4554,"output": 88}},- "d1b4efddf9cbdf8b21ebca18d5a73515bbc1ccd1": {+ "c03b93553be7a4f582a9481cd7ceef812cc3f1a2": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Table as the product type",- "copilotToolCallId": "toolu_vrtx_01CjJV9pdnAYmTgMmw2xqCik"+ "text": "Select Table as the first product",+ "copilotToolCallId": "toolu_vrtx_01URHnaVsJXwPYiCcwLfLqLP"},{"type": "tool_call",@@ -72,23 +57,23 @@"Table"]},- "id": "toolu_vrtx_01CjJV9pdnAYmTgMmw2xqCik"+ "id": "toolu_vrtx_01URHnaVsJXwPYiCcwLfLqLP"}]},"usage": {- "input": 5281,+ "input": 5263,"output": 116}},- "1340b24ab7000c3ecfe415e169728e151270b626": {+ "45c9e0f04d5ec6adea27cb529f321db6b7f3a217": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Blue as the color",- "copilotToolCallId": "toolu_vrtx_01HdgVvtbHAEew3V3SAhSE8e"+ "text": "Select Blue as the color for the table",+ "copilotToolCallId": "toolu_vrtx_01MUvkws5Pqa1NKuidpY27Js"},{"type": "tool_call",@@ -100,84 +85,179 @@"Blue"]},- "id": "toolu_vrtx_01HdgVvtbHAEew3V3SAhSE8e"+ "id": "toolu_vrtx_01MUvkws5Pqa1NKuidpY27Js"}]},"usage": {- "input": 5544,- "output": 114+ "input": 5526,+ "output": 117}},- "246c11ed0315d3cf3f8362a31b75f996e0b2c75b": {+ "f01b96c033547e823a688aaa59e57e45d581244f": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Fill in the customer details for the first order",- "copilotToolCallId": "toolu_vrtx_013EpJmdkVhMZ4vVp7rQcuj9"+ "text": "Fill in the Full Name field",+ "copilotToolCallId": "toolu_vrtx_01VzYhHy84gaE8mvmUgmPsoa"},{"type": "tool_call",- "name": "browser_fill_form",+ "name": "browser_type","arguments": {- "fields": [- {- "name": "Full Name",- "type": "textbox",- "ref": "e13",- "value": "John Doe"- },- {- "name": "Email Address",- "type": "textbox",- "ref": "e16",- "value": "[email protected]"- },- {- "name": "Street Address",- "type": "textbox",- "ref": "e19",- "value": "123 Main St"- },- {- "name": "City",- "type": "textbox",- "ref": "e22",- "value": "Anytown"- },- {- "name": "State/Province",- "type": "textbox",- "ref": "e25",- "value": "XYZ state"- },- {- "name": "ZIP/Postal Code",- "type": "textbox",- "ref": "e28",- "value": "12345"- }- ]+ "element": "Full Name textbox",+ "ref": "e13",+ "text": "John Doe"+ },+ "id": "toolu_vrtx_01VzYhHy84gaE8mvmUgmPsoa"+ }+ ]+ },+ "usage": {+ "input": 5792,+ "output": 115+ }+ },+ "6de46f95cbe9459b6af5f56cbab64e6a140afa14": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the Email Address field",+ "copilotToolCallId": "toolu_vrtx_019meT2mNudZyQwmT7fnCrsA"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "Email Address textbox",+ "ref": "e16",+ "text": "[email protected]"+ },+ "id": "toolu_vrtx_019meT2mNudZyQwmT7fnCrsA"+ }+ ]+ },+ "usage": {+ "input": 5953,+ "output": 117+ }+ },+ "e25f3a5f2011bdddcb17e6797577346ccef2899f": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the Street Address field",+ "copilotToolCallId": "toolu_vrtx_01LvRRG2trw2FbUE3AZSJk7Z"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "Street Address textbox",+ "ref": "e19",+ "text": "123 Main St"+ },+ "id": "toolu_vrtx_01LvRRG2trw2FbUE3AZSJk7Z"+ }+ ]+ },+ "usage": {+ "input": 6118,+ "output": 116+ }+ },+ "e939604b2f4bd1eafeb5be883ca4811d8af3c5f6": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the City field",+ "copilotToolCallId": "toolu_vrtx_01YcSZZtz11FVhUB5ZuCiVHT"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "City textbox",+ "ref": "e22",+ "text": "Anytown"+ },+ "id": "toolu_vrtx_01YcSZZtz11FVhUB5ZuCiVHT"+ }+ ]+ },+ "usage": {+ "input": 6281,+ "output": 113+ }+ },+ "b7ebb78e78e4c0e4cebfd3802160e994dff80374": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the State/Province field",+ "copilotToolCallId": "toolu_vrtx_01KR4DM6jFZFX1ze7kMisxa6"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "State/Province textbox",+ "ref": "e25",+ "text": "XYZ"+ },+ "id": "toolu_vrtx_01KR4DM6jFZFX1ze7kMisxa6"+ }+ ]+ },+ "usage": {+ "input": 6439,+ "output": 117+ }+ },+ "a808e5354acfb3617a6a9723dfe95478e2652e63": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the ZIP/Postal Code field",+ "copilotToolCallId": "toolu_vrtx_01PtJxTGzWz5ogj7d1nm2aKS"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "ZIP/Postal Code textbox",+ "ref": "e28",+ "text": "12345"},- "id": "toolu_vrtx_013EpJmdkVhMZ4vVp7rQcuj9"+ "id": "toolu_vrtx_01PtJxTGzWz5ogj7d1nm2aKS"}]},"usage": {- "input": 5807,- "output": 274+ "input": 6603,+ "output": 120}},- "90fd25992c1a647531b834bb4c37dd1aa507a875": {+ "d9b4832077829284be3acbe935349b329c47518c": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Submit the first order for the blue table",- "copilotToolCallId": "toolu_vrtx_019fyjP8YWhpbkWqCuWEaEEz"+ "text": "Submit the order for the blue table",+ "copilotToolCallId": "toolu_vrtx_01FDvdEQK9V9deeRt3WJLAok"},{"type": "tool_call",@@ -186,23 +266,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_019fyjP8YWhpbkWqCuWEaEEz"+ "id": "toolu_vrtx_01FDvdEQK9V9deeRt3WJLAok"}]},"usage": {- "input": 6260,- "output": 96+ "input": 6771,+ "output": 95}},- "95b5282664361cd6416210e38366b8aecce3022f": {+ "2cc8d0e79ffa0eaca9792d2b1e292688effcca6b": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Accept the confirmation dialog for the blue table order",- "copilotToolCallId": "toolu_vrtx_016sk3xCVtw6856vxqGGBdkB"+ "text": "Accept the order confirmation for blue table",+ "copilotToolCallId": "toolu_vrtx_014eUSnWVV85a5yhgtkhx7uN"},{"type": "tool_call",@@ -210,23 +290,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_016sk3xCVtw6856vxqGGBdkB"+ "id": "toolu_vrtx_014eUSnWVV85a5yhgtkhx7uN"}]},"usage": {- "input": 6431,- "output": 80+ "input": 6941,+ "output": 78}},- "db78696f00789b6fedb3d4f75b7acae7d43514ec": {+ "92e5599d7913f24dde5c2eedafede877dfc76594": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Click the button to place another order for the green desk",- "copilotToolCallId": "toolu_vrtx_01KmgEbnjXQaUcxsqucKXmKF"+ "text": "Click Place Another Order button to order the green desk",+ "copilotToolCallId": "toolu_vrtx_01VWmtWDYCiEQ51LqFSqRVyt"},{"type": "tool_call",@@ -235,23 +315,23 @@"element": "Place Another Order button","ref": "e58"},- "id": "toolu_vrtx_01KmgEbnjXQaUcxsqucKXmKF"+ "id": "toolu_vrtx_01VWmtWDYCiEQ51LqFSqRVyt"}]},"usage": {- "input": 7321,- "output": 101+ "input": 7826,+ "output": 100}},- "66d11ebd5aa97fcbf9002ac6cf7e25453bcca8f7": {+ "da4ec3f6c3c748eff025f787e4264ad01fcfb0f5": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Select Desk as the product type for the second order",- "copilotToolCallId": "toolu_vrtx_01Qwp1HZ551btMZqVfuhmg7K"+ "text": "Select Desk as the second product",+ "copilotToolCallId": "toolu_vrtx_018KwJC7fJJSrPKtknozFFFr"},{"type": "tool_call",@@ -263,23 +343,23 @@"Desk"]},- "id": "toolu_vrtx_01Qwp1HZ551btMZqVfuhmg7K"+ "id": "toolu_vrtx_018KwJC7fJJSrPKtknozFFFr"}]},"usage": {- "input": 8042,- "output": 122+ "input": 8546,+ "output": 118}},- "b9ea532ba3932051ca53f8995e4a84673ac15f15": {+ "6ee97d09a08742750496ab8b193d715eca619172": {"result": {"role": "assistant","content": [{"type": "text","text": "Select Green as the color for the desk",- "copilotToolCallId": "toolu_vrtx_01UcgZrsRtnDRtamzcXiyhEz"+ "copilotToolCallId": "toolu_vrtx_01PAWTp8xxLpWDcekkdVD9bF"},{"type": "tool_call",@@ -291,84 +371,179 @@"Green"]},- "id": "toolu_vrtx_01UcgZrsRtnDRtamzcXiyhEz"+ "id": "toolu_vrtx_01PAWTp8xxLpWDcekkdVD9bF"}]},"usage": {- "input": 8312,+ "input": 8812,"output": 117}},- "f062f1459e30461f856bac9eaf6c9ba4c0c83445": {+ "b6dfb1610253206d76e22c6cb14b79e1a8868dc5": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Fill in the customer details for the second order",- "copilotToolCallId": "toolu_vrtx_01S4ko4cV4txtLXSwhz12t1d"+ "text": "Fill in the Full Name field for second order",+ "copilotToolCallId": "toolu_vrtx_018Usc5oevz9wb9ZKpftC4sH"},{"type": "tool_call",- "name": "browser_fill_form",+ "name": "browser_type","arguments": {- "fields": [- {- "name": "Full Name",- "ref": "e13",- "type": "textbox",- "value": "John Doe"- },- {- "name": "Email Address",- "ref": "e16",- "type": "textbox",- "value": "[email protected]"- },- {- "name": "Street Address",- "ref": "e19",- "type": "textbox",- "value": "123 Main St"- },- {- "name": "City",- "ref": "e22",- "type": "textbox",- "value": "Anytown"- },- {- "name": "State/Province",- "ref": "e25",- "type": "textbox",- "value": "XYZ state"- },- {- "name": "ZIP/Postal Code",- "ref": "e28",- "type": "textbox",- "value": "12345"- }- ]+ "element": "Full Name textbox",+ "ref": "e13",+ "text": "John Doe"+ },+ "id": "toolu_vrtx_018Usc5oevz9wb9ZKpftC4sH"+ }+ ]+ },+ "usage": {+ "input": 9078,+ "output": 118+ }+ },+ "bb68d65b11d6f1744be333a30b4b2a2910559d8b": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the Email Address field for second order",+ "copilotToolCallId": "toolu_vrtx_0157WoyBbU9kXomtZMpLvnBd"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "Email Address textbox",+ "ref": "e16",+ "text": "[email protected]"+ },+ "id": "toolu_vrtx_0157WoyBbU9kXomtZMpLvnBd"+ }+ ]+ },+ "usage": {+ "input": 9242,+ "output": 120+ }+ },+ "a38c3850dcde9ef2dafcc13e03d17212dbd2de9d": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the Street Address field for second order",+ "copilotToolCallId": "toolu_vrtx_01F9YjYR5DXdVYf8bebw1fdG"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "Street Address textbox",+ "ref": "e19",+ "text": "123 Main St"+ },+ "id": "toolu_vrtx_01F9YjYR5DXdVYf8bebw1fdG"+ }+ ]+ },+ "usage": {+ "input": 9410,+ "output": 119+ }+ },+ "59648493dce87d10e25e5d50abebac4e2392ecc1": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the City field for second order",+ "copilotToolCallId": "toolu_vrtx_01BETdPBJWrtPMurfg49Vjoj"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "City textbox",+ "ref": "e22",+ "text": "Anytown"+ },+ "id": "toolu_vrtx_01BETdPBJWrtPMurfg49Vjoj"+ }+ ]+ },+ "usage": {+ "input": 9576,+ "output": 116+ }+ },+ "a87a7f06ffdf0d79613739ac83936a87142c6d7f": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the State/Province field for second order",+ "copilotToolCallId": "toolu_vrtx_01KXjCLqC74fLfD56Yb2rb3T"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "State/Province textbox",+ "ref": "e25",+ "text": "XYZ"+ },+ "id": "toolu_vrtx_01KXjCLqC74fLfD56Yb2rb3T"+ }+ ]+ },+ "usage": {+ "input": 9737,+ "output": 120+ }+ },+ "0edb63f376b23ae85e2319c8dcfd8167be315357": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Fill in the ZIP/Postal Code field for second order",+ "copilotToolCallId": "toolu_vrtx_01QdpV2UD1ynJBEz3t2vEBZY"+ },+ {+ "type": "tool_call",+ "name": "browser_type",+ "arguments": {+ "element": "ZIP/Postal Code textbox",+ "ref": "e28",+ "text": "12345"},- "id": "toolu_vrtx_01S4ko4cV4txtLXSwhz12t1d"+ "id": "toolu_vrtx_01QdpV2UD1ynJBEz3t2vEBZY"}]},"usage": {- "input": 8578,- "output": 266+ "input": 9904,+ "output": 123}},- "fdcd0330d001ae7fd2401858d3ece675b149b65d": {+ "4a49bf78bda39f22b05bb0dabdb778eab42f5d0e": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Submit the second order for the green desk",- "copilotToolCallId": "toolu_vrtx_015dGJ2kCyGMmwB7P6e3vRcE"+ "text": "Submit the order for the green desk",+ "copilotToolCallId": "toolu_vrtx_018WKajMd3gK6LsoHkMSqqF4"},{"type": "tool_call",@@ -377,23 +552,23 @@"element": "Order button","ref": "e29"},- "id": "toolu_vrtx_015dGJ2kCyGMmwB7P6e3vRcE"+ "id": "toolu_vrtx_018WKajMd3gK6LsoHkMSqqF4"}]},"usage": {- "input": 9031,- "output": 96+ "input": 10075,+ "output": 95}},- "09e90872eebfad23b65c6b8baf6a21f4d3e3be13": {+ "9a1806f90b31427e91b08c182a1f530b01687395": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Accept the confirmation dialog for the green desk order",- "copilotToolCallId": "toolu_vrtx_017Aese7AEKa1XYZoZjeTw6z"+ "text": "Accept the order confirmation for green desk",+ "copilotToolCallId": "toolu_vrtx_01958UH1ER5ox2DiKfqF6gSp"},{"type": "tool_call",@@ -401,23 +576,23 @@"arguments": {"accept": true},- "id": "toolu_vrtx_017Aese7AEKa1XYZoZjeTw6z"+ "id": "toolu_vrtx_01958UH1ER5ox2DiKfqF6gSp"}]},"usage": {- "input": 9202,- "output": 80+ "input": 10245,+ "output": 78}},- "28d9510654099e5564df19869f8609adff0bfaae": {+ "70d0aca5ce593777cf52a420da7aece1173004a1": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Report the total price and maximum delivery time for both orders. Blue table: $353.98, 3 days. Green desk: $299.98, 3 days. Total: $653.96, Max delivery: 3 days.",- "copilotToolCallId": "toolu_vrtx_01Umx8ecJT1VFHfNNJx6vcX5"+ "text": "Report the total price for both items ($353.98 + $299.98 = $653.96) and maximum estimated delivery (3 days for both)",+ "copilotToolCallId": "toolu_vrtx_01R97Mi28WVqRvEH25ky3aqa"},{"type": "tool_call",@@ -426,13 +601,13 @@"price": 653.96,"deliveryDays": 3},- "id": "toolu_vrtx_01Umx8ecJT1VFHfNNJx6vcX5"+ "id": "toolu_vrtx_01R97Mi28WVqRvEH25ky3aqa"}]},"usage": {- "input": 10093,- "output": 143+ "input": 11131,+ "output": 125}}}\ No newline at end of filediff --git a/tests/mcp/pdf.spec.ts b/tests/mcp/pdf.spec.tsindex 937eb16c87..80faf0b4b2 100644--- a/tests/mcp/pdf.spec.ts+++ b/tests/mcp/pdf.spec.ts@@ -51,7 +51,7 @@ test('save as pdf', async ({ startClient, mcpBrowser, server }, testInfo) => {name: 'browser_pdf_save',})).toHaveResponse({code: expect.stringContaining(`await page.pdf(`),- result: expect.stringMatching(/Saved page as.*page-[^:]+.pdf/),+ files: expect.stringMatching(/\[Page saved as PDF\]\(.*page-[^:]+.pdf\)/),});});@@ -75,7 +75,7 @@ test('save as pdf (filename: output.pdf)', async ({ startClient, mcpBrowser, serfilename: 'output.pdf',},})).toHaveResponse({- result: expect.stringContaining(`output.pdf`),+ files: expect.stringContaining(`output.pdf`),code: expect.stringContaining(`await page.pdf(`),});diff --git a/tests/mcp/snapshot-mode.spec.ts b/tests/mcp/snapshot-mode.spec.tsindex d9d68e3b5b..912f938e51 100644--- a/tests/mcp/snapshot-mode.spec.ts+++ b/tests/mcp/snapshot-mode.spec.ts@@ -99,3 +99,29 @@ test('should respect --snapshot-mode=none', async ({ startClient, server }) => {pageState: undefined});});++test('should respect snapshot[filename]', async ({ startClient, server }, testInfo) => {+ server.setContent('/', `<button>Button 1</button>`, 'text/html');++ const outputDir = testInfo.outputPath('output');+ const { client } = await startClient({+ config: { outputDir },+ });++ await client.callTool({+ name: 'browser_navigate',+ arguments: {+ url: server.PREFIX,+ },+ });++ expect(await client.callTool({+ name: 'browser_snapshot',+ arguments: {+ filename: 'snapshot1.md',+ },+ })).toHaveResponse({+ pageState: undefined,+ files: expect.stringMatching(/\[Saved snapshot\]\(.*md\)/)+ });+});