Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: allow saving snapshot instead of returning (#38499)

Pavel Feldman5216e334769a
diff --git a/packages/playwright/src/mcp/browser/config.ts b/packages/playwright/src/mcp/browser/config.ts
index 1099f3d281..73a69f5ad0 100644
--- a/packages/playwright/src/mcp/browser/config.ts
+++ b/packages/playwright/src/mcp/browser/config.ts
@@ -343,6 +343,7 @@ export function outputDir(config: FullConfig, clientInfo: ClientInfo): string {
export async function outputFile(config: FullConfig, clientInfo: ClientInfo, fileName: string, options: { origin: 'code' | 'llm' | 'web', reason: string }): Promise<string> {
const file = await resolveFile(config, clientInfo, fileName, options);
+ await fs.promises.mkdir(path.dirname(file), { recursive: true });
debug('pw:mcp:file')(options.reason, file);
return file;
}
diff --git a/packages/playwright/src/mcp/browser/context.ts b/packages/playwright/src/mcp/browser/context.ts
index 657194f8e9..9d98dd03b6 100644
--- a/packages/playwright/src/mcp/browser/context.ts
+++ b/packages/playwright/src/mcp/browser/context.ts
@@ -15,7 +15,6 @@
*/
import fs from 'fs';
-import path from 'path';
import { debug } from 'playwright-core/lib/utilsBundle';
import { selectors } from 'playwright-core';
@@ -172,7 +171,6 @@ export class Context {
await close(async () => {
for (const video of videos) {
const name = await this.outputFile(dateAsFileName('webm'), { origin: 'code', reason: 'Saving video' });
- await fs.promises.mkdir(path.dirname(name), { recursive: true });
const p = await video.path();
// video.saveAs() does not work for persistent contexts.
if (fs.existsSync(p)) {
diff --git a/packages/playwright/src/mcp/browser/response.ts b/packages/playwright/src/mcp/browser/response.ts
index 36aeed91b5..570805da5f 100644
--- a/packages/playwright/src/mcp/browser/response.ts
+++ b/packages/playwright/src/mcp/browser/response.ts
@@ -28,10 +28,12 @@ export class Response {
private _result: string[] = [];
private _code: string[] = [];
private _images: { contentType: string, data: Buffer }[] = [];
+ private _files: { fileName: string, title: string }[] = [];
private _context: Context;
private _includeSnapshot: 'none' | 'full' | 'incremental' = 'none';
private _includeTabs = false;
private _includeModalStates: ModalState[] | undefined;
+ private _includeMetaOnly: boolean = false;
private _tabSnapshot: TabSnapshot | undefined;
readonly toolName: string;
@@ -77,6 +79,12 @@ export class Response {
return this._images;
}
+ async addFile(fileName: string, options: { origin: 'code' | 'llm' | 'web', reason: string }) {
+ const resolvedFile = await this._context.outputFile(fileName, options);
+ this._files.push({ fileName: resolvedFile, title: options.reason });
+ return resolvedFile;
+ }
+
setIncludeSnapshot() {
this._includeSnapshot = this._context.config.snapshot.mode;
}
@@ -93,6 +101,10 @@ export class Response {
this._includeModalStates = modalStates;
}
+ setIncludeMetaOnly() {
+ this._includeMetaOnly = true;
+ }
+
async finish() {
// All the async snapshotting post-action is happening here.
// Everything below should race against modal states.
@@ -113,10 +125,10 @@ export class Response {
logEnd() {
if (requestDebug.enabled)
- requestDebug(this.serialize({ omitSnapshot: true, omitBlobs: true }));
+ requestDebug(this.serialize());
}
- serialize(options: { omitSnapshot?: boolean, omitBlobs?: boolean, _meta?: Record<string, any> } = {}): { content: (TextContent | ImageContent)[], isError?: boolean, _meta?: Record<string, any> } {
+ render(): RenderedResponse{
const renderedResponse = new RenderedResponse();
if (this._result.length)
@@ -138,28 +150,41 @@ export class Response {
const modalStatesMarkdown = renderModalStates(this._tabSnapshot.modalStates);
renderedResponse.states.modal = modalStatesMarkdown.join('\n');
} else if (this._tabSnapshot) {
- const includeSnapshot = options.omitSnapshot ? 'none' : this._includeSnapshot;
- renderTabSnapshot(this._tabSnapshot, includeSnapshot, renderedResponse);
+ renderTabSnapshot(this._tabSnapshot, this._includeSnapshot, renderedResponse);
} else if (this._includeModalStates) {
const modalStatesMarkdown = renderModalStates(this._includeModalStates);
renderedResponse.states.modal = modalStatesMarkdown.join('\n');
}
- const redactedResponse = this._context.config.secrets ? renderedResponse.redact(this._context.config.secrets) : renderedResponse;
+ if (this._files.length) {
+ const lines: string[] = [];
+ for (const file of this._files)
+ lines.push(`- [${file.title}](${file.fileName})`);
+ renderedResponse.updates.push({ category: 'files', content: lines.join('\n') });
+ }
- // Structured response.
+ return this._context.config.secrets ? renderedResponse.redact(this._context.config.secrets) : renderedResponse;
+ }
+
+ serialize(options: { _meta?: Record<string, any> } = {}): { content: (TextContent | ImageContent)[], isError?: boolean, _meta?: Record<string, any> } {
+ const renderedResponse = this.render();
const includeMeta = options._meta && 'dev.lowire/history' in options._meta && 'dev.lowire/state' in options._meta;
- const _meta: any = includeMeta ? redactedResponse.asMeta() : undefined;
+ const _meta: any = includeMeta ? renderedResponse.asMeta() : undefined;
- // Main response part
const content: (TextContent | ImageContent)[] = [
- { type: 'text', text: redactedResponse.asText() },
+ {
+ type: 'text',
+ text: renderedResponse.asText(this._includeMetaOnly ? { categories: ['files'] } : undefined)
+ },
];
+ if (this._includeMetaOnly)
+ return { _meta, content, isError: this._isError };
+
// Image attachments.
if (this._context.config.imageResponses !== 'omit') {
for (const image of this._images)
- content.push({ type: 'image', data: options.omitBlobs ? '<blob>' : image.data.toString('base64'), mimeType: image.contentType });
+ content.push({ type: 'image', data: image.data.toString('base64'), mimeType: image.contentType });
}
return {
@@ -234,11 +259,11 @@ function trim(text: string, maxLength: number) {
export class RenderedResponse {
readonly states: Partial<Record<'page' | 'tabs' | 'modal', string>> = {};
- readonly updates: { category: 'console' | 'downloads', content: string }[] = [];
+ readonly updates: { category: 'console' | 'downloads' | 'files', content: string }[] = [];
readonly results: string[] = [];
readonly code: string[] = [];
- constructor(copy?: { states: Partial<Record<'page' | 'tabs' | 'modal', string>>, updates: { category: 'console' | 'downloads', content: string }[], results: string[], code: string[] }) {
+ constructor(copy?: { states: Partial<Record<'page' | 'tabs' | 'modal', string>>, updates: { category: 'console' | 'downloads' | 'files', content: string }[], results: string[], code: string[] }) {
if (copy) {
this.states = copy.states;
this.updates = copy.updates;
@@ -247,7 +272,7 @@ export class RenderedResponse {
}
}
- asText(): string {
+ asText(filter?: { categories: string[] }): string {
const text: string[] = [];
if (this.results.length)
text.push(`### Result\n${this.results.join('\n')}\n`);
@@ -255,6 +280,8 @@ export class RenderedResponse {
text.push(`### Ran Playwright code\n${this.code.join('\n')}\n`);
for (const { category, content } of this.updates) {
+ if (filter && !filter.categories.includes(category))
+ continue;
if (!content.trim())
continue;
@@ -265,10 +292,15 @@ export class RenderedResponse {
case 'downloads':
text.push(`### Downloads\n${content}\n`);
break;
+ case 'files':
+ text.push(`### Files\n${content}\n`);
+ break;
}
}
for (const [category, value] of Object.entries(this.states)) {
+ if (filter && !filter.categories.includes(category))
+ continue;
if (!value.trim())
continue;
@@ -342,6 +374,7 @@ export function parseResponse(response: CallToolResult) {
const consoleMessages = sections.get('New console messages');
const modalState = sections.get('Modal state');
const downloads = sections.get('Downloads');
+ const files = sections.get('Files');
const codeNoFrame = code?.replace(/^```js\n/, '').replace(/\n```$/, '');
const isError = response.isError;
const attachments = response.content.slice(1);
@@ -354,6 +387,7 @@ export function parseResponse(response: CallToolResult) {
consoleMessages,
modalState,
downloads,
+ files,
isError,
attachments,
_meta: response._meta,
diff --git a/packages/playwright/src/mcp/browser/tools/pdf.ts b/packages/playwright/src/mcp/browser/tools/pdf.ts
index b6ca34a13b..18ae22627f 100644
--- a/packages/playwright/src/mcp/browser/tools/pdf.ts
+++ b/packages/playwright/src/mcp/browser/tools/pdf.ts
@@ -35,9 +35,8 @@ const pdf = defineTabTool({
},
handle: async (tab, params, response) => {
- const fileName = await tab.context.outputFile(params.filename ?? dateAsFileName('pdf'), { origin: 'llm', reason: 'Saving PDF' });
+ const fileName = await response.addFile(params.filename ?? dateAsFileName('pdf'), { origin: 'llm', reason: 'Page saved as PDF' });
response.addCode(`await page.pdf(${javascript.formatObject({ path: fileName })});`);
- response.addResult(`Saved page as ${fileName}`);
await tab.page.pdf({ path: fileName });
},
});
diff --git a/packages/playwright/src/mcp/browser/tools/screenshot.ts b/packages/playwright/src/mcp/browser/tools/screenshot.ts
index 8f07f2222b..ba069e1ecb 100644
--- a/packages/playwright/src/mcp/browser/tools/screenshot.ts
+++ b/packages/playwright/src/mcp/browser/tools/screenshot.ts
@@ -51,7 +51,6 @@ const screenshot = defineTabTool({
throw new Error('fullPage cannot be used with element screenshots.');
const fileType = params.type || 'png';
- const fileName = await tab.context.outputFile(params.filename || dateAsFileName(fileType), { origin: 'llm', reason: 'Saving screenshot' });
const options: playwright.PageScreenshotOptions = {
type: fileType,
quality: fileType === 'png' ? undefined : 90,
@@ -61,6 +60,7 @@ const screenshot = defineTabTool({
const isElementScreenshot = params.element && params.ref;
const screenshotTarget = isElementScreenshot ? params.element : (params.fullPage ? 'full page' : 'viewport');
+ const fileName = await response.addFile(params.filename || dateAsFileName(fileType), { origin: 'llm', reason: `Screenshot of ${screenshotTarget}` });
response.addCode(`// Screenshot ${screenshotTarget} and save it as ${fileName}`);
// Only get snapshot when element screenshot is needed
@@ -76,8 +76,6 @@ const screenshot = defineTabTool({
await mkdirIfNeeded(fileName);
await fs.promises.writeFile(fileName, buffer);
- response.addResult(`Took the ${screenshotTarget} screenshot and saved it as ${fileName}`);
-
response.addImage({
contentType: fileType === 'png' ? 'image/png' : 'image/jpeg',
data: scaleImageToFitMessage(buffer, fileType)
diff --git a/packages/playwright/src/mcp/browser/tools/snapshot.ts b/packages/playwright/src/mcp/browser/tools/snapshot.ts
index 06de18da58..58942eaf18 100644
--- a/packages/playwright/src/mcp/browser/tools/snapshot.ts
+++ b/packages/playwright/src/mcp/browser/tools/snapshot.ts
@@ -14,6 +14,8 @@
* limitations under the License.
*/
+import fs from 'fs';
+
import { z } from 'playwright-core/lib/mcpBundle';
import { defineTabTool, defineTool } from './tool';
import * as javascript from '../codegen';
@@ -24,13 +26,21 @@ const snapshot = defineTool({
name: 'browser_snapshot',
title: 'Page snapshot',
description: 'Capture accessibility snapshot of the current page, this is better than screenshot',
- inputSchema: z.object({}),
+ inputSchema: z.object({
+ filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'),
+ }),
type: 'readOnly',
},
handle: async (context, params, response) => {
await context.ensureTab();
response.setIncludeFullSnapshot();
+ if (params.filename) {
+ const renderedResponse = response.render();
+ const fileName = await response.addFile(params.filename, { origin: 'llm', reason: 'Saved snapshot' });
+ await fs.promises.writeFile(fileName, renderedResponse.asText());
+ response.setIncludeMetaOnly();
+ }
},
});
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
index 4634c34ee1..660806af61 100644
--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
@@ -1,42 +1,27 @@
{
- "3ead929e37fa1cb83a37e09ee827c20293652543": {
+ "9618bcb461bf695d7a9f9dff483361df53be8815": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll help you navigate to the form, fill it out to order a blue table and green desk, and report back the results."
+ "text": "I'll help you navigate to the form, fill it out with the specified details, and report back the price and delivery information."
}
]
},
"usage": {
- "input": 4471,
- "output": 116
+ "input": 4498,
+ "output": 115
}
},
- "3219899a2bc56b57af31754f9ca6180f988cf2fe": {
+ "cc75c9676a9e2450d7da588c814a28f5b464fafa": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll start by navigating to the form and then proceed with filling it out."
- }
- ]
- },
- "usage": {
- "input": 4529,
- "output": 107
- }
- },
- "c331f7d028fdc2c008445aeccf1381511958c1ec": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "Navigate to the form page",
- "copilotToolCallId": "toolu_vrtx_01F7ZmAjLNmvezxFfGNeXacs"
+ "text": "Navigate to the form page to start the ordering process",
+ "copilotToolCallId": "toolu_vrtx_019A43H2Z6Vk52FtTtmdgUkv"
},
{
"type": "tool_call",
@@ -44,23 +29,23 @@
"arguments": {
"url": "http://localhost:<PORT>/evals/fill-form.html"
},
- "id": "toolu_vrtx_01F7ZmAjLNmvezxFfGNeXacs"
+ "id": "toolu_vrtx_019A43H2Z6Vk52FtTtmdgUkv"
}
]
},
"usage": {
- "input": 4577,
- "output": 88
+ "input": 4555,
+ "output": 93
}
},
- "bc220b4e13c6dc33750280791d7e129da176ae08": {
+ "980f66d3b436de385561bc71b6f708714d55a480": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Table as the first product",
- "copilotToolCallId": "toolu_vrtx_018Nhst3Sa7BQMD21R1W5eAv"
+ "text": "Select 'Table' as the first product type",
+ "copilotToolCallId": "toolu_vrtx_01Q78j2chYdXjoS8zWxgbMsM"
},
{
"type": "tool_call",
@@ -72,23 +57,23 @@
"Table"
]
},
- "id": "toolu_vrtx_018Nhst3Sa7BQMD21R1W5eAv"
+ "id": "toolu_vrtx_01Q78j2chYdXjoS8zWxgbMsM"
}
]
},
"usage": {
- "input": 5286,
- "output": 116
+ "input": 5269,
+ "output": 120
}
},
- "aa16e56b7dbe11331771f4b202462a761adcb0d6": {
+ "e1ca8894598ec04fc844f0f661dfb7588695e652": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Blue as the color for the table",
- "copilotToolCallId": "toolu_vrtx_01Gk4qurt6wv2p5TXw2rfyk8"
+ "text": "Select 'Blue' as the color for the table",
+ "copilotToolCallId": "toolu_vrtx_01XL8rxxtEXgN9ykGXpj1cWu"
},
{
"type": "tool_call",
@@ -100,23 +85,23 @@
"Blue"
]
},
- "id": "toolu_vrtx_01Gk4qurt6wv2p5TXw2rfyk8"
+ "id": "toolu_vrtx_01XL8rxxtEXgN9ykGXpj1cWu"
}
]
},
"usage": {
- "input": 5549,
- "output": 117
+ "input": 5536,
+ "output": 120
}
},
- "19053de781776232ada37a39b113987cdfacdf41": {
+ "dd6dbb24cce4fe3129f0a830faf6048802a55c4b": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Fill in the customer details for the first order",
- "copilotToolCallId": "toolu_vrtx_01N527bhb1DzBQAs532f4WQJ"
+ "copilotToolCallId": "toolu_vrtx_013confcdST6e9sheNQFrmV9"
},
{
"type": "tool_call",
@@ -161,23 +146,23 @@
}
]
},
- "id": "toolu_vrtx_01N527bhb1DzBQAs532f4WQJ"
+ "id": "toolu_vrtx_013confcdST6e9sheNQFrmV9"
}
]
},
"usage": {
- "input": 5815,
+ "input": 5805,
"output": 274
}
},
- "d5ac4e87e2af7f0f6e80eabefb485dbe45c9921d": {
+ "e985be3f0fdacd8dcdb86bc7540bc0d4b102f4ab": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the first order for blue table",
- "copilotToolCallId": "toolu_vrtx_01NQqfZy2VZXGhaFASZqBcaF"
+ "text": "Submit the first order for a blue table",
+ "copilotToolCallId": "toolu_vrtx_01CaraEChMZBTSy5yBU2i5Tt"
},
{
"type": "tool_call",
@@ -186,23 +171,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_01NQqfZy2VZXGhaFASZqBcaF"
+ "id": "toolu_vrtx_01CaraEChMZBTSy5yBU2i5Tt"
}
]
},
"usage": {
- "input": 6268,
- "output": 95
+ "input": 6258,
+ "output": 96
}
},
- "4e2825b7b8d7ec20c52b14540dae9e12dcdfd743": {
+ "0fc3f5e5cde041ffb80ba10114d4d5eb35a5dfcd": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Accept the confirmation dialog for blue table",
- "copilotToolCallId": "toolu_vrtx_013gcm6ax8g2JWKssZfL3Sog"
+ "text": "Accept the confirmation dialog for the blue table order",
+ "copilotToolCallId": "toolu_vrtx_017SiTrBCbZmhLci7SSm123d"
},
{
"type": "tool_call",
@@ -210,23 +195,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_013gcm6ax8g2JWKssZfL3Sog"
+ "id": "toolu_vrtx_017SiTrBCbZmhLci7SSm123d"
}
]
},
"usage": {
- "input": 6438,
- "output": 78
+ "input": 6429,
+ "output": 80
}
},
- "8bfe6aa2bdaaedcef7a1c73f4d420d55e6c292d0": {
+ "55e321443a3b6141b36538f285e21731b68c337c": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click to place another order for the green desk",
- "copilotToolCallId": "toolu_vrtx_01HRHawjDCbqqVhW7PcxmY3r"
+ "text": "Click the button to place another order for the green desk",
+ "copilotToolCallId": "toolu_vrtx_01CxZFPgcehAJyZgFN8neKUk"
},
{
"type": "tool_call",
@@ -235,23 +220,23 @@
"element": "Place Another Order button",
"ref": "e58"
},
- "id": "toolu_vrtx_01HRHawjDCbqqVhW7PcxmY3r"
+ "id": "toolu_vrtx_01CxZFPgcehAJyZgFN8neKUk"
}
]
},
"usage": {
- "input": 7326,
- "output": 99
+ "input": 7319,
+ "output": 101
}
},
- "012fc6fa5f4a194ce969dbdece4f32835bbd5f3f": {
+ "4d1794ac41beff04dc0fa4145315a5bcc6ddf57f": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Desk for the second product",
- "copilotToolCallId": "toolu_vrtx_01YHy8qDmTAnrffRhxo8uneD"
+ "text": "Select 'Desk' as the second product type",
+ "copilotToolCallId": "toolu_vrtx_01SW8tZ18V51sx8UYX9SkVEH"
},
{
"type": "tool_call",
@@ -263,23 +248,23 @@
"Desk"
]
},
- "id": "toolu_vrtx_01YHy8qDmTAnrffRhxo8uneD"
+ "id": "toolu_vrtx_01SW8tZ18V51sx8UYX9SkVEH"
}
]
},
"usage": {
- "input": 8045,
- "output": 118
+ "input": 8040,
+ "output": 122
}
},
- "d5ee9fb0a787ecd5ee180fd181e4a3d5fb6a68f2": {
+ "db83906845c209b0970abe762d515c3eff3e157d": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Green as the color for the desk",
- "copilotToolCallId": "toolu_vrtx_016tGB5BYqmwKAxhynpyMmdm"
+ "text": "Select 'Green' as the color for the desk",
+ "copilotToolCallId": "toolu_vrtx_01FzNfhLHb8P6etWvaZfVoPd"
},
{
"type": "tool_call",
@@ -291,23 +276,23 @@
"Green"
]
},
- "id": "toolu_vrtx_016tGB5BYqmwKAxhynpyMmdm"
+ "id": "toolu_vrtx_01FzNfhLHb8P6etWvaZfVoPd"
}
]
},
"usage": {
- "input": 8311,
- "output": 117
+ "input": 8310,
+ "output": 120
}
},
- "440ee34411e291b4dbe9617db95c196eace37f1e": {
+ "e5766217902078a0f31d7d61a66d431fb8330064": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Fill in the customer details for the second order",
- "copilotToolCallId": "toolu_vrtx_01Af37pxR5WzUMaws88V3sa2"
+ "copilotToolCallId": "toolu_vrtx_01DTZk3sgWdkwEcTnQvNWk8u"
},
{
"type": "tool_call",
@@ -352,23 +337,23 @@
}
]
},
- "id": "toolu_vrtx_01Af37pxR5WzUMaws88V3sa2"
+ "id": "toolu_vrtx_01DTZk3sgWdkwEcTnQvNWk8u"
}
]
},
"usage": {
- "input": 8577,
+ "input": 8579,
"output": 266
}
},
- "81089209173d7b018430ef6887a408fcb2e24e84": {
+ "eb966490e95b464a6ab8834e3bd7cb2909ac17ea": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the second order for green desk",
- "copilotToolCallId": "toolu_vrtx_01JqdAnXaitbPWsyUAP4KuzW"
+ "text": "Submit the second order for a green desk",
+ "copilotToolCallId": "toolu_vrtx_01MhFtYhTdNE99Zzzzc7q2pF"
},
{
"type": "tool_call",
@@ -377,23 +362,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_01JqdAnXaitbPWsyUAP4KuzW"
+ "id": "toolu_vrtx_01MhFtYhTdNE99Zzzzc7q2pF"
}
]
},
"usage": {
- "input": 9030,
- "output": 95
+ "input": 9032,
+ "output": 96
}
},
- "74c9b7b48c20ea2d1385fba336bb20d392637d6c": {
+ "053fba57de7fbb68c913562b623324a34788df24": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Accept the confirmation dialog for green desk",
- "copilotToolCallId": "toolu_vrtx_01UHoxYquxpfX4meEBSstWZh"
+ "text": "Accept the confirmation dialog for the green desk order",
+ "copilotToolCallId": "toolu_vrtx_01CB3dJULpX59rXX3S9Spkbb"
},
{
"type": "tool_call",
@@ -401,23 +386,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01UHoxYquxpfX4meEBSstWZh"
+ "id": "toolu_vrtx_01CB3dJULpX59rXX3S9Spkbb"
}
]
},
"usage": {
- "input": 9200,
- "output": 78
+ "input": 9203,
+ "output": 80
}
},
- "90563878bf810e2606705227663679163b7359b0": {
+ "e62145520af148b1bb0e735f14cddab91f4bf639": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Report the combined total price and maximum delivery time for both orders",
- "copilotToolCallId": "toolu_vrtx_01La4TqsDPCAXcofLq5LApTh"
+ "text": "Report the total price for both items and maximum delivery time. Blue table: $353.98, Green desk: $299.98. Total: $653.96. Both orders have 3 days delivery.",
+ "copilotToolCallId": "toolu_vrtx_01J347sjiTcJ4P3WUq3ehW9R"
},
{
"type": "tool_call",
@@ -426,13 +411,13 @@
"price": 653.96,
"deliveryDays": 3
},
- "id": "toolu_vrtx_01La4TqsDPCAXcofLq5LApTh"
+ "id": "toolu_vrtx_01J347sjiTcJ4P3WUq3ehW9R"
}
]
},
"usage": {
- "input": 10089,
- "output": 103
+ "input": 10094,
+ "output": 134
}
}
}
\ No newline at end of file
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
index cf0083d92d..13df8bbaed 100644
--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
@@ -1,27 +1,27 @@
{
- "3ead929e37fa1cb83a37e09ee827c20293652543": {
+ "9618bcb461bf695d7a9f9dff483361df53be8815": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll help you navigate to the form, fill it out with the specified details, and report back the price and delivery information."
+ "text": "I'll help you navigate to the form, fill in the order details, and report back the results. Let me start by navigating to the page."
}
]
},
"usage": {
- "input": 4471,
- "output": 115
+ "input": 4498,
+ "output": 120
}
},
- "b863793e2b65f06307dd0e7f06d79c720ddf6518": {
+ "6dcc2c52acd06283151dfbd5e29f59186a89e2d5": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Navigate to the form page to start the task",
- "copilotToolCallId": "toolu_vrtx_01EtHjyUbs5JU6AqdzHh3k44"
+ "text": "Navigate to the form page to start the order process",
+ "copilotToolCallId": "toolu_vrtx_01Jekq7CtSogEHaP6U7MYCuZ"
},
{
"type": "tool_call",
@@ -29,23 +29,23 @@
"arguments": {
"url": "http://localhost:<PORT>/evals/fill-form.html"
},
- "id": "toolu_vrtx_01EtHjyUbs5JU6AqdzHh3k44"
+ "id": "toolu_vrtx_01Jekq7CtSogEHaP6U7MYCuZ"
}
]
},
"usage": {
- "input": 4528,
- "output": 92
+ "input": 4560,
+ "output": 93
}
},
- "874e465379d3c65b5c2be0a270730c1242c1dbde": {
+ "e0aa88b89e1a8fe978fbed56774a0f57621ac72d": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Select Table as the first product",
- "copilotToolCallId": "toolu_vrtx_012DUPssGjyvzP2i8Ga7e4LV"
+ "copilotToolCallId": "toolu_vrtx_01BaHkDosZGEZXs6YjuvLdFa"
},
{
"type": "tool_call",
@@ -57,23 +57,23 @@
"Table"
]
},
- "id": "toolu_vrtx_012DUPssGjyvzP2i8Ga7e4LV"
+ "id": "toolu_vrtx_01BaHkDosZGEZXs6YjuvLdFa"
}
]
},
"usage": {
- "input": 5241,
+ "input": 5274,
"output": 116
}
},
- "2b5eec7b6cf93f318562e07f971ac93aaf177fed": {
+ "61aba6d5fc9c3ac919b5336e19b1cd52fdd02121": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Blue color for the table",
- "copilotToolCallId": "toolu_vrtx_017NBjQL34v9FM1xUPp8mxYw"
+ "text": "Select Blue as the color for the table",
+ "copilotToolCallId": "toolu_vrtx_01WN9vgEzcxWmUgDADdX4b3T"
},
{
"type": "tool_call",
@@ -85,84 +85,179 @@
"Blue"
]
},
- "id": "toolu_vrtx_017NBjQL34v9FM1xUPp8mxYw"
+ "id": "toolu_vrtx_01WN9vgEzcxWmUgDADdX4b3T"
}
]
},
"usage": {
- "input": 5504,
- "output": 115
+ "input": 5537,
+ "output": 117
}
},
- "495aa09d1f5a15bd7d5ea2a13fc8e536a8553d8e": {
+ "33d14001ce5af7847b134df5adf2229124538057": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in all the customer information fields",
- "copilotToolCallId": "toolu_vrtx_01E2kVwYUZy4tYqUYc7CBku2"
+ "text": "Fill in the name field",
+ "copilotToolCallId": "toolu_vrtx_019UBYFGPaM1r1s7p2WJut1W"
},
{
"type": "tool_call",
- "name": "browser_fill_form",
+ "name": "browser_type",
"arguments": {
- "fields": [
- {
- "name": "Full Name",
- "type": "textbox",
- "ref": "e13",
- "value": "John Doe"
- },
- {
- "name": "Email Address",
- "type": "textbox",
- "ref": "e16",
- "value": "[email protected]"
- },
- {
- "name": "Street Address",
- "type": "textbox",
- "ref": "e19",
- "value": "123 Main St"
- },
- {
- "name": "City",
- "type": "textbox",
- "ref": "e22",
- "value": "Anytown"
- },
- {
- "name": "State/Province",
- "type": "textbox",
- "ref": "e25",
- "value": "XYZ state"
- },
- {
- "name": "ZIP/Postal Code",
- "type": "textbox",
- "ref": "e28",
- "value": "12345"
- }
- ]
+ "element": "Full Name textbox",
+ "ref": "e13",
+ "text": "John Doe"
+ },
+ "id": "toolu_vrtx_019UBYFGPaM1r1s7p2WJut1W"
+ }
+ ]
+ },
+ "usage": {
+ "input": 5803,
+ "output": 114
+ }
+ },
+ "74038cf37965dc7824a01a036e9a04e4bae6a721": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the email field",
+ "copilotToolCallId": "toolu_vrtx_01MEoP2sPB6DqaGGv3ERgcGs"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "Email Address textbox",
+ "ref": "e16",
+ "text": "[email protected]"
+ },
+ "id": "toolu_vrtx_01MEoP2sPB6DqaGGv3ERgcGs"
+ }
+ ]
+ },
+ "usage": {
+ "input": 5963,
+ "output": 116
+ }
+ },
+ "2136b8e6418e90e20ee465f406579da6bf065bd5": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the street address field",
+ "copilotToolCallId": "toolu_vrtx_011AAMYvdaj9v6Lj6MWRZRdh"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "Street Address textbox",
+ "ref": "e19",
+ "text": "123 Main St"
+ },
+ "id": "toolu_vrtx_011AAMYvdaj9v6Lj6MWRZRdh"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6127,
+ "output": 116
+ }
+ },
+ "29800f02b0f51a774d4e7f69f99b1c9ac4f26511": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the city field with Anytown",
+ "copilotToolCallId": "toolu_vrtx_01X3nicst7BVm8LWbSoHM59s"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "City textbox",
+ "ref": "e22",
+ "text": "Anytown"
+ },
+ "id": "toolu_vrtx_01X3nicst7BVm8LWbSoHM59s"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6290,
+ "output": 117
+ }
+ },
+ "1de325e1e70e801b92a1ad6e1c628fa084d862b8": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the state field with XYZ",
+ "copilotToolCallId": "toolu_vrtx_0154bhMfXbG9vcfEKYHKL8AJ"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "State/Province textbox",
+ "ref": "e25",
+ "text": "XYZ"
+ },
+ "id": "toolu_vrtx_0154bhMfXbG9vcfEKYHKL8AJ"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6452,
+ "output": 119
+ }
+ },
+ "f80886b488400b4a7840b02abf25cc7b8d334394": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the ZIP code field",
+ "copilotToolCallId": "toolu_vrtx_017KCKyqrm6Nr1RJ8MNVjYrU"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "ZIP/Postal Code textbox",
+ "ref": "e28",
+ "text": "12345"
},
- "id": "toolu_vrtx_01E2kVwYUZy4tYqUYc7CBku2"
+ "id": "toolu_vrtx_017KCKyqrm6Nr1RJ8MNVjYrU"
}
]
},
"usage": {
- "input": 5768,
- "output": 344
+ "input": 6618,
+ "output": 117
}
},
- "dcd381e74fa1687308551cdad91683e7b1b498ee": {
+ "6a4d45c6995b3070d336621bc3f3587d9bf23b1d": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click the Order button to submit the first item (blue table)",
- "copilotToolCallId": "toolu_vrtx_014nFz1pRcwFfs5vv4SjJhYL"
+ "text": "Submit the first order (blue table)",
+ "copilotToolCallId": "toolu_vrtx_0195LEteYBG6fxNVGxPuyb3k"
},
{
"type": "tool_call",
@@ -171,23 +266,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_014nFz1pRcwFfs5vv4SjJhYL"
+ "id": "toolu_vrtx_0195LEteYBG6fxNVGxPuyb3k"
}
]
},
"usage": {
- "input": 6219,
- "output": 100
+ "input": 6783,
+ "output": 95
}
},
- "19664a3e596c10157b109c88f2d66e4cb13cc5db": {
+ "7744f2df7189cd0edd36fd53671660cced79f716": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Accept the confirmation dialog for the blue table order",
- "copilotToolCallId": "toolu_vrtx_01NvPAjDPFV9kCUb4e34YMPV"
+ "copilotToolCallId": "toolu_vrtx_01SAAcwWFPqroJzW7My84G4z"
},
{
"type": "tool_call",
@@ -195,23 +290,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01NvPAjDPFV9kCUb4e34YMPV"
+ "id": "toolu_vrtx_01SAAcwWFPqroJzW7My84G4z"
}
]
},
"usage": {
- "input": 6394,
+ "input": 6953,
"output": 80
}
},
- "91a65f3c9adf4ecc0f2541db1da36e61844a3200": {
+ "e8ec627c81c07f9566337a80ee39323d3210abc0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click Place Another Order to order the second item (green desk)",
- "copilotToolCallId": "toolu_vrtx_01Ko8Nf61uLZmXBLU7usYUup"
+ "text": "Click Place Another Order button to order the second item (green desk)",
+ "copilotToolCallId": "toolu_vrtx_01A6XjxYpScgiKggZPmdyyBL"
},
{
"type": "tool_call",
@@ -220,23 +315,23 @@
"element": "Place Another Order button",
"ref": "e58"
},
- "id": "toolu_vrtx_01Ko8Nf61uLZmXBLU7usYUup"
+ "id": "toolu_vrtx_01A6XjxYpScgiKggZPmdyyBL"
}
]
},
"usage": {
- "input": 7284,
- "output": 102
+ "input": 7840,
+ "output": 103
}
},
- "5f0e607d8750e68007f731bdd9c82d1e73c3b8ff": {
+ "acbaa036380782cb995285df7bc2346ef2b6d756": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Select Desk as the second product",
- "copilotToolCallId": "toolu_vrtx_01Mci1LUTixN6P6X7jM5AXqi"
+ "copilotToolCallId": "toolu_vrtx_01Ko4WVozNnZTuvY9AP3YmeC"
},
{
"type": "tool_call",
@@ -248,23 +343,23 @@
"Desk"
]
},
- "id": "toolu_vrtx_01Mci1LUTixN6P6X7jM5AXqi"
+ "id": "toolu_vrtx_01Ko4WVozNnZTuvY9AP3YmeC"
}
]
},
"usage": {
- "input": 8006,
+ "input": 8563,
"output": 118
}
},
- "d4e8a88c5cd3e8c102920f0fd9b17f9fd932bf4d": {
+ "462192a6362e907a2b9ea0e6e545b521a3e3c09d": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Green color for the desk",
- "copilotToolCallId": "toolu_vrtx_018NKZFMWDtMrud3cxqcx7nu"
+ "text": "Select Green as the color for the desk",
+ "copilotToolCallId": "toolu_vrtx_01CDBDKtPA7ZwbTJJRHCUFoM"
},
{
"type": "tool_call",
@@ -276,84 +371,179 @@
"Green"
]
},
- "id": "toolu_vrtx_018NKZFMWDtMrud3cxqcx7nu"
+ "id": "toolu_vrtx_01CDBDKtPA7ZwbTJJRHCUFoM"
}
]
},
"usage": {
- "input": 8272,
- "output": 115
+ "input": 8829,
+ "output": 117
}
},
- "f02a2ac089582728acf943c2871a0791dfb6bcd4": {
+ "13567665222bacf36f5d7157ef02c0d22612676c": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in all the customer information fields for the second order",
- "copilotToolCallId": "toolu_vrtx_014FYeLbYMadQ89LzaQXFdoy"
+ "text": "Fill in the name field for second order",
+ "copilotToolCallId": "toolu_vrtx_01ARssnd9eTw1uHRbWLFQaAA"
},
{
"type": "tool_call",
- "name": "browser_fill_form",
+ "name": "browser_type",
"arguments": {
- "fields": [
- {
- "name": "Full Name",
- "ref": "e13",
- "type": "textbox",
- "value": "John Doe"
- },
- {
- "name": "Email Address",
- "ref": "e16",
- "type": "textbox",
- "value": "[email protected]"
- },
- {
- "name": "Street Address",
- "ref": "e19",
- "type": "textbox",
- "value": "123 Main St"
- },
- {
- "name": "City",
- "ref": "e22",
- "type": "textbox",
- "value": "Anytown"
- },
- {
- "name": "State/Province",
- "ref": "e25",
- "type": "textbox",
- "value": "XYZ state"
- },
- {
- "name": "ZIP/Postal Code",
- "ref": "e28",
- "type": "textbox",
- "value": "12345"
- }
- ]
+ "element": "Full Name textbox",
+ "ref": "e13",
+ "text": "John Doe"
+ },
+ "id": "toolu_vrtx_01ARssnd9eTw1uHRbWLFQaAA"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9095,
+ "output": 117
+ }
+ },
+ "7b8bd077cd1541bfca55303c45fd96a4fdd6337e": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the email field for second order",
+ "copilotToolCallId": "toolu_vrtx_01QwWGsWv9EKzghtCawineA5"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "Email Address textbox",
+ "ref": "e16",
+ "text": "[email protected]"
+ },
+ "id": "toolu_vrtx_01QwWGsWv9EKzghtCawineA5"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9258,
+ "output": 119
+ }
+ },
+ "aec3d303241453b82b622337aea0a774f1490908": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the street address field for second order",
+ "copilotToolCallId": "toolu_vrtx_01KXY1BeK5D62GkRwrKuTzoD"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "Street Address textbox",
+ "ref": "e19",
+ "text": "123 Main St"
+ },
+ "id": "toolu_vrtx_01KXY1BeK5D62GkRwrKuTzoD"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9425,
+ "output": 119
+ }
+ },
+ "ab3c87f72a039a7bd16601c4a77ef6c6f6c8d36f": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the city field for second order",
+ "copilotToolCallId": "toolu_vrtx_01AQ8ymjcSsF9inQD4Vdi9qx"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "City textbox",
+ "ref": "e22",
+ "text": "Anytown"
+ },
+ "id": "toolu_vrtx_01AQ8ymjcSsF9inQD4Vdi9qx"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9591,
+ "output": 116
+ }
+ },
+ "3444e227160908670bb9267f25ab95112550a5b6": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the state field for second order",
+ "copilotToolCallId": "toolu_vrtx_013sgmYiQHkztUVLRY8FUEKq"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "State/Province textbox",
+ "ref": "e25",
+ "text": "XYZ"
+ },
+ "id": "toolu_vrtx_013sgmYiQHkztUVLRY8FUEKq"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9752,
+ "output": 118
+ }
+ },
+ "a284cc1b4f7f15a31cea4f5b6301058cf0935ea4": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the ZIP code field for second order",
+ "copilotToolCallId": "toolu_vrtx_01Ep3dXycgf6gLsB45tQgisL"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "ZIP/Postal Code textbox",
+ "ref": "e28",
+ "text": "12345"
},
- "id": "toolu_vrtx_014FYeLbYMadQ89LzaQXFdoy"
+ "id": "toolu_vrtx_01Ep3dXycgf6gLsB45tQgisL"
}
]
},
"usage": {
- "input": 8536,
- "output": 268
+ "input": 9917,
+ "output": 120
}
},
- "50baf0fb89e0d4b60244120121fb12d9ec326131": {
+ "fd7b954cb041fcf5fa0fb998761dd9b398cc1cee": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click the Order button to submit the second item (green desk)",
- "copilotToolCallId": "toolu_vrtx_01WCTR9W4hh6CrYNvgm25Yjo"
+ "text": "Submit the second order (green desk)",
+ "copilotToolCallId": "toolu_vrtx_01Ugc3Z7yYLZtiTuMifEfhAZ"
},
{
"type": "tool_call",
@@ -362,23 +552,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_01WCTR9W4hh6CrYNvgm25Yjo"
+ "id": "toolu_vrtx_01Ugc3Z7yYLZtiTuMifEfhAZ"
}
]
},
"usage": {
- "input": 8991,
- "output": 100
+ "input": 10085,
+ "output": 95
}
},
- "f58af62434ae328de38eee3842082050991524de": {
+ "6c7a1d4b5bb21c55df6eaaeee1cc3fd93a20978c": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Accept the confirmation dialog for the green desk order",
- "copilotToolCallId": "toolu_vrtx_01D8bGvuBZnBaaHg9vEPhkto"
+ "copilotToolCallId": "toolu_vrtx_01Lpxhy6JBxhSTkJHFemHrEc"
},
{
"type": "tool_call",
@@ -386,23 +576,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01D8bGvuBZnBaaHg9vEPhkto"
+ "id": "toolu_vrtx_01Lpxhy6JBxhSTkJHFemHrEc"
}
]
},
"usage": {
- "input": 9166,
+ "input": 10255,
"output": 80
}
},
- "5c0839470e9ae67f3beeaa8113b7d39e52c616c9": {
+ "7a450713f8ee68d4c1e806ae3144c9c167335ba5": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Report the total price and maximum estimated delivery for both orders. Blue table: $353.98, 3 days. Green desk: $299.98, 3 days. Total: $653.96, max delivery: 3 days.",
- "copilotToolCallId": "toolu_vrtx_01KyKWN5mj68Pe9d7DUbiEpi"
+ "text": "Report the total price and maximum delivery time for both orders: Blue Table ($353.98, 3 days) and Green Desk ($299.98, 3 days). Total: $653.96, Max delivery: 3 days",
+ "copilotToolCallId": "toolu_vrtx_01UvTtaaPPEWqLR1mtJjLB3k"
},
{
"type": "tool_call",
@@ -411,12 +601,12 @@
"price": 653.96,
"deliveryDays": 3
},
- "id": "toolu_vrtx_01KyKWN5mj68Pe9d7DUbiEpi"
+ "id": "toolu_vrtx_01UvTtaaPPEWqLR1mtJjLB3k"
}
]
},
"usage": {
- "input": 10057,
+ "input": 11143,
"output": 143
}
}
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
index 56a5bf4b2f..2f7073a0bc 100644
--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
@@ -1,42 +1,27 @@
{
- "3ead929e37fa1cb83a37e09ee827c20293652543": {
+ "9618bcb461bf695d7a9f9dff483361df53be8815": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll help you navigate to the form, fill it out with the details provided, and report back the price and delivery information."
+ "text": "I'll help you navigate to the form, fill it out with the specified items and details, and report back the results."
}
]
},
"usage": {
- "input": 4471,
- "output": 115
- }
- },
- "f2ac9e2c1fb83e54daedc3a1607ee770dd6cd05c": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I'll navigate to the page and complete the form as requested."
- }
- ]
- },
- "usage": {
- "input": 4528,
- "output": 107
+ "input": 4498,
+ "output": 118
}
},
- "82c674fde5a1942a704bedf38590a35c19551537": {
+ "651ad41a348d6db6488f4746b0b4905094330b85": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Navigate to the form page",
- "copilotToolCallId": "toolu_vrtx_01PUGZnaWbXrBgkeYjhWXPui"
+ "copilotToolCallId": "toolu_vrtx_015A4Qxa4UTA1YLM31KPKBk5"
},
{
"type": "tool_call",
@@ -44,23 +29,23 @@
"arguments": {
"url": "http://localhost:<PORT>/evals/fill-form.html"
},
- "id": "toolu_vrtx_01PUGZnaWbXrBgkeYjhWXPui"
+ "id": "toolu_vrtx_015A4Qxa4UTA1YLM31KPKBk5"
}
]
},
"usage": {
- "input": 4572,
+ "input": 4554,
"output": 88
}
},
- "d1b4efddf9cbdf8b21ebca18d5a73515bbc1ccd1": {
+ "c03b93553be7a4f582a9481cd7ceef812cc3f1a2": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Table as the product type",
- "copilotToolCallId": "toolu_vrtx_01CjJV9pdnAYmTgMmw2xqCik"
+ "text": "Select Table as the first product",
+ "copilotToolCallId": "toolu_vrtx_01URHnaVsJXwPYiCcwLfLqLP"
},
{
"type": "tool_call",
@@ -72,23 +57,23 @@
"Table"
]
},
- "id": "toolu_vrtx_01CjJV9pdnAYmTgMmw2xqCik"
+ "id": "toolu_vrtx_01URHnaVsJXwPYiCcwLfLqLP"
}
]
},
"usage": {
- "input": 5281,
+ "input": 5263,
"output": 116
}
},
- "1340b24ab7000c3ecfe415e169728e151270b626": {
+ "45c9e0f04d5ec6adea27cb529f321db6b7f3a217": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Blue as the color",
- "copilotToolCallId": "toolu_vrtx_01HdgVvtbHAEew3V3SAhSE8e"
+ "text": "Select Blue as the color for the table",
+ "copilotToolCallId": "toolu_vrtx_01MUvkws5Pqa1NKuidpY27Js"
},
{
"type": "tool_call",
@@ -100,84 +85,179 @@
"Blue"
]
},
- "id": "toolu_vrtx_01HdgVvtbHAEew3V3SAhSE8e"
+ "id": "toolu_vrtx_01MUvkws5Pqa1NKuidpY27Js"
}
]
},
"usage": {
- "input": 5544,
- "output": 114
+ "input": 5526,
+ "output": 117
}
},
- "246c11ed0315d3cf3f8362a31b75f996e0b2c75b": {
+ "f01b96c033547e823a688aaa59e57e45d581244f": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in the customer details for the first order",
- "copilotToolCallId": "toolu_vrtx_013EpJmdkVhMZ4vVp7rQcuj9"
+ "text": "Fill in the Full Name field",
+ "copilotToolCallId": "toolu_vrtx_01VzYhHy84gaE8mvmUgmPsoa"
},
{
"type": "tool_call",
- "name": "browser_fill_form",
+ "name": "browser_type",
"arguments": {
- "fields": [
- {
- "name": "Full Name",
- "type": "textbox",
- "ref": "e13",
- "value": "John Doe"
- },
- {
- "name": "Email Address",
- "type": "textbox",
- "ref": "e16",
- "value": "[email protected]"
- },
- {
- "name": "Street Address",
- "type": "textbox",
- "ref": "e19",
- "value": "123 Main St"
- },
- {
- "name": "City",
- "type": "textbox",
- "ref": "e22",
- "value": "Anytown"
- },
- {
- "name": "State/Province",
- "type": "textbox",
- "ref": "e25",
- "value": "XYZ state"
- },
- {
- "name": "ZIP/Postal Code",
- "type": "textbox",
- "ref": "e28",
- "value": "12345"
- }
- ]
+ "element": "Full Name textbox",
+ "ref": "e13",
+ "text": "John Doe"
+ },
+ "id": "toolu_vrtx_01VzYhHy84gaE8mvmUgmPsoa"
+ }
+ ]
+ },
+ "usage": {
+ "input": 5792,
+ "output": 115
+ }
+ },
+ "6de46f95cbe9459b6af5f56cbab64e6a140afa14": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the Email Address field",
+ "copilotToolCallId": "toolu_vrtx_019meT2mNudZyQwmT7fnCrsA"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "Email Address textbox",
+ "ref": "e16",
+ "text": "[email protected]"
+ },
+ "id": "toolu_vrtx_019meT2mNudZyQwmT7fnCrsA"
+ }
+ ]
+ },
+ "usage": {
+ "input": 5953,
+ "output": 117
+ }
+ },
+ "e25f3a5f2011bdddcb17e6797577346ccef2899f": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the Street Address field",
+ "copilotToolCallId": "toolu_vrtx_01LvRRG2trw2FbUE3AZSJk7Z"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "Street Address textbox",
+ "ref": "e19",
+ "text": "123 Main St"
+ },
+ "id": "toolu_vrtx_01LvRRG2trw2FbUE3AZSJk7Z"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6118,
+ "output": 116
+ }
+ },
+ "e939604b2f4bd1eafeb5be883ca4811d8af3c5f6": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the City field",
+ "copilotToolCallId": "toolu_vrtx_01YcSZZtz11FVhUB5ZuCiVHT"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "City textbox",
+ "ref": "e22",
+ "text": "Anytown"
+ },
+ "id": "toolu_vrtx_01YcSZZtz11FVhUB5ZuCiVHT"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6281,
+ "output": 113
+ }
+ },
+ "b7ebb78e78e4c0e4cebfd3802160e994dff80374": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the State/Province field",
+ "copilotToolCallId": "toolu_vrtx_01KR4DM6jFZFX1ze7kMisxa6"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "State/Province textbox",
+ "ref": "e25",
+ "text": "XYZ"
+ },
+ "id": "toolu_vrtx_01KR4DM6jFZFX1ze7kMisxa6"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6439,
+ "output": 117
+ }
+ },
+ "a808e5354acfb3617a6a9723dfe95478e2652e63": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the ZIP/Postal Code field",
+ "copilotToolCallId": "toolu_vrtx_01PtJxTGzWz5ogj7d1nm2aKS"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "ZIP/Postal Code textbox",
+ "ref": "e28",
+ "text": "12345"
},
- "id": "toolu_vrtx_013EpJmdkVhMZ4vVp7rQcuj9"
+ "id": "toolu_vrtx_01PtJxTGzWz5ogj7d1nm2aKS"
}
]
},
"usage": {
- "input": 5807,
- "output": 274
+ "input": 6603,
+ "output": 120
}
},
- "90fd25992c1a647531b834bb4c37dd1aa507a875": {
+ "d9b4832077829284be3acbe935349b329c47518c": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the first order for the blue table",
- "copilotToolCallId": "toolu_vrtx_019fyjP8YWhpbkWqCuWEaEEz"
+ "text": "Submit the order for the blue table",
+ "copilotToolCallId": "toolu_vrtx_01FDvdEQK9V9deeRt3WJLAok"
},
{
"type": "tool_call",
@@ -186,23 +266,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_019fyjP8YWhpbkWqCuWEaEEz"
+ "id": "toolu_vrtx_01FDvdEQK9V9deeRt3WJLAok"
}
]
},
"usage": {
- "input": 6260,
- "output": 96
+ "input": 6771,
+ "output": 95
}
},
- "95b5282664361cd6416210e38366b8aecce3022f": {
+ "2cc8d0e79ffa0eaca9792d2b1e292688effcca6b": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Accept the confirmation dialog for the blue table order",
- "copilotToolCallId": "toolu_vrtx_016sk3xCVtw6856vxqGGBdkB"
+ "text": "Accept the order confirmation for blue table",
+ "copilotToolCallId": "toolu_vrtx_014eUSnWVV85a5yhgtkhx7uN"
},
{
"type": "tool_call",
@@ -210,23 +290,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_016sk3xCVtw6856vxqGGBdkB"
+ "id": "toolu_vrtx_014eUSnWVV85a5yhgtkhx7uN"
}
]
},
"usage": {
- "input": 6431,
- "output": 80
+ "input": 6941,
+ "output": 78
}
},
- "db78696f00789b6fedb3d4f75b7acae7d43514ec": {
+ "92e5599d7913f24dde5c2eedafede877dfc76594": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click the button to place another order for the green desk",
- "copilotToolCallId": "toolu_vrtx_01KmgEbnjXQaUcxsqucKXmKF"
+ "text": "Click Place Another Order button to order the green desk",
+ "copilotToolCallId": "toolu_vrtx_01VWmtWDYCiEQ51LqFSqRVyt"
},
{
"type": "tool_call",
@@ -235,23 +315,23 @@
"element": "Place Another Order button",
"ref": "e58"
},
- "id": "toolu_vrtx_01KmgEbnjXQaUcxsqucKXmKF"
+ "id": "toolu_vrtx_01VWmtWDYCiEQ51LqFSqRVyt"
}
]
},
"usage": {
- "input": 7321,
- "output": 101
+ "input": 7826,
+ "output": 100
}
},
- "66d11ebd5aa97fcbf9002ac6cf7e25453bcca8f7": {
+ "da4ec3f6c3c748eff025f787e4264ad01fcfb0f5": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Desk as the product type for the second order",
- "copilotToolCallId": "toolu_vrtx_01Qwp1HZ551btMZqVfuhmg7K"
+ "text": "Select Desk as the second product",
+ "copilotToolCallId": "toolu_vrtx_018KwJC7fJJSrPKtknozFFFr"
},
{
"type": "tool_call",
@@ -263,23 +343,23 @@
"Desk"
]
},
- "id": "toolu_vrtx_01Qwp1HZ551btMZqVfuhmg7K"
+ "id": "toolu_vrtx_018KwJC7fJJSrPKtknozFFFr"
}
]
},
"usage": {
- "input": 8042,
- "output": 122
+ "input": 8546,
+ "output": 118
}
},
- "b9ea532ba3932051ca53f8995e4a84673ac15f15": {
+ "6ee97d09a08742750496ab8b193d715eca619172": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Select Green as the color for the desk",
- "copilotToolCallId": "toolu_vrtx_01UcgZrsRtnDRtamzcXiyhEz"
+ "copilotToolCallId": "toolu_vrtx_01PAWTp8xxLpWDcekkdVD9bF"
},
{
"type": "tool_call",
@@ -291,84 +371,179 @@
"Green"
]
},
- "id": "toolu_vrtx_01UcgZrsRtnDRtamzcXiyhEz"
+ "id": "toolu_vrtx_01PAWTp8xxLpWDcekkdVD9bF"
}
]
},
"usage": {
- "input": 8312,
+ "input": 8812,
"output": 117
}
},
- "f062f1459e30461f856bac9eaf6c9ba4c0c83445": {
+ "b6dfb1610253206d76e22c6cb14b79e1a8868dc5": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in the customer details for the second order",
- "copilotToolCallId": "toolu_vrtx_01S4ko4cV4txtLXSwhz12t1d"
+ "text": "Fill in the Full Name field for second order",
+ "copilotToolCallId": "toolu_vrtx_018Usc5oevz9wb9ZKpftC4sH"
},
{
"type": "tool_call",
- "name": "browser_fill_form",
+ "name": "browser_type",
"arguments": {
- "fields": [
- {
- "name": "Full Name",
- "ref": "e13",
- "type": "textbox",
- "value": "John Doe"
- },
- {
- "name": "Email Address",
- "ref": "e16",
- "type": "textbox",
- "value": "[email protected]"
- },
- {
- "name": "Street Address",
- "ref": "e19",
- "type": "textbox",
- "value": "123 Main St"
- },
- {
- "name": "City",
- "ref": "e22",
- "type": "textbox",
- "value": "Anytown"
- },
- {
- "name": "State/Province",
- "ref": "e25",
- "type": "textbox",
- "value": "XYZ state"
- },
- {
- "name": "ZIP/Postal Code",
- "ref": "e28",
- "type": "textbox",
- "value": "12345"
- }
- ]
+ "element": "Full Name textbox",
+ "ref": "e13",
+ "text": "John Doe"
+ },
+ "id": "toolu_vrtx_018Usc5oevz9wb9ZKpftC4sH"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9078,
+ "output": 118
+ }
+ },
+ "bb68d65b11d6f1744be333a30b4b2a2910559d8b": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the Email Address field for second order",
+ "copilotToolCallId": "toolu_vrtx_0157WoyBbU9kXomtZMpLvnBd"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "Email Address textbox",
+ "ref": "e16",
+ "text": "[email protected]"
+ },
+ "id": "toolu_vrtx_0157WoyBbU9kXomtZMpLvnBd"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9242,
+ "output": 120
+ }
+ },
+ "a38c3850dcde9ef2dafcc13e03d17212dbd2de9d": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the Street Address field for second order",
+ "copilotToolCallId": "toolu_vrtx_01F9YjYR5DXdVYf8bebw1fdG"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "Street Address textbox",
+ "ref": "e19",
+ "text": "123 Main St"
+ },
+ "id": "toolu_vrtx_01F9YjYR5DXdVYf8bebw1fdG"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9410,
+ "output": 119
+ }
+ },
+ "59648493dce87d10e25e5d50abebac4e2392ecc1": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the City field for second order",
+ "copilotToolCallId": "toolu_vrtx_01BETdPBJWrtPMurfg49Vjoj"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "City textbox",
+ "ref": "e22",
+ "text": "Anytown"
+ },
+ "id": "toolu_vrtx_01BETdPBJWrtPMurfg49Vjoj"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9576,
+ "output": 116
+ }
+ },
+ "a87a7f06ffdf0d79613739ac83936a87142c6d7f": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the State/Province field for second order",
+ "copilotToolCallId": "toolu_vrtx_01KXjCLqC74fLfD56Yb2rb3T"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "State/Province textbox",
+ "ref": "e25",
+ "text": "XYZ"
+ },
+ "id": "toolu_vrtx_01KXjCLqC74fLfD56Yb2rb3T"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9737,
+ "output": 120
+ }
+ },
+ "0edb63f376b23ae85e2319c8dcfd8167be315357": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Fill in the ZIP/Postal Code field for second order",
+ "copilotToolCallId": "toolu_vrtx_01QdpV2UD1ynJBEz3t2vEBZY"
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "ZIP/Postal Code textbox",
+ "ref": "e28",
+ "text": "12345"
},
- "id": "toolu_vrtx_01S4ko4cV4txtLXSwhz12t1d"
+ "id": "toolu_vrtx_01QdpV2UD1ynJBEz3t2vEBZY"
}
]
},
"usage": {
- "input": 8578,
- "output": 266
+ "input": 9904,
+ "output": 123
}
},
- "fdcd0330d001ae7fd2401858d3ece675b149b65d": {
+ "4a49bf78bda39f22b05bb0dabdb778eab42f5d0e": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the second order for the green desk",
- "copilotToolCallId": "toolu_vrtx_015dGJ2kCyGMmwB7P6e3vRcE"
+ "text": "Submit the order for the green desk",
+ "copilotToolCallId": "toolu_vrtx_018WKajMd3gK6LsoHkMSqqF4"
},
{
"type": "tool_call",
@@ -377,23 +552,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_015dGJ2kCyGMmwB7P6e3vRcE"
+ "id": "toolu_vrtx_018WKajMd3gK6LsoHkMSqqF4"
}
]
},
"usage": {
- "input": 9031,
- "output": 96
+ "input": 10075,
+ "output": 95
}
},
- "09e90872eebfad23b65c6b8baf6a21f4d3e3be13": {
+ "9a1806f90b31427e91b08c182a1f530b01687395": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Accept the confirmation dialog for the green desk order",
- "copilotToolCallId": "toolu_vrtx_017Aese7AEKa1XYZoZjeTw6z"
+ "text": "Accept the order confirmation for green desk",
+ "copilotToolCallId": "toolu_vrtx_01958UH1ER5ox2DiKfqF6gSp"
},
{
"type": "tool_call",
@@ -401,23 +576,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_017Aese7AEKa1XYZoZjeTw6z"
+ "id": "toolu_vrtx_01958UH1ER5ox2DiKfqF6gSp"
}
]
},
"usage": {
- "input": 9202,
- "output": 80
+ "input": 10245,
+ "output": 78
}
},
- "28d9510654099e5564df19869f8609adff0bfaae": {
+ "70d0aca5ce593777cf52a420da7aece1173004a1": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Report the total price and maximum delivery time for both orders. Blue table: $353.98, 3 days. Green desk: $299.98, 3 days. Total: $653.96, Max delivery: 3 days.",
- "copilotToolCallId": "toolu_vrtx_01Umx8ecJT1VFHfNNJx6vcX5"
+ "text": "Report the total price for both items ($353.98 + $299.98 = $653.96) and maximum estimated delivery (3 days for both)",
+ "copilotToolCallId": "toolu_vrtx_01R97Mi28WVqRvEH25ky3aqa"
},
{
"type": "tool_call",
@@ -426,13 +601,13 @@
"price": 653.96,
"deliveryDays": 3
},
- "id": "toolu_vrtx_01Umx8ecJT1VFHfNNJx6vcX5"
+ "id": "toolu_vrtx_01R97Mi28WVqRvEH25ky3aqa"
}
]
},
"usage": {
- "input": 10093,
- "output": 143
+ "input": 11131,
+ "output": 125
}
}
}
\ No newline at end of file
diff --git a/tests/mcp/pdf.spec.ts b/tests/mcp/pdf.spec.ts
index 937eb16c87..80faf0b4b2 100644
--- a/tests/mcp/pdf.spec.ts
+++ b/tests/mcp/pdf.spec.ts
@@ -51,7 +51,7 @@ test('save as pdf', async ({ startClient, mcpBrowser, server }, testInfo) => {
name: 'browser_pdf_save',
})).toHaveResponse({
code: expect.stringContaining(`await page.pdf(`),
- result: expect.stringMatching(/Saved page as.*page-[^:]+.pdf/),
+ files: expect.stringMatching(/\[Page saved as PDF\]\(.*page-[^:]+.pdf\)/),
});
});
@@ -75,7 +75,7 @@ test('save as pdf (filename: output.pdf)', async ({ startClient, mcpBrowser, ser
filename: 'output.pdf',
},
})).toHaveResponse({
- result: expect.stringContaining(`output.pdf`),
+ files: expect.stringContaining(`output.pdf`),
code: expect.stringContaining(`await page.pdf(`),
});
diff --git a/tests/mcp/snapshot-mode.spec.ts b/tests/mcp/snapshot-mode.spec.ts
index d9d68e3b5b..912f938e51 100644
--- a/tests/mcp/snapshot-mode.spec.ts
+++ b/tests/mcp/snapshot-mode.spec.ts
@@ -99,3 +99,29 @@ test('should respect --snapshot-mode=none', async ({ startClient, server }) => {
pageState: undefined
});
});
+
+test('should respect snapshot[filename]', async ({ startClient, server }, testInfo) => {
+ server.setContent('/', `<button>Button 1</button>`, 'text/html');
+
+ const outputDir = testInfo.outputPath('output');
+ const { client } = await startClient({
+ config: { outputDir },
+ });
+
+ await client.callTool({
+ name: 'browser_navigate',
+ arguments: {
+ url: server.PREFIX,
+ },
+ });
+
+ expect(await client.callTool({
+ name: 'browser_snapshot',
+ arguments: {
+ filename: 'snapshot1.md',
+ },
+ })).toHaveResponse({
+ pageState: undefined,
+ files: expect.stringMatching(/\[Saved snapshot\]\(.*md\)/)
+ });
+});