Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(mcp): render structured response (#38419)

Pavel Feldmanacedc3fd6d64
diff --git a/packages/playwright/src/agents/performTask.ts b/packages/playwright/src/agents/performTask.ts
index 0c62846b07..d8da6ac287 100644
--- a/packages/playwright/src/agents/performTask.ts
+++ b/packages/playwright/src/agents/performTask.ts
@@ -25,21 +25,34 @@ import { wrapInClient } from '../mcp/sdk/server';
import type * as playwright from 'playwright-core';
import type * as lowireLoop from '@lowire/loop';
-export async function performTask(context: playwright.BrowserContext, task: string) {
+export type PerformTaskOptions = {
+ provider?: 'github' | 'openai' | 'anthropic' | 'google';
+ model?: string;
+ maxTokens?: number;
+ reasoning?: boolean;
+ temperature?: number;
+};
+
+export async function performTask(context: playwright.BrowserContext, task: string, options: PerformTaskOptions) {
const backend = new BrowserServerBackend(defaultConfig, identityBrowserContextFactory(context));
const client = await wrapInClient(backend, { name: 'Internal', version: '0.0.0' });
- const loop = new Loop('github', { model: 'claude-sonnet-4.5' });
-
const callTool: (params: { name: string, arguments: any}) => Promise<lowireLoop.ToolResult> = async params => {
return await client.callTool(params) as lowireLoop.ToolResult;
};
+ const loop = new Loop(options.provider ?? 'github', {
+ model: options.model ?? 'claude-sonnet-4.5',
+ reasoning: options.reasoning,
+ temperature: options.temperature,
+ maxTokens: options.maxTokens,
+ summarize: true,
+ debug,
+ callTool,
+ tools: await backend.listTools(),
+ });
+
try {
- return await loop.run(task, {
- tools: await backend.listTools(),
- callTool,
- debug,
- });
+ return await loop.run(task);
} finally {
await client.close();
}
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index 5265de9cd4..56c1fc9b3b 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -36,6 +36,7 @@ import type { APIRequestContext as APIRequestContextImpl } from '../../playwrigh
import type { ChannelOwner } from '../../playwright-core/src/client/channelOwner';
import type { Page as PageImpl } from '../../playwright-core/src/client/page';
import type { BrowserContext, BrowserContextOptions, LaunchOptions, Page, Tracing } from 'playwright-core';
+import type { PerformTaskOptions } from './agents/performTask';
export { expect } from './matchers/expect';
export const _baseTest: TestType<{}, {}> = rootTestType.test;
@@ -58,7 +59,7 @@ type TestFixtures = PlaywrightTestArgs & PlaywrightTestOptions & {
_combinedContextOptions: BrowserContextOptions,
_setupContextOptions: void;
_setupArtifacts: void;
- _perform: (task: string) => Promise<void>;
+ _perform: (task: string, options?: PerformTaskOptions) => Promise<void>;
_contextFactory: (options?: BrowserContextOptions) => Promise<{ context: BrowserContext, close: () => Promise<void> }>;
};
@@ -462,8 +463,8 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
},
_perform: async ({ context }, use) => {
- await use(async (task: string) => {
- await performTask(context, task);
+ await use(async (task: string, options?: PerformTaskOptions) => {
+ await performTask(context, task, options ?? {});
});
},
});
diff --git a/packages/playwright/src/mcp/browser/browserServerBackend.ts b/packages/playwright/src/mcp/browser/browserServerBackend.ts
index 9387f86f8a..dfc0ec82a8 100644
--- a/packages/playwright/src/mcp/browser/browserServerBackend.ts
+++ b/packages/playwright/src/mcp/browser/browserServerBackend.ts
@@ -73,7 +73,8 @@ export class BrowserServerBackend implements ServerBackend {
context.setRunningTool(undefined);
}
response.logEnd();
- return response.serialize();
+ const _meta = rawArguments?._meta as object | undefined;
+ return response.serialize({ _meta });
}
serverClosed() {
diff --git a/packages/playwright/src/mcp/browser/response.ts b/packages/playwright/src/mcp/browser/response.ts
index 1f6be9473b..36aeed91b5 100644
--- a/packages/playwright/src/mcp/browser/response.ts
+++ b/packages/playwright/src/mcp/browser/response.ts
@@ -20,6 +20,7 @@ import { renderModalStates } from './tab';
import type { Tab, TabSnapshot } from './tab';
import type { CallToolResult, ImageContent, TextContent } from '@modelcontextprotocol/sdk/types.js';
import type { Context } from './context';
+import type { ModalState } from './tools/tool';
export const requestDebug = debug('pw:mcp:request');
@@ -30,6 +31,7 @@ export class Response {
private _context: Context;
private _includeSnapshot: 'none' | 'full' | 'incremental' = 'none';
private _includeTabs = false;
+ private _includeModalStates: ModalState[] | undefined;
private _tabSnapshot: TabSnapshot | undefined;
readonly toolName: string;
@@ -87,6 +89,10 @@ export class Response {
this._includeTabs = true;
}
+ setIncludeModalStates(modalStates: ModalState[]) {
+ this._includeModalStates = modalStates;
+ }
+
async finish() {
// All the async snapshotting post-action is happening here.
// Everything below should race against modal states.
@@ -110,42 +116,44 @@ export class Response {
requestDebug(this.serialize({ omitSnapshot: true, omitBlobs: true }));
}
- serialize(options: { omitSnapshot?: boolean, omitBlobs?: boolean } = {}): { content: (TextContent | ImageContent)[], isError?: boolean } {
- const response: string[] = [];
+ serialize(options: { omitSnapshot?: boolean, omitBlobs?: boolean, _meta?: Record<string, any> } = {}): { content: (TextContent | ImageContent)[], isError?: boolean, _meta?: Record<string, any> } {
+ const renderedResponse = new RenderedResponse();
- // Start with command result.
- if (this._result.length) {
- response.push('### Result');
- response.push(this._result.join('\n'));
- response.push('');
- }
+ if (this._result.length)
+ renderedResponse.results.push(...this._result);
// Add code if it exists.
- if (this._code.length) {
- response.push(`### Ran Playwright code
-\`\`\`js
-${this._code.join('\n')}
-\`\`\``);
- response.push('');
- }
+ if (this._code.length)
+ renderedResponse.code.push(...this._code);
// List browser tabs.
- if (this._includeSnapshot !== 'none' || this._includeTabs)
- response.push(...renderTabsMarkdown(this._context.tabs(), this._includeTabs));
+ if (this._includeSnapshot !== 'none' || this._includeTabs) {
+ const tabsMarkdown = renderTabsMarkdown(this._context.tabs(), this._includeTabs);
+ if (tabsMarkdown.length)
+ renderedResponse.states.tabs = tabsMarkdown.join('\n');
+ }
// Add snapshot if provided.
if (this._tabSnapshot?.modalStates.length) {
- response.push(...renderModalStates(this._context, this._tabSnapshot.modalStates));
- response.push('');
+ const modalStatesMarkdown = renderModalStates(this._tabSnapshot.modalStates);
+ renderedResponse.states.modal = modalStatesMarkdown.join('\n');
} else if (this._tabSnapshot) {
const includeSnapshot = options.omitSnapshot ? 'none' : this._includeSnapshot;
- response.push(renderTabSnapshot(this._tabSnapshot, includeSnapshot));
- response.push('');
+ renderTabSnapshot(this._tabSnapshot, includeSnapshot, renderedResponse);
+ } else if (this._includeModalStates) {
+ const modalStatesMarkdown = renderModalStates(this._includeModalStates);
+ renderedResponse.states.modal = modalStatesMarkdown.join('\n');
}
+ const redactedResponse = this._context.config.secrets ? renderedResponse.redact(this._context.config.secrets) : renderedResponse;
+
+ // Structured response.
+ const includeMeta = options._meta && 'dev.lowire/history' in options._meta && 'dev.lowire/state' in options._meta;
+ const _meta: any = includeMeta ? redactedResponse.asMeta() : undefined;
+
// Main response part
const content: (TextContent | ImageContent)[] = [
- { type: 'text', text: response.join('\n') },
+ { type: 'text', text: redactedResponse.asText() },
];
// Image attachments.
@@ -154,50 +162,39 @@ ${this._code.join('\n')}
content.push({ type: 'image', data: options.omitBlobs ? '<blob>' : image.data.toString('base64'), mimeType: image.contentType });
}
- this._redactSecrets(content);
- return { content, isError: this._isError };
- }
-
- private _redactSecrets(content: (TextContent | ImageContent)[]) {
- if (!this._context.config.secrets)
- return;
-
- for (const item of content) {
- if (item.type !== 'text')
- continue;
- for (const [secretName, secretValue] of Object.entries(this._context.config.secrets))
- item.text = item.text.replaceAll(secretValue, `<secret>${secretName}</secret>`);
- }
+ return {
+ _meta,
+ content,
+ isError: this._isError
+ };
}
}
-function renderTabSnapshot(tabSnapshot: TabSnapshot, includeSnapshot: 'none' | 'full' | 'incremental'): string {
- const lines: string[] = [];
-
+function renderTabSnapshot(tabSnapshot: TabSnapshot, includeSnapshot: 'none' | 'full' | 'incremental', response: RenderedResponse) {
if (tabSnapshot.consoleMessages.length) {
- lines.push(`### New console messages`);
+ const lines: string[] = [];
for (const message of tabSnapshot.consoleMessages)
lines.push(`- ${trim(message.toString(), 100)}`);
- lines.push('');
+ response.updates.push({ category: 'console', content: lines.join('\n') });
}
if (tabSnapshot.downloads.length) {
- lines.push(`### Downloads`);
+ const lines: string[] = [];
for (const entry of tabSnapshot.downloads) {
if (entry.finished)
lines.push(`- Downloaded file ${entry.download.suggestedFilename()} to ${entry.outputFile}`);
else
lines.push(`- Downloading file ${entry.download.suggestedFilename()} ...`);
}
- lines.push('');
+ response.updates.push({ category: 'downloads', content: lines.join('\n') });
}
if (includeSnapshot === 'incremental' && tabSnapshot.ariaSnapshotDiff === '') {
// When incremental snapshot is present, but empty, do not render page state altogether.
- return lines.join('\n');
+ return;
}
- lines.push(`### Page state`);
+ const lines: string[] = [];
lines.push(`- Page URL: ${tabSnapshot.url}`);
lines.push(`- Page Title: ${tabSnapshot.title}`);
@@ -210,29 +207,22 @@ function renderTabSnapshot(tabSnapshot: TabSnapshot, includeSnapshot: 'none' | '
lines.push(tabSnapshot.ariaSnapshot);
lines.push('```');
}
-
- return lines.join('\n');
+ response.states.page = lines.join('\n');
}
function renderTabsMarkdown(tabs: Tab[], force: boolean = false): string[] {
if (tabs.length === 1 && !force)
return [];
- if (!tabs.length) {
- return [
- '### Open tabs',
- 'No open tabs. Use the "browser_navigate" tool to navigate to a page first.',
- '',
- ];
- }
+ if (!tabs.length)
+ return ['No open tabs. Use the "browser_navigate" tool to navigate to a page first.'];
- const lines: string[] = ['### Open tabs'];
+ const lines: string[] = [];
for (let i = 0; i < tabs.length; i++) {
const tab = tabs[i];
const current = tab.isCurrentTab() ? ' (current)' : '';
lines.push(`- ${i}:${current} [${tab.lastTitle()}] (${tab.page.url()})`);
}
- lines.push('');
return lines;
}
@@ -242,6 +232,86 @@ function trim(text: string, maxLength: number) {
return text.slice(0, maxLength) + '...';
}
+export class RenderedResponse {
+ readonly states: Partial<Record<'page' | 'tabs' | 'modal', string>> = {};
+ readonly updates: { category: 'console' | 'downloads', content: string }[] = [];
+ readonly results: string[] = [];
+ readonly code: string[] = [];
+
+ constructor(copy?: { states: Partial<Record<'page' | 'tabs' | 'modal', string>>, updates: { category: 'console' | 'downloads', content: string }[], results: string[], code: string[] }) {
+ if (copy) {
+ this.states = copy.states;
+ this.updates = copy.updates;
+ this.results = copy.results;
+ this.code = copy.code;
+ }
+ }
+
+ asText(): string {
+ const text: string[] = [];
+ if (this.results.length)
+ text.push(`### Result\n${this.results.join('\n')}\n`);
+ if (this.code.length)
+ text.push(`### Ran Playwright code\n${this.code.join('\n')}\n`);
+
+ for (const { category, content } of this.updates) {
+ if (!content.trim())
+ continue;
+
+ switch (category) {
+ case 'console':
+ text.push(`### New console messages\n${content}\n`);
+ break;
+ case 'downloads':
+ text.push(`### Downloads\n${content}\n`);
+ break;
+ }
+ }
+
+ for (const [category, value] of Object.entries(this.states)) {
+ if (!value.trim())
+ continue;
+
+ switch (category) {
+ case 'page':
+ text.push(`### Page state\n${value}\n`);
+ break;
+ case 'tabs':
+ text.push(`### Open tabs\n${value}\n`);
+ break;
+ case 'modal':
+ text.push(`### Modal state\n${value}\n`);
+ break;
+ }
+ }
+ return text.join('\n');
+ }
+
+ asMeta() {
+ const codeUpdate = this.code.length ? { category: 'code', content: this.code.join('\n') } : undefined;
+ const resultUpdate = this.results.length ? { category: 'result', content: this.results.join('\n') } : undefined;
+ const updates = [resultUpdate, codeUpdate, ...this.updates].filter(Boolean);
+ return {
+ 'dev.lowire/history': updates,
+ 'dev.lowire/state': { ...this.states },
+ };
+ }
+
+ redact(secrets: Record<string, string>): RenderedResponse {
+ const redactText = (text: string) => {
+ for (const [secretName, secretValue] of Object.entries(secrets))
+ text = text.replaceAll(secretValue, `<secret>${secretName}</secret>`);
+ return text;
+ };
+
+ const updates = this.updates.map(update => ({ ...update, content: redactText(update.content) }));
+ const results = this.results.map(result => redactText(result));
+ const code = this.code.map(code => redactText(code));
+ const states = Object.fromEntries(Object.entries(this.states).map(([key, value]) => [key, redactText(value)]));
+ return new RenderedResponse({ states, updates, results, code });
+ }
+}
+
function parseSections(text: string): Map<string, string> {
const sections = new Map<string, string>();
const sectionHeaders = text.split(/^### /m).slice(1); // Remove empty first element
@@ -286,5 +356,6 @@ export function parseResponse(response: CallToolResult) {
downloads,
isError,
attachments,
+ _meta: response._meta,
};
}
diff --git a/packages/playwright/src/mcp/browser/tab.ts b/packages/playwright/src/mcp/browser/tab.ts
index 6bf802c0a0..d254b115bf 100644
--- a/packages/playwright/src/mcp/browser/tab.ts
+++ b/packages/playwright/src/mcp/browser/tab.ts
@@ -132,10 +132,6 @@ export class Tab extends EventEmitter<TabEventsInterface> {
this._modalStates = this._modalStates.filter(state => state !== modalState);
}
- modalStatesMarkdown(): string[] {
- return renderModalStates(this.context, this.modalStates());
- }
-
private _dialogShown(dialog: playwright.Dialog) {
this.setModalState({
type: 'dialog',
@@ -349,8 +345,8 @@ function pageErrorToConsoleMessage(errorOrValue: Error | any): ConsoleMessage {
};
}
-export function renderModalStates(context: Context, modalStates: ModalState[]): string[] {
- const result: string[] = ['### Modal state'];
+export function renderModalStates(modalStates: ModalState[]): string[] {
+ const result: string[] = [];
if (modalStates.length === 0)
result.push('- There is no modal state present');
for (const state of modalStates)
diff --git a/packages/playwright/src/mcp/browser/tools/tool.ts b/packages/playwright/src/mcp/browser/tools/tool.ts
index 131c4f0115..a8452b882c 100644
--- a/packages/playwright/src/mcp/browser/tools/tool.ts
+++ b/packages/playwright/src/mcp/browser/tools/tool.ts
@@ -61,12 +61,15 @@ export function defineTabTool<Input extends z.Schema>(tool: TabTool<Input>): Too
handle: async (context, params, response) => {
const tab = await context.ensureTab();
const modalStates = tab.modalStates().map(state => state.type);
- if (tool.clearsModalState && !modalStates.includes(tool.clearsModalState))
- response.addError(`Error: The tool "${tool.schema.name}" can only be used when there is related modal state present.\n` + tab.modalStatesMarkdown().join('\n'));
- else if (!tool.clearsModalState && modalStates.length)
- response.addError(`Error: Tool "${tool.schema.name}" does not handle the modal state.\n` + tab.modalStatesMarkdown().join('\n'));
- else
+ if (tool.clearsModalState && !modalStates.includes(tool.clearsModalState)) {
+ response.setIncludeModalStates(tab.modalStates());
+ response.addError(`Error: The tool "${tool.schema.name}" can only be used when there is related modal state present.`);
+ } else if (!tool.clearsModalState && modalStates.length) {
+ response.setIncludeModalStates(tab.modalStates());
+ response.addError(`Error: Tool "${tool.schema.name}" does not handle the modal state.`);
+ } else {
return tool.handle(tab, params, response);
+ }
},
};
}
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
index 97698a4181..16c1d734a6 100644
--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
@@ -38,14 +38,14 @@
"output": 91
}
},
- "2ab792bc44bbdfeb68d86a9483a652ccff11cae6": {
+ "83da5a4a79d93a494d7f0f76d4758bc493d411f5": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Table as the product type",
- "copilotToolCallId": "toolu_vrtx_01DznCav5ifDdRS4RRAfr23m"
+ "text": "Select 'Table' as the product type for the first item",
+ "copilotToolCallId": "toolu_vrtx_011Vy4WSELJYN4U2tv9NgCLi"
},
{
"type": "tool_call",
@@ -57,23 +57,23 @@
"Table"
]
},
- "id": "toolu_vrtx_01DznCav5ifDdRS4RRAfr23m"
+ "id": "toolu_vrtx_011Vy4WSELJYN4U2tv9NgCLi"
}
]
},
"usage": {
- "input": 5171,
- "output": 116
+ "input": 5166,
+ "output": 123
}
},
- "46836443d1461732bfdc0e9af2de0eb77bd0f8d4": {
+ "0a13ca0491d6745366d7b71a32b864412e848d78": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Blue as the color",
- "copilotToolCallId": "toolu_vrtx_015cWm2XCDs1XNu1jRBeJiUY"
+ "text": "Select 'Blue' as the color for the table",
+ "copilotToolCallId": "toolu_vrtx_01AyhQJov7bZQ7dY9CmcZMeY"
},
{
"type": "tool_call",
@@ -85,23 +85,23 @@
"Blue"
]
},
- "id": "toolu_vrtx_015cWm2XCDs1XNu1jRBeJiUY"
+ "id": "toolu_vrtx_01AyhQJov7bZQ7dY9CmcZMeY"
}
]
},
"usage": {
- "input": 5439,
- "output": 114
+ "input": 5436,
+ "output": 120
}
},
- "3598c77979683f0f5defc155a6ae1adfc8420796": {
+ "0e1345afd5cefb63d40c0d9750aa6b73dfe36f38": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in all the form fields with customer information",
- "copilotToolCallId": "toolu_vrtx_013R8pvxWNKKEiGdH24u4RG7"
+ "text": "Fill in the customer details for the order",
+ "copilotToolCallId": "toolu_vrtx_011xPGae2W6mMbh3TXpbqjaH"
},
{
"type": "tool_call",
@@ -146,23 +146,23 @@
}
]
},
- "id": "toolu_vrtx_013R8pvxWNKKEiGdH24u4RG7"
+ "id": "toolu_vrtx_011xPGae2W6mMbh3TXpbqjaH"
}
]
},
"usage": {
- "input": 5707,
- "output": 277
+ "input": 5705,
+ "output": 348
}
},
- "e7b570342edf7c51d9f100944c8147ff8097531d": {
+ "90b95b16881abda623068841009641e51f32e7ae": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the order for the blue table",
- "copilotToolCallId": "toolu_vrtx_013EvTF42FSrCTpZNjTt1Jg9"
+ "text": "Click the Order button to submit the first order (blue table)",
+ "copilotToolCallId": "toolu_vrtx_01FwpyQSshn5YP7JyJbptXEh"
},
{
"type": "tool_call",
@@ -171,23 +171,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_013EvTF42FSrCTpZNjTt1Jg9"
+ "id": "toolu_vrtx_01FwpyQSshn5YP7JyJbptXEh"
}
]
},
"usage": {
- "input": 6171,
- "output": 95
+ "input": 6163,
+ "output": 100
}
},
- "91aacbc6d7de5840abbd469624b03f4acd2ea249": {
+ "7160f78eeba943a8bf0daac205c6b183839401ca": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Accept the confirmation dialog for the blue table order",
- "copilotToolCallId": "toolu_vrtx_01KbijiKyMFHXBkDykX2HEKF"
+ "copilotToolCallId": "toolu_vrtx_01LW5t6fGbyzj4ARj6wATf6y"
},
{
"type": "tool_call",
@@ -195,23 +195,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01KbijiKyMFHXBkDykX2HEKF"
+ "id": "toolu_vrtx_01LW5t6fGbyzj4ARj6wATf6y"
}
]
},
"usage": {
- "input": 6346,
+ "input": 6338,
"output": 80
}
},
- "a98731ce88937b3750262cf96e32915137da41f6": {
+ "80f4d8e16d86fb67f34dae4668d2ac84c71ac3ee": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click the button to place another order for the green desk",
- "copilotToolCallId": "toolu_vrtx_01Y1PPdE3BPW3QDvfGDfNvud"
+ "text": "Click 'Place Another Order' button to order the second item (green desk)",
+ "copilotToolCallId": "toolu_vrtx_01Y6z1hSQqTdsRs6VhoAkNrJ"
},
{
"type": "tool_call",
@@ -220,23 +220,23 @@
"element": "Place Another Order button",
"ref": "e58"
},
- "id": "toolu_vrtx_01Y1PPdE3BPW3QDvfGDfNvud"
+ "id": "toolu_vrtx_01Y6z1hSQqTdsRs6VhoAkNrJ"
}
]
},
"usage": {
- "input": 7244,
- "output": 101
+ "input": 7236,
+ "output": 106
}
},
- "6d42d0e007bf912126b91d0f5ff914f4f0ddd131": {
+ "8a7a2b461872a308c3c471069553fbb110b233e2": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Desk as the product type for the second order",
- "copilotToolCallId": "toolu_vrtx_01218mifcyQA471NRm6dUcRU"
+ "text": "Select 'Desk' as the product type for the second item",
+ "copilotToolCallId": "toolu_vrtx_01XFkp8Sb3zfxfSX56VB5Ukp"
},
{
"type": "tool_call",
@@ -248,23 +248,23 @@
"Desk"
]
},
- "id": "toolu_vrtx_01218mifcyQA471NRm6dUcRU"
+ "id": "toolu_vrtx_01XFkp8Sb3zfxfSX56VB5Ukp"
}
]
},
"usage": {
- "input": 7970,
- "output": 122
+ "input": 7962,
+ "output": 125
}
},
- "5025e2902f3df07923d7734a597be3713f677a74": {
+ "aa6c7116bcda3f6079c1d67f86e781f745b9511f": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select Green as the color for the desk",
- "copilotToolCallId": "toolu_vrtx_01FhmtZ6f2hJPYjjSoKC97y8"
+ "text": "Select 'Green' as the color for the desk",
+ "copilotToolCallId": "toolu_vrtx_01BBnzzEU2bFJdGS8XeWbKbU"
},
{
"type": "tool_call",
@@ -276,23 +276,23 @@
"Green"
]
},
- "id": "toolu_vrtx_01FhmtZ6f2hJPYjjSoKC97y8"
+ "id": "toolu_vrtx_01BBnzzEU2bFJdGS8XeWbKbU"
}
]
},
"usage": {
- "input": 8245,
- "output": 117
+ "input": 8235,
+ "output": 120
}
},
- "c12bab83290d808cdc2f8bd78770494f655d0f7b": {
+ "a30b487338609e8a5d8417c1f90419956bb4b727": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in the form fields again for the desk order",
- "copilotToolCallId": "toolu_vrtx_012BK22YLkPokRqtnZQLUxmg"
+ "text": "Fill in the customer details for the second order",
+ "copilotToolCallId": "toolu_vrtx_01W1QKYyFZvMCWhC3DwYQv2X"
},
{
"type": "tool_call",
@@ -337,23 +337,23 @@
}
]
},
- "id": "toolu_vrtx_012BK22YLkPokRqtnZQLUxmg"
+ "id": "toolu_vrtx_01W1QKYyFZvMCWhC3DwYQv2X"
}
]
},
"usage": {
- "input": 8516,
- "output": 270
+ "input": 8504,
+ "output": 269
}
},
- "97991c9176de4922376ca0724be0e0f08a663495": {
+ "7269d27383720010c65e7e530cf66cff736cb587": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the order for the green desk",
- "copilotToolCallId": "toolu_vrtx_01J4NkM1w1oVvGe669YpKznN"
+ "text": "Click the Order button to submit the second order (green desk)",
+ "copilotToolCallId": "toolu_vrtx_019EvisGfKGAHWu4Ehk6orM9"
},
{
"type": "tool_call",
@@ -362,23 +362,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_01J4NkM1w1oVvGe669YpKznN"
+ "id": "toolu_vrtx_019EvisGfKGAHWu4Ehk6orM9"
}
]
},
"usage": {
- "input": 8981,
- "output": 95
+ "input": 8963,
+ "output": 100
}
},
- "e21936aa617706f6d9db0cbd5c2ea8d50ace71c5": {
+ "28e8e2a0869e245c707a879c41bf7811648233ff": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Accept the confirmation dialog for the green desk order",
- "copilotToolCallId": "toolu_vrtx_01Xrhq5gNqNUZUbFLTA2fyt1"
+ "copilotToolCallId": "toolu_vrtx_01MawbHWCqo6WBwPh8YcEBm4"
},
{
"type": "tool_call",
@@ -386,23 +386,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01Xrhq5gNqNUZUbFLTA2fyt1"
+ "id": "toolu_vrtx_01MawbHWCqo6WBwPh8YcEBm4"
}
]
},
"usage": {
- "input": 9156,
+ "input": 9138,
"output": 80
}
},
- "8519a36f22606bdb77adbee31c1b2b3ff121583e": {
+ "cc4e5987be7c007db351125c487c40e5413aebf1": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Report the total price of both items and maximum estimated delivery time. Blue table: $353.98, Green desk: $299.98. Total: $653.96. Both have 3 days estimated delivery.",
- "copilotToolCallId": "toolu_vrtx_01X7E9MaaAgBgrjCRGx1BNpT"
+ "text": "Report the total price and maximum delivery time for both orders",
+ "copilotToolCallId": "toolu_vrtx_01DahshwKgzha389UJ8rq29Z"
},
{
"type": "tool_call",
@@ -411,13 +411,13 @@
"price": 653.96,
"deliveryDays": 3
},
- "id": "toolu_vrtx_01X7E9MaaAgBgrjCRGx1BNpT"
+ "id": "toolu_vrtx_01DahshwKgzha389UJ8rq29Z"
}
]
},
"usage": {
- "input": 10055,
- "output": 135
+ "input": 10037,
+ "output": 102
}
}
}
\ No newline at end of file
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
index 4fe31e8094..68b9b1d7cf 100644
--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
@@ -38,14 +38,14 @@
"output": 88
}
},
- "038e080c725fcc9423fd35bb7f3f66ef81a0123b": {
+ "7570cf20826150ea1678925d3c56cdaf6cc0ce0d": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Select 'Table' as the product type for the first order",
- "copilotToolCallId": "toolu_vrtx_01Kr5U1jWVpBh7SmizccRR3f"
+ "copilotToolCallId": "toolu_vrtx_01R9atbVxke4YwYTPtDeMwuK"
},
{
"type": "tool_call",
@@ -57,23 +57,23 @@
"Table"
]
},
- "id": "toolu_vrtx_01Kr5U1jWVpBh7SmizccRR3f"
+ "id": "toolu_vrtx_01R9atbVxke4YwYTPtDeMwuK"
}
]
},
"usage": {
- "input": 5160,
+ "input": 5155,
"output": 123
}
},
- "754d74faa1450471f76bd4fc78788cc8bc1ad71c": {
+ "1e9bce6dd706be6d7272ab962111b4e7eb5730d7": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Select 'Blue' as the color for the table",
- "copilotToolCallId": "toolu_vrtx_016C7VpVtiZQDYRWMiQqbq7j"
+ "copilotToolCallId": "toolu_vrtx_012uA5vSCDJPndFMhFP9z6AT"
},
{
"type": "tool_call",
@@ -85,23 +85,23 @@
"Blue"
]
},
- "id": "toolu_vrtx_016C7VpVtiZQDYRWMiQqbq7j"
+ "id": "toolu_vrtx_012uA5vSCDJPndFMhFP9z6AT"
}
]
},
"usage": {
- "input": 5435,
+ "input": 5425,
"output": 120
}
},
- "e523d834c1b2431a0897b1f72a82b1a41aeeeb19": {
+ "40f9be0c8022103c35d156209c8f35be0861e84e": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Fill in the customer details for the order",
- "copilotToolCallId": "toolu_vrtx_01CpuzPwzdWDkS9CAUrFzkZD"
+ "copilotToolCallId": "toolu_vrtx_01WGqTc3Hcmd1FTasnBTmwMv"
},
{
"type": "tool_call",
@@ -136,7 +136,7 @@
"name": "State/Province",
"type": "textbox",
"ref": "e25",
- "value": "XYZ state"
+ "value": "XYZ"
},
{
"name": "ZIP/Postal Code",
@@ -146,23 +146,23 @@
}
]
},
- "id": "toolu_vrtx_01CpuzPwzdWDkS9CAUrFzkZD"
+ "id": "toolu_vrtx_01WGqTc3Hcmd1FTasnBTmwMv"
}
]
},
"usage": {
- "input": 5709,
- "output": 345
+ "input": 5694,
+ "output": 344
}
},
- "80b96184b8d7012b4e08d83112649e7b59d205b3": {
+ "3f143bb307a820583389dcdc308cc4507dd72618": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Submit the order for the blue table",
- "copilotToolCallId": "toolu_vrtx_01HnWEUVGL6DezzGD9Ffm4fk"
+ "copilotToolCallId": "toolu_vrtx_01SLsAkw2y3eNVrRTPtAnfYx"
},
{
"type": "tool_call",
@@ -171,23 +171,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_01HnWEUVGL6DezzGD9Ffm4fk"
+ "id": "toolu_vrtx_01SLsAkw2y3eNVrRTPtAnfYx"
}
]
},
"usage": {
- "input": 6166,
+ "input": 6144,
"output": 95
}
},
- "0dde56ef8622cf938c1ee60b8a651c8259d0411f": {
+ "6e78e923857b521b4255a872588f577dee2d6795": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Accept the order confirmation for the blue table",
- "copilotToolCallId": "toolu_vrtx_01CR3bVABfhxB5qgt22Y4Pu1"
+ "text": "Confirm the order for the blue table",
+ "copilotToolCallId": "toolu_vrtx_01MESzajZqXLjdsQiKvVgUhF"
},
{
"type": "tool_call",
@@ -195,23 +195,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01CR3bVABfhxB5qgt22Y4Pu1"
+ "id": "toolu_vrtx_01MESzajZqXLjdsQiKvVgUhF"
}
]
},
"usage": {
- "input": 6341,
+ "input": 6314,
"output": 79
}
},
- "6caaa60190a581d2fbda0ce572ba1ab6dee6ebd5": {
+ "1b5ccd39ae43f194ba622a7bb4bdf639e69ecec0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click to place another order for the green desk",
- "copilotToolCallId": "toolu_vrtx_0151JgiQCvuneHYW3F8CGHR7"
+ "text": "Click the button to place the second order (green desk)",
+ "copilotToolCallId": "toolu_vrtx_01UmUQhxxbPHM7UUXudex9Zg"
},
{
"type": "tool_call",
@@ -220,23 +220,23 @@
"element": "Place Another Order button",
"ref": "e58"
},
- "id": "toolu_vrtx_0151JgiQCvuneHYW3F8CGHR7"
+ "id": "toolu_vrtx_01UmUQhxxbPHM7UUXudex9Zg"
}
]
},
"usage": {
- "input": 7230,
- "output": 99
+ "input": 7200,
+ "output": 101
}
},
- "713a5e3f378243a37c5a435f6aff1d556f697691": {
+ "4b74f6fa7afaa89403a0d6a70838971b04447ce6": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Select 'Desk' as the product type for the second order",
- "copilotToolCallId": "toolu_vrtx_01GnBKQrEkEretr3hXo8C3kN"
+ "copilotToolCallId": "toolu_vrtx_01FRxMd3DbZstMP7cdMg3qNE"
},
{
"type": "tool_call",
@@ -248,23 +248,23 @@
"Desk"
]
},
- "id": "toolu_vrtx_01GnBKQrEkEretr3hXo8C3kN"
+ "id": "toolu_vrtx_01FRxMd3DbZstMP7cdMg3qNE"
}
]
},
"usage": {
- "input": 7954,
+ "input": 7921,
"output": 125
}
},
- "32d80d3b3ea01b9adc6bb2311c0c01c8f1c02684": {
+ "0f35bef49760b5c9340ab795b04c5f4ccfb693af": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Select 'Green' as the color for the desk",
- "copilotToolCallId": "toolu_vrtx_012Dkh4SgZriiobB7rQoZ8cj"
+ "copilotToolCallId": "toolu_vrtx_01Ja8AwUYnJxFKNQbwvdTiQF"
},
{
"type": "tool_call",
@@ -276,23 +276,23 @@
"Green"
]
},
- "id": "toolu_vrtx_012Dkh4SgZriiobB7rQoZ8cj"
+ "id": "toolu_vrtx_01Ja8AwUYnJxFKNQbwvdTiQF"
}
]
},
"usage": {
- "input": 8232,
+ "input": 8194,
"output": 120
}
},
- "c0f6e0008f9b1ad8fe8831c50853f4cf5e03cfdb": {
+ "86eb224ddafacf54ef2fc05cf0db3893ef76e41e": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Fill in the customer details for the second order",
- "copilotToolCallId": "toolu_vrtx_01RaibhHtyqmX9VLm8w8TQkV"
+ "copilotToolCallId": "toolu_vrtx_012mxpfp6HJGg7pcLMxFt5fv"
},
{
"type": "tool_call",
@@ -327,7 +327,7 @@
"name": "State/Province",
"ref": "e25",
"type": "textbox",
- "value": "XYZ state"
+ "value": "XYZ"
},
{
"name": "ZIP/Postal Code",
@@ -337,23 +337,23 @@
}
]
},
- "id": "toolu_vrtx_01RaibhHtyqmX9VLm8w8TQkV"
+ "id": "toolu_vrtx_012mxpfp6HJGg7pcLMxFt5fv"
}
]
},
"usage": {
- "input": 8506,
- "output": 266
+ "input": 8463,
+ "output": 265
}
},
- "f4ba6d240b4bddc64099253f54166fea6fab8158": {
+ "d3c67880cd8b86dcc1311717192dbccd75f6f970": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
"text": "Submit the order for the green desk",
- "copilotToolCallId": "toolu_vrtx_017yfFSSUYR9n1rwX8tNsGgW"
+ "copilotToolCallId": "toolu_vrtx_01E69zhhreSssUMqZWhKaaPa"
},
{
"type": "tool_call",
@@ -362,23 +362,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_017yfFSSUYR9n1rwX8tNsGgW"
+ "id": "toolu_vrtx_01E69zhhreSssUMqZWhKaaPa"
}
]
},
"usage": {
- "input": 8964,
+ "input": 8914,
"output": 95
}
},
- "136809b18f9e4a99f8d1afd675e68a35527a6e75": {
+ "c9719d6d50a2845be611b84f4882c960a639850b": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Accept the order confirmation for the green desk",
- "copilotToolCallId": "toolu_vrtx_01Q5e5d5oBNXgczowvX1rs7h"
+ "text": "Confirm the order for the green desk",
+ "copilotToolCallId": "toolu_vrtx_01XTAjsdAD5dNfef84Dq8pSS"
},
{
"type": "tool_call",
@@ -386,23 +386,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01Q5e5d5oBNXgczowvX1rs7h"
+ "id": "toolu_vrtx_01XTAjsdAD5dNfef84Dq8pSS"
}
]
},
"usage": {
- "input": 9139,
+ "input": 9084,
"output": 79
}
},
- "e9c0de54e428999409f95e3e07f1ab1a2e5d8adf": {
+ "beae0a9e066cec2c0e6c14b1bab32ea9a85c0448": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Report the total price and maximum estimated delivery for both orders (blue table: $353.98 with 3 days delivery, green desk: $299.98 with 3 days delivery)",
- "copilotToolCallId": "toolu_vrtx_01MymHkTngjRe1vyd5KDLttU"
+ "text": "Report the total price and maximum delivery days for both orders. Blue table: $353.98, 3 days. Green desk: $299.98, 3 days. Total: $653.96, max delivery: 3 days.",
+ "copilotToolCallId": "toolu_vrtx_015FrLR8BJNkcxYXwgxkXkHk"
},
{
"type": "tool_call",
@@ -411,13 +411,13 @@
"price": 653.96,
"deliveryDays": 3
},
- "id": "toolu_vrtx_01MymHkTngjRe1vyd5KDLttU"
+ "id": "toolu_vrtx_015FrLR8BJNkcxYXwgxkXkHk"
}
]
},
"usage": {
- "input": 10029,
- "output": 132
+ "input": 9971,
+ "output": 143
}
}
}
\ No newline at end of file
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
index 0ef9973de6..2ffc395780 100644
--- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
@@ -38,36 +38,36 @@
"output": 92
}
},
- "f7e5a310544d7464ffffea2fe03932582d1bbab6": {
+ "7fcc4660d74d7989887daa17f960a092cfa6d75d": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Check the current state of the form before filling it out",
- "copilotToolCallId": "toolu_vrtx_018Z3r8uFf2n2mDskeb34NRs"
+ "text": "Get current page state to see the form",
+ "copilotToolCallId": "toolu_vrtx_01S1TPuFGdY3PpCwQ1MTs7iA"
},
{
"type": "tool_call",
"name": "browser_snapshot",
"arguments": {},
- "id": "toolu_vrtx_018Z3r8uFf2n2mDskeb34NRs"
+ "id": "toolu_vrtx_01S1TPuFGdY3PpCwQ1MTs7iA"
}
]
},
"usage": {
- "input": 5167,
- "output": 63
+ "input": 5162,
+ "output": 60
}
},
- "e4415a2e025d424030b34fde3d835d203f52bd34": {
+ "f7864b65b2ac8c7b1371d54672bc5cea4ac0ee3a": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Table' as the product type for the first order",
- "copilotToolCallId": "toolu_vrtx_01KYNS4MCaTXnsKn5KYVPznH"
+ "text": "Select Table as the first product",
+ "copilotToolCallId": "toolu_vrtx_01U7qs1Y5D27RMjy5vigdaXR"
},
{
"type": "tool_call",
@@ -79,23 +79,23 @@
"Table"
]
},
- "id": "toolu_vrtx_01KYNS4MCaTXnsKn5KYVPznH"
+ "id": "toolu_vrtx_01U7qs1Y5D27RMjy5vigdaXR"
}
]
},
"usage": {
- "input": 5821,
- "output": 123
+ "input": 5813,
+ "output": 116
}
},
- "99f72d9d743e48c4f9c3c10a22b63dfbdd277012": {
+ "7c78931db5b30156a7dc1e6612d870e8187a20d6": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Blue' as the color for the table",
- "copilotToolCallId": "toolu_vrtx_01H1M2gFHZdKEoiDYVVEimdm"
+ "text": "Select Blue color for the table",
+ "copilotToolCallId": "toolu_vrtx_01LXzd6tHpmw9uCZGrNc3TSi"
},
{
"type": "tool_call",
@@ -107,23 +107,23 @@
"Blue"
]
},
- "id": "toolu_vrtx_01H1M2gFHZdKEoiDYVVEimdm"
+ "id": "toolu_vrtx_01LXzd6tHpmw9uCZGrNc3TSi"
}
]
},
"usage": {
- "input": 6096,
- "output": 120
+ "input": 6076,
+ "output": 115
}
},
- "2963b09feb88a43912acb70deca9e945ed838170": {
+ "cd4bd5106418a526955afe6d3f7c7f2b53007346": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in all the customer details for the first order",
- "copilotToolCallId": "toolu_vrtx_0154BEz3qJ3AtgGtCRmyGaKt"
+ "text": "Fill in the customer information fields",
+ "copilotToolCallId": "toolu_vrtx_01YWjwfRNj3KaBujienmqBhV"
},
{
"type": "tool_call",
@@ -168,23 +168,23 @@
}
]
},
- "id": "toolu_vrtx_0154BEz3qJ3AtgGtCRmyGaKt"
+ "id": "toolu_vrtx_01YWjwfRNj3KaBujienmqBhV"
}
]
},
"usage": {
- "input": 6370,
- "output": 347
+ "input": 6340,
+ "output": 343
}
},
- "8252960e4ad2729cd515d1b93b9a319243cd5208": {
+ "41b9db56c70fcd4f84f8d3de9de0c799dce8db88": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the first order (blue table)",
- "copilotToolCallId": "toolu_vrtx_014JpUqZJ5qYQyNHt1SfGb3F"
+ "text": "Submit the first order for blue table",
+ "copilotToolCallId": "toolu_vrtx_01D6FAeFSJeXh67RCcBh5JMB"
},
{
"type": "tool_call",
@@ -193,23 +193,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_014JpUqZJ5qYQyNHt1SfGb3F"
+ "id": "toolu_vrtx_01D6FAeFSJeXh67RCcBh5JMB"
}
]
},
"usage": {
- "input": 6829,
+ "input": 6790,
"output": 95
}
},
- "cc501640fbc47d3305b5056d54348a6c99a98726": {
+ "e7da24b30926519b920dfff6d09d1f321ccb3ae7": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Accept the confirmation dialog for the blue table order",
- "copilotToolCallId": "toolu_vrtx_01ULFAfNeMTsQDvRm3o9jjog"
+ "text": "Confirm the order for blue table",
+ "copilotToolCallId": "toolu_vrtx_01TDuwkwXfeotUgRAu8fLUFD"
},
{
"type": "tool_call",
@@ -217,23 +217,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_01ULFAfNeMTsQDvRm3o9jjog"
+ "id": "toolu_vrtx_01TDuwkwXfeotUgRAu8fLUFD"
}
]
},
"usage": {
- "input": 7004,
- "output": 80
+ "input": 6960,
+ "output": 78
}
},
- "c8f6a897f0678c33d72b4f77ca10f52689f8b760": {
+ "b2200231c17c003417ced022a372a4b11edbf238": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Click the button to place another order for the green desk",
- "copilotToolCallId": "toolu_vrtx_01Y3noC5ecEE9LyHZQLuL6XR"
+ "text": "Click to place another order for the green desk",
+ "copilotToolCallId": "toolu_vrtx_014capLBfLNKY3rkD46m8bQ1"
},
{
"type": "tool_call",
@@ -242,23 +242,23 @@
"element": "Place Another Order button",
"ref": "e58"
},
- "id": "toolu_vrtx_01Y3noC5ecEE9LyHZQLuL6XR"
+ "id": "toolu_vrtx_014capLBfLNKY3rkD46m8bQ1"
}
]
},
"usage": {
- "input": 7894,
- "output": 101
+ "input": 7848,
+ "output": 99
}
},
- "1657a77de89340fe2f077023c29d0a4315ff2da2": {
+ "3eace161b63651422689b72dbf4e606a489163e6": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Desk' as the product type for the second order",
- "copilotToolCallId": "toolu_vrtx_01SSfRq5TCJMBUWkKoULWTfu"
+ "text": "Select Desk as the second product",
+ "copilotToolCallId": "toolu_vrtx_01CY7bczyF9uwULtuDbsv5Bi"
},
{
"type": "tool_call",
@@ -270,23 +270,23 @@
"Desk"
]
},
- "id": "toolu_vrtx_01SSfRq5TCJMBUWkKoULWTfu"
+ "id": "toolu_vrtx_01CY7bczyF9uwULtuDbsv5Bi"
}
]
},
"usage": {
- "input": 8620,
- "output": 125
+ "input": 8567,
+ "output": 118
}
},
- "ae643cadf2e2c3499a424fac62c9be17a1ca613a": {
+ "c59b631891d47364177fda353cfccb126dfbf570": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Select 'Green' as the color for the desk",
- "copilotToolCallId": "toolu_vrtx_01LvkZw7zRjt5enTXX2ZdsqN"
+ "text": "Select Green color for the desk",
+ "copilotToolCallId": "toolu_vrtx_019fxgUwj5JiAJFrCzxGXPSK"
},
{
"type": "tool_call",
@@ -298,23 +298,23 @@
"Green"
]
},
- "id": "toolu_vrtx_01LvkZw7zRjt5enTXX2ZdsqN"
+ "id": "toolu_vrtx_019fxgUwj5JiAJFrCzxGXPSK"
}
]
},
"usage": {
- "input": 8898,
- "output": 120
+ "input": 8833,
+ "output": 115
}
},
- "0423b4c46c9358cc18cbf3e19ac69bb7b32f1aec": {
+ "56e5cffa84ed757969f9abbbbe898dd6d0b2dbb6": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Fill in all the customer details for the second order",
- "copilotToolCallId": "toolu_vrtx_01BPw2B9YjakQKHJq7Y7qVYp"
+ "text": "Fill in the customer information for the second order",
+ "copilotToolCallId": "toolu_vrtx_016MMu8vjsBSdpwemMbNoXjV"
},
{
"type": "tool_call",
@@ -359,23 +359,23 @@
}
]
},
- "id": "toolu_vrtx_01BPw2B9YjakQKHJq7Y7qVYp"
+ "id": "toolu_vrtx_016MMu8vjsBSdpwemMbNoXjV"
}
]
},
"usage": {
- "input": 9172,
- "output": 267
+ "input": 9097,
+ "output": 266
}
},
- "fdc7144914501f04a01af4d951f27a3c0ec76fab": {
+ "d134b61bfaaea4e36415838a62db5cb13f888d20": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Submit the second order (green desk)",
- "copilotToolCallId": "toolu_vrtx_01WdPKRuBhfaQPxtY9EdxubM"
+ "text": "Submit the second order for green desk",
+ "copilotToolCallId": "toolu_vrtx_01TgMV6W9eKdLpPjDDDewJJf"
},
{
"type": "tool_call",
@@ -384,23 +384,23 @@
"element": "Order button",
"ref": "e29"
},
- "id": "toolu_vrtx_01WdPKRuBhfaQPxtY9EdxubM"
+ "id": "toolu_vrtx_01TgMV6W9eKdLpPjDDDewJJf"
}
]
},
"usage": {
- "input": 9631,
+ "input": 9550,
"output": 95
}
},
- "0db529b67667bc494d5b0e7605614a2c0497eef1": {
+ "de09913d7981ad51cc18c9ad9ecdf7d1982bb858": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Accept the confirmation dialog for the green desk order",
- "copilotToolCallId": "toolu_vrtx_0197daAGKqY1aXq8KxKFD6Rd"
+ "text": "Confirm the order for green desk",
+ "copilotToolCallId": "toolu_vrtx_018ksE83x6ekovoZuomxmrsv"
},
{
"type": "tool_call",
@@ -408,23 +408,23 @@
"arguments": {
"accept": true
},
- "id": "toolu_vrtx_0197daAGKqY1aXq8KxKFD6Rd"
+ "id": "toolu_vrtx_018ksE83x6ekovoZuomxmrsv"
}
]
},
"usage": {
- "input": 9806,
- "output": 80
+ "input": 9720,
+ "output": 78
}
},
- "c4624d908400df00c492cec467920acd3c98ef68": {
+ "987bfeb4500a519fde51250611923095075f6dd9": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Report the total price for both items and maximum estimated delivery time. Blue table: $353.98, Green desk: $299.98. Total: $653.96. Both have 3 days estimated delivery.",
- "copilotToolCallId": "toolu_vrtx_014zsVp7oDzvJJgsjg3LVazs"
+ "text": "Report the total price for both items and maximum estimated delivery time",
+ "copilotToolCallId": "toolu_vrtx_01JAZZ8bo7oMgM9FVCpBEvyf"
},
{
"type": "tool_call",
@@ -433,13 +433,13 @@
"price": 653.96,
"deliveryDays": 3
},
- "id": "toolu_vrtx_014zsVp7oDzvJJgsjg3LVazs"
+ "id": "toolu_vrtx_01JAZZ8bo7oMgM9FVCpBEvyf"
}
]
},
"usage": {
- "input": 10697,
- "output": 135
+ "input": 10609,
+ "output": 103
}
}
}
\ No newline at end of file
diff --git a/tests/mcp/fixtures.ts b/tests/mcp/fixtures.ts
index 8b78bf7403..874f1d1ecb 100644
--- a/tests/mcp/fixtures.ts
+++ b/tests/mcp/fixtures.ts
@@ -343,3 +343,8 @@ export async function prepareDebugTest(startClient: StartClient, testFile?: stri
function sanitizeFileName(name: string): string {
return name.replace('.ts', '').replace(/[^a-zA-Z0-9_]+/g, '-');
}
+
+export const lowireMeta = {
+ 'dev.lowire/history': true,
+ 'dev.lowire/state': true,
+};
diff --git a/tests/mcp/secrets.spec.ts b/tests/mcp/secrets.spec.ts
index b2e497f307..e207834a15 100644
--- a/tests/mcp/secrets.spec.ts
+++ b/tests/mcp/secrets.spec.ts
@@ -16,7 +16,7 @@
import fs from 'node:fs';
-import { test, expect } from './fixtures';
+import { test, expect, lowireMeta, parseResponse } from './fixtures';
test('browser_type', async ({ startClient, server }) => {
const secretsFile = test.info().outputPath('secrets.env');
@@ -126,3 +126,68 @@ await page.getByRole('textbox', { name: 'Password' }).fill(process.env['X-PASSWO
pageState: expect.stringContaining(`- textbox \"Password\" [active] [ref=e6]: <secret>X-PASSWORD</secret>`),
});
});
+
+
+test('structured response', async ({ startClient, server }) => {
+ const secretsFile = test.info().outputPath('secrets.env');
+ await fs.promises.writeFile(secretsFile, 'X-PASSWORD=password123');
+
+ const { client } = await startClient({
+ args: ['--secrets', secretsFile],
+ });
+
+ server.setContent('/', `
+ <!DOCTYPE html>
+ <html>
+ <input type='keypress' onkeypress="console.log('Key pressed:', event.key, ', Text:', event.target.value)"></input>
+ </html>
+ `, 'text/html');
+
+ await client.callTool({
+ name: 'browser_navigate',
+ arguments: {
+ url: server.PREFIX,
+ _meta: lowireMeta
+ },
+ });
+
+ {
+ const response = await client.callTool({
+ name: 'browser_type',
+ arguments: {
+ element: 'textbox',
+ ref: 'e2',
+ text: 'X-PASSWORD',
+ submit: true,
+ _meta: lowireMeta
+ },
+ });
+ const { _meta } = parseResponse(response);
+ expect(_meta).toEqual({
+ 'dev.lowire/history': [
+ { category: 'code', content: `await page.getByRole('textbox').fill(process.env['X-PASSWORD']);
+await page.getByRole('textbox').press('Enter');` },
+ { category: 'console', content: expect.stringContaining('[LOG] Key pressed: Enter , Text: <secret>X-PASSWORD</secret>') },
+ ],
+ 'dev.lowire/state': {
+ 'page': expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: <secret>X-PASSWORD<\/secret>/),
+ },
+ });
+ }
+
+ {
+ const response = await client.callTool({
+ name: 'browser_console_messages',
+ arguments: {
+ _meta: lowireMeta
+ },
+ });
+ const { _meta } = parseResponse(response);
+ expect(_meta).toEqual({
+ 'dev.lowire/history': [
+ { category: 'result', content: expect.stringContaining('[LOG] Key pressed: Enter , Text: <secret>X-PASSWORD</secret>') },
+ ],
+ 'dev.lowire/state': {},
+ });
+ }
+});
diff --git a/tests/page/perform-task.spec.ts b/tests/page/perform-task.spec.ts
new file mode 100644
index 0000000000..0da7e62466
--- /dev/null
+++ b/tests/page/perform-task.spec.ts
@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test as it } from './pageTest';
+
+// @ts-ignore
+it.skip('perform task', async ({ page, _perform }) => {
+ await page.goto(`https://playwright.dev/`);
+ await _perform('Click the get started button');
+});