Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: that-should-be-it perform mode (#38503)

Pavel Feldman8f97da76c67a
diff --git a/package-lock.json b/package-lock.json
index 1a695eb458..80c770a7e3 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -18,7 +18,7 @@
"@eslint/compat": "^1.3.2",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.34.0",
- "@lowire/loop": "^0.0.6",
+ "@lowire/loop": "^0.0.7",
"@modelcontextprotocol/sdk": "^1.17.5",
"@octokit/graphql-schema": "^15.26.0",
"@stylistic/eslint-plugin": "^5.2.3",
@@ -1064,9 +1064,9 @@
}
},
"node_modules/@lowire/loop": {
- "version": "0.0.6",
- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.6.tgz",
- "integrity": "sha512-mrQft4UUj4tWt8ikOvRjec88nLRt6XwEnfzyFlZggedV735OcKE5Bk0Yy2mr3QCRCVn4/aw91lugRBS9ftSSpg==",
+ "version": "0.0.7",
+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.7.tgz",
+ "integrity": "sha512-Hn/VU1v+Ld8m9y3AkFg3Naeht79Ub9dtefljGy5WEMZbrCJkONw4Q2GqihCWys8tcUQZUf9pUTyTIvC+A6HEQw==",
"dev": true,
"license": "Apache-2.0",
"engines": {
diff --git a/package.json b/package.json
index 21938cd139..334ad20a59 100644
--- a/package.json
+++ b/package.json
@@ -56,7 +56,7 @@
"@eslint/compat": "^1.3.2",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.34.0",
- "@lowire/loop": "^0.0.6",
+ "@lowire/loop": "^0.0.7",
"@modelcontextprotocol/sdk": "^1.17.5",
"@octokit/graphql-schema": "^15.26.0",
"@stylistic/eslint-plugin": "^5.2.3",
diff --git a/packages/playwright-core/ThirdPartyNotices.txt b/packages/playwright-core/ThirdPartyNotices.txt
index f0f493fa9c..4a0f753351 100644
--- a/packages/playwright-core/ThirdPartyNotices.txt
+++ b/packages/playwright-core/ThirdPartyNotices.txt
@@ -4,7 +4,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATION
This project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise.
-- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)
+- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)
- @modelcontextprotocol/[email protected] (https://github.com/modelcontextprotocol/typescript-sdk)
- [email protected] (https://github.com/jshttp/accepts)
- [email protected] (https://github.com/TooTallNate/proxy-agents)
@@ -135,7 +135,7 @@ This project incorporates components from the projects listed below. The origina
- [email protected] (https://github.com/StefanTerdell/zod-to-json-schema)
- [email protected] (https://github.com/colinhacks/zod)
-%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE
+%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE
=========================================
Apache License
Version 2.0, January 2004
@@ -339,7 +339,7 @@ Apache License
See the License for the specific language governing permissions and
limitations under the License.
=========================================
-END OF @lowire/[email protected] AND INFORMATION
+END OF @lowire/[email protected] AND INFORMATION
%% @modelcontextprotocol/[email protected] NOTICES AND INFORMATION BEGIN HERE
=========================================
diff --git a/packages/playwright-core/bundles/mcp/package-lock.json b/packages/playwright-core/bundles/mcp/package-lock.json
index e6b3091f7f..bf5ee6bf3a 100644
--- a/packages/playwright-core/bundles/mcp/package-lock.json
+++ b/packages/playwright-core/bundles/mcp/package-lock.json
@@ -8,16 +8,16 @@
"name": "mcp-bundle",
"version": "0.0.1",
"dependencies": {
- "@lowire/loop": "^0.0.6",
+ "@lowire/loop": "^0.0.7",
"@modelcontextprotocol/sdk": "^1.24.0",
"zod": "^3.25.76",
"zod-to-json-schema": "^3.24.6"
}
},
"node_modules/@lowire/loop": {
- "version": "0.0.6",
- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.6.tgz",
- "integrity": "sha512-mrQft4UUj4tWt8ikOvRjec88nLRt6XwEnfzyFlZggedV735OcKE5Bk0Yy2mr3QCRCVn4/aw91lugRBS9ftSSpg==",
+ "version": "0.0.7",
+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.7.tgz",
+ "integrity": "sha512-Hn/VU1v+Ld8m9y3AkFg3Naeht79Ub9dtefljGy5WEMZbrCJkONw4Q2GqihCWys8tcUQZUf9pUTyTIvC+A6HEQw==",
"license": "Apache-2.0",
"engines": {
"node": ">=20"
diff --git a/packages/playwright-core/bundles/mcp/package.json b/packages/playwright-core/bundles/mcp/package.json
index fa046d515d..eca0006903 100644
--- a/packages/playwright-core/bundles/mcp/package.json
+++ b/packages/playwright-core/bundles/mcp/package.json
@@ -4,7 +4,7 @@
"private": true,
"dependencies": {
"@modelcontextprotocol/sdk": "^1.24.0",
- "@lowire/loop": "^0.0.6",
+ "@lowire/loop": "^0.0.7",
"zod": "^3.25.76",
"zod-to-json-schema": "^3.24.6"
}
diff --git a/packages/playwright-core/src/server/agent/agent.ts b/packages/playwright-core/src/server/agent/agent.ts
index 27866b04cc..d952e814e3 100644
--- a/packages/playwright-core/src/server/agent/agent.ts
+++ b/packages/playwright-core/src/server/agent/agent.ts
@@ -18,7 +18,7 @@ import fs from 'fs';
import { toolsForLoop } from './backend';
import { debug } from '../../utilsBundle';
-import { Loop, z, zodToJsonSchema } from '../../mcpBundle';
+import { Loop } from '../../mcpBundle';
import { runAction } from './actionRunner';
import { Context } from './context';
@@ -34,9 +34,7 @@ export async function pagePerform(progress: Progress, page: Page, options: chann
if (await cachedPerform(context, options))
return;
- await perform(context, options.task, zodToJsonSchema(z.object({
- error: z.string().optional().describe('An error message if the task could not be completed successfully'),
- })) as loopTypes.Schema, options);
+ await perform(context, options.task, undefined, options);
await updateCache(context, options);
}
@@ -51,7 +49,7 @@ ${options.query}`;
return await perform(context, task, options.schema, options);
}
-async function perform(context: Context, userTask: string, resultSchema: loopTypes.Schema, options: { maxTurns?: number } = {}): Promise<any> {
+async function perform(context: Context, userTask: string, resultSchema: loopTypes.Schema | undefined, options: { maxTurns?: number } = {}): Promise<any> {
const { progress, page } = context;
const browserContext = page.browserContext;
if (!browserContext._options.agent)
@@ -66,6 +64,13 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp
debug,
callTool,
tools,
+ beforeTurn: params => {
+ const lastReply = params.conversation.messages.findLast(m => m.role === 'assistant');
+ const toolCall = lastReply?.content.find(c => c.type === 'tool_call');
+ if (!resultSchema && toolCall && toolCall.arguments.thatShouldBeIt)
+ return 'break';
+ return 'continue';
+ },
...options
});
@@ -75,9 +80,8 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp
${full}
`;
- return await loop.run(task, {
- resultSchema
- });
+ const { result } = await loop.run(task, { resultSchema });
+ return result;
}
type CachedActions = Record<string, {
@@ -121,8 +125,7 @@ async function updateCache(context: Context, options: channels.PagePerformParams
async function cachedActions(cacheFile: string): Promise<CachedActions> {
let cache = allCaches.get(cacheFile);
if (!cache) {
- const text = await fs.promises.readFile(cacheFile, 'utf-8').catch(() => '{}');
- cache = JSON.parse(text) as CachedActions;
+ cache = await fs.promises.readFile(cacheFile, 'utf-8').then(text => JSON.parse(text)).catch(() => ({})) as CachedActions;
allCaches.set(cacheFile, cache);
}
return cache;
diff --git a/packages/playwright-core/src/server/agent/tools.ts b/packages/playwright-core/src/server/agent/tools.ts
index 33125bc33b..7eac91aa07 100644
--- a/packages/playwright-core/src/server/agent/tools.ts
+++ b/packages/playwright-core/src/server/agent/tools.ts
@@ -35,12 +35,16 @@ function defineTool<Input extends zod.Schema>(tool: ToolDefinition<Input>): Tool
return tool;
}
+const baseSchema = z.object({
+ thatShouldBeIt: z.boolean().describe('Indicates that this tool call is sufficient to complete the task. If false, the task will continue with the next tool call'),
+});
+
const snapshot = defineTool({
schema: {
name: 'browser_snapshot',
title: 'Page snapshot',
description: 'Capture accessibility snapshot of the current page, this is better than screenshot',
- inputSchema: z.object({}),
+ inputSchema: baseSchema,
},
handle: async (context, params) => {
@@ -48,7 +52,7 @@ const snapshot = defineTool({
},
});
-const elementSchema = z.object({
+const elementSchema = baseSchema.extend({
element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
ref: z.string().describe('Exact target element reference from the page snapshot'),
});
@@ -86,7 +90,7 @@ const drag = defineTool({
name: 'browser_drag',
title: 'Drag mouse',
description: 'Perform drag and drop between two elements',
- inputSchema: z.object({
+ inputSchema: baseSchema.extend({
startElement: z.string().describe('Human-readable source element description used to obtain the permission to interact with the element'),
startRef: z.string().describe('Exact source element reference from the page snapshot'),
endElement: z.string().describe('Human-readable target element description used to obtain the permission to interact with the element'),
@@ -211,7 +215,7 @@ const fillForm = defineTool({
name: 'browser_fill_form',
title: 'Fill form',
description: 'Fill multiple form fields',
- inputSchema: z.object({
+ inputSchema: baseSchema.extend({
fields: z.array(z.object({
name: z.string().describe('Human-readable field name'),
type: z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider']).describe('Type of the field'),
diff --git a/packages/playwright/src/agents/agent.ts b/packages/playwright/src/agents/agent.ts
index b0b48b647d..c252eafba2 100644
--- a/packages/playwright/src/agents/agent.ts
+++ b/packages/playwright/src/agents/agent.ts
@@ -50,12 +50,13 @@ export class Agent<T extends z.ZodSchema<any>> {
const { clients, tools, callTool } = await this._initClients();
const prompt = this.spec.description;
try {
- return await this.loop.run<z.output<T>>(`${prompt}\n\nTask:\n${task}\n\nParams:\n${JSON.stringify(params, null, 2)}`, {
+ const { result } = await this.loop.run<z.output<T>>(`${prompt}\n\nTask:\n${task}\n\nParams:\n${JSON.stringify(params, null, 2)}`, {
...options,
tools,
callTool,
resultSchema: this.resultSchema
});
+ return result;
} finally {
await this._disconnectFromServers(clients);
}
diff --git a/tests/library/perform-task.spec.ts-cache.json b/tests/library/perform-task.spec.ts-cache.json
index c1deb2ea11..11b59d4b9d 100644
--- a/tests/library/perform-task.spec.ts-cache.json
+++ b/tests/library/perform-task.spec.ts-cache.json
@@ -1,6 +1,6 @@
{
"Enter x-secret-email into the email field": {
- "timestamp": 1765318973255,
+ "timestamp": 1765340032886,
"actions": [
{
"method": "fill",
@@ -10,7 +10,7 @@
]
},
"Fill out the form with the following details:\nName: John Smith\nAddress: 1045 La Avenida St, Mountain View, CA 94043\nEmail: [email protected]": {
- "timestamp": 1765318976120,
+ "timestamp": 1765340035431,
"actions": [
{
"method": "fill",
diff --git a/tests/mcp/browser.eval.ts b/tests/mcp/browser.eval.ts
index f15393abf4..28d528b87c 100644
--- a/tests/mcp/browser.eval.ts
+++ b/tests/mcp/browser.eval.ts
@@ -24,7 +24,7 @@ import type * as lowireLoop from '@lowire/loop';
test.use({ mcpHeadless: async ({ headless }, use) => use(headless) });
test('fill the form', async ({ loop, client, server }) => {
- expect(await loop.run<{ price: number, deliveryDays: number }>(`
+ const { result } = await loop.run<{ price: number, deliveryDays: number }>(`
Navigate to ${server.PREFIX + '/evals/fill-form.html'} via Playwright MCP.
Order a blue table and a green desk.
Use the following details for the order form:
@@ -41,5 +41,6 @@ test('fill the form', async ({ loop, client, server }) => {
price: z.number().describe('Total price in USD.'),
deliveryDays: z.number().describe('Maximum estimated delivery time in days.'),
})) as lowireLoop.Schema,
- })).toEqual({ price: 653.96, deliveryDays: 3 });
+ });
+ expect(result).toEqual({ price: 653.96, deliveryDays: 3 });
});