Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: that-should-be-it perform mode (#38503)
Pavel Feldman8f97da76c67a
diff --git a/package-lock.json b/package-lock.jsonindex 1a695eb458..80c770a7e3 100644--- a/package-lock.json+++ b/package-lock.json@@ -18,7 +18,7 @@"@eslint/compat": "^1.3.2","@eslint/eslintrc": "^3.3.1","@eslint/js": "^9.34.0",- "@lowire/loop": "^0.0.6",+ "@lowire/loop": "^0.0.7","@modelcontextprotocol/sdk": "^1.17.5","@octokit/graphql-schema": "^15.26.0","@stylistic/eslint-plugin": "^5.2.3",@@ -1064,9 +1064,9 @@}},"node_modules/@lowire/loop": {- "version": "0.0.6",- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.6.tgz",- "integrity": "sha512-mrQft4UUj4tWt8ikOvRjec88nLRt6XwEnfzyFlZggedV735OcKE5Bk0Yy2mr3QCRCVn4/aw91lugRBS9ftSSpg==",+ "version": "0.0.7",+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.7.tgz",+ "integrity": "sha512-Hn/VU1v+Ld8m9y3AkFg3Naeht79Ub9dtefljGy5WEMZbrCJkONw4Q2GqihCWys8tcUQZUf9pUTyTIvC+A6HEQw==","dev": true,"license": "Apache-2.0","engines": {diff --git a/package.json b/package.jsonindex 21938cd139..334ad20a59 100644--- a/package.json+++ b/package.json@@ -56,7 +56,7 @@"@eslint/compat": "^1.3.2","@eslint/eslintrc": "^3.3.1","@eslint/js": "^9.34.0",- "@lowire/loop": "^0.0.6",+ "@lowire/loop": "^0.0.7","@modelcontextprotocol/sdk": "^1.17.5","@octokit/graphql-schema": "^15.26.0","@stylistic/eslint-plugin": "^5.2.3",diff --git a/packages/playwright-core/ThirdPartyNotices.txt b/packages/playwright-core/ThirdPartyNotices.txtindex f0f493fa9c..4a0f753351 100644--- a/packages/playwright-core/ThirdPartyNotices.txt+++ b/packages/playwright-core/ThirdPartyNotices.txt@@ -4,7 +4,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATIONThis project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise.-- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)+- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)- @modelcontextprotocol/[email protected] (https://github.com/modelcontextprotocol/typescript-sdk)- [email protected] (https://github.com/jshttp/accepts)- [email protected] (https://github.com/TooTallNate/proxy-agents)@@ -135,7 +135,7 @@ This project incorporates components from the projects listed below. The origina- [email protected] (https://github.com/StefanTerdell/zod-to-json-schema)- [email protected] (https://github.com/colinhacks/zod)-%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE+%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE=========================================Apache LicenseVersion 2.0, January 2004@@ -339,7 +339,7 @@ Apache LicenseSee the License for the specific language governing permissions andlimitations under the License.=========================================-END OF @lowire/[email protected] AND INFORMATION+END OF @lowire/[email protected] AND INFORMATION%% @modelcontextprotocol/[email protected] NOTICES AND INFORMATION BEGIN HERE=========================================diff --git a/packages/playwright-core/bundles/mcp/package-lock.json b/packages/playwright-core/bundles/mcp/package-lock.jsonindex e6b3091f7f..bf5ee6bf3a 100644--- a/packages/playwright-core/bundles/mcp/package-lock.json+++ b/packages/playwright-core/bundles/mcp/package-lock.json@@ -8,16 +8,16 @@"name": "mcp-bundle","version": "0.0.1","dependencies": {- "@lowire/loop": "^0.0.6",+ "@lowire/loop": "^0.0.7","@modelcontextprotocol/sdk": "^1.24.0","zod": "^3.25.76","zod-to-json-schema": "^3.24.6"}},"node_modules/@lowire/loop": {- "version": "0.0.6",- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.6.tgz",- "integrity": "sha512-mrQft4UUj4tWt8ikOvRjec88nLRt6XwEnfzyFlZggedV735OcKE5Bk0Yy2mr3QCRCVn4/aw91lugRBS9ftSSpg==",+ "version": "0.0.7",+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.7.tgz",+ "integrity": "sha512-Hn/VU1v+Ld8m9y3AkFg3Naeht79Ub9dtefljGy5WEMZbrCJkONw4Q2GqihCWys8tcUQZUf9pUTyTIvC+A6HEQw==","license": "Apache-2.0","engines": {"node": ">=20"diff --git a/packages/playwright-core/bundles/mcp/package.json b/packages/playwright-core/bundles/mcp/package.jsonindex fa046d515d..eca0006903 100644--- a/packages/playwright-core/bundles/mcp/package.json+++ b/packages/playwright-core/bundles/mcp/package.json@@ -4,7 +4,7 @@"private": true,"dependencies": {"@modelcontextprotocol/sdk": "^1.24.0",- "@lowire/loop": "^0.0.6",+ "@lowire/loop": "^0.0.7","zod": "^3.25.76","zod-to-json-schema": "^3.24.6"}diff --git a/packages/playwright-core/src/server/agent/agent.ts b/packages/playwright-core/src/server/agent/agent.tsindex 27866b04cc..d952e814e3 100644--- a/packages/playwright-core/src/server/agent/agent.ts+++ b/packages/playwright-core/src/server/agent/agent.ts@@ -18,7 +18,7 @@ import fs from 'fs';import { toolsForLoop } from './backend';import { debug } from '../../utilsBundle';-import { Loop, z, zodToJsonSchema } from '../../mcpBundle';+import { Loop } from '../../mcpBundle';import { runAction } from './actionRunner';import { Context } from './context';@@ -34,9 +34,7 @@ export async function pagePerform(progress: Progress, page: Page, options: channif (await cachedPerform(context, options))return;- await perform(context, options.task, zodToJsonSchema(z.object({- error: z.string().optional().describe('An error message if the task could not be completed successfully'),- })) as loopTypes.Schema, options);+ await perform(context, options.task, undefined, options);await updateCache(context, options);}@@ -51,7 +49,7 @@ ${options.query}`;return await perform(context, task, options.schema, options);}-async function perform(context: Context, userTask: string, resultSchema: loopTypes.Schema, options: { maxTurns?: number } = {}): Promise<any> {+async function perform(context: Context, userTask: string, resultSchema: loopTypes.Schema | undefined, options: { maxTurns?: number } = {}): Promise<any> {const { progress, page } = context;const browserContext = page.browserContext;if (!browserContext._options.agent)@@ -66,6 +64,13 @@ async function perform(context: Context, userTask: string, resultSchema: loopTypdebug,callTool,tools,+ beforeTurn: params => {+ const lastReply = params.conversation.messages.findLast(m => m.role === 'assistant');+ const toolCall = lastReply?.content.find(c => c.type === 'tool_call');+ if (!resultSchema && toolCall && toolCall.arguments.thatShouldBeIt)+ return 'break';+ return 'continue';+ },...options});@@ -75,9 +80,8 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp${full}`;- return await loop.run(task, {- resultSchema- });+ const { result } = await loop.run(task, { resultSchema });+ return result;}type CachedActions = Record<string, {@@ -121,8 +125,7 @@ async function updateCache(context: Context, options: channels.PagePerformParamsasync function cachedActions(cacheFile: string): Promise<CachedActions> {let cache = allCaches.get(cacheFile);if (!cache) {- const text = await fs.promises.readFile(cacheFile, 'utf-8').catch(() => '{}');- cache = JSON.parse(text) as CachedActions;+ cache = await fs.promises.readFile(cacheFile, 'utf-8').then(text => JSON.parse(text)).catch(() => ({})) as CachedActions;allCaches.set(cacheFile, cache);}return cache;diff --git a/packages/playwright-core/src/server/agent/tools.ts b/packages/playwright-core/src/server/agent/tools.tsindex 33125bc33b..7eac91aa07 100644--- a/packages/playwright-core/src/server/agent/tools.ts+++ b/packages/playwright-core/src/server/agent/tools.ts@@ -35,12 +35,16 @@ function defineTool<Input extends zod.Schema>(tool: ToolDefinition<Input>): Toolreturn tool;}+const baseSchema = z.object({+ thatShouldBeIt: z.boolean().describe('Indicates that this tool call is sufficient to complete the task. If false, the task will continue with the next tool call'),+});+const snapshot = defineTool({schema: {name: 'browser_snapshot',title: 'Page snapshot',description: 'Capture accessibility snapshot of the current page, this is better than screenshot',- inputSchema: z.object({}),+ inputSchema: baseSchema,},handle: async (context, params) => {@@ -48,7 +52,7 @@ const snapshot = defineTool({},});-const elementSchema = z.object({+const elementSchema = baseSchema.extend({element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),ref: z.string().describe('Exact target element reference from the page snapshot'),});@@ -86,7 +90,7 @@ const drag = defineTool({name: 'browser_drag',title: 'Drag mouse',description: 'Perform drag and drop between two elements',- inputSchema: z.object({+ inputSchema: baseSchema.extend({startElement: z.string().describe('Human-readable source element description used to obtain the permission to interact with the element'),startRef: z.string().describe('Exact source element reference from the page snapshot'),endElement: z.string().describe('Human-readable target element description used to obtain the permission to interact with the element'),@@ -211,7 +215,7 @@ const fillForm = defineTool({name: 'browser_fill_form',title: 'Fill form',description: 'Fill multiple form fields',- inputSchema: z.object({+ inputSchema: baseSchema.extend({fields: z.array(z.object({name: z.string().describe('Human-readable field name'),type: z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider']).describe('Type of the field'),diff --git a/packages/playwright/src/agents/agent.ts b/packages/playwright/src/agents/agent.tsindex b0b48b647d..c252eafba2 100644--- a/packages/playwright/src/agents/agent.ts+++ b/packages/playwright/src/agents/agent.ts@@ -50,12 +50,13 @@ export class Agent<T extends z.ZodSchema<any>> {const { clients, tools, callTool } = await this._initClients();const prompt = this.spec.description;try {- return await this.loop.run<z.output<T>>(`${prompt}\n\nTask:\n${task}\n\nParams:\n${JSON.stringify(params, null, 2)}`, {+ const { result } = await this.loop.run<z.output<T>>(`${prompt}\n\nTask:\n${task}\n\nParams:\n${JSON.stringify(params, null, 2)}`, {...options,tools,callTool,resultSchema: this.resultSchema});+ return result;} finally {await this._disconnectFromServers(clients);}diff --git a/tests/library/perform-task.spec.ts-cache.json b/tests/library/perform-task.spec.ts-cache.jsonindex c1deb2ea11..11b59d4b9d 100644--- a/tests/library/perform-task.spec.ts-cache.json+++ b/tests/library/perform-task.spec.ts-cache.json@@ -1,6 +1,6 @@{"Enter x-secret-email into the email field": {- "timestamp": 1765318973255,+ "timestamp": 1765340032886,"actions": [{"method": "fill",@@ -10,7 +10,7 @@]},"Fill out the form with the following details:\nName: John Smith\nAddress: 1045 La Avenida St, Mountain View, CA 94043\nEmail: [email protected]": {- "timestamp": 1765318976120,+ "timestamp": 1765340035431,"actions": [{"method": "fill",diff --git a/tests/mcp/browser.eval.ts b/tests/mcp/browser.eval.tsindex f15393abf4..28d528b87c 100644--- a/tests/mcp/browser.eval.ts+++ b/tests/mcp/browser.eval.ts@@ -24,7 +24,7 @@ import type * as lowireLoop from '@lowire/loop';test.use({ mcpHeadless: async ({ headless }, use) => use(headless) });test('fill the form', async ({ loop, client, server }) => {- expect(await loop.run<{ price: number, deliveryDays: number }>(`+ const { result } = await loop.run<{ price: number, deliveryDays: number }>(`Navigate to ${server.PREFIX + '/evals/fill-form.html'} via Playwright MCP.Order a blue table and a green desk.Use the following details for the order form:@@ -41,5 +41,6 @@ test('fill the form', async ({ loop, client, server }) => {price: z.number().describe('Total price in USD.'),deliveryDays: z.number().describe('Maximum estimated delivery time in days.'),})) as lowireLoop.Schema,- })).toEqual({ price: 653.96, deliveryDays: 3 });+ });+ expect(result).toEqual({ price: 653.96, deliveryDays: 3 });});