Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: iterate on the perform cache (#38441)

Pavel Feldman40d8774a95a3
diff --git a/packages/playwright/src/agents/performTask.ts b/packages/playwright/src/agents/performTask.ts
index d8da6ac287..e95c2dc627 100644
--- a/packages/playwright/src/agents/performTask.ts
+++ b/packages/playwright/src/agents/performTask.ts
@@ -14,8 +14,11 @@
* limitations under the License.
*/
+import fs from 'fs';
+import path from 'path';
+
import { debug } from 'playwright-core/lib/utilsBundle';
-import { Loop } from 'playwright-core/lib/mcpBundle';
+import { z, zodToJsonSchema, Loop } from 'playwright-core/lib/mcpBundle';
import { identityBrowserContextFactory } from '../mcp/browser/browserContextFactory';
import { BrowserServerBackend } from '../mcp/browser/browserServerBackend';
@@ -24,6 +27,8 @@ import { wrapInClient } from '../mcp/sdk/server';
import type * as playwright from 'playwright-core';
import type * as lowireLoop from '@lowire/loop';
+import type * as zod from 'zod';
+import type { TestInfo } from '../../types/test';
export type PerformTaskOptions = {
provider?: 'github' | 'openai' | 'anthropic' | 'google';
@@ -33,7 +38,23 @@ export type PerformTaskOptions = {
temperature?: number;
};
-export async function performTask(context: playwright.BrowserContext, task: string, options: PerformTaskOptions) {
+const resultSchema = z.object({
+ code: z.string().optional().describe(`
+Generated code to perform the task using Playwright API.
+Check out the <code> blocks and combine them. Should be presented in the following form:
+
+perform(async ({ page }) => {
+ // generated code here.
+});
+`),
+ error: z.string().optional().describe('The error that occurred if execution failed.').optional(),
+});
+
+export async function performTask(testInfo: TestInfo, context: playwright.BrowserContext, userTask: string, options: PerformTaskOptions) {
+ const cacheStatus = await performTaskFromCache(testInfo, context, userTask);
+ if (cacheStatus === 'success')
+ return;
+
const backend = new BrowserServerBackend(defaultConfig, identityBrowserContextFactory(context));
const client = await wrapInClient(backend, { name: 'Internal', version: '0.0.0' });
const callTool: (params: { name: string, arguments: any}) => Promise<lowireLoop.ToolResult> = async params => {
@@ -52,8 +73,103 @@ export async function performTask(context: playwright.BrowserContext, task: stri
});
try {
- return await loop.run(task);
+ const result = await loop.run<zod.infer<typeof resultSchema>>(userTask, { resultSchema: zodToJsonSchema(resultSchema) as lowireLoop.Schema });
+ if (result.code)
+ await updatePerformFile(testInfo, userTask, result.code, options);
} finally {
await client.close();
}
}
+
+async function updatePerformFile(testInfo: TestInfo, userTask: string, taskCode: string, options: PerformTaskOptions) {
+ const relativeFile = path.relative(testInfo.project.testDir, testInfo.file);
+ const promptCacheFile = testInfo.file.replace('.spec.ts', '.cache.ts');
+ const testTitle = testInfo.title;
+
+ const loop = new Loop(options.provider ?? 'github', {
+ model: options.model ?? 'claude-sonnet-4.5',
+ reasoning: options.reasoning,
+ temperature: options.temperature,
+ maxTokens: options.maxTokens,
+ summarize: true,
+ debug,
+ callTool: async () => ({ content: [] }),
+ tools: [],
+ });
+
+ const resultSchema = z.object({
+ code: z.string().optional().describe(`
+Generated code with all the perofrm routines combined or updated into the following format:
+
+import { performCache } from '@playwright/test';
+
+performCache({
+ file: 'tests/page/perform-task.spec.ts',
+ test: 'perform task',
+ task: 'Click the learn more button',
+ code: async ({ page }) => {
+ await page.getByRole('link', { name: 'Learn more' }).click();
+ },
+});
+`),
+ });
+
+ const existingCode = await fs.promises.readFile(promptCacheFile, 'utf8').catch(() => '');
+ const task = `
+- Create or update a perform file to include performCache block for the given task and code.
+- Dedupe items with the same file, test, and task.
+- Should produce code in the following format
+
+import { performCache } from '@playwright/test';
+
+performCache({
+ file: '<file>',
+ test: '<test>',
+ task: '<task>',
+ code: async ({ page }) => {
+ <code>
+ },
+});
+
+performCache({
+...
+
+## Params for the new or updated performCache block
+<file-content>${existingCode}</file-content>
+<file>${relativeFile}</file>
+<test>${testTitle}</test>
+<task>${userTask}</task>
+<code>${taskCode}</code>
+`;
+
+ const result = await loop.run<zod.infer<typeof resultSchema>>(task, { resultSchema: zodToJsonSchema(resultSchema) as lowireLoop.Schema });
+ if (result.code)
+ await fs.promises.writeFile(promptCacheFile, result.code);
+}
+
+type PerformCacheEntry = {
+ file: string,
+ test: string,
+ task: string,
+ code: ({ page }: { page: playwright.Page }) => Promise<void>
+};
+
+const performCacheMap = new Map<string, PerformCacheEntry>();
+
+export function performCache(entry: PerformCacheEntry) {
+ performCacheMap.set(JSON.stringify({ ...entry, code: undefined }), entry);
+}
+
+async function performTaskFromCache(testInfo: TestInfo, context: playwright.BrowserContext, userTask: string): Promise<'success' | 'cache-miss' | Error> {
+ const relativeFile = path.relative(testInfo.project.testDir, testInfo.file);
+ const key = JSON.stringify({ file: relativeFile, test: testInfo.title, task: userTask });
+ const entry = performCacheMap.get(key);
+ if (!entry)
+ return 'cache-miss';
+ try {
+ await entry.code({ page: context.pages()[0] });
+ return 'success';
+ } catch (error) {
+ return error;
+ }
+}
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index 56c1fc9b3b..8868e54bc6 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -462,9 +462,9 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
}
},
- _perform: async ({ context }, use) => {
+ _perform: async ({ context }, use, testInfo) => {
await use(async (task: string, options?: PerformTaskOptions) => {
- await performTask(context, task, options ?? {});
+ await performTask(testInfo, context, task, options ?? {});
});
},
});
@@ -794,3 +794,4 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>(playwrightFix
export { defineConfig } from './common/configLoader';
export { mergeTests } from './common/testType';
export { mergeExpects } from './matchers/expect';
+export { performCache } from './agents/performTask';
diff --git a/tests/page/perform-task.cache.ts b/tests/page/perform-task.cache.ts
new file mode 100644
index 0000000000..e3bbc5875c
--- /dev/null
+++ b/tests/page/perform-task.cache.ts
@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-ignore
+import { performCache } from '@playwright/test';
+
+performCache({
+ file: 'perform-task.spec.ts',
+ test: 'perform task',
+ task: 'Add "Buy groceries" todo',
+ code: async ({ page }) => {
+ await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Buy groceries');
+ await page.keyboard.press('Enter');
+ },
+});
+
+performCache({
+ file: 'perform-task.spec.ts',
+ test: 'perform task',
+ task: 'Add "Walk the dog" todo',
+ code: async ({ page }) => {
+ await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Walk the dog');
+ await page.keyboard.press('Enter');
+ },
+});
+
+performCache({
+ file: 'perform-task.spec.ts',
+ test: 'perform task',
+ task: 'Add "Read a book" todo',
+ code: async ({ page }) => {
+ await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Read a book');
+ await page.getByRole('textbox', { name: 'What needs to be done?' }).press('Enter');
+ },
+});
diff --git a/tests/page/perform-task.spec.ts b/tests/page/perform-task.spec.ts
index 0da7e62466..f629795aa3 100644
--- a/tests/page/perform-task.spec.ts
+++ b/tests/page/perform-task.spec.ts
@@ -15,9 +15,12 @@
*/
import { test as it } from './pageTest';
+import './perform-task.cache';
// @ts-ignore
-it.skip('perform task', async ({ page, _perform }) => {
- await page.goto(`https://playwright.dev/`);
- await _perform('Click the get started button');
+it('perform task', async ({ page, _perform }) => {
+ await page.goto('https://demo.playwright.dev/todomvc');
+ await _perform('Add "Buy groceries" todo');
+ await _perform('Add "Walk the dog" todo');
+ await _perform('Add "Read a book" todo');
});