Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: add playwright-mdd experiment (#36430)

Pavel Feldman6b231cbf791a
diff --git a/package-lock.json b/package-lock.json
index 3ac58bc145..da1d9b6113 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1373,6 +1373,10 @@
"resolved": "packages/playwright-ct-vue",
"link": true
},
+ "node_modules/@playwright/mdd": {
+ "resolved": "packages/playwright-mdd",
+ "link": true
+ },
"node_modules/@playwright/test": {
"resolved": "packages/playwright-test",
"link": true
@@ -1754,6 +1758,16 @@
"@types/tern": "*"
}
},
+ "node_modules/@types/debug": {
+ "version": "4.1.12",
+ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
+ "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@types/ms": "*"
+ }
+ },
"node_modules/@types/estree": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz",
@@ -1812,6 +1826,13 @@
"@types/node": "*"
}
},
+ "node_modules/@types/ms": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz",
+ "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/@types/node": {
"version": "18.19.76",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.76.tgz",
@@ -2978,6 +2999,15 @@
"node": ">=0.1.90"
}
},
+ "node_modules/commander": {
+ "version": "13.1.0",
+ "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz",
+ "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=18"
+ }
+ },
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -3139,9 +3169,9 @@
}
},
"node_modules/debug": {
- "version": "4.4.0",
- "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
- "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
+ "version": "4.4.1",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
+ "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
@@ -3305,10 +3335,9 @@
}
},
"node_modules/dotenv": {
- "version": "16.4.7",
- "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz",
- "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==",
- "dev": true,
+ "version": "16.5.0",
+ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz",
+ "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=12"
@@ -5933,6 +5962,27 @@
"wrappy": "1"
}
},
+ "node_modules/openai": {
+ "version": "5.7.0",
+ "resolved": "https://registry.npmjs.org/openai/-/openai-5.7.0.tgz",
+ "integrity": "sha512-zXWawZl6J/P5Wz57/nKzVT3kJQZvogfuyuNVCdEp4/XU2UNrjL7SsuNpWAyLZbo6HVymwmnfno9toVzBhelygA==",
+ "license": "Apache-2.0",
+ "bin": {
+ "openai": "bin/cli"
+ },
+ "peerDependencies": {
+ "ws": "^8.18.0",
+ "zod": "^3.23.8"
+ },
+ "peerDependenciesMeta": {
+ "ws": {
+ "optional": true
+ },
+ "zod": {
+ "optional": true
+ }
+ }
+ },
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@@ -7809,10 +7859,10 @@
"license": "ISC"
},
"node_modules/ws": {
- "version": "8.18.0",
- "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
- "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
- "dev": true,
+ "version": "8.18.2",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz",
+ "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==",
+ "devOptional": true,
"license": "MIT",
"engines": {
"node": ">=10.0.0"
@@ -7956,12 +8006,20 @@
"version": "3.24.2",
"resolved": "https://registry.npmjs.org/zod/-/zod-3.24.2.tgz",
"integrity": "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==",
- "dev": true,
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/colinhacks"
}
},
+ "node_modules/zod-to-json-schema": {
+ "version": "3.24.5",
+ "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz",
+ "integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==",
+ "license": "ISC",
+ "peerDependencies": {
+ "zod": "^3.24.1"
+ }
+ },
"packages/html-reporter": {
"version": "0.0.0"
},
@@ -8706,6 +8764,44 @@
"node": ">=18"
}
},
+ "packages/playwright-mdd": {
+ "name": "@playwright/mdd",
+ "version": "0.0.1",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "commander": "^13.1.0",
+ "debug": "^4.4.1",
+ "dotenv": "^16.5.0",
+ "mime": "^4.0.7",
+ "openai": "^5.7.0",
+ "playwright-core": "1.54.0-next",
+ "zod-to-json-schema": "^3.24.4"
+ },
+ "bin": {
+ "playwright-mdd": "cli.js"
+ },
+ "devDependencies": {
+ "@types/debug": "^4.1.7"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "packages/playwright-mdd/node_modules/mime": {
+ "version": "4.0.7",
+ "resolved": "https://registry.npmjs.org/mime/-/mime-4.0.7.tgz",
+ "integrity": "sha512-2OfDPL+e03E0LrXaGYOtTFIYhiuzep94NSsuhrNULq+stylcJedcHdzHtz0atMUuGwJfFYs0YL5xeC/Ca2x0eQ==",
+ "funding": [
+ "https://github.com/sponsors/broofa"
+ ],
+ "license": "MIT",
+ "bin": {
+ "mime": "bin/cli.js"
+ },
+ "engines": {
+ "node": ">=16"
+ }
+ },
"packages/playwright-test": {
"name": "@playwright/test",
"version": "1.54.0-next",
diff --git a/packages/playwright-mdd/cli.js b/packages/playwright-mdd/cli.js
new file mode 100755
index 0000000000..7ce5384a0c
--- /dev/null
+++ b/packages/playwright-mdd/cli.js
@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { program } = require('./lib/program');
+void program.parseAsync(process.argv);
diff --git a/packages/playwright-mdd/package.json b/packages/playwright-mdd/package.json
new file mode 100644
index 0000000000..f5131e5ed0
--- /dev/null
+++ b/packages/playwright-mdd/package.json
@@ -0,0 +1,33 @@
+{
+ "name": "@playwright/mdd",
+ "version": "0.0.1",
+ "description": "Playwright MDD",
+ "private": true,
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/microsoft/playwright.git"
+ },
+ "homepage": "https://playwright.dev",
+ "engines": {
+ "node": ">=18"
+ },
+ "author": {
+ "name": "Microsoft Corporation"
+ },
+ "license": "Apache-2.0",
+ "dependencies": {
+ "commander": "^13.1.0",
+ "debug": "^4.4.1",
+ "dotenv": "^16.5.0",
+ "mime": "^4.0.7",
+ "openai": "^5.7.0",
+ "playwright-core": "1.54.0-next",
+ "zod-to-json-schema": "^3.24.4"
+ },
+ "devDependencies": {
+ "@types/debug": "^4.1.7"
+ },
+ "bin": {
+ "playwright-mdd": "cli.js"
+ }
+}
diff --git a/packages/playwright-mdd/src/context.ts b/packages/playwright-mdd/src/context.ts
new file mode 100644
index 0000000000..d998e51247
--- /dev/null
+++ b/packages/playwright-mdd/src/context.ts
@@ -0,0 +1,232 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import debug from 'debug';
+import * as playwright from 'playwright';
+
+import { callOnPageNoTrace, waitForCompletion } from './tools/utils';
+import { ManualPromise } from './manualPromise';
+
+import type { ModalState, Tool, ToolActionResult } from './tools/tool';
+
+type PendingAction = {
+ dialogShown: ManualPromise<void>;
+};
+
+type PageEx = playwright.Page & {
+ _snapshotForAI: () => Promise<string>;
+};
+
+const testDebug = debug('pw:mcp:test');
+
+export class Context {
+ readonly browser: playwright.Browser;
+ readonly page: playwright.Page;
+ readonly tools: Tool[];
+ private _modalStates: ModalState[] = [];
+ private _pendingAction: PendingAction | undefined;
+ private _downloads: { download: playwright.Download, finished: boolean, outputFile: string }[] = [];
+
+ constructor(browser: playwright.Browser, page: playwright.Page, tools: Tool[]) {
+ this.browser = browser;
+ this.page = page;
+ this.tools = tools;
+ testDebug('create context');
+ }
+
+ static async create(tools: Tool[]): Promise<Context> {
+ const browser = await playwright.chromium.launch({
+ headless: false,
+ });
+ const context = await browser.newContext();
+ const page = await context.newPage();
+ return new Context(browser, page, tools);
+ }
+
+ async close() {
+ await this.browser.close();
+ }
+
+ modalStates(): ModalState[] {
+ return this._modalStates;
+ }
+
+ setModalState(modalState: ModalState) {
+ this._modalStates.push(modalState);
+ }
+
+ clearModalState(modalState: ModalState) {
+ this._modalStates = this._modalStates.filter(state => state !== modalState);
+ }
+
+ modalStatesMarkdown(): string[] {
+ const result: string[] = ['### Modal state'];
+ if (this._modalStates.length === 0)
+ result.push('- There is no modal state present');
+ for (const state of this._modalStates) {
+ const tool = this.tools.find(tool => tool.clearsModalState === state.type);
+ result.push(`- [${state.description}]: can be handled by the "${tool?.schema.name}" tool`);
+ }
+ return result;
+ }
+
+ async run(tool: Tool, params: Record<string, unknown> | undefined): Promise<{ content: string, code: string[] }> {
+ const toolResult = await tool.handle(this, tool.schema.inputSchema.parse(params || {}));
+ const { code, action, waitForNetwork, captureSnapshot } = toolResult;
+ const racingAction = action ? () => this._raceAgainstModalDialogs(action) : undefined;
+
+ if (waitForNetwork)
+ await waitForCompletion(this, async () => racingAction?.());
+ else
+ await racingAction?.();
+
+ const result: string[] = [];
+
+ if (this.modalStates().length) {
+ result.push(...this.modalStatesMarkdown());
+ return {
+ code,
+ content: result.join('\n'),
+ };
+ }
+
+ if (this._downloads.length) {
+ result.push('', '### Downloads');
+ for (const entry of this._downloads) {
+ if (entry.finished)
+ result.push(`- Downloaded file ${entry.download.suggestedFilename()} to ${entry.outputFile}`);
+ else
+ result.push(`- Downloading file ${entry.download.suggestedFilename()} ...`);
+ }
+ result.push('');
+ }
+
+ result.push(
+ `- Page URL: ${this.page.url()}`,
+ `- Page Title: ${await this.title()}`
+ );
+
+ if (captureSnapshot && !this._javaScriptBlocked())
+ result.push(await this._snapshot());
+
+ return {
+ code,
+ content: result.join('\n'),
+ };
+ }
+
+ async title(): Promise<string> {
+ return await callOnPageNoTrace(this.page, page => page.title());
+ }
+
+ async waitForTimeout(time: number) {
+ if (this._javaScriptBlocked()) {
+ await new Promise(f => setTimeout(f, time));
+ return;
+ }
+
+ await callOnPageNoTrace(this.page, page => {
+ return page.evaluate(() => new Promise(f => setTimeout(f, 1000)));
+ });
+ }
+
+ async waitForLoadState(state: 'load', options?: { timeout?: number }): Promise<void> {
+ await callOnPageNoTrace(this.page, page => page.waitForLoadState(state, options).catch(() => {}));
+ }
+
+ async navigate(url: string) {
+ const downloadEvent = callOnPageNoTrace(this.page, page => page.waitForEvent('download').catch(() => {}));
+ try {
+ await this.page.goto(url, { waitUntil: 'domcontentloaded' });
+ } catch (_e: unknown) {
+ const e = _e as Error;
+ const mightBeDownload =
+ e.message.includes('net::ERR_ABORTED') // chromium
+ || e.message.includes('Download is starting'); // firefox + webkit
+ if (!mightBeDownload)
+ throw e;
+ // on chromium, the download event is fired *after* page.goto rejects, so we wait a lil bit
+ const download = await Promise.race([
+ downloadEvent,
+ new Promise(resolve => setTimeout(resolve, 1000)),
+ ]);
+ if (!download)
+ throw e;
+ }
+
+ // Cap load event to 5 seconds, the page is operational at this point.
+ await this.waitForLoadState('load', { timeout: 5000 });
+ }
+
+ refLocator(params: { element: string, ref: string }): playwright.Locator {
+ return this.page.locator(`aria-ref=${params.ref}`).describe(params.element);
+ }
+
+ private async _raceAgainstModalDialogs(action: () => Promise<ToolActionResult>): Promise<ToolActionResult> {
+ this._pendingAction = {
+ dialogShown: new ManualPromise(),
+ };
+
+ let result: ToolActionResult | undefined;
+ try {
+ await Promise.race([
+ action().then(r => result = r),
+ this._pendingAction.dialogShown,
+ ]);
+ } finally {
+ this._pendingAction = undefined;
+ }
+ return result;
+ }
+
+ private _javaScriptBlocked(): boolean {
+ return this._modalStates.some(state => state.type === 'dialog');
+ }
+
+ dialogShown(dialog: playwright.Dialog) {
+ this.setModalState({
+ type: 'dialog',
+ description: `"${dialog.type()}" dialog with message "${dialog.message()}"`,
+ dialog,
+ });
+ this._pendingAction?.dialogShown.resolve();
+ }
+
+ async downloadStarted(download: playwright.Download) {
+ const entry = {
+ download,
+ finished: false,
+ outputFile: this._outputFile(download.suggestedFilename())
+ };
+ this._downloads.push(entry);
+ await download.saveAs(entry.outputFile);
+ entry.finished = true;
+ }
+
+ private async _snapshot() {
+ const snapshot = await callOnPageNoTrace(this.page, page => (page as PageEx)._snapshotForAI());
+ return [
+ `- Page Snapshot`,
+ '```yaml',
+ snapshot,
+ '```',
+ ].join('\n');
+ }
+
+ private _outputFile(filename: string) {
+ return filename;
+ }
+}
diff --git a/packages/playwright-mdd/src/format.ts b/packages/playwright-mdd/src/format.ts
new file mode 100644
index 0000000000..a1fabbd97d
--- /dev/null
+++ b/packages/playwright-mdd/src/format.ts
@@ -0,0 +1,53 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// adapted from:
+// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/utils/isomorphic/stringUtils.ts
+// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/server/codegen/javascript.ts
+
+// NOTE: this function should not be used to escape any selectors.
+export function escapeWithQuotes(text: string, char: string = '\'') {
+ const stringified = JSON.stringify(text);
+ const escapedText = stringified.substring(1, stringified.length - 1).replace(/\\"/g, '"');
+ if (char === '\'')
+ return char + escapedText.replace(/[']/g, '\\\'') + char;
+ if (char === '"')
+ return char + escapedText.replace(/["]/g, '\\"') + char;
+ if (char === '`')
+ return char + escapedText.replace(/[`]/g, '`') + char;
+ throw new Error('Invalid escape char');
+}
+
+export function quote(text: string) {
+ return escapeWithQuotes(text, '\'');
+}
+
+export function formatObject(value: any, indent = ' '): string {
+ if (typeof value === 'string')
+ return quote(value);
+ if (Array.isArray(value))
+ return `[${value.map(o => formatObject(o)).join(', ')}]`;
+ if (typeof value === 'object') {
+ const keys = Object.keys(value).filter(key => value[key] !== undefined).sort();
+ if (!keys.length)
+ return '{}';
+ const tokens: string[] = [];
+ for (const key of keys)
+ tokens.push(`${key}: ${formatObject(value[key])}`);
+ return `{\n${indent}${tokens.join(`,\n${indent}`)}\n}`;
+ }
+ return String(value);
+}
diff --git a/packages/playwright-mdd/src/loop.ts b/packages/playwright-mdd/src/loop.ts
new file mode 100644
index 0000000000..64d715d3b2
--- /dev/null
+++ b/packages/playwright-mdd/src/loop.ts
@@ -0,0 +1,108 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import OpenAI from 'openai';
+import debug from 'debug';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+
+import { Tool } from './tools/tool';
+import { Context } from './context';
+
+/* eslint-disable no-console */
+
+export async function runTasks(context: Context, tasks: string[]): Promise<string> {
+ const openai = new OpenAI();
+ const allCode: string[] = [
+ `test('generated code', async ({ page }) => {`,
+ ];
+ for (const task of tasks) {
+ const { taskCode } = await runTask(openai, context, task);
+ if (taskCode.length)
+ allCode.push('', ...taskCode.map(code => ` ${code}`));
+ }
+ allCode.push('});');
+ return allCode.join('\n');
+}
+
+async function runTask(openai: OpenAI, context: Context, task: string): Promise<{ taskCode: string[] }> {
+ console.log('Perform task:', task);
+
+ const taskCode: string[] = [
+ `// ${task}`,
+ ];
+
+ const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [
+ {
+ role: 'user',
+ content: `Peform following task: ${task}. Once the task is complete, call the "done" tool.`
+ }
+ ];
+
+ for (let iteration = 0; iteration < 5; ++iteration) {
+ debug('history')(messages);
+ const response = await openai.chat.completions.create({
+ model: 'gpt-4.1',
+ messages,
+ tools: context.tools.map(asOpenAIDeclaration),
+ tool_choice: 'auto'
+ });
+
+ const message = response.choices[0].message;
+ if (!message.tool_calls?.length)
+ throw new Error('Unexpected response from LLM: ' + message.content);
+
+ messages.push({
+ role: 'assistant',
+ tool_calls: message.tool_calls
+ });
+
+ for (const toolCall of message.tool_calls) {
+ const functionCall = toolCall.function;
+ console.log('Call tool:', functionCall.name, functionCall.arguments);
+
+ if (functionCall.name === 'done')
+ return { taskCode };
+
+ const tool = context.tools.find(tool => tool.schema.name === functionCall.name);
+ if (!tool)
+ throw new Error('Unknown tool: ' + functionCall.name);
+
+ const { code, content } = await context.run(tool, JSON.parse(functionCall.arguments));
+ taskCode.push(...code);
+
+ messages.push({
+ role: 'tool',
+ tool_call_id: toolCall.id,
+ content,
+ });
+ }
+ }
+ throw new Error('Failed to perform step, max attempts reached');
+}
+
+function asOpenAIDeclaration(tool: Tool<any>): OpenAI.Chat.Completions.ChatCompletionTool {
+ const parameters = zodToJsonSchema(tool.schema.inputSchema);
+ delete parameters.$schema;
+ delete (parameters as any).additionalProperties;
+ return {
+ type: 'function',
+ function: {
+ name: tool.schema.name,
+ description: tool.schema.description,
+ parameters,
+ },
+ };
+}
diff --git a/packages/playwright-mdd/src/manualPromise.ts b/packages/playwright-mdd/src/manualPromise.ts
new file mode 100644
index 0000000000..a5034e05ec
--- /dev/null
+++ b/packages/playwright-mdd/src/manualPromise.ts
@@ -0,0 +1,127 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export class ManualPromise<T = void> extends Promise<T> {
+ private _resolve!: (t: T) => void;
+ private _reject!: (e: Error) => void;
+ private _isDone: boolean;
+
+ constructor() {
+ let resolve: (t: T) => void;
+ let reject: (e: Error) => void;
+ super((f, r) => {
+ resolve = f;
+ reject = r;
+ });
+ this._isDone = false;
+ this._resolve = resolve!;
+ this._reject = reject!;
+ }
+
+ isDone() {
+ return this._isDone;
+ }
+
+ resolve(t: T) {
+ this._isDone = true;
+ this._resolve(t);
+ }
+
+ reject(e: Error) {
+ this._isDone = true;
+ this._reject(e);
+ }
+
+ static override get [Symbol.species]() {
+ return Promise;
+ }
+
+ override get [Symbol.toStringTag]() {
+ return 'ManualPromise';
+ }
+}
+
+export class LongStandingScope {
+ private _terminateError: Error | undefined;
+ private _closeError: Error | undefined;
+ private _terminatePromises = new Map<ManualPromise<Error>, string[]>();
+ private _isClosed = false;
+
+ reject(error: Error) {
+ this._isClosed = true;
+ this._terminateError = error;
+ for (const p of this._terminatePromises.keys())
+ p.resolve(error);
+ }
+
+ close(error: Error) {
+ this._isClosed = true;
+ this._closeError = error;
+ for (const [p, frames] of this._terminatePromises)
+ p.resolve(cloneError(error, frames));
+ }
+
+ isClosed() {
+ return this._isClosed;
+ }
+
+ static async raceMultiple<T>(scopes: LongStandingScope[], promise: Promise<T>): Promise<T> {
+ return Promise.race(scopes.map(s => s.race(promise)));
+ }
+
+ async race<T>(promise: Promise<T> | Promise<T>[]): Promise<T> {
+ return this._race(Array.isArray(promise) ? promise : [promise], false) as Promise<T>;
+ }
+
+ async safeRace<T>(promise: Promise<T>, defaultValue?: T): Promise<T> {
+ return this._race([promise], true, defaultValue);
+ }
+
+ private async _race(promises: Promise<any>[], safe: boolean, defaultValue?: any): Promise<any> {
+ const terminatePromise = new ManualPromise<Error>();
+ const frames = captureRawStack();
+ if (this._terminateError)
+ terminatePromise.resolve(this._terminateError);
+ if (this._closeError)
+ terminatePromise.resolve(cloneError(this._closeError, frames));
+ this._terminatePromises.set(terminatePromise, frames);
+ try {
+ return await Promise.race([
+ terminatePromise.then(e => safe ? defaultValue : Promise.reject(e)),
+ ...promises
+ ]);
+ } finally {
+ this._terminatePromises.delete(terminatePromise);
+ }
+ }
+}
+
+function cloneError(error: Error, frames: string[]) {
+ const clone = new Error();
+ clone.name = error.name;
+ clone.message = error.message;
+ clone.stack = [error.name + ':' + error.message, ...frames].join('\n');
+ return clone;
+}
+
+function captureRawStack(): string[] {
+ const stackTraceLimit = Error.stackTraceLimit;
+ Error.stackTraceLimit = 50;
+ const error = new Error();
+ const stack = error.stack || '';
+ Error.stackTraceLimit = stackTraceLimit;
+ return stack.split('\n');
+}
diff --git a/packages/playwright-mdd/src/program.ts b/packages/playwright-mdd/src/program.ts
new file mode 100644
index 0000000000..aebfae0f12
--- /dev/null
+++ b/packages/playwright-mdd/src/program.ts
@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import dotenv from 'dotenv';
+import { program } from 'commander';
+
+import { runTasks } from './loop';
+import { Context } from './context';
+import { tools } from './tools';
+
+/* eslint-disable no-console */
+
+dotenv.config();
+
+const packageJSON = require('../package.json');
+
+program
+ .version('Version ' + packageJSON.version)
+ .name(packageJSON.name)
+ .action(async () => {
+ const context = await Context.create(tools);
+ const code = await runTasks(context, script);
+ console.log('Output code:');
+ console.log('```javascript');
+ console.log(code);
+ console.log('```');
+ await context.close();
+ });
+
+const script = [
+ 'Navigate to https://debs-obrien.github.io/playwright-movies-app',
+ 'Click search icon',
+ 'Type "Twister" in the search field and hit Enter',
+ 'Click on the link for the movie "Twisters"',
+];
+
+export { program };
diff --git a/packages/playwright-mdd/src/tools.ts b/packages/playwright-mdd/src/tools.ts
new file mode 100644
index 0000000000..e4b7c9a6ce
--- /dev/null
+++ b/packages/playwright-mdd/src/tools.ts
@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import snapshot from './tools/snapshot';
+import done from './tools/done';
+import navigate from './tools/navigate';
+
+import type { Tool } from './tools/tool.js';
+
+export const tools: Tool<any>[] = [
+ ...navigate,
+ ...snapshot,
+ ...done,
+];
diff --git a/packages/playwright-mdd/src/tools/done.ts b/packages/playwright-mdd/src/tools/done.ts
new file mode 100644
index 0000000000..8818d4368d
--- /dev/null
+++ b/packages/playwright-mdd/src/tools/done.ts
@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+
+import { defineTool } from './tool';
+
+const doneTool = defineTool({
+ schema: {
+ name: 'done',
+ description: 'Call this tool to indicate that the task is complete',
+ inputSchema: z.object({}),
+ },
+
+ handle: async () => {
+ return {
+ code: [],
+ captureSnapshot: false,
+ waitForNetwork: false,
+ };
+ },
+});
+
+export default [
+ doneTool,
+];
diff --git a/packages/playwright-mdd/src/tools/navigate.ts b/packages/playwright-mdd/src/tools/navigate.ts
new file mode 100644
index 0000000000..cf6f695fef
--- /dev/null
+++ b/packages/playwright-mdd/src/tools/navigate.ts
@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+
+import { defineTool } from './tool.js';
+
+const navigate = defineTool({
+ schema: {
+ name: 'browser_navigate',
+ description: 'Navigate to a URL',
+ inputSchema: z.object({
+ url: z.string().describe('The URL to navigate to'),
+ }),
+ },
+
+ handle: async (context, params) => {
+ const code = [
+ `await page.goto('${params.url}');`,
+ ];
+ await context.navigate(params.url);
+
+ return {
+ code,
+ captureSnapshot: true,
+ waitForNetwork: false,
+ };
+ },
+});
+
+const goBack = defineTool({
+ schema: {
+ name: 'browser_navigate_back',
+ description: 'Go back to the previous page',
+ inputSchema: z.object({}),
+ },
+
+ handle: async context => {
+ await context.page.goBack();
+ const code = [
+ `await page.goBack();`,
+ ];
+
+ return {
+ code,
+ captureSnapshot: true,
+ waitForNetwork: false,
+ };
+ },
+});
+
+const goForward = defineTool({
+ schema: {
+ name: 'browser_navigate_forward',
+ description: 'Go forward to the next page',
+ inputSchema: z.object({}),
+ },
+ handle: async context => {
+ await context.page.goForward();
+ const code = [
+ `await page.goForward();`,
+ ];
+ return {
+ code,
+ captureSnapshot: true,
+ waitForNetwork: false,
+ };
+ },
+});
+
+export default [
+ navigate,
+ goBack,
+ goForward,
+];
diff --git a/packages/playwright-mdd/src/tools/snapshot.ts b/packages/playwright-mdd/src/tools/snapshot.ts
new file mode 100644
index 0000000000..e2a90ad852
--- /dev/null
+++ b/packages/playwright-mdd/src/tools/snapshot.ts
@@ -0,0 +1,194 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+
+import { defineTool } from './tool';
+import * as format from '../format';
+import { generateLocator } from './utils';
+
+const snapshot = defineTool({
+ schema: {
+ name: 'browser_snapshot',
+ description: 'Capture accessibility snapshot of the current page, this is better than screenshot',
+ inputSchema: z.object({}),
+ },
+
+ handle: async () => {
+ return {
+ code: [],
+ captureSnapshot: true,
+ waitForNetwork: false,
+ };
+ },
+});
+
+const elementSchema = z.object({
+ element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
+ ref: z.string().describe('Exact target element reference from the page snapshot'),
+});
+
+const click = defineTool({
+ schema: {
+ name: 'browser_click',
+ description: 'Perform click on a web page',
+ inputSchema: elementSchema,
+ },
+
+ handle: async (context, params) => {
+ const locator = context.refLocator(params);
+
+ const code = [
+ `await page.${await generateLocator(locator)}.click();`
+ ];
+
+ return {
+ code,
+ action: () => locator.click(),
+ captureSnapshot: true,
+ waitForNetwork: true,
+ };
+ },
+});
+
+const drag = defineTool({
+ schema: {
+ name: 'browser_drag',
+ description: 'Perform drag and drop between two elements',
+ inputSchema: z.object({
+ startElement: z.string().describe('Human-readable source element description used to obtain the permission to interact with the element'),
+ startRef: z.string().describe('Exact source element reference from the page snapshot'),
+ endElement: z.string().describe('Human-readable target element description used to obtain the permission to interact with the element'),
+ endRef: z.string().describe('Exact target element reference from the page snapshot'),
+ }),
+ },
+
+ handle: async (context, params) => {
+ const startLocator = context.refLocator({ ref: params.startRef, element: params.startElement });
+ const endLocator = context.refLocator({ ref: params.endRef, element: params.endElement });
+
+ const code = [
+ `await page.${await generateLocator(startLocator)}.dragTo(page.${await generateLocator(endLocator)});`
+ ];
+
+ return {
+ code,
+ action: () => startLocator.dragTo(endLocator),
+ captureSnapshot: true,
+ waitForNetwork: true,
+ };
+ },
+});
+
+const hover = defineTool({
+ schema: {
+ name: 'browser_hover',
+ description: 'Hover over element on page',
+ inputSchema: elementSchema,
+ },
+
+ handle: async (context, params) => {
+ const locator = context.refLocator(params);
+
+ const code = [
+ `await page.${await generateLocator(locator)}.hover();`
+ ];
+
+ return {
+ code,
+ action: () => locator.hover(),
+ captureSnapshot: true,
+ waitForNetwork: true,
+ };
+ },
+});
+
+const typeSchema = elementSchema.extend({
+ text: z.string().describe('Text to type into the element'),
+ submit: z.boolean().optional().describe('Whether to submit entered text (press Enter after)'),
+ slowly: z.boolean().optional().describe('Whether to type one character at a time. Useful for triggering key handlers in the page. By default entire text is filled in at once.'),
+});
+
+const type = defineTool({
+ schema: {
+ name: 'browser_type',
+ description: 'Type text into editable element',
+ inputSchema: typeSchema,
+ },
+
+ handle: async (context, params) => {
+ const locator = context.refLocator(params);
+
+ const code: string[] = [];
+ const steps: (() => Promise<void>)[] = [];
+
+ if (params.slowly) {
+ code.push(`await page.${await generateLocator(locator)}.pressSequentially(${format.quote(params.text)});`);
+ steps.push(() => locator.pressSequentially(params.text));
+ } else {
+ code.push(`await page.${await generateLocator(locator)}.fill(${format.quote(params.text)});`);
+ steps.push(() => locator.fill(params.text));
+ }
+
+ if (params.submit) {
+ code.push(`await page.${await generateLocator(locator)}.press('Enter');`);
+ steps.push(() => locator.press('Enter'));
+ }
+
+ return {
+ code,
+ action: () => steps.reduce((acc, step) => acc.then(step), Promise.resolve()),
+ captureSnapshot: true,
+ waitForNetwork: true,
+ };
+ },
+});
+
+const selectOptionSchema = elementSchema.extend({
+ values: z.array(z.string()).describe('Array of values to select in the dropdown. This can be a single value or multiple values.'),
+});
+
+const selectOption = defineTool({
+ schema: {
+ name: 'browser_select_option',
+ description: 'Select an option in a dropdown',
+ inputSchema: selectOptionSchema,
+ },
+
+ handle: async (context, params) => {
+ const locator = context.refLocator(params);
+
+ const code = [
+ `await page.${await generateLocator(locator)}.selectOption(${format.formatObject(params.values)});`
+ ];
+
+ return {
+ code,
+ action: () => locator.selectOption(params.values).then(() => {}),
+ captureSnapshot: true,
+ waitForNetwork: true,
+ };
+ },
+});
+
+export default [
+ snapshot,
+ click,
+ drag,
+ hover,
+ type,
+ selectOption,
+];
diff --git a/packages/playwright-mdd/src/tools/tool.ts b/packages/playwright-mdd/src/tools/tool.ts
new file mode 100644
index 0000000000..9f840051eb
--- /dev/null
+++ b/packages/playwright-mdd/src/tools/tool.ts
@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { z } from 'zod';
+import type * as playwright from 'playwright-core';
+import type { Context } from '../context';
+
+export type ToolSchema<Input extends InputType> = {
+ name: string;
+ description: string;
+ inputSchema: Input;
+};
+
+type InputType = z.Schema;
+
+export type FileUploadModalState = {
+ type: 'fileChooser';
+ description: string;
+ fileChooser: playwright.FileChooser;
+};
+
+export type DialogModalState = {
+ type: 'dialog';
+ description: string;
+ dialog: playwright.Dialog;
+};
+
+export type ModalState = FileUploadModalState | DialogModalState;
+
+export type ToolActionResult = string | undefined | void;
+
+export type ToolResult = {
+ code: string[];
+ action?: () => Promise<void>;
+ captureSnapshot: boolean;
+ waitForNetwork: boolean;
+};
+
+export type Tool<Input extends InputType = InputType> = {
+ schema: ToolSchema<Input>;
+ clearsModalState?: ModalState['type'];
+ handle: (context: Context, params: z.output<Input>) => Promise<ToolResult>;
+};
+
+export function defineTool<Input extends InputType>(tool: Tool<Input>): Tool<Input> {
+ return tool;
+}
diff --git a/packages/playwright-mdd/src/tools/utils.ts b/packages/playwright-mdd/src/tools/utils.ts
new file mode 100644
index 0000000000..d82095edc8
--- /dev/null
+++ b/packages/playwright-mdd/src/tools/utils.ts
@@ -0,0 +1,91 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type * as playwright from 'playwright';
+import type { Context } from '../context';
+
+export async function waitForCompletion<R>(context: Context, callback: () => Promise<R>): Promise<R> {
+ const requests = new Set<playwright.Request>();
+ let frameNavigated = false;
+ let waitCallback: () => void = () => {};
+ const waitBarrier = new Promise<void>(f => { waitCallback = f; });
+
+ const requestListener = (request: playwright.Request) => requests.add(request);
+ const requestFinishedListener = (request: playwright.Request) => {
+ requests.delete(request);
+ if (!requests.size)
+ waitCallback();
+ };
+
+ const frameNavigateListener = (frame: playwright.Frame) => {
+ if (frame.parentFrame())
+ return;
+ frameNavigated = true;
+ dispose();
+ clearTimeout(timeout);
+ void context.waitForLoadState('load').then(waitCallback);
+ };
+
+ const onTimeout = () => {
+ dispose();
+ waitCallback();
+ };
+
+ context.page.on('request', requestListener);
+ context.page.on('requestfinished', requestFinishedListener);
+ context.page.on('framenavigated', frameNavigateListener);
+ const timeout = setTimeout(onTimeout, 10000);
+
+ const dispose = () => {
+ context.page.off('request', requestListener);
+ context.page.off('requestfinished', requestFinishedListener);
+ context.page.off('framenavigated', frameNavigateListener);
+ clearTimeout(timeout);
+ };
+
+ try {
+ const result = await callback();
+ if (!requests.size && !frameNavigated)
+ waitCallback();
+ await waitBarrier;
+ await context.page.waitForTimeout(1000);
+ return result;
+ } finally {
+ dispose();
+ }
+}
+
+export function sanitizeForFilePath(s: string) {
+ const sanitize = (s: string) => s.replace(/[\x00-\x2C\x2E-\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]+/g, '-');
+ const separator = s.lastIndexOf('.');
+ if (separator === -1)
+ return sanitize(s);
+ return sanitize(s.substring(0, separator)) + '.' + sanitize(s.substring(separator + 1));
+}
+
+export async function generateLocator(locator: playwright.Locator): Promise<string> {
+ try {
+ return await (locator as any)._generateLocatorString();
+ } catch (e) {
+ if (e instanceof Error && /locator._generateLocatorString: Timeout .* exceeded/.test(e.message))
+ throw new Error('Ref not found, likely because element was removed. Use browser_snapshot to see what elements are currently on the page.');
+ throw e;
+ }
+}
+
+export async function callOnPageNoTrace<T>(page: playwright.Page, callback: (page: playwright.Page) => Promise<T>): Promise<T> {
+ return await (page as any)._wrapApiCall(() => callback(page), { internal: true });
+}
diff --git a/utils/workspace.js b/utils/workspace.js
index 5331dc3335..928c61a0a0 100755
--- a/utils/workspace.js
+++ b/utils/workspace.js
@@ -219,6 +219,11 @@ const workspace = new Workspace(ROOT_PATH, [
path: path.join(ROOT_PATH, 'packages', 'playwright-ct-vue'),
files: ['LICENSE'],
}),
+ new PWPackage({
+ name: 'playwright-mdd',
+ path: path.join(ROOT_PATH, 'packages', 'playwright-mdd'),
+ files: ['LICENSE'],
+ }),
]);
if (require.main === module) {