Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: add playwright-mdd experiment (#36430)
Pavel Feldman6b231cbf791a
diff --git a/package-lock.json b/package-lock.jsonindex 3ac58bc145..da1d9b6113 100644--- a/package-lock.json+++ b/package-lock.json@@ -1373,6 +1373,10 @@"resolved": "packages/playwright-ct-vue","link": true},+ "node_modules/@playwright/mdd": {+ "resolved": "packages/playwright-mdd",+ "link": true+ },"node_modules/@playwright/test": {"resolved": "packages/playwright-test","link": true@@ -1754,6 +1758,16 @@"@types/tern": "*"}},+ "node_modules/@types/debug": {+ "version": "4.1.12",+ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",+ "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@types/ms": "*"+ }+ },"node_modules/@types/estree": {"version": "1.0.7","resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz",@@ -1812,6 +1826,13 @@"@types/node": "*"}},+ "node_modules/@types/ms": {+ "version": "2.1.0",+ "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz",+ "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==",+ "dev": true,+ "license": "MIT"+ },"node_modules/@types/node": {"version": "18.19.76","resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.76.tgz",@@ -2978,6 +2999,15 @@"node": ">=0.1.90"}},+ "node_modules/commander": {+ "version": "13.1.0",+ "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz",+ "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==",+ "license": "MIT",+ "engines": {+ "node": ">=18"+ }+ },"node_modules/concat-map": {"version": "0.0.1","resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",@@ -3139,9 +3169,9 @@}},"node_modules/debug": {- "version": "4.4.0",- "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",- "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",+ "version": "4.4.1",+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",+ "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==","license": "MIT","dependencies": {"ms": "^2.1.3"@@ -3305,10 +3335,9 @@}},"node_modules/dotenv": {- "version": "16.4.7",- "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz",- "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==",- "dev": true,+ "version": "16.5.0",+ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz",+ "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==","license": "BSD-2-Clause","engines": {"node": ">=12"@@ -5933,6 +5962,27 @@"wrappy": "1"}},+ "node_modules/openai": {+ "version": "5.7.0",+ "resolved": "https://registry.npmjs.org/openai/-/openai-5.7.0.tgz",+ "integrity": "sha512-zXWawZl6J/P5Wz57/nKzVT3kJQZvogfuyuNVCdEp4/XU2UNrjL7SsuNpWAyLZbo6HVymwmnfno9toVzBhelygA==",+ "license": "Apache-2.0",+ "bin": {+ "openai": "bin/cli"+ },+ "peerDependencies": {+ "ws": "^8.18.0",+ "zod": "^3.23.8"+ },+ "peerDependenciesMeta": {+ "ws": {+ "optional": true+ },+ "zod": {+ "optional": true+ }+ }+ },"node_modules/optionator": {"version": "0.9.4","resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",@@ -7809,10 +7859,10 @@"license": "ISC"},"node_modules/ws": {- "version": "8.18.0",- "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",- "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",- "dev": true,+ "version": "8.18.2",+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz",+ "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==",+ "devOptional": true,"license": "MIT","engines": {"node": ">=10.0.0"@@ -7956,12 +8006,20 @@"version": "3.24.2","resolved": "https://registry.npmjs.org/zod/-/zod-3.24.2.tgz","integrity": "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==",- "dev": true,"license": "MIT","funding": {"url": "https://github.com/sponsors/colinhacks"}},+ "node_modules/zod-to-json-schema": {+ "version": "3.24.5",+ "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz",+ "integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==",+ "license": "ISC",+ "peerDependencies": {+ "zod": "^3.24.1"+ }+ },"packages/html-reporter": {"version": "0.0.0"},@@ -8706,6 +8764,44 @@"node": ">=18"}},+ "packages/playwright-mdd": {+ "name": "@playwright/mdd",+ "version": "0.0.1",+ "license": "Apache-2.0",+ "dependencies": {+ "commander": "^13.1.0",+ "debug": "^4.4.1",+ "dotenv": "^16.5.0",+ "mime": "^4.0.7",+ "openai": "^5.7.0",+ "playwright-core": "1.54.0-next",+ "zod-to-json-schema": "^3.24.4"+ },+ "bin": {+ "playwright-mdd": "cli.js"+ },+ "devDependencies": {+ "@types/debug": "^4.1.7"+ },+ "engines": {+ "node": ">=18"+ }+ },+ "packages/playwright-mdd/node_modules/mime": {+ "version": "4.0.7",+ "resolved": "https://registry.npmjs.org/mime/-/mime-4.0.7.tgz",+ "integrity": "sha512-2OfDPL+e03E0LrXaGYOtTFIYhiuzep94NSsuhrNULq+stylcJedcHdzHtz0atMUuGwJfFYs0YL5xeC/Ca2x0eQ==",+ "funding": [+ "https://github.com/sponsors/broofa"+ ],+ "license": "MIT",+ "bin": {+ "mime": "bin/cli.js"+ },+ "engines": {+ "node": ">=16"+ }+ },"packages/playwright-test": {"name": "@playwright/test","version": "1.54.0-next",diff --git a/packages/playwright-mdd/cli.js b/packages/playwright-mdd/cli.jsnew file mode 100755index 0000000000..7ce5384a0c--- /dev/null+++ b/packages/playwright-mdd/cli.js@@ -0,0 +1,19 @@+#!/usr/bin/env node+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++const { program } = require('./lib/program');+void program.parseAsync(process.argv);diff --git a/packages/playwright-mdd/package.json b/packages/playwright-mdd/package.jsonnew file mode 100644index 0000000000..f5131e5ed0--- /dev/null+++ b/packages/playwright-mdd/package.json@@ -0,0 +1,33 @@+{+ "name": "@playwright/mdd",+ "version": "0.0.1",+ "description": "Playwright MDD",+ "private": true,+ "repository": {+ "type": "git",+ "url": "git+https://github.com/microsoft/playwright.git"+ },+ "homepage": "https://playwright.dev",+ "engines": {+ "node": ">=18"+ },+ "author": {+ "name": "Microsoft Corporation"+ },+ "license": "Apache-2.0",+ "dependencies": {+ "commander": "^13.1.0",+ "debug": "^4.4.1",+ "dotenv": "^16.5.0",+ "mime": "^4.0.7",+ "openai": "^5.7.0",+ "playwright-core": "1.54.0-next",+ "zod-to-json-schema": "^3.24.4"+ },+ "devDependencies": {+ "@types/debug": "^4.1.7"+ },+ "bin": {+ "playwright-mdd": "cli.js"+ }+}diff --git a/packages/playwright-mdd/src/context.ts b/packages/playwright-mdd/src/context.tsnew file mode 100644index 0000000000..d998e51247--- /dev/null+++ b/packages/playwright-mdd/src/context.ts@@ -0,0 +1,232 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import debug from 'debug';+import * as playwright from 'playwright';++import { callOnPageNoTrace, waitForCompletion } from './tools/utils';+import { ManualPromise } from './manualPromise';++import type { ModalState, Tool, ToolActionResult } from './tools/tool';++type PendingAction = {+ dialogShown: ManualPromise<void>;+};++type PageEx = playwright.Page & {+ _snapshotForAI: () => Promise<string>;+};++const testDebug = debug('pw:mcp:test');++export class Context {+ readonly browser: playwright.Browser;+ readonly page: playwright.Page;+ readonly tools: Tool[];+ private _modalStates: ModalState[] = [];+ private _pendingAction: PendingAction | undefined;+ private _downloads: { download: playwright.Download, finished: boolean, outputFile: string }[] = [];++ constructor(browser: playwright.Browser, page: playwright.Page, tools: Tool[]) {+ this.browser = browser;+ this.page = page;+ this.tools = tools;+ testDebug('create context');+ }++ static async create(tools: Tool[]): Promise<Context> {+ const browser = await playwright.chromium.launch({+ headless: false,+ });+ const context = await browser.newContext();+ const page = await context.newPage();+ return new Context(browser, page, tools);+ }++ async close() {+ await this.browser.close();+ }++ modalStates(): ModalState[] {+ return this._modalStates;+ }++ setModalState(modalState: ModalState) {+ this._modalStates.push(modalState);+ }++ clearModalState(modalState: ModalState) {+ this._modalStates = this._modalStates.filter(state => state !== modalState);+ }++ modalStatesMarkdown(): string[] {+ const result: string[] = ['### Modal state'];+ if (this._modalStates.length === 0)+ result.push('- There is no modal state present');+ for (const state of this._modalStates) {+ const tool = this.tools.find(tool => tool.clearsModalState === state.type);+ result.push(`- [${state.description}]: can be handled by the "${tool?.schema.name}" tool`);+ }+ return result;+ }++ async run(tool: Tool, params: Record<string, unknown> | undefined): Promise<{ content: string, code: string[] }> {+ const toolResult = await tool.handle(this, tool.schema.inputSchema.parse(params || {}));+ const { code, action, waitForNetwork, captureSnapshot } = toolResult;+ const racingAction = action ? () => this._raceAgainstModalDialogs(action) : undefined;++ if (waitForNetwork)+ await waitForCompletion(this, async () => racingAction?.());+ else+ await racingAction?.();++ const result: string[] = [];++ if (this.modalStates().length) {+ result.push(...this.modalStatesMarkdown());+ return {+ code,+ content: result.join('\n'),+ };+ }++ if (this._downloads.length) {+ result.push('', '### Downloads');+ for (const entry of this._downloads) {+ if (entry.finished)+ result.push(`- Downloaded file ${entry.download.suggestedFilename()} to ${entry.outputFile}`);+ else+ result.push(`- Downloading file ${entry.download.suggestedFilename()} ...`);+ }+ result.push('');+ }++ result.push(+ `- Page URL: ${this.page.url()}`,+ `- Page Title: ${await this.title()}`+ );++ if (captureSnapshot && !this._javaScriptBlocked())+ result.push(await this._snapshot());++ return {+ code,+ content: result.join('\n'),+ };+ }++ async title(): Promise<string> {+ return await callOnPageNoTrace(this.page, page => page.title());+ }++ async waitForTimeout(time: number) {+ if (this._javaScriptBlocked()) {+ await new Promise(f => setTimeout(f, time));+ return;+ }++ await callOnPageNoTrace(this.page, page => {+ return page.evaluate(() => new Promise(f => setTimeout(f, 1000)));+ });+ }++ async waitForLoadState(state: 'load', options?: { timeout?: number }): Promise<void> {+ await callOnPageNoTrace(this.page, page => page.waitForLoadState(state, options).catch(() => {}));+ }++ async navigate(url: string) {+ const downloadEvent = callOnPageNoTrace(this.page, page => page.waitForEvent('download').catch(() => {}));+ try {+ await this.page.goto(url, { waitUntil: 'domcontentloaded' });+ } catch (_e: unknown) {+ const e = _e as Error;+ const mightBeDownload =+ e.message.includes('net::ERR_ABORTED') // chromium+ || e.message.includes('Download is starting'); // firefox + webkit+ if (!mightBeDownload)+ throw e;+ // on chromium, the download event is fired *after* page.goto rejects, so we wait a lil bit+ const download = await Promise.race([+ downloadEvent,+ new Promise(resolve => setTimeout(resolve, 1000)),+ ]);+ if (!download)+ throw e;+ }++ // Cap load event to 5 seconds, the page is operational at this point.+ await this.waitForLoadState('load', { timeout: 5000 });+ }++ refLocator(params: { element: string, ref: string }): playwright.Locator {+ return this.page.locator(`aria-ref=${params.ref}`).describe(params.element);+ }++ private async _raceAgainstModalDialogs(action: () => Promise<ToolActionResult>): Promise<ToolActionResult> {+ this._pendingAction = {+ dialogShown: new ManualPromise(),+ };++ let result: ToolActionResult | undefined;+ try {+ await Promise.race([+ action().then(r => result = r),+ this._pendingAction.dialogShown,+ ]);+ } finally {+ this._pendingAction = undefined;+ }+ return result;+ }++ private _javaScriptBlocked(): boolean {+ return this._modalStates.some(state => state.type === 'dialog');+ }++ dialogShown(dialog: playwright.Dialog) {+ this.setModalState({+ type: 'dialog',+ description: `"${dialog.type()}" dialog with message "${dialog.message()}"`,+ dialog,+ });+ this._pendingAction?.dialogShown.resolve();+ }++ async downloadStarted(download: playwright.Download) {+ const entry = {+ download,+ finished: false,+ outputFile: this._outputFile(download.suggestedFilename())+ };+ this._downloads.push(entry);+ await download.saveAs(entry.outputFile);+ entry.finished = true;+ }++ private async _snapshot() {+ const snapshot = await callOnPageNoTrace(this.page, page => (page as PageEx)._snapshotForAI());+ return [+ `- Page Snapshot`,+ '```yaml',+ snapshot,+ '```',+ ].join('\n');+ }++ private _outputFile(filename: string) {+ return filename;+ }+}diff --git a/packages/playwright-mdd/src/format.ts b/packages/playwright-mdd/src/format.tsnew file mode 100644index 0000000000..a1fabbd97d--- /dev/null+++ b/packages/playwright-mdd/src/format.ts@@ -0,0 +1,53 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// adapted from:+// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/utils/isomorphic/stringUtils.ts+// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/server/codegen/javascript.ts++// NOTE: this function should not be used to escape any selectors.+export function escapeWithQuotes(text: string, char: string = '\'') {+ const stringified = JSON.stringify(text);+ const escapedText = stringified.substring(1, stringified.length - 1).replace(/\\"/g, '"');+ if (char === '\'')+ return char + escapedText.replace(/[']/g, '\\\'') + char;+ if (char === '"')+ return char + escapedText.replace(/["]/g, '\\"') + char;+ if (char === '`')+ return char + escapedText.replace(/[`]/g, '`') + char;+ throw new Error('Invalid escape char');+}++export function quote(text: string) {+ return escapeWithQuotes(text, '\'');+}++export function formatObject(value: any, indent = ' '): string {+ if (typeof value === 'string')+ return quote(value);+ if (Array.isArray(value))+ return `[${value.map(o => formatObject(o)).join(', ')}]`;+ if (typeof value === 'object') {+ const keys = Object.keys(value).filter(key => value[key] !== undefined).sort();+ if (!keys.length)+ return '{}';+ const tokens: string[] = [];+ for (const key of keys)+ tokens.push(`${key}: ${formatObject(value[key])}`);+ return `{\n${indent}${tokens.join(`,\n${indent}`)}\n}`;+ }+ return String(value);+}diff --git a/packages/playwright-mdd/src/loop.ts b/packages/playwright-mdd/src/loop.tsnew file mode 100644index 0000000000..64d715d3b2--- /dev/null+++ b/packages/playwright-mdd/src/loop.ts@@ -0,0 +1,108 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import OpenAI from 'openai';+import debug from 'debug';+import { zodToJsonSchema } from 'zod-to-json-schema';++import { Tool } from './tools/tool';+import { Context } from './context';++/* eslint-disable no-console */++export async function runTasks(context: Context, tasks: string[]): Promise<string> {+ const openai = new OpenAI();+ const allCode: string[] = [+ `test('generated code', async ({ page }) => {`,+ ];+ for (const task of tasks) {+ const { taskCode } = await runTask(openai, context, task);+ if (taskCode.length)+ allCode.push('', ...taskCode.map(code => ` ${code}`));+ }+ allCode.push('});');+ return allCode.join('\n');+}++async function runTask(openai: OpenAI, context: Context, task: string): Promise<{ taskCode: string[] }> {+ console.log('Perform task:', task);++ const taskCode: string[] = [+ `// ${task}`,+ ];++ const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [+ {+ role: 'user',+ content: `Peform following task: ${task}. Once the task is complete, call the "done" tool.`+ }+ ];++ for (let iteration = 0; iteration < 5; ++iteration) {+ debug('history')(messages);+ const response = await openai.chat.completions.create({+ model: 'gpt-4.1',+ messages,+ tools: context.tools.map(asOpenAIDeclaration),+ tool_choice: 'auto'+ });++ const message = response.choices[0].message;+ if (!message.tool_calls?.length)+ throw new Error('Unexpected response from LLM: ' + message.content);++ messages.push({+ role: 'assistant',+ tool_calls: message.tool_calls+ });++ for (const toolCall of message.tool_calls) {+ const functionCall = toolCall.function;+ console.log('Call tool:', functionCall.name, functionCall.arguments);++ if (functionCall.name === 'done')+ return { taskCode };++ const tool = context.tools.find(tool => tool.schema.name === functionCall.name);+ if (!tool)+ throw new Error('Unknown tool: ' + functionCall.name);++ const { code, content } = await context.run(tool, JSON.parse(functionCall.arguments));+ taskCode.push(...code);++ messages.push({+ role: 'tool',+ tool_call_id: toolCall.id,+ content,+ });+ }+ }+ throw new Error('Failed to perform step, max attempts reached');+}++function asOpenAIDeclaration(tool: Tool<any>): OpenAI.Chat.Completions.ChatCompletionTool {+ const parameters = zodToJsonSchema(tool.schema.inputSchema);+ delete parameters.$schema;+ delete (parameters as any).additionalProperties;+ return {+ type: 'function',+ function: {+ name: tool.schema.name,+ description: tool.schema.description,+ parameters,+ },+ };+}diff --git a/packages/playwright-mdd/src/manualPromise.ts b/packages/playwright-mdd/src/manualPromise.tsnew file mode 100644index 0000000000..a5034e05ec--- /dev/null+++ b/packages/playwright-mdd/src/manualPromise.ts@@ -0,0 +1,127 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++export class ManualPromise<T = void> extends Promise<T> {+ private _resolve!: (t: T) => void;+ private _reject!: (e: Error) => void;+ private _isDone: boolean;++ constructor() {+ let resolve: (t: T) => void;+ let reject: (e: Error) => void;+ super((f, r) => {+ resolve = f;+ reject = r;+ });+ this._isDone = false;+ this._resolve = resolve!;+ this._reject = reject!;+ }++ isDone() {+ return this._isDone;+ }++ resolve(t: T) {+ this._isDone = true;+ this._resolve(t);+ }++ reject(e: Error) {+ this._isDone = true;+ this._reject(e);+ }++ static override get [Symbol.species]() {+ return Promise;+ }++ override get [Symbol.toStringTag]() {+ return 'ManualPromise';+ }+}++export class LongStandingScope {+ private _terminateError: Error | undefined;+ private _closeError: Error | undefined;+ private _terminatePromises = new Map<ManualPromise<Error>, string[]>();+ private _isClosed = false;++ reject(error: Error) {+ this._isClosed = true;+ this._terminateError = error;+ for (const p of this._terminatePromises.keys())+ p.resolve(error);+ }++ close(error: Error) {+ this._isClosed = true;+ this._closeError = error;+ for (const [p, frames] of this._terminatePromises)+ p.resolve(cloneError(error, frames));+ }++ isClosed() {+ return this._isClosed;+ }++ static async raceMultiple<T>(scopes: LongStandingScope[], promise: Promise<T>): Promise<T> {+ return Promise.race(scopes.map(s => s.race(promise)));+ }++ async race<T>(promise: Promise<T> | Promise<T>[]): Promise<T> {+ return this._race(Array.isArray(promise) ? promise : [promise], false) as Promise<T>;+ }++ async safeRace<T>(promise: Promise<T>, defaultValue?: T): Promise<T> {+ return this._race([promise], true, defaultValue);+ }++ private async _race(promises: Promise<any>[], safe: boolean, defaultValue?: any): Promise<any> {+ const terminatePromise = new ManualPromise<Error>();+ const frames = captureRawStack();+ if (this._terminateError)+ terminatePromise.resolve(this._terminateError);+ if (this._closeError)+ terminatePromise.resolve(cloneError(this._closeError, frames));+ this._terminatePromises.set(terminatePromise, frames);+ try {+ return await Promise.race([+ terminatePromise.then(e => safe ? defaultValue : Promise.reject(e)),+ ...promises+ ]);+ } finally {+ this._terminatePromises.delete(terminatePromise);+ }+ }+}++function cloneError(error: Error, frames: string[]) {+ const clone = new Error();+ clone.name = error.name;+ clone.message = error.message;+ clone.stack = [error.name + ':' + error.message, ...frames].join('\n');+ return clone;+}++function captureRawStack(): string[] {+ const stackTraceLimit = Error.stackTraceLimit;+ Error.stackTraceLimit = 50;+ const error = new Error();+ const stack = error.stack || '';+ Error.stackTraceLimit = stackTraceLimit;+ return stack.split('\n');+}diff --git a/packages/playwright-mdd/src/program.ts b/packages/playwright-mdd/src/program.tsnew file mode 100644index 0000000000..aebfae0f12--- /dev/null+++ b/packages/playwright-mdd/src/program.ts@@ -0,0 +1,50 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import dotenv from 'dotenv';+import { program } from 'commander';++import { runTasks } from './loop';+import { Context } from './context';+import { tools } from './tools';++/* eslint-disable no-console */++dotenv.config();++const packageJSON = require('../package.json');++program+ .version('Version ' + packageJSON.version)+ .name(packageJSON.name)+ .action(async () => {+ const context = await Context.create(tools);+ const code = await runTasks(context, script);+ console.log('Output code:');+ console.log('```javascript');+ console.log(code);+ console.log('```');+ await context.close();+ });++const script = [+ 'Navigate to https://debs-obrien.github.io/playwright-movies-app',+ 'Click search icon',+ 'Type "Twister" in the search field and hit Enter',+ 'Click on the link for the movie "Twisters"',+];++export { program };diff --git a/packages/playwright-mdd/src/tools.ts b/packages/playwright-mdd/src/tools.tsnew file mode 100644index 0000000000..e4b7c9a6ce--- /dev/null+++ b/packages/playwright-mdd/src/tools.ts@@ -0,0 +1,27 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import snapshot from './tools/snapshot';+import done from './tools/done';+import navigate from './tools/navigate';++import type { Tool } from './tools/tool.js';++export const tools: Tool<any>[] = [+ ...navigate,+ ...snapshot,+ ...done,+];diff --git a/packages/playwright-mdd/src/tools/done.ts b/packages/playwright-mdd/src/tools/done.tsnew file mode 100644index 0000000000..8818d4368d--- /dev/null+++ b/packages/playwright-mdd/src/tools/done.ts@@ -0,0 +1,39 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { z } from 'zod';++import { defineTool } from './tool';++const doneTool = defineTool({+ schema: {+ name: 'done',+ description: 'Call this tool to indicate that the task is complete',+ inputSchema: z.object({}),+ },++ handle: async () => {+ return {+ code: [],+ captureSnapshot: false,+ waitForNetwork: false,+ };+ },+});++export default [+ doneTool,+];diff --git a/packages/playwright-mdd/src/tools/navigate.ts b/packages/playwright-mdd/src/tools/navigate.tsnew file mode 100644index 0000000000..cf6f695fef--- /dev/null+++ b/packages/playwright-mdd/src/tools/navigate.ts@@ -0,0 +1,88 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { z } from 'zod';++import { defineTool } from './tool.js';++const navigate = defineTool({+ schema: {+ name: 'browser_navigate',+ description: 'Navigate to a URL',+ inputSchema: z.object({+ url: z.string().describe('The URL to navigate to'),+ }),+ },++ handle: async (context, params) => {+ const code = [+ `await page.goto('${params.url}');`,+ ];+ await context.navigate(params.url);++ return {+ code,+ captureSnapshot: true,+ waitForNetwork: false,+ };+ },+});++const goBack = defineTool({+ schema: {+ name: 'browser_navigate_back',+ description: 'Go back to the previous page',+ inputSchema: z.object({}),+ },++ handle: async context => {+ await context.page.goBack();+ const code = [+ `await page.goBack();`,+ ];++ return {+ code,+ captureSnapshot: true,+ waitForNetwork: false,+ };+ },+});++const goForward = defineTool({+ schema: {+ name: 'browser_navigate_forward',+ description: 'Go forward to the next page',+ inputSchema: z.object({}),+ },+ handle: async context => {+ await context.page.goForward();+ const code = [+ `await page.goForward();`,+ ];+ return {+ code,+ captureSnapshot: true,+ waitForNetwork: false,+ };+ },+});++export default [+ navigate,+ goBack,+ goForward,+];diff --git a/packages/playwright-mdd/src/tools/snapshot.ts b/packages/playwright-mdd/src/tools/snapshot.tsnew file mode 100644index 0000000000..e2a90ad852--- /dev/null+++ b/packages/playwright-mdd/src/tools/snapshot.ts@@ -0,0 +1,194 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { z } from 'zod';++import { defineTool } from './tool';+import * as format from '../format';+import { generateLocator } from './utils';++const snapshot = defineTool({+ schema: {+ name: 'browser_snapshot',+ description: 'Capture accessibility snapshot of the current page, this is better than screenshot',+ inputSchema: z.object({}),+ },++ handle: async () => {+ return {+ code: [],+ captureSnapshot: true,+ waitForNetwork: false,+ };+ },+});++const elementSchema = z.object({+ element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),+ ref: z.string().describe('Exact target element reference from the page snapshot'),+});++const click = defineTool({+ schema: {+ name: 'browser_click',+ description: 'Perform click on a web page',+ inputSchema: elementSchema,+ },++ handle: async (context, params) => {+ const locator = context.refLocator(params);++ const code = [+ `await page.${await generateLocator(locator)}.click();`+ ];++ return {+ code,+ action: () => locator.click(),+ captureSnapshot: true,+ waitForNetwork: true,+ };+ },+});++const drag = defineTool({+ schema: {+ name: 'browser_drag',+ description: 'Perform drag and drop between two elements',+ inputSchema: z.object({+ startElement: z.string().describe('Human-readable source element description used to obtain the permission to interact with the element'),+ startRef: z.string().describe('Exact source element reference from the page snapshot'),+ endElement: z.string().describe('Human-readable target element description used to obtain the permission to interact with the element'),+ endRef: z.string().describe('Exact target element reference from the page snapshot'),+ }),+ },++ handle: async (context, params) => {+ const startLocator = context.refLocator({ ref: params.startRef, element: params.startElement });+ const endLocator = context.refLocator({ ref: params.endRef, element: params.endElement });++ const code = [+ `await page.${await generateLocator(startLocator)}.dragTo(page.${await generateLocator(endLocator)});`+ ];++ return {+ code,+ action: () => startLocator.dragTo(endLocator),+ captureSnapshot: true,+ waitForNetwork: true,+ };+ },+});++const hover = defineTool({+ schema: {+ name: 'browser_hover',+ description: 'Hover over element on page',+ inputSchema: elementSchema,+ },++ handle: async (context, params) => {+ const locator = context.refLocator(params);++ const code = [+ `await page.${await generateLocator(locator)}.hover();`+ ];++ return {+ code,+ action: () => locator.hover(),+ captureSnapshot: true,+ waitForNetwork: true,+ };+ },+});++const typeSchema = elementSchema.extend({+ text: z.string().describe('Text to type into the element'),+ submit: z.boolean().optional().describe('Whether to submit entered text (press Enter after)'),+ slowly: z.boolean().optional().describe('Whether to type one character at a time. Useful for triggering key handlers in the page. By default entire text is filled in at once.'),+});++const type = defineTool({+ schema: {+ name: 'browser_type',+ description: 'Type text into editable element',+ inputSchema: typeSchema,+ },++ handle: async (context, params) => {+ const locator = context.refLocator(params);++ const code: string[] = [];+ const steps: (() => Promise<void>)[] = [];++ if (params.slowly) {+ code.push(`await page.${await generateLocator(locator)}.pressSequentially(${format.quote(params.text)});`);+ steps.push(() => locator.pressSequentially(params.text));+ } else {+ code.push(`await page.${await generateLocator(locator)}.fill(${format.quote(params.text)});`);+ steps.push(() => locator.fill(params.text));+ }++ if (params.submit) {+ code.push(`await page.${await generateLocator(locator)}.press('Enter');`);+ steps.push(() => locator.press('Enter'));+ }++ return {+ code,+ action: () => steps.reduce((acc, step) => acc.then(step), Promise.resolve()),+ captureSnapshot: true,+ waitForNetwork: true,+ };+ },+});++const selectOptionSchema = elementSchema.extend({+ values: z.array(z.string()).describe('Array of values to select in the dropdown. This can be a single value or multiple values.'),+});++const selectOption = defineTool({+ schema: {+ name: 'browser_select_option',+ description: 'Select an option in a dropdown',+ inputSchema: selectOptionSchema,+ },++ handle: async (context, params) => {+ const locator = context.refLocator(params);++ const code = [+ `await page.${await generateLocator(locator)}.selectOption(${format.formatObject(params.values)});`+ ];++ return {+ code,+ action: () => locator.selectOption(params.values).then(() => {}),+ captureSnapshot: true,+ waitForNetwork: true,+ };+ },+});++export default [+ snapshot,+ click,+ drag,+ hover,+ type,+ selectOption,+];diff --git a/packages/playwright-mdd/src/tools/tool.ts b/packages/playwright-mdd/src/tools/tool.tsnew file mode 100644index 0000000000..9f840051eb--- /dev/null+++ b/packages/playwright-mdd/src/tools/tool.ts@@ -0,0 +1,60 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import type { z } from 'zod';+import type * as playwright from 'playwright-core';+import type { Context } from '../context';++export type ToolSchema<Input extends InputType> = {+ name: string;+ description: string;+ inputSchema: Input;+};++type InputType = z.Schema;++export type FileUploadModalState = {+ type: 'fileChooser';+ description: string;+ fileChooser: playwright.FileChooser;+};++export type DialogModalState = {+ type: 'dialog';+ description: string;+ dialog: playwright.Dialog;+};++export type ModalState = FileUploadModalState | DialogModalState;++export type ToolActionResult = string | undefined | void;++export type ToolResult = {+ code: string[];+ action?: () => Promise<void>;+ captureSnapshot: boolean;+ waitForNetwork: boolean;+};++export type Tool<Input extends InputType = InputType> = {+ schema: ToolSchema<Input>;+ clearsModalState?: ModalState['type'];+ handle: (context: Context, params: z.output<Input>) => Promise<ToolResult>;+};++export function defineTool<Input extends InputType>(tool: Tool<Input>): Tool<Input> {+ return tool;+}diff --git a/packages/playwright-mdd/src/tools/utils.ts b/packages/playwright-mdd/src/tools/utils.tsnew file mode 100644index 0000000000..d82095edc8--- /dev/null+++ b/packages/playwright-mdd/src/tools/utils.ts@@ -0,0 +1,91 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import type * as playwright from 'playwright';+import type { Context } from '../context';++export async function waitForCompletion<R>(context: Context, callback: () => Promise<R>): Promise<R> {+ const requests = new Set<playwright.Request>();+ let frameNavigated = false;+ let waitCallback: () => void = () => {};+ const waitBarrier = new Promise<void>(f => { waitCallback = f; });++ const requestListener = (request: playwright.Request) => requests.add(request);+ const requestFinishedListener = (request: playwright.Request) => {+ requests.delete(request);+ if (!requests.size)+ waitCallback();+ };++ const frameNavigateListener = (frame: playwright.Frame) => {+ if (frame.parentFrame())+ return;+ frameNavigated = true;+ dispose();+ clearTimeout(timeout);+ void context.waitForLoadState('load').then(waitCallback);+ };++ const onTimeout = () => {+ dispose();+ waitCallback();+ };++ context.page.on('request', requestListener);+ context.page.on('requestfinished', requestFinishedListener);+ context.page.on('framenavigated', frameNavigateListener);+ const timeout = setTimeout(onTimeout, 10000);++ const dispose = () => {+ context.page.off('request', requestListener);+ context.page.off('requestfinished', requestFinishedListener);+ context.page.off('framenavigated', frameNavigateListener);+ clearTimeout(timeout);+ };++ try {+ const result = await callback();+ if (!requests.size && !frameNavigated)+ waitCallback();+ await waitBarrier;+ await context.page.waitForTimeout(1000);+ return result;+ } finally {+ dispose();+ }+}++export function sanitizeForFilePath(s: string) {+ const sanitize = (s: string) => s.replace(/[\x00-\x2C\x2E-\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]+/g, '-');+ const separator = s.lastIndexOf('.');+ if (separator === -1)+ return sanitize(s);+ return sanitize(s.substring(0, separator)) + '.' + sanitize(s.substring(separator + 1));+}++export async function generateLocator(locator: playwright.Locator): Promise<string> {+ try {+ return await (locator as any)._generateLocatorString();+ } catch (e) {+ if (e instanceof Error && /locator._generateLocatorString: Timeout .* exceeded/.test(e.message))+ throw new Error('Ref not found, likely because element was removed. Use browser_snapshot to see what elements are currently on the page.');+ throw e;+ }+}++export async function callOnPageNoTrace<T>(page: playwright.Page, callback: (page: playwright.Page) => Promise<T>): Promise<T> {+ return await (page as any)._wrapApiCall(() => callback(page), { internal: true });+}diff --git a/utils/workspace.js b/utils/workspace.jsindex 5331dc3335..928c61a0a0 100755--- a/utils/workspace.js+++ b/utils/workspace.js@@ -219,6 +219,11 @@ const workspace = new Workspace(ROOT_PATH, [path: path.join(ROOT_PATH, 'packages', 'playwright-ct-vue'),files: ['LICENSE'],}),+ new PWPackage({+ name: 'playwright-mdd',+ path: path.join(ROOT_PATH, 'packages', 'playwright-mdd'),+ files: ['LICENSE'],+ }),]);if (require.main === module) {