Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: land experimental tools (#34503)
Pavel Feldmanbd2fdd0f2027
diff --git a/package-lock.json b/package-lock.jsonindex d7eae56cba..c26ed0b1f9 100644--- a/package-lock.json+++ b/package-lock.json@@ -111,6 +111,22 @@"node": ">=6.0.0"}},+ "node_modules/@anthropic-ai/sdk": {+ "version": "0.33.1",+ "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.33.1.tgz",+ "integrity": "sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@types/node": "^18.11.18",+ "@types/node-fetch": "^2.6.4",+ "abort-controller": "^3.0.0",+ "agentkeepalive": "^4.2.1",+ "form-data-encoder": "1.7.2",+ "formdata-node": "^4.3.2",+ "node-fetch": "^2.6.7"+ }+ },"node_modules/@babel/cli": {"version": "7.23.4","resolved": "https://registry.npmjs.org/@babel/cli/-/cli-7.23.4.tgz",@@ -1503,6 +1519,10 @@"resolved": "packages/playwright-ct-vue","link": true},+ "node_modules/@playwright/experimental-tools": {+ "resolved": "packages/playwright-tools",+ "link": true+ },"node_modules/@playwright/test": {"resolved": "packages/playwright-test","link": true@@ -1867,6 +1887,17 @@"undici-types": "~5.26.4"}},+ "node_modules/@types/node-fetch": {+ "version": "2.6.12",+ "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz",+ "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@types/node": "*",+ "form-data": "^4.0.0"+ }+ },"node_modules/@types/prop-types": {"version": "15.7.11","resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz",@@ -2331,6 +2362,19 @@"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==","dev": true},+ "node_modules/abort-controller": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",+ "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "event-target-shim": "^5.0.0"+ },+ "engines": {+ "node": ">=6.5"+ }+ },"node_modules/acorn": {"version": "8.11.3","resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",@@ -2351,6 +2395,19 @@"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"}},+ "node_modules/agentkeepalive": {+ "version": "4.6.0",+ "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz",+ "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "humanize-ms": "^1.2.1"+ },+ "engines": {+ "node": ">= 8.0.0"+ }+ },"node_modules/ajv": {"version": "6.12.6","resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",@@ -2590,6 +2647,13 @@"integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==","dev": true},+ "node_modules/asynckit": {+ "version": "0.4.0",+ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",+ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",+ "dev": true,+ "license": "MIT"+ },"node_modules/available-typed-arrays": {"version": "1.0.7","resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz",@@ -2898,6 +2962,19 @@"node": ">=0.1.90"}},+ "node_modules/combined-stream": {+ "version": "1.0.8",+ "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",+ "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "delayed-stream": "~1.0.0"+ },+ "engines": {+ "node": ">= 0.8"+ }+ },"node_modules/commander": {"version": "4.1.1","resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",@@ -3219,6 +3296,16 @@"url": "https://github.com/sponsors/ljharb"}},+ "node_modules/delayed-stream": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",+ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",+ "dev": true,+ "license": "MIT",+ "engines": {+ "node": ">=0.4.0"+ }+ },"node_modules/dequal": {"version": "2.0.3","resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",@@ -3930,6 +4017,16 @@"node": ">=0.10.0"}},+ "node_modules/event-target-shim": {+ "version": "5.0.1",+ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",+ "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",+ "dev": true,+ "license": "MIT",+ "engines": {+ "node": ">=6"+ }+ },"node_modules/extract-zip": {"version": "2.0.1","resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",@@ -4077,6 +4174,42 @@"is-callable": "^1.1.3"}},+ "node_modules/form-data": {+ "version": "4.0.1",+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",+ "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "asynckit": "^0.4.0",+ "combined-stream": "^1.0.8",+ "mime-types": "^2.1.12"+ },+ "engines": {+ "node": ">= 6"+ }+ },+ "node_modules/form-data-encoder": {+ "version": "1.7.2",+ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",+ "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",+ "dev": true,+ "license": "MIT"+ },+ "node_modules/formdata-node": {+ "version": "4.4.1",+ "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",+ "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "node-domexception": "1.0.0",+ "web-streams-polyfill": "4.0.0-beta.3"+ },+ "engines": {+ "node": ">= 12.20"+ }+ },"node_modules/formidable": {"version": "2.1.2","resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",@@ -4562,6 +4695,16 @@"node": ">=10.19.0"}},+ "node_modules/humanize-ms": {+ "version": "1.2.1",+ "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",+ "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "ms": "^2.0.0"+ }+ },"node_modules/ignore": {"version": "5.3.1","resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",@@ -5358,6 +5501,29 @@"node": ">=10.0.0"}},+ "node_modules/mime-db": {+ "version": "1.52.0",+ "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",+ "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",+ "dev": true,+ "license": "MIT",+ "engines": {+ "node": ">= 0.6"+ }+ },+ "node_modules/mime-types": {+ "version": "2.1.35",+ "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",+ "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "mime-db": "1.52.0"+ },+ "engines": {+ "node": ">= 0.6"+ }+ },"node_modules/mimic-response": {"version": "1.0.1","resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",@@ -5438,6 +5604,47 @@"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==","dev": true},+ "node_modules/node-domexception": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",+ "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",+ "dev": true,+ "funding": [+ {+ "type": "github",+ "url": "https://github.com/sponsors/jimmywarting"+ },+ {+ "type": "github",+ "url": "https://paypal.me/jimmywarting"+ }+ ],+ "license": "MIT",+ "engines": {+ "node": ">=10.5.0"+ }+ },+ "node_modules/node-fetch": {+ "version": "2.7.0",+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",+ "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "whatwg-url": "^5.0.0"+ },+ "engines": {+ "node": "4.x || >=6.0.0"+ },+ "peerDependencies": {+ "encoding": "^0.1.0"+ },+ "peerDependenciesMeta": {+ "encoding": {+ "optional": true+ }+ }+ },"node_modules/node-releases": {"version": "2.0.14","resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz",@@ -5637,6 +5844,37 @@"wrappy": "1"}},+ "node_modules/openai": {+ "version": "4.79.1",+ "resolved": "https://registry.npmjs.org/openai/-/openai-4.79.1.tgz",+ "integrity": "sha512-M7P5/PKnT/S/B5v0D64giC9mjyxFYkqlCuQFzR5hkdzMdqUuHf8T1gHhPGPF5oAvu4+PO3TvJv/qhZoS2bqAkw==",+ "dev": true,+ "license": "Apache-2.0",+ "dependencies": {+ "@types/node": "^18.11.18",+ "@types/node-fetch": "^2.6.4",+ "abort-controller": "^3.0.0",+ "agentkeepalive": "^4.2.1",+ "form-data-encoder": "1.7.2",+ "formdata-node": "^4.3.2",+ "node-fetch": "^2.6.7"+ },+ "bin": {+ "openai": "bin/cli"+ },+ "peerDependencies": {+ "ws": "^8.18.0",+ "zod": "^3.23.8"+ },+ "peerDependenciesMeta": {+ "ws": {+ "optional": true+ },+ "zod": {+ "optional": true+ }+ }+ },"node_modules/optionator": {"version": "0.9.3","resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",@@ -6754,6 +6992,13 @@"node": ">=8.0"}},+ "node_modules/tr46": {+ "version": "0.0.3",+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",+ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",+ "dev": true,+ "license": "MIT"+ },"node_modules/trace-viewer": {"resolved": "packages/trace-viewer","link": true@@ -7492,6 +7737,34 @@"resolved": "packages/web","link": true},+ "node_modules/web-streams-polyfill": {+ "version": "4.0.0-beta.3",+ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",+ "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",+ "dev": true,+ "license": "MIT",+ "engines": {+ "node": ">= 14"+ }+ },+ "node_modules/webidl-conversions": {+ "version": "3.0.1",+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",+ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",+ "dev": true,+ "license": "BSD-2-Clause"+ },+ "node_modules/whatwg-url": {+ "version": "5.0.0",+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",+ "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "tr46": "~0.0.3",+ "webidl-conversions": "^3.0.0"+ }+ },"node_modules/which": {"version": "2.0.2","resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",@@ -7610,10 +7883,11 @@"dev": true},"node_modules/ws": {- "version": "8.17.1",- "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",- "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",+ "version": "8.18.0",+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",+ "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==","dev": true,+ "license": "MIT","engines": {"node": ">=10.0.0"},@@ -7932,6 +8206,21 @@"node": ">=18"}},+ "packages/playwright-tools": {+ "name": "@playwright/experimental-tools",+ "version": "1.51.0-next",+ "license": "Apache-2.0",+ "dependencies": {+ "playwright": "1.51.0-next"+ },+ "devDependencies": {+ "@anthropic-ai/sdk": "^0.33.1",+ "openai": "^4.79.1"+ },+ "engines": {+ "node": ">=18"+ }+ },"packages/playwright-webkit": {"version": "1.51.0-next","hasInstallScript": true,diff --git a/packages/playwright-core/src/client/elementHandle.ts b/packages/playwright-core/src/client/elementHandle.tsindex 0a89d3091b..c6249e8078 100644--- a/packages/playwright-core/src/client/elementHandle.ts+++ b/packages/playwright-core/src/client/elementHandle.ts@@ -62,6 +62,11 @@ export class ElementHandle<T extends Node = Node> extends JSHandle<T> implementsreturn Frame.fromNullable((await this._elementChannel.contentFrame()).frame);}+ async _generateLocatorString(): Promise<string | null> {+ const value = (await this._elementChannel.generateLocatorString()).value;+ return value === undefined ? null : value;+ }+async getAttribute(name: string): Promise<string | null> {const value = (await this._elementChannel.getAttribute({ name })).value;return value === undefined ? null : value;diff --git a/packages/playwright-core/src/client/locator.ts b/packages/playwright-core/src/client/locator.tsindex 84e68e1b20..7784e1d685 100644--- a/packages/playwright-core/src/client/locator.ts+++ b/packages/playwright-core/src/client/locator.ts@@ -236,6 +236,10 @@ export class Locator implements api.Locator {return await this._frame._queryCount(this._selector);}+ async _generateLocatorString(): Promise<string | null> {+ return await this._withElement(h => h._generateLocatorString());+ }+async getAttribute(name: string, options?: TimeoutOptions): Promise<string | null> {return await this._frame.getAttribute(this._selector, name, { strict: true, ...options });}@@ -288,8 +292,8 @@ export class Locator implements api.Locator {return await this._withElement((h, timeout) => h.screenshot({ ...options, timeout }), options.timeout);}- async ariaSnapshot(options?: TimeoutOptions): Promise<string> {- const result = await this._frame._channel.ariaSnapshot({ ...options, selector: this._selector });+ async ariaSnapshot(options?: { _id?: boolean, _mode?: 'raw' | 'regex' } & TimeoutOptions): Promise<string> {+ const result = await this._frame._channel.ariaSnapshot({ ...options, id: options?._id, mode: options?._mode, selector: this._selector });return result.snapshot;}diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.tsindex 50e8b4f02a..cdcef14996 100644--- a/packages/playwright-core/src/protocol/validator.ts+++ b/packages/playwright-core/src/protocol/validator.ts@@ -1429,6 +1429,8 @@ scheme.FrameAddStyleTagResult = tObject({});scheme.FrameAriaSnapshotParams = tObject({selector: tString,+ id: tOptional(tBoolean),+ mode: tOptional(tEnum(['raw', 'regex'])),timeout: tOptional(tNumber),});scheme.FrameAriaSnapshotResult = tObject({@@ -1925,6 +1927,10 @@ scheme.ElementHandleFillParams = tObject({scheme.ElementHandleFillResult = tOptional(tObject({}));scheme.ElementHandleFocusParams = tOptional(tObject({}));scheme.ElementHandleFocusResult = tOptional(tObject({}));+scheme.ElementHandleGenerateLocatorStringParams = tOptional(tObject({}));+scheme.ElementHandleGenerateLocatorStringResult = tObject({+ value: tOptional(tString),+});scheme.ElementHandleGetAttributeParams = tObject({name: tString,});diff --git a/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts b/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.tsindex d4d0280ec4..8a6e4fcbaa 100644--- a/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts+++ b/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts@@ -63,6 +63,10 @@ export class ElementHandleDispatcher extends JSHandleDispatcher implements channreturn { frame: frame ? FrameDispatcher.from(this._browserContextDispatcher(), frame) : undefined };}+ async generateLocatorString(params: channels.ElementHandleGenerateLocatorStringParams, metadata: CallMetadata): Promise<channels.ElementHandleGenerateLocatorStringResult> {+ return { value: await this._elementHandle.generateLocatorString() };+ }+async getAttribute(params: channels.ElementHandleGetAttributeParams, metadata: CallMetadata): Promise<channels.ElementHandleGetAttributeResult> {const value = await this._elementHandle.getAttribute(metadata, params.name);return { value: value === null ? undefined : value };diff --git a/packages/playwright-core/src/server/dom.ts b/packages/playwright-core/src/server/dom.tsindex 962f385c90..288b8c2c8d 100644--- a/packages/playwright-core/src/server/dom.ts+++ b/packages/playwright-core/src/server/dom.ts@@ -28,7 +28,7 @@ import type { Progress } from './progress';import { ProgressController } from './progress';import type * as types from './types';import type { TimeoutOptions } from '../common/types';-import { isUnderTest } from '../utils';+import { asLocator, isUnderTest } from '../utils';import { prepareFilesForUpload } from './fileUploadUtils';export type InputFilesItems = {@@ -185,6 +185,15 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {return this._page._delegate.getContentFrame(this);}+ async generateLocatorString(): Promise<string | undefined> {+ const selector = await this.evaluateInUtility(async ([injected, node]) => {+ return injected.generateSelectorSimple(node as unknown as Element);+ }, {});+ if (selector === 'error:notconnected')+ return;+ return asLocator('javascript', selector);+ }+async getAttribute(metadata: CallMetadata, name: string): Promise<string | null> {return this._frame.getAttribute(metadata, ':scope', name, {}, this);}@@ -799,8 +808,8 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {return this._page._delegate.getBoundingBox(this);}- async ariaSnapshot(): Promise<string> {- return await this.evaluateInUtility(([injected, element]) => injected.ariaSnapshot(element), {});+ async ariaSnapshot(options: { id?: boolean, mode?: 'raw' | 'regex' }): Promise<string> {+ return await this.evaluateInUtility(([injected, element, options]) => injected.ariaSnapshot(element, options), options);}async screenshot(metadata: CallMetadata, options: ScreenshotOptions & TimeoutOptions = {}): Promise<Buffer> {diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.tsindex 1d2098b92c..1570df5770 100644--- a/packages/playwright-core/src/server/frames.ts+++ b/packages/playwright-core/src/server/frames.ts@@ -1408,10 +1408,10 @@ export class Frame extends SdkObject {});}- async ariaSnapshot(metadata: CallMetadata, selector: string, options: types.TimeoutOptions = {}): Promise<string> {+ async ariaSnapshot(metadata: CallMetadata, selector: string, options: { id?: boolean, mode?: 'raw' | 'regex' } & types.TimeoutOptions = {}): Promise<string> {const controller = new ProgressController(metadata, this);return controller.run(async progress => {- return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => handle.ariaSnapshot());+ return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => handle.ariaSnapshot(options));}, this._page._timeoutSettings.timeout(options));}diff --git a/packages/playwright-core/src/server/injected/consoleApi.ts b/packages/playwright-core/src/server/injected/consoleApi.tsindex eae874834e..faba90c57d 100644--- a/packages/playwright-core/src/server/injected/consoleApi.ts+++ b/packages/playwright-core/src/server/injected/consoleApi.ts@@ -85,10 +85,8 @@ class ConsoleAPI {inspect: (selector: string) => this._inspect(selector),selector: (element: Element) => this._selector(element),generateLocator: (element: Element, language?: Language) => this._generateLocator(element, language),- ariaSnapshot: (element?: Element) => {- const snapshot = this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body);- // eslint-disable-next-line no-console- console.log(snapshot);+ ariaSnapshot: (element?: Element, options?: { id?: boolean }) => {+ return this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body, options);},resume: () => this._resume(),...new Locator(injectedScript, ''),diff --git a/packages/playwright-core/src/server/injected/injectedScript.ts b/packages/playwright-core/src/server/injected/injectedScript.tsindex 43988d4c28..bb004590f5 100644--- a/packages/playwright-core/src/server/injected/injectedScript.ts+++ b/packages/playwright-core/src/server/injected/injectedScript.ts@@ -72,6 +72,7 @@ export class InjectedScript {// eslint-disable-next-line no-restricted-globalsreadonly window: Window & typeof globalThis;readonly document: Document;+ private _ariaElementById: Map<number, Element> | undefined;// Recorder must use any external dependencies through InjectedScript.// Otherwise it will end up with a copy of all modules it uses, and any@@ -130,6 +131,7 @@ export class InjectedScript {this._engines.set('internal:attr', this._createNamedAttributeEngine());this._engines.set('internal:testid', this._createNamedAttributeEngine());this._engines.set('internal:role', createRoleEngine(true));+ this._engines.set('internal:aria-id', this._createAriaIdEngine());for (const { name, engine } of customEngines)this._engines.set(name, engine);@@ -221,7 +223,8 @@ export class InjectedScript {if (node.nodeType !== Node.ELEMENT_NODE)throw this.createStacklessError('Can only capture aria snapshot of Element nodes.');const ariaSnapshot = generateAriaTree(node as Element);- return renderAriaTree(ariaSnapshot.root, options);+ this._ariaElementById = ariaSnapshot.elements;+ return renderAriaTree(ariaSnapshot.root, { ...options, ids: options?.id ? ariaSnapshot.ids : undefined });}ariaSnapshotAsObject(node: Node): AriaSnapshot {@@ -609,6 +612,15 @@ export class InjectedScript {return result;}+ _createAriaIdEngine() {+ const queryAll = (root: SelectorRoot, selector: string): Element[] => {+ const ariaId = parseInt(selector, 10);+ const result = this._ariaElementById?.get(ariaId);+ return result && result.isConnected ? [result] : [];+ };+ return { queryAll };+ }+elementState(node: Node, state: ElementStateWithoutStable): ElementStateQueryResult {const element = this.retarget(node, ['visible', 'hidden'].includes(state) ? 'none' : 'follow-label');if (!element || !element.isConnected) {diff --git a/packages/playwright-core/src/server/selectors.ts b/packages/playwright-core/src/server/selectors.tsindex b54a0269a5..edc11b9a86 100644--- a/packages/playwright-core/src/server/selectors.ts+++ b/packages/playwright-core/src/server/selectors.ts@@ -39,7 +39,7 @@ export class Selectors {'internal:has', 'internal:has-not','internal:has-text', 'internal:has-not-text','internal:and', 'internal:or', 'internal:chain',- 'role', 'internal:attr', 'internal:label', 'internal:text', 'internal:role', 'internal:testid',+ 'role', 'internal:attr', 'internal:label', 'internal:text', 'internal:role', 'internal:testid', 'internal:aria-id']);this._builtinEnginesInMainWorld = new Set(['_react', '_vue',diff --git a/packages/playwright-tools/.npmignore b/packages/playwright-tools/.npmignorenew file mode 100644index 0000000000..9b62c46c55--- /dev/null+++ b/packages/playwright-tools/.npmignore@@ -0,0 +1,8 @@+**/*+README.md+LICENSE+!lib/*+!browser.js+!browser.d.ts+!computer-20241022.js+!computer-20241022.d.tsdiff --git a/packages/playwright-tools/README.md b/packages/playwright-tools/README.mdnew file mode 100644index 0000000000..ce4a9dd7c6--- /dev/null+++ b/packages/playwright-tools/README.md@@ -0,0 +1 @@+> **BEWARE** This package is EXPERIMENTAL and does not respect semver.diff --git a/packages/playwright-tools/browser.d.ts b/packages/playwright-tools/browser.d.tsnew file mode 100644index 0000000000..e06589e947--- /dev/null+++ b/packages/playwright-tools/browser.d.ts@@ -0,0 +1,30 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import type playwright from 'playwright';+import { ToolDeclaration, JSONSchemaType } from './types';++export type ToolResult = {+ error?: string;+ code: Array<string>;+ snapshot: string;+}++export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise<ToolResult>;++export const schema: ToolDeclaration[];+export const call: ToolCall;+export const snapshot: (page) => Promise<string>;diff --git a/packages/playwright-tools/browser.js b/packages/playwright-tools/browser.jsnew file mode 100644index 0000000000..d0475e384b--- /dev/null+++ b/packages/playwright-tools/browser.js@@ -0,0 +1,19 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++const { schema, call, snapshot } = require('./lib/tools/browser');++module.exports = { schema, call, snapshot };diff --git a/packages/playwright-tools/computer-20241022.d.ts b/packages/playwright-tools/computer-20241022.d.tsnew file mode 100644index 0000000000..ce0f821d6d--- /dev/null+++ b/packages/playwright-tools/computer-20241022.d.ts@@ -0,0 +1,28 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import type playwright from 'playwright';+import { JSONSchemaType } from './types';++export type ToolResult = {+ output?: string;+ error?: string;+ base64_image?: string;+};++export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise<ToolResult>;++export const call: ToolCall;diff --git a/packages/playwright-tools/computer-20241022.js b/packages/playwright-tools/computer-20241022.jsnew file mode 100644index 0000000000..030d0e57e2--- /dev/null+++ b/packages/playwright-tools/computer-20241022.js@@ -0,0 +1,19 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++const { call } = require('./lib/tools/computer-20241022');++module.exports = { call };diff --git a/packages/playwright-tools/package.json b/packages/playwright-tools/package.jsonnew file mode 100644index 0000000000..c5366c2944--- /dev/null+++ b/packages/playwright-tools/package.json@@ -0,0 +1,35 @@+{+ "name": "@playwright/experimental-tools",+ "version": "1.51.0-next",+ "description": "Playwright Tools for AI",+ "repository": {+ "type": "git",+ "url": "git+https://github.com/microsoft/playwright.git"+ },+ "homepage": "https://playwright.dev",+ "engines": {+ "node": ">=18"+ },+ "author": {+ "name": "Microsoft Corporation"+ },+ "license": "Apache-2.0",+ "exports": {+ "./browser": {+ "types": "./browser.d.ts",+ "default": "./browser.js"+ },+ "./computer-20241022": {+ "types": "./computer-20241022.d.ts",+ "default": "./computer-20241022.js"+ },+ "./package.json": "./package.json"+ },+ "dependencies": {+ "playwright": "1.51.0-next"+ },+ "devDependencies": {+ "@anthropic-ai/sdk": "^0.33.1",+ "openai": "^4.79.1"+ }+}diff --git a/packages/playwright-tools/src/examples/browser-anthropic.ts b/packages/playwright-tools/src/examples/browser-anthropic.tsnew file mode 100644index 0000000000..0053c5749d--- /dev/null+++ b/packages/playwright-tools/src/examples/browser-anthropic.ts@@ -0,0 +1,143 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/* eslint-disable no-console */++import playwright from 'playwright';+import Anthropic from '@anthropic-ai/sdk';+import dotenv from 'dotenv';+import browser from '@playwright/experimental-tools/browser';++dotenv.config();++const anthropic = new Anthropic();++export const system = `+You are a web tester.++<Instructions>+- Perform test according to the provided checklist+- Use browser tools to perform actions on web page+- Never ask questions, always perform a best guess action+- Use one tool at a time, wait for its result before proceeding.+- When ready use "reportResult" tool to report result+</Instructions>`;++const reportTool: Anthropic.Tool = {+ name: 'reportResult',+ description: 'Submit test result',+ input_schema: {+ type: 'object',+ properties: {+ 'success': { type: 'boolean', description: 'Whether test passed' },+ 'result': { type: 'string', description: 'Result of the test if some information has been requested' },+ 'error': { type: 'string', description: 'Error message if test failed' }+ },+ required: ['success']+ }+};++type Message = Anthropic.Beta.Messages.BetaMessageParam & {+ history: Anthropic.Beta.Messages.BetaMessageParam['content']+};++async function anthropicAgentLoop(page: playwright.Page, task: string) {+ // Convert them into tools for Anthropic.+ const pageTools: Anthropic.Tool[] = browser.schema.map(tool => {+ return {+ name: tool.name,+ description: tool.description,+ input_schema: tool.parameters as any,+ };+ });++ // Add report tool.+ const tools = [reportTool, ...pageTools];++ const history: Message[] = [{+ role: 'user',+ history: `Task: ${task}`,+ content: `Task: ${task}\n\n${await browser.snapshot(page)}`,+ }];++ // Run agentic loop, cap steps.+ for (let i = 0; i < 50; i++) {+ const response = await anthropic.messages.create({+ model: 'claude-3-5-sonnet-20241022',+ max_tokens: 1024,+ temperature: 0,+ tools,+ system,+ messages: toAnthropicMessages(history),+ });+ history.push({ role: 'assistant', content: response.content, history: response.content });++ const toolUse = response.content.find(block => block.type === 'tool_use');+ if (!toolUse) {+ history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });+ continue;+ }++ if (toolUse.name === 'reportResult') {+ console.log(toolUse.input);+ return;+ }++ // Run the Playwright tool.+ const { error, snapshot, code } = await browser.call(page, toolUse.name, toolUse.input as any);+ if (code.length)+ console.log(code.join('\n'));++ // Report the result.+ const resultText = error ? `Error: ${error}\n` : 'Done\n';+ history.push({+ role: 'user',+ content: [{+ type: 'tool_result',+ tool_use_id: toolUse.id,+ content: [{ type: 'text', text: resultText + snapshot }],+ }],+ history: [{+ type: 'tool_result',+ tool_use_id: toolUse.id,+ content: [{ type: 'text', text: resultText }],+ }],+ });+ }+}++function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] {+ return messages.map((message, i) => {+ if (i === messages.length - 1)+ return { ...message, history: undefined };+ return { ...message, content: message.history, history: undefined };+ });+}++async function main() {+ const browser = await playwright.chromium.launch({ headless: false });+ const page = await browser.newPage();+ await anthropicAgentLoop(page, `+ - Go to http://github.com/microsoft+ - Search for "playwright" repository+ - Navigate to it+ - Switch into the Issues tab+ - Report 3 first issues+ `);+ await browser.close();+}++void main();diff --git a/packages/playwright-tools/src/examples/browser-openai.ts b/packages/playwright-tools/src/examples/browser-openai.tsnew file mode 100644index 0000000000..3d39618f5b--- /dev/null+++ b/packages/playwright-tools/src/examples/browser-openai.ts@@ -0,0 +1,161 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/* eslint-disable no-console */++import playwright from 'playwright';+import browser from '@playwright/experimental-tools/browser';+import dotenv from 'dotenv';+import OpenAI from 'openai';+import type { ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources';++dotenv.config();++const openai = new OpenAI();++export const system = `+You are a web tester.++<Instructions>to+- Perform test according to the provided checklist+- Use browser tools to perform actions on web page+- Never ask questions, always perform a best guess action+- When ready use "reportResult" tool to report result+- You can only make one tool call at a time.+</Instructions>`;++type Message = ChatCompletionMessageParam & {+ history: any+};++const reportTool: ChatCompletionTool = {+ type: 'function',+ function: {+ name: 'reportResult',+ description: 'Submit test result',+ parameters: {+ type: 'object',+ properties: {+ success: { type: 'boolean', description: 'Whether test passed' },+ result: { type: 'string', description: 'Result of the test if requested' },+ error: { type: 'string', description: 'Error if test failed' },+ },+ required: ['success'],+ additionalProperties: false,+ },+ }+};++async function openAIAgentLoop(page: playwright.Page, task: string) {+ const pageTools: ChatCompletionTool[] = browser.schema.map(tool => ({+ type: 'function',+ function: {+ name: tool.name,+ description: tool.description,+ parameters: {+ ...tool.parameters,+ additionalProperties: false,+ },+ }+ }));++ const tools = [reportTool, ...pageTools];++ const history: Message[] = [+ {+ role: 'system', content: system, history: system+ },+ {+ role: 'user',+ history: `Task: ${task}`,+ content: `Task: ${task}\n\n${await browser.snapshot(page)}`,+ }+ ];++ // Run agentic loop, cap steps.+ for (let i = 0; i < 50; i++) {+ const completion = await openai.chat.completions.create({+ model: 'gpt-4o',+ messages: toOpenAIMessages(history),+ tools,+ store: true,+ });++ console.log(JSON.stringify(completion, null, 2));++ const toolCalls = completion.choices[0]?.message?.tool_calls;+ if (!toolCalls || toolCalls.length !== 1 || toolCalls[0].type !== 'function') {+ history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });+ continue;+ }++ const toolCall = toolCalls[0];+ if (toolCall.function.name === 'reportResult') {+ console.log(JSON.parse(toolCall.function.arguments));+ return;+ }++ history.push({ ...completion.choices[0].message, history: null });++ // Run the Playwright tool.+ const params = JSON.parse(toolCall.function.arguments);+ const { error, snapshot, code } = await browser.call(page, toolCall.function.name, params);+ console.log({ error, code, snapshot });+ if (code.length)+ console.log(code.join('\n'));++ if (toolCall.function.name === 'log')+ return;++ // Report the result.+ const resultText = error ? `Error: ${error}\n` : 'Done\n';+ history.push({+ role: 'tool',+ tool_call_id: toolCall.id,+ content: resultText + snapshot,+ history: resultText,+ });+ }+}++function toOpenAIMessages(messages: Message[]): ChatCompletionMessageParam[] {+ return messages.map((message, i) => {+ const copy: Message = { ...message };+ delete copy.history;+ if (i === messages.length - 1)+ return copy;+ copy.content = message.history;+ return copy;+ });+}++async function main() {+ const browser = await playwright.chromium.launch({ headless: false });+ const page = await browser.newPage();+ await openAIAgentLoop(page, `+ - Go to http://github.com/microsoft+ - Search for "playwright" repository+ - Navigate to it+ - Capture snapshot for toolbar with Code, Issues, etc.+ - Capture snapshot for branch selector+ - Assert that number of Issues is present+ - Switch into the Issues tab+ - Report 3 first issues+ `);+ await browser.close();+}++void main();diff --git a/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts b/packages/playwright-tools/src/examples/computer-20241022-anthropic.tsnew file mode 100644index 0000000000..20028c564c--- /dev/null+++ b/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts@@ -0,0 +1,148 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/* eslint-disable no-console */++import playwright from 'playwright';+import Anthropic from '@anthropic-ai/sdk';+import dotenv from 'dotenv';+import computer, { type ToolResult } from '@playwright/experimental-tools/computer-20241022';+import type { BetaImageBlockParam, BetaTextBlockParam } from '@anthropic-ai/sdk/resources/beta/messages/messages';++dotenv.config();++const anthropic = new Anthropic();++export const system = `+You are a web tester.++<Instructions>+- Perform test according to the provided checklist+- Use browser tools to perform actions on web page+- Never ask questions, always perform a best guess action+- Use one tool at a time, wait for its result before proceeding.+- When ready use "reportResult" tool to report result+</Instructions>`;++const computerTool: Anthropic.Beta.BetaToolUnion = {+ type: 'computer_20241022',+ name: 'computer',+ display_width_px: 1920,+ display_height_px: 1080,+ display_number: 1,+};++const reportTool: Anthropic.Tool = {+ name: 'reportResult',+ description: 'Submit test result',+ input_schema: {+ type: 'object',+ properties: {+ 'success': { type: 'boolean', description: 'Whether test passed' },+ 'result': { type: 'string', description: 'Result of the test if some information has been requested' },+ 'error': { type: 'string', description: 'Error message if test failed' }+ },+ required: ['success']+ }+};++type Message = Anthropic.Beta.Messages.BetaMessageParam & {+ history: Anthropic.Beta.Messages.BetaMessageParam['content']+};++async function anthropicAgentLoop(page: playwright.Page, task: string) {+ // Add report tool.+ const tools = [reportTool, computerTool];++ const history: Message[] = [{+ role: 'user',+ history: `Task: ${task}`,+ content: `Task: ${task}`,+ }];++ // Run agentic loop, cap steps.+ for (let i = 0; i < 50; i++) {+ const response = await anthropic.beta.messages.create({+ model: 'claude-3-5-sonnet-20241022',+ max_tokens: 1024,+ temperature: 0,+ tools,+ system,+ messages: toAnthropicMessages(history),+ betas: ['computer-use-2024-10-22'],+ });++ history.push({ role: 'assistant', content: response.content, history: response.content });++ const toolUse = response.content.find(block => block.type === 'tool_use');+ if (!toolUse) {+ history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });+ continue;+ }++ if (toolUse.name === 'reportResult') {+ console.log(toolUse.input);+ return;+ }++ const result: ToolResult = await computer.call(page, toolUse.name, toolUse.input as any);+ const contentEntry: BetaTextBlockParam | BetaImageBlockParam = result.base64_image ? {+ type: 'image',+ source: { type: 'base64', media_type: 'image/jpeg', data: result.base64_image }+ } : {+ type: 'text',+ text: result.output || '',+ };+ history.push({+ role: 'user',+ content: [{+ type: 'tool_result',+ tool_use_id: toolUse.id,+ content: [contentEntry],+ }],+ history: [{+ type: 'tool_result',+ tool_use_id: toolUse.id,+ content: [{ type: 'text', text: '<redacted>' }],+ }],+ });+ }+}++function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] {+ return messages.map((message, i) => {+ if (i === messages.length - 1)+ return { ...message, history: undefined };+ return { ...message, content: message.history, history: undefined };+ });+}++const githubTask = `+ - Search for "playwright" repository+ - Navigate to it+ - Switch into the Issues tab+ - Report 3 first issues+`;++async function main() {+ const browser = await playwright.chromium.launch({ headless: false });+ const page = await browser.newPage();+ await page.goto('http://github.com/microsoft');+ await anthropicAgentLoop(page, githubTask);+ await browser.close();+}++void main();diff --git a/packages/playwright-tools/src/tools/browser.ts b/packages/playwright-tools/src/tools/browser.tsnew file mode 100644index 0000000000..267d03bcb3--- /dev/null+++ b/packages/playwright-tools/src/tools/browser.ts@@ -0,0 +1,146 @@+/**+ * Copyright 2017 Google Inc. All rights reserved.+ * Modifications copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import type playwright from 'playwright';+import type { JSONSchemaType, ToolDeclaration } from '../../types';+import type { ToolResult } from '../../browser';+import { waitForNetwork } from './utils';++type LocatorEx = playwright.Locator & {+ _generateLocatorString: () => Promise<string>;+};++const intentProperty = {+ intent: {+ type: 'string',+ description: 'Intent behind this particular action. Used as a comment.',+ }+};++const elementIdProperty = {+ elementId: {+ type: 'number',+ description: 'Target element',+ }+};++export const schema: ToolDeclaration[] = [+ {+ name: 'navigate',+ description: 'Navigate to a URL',+ parameters: {+ type: 'object',+ properties: {+ ...intentProperty,+ url: {+ type: 'string',+ description: 'URL to navigate to',+ },+ },+ required: ['intent', 'elementId'],+ }+ },+ {+ name: 'click',+ description: 'Perform click on a web page',+ parameters: {+ type: 'object',+ properties: {+ ...intentProperty,+ ...elementIdProperty,+ },+ required: ['intent', 'elementId'],+ }+ },+ {+ name: 'enterText',+ description: 'Enter text into editable element',+ parameters: {+ type: 'object',+ properties: {+ ...intentProperty,+ ...elementIdProperty,+ text: {+ type: 'string',+ description: 'Text to enter',+ },+ submit: {+ type: 'boolean',+ description: 'Whether to submit entered text (press Enter after)'+ }+ },+ required: ['intent', 'elementId', 'text'],+ }+ },+ {+ name: 'wait',+ description: `Wait for given amount of time to see if the page updates. Use it after action if you think page is not ready yet`,+ parameters: {+ type: 'object',+ properties: {+ ...intentProperty,+ time: {+ type: 'integer',+ description: 'Time to wait in seconds',+ },+ },+ required: ['intent', 'time'],+ }+ },+];++export async function call(page: playwright.Page, toolName: string, params: Record<string, JSONSchemaType>): Promise<ToolResult> {+ const code: string[] = [];+ try {+ await waitForNetwork(page, async () => {+ await performAction(page, toolName, params, code);+ });+ } catch (e) {+ return { error: e.message, snapshot: await snapshot(page), code };+ }+ return { snapshot: await snapshot(page), code };+}++export async function snapshot(page: playwright.Page) {+ const params = { _id: true } as any;+ return `<Page snapshot>\n${await page.locator('body').ariaSnapshot(params)}\n</Page snapshot>`;+}++async function performAction(page: playwright.Page, toolName: string, params: Record<string, JSONSchemaType>, code: string[]) {+ const locator = elementLocator(page, params);+ code.push((params.intent as string).split('\n').map(line => `// ${line}`).join('\n'));+ if (toolName === 'navigate') {+ code.push(`await page.goto(${JSON.stringify(params.url)})`);+ await page.goto(params.url as string);+ } else if (toolName === 'click') {+ code.push(`await page.${await locator._generateLocatorString()}.click()`);+ await locator.click();+ } else if (toolName === 'enterText') {+ code.push(`await page.${await locator._generateLocatorString()}.click()`);+ await locator.click();+ code.push(`await page.${await locator._generateLocatorString()}.fill(${JSON.stringify(params.text)})`);+ await locator.fill(params.text as string);+ if (params.submit) {+ code.push(`await page.${await locator._generateLocatorString()}.press("Enter")`);+ await locator.press('Enter');+ }+ }+}++function elementLocator(page: playwright.Page, params: any): LocatorEx {+ return page.locator(`internal:aria-id=${params.elementId}`) as LocatorEx;+}diff --git a/packages/playwright-tools/src/tools/computer-20241022.ts b/packages/playwright-tools/src/tools/computer-20241022.tsnew file mode 100644index 0000000000..e407d027ba--- /dev/null+++ b/packages/playwright-tools/src/tools/computer-20241022.ts@@ -0,0 +1,158 @@+/**+ * Copyright 2017 Google Inc. All rights reserved.+ * Modifications copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the 'License');+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an 'AS IS' BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import type playwright from 'playwright';+import type { JSONSchemaType } from '../../types';+import type { ToolResult } from '../../computer-20241022';+import { waitForNetwork } from './utils';++export async function call(page: playwright.Page, toolName: string, input: Record<string, JSONSchemaType>): Promise<ToolResult> {+ if (toolName !== 'computer')+ throw new Error('Unsupported tool');+ return await waitForNetwork(page, async () => {+ return await performAction(page, toolName, input);+ });+}++type PageState = {+ x: number;+ y: number;+};++const pageStateSymbol = Symbol('pageState');++function pageState(page: playwright.Page): PageState {+ if (!(page as any)[pageStateSymbol])+ (page as any)[pageStateSymbol] = { x: 0, y: 0 };+ return (page as any)[pageStateSymbol];+}++async function performAction(page: playwright.Page, toolName: string, input: Record<string, JSONSchemaType>): Promise<ToolResult> {+ const state = pageState(page);+ const { action } = input as { action: string };+ if (action === 'screenshot') {+ const screenshot = await page.screenshot({ type: 'jpeg', quality: 50, scale: 'css' });+ return {+ output: 'Screenshot',+ base64_image: screenshot.toString('base64'),+ };+ }+ if (action === 'mouse_move') {+ const { coordinate } = input as { coordinate: [number, number] };+ state.x = coordinate[0];+ state.y = coordinate[1];+ await page.mouse.move(state.x, state.y);+ return { output: 'Mouse moved' };+ }+ if (action === 'left_click') {+ await page.mouse.down();+ await page.mouse.up();+ return { output: 'Left clicked' };+ }+ if (action === 'left_click_drag') {+ await page.mouse.down();+ const { coordinate } = input as { coordinate: [number, number] };+ state.x = coordinate[0];+ state.y = coordinate[1];+ await page.mouse.move(state.x, state.y);+ await page.mouse.up();+ return { output: 'Left dragged' };+ }+ if (action === 'right_click') {+ await page.mouse.down({ button: 'right' });+ await page.mouse.up({ button: 'right' });+ return { output: 'Right clicked' };+ }+ if (action === 'double_click') {+ await page.mouse.down();+ await page.mouse.up();+ await page.mouse.down();+ await page.mouse.up();+ return { output: 'Double clicked' };+ }+ if (action === 'middle_click') {+ await page.mouse.down({ button: 'middle' });+ await page.mouse.up({ button: 'middle' });+ return { output: 'Middle clicked' };+ }+ if (action === 'key') {+ const { text } = input as { text: string };+ await page.keyboard.press(xToPlaywright(text));+ return { output: 'Text typed' };+ }+ if (action === 'cursor_position')+ return { output: `X=${state.x},Y=${state.y}` };+ throw new Error('Unimplemented tool: ' + toolName);+}++const xToPlaywrightKeyMap = new Map([+ ['BackSpace', 'Backspace'],+ ['Tab', 'Tab'],+ ['Return', 'Enter'],+ ['Escape', 'Escape'],+ ['space', ' '],+ ['Delete', 'Delete'],+ ['Home', 'Home'],+ ['End', 'End'],+ ['Left', 'ArrowLeft'],+ ['Up', 'ArrowUp'],+ ['Right', 'ArrowRight'],+ ['Down', 'ArrowDown'],+ ['Insert', 'Insert'],+ ['Page_Up', 'PageUp'],+ ['Page_Down', 'PageDown'],+ ['F1', 'F1'],+ ['F2', 'F2'],+ ['F3', 'F3'],+ ['F4', 'F4'],+ ['F5', 'F5'],+ ['F6', 'F6'],+ ['F7', 'F7'],+ ['F8', 'F8'],+ ['F9', 'F9'],+ ['F10', 'F10'],+ ['F11', 'F11'],+ ['F12', 'F12'],+ ['Shift_L', 'Shift'],+ ['Shift_R', 'Shift'],+ ['Control_L', 'Control'],+ ['Control_R', 'Control'],+ ['Alt_L', 'Alt'],+ ['Alt_R', 'Alt'],+ ['Super_L', 'Meta'],+ ['Super_R', 'Meta'],+]);++const xToPlaywrightModifierMap = new Map([+ ['alt', 'Alt'],+ ['control', 'Control'],+ ['meta', 'Meta'],+ ['shift', 'Shift'],+]);+++const xToPlaywright = (key: string) => {+ const tokens = key.split('+');+ if (tokens.length === 1)+ return xToPlaywrightKeyMap.get(key) || key;+ if (tokens.length === 2) {+ const modifier = xToPlaywrightModifierMap.get(tokens[0]);+ const key = xToPlaywrightKeyMap.get(tokens[1]) || tokens[1];+ return modifier + '+' + key;+ }+ throw new Error('Invalid key: ' + key);+};diff --git a/packages/playwright-tools/src/tools/utils.ts b/packages/playwright-tools/src/tools/utils.tsnew file mode 100644index 0000000000..08e674014d--- /dev/null+++ b/packages/playwright-tools/src/tools/utils.ts@@ -0,0 +1,71 @@+/**+ * Copyright 2017 Google Inc. All rights reserved.+ * Modifications copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import type playwright from 'playwright';+import { ManualPromise } from 'playwright-core/lib/utils';++export async function waitForNetwork<R>(page: playwright.Page, callback: () => Promise<R>): Promise<R> {+ const requests = new Set<playwright.Request>();+ let frameNavigated = false;+ const waitBarrier = new ManualPromise();++ const requestListener = (request: playwright.Request) => requests.add(request);+ const requestFinishedListener = (request: playwright.Request) => {+ requests.delete(request);+ if (!requests.size)+ waitBarrier.resolve();+ };++ const frameNavigateListener = (frame: playwright.Frame) => {+ if (frame.parentFrame())+ return;+ frameNavigated = true;+ dispose();+ clearTimeout(timeout);+ void frame.waitForLoadState('load').then(() => {+ waitBarrier.resolve();+ });+ };++ const onTimeout = () => {+ dispose();+ waitBarrier.resolve();+ };++ page.on('request', requestListener);+ page.on('requestfinished', requestFinishedListener);+ page.on('framenavigated', frameNavigateListener);+ const timeout = setTimeout(onTimeout, 10000);++ const dispose = () => {+ page.off('request', requestListener);+ page.off('requestfinished', requestFinishedListener);+ page.off('framenavigated', frameNavigateListener);+ clearTimeout(timeout);+ };++ try {+ const result = await callback();+ if (!requests.size && !frameNavigated)+ waitBarrier.resolve();+ await waitBarrier;+ await page.evaluate(() => new Promise(f => setTimeout(f, 1000)));+ return result;+ } finally {+ dispose();+ }+}diff --git a/packages/playwright-tools/types.d.ts b/packages/playwright-tools/types.d.tsnew file mode 100644index 0000000000..5fe61526c2--- /dev/null+++ b/packages/playwright-tools/types.d.ts@@ -0,0 +1,27 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import type playwright from 'playwright';++export type JSONSchemaType = string | number | boolean | JSONSchemaObject | JSONSchemaArray | null;+interface JSONSchemaObject { [key: string]: JSONSchemaType; }+interface JSONSchemaArray extends Array<JSONSchemaType> {}++export type ToolDeclaration = {+ name: string;+ description: string;+ parameters: any;+};diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.tsindex 526cc599ab..30f8c03088 100644--- a/packages/protocol/src/channels.d.ts+++ b/packages/protocol/src/channels.d.ts@@ -2619,9 +2619,13 @@ export type FrameAddStyleTagResult = {};export type FrameAriaSnapshotParams = {selector: string,+ id?: boolean,+ mode?: 'raw' | 'regex',timeout?: number,};export type FrameAriaSnapshotOptions = {+ id?: boolean,+ mode?: 'raw' | 'regex',timeout?: number,};export type FrameAriaSnapshotResult = {@@ -3311,6 +3315,7 @@ export interface ElementHandleChannel extends ElementHandleEventTarget, JSHandledispatchEvent(params: ElementHandleDispatchEventParams, metadata?: CallMetadata): Promise<ElementHandleDispatchEventResult>;fill(params: ElementHandleFillParams, metadata?: CallMetadata): Promise<ElementHandleFillResult>;focus(params?: ElementHandleFocusParams, metadata?: CallMetadata): Promise<ElementHandleFocusResult>;+ generateLocatorString(params?: ElementHandleGenerateLocatorStringParams, metadata?: CallMetadata): Promise<ElementHandleGenerateLocatorStringResult>;getAttribute(params: ElementHandleGetAttributeParams, metadata?: CallMetadata): Promise<ElementHandleGetAttributeResult>;hover(params: ElementHandleHoverParams, metadata?: CallMetadata): Promise<ElementHandleHoverResult>;innerHTML(params?: ElementHandleInnerHTMLParams, metadata?: CallMetadata): Promise<ElementHandleInnerHTMLResult>;@@ -3450,6 +3455,11 @@ export type ElementHandleFillResult = void;export type ElementHandleFocusParams = {};export type ElementHandleFocusOptions = {};export type ElementHandleFocusResult = void;+export type ElementHandleGenerateLocatorStringParams = {};+export type ElementHandleGenerateLocatorStringOptions = {};+export type ElementHandleGenerateLocatorStringResult = {+ value?: string,+};export type ElementHandleGetAttributeParams = {name: string,};diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.ymlindex df54dcbe1c..77501efa9b 100644--- a/packages/protocol/src/protocol.yml+++ b/packages/protocol/src/protocol.yml@@ -1879,6 +1879,12 @@ Frame:ariaSnapshot:parameters:selector: string+ id: boolean?+ mode:+ type: enum?+ literals:+ - raw+ - regextimeout: number?returns:snapshot: string@@ -2648,6 +2654,10 @@ ElementHandle:slowMo: truesnapshot: true+ generateLocatorString:+ returns:+ value: string?+getAttribute:parameters:name: stringdiff --git a/utils/workspace.js b/utils/workspace.jsindex 4fb69050b3..7e6850a262 100755--- a/utils/workspace.js+++ b/utils/workspace.js@@ -167,6 +167,11 @@ const workspace = new Workspace(ROOT_PATH, [path: path.join(ROOT_PATH, 'packages', 'playwright-chromium'),files: LICENCE_FILES,}),+ new PWPackage({+ name: '@playwright/experimental-tools',+ path: path.join(ROOT_PATH, 'packages', 'playwright-tools'),+ files: LICENCE_FILES,+ }),new PWPackage({name: '@playwright/browser-webkit',path: path.join(ROOT_PATH, 'packages', 'playwright-browser-webkit'),