Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: land experimental tools (#34503)

Pavel Feldmanbd2fdd0f2027
diff --git a/package-lock.json b/package-lock.json
index d7eae56cba..c26ed0b1f9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -111,6 +111,22 @@
"node": ">=6.0.0"
}
},
+ "node_modules/@anthropic-ai/sdk": {
+ "version": "0.33.1",
+ "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.33.1.tgz",
+ "integrity": "sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@types/node": "^18.11.18",
+ "@types/node-fetch": "^2.6.4",
+ "abort-controller": "^3.0.0",
+ "agentkeepalive": "^4.2.1",
+ "form-data-encoder": "1.7.2",
+ "formdata-node": "^4.3.2",
+ "node-fetch": "^2.6.7"
+ }
+ },
"node_modules/@babel/cli": {
"version": "7.23.4",
"resolved": "https://registry.npmjs.org/@babel/cli/-/cli-7.23.4.tgz",
@@ -1503,6 +1519,10 @@
"resolved": "packages/playwright-ct-vue",
"link": true
},
+ "node_modules/@playwright/experimental-tools": {
+ "resolved": "packages/playwright-tools",
+ "link": true
+ },
"node_modules/@playwright/test": {
"resolved": "packages/playwright-test",
"link": true
@@ -1867,6 +1887,17 @@
"undici-types": "~5.26.4"
}
},
+ "node_modules/@types/node-fetch": {
+ "version": "2.6.12",
+ "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz",
+ "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@types/node": "*",
+ "form-data": "^4.0.0"
+ }
+ },
"node_modules/@types/prop-types": {
"version": "15.7.11",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz",
@@ -2331,6 +2362,19 @@
"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
"dev": true
},
+ "node_modules/abort-controller": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
+ "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "event-target-shim": "^5.0.0"
+ },
+ "engines": {
+ "node": ">=6.5"
+ }
+ },
"node_modules/acorn": {
"version": "8.11.3",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
@@ -2351,6 +2395,19 @@
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
}
},
+ "node_modules/agentkeepalive": {
+ "version": "4.6.0",
+ "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz",
+ "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "humanize-ms": "^1.2.1"
+ },
+ "engines": {
+ "node": ">= 8.0.0"
+ }
+ },
"node_modules/ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
@@ -2590,6 +2647,13 @@
"integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
"dev": true
},
+ "node_modules/asynckit": {
+ "version": "0.4.0",
+ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/available-typed-arrays": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz",
@@ -2898,6 +2962,19 @@
"node": ">=0.1.90"
}
},
+ "node_modules/combined-stream": {
+ "version": "1.0.8",
+ "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+ "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "delayed-stream": "~1.0.0"
+ },
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
"node_modules/commander": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
@@ -3219,6 +3296,16 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/delayed-stream": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.4.0"
+ }
+ },
"node_modules/dequal": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
@@ -3930,6 +4017,16 @@
"node": ">=0.10.0"
}
},
+ "node_modules/event-target-shim": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
+ "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/extract-zip": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
@@ -4077,6 +4174,42 @@
"is-callable": "^1.1.3"
}
},
+ "node_modules/form-data": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",
+ "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "asynckit": "^0.4.0",
+ "combined-stream": "^1.0.8",
+ "mime-types": "^2.1.12"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
+ "node_modules/form-data-encoder": {
+ "version": "1.7.2",
+ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
+ "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",
+ "dev": true,
+ "license": "MIT"
+ },
+ "node_modules/formdata-node": {
+ "version": "4.4.1",
+ "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
+ "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "node-domexception": "1.0.0",
+ "web-streams-polyfill": "4.0.0-beta.3"
+ },
+ "engines": {
+ "node": ">= 12.20"
+ }
+ },
"node_modules/formidable": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",
@@ -4562,6 +4695,16 @@
"node": ">=10.19.0"
}
},
+ "node_modules/humanize-ms": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
+ "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "ms": "^2.0.0"
+ }
+ },
"node_modules/ignore": {
"version": "5.3.1",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",
@@ -5358,6 +5501,29 @@
"node": ">=10.0.0"
}
},
+ "node_modules/mime-db": {
+ "version": "1.52.0",
+ "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+ "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/mime-types": {
+ "version": "2.1.35",
+ "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+ "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "mime-db": "1.52.0"
+ },
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
"node_modules/mimic-response": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
@@ -5438,6 +5604,47 @@
"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
"dev": true
},
+ "node_modules/node-domexception": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
+ "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/jimmywarting"
+ },
+ {
+ "type": "github",
+ "url": "https://paypal.me/jimmywarting"
+ }
+ ],
+ "license": "MIT",
+ "engines": {
+ "node": ">=10.5.0"
+ }
+ },
+ "node_modules/node-fetch": {
+ "version": "2.7.0",
+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+ "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "whatwg-url": "^5.0.0"
+ },
+ "engines": {
+ "node": "4.x || >=6.0.0"
+ },
+ "peerDependencies": {
+ "encoding": "^0.1.0"
+ },
+ "peerDependenciesMeta": {
+ "encoding": {
+ "optional": true
+ }
+ }
+ },
"node_modules/node-releases": {
"version": "2.0.14",
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz",
@@ -5637,6 +5844,37 @@
"wrappy": "1"
}
},
+ "node_modules/openai": {
+ "version": "4.79.1",
+ "resolved": "https://registry.npmjs.org/openai/-/openai-4.79.1.tgz",
+ "integrity": "sha512-M7P5/PKnT/S/B5v0D64giC9mjyxFYkqlCuQFzR5hkdzMdqUuHf8T1gHhPGPF5oAvu4+PO3TvJv/qhZoS2bqAkw==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@types/node": "^18.11.18",
+ "@types/node-fetch": "^2.6.4",
+ "abort-controller": "^3.0.0",
+ "agentkeepalive": "^4.2.1",
+ "form-data-encoder": "1.7.2",
+ "formdata-node": "^4.3.2",
+ "node-fetch": "^2.6.7"
+ },
+ "bin": {
+ "openai": "bin/cli"
+ },
+ "peerDependencies": {
+ "ws": "^8.18.0",
+ "zod": "^3.23.8"
+ },
+ "peerDependenciesMeta": {
+ "ws": {
+ "optional": true
+ },
+ "zod": {
+ "optional": true
+ }
+ }
+ },
"node_modules/optionator": {
"version": "0.9.3",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
@@ -6754,6 +6992,13 @@
"node": ">=8.0"
}
},
+ "node_modules/tr46": {
+ "version": "0.0.3",
+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/trace-viewer": {
"resolved": "packages/trace-viewer",
"link": true
@@ -7492,6 +7737,34 @@
"resolved": "packages/web",
"link": true
},
+ "node_modules/web-streams-polyfill": {
+ "version": "4.0.0-beta.3",
+ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
+ "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">= 14"
+ }
+ },
+ "node_modules/webidl-conversions": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
+ "dev": true,
+ "license": "BSD-2-Clause"
+ },
+ "node_modules/whatwg-url": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+ "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "tr46": "~0.0.3",
+ "webidl-conversions": "^3.0.0"
+ }
+ },
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@@ -7610,10 +7883,11 @@
"dev": true
},
"node_modules/ws": {
- "version": "8.17.1",
- "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
- "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
+ "version": "8.18.0",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
+ "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
"dev": true,
+ "license": "MIT",
"engines": {
"node": ">=10.0.0"
},
@@ -7932,6 +8206,21 @@
"node": ">=18"
}
},
+ "packages/playwright-tools": {
+ "name": "@playwright/experimental-tools",
+ "version": "1.51.0-next",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "playwright": "1.51.0-next"
+ },
+ "devDependencies": {
+ "@anthropic-ai/sdk": "^0.33.1",
+ "openai": "^4.79.1"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
"packages/playwright-webkit": {
"version": "1.51.0-next",
"hasInstallScript": true,
diff --git a/packages/playwright-core/src/client/elementHandle.ts b/packages/playwright-core/src/client/elementHandle.ts
index 0a89d3091b..c6249e8078 100644
--- a/packages/playwright-core/src/client/elementHandle.ts
+++ b/packages/playwright-core/src/client/elementHandle.ts
@@ -62,6 +62,11 @@ export class ElementHandle<T extends Node = Node> extends JSHandle<T> implements
return Frame.fromNullable((await this._elementChannel.contentFrame()).frame);
}
+ async _generateLocatorString(): Promise<string | null> {
+ const value = (await this._elementChannel.generateLocatorString()).value;
+ return value === undefined ? null : value;
+ }
+
async getAttribute(name: string): Promise<string | null> {
const value = (await this._elementChannel.getAttribute({ name })).value;
return value === undefined ? null : value;
diff --git a/packages/playwright-core/src/client/locator.ts b/packages/playwright-core/src/client/locator.ts
index 84e68e1b20..7784e1d685 100644
--- a/packages/playwright-core/src/client/locator.ts
+++ b/packages/playwright-core/src/client/locator.ts
@@ -236,6 +236,10 @@ export class Locator implements api.Locator {
return await this._frame._queryCount(this._selector);
}
+ async _generateLocatorString(): Promise<string | null> {
+ return await this._withElement(h => h._generateLocatorString());
+ }
+
async getAttribute(name: string, options?: TimeoutOptions): Promise<string | null> {
return await this._frame.getAttribute(this._selector, name, { strict: true, ...options });
}
@@ -288,8 +292,8 @@ export class Locator implements api.Locator {
return await this._withElement((h, timeout) => h.screenshot({ ...options, timeout }), options.timeout);
}
- async ariaSnapshot(options?: TimeoutOptions): Promise<string> {
- const result = await this._frame._channel.ariaSnapshot({ ...options, selector: this._selector });
+ async ariaSnapshot(options?: { _id?: boolean, _mode?: 'raw' | 'regex' } & TimeoutOptions): Promise<string> {
+ const result = await this._frame._channel.ariaSnapshot({ ...options, id: options?._id, mode: options?._mode, selector: this._selector });
return result.snapshot;
}
diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts
index 50e8b4f02a..cdcef14996 100644
--- a/packages/playwright-core/src/protocol/validator.ts
+++ b/packages/playwright-core/src/protocol/validator.ts
@@ -1429,6 +1429,8 @@ scheme.FrameAddStyleTagResult = tObject({
});
scheme.FrameAriaSnapshotParams = tObject({
selector: tString,
+ id: tOptional(tBoolean),
+ mode: tOptional(tEnum(['raw', 'regex'])),
timeout: tOptional(tNumber),
});
scheme.FrameAriaSnapshotResult = tObject({
@@ -1925,6 +1927,10 @@ scheme.ElementHandleFillParams = tObject({
scheme.ElementHandleFillResult = tOptional(tObject({}));
scheme.ElementHandleFocusParams = tOptional(tObject({}));
scheme.ElementHandleFocusResult = tOptional(tObject({}));
+scheme.ElementHandleGenerateLocatorStringParams = tOptional(tObject({}));
+scheme.ElementHandleGenerateLocatorStringResult = tObject({
+ value: tOptional(tString),
+});
scheme.ElementHandleGetAttributeParams = tObject({
name: tString,
});
diff --git a/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts b/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts
index d4d0280ec4..8a6e4fcbaa 100644
--- a/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts
+++ b/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts
@@ -63,6 +63,10 @@ export class ElementHandleDispatcher extends JSHandleDispatcher implements chann
return { frame: frame ? FrameDispatcher.from(this._browserContextDispatcher(), frame) : undefined };
}
+ async generateLocatorString(params: channels.ElementHandleGenerateLocatorStringParams, metadata: CallMetadata): Promise<channels.ElementHandleGenerateLocatorStringResult> {
+ return { value: await this._elementHandle.generateLocatorString() };
+ }
+
async getAttribute(params: channels.ElementHandleGetAttributeParams, metadata: CallMetadata): Promise<channels.ElementHandleGetAttributeResult> {
const value = await this._elementHandle.getAttribute(metadata, params.name);
return { value: value === null ? undefined : value };
diff --git a/packages/playwright-core/src/server/dom.ts b/packages/playwright-core/src/server/dom.ts
index 962f385c90..288b8c2c8d 100644
--- a/packages/playwright-core/src/server/dom.ts
+++ b/packages/playwright-core/src/server/dom.ts
@@ -28,7 +28,7 @@ import type { Progress } from './progress';
import { ProgressController } from './progress';
import type * as types from './types';
import type { TimeoutOptions } from '../common/types';
-import { isUnderTest } from '../utils';
+import { asLocator, isUnderTest } from '../utils';
import { prepareFilesForUpload } from './fileUploadUtils';
export type InputFilesItems = {
@@ -185,6 +185,15 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
return this._page._delegate.getContentFrame(this);
}
+ async generateLocatorString(): Promise<string | undefined> {
+ const selector = await this.evaluateInUtility(async ([injected, node]) => {
+ return injected.generateSelectorSimple(node as unknown as Element);
+ }, {});
+ if (selector === 'error:notconnected')
+ return;
+ return asLocator('javascript', selector);
+ }
+
async getAttribute(metadata: CallMetadata, name: string): Promise<string | null> {
return this._frame.getAttribute(metadata, ':scope', name, {}, this);
}
@@ -799,8 +808,8 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
return this._page._delegate.getBoundingBox(this);
}
- async ariaSnapshot(): Promise<string> {
- return await this.evaluateInUtility(([injected, element]) => injected.ariaSnapshot(element), {});
+ async ariaSnapshot(options: { id?: boolean, mode?: 'raw' | 'regex' }): Promise<string> {
+ return await this.evaluateInUtility(([injected, element, options]) => injected.ariaSnapshot(element, options), options);
}
async screenshot(metadata: CallMetadata, options: ScreenshotOptions & TimeoutOptions = {}): Promise<Buffer> {
diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts
index 1d2098b92c..1570df5770 100644
--- a/packages/playwright-core/src/server/frames.ts
+++ b/packages/playwright-core/src/server/frames.ts
@@ -1408,10 +1408,10 @@ export class Frame extends SdkObject {
});
}
- async ariaSnapshot(metadata: CallMetadata, selector: string, options: types.TimeoutOptions = {}): Promise<string> {
+ async ariaSnapshot(metadata: CallMetadata, selector: string, options: { id?: boolean, mode?: 'raw' | 'regex' } & types.TimeoutOptions = {}): Promise<string> {
const controller = new ProgressController(metadata, this);
return controller.run(async progress => {
- return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => handle.ariaSnapshot());
+ return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => handle.ariaSnapshot(options));
}, this._page._timeoutSettings.timeout(options));
}
diff --git a/packages/playwright-core/src/server/injected/consoleApi.ts b/packages/playwright-core/src/server/injected/consoleApi.ts
index eae874834e..faba90c57d 100644
--- a/packages/playwright-core/src/server/injected/consoleApi.ts
+++ b/packages/playwright-core/src/server/injected/consoleApi.ts
@@ -85,10 +85,8 @@ class ConsoleAPI {
inspect: (selector: string) => this._inspect(selector),
selector: (element: Element) => this._selector(element),
generateLocator: (element: Element, language?: Language) => this._generateLocator(element, language),
- ariaSnapshot: (element?: Element) => {
- const snapshot = this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body);
- // eslint-disable-next-line no-console
- console.log(snapshot);
+ ariaSnapshot: (element?: Element, options?: { id?: boolean }) => {
+ return this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body, options);
},
resume: () => this._resume(),
...new Locator(injectedScript, ''),
diff --git a/packages/playwright-core/src/server/injected/injectedScript.ts b/packages/playwright-core/src/server/injected/injectedScript.ts
index 43988d4c28..bb004590f5 100644
--- a/packages/playwright-core/src/server/injected/injectedScript.ts
+++ b/packages/playwright-core/src/server/injected/injectedScript.ts
@@ -72,6 +72,7 @@ export class InjectedScript {
// eslint-disable-next-line no-restricted-globals
readonly window: Window & typeof globalThis;
readonly document: Document;
+ private _ariaElementById: Map<number, Element> | undefined;
// Recorder must use any external dependencies through InjectedScript.
// Otherwise it will end up with a copy of all modules it uses, and any
@@ -130,6 +131,7 @@ export class InjectedScript {
this._engines.set('internal:attr', this._createNamedAttributeEngine());
this._engines.set('internal:testid', this._createNamedAttributeEngine());
this._engines.set('internal:role', createRoleEngine(true));
+ this._engines.set('internal:aria-id', this._createAriaIdEngine());
for (const { name, engine } of customEngines)
this._engines.set(name, engine);
@@ -221,7 +223,8 @@ export class InjectedScript {
if (node.nodeType !== Node.ELEMENT_NODE)
throw this.createStacklessError('Can only capture aria snapshot of Element nodes.');
const ariaSnapshot = generateAriaTree(node as Element);
- return renderAriaTree(ariaSnapshot.root, options);
+ this._ariaElementById = ariaSnapshot.elements;
+ return renderAriaTree(ariaSnapshot.root, { ...options, ids: options?.id ? ariaSnapshot.ids : undefined });
}
ariaSnapshotAsObject(node: Node): AriaSnapshot {
@@ -609,6 +612,15 @@ export class InjectedScript {
return result;
}
+ _createAriaIdEngine() {
+ const queryAll = (root: SelectorRoot, selector: string): Element[] => {
+ const ariaId = parseInt(selector, 10);
+ const result = this._ariaElementById?.get(ariaId);
+ return result && result.isConnected ? [result] : [];
+ };
+ return { queryAll };
+ }
+
elementState(node: Node, state: ElementStateWithoutStable): ElementStateQueryResult {
const element = this.retarget(node, ['visible', 'hidden'].includes(state) ? 'none' : 'follow-label');
if (!element || !element.isConnected) {
diff --git a/packages/playwright-core/src/server/selectors.ts b/packages/playwright-core/src/server/selectors.ts
index b54a0269a5..edc11b9a86 100644
--- a/packages/playwright-core/src/server/selectors.ts
+++ b/packages/playwright-core/src/server/selectors.ts
@@ -39,7 +39,7 @@ export class Selectors {
'internal:has', 'internal:has-not',
'internal:has-text', 'internal:has-not-text',
'internal:and', 'internal:or', 'internal:chain',
- 'role', 'internal:attr', 'internal:label', 'internal:text', 'internal:role', 'internal:testid',
+ 'role', 'internal:attr', 'internal:label', 'internal:text', 'internal:role', 'internal:testid', 'internal:aria-id'
]);
this._builtinEnginesInMainWorld = new Set([
'_react', '_vue',
diff --git a/packages/playwright-tools/.npmignore b/packages/playwright-tools/.npmignore
new file mode 100644
index 0000000000..9b62c46c55
--- /dev/null
+++ b/packages/playwright-tools/.npmignore
@@ -0,0 +1,8 @@
+**/*
+README.md
+LICENSE
+!lib/*
+!browser.js
+!browser.d.ts
+!computer-20241022.js
+!computer-20241022.d.ts
diff --git a/packages/playwright-tools/README.md b/packages/playwright-tools/README.md
new file mode 100644
index 0000000000..ce4a9dd7c6
--- /dev/null
+++ b/packages/playwright-tools/README.md
@@ -0,0 +1 @@
+> **BEWARE** This package is EXPERIMENTAL and does not respect semver.
diff --git a/packages/playwright-tools/browser.d.ts b/packages/playwright-tools/browser.d.ts
new file mode 100644
index 0000000000..e06589e947
--- /dev/null
+++ b/packages/playwright-tools/browser.d.ts
@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type playwright from 'playwright';
+import { ToolDeclaration, JSONSchemaType } from './types';
+
+export type ToolResult = {
+ error?: string;
+ code: Array<string>;
+ snapshot: string;
+}
+
+export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise<ToolResult>;
+
+export const schema: ToolDeclaration[];
+export const call: ToolCall;
+export const snapshot: (page) => Promise<string>;
diff --git a/packages/playwright-tools/browser.js b/packages/playwright-tools/browser.js
new file mode 100644
index 0000000000..d0475e384b
--- /dev/null
+++ b/packages/playwright-tools/browser.js
@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { schema, call, snapshot } = require('./lib/tools/browser');
+
+module.exports = { schema, call, snapshot };
diff --git a/packages/playwright-tools/computer-20241022.d.ts b/packages/playwright-tools/computer-20241022.d.ts
new file mode 100644
index 0000000000..ce0f821d6d
--- /dev/null
+++ b/packages/playwright-tools/computer-20241022.d.ts
@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type playwright from 'playwright';
+import { JSONSchemaType } from './types';
+
+export type ToolResult = {
+ output?: string;
+ error?: string;
+ base64_image?: string;
+};
+
+export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise<ToolResult>;
+
+export const call: ToolCall;
diff --git a/packages/playwright-tools/computer-20241022.js b/packages/playwright-tools/computer-20241022.js
new file mode 100644
index 0000000000..030d0e57e2
--- /dev/null
+++ b/packages/playwright-tools/computer-20241022.js
@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { call } = require('./lib/tools/computer-20241022');
+
+module.exports = { call };
diff --git a/packages/playwright-tools/package.json b/packages/playwright-tools/package.json
new file mode 100644
index 0000000000..c5366c2944
--- /dev/null
+++ b/packages/playwright-tools/package.json
@@ -0,0 +1,35 @@
+{
+ "name": "@playwright/experimental-tools",
+ "version": "1.51.0-next",
+ "description": "Playwright Tools for AI",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/microsoft/playwright.git"
+ },
+ "homepage": "https://playwright.dev",
+ "engines": {
+ "node": ">=18"
+ },
+ "author": {
+ "name": "Microsoft Corporation"
+ },
+ "license": "Apache-2.0",
+ "exports": {
+ "./browser": {
+ "types": "./browser.d.ts",
+ "default": "./browser.js"
+ },
+ "./computer-20241022": {
+ "types": "./computer-20241022.d.ts",
+ "default": "./computer-20241022.js"
+ },
+ "./package.json": "./package.json"
+ },
+ "dependencies": {
+ "playwright": "1.51.0-next"
+ },
+ "devDependencies": {
+ "@anthropic-ai/sdk": "^0.33.1",
+ "openai": "^4.79.1"
+ }
+}
diff --git a/packages/playwright-tools/src/examples/browser-anthropic.ts b/packages/playwright-tools/src/examples/browser-anthropic.ts
new file mode 100644
index 0000000000..0053c5749d
--- /dev/null
+++ b/packages/playwright-tools/src/examples/browser-anthropic.ts
@@ -0,0 +1,143 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* eslint-disable no-console */
+
+import playwright from 'playwright';
+import Anthropic from '@anthropic-ai/sdk';
+import dotenv from 'dotenv';
+import browser from '@playwright/experimental-tools/browser';
+
+dotenv.config();
+
+const anthropic = new Anthropic();
+
+export const system = `
+You are a web tester.
+
+<Instructions>
+- Perform test according to the provided checklist
+- Use browser tools to perform actions on web page
+- Never ask questions, always perform a best guess action
+- Use one tool at a time, wait for its result before proceeding.
+- When ready use "reportResult" tool to report result
+</Instructions>`;
+
+const reportTool: Anthropic.Tool = {
+ name: 'reportResult',
+ description: 'Submit test result',
+ input_schema: {
+ type: 'object',
+ properties: {
+ 'success': { type: 'boolean', description: 'Whether test passed' },
+ 'result': { type: 'string', description: 'Result of the test if some information has been requested' },
+ 'error': { type: 'string', description: 'Error message if test failed' }
+ },
+ required: ['success']
+ }
+};
+
+type Message = Anthropic.Beta.Messages.BetaMessageParam & {
+ history: Anthropic.Beta.Messages.BetaMessageParam['content']
+};
+
+async function anthropicAgentLoop(page: playwright.Page, task: string) {
+ // Convert them into tools for Anthropic.
+ const pageTools: Anthropic.Tool[] = browser.schema.map(tool => {
+ return {
+ name: tool.name,
+ description: tool.description,
+ input_schema: tool.parameters as any,
+ };
+ });
+
+ // Add report tool.
+ const tools = [reportTool, ...pageTools];
+
+ const history: Message[] = [{
+ role: 'user',
+ history: `Task: ${task}`,
+ content: `Task: ${task}\n\n${await browser.snapshot(page)}`,
+ }];
+
+ // Run agentic loop, cap steps.
+ for (let i = 0; i < 50; i++) {
+ const response = await anthropic.messages.create({
+ model: 'claude-3-5-sonnet-20241022',
+ max_tokens: 1024,
+ temperature: 0,
+ tools,
+ system,
+ messages: toAnthropicMessages(history),
+ });
+ history.push({ role: 'assistant', content: response.content, history: response.content });
+
+ const toolUse = response.content.find(block => block.type === 'tool_use');
+ if (!toolUse) {
+ history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });
+ continue;
+ }
+
+ if (toolUse.name === 'reportResult') {
+ console.log(toolUse.input);
+ return;
+ }
+
+ // Run the Playwright tool.
+ const { error, snapshot, code } = await browser.call(page, toolUse.name, toolUse.input as any);
+ if (code.length)
+ console.log(code.join('\n'));
+
+ // Report the result.
+ const resultText = error ? `Error: ${error}\n` : 'Done\n';
+ history.push({
+ role: 'user',
+ content: [{
+ type: 'tool_result',
+ tool_use_id: toolUse.id,
+ content: [{ type: 'text', text: resultText + snapshot }],
+ }],
+ history: [{
+ type: 'tool_result',
+ tool_use_id: toolUse.id,
+ content: [{ type: 'text', text: resultText }],
+ }],
+ });
+ }
+}
+
+function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] {
+ return messages.map((message, i) => {
+ if (i === messages.length - 1)
+ return { ...message, history: undefined };
+ return { ...message, content: message.history, history: undefined };
+ });
+}
+
+async function main() {
+ const browser = await playwright.chromium.launch({ headless: false });
+ const page = await browser.newPage();
+ await anthropicAgentLoop(page, `
+ - Go to http://github.com/microsoft
+ - Search for "playwright" repository
+ - Navigate to it
+ - Switch into the Issues tab
+ - Report 3 first issues
+ `);
+ await browser.close();
+}
+
+void main();
diff --git a/packages/playwright-tools/src/examples/browser-openai.ts b/packages/playwright-tools/src/examples/browser-openai.ts
new file mode 100644
index 0000000000..3d39618f5b
--- /dev/null
+++ b/packages/playwright-tools/src/examples/browser-openai.ts
@@ -0,0 +1,161 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* eslint-disable no-console */
+
+import playwright from 'playwright';
+import browser from '@playwright/experimental-tools/browser';
+import dotenv from 'dotenv';
+import OpenAI from 'openai';
+import type { ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources';
+
+dotenv.config();
+
+const openai = new OpenAI();
+
+export const system = `
+You are a web tester.
+
+<Instructions>to
+- Perform test according to the provided checklist
+- Use browser tools to perform actions on web page
+- Never ask questions, always perform a best guess action
+- When ready use "reportResult" tool to report result
+- You can only make one tool call at a time.
+</Instructions>`;
+
+type Message = ChatCompletionMessageParam & {
+ history: any
+};
+
+const reportTool: ChatCompletionTool = {
+ type: 'function',
+ function: {
+ name: 'reportResult',
+ description: 'Submit test result',
+ parameters: {
+ type: 'object',
+ properties: {
+ success: { type: 'boolean', description: 'Whether test passed' },
+ result: { type: 'string', description: 'Result of the test if requested' },
+ error: { type: 'string', description: 'Error if test failed' },
+ },
+ required: ['success'],
+ additionalProperties: false,
+ },
+ }
+};
+
+async function openAIAgentLoop(page: playwright.Page, task: string) {
+ const pageTools: ChatCompletionTool[] = browser.schema.map(tool => ({
+ type: 'function',
+ function: {
+ name: tool.name,
+ description: tool.description,
+ parameters: {
+ ...tool.parameters,
+ additionalProperties: false,
+ },
+ }
+ }));
+
+ const tools = [reportTool, ...pageTools];
+
+ const history: Message[] = [
+ {
+ role: 'system', content: system, history: system
+ },
+ {
+ role: 'user',
+ history: `Task: ${task}`,
+ content: `Task: ${task}\n\n${await browser.snapshot(page)}`,
+ }
+ ];
+
+ // Run agentic loop, cap steps.
+ for (let i = 0; i < 50; i++) {
+ const completion = await openai.chat.completions.create({
+ model: 'gpt-4o',
+ messages: toOpenAIMessages(history),
+ tools,
+ store: true,
+ });
+
+ console.log(JSON.stringify(completion, null, 2));
+
+ const toolCalls = completion.choices[0]?.message?.tool_calls;
+ if (!toolCalls || toolCalls.length !== 1 || toolCalls[0].type !== 'function') {
+ history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });
+ continue;
+ }
+
+ const toolCall = toolCalls[0];
+ if (toolCall.function.name === 'reportResult') {
+ console.log(JSON.parse(toolCall.function.arguments));
+ return;
+ }
+
+ history.push({ ...completion.choices[0].message, history: null });
+
+ // Run the Playwright tool.
+ const params = JSON.parse(toolCall.function.arguments);
+ const { error, snapshot, code } = await browser.call(page, toolCall.function.name, params);
+ console.log({ error, code, snapshot });
+ if (code.length)
+ console.log(code.join('\n'));
+
+ if (toolCall.function.name === 'log')
+ return;
+
+ // Report the result.
+ const resultText = error ? `Error: ${error}\n` : 'Done\n';
+ history.push({
+ role: 'tool',
+ tool_call_id: toolCall.id,
+ content: resultText + snapshot,
+ history: resultText,
+ });
+ }
+}
+
+function toOpenAIMessages(messages: Message[]): ChatCompletionMessageParam[] {
+ return messages.map((message, i) => {
+ const copy: Message = { ...message };
+ delete copy.history;
+ if (i === messages.length - 1)
+ return copy;
+ copy.content = message.history;
+ return copy;
+ });
+}
+
+async function main() {
+ const browser = await playwright.chromium.launch({ headless: false });
+ const page = await browser.newPage();
+ await openAIAgentLoop(page, `
+ - Go to http://github.com/microsoft
+ - Search for "playwright" repository
+ - Navigate to it
+ - Capture snapshot for toolbar with Code, Issues, etc.
+ - Capture snapshot for branch selector
+ - Assert that number of Issues is present
+ - Switch into the Issues tab
+ - Report 3 first issues
+ `);
+ await browser.close();
+}
+
+void main();
diff --git a/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts b/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts
new file mode 100644
index 0000000000..20028c564c
--- /dev/null
+++ b/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts
@@ -0,0 +1,148 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* eslint-disable no-console */
+
+import playwright from 'playwright';
+import Anthropic from '@anthropic-ai/sdk';
+import dotenv from 'dotenv';
+import computer, { type ToolResult } from '@playwright/experimental-tools/computer-20241022';
+import type { BetaImageBlockParam, BetaTextBlockParam } from '@anthropic-ai/sdk/resources/beta/messages/messages';
+
+dotenv.config();
+
+const anthropic = new Anthropic();
+
+export const system = `
+You are a web tester.
+
+<Instructions>
+- Perform test according to the provided checklist
+- Use browser tools to perform actions on web page
+- Never ask questions, always perform a best guess action
+- Use one tool at a time, wait for its result before proceeding.
+- When ready use "reportResult" tool to report result
+</Instructions>`;
+
+const computerTool: Anthropic.Beta.BetaToolUnion = {
+ type: 'computer_20241022',
+ name: 'computer',
+ display_width_px: 1920,
+ display_height_px: 1080,
+ display_number: 1,
+};
+
+const reportTool: Anthropic.Tool = {
+ name: 'reportResult',
+ description: 'Submit test result',
+ input_schema: {
+ type: 'object',
+ properties: {
+ 'success': { type: 'boolean', description: 'Whether test passed' },
+ 'result': { type: 'string', description: 'Result of the test if some information has been requested' },
+ 'error': { type: 'string', description: 'Error message if test failed' }
+ },
+ required: ['success']
+ }
+};
+
+type Message = Anthropic.Beta.Messages.BetaMessageParam & {
+ history: Anthropic.Beta.Messages.BetaMessageParam['content']
+};
+
+async function anthropicAgentLoop(page: playwright.Page, task: string) {
+ // Add report tool.
+ const tools = [reportTool, computerTool];
+
+ const history: Message[] = [{
+ role: 'user',
+ history: `Task: ${task}`,
+ content: `Task: ${task}`,
+ }];
+
+ // Run agentic loop, cap steps.
+ for (let i = 0; i < 50; i++) {
+ const response = await anthropic.beta.messages.create({
+ model: 'claude-3-5-sonnet-20241022',
+ max_tokens: 1024,
+ temperature: 0,
+ tools,
+ system,
+ messages: toAnthropicMessages(history),
+ betas: ['computer-use-2024-10-22'],
+ });
+
+ history.push({ role: 'assistant', content: response.content, history: response.content });
+
+ const toolUse = response.content.find(block => block.type === 'tool_use');
+ if (!toolUse) {
+ history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });
+ continue;
+ }
+
+ if (toolUse.name === 'reportResult') {
+ console.log(toolUse.input);
+ return;
+ }
+
+ const result: ToolResult = await computer.call(page, toolUse.name, toolUse.input as any);
+ const contentEntry: BetaTextBlockParam | BetaImageBlockParam = result.base64_image ? {
+ type: 'image',
+ source: { type: 'base64', media_type: 'image/jpeg', data: result.base64_image }
+ } : {
+ type: 'text',
+ text: result.output || '',
+ };
+ history.push({
+ role: 'user',
+ content: [{
+ type: 'tool_result',
+ tool_use_id: toolUse.id,
+ content: [contentEntry],
+ }],
+ history: [{
+ type: 'tool_result',
+ tool_use_id: toolUse.id,
+ content: [{ type: 'text', text: '<redacted>' }],
+ }],
+ });
+ }
+}
+
+function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] {
+ return messages.map((message, i) => {
+ if (i === messages.length - 1)
+ return { ...message, history: undefined };
+ return { ...message, content: message.history, history: undefined };
+ });
+}
+
+const githubTask = `
+ - Search for "playwright" repository
+ - Navigate to it
+ - Switch into the Issues tab
+ - Report 3 first issues
+`;
+
+async function main() {
+ const browser = await playwright.chromium.launch({ headless: false });
+ const page = await browser.newPage();
+ await page.goto('http://github.com/microsoft');
+ await anthropicAgentLoop(page, githubTask);
+ await browser.close();
+}
+
+void main();
diff --git a/packages/playwright-tools/src/tools/browser.ts b/packages/playwright-tools/src/tools/browser.ts
new file mode 100644
index 0000000000..267d03bcb3
--- /dev/null
+++ b/packages/playwright-tools/src/tools/browser.ts
@@ -0,0 +1,146 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type playwright from 'playwright';
+import type { JSONSchemaType, ToolDeclaration } from '../../types';
+import type { ToolResult } from '../../browser';
+import { waitForNetwork } from './utils';
+
+type LocatorEx = playwright.Locator & {
+ _generateLocatorString: () => Promise<string>;
+};
+
+const intentProperty = {
+ intent: {
+ type: 'string',
+ description: 'Intent behind this particular action. Used as a comment.',
+ }
+};
+
+const elementIdProperty = {
+ elementId: {
+ type: 'number',
+ description: 'Target element',
+ }
+};
+
+export const schema: ToolDeclaration[] = [
+ {
+ name: 'navigate',
+ description: 'Navigate to a URL',
+ parameters: {
+ type: 'object',
+ properties: {
+ ...intentProperty,
+ url: {
+ type: 'string',
+ description: 'URL to navigate to',
+ },
+ },
+ required: ['intent', 'elementId'],
+ }
+ },
+ {
+ name: 'click',
+ description: 'Perform click on a web page',
+ parameters: {
+ type: 'object',
+ properties: {
+ ...intentProperty,
+ ...elementIdProperty,
+ },
+ required: ['intent', 'elementId'],
+ }
+ },
+ {
+ name: 'enterText',
+ description: 'Enter text into editable element',
+ parameters: {
+ type: 'object',
+ properties: {
+ ...intentProperty,
+ ...elementIdProperty,
+ text: {
+ type: 'string',
+ description: 'Text to enter',
+ },
+ submit: {
+ type: 'boolean',
+ description: 'Whether to submit entered text (press Enter after)'
+ }
+ },
+ required: ['intent', 'elementId', 'text'],
+ }
+ },
+ {
+ name: 'wait',
+ description: `Wait for given amount of time to see if the page updates. Use it after action if you think page is not ready yet`,
+ parameters: {
+ type: 'object',
+ properties: {
+ ...intentProperty,
+ time: {
+ type: 'integer',
+ description: 'Time to wait in seconds',
+ },
+ },
+ required: ['intent', 'time'],
+ }
+ },
+];
+
+export async function call(page: playwright.Page, toolName: string, params: Record<string, JSONSchemaType>): Promise<ToolResult> {
+ const code: string[] = [];
+ try {
+ await waitForNetwork(page, async () => {
+ await performAction(page, toolName, params, code);
+ });
+ } catch (e) {
+ return { error: e.message, snapshot: await snapshot(page), code };
+ }
+ return { snapshot: await snapshot(page), code };
+}
+
+export async function snapshot(page: playwright.Page) {
+ const params = { _id: true } as any;
+ return `<Page snapshot>\n${await page.locator('body').ariaSnapshot(params)}\n</Page snapshot>`;
+}
+
+async function performAction(page: playwright.Page, toolName: string, params: Record<string, JSONSchemaType>, code: string[]) {
+ const locator = elementLocator(page, params);
+ code.push((params.intent as string).split('\n').map(line => `// ${line}`).join('\n'));
+ if (toolName === 'navigate') {
+ code.push(`await page.goto(${JSON.stringify(params.url)})`);
+ await page.goto(params.url as string);
+ } else if (toolName === 'click') {
+ code.push(`await page.${await locator._generateLocatorString()}.click()`);
+ await locator.click();
+ } else if (toolName === 'enterText') {
+ code.push(`await page.${await locator._generateLocatorString()}.click()`);
+ await locator.click();
+ code.push(`await page.${await locator._generateLocatorString()}.fill(${JSON.stringify(params.text)})`);
+ await locator.fill(params.text as string);
+ if (params.submit) {
+ code.push(`await page.${await locator._generateLocatorString()}.press("Enter")`);
+ await locator.press('Enter');
+ }
+ }
+}
+
+function elementLocator(page: playwright.Page, params: any): LocatorEx {
+ return page.locator(`internal:aria-id=${params.elementId}`) as LocatorEx;
+}
diff --git a/packages/playwright-tools/src/tools/computer-20241022.ts b/packages/playwright-tools/src/tools/computer-20241022.ts
new file mode 100644
index 0000000000..e407d027ba
--- /dev/null
+++ b/packages/playwright-tools/src/tools/computer-20241022.ts
@@ -0,0 +1,158 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License');
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an 'AS IS' BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type playwright from 'playwright';
+import type { JSONSchemaType } from '../../types';
+import type { ToolResult } from '../../computer-20241022';
+import { waitForNetwork } from './utils';
+
+export async function call(page: playwright.Page, toolName: string, input: Record<string, JSONSchemaType>): Promise<ToolResult> {
+ if (toolName !== 'computer')
+ throw new Error('Unsupported tool');
+ return await waitForNetwork(page, async () => {
+ return await performAction(page, toolName, input);
+ });
+}
+
+type PageState = {
+ x: number;
+ y: number;
+};
+
+const pageStateSymbol = Symbol('pageState');
+
+function pageState(page: playwright.Page): PageState {
+ if (!(page as any)[pageStateSymbol])
+ (page as any)[pageStateSymbol] = { x: 0, y: 0 };
+ return (page as any)[pageStateSymbol];
+}
+
+async function performAction(page: playwright.Page, toolName: string, input: Record<string, JSONSchemaType>): Promise<ToolResult> {
+ const state = pageState(page);
+ const { action } = input as { action: string };
+ if (action === 'screenshot') {
+ const screenshot = await page.screenshot({ type: 'jpeg', quality: 50, scale: 'css' });
+ return {
+ output: 'Screenshot',
+ base64_image: screenshot.toString('base64'),
+ };
+ }
+ if (action === 'mouse_move') {
+ const { coordinate } = input as { coordinate: [number, number] };
+ state.x = coordinate[0];
+ state.y = coordinate[1];
+ await page.mouse.move(state.x, state.y);
+ return { output: 'Mouse moved' };
+ }
+ if (action === 'left_click') {
+ await page.mouse.down();
+ await page.mouse.up();
+ return { output: 'Left clicked' };
+ }
+ if (action === 'left_click_drag') {
+ await page.mouse.down();
+ const { coordinate } = input as { coordinate: [number, number] };
+ state.x = coordinate[0];
+ state.y = coordinate[1];
+ await page.mouse.move(state.x, state.y);
+ await page.mouse.up();
+ return { output: 'Left dragged' };
+ }
+ if (action === 'right_click') {
+ await page.mouse.down({ button: 'right' });
+ await page.mouse.up({ button: 'right' });
+ return { output: 'Right clicked' };
+ }
+ if (action === 'double_click') {
+ await page.mouse.down();
+ await page.mouse.up();
+ await page.mouse.down();
+ await page.mouse.up();
+ return { output: 'Double clicked' };
+ }
+ if (action === 'middle_click') {
+ await page.mouse.down({ button: 'middle' });
+ await page.mouse.up({ button: 'middle' });
+ return { output: 'Middle clicked' };
+ }
+ if (action === 'key') {
+ const { text } = input as { text: string };
+ await page.keyboard.press(xToPlaywright(text));
+ return { output: 'Text typed' };
+ }
+ if (action === 'cursor_position')
+ return { output: `X=${state.x},Y=${state.y}` };
+ throw new Error('Unimplemented tool: ' + toolName);
+}
+
+const xToPlaywrightKeyMap = new Map([
+ ['BackSpace', 'Backspace'],
+ ['Tab', 'Tab'],
+ ['Return', 'Enter'],
+ ['Escape', 'Escape'],
+ ['space', ' '],
+ ['Delete', 'Delete'],
+ ['Home', 'Home'],
+ ['End', 'End'],
+ ['Left', 'ArrowLeft'],
+ ['Up', 'ArrowUp'],
+ ['Right', 'ArrowRight'],
+ ['Down', 'ArrowDown'],
+ ['Insert', 'Insert'],
+ ['Page_Up', 'PageUp'],
+ ['Page_Down', 'PageDown'],
+ ['F1', 'F1'],
+ ['F2', 'F2'],
+ ['F3', 'F3'],
+ ['F4', 'F4'],
+ ['F5', 'F5'],
+ ['F6', 'F6'],
+ ['F7', 'F7'],
+ ['F8', 'F8'],
+ ['F9', 'F9'],
+ ['F10', 'F10'],
+ ['F11', 'F11'],
+ ['F12', 'F12'],
+ ['Shift_L', 'Shift'],
+ ['Shift_R', 'Shift'],
+ ['Control_L', 'Control'],
+ ['Control_R', 'Control'],
+ ['Alt_L', 'Alt'],
+ ['Alt_R', 'Alt'],
+ ['Super_L', 'Meta'],
+ ['Super_R', 'Meta'],
+]);
+
+const xToPlaywrightModifierMap = new Map([
+ ['alt', 'Alt'],
+ ['control', 'Control'],
+ ['meta', 'Meta'],
+ ['shift', 'Shift'],
+]);
+
+
+const xToPlaywright = (key: string) => {
+ const tokens = key.split('+');
+ if (tokens.length === 1)
+ return xToPlaywrightKeyMap.get(key) || key;
+ if (tokens.length === 2) {
+ const modifier = xToPlaywrightModifierMap.get(tokens[0]);
+ const key = xToPlaywrightKeyMap.get(tokens[1]) || tokens[1];
+ return modifier + '+' + key;
+ }
+ throw new Error('Invalid key: ' + key);
+};
diff --git a/packages/playwright-tools/src/tools/utils.ts b/packages/playwright-tools/src/tools/utils.ts
new file mode 100644
index 0000000000..08e674014d
--- /dev/null
+++ b/packages/playwright-tools/src/tools/utils.ts
@@ -0,0 +1,71 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type playwright from 'playwright';
+import { ManualPromise } from 'playwright-core/lib/utils';
+
+export async function waitForNetwork<R>(page: playwright.Page, callback: () => Promise<R>): Promise<R> {
+ const requests = new Set<playwright.Request>();
+ let frameNavigated = false;
+ const waitBarrier = new ManualPromise();
+
+ const requestListener = (request: playwright.Request) => requests.add(request);
+ const requestFinishedListener = (request: playwright.Request) => {
+ requests.delete(request);
+ if (!requests.size)
+ waitBarrier.resolve();
+ };
+
+ const frameNavigateListener = (frame: playwright.Frame) => {
+ if (frame.parentFrame())
+ return;
+ frameNavigated = true;
+ dispose();
+ clearTimeout(timeout);
+ void frame.waitForLoadState('load').then(() => {
+ waitBarrier.resolve();
+ });
+ };
+
+ const onTimeout = () => {
+ dispose();
+ waitBarrier.resolve();
+ };
+
+ page.on('request', requestListener);
+ page.on('requestfinished', requestFinishedListener);
+ page.on('framenavigated', frameNavigateListener);
+ const timeout = setTimeout(onTimeout, 10000);
+
+ const dispose = () => {
+ page.off('request', requestListener);
+ page.off('requestfinished', requestFinishedListener);
+ page.off('framenavigated', frameNavigateListener);
+ clearTimeout(timeout);
+ };
+
+ try {
+ const result = await callback();
+ if (!requests.size && !frameNavigated)
+ waitBarrier.resolve();
+ await waitBarrier;
+ await page.evaluate(() => new Promise(f => setTimeout(f, 1000)));
+ return result;
+ } finally {
+ dispose();
+ }
+}
diff --git a/packages/playwright-tools/types.d.ts b/packages/playwright-tools/types.d.ts
new file mode 100644
index 0000000000..5fe61526c2
--- /dev/null
+++ b/packages/playwright-tools/types.d.ts
@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type playwright from 'playwright';
+
+export type JSONSchemaType = string | number | boolean | JSONSchemaObject | JSONSchemaArray | null;
+interface JSONSchemaObject { [key: string]: JSONSchemaType; }
+interface JSONSchemaArray extends Array<JSONSchemaType> {}
+
+export type ToolDeclaration = {
+ name: string;
+ description: string;
+ parameters: any;
+};
diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts
index 526cc599ab..30f8c03088 100644
--- a/packages/protocol/src/channels.d.ts
+++ b/packages/protocol/src/channels.d.ts
@@ -2619,9 +2619,13 @@ export type FrameAddStyleTagResult = {
};
export type FrameAriaSnapshotParams = {
selector: string,
+ id?: boolean,
+ mode?: 'raw' | 'regex',
timeout?: number,
};
export type FrameAriaSnapshotOptions = {
+ id?: boolean,
+ mode?: 'raw' | 'regex',
timeout?: number,
};
export type FrameAriaSnapshotResult = {
@@ -3311,6 +3315,7 @@ export interface ElementHandleChannel extends ElementHandleEventTarget, JSHandle
dispatchEvent(params: ElementHandleDispatchEventParams, metadata?: CallMetadata): Promise<ElementHandleDispatchEventResult>;
fill(params: ElementHandleFillParams, metadata?: CallMetadata): Promise<ElementHandleFillResult>;
focus(params?: ElementHandleFocusParams, metadata?: CallMetadata): Promise<ElementHandleFocusResult>;
+ generateLocatorString(params?: ElementHandleGenerateLocatorStringParams, metadata?: CallMetadata): Promise<ElementHandleGenerateLocatorStringResult>;
getAttribute(params: ElementHandleGetAttributeParams, metadata?: CallMetadata): Promise<ElementHandleGetAttributeResult>;
hover(params: ElementHandleHoverParams, metadata?: CallMetadata): Promise<ElementHandleHoverResult>;
innerHTML(params?: ElementHandleInnerHTMLParams, metadata?: CallMetadata): Promise<ElementHandleInnerHTMLResult>;
@@ -3450,6 +3455,11 @@ export type ElementHandleFillResult = void;
export type ElementHandleFocusParams = {};
export type ElementHandleFocusOptions = {};
export type ElementHandleFocusResult = void;
+export type ElementHandleGenerateLocatorStringParams = {};
+export type ElementHandleGenerateLocatorStringOptions = {};
+export type ElementHandleGenerateLocatorStringResult = {
+ value?: string,
+};
export type ElementHandleGetAttributeParams = {
name: string,
};
diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml
index df54dcbe1c..77501efa9b 100644
--- a/packages/protocol/src/protocol.yml
+++ b/packages/protocol/src/protocol.yml
@@ -1879,6 +1879,12 @@ Frame:
ariaSnapshot:
parameters:
selector: string
+ id: boolean?
+ mode:
+ type: enum?
+ literals:
+ - raw
+ - regex
timeout: number?
returns:
snapshot: string
@@ -2648,6 +2654,10 @@ ElementHandle:
slowMo: true
snapshot: true
+ generateLocatorString:
+ returns:
+ value: string?
+
getAttribute:
parameters:
name: string
diff --git a/utils/workspace.js b/utils/workspace.js
index 4fb69050b3..7e6850a262 100755
--- a/utils/workspace.js
+++ b/utils/workspace.js
@@ -167,6 +167,11 @@ const workspace = new Workspace(ROOT_PATH, [
path: path.join(ROOT_PATH, 'packages', 'playwright-chromium'),
files: LICENCE_FILES,
}),
+ new PWPackage({
+ name: '@playwright/experimental-tools',
+ path: path.join(ROOT_PATH, 'packages', 'playwright-tools'),
+ files: LICENCE_FILES,
+ }),
new PWPackage({
name: '@playwright/browser-webkit',
path: path.join(ROOT_PATH, 'packages', 'playwright-browser-webkit'),