Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(aria-snapshot): add boxes option to include bounding boxes (#40389)
Pavel Feldman1b18f3bfd19b
diff --git a/docs/src/api/class-locator.md b/docs/src/api/class-locator.mdindex ad37165471..5d8a763480 100644--- a/docs/src/api/class-locator.md+++ b/docs/src/api/class-locator.md@@ -244,6 +244,12 @@ When set to `"ai"`, returns a snapshot optimized for AI consumption. Defaults toWhen specified, limits the depth of the snapshot.+### option: Locator.ariaSnapshot.boxes+* since: v1.60+- `boxes` <[boolean]>++When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.+## async method: Locator.blur* since: v1.28diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.mdindex 47e5626b72..b20c766651 100644--- a/docs/src/api/class-page.md+++ b/docs/src/api/class-page.md@@ -4253,6 +4253,12 @@ When set to `"ai"`, returns a snapshot optimized for AI consumption: including eWhen specified, limits the depth of the snapshot.+### option: Page.ariaSnapshot.boxes+* since: v1.60+- `boxes` <[boolean]>++When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.+## async method: Page.tap* since: v1.8* discouraged: Use locator-based [`method: Locator.tap`] instead. Read more about [locators](../locators.md).diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.tsindex 08e23d585e..17b438dc13 100644--- a/packages/injected/src/ariaSnapshot.ts+++ b/packages/injected/src/ariaSnapshot.ts@@ -41,6 +41,7 @@ export type AriaTreeOptions = {refPrefix?: string;doNotRenderActive?: boolean;depth?: number;+ boxes?: boolean;};type InternalOptions = {@@ -51,9 +52,11 @@ type InternalOptions = {renderCursorPointer?: boolean,renderActive?: boolean,renderStringsAsRegex?: boolean,+ renderBoxes?: boolean,};function toInternalOptions(options: AriaTreeOptions): InternalOptions {+ const renderBoxes = options.boxes;if (options.mode === 'ai') {// For AI consumption.return {@@ -63,18 +66,19 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions {includeGenericRole: true,renderActive: !options.doNotRenderActive,renderCursorPointer: true,+ renderBoxes,};}if (options.mode === 'autoexpect') {// To auto-generate assertions on visible elements.- return { visibility: 'ariaAndVisible', refs: 'none' };+ return { visibility: 'ariaAndVisible', refs: 'none', renderBoxes };}if (options.mode === 'codegen') {// To generate aria assertion with regex heurisitcs.- return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true };+ return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true, renderBoxes };}// To match aria snapshot.- return { visibility: 'aria', refs: 'none' };+ return { visibility: 'aria', refs: 'none', renderBoxes };}export function generateAriaTree(rootElement: Element, publicOptions: AriaTreeOptions): AriaSnapshot {@@ -624,6 +628,13 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTrif (renderCursorPointer && aria.hasPointerCursor(ariaNode))key += ' [cursor=pointer]';}+ if (options.renderBoxes) {+ const element = ariaNodeElement(ariaNode);+ if (element) {+ const r = element.getBoundingClientRect();+ key += ` [box=${Math.round(r.x)},${Math.round(r.y)},${Math.round(r.width)},${Math.round(r.height)}]`;+ }+ }return key;};diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.tsindex 30e475fcde..452fbe353a 100644--- a/packages/playwright-client/types/types.d.ts+++ b/packages/playwright-client/types/types.d.ts@@ -2051,6 +2051,11 @@ export interface Page {* @param options*/ariaSnapshot(options?: {+ /**+ * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.+ */+ boxes?: boolean;+/*** When specified, limits the depth of the snapshot.*/@@ -13057,6 +13062,11 @@ export interface Locator {* @param options*/ariaSnapshot(options?: {+ /**+ * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.+ */+ boxes?: boolean;+/*** When specified, limits the depth of the snapshot.*/diff --git a/packages/playwright-core/src/client/locator.ts b/packages/playwright-core/src/client/locator.tsindex 9de0f2b295..8dda1580c1 100644--- a/packages/playwright-core/src/client/locator.ts+++ b/packages/playwright-core/src/client/locator.ts@@ -329,8 +329,8 @@ export class Locator implements api.Locator {return ref ?? null;}- async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number } = {}): Promise<string> {- const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth });+ async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, boxes?: boolean } = {}): Promise<string> {+ const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth, boxes: options.boxes });return result.snapshot;}diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.tsindex 2d519d8095..1dc9817d36 100644--- a/packages/playwright-core/src/client/page.ts+++ b/packages/playwright-core/src/client/page.ts@@ -866,8 +866,8 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Pagereturn result.pdf;}- async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, _track?: string } = {}): Promise<string> {- const result = await this.mainFrame()._channel.ariaSnapshot({ timeout: this._timeoutSettings.timeout(options), track: options._track, mode: options.mode, depth: options.depth });+ async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, boxes?: boolean, _track?: string } = {}): Promise<string> {+ const result = await this.mainFrame()._channel.ariaSnapshot({ timeout: this._timeoutSettings.timeout(options), track: options._track, mode: options.mode, depth: options.depth, boxes: options.boxes });return result.snapshot;}diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.tsindex 3c24734e63..d9eaec067b 100644--- a/packages/playwright-core/src/protocol/validator.ts+++ b/packages/playwright-core/src/protocol/validator.ts@@ -1660,6 +1660,7 @@ scheme.FrameAriaSnapshotParams = tObject({track: tOptional(tString),selector: tOptional(tString),depth: tOptional(tInt),+ boxes: tOptional(tBoolean),timeout: tFloat,});scheme.FrameAriaSnapshotResult = tObject({diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.tsindex 90fafe6282..a298a0df92 100644--- a/packages/playwright-core/src/server/frames.ts+++ b/packages/playwright-core/src/server/frames.ts@@ -1774,14 +1774,14 @@ export class Frame extends SdkObject<FrameEventMap> {return { ref };}- async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number } = {}): Promise<{ snapshot: string }> {+ async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number, boxes?: boolean } = {}): Promise<{ snapshot: string }> {if (options.selector && options.track)throw new Error('Cannot specify both selector and track options');if (options.selector && options.mode !== 'ai') {// Non-ai locator snapshot is auto-waiting and does not include iframes.const snapshot = await this._retryWithProgressIfNotConnected(progress, options.selector, { strict: true, performActionPreChecks: true }, async (progress, handle) => {- return await progress.race(handle.evaluateInUtility(([injected, element, opts]) => injected.ariaSnapshot(element, opts), { mode: 'default' as const, depth: options.depth }));+ return await progress.race(handle.evaluateInUtility(([injected, element, opts]) => injected.ariaSnapshot(element, opts), { mode: 'default' as const, depth: options.depth, boxes: options.boxes }));});return { snapshot };}diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.tsindex 257b657ba3..281ad0a6a1 100644--- a/packages/playwright-core/src/server/page.ts+++ b/packages/playwright-core/src/server/page.ts@@ -1061,7 +1061,7 @@ export class InitScript extends DisposableObject {}}-export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Frame, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, info?: SelectorInfo, depth?: number } = {}): Promise<{ full: string[], incremental?: string[] }> {+export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Frame, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, info?: SelectorInfo, depth?: number, boxes?: boolean } = {}): Promise<{ full: string[], incremental?: string[] }> {// Only await the topmost navigations, inner frames will be empty when racing.const snapshot = await frame.retryWithProgressAndTimeouts(progress, [1000, 2000, 4000, 8000], async (progress, continuePolling) => {try {@@ -1085,6 +1085,7 @@ export async function ariaSnapshotForFrame(progress: Progress, frame: frames.FradoNotRenderActive: options.doNotRenderActive,info: options.info,depth: options.depth,+ boxes: options.boxes,}));if (snapshotOrRetry === true)return continuePolling;diff --git a/packages/playwright-core/src/tools/backend/response.ts b/packages/playwright-core/src/tools/backend/response.tsindex 6b83adc25a..eda678a846 100644--- a/packages/playwright-core/src/tools/backend/response.ts+++ b/packages/playwright-core/src/tools/backend/response.ts@@ -50,6 +50,7 @@ export class Response {private _includeSnapshotFileName: string | undefined;private _includeSnapshotRoot: playwright.Locator | undefined;private _includeSnapshotDepth: number | undefined;+ private _includeSnapshotBoxes: boolean | undefined;private _isClose: boolean = false;readonly toolName: string;@@ -142,10 +143,11 @@ export class Response {this._includeSnapshot = this._context.config.snapshot?.mode ?? 'full';}- setIncludeFullSnapshot(includeSnapshotFileName?: string, root?: playwright.Locator, depth?: number) {+ setIncludeFullSnapshot(includeSnapshotFileName?: string, root?: playwright.Locator, depth?: number, boxes?: boolean) {this._includeSnapshot = 'explicit';this._includeSnapshotFileName = includeSnapshotFileName;this._includeSnapshotDepth = depth;+ this._includeSnapshotBoxes = boxes;this._includeSnapshotRoot = root;}@@ -242,7 +244,7 @@ export class Response {addSection('Ran Playwright code', this._code, 'js');// Render tab titles upon changes or when more than one tab.- const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotRoot, this._includeSnapshotDepth, this._clientWorkspace) : undefined;+ const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotRoot, this._includeSnapshotDepth, this._includeSnapshotBoxes, this._clientWorkspace) : undefined;const tabHeaders = await Promise.all(this._context.tabs().map(tab => tab.headerSnapshot()));if (this._includeSnapshot !== 'none' || tabHeaders.some(header => header.changed)) {if (tabHeaders.length !== 1)diff --git a/packages/playwright-core/src/tools/backend/snapshot.ts b/packages/playwright-core/src/tools/backend/snapshot.tsindex 30730b33b4..516b17da43 100644--- a/packages/playwright-core/src/tools/backend/snapshot.ts+++ b/packages/playwright-core/src/tools/backend/snapshot.ts@@ -42,6 +42,7 @@ const snapshot = defineTabTool({target: z.string().optional().describe(elementTargetDescription),filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'),depth: z.number().optional().describe('Limit the depth of the snapshot tree'),+ boxes: z.boolean().optional().describe('Include each element\'s bounding box as [box=x,y,width,height] in the snapshot'),}),type: 'readOnly',},@@ -50,7 +51,7 @@ const snapshot = defineTabTool({let resolved: { locator: playwright.Locator | undefined, resolved: string } = { locator: undefined, resolved: '' };if (params.target)resolved = await tab.targetLocator({ target: params.target });- response.setIncludeFullSnapshot(params.filename, resolved.locator, params.depth);+ response.setIncludeFullSnapshot(params.filename, resolved.locator, params.depth, params.boxes);},});diff --git a/packages/playwright-core/src/tools/backend/tab.ts b/packages/playwright-core/src/tools/backend/tab.tsindex 0284509303..e409fa344d 100644--- a/packages/playwright-core/src/tools/backend/tab.ts+++ b/packages/playwright-core/src/tools/backend/tab.ts@@ -383,13 +383,13 @@ export class Tab extends EventEmitter<TabEventsInterface> {this._requests.length = 0;}- async captureSnapshot(root: playwright.Locator | undefined, depth: number | undefined, relativeTo: string | undefined): Promise<TabSnapshot> {+ async captureSnapshot(root: playwright.Locator | undefined, depth: number | undefined, boxes: boolean | undefined, relativeTo: string | undefined): Promise<TabSnapshot> {await this._initializedPromise;let tabSnapshot: TabSnapshot | undefined;const modalStates = await this._raceAgainstModalStates(async () => {const ariaSnapshot = root- ? await root.ariaSnapshot({ mode: 'ai', depth })- : await this.page.ariaSnapshot({ mode: 'ai', depth });+ ? await root.ariaSnapshot({ mode: 'ai', depth, boxes })+ : await this.page.ariaSnapshot({ mode: 'ai', depth, boxes });tabSnapshot = {ariaSnapshot,modalStates: [],diff --git a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md b/packages/playwright-core/src/tools/cli-client/skill/SKILL.mdindex 66b5ad4acd..9e70ca9322 100644--- a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md+++ b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md@@ -256,6 +256,9 @@ playwright-cli snapshot "#main"# limit snapshot depth for efficiency, take a partial snapshot afterwardsplaywright-cli snapshot --depth=4playwright-cli snapshot e34++# include each element's bounding box as [box=x,y,width,height]+playwright-cli snapshot --boxes```## Targeting elementsdiff --git a/packages/playwright-core/src/tools/cli-daemon/commands.ts b/packages/playwright-core/src/tools/cli-daemon/commands.tsindex d703773e87..594778e874 100644--- a/packages/playwright-core/src/tools/cli-daemon/commands.ts+++ b/packages/playwright-core/src/tools/cli-daemon/commands.ts@@ -367,9 +367,10 @@ const snapshot = declareCommand({options: z.object({filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'),depth: numberArg.optional().describe('Limit snapshot depth, unlimited by default.'),+ boxes: z.boolean().optional().describe('Include each element\'s bounding box as [box=x,y,width,height] in the snapshot.'),}),toolName: 'browser_snapshot',- toolParams: ({ filename, target, depth }) => ({ filename, target, depth }),+ toolParams: ({ filename, target, depth, boxes }) => ({ filename, target, depth, boxes }),});const generateLocator = declareCommand({diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.tsindex 30e475fcde..452fbe353a 100644--- a/packages/playwright-core/types/types.d.ts+++ b/packages/playwright-core/types/types.d.ts@@ -2051,6 +2051,11 @@ export interface Page {* @param options*/ariaSnapshot(options?: {+ /**+ * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.+ */+ boxes?: boolean;+/*** When specified, limits the depth of the snapshot.*/@@ -13057,6 +13062,11 @@ export interface Locator {* @param options*/ariaSnapshot(options?: {+ /**+ * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.+ */+ boxes?: boolean;+/*** When specified, limits the depth of the snapshot.*/diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.tsindex d03bbcdd71..8125a83d92 100644--- a/packages/protocol/src/channels.d.ts+++ b/packages/protocol/src/channels.d.ts@@ -2927,6 +2927,7 @@ export type FrameAriaSnapshotParams = {track?: string,selector?: string,depth?: number,+ boxes?: boolean,timeout: number,};export type FrameAriaSnapshotOptions = {@@ -2934,6 +2935,7 @@ export type FrameAriaSnapshotOptions = {track?: string,selector?: string,depth?: number,+ boxes?: boolean,};export type FrameAriaSnapshotResult = {snapshot: string,diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.ymlindex 45970c3cc1..4ec62cb7f1 100644--- a/packages/protocol/src/protocol.yml+++ b/packages/protocol/src/protocol.yml@@ -2324,6 +2324,7 @@ Frame:track: string?selector: string?depth: int?+ boxes: boolean?timeout: floatreturns:snapshot: stringdiff --git a/tests/mcp/cli-core.spec.ts b/tests/mcp/cli-core.spec.tsindex 89c472b887..24ef136ca5 100644--- a/tests/mcp/cli-core.spec.ts+++ b/tests/mcp/cli-core.spec.ts@@ -310,6 +310,20 @@ test('snapshot depth', async ({ cli, server }) => {- button "Cancel" [ref=e6]`);});+test('snapshot --boxes', async ({ cli, server }) => {+ server.setContent('/', `+ <style>body { margin: 0; }</style>+ <button style="position:absolute;left:100px;top:50px;width:80px;height:40px;margin:0;padding:0;border:0;">click</button>+ `, 'text/html');+ await cli('open', server.PREFIX);++ const { inlineSnapshot } = await cli('snapshot', '--boxes');+ expect(inlineSnapshot).toContain(`- button "click" [ref=e1] [box=100,50,80,40]`);++ const { inlineSnapshot: plain } = await cli('snapshot');+ expect(plain).not.toMatch(/\[box=/);+});+test('eval --raw', async ({ cli, server }) => {await cli('open', server.HELLO_WORLD);const { output } = await cli('eval', '--raw', '() => document.title');diff --git a/tests/mcp/core.spec.ts b/tests/mcp/core.spec.tsindex 42490d8fd8..06423357b5 100644--- a/tests/mcp/core.spec.ts+++ b/tests/mcp/core.spec.ts@@ -276,6 +276,31 @@ test('snapshot depth', async ({ client, server }) => {});});+test('snapshot with boxes', async ({ client, server }) => {+ server.setContent('/', `+ <style>body { margin: 0; }</style>+ <button style="position:absolute;left:100px;top:50px;width:80px;height:40px;margin:0;padding:0;border:0;">click</button>+ `, 'text/html');++ await client.callTool({+ name: 'browser_navigate',+ arguments: { url: server.PREFIX },+ });++ expect(await client.callTool({+ name: 'browser_snapshot',+ arguments: { boxes: true },+ })).toHaveResponse({+ inlineSnapshot: expect.stringContaining(`- button "click" [ref=e1] [box=100,50,80,40]`),+ });++ expect(await client.callTool({+ name: 'browser_snapshot',+ })).toHaveResponse({+ inlineSnapshot: expect.not.stringMatching(/\[box=/),+ });+});+test('snapshot by ref', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright-cli/issues/347' } }, async ({ client, server }) => {server.setContent('/', `<ul>diff --git a/tests/page/page-aria-snapshot.spec.ts b/tests/page/page-aria-snapshot.spec.tsindex 8b6bcf19e1..da9708d56c 100644--- a/tests/page/page-aria-snapshot.spec.ts+++ b/tests/page/page-aria-snapshot.spec.ts@@ -748,3 +748,29 @@ it('should snapshot a locator inside an iframe', async ({ page }) => {- listitem: Item 2`);});++it('should snapshot with box from page', async ({ page }) => {+ await page.setContent(`+ <button style="position:absolute;left:100px;top:50px;width:80px;height:40px;margin:0;padding:0;border:0;">click</button>+ `);++ const snapshot = await page.ariaSnapshot({ boxes: true });+ expect(snapshot).toBe(`- button "click" [box=100,50,80,40]`);+});++it('should snapshot with box from locator', async ({ page }) => {+ await page.setContent(`+ <div style="position:absolute;left:10px;top:20px;width:200px;height:100px;">+ <button style="position:absolute;left:5px;top:5px;width:60px;height:30px;margin:0;padding:0;border:0;">ok</button>+ </div>+ `);++ const snapshot = await page.locator('div').ariaSnapshot({ boxes: true });+ expect(snapshot).toBe(`- button "ok" [box=15,25,60,30]`);+});++it('should not include box when option is omitted', async ({ page }) => {+ await page.setContent(`<button>click</button>`);+ const snapshot = await page.ariaSnapshot();+ expect(snapshot).not.toMatch(/\[box=/);+});