Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(aria-snapshot): add boxes option to include bounding boxes (#40389)

Pavel Feldman1b18f3bfd19b
diff --git a/docs/src/api/class-locator.md b/docs/src/api/class-locator.md
index ad37165471..5d8a763480 100644
--- a/docs/src/api/class-locator.md
+++ b/docs/src/api/class-locator.md
@@ -244,6 +244,12 @@ When set to `"ai"`, returns a snapshot optimized for AI consumption. Defaults to
When specified, limits the depth of the snapshot.
+### option: Locator.ariaSnapshot.boxes
+* since: v1.60
+- `boxes` <[boolean]>
+
+When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+
## async method: Locator.blur
* since: v1.28
diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md
index 47e5626b72..b20c766651 100644
--- a/docs/src/api/class-page.md
+++ b/docs/src/api/class-page.md
@@ -4253,6 +4253,12 @@ When set to `"ai"`, returns a snapshot optimized for AI consumption: including e
When specified, limits the depth of the snapshot.
+### option: Page.ariaSnapshot.boxes
+* since: v1.60
+- `boxes` <[boolean]>
+
+When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+
## async method: Page.tap
* since: v1.8
* discouraged: Use locator-based [`method: Locator.tap`] instead. Read more about [locators](../locators.md).
diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.ts
index 08e23d585e..17b438dc13 100644
--- a/packages/injected/src/ariaSnapshot.ts
+++ b/packages/injected/src/ariaSnapshot.ts
@@ -41,6 +41,7 @@ export type AriaTreeOptions = {
refPrefix?: string;
doNotRenderActive?: boolean;
depth?: number;
+ boxes?: boolean;
};
type InternalOptions = {
@@ -51,9 +52,11 @@ type InternalOptions = {
renderCursorPointer?: boolean,
renderActive?: boolean,
renderStringsAsRegex?: boolean,
+ renderBoxes?: boolean,
};
function toInternalOptions(options: AriaTreeOptions): InternalOptions {
+ const renderBoxes = options.boxes;
if (options.mode === 'ai') {
// For AI consumption.
return {
@@ -63,18 +66,19 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions {
includeGenericRole: true,
renderActive: !options.doNotRenderActive,
renderCursorPointer: true,
+ renderBoxes,
};
}
if (options.mode === 'autoexpect') {
// To auto-generate assertions on visible elements.
- return { visibility: 'ariaAndVisible', refs: 'none' };
+ return { visibility: 'ariaAndVisible', refs: 'none', renderBoxes };
}
if (options.mode === 'codegen') {
// To generate aria assertion with regex heurisitcs.
- return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true };
+ return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true, renderBoxes };
}
// To match aria snapshot.
- return { visibility: 'aria', refs: 'none' };
+ return { visibility: 'aria', refs: 'none', renderBoxes };
}
export function generateAriaTree(rootElement: Element, publicOptions: AriaTreeOptions): AriaSnapshot {
@@ -624,6 +628,13 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTr
if (renderCursorPointer && aria.hasPointerCursor(ariaNode))
key += ' [cursor=pointer]';
}
+ if (options.renderBoxes) {
+ const element = ariaNodeElement(ariaNode);
+ if (element) {
+ const r = element.getBoundingClientRect();
+ key += ` [box=${Math.round(r.x)},${Math.round(r.y)},${Math.round(r.width)},${Math.round(r.height)}]`;
+ }
+ }
return key;
};
diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts
index 30e475fcde..452fbe353a 100644
--- a/packages/playwright-client/types/types.d.ts
+++ b/packages/playwright-client/types/types.d.ts
@@ -2051,6 +2051,11 @@ export interface Page {
* @param options
*/
ariaSnapshot(options?: {
+ /**
+ * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+ */
+ boxes?: boolean;
+
/**
* When specified, limits the depth of the snapshot.
*/
@@ -13057,6 +13062,11 @@ export interface Locator {
* @param options
*/
ariaSnapshot(options?: {
+ /**
+ * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+ */
+ boxes?: boolean;
+
/**
* When specified, limits the depth of the snapshot.
*/
diff --git a/packages/playwright-core/src/client/locator.ts b/packages/playwright-core/src/client/locator.ts
index 9de0f2b295..8dda1580c1 100644
--- a/packages/playwright-core/src/client/locator.ts
+++ b/packages/playwright-core/src/client/locator.ts
@@ -329,8 +329,8 @@ export class Locator implements api.Locator {
return ref ?? null;
}
- async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number } = {}): Promise<string> {
- const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth });
+ async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, boxes?: boolean } = {}): Promise<string> {
+ const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth, boxes: options.boxes });
return result.snapshot;
}
diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts
index 2d519d8095..1dc9817d36 100644
--- a/packages/playwright-core/src/client/page.ts
+++ b/packages/playwright-core/src/client/page.ts
@@ -866,8 +866,8 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
return result.pdf;
}
- async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, _track?: string } = {}): Promise<string> {
- const result = await this.mainFrame()._channel.ariaSnapshot({ timeout: this._timeoutSettings.timeout(options), track: options._track, mode: options.mode, depth: options.depth });
+ async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, boxes?: boolean, _track?: string } = {}): Promise<string> {
+ const result = await this.mainFrame()._channel.ariaSnapshot({ timeout: this._timeoutSettings.timeout(options), track: options._track, mode: options.mode, depth: options.depth, boxes: options.boxes });
return result.snapshot;
}
diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts
index 3c24734e63..d9eaec067b 100644
--- a/packages/playwright-core/src/protocol/validator.ts
+++ b/packages/playwright-core/src/protocol/validator.ts
@@ -1660,6 +1660,7 @@ scheme.FrameAriaSnapshotParams = tObject({
track: tOptional(tString),
selector: tOptional(tString),
depth: tOptional(tInt),
+ boxes: tOptional(tBoolean),
timeout: tFloat,
});
scheme.FrameAriaSnapshotResult = tObject({
diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts
index 90fafe6282..a298a0df92 100644
--- a/packages/playwright-core/src/server/frames.ts
+++ b/packages/playwright-core/src/server/frames.ts
@@ -1774,14 +1774,14 @@ export class Frame extends SdkObject<FrameEventMap> {
return { ref };
}
- async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number } = {}): Promise<{ snapshot: string }> {
+ async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number, boxes?: boolean } = {}): Promise<{ snapshot: string }> {
if (options.selector && options.track)
throw new Error('Cannot specify both selector and track options');
if (options.selector && options.mode !== 'ai') {
// Non-ai locator snapshot is auto-waiting and does not include iframes.
const snapshot = await this._retryWithProgressIfNotConnected(progress, options.selector, { strict: true, performActionPreChecks: true }, async (progress, handle) => {
- return await progress.race(handle.evaluateInUtility(([injected, element, opts]) => injected.ariaSnapshot(element, opts), { mode: 'default' as const, depth: options.depth }));
+ return await progress.race(handle.evaluateInUtility(([injected, element, opts]) => injected.ariaSnapshot(element, opts), { mode: 'default' as const, depth: options.depth, boxes: options.boxes }));
});
return { snapshot };
}
diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts
index 257b657ba3..281ad0a6a1 100644
--- a/packages/playwright-core/src/server/page.ts
+++ b/packages/playwright-core/src/server/page.ts
@@ -1061,7 +1061,7 @@ export class InitScript extends DisposableObject {
}
}
-export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Frame, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, info?: SelectorInfo, depth?: number } = {}): Promise<{ full: string[], incremental?: string[] }> {
+export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Frame, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, info?: SelectorInfo, depth?: number, boxes?: boolean } = {}): Promise<{ full: string[], incremental?: string[] }> {
// Only await the topmost navigations, inner frames will be empty when racing.
const snapshot = await frame.retryWithProgressAndTimeouts(progress, [1000, 2000, 4000, 8000], async (progress, continuePolling) => {
try {
@@ -1085,6 +1085,7 @@ export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Fra
doNotRenderActive: options.doNotRenderActive,
info: options.info,
depth: options.depth,
+ boxes: options.boxes,
}));
if (snapshotOrRetry === true)
return continuePolling;
diff --git a/packages/playwright-core/src/tools/backend/response.ts b/packages/playwright-core/src/tools/backend/response.ts
index 6b83adc25a..eda678a846 100644
--- a/packages/playwright-core/src/tools/backend/response.ts
+++ b/packages/playwright-core/src/tools/backend/response.ts
@@ -50,6 +50,7 @@ export class Response {
private _includeSnapshotFileName: string | undefined;
private _includeSnapshotRoot: playwright.Locator | undefined;
private _includeSnapshotDepth: number | undefined;
+ private _includeSnapshotBoxes: boolean | undefined;
private _isClose: boolean = false;
readonly toolName: string;
@@ -142,10 +143,11 @@ export class Response {
this._includeSnapshot = this._context.config.snapshot?.mode ?? 'full';
}
- setIncludeFullSnapshot(includeSnapshotFileName?: string, root?: playwright.Locator, depth?: number) {
+ setIncludeFullSnapshot(includeSnapshotFileName?: string, root?: playwright.Locator, depth?: number, boxes?: boolean) {
this._includeSnapshot = 'explicit';
this._includeSnapshotFileName = includeSnapshotFileName;
this._includeSnapshotDepth = depth;
+ this._includeSnapshotBoxes = boxes;
this._includeSnapshotRoot = root;
}
@@ -242,7 +244,7 @@ export class Response {
addSection('Ran Playwright code', this._code, 'js');
// Render tab titles upon changes or when more than one tab.
- const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotRoot, this._includeSnapshotDepth, this._clientWorkspace) : undefined;
+ const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotRoot, this._includeSnapshotDepth, this._includeSnapshotBoxes, this._clientWorkspace) : undefined;
const tabHeaders = await Promise.all(this._context.tabs().map(tab => tab.headerSnapshot()));
if (this._includeSnapshot !== 'none' || tabHeaders.some(header => header.changed)) {
if (tabHeaders.length !== 1)
diff --git a/packages/playwright-core/src/tools/backend/snapshot.ts b/packages/playwright-core/src/tools/backend/snapshot.ts
index 30730b33b4..516b17da43 100644
--- a/packages/playwright-core/src/tools/backend/snapshot.ts
+++ b/packages/playwright-core/src/tools/backend/snapshot.ts
@@ -42,6 +42,7 @@ const snapshot = defineTabTool({
target: z.string().optional().describe(elementTargetDescription),
filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'),
depth: z.number().optional().describe('Limit the depth of the snapshot tree'),
+ boxes: z.boolean().optional().describe('Include each element\'s bounding box as [box=x,y,width,height] in the snapshot'),
}),
type: 'readOnly',
},
@@ -50,7 +51,7 @@ const snapshot = defineTabTool({
let resolved: { locator: playwright.Locator | undefined, resolved: string } = { locator: undefined, resolved: '' };
if (params.target)
resolved = await tab.targetLocator({ target: params.target });
- response.setIncludeFullSnapshot(params.filename, resolved.locator, params.depth);
+ response.setIncludeFullSnapshot(params.filename, resolved.locator, params.depth, params.boxes);
},
});
diff --git a/packages/playwright-core/src/tools/backend/tab.ts b/packages/playwright-core/src/tools/backend/tab.ts
index 0284509303..e409fa344d 100644
--- a/packages/playwright-core/src/tools/backend/tab.ts
+++ b/packages/playwright-core/src/tools/backend/tab.ts
@@ -383,13 +383,13 @@ export class Tab extends EventEmitter<TabEventsInterface> {
this._requests.length = 0;
}
- async captureSnapshot(root: playwright.Locator | undefined, depth: number | undefined, relativeTo: string | undefined): Promise<TabSnapshot> {
+ async captureSnapshot(root: playwright.Locator | undefined, depth: number | undefined, boxes: boolean | undefined, relativeTo: string | undefined): Promise<TabSnapshot> {
await this._initializedPromise;
let tabSnapshot: TabSnapshot | undefined;
const modalStates = await this._raceAgainstModalStates(async () => {
const ariaSnapshot = root
- ? await root.ariaSnapshot({ mode: 'ai', depth })
- : await this.page.ariaSnapshot({ mode: 'ai', depth });
+ ? await root.ariaSnapshot({ mode: 'ai', depth, boxes })
+ : await this.page.ariaSnapshot({ mode: 'ai', depth, boxes });
tabSnapshot = {
ariaSnapshot,
modalStates: [],
diff --git a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md
index 66b5ad4acd..9e70ca9322 100644
--- a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md
+++ b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md
@@ -256,6 +256,9 @@ playwright-cli snapshot "#main"
# limit snapshot depth for efficiency, take a partial snapshot afterwards
playwright-cli snapshot --depth=4
playwright-cli snapshot e34
+
+# include each element's bounding box as [box=x,y,width,height]
+playwright-cli snapshot --boxes
```
## Targeting elements
diff --git a/packages/playwright-core/src/tools/cli-daemon/commands.ts b/packages/playwright-core/src/tools/cli-daemon/commands.ts
index d703773e87..594778e874 100644
--- a/packages/playwright-core/src/tools/cli-daemon/commands.ts
+++ b/packages/playwright-core/src/tools/cli-daemon/commands.ts
@@ -367,9 +367,10 @@ const snapshot = declareCommand({
options: z.object({
filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'),
depth: numberArg.optional().describe('Limit snapshot depth, unlimited by default.'),
+ boxes: z.boolean().optional().describe('Include each element\'s bounding box as [box=x,y,width,height] in the snapshot.'),
}),
toolName: 'browser_snapshot',
- toolParams: ({ filename, target, depth }) => ({ filename, target, depth }),
+ toolParams: ({ filename, target, depth, boxes }) => ({ filename, target, depth, boxes }),
});
const generateLocator = declareCommand({
diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts
index 30e475fcde..452fbe353a 100644
--- a/packages/playwright-core/types/types.d.ts
+++ b/packages/playwright-core/types/types.d.ts
@@ -2051,6 +2051,11 @@ export interface Page {
* @param options
*/
ariaSnapshot(options?: {
+ /**
+ * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+ */
+ boxes?: boolean;
+
/**
* When specified, limits the depth of the snapshot.
*/
@@ -13057,6 +13062,11 @@ export interface Locator {
* @param options
*/
ariaSnapshot(options?: {
+ /**
+ * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+ */
+ boxes?: boolean;
+
/**
* When specified, limits the depth of the snapshot.
*/
diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts
index d03bbcdd71..8125a83d92 100644
--- a/packages/protocol/src/channels.d.ts
+++ b/packages/protocol/src/channels.d.ts
@@ -2927,6 +2927,7 @@ export type FrameAriaSnapshotParams = {
track?: string,
selector?: string,
depth?: number,
+ boxes?: boolean,
timeout: number,
};
export type FrameAriaSnapshotOptions = {
@@ -2934,6 +2935,7 @@ export type FrameAriaSnapshotOptions = {
track?: string,
selector?: string,
depth?: number,
+ boxes?: boolean,
};
export type FrameAriaSnapshotResult = {
snapshot: string,
diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml
index 45970c3cc1..4ec62cb7f1 100644
--- a/packages/protocol/src/protocol.yml
+++ b/packages/protocol/src/protocol.yml
@@ -2324,6 +2324,7 @@ Frame:
track: string?
selector: string?
depth: int?
+ boxes: boolean?
timeout: float
returns:
snapshot: string
diff --git a/tests/mcp/cli-core.spec.ts b/tests/mcp/cli-core.spec.ts
index 89c472b887..24ef136ca5 100644
--- a/tests/mcp/cli-core.spec.ts
+++ b/tests/mcp/cli-core.spec.ts
@@ -310,6 +310,20 @@ test('snapshot depth', async ({ cli, server }) => {
- button "Cancel" [ref=e6]`);
});
+test('snapshot --boxes', async ({ cli, server }) => {
+ server.setContent('/', `
+ <style>body { margin: 0; }</style>
+ <button style="position:absolute;left:100px;top:50px;width:80px;height:40px;margin:0;padding:0;border:0;">click</button>
+ `, 'text/html');
+ await cli('open', server.PREFIX);
+
+ const { inlineSnapshot } = await cli('snapshot', '--boxes');
+ expect(inlineSnapshot).toContain(`- button "click" [ref=e1] [box=100,50,80,40]`);
+
+ const { inlineSnapshot: plain } = await cli('snapshot');
+ expect(plain).not.toMatch(/\[box=/);
+});
+
test('eval --raw', async ({ cli, server }) => {
await cli('open', server.HELLO_WORLD);
const { output } = await cli('eval', '--raw', '() => document.title');
diff --git a/tests/mcp/core.spec.ts b/tests/mcp/core.spec.ts
index 42490d8fd8..06423357b5 100644
--- a/tests/mcp/core.spec.ts
+++ b/tests/mcp/core.spec.ts
@@ -276,6 +276,31 @@ test('snapshot depth', async ({ client, server }) => {
});
});
+test('snapshot with boxes', async ({ client, server }) => {
+ server.setContent('/', `
+ <style>body { margin: 0; }</style>
+ <button style="position:absolute;left:100px;top:50px;width:80px;height:40px;margin:0;padding:0;border:0;">click</button>
+ `, 'text/html');
+
+ await client.callTool({
+ name: 'browser_navigate',
+ arguments: { url: server.PREFIX },
+ });
+
+ expect(await client.callTool({
+ name: 'browser_snapshot',
+ arguments: { boxes: true },
+ })).toHaveResponse({
+ inlineSnapshot: expect.stringContaining(`- button "click" [ref=e1] [box=100,50,80,40]`),
+ });
+
+ expect(await client.callTool({
+ name: 'browser_snapshot',
+ })).toHaveResponse({
+ inlineSnapshot: expect.not.stringMatching(/\[box=/),
+ });
+});
+
test('snapshot by ref', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright-cli/issues/347' } }, async ({ client, server }) => {
server.setContent('/', `
<ul>
diff --git a/tests/page/page-aria-snapshot.spec.ts b/tests/page/page-aria-snapshot.spec.ts
index 8b6bcf19e1..da9708d56c 100644
--- a/tests/page/page-aria-snapshot.spec.ts
+++ b/tests/page/page-aria-snapshot.spec.ts
@@ -748,3 +748,29 @@ it('should snapshot a locator inside an iframe', async ({ page }) => {
- listitem: Item 2
`);
});
+
+it('should snapshot with box from page', async ({ page }) => {
+ await page.setContent(`
+ <button style="position:absolute;left:100px;top:50px;width:80px;height:40px;margin:0;padding:0;border:0;">click</button>
+ `);
+
+ const snapshot = await page.ariaSnapshot({ boxes: true });
+ expect(snapshot).toBe(`- button "click" [box=100,50,80,40]`);
+});
+
+it('should snapshot with box from locator', async ({ page }) => {
+ await page.setContent(`
+ <div style="position:absolute;left:10px;top:20px;width:200px;height:100px;">
+ <button style="position:absolute;left:5px;top:5px;width:60px;height:30px;margin:0;padding:0;border:0;">ok</button>
+ </div>
+ `);
+
+ const snapshot = await page.locator('div').ariaSnapshot({ boxes: true });
+ expect(snapshot).toBe(`- button "ok" [box=15,25,60,30]`);
+});
+
+it('should not include box when option is omitted', async ({ page }) => {
+ await page.setContent(`<button>click</button>`);
+ const snapshot = await page.ariaSnapshot();
+ expect(snapshot).not.toMatch(/\[box=/);
+});