Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: upstream the frame tree snapshot (#35917)

Pavel Feldmana6b4b222ed67
diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.ts
index 923f6cf23b..a75a5cd185 100644
--- a/packages/injected/src/ariaSnapshot.ts
+++ b/packages/injected/src/ariaSnapshot.ts
@@ -26,7 +26,7 @@ import type { Box } from './domUtils';
export type AriaNode = AriaProps & {
role: AriaRole | 'fragment' | 'iframe';
name: string;
- ref?: number;
+ ref?: string;
children: (AriaNode | string)[];
element: Element;
box: Box;
@@ -36,23 +36,23 @@ export type AriaNode = AriaProps & {
export type AriaSnapshot = {
root: AriaNode;
- elements: Map<number, Element>;
+ elements: Map<string, Element>;
};
type AriaRef = {
role: string;
name: string;
- ref: number;
+ ref: string;
};
let lastRef = 0;
-export function generateAriaTree(rootElement: Element, options?: { forAI?: boolean }): AriaSnapshot {
+export function generateAriaTree(rootElement: Element, options?: { forAI?: boolean, refPrefix?: string }): AriaSnapshot {
const visited = new Set<Node>();
const snapshot: AriaSnapshot = {
root: { role: 'fragment', name: '', children: [], element: rootElement, props: {}, box: box(rootElement), receivesPointerEvents: true },
- elements: new Map<number, Element>(),
+ elements: new Map<string, Element>(),
};
const visit = (ariaNode: AriaNode, node: Node) => {
@@ -149,20 +149,20 @@ export function generateAriaTree(rootElement: Element, options?: { forAI?: boole
return snapshot;
}
-function ariaRef(element: Element, role: string, name: string, options?: { forAI?: boolean }): number | undefined {
+function ariaRef(element: Element, role: string, name: string, options?: { forAI?: boolean, refPrefix?: string }): string | undefined {
if (!options?.forAI)
return undefined;
let ariaRef: AriaRef | undefined;
ariaRef = (element as any)._ariaRef;
if (!ariaRef || ariaRef.role !== role || ariaRef.name !== name) {
- ariaRef = { role, name, ref: ++lastRef };
+ ariaRef = { role, name, ref: (options?.refPrefix ?? '') + 'e' + (++lastRef) };
(element as any)._ariaRef = ariaRef;
}
return ariaRef.ref;
}
-function toAriaNode(element: Element, options?: { forAI?: boolean }): AriaNode | null {
+function toAriaNode(element: Element, options?: { forAI?: boolean, refPrefix?: string }): AriaNode | null {
if (element.nodeName === 'IFRAME') {
return {
role: 'iframe',
@@ -443,7 +443,7 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, options?: { mode?: 'r
const ref = ariaNode.ref;
const cursor = hasPointerCursor(ariaNode) ? ' [cursor=pointer]' : '';
if (ref)
- key += ` [ref=e${ref}]${cursor}`;
+ key += ` [ref=${ref}]${cursor}`;
}
const escapedKey = indent + '- ' + yamlEscapeKeyIfNeeded(key);
diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.ts
index 0b77bd97ae..f276fdec5e 100644
--- a/packages/injected/src/injectedScript.ts
+++ b/packages/injected/src/injectedScript.ts
@@ -297,7 +297,7 @@ export class InjectedScript {
return new Set<Element>(result.map(r => r.element));
}
- ariaSnapshot(node: Node, options?: { mode?: 'raw' | 'regex', forAI?: boolean }): string {
+ ariaSnapshot(node: Node, options?: { mode?: 'raw' | 'regex', forAI?: boolean, refPrefix?: string }): string {
if (node.nodeType !== Node.ELEMENT_NODE)
throw this.createStacklessError('Can only capture aria snapshot of Element nodes.');
this._lastAriaSnapshot = generateAriaTree(node as Element, options);
@@ -675,10 +675,7 @@ export class InjectedScript {
_createAriaRefEngine() {
const queryAll = (root: SelectorRoot, selector: string): Element[] => {
- if (!selector.startsWith('e'))
- throw this.createStacklessError(`Invalid aria-ref selector "${selector}"`);
- const ref = +selector.substring(1);
- const result = this._lastAriaSnapshot?.elements?.get(ref);
+ const result = this._lastAriaSnapshot?.elements?.get(selector);
return result && result.isConnected ? [result] : [];
};
return { queryAll };
diff --git a/packages/playwright-core/src/client/locator.ts b/packages/playwright-core/src/client/locator.ts
index be42ffc60e..9a25653f7f 100644
--- a/packages/playwright-core/src/client/locator.ts
+++ b/packages/playwright-core/src/client/locator.ts
@@ -303,8 +303,8 @@ export class Locator implements api.Locator {
return await this._withElement((h, timeout) => h.screenshot({ ...options, mask, timeout }), options.timeout);
}
- async ariaSnapshot(options?: { _forAI?: boolean } & TimeoutOptions): Promise<string> {
- const result = await this._frame._channel.ariaSnapshot({ ...options, forAI: options?._forAI, selector: this._selector });
+ async ariaSnapshot(options?: TimeoutOptions): Promise<string> {
+ const result = await this._frame._channel.ariaSnapshot({ ...options, selector: this._selector });
return result.snapshot;
}
diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts
index b91b0b53cd..bdc36d3a6a 100644
--- a/packages/playwright-core/src/client/page.ts
+++ b/packages/playwright-core/src/client/page.ts
@@ -832,6 +832,11 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
}
return result.pdf;
}
+
+ async _snapshotForAI(): Promise<string> {
+ const result = await this._channel.snapshotForAI();
+ return result.snapshot;
+ }
}
export class BindingCall extends ChannelOwner<channels.BindingCallChannel> {
diff --git a/packages/playwright-core/src/protocol/debug.ts b/packages/playwright-core/src/protocol/debug.ts
index c58c3e4aaf..032e64c516 100644
--- a/packages/playwright-core/src/protocol/debug.ts
+++ b/packages/playwright-core/src/protocol/debug.ts
@@ -90,6 +90,7 @@ export const commandsWithTracingSnapshots = new Set([
'Page.mouseWheel',
'Page.touchscreenTap',
'Page.accessibilitySnapshot',
+ 'Page.snapshotForAI',
'Frame.evalOnSelector',
'Frame.evalOnSelectorAll',
'Frame.addScriptTag',
diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts
index 361742b8f6..aa8082bf54 100644
--- a/packages/playwright-core/src/protocol/validator.ts
+++ b/packages/playwright-core/src/protocol/validator.ts
@@ -1386,6 +1386,10 @@ scheme.PagePdfParams = tObject({
scheme.PagePdfResult = tObject({
pdf: tBinary,
});
+scheme.PageSnapshotForAIParams = tOptional(tObject({}));
+scheme.PageSnapshotForAIResult = tObject({
+ snapshot: tString,
+});
scheme.PageStartJSCoverageParams = tObject({
resetOnNavigation: tOptional(tBoolean),
reportAnonymousScripts: tOptional(tBoolean),
diff --git a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts
index bcb37e9ace..1c308a6d37 100644
--- a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts
+++ b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts
@@ -295,6 +295,10 @@ export class PageDispatcher extends Dispatcher<Page, channels.PageChannel, Brows
return { pdf: buffer };
}
+ async snapshotForAI(params: channels.PageSnapshotForAIParams, metadata: CallMetadata): Promise<channels.PageSnapshotForAIResult> {
+ return { snapshot: await this._page.snapshotForAI(metadata) };
+ }
+
async bringToFront(params: channels.PageBringToFrontParams, metadata: CallMetadata): Promise<void> {
await this._page.bringToFront();
}
diff --git a/packages/playwright-core/src/server/dom.ts b/packages/playwright-core/src/server/dom.ts
index a047680092..afcf0a087d 100644
--- a/packages/playwright-core/src/server/dom.ts
+++ b/packages/playwright-core/src/server/dom.ts
@@ -811,7 +811,7 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
return this._page.delegate.getBoundingBox(this);
}
- async ariaSnapshot(options: { forAI?: boolean }): Promise<string> {
+ async ariaSnapshot(options?: { forAI?: boolean, refPrefix?: string }): Promise<string> {
return await this.evaluateInUtility(([injected, element, options]) => injected.ariaSnapshot(element, options), options);
}
diff --git a/packages/playwright-core/src/server/frameSelectors.ts b/packages/playwright-core/src/server/frameSelectors.ts
index 90f24ca2f6..cf8e5eb971 100644
--- a/packages/playwright-core/src/server/frameSelectors.ts
+++ b/packages/playwright-core/src/server/frameSelectors.ts
@@ -111,7 +111,31 @@ export class FrameSelectors {
return Promise.all(result);
}
+ private _resolveForLastAISnapshot(selector: string, options: types.StrictOptions = {}): SelectorInFrame | null {
+ const match = selector.match(/^aria-ref=f(\d+)e\d+$/);
+ if (!match)
+ return null;
+
+ const frameIndex = +match[1];
+ const page = this.frame._page;
+ const frameId = page.lastSnapshotFrameIds[frameIndex - 1];
+ if (!frameId)
+ return null;
+ const frameManager = page.frameManager;
+ const frame = frameManager.frame(frameId);
+ if (!frame)
+ return null;
+ return {
+ frame,
+ info: frame.selectors._parseSelector(selector, options),
+ };
+ }
+
async resolveFrameForSelector(selector: string, options: types.StrictOptions = {}, scope?: ElementHandle): Promise<SelectorInFrame | null> {
+ const resolvedForSnapshot = this._resolveForLastAISnapshot(selector, options);
+ if (resolvedForSnapshot)
+ return resolvedForSnapshot;
+
let frame: Frame = this.frame;
const frameChunks = splitSelectorByFrame(selector);
diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts
index 487c2a700b..5ada5e2ab8 100644
--- a/packages/playwright-core/src/server/page.ts
+++ b/packages/playwright-core/src/server/page.ts
@@ -175,6 +175,7 @@ export class Page extends SdkObject {
// When throttling for tracing, 200ms between frames, except for 10 frames around the action.
private _frameThrottler = new FrameThrottler(10, 35, 200);
closeReason: string | undefined;
+ lastSnapshotFrameIds: string[] = [];
constructor(delegate: PageDelegate, browserContext: BrowserContext) {
super(browserContext, 'page');
@@ -807,6 +808,13 @@ export class Page extends SdkObject {
markAsServerSideOnly() {
this._isServerSideOnly = true;
}
+
+ async snapshotForAI(metadata: CallMetadata): Promise<string> {
+ const frameIds: string[] = [];
+ const snapshot = await snapshotFrameForAI(this.mainFrame(), 0, frameIds);
+ this.lastSnapshotFrameIds = frameIds;
+ return snapshot.join('\n');
+ }
}
export class Worker extends SdkObject {
@@ -988,3 +996,36 @@ class FrameThrottler {
}
}
}
+
+async function snapshotFrameForAI(frame: frames.Frame, frameOrdinal: number, frameIds: string[]): Promise<string[]> {
+ const context = await frame._utilityContext();
+ const injectedScript = await context.injectedScript();
+ const snapshot = await injectedScript.evaluate((injected, refPrefix) => {
+ return injected.ariaSnapshot(injected.document.body, { forAI: true, refPrefix });
+ }, frameOrdinal ? 'f' + frameOrdinal : '');
+
+ const lines = snapshot.split('\n');
+ const result = [];
+ for (const line of lines) {
+ const match = line.match(/^(\s*)- iframe \[ref=(.*)\]/);
+ if (!match) {
+ result.push(line);
+ continue;
+ }
+
+ const leadingSpace = match[1];
+ const ref = match[2];
+ const frameSelector = `aria-ref=${ref} >> internal:control=enter-frame`;
+ const frameBodySelector = `${frameSelector} >> body`;
+ const child = await frame.selectors.resolveFrameForSelector(frameBodySelector, { strict: true });
+ if (!child) {
+ result.push(line);
+ continue;
+ }
+ const frameOrdinal = frameIds.length + 1;
+ frameIds.push(child.frame._id);
+ const childSnapshot = await snapshotFrameForAI(child.frame, frameOrdinal, frameIds);
+ result.push(line + ':', ...childSnapshot.map(l => leadingSpace + ' ' + l));
+ }
+ return result;
+}
diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts
index 0b7fa280e6..62ea817120 100644
--- a/packages/protocol/src/channels.d.ts
+++ b/packages/protocol/src/channels.d.ts
@@ -2057,6 +2057,7 @@ export interface PageChannel extends PageEventTarget, EventTargetChannel {
touchscreenTap(params: PageTouchscreenTapParams, metadata?: CallMetadata): Promise<PageTouchscreenTapResult>;
accessibilitySnapshot(params: PageAccessibilitySnapshotParams, metadata?: CallMetadata): Promise<PageAccessibilitySnapshotResult>;
pdf(params: PagePdfParams, metadata?: CallMetadata): Promise<PagePdfResult>;
+ snapshotForAI(params?: PageSnapshotForAIParams, metadata?: CallMetadata): Promise<PageSnapshotForAIResult>;
startJSCoverage(params: PageStartJSCoverageParams, metadata?: CallMetadata): Promise<PageStartJSCoverageResult>;
stopJSCoverage(params?: PageStopJSCoverageParams, metadata?: CallMetadata): Promise<PageStopJSCoverageResult>;
startCSSCoverage(params: PageStartCSSCoverageParams, metadata?: CallMetadata): Promise<PageStartCSSCoverageResult>;
@@ -2497,6 +2498,11 @@ export type PagePdfOptions = {
export type PagePdfResult = {
pdf: Binary,
};
+export type PageSnapshotForAIParams = {};
+export type PageSnapshotForAIOptions = {};
+export type PageSnapshotForAIResult = {
+ snapshot: string,
+};
export type PageStartJSCoverageParams = {
resetOnNavigation?: boolean,
reportAnonymousScripts?: boolean,
diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml
index 94553efd79..2e83f20f94 100644
--- a/packages/protocol/src/protocol.yml
+++ b/packages/protocol/src/protocol.yml
@@ -1795,6 +1795,12 @@ Page:
returns:
pdf: binary
+ snapshotForAI:
+ returns:
+ snapshot: string
+ flags:
+ snapshot: true
+
startJSCoverage:
parameters:
resetOnNavigation: boolean?
diff --git a/tests/page/page-aria-snapshot-ai.spec.ts b/tests/page/page-aria-snapshot-ai.spec.ts
index 16872d2b5f..1491883b62 100644
--- a/tests/page/page-aria-snapshot-ai.spec.ts
+++ b/tests/page/page-aria-snapshot-ai.spec.ts
@@ -14,10 +14,11 @@
* limitations under the License.
*/
-import type { FrameLocator, Page } from '@playwright/test';
import { test as it, expect } from './pageTest';
-const forAI = { _forAI: true } as any;
+function snapshotForAI(page: any): Promise<string> {
+ return page._snapshotForAI();
+}
it('should generate refs', async ({ page }) => {
await page.setContent(`
@@ -26,7 +27,7 @@ it('should generate refs', async ({ page }) => {
<button>Three</button>
`);
- const snapshot1 = await page.locator('body').ariaSnapshot(forAI);
+ const snapshot1 = await snapshotForAI(page);
expect(snapshot1).toContainYaml(`
- generic [ref=e1]:
- button "One" [ref=e2]
@@ -41,7 +42,7 @@ it('should generate refs', async ({ page }) => {
e.textContent = 'Not Two';
});
- const snapshot2 = await page.locator('body').ariaSnapshot(forAI);
+ const snapshot2 = await snapshotForAI(page);
expect(snapshot2).toContainYaml(`
- generic [ref=e1]:
- button "One" [ref=e2]
@@ -56,47 +57,36 @@ it('should list iframes', async ({ page }) => {
<iframe name="foo" src="data:text/html,<h1>World</h1>">
`);
- const snapshot1 = await page.locator('body').ariaSnapshot(forAI);
+ const snapshot1 = await snapshotForAI(page);
expect(snapshot1).toContain('- iframe');
const frameSnapshot = await page.frameLocator(`iframe`).locator('body').ariaSnapshot();
expect(frameSnapshot).toEqual('- heading "World" [level=1]');
});
-it('ref mode can be used to stitch all frame snapshots', async ({ page, server }) => {
+it('should stitch all frame snapshots', async ({ page, server }) => {
await page.goto(server.PREFIX + '/frames/nested-frames.html');
-
- async function allFrameSnapshot(frame: Page | FrameLocator): Promise<string> {
- const snapshot = await frame.locator('body').ariaSnapshot(forAI);
- const lines = snapshot.split('\n');
- const result = [];
- for (const line of lines) {
- const match = line.match(/^(\s*)- iframe \[ref=(.*)\]/);
- if (!match) {
- result.push(line);
- continue;
- }
-
- const leadingSpace = match[1];
- const ref = match[2];
- const childFrame = frame.frameLocator(`aria-ref=${ref}`);
- const childSnapshot = await allFrameSnapshot(childFrame);
- result.push(line + ':', childSnapshot.split('\n').map(l => leadingSpace + ' ' + l).join('\n'));
- }
- return result.join('\n');
- }
-
- expect(await allFrameSnapshot(page)).toContainYaml(`
+ const snapshot = await snapshotForAI(page);
+ expect(snapshot).toContainYaml(`
- generic [ref=e1]:
- iframe [ref=e2]:
- - generic [ref=e1]:
- - iframe [ref=e2]:
- - generic [ref=e2]: Hi, I'm frame
- - iframe [ref=e3]:
- - generic [ref=e2]: Hi, I'm frame
+ - generic [ref=f1e1]:
+ - iframe [ref=f1e2]:
+ - generic [ref=f2e2]: Hi, I'm frame
+ - iframe [ref=f1e3]:
+ - generic [ref=f3e2]: Hi, I'm frame
- iframe [ref=e3]:
- - generic [ref=e2]: Hi, I'm frame
+ - generic [ref=f4e2]: Hi, I'm frame
`);
+
+ const href = await page.locator('aria-ref=e1').evaluate(e => e.ownerDocument.defaultView.location.href);
+ expect(href).toBe(server.PREFIX + '/frames/nested-frames.html');
+
+ const href2 = await page.locator('aria-ref=f1e2').evaluate(e => e.ownerDocument.defaultView.location.href);
+ expect(href2).toBe(server.PREFIX + '/frames/two-frames.html');
+
+ const href3 = await page.locator('aria-ref=f3e2').evaluate(e => e.ownerDocument.defaultView.location.href);
+ expect(href3).toBe(server.PREFIX + '/frames/frame.html');
});
it('should not generate refs for hidden elements', async ({ page }) => {
@@ -106,7 +96,7 @@ it('should not generate refs for hidden elements', async ({ page }) => {
<button>Three</button>
`);
- const snapshot = await page.locator('body').ariaSnapshot(forAI);
+ const snapshot = await snapshotForAI(page);
expect(snapshot).toContainYaml(`
- generic [ref=e1]:
- button "One" [ref=e2]
@@ -138,7 +128,7 @@ it('should not generate refs for elements with pointer-events:none', async ({ pa
</div>
`);
- const snapshot = await page.locator('body').ariaSnapshot(forAI);
+ const snapshot = await snapshotForAI(page);
expect(snapshot).toContainYaml(`
- generic [ref=e1]:
- button "no-ref"
@@ -182,7 +172,7 @@ it('emit generic roles for nodes w/o roles', async ({ page }) => {
</div>
`);
- const snapshot = await page.locator('body').ariaSnapshot(forAI);
+ const snapshot = await snapshotForAI(page);
expect(snapshot).toContainYaml(`
- generic [ref=e2]:
@@ -212,7 +202,7 @@ it('should collapse generic nodes', async ({ page }) => {
</div>
`);
- const snapshot = await page.locator('body').ariaSnapshot(forAI);
+ const snapshot = await snapshotForAI(page);
expect(snapshot).toContainYaml(`
- button \"Button\" [ref=e5]
`);
@@ -223,7 +213,7 @@ it('should include cursor pointer hint', async ({ page }) => {
<button style="cursor: pointer">Button</button>
`);
- const snapshot = await page.locator('body').ariaSnapshot(forAI);
+ const snapshot = await snapshotForAI(page);
expect(snapshot).toContainYaml(`
- button \"Button\" [ref=e2] [cursor=pointer]
`);