Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: upstream the frame tree snapshot (#35917)
Pavel Feldmana6b4b222ed67
diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.tsindex 923f6cf23b..a75a5cd185 100644--- a/packages/injected/src/ariaSnapshot.ts+++ b/packages/injected/src/ariaSnapshot.ts@@ -26,7 +26,7 @@ import type { Box } from './domUtils';export type AriaNode = AriaProps & {role: AriaRole | 'fragment' | 'iframe';name: string;- ref?: number;+ ref?: string;children: (AriaNode | string)[];element: Element;box: Box;@@ -36,23 +36,23 @@ export type AriaNode = AriaProps & {export type AriaSnapshot = {root: AriaNode;- elements: Map<number, Element>;+ elements: Map<string, Element>;};type AriaRef = {role: string;name: string;- ref: number;+ ref: string;};let lastRef = 0;-export function generateAriaTree(rootElement: Element, options?: { forAI?: boolean }): AriaSnapshot {+export function generateAriaTree(rootElement: Element, options?: { forAI?: boolean, refPrefix?: string }): AriaSnapshot {const visited = new Set<Node>();const snapshot: AriaSnapshot = {root: { role: 'fragment', name: '', children: [], element: rootElement, props: {}, box: box(rootElement), receivesPointerEvents: true },- elements: new Map<number, Element>(),+ elements: new Map<string, Element>(),};const visit = (ariaNode: AriaNode, node: Node) => {@@ -149,20 +149,20 @@ export function generateAriaTree(rootElement: Element, options?: { forAI?: boolereturn snapshot;}-function ariaRef(element: Element, role: string, name: string, options?: { forAI?: boolean }): number | undefined {+function ariaRef(element: Element, role: string, name: string, options?: { forAI?: boolean, refPrefix?: string }): string | undefined {if (!options?.forAI)return undefined;let ariaRef: AriaRef | undefined;ariaRef = (element as any)._ariaRef;if (!ariaRef || ariaRef.role !== role || ariaRef.name !== name) {- ariaRef = { role, name, ref: ++lastRef };+ ariaRef = { role, name, ref: (options?.refPrefix ?? '') + 'e' + (++lastRef) };(element as any)._ariaRef = ariaRef;}return ariaRef.ref;}-function toAriaNode(element: Element, options?: { forAI?: boolean }): AriaNode | null {+function toAriaNode(element: Element, options?: { forAI?: boolean, refPrefix?: string }): AriaNode | null {if (element.nodeName === 'IFRAME') {return {role: 'iframe',@@ -443,7 +443,7 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, options?: { mode?: 'rconst ref = ariaNode.ref;const cursor = hasPointerCursor(ariaNode) ? ' [cursor=pointer]' : '';if (ref)- key += ` [ref=e${ref}]${cursor}`;+ key += ` [ref=${ref}]${cursor}`;}const escapedKey = indent + '- ' + yamlEscapeKeyIfNeeded(key);diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.tsindex 0b77bd97ae..f276fdec5e 100644--- a/packages/injected/src/injectedScript.ts+++ b/packages/injected/src/injectedScript.ts@@ -297,7 +297,7 @@ export class InjectedScript {return new Set<Element>(result.map(r => r.element));}- ariaSnapshot(node: Node, options?: { mode?: 'raw' | 'regex', forAI?: boolean }): string {+ ariaSnapshot(node: Node, options?: { mode?: 'raw' | 'regex', forAI?: boolean, refPrefix?: string }): string {if (node.nodeType !== Node.ELEMENT_NODE)throw this.createStacklessError('Can only capture aria snapshot of Element nodes.');this._lastAriaSnapshot = generateAriaTree(node as Element, options);@@ -675,10 +675,7 @@ export class InjectedScript {_createAriaRefEngine() {const queryAll = (root: SelectorRoot, selector: string): Element[] => {- if (!selector.startsWith('e'))- throw this.createStacklessError(`Invalid aria-ref selector "${selector}"`);- const ref = +selector.substring(1);- const result = this._lastAriaSnapshot?.elements?.get(ref);+ const result = this._lastAriaSnapshot?.elements?.get(selector);return result && result.isConnected ? [result] : [];};return { queryAll };diff --git a/packages/playwright-core/src/client/locator.ts b/packages/playwright-core/src/client/locator.tsindex be42ffc60e..9a25653f7f 100644--- a/packages/playwright-core/src/client/locator.ts+++ b/packages/playwright-core/src/client/locator.ts@@ -303,8 +303,8 @@ export class Locator implements api.Locator {return await this._withElement((h, timeout) => h.screenshot({ ...options, mask, timeout }), options.timeout);}- async ariaSnapshot(options?: { _forAI?: boolean } & TimeoutOptions): Promise<string> {- const result = await this._frame._channel.ariaSnapshot({ ...options, forAI: options?._forAI, selector: this._selector });+ async ariaSnapshot(options?: TimeoutOptions): Promise<string> {+ const result = await this._frame._channel.ariaSnapshot({ ...options, selector: this._selector });return result.snapshot;}diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.tsindex b91b0b53cd..bdc36d3a6a 100644--- a/packages/playwright-core/src/client/page.ts+++ b/packages/playwright-core/src/client/page.ts@@ -832,6 +832,11 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page}return result.pdf;}++ async _snapshotForAI(): Promise<string> {+ const result = await this._channel.snapshotForAI();+ return result.snapshot;+ }}export class BindingCall extends ChannelOwner<channels.BindingCallChannel> {diff --git a/packages/playwright-core/src/protocol/debug.ts b/packages/playwright-core/src/protocol/debug.tsindex c58c3e4aaf..032e64c516 100644--- a/packages/playwright-core/src/protocol/debug.ts+++ b/packages/playwright-core/src/protocol/debug.ts@@ -90,6 +90,7 @@ export const commandsWithTracingSnapshots = new Set(['Page.mouseWheel','Page.touchscreenTap','Page.accessibilitySnapshot',+ 'Page.snapshotForAI','Frame.evalOnSelector','Frame.evalOnSelectorAll','Frame.addScriptTag',diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.tsindex 361742b8f6..aa8082bf54 100644--- a/packages/playwright-core/src/protocol/validator.ts+++ b/packages/playwright-core/src/protocol/validator.ts@@ -1386,6 +1386,10 @@ scheme.PagePdfParams = tObject({scheme.PagePdfResult = tObject({pdf: tBinary,});+scheme.PageSnapshotForAIParams = tOptional(tObject({}));+scheme.PageSnapshotForAIResult = tObject({+ snapshot: tString,+});scheme.PageStartJSCoverageParams = tObject({resetOnNavigation: tOptional(tBoolean),reportAnonymousScripts: tOptional(tBoolean),diff --git a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts b/packages/playwright-core/src/server/dispatchers/pageDispatcher.tsindex bcb37e9ace..1c308a6d37 100644--- a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts+++ b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts@@ -295,6 +295,10 @@ export class PageDispatcher extends Dispatcher<Page, channels.PageChannel, Browsreturn { pdf: buffer };}+ async snapshotForAI(params: channels.PageSnapshotForAIParams, metadata: CallMetadata): Promise<channels.PageSnapshotForAIResult> {+ return { snapshot: await this._page.snapshotForAI(metadata) };+ }+async bringToFront(params: channels.PageBringToFrontParams, metadata: CallMetadata): Promise<void> {await this._page.bringToFront();}diff --git a/packages/playwright-core/src/server/dom.ts b/packages/playwright-core/src/server/dom.tsindex a047680092..afcf0a087d 100644--- a/packages/playwright-core/src/server/dom.ts+++ b/packages/playwright-core/src/server/dom.ts@@ -811,7 +811,7 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {return this._page.delegate.getBoundingBox(this);}- async ariaSnapshot(options: { forAI?: boolean }): Promise<string> {+ async ariaSnapshot(options?: { forAI?: boolean, refPrefix?: string }): Promise<string> {return await this.evaluateInUtility(([injected, element, options]) => injected.ariaSnapshot(element, options), options);}diff --git a/packages/playwright-core/src/server/frameSelectors.ts b/packages/playwright-core/src/server/frameSelectors.tsindex 90f24ca2f6..cf8e5eb971 100644--- a/packages/playwright-core/src/server/frameSelectors.ts+++ b/packages/playwright-core/src/server/frameSelectors.ts@@ -111,7 +111,31 @@ export class FrameSelectors {return Promise.all(result);}+ private _resolveForLastAISnapshot(selector: string, options: types.StrictOptions = {}): SelectorInFrame | null {+ const match = selector.match(/^aria-ref=f(\d+)e\d+$/);+ if (!match)+ return null;++ const frameIndex = +match[1];+ const page = this.frame._page;+ const frameId = page.lastSnapshotFrameIds[frameIndex - 1];+ if (!frameId)+ return null;+ const frameManager = page.frameManager;+ const frame = frameManager.frame(frameId);+ if (!frame)+ return null;+ return {+ frame,+ info: frame.selectors._parseSelector(selector, options),+ };+ }+async resolveFrameForSelector(selector: string, options: types.StrictOptions = {}, scope?: ElementHandle): Promise<SelectorInFrame | null> {+ const resolvedForSnapshot = this._resolveForLastAISnapshot(selector, options);+ if (resolvedForSnapshot)+ return resolvedForSnapshot;+let frame: Frame = this.frame;const frameChunks = splitSelectorByFrame(selector);diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.tsindex 487c2a700b..5ada5e2ab8 100644--- a/packages/playwright-core/src/server/page.ts+++ b/packages/playwright-core/src/server/page.ts@@ -175,6 +175,7 @@ export class Page extends SdkObject {// When throttling for tracing, 200ms between frames, except for 10 frames around the action.private _frameThrottler = new FrameThrottler(10, 35, 200);closeReason: string | undefined;+ lastSnapshotFrameIds: string[] = [];constructor(delegate: PageDelegate, browserContext: BrowserContext) {super(browserContext, 'page');@@ -807,6 +808,13 @@ export class Page extends SdkObject {markAsServerSideOnly() {this._isServerSideOnly = true;}++ async snapshotForAI(metadata: CallMetadata): Promise<string> {+ const frameIds: string[] = [];+ const snapshot = await snapshotFrameForAI(this.mainFrame(), 0, frameIds);+ this.lastSnapshotFrameIds = frameIds;+ return snapshot.join('\n');+ }}export class Worker extends SdkObject {@@ -988,3 +996,36 @@ class FrameThrottler {}}}++async function snapshotFrameForAI(frame: frames.Frame, frameOrdinal: number, frameIds: string[]): Promise<string[]> {+ const context = await frame._utilityContext();+ const injectedScript = await context.injectedScript();+ const snapshot = await injectedScript.evaluate((injected, refPrefix) => {+ return injected.ariaSnapshot(injected.document.body, { forAI: true, refPrefix });+ }, frameOrdinal ? 'f' + frameOrdinal : '');++ const lines = snapshot.split('\n');+ const result = [];+ for (const line of lines) {+ const match = line.match(/^(\s*)- iframe \[ref=(.*)\]/);+ if (!match) {+ result.push(line);+ continue;+ }++ const leadingSpace = match[1];+ const ref = match[2];+ const frameSelector = `aria-ref=${ref} >> internal:control=enter-frame`;+ const frameBodySelector = `${frameSelector} >> body`;+ const child = await frame.selectors.resolveFrameForSelector(frameBodySelector, { strict: true });+ if (!child) {+ result.push(line);+ continue;+ }+ const frameOrdinal = frameIds.length + 1;+ frameIds.push(child.frame._id);+ const childSnapshot = await snapshotFrameForAI(child.frame, frameOrdinal, frameIds);+ result.push(line + ':', ...childSnapshot.map(l => leadingSpace + ' ' + l));+ }+ return result;+}diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.tsindex 0b7fa280e6..62ea817120 100644--- a/packages/protocol/src/channels.d.ts+++ b/packages/protocol/src/channels.d.ts@@ -2057,6 +2057,7 @@ export interface PageChannel extends PageEventTarget, EventTargetChannel {touchscreenTap(params: PageTouchscreenTapParams, metadata?: CallMetadata): Promise<PageTouchscreenTapResult>;accessibilitySnapshot(params: PageAccessibilitySnapshotParams, metadata?: CallMetadata): Promise<PageAccessibilitySnapshotResult>;pdf(params: PagePdfParams, metadata?: CallMetadata): Promise<PagePdfResult>;+ snapshotForAI(params?: PageSnapshotForAIParams, metadata?: CallMetadata): Promise<PageSnapshotForAIResult>;startJSCoverage(params: PageStartJSCoverageParams, metadata?: CallMetadata): Promise<PageStartJSCoverageResult>;stopJSCoverage(params?: PageStopJSCoverageParams, metadata?: CallMetadata): Promise<PageStopJSCoverageResult>;startCSSCoverage(params: PageStartCSSCoverageParams, metadata?: CallMetadata): Promise<PageStartCSSCoverageResult>;@@ -2497,6 +2498,11 @@ export type PagePdfOptions = {export type PagePdfResult = {pdf: Binary,};+export type PageSnapshotForAIParams = {};+export type PageSnapshotForAIOptions = {};+export type PageSnapshotForAIResult = {+ snapshot: string,+};export type PageStartJSCoverageParams = {resetOnNavigation?: boolean,reportAnonymousScripts?: boolean,diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.ymlindex 94553efd79..2e83f20f94 100644--- a/packages/protocol/src/protocol.yml+++ b/packages/protocol/src/protocol.yml@@ -1795,6 +1795,12 @@ Page:returns:pdf: binary+ snapshotForAI:+ returns:+ snapshot: string+ flags:+ snapshot: true+startJSCoverage:parameters:resetOnNavigation: boolean?diff --git a/tests/page/page-aria-snapshot-ai.spec.ts b/tests/page/page-aria-snapshot-ai.spec.tsindex 16872d2b5f..1491883b62 100644--- a/tests/page/page-aria-snapshot-ai.spec.ts+++ b/tests/page/page-aria-snapshot-ai.spec.ts@@ -14,10 +14,11 @@* limitations under the License.*/-import type { FrameLocator, Page } from '@playwright/test';import { test as it, expect } from './pageTest';-const forAI = { _forAI: true } as any;+function snapshotForAI(page: any): Promise<string> {+ return page._snapshotForAI();+}it('should generate refs', async ({ page }) => {await page.setContent(`@@ -26,7 +27,7 @@ it('should generate refs', async ({ page }) => {<button>Three</button>`);- const snapshot1 = await page.locator('body').ariaSnapshot(forAI);+ const snapshot1 = await snapshotForAI(page);expect(snapshot1).toContainYaml(`- generic [ref=e1]:- button "One" [ref=e2]@@ -41,7 +42,7 @@ it('should generate refs', async ({ page }) => {e.textContent = 'Not Two';});- const snapshot2 = await page.locator('body').ariaSnapshot(forAI);+ const snapshot2 = await snapshotForAI(page);expect(snapshot2).toContainYaml(`- generic [ref=e1]:- button "One" [ref=e2]@@ -56,47 +57,36 @@ it('should list iframes', async ({ page }) => {<iframe name="foo" src="data:text/html,<h1>World</h1>">`);- const snapshot1 = await page.locator('body').ariaSnapshot(forAI);+ const snapshot1 = await snapshotForAI(page);expect(snapshot1).toContain('- iframe');const frameSnapshot = await page.frameLocator(`iframe`).locator('body').ariaSnapshot();expect(frameSnapshot).toEqual('- heading "World" [level=1]');});-it('ref mode can be used to stitch all frame snapshots', async ({ page, server }) => {+it('should stitch all frame snapshots', async ({ page, server }) => {await page.goto(server.PREFIX + '/frames/nested-frames.html');-- async function allFrameSnapshot(frame: Page | FrameLocator): Promise<string> {- const snapshot = await frame.locator('body').ariaSnapshot(forAI);- const lines = snapshot.split('\n');- const result = [];- for (const line of lines) {- const match = line.match(/^(\s*)- iframe \[ref=(.*)\]/);- if (!match) {- result.push(line);- continue;- }-- const leadingSpace = match[1];- const ref = match[2];- const childFrame = frame.frameLocator(`aria-ref=${ref}`);- const childSnapshot = await allFrameSnapshot(childFrame);- result.push(line + ':', childSnapshot.split('\n').map(l => leadingSpace + ' ' + l).join('\n'));- }- return result.join('\n');- }-- expect(await allFrameSnapshot(page)).toContainYaml(`+ const snapshot = await snapshotForAI(page);+ expect(snapshot).toContainYaml(`- generic [ref=e1]:- iframe [ref=e2]:- - generic [ref=e1]:- - iframe [ref=e2]:- - generic [ref=e2]: Hi, I'm frame- - iframe [ref=e3]:- - generic [ref=e2]: Hi, I'm frame+ - generic [ref=f1e1]:+ - iframe [ref=f1e2]:+ - generic [ref=f2e2]: Hi, I'm frame+ - iframe [ref=f1e3]:+ - generic [ref=f3e2]: Hi, I'm frame- iframe [ref=e3]:- - generic [ref=e2]: Hi, I'm frame+ - generic [ref=f4e2]: Hi, I'm frame`);++ const href = await page.locator('aria-ref=e1').evaluate(e => e.ownerDocument.defaultView.location.href);+ expect(href).toBe(server.PREFIX + '/frames/nested-frames.html');++ const href2 = await page.locator('aria-ref=f1e2').evaluate(e => e.ownerDocument.defaultView.location.href);+ expect(href2).toBe(server.PREFIX + '/frames/two-frames.html');++ const href3 = await page.locator('aria-ref=f3e2').evaluate(e => e.ownerDocument.defaultView.location.href);+ expect(href3).toBe(server.PREFIX + '/frames/frame.html');});it('should not generate refs for hidden elements', async ({ page }) => {@@ -106,7 +96,7 @@ it('should not generate refs for hidden elements', async ({ page }) => {<button>Three</button>`);- const snapshot = await page.locator('body').ariaSnapshot(forAI);+ const snapshot = await snapshotForAI(page);expect(snapshot).toContainYaml(`- generic [ref=e1]:- button "One" [ref=e2]@@ -138,7 +128,7 @@ it('should not generate refs for elements with pointer-events:none', async ({ pa</div>`);- const snapshot = await page.locator('body').ariaSnapshot(forAI);+ const snapshot = await snapshotForAI(page);expect(snapshot).toContainYaml(`- generic [ref=e1]:- button "no-ref"@@ -182,7 +172,7 @@ it('emit generic roles for nodes w/o roles', async ({ page }) => {</div>`);- const snapshot = await page.locator('body').ariaSnapshot(forAI);+ const snapshot = await snapshotForAI(page);expect(snapshot).toContainYaml(`- generic [ref=e2]:@@ -212,7 +202,7 @@ it('should collapse generic nodes', async ({ page }) => {</div>`);- const snapshot = await page.locator('body').ariaSnapshot(forAI);+ const snapshot = await snapshotForAI(page);expect(snapshot).toContainYaml(`- button \"Button\" [ref=e5]`);@@ -223,7 +213,7 @@ it('should include cursor pointer hint', async ({ page }) => {<button style="cursor: pointer">Button</button>`);- const snapshot = await page.locator('body').ariaSnapshot(forAI);+ const snapshot = await snapshotForAI(page);expect(snapshot).toContainYaml(`- button \"Button\" [ref=e2] [cursor=pointer]`);