Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(trace-viewer): render action point and target box in aria mode (#42230)
Pavel Feldmanc7569128be79
diff --git a/packages/isomorphic/trace/traceModernizer.ts b/packages/isomorphic/trace/traceModernizer.tsindex fc28e2dc93..48012f9b0d 100644--- a/packages/isomorphic/trace/traceModernizer.ts+++ b/packages/isomorphic/trace/traceModernizer.ts@@ -122,6 +122,7 @@ export class TraceModernizer {const existing = this._actionMap.get(event.callId);existing!.inputSnapshot = event.inputSnapshot;existing!.point = event.point;+ existing!.box = event.box;break;}case 'log': {diff --git a/packages/playwright-core/src/server/dom.ts b/packages/playwright-core/src/server/dom.tsindex 9d1bc1d73b..0beae8a69f 100644--- a/packages/playwright-core/src/server/dom.ts+++ b/packages/playwright-core/src/server/dom.ts@@ -574,10 +574,10 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {return throwRetargetableDOMError(result);}- async _selectOption(progress: Progress, elements: ElementHandle[], values: types.SelectOption[], options: types.CommonActionOptions): Promise<string[] | 'error:notconnected'> {+ async _selectOption(progress: Progress, elements: ElementHandle[], values: types.SelectOption[], options: types.CommonActionOptions, box?: types.Rect): Promise<string[] | 'error:notconnected'> {let resultingOptions: string[] = [];const result = await this._retryAction(progress, 'select option', async progress => {- await this.instrumentation.onBeforeInputAction(progress, this);+ await this.instrumentation.onBeforeInputAction(progress, this, undefined, box);if (!options.force)progress.log(` waiting for element to be visible and enabled`);const optionsToSelect = [...elements, ...values];@@ -607,10 +607,10 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {assertDone(throwRetargetableDOMError(result));}- async _fill(progress: Progress, value: string, options: types.CommonActionOptions): Promise<'error:notconnected' | 'done'> {+ async _fill(progress: Progress, value: string, options: types.CommonActionOptions, box?: types.Rect): Promise<'error:notconnected' | 'done'> {progress.log(` fill("${value}")`);return await this._retryAction(progress, 'fill', async progress => {- await this.instrumentation.onBeforeInputAction(progress, this);+ await this.instrumentation.onBeforeInputAction(progress, this, undefined, box);if (!options.force)progress.log(' waiting for element to be visible, enabled and editable');const result = await progress.race(this.evaluateInUtility(async ([injected, node, { value, force }]) => {@@ -723,7 +723,7 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {}, { ...options, waitAfter: 'disabled' });}- async _setInputFiles(progress: Progress, items: InputFilesItems): Promise<'error:notconnected' | 'done'> {+ async _setInputFiles(progress: Progress, items: InputFilesItems, box?: types.Rect): Promise<'error:notconnected' | 'done'> {const { filePayloads, localPaths, localDirectory } = items;const multiple = filePayloads && filePayloads.length > 1 || localPaths && localPaths.length > 1;const result = await progress.race(this._evaluateHandleInUtility(([injected, node, { multiple, directoryUpload }]): Element | undefined => {@@ -744,7 +744,7 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {if (result === 'error:notconnected' || !result.asElement())return 'error:notconnected';const retargeted = result.asElement() as ElementHandle<HTMLInputElement>;- await this.instrumentation.onBeforeInputAction(progress, this);+ await this.instrumentation.onBeforeInputAction(progress, this, undefined, box);if (localPaths || localDirectory) {const localPathsOrDirectory = localDirectory ? [localDirectory] : localPaths!;await progress.race(Promise.all((localPathsOrDirectory).map(localPath => (@@ -783,9 +783,9 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {return assertDone(throwRetargetableDOMError(result));}- async _type(progress: Progress, text: string, options: { delay?: number } & types.StrictOptions): Promise<'error:notconnected' | 'done'> {+ async _type(progress: Progress, text: string, options: { delay?: number } & types.StrictOptions, box?: types.Rect): Promise<'error:notconnected' | 'done'> {progress.log(`elementHandle.type("${text}")`);- await this.instrumentation.onBeforeInputAction(progress, this);+ await this.instrumentation.onBeforeInputAction(progress, this, undefined, box);const result = await this._focus(progress, true /* resetSelectionIfNotFocused */);if (result !== 'done')return result;@@ -799,9 +799,9 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {return assertDone(throwRetargetableDOMError(result));}- async _press(progress: Progress, key: string, options: { delay?: number, noWaitAfter?: boolean } & types.StrictOptions): Promise<'error:notconnected' | 'done'> {+ async _press(progress: Progress, key: string, options: { delay?: number, noWaitAfter?: boolean } & types.StrictOptions, box?: types.Rect): Promise<'error:notconnected' | 'done'> {progress.log(`elementHandle.press("${key}")`);- await this.instrumentation.onBeforeInputAction(progress, this);+ await this.instrumentation.onBeforeInputAction(progress, this, undefined, box);return this._page.frameManager.waitForSignalsCreatedBy(progress, !options.noWaitAfter, async progress => {const result = await this._focus(progress, true /* resetSelectionIfNotFocused */);if (result !== 'done')diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.tsindex bad833ee84..6657843634 100644--- a/packages/playwright-core/src/server/frames.ts+++ b/packages/playwright-core/src/server/frames.ts@@ -1202,7 +1202,7 @@ export class Frame extends SdkObject<FrameEventMap> {progress: Progress,selector: string,options: { strict?: boolean, noAutoWaiting?: boolean, force?: boolean, performActionPreChecks?: boolean },- action: (progress: Progress, handle: dom.ElementHandle<Element>) => Promise<R | 'error:notconnected'>): Promise<R> {+ action: (progress: Progress, handle: dom.ElementHandle<Element>, box?: types.Rect) => Promise<R | 'error:notconnected'>): Promise<R> {progress.log(`waiting for ${this._asLocator(selector)}`);const noAutoWaiting = (options as any).__testHookNoAutoWaiting ?? options.noAutoWaiting;const performActionPreChecks = (options.performActionPreChecks ?? !options.force) && !noAutoWaiting;@@ -1217,7 +1217,8 @@ export class Frame extends SdkObject<FrameEventMap> {log = ` locator resolved to ${elements.length} elements. Proceeding with the first one: ${injected.previewNode(elements[0])}`;else if (element)log = ` locator resolved to ${injected.previewNode(element)}`;- return { log, success: !!element, element };+ const rect = element?.getBoundingClientRect();+ return { log, success: !!element, element, box: rect ? { x: rect.x, y: rect.y, width: rect.width, height: rect.height } : undefined };}, {}));if (!resolved) {if (noAutoWaiting)@@ -1225,7 +1226,7 @@ export class Frame extends SdkObject<FrameEventMap> {return continuePolling;}const result = resolved.result;- const { log, success } = await progress.race(result.evaluate(r => ({ log: r.log, success: r.success })));+ const { log, success, box } = await progress.race(result.evaluate(r => ({ log: r.log, success: r.success, box: r.box })));if (log)progress.log(log);if (!success) {@@ -1237,7 +1238,7 @@ export class Frame extends SdkObject<FrameEventMap> {const element = await progress.race(result.evaluateHandle(r => r.element)) as dom.ElementHandle<Element>;result.dispose();try {- const result = await action(progress, element);+ const result = await action(progress, element, box);if (result === 'error:notconnected') {if (noAutoWaiting)throw new dom.NonRecoverableDOMError('Element is not attached to the DOM');@@ -1297,7 +1298,7 @@ export class Frame extends SdkObject<FrameEventMap> {}async fill(progress: Progress, selector: string, value: string, options: types.CommonActionOptions) {- return dom.assertDone(await this._retryWithProgressIfNotConnected(progress, selector, options, (progress, handle) => handle._fill(progress, value, options)));+ return dom.assertDone(await this._retryWithProgressIfNotConnected(progress, selector, options, (progress, handle, box) => handle._fill(progress, value, options, box)));}async focus(progress: Progress, selector: string, options: types.StrictOptions & { noAutoWaiting?: boolean }) {@@ -1446,12 +1447,12 @@ export class Frame extends SdkObject<FrameEventMap> {}async selectOption(progress: Progress, selector: string, elements: dom.ElementHandle[], values: types.SelectOption[], options: types.CommonActionOptions): Promise<string[]> {- return await this._retryWithProgressIfNotConnected(progress, selector, options, (progress, handle) => handle._selectOption(progress, elements, values, options));+ return await this._retryWithProgressIfNotConnected(progress, selector, options, (progress, handle, box) => handle._selectOption(progress, elements, values, options, box));}async setInputFiles(progress: Progress, selector: string, params: Omit<channels.FrameSetInputFilesParams, 'timeout'> & { noAutoWaiting?: boolean }): Promise<channels.FrameSetInputFilesResult> {const inputFileItems = await progress.race(prepareFilesForUpload(this, params));- return dom.assertDone(await this._retryWithProgressIfNotConnected(progress, selector, params, (progress, handle) => handle._setInputFiles(progress, inputFileItems)));+ return dom.assertDone(await this._retryWithProgressIfNotConnected(progress, selector, params, (progress, handle, box) => handle._setInputFiles(progress, inputFileItems, box)));}async drop(progress: Progress, selector: string, params: Omit<channels.FrameDropParams, 'timeout' | 'selector'>, options: types.PointerActionWaitOptions): Promise<void> {@@ -1465,11 +1466,11 @@ export class Frame extends SdkObject<FrameEventMap> {}async type(progress: Progress, selector: string, text: string, options: { delay?: number, noAutoWaiting?: boolean } & types.StrictOptions) {- return dom.assertDone(await this._retryWithProgressIfNotConnected(progress, selector, options, (progress, handle) => handle._type(progress, text, options)));+ return dom.assertDone(await this._retryWithProgressIfNotConnected(progress, selector, options, (progress, handle, box) => handle._type(progress, text, options, box)));}async press(progress: Progress, selector: string, key: string, options: { delay?: number, noWaitAfter?: boolean, noAutoWaiting?: boolean } & types.StrictOptions) {- return dom.assertDone(await this._retryWithProgressIfNotConnected(progress, selector, options, (progress, handle) => handle._press(progress, key, options)));+ return dom.assertDone(await this._retryWithProgressIfNotConnected(progress, selector, options, (progress, handle, box) => handle._press(progress, key, options, box)));}async check(progress: Progress, selector: string, options: types.PointerActionWaitOptions) {diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.tsindex 8a8509f984..0575ebb978 100644--- a/packages/playwright-core/src/server/trace/recorder/tracing.ts+++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts@@ -544,12 +544,12 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snapsreturn this._captureSnapshot(progress, sdkObject, 'before', event.beforeSnapshot);}- onBeforeInputAction(progress: Progress, sdkObject: SdkObject, point?: types.Point) {+ onBeforeInputAction(progress: Progress, sdkObject: SdkObject, point?: types.Point, box?: types.Rect) {const { metadata } = progress;// IMPORTANT: no awaits in this method, this._appendTraceEvent must be called synchronously.if (!this._state?.callsInProgress.has(metadata.id))return Promise.resolve();- const event = createInputActionTraceEvent(metadata, point);+ const event = createInputActionTraceEvent(metadata, point, box);if (!event)return Promise.resolve();this._temporarilyDisableThrottling(sdkObject.attribution.page);@@ -802,13 +802,14 @@ function createBeforeActionTraceEvent(metadata: CallMetadata, parentId?: string)return event;}-function createInputActionTraceEvent(metadata: CallMetadata, point: types.Point | undefined): trace.InputActionTraceEvent | null {+function createInputActionTraceEvent(metadata: CallMetadata, point: types.Point | undefined, box: types.Rect | undefined): trace.InputActionTraceEvent | null {if (metadata.internal || metadata.method.startsWith('tracing'))return null;return {type: 'input',callId: metadata.id,point,+ box,};}diff --git a/packages/trace-viewer/src/ui/ariaModeView.css b/packages/trace-viewer/src/ui/ariaModeView.cssindex 45a1cae73b..852e6f8de8 100644--- a/packages/trace-viewer/src/ui/ariaModeView.css+++ b/packages/trace-viewer/src/ui/ariaModeView.css@@ -42,6 +42,21 @@outline: 1px solid #6fa8dc;}+.aria-mode-action-highlight {+ position: absolute;+ pointer-events: none;+ background-color: #f443361f;+ outline: 1px solid #f44336;+}++.aria-mode-action-point {+ position: absolute;+ pointer-events: none;+ transform: translate(-50%, -50%);+ border-radius: 50%;+ background-color: #f44336;+}+.aria-mode-snapshot {flex: 1 1 0;border-left: 1px solid var(--vscode-panel-border);diff --git a/packages/trace-viewer/src/ui/ariaModeView.tsx b/packages/trace-viewer/src/ui/ariaModeView.tsxindex 9e34e37c44..f9ed801ccf 100644--- a/packages/trace-viewer/src/ui/ariaModeView.tsx+++ b/packages/trace-viewer/src/ui/ariaModeView.tsx@@ -73,6 +73,7 @@ export function collectAriaModeTargets(model: TraceModel, action: ActionTraceEvereturn { action: actionTarget, before, after };}+type Point = { x: number, y: number };type Box = { x: number, y: number, width: number, height: number };type AriaSnapshotLine = {@@ -133,7 +134,9 @@ function renderLineTokens(text: string): React.ReactNode[] {export const AriaModeView: React.FunctionComponent<{model: TraceModel | undefined,target: AriaModeTarget | undefined,-}> = ({ model, target }) => {+ point?: Point,+ box?: Box,+}> = ({ model, target, point, box }) => {const screenshot = model && target ? model.screenshotForCall(target.callId, target.phase) : undefined;const ariaSnapshot = model && target ? model.ariaSnapshotForCall(target.callId, target.phase) : undefined;const [lines, setLines] = React.useState<AriaSnapshotLine[]>([]);@@ -165,7 +168,7 @@ export const AriaModeView: React.FunctionComponent<{return <PlaceholderPanel text='No aria snapshot' />;return <div className='aria-mode-view hbox'>- <AriaModeScreenshot model={model!} screenshot={screenshot} highlightedBox={highlightedBox} />+ <AriaModeScreenshot model={model!} screenshot={screenshot} highlightedBox={highlightedBox} point={point} box={box} /><div className='aria-mode-snapshot vbox'>{ariaSnapshot && <div className='aria-mode-lines' onMouseLeave={() => setHighlightedBox(undefined)}>{lines.map((line, index) => <div@@ -183,23 +186,40 @@ const AriaModeScreenshot: React.FunctionComponent<{model: TraceModel,screenshot: ScreenshotTraceEvent | undefined,highlightedBox: Box | undefined,-}> = ({ model, screenshot, highlightedBox }) => {+ point: Point | undefined,+ box: Box | undefined,+}> = ({ model, screenshot, highlightedBox, point, box }) => {const [measure, ref] = useMeasure<HTMLDivElement>();const [naturalSize, setNaturalSize] = React.useState<{ width: number, height: number } | undefined>();- // Trace screenshots are taken with css scale, so image pixels match the aria box viewport coordinates.- let highlightStyle: React.CSSProperties | undefined;- if (highlightedBox && naturalSize && measure.width) {+ // Trace screenshots are taken with css scale, so image pixels match the aria box viewport+ // coordinates. The image is scaled to fit into the available area, scale the boxes and+ // the action point to match the rendered image.+ let overlays: React.ReactNode;+ if (screenshot && naturalSize && measure.width) {const padding = 10;const availableWidth = measure.width - 2 * padding;const availableHeight = measure.height - 2 * padding;const scale = Math.min(availableWidth / naturalSize.width, availableHeight / naturalSize.height, 1);- highlightStyle = {- left: padding + (availableWidth - naturalSize.width * scale) / 2 + highlightedBox.x * scale + 'px',- top: padding + (availableHeight - naturalSize.height * scale) / 2 + highlightedBox.y * scale + 'px',- width: highlightedBox.width * scale + 'px',- height: highlightedBox.height * scale + 'px',- };+ const offsetX = padding + (availableWidth - naturalSize.width * scale) / 2;+ const offsetY = padding + (availableHeight - naturalSize.height * scale) / 2;+ const boxStyle = (b: Box): React.CSSProperties => ({+ left: offsetX + b.x * scale + 'px',+ top: offsetY + b.y * scale + 'px',+ width: b.width * scale + 'px',+ height: b.height * scale + 'px',+ });+ const pointStyle = (p: Point): React.CSSProperties => ({+ left: offsetX + p.x * scale + 'px',+ top: offsetY + p.y * scale + 'px',+ width: 20 * scale + 'px',+ height: 20 * scale + 'px',+ });+ overlays = <>+ {box && <div className='aria-mode-action-highlight' style={boxStyle(box)} />}+ {point && <div className='aria-mode-action-point' style={pointStyle(point)} />}+ {highlightedBox && <div className='aria-mode-highlight' style={boxStyle(highlightedBox)} />}+ </>;}return <div ref={ref} className='aria-mode-screenshot'>@@ -210,6 +230,6 @@ const AriaModeScreenshot: React.FunctionComponent<{onLoad={event => setNaturalSize({ width: event.currentTarget.naturalWidth, height: event.currentTarget.naturalHeight })}/>}{!screenshot && <PlaceholderPanel text='No screenshot' />}- {screenshot && highlightStyle && <div className='aria-mode-highlight' style={highlightStyle} />}+ {overlays}</div>;};diff --git a/packages/trace-viewer/src/ui/snapshotTab.tsx b/packages/trace-viewer/src/ui/snapshotTab.tsxindex 828daec4f9..4b3933720d 100644--- a/packages/trace-viewer/src/ui/snapshotTab.tsx+++ b/packages/trace-viewer/src/ui/snapshotTab.tsx@@ -104,6 +104,8 @@ export const SnapshotTabsView: React.FunctionComponent<{{displayAriaMode && <AriaModeViewmodel={model}target={ariaModeTargets[snapshotTab]}+ point={snapshotTab === 'action' ? action?.point : undefined}+ box={snapshotTab === 'action' ? action?.box : undefined}/>}{!displayAriaMode && <SnapshotViewsnapshotUrls={snapshotUrls}diff --git a/packages/trace/src/trace.ts b/packages/trace/src/trace.tsindex dd10371f94..46aa729f63 100644--- a/packages/trace/src/trace.ts+++ b/packages/trace/src/trace.ts@@ -16,7 +16,7 @@import type { FrameSnapshot, ResourceSnapshot } from './snapshot';import type { Language } from '@isomorphic/locatorGenerators';-import type { Point } from '@isomorphic/types';+import type { Point, Rect } from '@isomorphic/types';export type Size = { width: number, height: number };@@ -143,6 +143,7 @@ export type InputActionTraceEvent = {callId: string;inputSnapshot?: string;point?: Point;+ box?: Rect;};export type AfterActionTraceEventAttachment = {diff --git a/tests/library/trace-viewer.spec.ts b/tests/library/trace-viewer.spec.tsindex 69d37ac90f..06e5660e3e 100644--- a/tests/library/trace-viewer.spec.ts+++ b/tests/library/trace-viewer.spec.ts@@ -2108,9 +2108,12 @@ test('should toggle canvas rendering', async ({ runAndTrace, page }) => {});test('should display aria mode', async ({ runAndTrace, page }) => {+ let buttonBox: { x: number, y: number, width: number, height: number };const traceViewer = await runAndTrace(async () => {await page.setContent('<!DOCTYPE html><button>Click me</button>');+ buttonBox = (await page.locator('button').boundingBox())!;await page.locator('button').click();+ await page.locator('button').press('Enter');}, { snapshots: { dom: true, aria: true, screen: true } });await traceViewer.showSettings();@@ -2123,18 +2126,65 @@ test('should display aria mode', async ({ runAndTrace, page }) => {await expect(ariaModeView.locator('img')).toBeVisible();await expect(ariaModeView).toContainText('button "Click me"');- // Hovering an aria node highlights its box on the screenshot.+ // The screenshot is scaled to fit into the available area.+ const imgBox = (await ariaModeView.locator('img').boundingBox())!;+ const scale = imgBox.width / page.viewportSize()!.width;+ const toImage = (point: { x: number, y: number }) => ({ x: imgBox.x + point.x * scale, y: imgBox.y + point.y * scale });++ // Hovering an aria node highlights its box on the screenshot, scaled to match the image.const highlight = ariaModeView.locator('.aria-mode-highlight');await expect(highlight).not.toBeVisible();await ariaModeView.locator('.aria-mode-line', { hasText: 'button "Click me"' }).hover();await expect(highlight).toBeVisible();+ {+ const highlightBox = (await highlight.boundingBox())!;+ const expected = toImage(buttonBox!);+ expect(Math.abs(highlightBox.x - expected.x)).toBeLessThan(2);+ expect(Math.abs(highlightBox.y - expected.y)).toBeLessThan(2);+ expect(Math.abs(highlightBox.width - buttonBox!.width * scale)).toBeLessThan(2);+ expect(Math.abs(highlightBox.height - buttonBox!.height * scale)).toBeLessThan(2);+ }await ariaModeView.locator('img').hover();await expect(highlight).not.toBeVisible();- // The "Before" tab shows the state before the click.+ // The action point and the target box are rendered on the "Action" screenshot.+ const actionPoint = ariaModeView.locator('.aria-mode-action-point');+ const actionHighlight = ariaModeView.locator('.aria-mode-action-highlight');+ await expect(actionPoint).toBeVisible();+ await expect(actionHighlight).toBeVisible();+ {+ const pointBox = (await actionPoint.boundingBox())!;+ const expected = toImage({ x: buttonBox!.x + buttonBox!.width / 2, y: buttonBox!.y + buttonBox!.height / 2 });+ expect(Math.abs(pointBox.x + pointBox.width / 2 - expected.x)).toBeLessThan(2);+ expect(Math.abs(pointBox.y + pointBox.height / 2 - expected.y)).toBeLessThan(2);+ const actionHighlightBox = (await actionHighlight.boundingBox())!;+ const expectedBox = toImage(buttonBox!);+ expect(Math.abs(actionHighlightBox.x - expectedBox.x)).toBeLessThan(2);+ expect(Math.abs(actionHighlightBox.y - expectedBox.y)).toBeLessThan(2);+ expect(Math.abs(actionHighlightBox.width - buttonBox!.width * scale)).toBeLessThan(2);+ expect(Math.abs(actionHighlightBox.height - buttonBox!.height * scale)).toBeLessThan(2);+ }++ // Keyboard actions have no point, but still highlight the target box.+ await traceViewer.selectAction('Press');+ await expect(actionHighlight).toBeVisible();+ await expect(actionPoint).not.toBeVisible();+ {+ const actionHighlightBox = (await actionHighlight.boundingBox())!;+ const expectedBox = toImage(buttonBox!);+ expect(Math.abs(actionHighlightBox.x - expectedBox.x)).toBeLessThan(2);+ expect(Math.abs(actionHighlightBox.y - expectedBox.y)).toBeLessThan(2);+ expect(Math.abs(actionHighlightBox.width - buttonBox!.width * scale)).toBeLessThan(2);+ expect(Math.abs(actionHighlightBox.height - buttonBox!.height * scale)).toBeLessThan(2);+ }+ await traceViewer.selectAction('Click');++ // The "Before" tab shows the state before the click, without the action point.await traceViewer.selectSnapshot('Before');await expect(ariaModeView.locator('img')).toBeVisible();await expect(ariaModeView).toContainText('button "Click me"');+ await expect(actionPoint).not.toBeVisible();+ await expect(actionHighlight).not.toBeVisible();// Toggling the setting off restores the DOM snapshot.await traceViewer.showSettings();