Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(cli): include aria snapshot in annotate output (#40432)

Simon Knotta6786ebed5b9
diff --git a/packages/dashboard/src/dashboardChannel.ts b/packages/dashboard/src/dashboardChannel.ts
index 870cbc29d3..fb484cbfcb 100644
--- a/packages/dashboard/src/dashboardChannel.ts
+++ b/packages/dashboard/src/dashboardChannel.ts
@@ -64,8 +64,8 @@ export interface DashboardChannel {
startRecording(): Promise<void>;
stopRecording(): Promise<{ streamId: string }>;
readStream(params: { streamId: string }): Promise<{ data: string; eof: boolean }>;
- screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number }>;
- submitAnnotation(params: { data: string | undefined; annotations: AnnotationData[] }): Promise<void>;
+ screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number, ariaSnapshot: string }>;
+ submitAnnotation(params: { data: string | undefined; annotations: AnnotationData[], ariaSnapshot: string }): Promise<void>;
on<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;
off<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;
diff --git a/packages/dashboard/src/dashboardModel.ts b/packages/dashboard/src/dashboardModel.ts
index 858b89f15b..0817f3725d 100644
--- a/packages/dashboard/src/dashboardModel.ts
+++ b/packages/dashboard/src/dashboardModel.ts
@@ -27,7 +27,7 @@ export type RecordingState =
| { phase: 'recording' }
| { phase: 'stopped'; blob: Blob; blobUrl: string };
-export type AnnotateFrame = { data: string; viewportWidth: number; viewportHeight: number };
+export type AnnotateFrame = { data: string; viewportWidth: number; viewportHeight: number; ariaSnapshot: string };
export type DashboardState = {
// Session model state.
@@ -208,6 +208,7 @@ export class DashboardModel {
async submitAnnotation(data: string | undefined, annotations: Annotation[]) {
await this._client.submitAnnotation({
data,
+ ariaSnapshot: this.state.annotateFrame?.ariaSnapshot ?? '',
annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),
});
this.cancelAnnotate();
diff --git a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
index 4d9d6fa57f..db3b9af3b6 100644
--- a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
+++ b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
@@ -53,10 +53,10 @@ async function startDashboardServer(options: DashboardOptions): Promise<Dashboar
let pendingAnnotate = false;
const waitingSockets = new Set<net.Socket>();
- const submitAnnotation = (base64Png: string | undefined, annotations: AnnotationData[]) => {
+ const submitAnnotation = (base64Png: string | undefined, ariaSnapshot: string, annotations: AnnotationData[]) => {
if (waitingSockets.size === 0)
return;
- const payload = JSON.stringify({ png: base64Png, annotations });
+ const payload = JSON.stringify({ png: base64Png, ariaSnapshot, annotations });
for (const socket of waitingSockets) {
socket.write(payload);
socket.end();
@@ -388,17 +388,22 @@ async function runAnnotateClient(options: DashboardOptions): Promise<void> {
const text = Buffer.concat(chunks).toString();
if (!text)
return;
- const { png, annotations } = JSON.parse(text) as { png: string; annotations: AnnotationData[] };
+ const { png, annotations, ariaSnapshot } = JSON.parse(text) as { png: string; annotations: AnnotationData[], ariaSnapshot: string };
for (const a of annotations) {
// eslint-disable-next-line no-console
console.log(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);
}
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
if (png) {
- const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filePath = await saveOutputFile(`annotations-${timestamp}.png`, Buffer.from(png, 'base64'));
// eslint-disable-next-line no-console
console.log(`image: ${path.relative(process.cwd(), filePath)}`);
}
+ if (ariaSnapshot) {
+ const filePath = await saveOutputFile(`annotations-${timestamp}.yaml`, ariaSnapshot);
+ // eslint-disable-next-line no-console
+ console.log(`snapshot: ${path.relative(process.cwd(), filePath)}`);
+ }
}
function selfDestructOnParentGone() {
diff --git a/packages/playwright-core/src/tools/dashboard/dashboardController.ts b/packages/playwright-core/src/tools/dashboard/dashboardController.ts
index f5c5af5c9a..a52db4d9f8 100644
--- a/packages/playwright-core/src/tools/dashboard/dashboardController.ts
+++ b/packages/playwright-core/src/tools/dashboard/dashboardController.ts
@@ -110,7 +110,7 @@ export class DashboardConnection implements Transport {
private _attachedPage: AttachedPage | undefined;
private _onclose: () => void;
private _onconnected?: () => void;
- private _onAnnotationSubmit?: (base64Png: string | undefined, annotations: AnnotationData[]) => void;
+ private _onAnnotationSubmit?: (base64Png: string | undefined, ariaSnapshot: string, annotations: AnnotationData[]) => void;
private _serverRegistryDispose?: () => void;
private _pushSessionsScheduled = false;
private _pushTabsScheduled = false;
@@ -121,7 +121,7 @@ export class DashboardConnection implements Transport {
_recordingDir: string;
_streams = new Map<string, { handle: fs.promises.FileHandle; path: string }>();
- constructor(onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string | undefined, annotations: AnnotationData[]) => void) {
+ constructor(onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string | undefined, ariaSnapshot: string, annotations: AnnotationData[]) => void) {
this._onclose = onclose;
this._onconnected = onconnected;
this._onAnnotationSubmit = onAnnotationSubmit;
@@ -237,8 +237,8 @@ export class DashboardConnection implements Transport {
this._pushTabs();
}
- async submitAnnotation(params: { data: string | undefined; annotations: AnnotationData[] }) {
- this._onAnnotationSubmit?.(params.data, params.annotations);
+ async submitAnnotation(params: { data: string | undefined; ariaSnapshot: string; annotations: AnnotationData[] }) {
+ this._onAnnotationSubmit?.(params.data, params.ariaSnapshot, params.annotations);
}
async reveal(params: { path: string }) {
@@ -567,13 +567,15 @@ class AttachedPage {
return { streamId };
}
- async screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number }> {
+ async screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number, ariaSnapshot: string }> {
const buffer = await this._page.screenshot({ type: 'png' });
const vp = this._page.viewportSize();
+ const ariaSnapshot = await this._page.ariaSnapshot({ boxes: true, mode: 'ai' });
return {
data: buffer.toString('base64'),
viewportWidth: vp?.width ?? 0,
viewportHeight: vp?.height ?? 0,
+ ariaSnapshot,
};
}