Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(aria): only include changed chunks into snapshot diff (#37988)

Dmitry Gozman7249e2068ac8
diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.ts
index 1d41fc25ae..fb0c9e15a8 100644
--- a/packages/injected/src/ariaSnapshot.ts
+++ b/packages/injected/src/ariaSnapshot.ts
@@ -519,29 +519,35 @@ function buildByRefMap(root: AriaNode | undefined, map: Map<string | undefined,
function compareSnapshots(ariaSnapshot: AriaSnapshot, previousSnapshot: AriaSnapshot | undefined): Map<AriaNode, 'skip' | 'same' | 'changed'> {
const previousByRef = buildByRefMap(previousSnapshot?.root);
- const result = new Map<AriaNode, 'same' | 'changed'>();
+ const result = new Map<AriaNode, 'skip' | 'same' | 'changed'>();
// Returns whether ariaNode is the same as previousNode.
const visit = (ariaNode: AriaNode, previousNode: AriaNode | undefined): boolean => {
let same: boolean = ariaNode.children.length === previousNode?.children.length && ariaNodesEqual(ariaNode, previousNode);
if (ariaNode.role === 'iframe')
same = false;
+ let canBeSkipped = same;
for (let childIndex = 0 ; childIndex < ariaNode.children.length; childIndex++) {
const child = ariaNode.children[childIndex];
const previousChild = previousNode?.children[childIndex];
if (typeof child === 'string') {
same &&= child === previousChild;
+ canBeSkipped &&= child === previousChild;
} else {
let previous = typeof previousChild !== 'string' ? previousChild : undefined;
if (child.ref)
previous = previousByRef.get(child.ref);
const sameChild = visit(child, previous);
+ // New child, different order of children, or changed child with no ref -
+ // we have to include this node to list children in the right order.
+ if (!previous || (!sameChild && !child.ref) || (previous !== previousChild))
+ canBeSkipped = false;
same &&= (sameChild && previous === previousChild);
}
}
- result.set(ariaNode, same ? 'same' : 'changed');
+ result.set(ariaNode, same ? 'same' : (canBeSkipped ? 'skip' : 'changed'));
return same;
};
@@ -549,12 +555,47 @@ function compareSnapshots(ariaSnapshot: AriaSnapshot, previousSnapshot: AriaSnap
return result;
}
+// Chooses only the changed parts of the snapshot and returns them as new roots.
+function filterSnapshotDiff(nodes: (AriaNode | string)[], statusMap: Map<AriaNode, 'skip' | 'same' | 'changed'>): (AriaNode | string)[] {
+ const result: (AriaNode | string)[] = [];
+
+ const visit = (ariaNode: AriaNode) => {
+ const status = statusMap.get(ariaNode);
+ if (status === 'same') {
+ // No need to render unchanged root at all.
+ } else if (status === 'skip') {
+ // Only render changed children.
+ for (const child of ariaNode.children) {
+ if (typeof child !== 'string')
+ visit(child);
+ }
+ } else {
+ // Render this node's subtree.
+ result.push(ariaNode);
+ }
+ };
+
+ for (const node of nodes) {
+ if (typeof node === 'string')
+ result.push(node);
+ else
+ visit(node);
+ }
+ return result;
+}
+
export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTreeOptions, previousSnapshot?: AriaSnapshot): string {
const options = toInternalOptions(publicOptions);
const lines: string[] = [];
const includeText = options.renderStringsAsRegex ? textContributesInfo : () => true;
const renderString = options.renderStringsAsRegex ? convertToBestGuessRegex : (str: string) => str;
+
+ // Do not render the root fragment, just its children.
+ let nodesToRender = ariaSnapshot.root.role === 'fragment' ? ariaSnapshot.root.children : [ariaSnapshot.root];
+
const statusMap = compareSnapshots(ariaSnapshot, previousSnapshot);
+ if (previousSnapshot)
+ nodesToRender = filterSnapshotDiff(nodesToRender, statusMap);
const visitText = (text: string, indent: string) => {
const escaped = yamlEscapeValueIfNeeded(renderString(text));
@@ -604,15 +645,15 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTr
};
const visit = (ariaNode: AriaNode, indent: string, renderCursorPointer: boolean) => {
- const status = statusMap.get(ariaNode);
-
// Replace the whole subtree with a single reference when possible.
- if (status === 'same' && ariaNode.ref) {
+ if (statusMap.get(ariaNode) === 'same' && ariaNode.ref) {
lines.push(indent + `- ref=${ariaNode.ref} [unchanged]`);
return;
}
- const escapedKey = indent + '- ' + yamlEscapeKeyIfNeeded(createKey(ariaNode, renderCursorPointer));
+ // When producing a diff, add <changed> marker to all diff roots.
+ const isDiffRoot = !!previousSnapshot && !indent;
+ const escapedKey = indent + '- ' + (isDiffRoot ? '<changed> ' : '') + yamlEscapeKeyIfNeeded(createKey(ariaNode, renderCursorPointer));
const singleInlinedTextChild = getSingleInlinedTextChild(ariaNode);
if (!ariaNode.children.length && !Object.keys(ariaNode.props).length) {
@@ -630,22 +671,18 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTr
lines.push(escapedKey + ':');
for (const [name, value] of Object.entries(ariaNode.props))
lines.push(indent + ' - /' + name + ': ' + yamlEscapeValueIfNeeded(value));
- }
- indent += ' ';
- if (singleInlinedTextChild === undefined) {
+ const childIndent = indent + ' ';
const inCursorPointer = !!ariaNode.ref && renderCursorPointer && hasPointerCursor(ariaNode);
for (const child of ariaNode.children) {
if (typeof child === 'string')
- visitText(includeText(ariaNode, child) ? child : '', indent);
+ visitText(includeText(ariaNode, child) ? child : '', childIndent);
else
- visit(child, indent, renderCursorPointer && !inCursorPointer);
+ visit(child, childIndent, renderCursorPointer && !inCursorPointer);
}
}
};
- // Do not render the root fragment, just its children.
- const nodesToRender = ariaSnapshot.root.role === 'fragment' ? ariaSnapshot.root.children : [ariaSnapshot.root];
for (const nodeToRender of nodesToRender) {
if (typeof nodeToRender === 'string')
visitText(nodeToRender, '');
diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts
index 80270f2ea0..179332f683 100644
--- a/packages/playwright-core/src/server/page.ts
+++ b/packages/playwright-core/src/server/page.ts
@@ -1046,7 +1046,7 @@ async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, optio
if (!node)
return true;
return injected.ariaSnapshot(node, { mode: 'ai', ...options });
- }, { refPrefix: frame.seq ? 'f' + frame.seq : '', incremental: options.mode === 'incremental', track: options.track }));
+ }, { refPrefix: frame.seq ? 'f' + frame.seq : '', incremental: options.mode === 'incremental' && !frame.parentFrame(), track: options.track }));
if (snapshotOrRetry === true)
return continuePolling;
return snapshotOrRetry;
@@ -1060,7 +1060,7 @@ async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, optio
const lines = snapshot.split('\n');
const result = [];
for (const line of lines) {
- const match = line.match(/^(\s*)- iframe (?:\[active\] )?\[ref=([^\]]*)\]/);
+ const match = line.match(/^(\s*)-(?: <changed>)? iframe (?:\[active\] )?\[ref=([^\]]*)\]/);
if (!match) {
result.push(line);
continue;
diff --git a/packages/playwright/src/mcp/browser/response.ts b/packages/playwright/src/mcp/browser/response.ts
index a521291367..f4e40931aa 100644
--- a/packages/playwright/src/mcp/browser/response.ts
+++ b/packages/playwright/src/mcp/browser/response.ts
@@ -192,7 +192,8 @@ function renderTabSnapshot(tabSnapshot: TabSnapshot, options: { omitSnapshot?: b
lines.push(`- Page Title: ${tabSnapshot.title}`);
lines.push(`- Page Snapshot:`);
lines.push('```yaml');
- lines.push(options.omitSnapshot ? '<snapshot>' : tabSnapshot.ariaSnapshot);
+ // TODO: perhaps not render page state when there are no changes?
+ lines.push(options.omitSnapshot ? '<snapshot>' : (tabSnapshot.ariaSnapshot || '<no changes>'));
lines.push('```');
return lines.join('\n');
diff --git a/tests/mcp/click.spec.ts b/tests/mcp/click.spec.ts
index 8e64ad5c3c..72a37533a4 100644
--- a/tests/mcp/click.spec.ts
+++ b/tests/mcp/click.spec.ts
@@ -41,7 +41,7 @@ test('browser_click', async ({ client, server }) => {
},
})).toHaveResponse({
code: `await page.getByRole('button', { name: 'Submit' }).click();`,
- pageState: expect.stringContaining(`- button "Submit" [active] [ref=e2]`),
+ pageState: expect.stringContaining(`button "Submit" [active] [ref=e2]`),
});
});
@@ -70,7 +70,7 @@ test('browser_click (double)', async ({ client, server }) => {
},
})).toHaveResponse({
code: `await page.getByRole('heading', { name: 'Click me' }).dblclick();`,
- pageState: expect.stringContaining(`- heading "Double clicked" [level=1] [ref=e3]`),
+ pageState: expect.stringContaining(`heading "Double clicked" [level=1] [ref=e3]`),
});
});
@@ -100,7 +100,7 @@ test('browser_click (right)', async ({ client, server }) => {
});
expect(result).toHaveResponse({
code: `await page.getByRole('button', { name: 'Menu' }).click({ button: 'right' });`,
- pageState: expect.stringContaining(`- button "Right clicked"`),
+ pageState: expect.stringContaining(`button "Right clicked"`),
});
});
@@ -131,7 +131,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {
},
})).toHaveResponse({
code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Control'] });`,
- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:true metaKey:false shiftKey:false altKey:false`),
+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:true metaKey:false shiftKey:false altKey:false`),
});
}
@@ -144,7 +144,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {
},
})).toHaveResponse({
code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Shift'] });`,
- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:false`),
+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:false`),
});
expect(await client.callTool({
@@ -156,7 +156,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {
},
})).toHaveResponse({
code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Shift', 'Alt'] });`,
- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:true`),
+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:true`),
});
});
diff --git a/tests/mcp/core.spec.ts b/tests/mcp/core.spec.ts
index 13dde2ba84..7714bfa18d 100644
--- a/tests/mcp/core.spec.ts
+++ b/tests/mcp/core.spec.ts
@@ -58,7 +58,7 @@ test('browser_select_option', async ({ client, server }) => {
- Page Title: Title
- Page Snapshot:
\`\`\`yaml
-- combobox [ref=e2]:
+- <changed> combobox [ref=e2]:
- option "Foo"
- option "Bar" [selected]
\`\`\``,
@@ -90,8 +90,8 @@ test('browser_select_option (multiple)', async ({ client, server }) => {
})).toHaveResponse({
code: `await page.getByRole('listbox').selectOption(['bar', 'baz']);`,
pageState: expect.stringContaining(`
- - option "Bar" [selected] [ref=e4]
- - option "Baz" [selected] [ref=e5]`),
+- <changed> option "Bar" [selected] [ref=e4]
+- <changed> option "Baz" [selected] [ref=e5]`),
});
});
diff --git a/tests/mcp/dialogs.spec.ts b/tests/mcp/dialogs.spec.ts
index 548a33d608..79993805fe 100644
--- a/tests/mcp/dialogs.spec.ts
+++ b/tests/mcp/dialogs.spec.ts
@@ -139,7 +139,7 @@ test('confirm dialog (true)', async ({ client, server }) => {
},
})).toHaveResponse({
modalState: undefined,
- pageState: expect.stringContaining(`- generic [active] [ref=e1]: "true"`),
+ pageState: expect.stringContaining(`generic [active] [ref=e1]: "true"`),
});
});
@@ -175,7 +175,7 @@ test('confirm dialog (false)', async ({ client, server }) => {
},
})).toHaveResponse({
modalState: undefined,
- pageState: expect.stringContaining(`- generic [active] [ref=e1]: "false"`),
+ pageState: expect.stringContaining(`generic [active] [ref=e1]: "false"`),
});
});
@@ -213,7 +213,7 @@ test('prompt dialog', async ({ client, server }) => {
});
expect(result).toHaveResponse({
- pageState: expect.stringContaining(`- generic [active] [ref=e1]: Answer`),
+ pageState: expect.stringContaining(`generic [active] [ref=e1]: Answer`),
});
});
diff --git a/tests/mcp/secrets.spec.ts b/tests/mcp/secrets.spec.ts
index 0f8cb47095..b2e497f307 100644
--- a/tests/mcp/secrets.spec.ts
+++ b/tests/mcp/secrets.spec.ts
@@ -53,7 +53,7 @@ test('browser_type', async ({ startClient, server }) => {
expect(response).toHaveResponse({
code: `await page.getByRole('textbox').fill(process.env['X-PASSWORD']);
await page.getByRole('textbox').press('Enter');`,
- pageState: expect.stringContaining(`- textbox`),
+ pageState: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: <secret>X-PASSWORD<\/secret>/),
});
}
diff --git a/tests/mcp/snapshot-diff.spec.ts b/tests/mcp/snapshot-diff.spec.ts
index 4c4ffdae1e..a0654c8e27 100644
--- a/tests/mcp/snapshot-diff.spec.ts
+++ b/tests/mcp/snapshot-diff.spec.ts
@@ -65,7 +65,7 @@ test('should return aria snapshot diff', async ({ client, server }) => {
})).toHaveResponse({
pageState: expect.stringContaining(`Page Snapshot:
\`\`\`yaml
-- ref=e1 [unchanged]
+<no changes>
\`\`\``),
});
@@ -78,7 +78,7 @@ test('should return aria snapshot diff', async ({ client, server }) => {
})).toHaveResponse({
pageState: expect.stringContaining(`Page Snapshot:
\`\`\`yaml
-- generic [ref=e1]:
+- <changed> generic [ref=e1]:
- button "Button 1" [active] [ref=e2]
- button "Button 2new text" [ref=e105]
- ref=e4 [unchanged]
diff --git a/tests/mcp/type.spec.ts b/tests/mcp/type.spec.ts
index 7ac23141a3..70ddef9d31 100644
--- a/tests/mcp/type.spec.ts
+++ b/tests/mcp/type.spec.ts
@@ -44,7 +44,7 @@ test('browser_type', async ({ client, server }) => {
expect(response).toHaveResponse({
code: `await page.getByRole('textbox').fill('Hi!');
await page.getByRole('textbox').press('Enter');`,
- pageState: expect.stringContaining(`- textbox`),
+ pageState: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: Hi!/),
});
}
@@ -79,7 +79,7 @@ test('browser_type (slowly)', async ({ client, server }) => {
expect(response).toHaveResponse({
code: `await page.getByRole('textbox').pressSequentially('Hi!');`,
- pageState: expect.stringContaining(`- textbox`),
+ pageState: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: Hi!/),
});
}
const response = await client.callTool({
diff --git a/tests/page/page-aria-snapshot-ai.spec.ts b/tests/page/page-aria-snapshot-ai.spec.ts
index 96861d8190..920cfbbcba 100644
--- a/tests/page/page-aria-snapshot-ai.spec.ts
+++ b/tests/page/page-aria-snapshot-ai.spec.ts
@@ -510,7 +510,6 @@ it('should create incremental snapshots on multiple tracks', async ({ page }) =>
- listitem [ref=e5]: a span
`);
expect(await snapshotForAI(page, { track: 'first', mode: 'incremental' })).toContainYaml(`
- - ref=e2 [unchanged]
`);
await page.evaluate(() => {
@@ -518,7 +517,7 @@ it('should create incremental snapshots on multiple tracks', async ({ page }) =>
document.getElementById('hidden-li').style.display = 'inline';
});
expect(await snapshotForAI(page, { track: 'first', mode: 'incremental' })).toContainYaml(`
- - list [ref=e2]:
+ - <changed> list [ref=e2]:
- ref=e3 [unchanged]
- listitem [ref=e5]: changed span
- listitem [ref=e6]: some text
@@ -529,12 +528,11 @@ it('should create incremental snapshots on multiple tracks', async ({ page }) =>
document.getElementById('hidden-li').style.display = 'none';
});
expect(await snapshotForAI(page, { track: 'first', mode: 'incremental' })).toContainYaml(`
- - list [ref=e2]:
+ - <changed> list [ref=e2]:
- ref=e3 [unchanged]
- listitem [ref=e5]: a span
`);
expect(await snapshotForAI(page, { track: 'second', mode: 'incremental' })).toContainYaml(`
- - ref=e2 [unchanged]
`);
expect(await snapshotForAI(page, { track: 'second', mode: 'full' })).toContainYaml(`
@@ -554,7 +552,7 @@ it('should create incremental snapshot for attribute change', async ({ page }) =
await page.evaluate(() => document.querySelector('button').blur());
expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
- - button "a button" [ref=e2]
+ - <changed> button "a button" [ref=e2]
`);
});
@@ -568,7 +566,7 @@ it('should create incremental snapshot for child removal', async ({ page }) => {
await page.evaluate(() => document.querySelector('span').remove());
expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
- - listitem [ref=e2]:
+ - <changed> listitem [ref=e2]:
- ref=e3 [unchanged]
`);
});
@@ -582,7 +580,7 @@ it('should create incremental snapshot for child addition', async ({ page }) =>
await page.evaluate(() => document.querySelector('span').style.display = 'inline');
expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
- - listitem [ref=e2]:
+ - <changed> listitem [ref=e2]:
- ref=e3 [unchanged]
- text: some text
`);
@@ -597,7 +595,7 @@ it('should create incremental snapshot for prop change', async ({ page }) => {
await page.evaluate(() => document.querySelector('a').setAttribute('href', 'https://playwright.dev'));
expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
- - link "a link" [ref=e2] [cursor=pointer]:
+ - <changed> link "a link" [ref=e2] [cursor=pointer]:
- /url: https://playwright.dev
`);
});
@@ -611,7 +609,7 @@ it('should create incremental snapshot for cursor change', async ({ page }) => {
await page.evaluate(() => document.querySelector('a').style.cursor = 'default');
expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
- - link "a link" [ref=e2]:
+ - <changed> link "a link" [ref=e2]:
- /url: about:blank
`);
});
@@ -624,7 +622,7 @@ it('should create incremental snapshot for name change', async ({ page }) => {
await page.evaluate(() => document.querySelector('span').textContent = 'new button');
expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
- - button "new button" [ref=e3]
+ - <changed> button "new button" [ref=e3]
`);
});
@@ -636,7 +634,7 @@ it('should create incremental snapshot for text change', async ({ page }) => {
await page.evaluate(() => document.querySelector('span').textContent = 'new text');
expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
- - listitem [ref=e2]: new text
+ - <changed> listitem [ref=e2]: new text
`);
});
@@ -649,8 +647,50 @@ it('should not produce incremental snapshot for iframes', async ({ page }) => {
- heading "hello" [level=1] [ref=f1e2]
`);
expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
- - iframe [ref=e2]:
- - ref=f1e2 [unchanged]
+ - <changed> iframe [ref=e2]:
+ - heading "hello" [level=1] [ref=f1e2]
+ `);
+});
+
+it('should create multiple chunks in incremental snapshot', async ({ page }) => {
+ await page.setContent(`
+ <ul>
+ <li><span>item1</span></li>
+ <li><span>item2</span></li>
+ <li><div role=group><span>item3</span></div></li>
+ <ul>
+ <li id=to-remove>to be removed</li>
+ <li>one more</li>
+ </ul>
+ </ul>
+ `);
+ expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
+ - list [ref=e2]:
+ - listitem [ref=e3]: item1
+ - listitem [ref=e4]: item2
+ - listitem [ref=e5]:
+ - group [ref=e6]: item3
+ - list [ref=e7]:
+ - listitem [ref=e8]: to be removed
+ - listitem [ref=e9]: one more
+ `);
+
+ await page.evaluate(() => {
+ const spans = document.querySelectorAll('span');
+ spans[0].textContent = 'new item1';
+ spans[2].textContent = 'new item3';
+ const button = document.createElement('button');
+ button.textContent = 'button';
+ spans[2].parentElement.appendChild(button);
+ document.querySelector('#to-remove').remove();
+ });
+ expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
+ - <changed> listitem [ref=e3]: new item1
+ - <changed> group [ref=e6]:
+ - text: new item3
+ - button "button" [ref=e10]
+ - <changed> list [ref=e7]:
+ - ref=e9 [unchanged]
`);
});
@@ -686,7 +726,7 @@ it('should create incremental snapshot for children swap', async ({ page }) => {
await page.evaluate(() => document.querySelector('ul').appendChild(document.querySelector('li')));
expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`
- - list [ref=e2]:
+ - <changed> list [ref=e2]:
- ref=e4 [unchanged]
- ref=e3 [unchanged]
`);