Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(aria): only include changed chunks into snapshot diff (#37988)
Dmitry Gozman7249e2068ac8
diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.tsindex 1d41fc25ae..fb0c9e15a8 100644--- a/packages/injected/src/ariaSnapshot.ts+++ b/packages/injected/src/ariaSnapshot.ts@@ -519,29 +519,35 @@ function buildByRefMap(root: AriaNode | undefined, map: Map<string | undefined,function compareSnapshots(ariaSnapshot: AriaSnapshot, previousSnapshot: AriaSnapshot | undefined): Map<AriaNode, 'skip' | 'same' | 'changed'> {const previousByRef = buildByRefMap(previousSnapshot?.root);- const result = new Map<AriaNode, 'same' | 'changed'>();+ const result = new Map<AriaNode, 'skip' | 'same' | 'changed'>();// Returns whether ariaNode is the same as previousNode.const visit = (ariaNode: AriaNode, previousNode: AriaNode | undefined): boolean => {let same: boolean = ariaNode.children.length === previousNode?.children.length && ariaNodesEqual(ariaNode, previousNode);if (ariaNode.role === 'iframe')same = false;+ let canBeSkipped = same;for (let childIndex = 0 ; childIndex < ariaNode.children.length; childIndex++) {const child = ariaNode.children[childIndex];const previousChild = previousNode?.children[childIndex];if (typeof child === 'string') {same &&= child === previousChild;+ canBeSkipped &&= child === previousChild;} else {let previous = typeof previousChild !== 'string' ? previousChild : undefined;if (child.ref)previous = previousByRef.get(child.ref);const sameChild = visit(child, previous);+ // New child, different order of children, or changed child with no ref -+ // we have to include this node to list children in the right order.+ if (!previous || (!sameChild && !child.ref) || (previous !== previousChild))+ canBeSkipped = false;same &&= (sameChild && previous === previousChild);}}- result.set(ariaNode, same ? 'same' : 'changed');+ result.set(ariaNode, same ? 'same' : (canBeSkipped ? 'skip' : 'changed'));return same;};@@ -549,12 +555,47 @@ function compareSnapshots(ariaSnapshot: AriaSnapshot, previousSnapshot: AriaSnapreturn result;}+// Chooses only the changed parts of the snapshot and returns them as new roots.+function filterSnapshotDiff(nodes: (AriaNode | string)[], statusMap: Map<AriaNode, 'skip' | 'same' | 'changed'>): (AriaNode | string)[] {+ const result: (AriaNode | string)[] = [];++ const visit = (ariaNode: AriaNode) => {+ const status = statusMap.get(ariaNode);+ if (status === 'same') {+ // No need to render unchanged root at all.+ } else if (status === 'skip') {+ // Only render changed children.+ for (const child of ariaNode.children) {+ if (typeof child !== 'string')+ visit(child);+ }+ } else {+ // Render this node's subtree.+ result.push(ariaNode);+ }+ };++ for (const node of nodes) {+ if (typeof node === 'string')+ result.push(node);+ else+ visit(node);+ }+ return result;+}+export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTreeOptions, previousSnapshot?: AriaSnapshot): string {const options = toInternalOptions(publicOptions);const lines: string[] = [];const includeText = options.renderStringsAsRegex ? textContributesInfo : () => true;const renderString = options.renderStringsAsRegex ? convertToBestGuessRegex : (str: string) => str;++ // Do not render the root fragment, just its children.+ let nodesToRender = ariaSnapshot.root.role === 'fragment' ? ariaSnapshot.root.children : [ariaSnapshot.root];+const statusMap = compareSnapshots(ariaSnapshot, previousSnapshot);+ if (previousSnapshot)+ nodesToRender = filterSnapshotDiff(nodesToRender, statusMap);const visitText = (text: string, indent: string) => {const escaped = yamlEscapeValueIfNeeded(renderString(text));@@ -604,15 +645,15 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTr};const visit = (ariaNode: AriaNode, indent: string, renderCursorPointer: boolean) => {- const status = statusMap.get(ariaNode);-// Replace the whole subtree with a single reference when possible.- if (status === 'same' && ariaNode.ref) {+ if (statusMap.get(ariaNode) === 'same' && ariaNode.ref) {lines.push(indent + `- ref=${ariaNode.ref} [unchanged]`);return;}- const escapedKey = indent + '- ' + yamlEscapeKeyIfNeeded(createKey(ariaNode, renderCursorPointer));+ // When producing a diff, add <changed> marker to all diff roots.+ const isDiffRoot = !!previousSnapshot && !indent;+ const escapedKey = indent + '- ' + (isDiffRoot ? '<changed> ' : '') + yamlEscapeKeyIfNeeded(createKey(ariaNode, renderCursorPointer));const singleInlinedTextChild = getSingleInlinedTextChild(ariaNode);if (!ariaNode.children.length && !Object.keys(ariaNode.props).length) {@@ -630,22 +671,18 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTrlines.push(escapedKey + ':');for (const [name, value] of Object.entries(ariaNode.props))lines.push(indent + ' - /' + name + ': ' + yamlEscapeValueIfNeeded(value));- }- indent += ' ';- if (singleInlinedTextChild === undefined) {+ const childIndent = indent + ' ';const inCursorPointer = !!ariaNode.ref && renderCursorPointer && hasPointerCursor(ariaNode);for (const child of ariaNode.children) {if (typeof child === 'string')- visitText(includeText(ariaNode, child) ? child : '', indent);+ visitText(includeText(ariaNode, child) ? child : '', childIndent);else- visit(child, indent, renderCursorPointer && !inCursorPointer);+ visit(child, childIndent, renderCursorPointer && !inCursorPointer);}}};- // Do not render the root fragment, just its children.- const nodesToRender = ariaSnapshot.root.role === 'fragment' ? ariaSnapshot.root.children : [ariaSnapshot.root];for (const nodeToRender of nodesToRender) {if (typeof nodeToRender === 'string')visitText(nodeToRender, '');diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.tsindex 80270f2ea0..179332f683 100644--- a/packages/playwright-core/src/server/page.ts+++ b/packages/playwright-core/src/server/page.ts@@ -1046,7 +1046,7 @@ async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, optioif (!node)return true;return injected.ariaSnapshot(node, { mode: 'ai', ...options });- }, { refPrefix: frame.seq ? 'f' + frame.seq : '', incremental: options.mode === 'incremental', track: options.track }));+ }, { refPrefix: frame.seq ? 'f' + frame.seq : '', incremental: options.mode === 'incremental' && !frame.parentFrame(), track: options.track }));if (snapshotOrRetry === true)return continuePolling;return snapshotOrRetry;@@ -1060,7 +1060,7 @@ async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, optioconst lines = snapshot.split('\n');const result = [];for (const line of lines) {- const match = line.match(/^(\s*)- iframe (?:\[active\] )?\[ref=([^\]]*)\]/);+ const match = line.match(/^(\s*)-(?: <changed>)? iframe (?:\[active\] )?\[ref=([^\]]*)\]/);if (!match) {result.push(line);continue;diff --git a/packages/playwright/src/mcp/browser/response.ts b/packages/playwright/src/mcp/browser/response.tsindex a521291367..f4e40931aa 100644--- a/packages/playwright/src/mcp/browser/response.ts+++ b/packages/playwright/src/mcp/browser/response.ts@@ -192,7 +192,8 @@ function renderTabSnapshot(tabSnapshot: TabSnapshot, options: { omitSnapshot?: blines.push(`- Page Title: ${tabSnapshot.title}`);lines.push(`- Page Snapshot:`);lines.push('```yaml');- lines.push(options.omitSnapshot ? '<snapshot>' : tabSnapshot.ariaSnapshot);+ // TODO: perhaps not render page state when there are no changes?+ lines.push(options.omitSnapshot ? '<snapshot>' : (tabSnapshot.ariaSnapshot || '<no changes>'));lines.push('```');return lines.join('\n');diff --git a/tests/mcp/click.spec.ts b/tests/mcp/click.spec.tsindex 8e64ad5c3c..72a37533a4 100644--- a/tests/mcp/click.spec.ts+++ b/tests/mcp/click.spec.ts@@ -41,7 +41,7 @@ test('browser_click', async ({ client, server }) => {},})).toHaveResponse({code: `await page.getByRole('button', { name: 'Submit' }).click();`,- pageState: expect.stringContaining(`- button "Submit" [active] [ref=e2]`),+ pageState: expect.stringContaining(`button "Submit" [active] [ref=e2]`),});});@@ -70,7 +70,7 @@ test('browser_click (double)', async ({ client, server }) => {},})).toHaveResponse({code: `await page.getByRole('heading', { name: 'Click me' }).dblclick();`,- pageState: expect.stringContaining(`- heading "Double clicked" [level=1] [ref=e3]`),+ pageState: expect.stringContaining(`heading "Double clicked" [level=1] [ref=e3]`),});});@@ -100,7 +100,7 @@ test('browser_click (right)', async ({ client, server }) => {});expect(result).toHaveResponse({code: `await page.getByRole('button', { name: 'Menu' }).click({ button: 'right' });`,- pageState: expect.stringContaining(`- button "Right clicked"`),+ pageState: expect.stringContaining(`button "Right clicked"`),});});@@ -131,7 +131,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {},})).toHaveResponse({code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Control'] });`,- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:true metaKey:false shiftKey:false altKey:false`),+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:true metaKey:false shiftKey:false altKey:false`),});}@@ -144,7 +144,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {},})).toHaveResponse({code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Shift'] });`,- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:false`),+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:false`),});expect(await client.callTool({@@ -156,7 +156,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {},})).toHaveResponse({code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Shift', 'Alt'] });`,- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:true`),+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:true`),});});diff --git a/tests/mcp/core.spec.ts b/tests/mcp/core.spec.tsindex 13dde2ba84..7714bfa18d 100644--- a/tests/mcp/core.spec.ts+++ b/tests/mcp/core.spec.ts@@ -58,7 +58,7 @@ test('browser_select_option', async ({ client, server }) => {- Page Title: Title- Page Snapshot:\`\`\`yaml-- combobox [ref=e2]:+- <changed> combobox [ref=e2]:- option "Foo"- option "Bar" [selected]\`\`\``,@@ -90,8 +90,8 @@ test('browser_select_option (multiple)', async ({ client, server }) => {})).toHaveResponse({code: `await page.getByRole('listbox').selectOption(['bar', 'baz']);`,pageState: expect.stringContaining(`- - option "Bar" [selected] [ref=e4]- - option "Baz" [selected] [ref=e5]`),+- <changed> option "Bar" [selected] [ref=e4]+- <changed> option "Baz" [selected] [ref=e5]`),});});diff --git a/tests/mcp/dialogs.spec.ts b/tests/mcp/dialogs.spec.tsindex 548a33d608..79993805fe 100644--- a/tests/mcp/dialogs.spec.ts+++ b/tests/mcp/dialogs.spec.ts@@ -139,7 +139,7 @@ test('confirm dialog (true)', async ({ client, server }) => {},})).toHaveResponse({modalState: undefined,- pageState: expect.stringContaining(`- generic [active] [ref=e1]: "true"`),+ pageState: expect.stringContaining(`generic [active] [ref=e1]: "true"`),});});@@ -175,7 +175,7 @@ test('confirm dialog (false)', async ({ client, server }) => {},})).toHaveResponse({modalState: undefined,- pageState: expect.stringContaining(`- generic [active] [ref=e1]: "false"`),+ pageState: expect.stringContaining(`generic [active] [ref=e1]: "false"`),});});@@ -213,7 +213,7 @@ test('prompt dialog', async ({ client, server }) => {});expect(result).toHaveResponse({- pageState: expect.stringContaining(`- generic [active] [ref=e1]: Answer`),+ pageState: expect.stringContaining(`generic [active] [ref=e1]: Answer`),});});diff --git a/tests/mcp/secrets.spec.ts b/tests/mcp/secrets.spec.tsindex 0f8cb47095..b2e497f307 100644--- a/tests/mcp/secrets.spec.ts+++ b/tests/mcp/secrets.spec.ts@@ -53,7 +53,7 @@ test('browser_type', async ({ startClient, server }) => {expect(response).toHaveResponse({code: `await page.getByRole('textbox').fill(process.env['X-PASSWORD']);await page.getByRole('textbox').press('Enter');`,- pageState: expect.stringContaining(`- textbox`),+ pageState: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: <secret>X-PASSWORD<\/secret>/),});}diff --git a/tests/mcp/snapshot-diff.spec.ts b/tests/mcp/snapshot-diff.spec.tsindex 4c4ffdae1e..a0654c8e27 100644--- a/tests/mcp/snapshot-diff.spec.ts+++ b/tests/mcp/snapshot-diff.spec.ts@@ -65,7 +65,7 @@ test('should return aria snapshot diff', async ({ client, server }) => {})).toHaveResponse({pageState: expect.stringContaining(`Page Snapshot:\`\`\`yaml-- ref=e1 [unchanged]+<no changes>\`\`\``),});@@ -78,7 +78,7 @@ test('should return aria snapshot diff', async ({ client, server }) => {})).toHaveResponse({pageState: expect.stringContaining(`Page Snapshot:\`\`\`yaml-- generic [ref=e1]:+- <changed> generic [ref=e1]:- button "Button 1" [active] [ref=e2]- button "Button 2new text" [ref=e105]- ref=e4 [unchanged]diff --git a/tests/mcp/type.spec.ts b/tests/mcp/type.spec.tsindex 7ac23141a3..70ddef9d31 100644--- a/tests/mcp/type.spec.ts+++ b/tests/mcp/type.spec.ts@@ -44,7 +44,7 @@ test('browser_type', async ({ client, server }) => {expect(response).toHaveResponse({code: `await page.getByRole('textbox').fill('Hi!');await page.getByRole('textbox').press('Enter');`,- pageState: expect.stringContaining(`- textbox`),+ pageState: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: Hi!/),});}@@ -79,7 +79,7 @@ test('browser_type (slowly)', async ({ client, server }) => {expect(response).toHaveResponse({code: `await page.getByRole('textbox').pressSequentially('Hi!');`,- pageState: expect.stringContaining(`- textbox`),+ pageState: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: Hi!/),});}const response = await client.callTool({diff --git a/tests/page/page-aria-snapshot-ai.spec.ts b/tests/page/page-aria-snapshot-ai.spec.tsindex 96861d8190..920cfbbcba 100644--- a/tests/page/page-aria-snapshot-ai.spec.ts+++ b/tests/page/page-aria-snapshot-ai.spec.ts@@ -510,7 +510,6 @@ it('should create incremental snapshots on multiple tracks', async ({ page }) =>- listitem [ref=e5]: a span`);expect(await snapshotForAI(page, { track: 'first', mode: 'incremental' })).toContainYaml(`- - ref=e2 [unchanged]`);await page.evaluate(() => {@@ -518,7 +517,7 @@ it('should create incremental snapshots on multiple tracks', async ({ page }) =>document.getElementById('hidden-li').style.display = 'inline';});expect(await snapshotForAI(page, { track: 'first', mode: 'incremental' })).toContainYaml(`- - list [ref=e2]:+ - <changed> list [ref=e2]:- ref=e3 [unchanged]- listitem [ref=e5]: changed span- listitem [ref=e6]: some text@@ -529,12 +528,11 @@ it('should create incremental snapshots on multiple tracks', async ({ page }) =>document.getElementById('hidden-li').style.display = 'none';});expect(await snapshotForAI(page, { track: 'first', mode: 'incremental' })).toContainYaml(`- - list [ref=e2]:+ - <changed> list [ref=e2]:- ref=e3 [unchanged]- listitem [ref=e5]: a span`);expect(await snapshotForAI(page, { track: 'second', mode: 'incremental' })).toContainYaml(`- - ref=e2 [unchanged]`);expect(await snapshotForAI(page, { track: 'second', mode: 'full' })).toContainYaml(`@@ -554,7 +552,7 @@ it('should create incremental snapshot for attribute change', async ({ page }) =await page.evaluate(() => document.querySelector('button').blur());expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`- - button "a button" [ref=e2]+ - <changed> button "a button" [ref=e2]`);});@@ -568,7 +566,7 @@ it('should create incremental snapshot for child removal', async ({ page }) => {await page.evaluate(() => document.querySelector('span').remove());expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`- - listitem [ref=e2]:+ - <changed> listitem [ref=e2]:- ref=e3 [unchanged]`);});@@ -582,7 +580,7 @@ it('should create incremental snapshot for child addition', async ({ page }) =>await page.evaluate(() => document.querySelector('span').style.display = 'inline');expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`- - listitem [ref=e2]:+ - <changed> listitem [ref=e2]:- ref=e3 [unchanged]- text: some text`);@@ -597,7 +595,7 @@ it('should create incremental snapshot for prop change', async ({ page }) => {await page.evaluate(() => document.querySelector('a').setAttribute('href', 'https://playwright.dev'));expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`- - link "a link" [ref=e2] [cursor=pointer]:+ - <changed> link "a link" [ref=e2] [cursor=pointer]:- /url: https://playwright.dev`);});@@ -611,7 +609,7 @@ it('should create incremental snapshot for cursor change', async ({ page }) => {await page.evaluate(() => document.querySelector('a').style.cursor = 'default');expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`- - link "a link" [ref=e2]:+ - <changed> link "a link" [ref=e2]:- /url: about:blank`);});@@ -624,7 +622,7 @@ it('should create incremental snapshot for name change', async ({ page }) => {await page.evaluate(() => document.querySelector('span').textContent = 'new button');expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`- - button "new button" [ref=e3]+ - <changed> button "new button" [ref=e3]`);});@@ -636,7 +634,7 @@ it('should create incremental snapshot for text change', async ({ page }) => {await page.evaluate(() => document.querySelector('span').textContent = 'new text');expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`- - listitem [ref=e2]: new text+ - <changed> listitem [ref=e2]: new text`);});@@ -649,8 +647,50 @@ it('should not produce incremental snapshot for iframes', async ({ page }) => {- heading "hello" [level=1] [ref=f1e2]`);expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`- - iframe [ref=e2]:- - ref=f1e2 [unchanged]+ - <changed> iframe [ref=e2]:+ - heading "hello" [level=1] [ref=f1e2]+ `);+});++it('should create multiple chunks in incremental snapshot', async ({ page }) => {+ await page.setContent(`+ <ul>+ <li><span>item1</span></li>+ <li><span>item2</span></li>+ <li><div role=group><span>item3</span></div></li>+ <ul>+ <li id=to-remove>to be removed</li>+ <li>one more</li>+ </ul>+ </ul>+ `);+ expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`+ - list [ref=e2]:+ - listitem [ref=e3]: item1+ - listitem [ref=e4]: item2+ - listitem [ref=e5]:+ - group [ref=e6]: item3+ - list [ref=e7]:+ - listitem [ref=e8]: to be removed+ - listitem [ref=e9]: one more+ `);++ await page.evaluate(() => {+ const spans = document.querySelectorAll('span');+ spans[0].textContent = 'new item1';+ spans[2].textContent = 'new item3';+ const button = document.createElement('button');+ button.textContent = 'button';+ spans[2].parentElement.appendChild(button);+ document.querySelector('#to-remove').remove();+ });+ expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`+ - <changed> listitem [ref=e3]: new item1+ - <changed> group [ref=e6]:+ - text: new item3+ - button "button" [ref=e10]+ - <changed> list [ref=e7]:+ - ref=e9 [unchanged]`);});@@ -686,7 +726,7 @@ it('should create incremental snapshot for children swap', async ({ page }) => {await page.evaluate(() => document.querySelector('ul').appendChild(document.querySelector('li')));expect(await snapshotForAI(page, { track: 'track', mode: 'incremental' })).toContainYaml(`- - list [ref=e2]:+ - <changed> list [ref=e2]:- ref=e4 [unchanged]- ref=e3 [unchanged]`);