Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(mcp): respond with snapshot diff when beneficial (#37500)
Dmitry Gozmanc92b9a1e3ee4
diff --git a/packages/playwright-core/src/utils/isomorphic/ariaSnapshot.ts b/packages/playwright-core/src/utils/isomorphic/ariaSnapshot.tsindex 21c5f9a359..86050018f7 100644--- a/packages/playwright-core/src/utils/isomorphic/ariaSnapshot.ts+++ b/packages/playwright-core/src/utils/isomorphic/ariaSnapshot.ts@@ -43,9 +43,13 @@ export type AriaTextValue = {normalized: string;};+// Character offsets in the parsed source.+type SourceRange = { from: number, to: number };+export type AriaTemplateTextNode = {kind: 'text';text: AriaTextValue;+ sourceRange?: SourceRange;};export type AriaTemplateRoleNode = AriaProps & {@@ -55,6 +59,8 @@ export type AriaTemplateRoleNode = AriaProps & {children?: AriaTemplateNode[];props?: Record<string, AriaTextValue>;containerMode?: 'contain' | 'equal' | 'deep-equal';+ sourceRange?: SourceRange;+ subtreeSourceRange?: SourceRange; // only present when different from sourceRange};export type AriaTemplateNode = AriaTemplateRoleNode | AriaTemplateTextNode;@@ -70,7 +76,7 @@ type YamlLibrary = {};type ParsedYamlPosition = { line: number; col: number; };-type ParsingOptions = yamlTypes.ParseOptions;+type ParsingOptions = yamlTypes.ParseOptions & { laxProps?: boolean };export type ParsedYamlError = {message: string;@@ -98,6 +104,13 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Parsreturn [lineCounter.linePos(range[0]), lineCounter.linePos(range[1])];};+ type WithYamlRange = { range?: yamlTypes.Range | null };+ const computeRange = (firstToken: WithYamlRange, lastToken: WithYamlRange): SourceRange | undefined => {+ if (!firstToken.range)+ return;+ return { from: firstToken.range[0], to: lastToken.range ? lastToken.range[2] : firstToken.range[2] };+ };+const addError = (error: yamlTypes.YAMLError) => {errors.push({message: error.message,@@ -111,6 +124,7 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Parsif (itemIsString) {const childNode = KeyParser.parse(item, parseOptions, errors);if (childNode) {+ childNode.sourceRange = computeRange(item, item);container.children = container.children || [];container.children.push(childNode);}@@ -156,7 +170,8 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Pars}container.children.push({kind: 'text',- text: textValue(value.value)+ text: textValue(value.value),+ sourceRange: computeRange(key, value),});continue;}@@ -211,8 +226,11 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Pars...childNode,children: [{kind: 'text',- text: textValue(String(value.value))- }]+ text: textValue(String(value.value)),+ sourceRange: computeRange(value, value),+ }],+ sourceRange: computeRange(key, key),+ subtreeSourceRange: computeRange(key, value),});continue;}@@ -222,6 +240,8 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Parsconst valueIsSequence = value instanceof yaml.YAMLSeq;if (valueIsSequence) {container.children.push(childNode);+ childNode.sourceRange = computeRange(key, key);+ childNode.subtreeSourceRange = computeRange(key, value);convertSeq(childNode, value as yamlTypes.YAMLSeq);continue;}@@ -233,7 +253,13 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Pars}};- const fragment: AriaTemplateNode = { kind: 'role', role: 'fragment' };+ const emptyRange: WithYamlRange = { range: [0, 0, 0] }; // Fragment has no "self" source, only subtree.+ const fragment: AriaTemplateNode = {+ kind: 'role',+ role: 'fragment',+ sourceRange: computeRange(emptyRange, emptyRange),+ subtreeSourceRange: computeRange(emptyRange, yamlDoc),+ };yamlDoc.errors.forEach(addError);if (errors.length)@@ -272,13 +298,14 @@ export function textValue(value: string): AriaTextValue {}export class KeyParser {+ private _options: ParsingOptions;private _input: string;private _pos: number;private _length: number;static parse(text: yamlTypes.Scalar<string>, options: ParsingOptions, errors: ParsedYamlError[]): AriaTemplateRoleNode | null {try {- return new KeyParser(text.value)._parse();+ return new KeyParser(text.value, options)._parse();} catch (e) {if (e instanceof ParserError) {const message = options.prettyErrors === false ? e.message : e.message + ':\n\n' + text.value + '\n' + ' '.repeat(e.pos) + '^\n';@@ -292,7 +319,8 @@ export class KeyParser {}}- constructor(input: string) {+ constructor(input: string, options: ParsingOptions) {+ this._options = options;this._input = input;this._pos = 0;this._length = input.length;@@ -475,6 +503,11 @@ export class KeyParser {node.selected = value === 'true';return;}+ if (this._options.laxProps) {+ node.props = node.props || {};+ node.props[key] = textValue(value);+ return;+ }this._assert(false, `Unsupported attribute [${key}]`, errorPos);}@@ -492,3 +525,69 @@ export class ParserError extends Error {this.pos = pos;}}++type AriaSnapshotDiffResult = 'equal' | 'different' | { ref: string, newSource: string }[];++export function diffAriaSnapshots(yaml: YamlLibrary, oldSnapshot: string, newSnapshot: string): AriaSnapshotDiffResult {+ const diffTree = (oldNode: AriaTemplateNode, newNode: AriaTemplateNode): AriaSnapshotDiffResult => {+ if (!oldNode.sourceRange || !newNode.sourceRange)+ return 'different';++ const oldSelfSource = oldSnapshot.slice(oldNode.sourceRange.from, oldNode.sourceRange.to);+ const newSelfSource = newSnapshot.slice(newNode.sourceRange.from, newNode.sourceRange.to);+ if (oldNode.kind !== 'role' || newNode.kind !== 'role')+ return (oldNode.kind === newNode.kind && oldSelfSource === newSelfSource) ? 'equal' : 'different';++ const newNodeSubtreeSourceRange = newNode.subtreeSourceRange || newNode.sourceRange;+ const newSubtreeSource = newSnapshot.slice(newNodeSubtreeSourceRange.from, newNodeSubtreeSourceRange.to);++ const oldChildren = oldNode.children || [];+ const newChildren = newNode.children || [];+ const childrenDiffs = [];+ // When "self" is the same, we can diff children and try to find a small diff there.+ let useChildrenDiffs = oldSelfSource === newSelfSource && oldChildren.length === newChildren.length;+ let childrenTotalLength = 0;++ if (useChildrenDiffs) {+ for (let i = 0; i < oldChildren.length; i++) {+ const childDiff = diffTree(oldChildren[i], newChildren[i]);+ if (childDiff === 'equal')+ continue;+ if (childDiff === 'different') {+ useChildrenDiffs = false;+ break;+ }+ for (const diff of childDiff) {+ childrenTotalLength += diff.newSource.length;+ childrenDiffs.push(diff);+ }+ }+ }++ if (childrenDiffs.length > 1 && childrenTotalLength * 2 >= newSubtreeSource.length) {+ // Too many children diffs without too much of a benefit.+ useChildrenDiffs = false;+ }++ if (useChildrenDiffs) {+ if (!childrenDiffs.length)+ return 'equal';+ return childrenDiffs;+ }++ const oldRef = oldNode.props?.ref?.raw;+ const newRef = newNode.props?.ref?.raw;+ if (!oldRef || oldRef !== newRef)+ return 'different';++ return [{ ref: oldRef, newSource: newSubtreeSource }];+ };++ try {+ const oldParsed = parseAriaSnapshotUnsafe(yaml, oldSnapshot, { laxProps: true });+ const newParsed = parseAriaSnapshotUnsafe(yaml, newSnapshot, { laxProps: true });+ return diffTree(oldParsed, newParsed);+ } catch {+ return 'different';+ }+}diff --git a/packages/playwright/src/mcp/browser/response.ts b/packages/playwright/src/mcp/browser/response.tsindex db910cc222..348f03e355 100644--- a/packages/playwright/src/mcp/browser/response.ts+++ b/packages/playwright/src/mcp/browser/response.ts@@ -25,7 +25,7 @@ export class Response {private _code: string[] = [];private _images: { contentType: string, data: Buffer }[] = [];private _context: Context;- private _includeSnapshot = false;+ private _includeSnapshot: 'full' | 'partial' | 'none' = 'none';private _includeTabs = false;private _tabSnapshot: TabSnapshot | undefined;@@ -72,8 +72,8 @@ export class Response {return this._images;}- setIncludeSnapshot() {- this._includeSnapshot = true;+ setIncludeSnapshot(full?: 'full') {+ this._includeSnapshot = full ?? 'partial';}setIncludeTabs() {@@ -83,7 +83,7 @@ export class Response {async finish() {// All the async snapshotting post-action is happening here.// Everything below should race against modal states.- if (this._includeSnapshot && this._context.currentTab())+ if (this._includeSnapshot !== 'none' && this._context.currentTab())this._tabSnapshot = await this._context.currentTabOrDie().captureSnapshot();for (const tab of this._context.tabs())await tab.updateTitle();@@ -113,7 +113,7 @@ ${this._code.join('\n')}}// List browser tabs.- if (this._includeSnapshot || this._includeTabs)+ if (this._includeSnapshot !== 'none' || this._includeTabs)response.push(...renderTabsMarkdown(this._context.tabs(), this._includeTabs));// Add snapshot if provided.@@ -121,7 +121,7 @@ ${this._code.join('\n')}response.push(...renderModalStates(this._context, this._tabSnapshot.modalStates));response.push('');} else if (this._tabSnapshot) {- response.push(renderTabSnapshot(this._tabSnapshot));+ response.push(renderTabSnapshot(this._tabSnapshot, this._includeSnapshot === 'full'));response.push('');}@@ -153,7 +153,7 @@ ${this._code.join('\n')}}}-function renderTabSnapshot(tabSnapshot: TabSnapshot): string {+function renderTabSnapshot(tabSnapshot: TabSnapshot, fullSnapshot: boolean): string {const lines: string[] = [];if (tabSnapshot.consoleMessages.length) {@@ -177,10 +177,15 @@ function renderTabSnapshot(tabSnapshot: TabSnapshot): string {lines.push(`### Page state`);lines.push(`- Page URL: ${tabSnapshot.url}`);lines.push(`- Page Title: ${tabSnapshot.title}`);- lines.push(`- Page Snapshot:`);- lines.push('```yaml');- lines.push(tabSnapshot.ariaSnapshot);- lines.push('```');+ if (!fullSnapshot && tabSnapshot.formattedAriaSnapshotDiff) {+ lines.push(`- Page Snapshot Diff:`);+ lines.push(tabSnapshot.formattedAriaSnapshotDiff);+ } else {+ lines.push(`- Page Snapshot:`);+ lines.push('```yaml');+ lines.push(tabSnapshot.ariaSnapshot);+ lines.push('```');+ }return lines.join('\n');}diff --git a/packages/playwright/src/mcp/browser/tab.ts b/packages/playwright/src/mcp/browser/tab.tsindex 1669d51767..b0190b6c24 100644--- a/packages/playwright/src/mcp/browser/tab.ts+++ b/packages/playwright/src/mcp/browser/tab.ts@@ -16,7 +16,8 @@import { EventEmitter } from 'events';import * as playwright from 'playwright-core';-import { ManualPromise } from 'playwright-core/lib/utils';+import { ManualPromise, diffAriaSnapshots } from 'playwright-core/lib/utils';+import { yaml } from 'playwright-core/lib/utilsBundle';import { callOnPageNoTrace, waitForCompletion } from './tools/utils';import { logUnhandledError } from '../log';@@ -40,6 +41,7 @@ export type TabSnapshot = {url: string;title: string;ariaSnapshot: string;+ formattedAriaSnapshotDiff?: string;modalStates: ModalState[];consoleMessages: ConsoleMessage[];downloads: { download: playwright.Download, finished: boolean, outputFile: string }[];@@ -55,6 +57,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {private _onPageClose: (tab: Tab) => void;private _modalStates: ModalState[] = [];private _downloads: { download: playwright.Download, finished: boolean, outputFile: string }[] = [];+ private _lastAriaSnapshot: string | undefined;constructor(context: Context, page: playwright.Page, onPageClose: (tab: Tab) => void) {super();@@ -63,7 +66,14 @@ export class Tab extends EventEmitter<TabEventsInterface> {this._onPageClose = onPageClose;page.on('console', event => this._handleConsoleMessage(messageToConsoleMessage(event)));page.on('pageerror', error => this._handleConsoleMessage(pageErrorToConsoleMessage(error)));- page.on('request', request => this._requests.set(request, null));+ page.on('request', request => {+ this._requests.set(request, null);+ // Note: unfortunately, 'framenavigated' event does not differentiate between+ // a same-document (#hash or pushState) navigation and a new document being loaded.+ // Relying upon a navigation request heuristic.+ if (request.frame() === page.mainFrame() && request.isNavigationRequest())+ this._willNavigateMainFrameToNewDocument();+ });page.on('response', response => this._requests.set(response.request(), response));page.on('close', () => this._onClose());page.on('filechooser', chooser => {@@ -140,6 +150,10 @@ export class Tab extends EventEmitter<TabEventsInterface> {this._onPageClose(this);}+ private _willNavigateMainFrameToNewDocument() {+ this._lastAriaSnapshot = undefined;+ }+async updateTitle() {await this._raceAgainstModalStates(async () => {this._lastTitle = await callOnPageNoTrace(this.page, page => page.title());@@ -160,6 +174,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {async navigate(url: string) {this._clearCollectedArtifacts();+ this._willNavigateMainFrameToNewDocument();const downloadEvent = callOnPageNoTrace(this.page, page => page.waitForEvent('download').catch(logUnhandledError));try {@@ -203,6 +218,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {url: this.page.url(),title: await this.page.title(),ariaSnapshot: snapshot,+ formattedAriaSnapshotDiff: this._lastAriaSnapshot ? generateAriaSnapshotDiff(this._lastAriaSnapshot, snapshot) : undefined,modalStates: [],consoleMessages: [],downloads: this._downloads,@@ -212,6 +228,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {// Assign console message late so that we did not lose any to modal state.tabSnapshot.consoleMessages = this._recentConsoleMessages;this._recentConsoleMessages = [];+ this._lastAriaSnapshot = tabSnapshot.ariaSnapshot;}return tabSnapshot ?? {url: this.page.url(),@@ -312,3 +329,22 @@ export function renderModalStates(context: Context, modalStates: ModalState[]):}const tabSymbol = Symbol('tabSymbol');++function generateAriaSnapshotDiff(oldSnapshot: string, newSnapshot: string) {+ const diffs = diffAriaSnapshots(yaml, oldSnapshot, newSnapshot);+ if (diffs === 'equal')+ return '<no changes>';+ if (diffs === 'different')+ return;+ if (diffs.length > 3 || diffs.some(diff => diff.newSource.split('\n').length > 100)) {+ // Being conservative - up to 3 small diff changes, otherwise include full snapshot.+ return;+ }+ const lines = [`The following refs have changed`];+ for (const diff of diffs)+ lines.push('', '```yaml', diff.newSource.trimEnd(), '```');+ const combined = lines.join('\n');+ if (combined.length >= newSnapshot.length)+ return;+ return combined;+}diff --git a/packages/playwright/src/mcp/browser/tools/snapshot.ts b/packages/playwright/src/mcp/browser/tools/snapshot.tsindex bea116ee8f..c7e7cc539f 100644--- a/packages/playwright/src/mcp/browser/tools/snapshot.ts+++ b/packages/playwright/src/mcp/browser/tools/snapshot.ts@@ -31,7 +31,7 @@ const snapshot = defineTool({handle: async (context, params, response) => {await context.ensureTab();- response.setIncludeSnapshot();+ response.setIncludeSnapshot('full');},});diff --git a/tests/library/unit/parse-aria-snapshot.spec.ts b/tests/library/unit/parse-aria-snapshot.spec.tsnew file mode 100644index 0000000000..9aac6bee16--- /dev/null+++ b/tests/library/unit/parse-aria-snapshot.spec.ts@@ -0,0 +1,309 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { test, expect } from '@playwright/test';+import { parseAriaSnapshotUnsafe, diffAriaSnapshots } from '../../../packages/playwright-core/lib/utils/isomorphic/ariaSnapshot.js';+import { yaml } from 'playwright-core/lib/utilsBundle';++function addSource(text: string, node: any) {+ if (node.sourceRange)+ node.source = text.slice(node.sourceRange.from, node.sourceRange.to).trim();+ if (node.subtreeSourceRange)+ node.subtreeSource = text.slice(node.subtreeSourceRange.from, node.subtreeSourceRange.to).trim();+ (node.children || []).forEach(child => addSource(text, child));+}++function parse(text: string) {+ const result = parseAriaSnapshotUnsafe(yaml, text, { laxProps: true });+ addSource(text, result);+ return result;+}++function diff(oldSnapshot: string, newSnapshot: string) {+ const diffs = diffAriaSnapshots(yaml, oldSnapshot, newSnapshot);+ if (Array.isArray(diffs))+ diffs.forEach(diff => diff.newSource = diff.newSource.trimEnd());+ return diffs;+}++test('parse aria snapshot returns source ranges', () => {+ const listSnapshot = `list "list name" [ref=e1]:+ - listitem: item 1+ - listitem "item 2":+ - button "click me" [ref=e2]+ - text: hello+ - listitem:+ - generic: text1+ - generic [ref=e3] [active] [cursor=pointer]: text2`;++ const snapshot = `+- ${listSnapshot}+- button "another button" [ref=e4]+`;++ expect(parse(snapshot)).toEqual(expect.objectContaining({+ role: 'fragment',+ source: '',+ subtreeSource: snapshot.trim(),+ children: [+ expect.objectContaining({+ role: 'list',+ name: 'list name',+ props: { ref: expect.objectContaining({ raw: 'e1' }) },+ source: 'list "list name" [ref=e1]',+ subtreeSource: listSnapshot,+ children: [+ expect.objectContaining({+ role: 'listitem',+ source: 'listitem',+ subtreeSource: 'listitem: item 1',+ children: [+ expect.objectContaining({+ kind: 'text',+ text: expect.objectContaining({ raw: 'item 1' }),+ source: 'item 1',+ }),+ ],+ }),+ expect.objectContaining({+ role: 'listitem',+ name: 'item 2',+ source: 'listitem "item 2"',+ subtreeSource: 'listitem "item 2":\n - button "click me" [ref=e2]\n - text: hello',+ children: [+ expect.objectContaining({+ role: 'button',+ name: 'click me',+ props: { ref: expect.objectContaining({ raw: 'e2' }) },+ source: 'button "click me" [ref=e2]',+ }),+ expect.objectContaining({+ kind: 'text',+ text: expect.objectContaining({ raw: 'hello' }),+ source: 'text: hello',+ }),+ ],+ }),+ expect.objectContaining({+ role: 'listitem',+ source: 'listitem',+ subtreeSource: 'listitem:\n - generic: text1\n - generic [ref=e3] [active] [cursor=pointer]: text2',+ children: [+ expect.objectContaining({+ role: 'generic',+ source: 'generic',+ subtreeSource: 'generic: text1',+ children: [+ expect.objectContaining({+ kind: 'text',+ text: expect.objectContaining({ raw: 'text1' }),+ source: 'text1',+ }),+ ],+ }),+ expect.objectContaining({+ role: 'generic',+ props: { ref: expect.objectContaining({ raw: 'e3' }), cursor: expect.objectContaining({ raw: 'pointer' }) },+ active: true,+ source: 'generic [ref=e3] [active] [cursor=pointer]',+ subtreeSource: 'generic [ref=e3] [active] [cursor=pointer]: text2',+ children: [+ expect.objectContaining({+ kind: 'text',+ text: expect.objectContaining({ raw: 'text2' }),+ source: 'text2',+ }),+ ],+ }),+ ],+ }),+ ],+ }),+ expect.objectContaining({+ role: 'button',+ name: 'another button',+ props: { ref: expect.objectContaining({ raw: 'e4' }) },+ source: 'button "another button" [ref=e4]',+ }),+ ],+ }));+});++test('diff: equal', () => {+ expect(diff(`- button "name"`, `- button "name"`)).toBe('equal');+});++test('diff: different', () => {+ expect(diff(`- button "name"`, `- button "new name"`)).toBe('different');+});++test('diff: one ref', () => {+ expect(diff(`+- button "button 1" [ref=e1]+`, `+- button "button 2" [ref=e1]+`)).toEqual([+ {+ ref: 'e1',+ newSource: 'button "button 2" [ref=e1]',+ }+ ]);+});++test('diff: no ref upgrades to parent', () => {+ expect(diff(`+- listitem [ref=e6]:+ - generic: some text+`, `+- listitem [ref=e6]:+ - generic: some text has changed+`)).toEqual([+ {+ ref: 'e6',+ newSource: `listitem [ref=e6]:+ - generic: some text has changed`,+ }+ ]);+});++test('diff: two refs in a large parent', () => {+ expect(diff(`+- listitem [ref=e6]: some text+- listitem+- listitem+- listitem+- listitem+- listitem+- listitem+- listitem+- listitem [ref=e7]: other text+`, `+- listitem [ref=e6]: some new text+- listitem+- listitem+- listitem+- listitem+- listitem+- listitem+- listitem+- listitem [ref=e7]: other new text+`)).toEqual([+ {+ ref: 'e6',+ newSource: `listitem [ref=e6]: some new text`,+ },+ {+ ref: 'e7',+ newSource: `listitem [ref=e7]: other new text`,+ },+ ]);+});++test('diff: mixed', () => {+ const snapshot1 = `+ - button "button 1" [ref=e1] [active]+ - button "padding"+ - button "button 2" [ref=e2]+ - button "padding"+ - button "padding"+ - list "list name" [ref=e3]:+ - listitem [ref=e4]: item 1+ - listitem "item 2" [ref=e5]+ - button "padding"+ - button "padding"+ - button "padding"+ - button "padding"+ - listitem [ref=e6]:+ - generic: some text+ - button "padding"+ - button "padding"+ - button "padding"+ - button "padding"+ - generic:+ - listitem [ref=e10]: more text+ - listitem "item name" [ref=e7]+ - listitem [ref=e8]:+ - listitem "deep item" [ref=e9]+ `;+ const snapshot2 = `+ - button "button 1" [ref=e1]+ - button "padding"+ - button "button 2" [ref=e2] [active]+ - button "padding"+ - button "padding"+ - list "list name" [ref=e3]:+ - listitem [ref=e4]: item 1+ - listitem "item 2" [ref=e5]:+ - button "new button 1" [ref=e100]+ - button "padding"+ - button "padding"+ - button "padding"+ - button "padding"+ - listitem [ref=e6]:+ - generic: some text has changed+ - button "padding"+ - button "padding"+ - button "padding"+ - button "padding"+ - generic:+ - listitem [ref=e10]: more text has changed+ - listitem "new item name" [ref=e7]+ - listitem [ref=e8]:+ - listitem "new ref" [ref=e101]+ `;+ expect(diff(snapshot1, snapshot2)).toEqual([+ { ref: 'e1', newSource: 'button "button 1" [ref=e1]' },+ { ref: 'e2', newSource: 'button "button 2" [ref=e2] [active]' },+ { ref: 'e5', newSource: `listitem "item 2" [ref=e5]:+ - button "new button 1" [ref=e100]` },+ { ref: 'e6', newSource: `listitem [ref=e6]:+ - generic: some text has changed` },+ { ref: 'e10', newSource: 'listitem [ref=e10]: more text has changed' },+ { ref: 'e7', newSource: `listitem "new item name" [ref=e7]` },+ { ref: 'e8', newSource: `listitem [ref=e8]:+ - listitem "new ref" [ref=e101]` },+ ]);+});++test('diff: two refs in a small parent are combined', () => {+ expect(diff(`+- button "name"+- list [ref=e1]:+ - listitem [ref=e6]: some text+ - listitem+ - listitem+ - listitem+ - listitem [ref=e7]: other text+`, `+- button "name"+- list [ref=e1]:+ - listitem [ref=e6]: some new text+ - listitem+ - listitem+ - listitem+ - listitem [ref=e7]: other new text+`)).toEqual([+ {+ ref: 'e1',+ newSource: `list [ref=e1]:+ - listitem [ref=e6]: some new text+ - listitem+ - listitem+ - listitem+ - listitem [ref=e7]: other new text`,+ },+ ]);+});diff --git a/tests/mcp/click.spec.ts b/tests/mcp/click.spec.tsindex 6f6a06dbd0..169d57dca4 100644--- a/tests/mcp/click.spec.ts+++ b/tests/mcp/click.spec.ts@@ -16,10 +16,16 @@import { test, expect } from './fixtures';-test('browser_click', async ({ client, server, mcpBrowser }) => {+test('browser_click', async ({ client, server }) => {server.setContent('/', `<title>Title</title><button>Submit</button>+ <script>+ const button = document.querySelector('button');+ button.addEventListener('click', () => {+ button.focus(); // without manual focus, webkit focuses body+ });+ </script>`, 'text/html');await client.callTool({@@ -35,7 +41,7 @@ test('browser_click', async ({ client, server, mcpBrowser }) => {},})).toHaveResponse({code: `await page.getByRole('button', { name: 'Submit' }).click();`,- pageState: expect.stringContaining(`- button "Submit" ${mcpBrowser !== 'webkit' || process.platform === 'linux' ? '[active] ' : ''}[ref=e2]`),+ pageState: expect.stringContaining(`- button "Submit" [active] [ref=e2]`),});});@@ -125,7 +131,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {},})).toHaveResponse({code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Control'] });`,- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:true metaKey:false shiftKey:false altKey:false`),+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:true metaKey:false shiftKey:false altKey:false`),});}@@ -138,7 +144,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {},})).toHaveResponse({code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Shift'] });`,- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:false`),+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:false`),});expect(await client.callTool({@@ -150,6 +156,6 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {},})).toHaveResponse({code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Shift', 'Alt'] });`,- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:true`),+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:true`),});});diff --git a/tests/mcp/dialogs.spec.ts b/tests/mcp/dialogs.spec.tsindex 4c2bf98eb5..fa2be5cc73 100644--- a/tests/mcp/dialogs.spec.ts+++ b/tests/mcp/dialogs.spec.ts@@ -17,7 +17,7 @@import { test, expect } from './fixtures';test('alert dialog', async ({ client, server }) => {- server.setContent('/', `<button onclick="alert('Alert')">Button</button>`, 'text/html');+ server.setContent('/', `<title>Title</title><button onclick="alert('Alert')">Button</button>`, 'text/html');expect(await client.callTool({name: 'browser_navigate',arguments: { url: server.PREFIX },@@ -54,7 +54,7 @@ test('alert dialog', async ({ client, server }) => {},})).toHaveResponse({modalState: undefined,- pageState: expect.stringContaining(`- button "Button"`),+ pageState: expect.stringContaining(`Page Title: Title`),});});@@ -218,7 +218,7 @@ test('prompt dialog', async ({ client, server }) => {});test('alert dialog w/ race', async ({ client, server }) => {- server.setContent('/', `<button onclick="setTimeout(() => alert('Alert'), 100)">Button</button>`, 'text/html');+ server.setContent('/', `<title>Title</title><button onclick="setTimeout(() => alert('Alert'), 100)">Button</button>`, 'text/html');expect(await client.callTool({name: 'browser_navigate',arguments: { url: server.PREFIX },@@ -247,9 +247,6 @@ test('alert dialog w/ race', async ({ client, server }) => {expect(result).toHaveResponse({modalState: undefined,pageState: expect.stringContaining(`- Page URL: ${server.PREFIX}/-- Page Title:-- Page Snapshot:-\`\`\`yaml-- button "Button"`),+- Page Title: Title`),});});diff --git a/tests/mcp/session-log.spec.ts b/tests/mcp/session-log.spec.tsindex 6f9a98ee66..7e072aa55b 100644--- a/tests/mcp/session-log.spec.ts+++ b/tests/mcp/session-log.spec.ts@@ -42,7 +42,6 @@ test('session log should record tool calls', async ({ startClient, server }, tes},})).toHaveResponse({code: `await page.getByRole('button', { name: 'Submit' }).click();`,- pageState: expect.stringContaining(`- button "Submit"`),});const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];diff --git a/tests/mcp/snapshot-diff.spec.ts b/tests/mcp/snapshot-diff.spec.tsnew file mode 100644index 0000000000..f5ca5346c0--- /dev/null+++ b/tests/mcp/snapshot-diff.spec.ts@@ -0,0 +1,163 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { test, expect } from './fixtures';++test('should return aria snapshot diff', async ({ client, server }) => {+ server.setContent('/', `+ <button>Button 1</button>+ <button>Button 2</button>+ <ul id=filler></ul>+ <script>+ const filler = document.getElementById('filler');+ for (let i = 0; i < 100; i++) {+ const li = document.createElement('li');+ li.textContent = 'Filler ' + i;+ filler.appendChild(li);+ }+ const [button1, button2] = document.querySelectorAll('button');+ button1.addEventListener('click', () => {+ const button = document.createElement('button');+ button.textContent = 'new button';+ filler.firstChild.appendChild(button);+ button1.focus(); // without manual focus, webkit focuses body+ });+ button2.focus();+ button2.addEventListener('click', () => {+ button2.focus(); // without manual focus, webkit focuses body+ });+ </script>+ `, 'text/html');++ const listitems = new Array(100).fill(0).map((_, i) => `\n - listitem [ref=e${5 + i}]: Filler ${i}`).join('');++ expect(await client.callTool({+ name: 'browser_navigate',+ arguments: {+ url: server.PREFIX,+ },+ })).toHaveResponse({+ pageState: expect.stringContaining(`+ - button "Button 1" [ref=e2]+ - button "Button 2" [active] [ref=e3]+ - list [ref=e4]:${listitems}`),+ });++ expect(await client.callTool({+ name: 'browser_click',+ arguments: {+ element: 'Button 2',+ ref: 'e3',+ },+ })).toHaveResponse({+ pageState: expect.stringContaining(`+- Page Snapshot Diff:+<no changes>`),+ });++ expect(await client.callTool({+ name: 'browser_click',+ arguments: {+ element: 'Button 1',+ ref: 'e2',+ },+ })).toHaveResponse({+ pageState: expect.stringContaining(`+- Page Snapshot Diff:+The following refs have changed++\`\`\`yaml+button "Button 1" [active] [ref=e2]+\`\`\`++\`\`\`yaml+button "Button 2" [ref=e3]+\`\`\`++\`\`\`yaml+listitem [ref=e5]:+ - text: Filler 0+ - button "new button" [ref=e105]+\`\`\``),+ });++ expect(await client.callTool({+ name: 'browser_snapshot',+ })).toHaveResponse({+ pageState: expect.stringContaining(`+- Page Snapshot:`),+ });+});++test('should reset aria snapshot diff upon navigation', async ({ client, server }) => {+ server.setContent('/before', `+ <button>Button 1</button>+ <button>Button 2</button>+ <ul id=filler></ul>+ <script>+ const filler = document.getElementById('filler');+ for (let i = 0; i < 100; i++) {+ const li = document.createElement('li');+ li.textContent = 'Filler ' + i;+ filler.appendChild(li);+ }+ document.querySelector('button').addEventListener('click', () => {+ window.location.href = '/after';+ });+ document.querySelectorAll('button')[1].focus();+ </script>+ `, 'text/html');++ server.setContent('/after', `+ <button>Button 1</button>+ <button>Button 2</button>+ <ul id=filler></ul>+ <script>+ const filler = document.getElementById('filler');+ for (let i = 0; i < 100; i++) {+ const li = document.createElement('li');+ li.textContent = 'Filler ' + i;+ filler.appendChild(li);+ }+ const button = document.createElement('button');+ button.textContent = 'new button';+ filler.lastChild.appendChild(button);+ document.querySelectorAll('button')[0].focus();+ </script>+ `, 'text/html');++ expect(await client.callTool({+ name: 'browser_navigate',+ arguments: {+ url: server.PREFIX + '/before',+ },+ })).toHaveResponse({+ pageState: expect.stringContaining(`+ - button "Button 1" [ref=e2]+ - button "Button 2" [active] [ref=e3]`),+ });++ expect(await client.callTool({+ name: 'browser_click',+ arguments: {+ element: 'Button 1',+ ref: 'e2',+ },+ })).toHaveResponse({+ pageState: expect.stringContaining(`+- Page Snapshot:`),+ });+});diff --git a/tests/mcp/tabs.spec.ts b/tests/mcp/tabs.spec.tsindex a0b40d824b..bc59f2d7a5 100644--- a/tests/mcp/tabs.spec.ts+++ b/tests/mcp/tabs.spec.ts@@ -98,10 +98,8 @@ test('select tab', async ({ client }) => {- 2: [Tab two] (data:text/html,<title>Tab two</title><body>Body two</body>)`,pageState: expect.stringContaining(`- Page URL: data:text/html,<title>Tab one</title><body>Body one</body>- Page Title: Tab one-- Page Snapshot:-\`\`\`yaml-- generic [active] [ref=e1]: Body one-\`\`\``),+- Page Snapshot Diff:+<no changes>`),});expect(await client.callTool({@@ -133,10 +131,8 @@ test('close tab', async ({ client }) => {- 1: (current) [Tab one] (data:text/html,<title>Tab one</title><body>Body one</body>)`,pageState: expect.stringContaining(`- Page URL: data:text/html,<title>Tab one</title><body>Body one</body>- Page Title: Tab one-- Page Snapshot:-\`\`\`yaml-- generic [active] [ref=e1]: Body one-\`\`\``),+- Page Snapshot Diff:+<no changes>`),});});diff --git a/tests/mcp/wait.spec.ts b/tests/mcp/wait.spec.tsindex 0388faf259..e844eac4ea 100644--- a/tests/mcp/wait.spec.ts+++ b/tests/mcp/wait.spec.ts@@ -31,9 +31,11 @@ test('browser_wait_for(text)', async ({ client, server }) => {</body>`, 'text/html');- await client.callTool({+ expect(await client.callTool({name: 'browser_navigate',arguments: { url: server.PREFIX },+ })).toHaveResponse({+ pageState: expect.stringContaining(`- generic [ref=e3]: Text to disappear`),});await client.callTool({@@ -44,10 +46,14 @@ test('browser_wait_for(text)', async ({ client, server }) => {},});- expect(await client.callTool({+ await client.callTool({name: 'browser_wait_for',arguments: { text: 'Text to appear' },code: `await page.getByText("Text to appear").first().waitFor({ state: 'visible' });`,+ });++ expect(await client.callTool({+ name: 'browser_snapshot',})).toHaveResponse({pageState: expect.stringContaining(`- generic [ref=e3]: Text to appear`),});@@ -68,9 +74,11 @@ test('browser_wait_for(textGone)', async ({ client, server }) => {</body>`, 'text/html');- await client.callTool({+ expect(await client.callTool({name: 'browser_navigate',arguments: { url: server.PREFIX },+ })).toHaveResponse({+ pageState: expect.stringContaining(`- generic [ref=e3]: Text to disappear`),});await client.callTool({@@ -81,10 +89,14 @@ test('browser_wait_for(textGone)', async ({ client, server }) => {},});- expect(await client.callTool({+ await client.callTool({name: 'browser_wait_for',arguments: { textGone: 'Text to disappear' },code: `await page.getByText("Text to disappear").first().waitFor({ state: 'hidden' });`,+ });++ expect(await client.callTool({+ name: 'browser_snapshot',})).toHaveResponse({pageState: expect.stringContaining(`- generic [ref=e3]: Text to appear`),});