Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(mcp): respond with snapshot diff when beneficial (#37500)

Dmitry Gozmanc92b9a1e3ee4
diff --git a/packages/playwright-core/src/utils/isomorphic/ariaSnapshot.ts b/packages/playwright-core/src/utils/isomorphic/ariaSnapshot.ts
index 21c5f9a359..86050018f7 100644
--- a/packages/playwright-core/src/utils/isomorphic/ariaSnapshot.ts
+++ b/packages/playwright-core/src/utils/isomorphic/ariaSnapshot.ts
@@ -43,9 +43,13 @@ export type AriaTextValue = {
normalized: string;
};
+// Character offsets in the parsed source.
+type SourceRange = { from: number, to: number };
+
export type AriaTemplateTextNode = {
kind: 'text';
text: AriaTextValue;
+ sourceRange?: SourceRange;
};
export type AriaTemplateRoleNode = AriaProps & {
@@ -55,6 +59,8 @@ export type AriaTemplateRoleNode = AriaProps & {
children?: AriaTemplateNode[];
props?: Record<string, AriaTextValue>;
containerMode?: 'contain' | 'equal' | 'deep-equal';
+ sourceRange?: SourceRange;
+ subtreeSourceRange?: SourceRange; // only present when different from sourceRange
};
export type AriaTemplateNode = AriaTemplateRoleNode | AriaTemplateTextNode;
@@ -70,7 +76,7 @@ type YamlLibrary = {
};
type ParsedYamlPosition = { line: number; col: number; };
-type ParsingOptions = yamlTypes.ParseOptions;
+type ParsingOptions = yamlTypes.ParseOptions & { laxProps?: boolean };
export type ParsedYamlError = {
message: string;
@@ -98,6 +104,13 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Pars
return [lineCounter.linePos(range[0]), lineCounter.linePos(range[1])];
};
+ type WithYamlRange = { range?: yamlTypes.Range | null };
+ const computeRange = (firstToken: WithYamlRange, lastToken: WithYamlRange): SourceRange | undefined => {
+ if (!firstToken.range)
+ return;
+ return { from: firstToken.range[0], to: lastToken.range ? lastToken.range[2] : firstToken.range[2] };
+ };
+
const addError = (error: yamlTypes.YAMLError) => {
errors.push({
message: error.message,
@@ -111,6 +124,7 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Pars
if (itemIsString) {
const childNode = KeyParser.parse(item, parseOptions, errors);
if (childNode) {
+ childNode.sourceRange = computeRange(item, item);
container.children = container.children || [];
container.children.push(childNode);
}
@@ -156,7 +170,8 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Pars
}
container.children.push({
kind: 'text',
- text: textValue(value.value)
+ text: textValue(value.value),
+ sourceRange: computeRange(key, value),
});
continue;
}
@@ -211,8 +226,11 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Pars
...childNode,
children: [{
kind: 'text',
- text: textValue(String(value.value))
- }]
+ text: textValue(String(value.value)),
+ sourceRange: computeRange(value, value),
+ }],
+ sourceRange: computeRange(key, key),
+ subtreeSourceRange: computeRange(key, value),
});
continue;
}
@@ -222,6 +240,8 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Pars
const valueIsSequence = value instanceof yaml.YAMLSeq;
if (valueIsSequence) {
container.children.push(childNode);
+ childNode.sourceRange = computeRange(key, key);
+ childNode.subtreeSourceRange = computeRange(key, value);
convertSeq(childNode, value as yamlTypes.YAMLSeq);
continue;
}
@@ -233,7 +253,13 @@ export function parseAriaSnapshot(yaml: YamlLibrary, text: string, options: Pars
}
};
- const fragment: AriaTemplateNode = { kind: 'role', role: 'fragment' };
+ const emptyRange: WithYamlRange = { range: [0, 0, 0] }; // Fragment has no "self" source, only subtree.
+ const fragment: AriaTemplateNode = {
+ kind: 'role',
+ role: 'fragment',
+ sourceRange: computeRange(emptyRange, emptyRange),
+ subtreeSourceRange: computeRange(emptyRange, yamlDoc),
+ };
yamlDoc.errors.forEach(addError);
if (errors.length)
@@ -272,13 +298,14 @@ export function textValue(value: string): AriaTextValue {
}
export class KeyParser {
+ private _options: ParsingOptions;
private _input: string;
private _pos: number;
private _length: number;
static parse(text: yamlTypes.Scalar<string>, options: ParsingOptions, errors: ParsedYamlError[]): AriaTemplateRoleNode | null {
try {
- return new KeyParser(text.value)._parse();
+ return new KeyParser(text.value, options)._parse();
} catch (e) {
if (e instanceof ParserError) {
const message = options.prettyErrors === false ? e.message : e.message + ':\n\n' + text.value + '\n' + ' '.repeat(e.pos) + '^\n';
@@ -292,7 +319,8 @@ export class KeyParser {
}
}
- constructor(input: string) {
+ constructor(input: string, options: ParsingOptions) {
+ this._options = options;
this._input = input;
this._pos = 0;
this._length = input.length;
@@ -475,6 +503,11 @@ export class KeyParser {
node.selected = value === 'true';
return;
}
+ if (this._options.laxProps) {
+ node.props = node.props || {};
+ node.props[key] = textValue(value);
+ return;
+ }
this._assert(false, `Unsupported attribute [${key}]`, errorPos);
}
@@ -492,3 +525,69 @@ export class ParserError extends Error {
this.pos = pos;
}
}
+
+type AriaSnapshotDiffResult = 'equal' | 'different' | { ref: string, newSource: string }[];
+
+export function diffAriaSnapshots(yaml: YamlLibrary, oldSnapshot: string, newSnapshot: string): AriaSnapshotDiffResult {
+ const diffTree = (oldNode: AriaTemplateNode, newNode: AriaTemplateNode): AriaSnapshotDiffResult => {
+ if (!oldNode.sourceRange || !newNode.sourceRange)
+ return 'different';
+
+ const oldSelfSource = oldSnapshot.slice(oldNode.sourceRange.from, oldNode.sourceRange.to);
+ const newSelfSource = newSnapshot.slice(newNode.sourceRange.from, newNode.sourceRange.to);
+ if (oldNode.kind !== 'role' || newNode.kind !== 'role')
+ return (oldNode.kind === newNode.kind && oldSelfSource === newSelfSource) ? 'equal' : 'different';
+
+ const newNodeSubtreeSourceRange = newNode.subtreeSourceRange || newNode.sourceRange;
+ const newSubtreeSource = newSnapshot.slice(newNodeSubtreeSourceRange.from, newNodeSubtreeSourceRange.to);
+
+ const oldChildren = oldNode.children || [];
+ const newChildren = newNode.children || [];
+ const childrenDiffs = [];
+ // When "self" is the same, we can diff children and try to find a small diff there.
+ let useChildrenDiffs = oldSelfSource === newSelfSource && oldChildren.length === newChildren.length;
+ let childrenTotalLength = 0;
+
+ if (useChildrenDiffs) {
+ for (let i = 0; i < oldChildren.length; i++) {
+ const childDiff = diffTree(oldChildren[i], newChildren[i]);
+ if (childDiff === 'equal')
+ continue;
+ if (childDiff === 'different') {
+ useChildrenDiffs = false;
+ break;
+ }
+ for (const diff of childDiff) {
+ childrenTotalLength += diff.newSource.length;
+ childrenDiffs.push(diff);
+ }
+ }
+ }
+
+ if (childrenDiffs.length > 1 && childrenTotalLength * 2 >= newSubtreeSource.length) {
+ // Too many children diffs without too much of a benefit.
+ useChildrenDiffs = false;
+ }
+
+ if (useChildrenDiffs) {
+ if (!childrenDiffs.length)
+ return 'equal';
+ return childrenDiffs;
+ }
+
+ const oldRef = oldNode.props?.ref?.raw;
+ const newRef = newNode.props?.ref?.raw;
+ if (!oldRef || oldRef !== newRef)
+ return 'different';
+
+ return [{ ref: oldRef, newSource: newSubtreeSource }];
+ };
+
+ try {
+ const oldParsed = parseAriaSnapshotUnsafe(yaml, oldSnapshot, { laxProps: true });
+ const newParsed = parseAriaSnapshotUnsafe(yaml, newSnapshot, { laxProps: true });
+ return diffTree(oldParsed, newParsed);
+ } catch {
+ return 'different';
+ }
+}
diff --git a/packages/playwright/src/mcp/browser/response.ts b/packages/playwright/src/mcp/browser/response.ts
index db910cc222..348f03e355 100644
--- a/packages/playwright/src/mcp/browser/response.ts
+++ b/packages/playwright/src/mcp/browser/response.ts
@@ -25,7 +25,7 @@ export class Response {
private _code: string[] = [];
private _images: { contentType: string, data: Buffer }[] = [];
private _context: Context;
- private _includeSnapshot = false;
+ private _includeSnapshot: 'full' | 'partial' | 'none' = 'none';
private _includeTabs = false;
private _tabSnapshot: TabSnapshot | undefined;
@@ -72,8 +72,8 @@ export class Response {
return this._images;
}
- setIncludeSnapshot() {
- this._includeSnapshot = true;
+ setIncludeSnapshot(full?: 'full') {
+ this._includeSnapshot = full ?? 'partial';
}
setIncludeTabs() {
@@ -83,7 +83,7 @@ export class Response {
async finish() {
// All the async snapshotting post-action is happening here.
// Everything below should race against modal states.
- if (this._includeSnapshot && this._context.currentTab())
+ if (this._includeSnapshot !== 'none' && this._context.currentTab())
this._tabSnapshot = await this._context.currentTabOrDie().captureSnapshot();
for (const tab of this._context.tabs())
await tab.updateTitle();
@@ -113,7 +113,7 @@ ${this._code.join('\n')}
}
// List browser tabs.
- if (this._includeSnapshot || this._includeTabs)
+ if (this._includeSnapshot !== 'none' || this._includeTabs)
response.push(...renderTabsMarkdown(this._context.tabs(), this._includeTabs));
// Add snapshot if provided.
@@ -121,7 +121,7 @@ ${this._code.join('\n')}
response.push(...renderModalStates(this._context, this._tabSnapshot.modalStates));
response.push('');
} else if (this._tabSnapshot) {
- response.push(renderTabSnapshot(this._tabSnapshot));
+ response.push(renderTabSnapshot(this._tabSnapshot, this._includeSnapshot === 'full'));
response.push('');
}
@@ -153,7 +153,7 @@ ${this._code.join('\n')}
}
}
-function renderTabSnapshot(tabSnapshot: TabSnapshot): string {
+function renderTabSnapshot(tabSnapshot: TabSnapshot, fullSnapshot: boolean): string {
const lines: string[] = [];
if (tabSnapshot.consoleMessages.length) {
@@ -177,10 +177,15 @@ function renderTabSnapshot(tabSnapshot: TabSnapshot): string {
lines.push(`### Page state`);
lines.push(`- Page URL: ${tabSnapshot.url}`);
lines.push(`- Page Title: ${tabSnapshot.title}`);
- lines.push(`- Page Snapshot:`);
- lines.push('```yaml');
- lines.push(tabSnapshot.ariaSnapshot);
- lines.push('```');
+ if (!fullSnapshot && tabSnapshot.formattedAriaSnapshotDiff) {
+ lines.push(`- Page Snapshot Diff:`);
+ lines.push(tabSnapshot.formattedAriaSnapshotDiff);
+ } else {
+ lines.push(`- Page Snapshot:`);
+ lines.push('```yaml');
+ lines.push(tabSnapshot.ariaSnapshot);
+ lines.push('```');
+ }
return lines.join('\n');
}
diff --git a/packages/playwright/src/mcp/browser/tab.ts b/packages/playwright/src/mcp/browser/tab.ts
index 1669d51767..b0190b6c24 100644
--- a/packages/playwright/src/mcp/browser/tab.ts
+++ b/packages/playwright/src/mcp/browser/tab.ts
@@ -16,7 +16,8 @@
import { EventEmitter } from 'events';
import * as playwright from 'playwright-core';
-import { ManualPromise } from 'playwright-core/lib/utils';
+import { ManualPromise, diffAriaSnapshots } from 'playwright-core/lib/utils';
+import { yaml } from 'playwright-core/lib/utilsBundle';
import { callOnPageNoTrace, waitForCompletion } from './tools/utils';
import { logUnhandledError } from '../log';
@@ -40,6 +41,7 @@ export type TabSnapshot = {
url: string;
title: string;
ariaSnapshot: string;
+ formattedAriaSnapshotDiff?: string;
modalStates: ModalState[];
consoleMessages: ConsoleMessage[];
downloads: { download: playwright.Download, finished: boolean, outputFile: string }[];
@@ -55,6 +57,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {
private _onPageClose: (tab: Tab) => void;
private _modalStates: ModalState[] = [];
private _downloads: { download: playwright.Download, finished: boolean, outputFile: string }[] = [];
+ private _lastAriaSnapshot: string | undefined;
constructor(context: Context, page: playwright.Page, onPageClose: (tab: Tab) => void) {
super();
@@ -63,7 +66,14 @@ export class Tab extends EventEmitter<TabEventsInterface> {
this._onPageClose = onPageClose;
page.on('console', event => this._handleConsoleMessage(messageToConsoleMessage(event)));
page.on('pageerror', error => this._handleConsoleMessage(pageErrorToConsoleMessage(error)));
- page.on('request', request => this._requests.set(request, null));
+ page.on('request', request => {
+ this._requests.set(request, null);
+ // Note: unfortunately, 'framenavigated' event does not differentiate between
+ // a same-document (#hash or pushState) navigation and a new document being loaded.
+ // Relying upon a navigation request heuristic.
+ if (request.frame() === page.mainFrame() && request.isNavigationRequest())
+ this._willNavigateMainFrameToNewDocument();
+ });
page.on('response', response => this._requests.set(response.request(), response));
page.on('close', () => this._onClose());
page.on('filechooser', chooser => {
@@ -140,6 +150,10 @@ export class Tab extends EventEmitter<TabEventsInterface> {
this._onPageClose(this);
}
+ private _willNavigateMainFrameToNewDocument() {
+ this._lastAriaSnapshot = undefined;
+ }
+
async updateTitle() {
await this._raceAgainstModalStates(async () => {
this._lastTitle = await callOnPageNoTrace(this.page, page => page.title());
@@ -160,6 +174,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {
async navigate(url: string) {
this._clearCollectedArtifacts();
+ this._willNavigateMainFrameToNewDocument();
const downloadEvent = callOnPageNoTrace(this.page, page => page.waitForEvent('download').catch(logUnhandledError));
try {
@@ -203,6 +218,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {
url: this.page.url(),
title: await this.page.title(),
ariaSnapshot: snapshot,
+ formattedAriaSnapshotDiff: this._lastAriaSnapshot ? generateAriaSnapshotDiff(this._lastAriaSnapshot, snapshot) : undefined,
modalStates: [],
consoleMessages: [],
downloads: this._downloads,
@@ -212,6 +228,7 @@ export class Tab extends EventEmitter<TabEventsInterface> {
// Assign console message late so that we did not lose any to modal state.
tabSnapshot.consoleMessages = this._recentConsoleMessages;
this._recentConsoleMessages = [];
+ this._lastAriaSnapshot = tabSnapshot.ariaSnapshot;
}
return tabSnapshot ?? {
url: this.page.url(),
@@ -312,3 +329,22 @@ export function renderModalStates(context: Context, modalStates: ModalState[]):
}
const tabSymbol = Symbol('tabSymbol');
+
+function generateAriaSnapshotDiff(oldSnapshot: string, newSnapshot: string) {
+ const diffs = diffAriaSnapshots(yaml, oldSnapshot, newSnapshot);
+ if (diffs === 'equal')
+ return '<no changes>';
+ if (diffs === 'different')
+ return;
+ if (diffs.length > 3 || diffs.some(diff => diff.newSource.split('\n').length > 100)) {
+ // Being conservative - up to 3 small diff changes, otherwise include full snapshot.
+ return;
+ }
+ const lines = [`The following refs have changed`];
+ for (const diff of diffs)
+ lines.push('', '```yaml', diff.newSource.trimEnd(), '```');
+ const combined = lines.join('\n');
+ if (combined.length >= newSnapshot.length)
+ return;
+ return combined;
+}
diff --git a/packages/playwright/src/mcp/browser/tools/snapshot.ts b/packages/playwright/src/mcp/browser/tools/snapshot.ts
index bea116ee8f..c7e7cc539f 100644
--- a/packages/playwright/src/mcp/browser/tools/snapshot.ts
+++ b/packages/playwright/src/mcp/browser/tools/snapshot.ts
@@ -31,7 +31,7 @@ const snapshot = defineTool({
handle: async (context, params, response) => {
await context.ensureTab();
- response.setIncludeSnapshot();
+ response.setIncludeSnapshot('full');
},
});
diff --git a/tests/library/unit/parse-aria-snapshot.spec.ts b/tests/library/unit/parse-aria-snapshot.spec.ts
new file mode 100644
index 0000000000..9aac6bee16
--- /dev/null
+++ b/tests/library/unit/parse-aria-snapshot.spec.ts
@@ -0,0 +1,309 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from '@playwright/test';
+import { parseAriaSnapshotUnsafe, diffAriaSnapshots } from '../../../packages/playwright-core/lib/utils/isomorphic/ariaSnapshot.js';
+import { yaml } from 'playwright-core/lib/utilsBundle';
+
+function addSource(text: string, node: any) {
+ if (node.sourceRange)
+ node.source = text.slice(node.sourceRange.from, node.sourceRange.to).trim();
+ if (node.subtreeSourceRange)
+ node.subtreeSource = text.slice(node.subtreeSourceRange.from, node.subtreeSourceRange.to).trim();
+ (node.children || []).forEach(child => addSource(text, child));
+}
+
+function parse(text: string) {
+ const result = parseAriaSnapshotUnsafe(yaml, text, { laxProps: true });
+ addSource(text, result);
+ return result;
+}
+
+function diff(oldSnapshot: string, newSnapshot: string) {
+ const diffs = diffAriaSnapshots(yaml, oldSnapshot, newSnapshot);
+ if (Array.isArray(diffs))
+ diffs.forEach(diff => diff.newSource = diff.newSource.trimEnd());
+ return diffs;
+}
+
+test('parse aria snapshot returns source ranges', () => {
+ const listSnapshot = `list "list name" [ref=e1]:
+ - listitem: item 1
+ - listitem "item 2":
+ - button "click me" [ref=e2]
+ - text: hello
+ - listitem:
+ - generic: text1
+ - generic [ref=e3] [active] [cursor=pointer]: text2`;
+
+ const snapshot = `
+- ${listSnapshot}
+- button "another button" [ref=e4]
+`;
+
+ expect(parse(snapshot)).toEqual(expect.objectContaining({
+ role: 'fragment',
+ source: '',
+ subtreeSource: snapshot.trim(),
+ children: [
+ expect.objectContaining({
+ role: 'list',
+ name: 'list name',
+ props: { ref: expect.objectContaining({ raw: 'e1' }) },
+ source: 'list "list name" [ref=e1]',
+ subtreeSource: listSnapshot,
+ children: [
+ expect.objectContaining({
+ role: 'listitem',
+ source: 'listitem',
+ subtreeSource: 'listitem: item 1',
+ children: [
+ expect.objectContaining({
+ kind: 'text',
+ text: expect.objectContaining({ raw: 'item 1' }),
+ source: 'item 1',
+ }),
+ ],
+ }),
+ expect.objectContaining({
+ role: 'listitem',
+ name: 'item 2',
+ source: 'listitem "item 2"',
+ subtreeSource: 'listitem "item 2":\n - button "click me" [ref=e2]\n - text: hello',
+ children: [
+ expect.objectContaining({
+ role: 'button',
+ name: 'click me',
+ props: { ref: expect.objectContaining({ raw: 'e2' }) },
+ source: 'button "click me" [ref=e2]',
+ }),
+ expect.objectContaining({
+ kind: 'text',
+ text: expect.objectContaining({ raw: 'hello' }),
+ source: 'text: hello',
+ }),
+ ],
+ }),
+ expect.objectContaining({
+ role: 'listitem',
+ source: 'listitem',
+ subtreeSource: 'listitem:\n - generic: text1\n - generic [ref=e3] [active] [cursor=pointer]: text2',
+ children: [
+ expect.objectContaining({
+ role: 'generic',
+ source: 'generic',
+ subtreeSource: 'generic: text1',
+ children: [
+ expect.objectContaining({
+ kind: 'text',
+ text: expect.objectContaining({ raw: 'text1' }),
+ source: 'text1',
+ }),
+ ],
+ }),
+ expect.objectContaining({
+ role: 'generic',
+ props: { ref: expect.objectContaining({ raw: 'e3' }), cursor: expect.objectContaining({ raw: 'pointer' }) },
+ active: true,
+ source: 'generic [ref=e3] [active] [cursor=pointer]',
+ subtreeSource: 'generic [ref=e3] [active] [cursor=pointer]: text2',
+ children: [
+ expect.objectContaining({
+ kind: 'text',
+ text: expect.objectContaining({ raw: 'text2' }),
+ source: 'text2',
+ }),
+ ],
+ }),
+ ],
+ }),
+ ],
+ }),
+ expect.objectContaining({
+ role: 'button',
+ name: 'another button',
+ props: { ref: expect.objectContaining({ raw: 'e4' }) },
+ source: 'button "another button" [ref=e4]',
+ }),
+ ],
+ }));
+});
+
+test('diff: equal', () => {
+ expect(diff(`- button "name"`, `- button "name"`)).toBe('equal');
+});
+
+test('diff: different', () => {
+ expect(diff(`- button "name"`, `- button "new name"`)).toBe('different');
+});
+
+test('diff: one ref', () => {
+ expect(diff(`
+- button "button 1" [ref=e1]
+`, `
+- button "button 2" [ref=e1]
+`)).toEqual([
+ {
+ ref: 'e1',
+ newSource: 'button "button 2" [ref=e1]',
+ }
+ ]);
+});
+
+test('diff: no ref upgrades to parent', () => {
+ expect(diff(`
+- listitem [ref=e6]:
+ - generic: some text
+`, `
+- listitem [ref=e6]:
+ - generic: some text has changed
+`)).toEqual([
+ {
+ ref: 'e6',
+ newSource: `listitem [ref=e6]:
+ - generic: some text has changed`,
+ }
+ ]);
+});
+
+test('diff: two refs in a large parent', () => {
+ expect(diff(`
+- listitem [ref=e6]: some text
+- listitem
+- listitem
+- listitem
+- listitem
+- listitem
+- listitem
+- listitem
+- listitem [ref=e7]: other text
+`, `
+- listitem [ref=e6]: some new text
+- listitem
+- listitem
+- listitem
+- listitem
+- listitem
+- listitem
+- listitem
+- listitem [ref=e7]: other new text
+`)).toEqual([
+ {
+ ref: 'e6',
+ newSource: `listitem [ref=e6]: some new text`,
+ },
+ {
+ ref: 'e7',
+ newSource: `listitem [ref=e7]: other new text`,
+ },
+ ]);
+});
+
+test('diff: mixed', () => {
+ const snapshot1 = `
+ - button "button 1" [ref=e1] [active]
+ - button "padding"
+ - button "button 2" [ref=e2]
+ - button "padding"
+ - button "padding"
+ - list "list name" [ref=e3]:
+ - listitem [ref=e4]: item 1
+ - listitem "item 2" [ref=e5]
+ - button "padding"
+ - button "padding"
+ - button "padding"
+ - button "padding"
+ - listitem [ref=e6]:
+ - generic: some text
+ - button "padding"
+ - button "padding"
+ - button "padding"
+ - button "padding"
+ - generic:
+ - listitem [ref=e10]: more text
+ - listitem "item name" [ref=e7]
+ - listitem [ref=e8]:
+ - listitem "deep item" [ref=e9]
+ `;
+ const snapshot2 = `
+ - button "button 1" [ref=e1]
+ - button "padding"
+ - button "button 2" [ref=e2] [active]
+ - button "padding"
+ - button "padding"
+ - list "list name" [ref=e3]:
+ - listitem [ref=e4]: item 1
+ - listitem "item 2" [ref=e5]:
+ - button "new button 1" [ref=e100]
+ - button "padding"
+ - button "padding"
+ - button "padding"
+ - button "padding"
+ - listitem [ref=e6]:
+ - generic: some text has changed
+ - button "padding"
+ - button "padding"
+ - button "padding"
+ - button "padding"
+ - generic:
+ - listitem [ref=e10]: more text has changed
+ - listitem "new item name" [ref=e7]
+ - listitem [ref=e8]:
+ - listitem "new ref" [ref=e101]
+ `;
+ expect(diff(snapshot1, snapshot2)).toEqual([
+ { ref: 'e1', newSource: 'button "button 1" [ref=e1]' },
+ { ref: 'e2', newSource: 'button "button 2" [ref=e2] [active]' },
+ { ref: 'e5', newSource: `listitem "item 2" [ref=e5]:
+ - button "new button 1" [ref=e100]` },
+ { ref: 'e6', newSource: `listitem [ref=e6]:
+ - generic: some text has changed` },
+ { ref: 'e10', newSource: 'listitem [ref=e10]: more text has changed' },
+ { ref: 'e7', newSource: `listitem "new item name" [ref=e7]` },
+ { ref: 'e8', newSource: `listitem [ref=e8]:
+ - listitem "new ref" [ref=e101]` },
+ ]);
+});
+
+test('diff: two refs in a small parent are combined', () => {
+ expect(diff(`
+- button "name"
+- list [ref=e1]:
+ - listitem [ref=e6]: some text
+ - listitem
+ - listitem
+ - listitem
+ - listitem [ref=e7]: other text
+`, `
+- button "name"
+- list [ref=e1]:
+ - listitem [ref=e6]: some new text
+ - listitem
+ - listitem
+ - listitem
+ - listitem [ref=e7]: other new text
+`)).toEqual([
+ {
+ ref: 'e1',
+ newSource: `list [ref=e1]:
+ - listitem [ref=e6]: some new text
+ - listitem
+ - listitem
+ - listitem
+ - listitem [ref=e7]: other new text`,
+ },
+ ]);
+});
diff --git a/tests/mcp/click.spec.ts b/tests/mcp/click.spec.ts
index 6f6a06dbd0..169d57dca4 100644
--- a/tests/mcp/click.spec.ts
+++ b/tests/mcp/click.spec.ts
@@ -16,10 +16,16 @@
import { test, expect } from './fixtures';
-test('browser_click', async ({ client, server, mcpBrowser }) => {
+test('browser_click', async ({ client, server }) => {
server.setContent('/', `
<title>Title</title>
<button>Submit</button>
+ <script>
+ const button = document.querySelector('button');
+ button.addEventListener('click', () => {
+ button.focus(); // without manual focus, webkit focuses body
+ });
+ </script>
`, 'text/html');
await client.callTool({
@@ -35,7 +41,7 @@ test('browser_click', async ({ client, server, mcpBrowser }) => {
},
})).toHaveResponse({
code: `await page.getByRole('button', { name: 'Submit' }).click();`,
- pageState: expect.stringContaining(`- button "Submit" ${mcpBrowser !== 'webkit' || process.platform === 'linux' ? '[active] ' : ''}[ref=e2]`),
+ pageState: expect.stringContaining(`- button "Submit" [active] [ref=e2]`),
});
});
@@ -125,7 +131,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {
},
})).toHaveResponse({
code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Control'] });`,
- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:true metaKey:false shiftKey:false altKey:false`),
+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:true metaKey:false shiftKey:false altKey:false`),
});
}
@@ -138,7 +144,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {
},
})).toHaveResponse({
code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Shift'] });`,
- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:false`),
+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:false`),
});
expect(await client.callTool({
@@ -150,6 +156,6 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => {
},
})).toHaveResponse({
code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Shift', 'Alt'] });`,
- pageState: expect.stringContaining(`- generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:true`),
+ pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:true`),
});
});
diff --git a/tests/mcp/dialogs.spec.ts b/tests/mcp/dialogs.spec.ts
index 4c2bf98eb5..fa2be5cc73 100644
--- a/tests/mcp/dialogs.spec.ts
+++ b/tests/mcp/dialogs.spec.ts
@@ -17,7 +17,7 @@
import { test, expect } from './fixtures';
test('alert dialog', async ({ client, server }) => {
- server.setContent('/', `<button onclick="alert('Alert')">Button</button>`, 'text/html');
+ server.setContent('/', `<title>Title</title><button onclick="alert('Alert')">Button</button>`, 'text/html');
expect(await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
@@ -54,7 +54,7 @@ test('alert dialog', async ({ client, server }) => {
},
})).toHaveResponse({
modalState: undefined,
- pageState: expect.stringContaining(`- button "Button"`),
+ pageState: expect.stringContaining(`Page Title: Title`),
});
});
@@ -218,7 +218,7 @@ test('prompt dialog', async ({ client, server }) => {
});
test('alert dialog w/ race', async ({ client, server }) => {
- server.setContent('/', `<button onclick="setTimeout(() => alert('Alert'), 100)">Button</button>`, 'text/html');
+ server.setContent('/', `<title>Title</title><button onclick="setTimeout(() => alert('Alert'), 100)">Button</button>`, 'text/html');
expect(await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
@@ -247,9 +247,6 @@ test('alert dialog w/ race', async ({ client, server }) => {
expect(result).toHaveResponse({
modalState: undefined,
pageState: expect.stringContaining(`- Page URL: ${server.PREFIX}/
-- Page Title:
-- Page Snapshot:
-\`\`\`yaml
-- button "Button"`),
+- Page Title: Title`),
});
});
diff --git a/tests/mcp/session-log.spec.ts b/tests/mcp/session-log.spec.ts
index 6f9a98ee66..7e072aa55b 100644
--- a/tests/mcp/session-log.spec.ts
+++ b/tests/mcp/session-log.spec.ts
@@ -42,7 +42,6 @@ test('session log should record tool calls', async ({ startClient, server }, tes
},
})).toHaveResponse({
code: `await page.getByRole('button', { name: 'Submit' }).click();`,
- pageState: expect.stringContaining(`- button "Submit"`),
});
const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
diff --git a/tests/mcp/snapshot-diff.spec.ts b/tests/mcp/snapshot-diff.spec.ts
new file mode 100644
index 0000000000..f5ca5346c0
--- /dev/null
+++ b/tests/mcp/snapshot-diff.spec.ts
@@ -0,0 +1,163 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './fixtures';
+
+test('should return aria snapshot diff', async ({ client, server }) => {
+ server.setContent('/', `
+ <button>Button 1</button>
+ <button>Button 2</button>
+ <ul id=filler></ul>
+ <script>
+ const filler = document.getElementById('filler');
+ for (let i = 0; i < 100; i++) {
+ const li = document.createElement('li');
+ li.textContent = 'Filler ' + i;
+ filler.appendChild(li);
+ }
+ const [button1, button2] = document.querySelectorAll('button');
+ button1.addEventListener('click', () => {
+ const button = document.createElement('button');
+ button.textContent = 'new button';
+ filler.firstChild.appendChild(button);
+ button1.focus(); // without manual focus, webkit focuses body
+ });
+ button2.focus();
+ button2.addEventListener('click', () => {
+ button2.focus(); // without manual focus, webkit focuses body
+ });
+ </script>
+ `, 'text/html');
+
+ const listitems = new Array(100).fill(0).map((_, i) => `\n - listitem [ref=e${5 + i}]: Filler ${i}`).join('');
+
+ expect(await client.callTool({
+ name: 'browser_navigate',
+ arguments: {
+ url: server.PREFIX,
+ },
+ })).toHaveResponse({
+ pageState: expect.stringContaining(`
+ - button "Button 1" [ref=e2]
+ - button "Button 2" [active] [ref=e3]
+ - list [ref=e4]:${listitems}`),
+ });
+
+ expect(await client.callTool({
+ name: 'browser_click',
+ arguments: {
+ element: 'Button 2',
+ ref: 'e3',
+ },
+ })).toHaveResponse({
+ pageState: expect.stringContaining(`
+- Page Snapshot Diff:
+<no changes>`),
+ });
+
+ expect(await client.callTool({
+ name: 'browser_click',
+ arguments: {
+ element: 'Button 1',
+ ref: 'e2',
+ },
+ })).toHaveResponse({
+ pageState: expect.stringContaining(`
+- Page Snapshot Diff:
+The following refs have changed
+
+\`\`\`yaml
+button "Button 1" [active] [ref=e2]
+\`\`\`
+
+\`\`\`yaml
+button "Button 2" [ref=e3]
+\`\`\`
+
+\`\`\`yaml
+listitem [ref=e5]:
+ - text: Filler 0
+ - button "new button" [ref=e105]
+\`\`\``),
+ });
+
+ expect(await client.callTool({
+ name: 'browser_snapshot',
+ })).toHaveResponse({
+ pageState: expect.stringContaining(`
+- Page Snapshot:`),
+ });
+});
+
+test('should reset aria snapshot diff upon navigation', async ({ client, server }) => {
+ server.setContent('/before', `
+ <button>Button 1</button>
+ <button>Button 2</button>
+ <ul id=filler></ul>
+ <script>
+ const filler = document.getElementById('filler');
+ for (let i = 0; i < 100; i++) {
+ const li = document.createElement('li');
+ li.textContent = 'Filler ' + i;
+ filler.appendChild(li);
+ }
+ document.querySelector('button').addEventListener('click', () => {
+ window.location.href = '/after';
+ });
+ document.querySelectorAll('button')[1].focus();
+ </script>
+ `, 'text/html');
+
+ server.setContent('/after', `
+ <button>Button 1</button>
+ <button>Button 2</button>
+ <ul id=filler></ul>
+ <script>
+ const filler = document.getElementById('filler');
+ for (let i = 0; i < 100; i++) {
+ const li = document.createElement('li');
+ li.textContent = 'Filler ' + i;
+ filler.appendChild(li);
+ }
+ const button = document.createElement('button');
+ button.textContent = 'new button';
+ filler.lastChild.appendChild(button);
+ document.querySelectorAll('button')[0].focus();
+ </script>
+ `, 'text/html');
+
+ expect(await client.callTool({
+ name: 'browser_navigate',
+ arguments: {
+ url: server.PREFIX + '/before',
+ },
+ })).toHaveResponse({
+ pageState: expect.stringContaining(`
+ - button "Button 1" [ref=e2]
+ - button "Button 2" [active] [ref=e3]`),
+ });
+
+ expect(await client.callTool({
+ name: 'browser_click',
+ arguments: {
+ element: 'Button 1',
+ ref: 'e2',
+ },
+ })).toHaveResponse({
+ pageState: expect.stringContaining(`
+- Page Snapshot:`),
+ });
+});
diff --git a/tests/mcp/tabs.spec.ts b/tests/mcp/tabs.spec.ts
index a0b40d824b..bc59f2d7a5 100644
--- a/tests/mcp/tabs.spec.ts
+++ b/tests/mcp/tabs.spec.ts
@@ -98,10 +98,8 @@ test('select tab', async ({ client }) => {
- 2: [Tab two] (data:text/html,<title>Tab two</title><body>Body two</body>)`,
pageState: expect.stringContaining(`- Page URL: data:text/html,<title>Tab one</title><body>Body one</body>
- Page Title: Tab one
-- Page Snapshot:
-\`\`\`yaml
-- generic [active] [ref=e1]: Body one
-\`\`\``),
+- Page Snapshot Diff:
+<no changes>`),
});
expect(await client.callTool({
@@ -133,10 +131,8 @@ test('close tab', async ({ client }) => {
- 1: (current) [Tab one] (data:text/html,<title>Tab one</title><body>Body one</body>)`,
pageState: expect.stringContaining(`- Page URL: data:text/html,<title>Tab one</title><body>Body one</body>
- Page Title: Tab one
-- Page Snapshot:
-\`\`\`yaml
-- generic [active] [ref=e1]: Body one
-\`\`\``),
+- Page Snapshot Diff:
+<no changes>`),
});
});
diff --git a/tests/mcp/wait.spec.ts b/tests/mcp/wait.spec.ts
index 0388faf259..e844eac4ea 100644
--- a/tests/mcp/wait.spec.ts
+++ b/tests/mcp/wait.spec.ts
@@ -31,9 +31,11 @@ test('browser_wait_for(text)', async ({ client, server }) => {
</body>
`, 'text/html');
- await client.callTool({
+ expect(await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
+ })).toHaveResponse({
+ pageState: expect.stringContaining(`- generic [ref=e3]: Text to disappear`),
});
await client.callTool({
@@ -44,10 +46,14 @@ test('browser_wait_for(text)', async ({ client, server }) => {
},
});
- expect(await client.callTool({
+ await client.callTool({
name: 'browser_wait_for',
arguments: { text: 'Text to appear' },
code: `await page.getByText("Text to appear").first().waitFor({ state: 'visible' });`,
+ });
+
+ expect(await client.callTool({
+ name: 'browser_snapshot',
})).toHaveResponse({
pageState: expect.stringContaining(`- generic [ref=e3]: Text to appear`),
});
@@ -68,9 +74,11 @@ test('browser_wait_for(textGone)', async ({ client, server }) => {
</body>
`, 'text/html');
- await client.callTool({
+ expect(await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
+ })).toHaveResponse({
+ pageState: expect.stringContaining(`- generic [ref=e3]: Text to disappear`),
});
await client.callTool({
@@ -81,10 +89,14 @@ test('browser_wait_for(textGone)', async ({ client, server }) => {
},
});
- expect(await client.callTool({
+ await client.callTool({
name: 'browser_wait_for',
arguments: { textGone: 'Text to disappear' },
code: `await page.getByText("Text to disappear").first().waitFor({ state: 'hidden' });`,
+ });
+
+ expect(await client.callTool({
+ name: 'browser_snapshot',
})).toHaveResponse({
pageState: expect.stringContaining(`- generic [ref=e3]: Text to appear`),
});