Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore(agent): format expect errors in the same way (#38797)
Dmitry Gozmanc7c1c48003e1
diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.tsindex 4a46612a11..0d57a3cc0d 100644--- a/packages/playwright-core/src/server/agent/actionRunner.ts+++ b/packages/playwright-core/src/server/agent/actionRunner.ts@@ -14,10 +14,11 @@* limitations under the License.*/-import { serializeExpectedTextValues } from '../utils/expectUtils';+import { formatMatcherMessage, serializeExpectedTextValues, simpleMatcherUtils } from '../utils/expectUtils';import { monotonicTime } from '../../utils/isomorphic/time';import { createGuid } from '../utils/crypto';import { parseAriaSnapshotUnsafe } from '../../utils/isomorphic/ariaSnapshot';+import { asLocatorDescription } from '../../utils/isomorphic/locatorGenerators';import { yaml } from '../../utilsBundle';import { serializeError } from '../errors';@@ -25,9 +26,10 @@ import type * as actions from './actions';import type { Page } from '../page';import type { Progress } from '../progress';import type { NameValue } from '@protocol/channels';-import type { ExpectResult, Frame } from '../frames';+import type { Frame } from '../frames';import type { CallMetadata } from '../instrumentation';import type * as channels from '@protocol/channels';+import type { FrameExpectParams } from '@injected/injectedScript';export async function runAction(progress: Progress, mode: 'generate' | 'run', page: Page, action: actions.Action, secrets: NameValue[]) {const parentMetadata = progress.metadata;@@ -99,36 +101,49 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', pageawait frame.uncheck(progress, action.selector, { ...commonOptions });break;case 'expectVisible': {- const result = await frame.expect(progress, action.selector, { expression: 'to.be.visible', isNot: !!action.isNot });- if (!result.matches === !action.isNot)- throw new Error(result.errorMessage);+ await runExpect(frame, progress, action.selector, { expression: 'to.be.visible', isNot: !!action.isNot }, 'visible', 'toBeVisible', '');break;}case 'expectValue': {- let result: ExpectResult;if (action.type === 'textbox' || action.type === 'combobox' || action.type === 'slider') {const expectedText = serializeExpectedTextValues([action.value]);- result = await frame.expect(progress, action.selector, { expression: 'to.have.value', expectedText, isNot: !!action.isNot });+ await runExpect(frame, progress, action.selector, { expression: 'to.have.value', expectedText, isNot: !!action.isNot }, action.value, 'toHaveValue', 'expected');} else if (action.type === 'checkbox' || action.type === 'radio') {const expectedValue = { checked: action.value === 'true' };- result = await frame.expect(progress, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: !!action.isNot });+ await runExpect(frame, progress, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: !!action.isNot }, action.value ? 'checked' : 'unchecked', 'toBeChecked', '');} else {throw new Error(`Unsupported element type: ${action.type}`);}- if (!result.matches === !action.isNot)- throw new Error(result.errorMessage);break;}case 'expectAria': {const expectedValue = parseAriaSnapshotUnsafe(yaml, action.template);- const result = await frame.expect(progress, 'body', { expression: 'to.match.aria', expectedValue, isNot: !!action.isNot });- if (!result.matches === !action.isNot)- throw new Error(result.errorMessage);+ await runExpect(frame, progress, 'body', { expression: 'to.match.aria', expectedValue, isNot: !!action.isNot }, '\n' + action.template, 'toMatchAriaSnapshot', 'expected');break;}}}+async function runExpect(frame: Frame, progress: Progress, selector: string | undefined, options: FrameExpectParams, expected: string, matcherName: string, expectation: string) {+ const result = await frame.expect(progress, selector, options);+ if (!result.matches === !options.isNot) {+ const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received;+ throw new Error(formatMatcherMessage(simpleMatcherUtils, {+ isNot: options.isNot,+ matcherName,+ expectation,+ locator: selector ? asLocatorDescription('javascript', selector) : undefined,+ timedOut: result.timedOut,+ timeout: progress.timeout,+ printedExpected: options.isNot ? `Expected: not ${expected}` : `Expected: ${expected}`,+ printedReceived: result.errorMessage ? '' : `Received: ${received}`,+ errorMessage: result.errorMessage,+ // Note: we are not passing call log, because it will be automatically appended on the client side,+ // as a part of the agent.{perform,expect} call.+ }));+ }+}+export function traceParamsForAction(progress: Progress, action: actions.Action): { title?: string, type: string, method: string, params: any } {const timeout = progress.timeout;switch (action.method) {diff --git a/packages/playwright-core/src/server/utils/expectUtils.ts b/packages/playwright-core/src/server/utils/expectUtils.tsindex 3fac8db843..adfef78cac 100644--- a/packages/playwright-core/src/server/utils/expectUtils.ts+++ b/packages/playwright-core/src/server/utils/expectUtils.ts@@ -15,6 +15,7 @@*/import { isRegExp, isString } from '../../utils/isomorphic/rtti';+import { colors } from '../../utilsBundle';import type { ExpectedTextValue } from '@protocol/channels';@@ -144,3 +145,34 @@ Call log:${utils.DIM_COLOR(log.join('\n'))}`;};+++function printValue(value: unknown): string {+ try {+ return JSON.stringify(value);+ } catch {+ return String(value);+ }+}++function printReceived(value: unknown): string {+ return colors.red(printValue(value));+}++function printExpected(value: unknown): string {+ return colors.green(printValue(value));+}++export const simpleMatcherUtils: InternalMatcherUtils = {+ DIM_COLOR: colors.dim,+ RECEIVED_COLOR: colors.red,+ EXPECTED_COLOR: colors.green,+ INVERTED_COLOR: colors.inverse,+ printReceived,+ printExpected,+ printDiffOrStringify: (expected: unknown, received: unknown, expectedLabel: string, receivedLabel: string) => {+ const maxLength = Math.max(expectedLabel.length, receivedLabel.length) + 2;+ return `${expectedLabel}: `.padEnd(maxLength) + printExpected(expected) + `\n` ++ `${receivedLabel}: `.padEnd(maxLength) + printReceived(received);+ },+};diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.tsnew file mode 100644index 0000000000..d141a0d7fb--- /dev/null+++ b/tests/library/agent-expect.spec.ts@@ -0,0 +1,183 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { stripAnsi } from '../config/utils';+import { browserTest as test, expect } from '../config/browserTest';+import { setCacheObject, runAgent } from './agent-helpers';++// LOWIRE_NO_CACHE=1 to generate api caches+// LOWIRE_FORCE_CACHE=1 to force api caches++test('expectVisible not found error', async ({ context }) => {+ await setCacheObject({+ 'submit button is visible': {+ actions: [{+ code: '',+ method: 'expectVisible',+ selector: `internal:role=button`,+ }],+ },+ });+ const { page, agent } = await runAgent(context);+ await page.setContent(`<button hidden>Submit</button>`);+ const error = await agent.expect('submit button is visible', { timeout: 1000 }).catch(e => e);+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed++Locator: getByRole('button')+Expected: visible+Timeout: 1000ms+Error: element(s) not found++Call log:+ - Expect Visible+ - waiting for getByRole('button')`);+});++test('expectVisible not visible error', async ({ context }) => {+ await setCacheObject({+ 'submit button is visible': {+ actions: [{+ code: '',+ method: 'expectVisible',+ selector: `button`,+ }],+ },+ });+ const { page, agent } = await runAgent(context);+ await page.setContent(`<button hidden>Submit</button>`);+ const error = await agent.expect('submit button is visible', { timeout: 1000 }).catch(e => e);+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed++Locator: locator('button')+Expected: visible+Received: hidden+Timeout: 1000ms++Call log:+ - Expect Visible+ - waiting for locator('button')`);+});++test('not expectVisible visible error', async ({ context }) => {+ await setCacheObject({+ 'submit button is not visible': {+ actions: [{+ code: '',+ method: 'expectVisible',+ selector: `button`,+ isNot: true,+ }],+ },+ });+ const { page, agent } = await runAgent(context);+ await page.setContent(`<button>Submit</button>`);+ const error = await agent.expect('submit button is not visible', { timeout: 1000 }).catch(e => e);+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).not.toBeVisible() failed++Locator: locator('button')+Expected: not visible+Received: visible+Timeout: 1000ms++Call log:+ - Expect Visible+ - waiting for locator('button')`);+});++test('expectChecked not checked error', async ({ context }) => {+ await setCacheObject({+ 'checkbox is checked': {+ actions: [{+ code: '',+ method: 'expectValue',+ type: 'checkbox',+ value: 'true',+ selector: `input`,+ }],+ },+ });+ const { page, agent } = await runAgent(context);+ await page.setContent(`<input type=checkbox>`);+ const error = await agent.expect('checkbox is checked', { timeout: 1000 }).catch(e => e);+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeChecked() failed++Locator: locator('input')+Expected: checked+Received: unchecked+Timeout: 1000ms++Call log:+ - Expect Checked+ - waiting for locator('input')`);+});++test('expectValue wrong value error', async ({ context }) => {+ await setCacheObject({+ 'input has value "hello"': {+ actions: [{+ code: '',+ method: 'expectValue',+ type: 'textbox',+ value: 'hello',+ selector: `input`,+ }],+ },+ });+ const { page, agent } = await runAgent(context);+ await page.setContent(`<input type=text value="world">`);+ const error = await agent.expect('input has value "hello"', { timeout: 1000 }).catch(e => e);+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toHaveValue(expected) failed++Locator: locator('input')+Expected: hello+Received: world+Timeout: 1000ms++Call log:+ - Expect Value+ - waiting for locator('input')`);+});++test('expectAria wrong snapshot error', async ({ context }) => {+ await setCacheObject({+ 'two items are visible': {+ actions: [{+ code: '',+ method: 'expectAria',+ template: '- list:\n - listitem: one\n - listitem: two',+ }],+ },+ });+ const { page, agent } = await runAgent(context);+ await page.setContent(`<ul><li>one</li><li style="display:none">two</li></ul>`);+ const error = await agent.expect('two items are visible', { timeout: 1000 }).catch(e => e);+ const errorMessage = `pageAgent.expect: expect(locator).toMatchAriaSnapshot(expected) failed++Locator: locator('body')+Expected:+- list:+ - listitem: one+ - listitem: two+Received:+- list:+ - listitem: one+Timeout: 1000ms++Call log:+ - Expect Aria Snapshot+ - waiting for locator('body')`.replace('Expected:', 'Expected: ').replace('Received:', 'Received: ');+ expect(stripAnsi(error.message)).toContain(errorMessage);+});diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.tsindex af0920aeb8..b6d99de6a2 100644--- a/tests/library/agent-helpers.ts+++ b/tests/library/agent-helpers.ts@@ -28,6 +28,10 @@ export async function cacheObject() {return JSON.parse(await fs.promises.readFile(cacheFile(), 'utf8'));}+export async function setCacheObject(object: any) {+ await fs.promises.writeFile(cacheFile(), JSON.stringify(object, null, 2), 'utf8');+}+export async function generateAgent(context: BrowserContext, options: { secrets?: Record<string, string> } = {}) {const apiCacheFile = path.join(__dirname, '__llm_cache__', sanitizeFileName(test.info().titlePath.join(' ')) + '.json');diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.tsindex b57cbf1219..cae1477e41 100644--- a/tests/library/agent-perform.spec.ts+++ b/tests/library/agent-perform.spec.ts@@ -14,11 +14,10 @@* limitations under the License.*/-import fs from 'fs';import { z } from 'zod';import { browserTest as test, expect } from '../config/browserTest';-import { run, generateAgent, cacheObject, runAgent, cacheFile } from './agent-helpers';+import { run, generateAgent, cacheObject, runAgent, setCacheObject } from './agent-helpers';// LOWIRE_NO_CACHE=1 to generate api caches// LOWIRE_FORCE_CACHE=1 to force api caches@@ -157,13 +156,13 @@ test('perform run timeout', async ({ context }) => {});test('invalid cache file throws error', async ({ context }) => {- await fs.promises.writeFile(cacheFile(), JSON.stringify({+ await setCacheObject({'some key': {actions: [{method: 'invalid-method',}],},- }));+ });const { agent } = await runAgent(context);await expect(() => agent.perform('click the Test button')).rejects.toThrowError(/.*Failed to parse cache file .*: Invalid discriminator value*/