Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(agent): format expect errors in the same way (#38797)

Dmitry Gozmanc7c1c48003e1
diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts
index 4a46612a11..0d57a3cc0d 100644
--- a/packages/playwright-core/src/server/agent/actionRunner.ts
+++ b/packages/playwright-core/src/server/agent/actionRunner.ts
@@ -14,10 +14,11 @@
* limitations under the License.
*/
-import { serializeExpectedTextValues } from '../utils/expectUtils';
+import { formatMatcherMessage, serializeExpectedTextValues, simpleMatcherUtils } from '../utils/expectUtils';
import { monotonicTime } from '../../utils/isomorphic/time';
import { createGuid } from '../utils/crypto';
import { parseAriaSnapshotUnsafe } from '../../utils/isomorphic/ariaSnapshot';
+import { asLocatorDescription } from '../../utils/isomorphic/locatorGenerators';
import { yaml } from '../../utilsBundle';
import { serializeError } from '../errors';
@@ -25,9 +26,10 @@ import type * as actions from './actions';
import type { Page } from '../page';
import type { Progress } from '../progress';
import type { NameValue } from '@protocol/channels';
-import type { ExpectResult, Frame } from '../frames';
+import type { Frame } from '../frames';
import type { CallMetadata } from '../instrumentation';
import type * as channels from '@protocol/channels';
+import type { FrameExpectParams } from '@injected/injectedScript';
export async function runAction(progress: Progress, mode: 'generate' | 'run', page: Page, action: actions.Action, secrets: NameValue[]) {
const parentMetadata = progress.metadata;
@@ -99,36 +101,49 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
await frame.uncheck(progress, action.selector, { ...commonOptions });
break;
case 'expectVisible': {
- const result = await frame.expect(progress, action.selector, { expression: 'to.be.visible', isNot: !!action.isNot });
- if (!result.matches === !action.isNot)
- throw new Error(result.errorMessage);
+ await runExpect(frame, progress, action.selector, { expression: 'to.be.visible', isNot: !!action.isNot }, 'visible', 'toBeVisible', '');
break;
}
case 'expectValue': {
- let result: ExpectResult;
if (action.type === 'textbox' || action.type === 'combobox' || action.type === 'slider') {
const expectedText = serializeExpectedTextValues([action.value]);
- result = await frame.expect(progress, action.selector, { expression: 'to.have.value', expectedText, isNot: !!action.isNot });
+ await runExpect(frame, progress, action.selector, { expression: 'to.have.value', expectedText, isNot: !!action.isNot }, action.value, 'toHaveValue', 'expected');
} else if (action.type === 'checkbox' || action.type === 'radio') {
const expectedValue = { checked: action.value === 'true' };
- result = await frame.expect(progress, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: !!action.isNot });
+ await runExpect(frame, progress, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: !!action.isNot }, action.value ? 'checked' : 'unchecked', 'toBeChecked', '');
} else {
throw new Error(`Unsupported element type: ${action.type}`);
}
- if (!result.matches === !action.isNot)
- throw new Error(result.errorMessage);
break;
}
case 'expectAria': {
const expectedValue = parseAriaSnapshotUnsafe(yaml, action.template);
- const result = await frame.expect(progress, 'body', { expression: 'to.match.aria', expectedValue, isNot: !!action.isNot });
- if (!result.matches === !action.isNot)
- throw new Error(result.errorMessage);
+ await runExpect(frame, progress, 'body', { expression: 'to.match.aria', expectedValue, isNot: !!action.isNot }, '\n' + action.template, 'toMatchAriaSnapshot', 'expected');
break;
}
}
}
+async function runExpect(frame: Frame, progress: Progress, selector: string | undefined, options: FrameExpectParams, expected: string, matcherName: string, expectation: string) {
+ const result = await frame.expect(progress, selector, options);
+ if (!result.matches === !options.isNot) {
+ const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received;
+ throw new Error(formatMatcherMessage(simpleMatcherUtils, {
+ isNot: options.isNot,
+ matcherName,
+ expectation,
+ locator: selector ? asLocatorDescription('javascript', selector) : undefined,
+ timedOut: result.timedOut,
+ timeout: progress.timeout,
+ printedExpected: options.isNot ? `Expected: not ${expected}` : `Expected: ${expected}`,
+ printedReceived: result.errorMessage ? '' : `Received: ${received}`,
+ errorMessage: result.errorMessage,
+ // Note: we are not passing call log, because it will be automatically appended on the client side,
+ // as a part of the agent.{perform,expect} call.
+ }));
+ }
+}
+
export function traceParamsForAction(progress: Progress, action: actions.Action): { title?: string, type: string, method: string, params: any } {
const timeout = progress.timeout;
switch (action.method) {
diff --git a/packages/playwright-core/src/server/utils/expectUtils.ts b/packages/playwright-core/src/server/utils/expectUtils.ts
index 3fac8db843..adfef78cac 100644
--- a/packages/playwright-core/src/server/utils/expectUtils.ts
+++ b/packages/playwright-core/src/server/utils/expectUtils.ts
@@ -15,6 +15,7 @@
*/
import { isRegExp, isString } from '../../utils/isomorphic/rtti';
+import { colors } from '../../utilsBundle';
import type { ExpectedTextValue } from '@protocol/channels';
@@ -144,3 +145,34 @@ Call log:
${utils.DIM_COLOR(log.join('\n'))}
`;
};
+
+
+function printValue(value: unknown): string {
+ try {
+ return JSON.stringify(value);
+ } catch {
+ return String(value);
+ }
+}
+
+function printReceived(value: unknown): string {
+ return colors.red(printValue(value));
+}
+
+function printExpected(value: unknown): string {
+ return colors.green(printValue(value));
+}
+
+export const simpleMatcherUtils: InternalMatcherUtils = {
+ DIM_COLOR: colors.dim,
+ RECEIVED_COLOR: colors.red,
+ EXPECTED_COLOR: colors.green,
+ INVERTED_COLOR: colors.inverse,
+ printReceived,
+ printExpected,
+ printDiffOrStringify: (expected: unknown, received: unknown, expectedLabel: string, receivedLabel: string) => {
+ const maxLength = Math.max(expectedLabel.length, receivedLabel.length) + 2;
+ return `${expectedLabel}: `.padEnd(maxLength) + printExpected(expected) + `\n` +
+ `${receivedLabel}: `.padEnd(maxLength) + printReceived(received);
+ },
+};
diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.ts
new file mode 100644
index 0000000000..d141a0d7fb
--- /dev/null
+++ b/tests/library/agent-expect.spec.ts
@@ -0,0 +1,183 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { stripAnsi } from '../config/utils';
+import { browserTest as test, expect } from '../config/browserTest';
+import { setCacheObject, runAgent } from './agent-helpers';
+
+// LOWIRE_NO_CACHE=1 to generate api caches
+// LOWIRE_FORCE_CACHE=1 to force api caches
+
+test('expectVisible not found error', async ({ context }) => {
+ await setCacheObject({
+ 'submit button is visible': {
+ actions: [{
+ code: '',
+ method: 'expectVisible',
+ selector: `internal:role=button`,
+ }],
+ },
+ });
+ const { page, agent } = await runAgent(context);
+ await page.setContent(`<button hidden>Submit</button>`);
+ const error = await agent.expect('submit button is visible', { timeout: 1000 }).catch(e => e);
+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed
+
+Locator: getByRole('button')
+Expected: visible
+Timeout: 1000ms
+Error: element(s) not found
+
+Call log:
+ - Expect Visible
+ - waiting for getByRole('button')`);
+});
+
+test('expectVisible not visible error', async ({ context }) => {
+ await setCacheObject({
+ 'submit button is visible': {
+ actions: [{
+ code: '',
+ method: 'expectVisible',
+ selector: `button`,
+ }],
+ },
+ });
+ const { page, agent } = await runAgent(context);
+ await page.setContent(`<button hidden>Submit</button>`);
+ const error = await agent.expect('submit button is visible', { timeout: 1000 }).catch(e => e);
+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed
+
+Locator: locator('button')
+Expected: visible
+Received: hidden
+Timeout: 1000ms
+
+Call log:
+ - Expect Visible
+ - waiting for locator('button')`);
+});
+
+test('not expectVisible visible error', async ({ context }) => {
+ await setCacheObject({
+ 'submit button is not visible': {
+ actions: [{
+ code: '',
+ method: 'expectVisible',
+ selector: `button`,
+ isNot: true,
+ }],
+ },
+ });
+ const { page, agent } = await runAgent(context);
+ await page.setContent(`<button>Submit</button>`);
+ const error = await agent.expect('submit button is not visible', { timeout: 1000 }).catch(e => e);
+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).not.toBeVisible() failed
+
+Locator: locator('button')
+Expected: not visible
+Received: visible
+Timeout: 1000ms
+
+Call log:
+ - Expect Visible
+ - waiting for locator('button')`);
+});
+
+test('expectChecked not checked error', async ({ context }) => {
+ await setCacheObject({
+ 'checkbox is checked': {
+ actions: [{
+ code: '',
+ method: 'expectValue',
+ type: 'checkbox',
+ value: 'true',
+ selector: `input`,
+ }],
+ },
+ });
+ const { page, agent } = await runAgent(context);
+ await page.setContent(`<input type=checkbox>`);
+ const error = await agent.expect('checkbox is checked', { timeout: 1000 }).catch(e => e);
+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeChecked() failed
+
+Locator: locator('input')
+Expected: checked
+Received: unchecked
+Timeout: 1000ms
+
+Call log:
+ - Expect Checked
+ - waiting for locator('input')`);
+});
+
+test('expectValue wrong value error', async ({ context }) => {
+ await setCacheObject({
+ 'input has value "hello"': {
+ actions: [{
+ code: '',
+ method: 'expectValue',
+ type: 'textbox',
+ value: 'hello',
+ selector: `input`,
+ }],
+ },
+ });
+ const { page, agent } = await runAgent(context);
+ await page.setContent(`<input type=text value="world">`);
+ const error = await agent.expect('input has value "hello"', { timeout: 1000 }).catch(e => e);
+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toHaveValue(expected) failed
+
+Locator: locator('input')
+Expected: hello
+Received: world
+Timeout: 1000ms
+
+Call log:
+ - Expect Value
+ - waiting for locator('input')`);
+});
+
+test('expectAria wrong snapshot error', async ({ context }) => {
+ await setCacheObject({
+ 'two items are visible': {
+ actions: [{
+ code: '',
+ method: 'expectAria',
+ template: '- list:\n - listitem: one\n - listitem: two',
+ }],
+ },
+ });
+ const { page, agent } = await runAgent(context);
+ await page.setContent(`<ul><li>one</li><li style="display:none">two</li></ul>`);
+ const error = await agent.expect('two items are visible', { timeout: 1000 }).catch(e => e);
+ const errorMessage = `pageAgent.expect: expect(locator).toMatchAriaSnapshot(expected) failed
+
+Locator: locator('body')
+Expected:
+- list:
+ - listitem: one
+ - listitem: two
+Received:
+- list:
+ - listitem: one
+Timeout: 1000ms
+
+Call log:
+ - Expect Aria Snapshot
+ - waiting for locator('body')`.replace('Expected:', 'Expected: ').replace('Received:', 'Received: ');
+ expect(stripAnsi(error.message)).toContain(errorMessage);
+});
diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.ts
index af0920aeb8..b6d99de6a2 100644
--- a/tests/library/agent-helpers.ts
+++ b/tests/library/agent-helpers.ts
@@ -28,6 +28,10 @@ export async function cacheObject() {
return JSON.parse(await fs.promises.readFile(cacheFile(), 'utf8'));
}
+export async function setCacheObject(object: any) {
+ await fs.promises.writeFile(cacheFile(), JSON.stringify(object, null, 2), 'utf8');
+}
+
export async function generateAgent(context: BrowserContext, options: { secrets?: Record<string, string> } = {}) {
const apiCacheFile = path.join(__dirname, '__llm_cache__', sanitizeFileName(test.info().titlePath.join(' ')) + '.json');
diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts
index b57cbf1219..cae1477e41 100644
--- a/tests/library/agent-perform.spec.ts
+++ b/tests/library/agent-perform.spec.ts
@@ -14,11 +14,10 @@
* limitations under the License.
*/
-import fs from 'fs';
import { z } from 'zod';
import { browserTest as test, expect } from '../config/browserTest';
-import { run, generateAgent, cacheObject, runAgent, cacheFile } from './agent-helpers';
+import { run, generateAgent, cacheObject, runAgent, setCacheObject } from './agent-helpers';
// LOWIRE_NO_CACHE=1 to generate api caches
// LOWIRE_FORCE_CACHE=1 to force api caches
@@ -157,13 +156,13 @@ test('perform run timeout', async ({ context }) => {
});
test('invalid cache file throws error', async ({ context }) => {
- await fs.promises.writeFile(cacheFile(), JSON.stringify({
+ await setCacheObject({
'some key': {
actions: [{
method: 'invalid-method',
}],
},
- }));
+ });
const { agent } = await runAgent(context);
await expect(() => agent.perform('click the Test button')).rejects.toThrowError(
/.*Failed to parse cache file .*: Invalid discriminator value*/