Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: throw on Frame.expect / Page.expectScreenshot failure (#40801)
Dmitry Gozman7cc4a85b3728
diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.tsindex 22b118974a..c07b844f97 100644--- a/packages/injected/src/injectedScript.ts+++ b/packages/injected/src/injectedScript.ts@@ -48,7 +48,6 @@ import type { Builtins } from './utilityScript';export type FrameExpectParams = Omit<channels.FrameExpectParams, 'expectedValue' | 'timeout'> & {expectedValue?: any;- noAutoWaiting?: boolean;};export type ElementState = 'visible' | 'hidden' | 'enabled' | 'disabled' | 'editable' | 'checked' | 'unchecked' | 'indeterminate' | 'stable';diff --git a/packages/playwright-core/src/client/connection.ts b/packages/playwright-core/src/client/connection.tsindex 4e72253fb1..01c04284bb 100644--- a/packages/playwright-core/src/client/connection.ts+++ b/packages/playwright-core/src/client/connection.ts@@ -42,7 +42,7 @@ import { Stream } from './stream';import { Tracing } from './tracing';import { Worker } from './worker';import { WritableStream } from './writableStream';-import { ValidationError, findValidator } from '../protocol/validator';+import { ValidationError, findValidator, maybeFindValidator } from '../protocol/validator';import type { ClientInstrumentation } from './clientInstrumentation';import type { HeadersArray } from './types';import type { ValidatorContext } from '../protocol/validator';@@ -214,7 +214,7 @@ export class Connection extends EventEmitter {if (this._closedError)return;- const { id, guid, method, params, result, error, log } = message as any;+ const { id, guid, method, params, result, error, errorDetails, log } = message as any;if (id) {if (this._platform.isLogEnabled('channel'))this._platform.log('channel', '<RECV ' + JSON.stringify(message));@@ -224,7 +224,11 @@ export class Connection extends EventEmitter {this._callbacks.delete(id);if (error && !result) {const parsedError = parseError(error);+ parsedError.log = log || [];rewriteErrorMessage(parsedError, parsedError.message + formatCallLog(this._platform, log));+ const detailsValidator = maybeFindValidator(callback.type, callback.method, 'ErrorDetails');+ if (detailsValidator)+ parsedError.details = detailsValidator(errorDetails ?? {}, '', this._validatorFromWireContext());callback.reject(parsedError);} else {const validator = findValidator(callback.type, callback.method, 'Result');diff --git a/packages/playwright-core/src/client/errors.ts b/packages/playwright-core/src/client/errors.tsindex 7ee4438f99..ce4869d00d 100644--- a/packages/playwright-core/src/client/errors.ts+++ b/packages/playwright-core/src/client/errors.ts@@ -19,14 +19,19 @@ import { parseSerializedValue, serializeValue } from '../protocol/serializers';import type { SerializedError } from '@protocol/channels';-export class TimeoutError extends Error {+export class PlaywrightError extends Error {+ log: string[] = [];+ details?: any; // As declared in the protocol.+}++export class TimeoutError extends PlaywrightError {constructor(message: string) {super(message);this.name = 'TimeoutError';}}-export class TargetClosedError extends Error {+export class TargetClosedError extends PlaywrightError {constructor(cause?: string) {super(cause || 'Target page, context or browser has been closed');}@@ -42,24 +47,19 @@ export function serializeError(e: any): SerializedError {return { value: serializeValue(e, value => ({ fallThrough: value })) };}-export function parseError(error: SerializedError): Error {+export function parseError(error: SerializedError): PlaywrightError {if (!error.error) {if (error.value === undefined)throw new Error('Serialized error must have either an error or a value');return parseSerializedValue(error.value, undefined);}- if (error.error.name === 'TimeoutError') {- const e = new TimeoutError(error.error.message);- e.stack = error.error.stack || '';- return e;- }- if (error.error.name === 'TargetClosedError') {- const e = new TargetClosedError(error.error.message);- e.stack = error.error.stack || '';- return e;- }- const e = new Error(error.error.message);+ let e: PlaywrightError;+ if (error.error.name === 'TimeoutError')+ e = new TimeoutError(error.error.message);+ else if (error.error.name === 'TargetClosedError')+ e = new TargetClosedError(error.error.message);+ else+ e = Object.assign(new PlaywrightError(error.error.message), { name: error.error.name });e.stack = error.error.stack || '';- e.name = error.error.name;return e;}diff --git a/packages/playwright-core/src/client/frame.ts b/packages/playwright-core/src/client/frame.tsindex 9f5785c597..bb28e67377 100644--- a/packages/playwright-core/src/client/frame.ts+++ b/packages/playwright-core/src/client/frame.ts@@ -22,6 +22,7 @@ import { EventEmitter } from './eventEmitter';import { ChannelOwner } from './channelOwner';import { addSourceUrlToScript } from './clientHelper';import { ElementHandle, convertInputFiles, convertSelectOptionValues } from './elementHandle';+import { PlaywrightError } from './errors';import { Events } from './events';import { JSHandle, assertMaxArguments, parseResult, serializeArgument } from './jsHandle';import { FrameLocator, Locator, testIdAttributeName } from './locator';@@ -493,20 +494,25 @@ export class Frame extends ChannelOwner<channels.FrameChannel> implements api.Frasync _expect(expression: string, options: Omit<channels.FrameExpectParams, 'expression'>): Promise<ExpectResult> {const params: channels.FrameExpectParams = { expression, ...options, isNot: !!options.isNot };params.expectedValue = serializeArgument(options.expectedValue);- const channelResult = await this._channel.expect(params);- const result: ExpectResult = {- matches: channelResult.matches,- log: channelResult.log,- timedOut: channelResult.timedOut,- errorMessage: channelResult.errorMessage,- };- if (channelResult.received !== undefined && channelResult.matches === !!options.isNot) {- result.received = {- value: channelResult.received.value !== undefined ? parseResult(channelResult.received.value) : undefined,- ariaSnapshot: channelResult.received.ariaSnapshot,+ try {+ await this._channel.expect(params);+ return { matches: !params.isNot };+ } catch (e) {+ if (!(e instanceof PlaywrightError))+ throw e;+ const details = e.details as channels.FrameExpectErrorDetails;+ const received = details.received ? {+ value: details.received.value !== undefined ? parseResult(details.received.value) : undefined,+ ariaSnapshot: details.received.ariaSnapshot,+ } : undefined;+ return {+ matches: !!params.isNot,+ received,+ log: e.log,+ timedOut: details.timedOut,+ errorMessage: details.customErrorMessage ? 'Error: ' + details.customErrorMessage : undefined,};}- return result;}}diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.tsindex 64301fca54..b45b623083 100644--- a/packages/playwright-core/src/client/page.ts+++ b/packages/playwright-core/src/client/page.ts@@ -28,7 +28,7 @@ import { Coverage } from './coverage';import { DisposableObject, DisposableStub } from './disposable';import { Download } from './download';import { ElementHandle, determineScreenshotType } from './elementHandle';-import { TargetClosedError, isTargetClosedError, parseError, serializeError } from './errors';+import { PlaywrightError, TargetClosedError, isTargetClosedError, parseError, serializeError } from './errors';import { Events } from './events';import { FileChooser } from './fileChooser';import { Frame, verifyLoadState } from './frame';@@ -627,12 +627,20 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Pageframe: (options.locator as Locator)._frame._channel,selector: (options.locator as Locator)._selector,} : undefined;- return await this._channel.expectScreenshot({- ...options,- isNot: !!options.isNot,- locator,- mask,- });+ try {+ const result = await this._channel.expectScreenshot({+ ...options,+ isNot: !!options.isNot,+ locator,+ mask,+ });+ return { actual: result.actual };+ } catch (e) {+ if (!(e instanceof PlaywrightError))+ throw e;+ const details = e.details as channels.PageExpectScreenshotErrorDetails;+ return { ...details, errorMessage: details.customErrorMessage };+ }}async title(): Promise<string> {diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.tsindex 656817e780..bbdc20aebf 100644--- a/packages/playwright-core/src/protocol/validator.ts+++ b/packages/playwright-core/src/protocol/validator.ts@@ -1658,15 +1658,14 @@ scheme.FrameExpectParams = tObject({isNot: tBoolean,timeout: tFloat,});-scheme.FrameExpectResult = tObject({- matches: tBoolean,+scheme.FrameExpectResult = tOptional(tObject({}));+scheme.FrameExpectErrorDetails = tObject({received: tOptional(tObject({value: tOptional(tType('SerializedValue')),ariaSnapshot: tOptional(tString),})),timedOut: tOptional(tBoolean),- errorMessage: tOptional(tString),- log: tOptional(tArray(tString)),+ customErrorMessage: tOptional(tString),});scheme.JSHandleInitializer = tObject({preview: tString,@@ -2421,8 +2420,11 @@ scheme.PageExpectScreenshotParams = tObject({style: tOptional(tString),});scheme.PageExpectScreenshotResult = tObject({+ actual: tOptional(tBinary),+});+scheme.PageExpectScreenshotErrorDetails = tObject({diff: tOptional(tBinary),- errorMessage: tOptional(tString),+ customErrorMessage: tOptional(tString),actual: tOptional(tBinary),previous: tOptional(tBinary),timedOut: tOptional(tBoolean),diff --git a/packages/playwright-core/src/protocol/validatorPrimitives.ts b/packages/playwright-core/src/protocol/validatorPrimitives.tsindex ba15b856d8..551497237a 100644--- a/packages/playwright-core/src/protocol/validatorPrimitives.ts+++ b/packages/playwright-core/src/protocol/validatorPrimitives.ts@@ -23,13 +23,13 @@ export type ValidatorContext = {};export const scheme: { [key: string]: Validator } = {};-export function findValidator(type: string, method: string, kind: 'Initializer' | 'Event' | 'Params' | 'Result'): Validator {+export function findValidator(type: string, method: string, kind: 'Initializer' | 'Event' | 'Params' | 'Result' | 'ErrorDetails'): Validator {const validator = maybeFindValidator(type, method, kind);if (!validator)throw new ValidationError(`Unknown scheme for ${kind}: ${type}.${method}`);return validator;}-export function maybeFindValidator(type: string, method: string, kind: 'Initializer' | 'Event' | 'Params' | 'Result'): Validator | undefined {+export function maybeFindValidator(type: string, method: string, kind: 'Initializer' | 'Event' | 'Params' | 'Result' | 'ErrorDetails'): Validator | undefined {const schemeName = type + (kind === 'Initializer' ? '' : method[0].toUpperCase() + method.substring(1)) + kind;return scheme[schemeName];}diff --git a/packages/playwright-core/src/server/dispatchers/dispatcher.ts b/packages/playwright-core/src/server/dispatchers/dispatcher.tsindex 2a75e1d8d3..50460099f4 100644--- a/packages/playwright-core/src/server/dispatchers/dispatcher.ts+++ b/packages/playwright-core/src/server/dispatchers/dispatcher.ts@@ -22,7 +22,7 @@ import { isUnderTest } from '@utils/debug';import { assert } from '@isomorphic/assert';import { monotonicTime } from '@isomorphic/time';import { rewriteErrorMessage } from '@isomorphic/stackTrace';-import { ValidationError, createMetadataValidator, createWaitInfoValidator, findValidator } from '../../protocol/validator';+import { ValidationError, createMetadataValidator, createWaitInfoValidator, findValidator, maybeFindValidator } from '../../protocol/validator';import { TargetClosedError, isTargetClosedError, serializeError } from '../errors';import { createRootSdkObject, SdkObject } from '../instrumentation';import { isProtocolError } from '../protocolError';@@ -366,6 +366,9 @@ export class DispatcherConnection {rewriteErrorMessage(e, 'Target crashed ' + e.browserLogMessage());}response.error = serializeError(e);+ const detailsValidator = maybeFindValidator(dispatcher._type, method, 'ErrorDetails');+ if (detailsValidator)+ response.errorDetails = detailsValidator((e as any)?.details ?? {}, '', this._validatorToWireContext());// The command handler could have set error in the metadata, do not reset it if there was no exception.callMetadata.error = response.error;} finally {diff --git a/packages/playwright-core/src/server/dispatchers/frameDispatcher.ts b/packages/playwright-core/src/server/dispatchers/frameDispatcher.tsindex 9c038ccc20..8c7af45a22 100644--- a/packages/playwright-core/src/server/dispatchers/frameDispatcher.ts+++ b/packages/playwright-core/src/server/dispatchers/frameDispatcher.ts@@ -14,10 +14,8 @@* limitations under the License.*/-import yaml from 'yaml';-import { parseAriaSnapshotUnsafe } from '@isomorphic/ariaSnapshot';import { renderTitleForCall } from '@isomorphic/protocolFormatter';-import { Frame } from '../frames';+import { ExpectError, Frame } from '../frames';import { Dispatcher } from './dispatcher';import { ElementHandleDispatcher } from './elementHandlerDispatcher';import { parseArgument, serializeResult } from './jsHandleDispatcher';@@ -273,23 +271,14 @@ export class FrameDispatcher extends Dispatcher<Frame, channels.FrameChannel, Br}async expect(params: channels.FrameExpectParams, progress: Progress): Promise<channels.FrameExpectResult> {- let expectedValue = params.expectedValue ? parseArgument(params.expectedValue) : undefined;- if (params.expression === 'to.match.aria' && expectedValue)- expectedValue = parseAriaSnapshotUnsafe(yaml, expectedValue);progress.log(`${renderTitleForCall(progress.metadata)}${params.timeout ? ` with timeout ${params.timeout}ms` : ''}`);- const result = await this._frame.expect(progress, params.selector, { ...params, expectedValue });- const channelResult: channels.FrameExpectResult = {- matches: result.matches,- log: result.log,- timedOut: result.timedOut,- errorMessage: result.errorMessage,- };- if (result.received !== undefined) {- channelResult.received = {- value: result.received.value !== undefined ? serializeResult(result.received.value) : undefined,- ariaSnapshot: result.received.ariaSnapshot,- };+ const expectedValue = params.expectedValue ? parseArgument(params.expectedValue) : undefined;+ try {+ await this._frame.expect(progress, params.selector, { ...params, expectedValue });+ } catch (e) {+ if (e instanceof ExpectError && e.details.received && 'value' in e.details.received)+ e.details.received.value = serializeResult(e.details.received.value);+ throw e;}- return channelResult;}}diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.tsindex 92bb6801bc..48a352bcc1 100644--- a/packages/playwright-core/src/server/frames.ts+++ b/packages/playwright-core/src/server/frames.ts@@ -15,6 +15,8 @@* limitations under the License.*/+import yaml from 'yaml';+import { parseAriaSnapshotUnsafe } from '@isomorphic/ariaSnapshot';import { isInvalidSelectorError } from '@isomorphic/selectorParser';import { ManualPromise } from '@isomorphic/manualPromise';import { eventsHelper } from '@utils/eventsHelper';@@ -37,7 +39,6 @@ import { Page, ariaSnapshotForFrame } from './page';import { isAbortError, nullProgress, ProgressController } from './progress';import * as types from './types';import { isSessionClosedError } from './protocolError';-import { compressCallLog } from './callLog';import type { ConsoleMessage } from './console';import type { SelectorInfo } from './frameSelectors';@@ -95,8 +96,23 @@ export class NavigationAbortedError extends Error {}}-export type ExpectReceived = { value?: any, ariaSnapshot?: string };-export type ExpectResult = { matches: boolean, received?: ExpectReceived, log?: string[], timedOut?: boolean, errorMessage?: string };+type ExpectReceived = { value?: any, ariaSnapshot?: string };++type ExpectErrorDetails = {+ received?: ExpectReceived;+ timedOut?: boolean;+ customErrorMessage?: string;+};++export class ExpectError extends Error {+ readonly details: ExpectErrorDetails;++ constructor(details: ExpectErrorDetails) {+ super('Expect failed');+ this.name = 'ExpectError';+ this.details = details;+ }+}const kDummyFrameId = '<dummy>';@@ -1484,37 +1500,38 @@ export class Frame extends SdkObject<FrameEventMap> {}}- async expect(progress: Progress, selector: string | undefined, options: FrameExpectParams): Promise<ExpectResult> {- const lastIntermediateResult: { received?: ExpectReceived, isSet: boolean, errorMessage?: string } = { isSet: false };- const fixupMetadataError = (result: ExpectResult) => {- // Library mode special case for the expect errors which are return values, not exceptions.- if (result.matches === options.isNot)- progress.metadata.error = { error: { name: 'Expect', message: 'Expect failed' } };- };+ async expect(progress: Progress, selector: string | undefined, options: FrameExpectParams): Promise<void> {+ if (options.expression === 'to.match.aria' && options.expectedValue) {+ try {+ options = { ...options, expectedValue: parseAriaSnapshotUnsafe(yaml, options.expectedValue) };+ } catch (e) {+ throw new ExpectError({ customErrorMessage: e.message });+ }+ }+ // `isSet` distinguishes "not collected yet" from "collected with received: undefined".+ const lastIntermediateResult: { isSet: boolean, received?: ExpectReceived, errorMessage?: string } = { isSet: false };try {// Step 1: perform locator handlers checkpoint with a specified timeout.if (selector)progress.log(`waiting for ${this._asLocator(selector)}`);- if (!options.noAutoWaiting)- await this._page.performActionPreChecks(progress);+ await this._page.performActionPreChecks(progress);// Step 2: perform one-shot expect check without a timeout.// Supports the case of `expect(locator).toBeVisible({ timeout: 1 })`// that should succeed when the locator is already visible.try {const resultOneShot = await this._expectInternal(progress, selector, options, lastIntermediateResult, true);- if (options.noAutoWaiting || resultOneShot.matches !== options.isNot)- return resultOneShot;+ if (resultOneShot.matches !== options.isNot)+ return;} catch (e) {- if (options.noAutoWaiting || this.isNonRetriableError(e))+ if (this.isNonRetriableError(e))throw e;// Ignore any other errors from one-shot, we'll handle them during retries.}// Step 3: auto-retry expect with increasing timeouts. Bounded by the total remaining time.- const result = await this.retryWithProgressAndBackoff(progress, async (progress, continuePolling) => {- if (!options.noAutoWaiting)- await this._page.performActionPreChecks(progress);+ await this.retryWithProgressAndBackoff(progress, async (progress, continuePolling) => {+ await this._page.performActionPreChecks(progress);const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false);if (matches === options.isNot) {// Keep waiting in these cases:@@ -1524,24 +1541,19 @@ export class Frame extends SdkObject<FrameEventMap> {}return { matches, received };});- fixupMetadataError(result);- return result;} catch (e) {- // Q: Why not throw upon isNonRetriableError(e) as in other places?- // A: We want user to receive a friendly message containing the last intermediate result.- const result: ExpectResult = { matches: options.isNot, log: compressCallLog(progress.metadata.log) };+ const details: ExpectErrorDetails = {};if (isInvalidSelectorError(e)) {- result.errorMessage = 'Error: ' + e.message;+ details.customErrorMessage = e.message;} else if (js.isJavaScriptErrorInEvaluate(e)) {- result.errorMessage = e.message;+ details.customErrorMessage = e.message.startsWith('Error: ') ? e.message.substring('Error: '.length) : e.message;} else if (lastIntermediateResult.isSet) {- result.received = lastIntermediateResult.received;- result.errorMessage = lastIntermediateResult.errorMessage;+ details.received = lastIntermediateResult.received;+ details.customErrorMessage = lastIntermediateResult.errorMessage;}if (e instanceof TimeoutError)- result.timedOut = true;- fixupMetadataError(result);- return result;+ details.timedOut = true;+ throw new ExpectError(details);}}@@ -1577,7 +1589,7 @@ export class Frame extends SdkObject<FrameEventMap> {progressLog(log);// Note: missingReceived avoids `unexpected value "undefined"` when element was not found.if (matches === options.isNot) {- lastIntermediateResult.errorMessage = missingReceived ? 'Error: element(s) not found' : undefined;+ lastIntermediateResult.errorMessage = missingReceived ? 'element(s) not found' : undefined;lastIntermediateResult.received = received;lastIntermediateResult.isSet = true;if (!missingReceived && !Array.isArray(received?.value))diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.tsindex dbe1dbba12..c26d5d39dd 100644--- a/packages/playwright-core/src/server/page.ts+++ b/packages/playwright-core/src/server/page.ts@@ -705,7 +705,7 @@ export class Page extends SdkObject<PageEventMap> {await this.delegate.updateRequestInterception();}- async expectScreenshot(progress: Progress, options: ExpectScreenshotOptions): Promise<{ actual?: Buffer, previous?: Buffer, diff?: Buffer, errorMessage?: string, log?: string[], timedOut?: boolean }> {+ async expectScreenshot(progress: Progress, options: ExpectScreenshotOptions): Promise<{ actual?: Buffer }> {const locator = options.locator;const rafrafScreenshot = locator ? async (progress: Progress, timeout: number) => {return await locator.frame.rafrafTimeoutScreenshotElementWithProgress(progress, locator.selector, timeout, options || {});@@ -716,15 +716,6 @@ export class Page extends SdkObject<PageEventMap> {};const comparator = getComparator('image/png');- if (!options.expected && options.isNot)- return { errorMessage: '"not" matcher requires expected result' };- try {- const format = validateScreenshotOptions(options || {});- if (format !== 'png')- throw new Error('Only PNG screenshots are supported');- } catch (error) {- return { errorMessage: error.message };- }let intermediateResult: {actual?: Buffer,previous?: Buffer,@@ -741,6 +732,11 @@ export class Page extends SdkObject<PageEventMap> {};try {+ if (!options.expected && options.isNot)+ throw new Error('"not" matcher requires expected result');+ const format = validateScreenshotOptions(options || {});+ if (format !== 'png')+ throw new Error('Only PNG screenshots are supported');let actual: Buffer | undefined;let previous: Buffer | undefined;const pollIntervals = [0, 100, 250, 500];@@ -797,12 +793,17 @@ export class Page extends SdkObject<PageEventMap> {let errorMessage = e.message;if (e instanceof TimeoutError && intermediateResult?.previous)errorMessage = `Failed to take two consecutive stable screenshots.`;- return {+ const details: channels.PageExpectScreenshotErrorDetails = {log: compressCallLog(e.message ? [...progress.metadata.log, e.message] : progress.metadata.log),- ...intermediateResult,- errorMessage,+ actual: intermediateResult?.actual,+ previous: intermediateResult?.previous,+ diff: intermediateResult?.diff,timedOut: (e instanceof TimeoutError),+ customErrorMessage: errorMessage,};+ const error = new Error('Expect failed');+ (error as any).details = details;+ throw error;}}diff --git a/packages/playwright-core/src/server/recorder.ts b/packages/playwright-core/src/server/recorder.tsindex 43ad5d0df1..925520399b 100644--- a/packages/playwright-core/src/server/recorder.ts+++ b/packages/playwright-core/src/server/recorder.ts@@ -582,9 +582,12 @@ export class Recorder extends EventEmitter<RecorderEventMap> implements Instrumeprivate async _performAction(progress: Progress, frame: Frame, action: actions.PerformOnRecordAction) {const actionInContext = await this._createActionInContext(progress, frame, action);this._signalProcessor.addAction(actionInContext);- if (actionInContext.action.name !== 'openPage' && actionInContext.action.name !== 'closePage')- await performAction(progress, this._pageAliases, actionInContext);- actionInContext.endTime = monotonicTime();+ try {+ if (actionInContext.action.name !== 'openPage' && actionInContext.action.name !== 'closePage')+ await performAction(progress, this._pageAliases, actionInContext);+ } finally {+ actionInContext.endTime = monotonicTime();+ }}private async _recordAction(progress: Progress, frame: Frame, action: actions.Action) {diff --git a/packages/protocol/spec/frame.yml b/packages/protocol/spec/frame.ymlindex 4c92922cdf..a2ca0bfd71 100644--- a/packages/protocol/spec/frame.yml+++ b/packages/protocol/spec/frame.yml@@ -766,18 +766,14 @@ Frame:useInnerText: boolean?isNot: booleantimeout: float- returns:- matches: boolean+ errorDetails:received:type: object?properties:value: SerializedValue?ariaSnapshot: string?timedOut: boolean?- errorMessage: string?- log:- type: array?- items: string+ customErrorMessage: string?flags:snapshot: truepause: truediff --git a/packages/protocol/spec/page.yml b/packages/protocol/spec/page.ymlindex c1b077c04b..73a3adc218 100644--- a/packages/protocol/spec/page.yml+++ b/packages/protocol/spec/page.yml@@ -189,8 +189,10 @@ Page:clip: Rect?$mixin: CommonScreenshotOptionsreturns:+ actual: binary?+ errorDetails:diff: binary?- errorMessage: string?+ customErrorMessage: string?actual: binary?previous: binary?timedOut: boolean?diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.tsindex 02409015ca..09fb8655f9 100644--- a/packages/protocol/src/channels.d.ts+++ b/packages/protocol/src/channels.d.ts@@ -3030,15 +3030,14 @@ export type FrameExpectOptions = {expectedValue?: SerializedArgument,useInnerText?: boolean,};-export type FrameExpectResult = {- matches: boolean,+export type FrameExpectResult = void;+export type FrameExpectErrorDetails = {received?: {value?: SerializedValue,ariaSnapshot?: string,},timedOut?: boolean,- errorMessage?: string,- log?: string[],+ customErrorMessage?: string,};export interface FrameEvents {@@ -4325,8 +4324,11 @@ export type PageExpectScreenshotOptions = {style?: string,};export type PageExpectScreenshotResult = {+ actual?: Binary,+};+export type PageExpectScreenshotErrorDetails = {diff?: Binary,- errorMessage?: string,+ customErrorMessage?: string,actual?: Binary,previous?: Binary,timedOut?: boolean,diff --git a/tests/library/inspector/pause.spec.ts b/tests/library/inspector/pause.spec.tsindex a7c820343d..74c5dfd61e 100644--- a/tests/library/inspector/pause.spec.ts+++ b/tests/library/inspector/pause.spec.ts@@ -395,6 +395,27 @@ it.describe('pause', () => {expect(error.message).toContain('Not a checkbox or radio button');});+ it('should populate log with expect failure', async ({ page, recorderPageGetter }) => {+ await page.setContent('<button>Submit</button>');+ const scriptPromise = (async () => {+ // @ts-ignore+ await page.pause({ __testHookKeepTestTimeout: true });+ await expect(page.getByRole('button')).toHaveText('Other', { timeout: 1 });+ })().catch(e => e);+ const recorderPage = await recorderPageGetter();+ await recorderPage.click('[title="Resume (F8)"]');+ await recorderPage.waitForSelector('.source-line-error-underline');+ expect(await sanitizeLog(recorderPage)).toEqual([+ 'Pause- XXms',+ 'Expect "toHaveText"(page.getByRole(\'button\'))- XXms',+ 'Expect "toHaveText" with timeout 1ms',+ 'waiting for getByRole(\'button\')',+ 'error: Expect failed',+ ]);+ const error = await scriptPromise;+ expect(error.message).toContain('toHaveText');+ });+it('should populate log with error in waitForEvent', async ({ page, recorderPageGetter }) => {await page.setContent('<button>Submit</button>');const scriptPromise = (async () => {diff --git a/tests/library/trace-viewer.spec.ts b/tests/library/trace-viewer.spec.tsindex b5a2b3903b..1ea2337efc 100644--- a/tests/library/trace-viewer.spec.ts+++ b/tests/library/trace-viewer.spec.ts@@ -403,8 +403,6 @@ test('should show params and return value', async ({ showTraceViewer }) => {/locator:locator\('button'\)/,/expression:"to.have.text"/,/timeout:10000/,- /matches:true/,- /received:"Click"/,]);});diff --git a/tests/page/to-match-aria-snapshot.spec.ts b/tests/page/to-match-aria-snapshot.spec.tsindex fe951e052b..14e4fb1bb1 100644--- a/tests/page/to-match-aria-snapshot.spec.ts+++ b/tests/page/to-match-aria-snapshot.spec.ts@@ -609,17 +609,30 @@ test('should report error in YAML', async ({ page }) => {const error = await expect(page).toMatchAriaSnapshot(`heading "title"`).catch(e => e);- expect.soft(error.message).toBe(`expect.toMatchAriaSnapshot: Aria snapshot must be a YAML sequence, elements starting with " -"`);+ expect.soft(stripAnsi(error.message)).toBe(`expect(page).toMatchAriaSnapshot(expected) failed++Expected: "heading \\"title\\""+Error: Aria snapshot must be a YAML sequence, elements starting with " -"++Call log:+ - Expect "toMatchAriaSnapshot" with timeout 10000ms+`);}{const error = await expect(page).toMatchAriaSnapshot(`- heading: a:`).catch(e => e);- expect.soft(error.message).toBe(`expect.toMatchAriaSnapshot: Nested mappings are not allowed in compact mappings at line 1, column 12:+ expect.soft(stripAnsi(error.message)).toBe(`expect(page).toMatchAriaSnapshot(expected) failed++Expected: "- heading: a:"+Error: Nested mappings are not allowed in compact mappings at line 1, column 12:- heading: a:^++Call log:+ - Expect "toMatchAriaSnapshot" with timeout 10000ms`);}});@@ -631,10 +644,16 @@ test('should report error in YAML keys', async ({ page }) => {const error = await expect(page).toMatchAriaSnapshot(`- heading "title`).catch(e => e);- expect.soft(error.message).toBe(`expect.toMatchAriaSnapshot: Unterminated string:+ expect.soft(stripAnsi(error.message)).toBe(`expect(page).toMatchAriaSnapshot(expected) failed++Expected: "- heading \\"title"+Error: Unterminated string:heading "title^++Call log:+ - Expect "toMatchAriaSnapshot" with timeout 10000ms`);}@@ -642,10 +661,16 @@ heading "titleconst error = await expect(page).toMatchAriaSnapshot(`- heading /title`).catch(e => e);- expect.soft(error.message).toBe(`expect.toMatchAriaSnapshot: Unterminated regex:+ expect.soft(stripAnsi(error.message)).toBe(`expect(page).toMatchAriaSnapshot(expected) failed++Expected: "- heading /title"+Error: Unterminated regex:heading /title^++Call log:+ - Expect "toMatchAriaSnapshot" with timeout 10000ms`);}@@ -653,10 +678,16 @@ heading /titleconst error = await expect(page).toMatchAriaSnapshot(`- heading [level=a]`).catch(e => e);- expect.soft(error.message).toBe(`expect.toMatchAriaSnapshot: Value of "level" attribute must be a number:+ expect.soft(stripAnsi(error.message)).toBe(`expect(page).toMatchAriaSnapshot(expected) failed++Expected: "- heading [level=a]"+Error: Value of "level" attribute must be a number:heading [level=a]^++Call log:+ - Expect "toMatchAriaSnapshot" with timeout 10000ms`);}@@ -664,10 +695,16 @@ heading [level=a]const error = await expect(page).toMatchAriaSnapshot(`- heading [expanded=FALSE]`).catch(e => e);- expect.soft(error.message).toBe(`expect.toMatchAriaSnapshot: Value of "expanded" attribute must be a boolean:+ expect.soft(stripAnsi(error.message)).toBe(`expect(page).toMatchAriaSnapshot(expected) failed++Expected: "- heading [expanded=FALSE]"+Error: Value of "expanded" attribute must be a boolean:heading [expanded=FALSE]^++Call log:+ - Expect "toMatchAriaSnapshot" with timeout 10000ms`);}@@ -675,10 +712,16 @@ heading [expanded=FALSE]const error = await expect(page).toMatchAriaSnapshot(`- heading [checked=foo]`).catch(e => e);- expect.soft(error.message).toBe(`expect.toMatchAriaSnapshot: Value of "checked" attribute must be a boolean or "mixed":+ expect.soft(stripAnsi(error.message)).toBe(`expect(page).toMatchAriaSnapshot(expected) failed++Expected: "- heading [checked=foo]"+Error: Value of "checked" attribute must be a boolean or "mixed":heading [checked=foo]^++Call log:+ - Expect "toMatchAriaSnapshot" with timeout 10000ms`);}@@ -686,10 +729,16 @@ heading [checked=foo]const error = await expect(page).toMatchAriaSnapshot(`- heading [level=]`).catch(e => e);- expect.soft(error.message).toBe(`expect.toMatchAriaSnapshot: Value of "level" attribute must be a number:+ expect.soft(stripAnsi(error.message)).toBe(`expect(page).toMatchAriaSnapshot(expected) failed++Expected: "- heading [level=]"+Error: Value of "level" attribute must be a number:heading [level=]^++Call log:+ - Expect "toMatchAriaSnapshot" with timeout 10000ms`);}@@ -697,10 +746,16 @@ heading [level=]const error = await expect(page).toMatchAriaSnapshot(`- heading [bogus]`).catch(e => e);- expect.soft(error.message).toBe(`expect.toMatchAriaSnapshot: Unsupported attribute [bogus]:+ expect.soft(stripAnsi(error.message)).toBe(`expect(page).toMatchAriaSnapshot(expected) failed++Expected: "- heading [bogus]"+Error: Unsupported attribute [bogus]:heading [bogus]^++Call log:+ - Expect "toMatchAriaSnapshot" with timeout 10000ms`);}@@ -708,10 +763,16 @@ heading [bogus]const error = await expect(page).toMatchAriaSnapshot(`- heading invalid`).catch(e => e);- expect.soft(error.message).toBe(`expect.toMatchAriaSnapshot: Unexpected input:+ expect.soft(stripAnsi(error.message)).toBe(`expect(page).toMatchAriaSnapshot(expected) failed++Expected: "- heading invalid"+Error: Unexpected input:heading invalid^++Call log:+ - Expect "toMatchAriaSnapshot" with timeout 10000ms`);}});diff --git a/utils/generate_channels.js b/utils/generate_channels.jsindex cd33d24177..33856d271f 100755--- a/utils/generate_channels.js+++ b/utils/generate_channels.js@@ -341,6 +341,15 @@ for (const [name, item] of Object.entries(protocol)) {for (const derived of derivedClasses.get(channelName) || [])addScheme(`${derived}${titleCase(methodName)}Result`, `tType('${resultName}')`);+ if (method.errorDetails) {+ const errorDetailsName = `${channelName}${titleCase(methodName)}ErrorDetails`;+ const details = objectType(method.errorDetails, '');+ ts_types.set(errorDetailsName, details.ts);+ addScheme(errorDetailsName, details.scheme);+ for (const derived of derivedClasses.get(channelName) || [])+ addScheme(`${derived}${titleCase(methodName)}ErrorDetails`, `tType('${errorDetailsName}')`);+ }+channels_ts.push(` ${methodName}(params${method.parameters ? '' : '?'}: ${paramsName}, progress?: Progress): Promise<${resultName}>;`);}