Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: step error recovery framework (#36996)
Pavel Feldmand669fcde0ead
diff --git a/packages/playwright-core/src/client/channelOwner.ts b/packages/playwright-core/src/client/channelOwner.tsindex 7bd097b3ff..31adda7d17 100644--- a/packages/playwright-core/src/client/channelOwner.ts+++ b/packages/playwright-core/src/client/channelOwner.ts@@ -20,7 +20,7 @@ import { methodMetainfo } from '../utils/isomorphic/protocolMetainfo';import { captureLibraryStackTrace } from './clientStackTrace';import { stringifyStackFrames } from '../utils/isomorphic/stackTrace';-import type { ClientInstrumentation } from './clientInstrumentation';+import type { ClientInstrumentation, RecoverFromApiErrorHandler } from './clientInstrumentation';import type { Connection } from './connection';import type { Logger } from './types';import type { ValidatorContext } from '../protocol/validator';@@ -199,7 +199,14 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channelelsee.stack = '';if (!options?.internal) {+ const recoveryHandlers: RecoverFromApiErrorHandler[] = [];apiZone.error = e;+ this._instrumentation.onApiCallRecovery(apiZone, e, recoveryHandlers);+ for (const handler of recoveryHandlers) {+ const recoverResult = await handler();+ if (recoverResult.status === 'recovered')+ return recoverResult.value as R;+ }logApiCall(this._platform, logger, `<= ${apiZone.apiName} failed`);this._instrumentation.onApiCallEnd(apiZone);}diff --git a/packages/playwright-core/src/client/clientInstrumentation.ts b/packages/playwright-core/src/client/clientInstrumentation.tsindex b14a3de064..968e3d9810 100644--- a/packages/playwright-core/src/client/clientInstrumentation.ts+++ b/packages/playwright-core/src/client/clientInstrumentation.ts@@ -28,12 +28,20 @@ export interface ApiCallData {error?: Error;}+export type RecoverFromApiErrorResult = {+ status: 'recovered' | 'failed';+ value?: string | number | boolean | undefined;+};++export type RecoverFromApiErrorHandler = () => Promise<RecoverFromApiErrorResult>;+export interface ClientInstrumentation {addListener(listener: ClientInstrumentationListener): void;removeListener(listener: ClientInstrumentationListener): void;removeAllListeners(): void;onApiCallBegin(apiCall: ApiCallData, channel: { type: string, method: string, params?: Record<string, any> }): void;- onApiCallEnd(apiCal: ApiCallData): void;+ onApiCallRecovery(apiCall: ApiCallData, error: Error, recoveryHandlers: RecoverFromApiErrorHandler[]): void;+ onApiCallEnd(apiCall: ApiCallData): void;onWillPause(options: { keepTestTimeout: boolean }): void;runAfterCreateBrowserContext(context: BrowserContext): Promise<void>;@@ -44,6 +52,7 @@ export interface ClientInstrumentation {export interface ClientInstrumentationListener {onApiCallBegin?(apiCall: ApiCallData, channel: { type: string, method: string, params?: Record<string, any> }): void;+ onApiCallRecovery?(apiCall: ApiCallData, error: Error, recoveryHandlers: RecoverFromApiErrorHandler[]): void;onApiCallEnd?(apiCall: ApiCallData): void;onWillPause?(options: { keepTestTimeout: boolean }): void;diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.tsindex 806aed0532..e4645d1c69 100644--- a/packages/playwright/src/common/ipc.ts+++ b/packages/playwright/src/common/ipc.ts@@ -21,6 +21,7 @@ import { serializeCompilationCache } from '../transform/compilationCache';import type { ConfigLocation, FullConfigInternal } from './config';import type { ReporterDescription, TestInfoError, TestStatus } from '../../types/test';import type { SerializedCompilationCache } from '../transform/compilationCache';+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';export type ConfigCLIOverrides = {debug?: boolean;@@ -66,6 +67,7 @@ export type WorkerInitParams = {projectId: string;config: SerializedConfig;artifactsDir: string;+ recoverFromStepErrors: boolean;};export type TestBeginPayload = {@@ -105,6 +107,14 @@ export type StepBeginPayload = {location?: { file: string, line: number, column: number };};+export type StepRecoverFromErrorPayload = {+ testId: string;+ stepId: string;+ error: TestInfoErrorImpl;+};++export type ResumeAfterStepErrorPayload = RecoverFromStepErrorResult;+export type StepEndPayload = {testId: string;stepId: string;diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex 53f8bd0a36..26a5bf8f3a 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -286,6 +286,11 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({if (data.apiName === 'tracing.group')tracingGroupSteps.push(step);},+ onApiCallRecovery: (data, error, recoveryHandlers) => {+ const step = data.userData as TestStepInternal;+ if (step)+ recoveryHandlers.push(() => step.recoverFromStepError(error));+ },onApiCallEnd: data => {// "tracing.group" step will end later, when "tracing.groupEnd" finishes.if (data.apiName === 'tracing.group')diff --git a/packages/playwright/src/isomorphic/testServerConnection.ts b/packages/playwright/src/isomorphic/testServerConnection.tsindex 57a416a604..76c383bd36 100644--- a/packages/playwright/src/isomorphic/testServerConnection.ts+++ b/packages/playwright/src/isomorphic/testServerConnection.ts@@ -17,6 +17,7 @@import * as events from './events';import type { TestServerInterface, TestServerInterfaceEvents } from '@testIsomorphic/testServerInterface';+import type * as reporterTypes from '../../types/testReporter';// -- Reuse boundary -- Everything below this line is reused in the vscode extension.@@ -68,12 +69,14 @@ export class TestServerConnection implements TestServerInterface, TestServerIntereadonly onStdio: events.Event<{ type: 'stderr' | 'stdout'; text?: string | undefined; buffer?: string | undefined; }>;readonly onTestFilesChanged: events.Event<{ testFiles: string[] }>;readonly onLoadTraceRequested: events.Event<{ traceUrl: string }>;+ readonly onRecoverFromStepError: events.Event<{ stepId: string, message: string, location: reporterTypes.Location }>;private _onCloseEmitter = new events.EventEmitter<void>();private _onReportEmitter = new events.EventEmitter<any>();private _onStdioEmitter = new events.EventEmitter<{ type: 'stderr' | 'stdout'; text?: string | undefined; buffer?: string | undefined; }>();private _onTestFilesChangedEmitter = new events.EventEmitter<{ testFiles: string[] }>();private _onLoadTraceRequestedEmitter = new events.EventEmitter<{ traceUrl: string }>();+ private _onRecoverFromStepErrorEmitter = new events.EventEmitter<{ stepId: string, message: string, location: reporterTypes.Location }>();private _lastId = 0;private _transport: TestServerTransport;@@ -87,6 +90,7 @@ export class TestServerConnection implements TestServerInterface, TestServerIntethis.onStdio = this._onStdioEmitter.event;this.onTestFilesChanged = this._onTestFilesChangedEmitter.event;this.onLoadTraceRequested = this._onLoadTraceRequestedEmitter.event;+ this.onRecoverFromStepError = this._onRecoverFromStepErrorEmitter.event;this._transport = transport;this._transport.onmessage(data => {@@ -147,6 +151,8 @@ export class TestServerConnection implements TestServerInterface, TestServerIntethis._onTestFilesChangedEmitter.fire(params);else if (method === 'loadTraceRequested')this._onLoadTraceRequestedEmitter.fire(params);+ else if (method === 'recoverFromStepError')+ this._onRecoverFromStepErrorEmitter.fire(params);}async initialize(params: Parameters<TestServerInterface['initialize']>[0]): ReturnType<TestServerInterface['initialize']> {@@ -241,6 +247,10 @@ export class TestServerConnection implements TestServerInterface, TestServerInteawait this._sendMessage('closeGracefully', params);}+ async resumeAfterStepError(params: Parameters<TestServerInterface['resumeAfterStepError']>[0]): Promise<void> {+ await this._sendMessage('resumeAfterStepError', params);+ }+close() {try {this._transport.close();diff --git a/packages/playwright/src/isomorphic/testServerInterface.ts b/packages/playwright/src/isomorphic/testServerInterface.tsindex 66136bab2e..461b9230b5 100644--- a/packages/playwright/src/isomorphic/testServerInterface.ts+++ b/packages/playwright/src/isomorphic/testServerInterface.ts@@ -22,6 +22,12 @@ import type * as reporterTypes from '../../types/testReporter';export type ReportEntry = JsonEvent;+export type RecoverFromStepErrorResult = {+ stepId: string;+ status: 'recovered' | 'failed';+ value?: string | number | boolean | undefined;+};+export interface TestServerInterface {initialize(params: {serializer?: string,@@ -29,6 +35,7 @@ export interface TestServerInterface {interceptStdio?: boolean,watchTestDirs?: boolean,populateDependenciesOnList?: boolean,+ recoverFromStepErrors?: boolean,}): Promise<void>;ping(params: {}): Promise<void>;@@ -113,6 +120,8 @@ export interface TestServerInterface {stopTests(params: {}): Promise<void>;closeGracefully(params: {}): Promise<void>;++ resumeAfterStepError(params: RecoverFromStepErrorResult): Promise<void>;}export interface TestServerInterfaceEvents {@@ -127,4 +136,5 @@ export interface TestServerInterfaceEventEmitters {dispatchEvent(event: 'stdio', params: { type: 'stdout' | 'stderr', text?: string, buffer?: string }): void;dispatchEvent(event: 'testFilesChanged', params: { testFiles: string[] }): void;dispatchEvent(event: 'loadTraceRequested', params: { traceUrl: string }): void;+ dispatchEvent(event: 'recoverFromStepError', params: { stepId: string, message: string, location: reporterTypes.Location }): void;}diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.tsindex 507ca80bf8..b6e3c0dc2e 100644--- a/packages/playwright/src/matchers/expect.ts+++ b/packages/playwright/src/matchers/expect.ts@@ -365,20 +365,37 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {const step = testInfo._addStep(stepInfo);- const reportStepError = (e: Error | unknown) => {+ const reportStepError = (isAsync: boolean, e: Error | unknown) => {const jestError = isJestError(e) ? e : null;- const error = jestError ? new ExpectError(jestError, customMessage, stackFrames) : e;+ const expectError = jestError ? new ExpectError(jestError, customMessage, stackFrames) : undefined;if (jestError?.matcherResult.suggestedRebaseline) {// NOTE: this is a workaround for the fact that we can't pass the suggested rebaseline// for passing matchers. See toMatchAriaSnapshot for a counterpart.step.complete({ suggestedRebaseline: jestError?.matcherResult.suggestedRebaseline });return;}++ const error = expectError ?? e;step.complete({ error });- if (this._info.isSoft)- testInfo._failWithError(error);- else- throw error;++ if (!isAsync || !expectError) {+ if (this._info.isSoft)+ testInfo._failWithError(error);+ else+ throw error;+ return;+ }++ // Recoverable async failure.+ return (async () => {+ const recoveryResult = await step.recoverFromStepError(expectError);+ if (recoveryResult.status === 'recovered')+ return recoveryResult.value as any;+ if (this._info.isSoft)+ testInfo._failWithError(expectError);+ else+ throw expectError;+ })();};const finalizer = () => {@@ -390,11 +407,11 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {const callback = () => matcher.call(target, ...args);const result = currentZone().with('stepZone', step).run(callback);if (result instanceof Promise)- return result.then(finalizer).catch(reportStepError);+ return result.then(finalizer).catch(reportStepError.bind(null, true));finalizer();return result;} catch (e) {- reportStepError(e);+ void reportStepError(false, e);}};}diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex 22293b2a9d..5e0525e541 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -26,11 +26,12 @@ import type { ProcessExitData } from './processHost';import type { TestGroup } from './testGroups';import type { TestError, TestResult, TestStep } from '../../types/testReporter';import type { FullConfigInternal } from '../common/config';-import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload } from '../common/ipc';+import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, StepRecoverFromErrorPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload } from '../common/ipc';import type { Suite } from '../common/test';import type { TestCase } from '../common/test';import type { ReporterV2 } from '../reporters/reporterV2';import type { RegisteredListener } from 'playwright-core/lib/utils';+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';export type EnvByProjectId = Map<string, Record<string, string | undefined>>;@@ -218,7 +219,8 @@ export class Dispatcher {_createWorker(testGroup: TestGroup, parallelIndex: number, loaderData: SerializedConfig) {const projectConfig = this._config.projects.find(p => p.id === testGroup.projectId)!;const outputDir = projectConfig.project.outputDir;- const worker = new WorkerHost(testGroup, parallelIndex, loaderData, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir);+ const recoverFromStepErrors = this._failureTracker.canRecoverFromStepError();+ const worker = new WorkerHost(testGroup, parallelIndex, loaderData, recoverFromStepErrors, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir);const handleOutput = (params: TestOutputPayload) => {const chunk = chunkFromParams(params);if (worker.didFail()) {@@ -396,6 +398,26 @@ class JobDispatcher {this._reporter.onStepEnd?.(test, result, step);}+ private _onStepRecoverFromError(resumeAfterStepError: (result: RecoverFromStepErrorResult) => void, params: StepRecoverFromErrorPayload) {+ const data = this._dataByTestId.get(params.testId);+ if (!data) {+ resumeAfterStepError({ stepId: params.stepId, status: 'failed' });+ return;+ }+ const { steps } = data;+ const step = steps.get(params.stepId);+ if (!step) {+ resumeAfterStepError({ stepId: params.stepId, status: 'failed' });+ return;+ }++ const testError: TestError = {+ ...params.error,+ location: step.location,+ };+ this._failureTracker.recoverFromStepError(params.stepId, testError, resumeAfterStepError);+ }+private _onAttach(params: AttachmentPayload) {const data = this._dataByTestId.get(params.testId)!;if (!data) {@@ -560,12 +582,14 @@ class JobDispatcher {}),};worker.runTestGroup(runPayload);+ const resumeAfterStepError = worker.resumeAfterStepError.bind(worker);this._listeners = [eventsHelper.addEventListener(worker, 'testBegin', this._onTestBegin.bind(this)),eventsHelper.addEventListener(worker, 'testEnd', this._onTestEnd.bind(this)),eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)),eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)),+ eventsHelper.addEventListener(worker, 'stepRecoverFromError', this._onStepRecoverFromError.bind(this, resumeAfterStepError)),eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)),eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)),eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)),diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.tsindex dd8ce50587..42481e06ca 100644--- a/packages/playwright/src/runner/failureTracker.ts+++ b/packages/playwright/src/runner/failureTracker.ts@@ -14,18 +14,30 @@* limitations under the License.*/-import type { TestResult } from '../../types/testReporter';+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';+import type { TestResult, TestError } from '../../types/testReporter';import type { FullConfigInternal } from '../common/config';import type { Suite, TestCase } from '../common/test';+export type RecoverFromStepErrorHandler = (stepId: string, error: TestError) => Promise<RecoverFromStepErrorResult>;+export class FailureTracker {private _failureCount = 0;private _hasWorkerErrors = false;private _rootSuite: Suite | undefined;+ private _recoverFromStepErrorHandler: RecoverFromStepErrorHandler | undefined;constructor(private _config: FullConfigInternal) {}+ canRecoverFromStepError(): boolean {+ return !!this._recoverFromStepErrorHandler;+ }++ setRecoverFromStepErrorHandler(recoverFromStepErrorHandler: RecoverFromStepErrorHandler) {+ this._recoverFromStepErrorHandler = recoverFromStepErrorHandler;+ }+onRootSuite(rootSuite: Suite) {this._rootSuite = rootSuite;}@@ -36,6 +48,14 @@ export class FailureTracker {++this._failureCount;}+ recoverFromStepError(stepId: string, error: TestError, resumeAfterStepError: (result: RecoverFromStepErrorResult) => void) {+ if (!this._recoverFromStepErrorHandler) {+ resumeAfterStepError({ stepId, status: 'failed' });+ return;+ }+ void this._recoverFromStepErrorHandler(stepId, error).then(resumeAfterStepError).catch(() => {});+ }+onWorkerError() {this._hasWorkerErrors = true;}diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.tsindex 7a6129d5e1..1f8564f4f0 100644--- a/packages/playwright/src/runner/testServer.ts+++ b/packages/playwright/src/runner/testServer.ts@@ -44,6 +44,7 @@ import type { TestRunnerPluginRegistration } from '../plugins';import type { ReporterV2 } from '../reporters/reporterV2';import type { TraceViewerRedirectOptions, TraceViewerServerOptions } from 'playwright-core/lib/server/trace/viewer/traceViewer';import type { HttpServer, Transport } from 'playwright-core/lib/utils';+import type { RecoverFromStepErrorResult } from '../isomorphic/testServerInterface';const originalDebugLog = debug.log;const originalStdoutWrite = process.stdout.write;@@ -90,6 +91,8 @@ export class TestServerDispatcher implements TestServerInterface {private _watchTestDirs = false;private _closeOnDisconnect = false;private _populateDependenciesOnList = false;+ private _recoverFromStepErrors = false;+ private _resumeAfterStepErrors: Map<string, ManualPromise<RecoverFromStepErrorResult>> = new Map();constructor(configLocation: ConfigLocation, configCLIOverrides: ConfigCLIOverrides) {this._configLocation = configLocation;@@ -127,6 +130,7 @@ export class TestServerDispatcher implements TestServerInterface {await this._setInterceptStdio(!!params.interceptStdio);this._watchTestDirs = !!params.watchTestDirs;this._populateDependenciesOnList = !!params.populateDependenciesOnList;+ this._recoverFromStepErrors = !!params.recoverFromStepErrors;}async ping() {}@@ -351,7 +355,9 @@ export class TestServerDispatcher implements TestServerInterface {createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }),...createRunTestsTasks(config),];- const run = runTasks(new TestRun(config, reporter), tasks, 0, stop).then(async status => {+ const testRun = new TestRun(config, reporter);+ testRun.failureTracker.setRecoverFromStepErrorHandler(this._recoverFromStepError.bind(this));+ const run = runTasks(testRun, tasks, 0, stop).then(async status => {this._testRun = undefined;return status;});@@ -359,6 +365,26 @@ export class TestServerDispatcher implements TestServerInterface {return { status: await run };}+ private async _recoverFromStepError(stepId: string, error: reporterTypes.TestError): Promise<RecoverFromStepErrorResult> {+ if (!this._recoverFromStepErrors)+ return { stepId, status: 'failed' };+ const recoveryPromise = new ManualPromise<RecoverFromStepErrorResult>();+ this._resumeAfterStepErrors.set(stepId, recoveryPromise);+ if (!error?.message || !error?.location)+ return { stepId, status: 'failed' };+ this._dispatchEvent('recoverFromStepError', { stepId, message: error.message, location: error.location });+ const recoveredResult = await recoveryPromise;+ if (recoveredResult.stepId !== stepId)+ return { stepId, status: 'failed' };+ return recoveredResult;+ }++ async resumeAfterStepError(params: RecoverFromStepErrorResult): Promise<void> {+ const recoveryPromise = this._resumeAfterStepErrors.get(params.stepId);+ if (recoveryPromise)+ recoveryPromise.resolve(params);+ }+async watch(params: { fileNames: string[]; }) {this._watchedTestDependencies = new Set();for (const fileName of params.fileNames) {@@ -385,6 +411,7 @@ export class TestServerDispatcher implements TestServerInterface {async stopTests() {this._testRun?.stop?.resolve();await this._testRun?.run;+ this._resumeAfterStepErrors.clear();}async _setInterceptStdio(intercept: boolean) {diff --git a/packages/playwright/src/runner/watchMode.ts b/packages/playwright/src/runner/watchMode.tsindex d14d08d8b4..488a6cf888 100644--- a/packages/playwright/src/runner/watchMode.ts+++ b/packages/playwright/src/runner/watchMode.ts@@ -14,6 +14,7 @@* limitations under the License.*/+import fs from 'fs';import path from 'path';import readline from 'readline';import { EventEmitter } from 'stream';@@ -27,8 +28,10 @@ import { enquirer } from '../utilsBundle';import { TestServerDispatcher } from './testServer';import { TeleSuiteUpdater } from '../isomorphic/teleSuiteUpdater';import { TestServerConnection } from '../isomorphic/testServerConnection';+import { stripAnsiEscapes } from '../util';+import { codeFrameColumns } from '../transform/babelBundle';-import type { FullResult } from '../../types/testReporter';+import type * as reporterTypes from '../../types/testReporter';import type { ConfigLocation } from '../common/config';import type { TestServerTransport } from '../isomorphic/testServerConnection';@@ -72,7 +75,7 @@ interface WatchModeOptions {grep?: string;}-export async function runWatchModeLoop(configLocation: ConfigLocation, initialOptions: WatchModeOptions): Promise<FullResult['status']> {+export async function runWatchModeLoop(configLocation: ConfigLocation, initialOptions: WatchModeOptions): Promise<reporterTypes.FullResult['status']> {const options: WatchModeOptions = { ...initialOptions };let bufferMode = false;@@ -127,8 +130,28 @@ export async function runWatchModeLoop(configLocation: ConfigLocation, initialOp});});testServerConnection.onReport(report => teleSuiteUpdater.processTestReportEvent(report));+ testServerConnection.onRecoverFromStepError(({ stepId, message, location }) => {+ process.stdout.write(`\nTest error occurred.\n`);+ process.stdout.write('\n' + createErrorCodeframe(message, location) + '\n');+ process.stdout.write(`\n${colors.dim('Try recovering from the error. Press')} ${colors.bold('c')} ${colors.dim('to continue or')} ${colors.bold('t')} ${colors.dim('to throw the error')}\n`);+ readKeyPress(text => {+ if (text === 'c') {+ process.stdout.write(`\n${colors.dim('Continuing after recovery...')}\n`);+ testServerConnection.resumeAfterStepError({ stepId, status: 'recovered', value: undefined }).catch(() => {});+ } else if (text === 't') {+ process.stdout.write(`\n${colors.dim('Throwing error...')}\n`);+ testServerConnection.resumeAfterStepError({ stepId, status: 'failed' }).catch(() => {});+ }+ return text;+ });+ });- await testServerConnection.initialize({ interceptStdio: false, watchTestDirs: true, populateDependenciesOnList: true });+ await testServerConnection.initialize({+ interceptStdio: false,+ watchTestDirs: true,+ populateDependenciesOnList: true,+ recoverFromStepErrors: !process.env.PWTEST_RECOVERY_DISABLED,+ });await testServerConnection.runGlobalSetup({});const { report } = await testServerConnection.listTests({});@@ -137,7 +160,7 @@ export async function runWatchModeLoop(configLocation: ConfigLocation, initialOpconst projectNames = teleSuiteUpdater.rootSuite!.suites.map(s => s.title);let lastRun: { type: 'changed' | 'regular' | 'failed', failedTestIds?: string[], dirtyTestIds?: string[] } = { type: 'regular' };- let result: FullResult['status'] = 'passed';+ let result: reporterTypes.FullResult['status'] = 'passed';while (true) {if (bufferMode)@@ -426,3 +449,28 @@ async function toggleShowBrowser() {}type Command = 'run' | 'failed' | 'repeat' | 'changed' | 'project' | 'file' | 'grep' | 'exit' | 'interrupted' | 'toggle-show-browser' | 'toggle-buffer-mode';++function createErrorCodeframe(message: string, location: reporterTypes.Location) {+ let source: string;+ try {+ source = fs.readFileSync(location.file, 'utf-8') + '\n//';+ } catch (e) {+ return;+ }++ return codeFrameColumns(+ source,+ {+ start: {+ line: location.line,+ column: location.column,+ },+ },+ {+ highlightCode: true,+ linesAbove: 5,+ linesBelow: 5,+ message: stripAnsiEscapes(message).split('\n')[0] || undefined,+ }+ );+}diff --git a/packages/playwright/src/runner/workerHost.ts b/packages/playwright/src/runner/workerHost.tsindex 0ab8c54ba1..3879971ef0 100644--- a/packages/playwright/src/runner/workerHost.ts+++ b/packages/playwright/src/runner/workerHost.ts@@ -25,6 +25,7 @@ import { artifactsFolderName } from '../isomorphic/folders';import type { TestGroup } from './testGroups';import type { RunPayload, SerializedConfig, WorkerInitParams } from '../common/ipc';+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';let lastWorkerIndex = 0;@@ -36,7 +37,7 @@ export class WorkerHost extends ProcessHost {private _params: WorkerInitParams;private _didFail = false;- constructor(testGroup: TestGroup, parallelIndex: number, config: SerializedConfig, extraEnv: Record<string, string | undefined>, outputDir: string) {+ constructor(testGroup: TestGroup, parallelIndex: number, config: SerializedConfig, recoverFromStepErrors: boolean, extraEnv: Record<string, string | undefined>, outputDir: string) {const workerIndex = lastWorkerIndex++;super(require.resolve('../worker/workerMain.js'), `worker-${workerIndex}`, {...extraEnv,@@ -53,7 +54,8 @@ export class WorkerHost extends ProcessHost {repeatEachIndex: testGroup.repeatEachIndex,projectId: testGroup.projectId,config,- artifactsDir: path.join(outputDir, artifactsFolderName(workerIndex))+ artifactsDir: path.join(outputDir, artifactsFolderName(workerIndex)),+ recoverFromStepErrors,};}@@ -79,6 +81,10 @@ export class WorkerHost extends ProcessHost {this.sendMessageNoReply({ method: 'runTestGroup', params: runPayload });}+ resumeAfterStepError(result: RecoverFromStepErrorResult) {+ this.sendMessageNoReply({ method: 'resumeAfterStepError', params: result });+ }+hash() {return this._hash;}diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.tsindex eb590b0abd..344c50a64b 100644--- a/packages/playwright/src/worker/testInfo.ts+++ b/packages/playwright/src/worker/testInfo.ts@@ -17,10 +17,10 @@import fs from 'fs';import path from 'path';-import { captureRawStack, monotonicTime, sanitizeForFilePath, stringifyStackFrames, currentZone, createGuid, escapeWithQuotes } from 'playwright-core/lib/utils';+import { captureRawStack, monotonicTime, sanitizeForFilePath, stringifyStackFrames, currentZone, createGuid, escapeWithQuotes, ManualPromise } from 'playwright-core/lib/utils';import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutManager';-import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, sanitizeFilePathBeforeExtension, trimLongString, windowsFilesystemFriendlyLength } from '../util';+import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, sanitizeFilePathBeforeExtension, serializeError, trimLongString, windowsFilesystemFriendlyLength } from '../util';import { TestTracing } from './testTracing';import { testInfoError } from './util';import { wrapFunctionWithLocation } from '../transform/transform';@@ -29,9 +29,10 @@ import type { RunnableDescription } from './timeoutManager';import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test';import type { FullConfig, Location } from '../../types/testReporter';import type { FullConfigInternal, FullProjectInternal } from '../common/config';-import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc';+import type { AttachmentPayload, ResumeAfterStepErrorPayload, StepBeginPayload, StepEndPayload, StepRecoverFromErrorPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc';import type { TestCase } from '../common/test';import type { StackFrame } from '@protocol/channels';+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';export type TestStepVisibility = 'internal' | 'hidden';export type TestStepCategory = 'expect' | 'fixture' | 'hook' | 'pw:api' | 'test.step' | 'test.attach';@@ -48,6 +49,7 @@ interface TestStepData {}export interface TestStepInternal extends TestStepData {+ recoverFromStepError(error: Error): Promise<RecoverFromStepErrorResult>;complete(result: { error?: Error | unknown, suggestedRebaseline?: string }): void;info: TestStepInfoImpl;attachmentIndices: number[];@@ -65,6 +67,7 @@ type SnapshotNames = {export class TestInfoImpl implements TestInfo {private _onStepBegin: (payload: StepBeginPayload) => void;+ private _onStepRecoverFromError: (payload: StepRecoverFromErrorPayload) => void;private _onStepEnd: (payload: StepEndPayload) => void;private _onAttach: (payload: AttachmentPayload) => void;private _snapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} };@@ -118,6 +121,7 @@ export class TestInfoImpl implements TestInfo {readonly snapshotDir: string;errors: TestInfoErrorImpl[] = [];readonly _attachmentsPush: (...items: TestInfo['attachments']) => number;+ private _recoverFromStepErrorResults: Map<string, ManualPromise<ResumeAfterStepErrorPayload>> | undefined;get error(): TestInfoErrorImpl | undefined {return this.errors[0];@@ -157,11 +161,13 @@ export class TestInfoImpl implements TestInfo {test: TestCase | undefined,retry: number,onStepBegin: (payload: StepBeginPayload) => void,+ onStepRecoverFromError: (payload: StepRecoverFromErrorPayload) => void,onStepEnd: (payload: StepEndPayload) => void,onAttach: (payload: AttachmentPayload) => void,) {this.testId = test?.id ?? '';this._onStepBegin = onStepBegin;+ this._onStepRecoverFromError = onStepRecoverFromError;this._onStepEnd = onStepEnd;this._onAttach = onAttach;this._startTime = monotonicTime();@@ -185,6 +191,7 @@ export class TestInfoImpl implements TestInfo {this.tags = test?.tags ?? [];this.fn = test?.fn ?? (() => {});this.expectedStatus = test?.expectedStatus ?? 'skipped';+ this._recoverFromStepErrorResults = workerParams.recoverFromStepErrors ? new Map() : undefined;this._timeoutManager = new TimeoutManager(this.project.timeout);if (configInternal.configCLIOverrides.debug)@@ -298,6 +305,22 @@ export class TestInfoImpl implements TestInfo {steps: [],attachmentIndices: [],info: new TestStepInfoImpl(this, stepId, data.title, parentStep?.info),+ recoverFromStepError: async (error: Error) => {+ if (!this._recoverFromStepErrorResults)+ return { stepId, status: 'failed' };+ const payload: StepRecoverFromErrorPayload = {+ testId: this.testId,+ stepId,+ error: serializeError(error),+ };+ this._onStepRecoverFromError(payload);+ const recoveryPromise = new ManualPromise<RecoverFromStepErrorResult>();+ this._recoverFromStepErrorResults.set(stepId, recoveryPromise);+ const recoveryResult = await recoveryPromise;+ if (recoveryResult.stepId !== stepId)+ return { stepId, status: 'failed' };+ return recoveryResult;+ },complete: result => {if (step.endWallTime)return;@@ -374,6 +397,12 @@ export class TestInfoImpl implements TestInfo {return step;}+ resumeAfterStepError(result: ResumeAfterStepErrorPayload) {+ const recoveryPromise = this._recoverFromStepErrorResults?.get(result.stepId);+ if (recoveryPromise)+ recoveryPromise.resolve(result);+ }+_interrupt() {// Mark as interrupted so we can ignore TimeoutError thrown by interrupt() call.this._wasInterrupted = true;diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.tsindex e9f515a760..1fefb76a6d 100644--- a/packages/playwright/src/worker/workerMain.ts+++ b/packages/playwright/src/worker/workerMain.ts@@ -33,7 +33,7 @@ import { loadTestFile } from '../common/testLoader';import type { TimeSlot } from './timeoutManager';import type { Location } from '../../types/testReporter';import type { FullConfigInternal, FullProjectInternal } from '../common/config';-import type { DonePayload, RunPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc';+import type { DonePayload, ResumeAfterStepErrorPayload, RunPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc';import type { Suite, TestCase } from '../common/test';import type { TestAnnotation } from '../../types/test';@@ -113,7 +113,7 @@ export class WorkerMain extends ProcessRunner {return;}// Ignore top-level errors, they are already inside TestInfo.errors.- const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {});+ const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {}, () => {});const runnable = { type: 'teardown' } as const;// We have to load the project to get the right deadline below.await fakeTestInfo._runWithTimeout(runnable, () => this._loadIfNeeded()).catch(() => {});@@ -259,9 +259,14 @@ export class WorkerMain extends ProcessRunner {}}+ resumeAfterStepError(params: ResumeAfterStepErrorPayload): void {+ this._currentTest?.resumeAfterStepError(params);+ }+private async _runTest(test: TestCase, retry: number, nextTest: TestCase | undefined) {const testInfo = new TestInfoImpl(this._config, this._project, this._params, test, retry,stepBeginPayload => this.dispatchEvent('stepBegin', stepBeginPayload),+ stepRecoverFromErrorPayload => this.dispatchEvent('stepRecoverFromError', stepRecoverFromErrorPayload),stepEndPayload => this.dispatchEvent('stepEnd', stepEndPayload),attachment => this.dispatchEvent('attach', attachment));diff --git a/tests/playwright-test/watch.spec.ts b/tests/playwright-test/watch.spec.tsindex 431d5e2cc8..750c3f854d 100644--- a/tests/playwright-test/watch.spec.ts+++ b/tests/playwright-test/watch.spec.ts@@ -710,7 +710,7 @@ test('should run CT on changed deps', async ({ runWatchTest, writeFiles }) => {await expect(component).toHaveText('hello');});`,- });+ }, undefined, { PWTEST_RECOVERY_DISABLED: '1' });await testProcess.waitForOutput('Waiting for file changes.');await writeFiles({'src/button.tsx': `