Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: step error recovery framework (#36996)

Pavel Feldmand669fcde0ead
diff --git a/packages/playwright-core/src/client/channelOwner.ts b/packages/playwright-core/src/client/channelOwner.ts
index 7bd097b3ff..31adda7d17 100644
--- a/packages/playwright-core/src/client/channelOwner.ts
+++ b/packages/playwright-core/src/client/channelOwner.ts
@@ -20,7 +20,7 @@ import { methodMetainfo } from '../utils/isomorphic/protocolMetainfo';
import { captureLibraryStackTrace } from './clientStackTrace';
import { stringifyStackFrames } from '../utils/isomorphic/stackTrace';
-import type { ClientInstrumentation } from './clientInstrumentation';
+import type { ClientInstrumentation, RecoverFromApiErrorHandler } from './clientInstrumentation';
import type { Connection } from './connection';
import type { Logger } from './types';
import type { ValidatorContext } from '../protocol/validator';
@@ -199,7 +199,14 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
else
e.stack = '';
if (!options?.internal) {
+ const recoveryHandlers: RecoverFromApiErrorHandler[] = [];
apiZone.error = e;
+ this._instrumentation.onApiCallRecovery(apiZone, e, recoveryHandlers);
+ for (const handler of recoveryHandlers) {
+ const recoverResult = await handler();
+ if (recoverResult.status === 'recovered')
+ return recoverResult.value as R;
+ }
logApiCall(this._platform, logger, `<= ${apiZone.apiName} failed`);
this._instrumentation.onApiCallEnd(apiZone);
}
diff --git a/packages/playwright-core/src/client/clientInstrumentation.ts b/packages/playwright-core/src/client/clientInstrumentation.ts
index b14a3de064..968e3d9810 100644
--- a/packages/playwright-core/src/client/clientInstrumentation.ts
+++ b/packages/playwright-core/src/client/clientInstrumentation.ts
@@ -28,12 +28,20 @@ export interface ApiCallData {
error?: Error;
}
+export type RecoverFromApiErrorResult = {
+ status: 'recovered' | 'failed';
+ value?: string | number | boolean | undefined;
+};
+
+export type RecoverFromApiErrorHandler = () => Promise<RecoverFromApiErrorResult>;
+
export interface ClientInstrumentation {
addListener(listener: ClientInstrumentationListener): void;
removeListener(listener: ClientInstrumentationListener): void;
removeAllListeners(): void;
onApiCallBegin(apiCall: ApiCallData, channel: { type: string, method: string, params?: Record<string, any> }): void;
- onApiCallEnd(apiCal: ApiCallData): void;
+ onApiCallRecovery(apiCall: ApiCallData, error: Error, recoveryHandlers: RecoverFromApiErrorHandler[]): void;
+ onApiCallEnd(apiCall: ApiCallData): void;
onWillPause(options: { keepTestTimeout: boolean }): void;
runAfterCreateBrowserContext(context: BrowserContext): Promise<void>;
@@ -44,6 +52,7 @@ export interface ClientInstrumentation {
export interface ClientInstrumentationListener {
onApiCallBegin?(apiCall: ApiCallData, channel: { type: string, method: string, params?: Record<string, any> }): void;
+ onApiCallRecovery?(apiCall: ApiCallData, error: Error, recoveryHandlers: RecoverFromApiErrorHandler[]): void;
onApiCallEnd?(apiCall: ApiCallData): void;
onWillPause?(options: { keepTestTimeout: boolean }): void;
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index 806aed0532..e4645d1c69 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -21,6 +21,7 @@ import { serializeCompilationCache } from '../transform/compilationCache';
import type { ConfigLocation, FullConfigInternal } from './config';
import type { ReporterDescription, TestInfoError, TestStatus } from '../../types/test';
import type { SerializedCompilationCache } from '../transform/compilationCache';
+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';
export type ConfigCLIOverrides = {
debug?: boolean;
@@ -66,6 +67,7 @@ export type WorkerInitParams = {
projectId: string;
config: SerializedConfig;
artifactsDir: string;
+ recoverFromStepErrors: boolean;
};
export type TestBeginPayload = {
@@ -105,6 +107,14 @@ export type StepBeginPayload = {
location?: { file: string, line: number, column: number };
};
+export type StepRecoverFromErrorPayload = {
+ testId: string;
+ stepId: string;
+ error: TestInfoErrorImpl;
+};
+
+export type ResumeAfterStepErrorPayload = RecoverFromStepErrorResult;
+
export type StepEndPayload = {
testId: string;
stepId: string;
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index 53f8bd0a36..26a5bf8f3a 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -286,6 +286,11 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
if (data.apiName === 'tracing.group')
tracingGroupSteps.push(step);
},
+ onApiCallRecovery: (data, error, recoveryHandlers) => {
+ const step = data.userData as TestStepInternal;
+ if (step)
+ recoveryHandlers.push(() => step.recoverFromStepError(error));
+ },
onApiCallEnd: data => {
// "tracing.group" step will end later, when "tracing.groupEnd" finishes.
if (data.apiName === 'tracing.group')
diff --git a/packages/playwright/src/isomorphic/testServerConnection.ts b/packages/playwright/src/isomorphic/testServerConnection.ts
index 57a416a604..76c383bd36 100644
--- a/packages/playwright/src/isomorphic/testServerConnection.ts
+++ b/packages/playwright/src/isomorphic/testServerConnection.ts
@@ -17,6 +17,7 @@
import * as events from './events';
import type { TestServerInterface, TestServerInterfaceEvents } from '@testIsomorphic/testServerInterface';
+import type * as reporterTypes from '../../types/testReporter';
// -- Reuse boundary -- Everything below this line is reused in the vscode extension.
@@ -68,12 +69,14 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
readonly onStdio: events.Event<{ type: 'stderr' | 'stdout'; text?: string | undefined; buffer?: string | undefined; }>;
readonly onTestFilesChanged: events.Event<{ testFiles: string[] }>;
readonly onLoadTraceRequested: events.Event<{ traceUrl: string }>;
+ readonly onRecoverFromStepError: events.Event<{ stepId: string, message: string, location: reporterTypes.Location }>;
private _onCloseEmitter = new events.EventEmitter<void>();
private _onReportEmitter = new events.EventEmitter<any>();
private _onStdioEmitter = new events.EventEmitter<{ type: 'stderr' | 'stdout'; text?: string | undefined; buffer?: string | undefined; }>();
private _onTestFilesChangedEmitter = new events.EventEmitter<{ testFiles: string[] }>();
private _onLoadTraceRequestedEmitter = new events.EventEmitter<{ traceUrl: string }>();
+ private _onRecoverFromStepErrorEmitter = new events.EventEmitter<{ stepId: string, message: string, location: reporterTypes.Location }>();
private _lastId = 0;
private _transport: TestServerTransport;
@@ -87,6 +90,7 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
this.onStdio = this._onStdioEmitter.event;
this.onTestFilesChanged = this._onTestFilesChangedEmitter.event;
this.onLoadTraceRequested = this._onLoadTraceRequestedEmitter.event;
+ this.onRecoverFromStepError = this._onRecoverFromStepErrorEmitter.event;
this._transport = transport;
this._transport.onmessage(data => {
@@ -147,6 +151,8 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
this._onTestFilesChangedEmitter.fire(params);
else if (method === 'loadTraceRequested')
this._onLoadTraceRequestedEmitter.fire(params);
+ else if (method === 'recoverFromStepError')
+ this._onRecoverFromStepErrorEmitter.fire(params);
}
async initialize(params: Parameters<TestServerInterface['initialize']>[0]): ReturnType<TestServerInterface['initialize']> {
@@ -241,6 +247,10 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
await this._sendMessage('closeGracefully', params);
}
+ async resumeAfterStepError(params: Parameters<TestServerInterface['resumeAfterStepError']>[0]): Promise<void> {
+ await this._sendMessage('resumeAfterStepError', params);
+ }
+
close() {
try {
this._transport.close();
diff --git a/packages/playwright/src/isomorphic/testServerInterface.ts b/packages/playwright/src/isomorphic/testServerInterface.ts
index 66136bab2e..461b9230b5 100644
--- a/packages/playwright/src/isomorphic/testServerInterface.ts
+++ b/packages/playwright/src/isomorphic/testServerInterface.ts
@@ -22,6 +22,12 @@ import type * as reporterTypes from '../../types/testReporter';
export type ReportEntry = JsonEvent;
+export type RecoverFromStepErrorResult = {
+ stepId: string;
+ status: 'recovered' | 'failed';
+ value?: string | number | boolean | undefined;
+};
+
export interface TestServerInterface {
initialize(params: {
serializer?: string,
@@ -29,6 +35,7 @@ export interface TestServerInterface {
interceptStdio?: boolean,
watchTestDirs?: boolean,
populateDependenciesOnList?: boolean,
+ recoverFromStepErrors?: boolean,
}): Promise<void>;
ping(params: {}): Promise<void>;
@@ -113,6 +120,8 @@ export interface TestServerInterface {
stopTests(params: {}): Promise<void>;
closeGracefully(params: {}): Promise<void>;
+
+ resumeAfterStepError(params: RecoverFromStepErrorResult): Promise<void>;
}
export interface TestServerInterfaceEvents {
@@ -127,4 +136,5 @@ export interface TestServerInterfaceEventEmitters {
dispatchEvent(event: 'stdio', params: { type: 'stdout' | 'stderr', text?: string, buffer?: string }): void;
dispatchEvent(event: 'testFilesChanged', params: { testFiles: string[] }): void;
dispatchEvent(event: 'loadTraceRequested', params: { traceUrl: string }): void;
+ dispatchEvent(event: 'recoverFromStepError', params: { stepId: string, message: string, location: reporterTypes.Location }): void;
}
diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.ts
index 507ca80bf8..b6e3c0dc2e 100644
--- a/packages/playwright/src/matchers/expect.ts
+++ b/packages/playwright/src/matchers/expect.ts
@@ -365,20 +365,37 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {
const step = testInfo._addStep(stepInfo);
- const reportStepError = (e: Error | unknown) => {
+ const reportStepError = (isAsync: boolean, e: Error | unknown) => {
const jestError = isJestError(e) ? e : null;
- const error = jestError ? new ExpectError(jestError, customMessage, stackFrames) : e;
+ const expectError = jestError ? new ExpectError(jestError, customMessage, stackFrames) : undefined;
if (jestError?.matcherResult.suggestedRebaseline) {
// NOTE: this is a workaround for the fact that we can't pass the suggested rebaseline
// for passing matchers. See toMatchAriaSnapshot for a counterpart.
step.complete({ suggestedRebaseline: jestError?.matcherResult.suggestedRebaseline });
return;
}
+
+ const error = expectError ?? e;
step.complete({ error });
- if (this._info.isSoft)
- testInfo._failWithError(error);
- else
- throw error;
+
+ if (!isAsync || !expectError) {
+ if (this._info.isSoft)
+ testInfo._failWithError(error);
+ else
+ throw error;
+ return;
+ }
+
+ // Recoverable async failure.
+ return (async () => {
+ const recoveryResult = await step.recoverFromStepError(expectError);
+ if (recoveryResult.status === 'recovered')
+ return recoveryResult.value as any;
+ if (this._info.isSoft)
+ testInfo._failWithError(expectError);
+ else
+ throw expectError;
+ })();
};
const finalizer = () => {
@@ -390,11 +407,11 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {
const callback = () => matcher.call(target, ...args);
const result = currentZone().with('stepZone', step).run(callback);
if (result instanceof Promise)
- return result.then(finalizer).catch(reportStepError);
+ return result.then(finalizer).catch(reportStepError.bind(null, true));
finalizer();
return result;
} catch (e) {
- reportStepError(e);
+ void reportStepError(false, e);
}
};
}
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index 22293b2a9d..5e0525e541 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -26,11 +26,12 @@ import type { ProcessExitData } from './processHost';
import type { TestGroup } from './testGroups';
import type { TestError, TestResult, TestStep } from '../../types/testReporter';
import type { FullConfigInternal } from '../common/config';
-import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload } from '../common/ipc';
+import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, StepRecoverFromErrorPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload } from '../common/ipc';
import type { Suite } from '../common/test';
import type { TestCase } from '../common/test';
import type { ReporterV2 } from '../reporters/reporterV2';
import type { RegisteredListener } from 'playwright-core/lib/utils';
+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';
export type EnvByProjectId = Map<string, Record<string, string | undefined>>;
@@ -218,7 +219,8 @@ export class Dispatcher {
_createWorker(testGroup: TestGroup, parallelIndex: number, loaderData: SerializedConfig) {
const projectConfig = this._config.projects.find(p => p.id === testGroup.projectId)!;
const outputDir = projectConfig.project.outputDir;
- const worker = new WorkerHost(testGroup, parallelIndex, loaderData, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir);
+ const recoverFromStepErrors = this._failureTracker.canRecoverFromStepError();
+ const worker = new WorkerHost(testGroup, parallelIndex, loaderData, recoverFromStepErrors, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir);
const handleOutput = (params: TestOutputPayload) => {
const chunk = chunkFromParams(params);
if (worker.didFail()) {
@@ -396,6 +398,26 @@ class JobDispatcher {
this._reporter.onStepEnd?.(test, result, step);
}
+ private _onStepRecoverFromError(resumeAfterStepError: (result: RecoverFromStepErrorResult) => void, params: StepRecoverFromErrorPayload) {
+ const data = this._dataByTestId.get(params.testId);
+ if (!data) {
+ resumeAfterStepError({ stepId: params.stepId, status: 'failed' });
+ return;
+ }
+ const { steps } = data;
+ const step = steps.get(params.stepId);
+ if (!step) {
+ resumeAfterStepError({ stepId: params.stepId, status: 'failed' });
+ return;
+ }
+
+ const testError: TestError = {
+ ...params.error,
+ location: step.location,
+ };
+ this._failureTracker.recoverFromStepError(params.stepId, testError, resumeAfterStepError);
+ }
+
private _onAttach(params: AttachmentPayload) {
const data = this._dataByTestId.get(params.testId)!;
if (!data) {
@@ -560,12 +582,14 @@ class JobDispatcher {
}),
};
worker.runTestGroup(runPayload);
+ const resumeAfterStepError = worker.resumeAfterStepError.bind(worker);
this._listeners = [
eventsHelper.addEventListener(worker, 'testBegin', this._onTestBegin.bind(this)),
eventsHelper.addEventListener(worker, 'testEnd', this._onTestEnd.bind(this)),
eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)),
eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)),
+ eventsHelper.addEventListener(worker, 'stepRecoverFromError', this._onStepRecoverFromError.bind(this, resumeAfterStepError)),
eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)),
eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)),
eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)),
diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.ts
index dd8ce50587..42481e06ca 100644
--- a/packages/playwright/src/runner/failureTracker.ts
+++ b/packages/playwright/src/runner/failureTracker.ts
@@ -14,18 +14,30 @@
* limitations under the License.
*/
-import type { TestResult } from '../../types/testReporter';
+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';
+import type { TestResult, TestError } from '../../types/testReporter';
import type { FullConfigInternal } from '../common/config';
import type { Suite, TestCase } from '../common/test';
+export type RecoverFromStepErrorHandler = (stepId: string, error: TestError) => Promise<RecoverFromStepErrorResult>;
+
export class FailureTracker {
private _failureCount = 0;
private _hasWorkerErrors = false;
private _rootSuite: Suite | undefined;
+ private _recoverFromStepErrorHandler: RecoverFromStepErrorHandler | undefined;
constructor(private _config: FullConfigInternal) {
}
+ canRecoverFromStepError(): boolean {
+ return !!this._recoverFromStepErrorHandler;
+ }
+
+ setRecoverFromStepErrorHandler(recoverFromStepErrorHandler: RecoverFromStepErrorHandler) {
+ this._recoverFromStepErrorHandler = recoverFromStepErrorHandler;
+ }
+
onRootSuite(rootSuite: Suite) {
this._rootSuite = rootSuite;
}
@@ -36,6 +48,14 @@ export class FailureTracker {
++this._failureCount;
}
+ recoverFromStepError(stepId: string, error: TestError, resumeAfterStepError: (result: RecoverFromStepErrorResult) => void) {
+ if (!this._recoverFromStepErrorHandler) {
+ resumeAfterStepError({ stepId, status: 'failed' });
+ return;
+ }
+ void this._recoverFromStepErrorHandler(stepId, error).then(resumeAfterStepError).catch(() => {});
+ }
+
onWorkerError() {
this._hasWorkerErrors = true;
}
diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.ts
index 7a6129d5e1..1f8564f4f0 100644
--- a/packages/playwright/src/runner/testServer.ts
+++ b/packages/playwright/src/runner/testServer.ts
@@ -44,6 +44,7 @@ import type { TestRunnerPluginRegistration } from '../plugins';
import type { ReporterV2 } from '../reporters/reporterV2';
import type { TraceViewerRedirectOptions, TraceViewerServerOptions } from 'playwright-core/lib/server/trace/viewer/traceViewer';
import type { HttpServer, Transport } from 'playwright-core/lib/utils';
+import type { RecoverFromStepErrorResult } from '../isomorphic/testServerInterface';
const originalDebugLog = debug.log;
const originalStdoutWrite = process.stdout.write;
@@ -90,6 +91,8 @@ export class TestServerDispatcher implements TestServerInterface {
private _watchTestDirs = false;
private _closeOnDisconnect = false;
private _populateDependenciesOnList = false;
+ private _recoverFromStepErrors = false;
+ private _resumeAfterStepErrors: Map<string, ManualPromise<RecoverFromStepErrorResult>> = new Map();
constructor(configLocation: ConfigLocation, configCLIOverrides: ConfigCLIOverrides) {
this._configLocation = configLocation;
@@ -127,6 +130,7 @@ export class TestServerDispatcher implements TestServerInterface {
await this._setInterceptStdio(!!params.interceptStdio);
this._watchTestDirs = !!params.watchTestDirs;
this._populateDependenciesOnList = !!params.populateDependenciesOnList;
+ this._recoverFromStepErrors = !!params.recoverFromStepErrors;
}
async ping() {}
@@ -351,7 +355,9 @@ export class TestServerDispatcher implements TestServerInterface {
createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }),
...createRunTestsTasks(config),
];
- const run = runTasks(new TestRun(config, reporter), tasks, 0, stop).then(async status => {
+ const testRun = new TestRun(config, reporter);
+ testRun.failureTracker.setRecoverFromStepErrorHandler(this._recoverFromStepError.bind(this));
+ const run = runTasks(testRun, tasks, 0, stop).then(async status => {
this._testRun = undefined;
return status;
});
@@ -359,6 +365,26 @@ export class TestServerDispatcher implements TestServerInterface {
return { status: await run };
}
+ private async _recoverFromStepError(stepId: string, error: reporterTypes.TestError): Promise<RecoverFromStepErrorResult> {
+ if (!this._recoverFromStepErrors)
+ return { stepId, status: 'failed' };
+ const recoveryPromise = new ManualPromise<RecoverFromStepErrorResult>();
+ this._resumeAfterStepErrors.set(stepId, recoveryPromise);
+ if (!error?.message || !error?.location)
+ return { stepId, status: 'failed' };
+ this._dispatchEvent('recoverFromStepError', { stepId, message: error.message, location: error.location });
+ const recoveredResult = await recoveryPromise;
+ if (recoveredResult.stepId !== stepId)
+ return { stepId, status: 'failed' };
+ return recoveredResult;
+ }
+
+ async resumeAfterStepError(params: RecoverFromStepErrorResult): Promise<void> {
+ const recoveryPromise = this._resumeAfterStepErrors.get(params.stepId);
+ if (recoveryPromise)
+ recoveryPromise.resolve(params);
+ }
+
async watch(params: { fileNames: string[]; }) {
this._watchedTestDependencies = new Set();
for (const fileName of params.fileNames) {
@@ -385,6 +411,7 @@ export class TestServerDispatcher implements TestServerInterface {
async stopTests() {
this._testRun?.stop?.resolve();
await this._testRun?.run;
+ this._resumeAfterStepErrors.clear();
}
async _setInterceptStdio(intercept: boolean) {
diff --git a/packages/playwright/src/runner/watchMode.ts b/packages/playwright/src/runner/watchMode.ts
index d14d08d8b4..488a6cf888 100644
--- a/packages/playwright/src/runner/watchMode.ts
+++ b/packages/playwright/src/runner/watchMode.ts
@@ -14,6 +14,7 @@
* limitations under the License.
*/
+import fs from 'fs';
import path from 'path';
import readline from 'readline';
import { EventEmitter } from 'stream';
@@ -27,8 +28,10 @@ import { enquirer } from '../utilsBundle';
import { TestServerDispatcher } from './testServer';
import { TeleSuiteUpdater } from '../isomorphic/teleSuiteUpdater';
import { TestServerConnection } from '../isomorphic/testServerConnection';
+import { stripAnsiEscapes } from '../util';
+import { codeFrameColumns } from '../transform/babelBundle';
-import type { FullResult } from '../../types/testReporter';
+import type * as reporterTypes from '../../types/testReporter';
import type { ConfigLocation } from '../common/config';
import type { TestServerTransport } from '../isomorphic/testServerConnection';
@@ -72,7 +75,7 @@ interface WatchModeOptions {
grep?: string;
}
-export async function runWatchModeLoop(configLocation: ConfigLocation, initialOptions: WatchModeOptions): Promise<FullResult['status']> {
+export async function runWatchModeLoop(configLocation: ConfigLocation, initialOptions: WatchModeOptions): Promise<reporterTypes.FullResult['status']> {
const options: WatchModeOptions = { ...initialOptions };
let bufferMode = false;
@@ -127,8 +130,28 @@ export async function runWatchModeLoop(configLocation: ConfigLocation, initialOp
});
});
testServerConnection.onReport(report => teleSuiteUpdater.processTestReportEvent(report));
+ testServerConnection.onRecoverFromStepError(({ stepId, message, location }) => {
+ process.stdout.write(`\nTest error occurred.\n`);
+ process.stdout.write('\n' + createErrorCodeframe(message, location) + '\n');
+ process.stdout.write(`\n${colors.dim('Try recovering from the error. Press')} ${colors.bold('c')} ${colors.dim('to continue or')} ${colors.bold('t')} ${colors.dim('to throw the error')}\n`);
+ readKeyPress(text => {
+ if (text === 'c') {
+ process.stdout.write(`\n${colors.dim('Continuing after recovery...')}\n`);
+ testServerConnection.resumeAfterStepError({ stepId, status: 'recovered', value: undefined }).catch(() => {});
+ } else if (text === 't') {
+ process.stdout.write(`\n${colors.dim('Throwing error...')}\n`);
+ testServerConnection.resumeAfterStepError({ stepId, status: 'failed' }).catch(() => {});
+ }
+ return text;
+ });
+ });
- await testServerConnection.initialize({ interceptStdio: false, watchTestDirs: true, populateDependenciesOnList: true });
+ await testServerConnection.initialize({
+ interceptStdio: false,
+ watchTestDirs: true,
+ populateDependenciesOnList: true,
+ recoverFromStepErrors: !process.env.PWTEST_RECOVERY_DISABLED,
+ });
await testServerConnection.runGlobalSetup({});
const { report } = await testServerConnection.listTests({});
@@ -137,7 +160,7 @@ export async function runWatchModeLoop(configLocation: ConfigLocation, initialOp
const projectNames = teleSuiteUpdater.rootSuite!.suites.map(s => s.title);
let lastRun: { type: 'changed' | 'regular' | 'failed', failedTestIds?: string[], dirtyTestIds?: string[] } = { type: 'regular' };
- let result: FullResult['status'] = 'passed';
+ let result: reporterTypes.FullResult['status'] = 'passed';
while (true) {
if (bufferMode)
@@ -426,3 +449,28 @@ async function toggleShowBrowser() {
}
type Command = 'run' | 'failed' | 'repeat' | 'changed' | 'project' | 'file' | 'grep' | 'exit' | 'interrupted' | 'toggle-show-browser' | 'toggle-buffer-mode';
+
+function createErrorCodeframe(message: string, location: reporterTypes.Location) {
+ let source: string;
+ try {
+ source = fs.readFileSync(location.file, 'utf-8') + '\n//';
+ } catch (e) {
+ return;
+ }
+
+ return codeFrameColumns(
+ source,
+ {
+ start: {
+ line: location.line,
+ column: location.column,
+ },
+ },
+ {
+ highlightCode: true,
+ linesAbove: 5,
+ linesBelow: 5,
+ message: stripAnsiEscapes(message).split('\n')[0] || undefined,
+ }
+ );
+}
diff --git a/packages/playwright/src/runner/workerHost.ts b/packages/playwright/src/runner/workerHost.ts
index 0ab8c54ba1..3879971ef0 100644
--- a/packages/playwright/src/runner/workerHost.ts
+++ b/packages/playwright/src/runner/workerHost.ts
@@ -25,6 +25,7 @@ import { artifactsFolderName } from '../isomorphic/folders';
import type { TestGroup } from './testGroups';
import type { RunPayload, SerializedConfig, WorkerInitParams } from '../common/ipc';
+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';
let lastWorkerIndex = 0;
@@ -36,7 +37,7 @@ export class WorkerHost extends ProcessHost {
private _params: WorkerInitParams;
private _didFail = false;
- constructor(testGroup: TestGroup, parallelIndex: number, config: SerializedConfig, extraEnv: Record<string, string | undefined>, outputDir: string) {
+ constructor(testGroup: TestGroup, parallelIndex: number, config: SerializedConfig, recoverFromStepErrors: boolean, extraEnv: Record<string, string | undefined>, outputDir: string) {
const workerIndex = lastWorkerIndex++;
super(require.resolve('../worker/workerMain.js'), `worker-${workerIndex}`, {
...extraEnv,
@@ -53,7 +54,8 @@ export class WorkerHost extends ProcessHost {
repeatEachIndex: testGroup.repeatEachIndex,
projectId: testGroup.projectId,
config,
- artifactsDir: path.join(outputDir, artifactsFolderName(workerIndex))
+ artifactsDir: path.join(outputDir, artifactsFolderName(workerIndex)),
+ recoverFromStepErrors,
};
}
@@ -79,6 +81,10 @@ export class WorkerHost extends ProcessHost {
this.sendMessageNoReply({ method: 'runTestGroup', params: runPayload });
}
+ resumeAfterStepError(result: RecoverFromStepErrorResult) {
+ this.sendMessageNoReply({ method: 'resumeAfterStepError', params: result });
+ }
+
hash() {
return this._hash;
}
diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts
index eb590b0abd..344c50a64b 100644
--- a/packages/playwright/src/worker/testInfo.ts
+++ b/packages/playwright/src/worker/testInfo.ts
@@ -17,10 +17,10 @@
import fs from 'fs';
import path from 'path';
-import { captureRawStack, monotonicTime, sanitizeForFilePath, stringifyStackFrames, currentZone, createGuid, escapeWithQuotes } from 'playwright-core/lib/utils';
+import { captureRawStack, monotonicTime, sanitizeForFilePath, stringifyStackFrames, currentZone, createGuid, escapeWithQuotes, ManualPromise } from 'playwright-core/lib/utils';
import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutManager';
-import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, sanitizeFilePathBeforeExtension, trimLongString, windowsFilesystemFriendlyLength } from '../util';
+import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, sanitizeFilePathBeforeExtension, serializeError, trimLongString, windowsFilesystemFriendlyLength } from '../util';
import { TestTracing } from './testTracing';
import { testInfoError } from './util';
import { wrapFunctionWithLocation } from '../transform/transform';
@@ -29,9 +29,10 @@ import type { RunnableDescription } from './timeoutManager';
import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test';
import type { FullConfig, Location } from '../../types/testReporter';
import type { FullConfigInternal, FullProjectInternal } from '../common/config';
-import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc';
+import type { AttachmentPayload, ResumeAfterStepErrorPayload, StepBeginPayload, StepEndPayload, StepRecoverFromErrorPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc';
import type { TestCase } from '../common/test';
import type { StackFrame } from '@protocol/channels';
+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';
export type TestStepVisibility = 'internal' | 'hidden';
export type TestStepCategory = 'expect' | 'fixture' | 'hook' | 'pw:api' | 'test.step' | 'test.attach';
@@ -48,6 +49,7 @@ interface TestStepData {
}
export interface TestStepInternal extends TestStepData {
+ recoverFromStepError(error: Error): Promise<RecoverFromStepErrorResult>;
complete(result: { error?: Error | unknown, suggestedRebaseline?: string }): void;
info: TestStepInfoImpl;
attachmentIndices: number[];
@@ -65,6 +67,7 @@ type SnapshotNames = {
export class TestInfoImpl implements TestInfo {
private _onStepBegin: (payload: StepBeginPayload) => void;
+ private _onStepRecoverFromError: (payload: StepRecoverFromErrorPayload) => void;
private _onStepEnd: (payload: StepEndPayload) => void;
private _onAttach: (payload: AttachmentPayload) => void;
private _snapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} };
@@ -118,6 +121,7 @@ export class TestInfoImpl implements TestInfo {
readonly snapshotDir: string;
errors: TestInfoErrorImpl[] = [];
readonly _attachmentsPush: (...items: TestInfo['attachments']) => number;
+ private _recoverFromStepErrorResults: Map<string, ManualPromise<ResumeAfterStepErrorPayload>> | undefined;
get error(): TestInfoErrorImpl | undefined {
return this.errors[0];
@@ -157,11 +161,13 @@ export class TestInfoImpl implements TestInfo {
test: TestCase | undefined,
retry: number,
onStepBegin: (payload: StepBeginPayload) => void,
+ onStepRecoverFromError: (payload: StepRecoverFromErrorPayload) => void,
onStepEnd: (payload: StepEndPayload) => void,
onAttach: (payload: AttachmentPayload) => void,
) {
this.testId = test?.id ?? '';
this._onStepBegin = onStepBegin;
+ this._onStepRecoverFromError = onStepRecoverFromError;
this._onStepEnd = onStepEnd;
this._onAttach = onAttach;
this._startTime = monotonicTime();
@@ -185,6 +191,7 @@ export class TestInfoImpl implements TestInfo {
this.tags = test?.tags ?? [];
this.fn = test?.fn ?? (() => {});
this.expectedStatus = test?.expectedStatus ?? 'skipped';
+ this._recoverFromStepErrorResults = workerParams.recoverFromStepErrors ? new Map() : undefined;
this._timeoutManager = new TimeoutManager(this.project.timeout);
if (configInternal.configCLIOverrides.debug)
@@ -298,6 +305,22 @@ export class TestInfoImpl implements TestInfo {
steps: [],
attachmentIndices: [],
info: new TestStepInfoImpl(this, stepId, data.title, parentStep?.info),
+ recoverFromStepError: async (error: Error) => {
+ if (!this._recoverFromStepErrorResults)
+ return { stepId, status: 'failed' };
+ const payload: StepRecoverFromErrorPayload = {
+ testId: this.testId,
+ stepId,
+ error: serializeError(error),
+ };
+ this._onStepRecoverFromError(payload);
+ const recoveryPromise = new ManualPromise<RecoverFromStepErrorResult>();
+ this._recoverFromStepErrorResults.set(stepId, recoveryPromise);
+ const recoveryResult = await recoveryPromise;
+ if (recoveryResult.stepId !== stepId)
+ return { stepId, status: 'failed' };
+ return recoveryResult;
+ },
complete: result => {
if (step.endWallTime)
return;
@@ -374,6 +397,12 @@ export class TestInfoImpl implements TestInfo {
return step;
}
+ resumeAfterStepError(result: ResumeAfterStepErrorPayload) {
+ const recoveryPromise = this._recoverFromStepErrorResults?.get(result.stepId);
+ if (recoveryPromise)
+ recoveryPromise.resolve(result);
+ }
+
_interrupt() {
// Mark as interrupted so we can ignore TimeoutError thrown by interrupt() call.
this._wasInterrupted = true;
diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts
index e9f515a760..1fefb76a6d 100644
--- a/packages/playwright/src/worker/workerMain.ts
+++ b/packages/playwright/src/worker/workerMain.ts
@@ -33,7 +33,7 @@ import { loadTestFile } from '../common/testLoader';
import type { TimeSlot } from './timeoutManager';
import type { Location } from '../../types/testReporter';
import type { FullConfigInternal, FullProjectInternal } from '../common/config';
-import type { DonePayload, RunPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc';
+import type { DonePayload, ResumeAfterStepErrorPayload, RunPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc';
import type { Suite, TestCase } from '../common/test';
import type { TestAnnotation } from '../../types/test';
@@ -113,7 +113,7 @@ export class WorkerMain extends ProcessRunner {
return;
}
// Ignore top-level errors, they are already inside TestInfo.errors.
- const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {});
+ const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {}, () => {});
const runnable = { type: 'teardown' } as const;
// We have to load the project to get the right deadline below.
await fakeTestInfo._runWithTimeout(runnable, () => this._loadIfNeeded()).catch(() => {});
@@ -259,9 +259,14 @@ export class WorkerMain extends ProcessRunner {
}
}
+ resumeAfterStepError(params: ResumeAfterStepErrorPayload): void {
+ this._currentTest?.resumeAfterStepError(params);
+ }
+
private async _runTest(test: TestCase, retry: number, nextTest: TestCase | undefined) {
const testInfo = new TestInfoImpl(this._config, this._project, this._params, test, retry,
stepBeginPayload => this.dispatchEvent('stepBegin', stepBeginPayload),
+ stepRecoverFromErrorPayload => this.dispatchEvent('stepRecoverFromError', stepRecoverFromErrorPayload),
stepEndPayload => this.dispatchEvent('stepEnd', stepEndPayload),
attachment => this.dispatchEvent('attach', attachment));
diff --git a/tests/playwright-test/watch.spec.ts b/tests/playwright-test/watch.spec.ts
index 431d5e2cc8..750c3f854d 100644
--- a/tests/playwright-test/watch.spec.ts
+++ b/tests/playwright-test/watch.spec.ts
@@ -710,7 +710,7 @@ test('should run CT on changed deps', async ({ runWatchTest, writeFiles }) => {
await expect(component).toHaveText('hello');
});
`,
- });
+ }, undefined, { PWTEST_RECOVERY_DISABLED: '1' });
await testProcess.waitForOutput('Waiting for file changes.');
await writeFiles({
'src/button.tsx': `