Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat: pause on `--debug` (#38345)

Simon Knott71990226c3a3
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index 4d9ca7f846..89f17bad4e 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -84,11 +84,15 @@ export type AttachmentPayload = {
stepId?: string;
};
+export type TestErrorsPayload = {
+ testId: string;
+ errors: TestInfoErrorImpl[];
+};
+
export type TestInfoErrorImpl = TestInfoError;
export type TestPausedPayload = {
testId: string;
- errors: TestInfoErrorImpl[];
};
export type CustomMessageRequestPayload = {
@@ -105,7 +109,6 @@ export type TestEndPayload = {
testId: string;
duration: number;
status: TestStatus;
- errors: TestInfoErrorImpl[];
hasNonRetriableError: boolean;
expectedStatus: TestStatus;
annotations: { type: string, description?: string }[];
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index cef009c730..d0fced2ece 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -96,7 +96,8 @@ export type JsonTestResultEnd = {
id: string;
duration: number;
status: reporterTypes.TestStatus;
- errors: reporterTypes.TestError[];
+ /** No longer emitted, but kept for backwards compatibility */
+ errors?: reporterTypes.TestError[];
/** No longer emitted, but kept for backwards compatibility */
attachments?: JsonAttachment[];
annotations?: TestAnnotation[];
@@ -132,7 +133,7 @@ export type JsonFullResult = {
};
export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent
- | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnErrorEvent | JsonOnStdIOEvent;
+ | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnTestErrorEvent | JsonOnErrorEvent | JsonOnStdIOEvent;
export type JsonOnConfigureEvent = {
method: 'onConfigure';
@@ -198,6 +199,15 @@ export type JsonOnAttachEvent = {
params: JsonTestResultOnAttach;
};
+export type JsonOnTestErrorEvent = {
+ method: 'onTestError';
+ params: {
+ testId: string;
+ resultId: string;
+ error: reporterTypes.TestError;
+ }
+};
+
export type JsonOnErrorEvent = {
method: 'onError';
params: {
@@ -294,6 +304,10 @@ export class TeleReporterReceiver {
this._onAttach(params.testId, params.resultId, params.attachments);
return;
}
+ if (method === 'onTestError') {
+ this._onTestError(params.testId, params.resultId, params.error);
+ return;
+ }
if (method === 'onStepEnd') {
this._onStepEnd(params.testId, params.resultId, params.step);
return;
@@ -353,8 +367,11 @@ export class TeleReporterReceiver {
const result = test.results.find(r => r._id === payload.id)!;
result.duration = payload.duration;
result.status = payload.status;
- result.errors = payload.errors;
- result.error = result.errors?.[0];
+ // Errors are only present here from legacy blobs. These override all _onTestError events
+ if (!!payload.errors) {
+ result.errors = payload.errors;
+ result.error = result.errors[0];
+ }
// Attachments are only present here from legacy blobs. These override all _onAttach events
if (!!payload.attachments)
result.attachments = this._parseAttachments(payload.attachments);
@@ -404,6 +421,13 @@ export class TeleReporterReceiver {
})));
}
+ private _onTestError(testId: string, resultId: string, error: reporterTypes.TestError) {
+ const test = this._tests.get(testId)!;
+ const result = test.results.find(r => r._id === resultId)!;
+ result.errors.push(error);
+ result.error = result.errors[0];
+ }
+
private _onError(error: reporterTypes.TestError) {
this._reporter.onError?.(error);
}
diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts
index 946c16932b..c48bfdf70b 100644
--- a/packages/playwright/src/reporters/internalReporter.ts
+++ b/packages/playwright/src/reporters/internalReporter.ts
@@ -67,8 +67,12 @@ export class InternalReporter implements ReporterV2 {
this._reporter.onStdErr?.(chunk, test, result);
}
+ onTestError(test: TestCase, result: TestResult, error: TestError): void {
+ addLocationAndSnippetToError(this._config, error, test.location.file);
+ this._reporter.onTestError?.(test, result, error);
+ }
+
onTestEnd(test: TestCase, result: TestResult) {
- this._addSnippetToTestErrors(test, result);
this._reporter.onTestEnd?.(test, result);
}
@@ -89,7 +93,7 @@ export class InternalReporter implements ReporterV2 {
}
onError(error: TestError) {
- addLocationAndSnippetToError(this._config, error);
+ addLocationAndSnippetToError(this._config, error, undefined);
this._reporter.onError?.(error);
}
@@ -98,26 +102,17 @@ export class InternalReporter implements ReporterV2 {
}
onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
- this._addSnippetToStepError(test, step);
+ if (step.error)
+ addLocationAndSnippetToError(this._config, step.error, test.location.file);
this._reporter.onStepEnd?.(test, result, step);
}
printsToStdio() {
return this._reporter.printsToStdio ? this._reporter.printsToStdio() : true;
}
-
- private _addSnippetToTestErrors(test: TestCase, result: TestResult) {
- for (const error of result.errors)
- addLocationAndSnippetToError(this._config, error, test.location.file);
- }
-
- private _addSnippetToStepError(test: TestCase, step: TestStep) {
- if (step.error)
- addLocationAndSnippetToError(this._config, step.error, test.location.file);
- }
}
-export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file?: string) {
+export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file: string | undefined) {
if (error.stack && !error.location)
error.location = prepareErrorStack(error.stack).location;
const location = error.location;
diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts
index c0ed7fd6e9..3571f62972 100644
--- a/packages/playwright/src/reporters/merge.ts
+++ b/packages/playwright/src/reporters/merge.ts
@@ -404,6 +404,7 @@ class IdsPatcher {
case 'onProject':
this._onProject(params.project);
return;
+ case 'onTestError':
case 'onAttach':
case 'onTestBegin':
case 'onStepBegin':
@@ -498,7 +499,7 @@ class PathSeparatorPatcher {
test.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
const testResult = jsonEvent.params.result;
testResult.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
- testResult.errors.forEach(error => this._updateErrorLocations(error));
+ testResult.errors?.forEach(error => this._updateErrorLocations(error));
(testResult.attachments ?? []).forEach(attachment => {
if (attachment.path)
attachment.path = this._updatePath(attachment.path);
@@ -516,6 +517,10 @@ class PathSeparatorPatcher {
step.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
return;
}
+ if (jsonEvent.method === 'onTestError') {
+ this._updateErrorLocations(jsonEvent.params.error);
+ return;
+ }
if (jsonEvent.method === 'onAttach') {
const attach = jsonEvent.params;
attach.attachments.forEach(attachment => {
diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts
index 2a11b93583..0c8bb5ff14 100644
--- a/packages/playwright/src/reporters/multiplexer.ts
+++ b/packages/playwright/src/reporters/multiplexer.ts
@@ -54,6 +54,11 @@ export class Multiplexer implements ReporterV2 {
wrap(() => reporter.onStdErr?.(chunk, test, result));
}
+ onTestError(test: TestCase, result: TestResult, error: TestError) {
+ for (const reporter of this._reporters)
+ wrap(() => reporter.onTestError?.(test, result, error));
+ }
+
onTestEnd(test: TestCase, result: TestResult) {
for (const reporter of this._reporters)
wrap(() => reporter.onTestEnd?.(test, result));
diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts
index 280ef2bfd2..5085a63a85 100644
--- a/packages/playwright/src/reporters/reporterV2.ts
+++ b/packages/playwright/src/reporters/reporterV2.ts
@@ -22,6 +22,7 @@ export interface ReporterV2 {
onTestBegin?(test: TestCase, result: TestResult): void;
onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
+ onTestError?(test: TestCase, result: TestResult, error: TestError): void;
onTestEnd?(test: TestCase, result: TestResult): void;
onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;
onExit?(): void | Promise<void>;
diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts
index d191104b70..8a792d1849 100644
--- a/packages/playwright/src/reporters/teleEmitter.ts
+++ b/packages/playwright/src/reporters/teleEmitter.ts
@@ -71,6 +71,17 @@ export class TeleReporterEmitter implements ReporterV2 {
});
}
+ onTestError(test: reporterTypes.TestCase, result: reporterTypes.TestResult, error: reporterTypes.TestError): void {
+ this._messageSink({
+ method: 'onTestError',
+ params: {
+ testId: test.id,
+ resultId: (result as any)[this._idSymbol],
+ error,
+ }
+ });
+ }
+
onTestEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult): void {
const testEnd: teleReceiver.JsonTestEnd = {
testId: test.id,
@@ -248,7 +259,6 @@ export class TeleReporterEmitter implements ReporterV2 {
id: (result as any)[this._idSymbol],
duration: result.duration,
status: result.status,
- errors: result.errors,
annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined,
};
}
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index 358f639ac0..1e8958569f 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -28,7 +28,7 @@ import type { ProcessExitData } from './processHost';
import type { TestGroup } from './testGroups';
import type { TestError, TestResult, TestStep } from '../../types/testReporter';
import type { FullConfigInternal } from '../common/config';
-import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc';
+import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestErrorsPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc';
import type { Suite } from '../common/test';
import type { TestCase } from '../common/test';
import type { ReporterV2 } from '../reporters/reporterV2';
@@ -322,7 +322,6 @@ class JobDispatcher {
// Do not show more than one error to avoid confusion, but report
// as interrupted to indicate that we did actually start the test.
params.status = 'interrupted';
- params.errors = [];
}
const data = this._dataByTestId.get(params.testId);
if (!data) {
@@ -333,8 +332,6 @@ class JobDispatcher {
this._remainingByTestId.delete(params.testId);
const { result, test } = data;
result.duration = params.duration;
- result.errors = params.errors;
- result.error = result.errors[0];
result.status = params.status;
result.annotations = params.annotations;
test.annotations = [...params.annotations]; // last test result wins
@@ -429,6 +426,23 @@ class JobDispatcher {
}
}
+ private _onTestErrors(params: TestErrorsPayload) {
+ if (this._failureTracker.hasReachedMaxFailures()) {
+ // Do not show more than one error to avoid confusion.
+ return;
+ }
+
+ const data = this._dataByTestId.get(params.testId)!;
+ if (!data)
+ return;
+ const { test, result } = data;
+ for (const error of params.errors) {
+ result.errors.push(error);
+ result.error = result.errors[0];
+ this._reporter.onTestError?.(test, result, error);
+ }
+ }
+
private _failTestWithErrors(test: TestCase, errors: TestError[]) {
const runData = this._dataByTestId.get(test.id);
// There might be a single test that has started but has not finished yet.
@@ -439,8 +453,11 @@ class JobDispatcher {
result = test._appendTestResult();
this._reporter.onTestBegin?.(test, result);
}
- result.errors = [...errors];
- result.error = result.errors[0];
+ for (const error of errors) {
+ result.errors.push(error);
+ result.error = result.errors[0];
+ this._reporter.onTestError?.(test, result, error);
+ }
result.status = errors.length ? 'failed' : 'skipped';
this._reportTestEnd(test, result);
this._failedTests.add(test);
@@ -578,6 +595,7 @@ class JobDispatcher {
eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)),
eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)),
eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)),
+ eventsHelper.addEventListener(worker, 'testErrors', this._onTestErrors.bind(this)),
eventsHelper.addEventListener(worker, 'testPaused', this._onTestPaused.bind(this, worker)),
eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)),
eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)),
@@ -585,24 +603,28 @@ class JobDispatcher {
}
private _onTestPaused(worker: WorkerHost, params: TestPausedPayload) {
+ const data = this._dataByTestId.get(params.testId);
+ if (!data)
+ return;
+
+ const { test, result } = data;
+
const sendMessage = async (message: { request: any }) => {
try {
if (this.jobResult.isDone())
throw new Error('Test has already stopped');
const response = await worker.sendCustomMessage({ testId: params.testId, request: message.request });
if (response.error)
- addLocationAndSnippetToError(this._config.config, response.error);
+ addLocationAndSnippetToError(this._config.config, response.error, test.location.file);
return response;
} catch (e) {
const error = serializeError(e);
- addLocationAndSnippetToError(this._config.config, error);
+ addLocationAndSnippetToError(this._config.config, error, test.location.file);
return { response: undefined, error };
}
};
- for (const error of params.errors)
- addLocationAndSnippetToError(this._config.config, error);
- this._failureTracker.onTestPaused?.({ ...params, sendMessage });
+ this._failureTracker.onTestPaused?.({ errors: result.errors, sendMessage });
}
skipWholeJob(): boolean {
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index b7749748b5..0fd445df61 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -479,7 +479,9 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promise
createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }),
...createRunTestsTasks(config),
];
- const status = await runTasks(new TestRun(config, reporter), tasks, config.config.globalTimeout);
+
+ const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.debug, pauseOnError: config.configCLIOverrides.debug });
+ const status = await runTasks(testRun, tasks, config.config.globalTimeout);
// Calling process.exit() might truncate large stdout/stderr output.
// See https://github.com/nodejs/node/issues/6456.
diff --git a/packages/playwright/src/transform/babelHighlightUtils.ts b/packages/playwright/src/transform/babelHighlightUtils.ts
new file mode 100644
index 0000000000..4611233686
--- /dev/null
+++ b/packages/playwright/src/transform/babelHighlightUtils.ts
@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+import { traverse, babelParse, T, types as t } from './babelBundle';
+import type { Location } from '../../types/testReporter';
+
+function containsLocation(range: T.SourceLocation, location: Location): boolean {
+ if (location.line < range.start.line || location.line > range.end.line)
+ return false;
+ if (location.line === range.start.line && location.column < range.start.column)
+ return false;
+ if (location.line === range.end.line && location.column > range.end.column)
+ return false;
+ return true;
+}
+
+export function findTestEndLocation(text: string, testStartLocation: Location): Location | undefined {
+ const ast = babelParse(text, path.basename(testStartLocation.file), false);
+ let result: Location | undefined;
+ traverse(ast, {
+ enter(path) {
+ if (t.isCallExpression(path.node) && path.node.loc && containsLocation(path.node.loc, testStartLocation)) {
+ const callNode = path.node;
+ const funcNode = callNode.arguments[callNode.arguments.length - 1];
+ if (callNode.arguments.length >= 2 && t.isFunction(funcNode) && funcNode.body.loc)
+ result = { file: testStartLocation.file, ...funcNode.body.loc.end };
+ }
+ }
+ });
+ return result;
+}
diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts
index c9fd959dd9..5838b28a57 100644
--- a/packages/playwright/src/worker/testInfo.ts
+++ b/packages/playwright/src/worker/testInfo.ts
@@ -24,12 +24,13 @@ import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAnd
import { TestTracing } from './testTracing';
import { testInfoError } from './util';
import { wrapFunctionWithLocation } from '../transform/transform';
+import { findTestEndLocation } from '../transform/babelHighlightUtils';
import type { RunnableDescription } from './timeoutManager';
import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test';
import type { FullConfig, Location } from '../../types/testReporter';
import type { FullConfigInternal, FullProjectInternal } from '../common/config';
-import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestInfoErrorImpl, TestPausedPayload, WorkerInitParams } from '../common/ipc';
+import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestErrorsPayload, TestInfoErrorImpl, TestPausedPayload, WorkerInitParams } from '../common/ipc';
import type { TestCase } from '../common/test';
import type { StackFrame } from '@protocol/channels';
@@ -69,6 +70,7 @@ export class TestInfoImpl implements TestInfo {
private _onStepBegin: (payload: StepBeginPayload) => void;
private _onStepEnd: (payload: StepEndPayload) => void;
private _onAttach: (payload: AttachmentPayload) => void;
+ private _onErrors: (errors: TestErrorsPayload) => void;
private _onTestPaused: (payload: TestPausedPayload) => void;
private _snapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} };
private _ariaSnapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} };
@@ -121,6 +123,7 @@ export class TestInfoImpl implements TestInfo {
readonly outputDir: string;
readonly snapshotDir: string;
errors: TestInfoErrorImpl[] = [];
+ private _reportedErrorCount = 0;
readonly _attachmentsPush: (...items: TestInfo['attachments']) => number;
private _workerParams: WorkerInitParams;
@@ -164,12 +167,14 @@ export class TestInfoImpl implements TestInfo {
onStepBegin: (payload: StepBeginPayload) => void,
onStepEnd: (payload: StepEndPayload) => void,
onAttach: (payload: AttachmentPayload) => void,
+ onErrors: (payload: TestErrorsPayload) => void,
onTestPaused: (payload: TestPausedPayload) => void,
) {
this.testId = test?.id ?? '';
this._onStepBegin = onStepBegin;
this._onStepEnd = onStepEnd;
this._onAttach = onAttach;
+ this._onErrors = onErrors;
this._onTestPaused = onTestPaused;
this._startTime = monotonicTime();
this._startWallTime = Date.now();
@@ -406,7 +411,7 @@ export class TestInfoImpl implements TestInfo {
this._tracing.appendForError(serialized);
}
- async _runAsStep(stepInfo: { title: string, category: 'hook' | 'fixture', location?: Location, group?: string }, cb: () => Promise<any>) {
+ async _runAsStep(stepInfo: { title: string, category: 'hook' | 'fixture' | 'test.step', location?: Location, group?: string }, cb: () => Promise<any>) {
const step = this._addStep(stepInfo);
try {
await cb();
@@ -464,12 +469,35 @@ export class TestInfoImpl implements TestInfo {
async _didFinishTestFunction() {
const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure());
if (shouldPause) {
- this._onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] });
- await this._interruptedPromise;
+ const location = (this._isFailure() ? this._errorLocation() : await this._testEndLocation()) ?? { file: this.file, line: this.line, column: this.column };
+ this._emitErrors();
+ this._onTestPaused({ testId: this.testId });
+ await this._runAsStep({ title: this._isFailure() ? 'Paused on Error' : 'Paused at End', category: 'test.step', location }, async () => {
+ await this._interruptedPromise;
+ });
}
await this._onDidFinishTestFunctionCallback?.();
}
+ _emitErrors() {
+ const errors = this.errors.slice(this._reportedErrorCount);
+ this._reportedErrorCount = Math.max(this._reportedErrorCount, this.errors.length);
+ if (errors.length)
+ this._onErrors({ testId: this.testId, errors });
+ }
+
+ private _errorLocation(): Location | undefined {
+ if (this.error?.stack)
+ return filteredStackTrace(this.error.stack.split('\n'))[0];
+ }
+
+ private async _testEndLocation(): Promise<Location | undefined> {
+ try {
+ const source = await fs.promises.readFile(this.file, 'utf-8');
+ return findTestEndLocation(source, { file: this.file, line: this.line, column: this.column });
+ } catch {}
+ }
+
// ------------ TestInfo methods ------------
async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) {
diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts
index f1a96cbdd8..b49d6482da 100644
--- a/packages/playwright/src/worker/workerMain.ts
+++ b/packages/playwright/src/worker/workerMain.ts
@@ -118,7 +118,7 @@ export class WorkerMain extends ProcessRunner {
return;
}
// Ignore top-level errors, they are already inside TestInfo.errors.
- const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {}, () => {});
+ const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {}, () => {}, () => {});
const runnable = { type: 'teardown' } as const;
// We have to load the project to get the right deadline below.
await fakeTestInfo._runWithTimeout(runnable, () => this._loadIfNeeded()).catch(() => {});
@@ -283,6 +283,7 @@ export class WorkerMain extends ProcessRunner {
stepBeginPayload => this.dispatchEvent('stepBegin', stepBeginPayload),
stepEndPayload => this.dispatchEvent('stepEnd', stepEndPayload),
attachment => this.dispatchEvent('attach', attachment),
+ errors => this.dispatchEvent('testErrors', errors),
testPausedPayload => this.dispatchEvent('testPaused', testPausedPayload));
const processAnnotation = (annotation: TestAnnotation) => {
@@ -331,6 +332,7 @@ export class WorkerMain extends ProcessRunner {
if (isSkipped && nextTest && !hasAfterAllToRunBeforeNextTest) {
// Fast path - this test is skipped, and there are more tests that will handle cleanup.
testInfo.status = 'skipped';
+ testInfo._emitErrors();
this.dispatchEvent('testEnd', buildTestEndPayload(testInfo));
return;
}
@@ -497,6 +499,7 @@ export class WorkerMain extends ProcessRunner {
this._currentTest = null;
setCurrentTestInfo(null);
+ testInfo._emitErrors();
this.dispatchEvent('testEnd', buildTestEndPayload(testInfo));
const preserveOutput = this._config.config.preserveOutput === 'always' ||
@@ -622,7 +625,6 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {
testId: testInfo.testId,
duration: testInfo.duration,
status: testInfo.status!,
- errors: testInfo.errors,
hasNonRetriableError: testInfo._hasNonRetriableError,
expectedStatus: testInfo.expectedStatus,
annotations: testInfo.annotations,
diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts
new file mode 100644
index 0000000000..6021f70225
--- /dev/null
+++ b/tests/playwright-test/pause-at-end.spec.ts
@@ -0,0 +1,125 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Reporter, TestCase, TestResult, TestStep } from 'packages/playwright-test/reporter';
+import { test, expect, parseTestRunnerOutput } from './playwright-test-fixtures';
+
+class LocationReporter implements Reporter {
+ onStepBegin(test: TestCase, result: TestResult, step: TestStep): void {
+ if (step.title.startsWith('Paused')) {
+ console.log('\n');
+ console.log(`%%${step.title} at :${step.location?.line}:${step.location?.column}`);
+ if (result.error)
+ console.log(`%%result.error at :${result.error.location?.line}:${result.error.location?.column}`);
+ for (const [index, error] of result.errors.entries())
+ console.log(`%%result.errors[${index}] at :${error.location?.line}:${error.location?.column}`);
+ console.log('\n');
+ }
+ }
+}
+
+test('--debug should pause at end', async ({ interactWithTestRunner, }) => {
+ test.skip(process.platform === 'win32', 'No sending SIGINT on Windows');
+ const testProcess = await interactWithTestRunner({
+ 'location-reporter.js': `export default ${LocationReporter}`,
+ 'playwright.config.js': `
+ module.exports = { reporter: [['list'], ['./location-reporter.js']] };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('pass', () => {
+ });
+ test.afterEach(() => {
+ console.log('teardown'.toUpperCase()); // uppercase so we dont confuse it with source snippets
+ });
+ `
+ }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' });
+ await testProcess.waitForOutput('Paused at End');
+ await testProcess.kill('SIGINT');
+ expect(testProcess.output).toContain('TEARDOWN');
+ expect(testProcess.outputLines()).toEqual(['Paused at End at :4:7']);
+
+ const result = parseTestRunnerOutput(testProcess.output);
+ expect(result.interrupted).toBe(1);
+});
+
+test('--debug should pause at end with setup project', async ({ interactWithTestRunner, }) => {
+ test.skip(process.platform === 'win32', 'No sending SIGINT on Windows');
+ const testProcess = await interactWithTestRunner({
+ 'location-reporter.js': `export default ${LocationReporter}`,
+ 'playwright.config.js': `
+ module.exports = {
+ reporter: [['list'], ['./location-reporter.js']],
+ projects: [
+ { name: 'setup', testMatch: /setup\\.test\\.js/ },
+ { name: 'main', dependencies: ['setup'] }
+ ]
+ };
+ `,
+ 'setup.test.js': `
+ import { test } from '@playwright/test';
+ test('setup', () => {
+ });
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('pass', () => {
+ console.log('main test started');
+ });
+ `
+ }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' });
+ await testProcess.waitForOutput('main test started');
+ await testProcess.waitForOutput('Paused at End');
+ await testProcess.kill('SIGINT');
+ expect(testProcess.outputLines()).toEqual(['Paused at End at :5:7']);
+
+ const result = parseTestRunnerOutput(testProcess.output);
+ expect(result.interrupted).toBe(1);
+});
+
+test('--debug should pause on error', async ({ interactWithTestRunner, mergeReports }) => {
+ test.skip(process.platform === 'win32', 'No sending SIGINT on Windows');
+ const testProcess = await interactWithTestRunner({
+ 'location-reporter.js': `export default ${LocationReporter}`,
+ 'playwright.config.js': `
+ module.exports = { reporter: [['list'], ['blob'], ['./location-reporter.js']] };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('pass', () => {
+ expect.soft(1).toBe(2);
+ expect(2).toBe(3);
+ console.log('after error'.toUpperCase());
+ });
+ `
+ }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' });
+ await testProcess.waitForOutput('Paused on Error');
+ expect(testProcess.output).not.toContain('AFTER ERROR');
+ await testProcess.kill('SIGINT');
+ expect(testProcess.output).not.toContain('AFTER ERROR');
+ expect(testProcess.outputLines()).toEqual([
+ 'Paused on Error at :4:24',
+ 'result.error at :4:24',
+ 'result.errors[0] at :4:24',
+ 'result.errors[1] at :5:19',
+ ]);
+
+ const result = parseTestRunnerOutput(testProcess.output);
+ expect(result.failed).toBe(1);
+
+ const merged = await mergeReports('blob-report', undefined, { additionalArgs: ['--reporter', 'location-reporter.js'] });
+ expect(merged.outputLines).toEqual(testProcess.outputLines());
+});
diff --git a/tests/playwright-test/timeout.spec.ts b/tests/playwright-test/timeout.spec.ts
index 9d6001078d..03b626d9ce 100644
--- a/tests/playwright-test/timeout.spec.ts
+++ b/tests/playwright-test/timeout.spec.ts
@@ -143,8 +143,8 @@ test('should respect test.slow', async ({ runInlineTest }) => {
expect(result.output).toContain('Test timeout of 1000ms exceeded.');
});
-test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) => {
- const result = await runInlineTest({
+test('should ignore test.setTimeout when debugging', async ({ interactWithTestRunner }) => {
+ const testProcess = await interactWithTestRunner({
'a.spec.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
@@ -159,15 +159,15 @@ test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) =
await new Promise(f => setTimeout(f, 2000));
});
`
- }, { debug: true });
- expect(result.exitCode).toBe(0);
- expect(result.passed).toBe(1);
+ }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' });
+ await testProcess.waitForOutput('Paused at End');
+ await testProcess.kill('SIGINT');
});
test('should ignore globalTimeout when debugging', {
annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/34911' },
-}, async ({ runInlineTest }) => {
- const result = await runInlineTest({
+}, async ({ interactWithTestRunner }) => {
+ const testProcess = await interactWithTestRunner({
'playwright.config.ts': `
export default {
globalTimeout: 100,
@@ -179,9 +179,9 @@ test('should ignore globalTimeout when debugging', {
await new Promise(f => setTimeout(f, 2000));
});
`
- }, { debug: true });
- expect(result.exitCode).toBe(0);
- expect(result.passed).toBe(1);
+ }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' });
+ await testProcess.waitForOutput('Paused at End');
+ await testProcess.kill('SIGINT');
});
test('should respect fixture timeout', async ({ runInlineTest }) => {