Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: simplify reporter `onTestPaused` (#38679)

Simon Knottaa35f26b3f49
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index b22fdc036f..8c20d80e97 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -90,13 +90,10 @@ export type TestInfoErrorImpl = TestInfoError;
export type TestPausedPayload = {
testId: string;
- stepId: string;
errors: TestInfoErrorImpl[];
};
-export type ResumePayload = {
- action?: 'continue' | 'abort';
-};
+export type ResumePayload = {};
export type CloneStoragePayload = {
storageFile: string;
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index 657664199e..10c967d33c 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -172,7 +172,6 @@ export type JsonOnTestPausedEvent = {
params: {
testId: string;
resultId: string;
- stepId: string;
errors: reporterTypes.TestError[];
};
};
@@ -294,7 +293,7 @@ export class TeleReporterReceiver {
return;
}
if (method === 'onTestPaused') {
- this._onTestPaused(params.testId, params.resultId, params.stepId, params.errors);
+ this._onTestPaused(params.testId, params.resultId, params.errors);
return;
}
if (method === 'onTestEnd') {
@@ -362,14 +361,13 @@ export class TeleReporterReceiver {
this._reporter.onTestBegin?.(test, testResult);
}
- private _onTestPaused(testId: string, resultId: string, stepId: string, errors: reporterTypes.TestError[]) {
+ private _onTestPaused(testId: string, resultId: string, errors: reporterTypes.TestError[]) {
const test = this._tests.get(testId)!;
const result = test.results.find(r => r._id === resultId)!;
- const step = result._stepMap.get(stepId)!;
result.errors.push(...errors);
result.error = result.errors[0];
- void this._reporter.onTestPaused?.(test, result, step);
+ void this._reporter.onTestPaused?.(test, result);
}
private _onTestEnd(testEndPayload: JsonTestEnd, payload: JsonTestResultEnd) {
diff --git a/packages/playwright/src/mcp/test/testContext.ts b/packages/playwright/src/mcp/test/testContext.ts
index 4d5a8f3473..16fce05c55 100644
--- a/packages/playwright/src/mcp/test/testContext.ts
+++ b/packages/playwright/src/mcp/test/testContext.ts
@@ -205,7 +205,7 @@ export class TestContext {
claimStdio();
try {
- const setupReporter = new ListReporter({ configDir, screen, includeTestId: true });
+ const setupReporter = new MCPListReporter({ configDir, screen, includeTestId: true });
const { status } = await testRunner.runGlobalSetup([setupReporter]);
if (status !== 'passed')
return { output: testRunnerAndScreen.output.join('\n'), status };
@@ -227,7 +227,7 @@ export class TestContext {
};
try {
- const reporter = new ListReporter({ configDir, screen, includeTestId: true });
+ const reporter = new MCPListReporter({ configDir, screen, includeTestId: true });
status = await Promise.race([
testRunner.runTests(reporter, params).then(result => result.status),
testRunnerAndScreen.waitForTestPaused().then(() => 'paused' as const),
@@ -320,3 +320,10 @@ const bestPracticesMarkdown = `
- NEVER! use page.waitForTimeout()
- NEVER! use page.evaluate()
`;
+
+class MCPListReporter extends ListReporter {
+ async onTestPaused() {
+ // ListReporter waits for user input to resume, we don't want that in MCP.
+ await new Promise<void>(() => {});
+ }
+}
diff --git a/packages/playwright/src/reporters/blob.ts b/packages/playwright/src/reporters/blob.ts
index 324a19345d..50884a0ee9 100644
--- a/packages/playwright/src/reporters/blob.ts
+++ b/packages/playwright/src/reporters/blob.ts
@@ -27,7 +27,7 @@ import { resolveOutputFile, CommonReporterOptions } from './base';
import { TeleReporterEmitter } from './teleEmitter';
import type { BlobReporterOptions } from '../../types/test';
-import type { FullConfig, FullResult, TestCase, TestResult, TestStep } from '../../types/testReporter';
+import type { FullConfig, FullResult, TestCase, TestResult } from '../../types/testReporter';
import type { BlobReportMetadata, JsonAttachment, JsonEvent } from '../isomorphic/teleReceiver';
import type { EventEmitter } from 'events';
@@ -66,10 +66,9 @@ export class BlobReporter extends TeleReporterEmitter {
super.onConfigure(config);
}
- override async onTestPaused(test: TestCase, result: TestResult, step: TestStep) {
+ override async onTestPaused(test: TestCase, result: TestResult) {
// onTestPaused is only relevant for interactive use, not for blob replays.
// merge-reports still gets onStepBegin/onStepEnd for pausing, but not the interactive part, so this is a no-op.
- return { action: undefined };
}
override async onEnd(result: FullResult): Promise<void> {
diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts
index 147a837e52..7b0ac95b82 100644
--- a/packages/playwright/src/reporters/internalReporter.ts
+++ b/packages/playwright/src/reporters/internalReporter.ts
@@ -67,11 +67,9 @@ export class InternalReporter implements ReporterV2 {
this._reporter.onStdErr?.(chunk, test, result);
}
- async onTestPaused(test: TestCase, result: TestResult, step?: TestStep) {
+ async onTestPaused(test: TestCase, result: TestResult) {
this._addSnippetToTestErrors(test, result);
- if (step)
- this._addSnippetToStepError(test, step);
- return await this._reporter.onTestPaused?.(test, result, step) ?? { action: undefined };
+ return await this._reporter.onTestPaused?.(test, result);
}
onTestEnd(test: TestCase, result: TestResult) {
diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts
index ea5daa21a4..43864a7102 100644
--- a/packages/playwright/src/reporters/multiplexer.ts
+++ b/packages/playwright/src/reporters/multiplexer.ts
@@ -54,14 +54,9 @@ export class Multiplexer implements ReporterV2 {
wrap(() => reporter.onStdErr?.(chunk, test, result));
}
- async onTestPaused(test: TestCase, result: TestResult, step: TestStep) {
- // only one reporter can handle the pause. first reporter that cares wins.
- for (const reporter of this._reporters) {
- const disposition = await wrapAsync(() => reporter.onTestPaused?.(test, result, step));
- if (disposition?.action)
- return disposition;
- }
- return { action: undefined };
+ async onTestPaused(test: TestCase, result: TestResult) {
+ for (const reporter of this._reporters)
+ await wrapAsync(() => reporter.onTestPaused?.(test, result));
}
onTestEnd(test: TestCase, result: TestResult) {
diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts
index 555ba2a6a6..d36050a31f 100644
--- a/packages/playwright/src/reporters/reporterV2.ts
+++ b/packages/playwright/src/reporters/reporterV2.ts
@@ -22,7 +22,7 @@ export interface ReporterV2 {
onTestBegin?(test: TestCase, result: TestResult): void;
onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
- onTestPaused?(test: TestCase, result: TestResult, step?: TestStep): Promise<{ action?: 'continue' | 'abort' }>;
+ onTestPaused?(test: TestCase, result: TestResult): Promise<void>;
onTestEnd?(test: TestCase, result: TestResult): void;
onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;
onExit?(): void | Promise<void>;
diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts
index 4efd401309..2b65ed5ba9 100644
--- a/packages/playwright/src/reporters/teleEmitter.ts
+++ b/packages/playwright/src/reporters/teleEmitter.ts
@@ -72,20 +72,19 @@ export class TeleReporterEmitter implements ReporterV2 {
});
}
- async onTestPaused(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step?: reporterTypes.TestStep) {
+ async onTestPaused(test: reporterTypes.TestCase, result: reporterTypes.TestResult) {
const resultId = (result as any)[this._idSymbol];
- const stepId = (step as any)[this._idSymbol];
this._resultKnownErrorCounts.set(resultId, result.errors.length);
this._messageSink({
method: 'onTestPaused',
params: {
testId: test.id,
resultId,
- stepId,
errors: result.errors,
}
});
- return { action: undefined };
+ // keep the test paused until externally resumed
+ await new Promise<void>(() => {});
}
onTestEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult): void {
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index e00ba5b459..5c9f2ae834 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -598,10 +598,7 @@ class JobDispatcher {
if (!data)
return;
- const { result, test, steps } = data;
- const step = steps.get(params.stepId);
- if (!step)
- return;
+ const { result, test } = data;
const sendMessage = async (message: { request: any }) => {
try {
@@ -621,8 +618,8 @@ class JobDispatcher {
result.errors = params.errors;
result.error = result.errors[0];
- void this._reporter.onTestPaused?.(test, result, step).then(params => {
- worker.sendResume(params);
+ void this._reporter.onTestPaused?.(test, result).then(() => {
+ worker.sendResume({});
});
this._failureTracker.onTestPaused?.({ ...params, sendMessage });
}
diff --git a/packages/playwright/src/transform/babelHighlightUtils.ts b/packages/playwright/src/transform/babelHighlightUtils.ts
deleted file mode 100644
index 4611233686..0000000000
--- a/packages/playwright/src/transform/babelHighlightUtils.ts
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import path from 'path';
-import { traverse, babelParse, T, types as t } from './babelBundle';
-import type { Location } from '../../types/testReporter';
-
-function containsLocation(range: T.SourceLocation, location: Location): boolean {
- if (location.line < range.start.line || location.line > range.end.line)
- return false;
- if (location.line === range.start.line && location.column < range.start.column)
- return false;
- if (location.line === range.end.line && location.column > range.end.column)
- return false;
- return true;
-}
-
-export function findTestEndLocation(text: string, testStartLocation: Location): Location | undefined {
- const ast = babelParse(text, path.basename(testStartLocation.file), false);
- let result: Location | undefined;
- traverse(ast, {
- enter(path) {
- if (t.isCallExpression(path.node) && path.node.loc && containsLocation(path.node.loc, testStartLocation)) {
- const callNode = path.node;
- const funcNode = callNode.arguments[callNode.arguments.length - 1];
- if (callNode.arguments.length >= 2 && t.isFunction(funcNode) && funcNode.body.loc)
- result = { file: testStartLocation.file, ...funcNode.body.loc.end };
- }
- }
- });
- return result;
-}
diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts
index 1b37fd89d5..4d99313c05 100644
--- a/packages/playwright/src/worker/testInfo.ts
+++ b/packages/playwright/src/worker/testInfo.ts
@@ -24,7 +24,6 @@ import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAnd
import { TestTracing } from './testTracing';
import { testInfoError } from './util';
import { wrapFunctionWithLocation } from '../transform/transform';
-import { findTestEndLocation } from '../transform/babelHighlightUtils';
import type { RunnableDescription } from './timeoutManager';
import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test';
@@ -474,35 +473,14 @@ export class TestInfoImpl implements TestInfo {
async _didFinishTestFunction() {
const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure());
if (shouldPause) {
- const location = (this._isFailure() ? this._errorLocation() : await this._testEndLocation()) ?? { file: this.file, line: this.line, column: this.column };
- const step = this._addStep({ category: 'hook', title: 'Paused', location });
- const result = await Promise.race([
- this._callbacks.onTestPaused({ testId: this.testId, stepId: step.stepId, errors: this._isFailure() ? this.errors : [] }),
- this._interruptedPromise.then(() => 'interrupted' as const),
+ await Promise.race([
+ this._callbacks.onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] }),
+ this._interruptedPromise,
]);
- if (result !== 'interrupted') {
- if (result.action === 'abort')
- this._interrupt();
- if (result.action === undefined)
- await this._interruptedPromise;
- }
- step.complete({});
}
await this._onDidFinishTestFunctionCallback?.();
}
- private _errorLocation(): Location | undefined {
- if (this.error?.stack)
- return filteredStackTrace(this.error.stack.split('\n'))[0];
- }
-
- private async _testEndLocation(): Promise<Location | undefined> {
- try {
- const source = await fs.promises.readFile(this.file, 'utf-8');
- return findTestEndLocation(source, { file: this.file, line: this.line, column: this.column });
- } catch {}
- }
-
// ------------ TestInfo methods ------------
async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) {
diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts
index f590b84c7f..6e47a38fa3 100644
--- a/tests/playwright-test/pause-at-end.spec.ts
+++ b/tests/playwright-test/pause-at-end.spec.ts
@@ -18,6 +18,8 @@ import { TestCase, TestResult, TestStep } from 'packages/playwright-test/reporte
import { ReporterV2 } from 'packages/playwright/src/reporters/reporterV2';
import { test, expect } from './playwright-test-fixtures';
+test.describe.configure({ mode: 'parallel' });
+
class LocationReporter implements ReporterV2 {
private _foundErrors = 0;
version() {
@@ -29,10 +31,17 @@ class LocationReporter implements ReporterV2 {
this._printErrors(result);
}
}
- async onTestPaused(test: TestCase, result: TestResult, step?: TestStep) {
- console.log(`%%onTestPaused`);
+ async onTestPaused(test: TestCase, result: TestResult) {
+ if (result.error)
+ console.log(`%%onTestPaused on error at :${result.error.location?.line}:${result.error.location?.column}`);
+ else
+ console.log(`%%onTestPaused at end`);
this._printErrors(result);
- return { action: process.env.ACTION as 'continue' | 'abort' };
+ if (process.env.SIGINT_AFTER_PAUSE) {
+ console.log('%%SIGINT');
+ process.kill(process.pid, 'SIGINT');
+ await new Promise(() => {});
+ }
}
onStepEnd(test: TestCase, result: TestResult, step: TestStep): void {
if (step.title === 'Paused') {
@@ -70,11 +79,9 @@ test('--debug should pause at end', async ({ runInlineTest }) => {
console.log('%%teardown');
});
`
- }, { debug: true }, { ACTION: 'continue' });
+ }, { debug: true });
expect(result.outputLines).toEqual([
- 'onStepBegin: After Hooks > Paused at :4:7',
- 'onTestPaused',
- 'onStepEnd: After Hooks > Paused',
+ 'onTestPaused at end',
'teardown',
'onTestEnd',
]);
@@ -103,15 +110,15 @@ test('--debug should pause at end with setup project', async ({ runInlineTest })
console.log('main test started');
});
`
- }, { debug: true }, { ACTION: 'continue' });
- expect(result.outputLines).toContain('onStepBegin: After Hooks > Paused at :5:7');
+ }, { debug: true });
+ expect(result.outputLines).toContain('onTestPaused at end');
});
test('--debug should pause on error', async ({ runInlineTest, mergeReports }) => {
const result = await runInlineTest({
'location-reporter.js': `export default ${LocationReporter}`,
'playwright.config.js': `
- module.exports = { reporter: [['blob'], ['./location-reporter.js']] };
+ module.exports = { reporter: [['./location-reporter.js'], ['blob']] };
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
@@ -121,22 +128,69 @@ test('--debug should pause on error', async ({ runInlineTest, mergeReports }) =>
console.log('%%after error');
});
`
- }, { debug: true }, { ACTION: 'abort' });
+ }, { debug: true });
expect(result.outputLines).toEqual([
- 'onStepBegin: After Hooks > Paused at :4:24',
- 'onTestPaused',
+ 'onTestPaused on error at :4:24',
'result.errors[0] at :4:24',
'result.errors[1] at :5:19',
- 'onStepEnd: After Hooks > Paused',
'onTestEnd',
]);
const merged = await mergeReports('blob-report', undefined, { additionalArgs: ['--reporter', 'location-reporter.js'] });
expect(merged.outputLines, 'merge reporter doesnt get onTestPaused').toEqual([
- 'onStepBegin: After Hooks > Paused at :4:24',
- 'onStepEnd: After Hooks > Paused',
'onTestEnd',
'result.errors[0] at :4:24',
'result.errors[1] at :5:19',
]);
});
+
+test('SIGINT after pause at end should still run teardown', async ({ runInlineTest }) => {
+ test.skip(process.platform === 'win32', 'no SIGINT on windows');
+ const result = await runInlineTest({
+ 'location-reporter.js': `export default ${LocationReporter}`,
+ 'playwright.config.js': `
+ module.exports = { reporter: './location-reporter.js' };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('pass', () => {
+ });
+ test.afterEach(() => {
+ console.log('%%teardown');
+ });
+ `
+ }, { debug: true }, { SIGINT_AFTER_PAUSE: '1' });
+ expect(result.outputLines).toEqual([
+ 'onTestPaused at end',
+ 'SIGINT',
+ 'teardown',
+ 'onTestEnd',
+ ]);
+});
+
+test('SIGINT after pause on error should still run teardown', async ({ runInlineTest }) => {
+ test.skip(process.platform === 'win32', 'no SIGINT on windows');
+ const result = await runInlineTest({
+ 'location-reporter.js': `export default ${LocationReporter}`,
+ 'playwright.config.js': `
+ module.exports = { reporter: './location-reporter.js' };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('pass', () => {
+ expect(2).toBe(3);
+ console.log('%%after error');
+ });
+ test.afterEach(() => {
+ console.log('%%teardown');
+ });
+ `
+ }, { debug: true }, { SIGINT_AFTER_PAUSE: '1' });
+ expect(result.outputLines).toEqual([
+ 'onTestPaused on error at :4:19',
+ 'result.errors[0] at :4:19',
+ 'SIGINT',
+ 'teardown',
+ 'onTestEnd',
+ ]);
+});
diff --git a/tests/playwright-test/timeout.spec.ts b/tests/playwright-test/timeout.spec.ts
index 6726b677dd..9d6001078d 100644
--- a/tests/playwright-test/timeout.spec.ts
+++ b/tests/playwright-test/timeout.spec.ts
@@ -14,7 +14,6 @@
* limitations under the License.
*/
-import type { ReporterV2 } from 'packages/playwright/src/reporters/reporterV2';
import { test, expect } from './playwright-test-fixtures';
test('should run fixture teardown on timeout', async ({ runInlineTest }) => {
@@ -144,25 +143,8 @@ test('should respect test.slow', async ({ runInlineTest }) => {
expect(result.output).toContain('Test timeout of 1000ms exceeded.');
});
-// terminal reporters dont yet implement onTestPaused, so we default to continuing for these tests
-class TestPauseSkipReporter implements ReporterV2 {
- version() {
- return 'v2' as const;
- }
-
- async onTestPaused() {
- return { action: 'continue' as const };
- }
-}
-
test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) => {
const result = await runInlineTest({
- 'skip-pause.js': `export default ${TestPauseSkipReporter}`,
- 'playwright.config.ts': `
- export default {
- reporter: [['list'], ['./skip-pause.js']]
- };
- `,
'a.spec.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
@@ -186,11 +168,9 @@ test('should ignore globalTimeout when debugging', {
annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/34911' },
}, async ({ runInlineTest }) => {
const result = await runInlineTest({
- 'skip-pause.js': `export default ${TestPauseSkipReporter}`,
'playwright.config.ts': `
export default {
globalTimeout: 100,
- reporter: [['list'], ['./skip-pause.js']]
};
`,
'a.spec.ts': `