Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: simplify reporter `onTestPaused` (#38679)
Simon Knottaa35f26b3f49
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.tsindex b22fdc036f..8c20d80e97 100644--- a/packages/playwright/src/common/ipc.ts+++ b/packages/playwright/src/common/ipc.ts@@ -90,13 +90,10 @@ export type TestInfoErrorImpl = TestInfoError;export type TestPausedPayload = {testId: string;- stepId: string;errors: TestInfoErrorImpl[];};-export type ResumePayload = {- action?: 'continue' | 'abort';-};+export type ResumePayload = {};export type CloneStoragePayload = {storageFile: string;diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.tsindex 657664199e..10c967d33c 100644--- a/packages/playwright/src/isomorphic/teleReceiver.ts+++ b/packages/playwright/src/isomorphic/teleReceiver.ts@@ -172,7 +172,6 @@ export type JsonOnTestPausedEvent = {params: {testId: string;resultId: string;- stepId: string;errors: reporterTypes.TestError[];};};@@ -294,7 +293,7 @@ export class TeleReporterReceiver {return;}if (method === 'onTestPaused') {- this._onTestPaused(params.testId, params.resultId, params.stepId, params.errors);+ this._onTestPaused(params.testId, params.resultId, params.errors);return;}if (method === 'onTestEnd') {@@ -362,14 +361,13 @@ export class TeleReporterReceiver {this._reporter.onTestBegin?.(test, testResult);}- private _onTestPaused(testId: string, resultId: string, stepId: string, errors: reporterTypes.TestError[]) {+ private _onTestPaused(testId: string, resultId: string, errors: reporterTypes.TestError[]) {const test = this._tests.get(testId)!;const result = test.results.find(r => r._id === resultId)!;- const step = result._stepMap.get(stepId)!;result.errors.push(...errors);result.error = result.errors[0];- void this._reporter.onTestPaused?.(test, result, step);+ void this._reporter.onTestPaused?.(test, result);}private _onTestEnd(testEndPayload: JsonTestEnd, payload: JsonTestResultEnd) {diff --git a/packages/playwright/src/mcp/test/testContext.ts b/packages/playwright/src/mcp/test/testContext.tsindex 4d5a8f3473..16fce05c55 100644--- a/packages/playwright/src/mcp/test/testContext.ts+++ b/packages/playwright/src/mcp/test/testContext.ts@@ -205,7 +205,7 @@ export class TestContext {claimStdio();try {- const setupReporter = new ListReporter({ configDir, screen, includeTestId: true });+ const setupReporter = new MCPListReporter({ configDir, screen, includeTestId: true });const { status } = await testRunner.runGlobalSetup([setupReporter]);if (status !== 'passed')return { output: testRunnerAndScreen.output.join('\n'), status };@@ -227,7 +227,7 @@ export class TestContext {};try {- const reporter = new ListReporter({ configDir, screen, includeTestId: true });+ const reporter = new MCPListReporter({ configDir, screen, includeTestId: true });status = await Promise.race([testRunner.runTests(reporter, params).then(result => result.status),testRunnerAndScreen.waitForTestPaused().then(() => 'paused' as const),@@ -320,3 +320,10 @@ const bestPracticesMarkdown = `- NEVER! use page.waitForTimeout()- NEVER! use page.evaluate()`;++class MCPListReporter extends ListReporter {+ async onTestPaused() {+ // ListReporter waits for user input to resume, we don't want that in MCP.+ await new Promise<void>(() => {});+ }+}diff --git a/packages/playwright/src/reporters/blob.ts b/packages/playwright/src/reporters/blob.tsindex 324a19345d..50884a0ee9 100644--- a/packages/playwright/src/reporters/blob.ts+++ b/packages/playwright/src/reporters/blob.ts@@ -27,7 +27,7 @@ import { resolveOutputFile, CommonReporterOptions } from './base';import { TeleReporterEmitter } from './teleEmitter';import type { BlobReporterOptions } from '../../types/test';-import type { FullConfig, FullResult, TestCase, TestResult, TestStep } from '../../types/testReporter';+import type { FullConfig, FullResult, TestCase, TestResult } from '../../types/testReporter';import type { BlobReportMetadata, JsonAttachment, JsonEvent } from '../isomorphic/teleReceiver';import type { EventEmitter } from 'events';@@ -66,10 +66,9 @@ export class BlobReporter extends TeleReporterEmitter {super.onConfigure(config);}- override async onTestPaused(test: TestCase, result: TestResult, step: TestStep) {+ override async onTestPaused(test: TestCase, result: TestResult) {// onTestPaused is only relevant for interactive use, not for blob replays.// merge-reports still gets onStepBegin/onStepEnd for pausing, but not the interactive part, so this is a no-op.- return { action: undefined };}override async onEnd(result: FullResult): Promise<void> {diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.tsindex 147a837e52..7b0ac95b82 100644--- a/packages/playwright/src/reporters/internalReporter.ts+++ b/packages/playwright/src/reporters/internalReporter.ts@@ -67,11 +67,9 @@ export class InternalReporter implements ReporterV2 {this._reporter.onStdErr?.(chunk, test, result);}- async onTestPaused(test: TestCase, result: TestResult, step?: TestStep) {+ async onTestPaused(test: TestCase, result: TestResult) {this._addSnippetToTestErrors(test, result);- if (step)- this._addSnippetToStepError(test, step);- return await this._reporter.onTestPaused?.(test, result, step) ?? { action: undefined };+ return await this._reporter.onTestPaused?.(test, result);}onTestEnd(test: TestCase, result: TestResult) {diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.tsindex ea5daa21a4..43864a7102 100644--- a/packages/playwright/src/reporters/multiplexer.ts+++ b/packages/playwright/src/reporters/multiplexer.ts@@ -54,14 +54,9 @@ export class Multiplexer implements ReporterV2 {wrap(() => reporter.onStdErr?.(chunk, test, result));}- async onTestPaused(test: TestCase, result: TestResult, step: TestStep) {- // only one reporter can handle the pause. first reporter that cares wins.- for (const reporter of this._reporters) {- const disposition = await wrapAsync(() => reporter.onTestPaused?.(test, result, step));- if (disposition?.action)- return disposition;- }- return { action: undefined };+ async onTestPaused(test: TestCase, result: TestResult) {+ for (const reporter of this._reporters)+ await wrapAsync(() => reporter.onTestPaused?.(test, result));}onTestEnd(test: TestCase, result: TestResult) {diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.tsindex 555ba2a6a6..d36050a31f 100644--- a/packages/playwright/src/reporters/reporterV2.ts+++ b/packages/playwright/src/reporters/reporterV2.ts@@ -22,7 +22,7 @@ export interface ReporterV2 {onTestBegin?(test: TestCase, result: TestResult): void;onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;- onTestPaused?(test: TestCase, result: TestResult, step?: TestStep): Promise<{ action?: 'continue' | 'abort' }>;+ onTestPaused?(test: TestCase, result: TestResult): Promise<void>;onTestEnd?(test: TestCase, result: TestResult): void;onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;onExit?(): void | Promise<void>;diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.tsindex 4efd401309..2b65ed5ba9 100644--- a/packages/playwright/src/reporters/teleEmitter.ts+++ b/packages/playwright/src/reporters/teleEmitter.ts@@ -72,20 +72,19 @@ export class TeleReporterEmitter implements ReporterV2 {});}- async onTestPaused(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step?: reporterTypes.TestStep) {+ async onTestPaused(test: reporterTypes.TestCase, result: reporterTypes.TestResult) {const resultId = (result as any)[this._idSymbol];- const stepId = (step as any)[this._idSymbol];this._resultKnownErrorCounts.set(resultId, result.errors.length);this._messageSink({method: 'onTestPaused',params: {testId: test.id,resultId,- stepId,errors: result.errors,}});- return { action: undefined };+ // keep the test paused until externally resumed+ await new Promise<void>(() => {});}onTestEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult): void {diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex e00ba5b459..5c9f2ae834 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -598,10 +598,7 @@ class JobDispatcher {if (!data)return;- const { result, test, steps } = data;- const step = steps.get(params.stepId);- if (!step)- return;+ const { result, test } = data;const sendMessage = async (message: { request: any }) => {try {@@ -621,8 +618,8 @@ class JobDispatcher {result.errors = params.errors;result.error = result.errors[0];- void this._reporter.onTestPaused?.(test, result, step).then(params => {- worker.sendResume(params);+ void this._reporter.onTestPaused?.(test, result).then(() => {+ worker.sendResume({});});this._failureTracker.onTestPaused?.({ ...params, sendMessage });}diff --git a/packages/playwright/src/transform/babelHighlightUtils.ts b/packages/playwright/src/transform/babelHighlightUtils.tsdeleted file mode 100644index 4611233686..0000000000--- a/packages/playwright/src/transform/babelHighlightUtils.ts+++ /dev/null@@ -1,45 +0,0 @@-/**- * Copyright (c) Microsoft Corporation.- *- * Licensed under the Apache License, Version 2.0 (the "License");- * you may not use this file except in compliance with the License.- * You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import path from 'path';-import { traverse, babelParse, T, types as t } from './babelBundle';-import type { Location } from '../../types/testReporter';--function containsLocation(range: T.SourceLocation, location: Location): boolean {- if (location.line < range.start.line || location.line > range.end.line)- return false;- if (location.line === range.start.line && location.column < range.start.column)- return false;- if (location.line === range.end.line && location.column > range.end.column)- return false;- return true;-}--export function findTestEndLocation(text: string, testStartLocation: Location): Location | undefined {- const ast = babelParse(text, path.basename(testStartLocation.file), false);- let result: Location | undefined;- traverse(ast, {- enter(path) {- if (t.isCallExpression(path.node) && path.node.loc && containsLocation(path.node.loc, testStartLocation)) {- const callNode = path.node;- const funcNode = callNode.arguments[callNode.arguments.length - 1];- if (callNode.arguments.length >= 2 && t.isFunction(funcNode) && funcNode.body.loc)- result = { file: testStartLocation.file, ...funcNode.body.loc.end };- }- }- });- return result;-}diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.tsindex 1b37fd89d5..4d99313c05 100644--- a/packages/playwright/src/worker/testInfo.ts+++ b/packages/playwright/src/worker/testInfo.ts@@ -24,7 +24,6 @@ import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndimport { TestTracing } from './testTracing';import { testInfoError } from './util';import { wrapFunctionWithLocation } from '../transform/transform';-import { findTestEndLocation } from '../transform/babelHighlightUtils';import type { RunnableDescription } from './timeoutManager';import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test';@@ -474,35 +473,14 @@ export class TestInfoImpl implements TestInfo {async _didFinishTestFunction() {const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure());if (shouldPause) {- const location = (this._isFailure() ? this._errorLocation() : await this._testEndLocation()) ?? { file: this.file, line: this.line, column: this.column };- const step = this._addStep({ category: 'hook', title: 'Paused', location });- const result = await Promise.race([- this._callbacks.onTestPaused({ testId: this.testId, stepId: step.stepId, errors: this._isFailure() ? this.errors : [] }),- this._interruptedPromise.then(() => 'interrupted' as const),+ await Promise.race([+ this._callbacks.onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] }),+ this._interruptedPromise,]);- if (result !== 'interrupted') {- if (result.action === 'abort')- this._interrupt();- if (result.action === undefined)- await this._interruptedPromise;- }- step.complete({});}await this._onDidFinishTestFunctionCallback?.();}- private _errorLocation(): Location | undefined {- if (this.error?.stack)- return filteredStackTrace(this.error.stack.split('\n'))[0];- }-- private async _testEndLocation(): Promise<Location | undefined> {- try {- const source = await fs.promises.readFile(this.file, 'utf-8');- return findTestEndLocation(source, { file: this.file, line: this.line, column: this.column });- } catch {}- }-// ------------ TestInfo methods ------------async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) {diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.tsindex f590b84c7f..6e47a38fa3 100644--- a/tests/playwright-test/pause-at-end.spec.ts+++ b/tests/playwright-test/pause-at-end.spec.ts@@ -18,6 +18,8 @@ import { TestCase, TestResult, TestStep } from 'packages/playwright-test/reporteimport { ReporterV2 } from 'packages/playwright/src/reporters/reporterV2';import { test, expect } from './playwright-test-fixtures';+test.describe.configure({ mode: 'parallel' });+class LocationReporter implements ReporterV2 {private _foundErrors = 0;version() {@@ -29,10 +31,17 @@ class LocationReporter implements ReporterV2 {this._printErrors(result);}}- async onTestPaused(test: TestCase, result: TestResult, step?: TestStep) {- console.log(`%%onTestPaused`);+ async onTestPaused(test: TestCase, result: TestResult) {+ if (result.error)+ console.log(`%%onTestPaused on error at :${result.error.location?.line}:${result.error.location?.column}`);+ else+ console.log(`%%onTestPaused at end`);this._printErrors(result);- return { action: process.env.ACTION as 'continue' | 'abort' };+ if (process.env.SIGINT_AFTER_PAUSE) {+ console.log('%%SIGINT');+ process.kill(process.pid, 'SIGINT');+ await new Promise(() => {});+ }}onStepEnd(test: TestCase, result: TestResult, step: TestStep): void {if (step.title === 'Paused') {@@ -70,11 +79,9 @@ test('--debug should pause at end', async ({ runInlineTest }) => {console.log('%%teardown');});`- }, { debug: true }, { ACTION: 'continue' });+ }, { debug: true });expect(result.outputLines).toEqual([- 'onStepBegin: After Hooks > Paused at :4:7',- 'onTestPaused',- 'onStepEnd: After Hooks > Paused',+ 'onTestPaused at end','teardown','onTestEnd',]);@@ -103,15 +110,15 @@ test('--debug should pause at end with setup project', async ({ runInlineTest })console.log('main test started');});`- }, { debug: true }, { ACTION: 'continue' });- expect(result.outputLines).toContain('onStepBegin: After Hooks > Paused at :5:7');+ }, { debug: true });+ expect(result.outputLines).toContain('onTestPaused at end');});test('--debug should pause on error', async ({ runInlineTest, mergeReports }) => {const result = await runInlineTest({'location-reporter.js': `export default ${LocationReporter}`,'playwright.config.js': `- module.exports = { reporter: [['blob'], ['./location-reporter.js']] };+ module.exports = { reporter: [['./location-reporter.js'], ['blob']] };`,'a.test.js': `import { test, expect } from '@playwright/test';@@ -121,22 +128,69 @@ test('--debug should pause on error', async ({ runInlineTest, mergeReports }) =>console.log('%%after error');});`- }, { debug: true }, { ACTION: 'abort' });+ }, { debug: true });expect(result.outputLines).toEqual([- 'onStepBegin: After Hooks > Paused at :4:24',- 'onTestPaused',+ 'onTestPaused on error at :4:24','result.errors[0] at :4:24','result.errors[1] at :5:19',- 'onStepEnd: After Hooks > Paused','onTestEnd',]);const merged = await mergeReports('blob-report', undefined, { additionalArgs: ['--reporter', 'location-reporter.js'] });expect(merged.outputLines, 'merge reporter doesnt get onTestPaused').toEqual([- 'onStepBegin: After Hooks > Paused at :4:24',- 'onStepEnd: After Hooks > Paused','onTestEnd','result.errors[0] at :4:24','result.errors[1] at :5:19',]);});++test('SIGINT after pause at end should still run teardown', async ({ runInlineTest }) => {+ test.skip(process.platform === 'win32', 'no SIGINT on windows');+ const result = await runInlineTest({+ 'location-reporter.js': `export default ${LocationReporter}`,+ 'playwright.config.js': `+ module.exports = { reporter: './location-reporter.js' };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('pass', () => {+ });+ test.afterEach(() => {+ console.log('%%teardown');+ });+ `+ }, { debug: true }, { SIGINT_AFTER_PAUSE: '1' });+ expect(result.outputLines).toEqual([+ 'onTestPaused at end',+ 'SIGINT',+ 'teardown',+ 'onTestEnd',+ ]);+});++test('SIGINT after pause on error should still run teardown', async ({ runInlineTest }) => {+ test.skip(process.platform === 'win32', 'no SIGINT on windows');+ const result = await runInlineTest({+ 'location-reporter.js': `export default ${LocationReporter}`,+ 'playwright.config.js': `+ module.exports = { reporter: './location-reporter.js' };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('pass', () => {+ expect(2).toBe(3);+ console.log('%%after error');+ });+ test.afterEach(() => {+ console.log('%%teardown');+ });+ `+ }, { debug: true }, { SIGINT_AFTER_PAUSE: '1' });+ expect(result.outputLines).toEqual([+ 'onTestPaused on error at :4:19',+ 'result.errors[0] at :4:19',+ 'SIGINT',+ 'teardown',+ 'onTestEnd',+ ]);+});diff --git a/tests/playwright-test/timeout.spec.ts b/tests/playwright-test/timeout.spec.tsindex 6726b677dd..9d6001078d 100644--- a/tests/playwright-test/timeout.spec.ts+++ b/tests/playwright-test/timeout.spec.ts@@ -14,7 +14,6 @@* limitations under the License.*/-import type { ReporterV2 } from 'packages/playwright/src/reporters/reporterV2';import { test, expect } from './playwright-test-fixtures';test('should run fixture teardown on timeout', async ({ runInlineTest }) => {@@ -144,25 +143,8 @@ test('should respect test.slow', async ({ runInlineTest }) => {expect(result.output).toContain('Test timeout of 1000ms exceeded.');});-// terminal reporters dont yet implement onTestPaused, so we default to continuing for these tests-class TestPauseSkipReporter implements ReporterV2 {- version() {- return 'v2' as const;- }-- async onTestPaused() {- return { action: 'continue' as const };- }-}-test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) => {const result = await runInlineTest({- 'skip-pause.js': `export default ${TestPauseSkipReporter}`,- 'playwright.config.ts': `- export default {- reporter: [['list'], ['./skip-pause.js']]- };- `,'a.spec.ts': `import { test as base, expect } from '@playwright/test';const test = base.extend({@@ -186,11 +168,9 @@ test('should ignore globalTimeout when debugging', {annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/34911' },}, async ({ runInlineTest }) => {const result = await runInlineTest({- 'skip-pause.js': `export default ${TestPauseSkipReporter}`,'playwright.config.ts': `export default {globalTimeout: 100,- reporter: [['list'], ['./skip-pause.js']]};`,'a.spec.ts': `