Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore(line): report pause (#38717)
Simon Knottce4f0773d405
diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.tsindex adba12d73c..bafd9ecc8d 100644--- a/packages/playwright/src/common/config.ts+++ b/packages/playwright/src/common/config.ts@@ -118,7 +118,7 @@ export class FullConfigInternal {updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'),updateSourceMethod: takeFirst(configCLIOverrides.updateSourceMethod, userConfig.updateSourceMethod, 'patch'),version: require('../../package.json').version,- workers: resolveWorkers(takeFirst(configCLIOverrides.debug ? 1 : undefined, configCLIOverrides.workers, userConfig.workers, '50%')),+ workers: resolveWorkers(takeFirst(configCLIOverrides.pause ? 1 : undefined, configCLIOverrides.workers, userConfig.workers, '50%')),webServer: null,};for (const key in userConfig) {diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.tsindex 8c20d80e97..c6aa1b0a71 100644--- a/packages/playwright/src/common/ipc.ts+++ b/packages/playwright/src/common/ipc.ts@@ -31,6 +31,7 @@ export type ConfigCLIOverrides = {maxFailures?: number;outputDir?: string;preserveOutputDir?: boolean;+ pause?: boolean;quiet?: boolean;repeatEach?: number;retries?: number;@@ -91,6 +92,7 @@ export type TestInfoErrorImpl = TestInfoError;export type TestPausedPayload = {testId: string;errors: TestInfoErrorImpl[];+ status: TestStatus;};export type ResumePayload = {};diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex c48a605a8f..0df97a777c 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -308,6 +308,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverridupdateSourceMethod: options.updateSourceMethod,runAgents: options.runAgents,workers: options.workers,+ pause: options.pause ? true : undefined,};if (options.browser) {@@ -323,10 +324,11 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid});}- if (options.headed || options.debug)+ if (options.headed || options.debug || overrides.pause)overrides.use = { headless: false };if (!options.ui && options.debug) {overrides.debug = true;+ overrides.pause = true;process.env.PWDEBUG = '1';}if (!options.ui && options.trace) {@@ -401,7 +403,7 @@ const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retriesconst testOptions: [string, { description: string, choices?: string[], preset?: string }][] = [/* deprecated */ ['--browser <browser>', { description: `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")` }],['-c, --config <file>', { description: `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"` }],- ['--debug', { description: `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1" options` }],+ ['--debug', { description: `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1 --pause" options` }],['--fail-on-flaky-tests', { description: `Fail if any test is flagged as flaky (default: false)` }],['--forbid-only', { description: `Fail if test.only is called (default: false)` }],['--fully-parallel', { description: `Run all tests in parallel (default: false)` }],@@ -417,6 +419,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:['--output <dir>', { description: `Folder for output artifacts (default: "test-results")` }],['--only-changed [ref]', { description: `Only run test files that have been changed between 'HEAD' and 'ref'. Defaults to running all uncommitted changes. Only supports Git.` }],['--pass-with-no-tests', { description: `Makes test run succeed even if no tests were found` }],+ ['--pause', { description: `Run tests in headed mode and pause at the end of test execution` }],['--project <project-name...>', { description: `Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects)` }],['--quiet', { description: `Suppress stdio` }],['--repeat-each <N>', { description: `Run each test N times (default: 1)` }],diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.tsindex e3fde36e37..f94ffebf29 100644--- a/packages/playwright/src/reporters/base.ts+++ b/packages/playwright/src/reporters/base.ts@@ -359,20 +359,42 @@ export class TerminalReporter implements ReporterV2 {return formatError(this.screen, error);}+ formatSingleResult(test: TestCase, result: TestResult, index?: number): string {+ return formatSingleResult(this.screen, this.config, test, result, index);+ }+writeLine(line?: string) {this.screen.stdout?.write(line ? line + '\n' : '\n');}}+function formatSingleResult(screen: Screen, config: FullConfig, test: TestCase, result: TestResult, index?: number): string {+ const lines: string[] = [];+ const header = formatTestHeader(screen, config, test, { indent: ' ', index });+ lines.push(test.outcome() === 'unexpected' ? screen.colors.red(header) : screen.colors.yellow(header));+ if (test.outcome() === 'unexpected') {+ const errorDetails = formatResultFailure(screen, test, result, ' ');+ if (errorDetails.length > 0)+ lines.push('');+ for (const error of errorDetails)+ lines.push(error.message, '');+ }+ return lines.join('\n');+}+export function formatFailure(screen: Screen, config: FullConfig, test: TestCase, index?: number, options?: TerminalReporterOptions): string {const lines: string[] = [];- const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error', includeTestId: options?.includeTestId });- lines.push(screen.colors.red(header));+ let printedHeader = false;for (const result of test.results) {const resultLines: string[] = [];const errors = formatResultFailure(screen, test, result, ' ');if (!errors.length)continue;+ if (!printedHeader) {+ const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error', includeTestId: options?.includeTestId });+ lines.push(screen.colors.red(header));+ printedHeader = true;+ }if (result.retry) {resultLines.push('');resultLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`)));@@ -455,6 +477,12 @@ function quotePathIfNeeded(path: string): string {return path;}+const kReportedSymbol = Symbol('reported');++export function markErrorsAsReported(result: TestResult) {+ (result as any)[kReportedSymbol] = result.errors.length;+}+export function formatResultFailure(screen: Screen, test: TestCase, result: TestResult, initialIndent: string): ErrorDetails[] {const errorDetails: ErrorDetails[] = [];@@ -469,7 +497,8 @@ export function formatResultFailure(screen: Screen, test: TestCase, result: Test});}- for (const error of result.errors) {+ const reportedIndex = (result as any)[kReportedSymbol] || 0;+ for (const error of result.errors.slice(reportedIndex)) {const formattedError = formatError(screen, error);errorDetails.push({message: indent(formattedError.message, initialIndent),diff --git a/packages/playwright/src/reporters/line.ts b/packages/playwright/src/reporters/line.tsindex 3c34f22308..4bcda258f6 100644--- a/packages/playwright/src/reporters/line.ts+++ b/packages/playwright/src/reporters/line.ts@@ -14,7 +14,7 @@* limitations under the License.*/-import { TerminalReporter } from './base';+import { markErrorsAsReported, TerminalReporter } from './base';import type { FullResult, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';@@ -78,6 +78,23 @@ class LineReporter extends TerminalReporter {this._updateLine(test, result, step.parent);}+ async onTestPaused(test: TestCase, result: TestResult) {+ // without TTY, user cannot interrupt the pause. let's skip it.+ if (!process.stdin.isTTY && !process.env.PW_TEST_DEBUG_REPORTERS)+ return;++ if (!process.env.PW_TEST_DEBUG_REPORTERS)+ this.screen.stdout.write(`\u001B[1A\u001B[2K`);++ this.writeLine(this.formatSingleResult(test, result, test.outcome() === 'unexpected' ? ++this._failures : undefined));+ markErrorsAsReported(result);+ this.writeLine(this.screen.colors.yellow(` Paused ${test.outcome() === 'unexpected' ? 'on error' : 'at test end'}. Press Ctrl+C to end.`) + '\n\n');++ this._updateLine(test, result, undefined);++ await new Promise<void>(() => {});+ }+override onTestEnd(test: TestCase, result: TestResult) {super.onTestEnd(test, result);if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected' || result.status === 'interrupted')) {diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex dd8defedda..db5873fec3 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -615,6 +615,7 @@ class JobDispatcher {}};+ result.status = params.status;result.errors = params.errors;result.error = result.errors[0];diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.tsindex c75f75a815..564347ac78 100644--- a/packages/playwright/src/runner/testRunner.ts+++ b/packages/playwright/src/runner/testRunner.ts@@ -482,7 +482,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promise...createRunTestsTasks(config),];- const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.debug, pauseOnError: config.configCLIOverrides.debug });+ const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.pause, pauseOnError: config.configCLIOverrides.pause });const status = await runTasks(testRun, tasks, config.config.globalTimeout);// Calling process.exit() might truncate large stdout/stderr output.diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.tsindex 4d99313c05..db8a4d6e2e 100644--- a/packages/playwright/src/worker/testInfo.ts+++ b/packages/playwright/src/worker/testInfo.ts@@ -474,7 +474,7 @@ export class TestInfoImpl implements TestInfo {const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure());if (shouldPause) {await Promise.race([- this._callbacks.onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] }),+ this._callbacks.onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [], status: this.status }),this._interruptedPromise,]);}diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.tsindex 6e47a38fa3..60f2775864 100644--- a/tests/playwright-test/pause-at-end.spec.ts+++ b/tests/playwright-test/pause-at-end.spec.ts@@ -79,7 +79,7 @@ test('--debug should pause at end', async ({ runInlineTest }) => {console.log('%%teardown');});`- }, { debug: true });+ }, { pause: true });expect(result.outputLines).toEqual(['onTestPaused at end','teardown',@@ -110,7 +110,7 @@ test('--debug should pause at end with setup project', async ({ runInlineTest })console.log('main test started');});`- }, { debug: true });+ }, { pause: true });expect(result.outputLines).toContain('onTestPaused at end');});@@ -128,7 +128,7 @@ test('--debug should pause on error', async ({ runInlineTest, mergeReports }) =>console.log('%%after error');});`- }, { debug: true });+ }, { pause: true });expect(result.outputLines).toEqual(['onTestPaused on error at :4:24','result.errors[0] at :4:24',@@ -159,7 +159,7 @@ test('SIGINT after pause at end should still run teardown', async ({ runInlineTeconsole.log('%%teardown');});`- }, { debug: true }, { SIGINT_AFTER_PAUSE: '1' });+ }, { pause: true }, { SIGINT_AFTER_PAUSE: '1' });expect(result.outputLines).toEqual(['onTestPaused at end','SIGINT',@@ -185,7 +185,7 @@ test('SIGINT after pause on error should still run teardown', async ({ runInlineconsole.log('%%teardown');});`- }, { debug: true }, { SIGINT_AFTER_PAUSE: '1' });+ }, { pause: true }, { SIGINT_AFTER_PAUSE: '1' });expect(result.outputLines).toEqual(['onTestPaused on error at :4:19','result.errors[0] at :4:19',diff --git a/tests/playwright-test/reporter-line.spec.ts b/tests/playwright-test/reporter-line.spec.tsindex cd6fc6a65c..43c42c7081 100644--- a/tests/playwright-test/reporter-line.spec.ts+++ b/tests/playwright-test/reporter-line.spec.ts@@ -15,7 +15,7 @@*/import path from 'path';-import { test, expect } from './playwright-test-fixtures';+import { test, expect, stripAnsi } from './playwright-test-fixtures';for (const useIntermediateMergeReport of [false, true] as const) {test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {@@ -225,3 +225,151 @@ for (const useIntermediateMergeReport of [false, true] as const) {});});}++test.describe('onTestPaused', () => {+ test.skip(process.platform === 'win32', 'No SIGINT on windows');++ test('pause at end', async ({ interactWithTestRunner }) => {+ const runner = await interactWithTestRunner({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('foo', async ({}) => {+ });++ test.afterEach(() => {+ console.log('Running teardown');+ });+ `,+ }, { pause: true, reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' });++ await runner.waitForOutput('Paused at test end. Press Ctrl+C to end.');+ const { exitCode } = await runner.kill('SIGINT');+ expect(exitCode).toBe(130);++ expect(stripAnsi(runner.output)).toEqual(`+Running 1 test using 1 worker++[1/1] a.test.ts:3:13 › foo+ a.test.ts:3:13 › foo ─────────────────────────────────────────────────────────────────────────────+ Paused at test end. Press Ctrl+C to end.+++[1/1] a.test.ts:3:13 › foo+a.test.ts:3:13 › foo+Running teardown++ 1) a.test.ts:3:13 › foo ──────────────────────────────────────────────────────────────────────────++ Test was interrupted.+++ 1 interrupted+ a.test.ts:3:13 › foo ───────────────────────────────────────────────────────────────────────────+`);+ });++ test('pause at end - error in teardown', async ({ interactWithTestRunner }) => {+ const runner = await interactWithTestRunner({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('foo', async ({}) => {+ });++ test.afterEach(() => {+ throw new Error('teardown error');+ });+ `,+ }, { pause: true, reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' });++ await runner.waitForOutput('Paused at test end. Press Ctrl+C to end.');+ const { exitCode } = await runner.kill('SIGINT');+ expect(exitCode).toBe(130);++ expect(stripAnsi(runner.output)).toEqual(`+Running 1 test using 1 worker++[1/1] a.test.ts:3:13 › foo+ a.test.ts:3:13 › foo ─────────────────────────────────────────────────────────────────────────────+ Paused at test end. Press Ctrl+C to end.+++[1/1] a.test.ts:3:13 › foo+ 1) a.test.ts:3:13 › foo ──────────────────────────────────────────────────────────────────────────++ Test was interrupted.++ Error: teardown error++ 5 |+ 6 | test.afterEach(() => {+ > 7 | throw new Error('teardown error');+ | ^+ 8 | });+ 9 |+ at ${test.info().outputPath('a.test.ts')}:7:17+++ 1 interrupted+ a.test.ts:3:13 › foo ───────────────────────────────────────────────────────────────────────────+`);+ });++ test('pause on error', async ({ interactWithTestRunner }) => {+ const runner = await interactWithTestRunner({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('fails', async ({}) => {+ expect.soft(2).toBe(3);+ expect(3).toBe(4);+ });+ `,+ }, { pause: true, reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' });++ await runner.waitForOutput('Paused on error. Press Ctrl+C to end.');+ const { exitCode } = await runner.kill('SIGINT');+ expect(exitCode).toBe(130);++ expect(stripAnsi(runner.output)).toEqual(`+Running 1 test using 1 worker++[1/1] a.test.ts:3:13 › fails+ 1) a.test.ts:3:13 › fails ────────────────────────────────────────────────────────────────────────++ Error: expect(received).toBe(expected) // Object.is equality++ Expected: 3+ Received: 2++ 2 | import { test, expect } from '@playwright/test';+ 3 | test('fails', async ({}) => {+ > 4 | expect.soft(2).toBe(3);+ | ^+ 5 | expect(3).toBe(4);+ 6 | });+ 7 |+ at ${test.info().outputPath('a.test.ts')}:4:26++ Error: expect(received).toBe(expected) // Object.is equality++ Expected: 4+ Received: 3++ 3 | test('fails', async ({}) => {+ 4 | expect.soft(2).toBe(3);+ > 5 | expect(3).toBe(4);+ | ^+ 6 | });+ 7 |+ at ${test.info().outputPath('a.test.ts')}:5:21++ Paused on error. Press Ctrl+C to end.+++[1/1] a.test.ts:3:13 › fails+++ 1 failed+ a.test.ts:3:13 › fails ─────────────────────────────────────────────────────────────────────────+`);+ });+});