Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(line): report pause (#38717)

Simon Knottce4f0773d405
diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts
index adba12d73c..bafd9ecc8d 100644
--- a/packages/playwright/src/common/config.ts
+++ b/packages/playwright/src/common/config.ts
@@ -118,7 +118,7 @@ export class FullConfigInternal {
updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'),
updateSourceMethod: takeFirst(configCLIOverrides.updateSourceMethod, userConfig.updateSourceMethod, 'patch'),
version: require('../../package.json').version,
- workers: resolveWorkers(takeFirst(configCLIOverrides.debug ? 1 : undefined, configCLIOverrides.workers, userConfig.workers, '50%')),
+ workers: resolveWorkers(takeFirst(configCLIOverrides.pause ? 1 : undefined, configCLIOverrides.workers, userConfig.workers, '50%')),
webServer: null,
};
for (const key in userConfig) {
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index 8c20d80e97..c6aa1b0a71 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -31,6 +31,7 @@ export type ConfigCLIOverrides = {
maxFailures?: number;
outputDir?: string;
preserveOutputDir?: boolean;
+ pause?: boolean;
quiet?: boolean;
repeatEach?: number;
retries?: number;
@@ -91,6 +92,7 @@ export type TestInfoErrorImpl = TestInfoError;
export type TestPausedPayload = {
testId: string;
errors: TestInfoErrorImpl[];
+ status: TestStatus;
};
export type ResumePayload = {};
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index c48a605a8f..0df97a777c 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -308,6 +308,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
updateSourceMethod: options.updateSourceMethod,
runAgents: options.runAgents,
workers: options.workers,
+ pause: options.pause ? true : undefined,
};
if (options.browser) {
@@ -323,10 +324,11 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
});
}
- if (options.headed || options.debug)
+ if (options.headed || options.debug || overrides.pause)
overrides.use = { headless: false };
if (!options.ui && options.debug) {
overrides.debug = true;
+ overrides.pause = true;
process.env.PWDEBUG = '1';
}
if (!options.ui && options.trace) {
@@ -401,7 +403,7 @@ const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries
const testOptions: [string, { description: string, choices?: string[], preset?: string }][] = [
/* deprecated */ ['--browser <browser>', { description: `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")` }],
['-c, --config <file>', { description: `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"` }],
- ['--debug', { description: `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1" options` }],
+ ['--debug', { description: `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1 --pause" options` }],
['--fail-on-flaky-tests', { description: `Fail if any test is flagged as flaky (default: false)` }],
['--forbid-only', { description: `Fail if test.only is called (default: false)` }],
['--fully-parallel', { description: `Run all tests in parallel (default: false)` }],
@@ -417,6 +419,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
['--output <dir>', { description: `Folder for output artifacts (default: "test-results")` }],
['--only-changed [ref]', { description: `Only run test files that have been changed between 'HEAD' and 'ref'. Defaults to running all uncommitted changes. Only supports Git.` }],
['--pass-with-no-tests', { description: `Makes test run succeed even if no tests were found` }],
+ ['--pause', { description: `Run tests in headed mode and pause at the end of test execution` }],
['--project <project-name...>', { description: `Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects)` }],
['--quiet', { description: `Suppress stdio` }],
['--repeat-each <N>', { description: `Run each test N times (default: 1)` }],
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts
index e3fde36e37..f94ffebf29 100644
--- a/packages/playwright/src/reporters/base.ts
+++ b/packages/playwright/src/reporters/base.ts
@@ -359,20 +359,42 @@ export class TerminalReporter implements ReporterV2 {
return formatError(this.screen, error);
}
+ formatSingleResult(test: TestCase, result: TestResult, index?: number): string {
+ return formatSingleResult(this.screen, this.config, test, result, index);
+ }
+
writeLine(line?: string) {
this.screen.stdout?.write(line ? line + '\n' : '\n');
}
}
+function formatSingleResult(screen: Screen, config: FullConfig, test: TestCase, result: TestResult, index?: number): string {
+ const lines: string[] = [];
+ const header = formatTestHeader(screen, config, test, { indent: ' ', index });
+ lines.push(test.outcome() === 'unexpected' ? screen.colors.red(header) : screen.colors.yellow(header));
+ if (test.outcome() === 'unexpected') {
+ const errorDetails = formatResultFailure(screen, test, result, ' ');
+ if (errorDetails.length > 0)
+ lines.push('');
+ for (const error of errorDetails)
+ lines.push(error.message, '');
+ }
+ return lines.join('\n');
+}
+
export function formatFailure(screen: Screen, config: FullConfig, test: TestCase, index?: number, options?: TerminalReporterOptions): string {
const lines: string[] = [];
- const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error', includeTestId: options?.includeTestId });
- lines.push(screen.colors.red(header));
+ let printedHeader = false;
for (const result of test.results) {
const resultLines: string[] = [];
const errors = formatResultFailure(screen, test, result, ' ');
if (!errors.length)
continue;
+ if (!printedHeader) {
+ const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error', includeTestId: options?.includeTestId });
+ lines.push(screen.colors.red(header));
+ printedHeader = true;
+ }
if (result.retry) {
resultLines.push('');
resultLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`)));
@@ -455,6 +477,12 @@ function quotePathIfNeeded(path: string): string {
return path;
}
+const kReportedSymbol = Symbol('reported');
+
+export function markErrorsAsReported(result: TestResult) {
+ (result as any)[kReportedSymbol] = result.errors.length;
+}
+
export function formatResultFailure(screen: Screen, test: TestCase, result: TestResult, initialIndent: string): ErrorDetails[] {
const errorDetails: ErrorDetails[] = [];
@@ -469,7 +497,8 @@ export function formatResultFailure(screen: Screen, test: TestCase, result: Test
});
}
- for (const error of result.errors) {
+ const reportedIndex = (result as any)[kReportedSymbol] || 0;
+ for (const error of result.errors.slice(reportedIndex)) {
const formattedError = formatError(screen, error);
errorDetails.push({
message: indent(formattedError.message, initialIndent),
diff --git a/packages/playwright/src/reporters/line.ts b/packages/playwright/src/reporters/line.ts
index 3c34f22308..4bcda258f6 100644
--- a/packages/playwright/src/reporters/line.ts
+++ b/packages/playwright/src/reporters/line.ts
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-import { TerminalReporter } from './base';
+import { markErrorsAsReported, TerminalReporter } from './base';
import type { FullResult, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';
@@ -78,6 +78,23 @@ class LineReporter extends TerminalReporter {
this._updateLine(test, result, step.parent);
}
+ async onTestPaused(test: TestCase, result: TestResult) {
+ // without TTY, user cannot interrupt the pause. let's skip it.
+ if (!process.stdin.isTTY && !process.env.PW_TEST_DEBUG_REPORTERS)
+ return;
+
+ if (!process.env.PW_TEST_DEBUG_REPORTERS)
+ this.screen.stdout.write(`\u001B[1A\u001B[2K`);
+
+ this.writeLine(this.formatSingleResult(test, result, test.outcome() === 'unexpected' ? ++this._failures : undefined));
+ markErrorsAsReported(result);
+ this.writeLine(this.screen.colors.yellow(` Paused ${test.outcome() === 'unexpected' ? 'on error' : 'at test end'}. Press Ctrl+C to end.`) + '\n\n');
+
+ this._updateLine(test, result, undefined);
+
+ await new Promise<void>(() => {});
+ }
+
override onTestEnd(test: TestCase, result: TestResult) {
super.onTestEnd(test, result);
if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected' || result.status === 'interrupted')) {
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index dd8defedda..db5873fec3 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -615,6 +615,7 @@ class JobDispatcher {
}
};
+ result.status = params.status;
result.errors = params.errors;
result.error = result.errors[0];
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index c75f75a815..564347ac78 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -482,7 +482,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promise
...createRunTestsTasks(config),
];
- const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.debug, pauseOnError: config.configCLIOverrides.debug });
+ const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.pause, pauseOnError: config.configCLIOverrides.pause });
const status = await runTasks(testRun, tasks, config.config.globalTimeout);
// Calling process.exit() might truncate large stdout/stderr output.
diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts
index 4d99313c05..db8a4d6e2e 100644
--- a/packages/playwright/src/worker/testInfo.ts
+++ b/packages/playwright/src/worker/testInfo.ts
@@ -474,7 +474,7 @@ export class TestInfoImpl implements TestInfo {
const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure());
if (shouldPause) {
await Promise.race([
- this._callbacks.onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] }),
+ this._callbacks.onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [], status: this.status }),
this._interruptedPromise,
]);
}
diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts
index 6e47a38fa3..60f2775864 100644
--- a/tests/playwright-test/pause-at-end.spec.ts
+++ b/tests/playwright-test/pause-at-end.spec.ts
@@ -79,7 +79,7 @@ test('--debug should pause at end', async ({ runInlineTest }) => {
console.log('%%teardown');
});
`
- }, { debug: true });
+ }, { pause: true });
expect(result.outputLines).toEqual([
'onTestPaused at end',
'teardown',
@@ -110,7 +110,7 @@ test('--debug should pause at end with setup project', async ({ runInlineTest })
console.log('main test started');
});
`
- }, { debug: true });
+ }, { pause: true });
expect(result.outputLines).toContain('onTestPaused at end');
});
@@ -128,7 +128,7 @@ test('--debug should pause on error', async ({ runInlineTest, mergeReports }) =>
console.log('%%after error');
});
`
- }, { debug: true });
+ }, { pause: true });
expect(result.outputLines).toEqual([
'onTestPaused on error at :4:24',
'result.errors[0] at :4:24',
@@ -159,7 +159,7 @@ test('SIGINT after pause at end should still run teardown', async ({ runInlineTe
console.log('%%teardown');
});
`
- }, { debug: true }, { SIGINT_AFTER_PAUSE: '1' });
+ }, { pause: true }, { SIGINT_AFTER_PAUSE: '1' });
expect(result.outputLines).toEqual([
'onTestPaused at end',
'SIGINT',
@@ -185,7 +185,7 @@ test('SIGINT after pause on error should still run teardown', async ({ runInline
console.log('%%teardown');
});
`
- }, { debug: true }, { SIGINT_AFTER_PAUSE: '1' });
+ }, { pause: true }, { SIGINT_AFTER_PAUSE: '1' });
expect(result.outputLines).toEqual([
'onTestPaused on error at :4:19',
'result.errors[0] at :4:19',
diff --git a/tests/playwright-test/reporter-line.spec.ts b/tests/playwright-test/reporter-line.spec.ts
index cd6fc6a65c..43c42c7081 100644
--- a/tests/playwright-test/reporter-line.spec.ts
+++ b/tests/playwright-test/reporter-line.spec.ts
@@ -15,7 +15,7 @@
*/
import path from 'path';
-import { test, expect } from './playwright-test-fixtures';
+import { test, expect, stripAnsi } from './playwright-test-fixtures';
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
@@ -225,3 +225,151 @@ for (const useIntermediateMergeReport of [false, true] as const) {
});
});
}
+
+test.describe('onTestPaused', () => {
+ test.skip(process.platform === 'win32', 'No SIGINT on windows');
+
+ test('pause at end', async ({ interactWithTestRunner }) => {
+ const runner = await interactWithTestRunner({
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('foo', async ({}) => {
+ });
+
+ test.afterEach(() => {
+ console.log('Running teardown');
+ });
+ `,
+ }, { pause: true, reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' });
+
+ await runner.waitForOutput('Paused at test end. Press Ctrl+C to end.');
+ const { exitCode } = await runner.kill('SIGINT');
+ expect(exitCode).toBe(130);
+
+ expect(stripAnsi(runner.output)).toEqual(`
+Running 1 test using 1 worker
+
+[1/1] a.test.ts:3:13 › foo
+ a.test.ts:3:13 › foo ─────────────────────────────────────────────────────────────────────────────
+ Paused at test end. Press Ctrl+C to end.
+
+
+[1/1] a.test.ts:3:13 › foo
+a.test.ts:3:13 › foo
+Running teardown
+
+ 1) a.test.ts:3:13 › foo ──────────────────────────────────────────────────────────────────────────
+
+ Test was interrupted.
+
+
+ 1 interrupted
+ a.test.ts:3:13 › foo ───────────────────────────────────────────────────────────────────────────
+`);
+ });
+
+ test('pause at end - error in teardown', async ({ interactWithTestRunner }) => {
+ const runner = await interactWithTestRunner({
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('foo', async ({}) => {
+ });
+
+ test.afterEach(() => {
+ throw new Error('teardown error');
+ });
+ `,
+ }, { pause: true, reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' });
+
+ await runner.waitForOutput('Paused at test end. Press Ctrl+C to end.');
+ const { exitCode } = await runner.kill('SIGINT');
+ expect(exitCode).toBe(130);
+
+ expect(stripAnsi(runner.output)).toEqual(`
+Running 1 test using 1 worker
+
+[1/1] a.test.ts:3:13 › foo
+ a.test.ts:3:13 › foo ─────────────────────────────────────────────────────────────────────────────
+ Paused at test end. Press Ctrl+C to end.
+
+
+[1/1] a.test.ts:3:13 › foo
+ 1) a.test.ts:3:13 › foo ──────────────────────────────────────────────────────────────────────────
+
+ Test was interrupted.
+
+ Error: teardown error
+
+ 5 |
+ 6 | test.afterEach(() => {
+ > 7 | throw new Error('teardown error');
+ | ^
+ 8 | });
+ 9 |
+ at ${test.info().outputPath('a.test.ts')}:7:17
+
+
+ 1 interrupted
+ a.test.ts:3:13 › foo ───────────────────────────────────────────────────────────────────────────
+`);
+ });
+
+ test('pause on error', async ({ interactWithTestRunner }) => {
+ const runner = await interactWithTestRunner({
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('fails', async ({}) => {
+ expect.soft(2).toBe(3);
+ expect(3).toBe(4);
+ });
+ `,
+ }, { pause: true, reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' });
+
+ await runner.waitForOutput('Paused on error. Press Ctrl+C to end.');
+ const { exitCode } = await runner.kill('SIGINT');
+ expect(exitCode).toBe(130);
+
+ expect(stripAnsi(runner.output)).toEqual(`
+Running 1 test using 1 worker
+
+[1/1] a.test.ts:3:13 › fails
+ 1) a.test.ts:3:13 › fails ────────────────────────────────────────────────────────────────────────
+
+ Error: expect(received).toBe(expected) // Object.is equality
+
+ Expected: 3
+ Received: 2
+
+ 2 | import { test, expect } from '@playwright/test';
+ 3 | test('fails', async ({}) => {
+ > 4 | expect.soft(2).toBe(3);
+ | ^
+ 5 | expect(3).toBe(4);
+ 6 | });
+ 7 |
+ at ${test.info().outputPath('a.test.ts')}:4:26
+
+ Error: expect(received).toBe(expected) // Object.is equality
+
+ Expected: 4
+ Received: 3
+
+ 3 | test('fails', async ({}) => {
+ 4 | expect.soft(2).toBe(3);
+ > 5 | expect(3).toBe(4);
+ | ^
+ 6 | });
+ 7 |
+ at ${test.info().outputPath('a.test.ts')}:5:21
+
+ Paused on error. Press Ctrl+C to end.
+
+
+[1/1] a.test.ts:3:13 › fails
+
+
+ 1 failed
+ a.test.ts:3:13 › fails ─────────────────────────────────────────────────────────────────────────
+`);
+ });
+});