Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: merge FailureTracker into TestRun, pull more into TestRunOptions (#40150)

Dmitry Gozmanb82aa4973ec0
diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.ts
index 25eae98827..9e7e954ad8 100644
--- a/packages/playwright/src/cli/testActions.ts
+++ b/packages/playwright/src/cli/testActions.ts
@@ -41,6 +41,7 @@ export async function runTests(args: string[], opts: { [key: string]: any }) {
lastFailed: !!opts.lastFailed,
testList: opts.testList ? path.resolve(process.cwd(), opts.testList) : undefined,
testListInvert: opts.testListInvert ? path.resolve(process.cwd(), opts.testListInvert) : undefined,
+ shardWeights: resolveShardWeightsOption(),
};
// Evaluate project filters against config before starting execution. This enables a consistent error message across run modes
@@ -124,7 +125,6 @@ function overridesFromOptions(options: { [key: string]: any }): ipc.ConfigCLIOve
retries: options.retries ? parseInt(options.retries, 10) : undefined,
reporter: resolveReporterOption(options.reporter),
shard: resolveShardOption(options.shard),
- shardWeights: resolveShardWeightsOption(),
timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined,
ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
@@ -195,7 +195,7 @@ function resolveShardOption(shard?: string): ipc.ConfigCLIOverrides['shard'] {
return { current, total };
}
-function resolveShardWeightsOption(): ipc.ConfigCLIOverrides['shardWeights'] {
+function resolveShardWeightsOption(): TestRunOptions['shardWeights'] {
const shardWeights = process.env.PWTEST_SHARD_WEIGHTS;
if (!shardWeights)
return undefined;
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index e1774d2b91..42cc3bd6ab 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -30,15 +30,12 @@ export type ConfigCLIOverrides = {
globalTimeout?: number;
maxFailures?: number;
outputDir?: string;
- preserveOutputDir?: boolean;
pause?: boolean;
quiet?: boolean;
repeatEach?: number;
retries?: number;
reporter?: ReporterDescription[];
- additionalReporters?: ReporterDescription[];
shard?: { current: number, total: number };
- shardWeights?: number[];
timeout?: number;
tsconfig?: string;
ignoreSnapshots?: boolean;
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index b9afff3a5c..c48c009f51 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -20,16 +20,15 @@ import { eventsHelper } from '@utils/eventsHelper';
import { addSuggestedRebaseline } from './rebase';
import { WorkerHost } from './workerHost';
-import { FullConfigInternal, ipc, test as testNs } from '../common';
+import { ipc, test as testNs } from '../common';
import { addLocationAndSnippetToError } from '../reporters/internalReporter';
import { serializeError } from '../util';
import type { RegisteredListener } from '@utils/eventsHelper';
-import type { FailureTracker } from './failureTracker';
import type { ProcessExitData } from './processHost';
+import type { TestRun } from './tasks';
import type { TestGroup } from './testGroups';
import type { TestError, TestResult, TestStep } from '../../types/testReporter';
-import type { ReporterV2 } from '../reporters/reporterV2';
export type EnvByProjectId = Map<string, Record<string, string | undefined>>;
@@ -42,18 +41,14 @@ export class Dispatcher {
private _finished = new ManualPromise<void>();
private _isStopped = true;
- private _config: FullConfigInternal;
- private _reporter: ReporterV2;
- private _failureTracker: FailureTracker;
+ private _testRun: TestRun;
private _extraEnvByProjectId: EnvByProjectId = new Map();
private _producedEnvByProjectId: EnvByProjectId = new Map();
- constructor(config: FullConfigInternal, reporter: ReporterV2, failureTracker: FailureTracker) {
- this._config = config;
- this._reporter = reporter;
- this._failureTracker = failureTracker;
- for (const project of config.projects) {
+ constructor(testRun: TestRun) {
+ this._testRun = testRun;
+ for (const project of testRun.config.projects) {
if (project.workers)
this._workerLimitPerProjectId.set(project.id, project.workers);
}
@@ -97,7 +92,7 @@ export class Dispatcher {
// 3. Claim both the job and the worker slot.
this._queue.splice(jobIndex, 1);
- const jobDispatcher = new JobDispatcher(job, this._config, this._reporter, this._failureTracker, () => this.stop().catch(() => {}));
+ const jobDispatcher = new JobDispatcher(job, this._testRun, () => this.stop().catch(() => {}));
this._workerSlots[workerIndex].jobDispatcher = jobDispatcher;
// 4. Run the job. This is the only async operation.
@@ -132,7 +127,7 @@ export class Dispatcher {
// 2. Start the worker if it is down.
let startError;
if (!worker) {
- worker = this._createWorker(job, index, ipc.serializeConfig(this._config, true));
+ worker = this._createWorker(job, index, ipc.serializeConfig(this._testRun.config, true));
this._workerSlots[index].worker = worker;
worker.on('exit', () => this._workerSlots[index].worker = undefined);
startError = await worker.start();
@@ -198,10 +193,10 @@ export class Dispatcher {
this._isStopped = false;
this._workerSlots = [];
// 0. Stop right away if we have reached max failures.
- if (this._failureTracker.hasReachedMaxFailures())
+ if (this._testRun.hasReachedMaxFailures())
void this.stop();
// 1. Allocate workers.
- for (let i = 0; i < this._config.config.workers; i++)
+ for (let i = 0; i < this._testRun.config.config.workers; i++)
this._workerSlots.push({});
// 2. Schedule enough jobs.
for (let i = 0; i < this._workerSlots.length; i++)
@@ -213,15 +208,15 @@ export class Dispatcher {
}
_createWorker(testGroup: TestGroup, parallelIndex: number, loaderData: ipc.SerializedConfig) {
- const projectConfig = this._config.projects.find(p => p.id === testGroup.projectId)!;
- const outputDir = projectConfig.project.outputDir;
+ const project = this._testRun.config.projects.find(p => p.id === testGroup.projectId)!;
+ const pauseAtEnd = this._testRun.topLevelProjects.includes(project) && !!this._testRun.options.pauseAtEnd;
const worker = new WorkerHost(testGroup, {
parallelIndex,
config: loaderData,
extraEnv: this._extraEnvByProjectId.get(testGroup.projectId) || {},
- outputDir,
- pauseOnError: this._failureTracker.pauseOnError(),
- pauseAtEnd: this._failureTracker.pauseAtEnd(projectConfig),
+ outputDir: project.project.outputDir,
+ pauseOnError: !!this._testRun.options.pauseOnError,
+ pauseAtEnd,
});
const handleOutput = (params: ipc.TestOutputPayload) => {
const chunk = chunkFromParams(params);
@@ -239,17 +234,17 @@ export class Dispatcher {
worker.on('stdOut', (params: ipc.TestOutputPayload) => {
const { chunk, test, result } = handleOutput(params);
result?.stdout.push(chunk);
- this._reporter.onStdOut?.(chunk, test, result);
+ this._testRun.reporter.onStdOut?.(chunk, test, result);
});
worker.on('stdErr', (params: ipc.TestOutputPayload) => {
const { chunk, test, result } = handleOutput(params);
result?.stderr.push(chunk);
- this._reporter.onStdErr?.(chunk, test, result);
+ this._testRun.reporter.onStdErr?.(chunk, test, result);
});
worker.on('teardownErrors', (params: ipc.TeardownErrorsPayload) => {
- this._failureTracker.onWorkerError();
+ this._testRun.hasWorkerErrors = true;
for (const error of params.fatalErrors)
- this._reporter.onError?.(error);
+ this._testRun.reporter.onError?.(error);
});
worker.on('exit', () => {
const producedEnv = this._producedEnvByProjectId.get(testGroup.projectId) || {};
@@ -275,9 +270,7 @@ class JobDispatcher {
jobResult = new ManualPromise<{ newJob?: TestGroup, didFail: boolean }>();
readonly job: TestGroup;
- private _config: FullConfigInternal;
- private _reporter: ReporterV2;
- private _failureTracker: FailureTracker;
+ private _testRun: TestRun;
private _stopCallback: () => void;
private _listeners: RegisteredListener[] = [];
private _failedTests = new Set<testNs.TestCase>();
@@ -288,11 +281,9 @@ class JobDispatcher {
private _workerIndex = 0;
private _currentlyRunning: { test: testNs.TestCase, result: TestResult } | undefined;
- constructor(job: TestGroup, config: FullConfigInternal, reporter: ReporterV2, failureTracker: FailureTracker, stopCallback: () => void) {
+ constructor(job: TestGroup, testRun: TestRun, stopCallback: () => void) {
this.job = job;
- this._config = config;
- this._reporter = reporter;
- this._failureTracker = failureTracker;
+ this._testRun = testRun;
this._stopCallback = stopCallback;
this._remainingByTestId = new Map(this.job.tests.map(e => [e.id, e]));
}
@@ -308,12 +299,12 @@ class JobDispatcher {
result.parallelIndex = this._parallelIndex;
result.workerIndex = this._workerIndex;
result.startTime = new Date(params.startWallTime);
- this._reporter.onTestBegin?.(test, result);
+ this._testRun.reporter.onTestBegin?.(test, result);
this._currentlyRunning = { test, result };
}
private _onTestEnd(params: ipc.TestEndPayload) {
- if (this._failureTracker.hasReachedMaxFailures()) {
+ if (this._testRun.hasReachedMaxFailures()) {
// Do not show more than one error to avoid confusion, but report
// as interrupted to indicate that we did actually start the test.
params.status = 'interrupted';
@@ -377,7 +368,7 @@ class JobDispatcher {
};
steps.set(params.stepId, step);
(parentStep || result).steps.push(step);
- this._reporter.onStepBegin?.(test, result, step);
+ this._testRun.reporter.onStepBegin?.(test, result, step);
}
private _onStepEnd(params: ipc.StepEndPayload) {
@@ -389,7 +380,7 @@ class JobDispatcher {
const { result, steps, test } = data;
const step = steps.get(params.stepId);
if (!step) {
- this._reporter.onStdErr?.('Internal error: step end without step begin: ' + params.stepId, test, result);
+ this._testRun.reporter.onStdErr?.('Internal error: step end without step begin: ' + params.stepId, test, result);
return;
}
step.duration = params.wallTime - step.startTime.getTime();
@@ -399,7 +390,7 @@ class JobDispatcher {
addSuggestedRebaseline(step.location!, params.suggestedRebaseline);
step.annotations = params.annotations;
steps.delete(params.stepId);
- this._reporter.onStepEnd?.(test, result, step);
+ this._testRun.reporter.onStepEnd?.(test, result, step);
}
private _onAttach(params: ipc.AttachmentPayload) {
@@ -420,7 +411,7 @@ class JobDispatcher {
if (step)
step.attachments.push(attachment);
else
- this._reporter.onStdErr?.('Internal error: step id not found: ' + params.stepId);
+ this._testRun.reporter.onStdErr?.('Internal error: step id not found: ' + params.stepId);
}
}
@@ -432,7 +423,7 @@ class JobDispatcher {
result = runData.result;
} else {
result = test._appendTestResult();
- this._reporter.onTestBegin?.(test, result);
+ this._testRun.reporter.onTestBegin?.(test, result);
}
result.errors = [...errors];
result.error = result.errors[0];
@@ -445,7 +436,7 @@ class JobDispatcher {
for (const test of this._remainingByTestId.values()) {
if (!testIds.has(test.id))
continue;
- if (!this._failureTracker.hasReachedMaxFailures()) {
+ if (!this._testRun.hasReachedMaxFailures()) {
this._failTestWithErrors(test, errors);
errors = []; // Only report errors for the first test.
}
@@ -454,9 +445,9 @@ class JobDispatcher {
if (errors.length) {
// We had fatal errors after all tests have passed - most likely in some teardown.
// Let's just fail the test run.
- this._failureTracker.onWorkerError();
+ this._testRun.hasWorkerErrors = true;
for (const error of errors)
- this._reporter.onError?.(error);
+ this._testRun.reporter.onError?.(error);
}
}
@@ -592,11 +583,11 @@ class JobDispatcher {
throw new Error('Test has already stopped');
const response = await worker.sendCustomMessage({ testId: test.id, request: message.request });
if (response.error)
- addLocationAndSnippetToError(this._config.config, response.error);
+ addLocationAndSnippetToError(this._testRun.config.config, response.error);
return response;
} catch (e) {
const error = serializeError(e);
- addLocationAndSnippetToError(this._config.config, error);
+ addLocationAndSnippetToError(this._testRun.config.config, error);
return { response: undefined, error };
}
};
@@ -605,10 +596,10 @@ class JobDispatcher {
result.errors = params.errors;
result.error = result.errors[0];
- void this._reporter.onTestPaused?.(test, result).then(() => {
+ void this._testRun.reporter.onTestPaused?.(test, result).then(() => {
worker.sendResume({});
});
- this._failureTracker.onTestPaused?.({ ...params, sendMessage });
+ this._testRun.onTestPaused({ ...params, sendMessage });
}
skipWholeJob(): boolean {
@@ -620,10 +611,10 @@ class JobDispatcher {
// with skipped tests mixed in-between non-skipped. This makes
// for a better reporter experience.
const allTestsSkipped = this.job.tests.every(test => test.expectedStatus === 'skipped');
- if (allTestsSkipped && !this._failureTracker.hasReachedMaxFailures()) {
+ if (allTestsSkipped && !this._testRun.hasReachedMaxFailures()) {
for (const test of this.job.tests) {
const result = test._appendTestResult();
- this._reporter.onTestBegin?.(test, result);
+ this._testRun.reporter.onTestBegin?.(test, result);
result.status = 'skipped';
// This must mirror _onTestEnd() above
result.annotations = [...test.annotations];
@@ -639,13 +630,14 @@ class JobDispatcher {
}
private _reportTestEnd(test: testNs.TestCase, result: TestResult) {
- this._reporter.onTestEnd?.(test, result);
- const hadMaxFailures = this._failureTracker.hasReachedMaxFailures();
- this._failureTracker.onTestEnd(test, result);
- if (this._failureTracker.hasReachedMaxFailures()) {
+ this._testRun.reporter.onTestEnd?.(test, result);
+ const hadMaxFailures = this._testRun.hasReachedMaxFailures();
+ // Test is considered failing after the last retry.
+ if (test.outcome() === 'unexpected' && test.results.length > test.retries)
+ ++this._testRun.failedTestCount;
+ if (!hadMaxFailures && this._testRun.hasReachedMaxFailures()) {
this._stopCallback();
- if (!hadMaxFailures)
- this._reporter.onError?.({ message: colors.red(`Testing stopped early after ${this._failureTracker.maxFailures()} maximum allowed failures.`) });
+ this._testRun.reporter.onError?.({ message: colors.red(`Testing stopped early after ${this._testRun.config.config.maxFailures} maximum allowed failures.`) });
}
}
}
diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.ts
deleted file mode 100644
index 5b00d8cc4c..0000000000
--- a/packages/playwright/src/runner/failureTracker.ts
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Copyright Microsoft Corporation. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import type { TestResult, TestError } from '../../types/testReporter';
-import type { config as commonConfig, FullConfigInternal, test } from '../common';
-
-export type TestPausedParams = {
- errors: TestError[];
- sendMessage: (params: { request: any }) => Promise<{ response: any, error?: TestError }>;
-};
-
-export class FailureTracker {
- private _config: FullConfigInternal;
- private _failureCount = 0;
- private _hasWorkerErrors = false;
- private _rootSuite: test.Suite | undefined;
- private _topLevelProjects: commonConfig.FullProjectInternal[] = [];
- private _pauseOnError: boolean;
- private _pauseAtEnd: boolean;
- onTestPaused?: (params: TestPausedParams) => void;
-
- constructor(config: FullConfigInternal, options?: { pauseOnError?: boolean, pauseAtEnd?: boolean }) {
- this._config = config;
- this._pauseOnError = !!options?.pauseOnError;
- this._pauseAtEnd = !!options?.pauseAtEnd;
- }
-
- onRootSuite(rootSuite: test.Suite, topLevelProjects: commonConfig.FullProjectInternal[]) {
- this._rootSuite = rootSuite;
- this._topLevelProjects = topLevelProjects;
- }
-
- onTestEnd(test: test.TestCase, result: TestResult) {
- // Test is considered failing after the last retry.
- if (test.outcome() === 'unexpected' && test.results.length > test.retries)
- ++this._failureCount;
- }
-
- onWorkerError() {
- this._hasWorkerErrors = true;
- }
-
- pauseOnError() {
- return this._pauseOnError;
- }
-
- pauseAtEnd(inProject: commonConfig.FullProjectInternal) {
- return this._topLevelProjects.includes(inProject) && this._pauseAtEnd;
- }
-
- hasReachedMaxFailures() {
- return this.maxFailures() > 0 && this._failureCount >= this.maxFailures();
- }
-
- hasWorkerErrors() {
- return this._hasWorkerErrors;
- }
-
- result(): 'failed' | 'passed' {
- return this._hasWorkerErrors || this.hasReachedMaxFailures() || this.hasFailedTests() || (this._config.failOnFlakyTests && this.hasFlakyTests()) ? 'failed' : 'passed';
- }
-
- hasFailedTests() {
- return this._rootSuite?.allTests().some(test => !test.ok());
- }
-
- hasFlakyTests() {
- return this._rootSuite?.allTests().some(test => (test.outcome() === 'flaky'));
- }
-
- maxFailures() {
- return this._config.config.maxFailures;
- }
-
-}
diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts
index febdeffb24..a2cce3b386 100644
--- a/packages/playwright/src/runner/loadUtils.ts
+++ b/packages/playwright/src/runner/loadUtils.ts
@@ -117,7 +117,7 @@ export async function loadFileSuites(testRun: TestRun, mode: 'out-of-process' |
}
}
-export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean): Promise<{ rootSuite: testNs.Suite, topLevelProjects: commonConfig.FullProjectInternal[] }> {
+export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean) {
const config = testRun.config;
// Create root suite, where each child will be a project suite with cloned file suites inside it.
const rootSuite = new testNs.Suite('', 'root');
@@ -177,7 +177,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
}
// Shard test groups.
- const testGroupsInThisShard = filterForShard(config.config.shard, config.configCLIOverrides.shardWeights, testGroups);
+ const testGroupsInThisShard = filterForShard(config.config.shard, testRun.options.shardWeights, testGroups);
const testsInThisShard = new Set<testNs.TestCase>();
for (const group of testGroupsInThisShard) {
for (const test of group.tests)
@@ -207,7 +207,8 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
}
}
- return { rootSuite, topLevelProjects };
+ testRun.rootSuite = rootSuite;
+ testRun.topLevelProjects = topLevelProjects;
}
function createProjectSuite(project: commonConfig.FullProjectInternal, fileSuites: testNs.Suite[]): testNs.Suite {
diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts
index 253dbd4648..00dd732b63 100644
--- a/packages/playwright/src/runner/reporters.ts
+++ b/packages/playwright/src/runner/reporters.ts
@@ -51,8 +51,8 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' |
};
const reporters: ReporterV2[] = [];
descriptions ??= config.config.reporter;
- if (config.configCLIOverrides.additionalReporters)
- descriptions = [...descriptions, ...config.configCLIOverrides.additionalReporters];
+ if (runOptions?.additionalReporters)
+ descriptions = [...descriptions, ...runOptions.additionalReporters];
const reportOptions = reporterCommandOptions(config, mode, runOptions);
for (const r of descriptions) {
const [name, arg] = r;
diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts
index a26162ae88..6edd4e887c 100644
--- a/packages/playwright/src/runner/tasks.ts
+++ b/packages/playwright/src/runner/tasks.ts
@@ -24,7 +24,6 @@ import { monotonicTime } from '@isomorphic/time';
import { removeFolders } from '@utils/fileUtils';
import { Dispatcher } from './dispatcher';
-import { FailureTracker } from './failureTracker';
import { collectProjectsAndTestFiles, createRootSuite, loadFileSuites, loadGlobalHook, loadTestList } from './loadUtils';
import { buildDependentProjects, buildTeardownToSetupsMap, filterProjects } from './projectUtils';
import { applySuggestedRebaselines, clearSuggestedRebaselines } from './rebase';
@@ -38,7 +37,8 @@ import type { TestGroup } from '../runner/testGroups';
import type { EnvByProjectId } from './dispatcher';
import type { TestRunnerPluginRegistration } from '../plugins';
import type { Task } from './taskRunner';
-import type { FullResult } from '../../types/testReporter';
+import type { ReporterDescription } from '../../types/test';
+import type { FullResult, TestError } from '../../types/testReporter';
import type { Matcher, TestCaseFilter } from '../util';
import type { InternalReporter } from '../reporters/internalReporter';
@@ -69,19 +69,29 @@ export type TestRunOptions = {
lastFailedTestIds?: string[];
pauseOnError?: boolean;
pauseAtEnd?: boolean;
+ onTestPaused?: (params: TestPausedParams) => void;
+ preserveOutputDir?: boolean;
+ additionalReporters?: ReporterDescription[];
+ shardWeights?: number[];
+};
+
+export type TestPausedParams = {
+ errors: TestError[];
+ sendMessage: (params: { request: any }) => Promise<{ response: any, error?: TestError }>;
};
export class TestRun {
readonly config: FullConfigInternal;
readonly options: TestRunOptions;
readonly reporter: InternalReporter;
- readonly failureTracker: FailureTracker;
rootSuite: testNs.Suite | undefined = undefined;
readonly phases: Phase[] = [];
readonly filteredProjects: commonConfig.FullProjectInternal[];
projectFiles: Map<commonConfig.FullProjectInternal, string[]> = new Map();
projectSuites: Map<commonConfig.FullProjectInternal, testNs.Suite[]> = new Map();
topLevelProjects: commonConfig.FullProjectInternal[] = [];
+ hasWorkerErrors = false;
+ failedTestCount = 0;
readonly loadFileFilters: Matcher[] = [];
readonly preOnlyTestFilters: TestCaseFilter[] = [];
readonly postShardTestFilters: TestCaseFilter[] = [];
@@ -90,9 +100,23 @@ export class TestRun {
this.config = config;
this.options = options ?? {};
this.reporter = reporter;
- this.failureTracker = new FailureTracker(config, options);
this.filteredProjects = filterProjects(config.projects, this.options.projectFilter);
}
+
+ onTestPaused(params: TestPausedParams) {
+ this.options.onTestPaused?.(params);
+ }
+
+ hasReachedMaxFailures() {
+ const max = this.config.config.maxFailures;
+ return max > 0 && this.failedTestCount >= max;
+ }
+
+ result(): 'failed' | 'passed' {
+ const hasFailedTests = this.rootSuite?.allTests().some(test => !test.ok());
+ const hasFlakyTests = this.rootSuite?.allTests().some(test => test.outcome() === 'flaky');
+ return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';
+ }
}
export async function runTasks(testRun: TestRun, tasks: Task<TestRun>[], globalTimeout?: number, cancelPromise?: ManualPromise<void>) {
@@ -116,7 +140,7 @@ export async function runTasksDeferCleanup(testRun: TestRun, tasks: Task<TestRun
async function finishTaskRun(testRun: TestRun, status: FullResult['status']) {
if (status === 'passed')
- status = testRun.failureTracker.result();
+ status = testRun.result();
const modifiedResult = await testRun.reporter.onEnd({ status });
if (modifiedResult && modifiedResult.status)
status = modifiedResult.status;
@@ -125,15 +149,12 @@ async function finishTaskRun(testRun: TestRun, status: FullResult['status']) {
}
export function createGlobalSetupTasks(config: FullConfigInternal) {
- const tasks: Task<TestRun>[] = [];
- if (!config.configCLIOverrides.preserveOutputDir)
- tasks.push(createRemoveOutputDirsTask());
- tasks.push(
- ...createPluginSetupTasks(config),
- ...config.globalTeardowns.map(file => createGlobalTeardownTask(file, config)).reverse(),
- ...config.globalSetups.map(file => createGlobalSetupTask(file, config)),
- );
- return tasks;
+ return [
+ createRemoveOutputDirsTask(),
+ ...createPluginSetupTasks(config),
+ ...config.globalTeardowns.map(file => createGlobalTeardownTask(file, config)).reverse(),
+ ...config.globalSetups.map(file => createGlobalSetupTask(file, config)),
+ ];
}
export function createRunTestsTasks(config: FullConfigInternal) {
@@ -231,6 +252,8 @@ function createRemoveOutputDirsTask(): Task<TestRun> {
return {
title: 'clear output',
setup: async testRun => {
+ if (testRun.options.preserveOutputDir)
+ return;
const outputDirs = new Set<string>();
testRun.filteredProjects.forEach(p => outputDirs.add(p.project.outputDir));
@@ -255,14 +278,12 @@ export function createListFilesTask(): Task<TestRun> {
return {
title: 'load tests',
setup: async (testRun, errors) => {
- const { rootSuite, topLevelProjects } = await createRootSuite(testRun, errors, false);
- testRun.rootSuite = rootSuite;
- testRun.failureTracker.onRootSuite(rootSuite, topLevelProjects);
+ await createRootSuite(testRun, errors, false);
await collectProjectsAndTestFiles(testRun, false);
for (const [project, files] of testRun.projectFiles) {
const projectSuite = new testNs.Suite(project.project.name, 'project');
projectSuite._fullProject = project;
- testRun.rootSuite._addSuite(projectSuite);
+ testRun.rootSuite!._addSuite(projectSuite);
const suites = files.map(file => {
const title = path.relative(testRun.config.config.rootDir, file);
const suite = new testNs.Suite(title, 'file');
@@ -328,11 +349,9 @@ export function createLoadTask(mode: 'out-of-process' | 'in-process', options: {
testRun.preOnlyTestFilters.push(test => changedFiles.has(test.location.file));
}
- const { rootSuite, topLevelProjects } = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);
- testRun.rootSuite = rootSuite;
- testRun.failureTracker.onRootSuite(rootSuite, topLevelProjects);
+ await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);
// Fail when no tests.
- if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length
+ if (options.failOnLoadErrors && !testRun.rootSuite?.allTests().length
&& !testRun.options.passWithNoTests
&& !testRun.config.config.shard && !testRun.options.onlyChanged
&& !testRun.options.testList && !testRun.options.testListInvert) {
@@ -395,7 +414,7 @@ function createPhasesTask(): Task<TestRun> {
processed.add(project);
if (phaseProjects.length) {
let testGroupsInPhase = 0;
- const phase: Phase = { dispatcher: new Dispatcher(testRun.config, testRun.reporter, testRun.failureTracker), projects: [] };
+ const phase: Phase = { dispatcher: new Dispatcher(testRun), projects: [] };
testRun.phases.push(phase);
for (const project of phaseProjects) {
const projectSuite = projectToSuite.get(project)!;
@@ -416,12 +435,12 @@ function createPhasesTask(): Task<TestRun> {
function createRunTestsTask(): Task<TestRun> {
return {
title: 'test suite',
- setup: async ({ phases, failureTracker }) => {
+ setup: async testRun => {
const successfulProjects = new Set<commonConfig.FullProjectInternal>();
const extraEnvByProjectId: EnvByProjectId = new Map();
- const teardownToSetups = buildTeardownToSetupsMap(phases.map(phase => phase.projects.map(p => p.project)).flat());
+ const teardownToSetups = buildTeardownToSetupsMap(testRun.phases.map(phase => phase.projects.map(p => p.project)).flat());
- for (const { dispatcher, projects } of phases) {
+ for (const { dispatcher, projects } of testRun.phases) {
// Each phase contains dispatcher and a set of test groups.
// We don't want to run the test groups belonging to the projects
// that depend on the projects that failed previously.
@@ -451,7 +470,7 @@ function createRunTestsTask(): Task<TestRun> {
// If the worker broke, fail everything, we have no way of knowing which
// projects failed.
- if (!failureTracker.hasWorkerErrors()) {
+ if (!testRun.hasWorkerErrors) {
for (const { project, projectSuite } of projects) {
const hasFailedDeps = project.deps.some(p => !successfulProjects.has(p));
if (!hasFailedDeps && !projectSuite.allTests().some(test => !test.ok()))
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index 429b17638b..826763a66e 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -37,12 +37,11 @@ import { TestRun, createApplyRebaselinesTask, createClearCacheTask, createGlobal
import { LastRunReporter } from './lastRun';
import { filterProjects } from './projectUtils';
-import type { TestRunOptions } from './tasks';
+import type { TestPausedParams, TestRunOptions } from './tasks';
import type * as reporterTypes from '../../types/testReporter';
import type { ConfigLocation } from '../common';
import type { TestRunnerPluginRegistration } from '../plugins';
import type { AnyReporter } from '../reporters/reporterV2';
-import type { TestPausedParams } from './failureTracker';
export const TestRunnerEvent = {
TestFilesChanged: 'testFilesChanged',
@@ -296,7 +295,6 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {
repeatEach: 1,
retries: 0,
timeout: params.timeout,
- preserveOutputDir: true,
reporter: params.reporters ? params.reporters.map(r => [r]) : undefined,
use: {
...this._configCLIOverrides.use,
@@ -325,6 +323,8 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {
projectFilter: params.projects?.length ? params.projects : undefined,
pauseOnError: params.pauseOnError,
pauseAtEnd: params.pauseAtEnd,
+ preserveOutputDir: true,
+ onTestPaused: params => this.emit(TestRunnerEvent.TestPaused, params),
};
const configReporters = params.disableConfigReporters ? [] : await createReporters(config, 'test', undefined, options);
@@ -340,7 +340,6 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {
createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: !!params.failOnLoadErrors, doNotRunDepsOutsideProjectFilter: params.doNotRunDepsOutsideProjectFilter }),
...createRunTestsTasks(config),
];
- testRun.failureTracker.onTestPaused = params => this.emit(TestRunnerEvent.TestPaused, params);
const run = runTasks(testRun, tasks, 0, stop).then(async status => {
this._testRun = undefined;
return status;