Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: merge FailureTracker into TestRun, pull more into TestRunOptions (#40150)
Dmitry Gozmanb82aa4973ec0
diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.tsindex 25eae98827..9e7e954ad8 100644--- a/packages/playwright/src/cli/testActions.ts+++ b/packages/playwright/src/cli/testActions.ts@@ -41,6 +41,7 @@ export async function runTests(args: string[], opts: { [key: string]: any }) {lastFailed: !!opts.lastFailed,testList: opts.testList ? path.resolve(process.cwd(), opts.testList) : undefined,testListInvert: opts.testListInvert ? path.resolve(process.cwd(), opts.testListInvert) : undefined,+ shardWeights: resolveShardWeightsOption(),};// Evaluate project filters against config before starting execution. This enables a consistent error message across run modes@@ -124,7 +125,6 @@ function overridesFromOptions(options: { [key: string]: any }): ipc.ConfigCLIOveretries: options.retries ? parseInt(options.retries, 10) : undefined,reporter: resolveReporterOption(options.reporter),shard: resolveShardOption(options.shard),- shardWeights: resolveShardWeightsOption(),timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined,ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,@@ -195,7 +195,7 @@ function resolveShardOption(shard?: string): ipc.ConfigCLIOverrides['shard'] {return { current, total };}-function resolveShardWeightsOption(): ipc.ConfigCLIOverrides['shardWeights'] {+function resolveShardWeightsOption(): TestRunOptions['shardWeights'] {const shardWeights = process.env.PWTEST_SHARD_WEIGHTS;if (!shardWeights)return undefined;diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.tsindex e1774d2b91..42cc3bd6ab 100644--- a/packages/playwright/src/common/ipc.ts+++ b/packages/playwright/src/common/ipc.ts@@ -30,15 +30,12 @@ export type ConfigCLIOverrides = {globalTimeout?: number;maxFailures?: number;outputDir?: string;- preserveOutputDir?: boolean;pause?: boolean;quiet?: boolean;repeatEach?: number;retries?: number;reporter?: ReporterDescription[];- additionalReporters?: ReporterDescription[];shard?: { current: number, total: number };- shardWeights?: number[];timeout?: number;tsconfig?: string;ignoreSnapshots?: boolean;diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex b9afff3a5c..c48c009f51 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -20,16 +20,15 @@ import { eventsHelper } from '@utils/eventsHelper';import { addSuggestedRebaseline } from './rebase';import { WorkerHost } from './workerHost';-import { FullConfigInternal, ipc, test as testNs } from '../common';+import { ipc, test as testNs } from '../common';import { addLocationAndSnippetToError } from '../reporters/internalReporter';import { serializeError } from '../util';import type { RegisteredListener } from '@utils/eventsHelper';-import type { FailureTracker } from './failureTracker';import type { ProcessExitData } from './processHost';+import type { TestRun } from './tasks';import type { TestGroup } from './testGroups';import type { TestError, TestResult, TestStep } from '../../types/testReporter';-import type { ReporterV2 } from '../reporters/reporterV2';export type EnvByProjectId = Map<string, Record<string, string | undefined>>;@@ -42,18 +41,14 @@ export class Dispatcher {private _finished = new ManualPromise<void>();private _isStopped = true;- private _config: FullConfigInternal;- private _reporter: ReporterV2;- private _failureTracker: FailureTracker;+ private _testRun: TestRun;private _extraEnvByProjectId: EnvByProjectId = new Map();private _producedEnvByProjectId: EnvByProjectId = new Map();- constructor(config: FullConfigInternal, reporter: ReporterV2, failureTracker: FailureTracker) {- this._config = config;- this._reporter = reporter;- this._failureTracker = failureTracker;- for (const project of config.projects) {+ constructor(testRun: TestRun) {+ this._testRun = testRun;+ for (const project of testRun.config.projects) {if (project.workers)this._workerLimitPerProjectId.set(project.id, project.workers);}@@ -97,7 +92,7 @@ export class Dispatcher {// 3. Claim both the job and the worker slot.this._queue.splice(jobIndex, 1);- const jobDispatcher = new JobDispatcher(job, this._config, this._reporter, this._failureTracker, () => this.stop().catch(() => {}));+ const jobDispatcher = new JobDispatcher(job, this._testRun, () => this.stop().catch(() => {}));this._workerSlots[workerIndex].jobDispatcher = jobDispatcher;// 4. Run the job. This is the only async operation.@@ -132,7 +127,7 @@ export class Dispatcher {// 2. Start the worker if it is down.let startError;if (!worker) {- worker = this._createWorker(job, index, ipc.serializeConfig(this._config, true));+ worker = this._createWorker(job, index, ipc.serializeConfig(this._testRun.config, true));this._workerSlots[index].worker = worker;worker.on('exit', () => this._workerSlots[index].worker = undefined);startError = await worker.start();@@ -198,10 +193,10 @@ export class Dispatcher {this._isStopped = false;this._workerSlots = [];// 0. Stop right away if we have reached max failures.- if (this._failureTracker.hasReachedMaxFailures())+ if (this._testRun.hasReachedMaxFailures())void this.stop();// 1. Allocate workers.- for (let i = 0; i < this._config.config.workers; i++)+ for (let i = 0; i < this._testRun.config.config.workers; i++)this._workerSlots.push({});// 2. Schedule enough jobs.for (let i = 0; i < this._workerSlots.length; i++)@@ -213,15 +208,15 @@ export class Dispatcher {}_createWorker(testGroup: TestGroup, parallelIndex: number, loaderData: ipc.SerializedConfig) {- const projectConfig = this._config.projects.find(p => p.id === testGroup.projectId)!;- const outputDir = projectConfig.project.outputDir;+ const project = this._testRun.config.projects.find(p => p.id === testGroup.projectId)!;+ const pauseAtEnd = this._testRun.topLevelProjects.includes(project) && !!this._testRun.options.pauseAtEnd;const worker = new WorkerHost(testGroup, {parallelIndex,config: loaderData,extraEnv: this._extraEnvByProjectId.get(testGroup.projectId) || {},- outputDir,- pauseOnError: this._failureTracker.pauseOnError(),- pauseAtEnd: this._failureTracker.pauseAtEnd(projectConfig),+ outputDir: project.project.outputDir,+ pauseOnError: !!this._testRun.options.pauseOnError,+ pauseAtEnd,});const handleOutput = (params: ipc.TestOutputPayload) => {const chunk = chunkFromParams(params);@@ -239,17 +234,17 @@ export class Dispatcher {worker.on('stdOut', (params: ipc.TestOutputPayload) => {const { chunk, test, result } = handleOutput(params);result?.stdout.push(chunk);- this._reporter.onStdOut?.(chunk, test, result);+ this._testRun.reporter.onStdOut?.(chunk, test, result);});worker.on('stdErr', (params: ipc.TestOutputPayload) => {const { chunk, test, result } = handleOutput(params);result?.stderr.push(chunk);- this._reporter.onStdErr?.(chunk, test, result);+ this._testRun.reporter.onStdErr?.(chunk, test, result);});worker.on('teardownErrors', (params: ipc.TeardownErrorsPayload) => {- this._failureTracker.onWorkerError();+ this._testRun.hasWorkerErrors = true;for (const error of params.fatalErrors)- this._reporter.onError?.(error);+ this._testRun.reporter.onError?.(error);});worker.on('exit', () => {const producedEnv = this._producedEnvByProjectId.get(testGroup.projectId) || {};@@ -275,9 +270,7 @@ class JobDispatcher {jobResult = new ManualPromise<{ newJob?: TestGroup, didFail: boolean }>();readonly job: TestGroup;- private _config: FullConfigInternal;- private _reporter: ReporterV2;- private _failureTracker: FailureTracker;+ private _testRun: TestRun;private _stopCallback: () => void;private _listeners: RegisteredListener[] = [];private _failedTests = new Set<testNs.TestCase>();@@ -288,11 +281,9 @@ class JobDispatcher {private _workerIndex = 0;private _currentlyRunning: { test: testNs.TestCase, result: TestResult } | undefined;- constructor(job: TestGroup, config: FullConfigInternal, reporter: ReporterV2, failureTracker: FailureTracker, stopCallback: () => void) {+ constructor(job: TestGroup, testRun: TestRun, stopCallback: () => void) {this.job = job;- this._config = config;- this._reporter = reporter;- this._failureTracker = failureTracker;+ this._testRun = testRun;this._stopCallback = stopCallback;this._remainingByTestId = new Map(this.job.tests.map(e => [e.id, e]));}@@ -308,12 +299,12 @@ class JobDispatcher {result.parallelIndex = this._parallelIndex;result.workerIndex = this._workerIndex;result.startTime = new Date(params.startWallTime);- this._reporter.onTestBegin?.(test, result);+ this._testRun.reporter.onTestBegin?.(test, result);this._currentlyRunning = { test, result };}private _onTestEnd(params: ipc.TestEndPayload) {- if (this._failureTracker.hasReachedMaxFailures()) {+ if (this._testRun.hasReachedMaxFailures()) {// Do not show more than one error to avoid confusion, but report// as interrupted to indicate that we did actually start the test.params.status = 'interrupted';@@ -377,7 +368,7 @@ class JobDispatcher {};steps.set(params.stepId, step);(parentStep || result).steps.push(step);- this._reporter.onStepBegin?.(test, result, step);+ this._testRun.reporter.onStepBegin?.(test, result, step);}private _onStepEnd(params: ipc.StepEndPayload) {@@ -389,7 +380,7 @@ class JobDispatcher {const { result, steps, test } = data;const step = steps.get(params.stepId);if (!step) {- this._reporter.onStdErr?.('Internal error: step end without step begin: ' + params.stepId, test, result);+ this._testRun.reporter.onStdErr?.('Internal error: step end without step begin: ' + params.stepId, test, result);return;}step.duration = params.wallTime - step.startTime.getTime();@@ -399,7 +390,7 @@ class JobDispatcher {addSuggestedRebaseline(step.location!, params.suggestedRebaseline);step.annotations = params.annotations;steps.delete(params.stepId);- this._reporter.onStepEnd?.(test, result, step);+ this._testRun.reporter.onStepEnd?.(test, result, step);}private _onAttach(params: ipc.AttachmentPayload) {@@ -420,7 +411,7 @@ class JobDispatcher {if (step)step.attachments.push(attachment);else- this._reporter.onStdErr?.('Internal error: step id not found: ' + params.stepId);+ this._testRun.reporter.onStdErr?.('Internal error: step id not found: ' + params.stepId);}}@@ -432,7 +423,7 @@ class JobDispatcher {result = runData.result;} else {result = test._appendTestResult();- this._reporter.onTestBegin?.(test, result);+ this._testRun.reporter.onTestBegin?.(test, result);}result.errors = [...errors];result.error = result.errors[0];@@ -445,7 +436,7 @@ class JobDispatcher {for (const test of this._remainingByTestId.values()) {if (!testIds.has(test.id))continue;- if (!this._failureTracker.hasReachedMaxFailures()) {+ if (!this._testRun.hasReachedMaxFailures()) {this._failTestWithErrors(test, errors);errors = []; // Only report errors for the first test.}@@ -454,9 +445,9 @@ class JobDispatcher {if (errors.length) {// We had fatal errors after all tests have passed - most likely in some teardown.// Let's just fail the test run.- this._failureTracker.onWorkerError();+ this._testRun.hasWorkerErrors = true;for (const error of errors)- this._reporter.onError?.(error);+ this._testRun.reporter.onError?.(error);}}@@ -592,11 +583,11 @@ class JobDispatcher {throw new Error('Test has already stopped');const response = await worker.sendCustomMessage({ testId: test.id, request: message.request });if (response.error)- addLocationAndSnippetToError(this._config.config, response.error);+ addLocationAndSnippetToError(this._testRun.config.config, response.error);return response;} catch (e) {const error = serializeError(e);- addLocationAndSnippetToError(this._config.config, error);+ addLocationAndSnippetToError(this._testRun.config.config, error);return { response: undefined, error };}};@@ -605,10 +596,10 @@ class JobDispatcher {result.errors = params.errors;result.error = result.errors[0];- void this._reporter.onTestPaused?.(test, result).then(() => {+ void this._testRun.reporter.onTestPaused?.(test, result).then(() => {worker.sendResume({});});- this._failureTracker.onTestPaused?.({ ...params, sendMessage });+ this._testRun.onTestPaused({ ...params, sendMessage });}skipWholeJob(): boolean {@@ -620,10 +611,10 @@ class JobDispatcher {// with skipped tests mixed in-between non-skipped. This makes// for a better reporter experience.const allTestsSkipped = this.job.tests.every(test => test.expectedStatus === 'skipped');- if (allTestsSkipped && !this._failureTracker.hasReachedMaxFailures()) {+ if (allTestsSkipped && !this._testRun.hasReachedMaxFailures()) {for (const test of this.job.tests) {const result = test._appendTestResult();- this._reporter.onTestBegin?.(test, result);+ this._testRun.reporter.onTestBegin?.(test, result);result.status = 'skipped';// This must mirror _onTestEnd() aboveresult.annotations = [...test.annotations];@@ -639,13 +630,14 @@ class JobDispatcher {}private _reportTestEnd(test: testNs.TestCase, result: TestResult) {- this._reporter.onTestEnd?.(test, result);- const hadMaxFailures = this._failureTracker.hasReachedMaxFailures();- this._failureTracker.onTestEnd(test, result);- if (this._failureTracker.hasReachedMaxFailures()) {+ this._testRun.reporter.onTestEnd?.(test, result);+ const hadMaxFailures = this._testRun.hasReachedMaxFailures();+ // Test is considered failing after the last retry.+ if (test.outcome() === 'unexpected' && test.results.length > test.retries)+ ++this._testRun.failedTestCount;+ if (!hadMaxFailures && this._testRun.hasReachedMaxFailures()) {this._stopCallback();- if (!hadMaxFailures)- this._reporter.onError?.({ message: colors.red(`Testing stopped early after ${this._failureTracker.maxFailures()} maximum allowed failures.`) });+ this._testRun.reporter.onError?.({ message: colors.red(`Testing stopped early after ${this._testRun.config.config.maxFailures} maximum allowed failures.`) });}}}diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.tsdeleted file mode 100644index 5b00d8cc4c..0000000000--- a/packages/playwright/src/runner/failureTracker.ts+++ /dev/null@@ -1,88 +0,0 @@-/**- * Copyright Microsoft Corporation. All rights reserved.- *- * Licensed under the Apache License, Version 2.0 (the "License");- * you may not use this file except in compliance with the License.- * You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import type { TestResult, TestError } from '../../types/testReporter';-import type { config as commonConfig, FullConfigInternal, test } from '../common';--export type TestPausedParams = {- errors: TestError[];- sendMessage: (params: { request: any }) => Promise<{ response: any, error?: TestError }>;-};--export class FailureTracker {- private _config: FullConfigInternal;- private _failureCount = 0;- private _hasWorkerErrors = false;- private _rootSuite: test.Suite | undefined;- private _topLevelProjects: commonConfig.FullProjectInternal[] = [];- private _pauseOnError: boolean;- private _pauseAtEnd: boolean;- onTestPaused?: (params: TestPausedParams) => void;-- constructor(config: FullConfigInternal, options?: { pauseOnError?: boolean, pauseAtEnd?: boolean }) {- this._config = config;- this._pauseOnError = !!options?.pauseOnError;- this._pauseAtEnd = !!options?.pauseAtEnd;- }-- onRootSuite(rootSuite: test.Suite, topLevelProjects: commonConfig.FullProjectInternal[]) {- this._rootSuite = rootSuite;- this._topLevelProjects = topLevelProjects;- }-- onTestEnd(test: test.TestCase, result: TestResult) {- // Test is considered failing after the last retry.- if (test.outcome() === 'unexpected' && test.results.length > test.retries)- ++this._failureCount;- }-- onWorkerError() {- this._hasWorkerErrors = true;- }-- pauseOnError() {- return this._pauseOnError;- }-- pauseAtEnd(inProject: commonConfig.FullProjectInternal) {- return this._topLevelProjects.includes(inProject) && this._pauseAtEnd;- }-- hasReachedMaxFailures() {- return this.maxFailures() > 0 && this._failureCount >= this.maxFailures();- }-- hasWorkerErrors() {- return this._hasWorkerErrors;- }-- result(): 'failed' | 'passed' {- return this._hasWorkerErrors || this.hasReachedMaxFailures() || this.hasFailedTests() || (this._config.failOnFlakyTests && this.hasFlakyTests()) ? 'failed' : 'passed';- }-- hasFailedTests() {- return this._rootSuite?.allTests().some(test => !test.ok());- }-- hasFlakyTests() {- return this._rootSuite?.allTests().some(test => (test.outcome() === 'flaky'));- }-- maxFailures() {- return this._config.config.maxFailures;- }--}diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.tsindex febdeffb24..a2cce3b386 100644--- a/packages/playwright/src/runner/loadUtils.ts+++ b/packages/playwright/src/runner/loadUtils.ts@@ -117,7 +117,7 @@ export async function loadFileSuites(testRun: TestRun, mode: 'out-of-process' |}}-export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean): Promise<{ rootSuite: testNs.Suite, topLevelProjects: commonConfig.FullProjectInternal[] }> {+export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean) {const config = testRun.config;// Create root suite, where each child will be a project suite with cloned file suites inside it.const rootSuite = new testNs.Suite('', 'root');@@ -177,7 +177,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho}// Shard test groups.- const testGroupsInThisShard = filterForShard(config.config.shard, config.configCLIOverrides.shardWeights, testGroups);+ const testGroupsInThisShard = filterForShard(config.config.shard, testRun.options.shardWeights, testGroups);const testsInThisShard = new Set<testNs.TestCase>();for (const group of testGroupsInThisShard) {for (const test of group.tests)@@ -207,7 +207,8 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho}}- return { rootSuite, topLevelProjects };+ testRun.rootSuite = rootSuite;+ testRun.topLevelProjects = topLevelProjects;}function createProjectSuite(project: commonConfig.FullProjectInternal, fileSuites: testNs.Suite[]): testNs.Suite {diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.tsindex 253dbd4648..00dd732b63 100644--- a/packages/playwright/src/runner/reporters.ts+++ b/packages/playwright/src/runner/reporters.ts@@ -51,8 +51,8 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' |};const reporters: ReporterV2[] = [];descriptions ??= config.config.reporter;- if (config.configCLIOverrides.additionalReporters)- descriptions = [...descriptions, ...config.configCLIOverrides.additionalReporters];+ if (runOptions?.additionalReporters)+ descriptions = [...descriptions, ...runOptions.additionalReporters];const reportOptions = reporterCommandOptions(config, mode, runOptions);for (const r of descriptions) {const [name, arg] = r;diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex a26162ae88..6edd4e887c 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -24,7 +24,6 @@ import { monotonicTime } from '@isomorphic/time';import { removeFolders } from '@utils/fileUtils';import { Dispatcher } from './dispatcher';-import { FailureTracker } from './failureTracker';import { collectProjectsAndTestFiles, createRootSuite, loadFileSuites, loadGlobalHook, loadTestList } from './loadUtils';import { buildDependentProjects, buildTeardownToSetupsMap, filterProjects } from './projectUtils';import { applySuggestedRebaselines, clearSuggestedRebaselines } from './rebase';@@ -38,7 +37,8 @@ import type { TestGroup } from '../runner/testGroups';import type { EnvByProjectId } from './dispatcher';import type { TestRunnerPluginRegistration } from '../plugins';import type { Task } from './taskRunner';-import type { FullResult } from '../../types/testReporter';+import type { ReporterDescription } from '../../types/test';+import type { FullResult, TestError } from '../../types/testReporter';import type { Matcher, TestCaseFilter } from '../util';import type { InternalReporter } from '../reporters/internalReporter';@@ -69,19 +69,29 @@ export type TestRunOptions = {lastFailedTestIds?: string[];pauseOnError?: boolean;pauseAtEnd?: boolean;+ onTestPaused?: (params: TestPausedParams) => void;+ preserveOutputDir?: boolean;+ additionalReporters?: ReporterDescription[];+ shardWeights?: number[];+};++export type TestPausedParams = {+ errors: TestError[];+ sendMessage: (params: { request: any }) => Promise<{ response: any, error?: TestError }>;};export class TestRun {readonly config: FullConfigInternal;readonly options: TestRunOptions;readonly reporter: InternalReporter;- readonly failureTracker: FailureTracker;rootSuite: testNs.Suite | undefined = undefined;readonly phases: Phase[] = [];readonly filteredProjects: commonConfig.FullProjectInternal[];projectFiles: Map<commonConfig.FullProjectInternal, string[]> = new Map();projectSuites: Map<commonConfig.FullProjectInternal, testNs.Suite[]> = new Map();topLevelProjects: commonConfig.FullProjectInternal[] = [];+ hasWorkerErrors = false;+ failedTestCount = 0;readonly loadFileFilters: Matcher[] = [];readonly preOnlyTestFilters: TestCaseFilter[] = [];readonly postShardTestFilters: TestCaseFilter[] = [];@@ -90,9 +100,23 @@ export class TestRun {this.config = config;this.options = options ?? {};this.reporter = reporter;- this.failureTracker = new FailureTracker(config, options);this.filteredProjects = filterProjects(config.projects, this.options.projectFilter);}++ onTestPaused(params: TestPausedParams) {+ this.options.onTestPaused?.(params);+ }++ hasReachedMaxFailures() {+ const max = this.config.config.maxFailures;+ return max > 0 && this.failedTestCount >= max;+ }++ result(): 'failed' | 'passed' {+ const hasFailedTests = this.rootSuite?.allTests().some(test => !test.ok());+ const hasFlakyTests = this.rootSuite?.allTests().some(test => test.outcome() === 'flaky');+ return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';+ }}export async function runTasks(testRun: TestRun, tasks: Task<TestRun>[], globalTimeout?: number, cancelPromise?: ManualPromise<void>) {@@ -116,7 +140,7 @@ export async function runTasksDeferCleanup(testRun: TestRun, tasks: Task<TestRunasync function finishTaskRun(testRun: TestRun, status: FullResult['status']) {if (status === 'passed')- status = testRun.failureTracker.result();+ status = testRun.result();const modifiedResult = await testRun.reporter.onEnd({ status });if (modifiedResult && modifiedResult.status)status = modifiedResult.status;@@ -125,15 +149,12 @@ async function finishTaskRun(testRun: TestRun, status: FullResult['status']) {}export function createGlobalSetupTasks(config: FullConfigInternal) {- const tasks: Task<TestRun>[] = [];- if (!config.configCLIOverrides.preserveOutputDir)- tasks.push(createRemoveOutputDirsTask());- tasks.push(- ...createPluginSetupTasks(config),- ...config.globalTeardowns.map(file => createGlobalTeardownTask(file, config)).reverse(),- ...config.globalSetups.map(file => createGlobalSetupTask(file, config)),- );- return tasks;+ return [+ createRemoveOutputDirsTask(),+ ...createPluginSetupTasks(config),+ ...config.globalTeardowns.map(file => createGlobalTeardownTask(file, config)).reverse(),+ ...config.globalSetups.map(file => createGlobalSetupTask(file, config)),+ ];}export function createRunTestsTasks(config: FullConfigInternal) {@@ -231,6 +252,8 @@ function createRemoveOutputDirsTask(): Task<TestRun> {return {title: 'clear output',setup: async testRun => {+ if (testRun.options.preserveOutputDir)+ return;const outputDirs = new Set<string>();testRun.filteredProjects.forEach(p => outputDirs.add(p.project.outputDir));@@ -255,14 +278,12 @@ export function createListFilesTask(): Task<TestRun> {return {title: 'load tests',setup: async (testRun, errors) => {- const { rootSuite, topLevelProjects } = await createRootSuite(testRun, errors, false);- testRun.rootSuite = rootSuite;- testRun.failureTracker.onRootSuite(rootSuite, topLevelProjects);+ await createRootSuite(testRun, errors, false);await collectProjectsAndTestFiles(testRun, false);for (const [project, files] of testRun.projectFiles) {const projectSuite = new testNs.Suite(project.project.name, 'project');projectSuite._fullProject = project;- testRun.rootSuite._addSuite(projectSuite);+ testRun.rootSuite!._addSuite(projectSuite);const suites = files.map(file => {const title = path.relative(testRun.config.config.rootDir, file);const suite = new testNs.Suite(title, 'file');@@ -328,11 +349,9 @@ export function createLoadTask(mode: 'out-of-process' | 'in-process', options: {testRun.preOnlyTestFilters.push(test => changedFiles.has(test.location.file));}- const { rootSuite, topLevelProjects } = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);- testRun.rootSuite = rootSuite;- testRun.failureTracker.onRootSuite(rootSuite, topLevelProjects);+ await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);// Fail when no tests.- if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length+ if (options.failOnLoadErrors && !testRun.rootSuite?.allTests().length&& !testRun.options.passWithNoTests&& !testRun.config.config.shard && !testRun.options.onlyChanged&& !testRun.options.testList && !testRun.options.testListInvert) {@@ -395,7 +414,7 @@ function createPhasesTask(): Task<TestRun> {processed.add(project);if (phaseProjects.length) {let testGroupsInPhase = 0;- const phase: Phase = { dispatcher: new Dispatcher(testRun.config, testRun.reporter, testRun.failureTracker), projects: [] };+ const phase: Phase = { dispatcher: new Dispatcher(testRun), projects: [] };testRun.phases.push(phase);for (const project of phaseProjects) {const projectSuite = projectToSuite.get(project)!;@@ -416,12 +435,12 @@ function createPhasesTask(): Task<TestRun> {function createRunTestsTask(): Task<TestRun> {return {title: 'test suite',- setup: async ({ phases, failureTracker }) => {+ setup: async testRun => {const successfulProjects = new Set<commonConfig.FullProjectInternal>();const extraEnvByProjectId: EnvByProjectId = new Map();- const teardownToSetups = buildTeardownToSetupsMap(phases.map(phase => phase.projects.map(p => p.project)).flat());+ const teardownToSetups = buildTeardownToSetupsMap(testRun.phases.map(phase => phase.projects.map(p => p.project)).flat());- for (const { dispatcher, projects } of phases) {+ for (const { dispatcher, projects } of testRun.phases) {// Each phase contains dispatcher and a set of test groups.// We don't want to run the test groups belonging to the projects// that depend on the projects that failed previously.@@ -451,7 +470,7 @@ function createRunTestsTask(): Task<TestRun> {// If the worker broke, fail everything, we have no way of knowing which// projects failed.- if (!failureTracker.hasWorkerErrors()) {+ if (!testRun.hasWorkerErrors) {for (const { project, projectSuite } of projects) {const hasFailedDeps = project.deps.some(p => !successfulProjects.has(p));if (!hasFailedDeps && !projectSuite.allTests().some(test => !test.ok()))diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.tsindex 429b17638b..826763a66e 100644--- a/packages/playwright/src/runner/testRunner.ts+++ b/packages/playwright/src/runner/testRunner.ts@@ -37,12 +37,11 @@ import { TestRun, createApplyRebaselinesTask, createClearCacheTask, createGlobalimport { LastRunReporter } from './lastRun';import { filterProjects } from './projectUtils';-import type { TestRunOptions } from './tasks';+import type { TestPausedParams, TestRunOptions } from './tasks';import type * as reporterTypes from '../../types/testReporter';import type { ConfigLocation } from '../common';import type { TestRunnerPluginRegistration } from '../plugins';import type { AnyReporter } from '../reporters/reporterV2';-import type { TestPausedParams } from './failureTracker';export const TestRunnerEvent = {TestFilesChanged: 'testFilesChanged',@@ -296,7 +295,6 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {repeatEach: 1,retries: 0,timeout: params.timeout,- preserveOutputDir: true,reporter: params.reporters ? params.reporters.map(r => [r]) : undefined,use: {...this._configCLIOverrides.use,@@ -325,6 +323,8 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {projectFilter: params.projects?.length ? params.projects : undefined,pauseOnError: params.pauseOnError,pauseAtEnd: params.pauseAtEnd,+ preserveOutputDir: true,+ onTestPaused: params => this.emit(TestRunnerEvent.TestPaused, params),};const configReporters = params.disableConfigReporters ? [] : await createReporters(config, 'test', undefined, options);@@ -340,7 +340,6 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: !!params.failOnLoadErrors, doNotRunDepsOutsideProjectFilter: params.doNotRunDepsOutsideProjectFilter }),...createRunTestsTasks(config),];- testRun.failureTracker.onTestPaused = params => this.emit(TestRunnerEvent.TestPaused, params);const run = runTasks(testRun, tasks, 0, stop).then(async status => {this._testRun = undefined;return status;