Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(reporter): expose failOnFlakyTests on FullConfig (#40935)

Dmitry Gozman3acf04517034
diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md
index 627218db7b..e3097cb16f 100644
--- a/docs/src/test-api/class-fullconfig.md
+++ b/docs/src/test-api/class-fullconfig.md
@@ -21,6 +21,12 @@ any argument-parsing library.
Path to the configuration file used to run the tests. The value is an empty string if no config file was used.
+## property: FullConfig.failOnFlakyTests
+* since: v1.61
+- type: <[boolean]>
+
+See [`property: TestConfig.failOnFlakyTests`].
+
## property: FullConfig.forbidOnly
* since: v1.10
- type: <[boolean]>
diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts
index 475386f853..2b8ae1a60f 100644
--- a/packages/playwright/src/common/config.ts
+++ b/packages/playwright/src/common/config.ts
@@ -48,7 +48,6 @@ export class FullConfigInternal {
readonly projects: FullProjectInternal[] = [];
readonly singleTSConfigPath?: string;
readonly captureGitInfo: Config['captureGitInfo'];
- readonly failOnFlakyTests: boolean;
defineConfigWasUsed = false;
globalSetups: string[] = [];
@@ -68,7 +67,6 @@ export class FullConfigInternal {
this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p }));
this.singleTSConfigPath = pathResolve(configDir, userConfig.tsconfig);
this.captureGitInfo = userConfig.captureGitInfo;
- this.failOnFlakyTests = takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false);
this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);
this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);
@@ -87,6 +85,7 @@ export class FullConfigInternal {
argv: configCLIOverrides.argv ?? [],
configFile: resolvedConfigFile,
rootDir: pathResolve(configDir, userConfig.testDir) || configDir,
+ failOnFlakyTests: takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false),
forbidOnly: takeFirst(configCLIOverrides.forbidOnly, userConfig.forbidOnly, false),
fullyParallel: takeFirst(configCLIOverrides.fullyParallel, userConfig.fullyParallel, false),
globalSetup: this.globalSetups[0] ?? null,
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index bd534c9216..bfde074d8d 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -29,6 +29,7 @@ export type JsonConfig = Pick<reporterTypes.FullConfig, 'configFile' | 'globalTi
// optional for backwards compatibility
tags?: reporterTypes.FullConfig['tags'],
webServer?: reporterTypes.FullConfig['webServer'],
+ failOnFlakyTests?: reporterTypes.FullConfig['failOnFlakyTests'],
};
export type JsonPattern = {
@@ -780,6 +781,7 @@ export type TeleFullProject = reporterTypes.FullProject;
export const baseFullConfig: reporterTypes.FullConfig = {
argv: [],
+ failOnFlakyTests: false,
forbidOnly: false,
fullyParallel: false,
globalSetup: null,
diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts
index fddfc7b071..808239bee5 100644
--- a/packages/playwright/src/reporters/teleEmitter.ts
+++ b/packages/playwright/src/reporters/teleEmitter.ts
@@ -200,6 +200,7 @@ export class TeleReporterEmitter implements ReporterV2 {
globalTeardown: config.globalTeardown,
tags: config.tags,
webServer: config.webServer,
+ failOnFlakyTests: config.failOnFlakyTests,
};
}
diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts
index a6196a8115..50b6951177 100644
--- a/packages/playwright/src/runner/tasks.ts
+++ b/packages/playwright/src/runner/tasks.ts
@@ -120,7 +120,7 @@ export class TestRun {
result(): 'failed' | 'passed' {
const hasFailedTests = this.rootSuite?.allTests().some(test => !test.ok());
const hasFlakyTests = this.rootSuite?.allTests().some(test => test.outcome() === 'flaky');
- return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';
+ return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';
}
}
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index 52e56752ee..1e292f153a 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -2079,6 +2079,12 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
*/
configFile?: string;
+ /**
+ * See
+ * [testConfig.failOnFlakyTests](https://playwright.dev/docs/api/class-testconfig#test-config-fail-on-flaky-tests).
+ */
+ failOnFlakyTests: boolean;
+
/**
* See [testConfig.forbidOnly](https://playwright.dev/docs/api/class-testconfig#test-config-forbid-only).
*/
diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts
index fc83a7eb52..ab04bf8970 100644
--- a/tests/playwright-test/config.spec.ts
+++ b/tests/playwright-test/config.spec.ts
@@ -77,9 +77,17 @@ test('should support failOnFlakyTests config option', async ({ runInlineTest })
'playwright.config.ts': `
module.exports = {
failOnFlakyTests: true,
- retries: 1
+ retries: 1,
+ reporter: [['line'], ['./reporter.js']],
};
`,
+ 'reporter.js': `
+ module.exports = class Reporter {
+ onBegin(config) {
+ console.log('reporter.failOnFlakyTests:', config.failOnFlakyTests);
+ }
+ };
+ `,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('flake', async ({}, testInfo) => {
@@ -89,6 +97,7 @@ test('should support failOnFlakyTests config option', async ({ runInlineTest })
}, { 'retries': 1 });
expect(result.exitCode).not.toBe(0);
expect(result.flaky).toBe(1);
+ expect(result.output).toContain('reporter.failOnFlakyTests: true');
});
test('should read config from --config, resolve relative testDir', async ({ runInlineTest }) => {
diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts
index 7c620d85f5..9ded1e17cc 100644
--- a/tests/playwright-test/reporter-blob.spec.ts
+++ b/tests/playwright-test/reporter-blob.spec.ts
@@ -972,6 +972,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {
const config: PlaywrightTestConfig = {
// Runner options:
globalTimeout: 202300,
+ failOnFlakyTests: true,
maxFailures: 3,
metadata: {
'a': 'b',
@@ -1043,6 +1044,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {
expect(json.rootDir).toBe(test.info().outputDir);
expect(json.globalTimeout).toBe(config.globalTimeout);
expect(json.maxFailures).toBe(config.maxFailures);
+ expect(json.failOnFlakyTests).toBe(config.failOnFlakyTests);
expect(json.metadata).toEqual(expect.objectContaining(config.metadata));
expect(json.workers).toBe(2);
expect(json.version).toBeTruthy();