Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(reporter): expose failOnFlakyTests on FullConfig (#40935)
Dmitry Gozman3acf04517034
diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.mdindex 627218db7b..e3097cb16f 100644--- a/docs/src/test-api/class-fullconfig.md+++ b/docs/src/test-api/class-fullconfig.md@@ -21,6 +21,12 @@ any argument-parsing library.Path to the configuration file used to run the tests. The value is an empty string if no config file was used.+## property: FullConfig.failOnFlakyTests+* since: v1.61+- type: <[boolean]>++See [`property: TestConfig.failOnFlakyTests`].+## property: FullConfig.forbidOnly* since: v1.10- type: <[boolean]>diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.tsindex 475386f853..2b8ae1a60f 100644--- a/packages/playwright/src/common/config.ts+++ b/packages/playwright/src/common/config.ts@@ -48,7 +48,6 @@ export class FullConfigInternal {readonly projects: FullProjectInternal[] = [];readonly singleTSConfigPath?: string;readonly captureGitInfo: Config['captureGitInfo'];- readonly failOnFlakyTests: boolean;defineConfigWasUsed = false;globalSetups: string[] = [];@@ -68,7 +67,6 @@ export class FullConfigInternal {this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p }));this.singleTSConfigPath = pathResolve(configDir, userConfig.tsconfig);this.captureGitInfo = userConfig.captureGitInfo;- this.failOnFlakyTests = takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false);this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);@@ -87,6 +85,7 @@ export class FullConfigInternal {argv: configCLIOverrides.argv ?? [],configFile: resolvedConfigFile,rootDir: pathResolve(configDir, userConfig.testDir) || configDir,+ failOnFlakyTests: takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false),forbidOnly: takeFirst(configCLIOverrides.forbidOnly, userConfig.forbidOnly, false),fullyParallel: takeFirst(configCLIOverrides.fullyParallel, userConfig.fullyParallel, false),globalSetup: this.globalSetups[0] ?? null,diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.tsindex bd534c9216..bfde074d8d 100644--- a/packages/playwright/src/isomorphic/teleReceiver.ts+++ b/packages/playwright/src/isomorphic/teleReceiver.ts@@ -29,6 +29,7 @@ export type JsonConfig = Pick<reporterTypes.FullConfig, 'configFile' | 'globalTi// optional for backwards compatibilitytags?: reporterTypes.FullConfig['tags'],webServer?: reporterTypes.FullConfig['webServer'],+ failOnFlakyTests?: reporterTypes.FullConfig['failOnFlakyTests'],};export type JsonPattern = {@@ -780,6 +781,7 @@ export type TeleFullProject = reporterTypes.FullProject;export const baseFullConfig: reporterTypes.FullConfig = {argv: [],+ failOnFlakyTests: false,forbidOnly: false,fullyParallel: false,globalSetup: null,diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.tsindex fddfc7b071..808239bee5 100644--- a/packages/playwright/src/reporters/teleEmitter.ts+++ b/packages/playwright/src/reporters/teleEmitter.ts@@ -200,6 +200,7 @@ export class TeleReporterEmitter implements ReporterV2 {globalTeardown: config.globalTeardown,tags: config.tags,webServer: config.webServer,+ failOnFlakyTests: config.failOnFlakyTests,};}diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex a6196a8115..50b6951177 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -120,7 +120,7 @@ export class TestRun {result(): 'failed' | 'passed' {const hasFailedTests = this.rootSuite?.allTests().some(test => !test.ok());const hasFlakyTests = this.rootSuite?.allTests().some(test => test.outcome() === 'flaky');- return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';+ return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';}}diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex 52e56752ee..1e292f153a 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -2079,6 +2079,12 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {*/configFile?: string;+ /**+ * See+ * [testConfig.failOnFlakyTests](https://playwright.dev/docs/api/class-testconfig#test-config-fail-on-flaky-tests).+ */+ failOnFlakyTests: boolean;+/*** See [testConfig.forbidOnly](https://playwright.dev/docs/api/class-testconfig#test-config-forbid-only).*/diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.tsindex fc83a7eb52..ab04bf8970 100644--- a/tests/playwright-test/config.spec.ts+++ b/tests/playwright-test/config.spec.ts@@ -77,9 +77,17 @@ test('should support failOnFlakyTests config option', async ({ runInlineTest })'playwright.config.ts': `module.exports = {failOnFlakyTests: true,- retries: 1+ retries: 1,+ reporter: [['line'], ['./reporter.js']],};`,+ 'reporter.js': `+ module.exports = class Reporter {+ onBegin(config) {+ console.log('reporter.failOnFlakyTests:', config.failOnFlakyTests);+ }+ };+ `,'a.test.js': `import { test, expect } from '@playwright/test';test('flake', async ({}, testInfo) => {@@ -89,6 +97,7 @@ test('should support failOnFlakyTests config option', async ({ runInlineTest })}, { 'retries': 1 });expect(result.exitCode).not.toBe(0);expect(result.flaky).toBe(1);+ expect(result.output).toContain('reporter.failOnFlakyTests: true');});test('should read config from --config, resolve relative testDir', async ({ runInlineTest }) => {diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.tsindex 7c620d85f5..9ded1e17cc 100644--- a/tests/playwright-test/reporter-blob.spec.ts+++ b/tests/playwright-test/reporter-blob.spec.ts@@ -972,6 +972,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {const config: PlaywrightTestConfig = {// Runner options:globalTimeout: 202300,+ failOnFlakyTests: true,maxFailures: 3,metadata: {'a': 'b',@@ -1043,6 +1044,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {expect(json.rootDir).toBe(test.info().outputDir);expect(json.globalTimeout).toBe(config.globalTimeout);expect(json.maxFailures).toBe(config.maxFailures);+ expect(json.failOnFlakyTests).toBe(config.failOnFlakyTests);expect(json.metadata).toEqual(expect.objectContaining(config.metadata));expect(json.workers).toBe(2);expect(json.version).toBeTruthy();