Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(config): failOnFlakyTests option (#35109)
Jean-François Greffier85a66912c65b
diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.mdindex edbc991bfd..fe80475ed6 100644--- a/docs/src/test-api/class-fullconfig.md+++ b/docs/src/test-api/class-fullconfig.md@@ -10,6 +10,12 @@ Resolved configuration which is accessible via [`property: TestInfo.config`] andPath to the configuration file used to run the tests. The value is an empty string if no config file was used.+## property: FullConfig.failOnFlakyTests+* since: v1.52+- type: <[boolean]>++See [`property: TestConfig.failOnFlakyTests`].+## property: FullConfig.forbidOnly* since: v1.10- type: <[boolean]>diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.mdindex d5c2eb1ae6..b0cf02ccc2 100644--- a/docs/src/test-api/class-testconfig.md+++ b/docs/src/test-api/class-testconfig.md@@ -108,6 +108,24 @@ export default defineConfig({});```+## property: TestConfig.failOnFlakyTests+* since: v1.52+- type: ?<[boolean]>++Whether to exit with an error if any tests are marked as flaky. Useful on CI.++Also available in the [command line](../test-cli.md) with the `--fail-on-flaky-tests` option.++**Usage**++```js title="playwright.config.ts"+import { defineConfig } from '@playwright/test';++export default defineConfig({+ failOnFlakyTests: !!process.env.CI,+});+```+## property: TestConfig.forbidOnly* since: v1.10- type: ?<[boolean]>diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.tsindex f91f9d3989..c41af72843 100644--- a/packages/playwright/src/common/config.ts+++ b/packages/playwright/src/common/config.ts@@ -56,7 +56,6 @@ export class FullConfigInternal {cliProjectFilter?: string[];cliListOnly = false;cliPassWithNoTests?: boolean;- cliFailOnFlakyTests?: boolean;cliLastFailed?: boolean;testIdMatcher?: Matcher;lastFailedTestIdMatcher?: Matcher;@@ -90,6 +89,7 @@ export class FullConfigInternal {this.config = {configFile: resolvedConfigFile,rootDir: pathResolve(configDir, userConfig.testDir) || configDir,+ failOnFlakyTests: takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false),forbidOnly: takeFirst(configCLIOverrides.forbidOnly, userConfig.forbidOnly, false),fullyParallel: takeFirst(configCLIOverrides.fullyParallel, userConfig.fullyParallel, false),globalSetup: this.globalSetups[0] ?? null,diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.tsindex b13ba4df72..9fd819c6bc 100644--- a/packages/playwright/src/common/ipc.ts+++ b/packages/playwright/src/common/ipc.ts@@ -25,6 +25,7 @@ import type { SerializedCompilationCache } from '../transform/compilationCache'export type ConfigCLIOverrides = {debug?: boolean;+ failOnFlakyTests?: boolean;forbidOnly?: boolean;fullyParallel?: boolean;globalTimeout?: number;diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.tsindex 29b1beb74d..7143fcf822 100644--- a/packages/playwright/src/isomorphic/teleReceiver.ts+++ b/packages/playwright/src/isomorphic/teleReceiver.ts@@ -593,6 +593,7 @@ export class TeleTestResult implements reporterTypes.TestResult {export type TeleFullProject = reporterTypes.FullProject;export const baseFullConfig: reporterTypes.FullConfig = {+ failOnFlakyTests: false,forbidOnly: false,fullyParallel: false,globalSetup: null,diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex bc899b48c6..eb9bb84eec 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -172,7 +172,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) {config.cliListOnly = !!opts.list;config.cliProjectFilter = opts.project || undefined;config.cliPassWithNoTests = !!opts.passWithNoTests;- config.cliFailOnFlakyTests = !!opts.failOnFlakyTests;config.cliLastFailed = !!opts.lastFailed;// Evaluate project filters against config before starting execution. This enables a consistent error message across run modes@@ -294,6 +293,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverridupdateSnapshots = 'updateSnapshots' in options ? 'changed' : undefined;const overrides: ConfigCLIOverrides = {+ failOnFlakyTests: options.failOnFlakyTests ? true : undefined,forbidOnly: options.forbidOnly ? true : undefined,fullyParallel: options.fullyParallel ? true : undefined,globalTimeout: options.globalTimeout ? parseInt(options.globalTimeout, 10) : undefined,diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.tsindex ac62677571..afc4548a66 100644--- a/packages/playwright/src/runner/failureTracker.ts+++ b/packages/playwright/src/runner/failureTracker.ts@@ -49,7 +49,7 @@ export class FailureTracker {}result(): 'failed' | 'passed' {- return this._hasWorkerErrors || this.hasReachedMaxFailures() || this.hasFailedTests() || (this._config.cliFailOnFlakyTests && this.hasFlakyTests()) ? 'failed' : 'passed';+ return this._hasWorkerErrors || this.hasReachedMaxFailures() || this.hasFailedTests() || (this._config.config.failOnFlakyTests && this.hasFlakyTests()) ? 'failed' : 'passed';}hasFailedTests() {@@ -63,4 +63,5 @@ export class FailureTracker {maxFailures() {return this._config.config.maxFailures;}+}diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex 41505270af..54725c0bc7 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -1177,6 +1177,25 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {};};+ /**+ * Whether to exit with an error if any tests are marked as flaky. Useful on CI.+ *+ * Also available in the [command line](https://playwright.dev/docs/test-cli) with the `--fail-on-flaky-tests` option.+ *+ * **Usage**+ *+ * ```js+ * // playwright.config.ts+ * import { defineConfig } from '@playwright/test';+ *+ * export default defineConfig({+ * failOnFlakyTests: !!process.env.CI,+ * });+ * ```+ *+ */+ failOnFlakyTests?: boolean;+/*** Whether to exit with an error if any tests or groups are marked as* [test.only(title[, details, body])](https://playwright.dev/docs/api/class-test#test-only) or@@ -1911,6 +1930,12 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {*/configFile?: string;+ /**+ * See+ * [testConfig.failOnFlakyTests](https://playwright.dev/docs/api/class-testconfig#test-config-fail-on-flaky-tests).+ */+ failOnFlakyTests: boolean;+/*** See [testConfig.forbidOnly](https://playwright.dev/docs/api/class-testconfig#test-config-forbid-only).*/diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.tsindex 77e5fb23f6..30945fdc1e 100644--- a/tests/playwright-test/config.spec.ts+++ b/tests/playwright-test/config.spec.ts@@ -72,6 +72,25 @@ test('should prioritize command line timeout over project timeout', async ({ runexpect(result.output).toContain('Test timeout of 500ms exceeded.');});+test('should support failOnFlakyTests config option', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = {+ failOnFlakyTests: true,+ retries: 1+ };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('flake', async ({}, testInfo) => {+ expect(testInfo.retry).toBe(1);+ });+ `,+ }, { 'retries': 1 });+ expect(result.exitCode).not.toBe(0);+ expect(result.flaky).toBe(1);+});+test('should read config from --config, resolve relative testDir', async ({ runInlineTest }) => {const result = await runInlineTest({'my.config.ts': `