Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(config): failOnFlakyTests option (#35109)

Jean-François Greffier85a66912c65b
diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md
index edbc991bfd..fe80475ed6 100644
--- a/docs/src/test-api/class-fullconfig.md
+++ b/docs/src/test-api/class-fullconfig.md
@@ -10,6 +10,12 @@ Resolved configuration which is accessible via [`property: TestInfo.config`] and
Path to the configuration file used to run the tests. The value is an empty string if no config file was used.
+## property: FullConfig.failOnFlakyTests
+* since: v1.52
+- type: <[boolean]>
+
+See [`property: TestConfig.failOnFlakyTests`].
+
## property: FullConfig.forbidOnly
* since: v1.10
- type: <[boolean]>
diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md
index d5c2eb1ae6..b0cf02ccc2 100644
--- a/docs/src/test-api/class-testconfig.md
+++ b/docs/src/test-api/class-testconfig.md
@@ -108,6 +108,24 @@ export default defineConfig({
});
```
+## property: TestConfig.failOnFlakyTests
+* since: v1.52
+- type: ?<[boolean]>
+
+Whether to exit with an error if any tests are marked as flaky. Useful on CI.
+
+Also available in the [command line](../test-cli.md) with the `--fail-on-flaky-tests` option.
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+ failOnFlakyTests: !!process.env.CI,
+});
+```
+
## property: TestConfig.forbidOnly
* since: v1.10
- type: ?<[boolean]>
diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts
index f91f9d3989..c41af72843 100644
--- a/packages/playwright/src/common/config.ts
+++ b/packages/playwright/src/common/config.ts
@@ -56,7 +56,6 @@ export class FullConfigInternal {
cliProjectFilter?: string[];
cliListOnly = false;
cliPassWithNoTests?: boolean;
- cliFailOnFlakyTests?: boolean;
cliLastFailed?: boolean;
testIdMatcher?: Matcher;
lastFailedTestIdMatcher?: Matcher;
@@ -90,6 +89,7 @@ export class FullConfigInternal {
this.config = {
configFile: resolvedConfigFile,
rootDir: pathResolve(configDir, userConfig.testDir) || configDir,
+ failOnFlakyTests: takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false),
forbidOnly: takeFirst(configCLIOverrides.forbidOnly, userConfig.forbidOnly, false),
fullyParallel: takeFirst(configCLIOverrides.fullyParallel, userConfig.fullyParallel, false),
globalSetup: this.globalSetups[0] ?? null,
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index b13ba4df72..9fd819c6bc 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -25,6 +25,7 @@ import type { SerializedCompilationCache } from '../transform/compilationCache'
export type ConfigCLIOverrides = {
debug?: boolean;
+ failOnFlakyTests?: boolean;
forbidOnly?: boolean;
fullyParallel?: boolean;
globalTimeout?: number;
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index 29b1beb74d..7143fcf822 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -593,6 +593,7 @@ export class TeleTestResult implements reporterTypes.TestResult {
export type TeleFullProject = reporterTypes.FullProject;
export const baseFullConfig: reporterTypes.FullConfig = {
+ failOnFlakyTests: false,
forbidOnly: false,
fullyParallel: false,
globalSetup: null,
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index bc899b48c6..eb9bb84eec 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -172,7 +172,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
config.cliListOnly = !!opts.list;
config.cliProjectFilter = opts.project || undefined;
config.cliPassWithNoTests = !!opts.passWithNoTests;
- config.cliFailOnFlakyTests = !!opts.failOnFlakyTests;
config.cliLastFailed = !!opts.lastFailed;
// Evaluate project filters against config before starting execution. This enables a consistent error message across run modes
@@ -294,6 +293,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
updateSnapshots = 'updateSnapshots' in options ? 'changed' : undefined;
const overrides: ConfigCLIOverrides = {
+ failOnFlakyTests: options.failOnFlakyTests ? true : undefined,
forbidOnly: options.forbidOnly ? true : undefined,
fullyParallel: options.fullyParallel ? true : undefined,
globalTimeout: options.globalTimeout ? parseInt(options.globalTimeout, 10) : undefined,
diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.ts
index ac62677571..afc4548a66 100644
--- a/packages/playwright/src/runner/failureTracker.ts
+++ b/packages/playwright/src/runner/failureTracker.ts
@@ -49,7 +49,7 @@ export class FailureTracker {
}
result(): 'failed' | 'passed' {
- return this._hasWorkerErrors || this.hasReachedMaxFailures() || this.hasFailedTests() || (this._config.cliFailOnFlakyTests && this.hasFlakyTests()) ? 'failed' : 'passed';
+ return this._hasWorkerErrors || this.hasReachedMaxFailures() || this.hasFailedTests() || (this._config.config.failOnFlakyTests && this.hasFlakyTests()) ? 'failed' : 'passed';
}
hasFailedTests() {
@@ -63,4 +63,5 @@ export class FailureTracker {
maxFailures() {
return this._config.config.maxFailures;
}
+
}
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index 41505270af..54725c0bc7 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -1177,6 +1177,25 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
};
};
+ /**
+ * Whether to exit with an error if any tests are marked as flaky. Useful on CI.
+ *
+ * Also available in the [command line](https://playwright.dev/docs/test-cli) with the `--fail-on-flaky-tests` option.
+ *
+ * **Usage**
+ *
+ * ```js
+ * // playwright.config.ts
+ * import { defineConfig } from '@playwright/test';
+ *
+ * export default defineConfig({
+ * failOnFlakyTests: !!process.env.CI,
+ * });
+ * ```
+ *
+ */
+ failOnFlakyTests?: boolean;
+
/**
* Whether to exit with an error if any tests or groups are marked as
* [test.only(title[, details, body])](https://playwright.dev/docs/api/class-test#test-only) or
@@ -1911,6 +1930,12 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
*/
configFile?: string;
+ /**
+ * See
+ * [testConfig.failOnFlakyTests](https://playwright.dev/docs/api/class-testconfig#test-config-fail-on-flaky-tests).
+ */
+ failOnFlakyTests: boolean;
+
/**
* See [testConfig.forbidOnly](https://playwright.dev/docs/api/class-testconfig#test-config-forbid-only).
*/
diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts
index 77e5fb23f6..30945fdc1e 100644
--- a/tests/playwright-test/config.spec.ts
+++ b/tests/playwright-test/config.spec.ts
@@ -72,6 +72,25 @@ test('should prioritize command line timeout over project timeout', async ({ run
expect(result.output).toContain('Test timeout of 500ms exceeded.');
});
+test('should support failOnFlakyTests config option', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = {
+ failOnFlakyTests: true,
+ retries: 1
+ };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('flake', async ({}, testInfo) => {
+ expect(testInfo.retry).toBe(1);
+ });
+ `,
+ }, { 'retries': 1 });
+ expect(result.exitCode).not.toBe(0);
+ expect(result.flaky).toBe(1);
+});
+
test('should read config from --config, resolve relative testDir', async ({ runInlineTest }) => {
const result = await runInlineTest({
'my.config.ts': `