Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(shard): fail when no tests match across all shards (#43173)

Pavel Feldman4dd2a65fe881
diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts
index 80553cb8e2..55759c8ca4 100644
--- a/packages/playwright/src/runner/loadUtils.ts
+++ b/packages/playwright/src/runner/loadUtils.ts
@@ -155,6 +155,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
rootSuite._addSuite(buildProjectSuite(project, filteredProjectSuites.get(project)!));
}
}
+ testRun.testCountBeforeSharding = rootSuite.allTests().length;
// Complain about only.
if (config.config.forbidOnly) {
diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts
index 1361860eab..462dfa5a05 100644
--- a/packages/playwright/src/runner/tasks.ts
+++ b/packages/playwright/src/runner/tasks.ts
@@ -91,6 +91,7 @@ export class TestRun {
topLevelProjects: commonConfig.FullProjectInternal[] = [];
hasWorkerErrors = false;
failedTestCount = 0;
+ testCountBeforeSharding = 0;
readonly loadFileFilters: Matcher[] = [];
readonly preOnlyTestFilters: TestCaseFilter[] = [];
readonly postShardTestFilters: TestCaseFilter[] = [];
@@ -335,9 +336,10 @@ export function createLoadTask(mode: 'out-of-process' | 'in-process', options: {
await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);
// Fail when no tests.
- if (options.failOnLoadErrors && !testRun.rootSuite?.allTests().length
+ const testCount = testRun.config.config.shard ? testRun.testCountBeforeSharding : testRun.rootSuite?.allTests().length;
+ if (options.failOnLoadErrors && !testCount
&& !testRun.options.passWithNoTests
- && !testRun.config.config.shard && !testRun.options.onlyChanged
+ && !testRun.options.onlyChanged
&& !testRun.options.testList && !testRun.options.testListInvert) {
if (testRun.options.locations?.length) {
throw new Error([
diff --git a/tests/playwright-test/reporter-preprocess.spec.ts b/tests/playwright-test/reporter-preprocess.spec.ts
index 66a84ba4fb..6ff4e2c790 100644
--- a/tests/playwright-test/reporter-preprocess.spec.ts
+++ b/tests/playwright-test/reporter-preprocess.spec.ts
@@ -369,6 +369,30 @@ test('skipSharding disables the built-in shard filter; preprocess sees the full
expect(result.outputLines).toContain('begin: t0,t2');
});
+test('custom sharding may produce an empty shard', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'reporter.ts': `
+ class R {
+ async preprocess({ suite, testRun }) {
+ testRun.skipSharding();
+ for (const t of suite.allTests())
+ testRun.exclude(t);
+ }
+ }
+ module.exports = R;
+ `,
+ 'playwright.config.ts': `module.exports = { reporter: './reporter.ts', shard: { current: 2, total: 2 } };`,
+ 'a.test.ts': `
+ import { test } from '@playwright/test';
+ test('t0', async () => {});
+ `,
+ }, { reporter: '', workers: 1 });
+
+ expect(result.exitCode).toBe(0);
+ expect(result.passed).toBe(0);
+ expect(result.output).not.toContain('No tests found');
+});
+
test('multiple reporters declaring custom sharding throws', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter-a.ts': `
diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.ts
index 15774e8e37..a700812e0b 100644
--- a/tests/playwright-test/shard.spec.ts
+++ b/tests/playwright-test/shard.spec.ts
@@ -156,6 +156,25 @@ test('should not produce skipped tests for zero-sized shards', async ({ runInlin
expect(result.outputLines).toEqual([]);
});
+test('should fail when no tests match across all shards', async ({ runInlineTest }) => {
+ const result = await runInlineTest(tests, { shard: '1/2', grep: 'does-not-match' });
+ expect(result.exitCode).toBe(1);
+ expect(result.output).toContain('No tests found');
+});
+
+test('should fail when no test files match across all shards', async ({ runInlineTest }) => {
+ const result = await runInlineTest(tests, { shard: '1/2' }, {}, { additionalArgs: ['does-not-match'] });
+ expect(result.exitCode).toBe(1);
+ expect(result.output).toContain('No tests found');
+ expect(result.output).toContain('Make sure that arguments are regular expressions matching test files');
+});
+
+test('should pass with no tests across all shards and --pass-with-no-tests', async ({ runInlineTest }) => {
+ const result = await runInlineTest(tests, { 'shard': '1/2', 'grep': 'does-not-match', 'pass-with-no-tests': true });
+ expect(result.exitCode).toBe(0);
+ expect(result.passed).toBe(0);
+});
+
test('should respect shard=1/2 in config', async ({ runInlineTest }) => {
const result = await runInlineTest({
...tests,