Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(runner): display no projects error across all `test` modes (#34676)

Adam Gastineauaeed1f590934
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index a759e6f366..7aef21faee 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -29,6 +29,7 @@ export { program } from 'playwright-core/lib/cli/program';
import { prepareErrorStack } from './reporters/base';
import { showHTMLReport } from './reporters/html';
import { createMergedReport } from './reporters/merge';
+import { filterProjects } from './runner/projectUtils';
import { Runner } from './runner/runner';
import * as testServer from './runner/testServer';
import { runWatchModeLoop } from './runner/watchMode';
@@ -161,6 +162,23 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
await startProfiling();
const cliOverrides = overridesFromOptions(opts);
+ const config = await loadConfigFromFileRestartIfNeeded(opts.config, cliOverrides, opts.deps === false);
+ if (!config)
+ return;
+
+ config.cliArgs = args;
+ config.cliGrep = opts.grep as string | undefined;
+ config.cliOnlyChanged = opts.onlyChanged === true ? 'HEAD' : opts.onlyChanged;
+ config.cliGrepInvert = opts.grepInvert as string | undefined;
+ config.cliListOnly = !!opts.list;
+ config.cliProjectFilter = opts.project || undefined;
+ config.cliPassWithNoTests = !!opts.passWithNoTests;
+ config.cliFailOnFlakyTests = !!opts.failOnFlakyTests;
+ config.cliLastFailed = !!opts.lastFailed;
+
+ // Evaluate project filters against config before starting execution. This enables a consistent error message across run modes
+ filterProjects(config.projects, config.cliProjectFilter);
+
if (opts.ui || opts.uiHost || opts.uiPort) {
if (opts.onlyChanged)
throw new Error(`--only-changed is not supported in UI mode. If you'd like that to change, see https://github.com/microsoft/playwright/issues/15075 for more details.`);
@@ -202,20 +220,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
return;
}
- const config = await loadConfigFromFileRestartIfNeeded(opts.config, cliOverrides, opts.deps === false);
- if (!config)
- return;
-
- config.cliArgs = args;
- config.cliGrep = opts.grep as string | undefined;
- config.cliOnlyChanged = opts.onlyChanged === true ? 'HEAD' : opts.onlyChanged;
- config.cliGrepInvert = opts.grepInvert as string | undefined;
- config.cliListOnly = !!opts.list;
- config.cliProjectFilter = opts.project || undefined;
- config.cliPassWithNoTests = !!opts.passWithNoTests;
- config.cliFailOnFlakyTests = !!opts.failOnFlakyTests;
- config.cliLastFailed = !!opts.lastFailed;
-
const runner = new Runner(config);
const status = await runner.runAllTests();
await stopProfiling('runner');
diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts
index 7870d74706..518bf33839 100644
--- a/tests/playwright-test/config.spec.ts
+++ b/tests/playwright-test/config.spec.ts
@@ -327,6 +327,25 @@ test('should print nice error when project is unknown', async ({ runInlineTest }
expect(output).toContain('Project(s) "suite3" not found. Available projects: "suite1", "suite2"');
});
+test('should print nice error when project is unknown and launching UI mode', async ({ runInlineTest }) => {
+ // Prevent UI mode from opening and the test never finishing
+ test.setTimeout(5000);
+ const { output, exitCode } = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = { projects: [
+ { name: 'suite1' },
+ { name: 'suite2' },
+ ] };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('pass', async ({}, testInfo) => {});
+ `
+ }, { project: 'suite3', ui: true });
+ expect(exitCode).toBe(1);
+ expect(output).toContain('Project(s) "suite3" not found. Available projects: "suite1", "suite2"');
+});
+
test('should filter by project list, case-insensitive', async ({ runInlineTest }) => {
const { passed, failed, outputLines, skipped } = await runInlineTest({
'playwright.config.ts': `
diff --git a/tests/playwright-test/ui-mode-test-setup.spec.ts b/tests/playwright-test/ui-mode-test-setup.spec.ts
index d3fefb27fc..c912038571 100644
--- a/tests/playwright-test/ui-mode-test-setup.spec.ts
+++ b/tests/playwright-test/ui-mode-test-setup.spec.ts
@@ -95,17 +95,6 @@ test('should teardown on sigint', async ({ runUITest, nodeVersion }) => {
]);
});
-test('should show errors in config', async ({ runUITest }) => {
- const { page } = await runUITest({
- 'playwright.config.ts': `
- import { defineConfig, devices } from '@playwright/test';
- throw new Error("URL is empty")
- `,
- });
- await page.getByText('playwright.config.ts').click();
- await expect(page.getByText('Error: URL is empty')).toBeInViewport();
-});
-
const testsWithSetup = {
'playwright.config.ts': `
import { defineConfig } from '@playwright/test';