Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(ui): always pass test locations to skip loading unnecessary test files (#38235)

Simon Knott0b4873533bfe
diff --git a/packages/playwright/src/isomorphic/testServerInterface.ts b/packages/playwright/src/isomorphic/testServerInterface.ts
index 666ea6c5ba..fcc34bb0c9 100644
--- a/packages/playwright/src/isomorphic/testServerInterface.ts
+++ b/packages/playwright/src/isomorphic/testServerInterface.ts
@@ -88,7 +88,7 @@ export interface TestServerInterface {
}>;
runTests(params: {
- locations?: string[];
+ locations: string[];
grep?: string;
grepInvert?: string;
testIds?: string[];
diff --git a/packages/playwright/src/isomorphic/testTree.ts b/packages/playwright/src/isomorphic/testTree.ts
index 713b0d6d5d..419c3e7d04 100644
--- a/packages/playwright/src/isomorphic/testTree.ts
+++ b/packages/playwright/src/isomorphic/testTree.ts
@@ -292,17 +292,6 @@ export class TestTree {
this.rootItem = shortRoot;
}
- testIds(): Set<string> {
- const result = new Set<string>();
- const visit = (treeItem: TreeItem) => {
- if (treeItem.kind === 'case')
- treeItem.tests.forEach(t => result.add(t.id));
- treeItem.children.forEach(visit);
- };
- visit(this.rootItem);
- return result;
- }
-
fileNames(): string[] {
const result = new Set<string>();
const visit = (treeItem: TreeItem) => {
@@ -329,8 +318,8 @@ export class TestTree {
return this._treeItemById.get(id);
}
- collectTestIds(treeItem?: TreeItem): Set<string> {
- return treeItem ? collectTestIds(treeItem) : new Set();
+ collectTestIds(treeItem: TreeItem) {
+ return collectTestIds(treeItem);
}
}
@@ -371,18 +360,27 @@ export function sortAndPropagateStatus(treeItem: TreeItem) {
treeItem.status = 'passed';
}
-export function collectTestIds(treeItem: TreeItem): Set<string> {
+function collectTestIds(treeItem: TreeItem): { testIds: Set<string>, locations: Set<string> } {
const testIds = new Set<string>();
+ const locations = new Set<string>();
const visit = (treeItem: TreeItem) => {
+ if (treeItem.kind !== 'test' && treeItem.kind !== 'case') {
+ treeItem.children.forEach(visit);
+ return;
+ }
+
+ let fileItem: TreeItem = treeItem;
+ while (fileItem && fileItem.parent && !(fileItem.kind === 'group' && fileItem.subKind === 'file'))
+ fileItem = fileItem.parent;
+ locations.add(fileItem.location.file);
+
if (treeItem.kind === 'case')
- treeItem.tests.map(t => t.id).forEach(id => testIds.add(id));
- else if (treeItem.kind === 'test')
- testIds.add(treeItem.id);
+ treeItem.tests.forEach(test => testIds.add(test.id));
else
- treeItem.children?.forEach(visit);
+ testIds.add(treeItem.id);
};
visit(treeItem);
- return testIds;
+ return { testIds, locations };
}
export const statusEx = Symbol('statusEx');
diff --git a/packages/playwright/src/mcp/test/testTools.ts b/packages/playwright/src/mcp/test/testTools.ts
index 2c57ecc81c..29b185048c 100644
--- a/packages/playwright/src/mcp/test/testTools.ts
+++ b/packages/playwright/src/mcp/test/testTools.ts
@@ -49,7 +49,7 @@ export const runTests = defineTestTool({
handle: async (context, params) => {
const { output } = await context.runTestsWithGlobalSetupAndPossiblePause({
- locations: params.locations,
+ locations: params.locations ?? [],
projects: params.projects,
disableConfigReporters: true,
});
@@ -74,6 +74,7 @@ export const debugTest = defineTestTool({
handle: async (context, params) => {
const { output, status } = await context.runTestsWithGlobalSetupAndPossiblePause({
headed: context.computedHeaded,
+ locations: [], // we can make this faster by passing the test's location, so we don't need to scan all tests to find the ID
testIds: [params.test.id],
// For automatic recovery
timeout: 0,
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index 596ac10a50..ec2397e31f 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -60,7 +60,7 @@ export type ListTestsParams = {
export type RunTestsParams = {
timeout?: number;
- locations?: string[];
+ locations: string[];
grep?: string;
grepInvert?: string;
testIds?: string[];
@@ -331,7 +331,7 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {
config.cliListOnly = false;
config.cliPassWithNoTests = true;
- config.cliArgs = params.locations || [];
+ config.cliArgs = params.locations;
config.cliGrep = params.grep;
config.cliGrepInvert = params.grepInvert;
config.cliProjectFilter = params.projects?.length ? params.projects : undefined;
diff --git a/packages/playwright/src/runner/watchMode.ts b/packages/playwright/src/runner/watchMode.ts
index ce147dc338..e63cf174b9 100644
--- a/packages/playwright/src/runner/watchMode.ts
+++ b/packages/playwright/src/runner/watchMode.ts
@@ -316,7 +316,7 @@ async function runTests(watchOptions: WatchModeOptions, testServerConnection: Te
await testServerConnection.runTests({
grep: watchOptions.grep,
testIds: options?.testIds,
- locations: watchOptions?.files,
+ locations: watchOptions?.files ?? [], // TODO: always collect locations based on knowledge about tree, so that we don't have to load all tests
projects: watchOptions.projects,
connectWsEndpoint,
reuseContext: connectWsEndpoint ? true : undefined,
diff --git a/packages/trace-viewer/src/ui/uiModeTestListView.tsx b/packages/trace-viewer/src/ui/uiModeTestListView.tsx
index 1d6ddd850c..0c900b0f3c 100644
--- a/packages/trace-viewer/src/ui/uiModeTestListView.tsx
+++ b/packages/trace-viewer/src/ui/uiModeTestListView.tsx
@@ -39,7 +39,7 @@ export const TestListView: React.FC<{
testTree: TestTree,
testServerConnection: TestServerConnection | undefined,
testModel?: TeleSuiteUpdaterTestModel,
- runTests: (mode: 'bounce-if-busy' | 'queue-if-busy', testIds: Set<string>) => void,
+ runTests: (mode: 'bounce-if-busy' | 'queue-if-busy', filter: { testIds: Iterable<string>, locations: Iterable<string> }) => void,
runningState?: { testIds: Set<string>, itemSelectedByUser?: boolean, completed?: boolean },
watchAll: boolean,
watchedTreeIds: { value: Set<string> },
diff --git a/packages/trace-viewer/src/ui/uiModeView.tsx b/packages/trace-viewer/src/ui/uiModeView.tsx
index c81726e803..227c0ef332 100644
--- a/packages/trace-viewer/src/ui/uiModeView.tsx
+++ b/packages/trace-viewer/src/ui/uiModeView.tsx
@@ -66,6 +66,14 @@ if (queryParams.updateSnapshots && !['all', 'changed', 'none', 'missing'].includ
const isMac = navigator.platform === 'MacIntel';
+function escapeRegex(text: string) {
+ // playwright interprets absolute paths as regex,
+ // removing the leading slash prevents that.
+ if (text.startsWith('/'))
+ text = text.substring(1);
+ return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
export const UIModeView: React.FC<{}> = ({
}) => {
const [filterText, setFilterText] = React.useState<string>('');
@@ -80,7 +88,6 @@ export const UIModeView: React.FC<{}> = ({
const [testModel, setTestModel] = React.useState<TeleSuiteUpdaterTestModel>();
const [progress, setProgress] = React.useState<TeleSuiteUpdaterProgress & { total: number } | undefined>();
const [selectedItem, setSelectedItem] = React.useState<{ treeItem?: TreeItem, testFile?: SourceLocation, testCase?: reporterTypes.TestCase }>({});
- const [visibleTestIds, setVisibleTestIds] = React.useState<Set<string>>(new Set());
const [isLoading, setIsLoading] = React.useState<boolean>(false);
const [runningState, setRunningState] = React.useState<{ testIds: Set<string>, itemSelectedByUser?: boolean, completed?: boolean } | undefined>();
const isRunningTest = runningState && !runningState.completed;
@@ -88,7 +95,7 @@ export const UIModeView: React.FC<{}> = ({
const [watchAll, setWatchAll] = useSetting<boolean>('watch-all', false);
const [watchedTreeIds, setWatchedTreeIds] = React.useState<{ value: Set<string> }>({ value: new Set() });
const commandQueue = React.useRef(Promise.resolve());
- const runTestBacklog = React.useRef<Set<string>>(new Set());
+ const runTestBacklog = React.useRef<{ testIds: Set<string>, locations: Set<string> }>({ testIds: new Set(), locations: new Set() });
const [collapseAllCount, setCollapseAllCount] = React.useState(0);
const [expandAllCount, setExpandAllCount] = React.useState(0);
const [isDisconnected, setIsDisconnected] = React.useState(false);
@@ -247,20 +254,22 @@ export const UIModeView: React.FC<{}> = ({
testTree.sortAndPropagateStatus();
testTree.shortenRoot();
testTree.flattenForSingleProject();
- setVisibleTestIds(testTree.testIds());
return { testTree };
- }, [filterText, testModel, statusFilters, projectFilters, setVisibleTestIds, runningState, isRunningTest, mergeFiles]);
+ }, [filterText, testModel, statusFilters, projectFilters, runningState, isRunningTest, mergeFiles]);
- const runTests = React.useCallback((mode: 'queue-if-busy' | 'bounce-if-busy', testIds: Set<string>) => {
+ const runTests = React.useCallback((mode: 'queue-if-busy' | 'bounce-if-busy', filter: { testIds: Iterable<string>, locations: Iterable<string> }) => {
if (!testServerConnection || !testModel)
return;
if (mode === 'bounce-if-busy' && isRunningTest)
return;
- runTestBacklog.current = new Set([...runTestBacklog.current, ...testIds]);
+ for (const testId of filter.testIds)
+ runTestBacklog.current.testIds.add(testId);
+ for (const location of filter.locations)
+ runTestBacklog.current.locations.add(location);
commandQueue.current = commandQueue.current.then(async () => {
- const testIds = runTestBacklog.current;
- runTestBacklog.current = new Set();
+ const { testIds, locations } = runTestBacklog.current;
+ runTestBacklog.current = { testIds: new Set(), locations: new Set() };
if (!testIds.size)
return;
@@ -282,7 +291,7 @@ export const UIModeView: React.FC<{}> = ({
setRunningState({ testIds });
await testServerConnection.runTests({
- locations: queryParams.args,
+ locations: [...locations].map(escapeRegex),
grep: queryParams.grep,
grepInvert: queryParams.grepInvert,
testIds: [...testIds],
@@ -302,6 +311,8 @@ export const UIModeView: React.FC<{}> = ({
});
}, [projectFilters, isRunningTest, testModel, testServerConnection, updateSnapshots, singleWorker]);
+ const runVisibleTests = React.useCallback(() => runTests('bounce-if-busy', testTree.collectTestIds(testTree.rootItem)), [runTests, testTree]);
+
React.useEffect(() => {
if (!testServerConnection || !teleSuiteUpdater)
return;
@@ -328,13 +339,17 @@ export const UIModeView: React.FC<{}> = ({
const testModel = teleSuiteUpdater.asModel();
const testTree = new TestTree('', testModel.rootSuite, testModel.loadErrors, projectFilters, queryParams.pathSeparator, mergeFiles);
+ const locations: string[] = [];
const testIds: string[] = [];
const set = new Set(params.testFiles);
if (watchAll) {
const visit = (treeItem: TreeItem) => {
const fileName = treeItem.location.file;
- if (fileName && set.has(fileName))
- testIds.push(...testTree.collectTestIds(treeItem));
+ if (fileName && set.has(fileName)) {
+ const filter = testTree.collectTestIds(treeItem);
+ locations.push(...filter.locations);
+ testIds.push(...filter.testIds);
+ }
if (treeItem.kind === 'group' && treeItem.subKind === 'folder')
treeItem.children.forEach(visit);
};
@@ -342,12 +357,22 @@ export const UIModeView: React.FC<{}> = ({
} else {
for (const treeId of watchedTreeIds.value) {
const treeItem = testTree.treeItemById(treeId);
- const fileName = treeItem?.location.file;
- if (fileName && set.has(fileName))
- testIds.push(...testTree.collectTestIds(treeItem));
+ if (!treeItem)
+ continue;
+
+ let fileItem = treeItem;
+ while (!(fileItem.kind === 'group' && (fileItem.subKind === 'file' || fileItem.subKind === 'folder')) && fileItem.parent)
+ fileItem = fileItem.parent;
+ const fileName = fileItem?.location.file;
+ if (fileName && set.has(fileName)) {
+ const filter = testTree.collectTestIds(treeItem);
+ locations.push(...filter.locations);
+ testIds.push(...filter.testIds);
+ }
}
}
- runTests('queue-if-busy', new Set(testIds));
+
+ runTests('queue-if-busy', { locations, testIds });
});
return () => disposable.dispose();
}, [runTests, testServerConnection, watchAll, watchedTreeIds, teleSuiteUpdater, projectFilters, mergeFiles]);
@@ -365,14 +390,14 @@ export const UIModeView: React.FC<{}> = ({
testServerConnection?.stopTestsNoReply({});
} else if (e.code === 'F5') {
e.preventDefault();
- runTests('bounce-if-busy', visibleTestIds);
+ runVisibleTests();
}
};
addEventListener('keydown', onShortcutEvent);
return () => {
removeEventListener('keydown', onShortcutEvent);
};
- }, [runTests, reloadTests, testServerConnection, visibleTestIds, isShowingOutput]);
+ }, [runVisibleTests, reloadTests, testServerConnection, isShowingOutput]);
const dialogRef = React.useRef<HTMLDialogElement>(null);
const openInstallDialog = React.useCallback((e: React.MouseEvent) => {
@@ -456,7 +481,7 @@ export const UIModeView: React.FC<{}> = ({
projectFilters={projectFilters}
setProjectFilters={setProjectFilters}
testModel={testModel}
- runTests={() => runTests('bounce-if-busy', visibleTestIds)} />
+ runTests={runVisibleTests} />
<Toolbar className='section-toolbar' noMinHeight={true}>
{!isRunningTest && !progress && <div className='section-title'>Tests</div>}
{!isRunningTest && progress && <div data-testid='status-line' className='status-line'>
@@ -465,7 +490,7 @@ export const UIModeView: React.FC<{}> = ({
{isRunningTest && progress && <div data-testid='status-line' className='status-line'>
<div>Running {progress.passed}/{runningState.testIds.size} passed ({(progress.passed / runningState.testIds.size) * 100 | 0}%)</div>
</div>}
- <ToolbarButton icon='play' title='Run all — F5' onClick={() => runTests('bounce-if-busy', visibleTestIds)} disabled={isRunningTest || isLoading}></ToolbarButton>
+ <ToolbarButton icon='play' title='Run all — F5' onClick={runVisibleTests} disabled={isRunningTest || isLoading}></ToolbarButton>
<ToolbarButton icon='debug-stop' title={'Stop — ' + (isMac ? '⇧F5' : 'Shift + F5')} onClick={() => testServerConnection?.stopTests({})} disabled={!isRunningTest || isLoading}></ToolbarButton>
<ToolbarButton icon='eye' title='Watch all' toggled={watchAll} onClick={() => {
setWatchedTreeIds({ value: new Set() });
diff --git a/tests/playwright-test/test-server.spec.ts b/tests/playwright-test/test-server.spec.ts
index c70eb1fd8b..a83ca3910a 100644
--- a/tests/playwright-test/test-server.spec.ts
+++ b/tests/playwright-test/test-server.spec.ts
@@ -60,6 +60,7 @@ class TestServerConnectionUnderTest extends TestServerConnection {
this.onStdio(params => this.events.push(['stdio', params]));
this.onLoadTraceRequested(params => this.events.push(['loadTraceRequested', params]));
this.onTestPaused(params => this.events.push(['testPaused', params]));
+ this.onReport(params => this.events.push(['report', params]));
}
}
@@ -166,7 +167,7 @@ test('stdio interception', async ({ startTestServer, writeFiles }) => {
`,
});
- const tests = await testServerConnection.runTests({ trace: 'on' });
+ const tests = await testServerConnection.runTests({ trace: 'on', locations: [] });
expect(tests).toEqual({ status: 'passed' });
await expect.poll(() => testServerConnection.events).toEqual(expect.arrayContaining([
['stdio', { type: 'stderr', text: 'this goes to stderr\n' }],
@@ -247,7 +248,7 @@ test('timeout override', async ({ startTestServer, writeFiles }) => {
`,
});
- expect(await testServerConnection.runTests({ timeout: 42 })).toEqual({ status: 'passed' });
+ expect(await testServerConnection.runTests({ timeout: 42, locations: [] })).toEqual({ status: 'passed' });
});
test('PLAYWRIGHT_TEST environment variable', async ({ startTestServer, writeFiles }) => {
@@ -261,7 +262,7 @@ test('PLAYWRIGHT_TEST environment variable', async ({ startTestServer, writeFile
});
`,
});
- expect(await testServerConnection.runTests({})).toEqual({ status: 'passed' });
+ expect(await testServerConnection.runTests({ locations: [] })).toEqual({ status: 'passed' });
});
test('pauseAtEnd', async ({ startTestServer, writeFiles }) => {
@@ -275,7 +276,7 @@ test('pauseAtEnd', async ({ startTestServer, writeFiles }) => {
`,
});
- const promise = testServerConnection.runTests({ pauseAtEnd: true });
+ const promise = testServerConnection.runTests({ pauseAtEnd: true, locations: [] });
await expect.poll(() => testServerConnection.events.find(e => e[0] === 'testPaused')).toEqual(['testPaused', { errors: [] }]);
await testServerConnection.stopTests({});
expect(await promise).toEqual({ status: 'interrupted' });
@@ -293,7 +294,7 @@ test('pauseOnError', async ({ startTestServer, writeFiles }) => {
`,
});
- const promise = testServerConnection.runTests({ pauseOnError: true });
+ const promise = testServerConnection.runTests({ pauseOnError: true, locations: [] });
await expect.poll(() => testServerConnection.events.some(e => e[0] === 'testPaused')).toBeTruthy();
expect(testServerConnection.events.find(e => e[0] === 'testPaused')[1]).toEqual({
errors: [
@@ -324,6 +325,6 @@ test('pauseOnError no errors', async ({ startTestServer, writeFiles }) => {
`,
});
- expect(await testServerConnection.runTests({ pauseOnError: true })).toEqual({ status: 'passed' });
+ expect(await testServerConnection.runTests({ pauseOnError: true, locations: [] })).toEqual({ status: 'passed' });
expect(testServerConnection.events.filter(e => e[0] === 'testPaused')).toEqual([]);
});
diff --git a/tests/playwright-test/ui-mode-test-run.spec.ts b/tests/playwright-test/ui-mode-test-run.spec.ts
index 3b2351083a..e99c13b9a5 100644
--- a/tests/playwright-test/ui-mode-test-run.spec.ts
+++ b/tests/playwright-test/ui-mode-test-run.spec.ts
@@ -798,3 +798,38 @@ test('should not leak websocket connections', {
await expect.poll(() => ws1.isClosed()).toBe(true);
});
+
+test('should run test defined outside of .spec.ts file', async ({ runUITest }) => {
+ const { page } = await runUITest({
+ 'example.spec.ts': `
+ import './impl';
+ `,
+ 'impl.ts': `
+ import { test } from '@playwright/test';
+ test('one', async () => {});
+ `,
+ });
+ await page.getByRole('treeitem', { name: 'one' }).dblclick();
+ await expect.poll(dumpTestTree(page)).toBe(`
+ ▼ ✅ example.spec.ts
+ ✅ one <=
+ `);
+});
+
+test('should run test defined outside of testdir', async ({ runUITest }) => {
+ const { page } = await runUITest({
+ 'playwright.config.ts': `export default { testDir: 'tests' };`,
+ 'tests/example.spec.ts': `
+ import '../src/impl';
+ `,
+ 'src/impl.ts': `
+ import { test } from '@playwright/test';
+ test('external', async () => {});
+ `,
+ });
+ await page.getByRole('treeitem', { name: 'external' }).dblclick();
+ await expect.poll(dumpTestTree(page)).toBe(`
+ ▼ ✅ example.spec.ts
+ ✅ external <=
+ `);
+});
diff --git a/tests/playwright-test/ui-mode-test-watch.spec.ts b/tests/playwright-test/ui-mode-test-watch.spec.ts
index da1920b120..851eefd804 100644
--- a/tests/playwright-test/ui-mode-test-watch.spec.ts
+++ b/tests/playwright-test/ui-mode-test-watch.spec.ts
@@ -350,3 +350,35 @@ test('should have watch icon highlighted when a test is focused and watch on the
◯ passes 👁
`);
});
+
+test('should watch test defined outside of .spec.ts file', async ({ runUITest, writeFiles }) => {
+ const { page } = await runUITest({
+ 'example.spec.ts': `
+ import './impl';
+ `,
+ 'impl.ts': `
+ import { test } from '@playwright/test';
+ test('one', async () => {});
+ `,
+ });
+
+ await page.getByRole('treeitem', { name: 'one' }).click();
+ await page.getByRole('treeitem', { name: 'one' }).getByRole('button', { name: 'Watch' }).click();
+
+ await expect.poll(dumpTestTree(page)).toBe(`
+ ▼ ◯ example.spec.ts
+ ◯ one 👁 <=
+ `);
+
+ await writeFiles({
+ 'impl.ts': `
+ import { test } from '@playwright/test';
+ test('one', async () => { /* modified */ });
+ `,
+ });
+
+ await expect.poll(dumpTestTree(page)).toBe(`
+ ▼ ✅ example.spec.ts
+ ✅ one 👁 <=
+ `);
+});