Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(html): show the list of runs from the same worker (#39103)

Dmitry Gozman5d5b28f782e1
diff --git a/packages/html-reporter/src/chip.tsx b/packages/html-reporter/src/chip.tsx
index 93e79abd4b..f0dce3e48f 100644
--- a/packages/html-reporter/src/chip.tsx
+++ b/packages/html-reporter/src/chip.tsx
@@ -28,9 +28,10 @@ export const Chip: React.FC<{
expanded?: boolean,
noInsets?: boolean,
setExpanded?: (expanded: boolean) => void,
- children?: any,
+ children?: React.ReactNode,
+ body?: () => React.ReactNode | undefined,
dataTestId?: string,
-}> = ({ header, footer, expanded, setExpanded, children, noInsets, dataTestId }) => {
+}> = ({ header, footer, expanded, setExpanded, children, noInsets, body, dataTestId }) => {
const id = React.useId();
return <div className='chip' data-testid={dataTestId}>
<div
@@ -45,6 +46,7 @@ export const Chip: React.FC<{
</div>
{(!setExpanded || expanded) && <div id={id} role='region' className={clsx('chip-body', noInsets && 'chip-body-no-insets')}>
{children}
+ {body && body()}
{footer && <div className='chip-footer'>{footer}</div>}
</div>}
</div>;
@@ -54,10 +56,11 @@ export const AutoChip: React.FC<{
header: React.JSX.Element | string,
initialExpanded?: boolean,
noInsets?: boolean,
- children?: any,
+ children?: React.ReactNode,
+ body?: () => React.ReactNode | undefined,
dataTestId?: string,
revealOnAnchorId?: AnchorID,
-}> = ({ header, initialExpanded, noInsets, children, dataTestId, revealOnAnchorId }) => {
+}> = ({ header, initialExpanded, noInsets, children, body, dataTestId, revealOnAnchorId }) => {
const [expanded, setExpanded] = React.useState(initialExpanded ?? true);
const onReveal = React.useCallback(() => setExpanded(true), []);
useAnchor(revealOnAnchorId, onReveal);
@@ -66,6 +69,7 @@ export const AutoChip: React.FC<{
expanded={expanded}
setExpanded={setExpanded}
noInsets={noInsets}
+ body={body}
dataTestId={dataTestId}
>
{children}
diff --git a/packages/html-reporter/src/reportView.tsx b/packages/html-reporter/src/reportView.tsx
index 0bc6144802..b514e67ace 100644
--- a/packages/html-reporter/src/reportView.tsx
+++ b/packages/html-reporter/src/reportView.tsx
@@ -193,12 +193,9 @@ const TestCaseViewLoader: React.FC<{
</div>;
}
- const { projectNames, metadata, options } = report.json();
return <div className='test-case-column'>
<TestCaseView
- projectNames={projectNames}
- testRunMetadata={metadata}
- options={options}
+ report={report}
next={next}
prev={prev}
test={test}
diff --git a/packages/html-reporter/src/testCaseView.tsx b/packages/html-reporter/src/testCaseView.tsx
index b7a5c8f65f..aa85522a8f 100644
--- a/packages/html-reporter/src/testCaseView.tsx
+++ b/packages/html-reporter/src/testCaseView.tsx
@@ -15,7 +15,7 @@
*/
import type { TestAnnotation } from '@playwright/test';
-import type { HTMLReportOptions, TestCase, TestCaseSummary } from './types';
+import type { TestCase, TestCaseSummary } from './types';
import * as React from 'react';
import { TabbedPane } from './tabbedPane';
import { AutoChip } from './chip';
@@ -29,18 +29,16 @@ import { msToString } from './utils';
import { clsx } from '@web/uiUtils';
import { CopyToClipboardContainer } from './copyToClipboard';
import { HeaderView } from './headerView';
-import type { MetadataWithCommitInfo } from '@playwright/isomorphic/types';
import { ProjectAndTagLabelsView } from './labels';
+import type { LoadedReport } from './loadedReport';
export const TestCaseView: React.FC<{
- projectNames: string[],
+ report: LoadedReport,
test: TestCase,
- testRunMetadata: MetadataWithCommitInfo | undefined,
next: TestCaseSummary | undefined,
prev: TestCaseSummary | undefined,
run: number,
- options?: HTMLReportOptions,
-}> = ({ projectNames, test, testRunMetadata, run, next, prev, options }) => {
+}> = ({ report, test, run, next, prev }) => {
const [selectedResultIndex, setSelectedResultIndex] = React.useState(run);
const searchParams = useSearchParams();
@@ -66,7 +64,7 @@ export const TestCaseView: React.FC<{
<TraceLink test={test} trailingSeparator={true} />
<div className='test-case-duration'>{msToString(test.duration)}</div>
</div>
- <ProjectAndTagLabelsView style={{ marginLeft: '6px' }} projectNames={projectNames} activeProjectName={test.projectName} otherLabels={test.tags} />
+ <ProjectAndTagLabelsView style={{ marginLeft: '6px' }} projectNames={report.json().projectNames} activeProjectName={test.projectName} otherLabels={test.tags} />
{/* If there are no results, display test annotations. Otherwise test annotations will be displayed alongside runtime annotations in individual result pane */}
{test.results.length === 0 && visibleTestAnnotations.length !== 0 && <AutoChip header='Annotations' dataTestId='test-case-annotations'>
{visibleTestAnnotations.map((annotation, index) => <TestCaseAnnotationView key={index} annotation={annotation} />)}
@@ -84,7 +82,7 @@ export const TestCaseView: React.FC<{
{!!visibleAnnotations.length && <AutoChip header='Annotations' dataTestId='test-case-annotations'>
{visibleAnnotations.map((annotation, index) => <TestCaseAnnotationView key={index} annotation={annotation} />)}
</AutoChip>}
- <TestResultView test={test!} result={result} testRunMetadata={testRunMetadata} options={options} />
+ <TestResultView test={test!} result={result} report={report} />
</>;
},
})) || []} selectedTab={String(selectedResultIndex)} setSelectedTab={id => setSelectedResultIndex(+id)} />
diff --git a/packages/html-reporter/src/testFileView.css b/packages/html-reporter/src/testFileView.css
index f081c696b9..021f4c6250 100644
--- a/packages/html-reporter/src/testFileView.css
+++ b/packages/html-reporter/src/testFileView.css
@@ -22,7 +22,7 @@
text-overflow: ellipsis;
}
-.test-file-test:hover {
+.test-file-test-selected, .test-file-test:hover {
background-color: var(--color-canvas-subtle);
}
diff --git a/packages/html-reporter/src/testFileView.tsx b/packages/html-reporter/src/testFileView.tsx
index 1a5a9dc9ef..4022094bf6 100644
--- a/packages/html-reporter/src/testFileView.tsx
+++ b/packages/html-reporter/src/testFileView.tsx
@@ -25,14 +25,13 @@ import { video, image } from './icons';
import { clsx } from '@web/uiUtils';
import { ProjectAndTagLabelsView } from './labels';
-export const TestFileView: React.FC<React.PropsWithChildren<{
+export const TestFileView: React.FC<{
file: TestFileSummary;
projectNames: string[];
isFileExpanded?: (fileId: string) => boolean;
setFileExpanded?: (fileId: string, expanded: boolean) => void;
footer?: React.JSX.Element | string;
-}>> = ({ file, projectNames, isFileExpanded, setFileExpanded, footer }) => {
- const searchParams = useSearchParams();
+}> = ({ file, projectNames, isFileExpanded, setFileExpanded, footer }) => {
return <Chip
expanded={isFileExpanded ? isFileExpanded(file.fileId) : undefined}
noInsets={true}
@@ -42,15 +41,31 @@ export const TestFileView: React.FC<React.PropsWithChildren<{
</span>}
footer={footer}
>
- {file.tests.map(test =>
- <div key={`test-${test.testId}`} className={clsx('test-file-test', 'test-file-test-outcome-' + test.outcome)}>
+ <TestCaseListView tests={file.tests} projectNames={projectNames} />
+ </Chip>;
+};
+
+export const TestCaseListView: React.FC<{
+ tests: TestCaseSummary[];
+ projectNames: string[];
+ runs?: number[];
+ selectedTestId?: string;
+}> = ({ tests, projectNames, runs, selectedTestId }) => {
+ const searchParams = useSearchParams();
+ return <div role='list'>
+ {tests.map((test, index) => {
+ const run = runs?.[index];
+ const result = run !== undefined ? test.results[run] : undefined;
+ const href = testResultHref({ test, result }, searchParams);
+ const selected = selectedTestId === test.testId;
+ return <div key={`test-${test.testId}`} className={clsx('test-file-test', 'test-file-test-outcome-' + test.outcome, selected && 'test-file-test-selected')} role='listitem' aria-current={selected}>
<div className='hbox' style={{ alignItems: 'flex-start' }}>
<div className='hbox'>
<span className='test-file-test-status-icon'>
{statusIcon(test.outcome)}
</span>
<span>
- <Link href={testResultHref({ test }, searchParams)} title={[...test.path, test.title].join(' › ')}>
+ <Link href={href} title={[...test.path, test.title].join(' › ')}>
<span className='test-file-title'>{[...test.path, test.title].join(' › ')}</span>
</Link>
<ProjectAndTagLabelsView style={{ marginLeft: '6px' }} projectNames={projectNames} activeProjectName={test.projectName} otherLabels={test.tags} />
@@ -60,7 +75,7 @@ export const TestFileView: React.FC<React.PropsWithChildren<{
</div>
<div className='test-file-details-row'>
<div className='test-file-details-row-items'>
- <Link href={testResultHref({ test }, searchParams)} title={[...test.path, test.title].join(' › ')} className='test-file-path-link'>
+ <Link href={href} title={[...test.path, test.title].join(' › ')} className='test-file-path-link'>
<span className='test-file-path'>{test.location.file}:{test.location.line}</span>
</Link>
<ImageDiffBadge test={test} />
@@ -68,9 +83,9 @@ export const TestFileView: React.FC<React.PropsWithChildren<{
<TraceLink test={test} dim={true} />
</div>
</div>
- </div>
- )}
- </Chip>;
+ </div>;
+ })}
+ </div>;
};
function ImageDiffBadge({ test }: { test: TestCaseSummary }) {
diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx
index 08db1aad95..0a689be253 100644
--- a/packages/html-reporter/src/testResultView.tsx
+++ b/packages/html-reporter/src/testResultView.tsx
@@ -14,7 +14,7 @@
limitations under the License.
*/
-import type { HTMLReportOptions, TestAttachment, TestCase, TestResult, TestStep } from './types';
+import type { TestAttachment, TestCase, TestCaseSummary, TestResult, TestStep } from './types';
import * as React from 'react';
import { TreeItem } from './treeItem';
import { formatUrl, msToString } from './utils';
@@ -29,7 +29,8 @@ import * as icons from './icons';
import './testResultView.css';
import { useAsyncMemo } from '@web/uiUtils';
import { copyPrompt } from '@web/shared/prompts';
-import type { MetadataWithCommitInfo } from '@playwright/isomorphic/types';
+import type { LoadedReport } from './loadedReport';
+import { TestCaseListView } from './testFileView';
interface ImageDiffWithAnchors extends ImageDiff {
anchors: string[];
@@ -71,11 +72,10 @@ function groupImageDiffs(screenshots: Set<TestAttachment>, result: TestResult):
}
export const TestResultView: React.FC<{
+ report: LoadedReport,
test: TestCase,
result: TestResult,
- testRunMetadata: MetadataWithCommitInfo | undefined,
- options?: HTMLReportOptions,
-}> = ({ test, result, testRunMetadata, options }) => {
+}> = ({ report, test, result }) => {
const { screenshots, videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors, errorContext } = React.useMemo(() => {
const attachments = result.attachments.filter(a => !a.name.startsWith('_'));
const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/')));
@@ -92,7 +92,7 @@ export const TestResultView: React.FC<{
}, [result]);
const prompt = useAsyncMemo(async () => {
- if (options?.noCopyPrompt)
+ if (report.json().options?.noCopyPrompt)
return undefined;
const stdoutAttachment = result.attachments.find(a => a.name === 'stdout');
@@ -105,14 +105,14 @@ export const TestResultView: React.FC<{
`- Name: ${test.path.join(' >> ')} >> ${test.title}`,
`- Location: ${test.location.file}:${test.location.line}:${test.location.column}`
].join('\n'),
- metadata: testRunMetadata,
+ metadata: report.json().metadata,
errorContext: errorContext?.path ? await fetch(errorContext.path!).then(r => r.text()) : errorContext?.body,
errors: result.errors,
buildCodeFrame: async error => error.codeframe,
stdout,
stderr,
});
- }, [test, errorContext, testRunMetadata, result], undefined);
+ }, [test, errorContext, report, result], undefined);
return <div className='test-result'>
{!!errors.length && <AutoChip header='Errors'>
@@ -177,6 +177,16 @@ export const TestResultView: React.FC<{
</Anchor>
)}
</AutoChip>}
+
+ <AutoChip header={`Executed in Worker #${result.workerIndex}`} dataTestId='worker-test-list' initialExpanded={false} noInsets={true} body={() => {
+ const list = buildWorkerLists(report).get(result.workerIndex) || { tests: [], runs: [] };
+ return <TestCaseListView
+ tests={list.tests}
+ runs={list.runs}
+ projectNames={report.json().projectNames}
+ selectedTestId={test.testId}
+ />;
+ }}/>
</div>;
};
@@ -216,3 +226,33 @@ const StepTreeItem: React.FC<{
return snippet.concat(steps);
} : undefined} depth={depth}/>;
};
+
+type WorkerLists = Map<number, { tests: TestCaseSummary[], runs: number[] }>;
+const kWorkerListsSymbol = Symbol('workerLists');
+
+function buildWorkerLists(report: LoadedReport): WorkerLists {
+ let data: WorkerLists | undefined = (report as any)[kWorkerListsSymbol];
+ if (!data) {
+ const lists = new Map<number, { test: TestCaseSummary, time: number, run: number }[]>();
+ for (const file of report.json().files) {
+ for (const test of file.tests) {
+ for (let index = 0; index < test.results.length; index++) {
+ let list = lists.get(test.results[index].workerIndex);
+ if (!list) {
+ list = [];
+ lists.set(test.results[index].workerIndex, list);
+ }
+ list.push({ test, time: new Date(test.results[index].startTime).valueOf(), run: index });
+ }
+ }
+ }
+
+ data = new Map();
+ for (const [workerIndex, list] of lists) {
+ list.sort((a, b) => a.time - b.time);
+ data.set(workerIndex, { tests: list.map(t => t.test), runs: list.map(t => t.run) });
+ }
+ (report as any)[kWorkerListsSymbol] = data;
+ }
+ return data;
+}
diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.ts
index ddb273bf52..dd7927e09d 100644
--- a/packages/html-reporter/src/types.d.ts
+++ b/packages/html-reporter/src/types.d.ts
@@ -88,6 +88,8 @@ export type TestCaseSummary = {
export type TestResultSummary = {
attachments: { name: string, contentType: string, path?: string }[];
+ startTime: string;
+ workerIndex: number;
};
export type TestCase = Omit<TestCaseSummary, 'results'> & {
@@ -110,6 +112,7 @@ export type TestResult = {
attachments: TestAttachment[];
status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
annotations: TestAnnotation[];
+ workerIndex: number;
};
export type TestStep = {
diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts
index cd0fc9fd53..31548f6ec1 100644
--- a/packages/playwright/src/reporters/html.ts
+++ b/packages/playwright/src/reporters/html.ts
@@ -443,7 +443,11 @@ class HtmlBuilder {
path,
ok: test.outcome() === 'expected' || test.outcome() === 'flaky',
results: results.map(result => {
- return { attachments: result.attachments.map(a => ({ name: a.name, contentType: a.contentType, path: a.path })) };
+ return {
+ attachments: result.attachments.map(a => ({ name: a.name, contentType: a.contentType, path: a.path })),
+ startTime: result.startTime,
+ workerIndex: result.workerIndex,
+ };
}),
},
};
@@ -552,6 +556,7 @@ class HtmlBuilder {
...result.attachments,
...result.stdout.map(m => stdioAttachment(m, 'stdout')),
...result.stderr.map(m => stdioAttachment(m, 'stderr'))]),
+ workerIndex: result.workerIndex,
};
}
diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts
index 13d662cc5a..e8a7945396 100644
--- a/packages/playwright/src/reporters/merge.ts
+++ b/packages/playwright/src/reporters/merge.ts
@@ -84,6 +84,7 @@ export async function createMergedReport(config: FullConfigInternal, dir: string
};
await dispatchEvents(eventData.prologue);
+ let usedWorkers = 0;
for (const { reportFile, zipFile, eventPatchers, metadata, config, fullResult } of eventData.reports) {
multiplexer.onReportConfigure({
reportPath: zipFile,
@@ -97,7 +98,10 @@ export async function createMergedReport(config: FullConfigInternal, dir: string
eventPatchers.patchers.push(new GlobalErrorPatcher(metadata.name));
if (config?.tags?.length)
eventPatchers.patchers.push(new GlobalErrorPatcher(config.tags.join(' ')));
+ const workerIndexPatcher = new WorkerIndexPatcher(usedWorkers);
+ eventPatchers.patchers.push(workerIndexPatcher);
eventPatchers.patchEvents(events);
+ usedWorkers += workerIndexPatcher.usedWorkers();
await dispatchEvents(events);
multiplexer.onReportEnd({
reportPath: zipFile,
@@ -607,6 +611,26 @@ class GlobalErrorPatcher {
}
}
+class WorkerIndexPatcher {
+ private _baseWorkerIndex: number;
+ private _maxWorkerIndex = 0;
+
+ constructor(baseWorkerIndex: number) {
+ this._baseWorkerIndex = baseWorkerIndex;
+ }
+
+ patchEvent(event: JsonEvent) {
+ if (event.method === 'onTestBegin') {
+ this._maxWorkerIndex = Math.max(this._maxWorkerIndex, event.params.result.workerIndex);
+ event.params.result.workerIndex += this._baseWorkerIndex;
+ }
+ }
+
+ usedWorkers() {
+ return this._maxWorkerIndex + 1;
+ }
+}
+
interface JsonEventPatcher {
patchEvent(event: JsonEvent): void;
}
diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts
index a1d60d2d69..2cf828cec3 100644
--- a/tests/playwright-test/reporter-blob.spec.ts
+++ b/tests/playwright-test/reporter-blob.spec.ts
@@ -2326,3 +2326,39 @@ test('should populate projects in config when merging reports', async ({ runInli
const projectNames = JSON.parse(outputLines[0]);
expect(projectNames).toEqual(['setup', 'p1', 'setup', 'p2']);
});
+
+test('workerIndex is rebased', async ({ runInlineTest, writeFiles, showReport, page, mergeReports }) => {
+ const reportDir = test.info().outputPath('blob-reports');
+ await writeFiles({
+ 'playwright.config.ts': `
+ module.exports = {
+ fullyParallel: true,
+ reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
+ };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('test-one', async () => {});
+ test('test-two', async () => {});
+ `,
+ });
+
+ await runInlineTest({}, { shard: '1/2', workers: '1' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest({}, { shard: '2/2', workers: '1' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+
+ const { exitCode } = await mergeReports(reportDir, { 'PLAYWRIGHT_HTML_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
+ expect(exitCode).toBe(0);
+ await showReport();
+
+ await page.getByRole('link', { name: 'test-two', exact: true }).click();
+
+ await page.getByRole('button', { name: 'Executed in Worker #1' }).click();
+ await expect(page.getByTestId('worker-test-list')).toMatchAriaSnapshot(`
+ - 'button "Executed in Worker #1" [expanded]'
+ - region:
+ - list:
+ - listitem:
+ - link "test-two"
+ - link "a.test.js:4"
+ `);
+});
diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts
index 262f927410..e2dbfadef1 100644
--- a/tests/playwright-test/reporter-html.spec.ts
+++ b/tests/playwright-test/reporter-html.spec.ts
@@ -3122,6 +3122,43 @@ for (const useIntermediateMergeReport of [true, false] as const) {
await expect(page.getByTestId('test-snippet')).not.toBeVisible();
});
+ test('worker test list', async ({ runInlineTest, showReport, page }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ export default { reporter: [['html']], retries: 1 }
+ `,
+ 'example.spec.ts': `
+ import { test, expect } from '@playwright/test';
+ test('success1', () => {});
+ test('flaky1', () => {
+ expect(test.info().retry).toBe(1);
+ });
+ test('success2', () => {});
+ `,
+ }, { reporter: 'html', workers: '1' }, { PLAYWRIGHT_HTML_OPEN: 'never' });
+ expect(result.exitCode).toBe(0);
+ await showReport();
+
+ await page.getByRole('link', { name: 'flaky1', exact: true }).click();
+
+ await page.getByRole('button', { name: 'Executed in Worker #0' }).click();
+ await expect(page.getByTestId('worker-test-list')).toMatchAriaSnapshot(`
+ - 'button "Executed in Worker #0" [expanded]'
+ - region:
+ - list:
+ - listitem:
+ - link "success1"
+ - link "example.spec.ts:3"
+ - listitem:
+ - link "flaky1"
+ - link "example.spec.ts:4"
+ `);
+ await expect(page.getByRole('listitem').filter({ hasText: 'flaky1' })).toHaveAttribute('aria-current', 'true');
+
+ await page.getByRole('link', { name: 'success1', exact: true }).click();
+ await expect(page.locator('.test-case-location')).toHaveText('example.spec.ts:3');
+ });
+
test('should support keyboard shortcuts', async ({ runInlineTest, showReport, page }) => {
await runInlineTest({
'playwright.config.ts': `
@@ -3243,18 +3280,25 @@ for (const useIntermediateMergeReport of [true, false] as const) {
await expect(page.getByRole('main')).toMatchAriaSnapshot(`
- button "Slowest Tests"
- region:
- - link "three"
- - text: /foo/
- - link "two"
- - text: /foo/
- - link "one"
- - text: /foo/
- - link "three"
- - text: /bar/
- - link "two"
- - text: /bar/
- - link "one"
- - text: /bar/
+ - list:
+ - listitem:
+ - link "three"
+ - text: /foo/
+ - listitem:
+ - link "two"
+ - text: /foo/
+ - listitem:
+ - link "one"
+ - text: /foo/
+ - listitem:
+ - link "three"
+ - text: /bar/
+ - listitem:
+ - link "two"
+ - text: /bar/
+ - listitem:
+ - link "one"
+ - text: /bar/
`);
await page.getByText('foo').first().click();
await expect(page.getByRole('main')).toMatchAriaSnapshot(`
@@ -3334,14 +3378,19 @@ test('should support merge files option', async ({ runInlineTest, showReport, pa
await expect(page.locator('body')).toMatchAriaSnapshot(`
- button "<anonymous>" [expanded]
- region:
- - link "test 2"
- - link "a.test.js:6"
+ - list:
+ - listitem:
+ - link "test 2"
+ - link "a.test.js:6"
- button "describe" [expanded]
- region:
- - link "test 1"
- - link "a.test.js:4"
- - link "test 3"
- - link "b.test.js:4"
+ - list:
+ - listitem:
+ - link "test 1"
+ - link "a.test.js:4"
+ - listitem:
+ - link "test 3"
+ - link "b.test.js:4"
`);
});