Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(html): show the list of runs from the same worker (#39103)
Dmitry Gozman5d5b28f782e1
diff --git a/packages/html-reporter/src/chip.tsx b/packages/html-reporter/src/chip.tsxindex 93e79abd4b..f0dce3e48f 100644--- a/packages/html-reporter/src/chip.tsx+++ b/packages/html-reporter/src/chip.tsx@@ -28,9 +28,10 @@ export const Chip: React.FC<{expanded?: boolean,noInsets?: boolean,setExpanded?: (expanded: boolean) => void,- children?: any,+ children?: React.ReactNode,+ body?: () => React.ReactNode | undefined,dataTestId?: string,-}> = ({ header, footer, expanded, setExpanded, children, noInsets, dataTestId }) => {+}> = ({ header, footer, expanded, setExpanded, children, noInsets, body, dataTestId }) => {const id = React.useId();return <div className='chip' data-testid={dataTestId}><div@@ -45,6 +46,7 @@ export const Chip: React.FC<{</div>{(!setExpanded || expanded) && <div id={id} role='region' className={clsx('chip-body', noInsets && 'chip-body-no-insets')}>{children}+ {body && body()}{footer && <div className='chip-footer'>{footer}</div>}</div>}</div>;@@ -54,10 +56,11 @@ export const AutoChip: React.FC<{header: React.JSX.Element | string,initialExpanded?: boolean,noInsets?: boolean,- children?: any,+ children?: React.ReactNode,+ body?: () => React.ReactNode | undefined,dataTestId?: string,revealOnAnchorId?: AnchorID,-}> = ({ header, initialExpanded, noInsets, children, dataTestId, revealOnAnchorId }) => {+}> = ({ header, initialExpanded, noInsets, children, body, dataTestId, revealOnAnchorId }) => {const [expanded, setExpanded] = React.useState(initialExpanded ?? true);const onReveal = React.useCallback(() => setExpanded(true), []);useAnchor(revealOnAnchorId, onReveal);@@ -66,6 +69,7 @@ export const AutoChip: React.FC<{expanded={expanded}setExpanded={setExpanded}noInsets={noInsets}+ body={body}dataTestId={dataTestId}>{children}diff --git a/packages/html-reporter/src/reportView.tsx b/packages/html-reporter/src/reportView.tsxindex 0bc6144802..b514e67ace 100644--- a/packages/html-reporter/src/reportView.tsx+++ b/packages/html-reporter/src/reportView.tsx@@ -193,12 +193,9 @@ const TestCaseViewLoader: React.FC<{</div>;}- const { projectNames, metadata, options } = report.json();return <div className='test-case-column'><TestCaseView- projectNames={projectNames}- testRunMetadata={metadata}- options={options}+ report={report}next={next}prev={prev}test={test}diff --git a/packages/html-reporter/src/testCaseView.tsx b/packages/html-reporter/src/testCaseView.tsxindex b7a5c8f65f..aa85522a8f 100644--- a/packages/html-reporter/src/testCaseView.tsx+++ b/packages/html-reporter/src/testCaseView.tsx@@ -15,7 +15,7 @@*/import type { TestAnnotation } from '@playwright/test';-import type { HTMLReportOptions, TestCase, TestCaseSummary } from './types';+import type { TestCase, TestCaseSummary } from './types';import * as React from 'react';import { TabbedPane } from './tabbedPane';import { AutoChip } from './chip';@@ -29,18 +29,16 @@ import { msToString } from './utils';import { clsx } from '@web/uiUtils';import { CopyToClipboardContainer } from './copyToClipboard';import { HeaderView } from './headerView';-import type { MetadataWithCommitInfo } from '@playwright/isomorphic/types';import { ProjectAndTagLabelsView } from './labels';+import type { LoadedReport } from './loadedReport';export const TestCaseView: React.FC<{- projectNames: string[],+ report: LoadedReport,test: TestCase,- testRunMetadata: MetadataWithCommitInfo | undefined,next: TestCaseSummary | undefined,prev: TestCaseSummary | undefined,run: number,- options?: HTMLReportOptions,-}> = ({ projectNames, test, testRunMetadata, run, next, prev, options }) => {+}> = ({ report, test, run, next, prev }) => {const [selectedResultIndex, setSelectedResultIndex] = React.useState(run);const searchParams = useSearchParams();@@ -66,7 +64,7 @@ export const TestCaseView: React.FC<{<TraceLink test={test} trailingSeparator={true} /><div className='test-case-duration'>{msToString(test.duration)}</div></div>- <ProjectAndTagLabelsView style={{ marginLeft: '6px' }} projectNames={projectNames} activeProjectName={test.projectName} otherLabels={test.tags} />+ <ProjectAndTagLabelsView style={{ marginLeft: '6px' }} projectNames={report.json().projectNames} activeProjectName={test.projectName} otherLabels={test.tags} />{/* If there are no results, display test annotations. Otherwise test annotations will be displayed alongside runtime annotations in individual result pane */}{test.results.length === 0 && visibleTestAnnotations.length !== 0 && <AutoChip header='Annotations' dataTestId='test-case-annotations'>{visibleTestAnnotations.map((annotation, index) => <TestCaseAnnotationView key={index} annotation={annotation} />)}@@ -84,7 +82,7 @@ export const TestCaseView: React.FC<{{!!visibleAnnotations.length && <AutoChip header='Annotations' dataTestId='test-case-annotations'>{visibleAnnotations.map((annotation, index) => <TestCaseAnnotationView key={index} annotation={annotation} />)}</AutoChip>}- <TestResultView test={test!} result={result} testRunMetadata={testRunMetadata} options={options} />+ <TestResultView test={test!} result={result} report={report} /></>;},})) || []} selectedTab={String(selectedResultIndex)} setSelectedTab={id => setSelectedResultIndex(+id)} />diff --git a/packages/html-reporter/src/testFileView.css b/packages/html-reporter/src/testFileView.cssindex f081c696b9..021f4c6250 100644--- a/packages/html-reporter/src/testFileView.css+++ b/packages/html-reporter/src/testFileView.css@@ -22,7 +22,7 @@text-overflow: ellipsis;}-.test-file-test:hover {+.test-file-test-selected, .test-file-test:hover {background-color: var(--color-canvas-subtle);}diff --git a/packages/html-reporter/src/testFileView.tsx b/packages/html-reporter/src/testFileView.tsxindex 1a5a9dc9ef..4022094bf6 100644--- a/packages/html-reporter/src/testFileView.tsx+++ b/packages/html-reporter/src/testFileView.tsx@@ -25,14 +25,13 @@ import { video, image } from './icons';import { clsx } from '@web/uiUtils';import { ProjectAndTagLabelsView } from './labels';-export const TestFileView: React.FC<React.PropsWithChildren<{+export const TestFileView: React.FC<{file: TestFileSummary;projectNames: string[];isFileExpanded?: (fileId: string) => boolean;setFileExpanded?: (fileId: string, expanded: boolean) => void;footer?: React.JSX.Element | string;-}>> = ({ file, projectNames, isFileExpanded, setFileExpanded, footer }) => {- const searchParams = useSearchParams();+}> = ({ file, projectNames, isFileExpanded, setFileExpanded, footer }) => {return <Chipexpanded={isFileExpanded ? isFileExpanded(file.fileId) : undefined}noInsets={true}@@ -42,15 +41,31 @@ export const TestFileView: React.FC<React.PropsWithChildren<{</span>}footer={footer}>- {file.tests.map(test =>- <div key={`test-${test.testId}`} className={clsx('test-file-test', 'test-file-test-outcome-' + test.outcome)}>+ <TestCaseListView tests={file.tests} projectNames={projectNames} />+ </Chip>;+};++export const TestCaseListView: React.FC<{+ tests: TestCaseSummary[];+ projectNames: string[];+ runs?: number[];+ selectedTestId?: string;+}> = ({ tests, projectNames, runs, selectedTestId }) => {+ const searchParams = useSearchParams();+ return <div role='list'>+ {tests.map((test, index) => {+ const run = runs?.[index];+ const result = run !== undefined ? test.results[run] : undefined;+ const href = testResultHref({ test, result }, searchParams);+ const selected = selectedTestId === test.testId;+ return <div key={`test-${test.testId}`} className={clsx('test-file-test', 'test-file-test-outcome-' + test.outcome, selected && 'test-file-test-selected')} role='listitem' aria-current={selected}><div className='hbox' style={{ alignItems: 'flex-start' }}><div className='hbox'><span className='test-file-test-status-icon'>{statusIcon(test.outcome)}</span><span>- <Link href={testResultHref({ test }, searchParams)} title={[...test.path, test.title].join(' › ')}>+ <Link href={href} title={[...test.path, test.title].join(' › ')}><span className='test-file-title'>{[...test.path, test.title].join(' › ')}</span></Link><ProjectAndTagLabelsView style={{ marginLeft: '6px' }} projectNames={projectNames} activeProjectName={test.projectName} otherLabels={test.tags} />@@ -60,7 +75,7 @@ export const TestFileView: React.FC<React.PropsWithChildren<{</div><div className='test-file-details-row'><div className='test-file-details-row-items'>- <Link href={testResultHref({ test }, searchParams)} title={[...test.path, test.title].join(' › ')} className='test-file-path-link'>+ <Link href={href} title={[...test.path, test.title].join(' › ')} className='test-file-path-link'><span className='test-file-path'>{test.location.file}:{test.location.line}</span></Link><ImageDiffBadge test={test} />@@ -68,9 +83,9 @@ export const TestFileView: React.FC<React.PropsWithChildren<{<TraceLink test={test} dim={true} /></div></div>- </div>- )}- </Chip>;+ </div>;+ })}+ </div>;};function ImageDiffBadge({ test }: { test: TestCaseSummary }) {diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsxindex 08db1aad95..0a689be253 100644--- a/packages/html-reporter/src/testResultView.tsx+++ b/packages/html-reporter/src/testResultView.tsx@@ -14,7 +14,7 @@limitations under the License.*/-import type { HTMLReportOptions, TestAttachment, TestCase, TestResult, TestStep } from './types';+import type { TestAttachment, TestCase, TestCaseSummary, TestResult, TestStep } from './types';import * as React from 'react';import { TreeItem } from './treeItem';import { formatUrl, msToString } from './utils';@@ -29,7 +29,8 @@ import * as icons from './icons';import './testResultView.css';import { useAsyncMemo } from '@web/uiUtils';import { copyPrompt } from '@web/shared/prompts';-import type { MetadataWithCommitInfo } from '@playwright/isomorphic/types';+import type { LoadedReport } from './loadedReport';+import { TestCaseListView } from './testFileView';interface ImageDiffWithAnchors extends ImageDiff {anchors: string[];@@ -71,11 +72,10 @@ function groupImageDiffs(screenshots: Set<TestAttachment>, result: TestResult):}export const TestResultView: React.FC<{+ report: LoadedReport,test: TestCase,result: TestResult,- testRunMetadata: MetadataWithCommitInfo | undefined,- options?: HTMLReportOptions,-}> = ({ test, result, testRunMetadata, options }) => {+}> = ({ report, test, result }) => {const { screenshots, videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors, errorContext } = React.useMemo(() => {const attachments = result.attachments.filter(a => !a.name.startsWith('_'));const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/')));@@ -92,7 +92,7 @@ export const TestResultView: React.FC<{}, [result]);const prompt = useAsyncMemo(async () => {- if (options?.noCopyPrompt)+ if (report.json().options?.noCopyPrompt)return undefined;const stdoutAttachment = result.attachments.find(a => a.name === 'stdout');@@ -105,14 +105,14 @@ export const TestResultView: React.FC<{`- Name: ${test.path.join(' >> ')} >> ${test.title}`,`- Location: ${test.location.file}:${test.location.line}:${test.location.column}`].join('\n'),- metadata: testRunMetadata,+ metadata: report.json().metadata,errorContext: errorContext?.path ? await fetch(errorContext.path!).then(r => r.text()) : errorContext?.body,errors: result.errors,buildCodeFrame: async error => error.codeframe,stdout,stderr,});- }, [test, errorContext, testRunMetadata, result], undefined);+ }, [test, errorContext, report, result], undefined);return <div className='test-result'>{!!errors.length && <AutoChip header='Errors'>@@ -177,6 +177,16 @@ export const TestResultView: React.FC<{</Anchor>)}</AutoChip>}++ <AutoChip header={`Executed in Worker #${result.workerIndex}`} dataTestId='worker-test-list' initialExpanded={false} noInsets={true} body={() => {+ const list = buildWorkerLists(report).get(result.workerIndex) || { tests: [], runs: [] };+ return <TestCaseListView+ tests={list.tests}+ runs={list.runs}+ projectNames={report.json().projectNames}+ selectedTestId={test.testId}+ />;+ }}/></div>;};@@ -216,3 +226,33 @@ const StepTreeItem: React.FC<{return snippet.concat(steps);} : undefined} depth={depth}/>;};++type WorkerLists = Map<number, { tests: TestCaseSummary[], runs: number[] }>;+const kWorkerListsSymbol = Symbol('workerLists');++function buildWorkerLists(report: LoadedReport): WorkerLists {+ let data: WorkerLists | undefined = (report as any)[kWorkerListsSymbol];+ if (!data) {+ const lists = new Map<number, { test: TestCaseSummary, time: number, run: number }[]>();+ for (const file of report.json().files) {+ for (const test of file.tests) {+ for (let index = 0; index < test.results.length; index++) {+ let list = lists.get(test.results[index].workerIndex);+ if (!list) {+ list = [];+ lists.set(test.results[index].workerIndex, list);+ }+ list.push({ test, time: new Date(test.results[index].startTime).valueOf(), run: index });+ }+ }+ }++ data = new Map();+ for (const [workerIndex, list] of lists) {+ list.sort((a, b) => a.time - b.time);+ data.set(workerIndex, { tests: list.map(t => t.test), runs: list.map(t => t.run) });+ }+ (report as any)[kWorkerListsSymbol] = data;+ }+ return data;+}diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.tsindex ddb273bf52..dd7927e09d 100644--- a/packages/html-reporter/src/types.d.ts+++ b/packages/html-reporter/src/types.d.ts@@ -88,6 +88,8 @@ export type TestCaseSummary = {export type TestResultSummary = {attachments: { name: string, contentType: string, path?: string }[];+ startTime: string;+ workerIndex: number;};export type TestCase = Omit<TestCaseSummary, 'results'> & {@@ -110,6 +112,7 @@ export type TestResult = {attachments: TestAttachment[];status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';annotations: TestAnnotation[];+ workerIndex: number;};export type TestStep = {diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.tsindex cd0fc9fd53..31548f6ec1 100644--- a/packages/playwright/src/reporters/html.ts+++ b/packages/playwright/src/reporters/html.ts@@ -443,7 +443,11 @@ class HtmlBuilder {path,ok: test.outcome() === 'expected' || test.outcome() === 'flaky',results: results.map(result => {- return { attachments: result.attachments.map(a => ({ name: a.name, contentType: a.contentType, path: a.path })) };+ return {+ attachments: result.attachments.map(a => ({ name: a.name, contentType: a.contentType, path: a.path })),+ startTime: result.startTime,+ workerIndex: result.workerIndex,+ };}),},};@@ -552,6 +556,7 @@ class HtmlBuilder {...result.attachments,...result.stdout.map(m => stdioAttachment(m, 'stdout')),...result.stderr.map(m => stdioAttachment(m, 'stderr'))]),+ workerIndex: result.workerIndex,};}diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.tsindex 13d662cc5a..e8a7945396 100644--- a/packages/playwright/src/reporters/merge.ts+++ b/packages/playwright/src/reporters/merge.ts@@ -84,6 +84,7 @@ export async function createMergedReport(config: FullConfigInternal, dir: string};await dispatchEvents(eventData.prologue);+ let usedWorkers = 0;for (const { reportFile, zipFile, eventPatchers, metadata, config, fullResult } of eventData.reports) {multiplexer.onReportConfigure({reportPath: zipFile,@@ -97,7 +98,10 @@ export async function createMergedReport(config: FullConfigInternal, dir: stringeventPatchers.patchers.push(new GlobalErrorPatcher(metadata.name));if (config?.tags?.length)eventPatchers.patchers.push(new GlobalErrorPatcher(config.tags.join(' ')));+ const workerIndexPatcher = new WorkerIndexPatcher(usedWorkers);+ eventPatchers.patchers.push(workerIndexPatcher);eventPatchers.patchEvents(events);+ usedWorkers += workerIndexPatcher.usedWorkers();await dispatchEvents(events);multiplexer.onReportEnd({reportPath: zipFile,@@ -607,6 +611,26 @@ class GlobalErrorPatcher {}}+class WorkerIndexPatcher {+ private _baseWorkerIndex: number;+ private _maxWorkerIndex = 0;++ constructor(baseWorkerIndex: number) {+ this._baseWorkerIndex = baseWorkerIndex;+ }++ patchEvent(event: JsonEvent) {+ if (event.method === 'onTestBegin') {+ this._maxWorkerIndex = Math.max(this._maxWorkerIndex, event.params.result.workerIndex);+ event.params.result.workerIndex += this._baseWorkerIndex;+ }+ }++ usedWorkers() {+ return this._maxWorkerIndex + 1;+ }+}+interface JsonEventPatcher {patchEvent(event: JsonEvent): void;}diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.tsindex a1d60d2d69..2cf828cec3 100644--- a/tests/playwright-test/reporter-blob.spec.ts+++ b/tests/playwright-test/reporter-blob.spec.ts@@ -2326,3 +2326,39 @@ test('should populate projects in config when merging reports', async ({ runInliconst projectNames = JSON.parse(outputLines[0]);expect(projectNames).toEqual(['setup', 'p1', 'setup', 'p2']);});++test('workerIndex is rebased', async ({ runInlineTest, writeFiles, showReport, page, mergeReports }) => {+ const reportDir = test.info().outputPath('blob-reports');+ await writeFiles({+ 'playwright.config.ts': `+ module.exports = {+ fullyParallel: true,+ reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]+ };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('test-one', async () => {});+ test('test-two', async () => {});+ `,+ });++ await runInlineTest({}, { shard: '1/2', workers: '1' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });+ await runInlineTest({}, { shard: '2/2', workers: '1' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });++ const { exitCode } = await mergeReports(reportDir, { 'PLAYWRIGHT_HTML_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });+ expect(exitCode).toBe(0);+ await showReport();++ await page.getByRole('link', { name: 'test-two', exact: true }).click();++ await page.getByRole('button', { name: 'Executed in Worker #1' }).click();+ await expect(page.getByTestId('worker-test-list')).toMatchAriaSnapshot(`+ - 'button "Executed in Worker #1" [expanded]'+ - region:+ - list:+ - listitem:+ - link "test-two"+ - link "a.test.js:4"+ `);+});diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.tsindex 262f927410..e2dbfadef1 100644--- a/tests/playwright-test/reporter-html.spec.ts+++ b/tests/playwright-test/reporter-html.spec.ts@@ -3122,6 +3122,43 @@ for (const useIntermediateMergeReport of [true, false] as const) {await expect(page.getByTestId('test-snippet')).not.toBeVisible();});+ test('worker test list', async ({ runInlineTest, showReport, page }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ export default { reporter: [['html']], retries: 1 }+ `,+ 'example.spec.ts': `+ import { test, expect } from '@playwright/test';+ test('success1', () => {});+ test('flaky1', () => {+ expect(test.info().retry).toBe(1);+ });+ test('success2', () => {});+ `,+ }, { reporter: 'html', workers: '1' }, { PLAYWRIGHT_HTML_OPEN: 'never' });+ expect(result.exitCode).toBe(0);+ await showReport();++ await page.getByRole('link', { name: 'flaky1', exact: true }).click();++ await page.getByRole('button', { name: 'Executed in Worker #0' }).click();+ await expect(page.getByTestId('worker-test-list')).toMatchAriaSnapshot(`+ - 'button "Executed in Worker #0" [expanded]'+ - region:+ - list:+ - listitem:+ - link "success1"+ - link "example.spec.ts:3"+ - listitem:+ - link "flaky1"+ - link "example.spec.ts:4"+ `);+ await expect(page.getByRole('listitem').filter({ hasText: 'flaky1' })).toHaveAttribute('aria-current', 'true');++ await page.getByRole('link', { name: 'success1', exact: true }).click();+ await expect(page.locator('.test-case-location')).toHaveText('example.spec.ts:3');+ });+test('should support keyboard shortcuts', async ({ runInlineTest, showReport, page }) => {await runInlineTest({'playwright.config.ts': `@@ -3243,18 +3280,25 @@ for (const useIntermediateMergeReport of [true, false] as const) {await expect(page.getByRole('main')).toMatchAriaSnapshot(`- button "Slowest Tests"- region:- - link "three"- - text: /foo/- - link "two"- - text: /foo/- - link "one"- - text: /foo/- - link "three"- - text: /bar/- - link "two"- - text: /bar/- - link "one"- - text: /bar/+ - list:+ - listitem:+ - link "three"+ - text: /foo/+ - listitem:+ - link "two"+ - text: /foo/+ - listitem:+ - link "one"+ - text: /foo/+ - listitem:+ - link "three"+ - text: /bar/+ - listitem:+ - link "two"+ - text: /bar/+ - listitem:+ - link "one"+ - text: /bar/`);await page.getByText('foo').first().click();await expect(page.getByRole('main')).toMatchAriaSnapshot(`@@ -3334,14 +3378,19 @@ test('should support merge files option', async ({ runInlineTest, showReport, paawait expect(page.locator('body')).toMatchAriaSnapshot(`- button "<anonymous>" [expanded]- region:- - link "test 2"- - link "a.test.js:6"+ - list:+ - listitem:+ - link "test 2"+ - link "a.test.js:6"- button "describe" [expanded]- region:- - link "test 1"- - link "a.test.js:4"- - link "test 3"- - link "b.test.js:4"+ - list:+ - listitem:+ - link "test 1"+ - link "a.test.js:4"+ - listitem:+ - link "test 3"+ - link "b.test.js:4"`);});