Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat: split up static and dynamic annotations (#35292)
Simon Knott45fa3d17fccc
diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.mdindex 82ab6ca521..1aa5379ac4 100644--- a/docs/src/test-reporter-api/class-testcase.md+++ b/docs/src/test-reporter-api/class-testcase.md@@ -12,8 +12,7 @@The list of annotations applicable to the current test. Includes:* annotations defined on the test or suite via [`method: Test.(call)`] and [`method: Test.describe`];-* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`];-* annotations appended to [`property: TestInfo.annotations`] during the test execution.+* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`] prior to test execution.Annotations are available during test execution through [`property: TestInfo.annotations`].diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.mdindex 7674ac333d..0e60558632 100644--- a/docs/src/test-reporter-api/class-testresult.md+++ b/docs/src/test-reporter-api/class-testresult.md@@ -14,6 +14,20 @@ A result of a single [TestCase] run.The list of files or buffers attached during the test execution through [`property: TestInfo.attachments`].+## property: TestResult.annotations+* since: v1.52+- type: <[Array]<[Object]>>+ - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.+ - `description` ?<[string]> Optional description.++The list of annotations appended during test execution. Includes:+* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`] during test execution;+* annotations appended to [`property: TestInfo.annotations`].++Annotations are available during test execution through [`property: TestInfo.annotations`].++Learn more about [test annotations](../test-annotations.md).+## property: TestResult.duration* since: v1.10- type: <[float]>diff --git a/packages/html-reporter/src/tabbedPane.tsx b/packages/html-reporter/src/tabbedPane.tsxindex 02d0c6f3b1..af15b6b4fa 100644--- a/packages/html-reporter/src/tabbedPane.tsx+++ b/packages/html-reporter/src/tabbedPane.tsx@@ -30,14 +30,18 @@ export const TabbedPane: React.FunctionComponent<{selectedTab: string,setSelectedTab: (tab: string) => void}> = ({ tabs, selectedTab, setSelectedTab }) => {+ const idPrefix = React.useId();return <div className='tabbed-pane'><div className='vbox'><div className='hbox' style={{ flex: 'none' }}>- <div className='tabbed-pane-tab-strip'>{+ <div className='tabbed-pane-tab-strip' role='tablist'>{tabs.map(tab => (<div className={clsx('tabbed-pane-tab-element', selectedTab === tab.id && 'selected')}onClick={() => setSelectedTab(tab.id)}- key={tab.id}>+ id={`${idPrefix}-${tab.id}`}+ key={tab.id}+ role='tab'+ aria-selected={selectedTab === tab.id}><div className='tabbed-pane-tab-label'>{tab.title}</div></div>))@@ -46,7 +50,7 @@ export const TabbedPane: React.FunctionComponent<{{tabs.map(tab => {if (selectedTab === tab.id)- return <div key={tab.id} className='tab-content'>{tab.render()}</div>;+ return <div key={tab.id} className='tab-content' role='tabpanel' aria-labelledby={`${idPrefix}-${tab.id}`}>{tab.render()}</div>;})}</div>diff --git a/packages/html-reporter/src/testCaseView.spec.tsx b/packages/html-reporter/src/testCaseView.spec.tsxindex 8aa3fafc8f..3f6c7f2d05 100644--- a/packages/html-reporter/src/testCaseView.spec.tsx+++ b/packages/html-reporter/src/testCaseView.spec.tsx@@ -42,6 +42,7 @@ const result: TestResult = {}],attachments: [],}],+ annotations: [],attachments: [],status: 'passed',};@@ -151,6 +152,7 @@ const resultWithAttachment: TestResult = {name: 'attachment with inline link https://github.com/microsoft/playwright/issues/31284',contentType: 'text/plain'}],+ annotations: [],status: 'passed',};@@ -238,13 +240,15 @@ test('total duration is selected run duration', async ({ mount, page }) => {const component = await mount(<TestCaseView projectNames={['chromium', 'webkit']} test={testCaseWithTwoAttempts} prev={undefined} next={undefined} run={0}></TestCaseView>);await expect(component).toMatchAriaSnapshot(`- text: "My test test.spec.ts:42 200ms"- - text: "Run 50ms Retry #1 150ms"+ - tablist:+ - tab "Run 50ms"+ - 'tab "Retry #1 150ms"'`);- await page.locator('.tabbed-pane-tab-label', { hasText: 'Run50ms' }).click();+ await page.getByRole('tab', { name: 'Run' }).click();await expect(component).toMatchAriaSnapshot(`- text: "My test test.spec.ts:42 200ms"`);- await page.locator('.tabbed-pane-tab-label', { hasText: 'Retry #1150ms' }).click();+ await page.getByRole('tab', { name: 'Retry' }).click();await expect(component).toMatchAriaSnapshot(`- text: "My test test.spec.ts:42 200ms"`);diff --git a/packages/html-reporter/src/testCaseView.tsx b/packages/html-reporter/src/testCaseView.tsxindex ca5ff6d2e2..56ad193c8c 100644--- a/packages/html-reporter/src/testCaseView.tsx+++ b/packages/html-reporter/src/testCaseView.tsx@@ -14,7 +14,7 @@limitations under the License.*/-import type { TestCase, TestCaseAnnotation, TestCaseSummary } from './types';+import type { TestCase, TestAnnotation, TestCaseSummary } from './types';import * as React from 'react';import { TabbedPane } from './tabbedPane';import { AutoChip } from './chip';@@ -46,8 +46,13 @@ export const TestCaseView: React.FC<{}, [test]);const visibleAnnotations = React.useMemo(() => {- return test?.annotations?.filter(annotation => !annotation.type.startsWith('_')) || [];- }, [test?.annotations]);+ if (!test)+ return [];+ const annotations = [...test.annotations];+ if (test.results[selectedResultIndex])+ annotations.push(...test.results[selectedResultIndex].annotations);+ return annotations.filter(annotation => !annotation.type.startsWith('_'));+ }, [test, selectedResultIndex]);return <div className='test-case-column vbox'>{test && <div className='hbox'>@@ -71,7 +76,7 @@ export const TestCaseView: React.FC<{{test && !!test.projectName && <ProjectLink projectNames={projectNames} projectName={test.projectName}></ProjectLink>}{labels && <LabelsLinkView labels={labels} />}</div>}- {!!visibleAnnotations.length && <AutoChip header='Annotations'>+ {!!visibleAnnotations.length && <AutoChip header='Annotations' dataTestId='test-case-annotations'>{visibleAnnotations.map((annotation, index) => <TestCaseAnnotationView key={index} annotation={annotation} />)}</AutoChip>}{test && <TabbedPane tabs={@@ -86,7 +91,7 @@ export const TestCaseView: React.FC<{</div>;};-function TestCaseAnnotationView({ annotation: { type, description } }: { annotation: TestCaseAnnotation }) {+function TestCaseAnnotationView({ annotation: { type, description } }: { annotation: TestAnnotation }) {return (<div className='test-case-annotation'><span style={{ fontWeight: 'bold' }}>{type}</span>diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.tsindex 5db8199d34..17c5a3b373 100644--- a/packages/html-reporter/src/types.d.ts+++ b/packages/html-reporter/src/types.d.ts@@ -59,7 +59,7 @@ export type TestFileSummary = {stats: Stats;};-export type TestCaseAnnotation = { type: string, description?: string };+export type TestAnnotation = { type: string, description?: string };export type TestCaseSummary = {testId: string,@@ -67,7 +67,7 @@ export type TestCaseSummary = {path: string[];projectName: string;location: Location;- annotations: TestCaseAnnotation[];+ annotations: TestAnnotation[];tags: string[];outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';duration: number;@@ -98,6 +98,7 @@ export type TestResult = {errors: string[];attachments: TestAttachment[];status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';+ annotations: TestAnnotation[];};export type TestStep = {diff --git a/packages/playwright/src/common/suiteUtils.ts b/packages/playwright/src/common/suiteUtils.tsindex ab676548b5..20c46e3157 100644--- a/packages/playwright/src/common/suiteUtils.ts+++ b/packages/playwright/src/common/suiteUtils.ts@@ -64,10 +64,9 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit// Inherit properties from parent suites.let inheritedRetries: number | undefined;let inheritedTimeout: number | undefined;- test.annotations = [];for (let parentSuite: Suite | undefined = suite; parentSuite; parentSuite = parentSuite.parent) {if (parentSuite._staticAnnotations.length)- test.annotations = [...parentSuite._staticAnnotations, ...test.annotations];+ test.annotations.unshift(...parentSuite._staticAnnotations);if (inheritedRetries === undefined && parentSuite._retries !== undefined)inheritedRetries = parentSuite._retries;if (inheritedTimeout === undefined && parentSuite._timeout !== undefined)@@ -75,7 +74,6 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit}test.retries = inheritedRetries ?? project.project.retries;test.timeout = inheritedTimeout ?? project.project.timeout;- test.annotations.push(...test._staticAnnotations);// Skip annotations imply skipped expectedStatus.if (test.annotations.some(a => a.type === 'skip' || a.type === 'fixme'))diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.tsindex 7b54f0c291..2b6434a2ef 100644--- a/packages/playwright/src/common/test.ts+++ b/packages/playwright/src/common/test.ts@@ -262,8 +262,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {_poolDigest = '';_workerHash = '';_projectId = '';- // Annotations known statically before running the test, e.g. `test.skip()` or `test(title, { annotation }, body)`.- _staticAnnotations: Annotation[] = [];// Explicitly declared tags that are not a part of the title._tags: string[] = [];@@ -306,7 +304,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {requireFile: this._requireFile,poolDigest: this._poolDigest,workerHash: this._workerHash,- staticAnnotations: this._staticAnnotations.slice(),annotations: this.annotations.slice(),tags: this._tags.slice(),projectId: this._projectId,@@ -323,7 +320,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {test._requireFile = data.requireFile;test._poolDigest = data.poolDigest;test._workerHash = data.workerHash;- test._staticAnnotations = data.staticAnnotations;test.annotations = data.annotations;test._tags = data.tags;test._projectId = data.projectId;@@ -351,6 +347,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {status: 'skipped',steps: [],errors: [],+ annotations: [],};this.results.push(result);return result;diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.tsindex 565ed58321..1fa0f4ce6c 100644--- a/packages/playwright/src/common/testType.ts+++ b/packages/playwright/src/common/testType.ts@@ -107,16 +107,16 @@ export class TestTypeImpl {const validatedDetails = validateTestDetails(details);const test = new TestCase(title, body, this, location);test._requireFile = suite._requireFile;- test._staticAnnotations.push(...validatedDetails.annotations);+ test.annotations.push(...validatedDetails.annotations);test._tags.push(...validatedDetails.tags);suite._addTest(test);if (type === 'only' || type === 'fail.only')test._only = true;if (type === 'skip' || type === 'fixme' || type === 'fail')- test._staticAnnotations.push({ type });+ test.annotations.push({ type });else if (type === 'fail.only')- test._staticAnnotations.push({ type: 'fail' });+ test.annotations.push({ type: 'fail' });}private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) {diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.tsindex 7143fcf822..4504c160fd 100644--- a/packages/playwright/src/isomorphic/teleReceiver.ts+++ b/packages/playwright/src/isomorphic/teleReceiver.ts@@ -68,14 +68,14 @@ export type JsonTestCase = {retries: number;tags?: string[];repeatEachIndex: number;- annotations?: { type: string, description?: string }[];+ annotations?: Annotation[];};export type JsonTestEnd = {testId: string;expectedStatus: reporterTypes.TestStatus;timeout: number;- annotations: { type: string, description?: string }[];+ annotations: Annotation[];};export type JsonTestResultStart = {@@ -94,6 +94,7 @@ export type JsonTestResultEnd = {status: reporterTypes.TestStatus;errors: reporterTypes.TestError[];attachments: JsonAttachment[];+ annotations?: Annotation[];};export type JsonTestStepStart = {@@ -241,6 +242,8 @@ export class TeleReporterReceiver {result.errors = payload.errors;result.error = result.errors?.[0];result.attachments = this._parseAttachments(payload.attachments);+ if (payload.annotations)+ result.annotations = payload.annotations;this._reporter.onTestEnd?.(test, result);// Free up the memory as won't see these step ids.result._stepMap = new Map();@@ -562,6 +565,7 @@ export class TeleTestResult implements reporterTypes.TestResult {stdout: reporterTypes.TestResult['stdout'] = [];stderr: reporterTypes.TestResult['stderr'] = [];attachments: reporterTypes.TestResult['attachments'] = [];+ annotations: reporterTypes.TestResult['annotations'] = [];status: reporterTypes.TestStatus = 'skipped';steps: TeleTestStep[] = [];errors: reporterTypes.TestResult['errors'] = [];diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.tsindex 30534d7e8f..c8326ece31 100644--- a/packages/playwright/src/reporters/base.ts+++ b/packages/playwright/src/reporters/base.ts@@ -291,10 +291,13 @@ export class TerminalReporter implements ReporterV2 {}private _printWarnings() {- const warningTests = this.suite.allTests().filter(test => test.annotations.some(a => a.type === 'warning'));+ const warningTests = this.suite.allTests().filter(test => {+ const annotations = [...test.annotations, ...test.results.flatMap(r => r.annotations)];+ return annotations.some(a => a.type === 'warning');+ });const encounteredWarnings = new Map<string, Array<TestCase>>();for (const test of warningTests) {- for (const annotation of test.annotations) {+ for (const annotation of [...test.annotations, ...test.results.flatMap(r => r.annotations)]) {if (annotation.type !== 'warning' || annotation.description === undefined)continue;let tests = encounteredWarnings.get(annotation.description);diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.tsindex ec83a58046..dfe51127cc 100644--- a/packages/playwright/src/reporters/html.ts+++ b/packages/playwright/src/reporters/html.ts@@ -31,7 +31,7 @@ import { resolveReporterOutputPath, stripAnsiEscapes } from '../util';import type { ReporterV2 } from './reporterV2';import type { Metadata } from '../../types/test';import type * as api from '../../types/testReporter';-import type { HTMLReport, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types';+import type { HTMLReport, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep, TestAnnotation } from '@html-reporter/types';import type { ZipFile } from 'playwright-core/lib/zipBundle';import type { TransformCallback } from 'stream';@@ -404,8 +404,7 @@ class HtmlBuilder {projectName,location,duration,- // Annotations can be pushed directly, with a wrong type.- annotations: test.annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })),+ annotations: this._serializeAnnotations(test.annotations),tags: test.tags,outcome: test.outcome(),path,@@ -418,8 +417,7 @@ class HtmlBuilder {projectName,location,duration,- // Annotations can be pushed directly, with a wrong type.- annotations: test.annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })),+ annotations: this._serializeAnnotations([...test.annotations, ...results.flatMap(r => r.annotations)]),tags: test.tags,outcome: test.outcome(),path,@@ -503,6 +501,11 @@ class HtmlBuilder {}).filter(Boolean) as TestAttachment[];}+ private _serializeAnnotations(annotations: api.TestCase['annotations']): TestAnnotation[] {+ // Annotations can be pushed directly, with a wrong type.+ return annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description }));+ }+private _createTestResult(test: api.TestCase, result: api.TestResult): TestResult {return {duration: result.duration,@@ -511,6 +514,7 @@ class HtmlBuilder {steps: dedupeSteps(result.steps).map(s => this._createTestStep(s, result)),errors: formatResultFailure(internalScreen, test, result, '').map(error => error.message),status: result.status,+ annotations: this._serializeAnnotations(result.annotations),attachments: this._serializeAttachments([...result.attachments,...result.stdout.map(m => stdioAttachment(m, 'stdout')),diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.tsindex ac33d25d9a..6cdd994531 100644--- a/packages/playwright/src/reporters/json.ts+++ b/packages/playwright/src/reporters/json.ts@@ -213,6 +213,7 @@ class JSONReporter implements ReporterV2 {retry: result.retry,steps: steps.length ? steps.map(s => this._serializeTestStep(s)) : undefined,startTime: result.startTime.toISOString(),+ annotations: result.annotations,attachments: result.attachments.map(a => ({name: a.name,contentType: a.contentType,diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.tsindex a89ebd7df7..1ae8d4f133 100644--- a/packages/playwright/src/reporters/junit.ts+++ b/packages/playwright/src/reporters/junit.ts@@ -166,7 +166,8 @@ class JUnitReporter implements ReporterV2 {children: [] as XMLEntry[]};- for (const annotation of test.annotations) {+ const annotations = [...test.annotations, ...test.results.flatMap(r => r.annotations)];+ for (const annotation of annotations) {const property: XMLEntry = {name: 'property',attributes: {diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.tsindex 7b1571f12b..45532683cf 100644--- a/packages/playwright/src/reporters/teleEmitter.ts+++ b/packages/playwright/src/reporters/teleEmitter.ts@@ -237,6 +237,7 @@ export class TeleReporterEmitter implements ReporterV2 {status: result.status,errors: result.errors,attachments: this._serializeAttachments(result.attachments),+ annotations: result.annotations?.length ? result.annotations : undefined,};}diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex 462f7551c4..ec654de091 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -325,8 +325,8 @@ class JobDispatcher {result.errors = params.errors;result.error = result.errors[0];result.status = params.status;+ result.annotations = params.annotations;test.expectedStatus = params.expectedStatus;- test.annotations = params.annotations;test.timeout = params.timeout;const isFailure = result.status !== 'skipped' && result.status !== test.expectedStatus;if (isFailure)diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.tsindex 9b61c4b0b0..54f5bf6968 100644--- a/packages/playwright/src/worker/workerMain.ts+++ b/packages/playwright/src/worker/workerMain.ts@@ -292,6 +292,8 @@ export class WorkerMain extends ProcessRunner {for (const annotation of test.annotations)processAnnotation(annotation);+ const staticAnnotations = new Set(testInfo.annotations);+// Process existing annotations dynamically set for parent suites.for (const suite of suites) {const extraAnnotations = this._activeSuites.get(suite) || [];@@ -310,7 +312,7 @@ export class WorkerMain extends ProcessRunner {if (isSkipped && nextTest && !hasAfterAllToRunBeforeNextTest) {// Fast path - this test is skipped, and there are more tests that will handle cleanup.testInfo.status = 'skipped';- this.dispatchEvent('testEnd', buildTestEndPayload(testInfo));+ this.dispatchEvent('testEnd', buildTestEndPayload(testInfo, staticAnnotations));return;}@@ -492,7 +494,7 @@ export class WorkerMain extends ProcessRunner {this._currentTest = null;setCurrentTestInfo(null);- this.dispatchEvent('testEnd', buildTestEndPayload(testInfo));+ this.dispatchEvent('testEnd', buildTestEndPayload(testInfo, staticAnnotations));const preserveOutput = this._config.config.preserveOutput === 'always' ||(this._config.config.preserveOutput === 'failures-only' && testInfo._isFailure());@@ -612,7 +614,7 @@ function buildTestBeginPayload(testInfo: TestInfoImpl): TestBeginPayload {};}-function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {+function buildTestEndPayload(testInfo: TestInfoImpl, staticAnnotations: Set<Annotation>): TestEndPayload {return {testId: testInfo.testId,duration: testInfo.duration,@@ -620,7 +622,7 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {errors: testInfo.errors,hasNonRetriableError: testInfo._hasNonRetriableError,expectedStatus: testInfo.expectedStatus,- annotations: testInfo.annotations,+ annotations: testInfo.annotations.filter(a => !staticAnnotations.has(a)),timeout: testInfo.timeout,};}diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.tsindex 2abfa9592e..0c27925be2 100644--- a/packages/playwright/types/testReporter.d.ts+++ b/packages/playwright/types/testReporter.d.ts@@ -307,6 +307,7 @@ export interface JSONReportTestResult {body?: string;contentType: string;}[];+ annotations: { type: string, description?: string }[];errorLocation?: Location;}@@ -445,10 +446,8 @@ export interface TestCase {* [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip),* [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme)* and- * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail);- * - annotations appended to- * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations) during the test- * execution.+ * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail)+ * prior to test execution.** Annotations are available during test execution through* [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations).@@ -592,6 +591,34 @@ export interface TestError {* A result of a single [TestCase](https://playwright.dev/docs/api/class-testcase) run.*/export interface TestResult {+ /**+ * The list of annotations appended during test execution. Includes:+ * - annotations implicitly added by methods+ * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip),+ * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme)+ * and+ * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail)+ * during test execution;+ * - annotations appended to+ * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations).+ *+ * Annotations are available during test execution through+ * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations).+ *+ * Learn more about [test annotations](https://playwright.dev/docs/test-annotations).+ */+ annotations: Array<{+ /**+ * Annotation type, for example `'skip'` or `'fail'`.+ */+ type: string;++ /**+ * Optional description.+ */+ description?: string;+ }>;+/*** The list of files or buffers attached during the test execution through* [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments).diff --git a/packages/trace-viewer/src/ui/uiModeTraceView.tsx b/packages/trace-viewer/src/ui/uiModeTraceView.tsxindex cf35d89007..84e2e92cb3 100644--- a/packages/trace-viewer/src/ui/uiModeTraceView.tsx+++ b/packages/trace-viewer/src/ui/uiModeTraceView.tsx@@ -86,6 +86,8 @@ export const TraceView: React.FC<{};}, [outputDir, item, setModel, counter, setCounter, pathSeparator]);+ const annotations = item.testCase ? [...item.testCase.annotations, ...(item.testCase.results[0]?.annotations ?? [])] : [];+return <Workbenchkey='workbench'model={model?.model}@@ -94,7 +96,7 @@ export const TraceView: React.FC<{fallbackLocation={item.testFile}isLive={model?.isLive}status={item.treeItem?.status}- annotations={item.testCase?.annotations || []}+ annotations={annotations}onOpenExternally={onOpenExternally}revealSource={revealSource}/>;diff --git a/tests/playwright-test/access-data.spec.ts b/tests/playwright-test/access-data.spec.tsindex 67958003be..90a12942b1 100644--- a/tests/playwright-test/access-data.spec.ts+++ b/tests/playwright-test/access-data.spec.ts@@ -60,7 +60,7 @@ test('should access annotations in fixture', async ({ runInlineTest }) => {});expect(exitCode).toBe(0);const test = report.suites[0].specs[0].tests[0];- expect(test.annotations).toEqual([{ type: 'slow', description: 'just slow' }, { type: 'myname', description: 'hello' }]);+ expect(test.results[0].annotations).toEqual([{ type: 'slow', description: 'just slow' }, { type: 'myname', description: 'hello' }]);expect(test.results[0].stdout).toEqual([{ text: 'console.log\n' }]);expect(test.results[0].stderr).toEqual([{ text: 'console.error\n' }]);});diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.tsindex c06019a3b2..168d072120 100644--- a/tests/playwright-test/playwright-test-fixtures.ts+++ b/tests/playwright-test/playwright-test-fixtures.ts@@ -434,7 +434,7 @@ export function expectTestHelper(result: RunResult) {for (const test of tests) {expect(test.expectedStatus, `title: ${title}`).toBe(expectedStatus);expect(test.status, `title: ${title}`).toBe(status);- expect(test.annotations.map(a => a.type), `title: ${title}`).toEqual(annotations);+ expect([...test.annotations, ...test.results.flatMap(r => r.annotations)].map(a => a.type), `title: ${title}`).toEqual(annotations);}};}diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.tsindex 727cbe4077..b4e1c5fa5e 100644--- a/tests/playwright-test/reporter-html.spec.ts+++ b/tests/playwright-test/reporter-html.spec.ts@@ -829,6 +829,33 @@ for (const useIntermediateMergeReport of [true, false] as const) {await expect(page.locator('.test-case-annotation')).toHaveText('issue: I am not interested in this test');});+ test('should render dynamic annotations at test result level', async ({ runInlineTest, page, showReport }) => {+ const result = await runInlineTest({+ 'playwright.config.js': `+ module.exports = { timeout: 1500, retries: 3 };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('annotated test', async ({}) => {+ test.info().annotations.push({ type: 'foo', description: 'retry #' + test.info().retry });+ test.info().annotations.push({ type: 'bar', description: 'static value' });+ throw new Error('fail');+ });+ `,+ }, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' });+ expect(result.failed).toBe(1);++ await showReport();+ await page.getByRole('link', { name: 'annotated test' }).click();+ await page.getByRole('tab', { name: 'Retry #1' }).click();+ await expect(page.getByTestId('test-case-annotations')).toContainText('foo: retry #1');++ await page.getByRole('tab', { name: 'Retry #3' }).click();+ await expect(page.getByTestId('test-case-annotations')).toContainText('foo: retry #3');++ await expect(page.getByTestId('test-case-annotations').getByText('static value')).toHaveCount(1);+ });+test('should render annotations as link if needed', async ({ runInlineTest, page, showReport, server }) => {const result = await runInlineTest({'playwright.config.js': `@@ -2431,12 +2458,13 @@ for (const useIntermediateMergeReport of [true, false] as const) {'a.test.js': `const { test, expect } = require('@playwright/test');test('annotated test',{ annotation :[{type:'key',description:'value'}]}, async ({}) => {expect(1).toBe(1);});+ test('slow test', () => { test.slow(); });test('non-annotated test', async ({}) => {expect(1).toBe(2);});`,}, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' });expect(result.exitCode).toBe(1);- expect(result.passed).toBe(1);+ expect(result.passed).toBe(2);expect(result.failed).toBe(1);await showReport();@@ -2456,6 +2484,11 @@ for (const useIntermediateMergeReport of [true, false] as const) {await expect(page.getByText('non-annotated test')).not.toBeVisible();await expect(page.getByText('annotated test')).toBeVisible();});++ await test.step('filter by result annotation', async () => {+ await searchInput.fill('annot:slow');+ await expect(page.getByText('slow test')).toBeVisible();+ });});test('tests should filter by fileName:line/column', async ({ runInlineTest, showReport, page }) => {diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.tsindex b5dfc7a347..80e4d23123 100644--- a/tests/playwright-test/retry.spec.ts+++ b/tests/playwright-test/retry.spec.ts@@ -260,5 +260,5 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInlineexpect(result.failed).toBe(0);expect(result.flaky).toBe(1);expect(result.output).toContain('Failed on first run');- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]);+ expect(result.report.suites[0].specs[0].tests[0].results[1].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]);});diff --git a/tests/playwright-test/test-modifiers.spec.ts b/tests/playwright-test/test-modifiers.spec.tsindex f7fb9a6ae0..86b9c2233c 100644--- a/tests/playwright-test/test-modifiers.spec.ts+++ b/tests/playwright-test/test-modifiers.spec.ts@@ -106,7 +106,7 @@ test('test modifiers should work', async ({ runInlineTest }) => {const test = spec.tests[0];expect(test.expectedStatus).toBe(expectedStatus);expect(test.results[0].status).toBe(status);- expect(test.annotations).toEqual(annotations);+ expect([...test.annotations, ...test.results.flatMap(r => r.annotations)]).toEqual(annotations);};expectTest('passed1', 'passed', 'passed', []);expectTest('passed2', 'passed', 'passed', []);@@ -306,23 +306,6 @@ test.describe('test modifier annotations', () => {expectTest('focused fail by suite', 'failed', 'expected', ['fail']);});- test('should not multiple on retry', async ({ runInlineTest }) => {- const result = await runInlineTest({- 'a.test.ts': `- import { test, expect } from '@playwright/test';- test('retry', () => {- test.info().annotations.push({ type: 'example' });- expect(1).toBe(2);- });- `,- }, { retries: 3 });- const expectTest = expectTestHelper(result);-- expect(result.exitCode).toBe(1);- expect(result.passed).toBe(0);- expectTest('retry', 'passed', 'unexpected', ['example']);- });-test('should not multiply on repeat-each', async ({ runInlineTest }) => {const result = await runInlineTest({'a.test.ts': `@@ -424,7 +407,7 @@ test('should skip inside fixture', async ({ runInlineTest }) => {});expect(result.exitCode).toBe(0);expect(result.skipped).toBe(1);- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);+ expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);});test('modifier with a function should throw in the test', async ({ runInlineTest }) => {@@ -477,8 +460,8 @@ test('test.skip with worker fixtures only should skip before hooks and tests', aexpect(result.passed).toBe(1);expect(result.skipped).toBe(2);expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([]);- expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);- expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);+ expect(result.report.suites[0].suites![0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);+ expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);expect(result.outputLines).toEqual(['beforeEach','passed',@@ -615,8 +598,8 @@ test('should skip all tests from beforeAll', async ({ runInlineTest }) => {'beforeAll','afterAll',]);- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);- expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);+ expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);+ expect(result.report.suites[0].specs[1].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);});test('should report skipped tests in-order with correct properties', async ({ runInlineTest }) => {@@ -712,7 +695,7 @@ test('static modifiers should be added in serial mode', async ({ runInlineTest }expect(result.passed).toBe(0);expect(result.skipped).toBe(2);expect(result.didNotRun).toBe(1);- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }]);+ expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'slow' }]);expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme' }]);expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip' }]);expect(result.report.suites[0].specs[3].tests[0].annotations).toEqual([]);diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.tsindex 70365a911e..b949dd3d0c 100644--- a/utils/generate_types/overrides-testReporter.d.ts+++ b/utils/generate_types/overrides-testReporter.d.ts@@ -125,6 +125,7 @@ export interface JSONReportTestResult {body?: string;contentType: string;}[];+ annotations: { type: string, description?: string }[];errorLocation?: Location;}