Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat: split up static and dynamic annotations (#35292)

Simon Knott45fa3d17fccc
diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md
index 82ab6ca521..1aa5379ac4 100644
--- a/docs/src/test-reporter-api/class-testcase.md
+++ b/docs/src/test-reporter-api/class-testcase.md
@@ -12,8 +12,7 @@
The list of annotations applicable to the current test. Includes:
* annotations defined on the test or suite via [`method: Test.(call)`] and [`method: Test.describe`];
-* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`];
-* annotations appended to [`property: TestInfo.annotations`] during the test execution.
+* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`] prior to test execution.
Annotations are available during test execution through [`property: TestInfo.annotations`].
diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md
index 7674ac333d..0e60558632 100644
--- a/docs/src/test-reporter-api/class-testresult.md
+++ b/docs/src/test-reporter-api/class-testresult.md
@@ -14,6 +14,20 @@ A result of a single [TestCase] run.
The list of files or buffers attached during the test execution through [`property: TestInfo.attachments`].
+## property: TestResult.annotations
+* since: v1.52
+- type: <[Array]<[Object]>>
+ - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.
+ - `description` ?<[string]> Optional description.
+
+The list of annotations appended during test execution. Includes:
+* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`] during test execution;
+* annotations appended to [`property: TestInfo.annotations`].
+
+Annotations are available during test execution through [`property: TestInfo.annotations`].
+
+Learn more about [test annotations](../test-annotations.md).
+
## property: TestResult.duration
* since: v1.10
- type: <[float]>
diff --git a/packages/html-reporter/src/tabbedPane.tsx b/packages/html-reporter/src/tabbedPane.tsx
index 02d0c6f3b1..af15b6b4fa 100644
--- a/packages/html-reporter/src/tabbedPane.tsx
+++ b/packages/html-reporter/src/tabbedPane.tsx
@@ -30,14 +30,18 @@ export const TabbedPane: React.FunctionComponent<{
selectedTab: string,
setSelectedTab: (tab: string) => void
}> = ({ tabs, selectedTab, setSelectedTab }) => {
+ const idPrefix = React.useId();
return <div className='tabbed-pane'>
<div className='vbox'>
<div className='hbox' style={{ flex: 'none' }}>
- <div className='tabbed-pane-tab-strip'>{
+ <div className='tabbed-pane-tab-strip' role='tablist'>{
tabs.map(tab => (
<div className={clsx('tabbed-pane-tab-element', selectedTab === tab.id && 'selected')}
onClick={() => setSelectedTab(tab.id)}
- key={tab.id}>
+ id={`${idPrefix}-${tab.id}`}
+ key={tab.id}
+ role='tab'
+ aria-selected={selectedTab === tab.id}>
<div className='tabbed-pane-tab-label'>{tab.title}</div>
</div>
))
@@ -46,7 +50,7 @@ export const TabbedPane: React.FunctionComponent<{
{
tabs.map(tab => {
if (selectedTab === tab.id)
- return <div key={tab.id} className='tab-content'>{tab.render()}</div>;
+ return <div key={tab.id} className='tab-content' role='tabpanel' aria-labelledby={`${idPrefix}-${tab.id}`}>{tab.render()}</div>;
})
}
</div>
diff --git a/packages/html-reporter/src/testCaseView.spec.tsx b/packages/html-reporter/src/testCaseView.spec.tsx
index 8aa3fafc8f..3f6c7f2d05 100644
--- a/packages/html-reporter/src/testCaseView.spec.tsx
+++ b/packages/html-reporter/src/testCaseView.spec.tsx
@@ -42,6 +42,7 @@ const result: TestResult = {
}],
attachments: [],
}],
+ annotations: [],
attachments: [],
status: 'passed',
};
@@ -151,6 +152,7 @@ const resultWithAttachment: TestResult = {
name: 'attachment with inline link https://github.com/microsoft/playwright/issues/31284',
contentType: 'text/plain'
}],
+ annotations: [],
status: 'passed',
};
@@ -238,13 +240,15 @@ test('total duration is selected run duration', async ({ mount, page }) => {
const component = await mount(<TestCaseView projectNames={['chromium', 'webkit']} test={testCaseWithTwoAttempts} prev={undefined} next={undefined} run={0}></TestCaseView>);
await expect(component).toMatchAriaSnapshot(`
- text: "My test test.spec.ts:42 200ms"
- - text: "Run 50ms Retry #1 150ms"
+ - tablist:
+ - tab "Run 50ms"
+ - 'tab "Retry #1 150ms"'
`);
- await page.locator('.tabbed-pane-tab-label', { hasText: 'Run50ms' }).click();
+ await page.getByRole('tab', { name: 'Run' }).click();
await expect(component).toMatchAriaSnapshot(`
- text: "My test test.spec.ts:42 200ms"
`);
- await page.locator('.tabbed-pane-tab-label', { hasText: 'Retry #1150ms' }).click();
+ await page.getByRole('tab', { name: 'Retry' }).click();
await expect(component).toMatchAriaSnapshot(`
- text: "My test test.spec.ts:42 200ms"
`);
diff --git a/packages/html-reporter/src/testCaseView.tsx b/packages/html-reporter/src/testCaseView.tsx
index ca5ff6d2e2..56ad193c8c 100644
--- a/packages/html-reporter/src/testCaseView.tsx
+++ b/packages/html-reporter/src/testCaseView.tsx
@@ -14,7 +14,7 @@
limitations under the License.
*/
-import type { TestCase, TestCaseAnnotation, TestCaseSummary } from './types';
+import type { TestCase, TestAnnotation, TestCaseSummary } from './types';
import * as React from 'react';
import { TabbedPane } from './tabbedPane';
import { AutoChip } from './chip';
@@ -46,8 +46,13 @@ export const TestCaseView: React.FC<{
}, [test]);
const visibleAnnotations = React.useMemo(() => {
- return test?.annotations?.filter(annotation => !annotation.type.startsWith('_')) || [];
- }, [test?.annotations]);
+ if (!test)
+ return [];
+ const annotations = [...test.annotations];
+ if (test.results[selectedResultIndex])
+ annotations.push(...test.results[selectedResultIndex].annotations);
+ return annotations.filter(annotation => !annotation.type.startsWith('_'));
+ }, [test, selectedResultIndex]);
return <div className='test-case-column vbox'>
{test && <div className='hbox'>
@@ -71,7 +76,7 @@ export const TestCaseView: React.FC<{
{test && !!test.projectName && <ProjectLink projectNames={projectNames} projectName={test.projectName}></ProjectLink>}
{labels && <LabelsLinkView labels={labels} />}
</div>}
- {!!visibleAnnotations.length && <AutoChip header='Annotations'>
+ {!!visibleAnnotations.length && <AutoChip header='Annotations' dataTestId='test-case-annotations'>
{visibleAnnotations.map((annotation, index) => <TestCaseAnnotationView key={index} annotation={annotation} />)}
</AutoChip>}
{test && <TabbedPane tabs={
@@ -86,7 +91,7 @@ export const TestCaseView: React.FC<{
</div>;
};
-function TestCaseAnnotationView({ annotation: { type, description } }: { annotation: TestCaseAnnotation }) {
+function TestCaseAnnotationView({ annotation: { type, description } }: { annotation: TestAnnotation }) {
return (
<div className='test-case-annotation'>
<span style={{ fontWeight: 'bold' }}>{type}</span>
diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.ts
index 5db8199d34..17c5a3b373 100644
--- a/packages/html-reporter/src/types.d.ts
+++ b/packages/html-reporter/src/types.d.ts
@@ -59,7 +59,7 @@ export type TestFileSummary = {
stats: Stats;
};
-export type TestCaseAnnotation = { type: string, description?: string };
+export type TestAnnotation = { type: string, description?: string };
export type TestCaseSummary = {
testId: string,
@@ -67,7 +67,7 @@ export type TestCaseSummary = {
path: string[];
projectName: string;
location: Location;
- annotations: TestCaseAnnotation[];
+ annotations: TestAnnotation[];
tags: string[];
outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';
duration: number;
@@ -98,6 +98,7 @@ export type TestResult = {
errors: string[];
attachments: TestAttachment[];
status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
+ annotations: TestAnnotation[];
};
export type TestStep = {
diff --git a/packages/playwright/src/common/suiteUtils.ts b/packages/playwright/src/common/suiteUtils.ts
index ab676548b5..20c46e3157 100644
--- a/packages/playwright/src/common/suiteUtils.ts
+++ b/packages/playwright/src/common/suiteUtils.ts
@@ -64,10 +64,9 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit
// Inherit properties from parent suites.
let inheritedRetries: number | undefined;
let inheritedTimeout: number | undefined;
- test.annotations = [];
for (let parentSuite: Suite | undefined = suite; parentSuite; parentSuite = parentSuite.parent) {
if (parentSuite._staticAnnotations.length)
- test.annotations = [...parentSuite._staticAnnotations, ...test.annotations];
+ test.annotations.unshift(...parentSuite._staticAnnotations);
if (inheritedRetries === undefined && parentSuite._retries !== undefined)
inheritedRetries = parentSuite._retries;
if (inheritedTimeout === undefined && parentSuite._timeout !== undefined)
@@ -75,7 +74,6 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit
}
test.retries = inheritedRetries ?? project.project.retries;
test.timeout = inheritedTimeout ?? project.project.timeout;
- test.annotations.push(...test._staticAnnotations);
// Skip annotations imply skipped expectedStatus.
if (test.annotations.some(a => a.type === 'skip' || a.type === 'fixme'))
diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts
index 7b54f0c291..2b6434a2ef 100644
--- a/packages/playwright/src/common/test.ts
+++ b/packages/playwright/src/common/test.ts
@@ -262,8 +262,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {
_poolDigest = '';
_workerHash = '';
_projectId = '';
- // Annotations known statically before running the test, e.g. `test.skip()` or `test(title, { annotation }, body)`.
- _staticAnnotations: Annotation[] = [];
// Explicitly declared tags that are not a part of the title.
_tags: string[] = [];
@@ -306,7 +304,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {
requireFile: this._requireFile,
poolDigest: this._poolDigest,
workerHash: this._workerHash,
- staticAnnotations: this._staticAnnotations.slice(),
annotations: this.annotations.slice(),
tags: this._tags.slice(),
projectId: this._projectId,
@@ -323,7 +320,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {
test._requireFile = data.requireFile;
test._poolDigest = data.poolDigest;
test._workerHash = data.workerHash;
- test._staticAnnotations = data.staticAnnotations;
test.annotations = data.annotations;
test._tags = data.tags;
test._projectId = data.projectId;
@@ -351,6 +347,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
status: 'skipped',
steps: [],
errors: [],
+ annotations: [],
};
this.results.push(result);
return result;
diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts
index 565ed58321..1fa0f4ce6c 100644
--- a/packages/playwright/src/common/testType.ts
+++ b/packages/playwright/src/common/testType.ts
@@ -107,16 +107,16 @@ export class TestTypeImpl {
const validatedDetails = validateTestDetails(details);
const test = new TestCase(title, body, this, location);
test._requireFile = suite._requireFile;
- test._staticAnnotations.push(...validatedDetails.annotations);
+ test.annotations.push(...validatedDetails.annotations);
test._tags.push(...validatedDetails.tags);
suite._addTest(test);
if (type === 'only' || type === 'fail.only')
test._only = true;
if (type === 'skip' || type === 'fixme' || type === 'fail')
- test._staticAnnotations.push({ type });
+ test.annotations.push({ type });
else if (type === 'fail.only')
- test._staticAnnotations.push({ type: 'fail' });
+ test.annotations.push({ type: 'fail' });
}
private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) {
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index 7143fcf822..4504c160fd 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -68,14 +68,14 @@ export type JsonTestCase = {
retries: number;
tags?: string[];
repeatEachIndex: number;
- annotations?: { type: string, description?: string }[];
+ annotations?: Annotation[];
};
export type JsonTestEnd = {
testId: string;
expectedStatus: reporterTypes.TestStatus;
timeout: number;
- annotations: { type: string, description?: string }[];
+ annotations: Annotation[];
};
export type JsonTestResultStart = {
@@ -94,6 +94,7 @@ export type JsonTestResultEnd = {
status: reporterTypes.TestStatus;
errors: reporterTypes.TestError[];
attachments: JsonAttachment[];
+ annotations?: Annotation[];
};
export type JsonTestStepStart = {
@@ -241,6 +242,8 @@ export class TeleReporterReceiver {
result.errors = payload.errors;
result.error = result.errors?.[0];
result.attachments = this._parseAttachments(payload.attachments);
+ if (payload.annotations)
+ result.annotations = payload.annotations;
this._reporter.onTestEnd?.(test, result);
// Free up the memory as won't see these step ids.
result._stepMap = new Map();
@@ -562,6 +565,7 @@ export class TeleTestResult implements reporterTypes.TestResult {
stdout: reporterTypes.TestResult['stdout'] = [];
stderr: reporterTypes.TestResult['stderr'] = [];
attachments: reporterTypes.TestResult['attachments'] = [];
+ annotations: reporterTypes.TestResult['annotations'] = [];
status: reporterTypes.TestStatus = 'skipped';
steps: TeleTestStep[] = [];
errors: reporterTypes.TestResult['errors'] = [];
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts
index 30534d7e8f..c8326ece31 100644
--- a/packages/playwright/src/reporters/base.ts
+++ b/packages/playwright/src/reporters/base.ts
@@ -291,10 +291,13 @@ export class TerminalReporter implements ReporterV2 {
}
private _printWarnings() {
- const warningTests = this.suite.allTests().filter(test => test.annotations.some(a => a.type === 'warning'));
+ const warningTests = this.suite.allTests().filter(test => {
+ const annotations = [...test.annotations, ...test.results.flatMap(r => r.annotations)];
+ return annotations.some(a => a.type === 'warning');
+ });
const encounteredWarnings = new Map<string, Array<TestCase>>();
for (const test of warningTests) {
- for (const annotation of test.annotations) {
+ for (const annotation of [...test.annotations, ...test.results.flatMap(r => r.annotations)]) {
if (annotation.type !== 'warning' || annotation.description === undefined)
continue;
let tests = encounteredWarnings.get(annotation.description);
diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts
index ec83a58046..dfe51127cc 100644
--- a/packages/playwright/src/reporters/html.ts
+++ b/packages/playwright/src/reporters/html.ts
@@ -31,7 +31,7 @@ import { resolveReporterOutputPath, stripAnsiEscapes } from '../util';
import type { ReporterV2 } from './reporterV2';
import type { Metadata } from '../../types/test';
import type * as api from '../../types/testReporter';
-import type { HTMLReport, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types';
+import type { HTMLReport, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep, TestAnnotation } from '@html-reporter/types';
import type { ZipFile } from 'playwright-core/lib/zipBundle';
import type { TransformCallback } from 'stream';
@@ -404,8 +404,7 @@ class HtmlBuilder {
projectName,
location,
duration,
- // Annotations can be pushed directly, with a wrong type.
- annotations: test.annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })),
+ annotations: this._serializeAnnotations(test.annotations),
tags: test.tags,
outcome: test.outcome(),
path,
@@ -418,8 +417,7 @@ class HtmlBuilder {
projectName,
location,
duration,
- // Annotations can be pushed directly, with a wrong type.
- annotations: test.annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })),
+ annotations: this._serializeAnnotations([...test.annotations, ...results.flatMap(r => r.annotations)]),
tags: test.tags,
outcome: test.outcome(),
path,
@@ -503,6 +501,11 @@ class HtmlBuilder {
}).filter(Boolean) as TestAttachment[];
}
+ private _serializeAnnotations(annotations: api.TestCase['annotations']): TestAnnotation[] {
+ // Annotations can be pushed directly, with a wrong type.
+ return annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description }));
+ }
+
private _createTestResult(test: api.TestCase, result: api.TestResult): TestResult {
return {
duration: result.duration,
@@ -511,6 +514,7 @@ class HtmlBuilder {
steps: dedupeSteps(result.steps).map(s => this._createTestStep(s, result)),
errors: formatResultFailure(internalScreen, test, result, '').map(error => error.message),
status: result.status,
+ annotations: this._serializeAnnotations(result.annotations),
attachments: this._serializeAttachments([
...result.attachments,
...result.stdout.map(m => stdioAttachment(m, 'stdout')),
diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts
index ac33d25d9a..6cdd994531 100644
--- a/packages/playwright/src/reporters/json.ts
+++ b/packages/playwright/src/reporters/json.ts
@@ -213,6 +213,7 @@ class JSONReporter implements ReporterV2 {
retry: result.retry,
steps: steps.length ? steps.map(s => this._serializeTestStep(s)) : undefined,
startTime: result.startTime.toISOString(),
+ annotations: result.annotations,
attachments: result.attachments.map(a => ({
name: a.name,
contentType: a.contentType,
diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts
index a89ebd7df7..1ae8d4f133 100644
--- a/packages/playwright/src/reporters/junit.ts
+++ b/packages/playwright/src/reporters/junit.ts
@@ -166,7 +166,8 @@ class JUnitReporter implements ReporterV2 {
children: [] as XMLEntry[]
};
- for (const annotation of test.annotations) {
+ const annotations = [...test.annotations, ...test.results.flatMap(r => r.annotations)];
+ for (const annotation of annotations) {
const property: XMLEntry = {
name: 'property',
attributes: {
diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts
index 7b1571f12b..45532683cf 100644
--- a/packages/playwright/src/reporters/teleEmitter.ts
+++ b/packages/playwright/src/reporters/teleEmitter.ts
@@ -237,6 +237,7 @@ export class TeleReporterEmitter implements ReporterV2 {
status: result.status,
errors: result.errors,
attachments: this._serializeAttachments(result.attachments),
+ annotations: result.annotations?.length ? result.annotations : undefined,
};
}
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index 462f7551c4..ec654de091 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -325,8 +325,8 @@ class JobDispatcher {
result.errors = params.errors;
result.error = result.errors[0];
result.status = params.status;
+ result.annotations = params.annotations;
test.expectedStatus = params.expectedStatus;
- test.annotations = params.annotations;
test.timeout = params.timeout;
const isFailure = result.status !== 'skipped' && result.status !== test.expectedStatus;
if (isFailure)
diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts
index 9b61c4b0b0..54f5bf6968 100644
--- a/packages/playwright/src/worker/workerMain.ts
+++ b/packages/playwright/src/worker/workerMain.ts
@@ -292,6 +292,8 @@ export class WorkerMain extends ProcessRunner {
for (const annotation of test.annotations)
processAnnotation(annotation);
+ const staticAnnotations = new Set(testInfo.annotations);
+
// Process existing annotations dynamically set for parent suites.
for (const suite of suites) {
const extraAnnotations = this._activeSuites.get(suite) || [];
@@ -310,7 +312,7 @@ export class WorkerMain extends ProcessRunner {
if (isSkipped && nextTest && !hasAfterAllToRunBeforeNextTest) {
// Fast path - this test is skipped, and there are more tests that will handle cleanup.
testInfo.status = 'skipped';
- this.dispatchEvent('testEnd', buildTestEndPayload(testInfo));
+ this.dispatchEvent('testEnd', buildTestEndPayload(testInfo, staticAnnotations));
return;
}
@@ -492,7 +494,7 @@ export class WorkerMain extends ProcessRunner {
this._currentTest = null;
setCurrentTestInfo(null);
- this.dispatchEvent('testEnd', buildTestEndPayload(testInfo));
+ this.dispatchEvent('testEnd', buildTestEndPayload(testInfo, staticAnnotations));
const preserveOutput = this._config.config.preserveOutput === 'always' ||
(this._config.config.preserveOutput === 'failures-only' && testInfo._isFailure());
@@ -612,7 +614,7 @@ function buildTestBeginPayload(testInfo: TestInfoImpl): TestBeginPayload {
};
}
-function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {
+function buildTestEndPayload(testInfo: TestInfoImpl, staticAnnotations: Set<Annotation>): TestEndPayload {
return {
testId: testInfo.testId,
duration: testInfo.duration,
@@ -620,7 +622,7 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {
errors: testInfo.errors,
hasNonRetriableError: testInfo._hasNonRetriableError,
expectedStatus: testInfo.expectedStatus,
- annotations: testInfo.annotations,
+ annotations: testInfo.annotations.filter(a => !staticAnnotations.has(a)),
timeout: testInfo.timeout,
};
}
diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts
index 2abfa9592e..0c27925be2 100644
--- a/packages/playwright/types/testReporter.d.ts
+++ b/packages/playwright/types/testReporter.d.ts
@@ -307,6 +307,7 @@ export interface JSONReportTestResult {
body?: string;
contentType: string;
}[];
+ annotations: { type: string, description?: string }[];
errorLocation?: Location;
}
@@ -445,10 +446,8 @@ export interface TestCase {
* [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip),
* [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme)
* and
- * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail);
- * - annotations appended to
- * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations) during the test
- * execution.
+ * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail)
+ * prior to test execution.
*
* Annotations are available during test execution through
* [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations).
@@ -592,6 +591,34 @@ export interface TestError {
* A result of a single [TestCase](https://playwright.dev/docs/api/class-testcase) run.
*/
export interface TestResult {
+ /**
+ * The list of annotations appended during test execution. Includes:
+ * - annotations implicitly added by methods
+ * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip),
+ * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme)
+ * and
+ * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail)
+ * during test execution;
+ * - annotations appended to
+ * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations).
+ *
+ * Annotations are available during test execution through
+ * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations).
+ *
+ * Learn more about [test annotations](https://playwright.dev/docs/test-annotations).
+ */
+ annotations: Array<{
+ /**
+ * Annotation type, for example `'skip'` or `'fail'`.
+ */
+ type: string;
+
+ /**
+ * Optional description.
+ */
+ description?: string;
+ }>;
+
/**
* The list of files or buffers attached during the test execution through
* [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments).
diff --git a/packages/trace-viewer/src/ui/uiModeTraceView.tsx b/packages/trace-viewer/src/ui/uiModeTraceView.tsx
index cf35d89007..84e2e92cb3 100644
--- a/packages/trace-viewer/src/ui/uiModeTraceView.tsx
+++ b/packages/trace-viewer/src/ui/uiModeTraceView.tsx
@@ -86,6 +86,8 @@ export const TraceView: React.FC<{
};
}, [outputDir, item, setModel, counter, setCounter, pathSeparator]);
+ const annotations = item.testCase ? [...item.testCase.annotations, ...(item.testCase.results[0]?.annotations ?? [])] : [];
+
return <Workbench
key='workbench'
model={model?.model}
@@ -94,7 +96,7 @@ export const TraceView: React.FC<{
fallbackLocation={item.testFile}
isLive={model?.isLive}
status={item.treeItem?.status}
- annotations={item.testCase?.annotations || []}
+ annotations={annotations}
onOpenExternally={onOpenExternally}
revealSource={revealSource}
/>;
diff --git a/tests/playwright-test/access-data.spec.ts b/tests/playwright-test/access-data.spec.ts
index 67958003be..90a12942b1 100644
--- a/tests/playwright-test/access-data.spec.ts
+++ b/tests/playwright-test/access-data.spec.ts
@@ -60,7 +60,7 @@ test('should access annotations in fixture', async ({ runInlineTest }) => {
});
expect(exitCode).toBe(0);
const test = report.suites[0].specs[0].tests[0];
- expect(test.annotations).toEqual([{ type: 'slow', description: 'just slow' }, { type: 'myname', description: 'hello' }]);
+ expect(test.results[0].annotations).toEqual([{ type: 'slow', description: 'just slow' }, { type: 'myname', description: 'hello' }]);
expect(test.results[0].stdout).toEqual([{ text: 'console.log\n' }]);
expect(test.results[0].stderr).toEqual([{ text: 'console.error\n' }]);
});
diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts
index c06019a3b2..168d072120 100644
--- a/tests/playwright-test/playwright-test-fixtures.ts
+++ b/tests/playwright-test/playwright-test-fixtures.ts
@@ -434,7 +434,7 @@ export function expectTestHelper(result: RunResult) {
for (const test of tests) {
expect(test.expectedStatus, `title: ${title}`).toBe(expectedStatus);
expect(test.status, `title: ${title}`).toBe(status);
- expect(test.annotations.map(a => a.type), `title: ${title}`).toEqual(annotations);
+ expect([...test.annotations, ...test.results.flatMap(r => r.annotations)].map(a => a.type), `title: ${title}`).toEqual(annotations);
}
};
}
diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts
index 727cbe4077..b4e1c5fa5e 100644
--- a/tests/playwright-test/reporter-html.spec.ts
+++ b/tests/playwright-test/reporter-html.spec.ts
@@ -829,6 +829,33 @@ for (const useIntermediateMergeReport of [true, false] as const) {
await expect(page.locator('.test-case-annotation')).toHaveText('issue: I am not interested in this test');
});
+ test('should render dynamic annotations at test result level', async ({ runInlineTest, page, showReport }) => {
+ const result = await runInlineTest({
+ 'playwright.config.js': `
+ module.exports = { timeout: 1500, retries: 3 };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('annotated test', async ({}) => {
+ test.info().annotations.push({ type: 'foo', description: 'retry #' + test.info().retry });
+ test.info().annotations.push({ type: 'bar', description: 'static value' });
+ throw new Error('fail');
+ });
+ `,
+ }, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' });
+ expect(result.failed).toBe(1);
+
+ await showReport();
+ await page.getByRole('link', { name: 'annotated test' }).click();
+ await page.getByRole('tab', { name: 'Retry #1' }).click();
+ await expect(page.getByTestId('test-case-annotations')).toContainText('foo: retry #1');
+
+ await page.getByRole('tab', { name: 'Retry #3' }).click();
+ await expect(page.getByTestId('test-case-annotations')).toContainText('foo: retry #3');
+
+ await expect(page.getByTestId('test-case-annotations').getByText('static value')).toHaveCount(1);
+ });
+
test('should render annotations as link if needed', async ({ runInlineTest, page, showReport, server }) => {
const result = await runInlineTest({
'playwright.config.js': `
@@ -2431,12 +2458,13 @@ for (const useIntermediateMergeReport of [true, false] as const) {
'a.test.js': `
const { test, expect } = require('@playwright/test');
test('annotated test',{ annotation :[{type:'key',description:'value'}]}, async ({}) => {expect(1).toBe(1);});
+ test('slow test', () => { test.slow(); });
test('non-annotated test', async ({}) => {expect(1).toBe(2);});
`,
}, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' });
expect(result.exitCode).toBe(1);
- expect(result.passed).toBe(1);
+ expect(result.passed).toBe(2);
expect(result.failed).toBe(1);
await showReport();
@@ -2456,6 +2484,11 @@ for (const useIntermediateMergeReport of [true, false] as const) {
await expect(page.getByText('non-annotated test')).not.toBeVisible();
await expect(page.getByText('annotated test')).toBeVisible();
});
+
+ await test.step('filter by result annotation', async () => {
+ await searchInput.fill('annot:slow');
+ await expect(page.getByText('slow test')).toBeVisible();
+ });
});
test('tests should filter by fileName:line/column', async ({ runInlineTest, showReport, page }) => {
diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.ts
index b5dfc7a347..80e4d23123 100644
--- a/tests/playwright-test/retry.spec.ts
+++ b/tests/playwright-test/retry.spec.ts
@@ -260,5 +260,5 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInline
expect(result.failed).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('Failed on first run');
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]);
+ expect(result.report.suites[0].specs[0].tests[0].results[1].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]);
});
diff --git a/tests/playwright-test/test-modifiers.spec.ts b/tests/playwright-test/test-modifiers.spec.ts
index f7fb9a6ae0..86b9c2233c 100644
--- a/tests/playwright-test/test-modifiers.spec.ts
+++ b/tests/playwright-test/test-modifiers.spec.ts
@@ -106,7 +106,7 @@ test('test modifiers should work', async ({ runInlineTest }) => {
const test = spec.tests[0];
expect(test.expectedStatus).toBe(expectedStatus);
expect(test.results[0].status).toBe(status);
- expect(test.annotations).toEqual(annotations);
+ expect([...test.annotations, ...test.results.flatMap(r => r.annotations)]).toEqual(annotations);
};
expectTest('passed1', 'passed', 'passed', []);
expectTest('passed2', 'passed', 'passed', []);
@@ -306,23 +306,6 @@ test.describe('test modifier annotations', () => {
expectTest('focused fail by suite', 'failed', 'expected', ['fail']);
});
- test('should not multiple on retry', async ({ runInlineTest }) => {
- const result = await runInlineTest({
- 'a.test.ts': `
- import { test, expect } from '@playwright/test';
- test('retry', () => {
- test.info().annotations.push({ type: 'example' });
- expect(1).toBe(2);
- });
- `,
- }, { retries: 3 });
- const expectTest = expectTestHelper(result);
-
- expect(result.exitCode).toBe(1);
- expect(result.passed).toBe(0);
- expectTest('retry', 'passed', 'unexpected', ['example']);
- });
-
test('should not multiply on repeat-each', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
@@ -424,7 +407,7 @@ test('should skip inside fixture', async ({ runInlineTest }) => {
});
expect(result.exitCode).toBe(0);
expect(result.skipped).toBe(1);
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+ expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
});
test('modifier with a function should throw in the test', async ({ runInlineTest }) => {
@@ -477,8 +460,8 @@ test('test.skip with worker fixtures only should skip before hooks and tests', a
expect(result.passed).toBe(1);
expect(result.skipped).toBe(2);
expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([]);
- expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
- expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+ expect(result.report.suites[0].suites![0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+ expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
expect(result.outputLines).toEqual([
'beforeEach',
'passed',
@@ -615,8 +598,8 @@ test('should skip all tests from beforeAll', async ({ runInlineTest }) => {
'beforeAll',
'afterAll',
]);
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
- expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+ expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+ expect(result.report.suites[0].specs[1].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
});
test('should report skipped tests in-order with correct properties', async ({ runInlineTest }) => {
@@ -712,7 +695,7 @@ test('static modifiers should be added in serial mode', async ({ runInlineTest }
expect(result.passed).toBe(0);
expect(result.skipped).toBe(2);
expect(result.didNotRun).toBe(1);
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }]);
+ expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'slow' }]);
expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme' }]);
expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip' }]);
expect(result.report.suites[0].specs[3].tests[0].annotations).toEqual([]);
diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts
index 70365a911e..b949dd3d0c 100644
--- a/utils/generate_types/overrides-testReporter.d.ts
+++ b/utils/generate_types/overrides-testReporter.d.ts
@@ -125,6 +125,7 @@ export interface JSONReportTestResult {
body?: string;
contentType: string;
}[];
+ annotations: { type: string, description?: string }[];
errorLocation?: Location;
}