Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat: add location information for annotations (#36542)

Andrey Lushnikovdb2a52de84d4
diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md
index 541a547bc1..9f577cad9c 100644
--- a/docs/src/test-api/class-testinfo.md
+++ b/docs/src/test-api/class-testinfo.md
@@ -19,6 +19,7 @@ test('basic test', async ({ page }, testInfo) => {
- type: <[Array]<[Object]>>
- `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.
- `description` ?<[string]> Optional description.
+ - `location` ?<[Location]> Optional location in the source where the annotation is added.
The list of annotations applicable to the current test. Includes annotations from the test, annotations from all [`method: Test.describe`] groups the test belongs to and file-level annotations for the test file.
diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md
index a3c9c2db99..22a8588fb0 100644
--- a/docs/src/test-reporter-api/class-testcase.md
+++ b/docs/src/test-reporter-api/class-testcase.md
@@ -9,6 +9,7 @@
- type: <[Array]<[Object]>>
- `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.
- `description` ?<[string]> Optional description.
+ - `location` ?<[Location]> Optional location in the source where the annotation is added.
[`property: TestResult.annotations`] of the last test run.
diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md
index 263b299c49..ef41200b40 100644
--- a/docs/src/test-reporter-api/class-testresult.md
+++ b/docs/src/test-reporter-api/class-testresult.md
@@ -19,6 +19,7 @@ The list of files or buffers attached during the test execution through [`proper
- type: <[Array]<[Object]>>
- `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.
- `description` ?<[string]> Optional description.
+ - `location` ?<[Location]> Optional location in the source where the annotation is added.
The list of annotations applicable to the current test. Includes:
* annotations defined on the test or suite via [`method: Test.(call)`] and [`method: Test.describe`];
diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md
index a8d1c12110..84ba3abbda 100644
--- a/docs/src/test-reporter-api/class-teststep.md
+++ b/docs/src/test-reporter-api/class-teststep.md
@@ -58,6 +58,7 @@ List of steps inside this step.
- type: <[Array]<[Object]>>
- `type` <[string]> Annotation type, for example `'skip'`.
- `description` ?<[string]> Optional description.
+ - `location` ?<[Location]> Optional location in the source where the annotation is added.
The list of annotations applicable to the current test step.
diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts
index 19ffb9fb61..b7ab5648f2 100644
--- a/packages/playwright/src/common/testType.ts
+++ b/packages/playwright/src/common/testType.ts
@@ -102,7 +102,7 @@ export class TestTypeImpl {
details = fnOrDetails;
}
- const validatedDetails = validateTestDetails(details);
+ const validatedDetails = validateTestDetails(details, location);
const test = new TestCase(title, body, this, location);
test._requireFile = suite._requireFile;
test.annotations.push(...validatedDetails.annotations);
@@ -112,9 +112,9 @@ export class TestTypeImpl {
if (type === 'only' || type === 'fail.only')
test._only = true;
if (type === 'skip' || type === 'fixme' || type === 'fail')
- test.annotations.push({ type });
+ test.annotations.push({ type, location });
else if (type === 'fail.only')
- test.annotations.push({ type: 'fail' });
+ test.annotations.push({ type: 'fail', location });
}
private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) {
@@ -141,7 +141,7 @@ export class TestTypeImpl {
body = fn!;
}
- const validatedDetails = validateTestDetails(details);
+ const validatedDetails = validateTestDetails(details, location);
const child = new Suite(title, 'describe');
child._requireFile = suite._requireFile;
child.location = location;
@@ -156,7 +156,7 @@ export class TestTypeImpl {
if (type === 'parallel' || type === 'parallel.only')
child._parallelMode = 'parallel';
if (type === 'skip' || type === 'fixme')
- child._staticAnnotations.push({ type });
+ child._staticAnnotations.push({ type, location });
for (let parent: Suite | undefined = suite; parent; parent = parent.parent) {
if (parent._parallelMode === 'serial' && child._parallelMode === 'parallel')
@@ -227,7 +227,7 @@ export class TestTypeImpl {
if (modifierArgs.length >= 1 && !modifierArgs[0])
return;
const description = modifierArgs[1];
- suite._staticAnnotations.push({ type, description });
+ suite._staticAnnotations.push({ type, description, location });
}
return;
}
@@ -237,7 +237,7 @@ export class TestTypeImpl {
throw new Error(`test.${type}() can only be called inside test, describe block or fixture`);
if (typeof modifierArgs[0] === 'function')
throw new Error(`test.${type}() with a function can only be called inside describe block`);
- testInfo[type](...modifierArgs as [any, any]);
+ testInfo._modifier(type, location, modifierArgs as [any, any]);
}
private _setTimeout(location: Location, timeout: number) {
@@ -270,7 +270,7 @@ export class TestTypeImpl {
let result: Awaited<ReturnType<typeof raceAgainstDeadline<T>>> | undefined = undefined;
result = await raceAgainstDeadline(async () => {
try {
- return await step.info._runStepBody(expectation === 'skip', body);
+ return await step.info._runStepBody(expectation === 'skip', body, step.location);
} catch (e) {
// If the step timed out, the test fixtures will tear down, which in turn
// will abort unfinished actions in the step body. Record such errors here.
@@ -309,8 +309,9 @@ function throwIfRunningInsideJest() {
}
}
-function validateTestDetails(details: TestDetails) {
- const annotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []);
+function validateTestDetails(details: TestDetails, location: Location) {
+ const originalAnnotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []);
+ const annotations = originalAnnotations.map(annotation => ({ ...annotation, location }));
const tags = Array.isArray(details.tag) ? details.tag : (details.tag ? [details.tag] : []);
for (const tag of tags) {
if (tag[0] !== '@')
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index 5f0ac0f693..df3e1c84a4 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -355,8 +355,9 @@ export class TeleReporterReceiver {
if (!!payload.attachments)
result.attachments = this._parseAttachments(payload.attachments);
if (payload.annotations) {
+ this._absoluteAnnotationLocationsInplace(payload.annotations);
result.annotations = payload.annotations;
- test.annotations = result.annotations;
+ test.annotations = payload.annotations;
}
this._reporter.onTestEnd?.(test, result);
// Free up the memory as won't see these step ids.
@@ -499,9 +500,17 @@ export class TeleReporterReceiver {
test.retries = payload.retries;
test.tags = payload.tags ?? [];
test.annotations = payload.annotations ?? [];
+ this._absoluteAnnotationLocationsInplace(test.annotations);
return test;
}
+ private _absoluteAnnotationLocationsInplace(annotations: TestAnnotation[]) {
+ for (const annotation of annotations) {
+ if (annotation.location)
+ annotation.location = this._absoluteLocation(annotation.location);
+ }
+ }
+
private _absoluteLocation(location: reporterTypes.Location): reporterTypes.Location;
private _absoluteLocation(location?: reporterTypes.Location): reporterTypes.Location | undefined;
private _absoluteLocation(location: reporterTypes.Location | undefined): reporterTypes.Location | undefined {
diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts
index f955bf8070..2946ab676f 100644
--- a/packages/playwright/src/reporters/html.ts
+++ b/packages/playwright/src/reporters/html.ts
@@ -502,7 +502,15 @@ class HtmlBuilder {
private _serializeAnnotations(annotations: api.TestCase['annotations']): TestAnnotation[] {
// Annotations can be pushed directly, with a wrong type.
- return annotations.map(a => ({ type: a.type, description: a.description === undefined ? undefined : String(a.description) }));
+ return annotations.map(a => ({
+ type: a.type,
+ description: a.description === undefined ? undefined : String(a.description),
+ location: a.location ? {
+ file: a.location.file,
+ line: a.location.line,
+ column: a.location.column,
+ } : undefined,
+ }));
}
private _createTestResult(test: api.TestCase, result: api.TestResult): TestResult {
diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts
index 10f12a5a54..719cf3c567 100644
--- a/packages/playwright/src/reporters/merge.ts
+++ b/packages/playwright/src/reporters/merge.ts
@@ -26,7 +26,7 @@ import { TeleReporterReceiver } from '../isomorphic/teleReceiver';
import { createReporters } from '../runner/reporters';
import { relativeFilePath } from '../util';
-import type { ReporterDescription } from '../../types/test';
+import type { ReporterDescription, TestAnnotation } from '../../types/test';
import type { TestError } from '../../types/testReporter';
import type { FullConfigInternal } from '../common/config';
import type { BlobReportMetadata, JsonAttachment, JsonConfig, JsonEvent, JsonFullResult, JsonLocation, JsonOnConfigureEvent, JsonOnEndEvent, JsonOnProjectEvent, JsonProject, JsonSuite, JsonTestCase } from '../isomorphic/teleReceiver';
@@ -484,7 +484,10 @@ class PathSeparatorPatcher {
return;
}
if (jsonEvent.method === 'onTestEnd') {
+ const test = jsonEvent.params.test;
+ test.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
const testResult = jsonEvent.params.result;
+ testResult.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
testResult.errors.forEach(error => this._updateErrorLocations(error));
(testResult.attachments ?? []).forEach(attachment => {
if (attachment.path)
@@ -500,6 +503,7 @@ class PathSeparatorPatcher {
if (jsonEvent.method === 'onStepEnd') {
const step = jsonEvent.params.step;
this._updateErrorLocations(step.error);
+ step.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
return;
}
if (jsonEvent.method === 'onAttach') {
@@ -524,10 +528,12 @@ class PathSeparatorPatcher {
if (isFileSuite)
suite.title = this._updatePath(suite.title);
for (const entry of suite.entries) {
- if ('testId' in entry)
+ if ('testId' in entry) {
this._updateLocation(entry.location);
- else
+ entry.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
+ } else {
this._updateSuite(entry);
+ }
}
}
@@ -538,6 +544,10 @@ class PathSeparatorPatcher {
}
}
+ private _updateAnnotationLocation(annotation: TestAnnotation) {
+ this._updateLocation(annotation.location);
+ }
+
private _updateLocation(location?: JsonLocation) {
if (location)
location.file = this._updatePath(location.file);
diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts
index b403cab303..44fbcec723 100644
--- a/packages/playwright/src/reporters/teleEmitter.ts
+++ b/packages/playwright/src/reporters/teleEmitter.ts
@@ -22,6 +22,7 @@ import { serializeRegexPatterns } from '../isomorphic/teleReceiver';
import type { ReporterV2 } from './reporterV2';
import type * as reporterTypes from '../../types/testReporter';
+import type { TestAnnotation } from '../../types/test';
import type * as teleReceiver from '../isomorphic/teleReceiver';
export type TeleReporterEmitterOptions = {
@@ -224,7 +225,7 @@ export class TeleReporterEmitter implements ReporterV2 {
retries: test.retries,
tags: test.tags,
repeatEachIndex: test.repeatEachIndex,
- annotations: test.annotations,
+ annotations: this._relativeAnnotationLocations(test.annotations),
};
}
@@ -244,7 +245,7 @@ export class TeleReporterEmitter implements ReporterV2 {
duration: result.duration,
status: result.status,
errors: result.errors,
- annotations: result.annotations?.length ? result.annotations : undefined,
+ annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined,
};
}
@@ -294,10 +295,17 @@ export class TeleReporterEmitter implements ReporterV2 {
duration: step.duration,
error: step.error,
attachments: step.attachments.length ? step.attachments.map(a => result.attachments.indexOf(a)) : undefined,
- annotations: step.annotations.length ? step.annotations : undefined,
+ annotations: step.annotations.length ? this._relativeAnnotationLocations(step.annotations) : undefined,
};
}
+ private _relativeAnnotationLocations(annotations: TestAnnotation[]): TestAnnotation[] {
+ return annotations.map(annotation => ({
+ ...annotation,
+ location: annotation.location ? this._relativeLocation(annotation.location) : undefined,
+ }));
+ }
+
private _relativeLocation(location: reporterTypes.Location): reporterTypes.Location;
private _relativeLocation(location?: reporterTypes.Location): reporterTypes.Location | undefined;
private _relativeLocation(location: reporterTypes.Location | undefined): reporterTypes.Location | undefined {
diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts
index 0a7736d3a6..19a2fa5071 100644
--- a/packages/playwright/src/worker/testInfo.ts
+++ b/packages/playwright/src/worker/testInfo.ts
@@ -23,6 +23,7 @@ import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutMana
import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, sanitizeFilePathBeforeExtension, trimLongString, windowsFilesystemFriendlyLength } from '../util';
import { TestTracing } from './testTracing';
import { testInfoError } from './util';
+import { wrapFunctionWithLocation } from '../transform/transform';
import type { RunnableDescription } from './timeoutManager';
import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test';
@@ -80,6 +81,12 @@ export class TestInfoImpl implements TestInfo {
_hasUnhandledError = false;
_allowSkips = false;
+ // ------------ Main methods ------------
+ skip: (arg?: any, description?: string) => void;
+ fixme: (arg?: any, description?: string) => void;
+ fail: (arg?: any, description?: string) => void;
+ slow: (arg?: any, description?: string) => void;
+
// ------------ TestInfo fields ------------
readonly testId: string;
readonly repeatEachIndex: number;
@@ -205,9 +212,14 @@ export class TestInfoImpl implements TestInfo {
};
this._tracing = new TestTracing(this, workerParams.artifactsDir);
+
+ this.skip = wrapFunctionWithLocation((location, ...args) => this._modifier('skip', location, args));
+ this.fixme = wrapFunctionWithLocation((location, ...args) => this._modifier('fixme', location, args));
+ this.fail = wrapFunctionWithLocation((location, ...args) => this._modifier('fail', location, args));
+ this.slow = wrapFunctionWithLocation((location, ...args) => this._modifier('slow', location, args));
}
- private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) {
+ _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', location: Location, modifierArgs: [arg?: any, description?: string]) {
if (typeof modifierArgs[1] === 'function') {
throw new Error([
'It looks like you are calling test.skip() inside the test and pass a callback.',
@@ -222,7 +234,7 @@ export class TestInfoImpl implements TestInfo {
return;
const description = modifierArgs[1];
- this.annotations.push({ type, description });
+ this.annotations.push({ type, description, location });
if (type === 'slow') {
this._timeoutManager.slow();
} else if (type === 'skip' || type === 'fixme') {
@@ -567,22 +579,6 @@ export class TestInfoImpl implements TestInfo {
return this._resolveSnapshotPaths(kind, name.length <= 1 ? name[0] : name, 'dontUpdateSnapshotIndex').absoluteSnapshotPath;
}
- skip(...args: [arg?: any, description?: string]) {
- this._modifier('skip', args);
- }
-
- fixme(...args: [arg?: any, description?: string]) {
- this._modifier('fixme', args);
- }
-
- fail(...args: [arg?: any, description?: string]) {
- this._modifier('fail', args);
- }
-
- slow(...args: [arg?: any, description?: string]) {
- this._modifier('slow', args);
- }
-
setTimeout(timeout: number) {
this._timeoutManager.setTimeout(timeout);
}
@@ -594,14 +590,25 @@ export class TestStepInfoImpl implements TestStepInfo {
private _testInfo: TestInfoImpl;
private _stepId: string;
+ skip: (arg?: any, description?: string) => void;
+
constructor(testInfo: TestInfoImpl, stepId: string) {
this._testInfo = testInfo;
this._stepId = stepId;
+ this.skip = wrapFunctionWithLocation((location: Location, ...args: unknown[]) => {
+ // skip();
+ // skip(condition: boolean, description: string);
+ if (args.length > 0 && !args[0])
+ return;
+ const description = args[1] as (string|undefined);
+ this.annotations.push({ type: 'skip', description, location });
+ throw new StepSkipError(description);
+ });
}
- async _runStepBody<T>(skip: boolean, body: (step: TestStepInfo) => T | Promise<T>) {
+ async _runStepBody<T>(skip: boolean, body: (step: TestStepInfo) => T | Promise<T>, location?: Location) {
if (skip) {
- this.annotations.push({ type: 'skip' });
+ this.annotations.push({ type: 'skip', location });
return undefined as T;
}
try {
@@ -620,16 +627,6 @@ export class TestStepInfoImpl implements TestStepInfo {
async attach(name: string, options?: { body?: string | Buffer; contentType?: string; path?: string; }): Promise<void> {
this._attachToStep(await normalizeAndSaveAttachment(this._testInfo.outputPath(), name, options));
}
-
- skip(...args: unknown[]) {
- // skip();
- // skip(condition: boolean, description: string);
- if (args.length > 0 && !args[0])
- return;
- const description = args[1] as (string|undefined);
- this.annotations.push({ type: 'skip', description });
- throw new StepSkipError(description);
- }
}
export class TestSkipError extends Error {
diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts
index 59dbae0f4f..e9f515a760 100644
--- a/packages/playwright/src/worker/workerMain.ts
+++ b/packages/playwright/src/worker/workerMain.ts
@@ -494,7 +494,7 @@ export class WorkerMain extends ProcessRunner {
continue;
const fn = async (fixtures: any) => {
const result = await modifier.fn(fixtures);
- testInfo[modifier.type](!!result, modifier.description);
+ testInfo._modifier(modifier.type, modifier.location, [!!result, modifier.description]);
};
inheritFixtureNames(modifier.fn, fn);
runnables.push({
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index fe1e3ba1ff..8390543f34 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -2325,6 +2325,11 @@ export interface TestInfo {
* Optional description.
*/
description?: string;
+
+ /**
+ * Optional location in the source where the annotation is added.
+ */
+ location?: Location;
}>;
/**
@@ -2575,7 +2580,9 @@ export type TestDetailsAnnotation = {
description?: string;
};
-export type TestAnnotation = TestDetailsAnnotation;
+export type TestAnnotation = TestDetailsAnnotation & {
+ location?: Location;
+};
export type TestDetails = {
tag?: string | string[];
diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts
index bbf22d3bc6..d03244a1ec 100644
--- a/packages/playwright/types/testReporter.d.ts
+++ b/packages/playwright/types/testReporter.d.ts
@@ -451,6 +451,11 @@ export interface TestCase {
* Optional description.
*/
description?: string;
+
+ /**
+ * Optional location in the source where the annotation is added.
+ */
+ location?: Location;
}>;
/**
@@ -607,6 +612,11 @@ export interface TestResult {
* Optional description.
*/
description?: string;
+
+ /**
+ * Optional location in the source where the annotation is added.
+ */
+ location?: Location;
}>;
/**
@@ -722,6 +732,11 @@ export interface TestStep {
* Optional description.
*/
description?: string;
+
+ /**
+ * Optional location in the source where the annotation is added.
+ */
+ location?: Location;
}>;
/**
diff --git a/tests/playwright-test/access-data.spec.ts b/tests/playwright-test/access-data.spec.ts
index ab5d3e4584..4da80d8de0 100644
--- a/tests/playwright-test/access-data.spec.ts
+++ b/tests/playwright-test/access-data.spec.ts
@@ -61,7 +61,7 @@ test('should access annotations in fixture', async ({ runInlineTest }) => {
expect(exitCode).toBe(0);
const test = report.suites[0].specs[0].tests[0];
expect(test.annotations).toEqual([
- { type: 'slow', description: 'just slow' },
+ { type: 'slow', description: 'just slow', location: { file: expect.any(String), line: 10, column: 14 } },
{ type: 'myname', description: 'hello' }
]);
expect(test.results[0].stdout).toEqual([{ text: 'console.log\n' }]);
diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts
index 26cd4082db..de7c1a10e4 100644
--- a/tests/playwright-test/reporter-blob.spec.ts
+++ b/tests/playwright-test/reporter-blob.spec.ts
@@ -1758,6 +1758,9 @@ test('merge reports with different rootDirs and path separators', async ({ runIn
console.log('test:', test.location.file);
console.log('test title:', test.titlePath()[2]);
}
+ onTestEnd(test) {
+ console.log('annotations:', test.annotations.map(a => 'type: ' + a.type + ', description: ' + a.description + ', file: ' + a.location.file).join(','));
+ }
};
`,
'merge.config.ts': `module.exports = {
@@ -1769,7 +1772,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn
};`,
'dir1/tests1/a.test.js': `
import { test, expect } from '@playwright/test';
- test('math 1', async ({}) => { });
+ test('math 1', { annotation: { type: 'warning', description: 'Some warning' } }, async ({}) => { });
`,
};
await runInlineTest(files1, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir1/playwright.config.ts')] });
@@ -1780,7 +1783,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn
};`,
'dir2/tests2/b.test.js': `
import { test, expect } from '@playwright/test';
- test('math 2', async ({}) => { });
+ test('math 2', { annotation: { type: 'issue' } }, async ({}) => { });
`,
};
await runInlineTest(files2, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir2/playwright.config.ts')] });
@@ -1802,12 +1805,16 @@ test('merge reports with different rootDirs and path separators', async ({ runIn
{
const { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--config', 'merge.config.ts'] });
+ const testPath1 = test.info().outputPath('mergeRoot', 'tests1', 'a.test.js');
+ const testPath2 = test.info().outputPath('mergeRoot', 'tests2', 'b.test.js');
expect(exitCode).toBe(0);
expect(output).toContain(`rootDir: ${test.info().outputPath('mergeRoot')}`);
- expect(output).toContain(`test: ${test.info().outputPath('mergeRoot', 'tests1', 'a.test.js')}`);
+ expect(output).toContain(`test: ${testPath1}`);
expect(output).toContain(`test title: ${'tests1' + path.sep + 'a.test.js'}`);
- expect(output).toContain(`test: ${test.info().outputPath('mergeRoot', 'tests2', 'b.test.js')}`);
+ expect(output).toContain(`annotations: type: warning, description: Some warning, file: ${testPath1}`);
+ expect(output).toContain(`test: ${testPath2}`);
expect(output).toContain(`test title: ${'tests2' + path.sep + 'b.test.js'}`);
+ expect(output).toContain(`annotations: type: issue, description: undefined, file: ${testPath2}`);
}
});
@@ -1823,6 +1830,9 @@ test('merge reports without --config preserves path separators', async ({ runInl
console.log('test:', test.location.file);
console.log('test title:', test.titlePath()[2]);
}
+ onTestEnd(test) {
+ console.log('annotations:', test.annotations.map(a => 'type: ' + a.type + ', description: ' + a.description + ', file: ' + a.location.file).join(','));
+ }
};
`,
'dir1/playwright.config.ts': `module.exports = {
@@ -1830,11 +1840,11 @@ test('merge reports without --config preserves path separators', async ({ runInl
};`,
'dir1/tests1/a.test.js': `
import { test, expect } from '@playwright/test';
- test('math 1', async ({}) => { });
+ test('math 1', { annotation: { type: 'warning', description: 'Some warning' } }, async ({}) => { });
`,
'dir1/tests2/b.test.js': `
import { test, expect } from '@playwright/test';
- test('math 2', async ({}) => { });
+ test('math 2', { annotation: { type: 'issue' } }, async ({}) => { });
`,
};
await runInlineTest(files1, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir1/playwright.config.ts')] });
@@ -1854,11 +1864,15 @@ test('merge reports without --config preserves path separators', async ({ runInl
const { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--reporter', './echo-reporter.js'] });
expect(exitCode).toBe(0);
const otherSeparator = path.sep === '/' ? '\\' : '/';
+ const testPath1 = test.info().outputPath('dir1', 'tests1', 'a.test.js').replaceAll(path.sep, otherSeparator);
+ const testPath2 = test.info().outputPath('dir1', 'tests2', 'b.test.js').replaceAll(path.sep, otherSeparator);
expect(output).toContain(`rootDir: ${test.info().outputPath('dir1').replaceAll(path.sep, otherSeparator)}`);
- expect(output).toContain(`test: ${test.info().outputPath('dir1', 'tests1', 'a.test.js').replaceAll(path.sep, otherSeparator)}`);
+ expect(output).toContain(`test: ${testPath1}`);
expect(output).toContain(`test title: ${'tests1' + otherSeparator + 'a.test.js'}`);
- expect(output).toContain(`test: ${test.info().outputPath('dir1', 'tests2', 'b.test.js').replaceAll(path.sep, otherSeparator)}`);
+ expect(output).toContain(`annotations: type: warning, description: Some warning, file: ${testPath1}`);
+ expect(output).toContain(`test: ${testPath2}`);
expect(output).toContain(`test title: ${'tests2' + otherSeparator + 'b.test.js'}`);
+ expect(output).toContain(`annotations: type: issue, description: undefined, file: ${testPath2}`);
});
test('merge reports should preserve attachments', async ({ runInlineTest, mergeReports, showReport, page }) => {
diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts
index 1e28e8b288..afe152ff7e 100644
--- a/tests/playwright-test/reporter.spec.ts
+++ b/tests/playwright-test/reporter.spec.ts
@@ -584,7 +584,9 @@ test('should report annotations from test declaration', async ({ runInlineTest }
const visit = suite => {
for (const test of suite.tests || []) {
const annotations = test.annotations.map(a => {
- return a.description ? a.type + '=' + a.description : a.type;
+ const description = a.description ? a.type + '=' + a.description : a.type;
+ const location = a.location ? '(' + a.location.line + ':' + a.location.column + ')' : '';
+ return description + location;
});
console.log('\\n%%title=' + test.title + ', annotations=' + annotations.join(','));
}
@@ -609,7 +611,7 @@ test('should report annotations from test declaration', async ({ runInlineTest }
expect(test.info().annotations).toEqual([]);
});
test('foo', { annotation: { type: 'foo' } }, () => {
- expect(test.info().annotations).toEqual([{ type: 'foo' }]);
+ expect(test.info().annotations).toEqual([{ type: 'foo', location: { file: expect.any(String), line: 6, column: 11 } }]);
});
test('foo-bar', {
annotation: [
@@ -618,8 +620,8 @@ test('should report annotations from test declaration', async ({ runInlineTest }
],
}, () => {
expect(test.info().annotations).toEqual([
- { type: 'foo', description: 'desc' },
- { type: 'bar' },
+ { type: 'foo', description: 'desc', location: { file: expect.any(String), line: 9, column: 11 } },
+ { type: 'bar', location: { file: expect.any(String), line: 9, column: 11 } },
]);
});
test.skip('skip-foo', { annotation: { type: 'foo' } }, () => {
@@ -636,11 +638,14 @@ test('should report annotations from test declaration', async ({ runInlineTest }
});
test.describe('suite', { annotation: { type: 'foo' } }, () => {
test('foo-suite', () => {
- expect(test.info().annotations).toEqual([{ type: 'foo' }]);
+ expect(test.info().annotations).toEqual([{ type: 'foo', location: { file: expect.any(String), line: 32, column: 12 } }]);
});
test.describe('inner', { annotation: { type: 'bar' } }, () => {
test('foo-bar-suite', () => {
- expect(test.info().annotations).toEqual([{ type: 'foo' }, { type: 'bar' }]);
+ expect(test.info().annotations).toEqual([
+ { type: 'foo', location: { file: expect.any(String), line: 32, column: 12 } },
+ { type: 'bar', location: { file: expect.any(String), line: 36, column: 14 } }
+ ]);
});
});
});
@@ -657,15 +662,15 @@ test('should report annotations from test declaration', async ({ runInlineTest }
expect(result.exitCode).toBe(0);
expect(result.outputLines).toEqual([
`title=none, annotations=`,
- `title=foo, annotations=foo`,
- `title=foo-bar, annotations=foo=desc,bar`,
- `title=skip-foo, annotations=foo,skip`,
- `title=fixme-bar, annotations=bar,fixme`,
- `title=fail-foo-bar, annotations=foo,bar=desc,fail`,
- `title=foo-suite, annotations=foo`,
- `title=foo-bar-suite, annotations=foo,bar`,
- `title=skip-foo-suite, annotations=foo,skip`,
- `title=fixme-bar-suite, annotations=bar,fixme`,
+ `title=foo, annotations=foo(6:11)`,
+ `title=foo-bar, annotations=foo=desc(9:11),bar(9:11)`,
+ `title=skip-foo, annotations=foo(20:12),skip(20:12)`,
+ `title=fixme-bar, annotations=bar(22:12),fixme(22:12)`,
+ `title=fail-foo-bar, annotations=foo(24:12),bar=desc(24:12),fail(24:12)`,
+ `title=foo-suite, annotations=foo(32:12)`,
+ `title=foo-bar-suite, annotations=foo(32:12),bar(36:14)`,
+ `title=skip-foo-suite, annotations=foo(45:21),skip(45:21)`,
+ `title=fixme-bar-suite, annotations=bar(49:21),fixme(49:21)`,
]);
});
diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.ts
index b5dfc7a347..c82a61b20f 100644
--- a/tests/playwright-test/retry.spec.ts
+++ b/tests/playwright-test/retry.spec.ts
@@ -260,5 +260,5 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInline
expect(result.failed).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('Failed on first run');
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]);
+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry', location: expect.anything() }]);
});
diff --git a/tests/playwright-test/test-modifiers.spec.ts b/tests/playwright-test/test-modifiers.spec.ts
index e992fae2e7..e73e483b27 100644
--- a/tests/playwright-test/test-modifiers.spec.ts
+++ b/tests/playwright-test/test-modifiers.spec.ts
@@ -113,19 +113,19 @@ test('test modifiers should work', async ({ runInlineTest }) => {
expectTest('passed3', 'passed', 'passed', []);
expectTest('passed4', 'passed', 'passed', []);
expectTest('passed5', 'passed', 'passed', []);
- expectTest('skipped1', 'skipped', 'skipped', [{ type: 'skip' }]);
- expectTest('skipped2', 'skipped', 'skipped', [{ type: 'skip' }]);
- expectTest('skipped3', 'skipped', 'skipped', [{ type: 'skip' }]);
- expectTest('skipped4', 'skipped', 'skipped', [{ type: 'skip', description: 'reason' }]);
- expectTest('skipped5', 'skipped', 'skipped', [{ type: 'fixme' }]);
- expectTest('skipped6', 'skipped', 'skipped', [{ type: 'fixme', description: 'reason' }]);
- expectTest('failed1', 'failed', 'failed', [{ type: 'fail' }]);
- expectTest('failed2', 'failed', 'failed', [{ type: 'fail' }]);
- expectTest('failed3', 'failed', 'failed', [{ type: 'fail' }]);
- expectTest('failed4', 'failed', 'failed', [{ type: 'fail', description: 'reason' }]);
- expectTest('suite1', 'skipped', 'skipped', [{ type: 'skip' }]);
- expectTest('suite2', 'skipped', 'skipped', [{ type: 'skip' }]);
- expectTest('suite3', 'skipped', 'skipped', [{ type: 'skip', description: 'reason' }]);
+ expectTest('skipped1', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 20, column: 14 } }]);
+ expectTest('skipped2', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 23, column: 14 } }]);
+ expectTest('skipped3', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 26, column: 14 } }]);
+ expectTest('skipped4', 'skipped', 'skipped', [{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 29, column: 14 } }]);
+ expectTest('skipped5', 'skipped', 'skipped', [{ type: 'fixme', location: { file: expect.any(String), line: 32, column: 14 } }]);
+ expectTest('skipped6', 'skipped', 'skipped', [{ type: 'fixme', description: 'reason', location: { file: expect.any(String), line: 35, column: 14 } }]);
+ expectTest('failed1', 'failed', 'failed', [{ type: 'fail', location: { file: expect.any(String), line: 39, column: 14 } }]);
+ expectTest('failed2', 'failed', 'failed', [{ type: 'fail', location: { file: expect.any(String), line: 43, column: 14 } }]);
+ expectTest('failed3', 'failed', 'failed', [{ type: 'fail', location: { file: expect.any(String), line: 47, column: 14 } }]);
+ expectTest('failed4', 'failed', 'failed', [{ type: 'fail', description: 'reason', location: { file: expect.any(String), line: 51, column: 14 } }]);
+ expectTest('suite1', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 56, column: 14 } }]);
+ expectTest('suite2', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 61, column: 14 } }]);
+ expectTest('suite3', 'skipped', 'skipped', [{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 66, column: 14 } }]);
expectTest('suite4', 'passed', 'passed', []);
expect(result.passed).toBe(10);
expect(result.skipped).toBe(9);
@@ -407,7 +407,7 @@ test('should skip inside fixture', async ({ runInlineTest }) => {
});
expect(result.exitCode).toBe(0);
expect(result.skipped).toBe(1);
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 5, column: 20 } }]);
});
test('modifier with a function should throw in the test', async ({ runInlineTest }) => {
@@ -460,8 +460,8 @@ test('test.skip with worker fixtures only should skip before hooks and tests', a
expect(result.passed).toBe(1);
expect(result.skipped).toBe(2);
expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([]);
- expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
- expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+ expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 14, column: 14 } }]);
+ expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 14, column: 14 } }]);
expect(result.outputLines).toEqual([
'beforeEach',
'passed',
@@ -493,8 +493,8 @@ test('test.skip without a callback in describe block should skip hooks', async (
});
expect(result.exitCode).toBe(0);
expect(result.skipped).toBe(2);
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
- expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 10, column: 12 } }]);
+ expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 10, column: 12 } }]);
expect(result.output).not.toContain('%%');
});
@@ -598,8 +598,8 @@ test('should skip all tests from beforeAll', async ({ runInlineTest }) => {
'beforeAll',
'afterAll',
]);
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
- expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 5, column: 14 } }]);
+ expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 5, column: 14 } }]);
});
test('should report skipped tests in-order with correct properties', async ({ runInlineTest }) => {
@@ -695,9 +695,9 @@ test('static modifiers should be added in serial mode', async ({ runInlineTest }
expect(result.passed).toBe(0);
expect(result.skipped).toBe(2);
expect(result.didNotRun).toBe(1);
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }]);
- expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme' }]);
- expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip' }]);
+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow', location: { file: expect.any(String), line: 6, column: 14 } }]);
+ expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme', location: { file: expect.any(String), line: 9, column: 12 } }]);
+ expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip', location: { file: expect.any(String), line: 11, column: 12 } }]);
expect(result.report.suites[0].specs[3].tests[0].annotations).toEqual([]);
});
@@ -721,9 +721,18 @@ test('should contain only one slow modifier', async ({ runInlineTest }) => {
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'fixme' }, { type: 'issue', description: 'my-value' }]);
- expect(result.report.suites[1].specs[0].tests[0].annotations).toEqual([{ type: 'skip' }, { type: 'issue', description: 'my-value' }]);
- expect(result.report.suites[2].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }, { type: 'issue', description: 'my-value' }]);
+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([
+ { type: 'fixme', location: { file: expect.any(String), line: 3, column: 12 } },
+ { type: 'issue', description: 'my-value', location: { file: expect.any(String), line: 4, column: 11 } }
+ ]);
+ expect(result.report.suites[1].specs[0].tests[0].annotations).toEqual([
+ { type: 'skip', location: { file: expect.any(String), line: 3, column: 12 } },
+ { type: 'issue', description: 'my-value', location: { file: expect.any(String), line: 4, column: 11 } }
+ ]);
+ expect(result.report.suites[2].specs[0].tests[0].annotations).toEqual([
+ { type: 'slow', location: { file: expect.any(String), line: 3, column: 12 } },
+ { type: 'issue', description: 'my-value', location: { file: expect.any(String), line: 4, column: 11 } }
+ ]);
});
test('should skip beforeEach hooks upon modifiers', async ({ runInlineTest }) => {
diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts
index d00c5eaadb..2dfd024bfc 100644
--- a/utils/generate_types/overrides-test.d.ts
+++ b/utils/generate_types/overrides-test.d.ts
@@ -81,7 +81,9 @@ export type TestDetailsAnnotation = {
description?: string;
};
-export type TestAnnotation = TestDetailsAnnotation;
+export type TestAnnotation = TestDetailsAnnotation & {
+ location?: Location;
+};
export type TestDetails = {
tag?: string | string[];