Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat: add location information for annotations (#36542)
Andrey Lushnikovdb2a52de84d4
diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.mdindex 541a547bc1..9f577cad9c 100644--- a/docs/src/test-api/class-testinfo.md+++ b/docs/src/test-api/class-testinfo.md@@ -19,6 +19,7 @@ test('basic test', async ({ page }, testInfo) => {- type: <[Array]<[Object]>>- `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.- `description` ?<[string]> Optional description.+ - `location` ?<[Location]> Optional location in the source where the annotation is added.The list of annotations applicable to the current test. Includes annotations from the test, annotations from all [`method: Test.describe`] groups the test belongs to and file-level annotations for the test file.diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.mdindex a3c9c2db99..22a8588fb0 100644--- a/docs/src/test-reporter-api/class-testcase.md+++ b/docs/src/test-reporter-api/class-testcase.md@@ -9,6 +9,7 @@- type: <[Array]<[Object]>>- `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.- `description` ?<[string]> Optional description.+ - `location` ?<[Location]> Optional location in the source where the annotation is added.[`property: TestResult.annotations`] of the last test run.diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.mdindex 263b299c49..ef41200b40 100644--- a/docs/src/test-reporter-api/class-testresult.md+++ b/docs/src/test-reporter-api/class-testresult.md@@ -19,6 +19,7 @@ The list of files or buffers attached during the test execution through [`proper- type: <[Array]<[Object]>>- `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.- `description` ?<[string]> Optional description.+ - `location` ?<[Location]> Optional location in the source where the annotation is added.The list of annotations applicable to the current test. Includes:* annotations defined on the test or suite via [`method: Test.(call)`] and [`method: Test.describe`];diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.mdindex a8d1c12110..84ba3abbda 100644--- a/docs/src/test-reporter-api/class-teststep.md+++ b/docs/src/test-reporter-api/class-teststep.md@@ -58,6 +58,7 @@ List of steps inside this step.- type: <[Array]<[Object]>>- `type` <[string]> Annotation type, for example `'skip'`.- `description` ?<[string]> Optional description.+ - `location` ?<[Location]> Optional location in the source where the annotation is added.The list of annotations applicable to the current test step.diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.tsindex 19ffb9fb61..b7ab5648f2 100644--- a/packages/playwright/src/common/testType.ts+++ b/packages/playwright/src/common/testType.ts@@ -102,7 +102,7 @@ export class TestTypeImpl {details = fnOrDetails;}- const validatedDetails = validateTestDetails(details);+ const validatedDetails = validateTestDetails(details, location);const test = new TestCase(title, body, this, location);test._requireFile = suite._requireFile;test.annotations.push(...validatedDetails.annotations);@@ -112,9 +112,9 @@ export class TestTypeImpl {if (type === 'only' || type === 'fail.only')test._only = true;if (type === 'skip' || type === 'fixme' || type === 'fail')- test.annotations.push({ type });+ test.annotations.push({ type, location });else if (type === 'fail.only')- test.annotations.push({ type: 'fail' });+ test.annotations.push({ type: 'fail', location });}private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) {@@ -141,7 +141,7 @@ export class TestTypeImpl {body = fn!;}- const validatedDetails = validateTestDetails(details);+ const validatedDetails = validateTestDetails(details, location);const child = new Suite(title, 'describe');child._requireFile = suite._requireFile;child.location = location;@@ -156,7 +156,7 @@ export class TestTypeImpl {if (type === 'parallel' || type === 'parallel.only')child._parallelMode = 'parallel';if (type === 'skip' || type === 'fixme')- child._staticAnnotations.push({ type });+ child._staticAnnotations.push({ type, location });for (let parent: Suite | undefined = suite; parent; parent = parent.parent) {if (parent._parallelMode === 'serial' && child._parallelMode === 'parallel')@@ -227,7 +227,7 @@ export class TestTypeImpl {if (modifierArgs.length >= 1 && !modifierArgs[0])return;const description = modifierArgs[1];- suite._staticAnnotations.push({ type, description });+ suite._staticAnnotations.push({ type, description, location });}return;}@@ -237,7 +237,7 @@ export class TestTypeImpl {throw new Error(`test.${type}() can only be called inside test, describe block or fixture`);if (typeof modifierArgs[0] === 'function')throw new Error(`test.${type}() with a function can only be called inside describe block`);- testInfo[type](...modifierArgs as [any, any]);+ testInfo._modifier(type, location, modifierArgs as [any, any]);}private _setTimeout(location: Location, timeout: number) {@@ -270,7 +270,7 @@ export class TestTypeImpl {let result: Awaited<ReturnType<typeof raceAgainstDeadline<T>>> | undefined = undefined;result = await raceAgainstDeadline(async () => {try {- return await step.info._runStepBody(expectation === 'skip', body);+ return await step.info._runStepBody(expectation === 'skip', body, step.location);} catch (e) {// If the step timed out, the test fixtures will tear down, which in turn// will abort unfinished actions in the step body. Record such errors here.@@ -309,8 +309,9 @@ function throwIfRunningInsideJest() {}}-function validateTestDetails(details: TestDetails) {- const annotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []);+function validateTestDetails(details: TestDetails, location: Location) {+ const originalAnnotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []);+ const annotations = originalAnnotations.map(annotation => ({ ...annotation, location }));const tags = Array.isArray(details.tag) ? details.tag : (details.tag ? [details.tag] : []);for (const tag of tags) {if (tag[0] !== '@')diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.tsindex 5f0ac0f693..df3e1c84a4 100644--- a/packages/playwright/src/isomorphic/teleReceiver.ts+++ b/packages/playwright/src/isomorphic/teleReceiver.ts@@ -355,8 +355,9 @@ export class TeleReporterReceiver {if (!!payload.attachments)result.attachments = this._parseAttachments(payload.attachments);if (payload.annotations) {+ this._absoluteAnnotationLocationsInplace(payload.annotations);result.annotations = payload.annotations;- test.annotations = result.annotations;+ test.annotations = payload.annotations;}this._reporter.onTestEnd?.(test, result);// Free up the memory as won't see these step ids.@@ -499,9 +500,17 @@ export class TeleReporterReceiver {test.retries = payload.retries;test.tags = payload.tags ?? [];test.annotations = payload.annotations ?? [];+ this._absoluteAnnotationLocationsInplace(test.annotations);return test;}+ private _absoluteAnnotationLocationsInplace(annotations: TestAnnotation[]) {+ for (const annotation of annotations) {+ if (annotation.location)+ annotation.location = this._absoluteLocation(annotation.location);+ }+ }+private _absoluteLocation(location: reporterTypes.Location): reporterTypes.Location;private _absoluteLocation(location?: reporterTypes.Location): reporterTypes.Location | undefined;private _absoluteLocation(location: reporterTypes.Location | undefined): reporterTypes.Location | undefined {diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.tsindex f955bf8070..2946ab676f 100644--- a/packages/playwright/src/reporters/html.ts+++ b/packages/playwright/src/reporters/html.ts@@ -502,7 +502,15 @@ class HtmlBuilder {private _serializeAnnotations(annotations: api.TestCase['annotations']): TestAnnotation[] {// Annotations can be pushed directly, with a wrong type.- return annotations.map(a => ({ type: a.type, description: a.description === undefined ? undefined : String(a.description) }));+ return annotations.map(a => ({+ type: a.type,+ description: a.description === undefined ? undefined : String(a.description),+ location: a.location ? {+ file: a.location.file,+ line: a.location.line,+ column: a.location.column,+ } : undefined,+ }));}private _createTestResult(test: api.TestCase, result: api.TestResult): TestResult {diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.tsindex 10f12a5a54..719cf3c567 100644--- a/packages/playwright/src/reporters/merge.ts+++ b/packages/playwright/src/reporters/merge.ts@@ -26,7 +26,7 @@ import { TeleReporterReceiver } from '../isomorphic/teleReceiver';import { createReporters } from '../runner/reporters';import { relativeFilePath } from '../util';-import type { ReporterDescription } from '../../types/test';+import type { ReporterDescription, TestAnnotation } from '../../types/test';import type { TestError } from '../../types/testReporter';import type { FullConfigInternal } from '../common/config';import type { BlobReportMetadata, JsonAttachment, JsonConfig, JsonEvent, JsonFullResult, JsonLocation, JsonOnConfigureEvent, JsonOnEndEvent, JsonOnProjectEvent, JsonProject, JsonSuite, JsonTestCase } from '../isomorphic/teleReceiver';@@ -484,7 +484,10 @@ class PathSeparatorPatcher {return;}if (jsonEvent.method === 'onTestEnd') {+ const test = jsonEvent.params.test;+ test.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));const testResult = jsonEvent.params.result;+ testResult.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));testResult.errors.forEach(error => this._updateErrorLocations(error));(testResult.attachments ?? []).forEach(attachment => {if (attachment.path)@@ -500,6 +503,7 @@ class PathSeparatorPatcher {if (jsonEvent.method === 'onStepEnd') {const step = jsonEvent.params.step;this._updateErrorLocations(step.error);+ step.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));return;}if (jsonEvent.method === 'onAttach') {@@ -524,10 +528,12 @@ class PathSeparatorPatcher {if (isFileSuite)suite.title = this._updatePath(suite.title);for (const entry of suite.entries) {- if ('testId' in entry)+ if ('testId' in entry) {this._updateLocation(entry.location);- else+ entry.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));+ } else {this._updateSuite(entry);+ }}}@@ -538,6 +544,10 @@ class PathSeparatorPatcher {}}+ private _updateAnnotationLocation(annotation: TestAnnotation) {+ this._updateLocation(annotation.location);+ }+private _updateLocation(location?: JsonLocation) {if (location)location.file = this._updatePath(location.file);diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.tsindex b403cab303..44fbcec723 100644--- a/packages/playwright/src/reporters/teleEmitter.ts+++ b/packages/playwright/src/reporters/teleEmitter.ts@@ -22,6 +22,7 @@ import { serializeRegexPatterns } from '../isomorphic/teleReceiver';import type { ReporterV2 } from './reporterV2';import type * as reporterTypes from '../../types/testReporter';+import type { TestAnnotation } from '../../types/test';import type * as teleReceiver from '../isomorphic/teleReceiver';export type TeleReporterEmitterOptions = {@@ -224,7 +225,7 @@ export class TeleReporterEmitter implements ReporterV2 {retries: test.retries,tags: test.tags,repeatEachIndex: test.repeatEachIndex,- annotations: test.annotations,+ annotations: this._relativeAnnotationLocations(test.annotations),};}@@ -244,7 +245,7 @@ export class TeleReporterEmitter implements ReporterV2 {duration: result.duration,status: result.status,errors: result.errors,- annotations: result.annotations?.length ? result.annotations : undefined,+ annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined,};}@@ -294,10 +295,17 @@ export class TeleReporterEmitter implements ReporterV2 {duration: step.duration,error: step.error,attachments: step.attachments.length ? step.attachments.map(a => result.attachments.indexOf(a)) : undefined,- annotations: step.annotations.length ? step.annotations : undefined,+ annotations: step.annotations.length ? this._relativeAnnotationLocations(step.annotations) : undefined,};}+ private _relativeAnnotationLocations(annotations: TestAnnotation[]): TestAnnotation[] {+ return annotations.map(annotation => ({+ ...annotation,+ location: annotation.location ? this._relativeLocation(annotation.location) : undefined,+ }));+ }+private _relativeLocation(location: reporterTypes.Location): reporterTypes.Location;private _relativeLocation(location?: reporterTypes.Location): reporterTypes.Location | undefined;private _relativeLocation(location: reporterTypes.Location | undefined): reporterTypes.Location | undefined {diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.tsindex 0a7736d3a6..19a2fa5071 100644--- a/packages/playwright/src/worker/testInfo.ts+++ b/packages/playwright/src/worker/testInfo.ts@@ -23,6 +23,7 @@ import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutManaimport { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, sanitizeFilePathBeforeExtension, trimLongString, windowsFilesystemFriendlyLength } from '../util';import { TestTracing } from './testTracing';import { testInfoError } from './util';+import { wrapFunctionWithLocation } from '../transform/transform';import type { RunnableDescription } from './timeoutManager';import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test';@@ -80,6 +81,12 @@ export class TestInfoImpl implements TestInfo {_hasUnhandledError = false;_allowSkips = false;+ // ------------ Main methods ------------+ skip: (arg?: any, description?: string) => void;+ fixme: (arg?: any, description?: string) => void;+ fail: (arg?: any, description?: string) => void;+ slow: (arg?: any, description?: string) => void;+// ------------ TestInfo fields ------------readonly testId: string;readonly repeatEachIndex: number;@@ -205,9 +212,14 @@ export class TestInfoImpl implements TestInfo {};this._tracing = new TestTracing(this, workerParams.artifactsDir);++ this.skip = wrapFunctionWithLocation((location, ...args) => this._modifier('skip', location, args));+ this.fixme = wrapFunctionWithLocation((location, ...args) => this._modifier('fixme', location, args));+ this.fail = wrapFunctionWithLocation((location, ...args) => this._modifier('fail', location, args));+ this.slow = wrapFunctionWithLocation((location, ...args) => this._modifier('slow', location, args));}- private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) {+ _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', location: Location, modifierArgs: [arg?: any, description?: string]) {if (typeof modifierArgs[1] === 'function') {throw new Error(['It looks like you are calling test.skip() inside the test and pass a callback.',@@ -222,7 +234,7 @@ export class TestInfoImpl implements TestInfo {return;const description = modifierArgs[1];- this.annotations.push({ type, description });+ this.annotations.push({ type, description, location });if (type === 'slow') {this._timeoutManager.slow();} else if (type === 'skip' || type === 'fixme') {@@ -567,22 +579,6 @@ export class TestInfoImpl implements TestInfo {return this._resolveSnapshotPaths(kind, name.length <= 1 ? name[0] : name, 'dontUpdateSnapshotIndex').absoluteSnapshotPath;}- skip(...args: [arg?: any, description?: string]) {- this._modifier('skip', args);- }-- fixme(...args: [arg?: any, description?: string]) {- this._modifier('fixme', args);- }-- fail(...args: [arg?: any, description?: string]) {- this._modifier('fail', args);- }-- slow(...args: [arg?: any, description?: string]) {- this._modifier('slow', args);- }-setTimeout(timeout: number) {this._timeoutManager.setTimeout(timeout);}@@ -594,14 +590,25 @@ export class TestStepInfoImpl implements TestStepInfo {private _testInfo: TestInfoImpl;private _stepId: string;+ skip: (arg?: any, description?: string) => void;+constructor(testInfo: TestInfoImpl, stepId: string) {this._testInfo = testInfo;this._stepId = stepId;+ this.skip = wrapFunctionWithLocation((location: Location, ...args: unknown[]) => {+ // skip();+ // skip(condition: boolean, description: string);+ if (args.length > 0 && !args[0])+ return;+ const description = args[1] as (string|undefined);+ this.annotations.push({ type: 'skip', description, location });+ throw new StepSkipError(description);+ });}- async _runStepBody<T>(skip: boolean, body: (step: TestStepInfo) => T | Promise<T>) {+ async _runStepBody<T>(skip: boolean, body: (step: TestStepInfo) => T | Promise<T>, location?: Location) {if (skip) {- this.annotations.push({ type: 'skip' });+ this.annotations.push({ type: 'skip', location });return undefined as T;}try {@@ -620,16 +627,6 @@ export class TestStepInfoImpl implements TestStepInfo {async attach(name: string, options?: { body?: string | Buffer; contentType?: string; path?: string; }): Promise<void> {this._attachToStep(await normalizeAndSaveAttachment(this._testInfo.outputPath(), name, options));}-- skip(...args: unknown[]) {- // skip();- // skip(condition: boolean, description: string);- if (args.length > 0 && !args[0])- return;- const description = args[1] as (string|undefined);- this.annotations.push({ type: 'skip', description });- throw new StepSkipError(description);- }}export class TestSkipError extends Error {diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.tsindex 59dbae0f4f..e9f515a760 100644--- a/packages/playwright/src/worker/workerMain.ts+++ b/packages/playwright/src/worker/workerMain.ts@@ -494,7 +494,7 @@ export class WorkerMain extends ProcessRunner {continue;const fn = async (fixtures: any) => {const result = await modifier.fn(fixtures);- testInfo[modifier.type](!!result, modifier.description);+ testInfo._modifier(modifier.type, modifier.location, [!!result, modifier.description]);};inheritFixtureNames(modifier.fn, fn);runnables.push({diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex fe1e3ba1ff..8390543f34 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -2325,6 +2325,11 @@ export interface TestInfo {* Optional description.*/description?: string;++ /**+ * Optional location in the source where the annotation is added.+ */+ location?: Location;}>;/**@@ -2575,7 +2580,9 @@ export type TestDetailsAnnotation = {description?: string;};-export type TestAnnotation = TestDetailsAnnotation;+export type TestAnnotation = TestDetailsAnnotation & {+ location?: Location;+};export type TestDetails = {tag?: string | string[];diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.tsindex bbf22d3bc6..d03244a1ec 100644--- a/packages/playwright/types/testReporter.d.ts+++ b/packages/playwright/types/testReporter.d.ts@@ -451,6 +451,11 @@ export interface TestCase {* Optional description.*/description?: string;++ /**+ * Optional location in the source where the annotation is added.+ */+ location?: Location;}>;/**@@ -607,6 +612,11 @@ export interface TestResult {* Optional description.*/description?: string;++ /**+ * Optional location in the source where the annotation is added.+ */+ location?: Location;}>;/**@@ -722,6 +732,11 @@ export interface TestStep {* Optional description.*/description?: string;++ /**+ * Optional location in the source where the annotation is added.+ */+ location?: Location;}>;/**diff --git a/tests/playwright-test/access-data.spec.ts b/tests/playwright-test/access-data.spec.tsindex ab5d3e4584..4da80d8de0 100644--- a/tests/playwright-test/access-data.spec.ts+++ b/tests/playwright-test/access-data.spec.ts@@ -61,7 +61,7 @@ test('should access annotations in fixture', async ({ runInlineTest }) => {expect(exitCode).toBe(0);const test = report.suites[0].specs[0].tests[0];expect(test.annotations).toEqual([- { type: 'slow', description: 'just slow' },+ { type: 'slow', description: 'just slow', location: { file: expect.any(String), line: 10, column: 14 } },{ type: 'myname', description: 'hello' }]);expect(test.results[0].stdout).toEqual([{ text: 'console.log\n' }]);diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.tsindex 26cd4082db..de7c1a10e4 100644--- a/tests/playwright-test/reporter-blob.spec.ts+++ b/tests/playwright-test/reporter-blob.spec.ts@@ -1758,6 +1758,9 @@ test('merge reports with different rootDirs and path separators', async ({ runInconsole.log('test:', test.location.file);console.log('test title:', test.titlePath()[2]);}+ onTestEnd(test) {+ console.log('annotations:', test.annotations.map(a => 'type: ' + a.type + ', description: ' + a.description + ', file: ' + a.location.file).join(','));+ }};`,'merge.config.ts': `module.exports = {@@ -1769,7 +1772,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn};`,'dir1/tests1/a.test.js': `import { test, expect } from '@playwright/test';- test('math 1', async ({}) => { });+ test('math 1', { annotation: { type: 'warning', description: 'Some warning' } }, async ({}) => { });`,};await runInlineTest(files1, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir1/playwright.config.ts')] });@@ -1780,7 +1783,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn};`,'dir2/tests2/b.test.js': `import { test, expect } from '@playwright/test';- test('math 2', async ({}) => { });+ test('math 2', { annotation: { type: 'issue' } }, async ({}) => { });`,};await runInlineTest(files2, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir2/playwright.config.ts')] });@@ -1802,12 +1805,16 @@ test('merge reports with different rootDirs and path separators', async ({ runIn{const { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--config', 'merge.config.ts'] });+ const testPath1 = test.info().outputPath('mergeRoot', 'tests1', 'a.test.js');+ const testPath2 = test.info().outputPath('mergeRoot', 'tests2', 'b.test.js');expect(exitCode).toBe(0);expect(output).toContain(`rootDir: ${test.info().outputPath('mergeRoot')}`);- expect(output).toContain(`test: ${test.info().outputPath('mergeRoot', 'tests1', 'a.test.js')}`);+ expect(output).toContain(`test: ${testPath1}`);expect(output).toContain(`test title: ${'tests1' + path.sep + 'a.test.js'}`);- expect(output).toContain(`test: ${test.info().outputPath('mergeRoot', 'tests2', 'b.test.js')}`);+ expect(output).toContain(`annotations: type: warning, description: Some warning, file: ${testPath1}`);+ expect(output).toContain(`test: ${testPath2}`);expect(output).toContain(`test title: ${'tests2' + path.sep + 'b.test.js'}`);+ expect(output).toContain(`annotations: type: issue, description: undefined, file: ${testPath2}`);}});@@ -1823,6 +1830,9 @@ test('merge reports without --config preserves path separators', async ({ runInlconsole.log('test:', test.location.file);console.log('test title:', test.titlePath()[2]);}+ onTestEnd(test) {+ console.log('annotations:', test.annotations.map(a => 'type: ' + a.type + ', description: ' + a.description + ', file: ' + a.location.file).join(','));+ }};`,'dir1/playwright.config.ts': `module.exports = {@@ -1830,11 +1840,11 @@ test('merge reports without --config preserves path separators', async ({ runInl};`,'dir1/tests1/a.test.js': `import { test, expect } from '@playwright/test';- test('math 1', async ({}) => { });+ test('math 1', { annotation: { type: 'warning', description: 'Some warning' } }, async ({}) => { });`,'dir1/tests2/b.test.js': `import { test, expect } from '@playwright/test';- test('math 2', async ({}) => { });+ test('math 2', { annotation: { type: 'issue' } }, async ({}) => { });`,};await runInlineTest(files1, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir1/playwright.config.ts')] });@@ -1854,11 +1864,15 @@ test('merge reports without --config preserves path separators', async ({ runInlconst { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--reporter', './echo-reporter.js'] });expect(exitCode).toBe(0);const otherSeparator = path.sep === '/' ? '\\' : '/';+ const testPath1 = test.info().outputPath('dir1', 'tests1', 'a.test.js').replaceAll(path.sep, otherSeparator);+ const testPath2 = test.info().outputPath('dir1', 'tests2', 'b.test.js').replaceAll(path.sep, otherSeparator);expect(output).toContain(`rootDir: ${test.info().outputPath('dir1').replaceAll(path.sep, otherSeparator)}`);- expect(output).toContain(`test: ${test.info().outputPath('dir1', 'tests1', 'a.test.js').replaceAll(path.sep, otherSeparator)}`);+ expect(output).toContain(`test: ${testPath1}`);expect(output).toContain(`test title: ${'tests1' + otherSeparator + 'a.test.js'}`);- expect(output).toContain(`test: ${test.info().outputPath('dir1', 'tests2', 'b.test.js').replaceAll(path.sep, otherSeparator)}`);+ expect(output).toContain(`annotations: type: warning, description: Some warning, file: ${testPath1}`);+ expect(output).toContain(`test: ${testPath2}`);expect(output).toContain(`test title: ${'tests2' + otherSeparator + 'b.test.js'}`);+ expect(output).toContain(`annotations: type: issue, description: undefined, file: ${testPath2}`);});test('merge reports should preserve attachments', async ({ runInlineTest, mergeReports, showReport, page }) => {diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.tsindex 1e28e8b288..afe152ff7e 100644--- a/tests/playwright-test/reporter.spec.ts+++ b/tests/playwright-test/reporter.spec.ts@@ -584,7 +584,9 @@ test('should report annotations from test declaration', async ({ runInlineTest }const visit = suite => {for (const test of suite.tests || []) {const annotations = test.annotations.map(a => {- return a.description ? a.type + '=' + a.description : a.type;+ const description = a.description ? a.type + '=' + a.description : a.type;+ const location = a.location ? '(' + a.location.line + ':' + a.location.column + ')' : '';+ return description + location;});console.log('\\n%%title=' + test.title + ', annotations=' + annotations.join(','));}@@ -609,7 +611,7 @@ test('should report annotations from test declaration', async ({ runInlineTest }expect(test.info().annotations).toEqual([]);});test('foo', { annotation: { type: 'foo' } }, () => {- expect(test.info().annotations).toEqual([{ type: 'foo' }]);+ expect(test.info().annotations).toEqual([{ type: 'foo', location: { file: expect.any(String), line: 6, column: 11 } }]);});test('foo-bar', {annotation: [@@ -618,8 +620,8 @@ test('should report annotations from test declaration', async ({ runInlineTest }],}, () => {expect(test.info().annotations).toEqual([- { type: 'foo', description: 'desc' },- { type: 'bar' },+ { type: 'foo', description: 'desc', location: { file: expect.any(String), line: 9, column: 11 } },+ { type: 'bar', location: { file: expect.any(String), line: 9, column: 11 } },]);});test.skip('skip-foo', { annotation: { type: 'foo' } }, () => {@@ -636,11 +638,14 @@ test('should report annotations from test declaration', async ({ runInlineTest }});test.describe('suite', { annotation: { type: 'foo' } }, () => {test('foo-suite', () => {- expect(test.info().annotations).toEqual([{ type: 'foo' }]);+ expect(test.info().annotations).toEqual([{ type: 'foo', location: { file: expect.any(String), line: 32, column: 12 } }]);});test.describe('inner', { annotation: { type: 'bar' } }, () => {test('foo-bar-suite', () => {- expect(test.info().annotations).toEqual([{ type: 'foo' }, { type: 'bar' }]);+ expect(test.info().annotations).toEqual([+ { type: 'foo', location: { file: expect.any(String), line: 32, column: 12 } },+ { type: 'bar', location: { file: expect.any(String), line: 36, column: 14 } }+ ]);});});});@@ -657,15 +662,15 @@ test('should report annotations from test declaration', async ({ runInlineTest }expect(result.exitCode).toBe(0);expect(result.outputLines).toEqual([`title=none, annotations=`,- `title=foo, annotations=foo`,- `title=foo-bar, annotations=foo=desc,bar`,- `title=skip-foo, annotations=foo,skip`,- `title=fixme-bar, annotations=bar,fixme`,- `title=fail-foo-bar, annotations=foo,bar=desc,fail`,- `title=foo-suite, annotations=foo`,- `title=foo-bar-suite, annotations=foo,bar`,- `title=skip-foo-suite, annotations=foo,skip`,- `title=fixme-bar-suite, annotations=bar,fixme`,+ `title=foo, annotations=foo(6:11)`,+ `title=foo-bar, annotations=foo=desc(9:11),bar(9:11)`,+ `title=skip-foo, annotations=foo(20:12),skip(20:12)`,+ `title=fixme-bar, annotations=bar(22:12),fixme(22:12)`,+ `title=fail-foo-bar, annotations=foo(24:12),bar=desc(24:12),fail(24:12)`,+ `title=foo-suite, annotations=foo(32:12)`,+ `title=foo-bar-suite, annotations=foo(32:12),bar(36:14)`,+ `title=skip-foo-suite, annotations=foo(45:21),skip(45:21)`,+ `title=fixme-bar-suite, annotations=bar(49:21),fixme(49:21)`,]);});diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.tsindex b5dfc7a347..c82a61b20f 100644--- a/tests/playwright-test/retry.spec.ts+++ b/tests/playwright-test/retry.spec.ts@@ -260,5 +260,5 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInlineexpect(result.failed).toBe(0);expect(result.flaky).toBe(1);expect(result.output).toContain('Failed on first run');- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]);+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry', location: expect.anything() }]);});diff --git a/tests/playwright-test/test-modifiers.spec.ts b/tests/playwright-test/test-modifiers.spec.tsindex e992fae2e7..e73e483b27 100644--- a/tests/playwright-test/test-modifiers.spec.ts+++ b/tests/playwright-test/test-modifiers.spec.ts@@ -113,19 +113,19 @@ test('test modifiers should work', async ({ runInlineTest }) => {expectTest('passed3', 'passed', 'passed', []);expectTest('passed4', 'passed', 'passed', []);expectTest('passed5', 'passed', 'passed', []);- expectTest('skipped1', 'skipped', 'skipped', [{ type: 'skip' }]);- expectTest('skipped2', 'skipped', 'skipped', [{ type: 'skip' }]);- expectTest('skipped3', 'skipped', 'skipped', [{ type: 'skip' }]);- expectTest('skipped4', 'skipped', 'skipped', [{ type: 'skip', description: 'reason' }]);- expectTest('skipped5', 'skipped', 'skipped', [{ type: 'fixme' }]);- expectTest('skipped6', 'skipped', 'skipped', [{ type: 'fixme', description: 'reason' }]);- expectTest('failed1', 'failed', 'failed', [{ type: 'fail' }]);- expectTest('failed2', 'failed', 'failed', [{ type: 'fail' }]);- expectTest('failed3', 'failed', 'failed', [{ type: 'fail' }]);- expectTest('failed4', 'failed', 'failed', [{ type: 'fail', description: 'reason' }]);- expectTest('suite1', 'skipped', 'skipped', [{ type: 'skip' }]);- expectTest('suite2', 'skipped', 'skipped', [{ type: 'skip' }]);- expectTest('suite3', 'skipped', 'skipped', [{ type: 'skip', description: 'reason' }]);+ expectTest('skipped1', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 20, column: 14 } }]);+ expectTest('skipped2', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 23, column: 14 } }]);+ expectTest('skipped3', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 26, column: 14 } }]);+ expectTest('skipped4', 'skipped', 'skipped', [{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 29, column: 14 } }]);+ expectTest('skipped5', 'skipped', 'skipped', [{ type: 'fixme', location: { file: expect.any(String), line: 32, column: 14 } }]);+ expectTest('skipped6', 'skipped', 'skipped', [{ type: 'fixme', description: 'reason', location: { file: expect.any(String), line: 35, column: 14 } }]);+ expectTest('failed1', 'failed', 'failed', [{ type: 'fail', location: { file: expect.any(String), line: 39, column: 14 } }]);+ expectTest('failed2', 'failed', 'failed', [{ type: 'fail', location: { file: expect.any(String), line: 43, column: 14 } }]);+ expectTest('failed3', 'failed', 'failed', [{ type: 'fail', location: { file: expect.any(String), line: 47, column: 14 } }]);+ expectTest('failed4', 'failed', 'failed', [{ type: 'fail', description: 'reason', location: { file: expect.any(String), line: 51, column: 14 } }]);+ expectTest('suite1', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 56, column: 14 } }]);+ expectTest('suite2', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 61, column: 14 } }]);+ expectTest('suite3', 'skipped', 'skipped', [{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 66, column: 14 } }]);expectTest('suite4', 'passed', 'passed', []);expect(result.passed).toBe(10);expect(result.skipped).toBe(9);@@ -407,7 +407,7 @@ test('should skip inside fixture', async ({ runInlineTest }) => {});expect(result.exitCode).toBe(0);expect(result.skipped).toBe(1);- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 5, column: 20 } }]);});test('modifier with a function should throw in the test', async ({ runInlineTest }) => {@@ -460,8 +460,8 @@ test('test.skip with worker fixtures only should skip before hooks and tests', aexpect(result.passed).toBe(1);expect(result.skipped).toBe(2);expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([]);- expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);- expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);+ expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 14, column: 14 } }]);+ expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 14, column: 14 } }]);expect(result.outputLines).toEqual(['beforeEach','passed',@@ -493,8 +493,8 @@ test('test.skip without a callback in describe block should skip hooks', async (});expect(result.exitCode).toBe(0);expect(result.skipped).toBe(2);- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);- expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 10, column: 12 } }]);+ expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 10, column: 12 } }]);expect(result.output).not.toContain('%%');});@@ -598,8 +598,8 @@ test('should skip all tests from beforeAll', async ({ runInlineTest }) => {'beforeAll','afterAll',]);- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);- expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 5, column: 14 } }]);+ expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 5, column: 14 } }]);});test('should report skipped tests in-order with correct properties', async ({ runInlineTest }) => {@@ -695,9 +695,9 @@ test('static modifiers should be added in serial mode', async ({ runInlineTest }expect(result.passed).toBe(0);expect(result.skipped).toBe(2);expect(result.didNotRun).toBe(1);- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }]);- expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme' }]);- expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip' }]);+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow', location: { file: expect.any(String), line: 6, column: 14 } }]);+ expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme', location: { file: expect.any(String), line: 9, column: 12 } }]);+ expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip', location: { file: expect.any(String), line: 11, column: 12 } }]);expect(result.report.suites[0].specs[3].tests[0].annotations).toEqual([]);});@@ -721,9 +721,18 @@ test('should contain only one slow modifier', async ({ runInlineTest }) => {});expect(result.exitCode).toBe(0);expect(result.passed).toBe(1);- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'fixme' }, { type: 'issue', description: 'my-value' }]);- expect(result.report.suites[1].specs[0].tests[0].annotations).toEqual([{ type: 'skip' }, { type: 'issue', description: 'my-value' }]);- expect(result.report.suites[2].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }, { type: 'issue', description: 'my-value' }]);+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([+ { type: 'fixme', location: { file: expect.any(String), line: 3, column: 12 } },+ { type: 'issue', description: 'my-value', location: { file: expect.any(String), line: 4, column: 11 } }+ ]);+ expect(result.report.suites[1].specs[0].tests[0].annotations).toEqual([+ { type: 'skip', location: { file: expect.any(String), line: 3, column: 12 } },+ { type: 'issue', description: 'my-value', location: { file: expect.any(String), line: 4, column: 11 } }+ ]);+ expect(result.report.suites[2].specs[0].tests[0].annotations).toEqual([+ { type: 'slow', location: { file: expect.any(String), line: 3, column: 12 } },+ { type: 'issue', description: 'my-value', location: { file: expect.any(String), line: 4, column: 11 } }+ ]);});test('should skip beforeEach hooks upon modifiers', async ({ runInlineTest }) => {diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.tsindex d00c5eaadb..2dfd024bfc 100644--- a/utils/generate_types/overrides-test.d.ts+++ b/utils/generate_types/overrides-test.d.ts@@ -81,7 +81,9 @@ export type TestDetailsAnnotation = {description?: string;};-export type TestAnnotation = TestDetailsAnnotation;+export type TestAnnotation = TestDetailsAnnotation & {+ location?: Location;+};export type TestDetails = {tag?: string | string[];