Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(coverage): experimental istanbul coverage collection via tracing (#42684)

Yury Semikhatsky05c126b8dcc7
diff --git a/docs/src/api/class-tracing.md b/docs/src/api/class-tracing.md
index 3d46033821..906aece296 100644
--- a/docs/src/api/class-tracing.md
+++ b/docs/src/api/class-tracing.md
@@ -146,6 +146,13 @@ To specify the final trace zip file name, you need to pass `path` option to
Whether to capture screenshots during tracing. Screenshots are used to build
a timeline preview.
+### option: Tracing.start.coverage
+* since: v1.64
+* langs: js
+- `coverage` <[boolean]>
+
+Whether to collect code coverage from istanbul-instrumented application code. Build the application with an istanbul instrumentation plugin, for example [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right before navigations and page closes, and stores them in istanbul format inside the trace file.
+
### option: Tracing.start.snapshots
* since: v1.12
* langs: js
diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md
index 5777731f99..06cfe0e883 100644
--- a/docs/src/test-api/class-testoptions.md
+++ b/docs/src/test-api/class-testoptions.md
@@ -673,6 +673,7 @@ export default defineConfig({
- `dom` ?<[boolean]> Capture DOM snapshot on every action and record network activity. Optional.
- `aria` ?<[boolean]> Capture aria snapshot of the page on every action. Optional.
- `screen` ?<[boolean]> Capture a screenshot of the page on every action. Optional.
+ - `coverage` ?<[boolean]> Whether to collect coverage from istanbul-instrumented application code into the trace. Defaults to false. Optional.
- `sources` ?<[boolean]> Whether to include source files for trace actions. Defaults to true. Optional.
Whether to record trace for each test. Defaults to `'off'`. The initial run of a test is the "first run"; subsequent runs caused by [retries](../test-retries.md) are "retries".
diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts
new file mode 100644
index 0000000000..1852ece160
--- /dev/null
+++ b/packages/injected/src/coverageScript.ts
@@ -0,0 +1,165 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { kCoverageStashError, kCoverageStashPrefix } from '@isomorphic/istanbulCoverage';
+
+import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta } from '@isomorphic/istanbulCoverage';
+
+// Reports `__coverage__` as a delta: reading resets the counters, maps are sent once per file.
+export class CoverageScript {
+ private _global: typeof globalThis;
+ private _takeName: string;
+ private _sessionId: string;
+ private _reportedFiles = new Set<string>();
+ private _onPageHide = () => this._stashCurrent();
+
+ constructor(global: typeof globalThis, takeName: string, sessionId: string) {
+ this._global = global;
+ this._takeName = takeName;
+ this._sessionId = sessionId;
+ (global as any)[takeName] = Object.assign(() => this.take(), { dispose: () => this.dispose() });
+ this._resetCounters();
+ // Counters die with the document and unload-time calls are not delivered.
+ global.addEventListener('pagehide', this._onPageHide);
+ }
+
+ private _resetCounters() {
+ const coverage: IstanbulCoverage | undefined = (this._global as any).__coverage__;
+ for (const fileCoverage of Object.values(coverage || {})) {
+ takeCounters(fileCoverage.s);
+ takeCounters(fileCoverage.f);
+ takeBranchCounters(fileCoverage.b);
+ }
+ }
+
+ dispose() {
+ this._global.removeEventListener('pagehide', this._onPageHide);
+ delete (this._global as any)[this._takeName];
+ }
+
+ take(): string[] {
+ const chunks = takeCoverageStashes(this._global, this._sessionId);
+ const delta = this._takeCurrent();
+ if (delta)
+ chunks.push(JSON.stringify({ data: delta }));
+ return chunks;
+ }
+
+ private _takeCurrent(): IstanbulCoverageDelta | undefined {
+ const coverage: IstanbulCoverage | undefined = (this._global as any).__coverage__;
+ if (!coverage)
+ return undefined;
+ const delta: IstanbulCoverageDelta = {};
+ for (const [file, fileCoverage] of Object.entries(coverage)) {
+ const s = takeCounters(fileCoverage.s);
+ const f = takeCounters(fileCoverage.f);
+ const b = takeBranchCounters(fileCoverage.b);
+ // Every file is reported once, so that the never hit ones are accounted for.
+ const isFirstReport = !this._reportedFiles.has(file);
+ if (!isFirstReport && !s && !f && !b)
+ continue;
+ const entry: IstanbulFileCoverageDelta = { path: fileCoverage.path, s: s || {}, f: f || {}, b: b || {} };
+ if (isFirstReport) {
+ entry.statementMap = fileCoverage.statementMap;
+ entry.fnMap = fileCoverage.fnMap;
+ entry.branchMap = fileCoverage.branchMap;
+ this._reportedFiles.add(file);
+ }
+ delta[file] = entry;
+ }
+ return Object.keys(delta).length ? delta : undefined;
+ }
+
+ private _stashCurrent() {
+ const delta = this._takeCurrent();
+ if (!delta)
+ return;
+ // Several documents can pick up the same stash, the id discards the copies.
+ const id = Math.random().toString(36).slice(2);
+ try {
+ this._global.localStorage.setItem(kCoverageStashPrefix + this._sessionId + '.' + id, JSON.stringify({ id, data: delta }));
+ } catch (error) {
+ // The counters are lost, the next take surfaces the error.
+ try {
+ this._global.localStorage.setItem(stashErrorKey(this._sessionId), String(error));
+ } catch {
+ }
+ }
+ }
+}
+
+function stashErrorKey(sessionId: string) {
+ return kCoverageStashPrefix + 'error.' + sessionId;
+}
+
+// Stashes of other sessions are stale, e.g. left by a previous run in a persistent profile.
+export function takeCoverageStashes(global: typeof globalThis, sessionId: string): string[] {
+ const result: string[] = [];
+ let storage: Storage;
+ try {
+ storage = global.localStorage;
+ } catch {
+ return result;
+ }
+ const error = storage.getItem(stashErrorKey(sessionId));
+ if (error) {
+ storage.removeItem(stashErrorKey(sessionId));
+ throw new Error(kCoverageStashError + ': ' + error);
+ }
+ const sessionPrefix = kCoverageStashPrefix + sessionId + '.';
+ const keys: string[] = [];
+ for (let i = 0; i < storage.length; i++) {
+ const key = storage.key(i);
+ if (key && key.startsWith(kCoverageStashPrefix))
+ keys.push(key);
+ }
+ for (const key of keys) {
+ const json = key.startsWith(sessionPrefix) ? storage.getItem(key) : undefined;
+ storage.removeItem(key);
+ if (json)
+ result.push(json);
+ }
+ return result;
+}
+
+function takeCounters(counters: { [key: string]: number }): { [key: string]: number } | undefined {
+ let result: { [key: string]: number } | undefined;
+ for (const key of Object.keys(counters)) {
+ const count = counters[key];
+ if (!count)
+ continue;
+ if (!result)
+ result = {};
+ result[key] = count;
+ counters[key] = 0;
+ }
+ return result;
+}
+
+// Branch counters are positional, so a hit branch is reported with the whole array.
+function takeBranchCounters(counters: { [key: string]: number[] }): { [key: string]: number[] } | undefined {
+ let result: { [key: string]: number[] } | undefined;
+ for (const key of Object.keys(counters)) {
+ const counts = counters[key];
+ if (!counts.some(Boolean))
+ continue;
+ if (!result)
+ result = {};
+ result[key] = counts.slice();
+ counts.fill(0);
+ }
+ return result;
+}
diff --git a/packages/injected/src/storageScript.ts b/packages/injected/src/storageScript.ts
index f7e4d37809..efb3054cef 100644
--- a/packages/injected/src/storageScript.ts
+++ b/packages/injected/src/storageScript.ts
@@ -14,6 +14,7 @@
* limitations under the License.
*/
+import { kCoverageStashPrefix } from '@isomorphic/istanbulCoverage';
import { parseEvaluationResultValue, serializeAsCallArgument, typedArrayToBase64 } from '@isomorphic/utilityScriptSerializers';
import type { IndexedDBDatabase, OPFSEntry, OriginStorage, SetOriginStorage } from '@protocol/structs';
@@ -170,7 +171,9 @@ export class StorageScript {
}
async collect(record: { indexedDB: boolean, opfs: boolean }): Promise<SerializedStorage> {
- const localStorage = Object.keys(this._global.localStorage).map(name => ({ name, value: this._global.localStorage.getItem(name)! }));
+ const localStorage = Object.keys(this._global.localStorage)
+ .filter(name => !name.startsWith(kCoverageStashPrefix))
+ .map(name => ({ name, value: this._global.localStorage.getItem(name)! }));
const result: SerializedStorage = { localStorage };
if (record.indexedDB) {
try {
diff --git a/packages/isomorphic/index.ts b/packages/isomorphic/index.ts
index bcd969c0ff..146759e178 100644
--- a/packages/isomorphic/index.ts
+++ b/packages/isomorphic/index.ts
@@ -21,6 +21,7 @@ export * from './base64';
export * from './colors';
export * from './headers';
export * from './imageUtils';
+export * from './istanbulCoverage';
export * from './jsonSchema';
export * from './locatorGenerators';
export * from './manualPromise';
diff --git a/packages/isomorphic/istanbulCoverage.ts b/packages/isomorphic/istanbulCoverage.ts
new file mode 100644
index 0000000000..7c0111c6f4
--- /dev/null
+++ b/packages/isomorphic/istanbulCoverage.ts
@@ -0,0 +1,101 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// The istanbul coverage format, mirrors the shapes from istanbul-lib-coverage.
+
+export type IstanbulLocation = {
+ line: number;
+ column: number;
+};
+
+export type IstanbulRange = {
+ start: IstanbulLocation;
+ end: IstanbulLocation;
+};
+
+export type IstanbulFunctionMapping = {
+ name: string;
+ decl: IstanbulRange;
+ loc: IstanbulRange;
+ line?: number;
+};
+
+export type IstanbulBranchMapping = {
+ type: string;
+ loc: IstanbulRange;
+ locations: IstanbulRange[];
+ line?: number;
+};
+
+export type IstanbulFileCoverage = {
+ path: string;
+ statementMap: { [key: string]: IstanbulRange };
+ fnMap: { [key: string]: IstanbulFunctionMapping };
+ branchMap: { [key: string]: IstanbulBranchMapping };
+ s: { [key: string]: number };
+ f: { [key: string]: number };
+ b: { [key: string]: number[] };
+};
+
+export type IstanbulCoverage = { [file: string]: IstanbulFileCoverage };
+
+// The maps are only sent with the first report of each file.
+export type IstanbulFileCoverageDelta = Partial<IstanbulFileCoverage> & Pick<IstanbulFileCoverage, 'path' | 's' | 'f' | 'b'>;
+
+export type IstanbulCoverageDelta = { [file: string]: IstanbulFileCoverageDelta };
+
+export const kCoverageStashPrefix = '__pwCoverage.';
+export const kCoverageStashError = 'Failed to stash the coverage';
+
+export type IstanbulCoverageChunk = {
+ data: IstanbulCoverageDelta;
+ id?: string;
+};
+
+export function sortedIstanbulCoverage(coverage: Map<string, IstanbulFileCoverage>): IstanbulCoverage {
+ return Object.fromEntries([...coverage.entries()].sort(([a], [b]) => a.localeCompare(b)));
+}
+
+export function mergeIstanbulCoverage(into: Map<string, IstanbulFileCoverage>, data: IstanbulCoverageDelta) {
+ for (const [file, fileCov] of Object.entries(data)) {
+ let existing = into.get(file);
+ if (!existing) {
+ existing = { path: fileCov.path, statementMap: {}, fnMap: {}, branchMap: {}, s: {}, f: {}, b: {} };
+ into.set(file, existing);
+ }
+ // The maps can arrive late if the report that carried them was lost.
+ if (fileCov.statementMap && !Object.keys(existing.statementMap).length) {
+ existing.statementMap = fileCov.statementMap;
+ existing.fnMap = fileCov.fnMap || {};
+ existing.branchMap = fileCov.branchMap || {};
+ // Reports only carry the counters that were hit, zero fill the rest.
+ for (const key of Object.keys(existing.statementMap))
+ existing.s[key] = existing.s[key] || 0;
+ for (const key of Object.keys(existing.fnMap))
+ existing.f[key] = existing.f[key] || 0;
+ for (const [key, branch] of Object.entries(existing.branchMap))
+ existing.b[key] = existing.b[key] || branch.locations.map(() => 0);
+ }
+ for (const key of Object.keys(fileCov.s))
+ existing.s[key] = (existing.s[key] || 0) + fileCov.s[key];
+ for (const key of Object.keys(fileCov.f))
+ existing.f[key] = (existing.f[key] || 0) + fileCov.f[key];
+ for (const key of Object.keys(fileCov.b)) {
+ const branches = existing.b[key] || (existing.b[key] = []);
+ fileCov.b[key].forEach((count, i) => branches[i] = (branches[i] || 0) + count);
+ }
+ }
+}
diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts
index be0790ba1a..5fcd6b1c96 100644
--- a/packages/playwright-client/types/types.d.ts
+++ b/packages/playwright-client/types/types.d.ts
@@ -23268,6 +23268,16 @@ export interface Tracing {
* @param options
*/
start(options?: {
+ /**
+ * Whether to collect code coverage from istanbul-instrumented application code. Build the application with an
+ * istanbul instrumentation plugin, for example
+ * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or
+ * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the
+ * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right
+ * before navigations and page closes, and stores them in istanbul format inside the trace file.
+ */
+ coverage?: boolean;
+
/**
* When enabled, the trace is written to an unarchived file that is updated in real time as actions occur, instead of
* caching changes and archiving them into a zip file at the end. This is useful for live trace viewing during test
diff --git a/packages/playwright-core/src/client/channels.d.ts b/packages/playwright-core/src/client/channels.d.ts
index a645cd61a8..6eee244b3a 100644
--- a/packages/playwright-core/src/client/channels.d.ts
+++ b/packages/playwright-core/src/client/channels.d.ts
@@ -5075,6 +5075,7 @@ export type TracingTracingStartParams = {
snapshotAria?: boolean,
snapshotScreen?: boolean,
screencast?: boolean,
+ coverage?: boolean,
live?: boolean,
};
export type TracingTracingStartOptions = {
@@ -5083,6 +5084,7 @@ export type TracingTracingStartOptions = {
snapshotAria?: boolean,
snapshotScreen?: boolean,
screencast?: boolean,
+ coverage?: boolean,
live?: boolean,
};
export type TracingTracingStartResult = void;
diff --git a/packages/playwright-core/src/client/tracing.ts b/packages/playwright-core/src/client/tracing.ts
index 4f403f5679..928f8615e8 100644
--- a/packages/playwright-core/src/client/tracing.ts
+++ b/packages/playwright-core/src/client/tracing.ts
@@ -42,7 +42,7 @@ export class Tracing extends ChannelOwner<channels.TracingChannel> implements ap
super(parent, type, guid, initializer);
}
- async start(options: { name?: string, title?: string, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, live?: boolean } = {}) {
+ async start(options: { name?: string, title?: string, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, coverage?: boolean, sources?: boolean, live?: boolean } = {}) {
await this._wrapApiCall(async () => {
this._includeSources = !!options.sources;
this._isLive = !!options.live;
@@ -53,6 +53,7 @@ export class Tracing extends ChannelOwner<channels.TracingChannel> implements ap
snapshotAria: snapshots.aria,
snapshotScreen: snapshots.screen,
screencast: options.screenshots,
+ coverage: options.coverage,
live: options.live,
}, kNoTimeout);
const { traceName } = await this._channel.tracingStartChunk({ name: options.name, title: options.title }, kNoTimeout);
diff --git a/packages/playwright-core/src/server/browserContext.ts b/packages/playwright-core/src/server/browserContext.ts
index e510a16e64..f3e399f5b3 100644
--- a/packages/playwright-core/src/server/browserContext.ts
+++ b/packages/playwright-core/src/server/browserContext.ts
@@ -614,6 +614,10 @@ export abstract class BrowserContext<EM extends EventMap = EventMap> extends Sdk
this._origins.add(origin);
}
+ visitedOrigins(): ReadonlySet<string> {
+ return this._origins;
+ }
+
async storageState(progress: Progress, { indexedDB = false, opfs = false, credentials = false }: { indexedDB?: boolean, opfs?: boolean, credentials?: boolean } = {}): Promise<channels.BrowserContextStorageStateResult> {
const result: channels.BrowserContextStorageStateResult = {
cookies: await this.cookies(progress),
@@ -648,25 +652,31 @@ export abstract class BrowserContext<EM extends EventMap = EventMap> extends Sdk
// If there are still origins to save, create a blank page to iterate over origins.
if (originsToSave.size) {
- const page = await this.newPage(progress, true /* forStorageState */);
- try {
- await page.addRequestInterceptor(progress, route => {
- route.fulfill({ body: '<html></html>' }).catch(() => {});
- }, 'prepend');
- for (const origin of originsToSave) {
- const frame = page.mainFrame();
- await frame.gotoImpl(progress, origin, {});
- const storage: SerializedStorage = await frame.evaluateExpression(progress, collectScript, { world: 'utility' });
- if (hasStorage(storage))
- result.origins.push({ origin, ...storage });
- }
- } finally {
- await page.close(progress);
- }
+ await this.visitOrigins(progress, originsToSave, async (frame, origin) => {
+ const storage: SerializedStorage = await frame.evaluateExpression(progress, collectScript, { world: 'utility' });
+ if (hasStorage(storage))
+ result.origins.push({ origin, ...storage });
+ });
}
return result;
}
+ async visitOrigins(progress: Progress, origins: Set<string>, callback: (frame: frames.Frame, origin: string) => Promise<void>) {
+ const page = await this.newPage(progress, true /* forStorageState */);
+ try {
+ await page.addRequestInterceptor(progress, route => {
+ route.fulfill({ body: '<html></html>' }).catch(() => {});
+ }, 'prepend');
+ for (const origin of origins) {
+ const frame = page.mainFrame();
+ await frame.gotoImpl(progress, origin, {});
+ await progress.race(callback(frame, origin));
+ }
+ } finally {
+ await page.close(progress);
+ }
+ }
+
isCreatingStorageStatePage(): boolean {
return this._creatingStorageStatePage;
}
diff --git a/packages/playwright-core/src/server/channels.d.ts b/packages/playwright-core/src/server/channels.d.ts
index f2fc853c42..7edbe1ba7f 100644
--- a/packages/playwright-core/src/server/channels.d.ts
+++ b/packages/playwright-core/src/server/channels.d.ts
@@ -5076,6 +5076,7 @@ export type TracingTracingStartParams = {
snapshotAria?: boolean,
snapshotScreen?: boolean,
screencast?: boolean,
+ coverage?: boolean,
live?: boolean,
};
export type TracingTracingStartOptions = {
@@ -5084,6 +5085,7 @@ export type TracingTracingStartOptions = {
snapshotAria?: boolean,
snapshotScreen?: boolean,
screencast?: boolean,
+ coverage?: boolean,
live?: boolean,
};
export type TracingTracingStartResult = void;
diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts
new file mode 100644
index 0000000000..2726546c6c
--- /dev/null
+++ b/packages/playwright-core/src/server/coverageRecorder.ts
@@ -0,0 +1,147 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { kCoverageStashError, mergeIstanbulCoverage, sortedIstanbulCoverage } from '@isomorphic/istanbulCoverage';
+import * as rawCoverageSource from '../generated/coverageScriptSource';
+
+import type { BrowserContext } from './browserContext';
+import type { InitScript, Page } from './page';
+import type { Progress } from './progress';
+import type { IstanbulCoverageChunk, IstanbulFileCoverage } from '@isomorphic/istanbulCoverage';
+
+const kCoverageTakeName = '__pwCoverageTake';
+
+const coverageTakeExpression = `window[${JSON.stringify(kCoverageTakeName)}] ? window[${JSON.stringify(kCoverageTakeName)}]() : []`;
+const coverageDisposeExpression = `window[${JSON.stringify(kCoverageTakeName)}]?.dispose()`;
+
+export class CoverageRecorder {
+ private _context: BrowserContext;
+ private _coverage = new Map<string, IstanbulFileCoverage>();
+ private _stashedChunkIds = new Set<string>();
+ private _stashError: Error | undefined;
+ private _initScript: InitScript | undefined;
+
+ constructor(context: BrowserContext) {
+ this._context = context;
+ }
+
+ private _moduleExpression(call: string) {
+ return `(() => {
+ const module = {};
+ ${rawCoverageSource.source}
+ ${call}
+ })()`;
+ }
+
+ private _sessionId() {
+ return JSON.stringify(this._context.guid);
+ }
+
+ async install(progress: Progress) {
+ if (this._initScript)
+ return;
+ const source = this._moduleExpression(`new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageTakeName)}, ${this._sessionId()});`);
+ this._initScript = await this._context.addInitScript(progress, source);
+ await progress.race(this._context.safeNonStallingEvaluateInAllFrames(source, 'main'));
+ }
+
+ async uninstall() {
+ await Promise.all([
+ this._initScript?.dispose(),
+ this._context.safeNonStallingEvaluateInAllFrames(coverageDisposeExpression, 'main'),
+ ]).catch(() => {});
+ }
+
+ async flush(progress: Progress) {
+ await progress.race(Promise.all(this._context.pages().map(page => this.collectFromPage(page))));
+ await this._harvestOriginsWithoutPage(progress);
+ }
+
+ async take(progress: Progress, mode: 'keep' | 'discard'): Promise<string | undefined> {
+ if (mode === 'discard') {
+ this._coverage.clear();
+ return;
+ }
+ await this.flush(progress);
+ if (this._stashError)
+ throw this._stashError;
+ this._stashedChunkIds.clear();
+ if (!this._coverage.size)
+ return;
+ const sorted = sortedIstanbulCoverage(this._coverage);
+ this._coverage.clear();
+ return JSON.stringify(sorted);
+ }
+
+ async collectFromPage(page: Page) {
+ if (page.isStorageStatePage)
+ return;
+ // A page that is going away may never answer.
+ await Promise.race([page.closedPromise, this._collectFromPage(page)]);
+ }
+
+ private async _collectFromPage(page: Page) {
+ await Promise.all(page.frames().map(async frame => {
+ const chunks: string[] = await frame.nonStallingRawEvaluateInExistingMainContext(coverageTakeExpression).catch(error => this._takeFailed(error));
+ for (const json of chunks)
+ this._append(json);
+ }));
+ }
+
+ private async _harvestOriginsWithoutPage(progress: Progress) {
+ if (this._context.isClosingOrClosed())
+ return;
+ const liveOrigins = new Set<string>();
+ for (const page of this._context.pages()) {
+ for (const frame of page.frames()) {
+ const origin = frame.origin();
+ if (origin)
+ liveOrigins.add(origin);
+ }
+ }
+ // The origins that still have a page were drained by the sweep above.
+ const origins = new Set([...this._context.visitedOrigins()].filter(origin => !liveOrigins.has(origin)));
+ if (!origins.size)
+ return;
+ const source = this._moduleExpression(`return (module.exports.takeCoverageStashes())(window, ${this._sessionId()});`);
+ await this._context.visitOrigins(progress, origins, async frame => {
+ const chunks: string[] = await frame.evaluateExpression(progress, source, { world: 'main' }).catch(error => this._takeFailed(error));
+ for (const json of chunks)
+ this._append(json);
+ });
+ }
+
+ // Evaluation fails for pages on their way out, only the stash error matters.
+ private _takeFailed(error: Error): string[] {
+ const message = error.message?.split('\n')[0].replace(/^Error: /, '');
+ if (message?.includes(kCoverageStashError))
+ this._stashError ??= new Error(message);
+ return [];
+ }
+
+ private _append(json: string) {
+ try {
+ const chunk: IstanbulCoverageChunk = JSON.parse(json);
+ if (chunk.id) {
+ if (this._stashedChunkIds.has(chunk.id))
+ return;
+ this._stashedChunkIds.add(chunk.id);
+ }
+ mergeIstanbulCoverage(this._coverage, chunk.data);
+ } catch {
+ }
+ }
+}
diff --git a/packages/playwright-core/src/server/debugController.ts b/packages/playwright-core/src/server/debugController.ts
index 7ad8b2d4cc..48bca18fcc 100644
--- a/packages/playwright-core/src/server/debugController.ts
+++ b/packages/playwright-core/src/server/debugController.ts
@@ -68,7 +68,7 @@ export class DebugController extends SdkObject {
if (enabled && !this._trackHierarchyListener) {
this._trackHierarchyListener = {
onPageOpen: () => this._emitSnapshot(false),
- onPageClose: () => this._emitSnapshot(false),
+ onPageDidClose: () => this._emitSnapshot(false),
};
this._playwright.instrumentation.addListener(this._trackHierarchyListener, null);
this._emitSnapshot(true);
diff --git a/packages/playwright-core/src/server/instrumentation.ts b/packages/playwright-core/src/server/instrumentation.ts
index a2de6a9806..a9aa05b3f9 100644
--- a/packages/playwright-core/src/server/instrumentation.ts
+++ b/packages/playwright-core/src/server/instrumentation.ts
@@ -111,7 +111,8 @@ export interface Instrumentation {
onCallLog(sdkObject: SdkObject, metadata: CallMetadata, logName: string, message: string): void;
onAfterCall(progress: Progress, sdkObject: SdkObject): Promise<void>;
onPageOpen(page: Page): void;
- onPageClose(page: Page): void;
+ onPageWillClose(page: Page): Promise<void>;
+ onPageDidClose(page: Page): void;
onBrowserOpen(browser: Browser): void;
onBrowserClose(browser: Browser): void;
onDialog(dialog: Dialog): void;
@@ -124,7 +125,8 @@ export interface InstrumentationListener {
onCallLog?(sdkObject: SdkObject, metadata: CallMetadata, logName: string, message: string): void;
onAfterCall?(progress: Progress, sdkObject: SdkObject): Promise<void>;
onPageOpen?(page: Page): void;
- onPageClose?(page: Page): void;
+ onPageWillClose?(page: Page): Promise<void>;
+ onPageDidClose?(page: Page): void;
onBrowserOpen?(browser: Browser): void;
onBrowserClose?(browser: Browser): void;
onDialog?(dialog: Dialog): void;
diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts
index 00ee7449d2..0c5d103981 100644
--- a/packages/playwright-core/src/server/page.ts
+++ b/packages/playwright-core/src/server/page.ts
@@ -316,7 +316,7 @@ export class Page extends SdkObject<PageEventMap> {
this.browserContext.emit(BrowserContext.Events.PageClosed, this);
this.closedPromise.resolve();
if (!this.isStorageStatePage)
- this.instrumentation.onPageClose(this);
+ this.instrumentation.onPageDidClose(this);
}
_didCrash() {
@@ -831,6 +831,9 @@ export class Page extends SdkObject<PageEventMap> {
if (options.reason)
this._closeReason = options.reason;
+ if (!this.isStorageStatePage)
+ await this.instrumentation.onPageWillClose(this).catch(() => {});
+
await this.screencast.handlePageOrContextClose();
if (this._lifecycle !== 'closing') {
diff --git a/packages/playwright-core/src/server/playwright.ts b/packages/playwright-core/src/server/playwright.ts
index dc129c7e23..9b04c951a0 100644
--- a/packages/playwright-core/src/server/playwright.ts
+++ b/packages/playwright-core/src/server/playwright.ts
@@ -56,7 +56,7 @@ export class Playwright extends SdkObject {
onBrowserOpen: browser => this._allBrowsers.add(browser),
onBrowserClose: browser => this._allBrowsers.delete(browser),
onPageOpen: page => this._allPages.add(page),
- onPageClose: page => this._allPages.delete(page),
+ onPageDidClose: page => this._allPages.delete(page),
}, null);
this.chromium = new Chromium(this, new BidiChromium(this));
this.firefox = new Firefox(this, new BidiFirefox(this));
diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts
index 3ba35b8348..67b163b1fe 100644
--- a/packages/playwright-core/src/server/trace/recorder/tracing.ts
+++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts
@@ -30,6 +30,7 @@ import { getPlaywrightVersion } from '../../userAgent';
import { Snapshotter } from './snapshotter';
import { Artifact } from '../../artifact';
import { BrowserContext } from '../../browserContext';
+import { CoverageRecorder } from '../../coverageRecorder';
import { Dispatcher } from '../../dispatchers/dispatcher';
import { serializeError } from '../../errors';
import { HarRecorder } from '../../har/harRecorder';
@@ -63,6 +64,7 @@ export type TracerOptions = {
snapshotAria?: boolean;
snapshotScreen?: boolean;
screencast?: boolean;
+ coverage?: boolean;
live?: boolean;
};
@@ -100,6 +102,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
private _contextCreatedEvent: trace.ContextCreatedTraceEvent;
private _pendingHarEntries = new Set<har.Entry>();
private _started = false;
+ private _coverageRecorder: CoverageRecorder | undefined;
readonly harRecorders = new Map<string, HarRecorder>();
constructor(context: BrowserContext | APIRequestContext, tracesDir: string | undefined) {
@@ -184,6 +187,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
// Tracing is 10x bigger if we include scripts in every trace.
if (options.snapshotDom)
this._harTracer.start({ omitScripts: !options.live });
+ if (options.coverage && this._context instanceof BrowserContext)
+ this._coverageRecorder = new CoverageRecorder(this._context);
this._started = true;
}
@@ -235,6 +240,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
this._harTracer.setOmitWebSocketFrames(!!process.env.PLAYWRIGHT_TRACING_NO_WEBSOCKET_FRAMES);
if (this._state.options.snapshotDom)
await this._snapshotter?.start(progress);
+ if (this._state.options.coverage)
+ await this._coverageRecorder?.install(progress);
return { traceName: this._state.traceName };
}
@@ -339,6 +346,9 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
this._closeAllGroups();
this._harTracer.stop();
this.flushHarEntries();
+ const coverageRecorder = this._coverageRecorder;
+ this._coverageRecorder = undefined;
+ await coverageRecorder?.uninstall();
await this._fs.sync().finally(() => {
this._state = undefined;
});
@@ -387,12 +397,35 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
this._groupEnd();
}
+ // Collected before the chunk stops, while the pages can still be evaluated in.
+ private async _takeCoverage(progress: Progress, mode: TracingTracingStopChunkParams['mode']): Promise<string | undefined> {
+ const recorder = this._coverageRecorder;
+ if (!this._state?.recording || !recorder)
+ return;
+ const coverageJson = await recorder.take(progress, mode === 'discard' ? 'discard' : 'keep');
+ if (coverageJson === undefined)
+ return;
+ const coverageFile = path.join(this._state.tracesDir, `${this._state.traceName}-coverage-${this._state.chunkOrdinal}.json`);
+ this._fs.writeFile(coverageFile, coverageJson);
+ return coverageFile;
+ }
+
async stopChunk(progress: Progress, params: TracingTracingStopChunkParams): Promise<{ artifact?: Artifact, entries?: NameValue[] }> {
if (this._isStopping)
throw new Error(`Tracing is already stopping`);
this._isStopping = true;
try {
- const result = this._stopChunk(params);
+ let coverageFile: string | undefined;
+ let coverageError: Error | undefined;
+ try {
+ coverageFile = await this._takeCoverage(progress, params.mode);
+ } catch (error) {
+ coverageError = error;
+ }
+ // The chunk is torn down either way, the coverage error is reported after.
+ const result = this._stopChunk(params, coverageFile);
+ if (coverageError)
+ throw coverageError;
if (!result)
return {};
@@ -421,7 +454,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
}
}
- private _stopChunk(params: TracingTracingStopChunkParams): { entries: NameValue[], zipFileName: string } | undefined {
+ private _stopChunk(params: TracingTracingStopChunkParams, coverageFile: string | undefined): { entries: NameValue[], zipFileName: string } | undefined {
if (!this._state || !this._state.recording) {
if (params.mode !== 'discard')
throw new Error(`Must start tracing before stopping`);
@@ -453,6 +486,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
const entries: NameValue[] = [];
entries.push({ name: 'trace.trace', value: this._state.traceFile });
entries.push({ name: 'trace.network', value: newNetworkFile });
+ if (coverageFile)
+ entries.push({ name: 'trace.coverage', value: coverageFile });
for (const file of new Set([...this._state.chunkFiles, ...this._state.crossChunkFiles]))
entries.push({ name: file, value: path.join(this._state.tracesDir, file) });
@@ -488,6 +523,18 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
await this._captureScreenshot(progress, page, phase);
if (options?.snapshotAria)
await this._captureAriaSnapshot(progress, page, phase);
+ if (phase === 'after')
+ await this._captureCoverage(progress, page);
+ }
+
+ private async _captureCoverage(progress: Progress, page: Page): Promise<void> {
+ const recorder = this._coverageRecorder;
+ if (!recorder)
+ return;
+ try {
+ await progress.race(recorder.collectFromPage(page));
+ } catch {
+ }
}
private _shouldCaptureAtPhase(metadata: CallMetadata, phase: trace.ActionPhase) {
@@ -574,6 +621,10 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
return this._captureSnapshot(progress, sdkObject, 'after');
}
+ async onPageWillClose(page: Page) {
+ await this._coverageRecorder?.collectFromPage(page).catch(() => {});
+ }
+
onEntryStarted(entry: har.Entry) {
this._pendingHarEntries.add(entry);
}
@@ -670,7 +721,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
this._appendTraceEvent(event);
}
- onPageClose(page: Page) {
+ onPageDidClose(page: Page) {
const event: trace.EventTraceEvent = {
type: 'event',
time: monotonicTime(),
diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts
index be0790ba1a..5fcd6b1c96 100644
--- a/packages/playwright-core/types/types.d.ts
+++ b/packages/playwright-core/types/types.d.ts
@@ -23268,6 +23268,16 @@ export interface Tracing {
* @param options
*/
start(options?: {
+ /**
+ * Whether to collect code coverage from istanbul-instrumented application code. Build the application with an
+ * istanbul instrumentation plugin, for example
+ * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or
+ * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the
+ * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right
+ * before navigations and page closes, and stores them in istanbul format inside the trace file.
+ */
+ coverage?: boolean;
+
/**
* When enabled, the trace is written to an unarchived file that is updated in real time as actions occur, instead of
* caching changes and archiving them into a zip file at the end. This is useful for live trace viewing during test
diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts
index e139194f08..a237ccd041 100644
--- a/packages/playwright/src/common/config.ts
+++ b/packages/playwright/src/common/config.ts
@@ -291,7 +291,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] |
return reporters;
}
-export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'perfetto'] as const;
+export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'perfetto', 'coverage'] as const;
export type BuiltInReporter = typeof builtInReporters[number];
export type ContextReuseMode = 'none' | 'when-possible';
diff --git a/packages/playwright/src/reporters/coverage.ts b/packages/playwright/src/reporters/coverage.ts
new file mode 100644
index 0000000000..f392f5b18d
--- /dev/null
+++ b/packages/playwright/src/reporters/coverage.ts
@@ -0,0 +1,220 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+import { mergeIstanbulCoverage, sortedIstanbulCoverage } from '@isomorphic/istanbulCoverage';
+import { ZipFile } from '@utils/zipFile';
+import { terminalScreen } from './base';
+import { resolveReporterOutputPath } from '../util';
+
+import type { IstanbulCoverage, IstanbulFileCoverage } from '@isomorphic/istanbulCoverage';
+import type { CommonReporterOptions } from './base';
+import type { ReporterV2 } from './reporterV2';
+import type { TestCase, TestResult } from '../../types/testReporter';
+
+type CoverageReporterOptions = {
+ outputDir?: string,
+};
+
+class CoverageReporter implements ReporterV2 {
+ private _options: CoverageReporterOptions & CommonReporterOptions;
+ private _traceFiles = new Set<string>();
+
+ constructor(options: CoverageReporterOptions & CommonReporterOptions) {
+ this._options = options;
+ }
+
+ version(): 'v2' {
+ return 'v2';
+ }
+
+ printsToStdio() {
+ return false;
+ }
+
+ onTestEnd(test: TestCase, result: TestResult) {
+ for (const attachment of result.attachments) {
+ if (attachment.name === 'trace' && attachment.path)
+ this._traceFiles.add(attachment.path);
+ }
+ }
+
+ async onEnd() {
+ const coverage = new Map<string, IstanbulFileCoverage>();
+ // Traces of several contexts are merged into one, each keeping its own coverage entry.
+ const traceFiles = [...this._traceFiles];
+ while (traceFiles.length) {
+ const batch = traceFiles.splice(0, 16).map(async traceFile => {
+ const zipFile = new ZipFile(traceFile);
+ try {
+ const entries = (await zipFile.entries()).filter(entry => entry.match(/(^|-)trace\.coverage$/));
+ return await Promise.all(entries.map(entry => zipFile.read(entry)));
+ } catch {
+ return [];
+ } finally {
+ zipFile.close();
+ }
+ });
+ for (const buffers of await Promise.all(batch)) {
+ for (const buffer of buffers) {
+ try {
+ mergeIstanbulCoverage(coverage, JSON.parse(buffer.toString('utf8')));
+ } catch {
+ }
+ }
+ }
+ }
+
+ if (!coverage.size) {
+ writeLine([
+ `\nNo code coverage was collected.`,
+ `Make sure the application under test is built with istanbul instrumentation`,
+ `(e.g. vite-plugin-istanbul or babel-plugin-istanbul) so that pages expose window.__coverage__,`,
+ `and that tracing is configured with coverage, e.g. trace: { mode: 'on', coverage: true }.`,
+ `Coverage is aggregated from the tests whose traces are kept.`,
+ ].join('\n'));
+ return;
+ }
+
+ const outputDir = resolveReporterOutputPath('coverage', this._options.configDir, this._options.outputDir);
+ await fs.promises.mkdir(outputDir, { recursive: true });
+ const mergedJson = sortedIstanbulCoverage(coverage);
+ await fs.promises.writeFile(path.join(outputDir, 'coverage-final.json'), JSON.stringify(mergedJson));
+ await fs.promises.writeFile(path.join(outputDir, 'lcov.info'), lcovReport(mergedJson));
+ const hasHtml = this._tryWriteHtmlReport(mergedJson, outputDir);
+
+ const summary = computeSummary(mergedJson);
+ const lines = [
+ ``,
+ `Code coverage (${coverage.size} files):`,
+ ` statements: ${formatMetric(summary.statements)}`,
+ ` branches: ${formatMetric(summary.branches)}`,
+ ` functions: ${formatMetric(summary.functions)}`,
+ ` lines: ${formatMetric(summary.lines)}`,
+ `Coverage report written to ${path.relative(process.cwd(), outputDir)}`,
+ ];
+ if (!hasHtml)
+ lines.push(`For an HTML report, install istanbul-lib-coverage, istanbul-lib-report and istanbul-reports, or run 'npx nyc report --temp-dir=${path.relative(process.cwd(), outputDir)} --reporter=html'.`);
+ writeLine(lines.join('\n'));
+ }
+
+ // Istanbul report libraries are not shipped with Playwright, use the project's if any.
+ private _tryWriteHtmlReport(mergedJson: IstanbulCoverage, outputDir: string): boolean {
+ try {
+ const resolveFrom = (name: string) => require.resolve(name, { paths: [this._options.configDir] });
+ const libCoverage = require(resolveFrom('istanbul-lib-coverage'));
+ const libReport = require(resolveFrom('istanbul-lib-report'));
+ const reports = require(resolveFrom('istanbul-reports'));
+ const context = libReport.createContext({
+ dir: outputDir,
+ coverageMap: libCoverage.createCoverageMap(mergedJson),
+ });
+ reports.create('html').execute(context);
+ return true;
+ } catch {
+ return false;
+ }
+ }
+}
+
+type Metric = { covered: number, total: number };
+
+function computeSummary(mergedJson: IstanbulCoverage) {
+ const statements: Metric = { covered: 0, total: 0 };
+ const branches: Metric = { covered: 0, total: 0 };
+ const functions: Metric = { covered: 0, total: 0 };
+ const lines: Metric = { covered: 0, total: 0 };
+ for (const fileCov of Object.values(mergedJson)) {
+ countHits(statements, Object.values(fileCov.s));
+ countHits(functions, Object.values(fileCov.f));
+ for (const counts of Object.values(fileCov.b))
+ countHits(branches, counts);
+ countHits(lines, lineCoverage(fileCov).values());
+ }
+ return { statements, branches, functions, lines };
+}
+
+function countHits(metric: Metric, counts: Iterable<number>) {
+ for (const count of counts) {
+ ++metric.total;
+ if (count > 0)
+ ++metric.covered;
+ }
+}
+
+function lineCoverage(fileCov: IstanbulFileCoverage): Map<number, number> {
+ const lines = new Map<number, number>();
+ for (const [key, statement] of Object.entries(fileCov.statementMap)) {
+ const line = statement.start.line;
+ const count = fileCov.s[key] || 0;
+ lines.set(line, Math.max(lines.get(line) || 0, count));
+ }
+ return lines;
+}
+
+function formatMetric(metric: Metric) {
+ const percent = metric.total ? (metric.covered / metric.total * 100).toFixed(2) : '100.00';
+ return `${percent}% (${metric.covered}/${metric.total})`;
+}
+
+function lcovReport(mergedJson: IstanbulCoverage): string {
+ const out: string[] = [];
+ for (const [file, fileCov] of Object.entries(mergedJson)) {
+ out.push(`SF:${file}`);
+ let functionsCovered = 0;
+ for (const [key, fn] of Object.entries(fileCov.fnMap)) {
+ out.push(`FN:${fn.decl.start.line},${fn.name}`);
+ out.push(`FNDA:${fileCov.f[key] || 0},${fn.name}`);
+ if (fileCov.f[key] > 0)
+ ++functionsCovered;
+ }
+ out.push(`FNF:${Object.keys(fileCov.fnMap).length}`);
+ out.push(`FNH:${functionsCovered}`);
+ let branchesFound = 0;
+ let branchesCovered = 0;
+ for (const [key, branch] of Object.entries(fileCov.branchMap)) {
+ const counts: number[] = fileCov.b[key] || [];
+ branch.locations.forEach((location, i) => {
+ const count = counts[i] || 0;
+ out.push(`BRDA:${location.start.line ?? branch.loc.start.line},${key},${i},${count}`);
+ ++branchesFound;
+ if (count > 0)
+ ++branchesCovered;
+ });
+ }
+ out.push(`BRF:${branchesFound}`);
+ out.push(`BRH:${branchesCovered}`);
+ const lines = lineCoverage(fileCov);
+ let linesCovered = 0;
+ for (const [line, count] of [...lines.entries()].sort((a, b) => a[0] - b[0])) {
+ out.push(`DA:${line},${count}`);
+ if (count > 0)
+ ++linesCovered;
+ }
+ out.push(`LF:${lines.size}`);
+ out.push(`LH:${linesCovered}`);
+ out.push(`end_of_record`);
+ }
+ return out.join('\n') + '\n';
+}
+
+function writeLine(line: string) {
+ terminalScreen.stdout.write(line + '\n');
+}
+
+export default CoverageReporter;
diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts
index 01966f8b3d..34cff70b12 100644
--- a/packages/playwright/src/runner/reporters.ts
+++ b/packages/playwright/src/runner/reporters.ts
@@ -19,6 +19,7 @@ import { calculateSha1 } from '@utils/crypto';
import { loadReporter } from './loadUtils';
import { formatError } from '../reporters/base';
import { BlobReporter } from '../reporters/blob';
+import CoverageReporter from '../reporters/coverage';
import DotReporter from '../reporters/dot';
import EmptyReporter from '../reporters/empty';
import GitHubReporter from '../reporters/github';
@@ -50,6 +51,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' |
'junit': JUnitReporter,
'null': EmptyReporter,
'html': HtmlReporter,
+ 'coverage': CoverageReporter,
};
const reporters: ReporterV2[] = [];
descriptions ??= config.config.reporter;
diff --git a/packages/playwright/src/worker/testTracing.ts b/packages/playwright/src/worker/testTracing.ts
index fed713a572..8b8637544e 100644
--- a/packages/playwright/src/worker/testTracing.ts
+++ b/packages/playwright/src/worker/testTracing.ts
@@ -38,7 +38,7 @@ const version: trace.VERSION = 10;
let traceOrdinal = 0;
type TraceFixtureValue = PlaywrightWorkerOptions['trace'] | undefined;
-type TraceOptions = { screenshots: boolean, snapshots: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, sources: boolean, attachments: boolean, live: boolean, mode: TraceMode };
+type TraceOptions = { screenshots: boolean, snapshots: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, coverage: boolean, sources: boolean, attachments: boolean, live: boolean, mode: TraceMode };
export class TestTracing {
private _testInfo: TestInfoImpl;
@@ -93,7 +93,7 @@ export class TestTracing {
}
async startIfNeeded(value: TraceFixtureValue) {
- const defaultTraceOptions: TraceOptions = { screenshots: true, snapshots: true, sources: true, attachments: true, live: false, mode: 'off' };
+ const defaultTraceOptions: TraceOptions = { screenshots: true, snapshots: true, coverage: false, sources: true, attachments: true, live: false, mode: 'off' };
if (!value) {
this._options = defaultTraceOptions;
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index 8d0eef73cc..07effc78c1 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -919,7 +919,7 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
* ```
*
*/
- reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto', string> | ReporterDescription[];
+ reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto'|'coverage', string> | ReporterDescription[];
/**
* Global options for all tests, for example
* [testOptions.browserName](https://playwright.dev/docs/api/class-testoptions#test-options-browser-name). Learn more
@@ -7202,7 +7202,7 @@ export interface PlaywrightWorkerOptions {
*
* Learn more about [recording trace](https://playwright.dev/docs/test-use-options#recording-options).
*/
- trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean };
+ trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, coverage?: boolean, sources?: boolean, attachments?: boolean };
/**
* Whether to record video for each test. Defaults to `'off'`. The initial run of a test is the "first run";
* subsequent runs caused by [retries](https://playwright.dev/docs/test-retries) are "retries".
diff --git a/packages/protocol/spec/tracing.yml b/packages/protocol/spec/tracing.yml
index de9a644d9d..054e06d8bb 100644
--- a/packages/protocol/spec/tracing.yml
+++ b/packages/protocol/spec/tracing.yml
@@ -26,6 +26,7 @@ Tracing:
snapshotAria: boolean?
snapshotScreen: boolean?
screencast: boolean?
+ coverage: boolean?
live: boolean?
tracingStartChunk:
diff --git a/packages/protocol/src/validator.ts b/packages/protocol/src/validator.ts
index 21e7cbe603..86f1e91255 100644
--- a/packages/protocol/src/validator.ts
+++ b/packages/protocol/src/validator.ts
@@ -3065,6 +3065,7 @@ scheme.TracingTracingStartParams = tObject({
snapshotAria: tOptional(tBoolean),
snapshotScreen: tOptional(tBoolean),
screencast: tOptional(tBoolean),
+ coverage: tOptional(tBoolean),
live: tOptional(tBoolean),
});
scheme.TracingTracingStartResult = tOptional(tObject({}));
diff --git a/tests/config/utils.ts b/tests/config/utils.ts
index b28fc36365..0a4148ba77 100644
--- a/tests/config/utils.ts
+++ b/tests/config/utils.ts
@@ -104,6 +104,18 @@ export function suppressCertificateWarning() {
};
}
+// Traces of several contexts are merged into one, each keeping its own coverage entry.
+export async function parseTraceCoverage(file: string): Promise<any | undefined> {
+ const { resources } = await parseTraceRaw(file);
+ const entries = [...resources.keys()].filter(name => name.match(/(^|-)trace\.coverage$/));
+ if (!entries.length)
+ return undefined;
+ const coverage = new Map<string, any>();
+ for (const entry of entries)
+ iso.mergeIstanbulCoverage(coverage, JSON.parse(resources.get(entry)!.toString()));
+ return iso.sortedIstanbulCoverage(coverage);
+}
+
export async function parseTraceRaw(file: string): Promise<{ events: any[], resources: Map<string, Buffer>, actions: string[], actionObjects: ActionTraceEvent[], stacks: Map<string, StackFrame[]> }> {
const zipFS = new utils.ZipFile(file);
const resources = new Map<string, Buffer>();
diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts
new file mode 100644
index 0000000000..53441d41b1
--- /dev/null
+++ b/tests/library/coverage.spec.ts
@@ -0,0 +1,312 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { browserTest as it, expect } from '../config/browserTest';
+import { parseTraceCoverage } from '../config/utils';
+
+const fileCoverage = (file: string, s0: number) => ({
+ [file]: {
+ path: file,
+ statementMap: { '0': { start: { line: 1, column: 0 }, end: { line: 1, column: 20 } } },
+ fnMap: {},
+ branchMap: {},
+ s: { '0': s0 },
+ f: {},
+ b: {},
+ },
+});
+
+const coverageScript = (file: string, s0: number) => `<script>window.__coverage__ = ${JSON.stringify(fileCoverage(file, s0))}</script>`;
+
+it('should collect istanbul coverage into the trace', async ({ browser }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+
+ const page = await context.newPage();
+ await page.setContent(coverageScript('a.js', 3));
+
+ // Coverage of a closed page is collected automatically.
+ const page2 = await context.newPage();
+ await page2.setContent(coverageScript('b.js', 2));
+ await page2.close();
+
+ const traceFile = testInfo.outputPath('trace.zip');
+ await context.tracing.stop({ path: traceFile });
+ await context.close();
+
+ const data = await parseTraceCoverage(traceFile);
+ expect(data['a.js'].s['0']).toBe(3);
+ expect(data['b.js'].s['0']).toBe(2);
+});
+
+it('should collect coverage per trace chunk', async ({ browser }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+ const page = await context.newPage();
+
+ await page.setContent(coverageScript('a.js', 5));
+ const traceFile1 = testInfo.outputPath('trace1.zip');
+ await context.tracing.stopChunk({ path: traceFile1 });
+
+ await context.tracing.startChunk();
+ await page.setContent(coverageScript('b.js', 7));
+ const traceFile2 = testInfo.outputPath('trace2.zip');
+ await context.tracing.stopChunk({ path: traceFile2 });
+
+ await context.tracing.stop();
+ await context.close();
+
+ const data1 = await parseTraceCoverage(traceFile1);
+ expect(data1['a.js'].s['0']).toBe(5);
+ expect(data1['b.js']).toBe(undefined);
+
+ const data2 = await parseTraceCoverage(traceFile2);
+ expect(data2['b.js'].s['0']).toBe(7);
+ expect(data2['a.js']).toBe(undefined);
+});
+
+it('should report maps once and counters incrementally', async ({ browser, server }) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+ const page = await context.newPage();
+ await page.goto(server.EMPTY_PAGE);
+
+ // Actions collect the counters on their own, so hit and collect in one evaluate.
+ const first = await page.evaluate(coverage => {
+ (window as any).__coverage__ = JSON.parse(coverage);
+ return (window as any).__pwCoverageTake().map((json: string) => JSON.parse(json));
+ }, JSON.stringify(fileCoverage('a.js', 3)));
+ expect(first[0].data['a.js'].statementMap).toBeTruthy();
+ expect(first[0].data['a.js'].s).toEqual({ '0': 3 });
+
+ const second = await page.evaluate(() => {
+ (window as any).__coverage__['a.js'].s['0'] += 2;
+ return (window as any).__pwCoverageTake().map((json: string) => JSON.parse(json));
+ });
+ expect(second[0].data['a.js'].statementMap).toBe(undefined);
+ expect(second[0].data['a.js'].s).toEqual({ '0': 2 });
+
+ // Nothing was hit since the last report.
+ expect(await page.evaluate(() => (window as any).__pwCoverageTake())).toEqual([]);
+
+ await context.tracing.stop();
+ await context.close();
+});
+
+it('should accumulate counters across pulls and keep never hit files', async ({ browser }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+ const page = await context.newPage();
+ await page.setContent(coverageScript('a.js', 0));
+ await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 4);
+ await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 3);
+
+ const traceFile = testInfo.outputPath('trace.zip');
+ await context.tracing.stop({ path: traceFile });
+ await context.close();
+
+ const data = await parseTraceCoverage(traceFile);
+ expect(data['a.js'].s['0']).toBe(7);
+ expect(Object.keys(data['a.js'].statementMap)).toEqual(['0']);
+});
+
+async function openPopup(page: any, server: any, file: string, s0: number) {
+ const [popup] = await Promise.all([
+ page.waitForEvent('popup'),
+ page.evaluate(url => (window as any).__popup = window.open(url), server.EMPTY_PAGE),
+ ]);
+ await popup.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage(file, s0)));
+ return popup;
+}
+
+it('should collect coverage of a page closed by in-page script', async ({ browser, server }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+ const page = await context.newPage();
+ await page.goto(server.EMPTY_PAGE);
+
+ const popup = await openPopup(page, server, 'popup.js', 5);
+ // An in-page close never reaches Playwright as page.close(), the counters
+ // are preserved because the actions collect them as they go.
+ await Promise.all([
+ popup.waitForEvent('close'),
+ popup.evaluate(() => setTimeout(() => window.close(), 0)),
+ ]);
+
+ const traceFile = testInfo.outputPath('trace.zip');
+ await context.tracing.stop({ path: traceFile });
+ await context.close();
+
+ const data = await parseTraceCoverage(traceFile);
+ expect(data['popup.js'].s['0']).toBe(5);
+});
+
+it('should not double count a stash picked up twice', async ({ browser, server }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+ const page = await context.newPage();
+ await page.goto(server.EMPTY_PAGE);
+
+ // Emulate two documents picking up the same stash before either removes it.
+ const prefix = '__pwCoverage.' + (context as any)._guid + '.';
+ await page.evaluate(({ prefix, coverage }) => {
+ const chunk = JSON.stringify({ id: 'stash-id', data: JSON.parse(coverage) });
+ localStorage.setItem(prefix + 'one', chunk);
+ localStorage.setItem(prefix + 'two', chunk);
+ }, { prefix, coverage: JSON.stringify(fileCoverage('stashed.js', 4)) });
+
+ const traceFile = testInfo.outputPath('trace.zip');
+ await context.tracing.stop({ path: traceFile });
+ await context.close();
+
+ const data = await parseTraceCoverage(traceFile);
+ expect(data['stashed.js'].s['0']).toBe(4);
+});
+
+it('should discard stashes of other sessions', async ({ browser, server }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+ const page = await context.newPage();
+ await page.goto(server.EMPTY_PAGE);
+ await page.evaluate(coverage => {
+ localStorage.setItem('__pwCoverage.other-session.1', JSON.stringify({ id: 'other', data: JSON.parse(coverage) }));
+ }, JSON.stringify(fileCoverage('stale.js', 7)));
+
+ const remaining = await page.evaluate(() => Object.keys(localStorage).filter(key => key.startsWith('__pwCoverage.')));
+ expect(remaining).toEqual([]);
+
+ const traceFile = testInfo.outputPath('trace.zip');
+ await context.tracing.stop({ path: traceFile });
+ await context.close();
+
+ // The stale stash was the only coverage, so nothing was collected at all.
+ expect(await parseTraceCoverage(traceFile)).toBe(undefined);
+});
+
+it('should collect coverage of an origin left without a page', async ({ browser, server }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+ const page = await context.newPage();
+ await page.goto(server.EMPTY_PAGE);
+
+ const popup = await openPopup(page, server, 'popup.js', 0);
+ await popup.evaluate(url => {
+ (window as any).__go = () => {
+ (window as any).__coverage__['popup.js'].s['0'] = 5;
+ window.location.href = url;
+ };
+ }, server.CROSS_PROCESS_PREFIX + '/empty.html');
+ // The popup is hit and navigates away, leaving a stash behind. Triggered from
+ // the opener, so that the pull after the action does not race the navigation.
+ await Promise.all([
+ popup.waitForURL(server.CROSS_PROCESS_PREFIX + '/empty.html'),
+ page.evaluate(() => (window as any).__popup.__go()),
+ ]);
+ // Leave the origin of the stash without a page to relay it.
+ await page.goto(server.CROSS_PROCESS_PREFIX + '/empty.html');
+
+ const traceFile = testInfo.outputPath('trace.zip');
+ await context.tracing.stop({ path: traceFile });
+ await context.close();
+
+ const data = await parseTraceCoverage(traceFile);
+ expect(data['popup.js'].s['0']).toBe(5);
+});
+
+it('should pull counters as the actions go', async ({ browser, server }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+ const page = await context.newPage();
+ await page.goto(server.EMPTY_PAGE);
+ await page.setContent(coverageScript('a.js', 3));
+ // The counters are collected by the action itself, not only by the final flush.
+ expect(await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'])).toBe(0);
+
+ const traceFile = testInfo.outputPath('trace.zip');
+ await context.tracing.stop({ path: traceFile });
+ await context.close();
+
+ const data = await parseTraceCoverage(traceFile);
+ expect(data['a.js'].s['0']).toBe(3);
+});
+
+it('should not collect coverage without the option', async ({ browser }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start();
+ const page = await context.newPage();
+ await page.setContent(coverageScript('a.js', 1));
+
+ const traceFile = testInfo.outputPath('trace.zip');
+ await context.tracing.stop({ path: traceFile });
+ await context.close();
+
+ expect(await parseTraceCoverage(traceFile)).toBe(undefined);
+});
+
+it('should count only the hits after start', async ({ browser, server }, testInfo) => {
+ const context = await browser.newContext();
+ const page = await context.newPage();
+ await page.goto(server.EMPTY_PAGE);
+ await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('a.js', 3)));
+
+ await context.tracing.start({ coverage: true });
+ await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 2);
+ const traceFile = testInfo.outputPath('trace.zip');
+ await context.tracing.stop({ path: traceFile });
+ await context.close();
+
+ const data = await parseTraceCoverage(traceFile);
+ expect(data['a.js'].s['0']).toBe(2);
+ expect(Object.keys(data['a.js'].statementMap)).toEqual(['0']);
+});
+
+it('should stop collecting when tracing stops', async ({ browser, server }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+ const page = await context.newPage();
+ await page.goto(server.EMPTY_PAGE);
+ await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('a.js', 1)));
+ await context.tracing.stop();
+
+ expect(await page.evaluate(() => typeof (window as any).__pwCoverageTake)).toBe('undefined');
+ // Leaving the document no longer stashes the counters.
+ await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] = 5);
+ await page.goto(server.PREFIX + '/title.html');
+ expect(await page.evaluate(() => Object.keys(localStorage).filter(key => key.startsWith('__pwCoverage.')))).toEqual([]);
+ expect(await page.evaluate(() => typeof (window as any).__pwCoverageTake)).toBe('undefined');
+ await context.close();
+});
+
+it('should surface a failure to stash the coverage', async ({ browser, server }, testInfo) => {
+ const context = await browser.newContext();
+ await context.tracing.start({ coverage: true });
+ const page = await context.newPage();
+ await page.goto(server.EMPTY_PAGE);
+ await page.evaluate(coverage => {
+ (window as any).__coverage__ = JSON.parse(coverage);
+ // The stash does not fit, the error record still does.
+ const setItem = Storage.prototype.setItem;
+ Storage.prototype.setItem = function(key: string, value: string) {
+ if (value.length > 100)
+ throw new DOMException('The quota has been exceeded.', 'QuotaExceededError');
+ setItem.call(this, key, value);
+ };
+ window.dispatchEvent(new Event('pagehide'));
+ }, JSON.stringify(fileCoverage('a.js', 3)));
+
+ await expect(context.tracing.stop({ path: testInfo.outputPath('trace.zip') })).rejects.toThrow('Failed to stash the coverage: QuotaExceededError: The quota has been exceeded.');
+ await context.close();
+});
diff --git a/tests/playwright-test/coverage.spec.ts b/tests/playwright-test/coverage.spec.ts
new file mode 100644
index 0000000000..f00a17b582
--- /dev/null
+++ b/tests/playwright-test/coverage.spec.ts
@@ -0,0 +1,150 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import { test, expect } from './playwright-test-fixtures';
+import { parseTraceCoverage } from '../config/utils';
+
+const coverageHelper = `
+ const makeCoverage = (file, s0) => ({
+ [file]: {
+ path: file,
+ statementMap: {
+ '0': { start: { line: 1, column: 0 }, end: { line: 1, column: 20 } },
+ '1': { start: { line: 2, column: 0 }, end: { line: 2, column: 20 } },
+ },
+ fnMap: {
+ '0': { name: 'foo', decl: { start: { line: 1, column: 0 }, end: { line: 1, column: 3 } }, loc: { start: { line: 1, column: 0 }, end: { line: 3, column: 1 } }, line: 1 },
+ },
+ branchMap: {
+ '0': { loc: { start: { line: 2, column: 0 }, end: { line: 2, column: 20 } }, type: 'if', locations: [{ start: { line: 2, column: 0 }, end: { line: 2, column: 10 } }, { start: { line: 2, column: 10 }, end: { line: 2, column: 20 } }], line: 2 },
+ },
+ s: { '0': s0, '1': 0 },
+ f: { '0': 1 },
+ b: { '0': [1, 0] },
+ },
+ });
+ const coverageScript = (file, s0) => '<script>window.__coverage__ = ' + JSON.stringify(makeCoverage(file, s0)) + '</script>';
+`;
+
+test('should collect istanbul coverage into the trace', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = {
+ use: { trace: { mode: 'on', coverage: true } },
+ };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ ${coverageHelper}
+ test('pass', async ({ page }) => {
+ const http = require('http');
+ const server = http.createServer((req, res) => {
+ res.setHeader('Content-Type', 'text/html');
+ if (req.url === '/frame')
+ res.end(coverageScript('src/frame.js', 3));
+ else if (req.url === '/page1')
+ res.end(coverageScript('src/app.js', 5) + '<iframe src="/frame"></iframe>');
+ else
+ res.end(coverageScript('src/app.js', 2));
+ });
+ await new Promise(f => server.listen(0, '127.0.0.1', f));
+ const origin = 'http://127.0.0.1:' + server.address().port;
+ await page.goto(origin + '/page1');
+ await expect(page.locator('iframe')).toBeVisible();
+ // Navigating away parks the first document's counters for the next one.
+ await page.goto(origin + '/page2');
+ server.close();
+ });
+ `,
+ });
+ expect(result.exitCode).toBe(0);
+ expect(result.passed).toBe(1);
+
+ const data = await parseTraceCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip'));
+ expect(data['src/app.js'].s['0']).toBe(7);
+ expect(data['src/app.js'].s['1']).toBe(0);
+ expect(data['src/app.js'].f['0']).toBe(2);
+ expect(data['src/app.js'].b['0']).toEqual([2, 0]);
+ expect(data['src/frame.js'].s['0']).toBe(3);
+});
+
+test('should aggregate coverage with the coverage reporter', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = {
+ use: { trace: { mode: 'on', coverage: true } },
+ reporter: [['dot'], ['coverage', { outputDir: 'coverage-report' }]],
+ };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ ${coverageHelper}
+ test('first', async ({ page }) => {
+ await page.setContent(coverageScript('src/app.js', 5));
+ });
+ `,
+ 'b.test.ts': `
+ import { test, expect } from '@playwright/test';
+ ${coverageHelper}
+ test('second', async ({ page, browser }) => {
+ await page.setContent(coverageScript('src/app.js', 2));
+ // Every context of a test keeps its own coverage entry in the trace.
+ const second = await browser.newContext();
+ await (await second.newPage()).setContent(coverageScript('src/second.js', 4));
+ await second.close();
+ });
+ `,
+ });
+ expect(result.exitCode).toBe(0);
+ expect(result.passed).toBe(2);
+
+ const final = JSON.parse(fs.readFileSync(test.info().outputPath('coverage-report', 'coverage-final.json'), 'utf8'));
+ expect(final['src/app.js'].s['0']).toBe(7);
+ expect(final['src/app.js'].s['1']).toBe(0);
+ expect(final['src/app.js'].f['0']).toBe(2);
+ expect(final['src/app.js'].b['0']).toEqual([2, 0]);
+ expect(final['src/second.js'].s['0']).toBe(4);
+
+ const lcov = fs.readFileSync(test.info().outputPath('coverage-report', 'lcov.info'), 'utf8');
+ expect(lcov).toContain('SF:src/app.js');
+ expect(lcov).toContain('DA:1,7');
+ expect(lcov).toContain('FNDA:2,foo');
+ expect(lcov).toContain('SF:src/second.js');
+
+ expect(result.output).toContain('Code coverage (2 files)');
+ expect(result.output).toContain('statements: 50.00% (2/4)');
+});
+
+test('should warn when no coverage was collected', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = {
+ use: { trace: { mode: 'on', coverage: true } },
+ reporter: [['dot'], ['coverage', { outputDir: 'coverage-report' }]],
+ };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('no instrumented code', async ({ page }) => {
+ await page.setContent('<div>hello</div>');
+ });
+ `,
+ });
+ expect(result.exitCode).toBe(0);
+ expect(result.output).toContain('No code coverage was collected');
+ expect(fs.existsSync(test.info().outputPath('coverage-report', 'coverage-final.json'))).toBe(false);
+});
diff --git a/utils/generate_injected.js b/utils/generate_injected.js
index 4162e312d2..86fb7cb39d 100644
--- a/utils/generate_injected.js
+++ b/utils/generate_injected.js
@@ -81,6 +81,11 @@ const injectedScripts = [
path.join(ROOT, 'packages', 'injected', 'lib'),
path.join(ROOT, 'packages', 'playwright-core', 'src', 'generated'),
],
+ [
+ path.join(ROOT, 'packages', 'injected', 'src', 'coverageScript.ts'),
+ path.join(ROOT, 'packages', 'injected', 'lib'),
+ path.join(ROOT, 'packages', 'playwright-core', 'src', 'generated'),
+ ],
];
const modulePrefix = `
diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts
index 90a983fdce..4a829c80ff 100644
--- a/utils/generate_types/overrides-test.d.ts
+++ b/utils/generate_types/overrides-test.d.ts
@@ -69,7 +69,7 @@ type LiteralUnion<T extends U, U = string> = T | (U & { zz_IGNORE_ME?: never });
interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
projects?: Project<TestArgs, WorkerArgs>[];
- reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto', string> | ReporterDescription[];
+ reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto'|'coverage', string> | ReporterDescription[];
use?: UseOptions<TestArgs, WorkerArgs>;
webServer?: TestConfigWebServer | TestConfigWebServer[];
}
@@ -272,7 +272,7 @@ export interface PlaywrightWorkerOptions {
connectOptions: ConnectOptions | undefined;
reuseContext: boolean;
screenshot: ScreenshotMode | { mode: ScreenshotMode } & Pick<PageScreenshotOptions, 'fullPage' | 'omitBackground'>;
- trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean };
+ trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, coverage?: boolean, sources?: boolean, attachments?: boolean };
video: VideoMode | /** deprecated */ 'retry-with-video' | { mode: VideoMode, size?: ViewportSize, show?: { actions?: { duration?: number, position?: 'top-left' | 'top' | 'top-right' | 'bottom-left' | 'bottom' | 'bottom-right', fontSize?: number, cursor?: 'none' | 'pointer' }, test?: { level?: 'file' | 'title' | 'step', position?: 'top-left' | 'top' | 'top-right' | 'bottom-left' | 'bottom' | 'bottom-right', fontSize?: number } } };
}