Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(blob): do not remove existing reports from output dir (#43056)

Yury Semikhatskye54293a3280f
diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md
index f05ac3568e..e78805094e 100644
--- a/docs/src/test-reporters-js.md
+++ b/docs/src/test-reporters-js.md
@@ -357,7 +357,7 @@ Blob report supports following configuration options and environment variables:
| Environment Variable Name | Reporter Config Option| Description | Default
|---|---|---|---|
-| `PLAYWRIGHT_BLOB_OUTPUT_DIR` | `outputDir` | Directory to save the output. Existing content is deleted before writing the new report. | `blob-report`
+| `PLAYWRIGHT_BLOB_OUTPUT_DIR` | `outputDir` | Directory to save the output. Existing reports with other names are preserved, a report with the same name is overwritten. | `blob-report`
| `PLAYWRIGHT_BLOB_OUTPUT_NAME` | `fileName` | Report file name. | `report-<hash>-<shard_number>.zip`
| `PLAYWRIGHT_BLOB_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `outputDir` and `fileName` will be ignored. | `undefined`
diff --git a/packages/playwright/src/reporters/blob.ts b/packages/playwright/src/reporters/blob.ts
index 9590a9e1b2..ceb8769441 100644
--- a/packages/playwright/src/reporters/blob.ts
+++ b/packages/playwright/src/reporters/blob.ts
@@ -22,7 +22,7 @@ import mime from 'mime';
import * as yazl from 'yazl';
import { ManualPromise } from '@isomorphic/manualPromise';
import { calculateSha1, createGuid } from '@utils/crypto';
-import { removeFolders, sanitizeForFilePath } from '@utils/fileUtils';
+import { sanitizeForFilePath } from '@utils/fileUtils';
import { getUserAgent } from 'playwright-core/lib/coreBundle';
import { resolveOutputFile, CommonReporterOptions } from './base';
@@ -104,15 +104,13 @@ export class BlobReporter extends TeleReporterEmitter {
}
private async _prepareOutputFile() {
- const { outputFile, outputDir } = resolveOutputFile('BLOB', {
+ const { outputFile } = resolveOutputFile('BLOB', {
...this._options,
default: {
fileName: this._defaultReportName(this._config),
outputDir: 'blob-report',
}
})!;
- if (!process.env.PWTEST_BLOB_DO_NOT_REMOVE)
- await removeFolders([outputDir!]);
await fs.promises.mkdir(path.dirname(outputFile), { recursive: true });
return outputFile;
}
diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts
index d6899c5204..ddde575cad 100644
--- a/tests/playwright-test/reporter-blob.spec.ts
+++ b/tests/playwright-test/reporter-blob.spec.ts
@@ -131,7 +131,7 @@ test('should call methods in right order', async ({ runInlineTest, mergeReports
`
};
await runInlineTest(files, { shard: `1/3` });
- await runInlineTest(files, { shard: `3/3` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `3/3` });
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-.*.zip/), expect.stringMatching(/report-.*.zip/)]);
@@ -174,7 +174,7 @@ test('should fail merge and report error when reporter throws in onEnd', async (
`,
};
await runInlineTest(files, { shard: `1/2` });
- await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `2/2` });
// Merge through the dot reporter so the reporter error is surfaced, and the
// throwing reporter to fail the merge command.
@@ -238,7 +238,7 @@ test('should merge into html with dependencies', async ({ runInlineTest, mergeRe
};
const totalShards = 3;
for (let i = 0; i < totalShards; i++)
- await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `${i + 1}/${totalShards}` });
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-.*.zip/), expect.stringMatching(/report-.*.zip/), expect.stringMatching(/report-.*.zip/)]);
@@ -302,8 +302,8 @@ test('should merge blob into blob', async ({ runInlineTest, mergeReports, showRe
test.skip('skipped 2', async ({}) => {});
`
};
- await runInlineTest(files, { shard: `1/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
- await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `1/2` });
+ await runInlineTest(files, { shard: `2/2` });
{
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
@@ -395,7 +395,7 @@ test('be able to merge incomplete shards', async ({ runInlineTest, mergeReports,
`
};
await runInlineTest(files, { shard: `1/3` });
- await runInlineTest(files, { shard: `3/3` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `3/3` });
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
@@ -437,7 +437,7 @@ test('total time is from test run not from merge', async ({ runInlineTest, merge
`,
};
await runInlineTest(files, { shard: `1/2` });
- await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `2/2` });
const { exitCode, output } = await mergeReports(reportDir, { 'PLAYWRIGHT_HTML_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0);
@@ -502,7 +502,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports }
const totalShards = 3;
for (let i = 0; i < totalShards; i++)
- await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `${i + 1}/${totalShards}` });
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip', 'report-3.zip']);
@@ -588,8 +588,8 @@ test('should print progress', async ({ runInlineTest, mergeReports }) => {
`
};
- await runInlineTest(files, { shard: `1/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
- await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `1/2` });
+ await runInlineTest(files, { shard: `2/2` });
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
@@ -779,7 +779,7 @@ test('resource names should not clash between runs', async ({ runInlineTest, sho
`
};
await runInlineTest(files, { shard: `1/2` });
- await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `2/2` });
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
@@ -913,7 +913,7 @@ test('multiple output reports based on config', async ({ runInlineTest, mergeRep
`
};
await runInlineTest(files, { shard: `1/2` });
- await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `2/2` });
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
@@ -1056,7 +1056,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {
};
await runInlineTest(files, { shard: `1/3`, workers: 1 });
- await runInlineTest(files, { shard: `3/3`, workers: 1 }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `3/3`, workers: 1 });
const mergeConfig = {
reportSlowTests: {
@@ -1307,7 +1307,7 @@ test('support fileName option', async ({ runInlineTest, mergeReports }) => {
});
await runInlineTest(files('one'));
- await runInlineTest(files('two'), undefined, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files('two'), undefined);
const reportDir = test.info().outputPath('blob-report');
const reportFiles = await fs.promises.readdir(reportDir);
@@ -1338,6 +1338,28 @@ test('support PLAYWRIGHT_BLOB_OUTPUT_DIR env variable', async ({ runInlineTest,
expect(reportFiles.sort()).toEqual(['report.zip']);
});
+test('should preserve reports from other shards in PLAYWRIGHT_BLOB_OUTPUT_DIR', async ({ runInlineTest }) => {
+ test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/43013' });
+ const files = {
+ 'playwright.config.ts': `
+ module.exports = {
+ reporter: [['blob']],
+ };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('math 1', async ({}) => {});
+ test('math 2', async ({}) => {});
+ `,
+ };
+
+ await runInlineTest(files, { shard: '1/2' }, { PLAYWRIGHT_BLOB_OUTPUT_DIR: 'my/dir' });
+ await runInlineTest(files, { shard: '2/2' }, { PLAYWRIGHT_BLOB_OUTPUT_DIR: 'my/dir' });
+
+ const reportFiles = await fs.promises.readdir(test.info().outputPath('my', 'dir'));
+ expect(reportFiles.sort()).toEqual(['report-1.zip', 'report-2.zip']);
+});
+
test('support PLAYWRIGHT_BLOB_OUTPUT_NAME env variable', async ({ runInlineTest, mergeReports }) => {
test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/30091' });
const files = {
@@ -1356,7 +1378,7 @@ test('support PLAYWRIGHT_BLOB_OUTPUT_NAME env variable', async ({ runInlineTest,
};
await runInlineTest(files, undefined, { PLAYWRIGHT_BLOB_OUTPUT_NAME: 'report-one.zip' });
- await runInlineTest(files, undefined, { PLAYWRIGHT_BLOB_OUTPUT_NAME: 'report-two.zip', PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, undefined, { PLAYWRIGHT_BLOB_OUTPUT_NAME: 'report-two.zip' });
const reportDir = test.info().outputPath('blob-report');
const reportFiles = await fs.promises.readdir(reportDir);
@@ -1440,7 +1462,7 @@ test('keep projects with same name different global tag separate', async ({ runI
});
await runInlineTest(files('first'), undefined, { GLOBAL_TAG: '@first' });
- await runInlineTest(files('second'), undefined, { GLOBAL_TAG: '@second', PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files('second'), undefined, { GLOBAL_TAG: '@second' });
const reportDir = test.info().outputPath('blob-report');
const reportFiles = await fs.promises.readdir(reportDir);
@@ -1555,7 +1577,7 @@ test('merge-reports should throw if report version is from the future', async ({
`,
};
await runInlineTest(files, { shard: `1/2` });
- await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `2/2` });
const reportFiles = await fs.promises.readdir(reportDir);
expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
@@ -1791,7 +1813,7 @@ test('merge reports same rootDirs', async ({ runInlineTest, mergeReports }) => {
`,
};
await runInlineTest(files, { shard: `1/2` });
- await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `2/2` });
const allReportsDir = test.info().outputPath('blob-report');
@@ -1983,8 +2005,8 @@ test('merge reports should preserve attachments', async ({ runInlineTest, mergeR
});
`
};
- await runInlineTest(files, { shard: `1/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
- await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `1/2` });
+ await runInlineTest(files, { shard: `2/2` });
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
@@ -2092,7 +2114,7 @@ test('merge reports must not change test ids when there is no need to', async ({
testIdsFromShard1 = outputLines.sort();
}
{
- const { exitCode, outputLines } = await runInlineTest(files, { workers: 1 }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }, { additionalArgs: ['--config', test.info().outputPath('shard-2.config.ts')] });
+ const { exitCode, outputLines } = await runInlineTest(files, { workers: 1 }, {}, { additionalArgs: ['--config', test.info().outputPath('shard-2.config.ts')] });
expect(exitCode).toBe(0);
testIdsFromShard2 = outputLines.sort();
expect([...testIdsFromShard1, ...testIdsFromShard2].sort()).toEqual(testIdsFromSingleRun);
@@ -2263,7 +2285,7 @@ test('project filter in report name', async ({ runInlineTest }) => {
const result = await runInlineTest(files, { shard: `1/2`, project: ['foo', 'b*r'], grep: 'smoke' });
expect(result.exitCode).toBe(0);
const reportFiles = await fs.promises.readdir(reportDir);
- expect(reportFiles.sort()).toEqual(['report-foo-b-r-c29b5fa-1.zip']);
+ expect(reportFiles.sort()).toEqual(['report-foo-2.zip', 'report-foo-b-r-c29b5fa-1.zip']);
}
{
@@ -2310,7 +2332,7 @@ test('should report duration across all shards', async ({ runInlineTest, mergeRe
await runInlineTest(files, { shard: `1/2`, workers: 1 });
await new Promise(f => setTimeout(f, 1500)); // Ensure different start times.
- await runInlineTest(files, { shard: `2/2`, workers: 1 }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `2/2`, workers: 1 });
const { exitCode, outputLines } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('reporter.js'), '-c', test.info().outputPath('playwright.config.ts')] });
expect(exitCode).toBe(0);
@@ -2344,12 +2366,12 @@ test('shard chart', async ({ runInlineTest, writeFiles, showReport, page, mergeR
`,
});
- await runInlineTest({}, { shard: '1/3' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });
- await runInlineTest({}, { shard: '2/3' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });
- await runInlineTest({}, { shard: '3/3' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });
+ await runInlineTest({}, { shard: '1/3' }, { BOT_TAG: '@linux' });
+ await runInlineTest({}, { shard: '2/3' }, { BOT_TAG: '@linux' });
+ await runInlineTest({}, { shard: '3/3' }, { BOT_TAG: '@linux' });
- await runInlineTest({}, { shard: '1/2' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });
- await runInlineTest({}, { shard: '2/2' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });
+ await runInlineTest({}, { shard: '1/2' }, { BOT_TAG: '@mac' });
+ await runInlineTest({}, { shard: '2/2' }, { BOT_TAG: '@mac' });
const { exitCode } = await mergeReports(reportDir, { 'PLAYWRIGHT_HTML_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0);
@@ -2396,7 +2418,7 @@ test('should populate projects in config when merging reports', async ({ runInli
};
await runInlineTest(files, { shard: `1/2`, workers: 1 });
- await runInlineTest(files, { shard: `2/2`, workers: 1 }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest(files, { shard: `2/2`, workers: 1 });
const reportFiles = await fs.promises.readdir(reportDir);
expect(reportFiles).toHaveLength(2);
@@ -2424,8 +2446,8 @@ test('workerIndex is rebased', async ({ runInlineTest, writeFiles, showReport, p
`,
});
- await runInlineTest({}, { shard: '1/2', workers: '1' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
- await runInlineTest({}, { shard: '2/2', workers: '1' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+ await runInlineTest({}, { shard: '1/2', workers: '1' });
+ await runInlineTest({}, { shard: '2/2', workers: '1' });
const { exitCode } = await mergeReports(reportDir, { 'PLAYWRIGHT_HTML_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0);