Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(test runner): add secrets option to trace to redact sensitive values (#43423)
Dmitry Gozman98891e9a781b
diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.mdindex e4241bb9e2..94e09e56b9 100644--- a/docs/src/test-api/class-testoptions.md+++ b/docs/src/test-api/class-testoptions.md@@ -706,6 +706,9 @@ export default defineConfig({- `aria` ?<[boolean]> Capture aria snapshot of the page on every action. Optional.- `screen` ?<[boolean]> Capture a screenshot of the page on every action. Optional.- `sources` ?<[boolean]> Whether to include source files for trace actions. Defaults to true. Optional.+ - `secrets` ?<[Array]<[Object]>> Sensitive values to hide in the trace. Optional.+ - `name` ?<[string]> Name of the secret. The trace shows `<secret>NAME</secret>` in place of the value, or `<secret/>` when the name is not set. Optional.+ - `value` <[string]> Sensitive text to hide.Whether to record trace for each test. Defaults to `'off'`. The initial run of a test is the "first run"; subsequent runs caused by [retries](../test-retries.md) are "retries".* `'off'`: Do not record trace.diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.mdindex e811dcc39c..3a5322447b 100644--- a/docs/src/test-reporters-js.md+++ b/docs/src/test-reporters-js.md@@ -96,7 +96,7 @@ export default defineConfig({In this example, the terminal output is not changed, while the HTML report never contains the password.-Values are matched as plain text, so a short value could match unrelated text. Screenshots, videos, trace files, and attachments referenced by path are not changed.+Values are matched as plain text, so a short value could match unrelated text. Screenshots, videos, and attachments referenced by path, including trace files, are not changed. To hide secrets in traces, use the `secrets` option of [`property: TestOptions.trace`].## Built-in reportersdiff --git a/packages/isomorphic/secrets.ts b/packages/isomorphic/secrets.tsindex 3b5a8a9fb5..7c201e948d 100644--- a/packages/isomorphic/secrets.ts+++ b/packages/isomorphic/secrets.ts@@ -29,23 +29,20 @@ export function secretsKey(secrets: Secrets | undefined): string {return JSON.stringify(normalizeSecrets(secrets).map(secret => JSON.stringify([secret.name ?? null, secret.value])).sort());}-export function secretPlaceholder(name: string | undefined): string {+function secretPlaceholder(name: string | undefined): string {return name ? `<secret>${name}</secret>` : `<secret/>`;}-export function redactSecrets(text: string, secrets: Secrets): string {+export function redactSecretsInText(text: string, secrets: Secrets): string {for (const { name, value } of secrets)text = text.replaceAll(value, secretPlaceholder(name));return text;}-export function redactSecretsInJSON(json: string, secrets: Secrets): string {- for (const { name, value } of secrets) {- const escapedValue = JSON.stringify(value).slice(1, -1);- const escapedPlaceholder = JSON.stringify(secretPlaceholder(name)).slice(1, -1);- json = json.replaceAll(escapedValue, escapedPlaceholder);- }- return json;+export function escapeSecretsForJSON(secrets: Secrets): Secrets {+ const escape = (text: string) => JSON.stringify(text).slice(1, -1);+ const escaped = secrets.map(({ name, value }) => ({ name: name === undefined ? undefined : escape(name), value: escape(value) }));+ return normalizeSecrets(escaped.filter((secret, index) => secret.name !== secrets[index].name || secret.value !== secrets[index].value));}// The buffer is not decoded, because it may not be a valid utf-8 string.diff --git a/packages/playwright/src/reporters/redactingReporter.ts b/packages/playwright/src/reporters/redactingReporter.tsindex fee40c73b5..1c427186ec 100644--- a/packages/playwright/src/reporters/redactingReporter.ts+++ b/packages/playwright/src/reporters/redactingReporter.ts@@ -14,7 +14,7 @@* limitations under the License.*/-import { normalizeSecrets, redactSecrets, redactSecretsInBuffer } from '@isomorphic/secrets';+import { normalizeSecrets, redactSecretsInBuffer, redactSecretsInText } from '@isomorphic/secrets';import type { ReportConfigureParams, ReportEndParams, ReporterPreprocessParams, ReporterV2 } from './reporterV2';import type { FullConfig, FullResult, Suite, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter';@@ -41,7 +41,7 @@ export class SecretsRedactor {redact<T>(value: T): T {if (typeof value === 'string')- return redactSecrets(value, this._secrets) as T;+ return redactSecretsInText(value, this._secrets) as T;if (typeof value !== 'object' || value === null)return value;if (Buffer.isBuffer(value)) {diff --git a/packages/playwright/src/worker/testTracing.ts b/packages/playwright/src/worker/testTracing.tsindex 700be901a7..0bf6f3a776 100644--- a/packages/playwright/src/worker/testTracing.ts+++ b/packages/playwright/src/worker/testTracing.ts@@ -20,6 +20,7 @@ import path from 'path';import * as yazl from 'yazl';import * as yauzl from 'yauzl';import { ManualPromise } from '@isomorphic/manualPromise';+import { escapeSecretsForJSON, normalizeSecrets, redactSecretsInBuffer } from '@isomorphic/secrets';import { monotonicTime } from '@isomorphic/time';import { calculateSha1, createGuid } from '@utils/crypto';import { SerializedFS } from '@utils/serializedFS';@@ -29,6 +30,7 @@ import { filteredStackTrace } from '@utils/stackTrace';import type { TestStepCategory, TestInfoImpl } from './testInfo';import type { PlaywrightWorkerOptions, TestInfo, TestInfoError, TraceMode } from '../../types/test';import type { StackFrame } from '@utils/stackTrace';+import type { Secrets } from '@isomorphic/secrets';import type * as trace from '@isomorphic/trace/trace';import type EventEmitter from 'events';@@ -43,6 +45,7 @@ type TraceOptions = { screenshots: boolean, snapshots: boolean | { dom?: booleanexport class TestTracing {private _testInfo: TestInfoImpl;private _options: TraceOptions | undefined;+ private _secrets: Secrets = [];private _liveTraceFile: { file: string, fs: SerializedFS } | undefined;private _traceEvents: trace.TraceEvent[] = [];private _temporaryTraceFiles: string[] = [];@@ -100,8 +103,10 @@ export class TestTracing {} else if (typeof value === 'string') {this._options = { ...defaultTraceOptions, mode: value === 'retry-with-trace' ? 'on-first-retry' : value as TraceMode };} else {- const mode = value.mode || 'off';- this._options = { ...defaultTraceOptions, ...value, mode: (mode as string) === 'retry-with-trace' ? 'on-first-retry' : mode };+ const { secrets, ...traceOptions } = value;+ const mode = traceOptions.mode || 'off';+ this._options = { ...defaultTraceOptions, ...traceOptions, mode: (mode as string) === 'retry-with-trace' ? 'on-first-retry' : mode };+ this._secrets = normalizeSecrets(secrets);}if (!this._shouldCaptureTrace()) {@@ -243,7 +248,7 @@ export class TestTracing {});const tracePath = this._testInfo.outputPath('trace.zip');- await mergeTraceFiles(tracePath, this._temporaryTraceFiles);+ await mergeTraceFiles(tracePath, this._temporaryTraceFiles, this._secrets);this._testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' });}@@ -265,6 +270,9 @@ export class TestTracing {}appendStdioToTrace(type: 'stdout' | 'stderr', chunk: string | Buffer) {+ // base64-encoded chunks must be redacted before writing.+ if (typeof chunk !== 'string')+ chunk = redactSecretsInBuffer(chunk, this._secrets);this._appendTraceEvent({type,timestamp: monotonicTime(),@@ -341,12 +349,13 @@ function generatePreview(value: any, visited = new Set<any>()): string {return String(value);}-async function mergeTraceFiles(fileName: string, temporaryTraceFiles: string[]) {+async function mergeTraceFiles(fileName: string, temporaryTraceFiles: string[], secrets: Secrets) {temporaryTraceFiles = temporaryTraceFiles.filter(file => fs.existsSync(file));- if (temporaryTraceFiles.length === 1) {+ if (temporaryTraceFiles.length === 1 && !secrets.length) {await fs.promises.rename(temporaryTraceFiles[0], fileName);return;}+ const escapedSecrets = escapeSecretsForJSON(secrets);const mergePromise = new ManualPromise();const zipFile = new yazl.ZipFile();@@ -382,9 +391,17 @@ async function mergeTraceFiles(fileName: string, temporaryTraceFiles: string[])promise.reject(err);return;}- zipFile.addReadStream(readStream!, entryName);- if (--pendingEntries === 0)- promise.resolve();+ if (!secrets.length) {+ zipFile.addReadStream(readStream!, entryName);+ if (--pendingEntries === 0)+ promise.resolve();+ return;+ }+ readStreamToBuffer(readStream!).then(buffer => {+ zipFile.addBuffer(redactSecretsInTraceEntry(buffer, secrets, escapedSecrets), entryName);+ if (--pendingEntries === 0)+ promise.resolve();+ }).catch(error => promise.reject(error));});});});@@ -400,3 +417,15 @@ async function mergeTraceFiles(fileName: string, temporaryTraceFiles: string[])});await mergePromise;}++async function readStreamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {+ const chunks: Buffer[] = [];+ for await (const chunk of stream)+ chunks.push(chunk as Buffer);+ return Buffer.concat(chunks);+}++function redactSecretsInTraceEntry(buffer: Buffer, secrets: Secrets, escapedSecrets: Secrets): Buffer {+ buffer = redactSecretsInBuffer(buffer, escapedSecrets);+ return redactSecretsInBuffer(buffer, secrets);+}diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex 810f31340c..3224b88103 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -7579,7 +7579,7 @@ export interface PlaywrightWorkerOptions {** Learn more about [recording trace](https://playwright.dev/docs/test-use-options#recording-options).*/- trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean };+ trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean, secrets?: { name?: string, value: string }[] };/*** Whether to record video for each test. Defaults to `'off'`. The initial run of a test is the "first run";* subsequent runs caused by [retries](https://playwright.dev/docs/test-retries) are "retries".diff --git a/tests/playwright-test/bun-expectations.conf b/tests/playwright-test/bun-expectations.confindex deda270dcc..abcecbe150 100644--- a/tests/playwright-test/bun-expectations.conf+++ b/tests/playwright-test/bun-expectations.conf@@ -73,6 +73,7 @@ test-step.spec.ts > should report api step failure [fixme]test-step.spec.ts > step timeout includes interrupted action errors [fixme]# Bun's console methods do not call process.stdout.write, so console output of the test does not get into the trace.+playwright.trace.spec.ts > should redact secrets [fixme]ui-mode-test-output.spec.ts > should collapse repeated console messages for test [fixme]ui-mode-test-output.spec.ts > should print beforeAll console messages once [fixme]ui-mode-test-output.spec.ts > should show console messages for test [fixme]diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.tsindex 8994742c27..dd836bbd9c 100644--- a/tests/playwright-test/playwright.trace.spec.ts+++ b/tests/playwright-test/playwright.trace.spec.ts@@ -14,7 +14,7 @@* limitations under the License.*/-import { test, expect } from './playwright-test-fixtures';+import { test, expect, stripAnsi } from './playwright-test-fixtures';import { parseTrace, parseTraceRaw } from '../config/utils';import fs from 'fs';@@ -618,6 +618,79 @@ test('should include attachments by default', async ({ runInlineTest, server },expect([...resources.keys()]).toContain(trace.model.actions[1].attachments[0].file);});+test('should redact secrets', async ({ runInlineTest, server }, testInfo) => {+ server.setRoute('/secret', (req, res) => {+ res.setHeader('Content-Type', 'application/json');+ res.end(JSON.stringify({ password: req.headers['x-secret'] }));+ });+ const result = await runInlineTest({+ 'playwright.config.ts': `+ export default { use: { trace: { mode: 'on', secrets: [{ name: 'PASSWORD', value: process.env.PASSWORD }, { name: 'QUOTED', value: process.env.QUOTED }] } } };+ `,+ 'a.spec.ts': `+ import { test, expect } from '@playwright/test';+ const password = process.env.PASSWORD;++ test('with page', async ({ page }, testInfo) => {+ await page.goto('${server.EMPTY_PAGE}');+ await page.setContent('<input>');+ await page.locator('input').fill(password);+ await page.evaluate(password => console.log('console ' + password), password);+ await page.evaluate(async password => {+ const response = await fetch('/secret', { method: 'POST', headers: { 'x-secret': password }, body: 'body ' + password });+ await response.text();+ }, password);+ console.log('stdout ' + password);+ console.log('quoted ' + process.env.QUOTED);+ process.stdout.write(Buffer.from('buffer ' + password + '\\n'));+ await testInfo.attach('note ' + password, { body: 'body ' + password });+ await test.step('step ' + password, async () => {+ expect(password).toBe('wrong ' + 'password123');+ });+ });++ test('without page', async ({}, testInfo) => {+ await testInfo.attach('note ' + password, { body: 'body ' + password });+ });+ `,+ }, { workers: 1 }, { PASSWORD: 'password123', QUOTED: 'say "hi"' });++ expect(result.exitCode).toBe(1);+ expect(result.passed).toBe(1);+ expect(result.failed).toBe(1);++ const { events, resources, actions } = await parseTraceRaw(testInfo.outputPath('test-results', 'a-with-page', 'trace.zip'));+ for (const [name, buffer] of resources) {+ for (const needle of ['password123', 'say "hi"', 'say \\"hi\\"'])+ expect(buffer.includes(needle), `${name} contains ${needle}`).toBe(false);+ }+ expect(actions).toContain(`Fill "<secret>PASSWORD</secret>" locator('input')`);+ expect(actions).toContain(`step <secret>PASSWORD</secret>`);+ expect(events.filter(e => e.type === 'console').map(e => e.text)).toContain('console <secret>PASSWORD</secret>');+ expect(events.filter(e => e.type === 'stdout').map(e => e.text ?? Buffer.from(e.base64, 'base64').toString())).toEqual([+ 'stdout <secret>PASSWORD</secret>\n',+ 'quoted <secret>QUOTED</secret>\n',+ 'buffer <secret>PASSWORD</secret>\n',+ ]);+ expect(stripAnsi(events.filter(e => e.type === 'error').map(e => e.message).join('\n'))).toContain('Expected: "wrong <secret>PASSWORD</secret>"');+ expect(events.filter(e => e.type === 'after').flatMap(e => e.attachments ?? []).map(a => a.name)).toContain('note <secret>PASSWORD</secret>');+ const traceFile = [...resources.keys()].find(name => name.endsWith('.trace') && name !== 'test.trace')!;+ expect(resources.get(traceFile)!.toString()).toContain('"__playwright_value_":"<secret>PASSWORD</secret>"');+ const networkFile = [...resources.keys()].find(name => name.endsWith('.network'))!;+ expect(resources.get(networkFile)!.toString()).toContain('"x-secret","value":"<secret>PASSWORD</secret>"');+ const bodies = [...resources.values()].map(buffer => buffer.toString());+ expect(bodies).toContain('body <secret>PASSWORD</secret>');+ expect(bodies).toContain('{"password":"<secret>PASSWORD</secret>"}');+ const sourceFile = [...resources.keys()].find(name => name.startsWith('src/'))!;+ expect(resources.get(sourceFile)!.toString()).toContain(`'wrong ' + '<secret>PASSWORD</secret>'`);++ const single = await parseTraceRaw(testInfo.outputPath('test-results', 'a-without-page', 'trace.zip'));+ for (const [name, buffer] of single.resources)+ expect(buffer.includes('password123'), `${name} contains the secret`).toBe(false);+ expect(single.actions).toContain(`Attach "note <secret>PASSWORD</secret>"`);+ expect([...single.resources.values()].map(buffer => buffer.toString())).toContain('body <secret>PASSWORD</secret>');+});+test('should opt out of attachments', async ({ runInlineTest, server }, testInfo) => {const result = await runInlineTest({'playwright.config.ts': `diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.tsindex 6ccb2a602e..eccd7301e3 100644--- a/utils/generate_types/overrides-test.d.ts+++ b/utils/generate_types/overrides-test.d.ts@@ -280,7 +280,7 @@ export interface PlaywrightWorkerOptions {connectOptions: ConnectOptions | undefined;reuseContext: boolean;screenshot: ScreenshotMode | { mode: ScreenshotMode } & Pick<PageScreenshotOptions, 'fullPage' | 'omitBackground'>;- trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean };+ trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean, secrets?: { name?: string, value: string }[] };video: VideoMode | /** deprecated */ 'retry-with-video' | { mode: VideoMode, size?: ViewportSize, fps?: number, show?: { actions?: { duration?: number, position?: 'top-left' | 'top' | 'top-right' | 'bottom-left' | 'bottom' | 'bottom-right', /** @deprecated Use `style.title` instead. */ fontSize?: number, cursor?: 'none' | 'pointer', style?: { point?: string, highlight?: string, title?: string } }, test?: { level?: 'file' | 'title' | 'step', position?: 'top-left' | 'top' | 'top-right' | 'bottom-left' | 'bottom' | 'bottom-right', fontSize?: number } } };}