Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(test runner): add secrets option to trace to redact sensitive values (#43423)

Dmitry Gozman98891e9a781b
diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md
index e4241bb9e2..94e09e56b9 100644
--- a/docs/src/test-api/class-testoptions.md
+++ b/docs/src/test-api/class-testoptions.md
@@ -706,6 +706,9 @@ export default defineConfig({
- `aria` ?<[boolean]> Capture aria snapshot of the page on every action. Optional.
- `screen` ?<[boolean]> Capture a screenshot of the page on every action. Optional.
- `sources` ?<[boolean]> Whether to include source files for trace actions. Defaults to true. Optional.
+ - `secrets` ?<[Array]<[Object]>> Sensitive values to hide in the trace. Optional.
+ - `name` ?<[string]> Name of the secret. The trace shows `<secret>NAME</secret>` in place of the value, or `<secret/>` when the name is not set. Optional.
+ - `value` <[string]> Sensitive text to hide.
Whether to record trace for each test. Defaults to `'off'`. The initial run of a test is the "first run"; subsequent runs caused by [retries](../test-retries.md) are "retries".
* `'off'`: Do not record trace.
diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md
index e811dcc39c..3a5322447b 100644
--- a/docs/src/test-reporters-js.md
+++ b/docs/src/test-reporters-js.md
@@ -96,7 +96,7 @@ export default defineConfig({
In this example, the terminal output is not changed, while the HTML report never contains the password.
-Values are matched as plain text, so a short value could match unrelated text. Screenshots, videos, trace files, and attachments referenced by path are not changed.
+Values are matched as plain text, so a short value could match unrelated text. Screenshots, videos, and attachments referenced by path, including trace files, are not changed. To hide secrets in traces, use the `secrets` option of [`property: TestOptions.trace`].
## Built-in reporters
diff --git a/packages/isomorphic/secrets.ts b/packages/isomorphic/secrets.ts
index 3b5a8a9fb5..7c201e948d 100644
--- a/packages/isomorphic/secrets.ts
+++ b/packages/isomorphic/secrets.ts
@@ -29,23 +29,20 @@ export function secretsKey(secrets: Secrets | undefined): string {
return JSON.stringify(normalizeSecrets(secrets).map(secret => JSON.stringify([secret.name ?? null, secret.value])).sort());
}
-export function secretPlaceholder(name: string | undefined): string {
+function secretPlaceholder(name: string | undefined): string {
return name ? `<secret>${name}</secret>` : `<secret/>`;
}
-export function redactSecrets(text: string, secrets: Secrets): string {
+export function redactSecretsInText(text: string, secrets: Secrets): string {
for (const { name, value } of secrets)
text = text.replaceAll(value, secretPlaceholder(name));
return text;
}
-export function redactSecretsInJSON(json: string, secrets: Secrets): string {
- for (const { name, value } of secrets) {
- const escapedValue = JSON.stringify(value).slice(1, -1);
- const escapedPlaceholder = JSON.stringify(secretPlaceholder(name)).slice(1, -1);
- json = json.replaceAll(escapedValue, escapedPlaceholder);
- }
- return json;
+export function escapeSecretsForJSON(secrets: Secrets): Secrets {
+ const escape = (text: string) => JSON.stringify(text).slice(1, -1);
+ const escaped = secrets.map(({ name, value }) => ({ name: name === undefined ? undefined : escape(name), value: escape(value) }));
+ return normalizeSecrets(escaped.filter((secret, index) => secret.name !== secrets[index].name || secret.value !== secrets[index].value));
}
// The buffer is not decoded, because it may not be a valid utf-8 string.
diff --git a/packages/playwright/src/reporters/redactingReporter.ts b/packages/playwright/src/reporters/redactingReporter.ts
index fee40c73b5..1c427186ec 100644
--- a/packages/playwright/src/reporters/redactingReporter.ts
+++ b/packages/playwright/src/reporters/redactingReporter.ts
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-import { normalizeSecrets, redactSecrets, redactSecretsInBuffer } from '@isomorphic/secrets';
+import { normalizeSecrets, redactSecretsInBuffer, redactSecretsInText } from '@isomorphic/secrets';
import type { ReportConfigureParams, ReportEndParams, ReporterPreprocessParams, ReporterV2 } from './reporterV2';
import type { FullConfig, FullResult, Suite, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter';
@@ -41,7 +41,7 @@ export class SecretsRedactor {
redact<T>(value: T): T {
if (typeof value === 'string')
- return redactSecrets(value, this._secrets) as T;
+ return redactSecretsInText(value, this._secrets) as T;
if (typeof value !== 'object' || value === null)
return value;
if (Buffer.isBuffer(value)) {
diff --git a/packages/playwright/src/worker/testTracing.ts b/packages/playwright/src/worker/testTracing.ts
index 700be901a7..0bf6f3a776 100644
--- a/packages/playwright/src/worker/testTracing.ts
+++ b/packages/playwright/src/worker/testTracing.ts
@@ -20,6 +20,7 @@ import path from 'path';
import * as yazl from 'yazl';
import * as yauzl from 'yauzl';
import { ManualPromise } from '@isomorphic/manualPromise';
+import { escapeSecretsForJSON, normalizeSecrets, redactSecretsInBuffer } from '@isomorphic/secrets';
import { monotonicTime } from '@isomorphic/time';
import { calculateSha1, createGuid } from '@utils/crypto';
import { SerializedFS } from '@utils/serializedFS';
@@ -29,6 +30,7 @@ import { filteredStackTrace } from '@utils/stackTrace';
import type { TestStepCategory, TestInfoImpl } from './testInfo';
import type { PlaywrightWorkerOptions, TestInfo, TestInfoError, TraceMode } from '../../types/test';
import type { StackFrame } from '@utils/stackTrace';
+import type { Secrets } from '@isomorphic/secrets';
import type * as trace from '@isomorphic/trace/trace';
import type EventEmitter from 'events';
@@ -43,6 +45,7 @@ type TraceOptions = { screenshots: boolean, snapshots: boolean | { dom?: boolean
export class TestTracing {
private _testInfo: TestInfoImpl;
private _options: TraceOptions | undefined;
+ private _secrets: Secrets = [];
private _liveTraceFile: { file: string, fs: SerializedFS } | undefined;
private _traceEvents: trace.TraceEvent[] = [];
private _temporaryTraceFiles: string[] = [];
@@ -100,8 +103,10 @@ export class TestTracing {
} else if (typeof value === 'string') {
this._options = { ...defaultTraceOptions, mode: value === 'retry-with-trace' ? 'on-first-retry' : value as TraceMode };
} else {
- const mode = value.mode || 'off';
- this._options = { ...defaultTraceOptions, ...value, mode: (mode as string) === 'retry-with-trace' ? 'on-first-retry' : mode };
+ const { secrets, ...traceOptions } = value;
+ const mode = traceOptions.mode || 'off';
+ this._options = { ...defaultTraceOptions, ...traceOptions, mode: (mode as string) === 'retry-with-trace' ? 'on-first-retry' : mode };
+ this._secrets = normalizeSecrets(secrets);
}
if (!this._shouldCaptureTrace()) {
@@ -243,7 +248,7 @@ export class TestTracing {
});
const tracePath = this._testInfo.outputPath('trace.zip');
- await mergeTraceFiles(tracePath, this._temporaryTraceFiles);
+ await mergeTraceFiles(tracePath, this._temporaryTraceFiles, this._secrets);
this._testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' });
}
@@ -265,6 +270,9 @@ export class TestTracing {
}
appendStdioToTrace(type: 'stdout' | 'stderr', chunk: string | Buffer) {
+ // base64-encoded chunks must be redacted before writing.
+ if (typeof chunk !== 'string')
+ chunk = redactSecretsInBuffer(chunk, this._secrets);
this._appendTraceEvent({
type,
timestamp: monotonicTime(),
@@ -341,12 +349,13 @@ function generatePreview(value: any, visited = new Set<any>()): string {
return String(value);
}
-async function mergeTraceFiles(fileName: string, temporaryTraceFiles: string[]) {
+async function mergeTraceFiles(fileName: string, temporaryTraceFiles: string[], secrets: Secrets) {
temporaryTraceFiles = temporaryTraceFiles.filter(file => fs.existsSync(file));
- if (temporaryTraceFiles.length === 1) {
+ if (temporaryTraceFiles.length === 1 && !secrets.length) {
await fs.promises.rename(temporaryTraceFiles[0], fileName);
return;
}
+ const escapedSecrets = escapeSecretsForJSON(secrets);
const mergePromise = new ManualPromise();
const zipFile = new yazl.ZipFile();
@@ -382,9 +391,17 @@ async function mergeTraceFiles(fileName: string, temporaryTraceFiles: string[])
promise.reject(err);
return;
}
- zipFile.addReadStream(readStream!, entryName);
- if (--pendingEntries === 0)
- promise.resolve();
+ if (!secrets.length) {
+ zipFile.addReadStream(readStream!, entryName);
+ if (--pendingEntries === 0)
+ promise.resolve();
+ return;
+ }
+ readStreamToBuffer(readStream!).then(buffer => {
+ zipFile.addBuffer(redactSecretsInTraceEntry(buffer, secrets, escapedSecrets), entryName);
+ if (--pendingEntries === 0)
+ promise.resolve();
+ }).catch(error => promise.reject(error));
});
});
});
@@ -400,3 +417,15 @@ async function mergeTraceFiles(fileName: string, temporaryTraceFiles: string[])
});
await mergePromise;
}
+
+async function readStreamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
+ const chunks: Buffer[] = [];
+ for await (const chunk of stream)
+ chunks.push(chunk as Buffer);
+ return Buffer.concat(chunks);
+}
+
+function redactSecretsInTraceEntry(buffer: Buffer, secrets: Secrets, escapedSecrets: Secrets): Buffer {
+ buffer = redactSecretsInBuffer(buffer, escapedSecrets);
+ return redactSecretsInBuffer(buffer, secrets);
+}
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index 810f31340c..3224b88103 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -7579,7 +7579,7 @@ export interface PlaywrightWorkerOptions {
*
* Learn more about [recording trace](https://playwright.dev/docs/test-use-options#recording-options).
*/
- trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean };
+ trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean, secrets?: { name?: string, value: string }[] };
/**
* Whether to record video for each test. Defaults to `'off'`. The initial run of a test is the "first run";
* subsequent runs caused by [retries](https://playwright.dev/docs/test-retries) are "retries".
diff --git a/tests/playwright-test/bun-expectations.conf b/tests/playwright-test/bun-expectations.conf
index deda270dcc..abcecbe150 100644
--- a/tests/playwright-test/bun-expectations.conf
+++ b/tests/playwright-test/bun-expectations.conf
@@ -73,6 +73,7 @@ test-step.spec.ts > should report api step failure [fixme]
test-step.spec.ts > step timeout includes interrupted action errors [fixme]
# Bun's console methods do not call process.stdout.write, so console output of the test does not get into the trace.
+playwright.trace.spec.ts > should redact secrets [fixme]
ui-mode-test-output.spec.ts > should collapse repeated console messages for test [fixme]
ui-mode-test-output.spec.ts > should print beforeAll console messages once [fixme]
ui-mode-test-output.spec.ts > should show console messages for test [fixme]
diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts
index 8994742c27..dd836bbd9c 100644
--- a/tests/playwright-test/playwright.trace.spec.ts
+++ b/tests/playwright-test/playwright.trace.spec.ts
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-import { test, expect } from './playwright-test-fixtures';
+import { test, expect, stripAnsi } from './playwright-test-fixtures';
import { parseTrace, parseTraceRaw } from '../config/utils';
import fs from 'fs';
@@ -618,6 +618,79 @@ test('should include attachments by default', async ({ runInlineTest, server },
expect([...resources.keys()]).toContain(trace.model.actions[1].attachments[0].file);
});
+test('should redact secrets', async ({ runInlineTest, server }, testInfo) => {
+ server.setRoute('/secret', (req, res) => {
+ res.setHeader('Content-Type', 'application/json');
+ res.end(JSON.stringify({ password: req.headers['x-secret'] }));
+ });
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ export default { use: { trace: { mode: 'on', secrets: [{ name: 'PASSWORD', value: process.env.PASSWORD }, { name: 'QUOTED', value: process.env.QUOTED }] } } };
+ `,
+ 'a.spec.ts': `
+ import { test, expect } from '@playwright/test';
+ const password = process.env.PASSWORD;
+
+ test('with page', async ({ page }, testInfo) => {
+ await page.goto('${server.EMPTY_PAGE}');
+ await page.setContent('<input>');
+ await page.locator('input').fill(password);
+ await page.evaluate(password => console.log('console ' + password), password);
+ await page.evaluate(async password => {
+ const response = await fetch('/secret', { method: 'POST', headers: { 'x-secret': password }, body: 'body ' + password });
+ await response.text();
+ }, password);
+ console.log('stdout ' + password);
+ console.log('quoted ' + process.env.QUOTED);
+ process.stdout.write(Buffer.from('buffer ' + password + '\\n'));
+ await testInfo.attach('note ' + password, { body: 'body ' + password });
+ await test.step('step ' + password, async () => {
+ expect(password).toBe('wrong ' + 'password123');
+ });
+ });
+
+ test('without page', async ({}, testInfo) => {
+ await testInfo.attach('note ' + password, { body: 'body ' + password });
+ });
+ `,
+ }, { workers: 1 }, { PASSWORD: 'password123', QUOTED: 'say "hi"' });
+
+ expect(result.exitCode).toBe(1);
+ expect(result.passed).toBe(1);
+ expect(result.failed).toBe(1);
+
+ const { events, resources, actions } = await parseTraceRaw(testInfo.outputPath('test-results', 'a-with-page', 'trace.zip'));
+ for (const [name, buffer] of resources) {
+ for (const needle of ['password123', 'say "hi"', 'say \\"hi\\"'])
+ expect(buffer.includes(needle), `${name} contains ${needle}`).toBe(false);
+ }
+ expect(actions).toContain(`Fill "<secret>PASSWORD</secret>" locator('input')`);
+ expect(actions).toContain(`step <secret>PASSWORD</secret>`);
+ expect(events.filter(e => e.type === 'console').map(e => e.text)).toContain('console <secret>PASSWORD</secret>');
+ expect(events.filter(e => e.type === 'stdout').map(e => e.text ?? Buffer.from(e.base64, 'base64').toString())).toEqual([
+ 'stdout <secret>PASSWORD</secret>\n',
+ 'quoted <secret>QUOTED</secret>\n',
+ 'buffer <secret>PASSWORD</secret>\n',
+ ]);
+ expect(stripAnsi(events.filter(e => e.type === 'error').map(e => e.message).join('\n'))).toContain('Expected: "wrong <secret>PASSWORD</secret>"');
+ expect(events.filter(e => e.type === 'after').flatMap(e => e.attachments ?? []).map(a => a.name)).toContain('note <secret>PASSWORD</secret>');
+ const traceFile = [...resources.keys()].find(name => name.endsWith('.trace') && name !== 'test.trace')!;
+ expect(resources.get(traceFile)!.toString()).toContain('"__playwright_value_":"<secret>PASSWORD</secret>"');
+ const networkFile = [...resources.keys()].find(name => name.endsWith('.network'))!;
+ expect(resources.get(networkFile)!.toString()).toContain('"x-secret","value":"<secret>PASSWORD</secret>"');
+ const bodies = [...resources.values()].map(buffer => buffer.toString());
+ expect(bodies).toContain('body <secret>PASSWORD</secret>');
+ expect(bodies).toContain('{"password":"<secret>PASSWORD</secret>"}');
+ const sourceFile = [...resources.keys()].find(name => name.startsWith('src/'))!;
+ expect(resources.get(sourceFile)!.toString()).toContain(`'wrong ' + '<secret>PASSWORD</secret>'`);
+
+ const single = await parseTraceRaw(testInfo.outputPath('test-results', 'a-without-page', 'trace.zip'));
+ for (const [name, buffer] of single.resources)
+ expect(buffer.includes('password123'), `${name} contains the secret`).toBe(false);
+ expect(single.actions).toContain(`Attach "note <secret>PASSWORD</secret>"`);
+ expect([...single.resources.values()].map(buffer => buffer.toString())).toContain('body <secret>PASSWORD</secret>');
+});
+
test('should opt out of attachments', async ({ runInlineTest, server }, testInfo) => {
const result = await runInlineTest({
'playwright.config.ts': `
diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts
index 6ccb2a602e..eccd7301e3 100644
--- a/utils/generate_types/overrides-test.d.ts
+++ b/utils/generate_types/overrides-test.d.ts
@@ -280,7 +280,7 @@ export interface PlaywrightWorkerOptions {
connectOptions: ConnectOptions | undefined;
reuseContext: boolean;
screenshot: ScreenshotMode | { mode: ScreenshotMode } & Pick<PageScreenshotOptions, 'fullPage' | 'omitBackground'>;
- trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean };
+ trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean, secrets?: { name?: string, value: string }[] };
video: VideoMode | /** deprecated */ 'retry-with-video' | { mode: VideoMode, size?: ViewportSize, fps?: number, show?: { actions?: { duration?: number, position?: 'top-left' | 'top' | 'top-right' | 'bottom-left' | 'bottom' | 'bottom-right', /** @deprecated Use `style.title` instead. */ fontSize?: number, cursor?: 'none' | 'pointer', style?: { point?: string, highlight?: string, title?: string } }, test?: { level?: 'file' | 'title' | 'step', position?: 'top-left' | 'top' | 'top-right' | 'bottom-left' | 'bottom' | 'bottom-right', fontSize?: number } } };
}