Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(har): stream HAR to disk to support files >512MB (#40616)
Yury Semikhatskyc025980e1372
diff --git a/packages/playwright-core/src/server/har/harRecorder.ts b/packages/playwright-core/src/server/har/harRecorder.tsindex 3367c1acdb..b81a287a1e 100644--- a/packages/playwright-core/src/server/har/harRecorder.ts+++ b/packages/playwright-core/src/server/har/harRecorder.ts@@ -87,7 +87,34 @@ export class HarRecorder implements HarTracerDelegate {log.entries = this._entries;this._fs.mkdir(path.dirname(this._harFilePath));- this._fs.writeFile(this._harFilePath, jsonStringify({ log }));+ // Stream the HAR field-by-field and entry-by-entry so the document is+ // never held in memory in one piece - it can exceed V8's ~512MB string+ // length limit.+ const file = this._harFilePath;+ this._fs.writeFile(file, '');+ this._fs.appendFile(file, '{"log":{"version":' + JSON.stringify(log.version));+ this._fs.appendFile(file, ',"creator":' + JSON.stringify(log.creator));+ if (log.browser)+ this._fs.appendFile(file, ',"browser":' + JSON.stringify(log.browser));+ if (log.pages) {+ this._fs.appendFile(file, ',"pages":[');+ for (let i = 0; i < log.pages.length; i++) {+ if (i)+ this._fs.appendFile(file, ',');+ this._fs.appendFile(file, JSON.stringify(log.pages[i]));+ }+ this._fs.appendFile(file, ']');+ }+ this._fs.appendFile(file, ',"entries":[');+ for (let i = 0; i < log.entries.length; i++) {+ if (i)+ this._fs.appendFile(file, ',');+ this._fs.appendFile(file, JSON.stringify(log.entries[i]));+ }+ this._fs.appendFile(file, ']');+ if (log.comment !== undefined)+ this._fs.appendFile(file, ',"comment":' + JSON.stringify(log.comment));+ this._fs.appendFile(file, '}}', true /* flush */);}async flush() {@@ -115,50 +142,3 @@ export class HarRecorder implements HarTracerDelegate {return { artifact };}}--function jsonStringify(object: any): string {- const tokens: string[] = [];- innerJsonStringify(object, tokens, '', false, undefined);- return tokens.join('');-}--function innerJsonStringify(object: any, tokens: string[], indent: string, flat: boolean, parentKey: string | undefined) {- if (typeof object !== 'object' || object === null) {- tokens.push(JSON.stringify(object));- return;- }-- const isArray = Array.isArray(object);- if (!isArray && object.constructor.name !== 'Object') {- tokens.push(JSON.stringify(object));- return;- }-- const entries = isArray ? object : Object.entries(object).filter(e => e[1] !== undefined);- if (!entries.length) {- tokens.push(isArray ? `[]` : `{}`);- return;- }-- const childIndent = `${indent} `;- let brackets: { open: string, close: string };- if (isArray)- brackets = flat ? { open: '[', close: ']' } : { open: `[\n${childIndent}`, close: `\n${indent}]` };- else- brackets = flat ? { open: '{ ', close: ' }' } : { open: `{\n${childIndent}`, close: `\n${indent}}` };-- tokens.push(brackets.open);-- for (let i = 0; i < entries.length; ++i) {- const entry = entries[i];- if (i)- tokens.push(flat ? `, ` : `,\n${childIndent}`);- if (!isArray)- tokens.push(`${JSON.stringify(entry[0])}: `);- const key = isArray ? undefined : entry[0];- const flatten = flat || key === 'timings' || parentKey === 'headers';- innerJsonStringify(isArray ? entry : entry[1], tokens, childIndent, flatten, key);- }-- tokens.push(brackets.close);-}diff --git a/packages/utils/serializedFS.ts b/packages/utils/serializedFS.tsindex 711d13126a..994ebf22e1 100644--- a/packages/utils/serializedFS.ts+++ b/packages/utils/serializedFS.ts@@ -34,6 +34,11 @@ type SerializedFSOperation = {op: 'zip', entries: NameValue[], zipFileName: string,};+// Per-file pending appendFile chunks are flushed once they accumulate to this+// size, and the merge in _appendOperation stops merging past it. Bounding+// merged content keeps any single op below V8's ~512MB max string length.+const APPEND_CHUNK_SIZE = 64 * 1024;+export class SerializedFS {private _buffers = new Map<string, string[]>(); // Should never be accessed from within appendOperation.private _error: Error | undefined;@@ -57,8 +62,12 @@ export class SerializedFS {appendFile(file: string, text: string, flush?: boolean) {if (!this._buffers.has(file))this._buffers.set(file, []);- this._buffers.get(file)!.push(text);- if (flush)+ const buffer = this._buffers.get(file)!;+ buffer.push(text);+ let size = 0;+ for (const chunk of buffer)+ size += chunk.length;+ if (flush || size >= APPEND_CHUNK_SIZE)this._flushFile(file);}@@ -96,8 +105,10 @@ export class SerializedFS {// This method serializes all writes to the trace.private _appendOperation(op: SerializedFSOperation): void {const last = this._operations[this._operations.length - 1];- if (last?.op === 'appendFile' && op.op === 'appendFile' && last.file === op.file) {- // Merge pending appendFile operations for performance.+ if (last?.op === 'appendFile' && op.op === 'appendFile' && last.file === op.file+ && last.content.length < APPEND_CHUNK_SIZE) {+ // Merge pending appendFile operations for performance, but cap merging at+ // APPEND_CHUNK_SIZE so a single op never grows past V8's max string length.last.content += op.content;return;}diff --git a/tests/library/har.spec.ts b/tests/library/har.spec.tsindex 6deab064cf..dbd00adfb5 100644--- a/tests/library/har.spec.ts+++ b/tests/library/har.spec.ts@@ -916,6 +916,41 @@ it('should not hang on slow chunked response', async ({ browserName, browser, coexpect(log.browser!.version).toBe(browser.version());});+it('should support HAR larger than 512MB', async ({ contextFactory, server, browserName }, testInfo) => {+ it.skip(browserName !== 'chromium', 'serializer is browser-agnostic; one browser is enough');+ it.slow();+ it.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/36707' });++ const harPath = testInfo.outputPath('test.har');+ const context = await contextFactory({ recordHar: { path: harPath } });++ // 30 x 20MB textual responses push the HAR JSON past V8's ~512MB max+ // string length. Each body still fits in a single string; only the+ // aggregate would overflow the previous tokens.join('').+ const body = 'a'.repeat(20 * 1024 * 1024);+ server.setRoute('/large', (req, res) => {+ res.writeHead(200, { 'Content-Type': 'text/plain' });+ res.end(body);+ });+ for (let i = 0; i < 30; i++)+ await context.request.get(`${server.PREFIX}/large`);+ await context.close();++ const stats = fs.statSync(harPath);+ expect(stats.size).toBeGreaterThan(512 * 1024 * 1024);++ // Reading the whole file as a string would re-hit V8's limit, so+ // sample the head and tail to verify structural sanity.+ const fd = fs.openSync(harPath, 'r');+ const head = Buffer.alloc(64);+ fs.readSync(fd, head, 0, 64, 0);+ const tail = Buffer.alloc(64);+ fs.readSync(fd, tail, 0, 64, stats.size - 64);+ fs.closeSync(fd);+ expect(head.toString()).toMatch(/^\{\s*"log"\s*:\s*\{/);+ expect(tail.toString()).toMatch(/\}\s*\}\s*$/);+});+it.describe('tracing.startHar', () => {it('should record a HAR with options', async ({ contextFactory, server }, testInfo) => {const context = await contextFactory();