Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(har): stream HAR to disk to support files >512MB (#40616)

Yury Semikhatskyc025980e1372
diff --git a/packages/playwright-core/src/server/har/harRecorder.ts b/packages/playwright-core/src/server/har/harRecorder.ts
index 3367c1acdb..b81a287a1e 100644
--- a/packages/playwright-core/src/server/har/harRecorder.ts
+++ b/packages/playwright-core/src/server/har/harRecorder.ts
@@ -87,7 +87,34 @@ export class HarRecorder implements HarTracerDelegate {
log.entries = this._entries;
this._fs.mkdir(path.dirname(this._harFilePath));
- this._fs.writeFile(this._harFilePath, jsonStringify({ log }));
+ // Stream the HAR field-by-field and entry-by-entry so the document is
+ // never held in memory in one piece - it can exceed V8's ~512MB string
+ // length limit.
+ const file = this._harFilePath;
+ this._fs.writeFile(file, '');
+ this._fs.appendFile(file, '{"log":{"version":' + JSON.stringify(log.version));
+ this._fs.appendFile(file, ',"creator":' + JSON.stringify(log.creator));
+ if (log.browser)
+ this._fs.appendFile(file, ',"browser":' + JSON.stringify(log.browser));
+ if (log.pages) {
+ this._fs.appendFile(file, ',"pages":[');
+ for (let i = 0; i < log.pages.length; i++) {
+ if (i)
+ this._fs.appendFile(file, ',');
+ this._fs.appendFile(file, JSON.stringify(log.pages[i]));
+ }
+ this._fs.appendFile(file, ']');
+ }
+ this._fs.appendFile(file, ',"entries":[');
+ for (let i = 0; i < log.entries.length; i++) {
+ if (i)
+ this._fs.appendFile(file, ',');
+ this._fs.appendFile(file, JSON.stringify(log.entries[i]));
+ }
+ this._fs.appendFile(file, ']');
+ if (log.comment !== undefined)
+ this._fs.appendFile(file, ',"comment":' + JSON.stringify(log.comment));
+ this._fs.appendFile(file, '}}', true /* flush */);
}
async flush() {
@@ -115,50 +142,3 @@ export class HarRecorder implements HarTracerDelegate {
return { artifact };
}
}
-
-function jsonStringify(object: any): string {
- const tokens: string[] = [];
- innerJsonStringify(object, tokens, '', false, undefined);
- return tokens.join('');
-}
-
-function innerJsonStringify(object: any, tokens: string[], indent: string, flat: boolean, parentKey: string | undefined) {
- if (typeof object !== 'object' || object === null) {
- tokens.push(JSON.stringify(object));
- return;
- }
-
- const isArray = Array.isArray(object);
- if (!isArray && object.constructor.name !== 'Object') {
- tokens.push(JSON.stringify(object));
- return;
- }
-
- const entries = isArray ? object : Object.entries(object).filter(e => e[1] !== undefined);
- if (!entries.length) {
- tokens.push(isArray ? `[]` : `{}`);
- return;
- }
-
- const childIndent = `${indent} `;
- let brackets: { open: string, close: string };
- if (isArray)
- brackets = flat ? { open: '[', close: ']' } : { open: `[\n${childIndent}`, close: `\n${indent}]` };
- else
- brackets = flat ? { open: '{ ', close: ' }' } : { open: `{\n${childIndent}`, close: `\n${indent}}` };
-
- tokens.push(brackets.open);
-
- for (let i = 0; i < entries.length; ++i) {
- const entry = entries[i];
- if (i)
- tokens.push(flat ? `, ` : `,\n${childIndent}`);
- if (!isArray)
- tokens.push(`${JSON.stringify(entry[0])}: `);
- const key = isArray ? undefined : entry[0];
- const flatten = flat || key === 'timings' || parentKey === 'headers';
- innerJsonStringify(isArray ? entry : entry[1], tokens, childIndent, flatten, key);
- }
-
- tokens.push(brackets.close);
-}
diff --git a/packages/utils/serializedFS.ts b/packages/utils/serializedFS.ts
index 711d13126a..994ebf22e1 100644
--- a/packages/utils/serializedFS.ts
+++ b/packages/utils/serializedFS.ts
@@ -34,6 +34,11 @@ type SerializedFSOperation = {
op: 'zip', entries: NameValue[], zipFileName: string,
};
+// Per-file pending appendFile chunks are flushed once they accumulate to this
+// size, and the merge in _appendOperation stops merging past it. Bounding
+// merged content keeps any single op below V8's ~512MB max string length.
+const APPEND_CHUNK_SIZE = 64 * 1024;
+
export class SerializedFS {
private _buffers = new Map<string, string[]>(); // Should never be accessed from within appendOperation.
private _error: Error | undefined;
@@ -57,8 +62,12 @@ export class SerializedFS {
appendFile(file: string, text: string, flush?: boolean) {
if (!this._buffers.has(file))
this._buffers.set(file, []);
- this._buffers.get(file)!.push(text);
- if (flush)
+ const buffer = this._buffers.get(file)!;
+ buffer.push(text);
+ let size = 0;
+ for (const chunk of buffer)
+ size += chunk.length;
+ if (flush || size >= APPEND_CHUNK_SIZE)
this._flushFile(file);
}
@@ -96,8 +105,10 @@ export class SerializedFS {
// This method serializes all writes to the trace.
private _appendOperation(op: SerializedFSOperation): void {
const last = this._operations[this._operations.length - 1];
- if (last?.op === 'appendFile' && op.op === 'appendFile' && last.file === op.file) {
- // Merge pending appendFile operations for performance.
+ if (last?.op === 'appendFile' && op.op === 'appendFile' && last.file === op.file
+ && last.content.length < APPEND_CHUNK_SIZE) {
+ // Merge pending appendFile operations for performance, but cap merging at
+ // APPEND_CHUNK_SIZE so a single op never grows past V8's max string length.
last.content += op.content;
return;
}
diff --git a/tests/library/har.spec.ts b/tests/library/har.spec.ts
index 6deab064cf..dbd00adfb5 100644
--- a/tests/library/har.spec.ts
+++ b/tests/library/har.spec.ts
@@ -916,6 +916,41 @@ it('should not hang on slow chunked response', async ({ browserName, browser, co
expect(log.browser!.version).toBe(browser.version());
});
+it('should support HAR larger than 512MB', async ({ contextFactory, server, browserName }, testInfo) => {
+ it.skip(browserName !== 'chromium', 'serializer is browser-agnostic; one browser is enough');
+ it.slow();
+ it.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/36707' });
+
+ const harPath = testInfo.outputPath('test.har');
+ const context = await contextFactory({ recordHar: { path: harPath } });
+
+ // 30 x 20MB textual responses push the HAR JSON past V8's ~512MB max
+ // string length. Each body still fits in a single string; only the
+ // aggregate would overflow the previous tokens.join('').
+ const body = 'a'.repeat(20 * 1024 * 1024);
+ server.setRoute('/large', (req, res) => {
+ res.writeHead(200, { 'Content-Type': 'text/plain' });
+ res.end(body);
+ });
+ for (let i = 0; i < 30; i++)
+ await context.request.get(`${server.PREFIX}/large`);
+ await context.close();
+
+ const stats = fs.statSync(harPath);
+ expect(stats.size).toBeGreaterThan(512 * 1024 * 1024);
+
+ // Reading the whole file as a string would re-hit V8's limit, so
+ // sample the head and tail to verify structural sanity.
+ const fd = fs.openSync(harPath, 'r');
+ const head = Buffer.alloc(64);
+ fs.readSync(fd, head, 0, 64, 0);
+ const tail = Buffer.alloc(64);
+ fs.readSync(fd, tail, 0, 64, stats.size - 64);
+ fs.closeSync(fd);
+ expect(head.toString()).toMatch(/^\{\s*"log"\s*:\s*\{/);
+ expect(tail.toString()).toMatch(/\}\s*\}\s*$/);
+});
+
it.describe('tracing.startHar', () => {
it('should record a HAR with options', async ({ contextFactory, server }, testInfo) => {
const context = await contextFactory();