Skip to content

Lightpanda: the headless browser designed for AI and automation

update tests for brotli coverage

Halil Durak54d7902cd259
diff --git a/src/browser/tests/compression/compression_stream.html b/src/browser/tests/compression/compression_stream.html
index 31004e32e..fe3ff367f 100644
--- a/src/browser/tests/compression/compression_stream.html
+++ b/src/browser/tests/compression/compression_stream.html
@@ -2,7 +2,7 @@
<script src="../testing.js"></script>
<script>
- const FORMATS = ['gzip', 'deflate', 'deflate-raw'];
+ const FORMATS = ['gzip', 'deflate', 'deflate-raw', 'brotli'];
async function readAll(readable) {
return readRest(readable.getReader(), []);
@@ -52,7 +52,9 @@
const out = new Uint8Array(len);
let x = 12345;
for (let i = 0; i < len; i++) {
- x = (x * 1103515245 + 12345) >>> 0;
+ // Math.imul keeps the low 32 bits exact; a plain `*` rounds to a double
+ // and the sequence falls into a ~6 KiB cycle.
+ x = (Math.imul(x, 1103515245) + 12345) >>> 0;
out[i] = x >>> 24;
}
return out;
@@ -66,6 +68,13 @@
}
return 'resolved';
}
+
+ // Decodes brotli with the HTTP client: the test server echoes `compressed`
+ // back as `Content-Encoding: br`.
+ async function unbrotli(compressed) {
+ const response = await fetch('http://127.0.0.1:9582/echo_brotli', { method: 'POST', body: compressed });
+ return new Uint8Array(await response.arrayBuffer());
+ }
</script>
<script id=constructor>
@@ -76,7 +85,7 @@
testing.expectTrue(cs.writable instanceof WritableStream);
}
- for (const format of ['GZIP', 'zlib', '', undefined]) {
+ for (const format of ['GZIP', 'zlib', 'BROTLI', 'br', '', undefined]) {
testing.withError((err) => testing.expectEqual('TypeError', err.name), () => new CompressionStream(format));
}
testing.withError((err) => testing.expectEqual('TypeError', err.name), () => new CompressionStream());
@@ -145,6 +154,7 @@
testing.expectEqual(20, results['gzip'].length);
testing.expectEqual(8, results['deflate'].length);
testing.expectEqual(2, results['deflate-raw'].length);
+ testing.expectEqual(1, results['brotli'].length);
for (const format of FORMATS) {
testing.expectEqual([], results[format].decompressed);
}
@@ -180,17 +190,19 @@
const state = await testing.async();
const results = [];
- for (const chunk of ['hello', undefined, null, 42, {}, [1, 2, 3]]) {
- const cs = new CompressionStream('gzip');
- const writer = cs.writable.getWriter();
- const reader = cs.readable.getReader();
- const pending_read = rejection(reader.read());
- results.push([
- await rejection(writer.write(chunk)),
- await pending_read,
- await rejection(reader.read()),
- await rejection(writer.write(new Uint8Array([1]))),
- ]);
+ for (const format of ['gzip', 'brotli']) {
+ for (const chunk of ['hello', undefined, null, 42, {}, [1, 2, 3]]) {
+ const cs = new CompressionStream(format);
+ const writer = cs.writable.getWriter();
+ const reader = cs.readable.getReader();
+ const pending_read = rejection(reader.read());
+ results.push([
+ await rejection(writer.write(chunk)),
+ await pending_read,
+ await rejection(reader.read()),
+ await rejection(writer.write(new Uint8Array([1]))),
+ ]);
+ }
}
state.resolve();
@@ -365,3 +377,81 @@
});
}
</script>
+
+<script id=brotli_matches_cli type=module>
+ const state = await testing.async();
+
+ const encode = (text) => new TextEncoder().encode(text);
+ const results = {
+ hello: Array.from(await run(new CompressionStream('brotli'), [encode('hello')])),
+ repeated: Array.from(await run(new CompressionStream('brotli'), [encode('hello brotli, '.repeat(8))])),
+ // Only empty chunks: still a complete, valid stream.
+ empty: Array.from(await run(new CompressionStream('brotli'), [new Uint8Array(0), new ArrayBuffer(0)])),
+ };
+
+ state.resolve();
+ await state.done(() => {
+ // Byte-for-byte what `brotli -q 11 -w 22` (v1.2.0) gives, the encoder's
+ // defaults. 'hello' is too short to compress, so it's stored as is.
+ testing.expectEqual([11, 2, 128, 104, 101, 108, 108, 111, 3], results.hello);
+ testing.expectEqual([
+ 27, 111, 0, 248, 29, 9, 54, 78, 116, 111, 36, 134, 215, 98, 101, 104, 114,
+ 26, 132, 133, 141, 78, 178, 165, 189, 5, 130, 241, 0, 60, 51, 252, 18,
+ ], results.repeated);
+ testing.expectEqual([59], results.empty);
+ });
+</script>
+
+<script id=brotli_round_trip type=module>
+ const state = await testing.async();
+
+ const noisy = noise(200 * 1024);
+ const tiny = noise(4096);
+ const text = new TextEncoder().encode('hello world '.repeat(1000));
+ const cases = {
+ noise: [noisy, [noisy.subarray(0, 1), noisy.subarray(1, 70000), noisy.subarray(70000)]],
+ tiny_writes: [tiny, Array.from(tiny, (_, i) => tiny.subarray(i, i + 1))],
+ text: [text, [text]],
+ // Closed without a single write.
+ empty: [new Uint8Array(0), []],
+ };
+
+ const results = {};
+ for (const [name, [input, chunks]] of Object.entries(cases)) {
+ const compressed = await run(new CompressionStream('brotli'), chunks);
+ results[name] = equal(await unbrotli(compressed), input);
+ }
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual({ noise: true, tiny_writes: true, text: true, empty: true }, results);
+ });
+</script>
+
+<script id=brotli_output_before_close type=module>
+ const state = await testing.async();
+
+ // The encoder holds input back until it has a whole meta-block; for this
+ // noise that's after 1.25 MiB, so the write produces output by itself. It
+ // then stops taking input until that output is taken, so the rest of the
+ // write needs another call into the encoder.
+ const input = noise(1664 * 1024);
+ const cs = new CompressionStream('brotli');
+ const writer = cs.writable.getWriter();
+ const reader = cs.readable.getReader();
+
+ let first = null;
+ reader.read().then(({ value }) => {
+ first = value;
+ });
+ await writer.write(input);
+ const before_close = first !== null;
+ writer.close();
+ const compressed = await readRest(reader, before_close ? [first] : []);
+ const same = equal(await unbrotli(compressed), input);
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual({ before_close: true, same: true }, { before_close, same });
+ });
+</script>
diff --git a/src/browser/tests/compression/decompression_stream.html b/src/browser/tests/compression/decompression_stream.html
index 5239bbded..4a9beba09 100644
--- a/src/browser/tests/compression/decompression_stream.html
+++ b/src/browser/tests/compression/decompression_stream.html
@@ -2,13 +2,15 @@
<script src="../testing.js"></script>
<script>
- const FORMATS = ['gzip', 'deflate', 'deflate-raw'];
+ const FORMATS = ['gzip', 'deflate', 'deflate-raw', 'brotli'];
- // 'hello', compressed by Python's zlib/gzip modules.
+ // 'hello', compressed by Python's zlib/gzip modules and the brotli CLI
+ // (v1.2.0).
const HELLO = {
'gzip': [31, 139, 8, 0, 0, 0, 0, 0, 2, 255, 203, 72, 205, 201, 201, 7, 0, 134, 166, 16, 54, 5, 0, 0, 0],
'deflate': [120, 156, 203, 72, 205, 201, 201, 7, 0, 6, 44, 2, 21],
'deflate-raw': [203, 72, 205, 201, 201, 7, 0],
+ 'brotli': [15, 2, 128, 104, 101, 108, 108, 111, 3],
};
// '', compressed.
@@ -16,6 +18,7 @@
'gzip': [31, 139, 8, 0, 0, 0, 0, 0, 2, 255, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'deflate': [120, 156, 3, 0, 0, 0, 0, 1],
'deflate-raw': [3, 0],
+ 'brotli': [63],
};
async function readAll(readable) {
@@ -104,7 +107,7 @@
testing.expectTrue(ds.writable instanceof WritableStream);
}
- for (const format of ['GZIP', 'zlib', '', undefined]) {
+ for (const format of ['GZIP', 'zlib', 'BROTLI', 'br', '', undefined]) {
testing.withError((err) => testing.expectEqual('TypeError', err.name), () => new DecompressionStream(format));
}
testing.withError((err) => testing.expectEqual('TypeError', err.name), () => new DecompressionStream());
@@ -188,6 +191,10 @@
// A reserved block type
results['deflate-raw'] = await outcome('deflate-raw', [[7, 0]]);
+ // A reserved window size (RFC 7932, section 9.1). brotli has no checksum, so
+ // flipped bytes often go unnoticed.
+ results['brotli'] = await outcome('brotli', [[17, 0, 0, 0]]);
+
state.resolve();
await state.done(() => {
for (const result of Object.values(results)) {
@@ -223,12 +230,14 @@
const state = await testing.async();
const results = [];
- for (const chunk of ['hello', undefined, null, 42, {}, [1, 2, 3]]) {
- const ds = new DecompressionStream('gzip');
- const writer = ds.writable.getWriter();
- const reader = ds.readable.getReader();
- const pending_read = rejection(reader.read());
- results.push([await rejection(writer.write(chunk)), await pending_read, await rejection(reader.read())]);
+ for (const format of ['gzip', 'brotli']) {
+ for (const chunk of ['hello', undefined, null, 42, {}, [1, 2, 3]]) {
+ const ds = new DecompressionStream(format);
+ const writer = ds.writable.getWriter();
+ const reader = ds.readable.getReader();
+ const pending_read = rejection(reader.read());
+ results.push([await rejection(writer.write(chunk)), await pending_read, await rejection(reader.read())]);
+ }
}
state.resolve();
@@ -328,14 +337,45 @@
['gzip', 'deflate-raw'],
['deflate-raw', 'gzip'],
['deflate-raw', 'deflate'],
+ ['brotli', 'gzip'],
+ ['brotli', 'deflate'],
+ ['gzip', 'brotli'],
+ ['deflate', 'brotli'],
+ ['deflate-raw', 'brotli'],
]) {
results[`${data} as ${format}`] = await outcome(format, [HELLO[data]]);
}
+ // brotli has no magic number: these bytes start a valid-looking stream that
+ // never ends, so only close() notices.
+ const raw_as_brotli = await outcome('brotli', [HELLO['deflate-raw']]);
+
state.resolve();
await state.done(() => {
for (const result of Object.values(results)) {
testing.expectEqual({ writes: ['TypeError'], close: 'TypeError', read: 'TypeError' }, result);
}
+ testing.expectEqual({ writes: ['resolved'], close: 'TypeError', read: 'TypeError' }, raw_as_brotli);
+ });
+</script>
+
+<script id=brotli_small_window type=module>
+ const state = await testing.async();
+
+ // `brotli -q 11 -w 10` (v1.2.0) of 'hello world ' x 1000: with a 1 KiB
+ // window the decoder's ring buffer fills a dozen times within one write.
+ const compressed = [
+ 161, 248, 118, 193, 111, 164, 116, 243, 38, 170, 50, 180, 4, 97, 123, 171, 89, 30, 171, 5, 50, 160, 218, 142, 38, 0,
+ 0,
+ ];
+ const expected = 'hello world '.repeat(1000);
+ const results = {
+ whole: (await decompress('brotli', [compressed])) === expected,
+ split: (await decompress('brotli', compressed.map((b) => [b]))) === expected,
+ };
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual({ whole: true, split: true }, results);
});
</script>
diff --git a/src/testing.zig b/src/testing.zig
index c738167f6..4a69b039b 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -1269,6 +1269,20 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
});
}
+ if (std.mem.eql(u8, path, "/echo_brotli")) {
+ // Echo the request body back as `Content-Encoding: br`, so the HTTP
+ // client decodes it; checks CompressionStream('brotli') output with a
+ // decoder that isn't ours.
+ var body_buf: [4096]u8 = undefined;
+ const body = try req.readerExpectNone(&body_buf).allocRemaining(arena_allocator, .limited(4 * 1024 * 1024));
+ return req.respond(body, .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "application/octet-stream" },
+ .{ .name = "Content-Encoding", .value = "br" },
+ },
+ });
+ }
+
if (std.mem.eql(u8, path, "/echo_headers")) {
if (req.head.method == .OPTIONS) {
return req.respond("", .{