Skip to content

Lightpanda: the headless browser designed for AI and automation

add tests for compression streams

Halil Durakc427682c31bc
diff --git a/src/browser/tests/compression/compression_stream.html b/src/browser/tests/compression/compression_stream.html
new file mode 100644
index 000000000..31004e32e
--- /dev/null
+++ b/src/browser/tests/compression/compression_stream.html
@@ -0,0 +1,367 @@
+<!DOCTYPE html>
+<script src="../testing.js"></script>
+
+<script>
+ const FORMATS = ['gzip', 'deflate', 'deflate-raw'];
+
+ async function readAll(readable) {
+ return readRest(readable.getReader(), []);
+ }
+
+ // Reads `reader` to the end, after the `chunks` already read from it.
+ async function readRest(reader, chunks) {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ chunks.push(value);
+ }
+ return concat(chunks);
+ }
+
+ function concat(chunks) {
+ let length = 0;
+ for (const chunk of chunks) {
+ if (!(chunk instanceof Uint8Array)) throw new Error('chunk is not a Uint8Array');
+ // The spec only enqueues non-empty pieces.
+ if (chunk.length === 0) throw new Error('chunk is empty');
+ length += chunk.length;
+ }
+ const out = new Uint8Array(length);
+ let offset = 0;
+ for (const chunk of chunks) {
+ out.set(chunk, offset);
+ offset += chunk.length;
+ }
+ return out;
+ }
+
+ function equal(a, b) {
+ return a.length === b.length && a.every((byte, i) => byte === b[i]);
+ }
+
+ async function run(stream, chunks) {
+ const writer = stream.writable.getWriter();
+ const output = readAll(stream.readable);
+ for (const chunk of chunks) writer.write(chunk);
+ writer.close();
+ return output;
+ }
+
+ // Deterministic, poorly compressible bytes
+ function noise(len) {
+ const out = new Uint8Array(len);
+ let x = 12345;
+ for (let i = 0; i < len; i++) {
+ x = (x * 1103515245 + 12345) >>> 0;
+ out[i] = x >>> 24;
+ }
+ return out;
+ }
+
+ async function rejection(promise) {
+ try {
+ await promise;
+ } catch (e) {
+ return e.name;
+ }
+ return 'resolved';
+ }
+</script>
+
+<script id=constructor>
+ {
+ for (const format of FORMATS) {
+ const cs = new CompressionStream(format);
+ testing.expectTrue(cs.readable instanceof ReadableStream);
+ testing.expectTrue(cs.writable instanceof WritableStream);
+ }
+
+ for (const format of ['GZIP', 'zlib', '', undefined]) {
+ testing.withError((err) => testing.expectEqual('TypeError', err.name), () => new CompressionStream(format));
+ }
+ testing.withError((err) => testing.expectEqual('TypeError', err.name), () => new CompressionStream());
+ }
+</script>
+
+<script id=matches_zlib type=module>
+ const state = await testing.async();
+
+ const hello = new TextEncoder().encode('hello');
+ const results = {};
+ for (const format of FORMATS) {
+ results[format] = await run(new CompressionStream(format), [hello]);
+ }
+
+ state.resolve();
+ await state.done(() => {
+ // Byte-for-byte what zlib.compress(b'hello') gives in Python.
+ testing.expectEqual([120, 156, 203, 72, 205, 201, 201, 7, 0, 6, 44, 2, 21], Array.from(results['deflate']));
+ testing.expectEqual([203, 72, 205, 201, 201, 7, 0], Array.from(results['deflate-raw']));
+
+ // gzip: magic + CM, then (after mtime, xfl, os) the same body, CRC32 and ISIZE.
+ const gzip = results['gzip'];
+ testing.expectEqual([31, 139, 8, 0], Array.from(gzip.slice(0, 4)));
+ testing.expectEqual([203, 72, 205, 201, 201, 7, 0, 134, 166, 16, 54, 5, 0, 0, 0], Array.from(gzip.slice(10)));
+ });
+</script>
+
+<script id=round_trip type=module>
+ const state = await testing.async();
+
+ // Big enough for the output to span several internal buffers.
+ const input = noise(200 * 1024);
+ const chunks = [input.subarray(0, 1), input.subarray(1, 70000), input.subarray(70000)];
+
+ const results = {};
+ for (const format of FORMATS) {
+ const compressed = await run(new CompressionStream(format), chunks);
+ const decompressed = await run(new DecompressionStream(format), [compressed]);
+ results[format] = decompressed.length === input.length && decompressed.every((b, i) => b === input[i]);
+ }
+
+ state.resolve();
+ await state.done(() => {
+ for (const format of FORMATS) {
+ testing.expectEqual(true, results[format]);
+ }
+ });
+</script>
+
+<script id=empty_input type=module>
+ const state = await testing.async();
+
+ const results = {};
+ for (const format of FORMATS) {
+ // Only empty chunks: still a complete, valid stream.
+ const compressed = await run(new CompressionStream(format), [new Uint8Array(0), new ArrayBuffer(0)]);
+ results[format] = {
+ length: compressed.length,
+ decompressed: Array.from(await run(new DecompressionStream(format), [compressed])),
+ };
+ }
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual(20, results['gzip'].length);
+ testing.expectEqual(8, results['deflate'].length);
+ testing.expectEqual(2, results['deflate-raw'].length);
+ for (const format of FORMATS) {
+ testing.expectEqual([], results[format].decompressed);
+ }
+ });
+</script>
+
+<script id=buffer_sources type=module>
+ const state = await testing.async();
+
+ const bytes = new TextEncoder().encode('xhello!');
+ const view = bytes.subarray(1, 6);
+ const sources = [
+ view,
+ view.slice().buffer,
+ new DataView(bytes.buffer, 1, 5),
+ new Uint16Array(new Uint8Array([104, 101, 108, 108, 111, 0]).buffer).subarray(0, 2),
+ ];
+
+ const results = [];
+ for (const source of sources) {
+ const compressed = await run(new CompressionStream('deflate-raw'), [source]);
+ const decompressed = await run(new DecompressionStream('deflate-raw'), [compressed]);
+ results.push(new TextDecoder().decode(decompressed));
+ }
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual(['hello', 'hello', 'hello', 'hell'], results);
+ });
+</script>
+
+<script id=bad_chunks type=module>
+ const state = await testing.async();
+
+ const results = [];
+ for (const chunk of ['hello', undefined, null, 42, {}, [1, 2, 3]]) {
+ const cs = new CompressionStream('gzip');
+ const writer = cs.writable.getWriter();
+ const reader = cs.readable.getReader();
+ const pending_read = rejection(reader.read());
+ results.push([
+ await rejection(writer.write(chunk)),
+ await pending_read,
+ await rejection(reader.read()),
+ await rejection(writer.write(new Uint8Array([1]))),
+ ]);
+ }
+
+ state.resolve();
+ await state.done(() => {
+ for (const result of results) {
+ testing.expectEqual(['TypeError', 'TypeError', 'TypeError', 'TypeError'], result);
+ }
+ });
+</script>
+
+<script id=cancelled_readable type=module>
+ const state = await testing.async();
+
+ const cs = new CompressionStream('gzip');
+ const writer = cs.writable.getWriter();
+ await cs.readable.cancel();
+
+ // Enough poorly compressible data that deflate has output to enqueue.
+ const write = await rejection(writer.write(noise(200 * 1024)));
+ const next_write = await rejection(writer.write(new Uint8Array([1])));
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual('TypeError', write);
+ testing.expectEqual('TypeError', next_write);
+ });
+</script>
+
+<script id=pipe_through type=module>
+ const state = await testing.async();
+
+ const text = 'hello world '.repeat(1000);
+ const compressed = new Blob([text]).stream().pipeThrough(new CompressionStream('gzip'));
+ const bytes = await new Response(compressed).arrayBuffer();
+
+ const decompressed = new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip'));
+ const result = await new Response(decompressed).text();
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectTrue(bytes.byteLength < 200);
+ testing.expectEqual(text, result);
+ });
+</script>
+
+<script id=tiny_writes type=module>
+ const state = await testing.async();
+
+ // deflate buffers most one-byte writes without producing anything; none of
+ // that may surface as an empty chunk (concat rejects those).
+ const input = noise(4096);
+ const chunks = Array.from(input, (_, i) => input.subarray(i, i + 1));
+
+ const results = {};
+ for (const format of FORMATS) {
+ const compressed = await run(new CompressionStream(format), chunks);
+ results[format] = equal(await run(new DecompressionStream(format), [compressed]), input);
+ }
+
+ state.resolve();
+ await state.done(() => {
+ for (const format of FORMATS) {
+ testing.expectEqual(true, results[format]);
+ }
+ });
+</script>
+
+<script id=reentrant_write>
+ {
+ // A classic script runs as a task, so resolving the pending read runs its
+ // callback right away: it writes again and closes while the first write
+ // is still handing out its output.
+ const state_promise = testing.async();
+
+ const first = noise(64 * 1024);
+ const second = new TextEncoder().encode('second write');
+ const cs = new CompressionStream('deflate');
+ const writer = cs.writable.getWriter();
+ const reader = cs.readable.getReader();
+
+ const chunks = [];
+ let nested = null;
+ reader.read().then(({ value }) => {
+ chunks.push(value);
+ nested = Promise.all([rejection(writer.write(second)), rejection(writer.close())]);
+ });
+ const outer = rejection(writer.write(first));
+
+ state_promise.then(async (state) => {
+ const compressed = await readRest(reader, chunks);
+ const decompressed = await run(new DecompressionStream('deflate'), [compressed]);
+ const results = {
+ outer: await outer,
+ nested: await nested,
+ same: equal(decompressed, concat([first, second])),
+ };
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual({ outer: 'resolved', nested: ['resolved', 'resolved'], same: true }, results);
+ });
+ });
+ }
+</script>
+
+<script id=close_after_error>
+ {
+ // A bad chunk errors the stream. Rejecting the pending read runs this
+ // callback right away (a classic script runs as a task), and it closes
+ // the writable while the stream is still being errored.
+ const state_promise = testing.async();
+
+ const cs = new CompressionStream('gzip');
+ const writer = cs.writable.getWriter();
+ let close = null;
+ const read = cs.readable.getReader().read().catch((e) => {
+ close = rejection(writer.close());
+ return e.name;
+ });
+ const write = rejection(writer.write('not a BufferSource'));
+
+ state_promise.then(async (state) => {
+ const results = { write: await write, read: await read, close: await close };
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual({ write: 'TypeError', read: 'TypeError', close: 'TypeError' }, results);
+ });
+ });
+ }
+</script>
+
+<script id=detached_input>
+ {
+ // A classic script runs as a task, so the pending read is resolved inside
+ // write(). Looking up `then` on its result detaches the input: zlib must
+ // be done with it by then.
+ const state_promise = testing.async();
+
+ const input = noise(200 * 1024);
+ const copy = input.slice();
+ const cs = new CompressionStream('deflate');
+ const writer = cs.writable.getWriter();
+ const reader = cs.readable.getReader();
+ const first = reader.read();
+
+ let detached = false;
+ Object.defineProperty(Object.prototype, 'then', {
+ configurable: true,
+ get() {
+ delete Object.prototype.then;
+ input.buffer.transfer();
+ detached = true;
+ },
+ });
+ const write = rejection(writer.write(input));
+ writer.close();
+
+ state_promise.then(async (state) => {
+ const compressed = await readRest(reader, [(await first).value]);
+ const decompressed = await run(new DecompressionStream('deflate'), [compressed]);
+ const results = {
+ detached: detached && input.byteLength === 0,
+ write: await write,
+ same: equal(decompressed, copy),
+ };
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual({ detached: true, write: 'resolved', same: true }, results);
+ });
+ });
+ }
+</script>
diff --git a/src/browser/tests/compression/decompression_stream.html b/src/browser/tests/compression/decompression_stream.html
new file mode 100644
index 000000000..5239bbded
--- /dev/null
+++ b/src/browser/tests/compression/decompression_stream.html
@@ -0,0 +1,341 @@
+<!DOCTYPE html>
+<script src="../testing.js"></script>
+
+<script>
+ const FORMATS = ['gzip', 'deflate', 'deflate-raw'];
+
+ // 'hello', compressed by Python's zlib/gzip modules.
+ const HELLO = {
+ 'gzip': [31, 139, 8, 0, 0, 0, 0, 0, 2, 255, 203, 72, 205, 201, 201, 7, 0, 134, 166, 16, 54, 5, 0, 0, 0],
+ 'deflate': [120, 156, 203, 72, 205, 201, 201, 7, 0, 6, 44, 2, 21],
+ 'deflate-raw': [203, 72, 205, 201, 201, 7, 0],
+ };
+
+ // '', compressed.
+ const EMPTY = {
+ 'gzip': [31, 139, 8, 0, 0, 0, 0, 0, 2, 255, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+ 'deflate': [120, 156, 3, 0, 0, 0, 0, 1],
+ 'deflate-raw': [3, 0],
+ };
+
+ async function readAll(readable) {
+ return readRest(readable.getReader(), []);
+ }
+
+ // Reads `reader` to the end, after the `chunks` already read from it.
+ async function readRest(reader, chunks) {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ chunks.push(value);
+ }
+ return concat(chunks);
+ }
+
+ function concat(chunks) {
+ let length = 0;
+ for (const chunk of chunks) {
+ if (!(chunk instanceof Uint8Array)) throw new Error('chunk is not a Uint8Array');
+ // The spec only enqueues non-empty pieces.
+ if (chunk.length === 0) throw new Error('chunk is empty');
+ length += chunk.length;
+ }
+ const out = new Uint8Array(length);
+ let offset = 0;
+ for (const chunk of chunks) {
+ out.set(chunk, offset);
+ offset += chunk.length;
+ }
+ return out;
+ }
+
+ function equal(a, b) {
+ return a.length === b.length && a.every((byte, i) => byte === b[i]);
+ }
+
+ async function compress(format, bytes) {
+ const cs = new CompressionStream(format);
+ const writer = cs.writable.getWriter();
+ const output = readAll(cs.readable);
+ writer.write(bytes);
+ writer.close();
+ return output;
+ }
+
+ async function decompress(format, chunks) {
+ const ds = new DecompressionStream(format);
+ const writer = ds.writable.getWriter();
+ const output = readAll(ds.readable);
+ for (const chunk of chunks) writer.write(new Uint8Array(chunk));
+ writer.close();
+ return new TextDecoder().decode(await output);
+ }
+
+ async function rejection(promise) {
+ try {
+ await promise;
+ } catch (e) {
+ return e.name;
+ }
+ return 'resolved';
+ }
+
+ // Writes each chunk and closes. Returns how each write, the close, and a
+ // read after all of them settled.
+ async function outcome(format, chunks) {
+ const ds = new DecompressionStream(format);
+ const writer = ds.writable.getWriter();
+ const writes = [];
+ for (const chunk of chunks) writes.push(rejection(writer.write(new Uint8Array(chunk))));
+ const close = rejection(writer.close());
+ return {
+ writes: await Promise.all(writes),
+ close: await close,
+ read: await rejection(ds.readable.getReader().read()),
+ };
+ }
+</script>
+
+<script id=constructor>
+ {
+ for (const format of FORMATS) {
+ const ds = new DecompressionStream(format);
+ testing.expectTrue(ds.readable instanceof ReadableStream);
+ testing.expectTrue(ds.writable instanceof WritableStream);
+ }
+
+ for (const format of ['GZIP', 'zlib', '', undefined]) {
+ testing.withError((err) => testing.expectEqual('TypeError', err.name), () => new DecompressionStream(format));
+ }
+ testing.withError((err) => testing.expectEqual('TypeError', err.name), () => new DecompressionStream());
+ }
+</script>
+
+<script id=decompress type=module>
+ const state = await testing.async();
+
+ const results = {};
+ for (const format of FORMATS) {
+ results[format] = {
+ whole: await decompress(format, [HELLO[format]]),
+ // one byte per chunk
+ split: await decompress(format, HELLO[format].map((b) => [b])),
+ // empty chunks are fine anywhere, including after the end
+ padded: await decompress(format, [[], HELLO[format], [], []]),
+ empty: await decompress(format, [EMPTY[format]]),
+ };
+ }
+
+ state.resolve();
+ await state.done(() => {
+ for (const format of FORMATS) {
+ testing.expectEqual({ whole: 'hello', split: 'hello', padded: 'hello', empty: '' }, results[format]);
+ }
+ });
+</script>
+
+<script id=no_input type=module>
+ const state = await testing.async();
+
+ const results = {};
+ for (const format of FORMATS) {
+ results[format] = await outcome(format, []);
+ }
+
+ state.resolve();
+ await state.done(() => {
+ for (const format of FORMATS) {
+ testing.expectEqual({ writes: [], close: 'TypeError', read: 'TypeError' }, results[format]);
+ }
+ });
+</script>
+
+<script id=truncated type=module>
+ const state = await testing.async();
+
+ const results = {};
+ for (const format of FORMATS) {
+ results[format] = await outcome(format, [HELLO[format].slice(0, -1)]);
+ }
+
+ state.resolve();
+ await state.done(() => {
+ for (const format of FORMATS) {
+ testing.expectEqual({ writes: ['resolved'], close: 'TypeError', read: 'TypeError' }, results[format]);
+ }
+ });
+</script>
+
+<script id=corrupt type=module>
+ const state = await testing.async();
+
+ const results = {};
+ for (const format of ['gzip', 'deflate']) {
+ results[format] = await outcome(format, [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]);
+ }
+
+ // Flip a byte of the deflate data: the CRC32 / ADLER32 check fails.
+ const bad_gzip = HELLO['gzip'].slice();
+ bad_gzip[17] ^= 1;
+ results['gzip crc'] = await outcome('gzip', [bad_gzip]);
+ const bad_deflate = HELLO['deflate'].slice();
+ bad_deflate[12] ^= 1;
+ results['deflate adler'] = await outcome('deflate', [bad_deflate]);
+
+ // FDICT (a preset dictionary) isn't supported.
+ results['fdict'] = await outcome('deflate', [[120, 187, 6, 44, 2, 21, 203, 0, 17, 0, 6, 44, 2, 21]]);
+
+ // A reserved block type
+ results['deflate-raw'] = await outcome('deflate-raw', [[7, 0]]);
+
+ state.resolve();
+ await state.done(() => {
+ for (const result of Object.values(results)) {
+ testing.expectEqual({ writes: ['TypeError'], close: 'TypeError', read: 'TypeError' }, result);
+ }
+ });
+</script>
+
+<script id=trailing_data type=module>
+ const state = await testing.async();
+
+ const results = {};
+ for (const format of FORMATS) {
+ results[format] = await outcome(format, [HELLO[format].concat([0])]);
+ results[format + ' later write'] = await outcome(format, [HELLO[format], [0]]);
+ }
+
+ // A second gzip member is trailing data too.
+ results['gzip members'] = await outcome('gzip', [HELLO['gzip'].concat(HELLO['gzip'])]);
+
+ state.resolve();
+ await state.done(() => {
+ for (const format of FORMATS) {
+ testing.expectEqual({ writes: ['TypeError'], close: 'TypeError', read: 'TypeError' }, results[format]);
+ const later = results[format + ' later write'];
+ testing.expectEqual({ writes: ['resolved', 'TypeError'], close: 'TypeError', read: 'TypeError' }, later);
+ }
+ testing.expectEqual({ writes: ['TypeError'], close: 'TypeError', read: 'TypeError' }, results['gzip members']);
+ });
+</script>
+
+<script id=bad_chunks type=module>
+ const state = await testing.async();
+
+ const results = [];
+ for (const chunk of ['hello', undefined, null, 42, {}, [1, 2, 3]]) {
+ const ds = new DecompressionStream('gzip');
+ const writer = ds.writable.getWriter();
+ const reader = ds.readable.getReader();
+ const pending_read = rejection(reader.read());
+ results.push([await rejection(writer.write(chunk)), await pending_read, await rejection(reader.read())]);
+ }
+
+ state.resolve();
+ await state.done(() => {
+ for (const result of results) {
+ testing.expectEqual(['TypeError', 'TypeError', 'TypeError'], result);
+ }
+ });
+</script>
+
+<script id=large_output type=module>
+ const state = await testing.async();
+
+ // A few hundred bytes that inflate to 1 MiB within a single write.
+ const zeros = new Uint8Array(1024 * 1024);
+ const results = {};
+ for (const format of FORMATS) {
+ const compressed = await compress(format, zeros);
+ const ds = new DecompressionStream(format);
+ const writer = ds.writable.getWriter();
+ const output = readAll(ds.readable);
+ const write = rejection(writer.write(compressed));
+ writer.close();
+ results[format] = { small: compressed.length < 4096, write: await write, same: equal(await output, zeros) };
+ }
+
+ state.resolve();
+ await state.done(() => {
+ for (const format of FORMATS) {
+ testing.expectEqual({ small: true, write: 'resolved', same: true }, results[format]);
+ }
+ });
+</script>
+
+<script id=pending_reads type=module>
+ const state = await testing.async();
+
+ // Reads waiting before a write that produces many chunks get the first
+ // ones, in order; the rest stay queued for later reads.
+ const input = new Uint8Array(256 * 1024).map((_, i) => i % 251);
+ const compressed = await compress('deflate', input);
+
+ const ds = new DecompressionStream('deflate');
+ const writer = ds.writable.getWriter();
+ const reader = ds.readable.getReader();
+ const reads = [reader.read(), reader.read(), reader.read()];
+ writer.write(compressed);
+ writer.close();
+
+ const first = await Promise.all(reads);
+ const output = await readRest(reader, first.map((r) => r.value));
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual([false, false, false], first.map((r) => r.done));
+ testing.expectEqual(true, equal(output, input));
+ });
+</script>
+
+<script id=extra_input_output type=module>
+ const state = await testing.async();
+
+ // What was inflated before the trailing junk still reaches a waiting read;
+ // only then does the stream error.
+ const results = {};
+ for (const format of FORMATS) {
+ const ds = new DecompressionStream(format);
+ const writer = ds.writable.getWriter();
+ const reader = ds.readable.getReader();
+ const read = reader.read();
+ const write = rejection(writer.write(new Uint8Array(HELLO[format].concat([0]))));
+ const { value } = await read;
+ results[format] = {
+ value: new TextDecoder().decode(value),
+ write: await write,
+ next: await rejection(reader.read()),
+ };
+ }
+
+ state.resolve();
+ await state.done(() => {
+ for (const format of FORMATS) {
+ testing.expectEqual({ value: 'hello', write: 'TypeError', next: 'TypeError' }, results[format]);
+ }
+ });
+</script>
+
+<script id=format_mismatch type=module>
+ const state = await testing.async();
+
+ // Each format only accepts its own framing; nothing is auto-detected.
+ const results = {};
+ for (const [format, data] of [
+ ['deflate', 'gzip'],
+ ['gzip', 'deflate'],
+ ['deflate', 'deflate-raw'],
+ ['gzip', 'deflate-raw'],
+ ['deflate-raw', 'gzip'],
+ ['deflate-raw', 'deflate'],
+ ]) {
+ results[`${data} as ${format}`] = await outcome(format, [HELLO[data]]);
+ }
+
+ state.resolve();
+ await state.done(() => {
+ for (const result of Object.values(results)) {
+ testing.expectEqual({ writes: ['TypeError'], close: 'TypeError', read: 'TypeError' }, result);
+ }
+ });
+</script>