Lightpanda: the headless browser designed for AI and automation
TextDecoderStream: decode any encoding, with state across chunks
orezraeyf506c65efdc3
diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zigindex e3e538e87..33c4211d7 100644--- a/src/browser/parser/html5ever.zig+++ b/src/browser/parser/html5ever.zig@@ -350,6 +350,13 @@ pub extern "c" fn encoding_decoder_decode(pub extern "c" fn encoding_decoder_free(decoder: *anyopaque) void;+pub extern "c" fn encoding_decoder_max_utf8_buffer_length(decoder: *anyopaque, input_len: usize) usize;++// A decoder in caller-owned memory, never freed (see lib.rs).+pub extern "c" fn encoding_decoder_size() usize;+pub extern "c" fn encoding_decoder_align() usize;+pub extern "c" fn encoding_decoder_init(handle: *anyopaque, out: *anyopaque) u8;+// Encoding API (UTF-8 to legacy encoding with NCR fallback)const EncodeResult = extern struct {status: u8,diff --git a/src/browser/tests/encoding/text_decoder.html b/src/browser/tests/encoding/text_decoder.htmlindex 914f26b3d..47fb2277d 100644--- a/src/browser/tests/encoding/text_decoder.html+++ b/src/browser/tests/encoding/text_decoder.html@@ -170,3 +170,19 @@testing.expectEqual('B', d.decode(new Uint8Array([66])));}</script>++<script id=stream_pending_bytes>+ {+ // Bytes held from the last call can decode to more than the new input+ // alone would: here one byte gives two U+FFFD.+ let d = new TextDecoder();+ testing.expectEqual('', d.decode(new Uint8Array([0xF0]), { stream: true }));+ testing.expectEqual('\uFFFD\uFFFD', d.decode(new Uint8Array([0x8F]), { stream: true }));+ testing.expectEqual('\uFFFD', d.decode(new Uint8Array([0x92]), { stream: true }));+ testing.expectEqual('', d.decode());++ testing.expectEqual('', d.decode(new Uint8Array([0xF0, 0x9F]), { stream: true }));+ testing.expectEqual('\uFFFDA', d.decode(new Uint8Array([0x41]), { stream: true }));+ testing.expectEqual('', d.decode());+ }+</script>diff --git a/src/browser/tests/streams/text_decoder_stream.html b/src/browser/tests/streams/text_decoder_stream.htmlindex cf38a2fdc..be3901f28 100644--- a/src/browser/tests/streams/text_decoder_stream.html+++ b/src/browser/tests/streams/text_decoder_stream.html@@ -27,15 +27,17 @@}</script>-<script id=text_decoder_stream_invalid_label>+<script id=text_decoder_stream_labels>{- let errorThrown = false;- try {- new TextDecoderStream('windows-1252');- } catch (e) {- errorThrown = true;- }- testing.expectEqual(true, errorThrown);+ // Any encoding a TextDecoder supports, with its lowercase name.+ testing.expectEqual('windows-1252', new TextDecoderStream('latin1').encoding);+ testing.expectEqual('utf-16le', new TextDecoderStream('UTF-16').encoding);+ testing.expectEqual('shift_jis', new TextDecoderStream('sjis').encoding);+ testing.expectEqual('gbk', new TextDecoderStream('gb2312').encoding);++ testing.expectError('RangeError', () => new TextDecoderStream('nope'));+ // A label of the replacement encoding can't decode.+ testing.expectError('RangeError', () => new TextDecoderStream('iso-2022-kr'));}</script>@@ -80,3 +82,85 @@testing.expectEqual(true, result2.done);})();</script>+++<script id=text_decoder_stream_chunks type=module>+ const state = await testing.async();++ // Writes the chunks, then closes: the strings read, or the error's name.+ async function decode(chunks, label, opts) {+ const tds = new TextDecoderStream(label, opts);+ const writer = tds.writable.getWriter();+ const reader = tds.readable.getReader();+ for (const chunk of chunks) {+ writer.write(chunk).catch(() => {});+ }+ writer.close().catch(() => {});+ const out = [];+ try {+ for (;;) {+ const { done, value } = await reader.read();+ if (done) return out;+ out.push(value);+ }+ } catch (e) {+ return e.name;+ }+ }+ const bytes = (...b) => new Uint8Array(b);++ const results = {+ // A character split across chunks decodes once complete.+ split: await decode([bytes(0xE2, 0x82), bytes(0xAC, 0x61)]),+ // So does a BOM, which is stripped unless ignoreBOM.+ split_bom: await decode([bytes(0xEF), bytes(0xBB, 0xBF, 0x61)]),+ kept_bom: await decode([bytes(0xEF, 0xBB, 0xBF, 0x61)], 'utf-8', { ignoreBOM: true }),+ // Only a leading BOM.+ second_bom: await decode([bytes(0x61), bytes(0xEF, 0xBB, 0xBF)]),+ utf16: await decode([bytes(0xFF, 0xFE, 0x61, 0x00, 0x62), bytes(0x00)], 'utf-16le'),+ utf16be: await decode([bytes(0x00, 0x61, 0x00), bytes(0x62)], 'utf-16be'),+ windows1252: await decode([bytes(0x80, 0x41)], 'windows-1252'),+ shift_jis: await decode([bytes(0x82), bytes(0xA0)], 'shift_jis'),+ // An ArrayBuffer and any view, of a SharedArrayBuffer too.+ sources: await decode([bytes(0x61).buffer, new DataView(bytes(0x62).buffer), new Uint16Array([0x6463])]),+ shared: await decode([new Uint8Array(new SharedArrayBuffer(1)).fill(0x61)]),+ // What's left of a split character at the end is U+FFFD.+ truncated: await decode([bytes(0x61, 0xE2, 0x82)]),+ invalid: await decode([bytes(0x61, 0xFF, 0x62)]),+ fatal_invalid: await decode([bytes(0x61), bytes(0xFF)], 'utf-8', { fatal: true }),+ fatal_truncated: await decode([bytes(0xE2, 0x82)], 'utf-8', { fatal: true }),+ fatal_split: await decode([bytes(0xE2), bytes(0x82, 0xAC)], 'utf-8', { fatal: true }),+ string_chunk: await decode(['a']),+ number_chunk: await decode([bytes(0x61), 1]),+ };++ // A rejected write rejects the next ones too, with the same TypeError.+ const tds = new TextDecoderStream('utf-8', { fatal: true });+ const writer = tds.writable.getWriter();+ tds.readable.getReader().read().catch(() => {});+ const first = await writer.write(bytes(0xFF)).catch((e) => e);+ const second = await writer.write(bytes(0x61)).catch((e) => e);++ state.resolve();+ await state.done(() => {+ testing.expectEqual(['\u20ACa'], results.split);+ testing.expectEqual(['a'], results.split_bom);+ testing.expectEqual(['\uFEFFa'], results.kept_bom);+ testing.expectEqual(['a', '\uFEFF'], results.second_bom);+ testing.expectEqual(['a', 'b'], results.utf16);+ testing.expectEqual(['a', 'b'], results.utf16be);+ testing.expectEqual(['\u20ACA'], results.windows1252);+ testing.expectEqual(['\u3042'], results.shift_jis);+ testing.expectEqual(['a', 'b', 'cd'], results.sources);+ testing.expectEqual(['a'], results.shared);+ testing.expectEqual(['a', '\uFFFD'], results.truncated);+ testing.expectEqual(['a\uFFFDb'], results.invalid);+ testing.expectEqual('TypeError', results.fatal_invalid);+ testing.expectEqual('TypeError', results.fatal_truncated);+ testing.expectEqual(['\u20AC'], results.fatal_split);+ testing.expectEqual('TypeError', results.string_chunk);+ testing.expectEqual('TypeError', results.number_chunk);+ testing.expectEqual('TypeError', first.name);+ testing.expectEqual(true, first === second);+ });+</script>diff --git a/src/browser/webapi/encoding/TextDecoder.zig b/src/browser/webapi/encoding/TextDecoder.zigindex a99800077..e052bdff5 100644--- a/src/browser/webapi/encoding/TextDecoder.zig+++ b/src/browser/webapi/encoding/TextDecoder.zig@@ -41,19 +41,19 @@ const InitOpts = struct {ignoreBOM: bool = false,};-pub fn init(label_: ?[]const u8, opts_: ?InitOpts, page: *Page) !*TextDecoder {- const label = label_ orelse "utf-8";-- const info = html5ever.encoding_for_label(label.ptr, label.len);- if (!info.isValid()) {+/// The encoding a TextDecoder or TextDecoderStream label names: a RangeError+/// for an unknown label, and for "replacement", which can't decode.+pub fn encodingForLabel(label_: ?[]const u8) !html5ever.EncodingInfo {+ const info = html5ever.EncodingInfo.forLabel(label_ orelse "utf-8") orelse return error.RangeError;+ if (std.mem.eql(u8, info.name(), "replacement")) {return error.RangeError;}+ return info;+}- // Check for "replacement" encoding - it's not usable for decoding per spec+pub fn init(label_: ?[]const u8, opts_: ?InitOpts, page: *Page) !*TextDecoder {+ const info = try encodingForLabel(label_);const enc_name = info.name();- if (std.mem.eql(u8, enc_name, "replacement")) {- return error.RangeError;- }// Only ever holds the decoder itself and the lazily-lowercased encoding// name; decode output comes from the caller's call_arena.@@ -139,15 +139,39 @@ pub fn decode(self: *TextDecoder, input_: ?[]const u8, opts_: ?DecodeOpts, exec:}fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, decoder: *anyopaque, is_last: bool) ![]const u8 {+ return decodeChunk(arena, .{+ .encoding_name = self._encoding_name,+ .fatal = self._fatal,+ .ignore_bom = self._ignore_bom,+ .bom_seen = &self._bom_seen,+ }, decoder, input, is_last) catch |err| {+ if (err == error.TypeError) {+ // Reset decoder on error+ html5ever.encoding_decoder_free(decoder);+ self._decoder = null;+ self._bom_seen = false;+ }+ return err;+ };+}++pub const ChunkOpts = struct {+ encoding_name: []const u8,+ fatal: bool,+ ignore_bom: bool,+ // Whether output has come out yet, for the BOM.+ bom_seen: *bool,+};++/// Decodes `input` with the streaming `decoder` into `arena`, as TextDecoder+/// and TextDecoderStream do. A decoding error in fatal mode is a TypeError.+pub fn decodeChunk(arena: std.mem.Allocator, opts: ChunkOpts, decoder: *anyopaque, input: []const u8, is_last: bool) ![]const u8 {if (input.len == 0 and !is_last) {return "";}- // Calculate max output size (add extra for potential buffered bytes when finishing)- const max_out = html5ever.encoding_max_utf8_buffer_length(- self._encoding_handle,- if (input.len == 0) 4 else input.len,- );+ // Counts the bytes the decoder holds from earlier input too.+ const max_out = html5ever.encoding_decoder_max_utf8_buffer_length(decoder, input.len);if (max_out == 0) {return "";@@ -165,12 +189,7 @@ fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, deco@intFromBool(is_last),);- // Handle errors in fatal mode- if (self._fatal and result.hadErrors()) {- // Reset decoder on error- html5ever.encoding_decoder_free(decoder);- self._decoder = null;- self._bom_seen = false;+ if (opts.fatal and result.hadErrors()) {return error.TypeError;}@@ -179,11 +198,11 @@ fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, deco// The decoder does no BOM sniffing, so a BOM of this encoding is a// leading U+FEFF in the first output; other bytes stay as decoded.// Only UTF-8 and UTF-16 have a BOM: in gb18030, U+FEFF is a character.- if (!self._bom_seen and decoded.len > 0) {- if (self._ignore_bom == false and hasBom(self._encoding_name)) {+ if (!opts.bom_seen.* and decoded.len > 0) {+ if (opts.ignore_bom == false and hasBom(opts.encoding_name)) {decoded = stripBom(decoded);}- self._bom_seen = true;+ opts.bom_seen.* = true;}return decoded;diff --git a/src/browser/webapi/encoding/TextDecoderStream.zig b/src/browser/webapi/encoding/TextDecoderStream.zigindex e71567768..fe2eeb118 100644--- a/src/browser/webapi/encoding/TextDecoderStream.zig+++ b/src/browser/webapi/encoding/TextDecoderStream.zig@@ -19,59 +19,106 @@const std = @import("std");const js = @import("../../js/js.zig");+const html5ever = @import("../../parser/html5ever.zig");++const TextDecoder = @import("TextDecoder.zig");const ReadableStream = @import("../streams/ReadableStream.zig");const WritableStream = @import("../streams/WritableStream.zig");const TransformStream = @import("../streams/TransformStream.zig");const Execution = js.Execution;+// https://encoding.spec.whatwg.org/#interface-textdecoderstreamconst TextDecoderStream = @This();_transform: *TransformStream,-_fatal: bool,-_ignore_bom: bool,--const Label = enum {- utf8,- @"utf-8",- @"unicode-1-1-utf-8",-};+_decoder: *Decoder,+_encoding: []const u8,const InitOpts = struct {fatal: bool = false,ignoreBOM: bool = false,};-pub fn init(label_: ?[]const u8, opts_: ?InitOpts, exec: *const Execution) !TextDecoderStream {- if (label_) |label| {- _ = std.meta.stringToEnum(Label, label) orelse return error.RangeError;- }-+pub fn init(label: ?[]const u8, opts_: ?InitOpts, exec: *const Execution) !TextDecoderStream {+ const info = try TextDecoder.encodingForLabel(label);const opts = opts_ orelse InitOpts{};- const transform = try TransformStream.initWithZigTransformer(.{ .text_decoder = .{ .ignore_bom = opts.ignoreBOM } }, exec);++ const decoder = try Decoder.init(info, opts.fatal, opts.ignoreBOM, exec);+ const transform = try TransformStream.initWithZigTransformer(.{ .text_decoder = decoder }, exec);return .{._transform = transform,- ._fatal = opts.fatal,- ._ignore_bom = opts.ignoreBOM,+ ._decoder = decoder,+ ._encoding = try std.ascii.allocLowerString(exec.page_arena, info.name()),};}-pub fn decodeTransform(controller: *TransformStream.DefaultController, chunk: js.Value, ignoreBOM: bool) !void {- // chunk should be a Uint8Array; decode it as UTF-8 string- const typed_array = try chunk.toZig(js.TypedArray(u8));- var input = typed_array.values;+/// The stream's decoder, in the frame's memory like the TransformStream that+/// uses it: encoding_rs's state is built in place and holds no heap memory,+/// so a stream that's never closed leaks nothing.+pub const Decoder = struct {+ exec: *const Execution,+ encoding_name: []const u8,+ fatal: bool,+ ignore_bom: bool,+ bom_seen: bool = false,+ state: *anyopaque,++ fn init(info: html5ever.EncodingInfo, fatal: bool, ignore_bom: bool, exec: *const Execution) !*Decoder {+ const size = html5ever.encoding_decoder_size();+ const alignment = html5ever.encoding_decoder_align();+ std.debug.assert(alignment <= 16);+ const state = try exec.page_arena.alignedAlloc(u8, .@"16", size);+ // Only fails for a null pointer.+ const ok = html5ever.encoding_decoder_init(info.handle.?, state.ptr);+ std.debug.assert(ok == 1);+ return exec._factory.create(Decoder{+ .exec = exec,+ .encoding_name = info.name(),+ .fatal = fatal,+ .ignore_bom = ignore_bom,+ .state = state.ptr,+ });+ }- // Strip UTF-8 BOM if present- if (ignoreBOM == false and std.mem.startsWith(u8, input, &.{ 0xEF, 0xBB, 0xBF })) {- input = input[3..];+ /// Each chunk must be a BufferSource, shared or not; its bytes decode with what the last+ /// chunk left of a split sequence.+ pub fn transform(self: *Decoder, controller: *TransformStream.DefaultController, chunk: js.Value) !void {+ const input = chunk.toZig(js.AllowSharedBufferSource) catch {+ return fail(controller, "The chunk is not an ArrayBuffer or ArrayBufferView");+ };+ return self.decode(controller, input.bytes, false);}- // Per spec, empty chunks produce no output- if (input.len == 0) return;+ /// What a split sequence left at the end decodes now: U+FFFD, or a+ /// TypeError in fatal mode.+ pub fn flush(self: *Decoder, controller: *TransformStream.DefaultController) !void {+ return self.decode(controller, "", true);+ }- try controller.enqueue(.{ .string = input });-}+ // The output only lives for the call: enqueue copies it into a JS string.+ fn decode(self: *Decoder, controller: *TransformStream.DefaultController, input: []const u8, is_last: bool) !void {+ const decoded = TextDecoder.decodeChunk(self.exec.call_arena, .{+ .encoding_name = self.encoding_name,+ .fatal = self.fatal,+ .ignore_bom = self.ignore_bom,+ .bom_seen = &self.bom_seen,+ }, self.state, input, is_last) catch |err| switch (err) {+ error.TypeError => return fail(controller, "The encoded data is not valid"),+ else => return err,+ };+ // Per spec, a chunk that decodes to nothing produces no output.+ if (decoded.len > 0) {+ try controller.enqueue(.{ .string = decoded });+ }+ }++ fn fail(controller: *TransformStream.DefaultController, message: []const u8) !void {+ try controller.typeError(message);+ return error.StreamError;+ }+};pub fn getReadable(self: *const TextDecoderStream) *ReadableStream {return self._transform.getReadable();@@ -81,12 +128,16 @@ pub fn getWritable(self: *const TextDecoderStream) *WritableStream {return self._transform.getWritable();}+fn getEncoding(self: *const TextDecoderStream) []const u8 {+ return self._encoding;+}+fn getFatal(self: *const TextDecoderStream) bool {- return self._fatal;+ return self._decoder.fatal;}fn getIgnoreBOM(self: *const TextDecoderStream) bool {- return self._ignore_bom;+ return self._decoder.ignore_bom;}pub const JsApi = struct {@@ -98,7 +149,7 @@ pub const JsApi = struct {};pub const constructor = bridge.constructor(TextDecoderStream.init, .{});- pub const encoding = bridge.property("utf-8", .{ .template = false });+ pub const encoding = bridge.accessor(TextDecoderStream.getEncoding, null, .{});pub const readable = bridge.accessor(TextDecoderStream.getReadable, null, .{});pub const writable = bridge.accessor(TextDecoderStream.getWritable, null, .{});pub const fatal = bridge.accessor(TextDecoderStream.getFatal, null, .{});diff --git a/src/browser/webapi/streams/TransformStream.zig b/src/browser/webapi/streams/TransformStream.zigindex cc895e957..53165eaa5 100644--- a/src/browser/webapi/streams/TransformStream.zig+++ b/src/browser/webapi/streams/TransformStream.zig@@ -34,22 +34,21 @@ pub const DefaultController = TransformStreamDefaultController;/// The transformers implemented in Zig.pub const ZigTransformer = union(enum) {text_encoder,- text_decoder: struct { ignore_bom: bool },+ text_decoder: *TextDecoderStream.Decoder,compressor: *compress.Compressor,decompressor: *compress.Decompressor,fn transform(self: ZigTransformer, controller: *TransformStreamDefaultController, chunk: js.Value) !void {return switch (self) {.text_encoder => TextEncoderStream.encodeTransform(controller, chunk),- .text_decoder => |opts| TextDecoderStream.decodeTransform(controller, chunk, opts.ignore_bom),- inline .compressor, .decompressor => |t| t.transform(controller, chunk),+ inline .text_decoder, .compressor, .decompressor => |t| t.transform(controller, chunk),};}fn flush(self: ZigTransformer, controller: *TransformStreamDefaultController) !void {return switch (self) {- .text_encoder, .text_decoder => {},- inline .compressor, .decompressor => |t| t.flush(controller),+ .text_encoder => {},+ inline .text_decoder, .compressor, .decompressor => |t| t.flush(controller),};}};diff --git a/src/rust/html5ever/lib.rs b/src/rust/html5ever/lib.rsindex 1340b4894..ce259ac4a 100644--- a/src/rust/html5ever/lib.rs+++ b/src/rust/html5ever/lib.rs@@ -289,6 +289,44 @@ pub extern "C" fn encoding_decoder_new(handle: *const c_void) -> *mut c_void {Box::into_raw(decoder) as *mut c_void}+/// The size and alignment of a streaming decoder, for `encoding_decoder_init`.+#[no_mangle]+pub extern "C" fn encoding_decoder_size() -> usize {+ std::mem::size_of::<Decoder>()+}++#[no_mangle]+pub extern "C" fn encoding_decoder_align() -> usize {+ std::mem::align_of::<Decoder>()+}++/// Like `encoding_decoder_new`, but in `out`, which the caller owns: at least+/// `encoding_decoder_size()` bytes aligned to `encoding_decoder_align()`. A+/// Decoder holds no heap memory, so it's never freed.+#[no_mangle]+pub extern "C" fn encoding_decoder_init(handle: *const c_void, out: *mut c_void) -> u8 {+ if handle.is_null() || out.is_null() {+ return 0;+ }+ let encoding: &'static Encoding = unsafe { &*(handle as *const Encoding) };+ unsafe { std::ptr::write(out as *mut Decoder, encoding.new_decoder_without_bom_handling()) };+ 1+}++/// The most UTF-8 a streaming decoder can write for `input_len` more bytes,+/// counting what it holds from earlier input.+#[no_mangle]+pub extern "C" fn encoding_decoder_max_utf8_buffer_length(+ decoder_ptr: *const c_void,+ input_len: usize,+) -> usize {+ if decoder_ptr.is_null() {+ return 0;+ }+ let decoder: &Decoder = unsafe { &*(decoder_ptr as *const Decoder) };+ decoder.max_utf8_buffer_length(input_len).unwrap_or(0)+}+/// Decode using a streaming decoder (maintains state for incomplete sequences)#[no_mangle]pub extern "C" fn encoding_decoder_decode(