Skip to content

Lightpanda: the headless browser designed for AI and automation

TextDecoderStream: decode any encoding, with state across chunks

orezraeyf506c65efdc3
diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zig
index e3e538e87..33c4211d7 100644
--- a/src/browser/parser/html5ever.zig
+++ b/src/browser/parser/html5ever.zig
@@ -350,6 +350,13 @@ pub extern "c" fn encoding_decoder_decode(
pub extern "c" fn encoding_decoder_free(decoder: *anyopaque) void;
+pub extern "c" fn encoding_decoder_max_utf8_buffer_length(decoder: *anyopaque, input_len: usize) usize;
+
+// A decoder in caller-owned memory, never freed (see lib.rs).
+pub extern "c" fn encoding_decoder_size() usize;
+pub extern "c" fn encoding_decoder_align() usize;
+pub extern "c" fn encoding_decoder_init(handle: *anyopaque, out: *anyopaque) u8;
+
// Encoding API (UTF-8 to legacy encoding with NCR fallback)
const EncodeResult = extern struct {
status: u8,
diff --git a/src/browser/tests/encoding/text_decoder.html b/src/browser/tests/encoding/text_decoder.html
index 914f26b3d..47fb2277d 100644
--- a/src/browser/tests/encoding/text_decoder.html
+++ b/src/browser/tests/encoding/text_decoder.html
@@ -170,3 +170,19 @@
testing.expectEqual('B', d.decode(new Uint8Array([66])));
}
</script>
+
+<script id=stream_pending_bytes>
+ {
+ // Bytes held from the last call can decode to more than the new input
+ // alone would: here one byte gives two U+FFFD.
+ let d = new TextDecoder();
+ testing.expectEqual('', d.decode(new Uint8Array([0xF0]), { stream: true }));
+ testing.expectEqual('\uFFFD\uFFFD', d.decode(new Uint8Array([0x8F]), { stream: true }));
+ testing.expectEqual('\uFFFD', d.decode(new Uint8Array([0x92]), { stream: true }));
+ testing.expectEqual('', d.decode());
+
+ testing.expectEqual('', d.decode(new Uint8Array([0xF0, 0x9F]), { stream: true }));
+ testing.expectEqual('\uFFFDA', d.decode(new Uint8Array([0x41]), { stream: true }));
+ testing.expectEqual('', d.decode());
+ }
+</script>
diff --git a/src/browser/tests/streams/text_decoder_stream.html b/src/browser/tests/streams/text_decoder_stream.html
index cf38a2fdc..be3901f28 100644
--- a/src/browser/tests/streams/text_decoder_stream.html
+++ b/src/browser/tests/streams/text_decoder_stream.html
@@ -27,15 +27,17 @@
}
</script>
-<script id=text_decoder_stream_invalid_label>
+<script id=text_decoder_stream_labels>
{
- let errorThrown = false;
- try {
- new TextDecoderStream('windows-1252');
- } catch (e) {
- errorThrown = true;
- }
- testing.expectEqual(true, errorThrown);
+ // Any encoding a TextDecoder supports, with its lowercase name.
+ testing.expectEqual('windows-1252', new TextDecoderStream('latin1').encoding);
+ testing.expectEqual('utf-16le', new TextDecoderStream('UTF-16').encoding);
+ testing.expectEqual('shift_jis', new TextDecoderStream('sjis').encoding);
+ testing.expectEqual('gbk', new TextDecoderStream('gb2312').encoding);
+
+ testing.expectError('RangeError', () => new TextDecoderStream('nope'));
+ // A label of the replacement encoding can't decode.
+ testing.expectError('RangeError', () => new TextDecoderStream('iso-2022-kr'));
}
</script>
@@ -80,3 +82,85 @@
testing.expectEqual(true, result2.done);
})();
</script>
+
+
+<script id=text_decoder_stream_chunks type=module>
+ const state = await testing.async();
+
+ // Writes the chunks, then closes: the strings read, or the error's name.
+ async function decode(chunks, label, opts) {
+ const tds = new TextDecoderStream(label, opts);
+ const writer = tds.writable.getWriter();
+ const reader = tds.readable.getReader();
+ for (const chunk of chunks) {
+ writer.write(chunk).catch(() => {});
+ }
+ writer.close().catch(() => {});
+ const out = [];
+ try {
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) return out;
+ out.push(value);
+ }
+ } catch (e) {
+ return e.name;
+ }
+ }
+ const bytes = (...b) => new Uint8Array(b);
+
+ const results = {
+ // A character split across chunks decodes once complete.
+ split: await decode([bytes(0xE2, 0x82), bytes(0xAC, 0x61)]),
+ // So does a BOM, which is stripped unless ignoreBOM.
+ split_bom: await decode([bytes(0xEF), bytes(0xBB, 0xBF, 0x61)]),
+ kept_bom: await decode([bytes(0xEF, 0xBB, 0xBF, 0x61)], 'utf-8', { ignoreBOM: true }),
+ // Only a leading BOM.
+ second_bom: await decode([bytes(0x61), bytes(0xEF, 0xBB, 0xBF)]),
+ utf16: await decode([bytes(0xFF, 0xFE, 0x61, 0x00, 0x62), bytes(0x00)], 'utf-16le'),
+ utf16be: await decode([bytes(0x00, 0x61, 0x00), bytes(0x62)], 'utf-16be'),
+ windows1252: await decode([bytes(0x80, 0x41)], 'windows-1252'),
+ shift_jis: await decode([bytes(0x82), bytes(0xA0)], 'shift_jis'),
+ // An ArrayBuffer and any view, of a SharedArrayBuffer too.
+ sources: await decode([bytes(0x61).buffer, new DataView(bytes(0x62).buffer), new Uint16Array([0x6463])]),
+ shared: await decode([new Uint8Array(new SharedArrayBuffer(1)).fill(0x61)]),
+ // What's left of a split character at the end is U+FFFD.
+ truncated: await decode([bytes(0x61, 0xE2, 0x82)]),
+ invalid: await decode([bytes(0x61, 0xFF, 0x62)]),
+ fatal_invalid: await decode([bytes(0x61), bytes(0xFF)], 'utf-8', { fatal: true }),
+ fatal_truncated: await decode([bytes(0xE2, 0x82)], 'utf-8', { fatal: true }),
+ fatal_split: await decode([bytes(0xE2), bytes(0x82, 0xAC)], 'utf-8', { fatal: true }),
+ string_chunk: await decode(['a']),
+ number_chunk: await decode([bytes(0x61), 1]),
+ };
+
+ // A rejected write rejects the next ones too, with the same TypeError.
+ const tds = new TextDecoderStream('utf-8', { fatal: true });
+ const writer = tds.writable.getWriter();
+ tds.readable.getReader().read().catch(() => {});
+ const first = await writer.write(bytes(0xFF)).catch((e) => e);
+ const second = await writer.write(bytes(0x61)).catch((e) => e);
+
+ state.resolve();
+ await state.done(() => {
+ testing.expectEqual(['\u20ACa'], results.split);
+ testing.expectEqual(['a'], results.split_bom);
+ testing.expectEqual(['\uFEFFa'], results.kept_bom);
+ testing.expectEqual(['a', '\uFEFF'], results.second_bom);
+ testing.expectEqual(['a', 'b'], results.utf16);
+ testing.expectEqual(['a', 'b'], results.utf16be);
+ testing.expectEqual(['\u20ACA'], results.windows1252);
+ testing.expectEqual(['\u3042'], results.shift_jis);
+ testing.expectEqual(['a', 'b', 'cd'], results.sources);
+ testing.expectEqual(['a'], results.shared);
+ testing.expectEqual(['a', '\uFFFD'], results.truncated);
+ testing.expectEqual(['a\uFFFDb'], results.invalid);
+ testing.expectEqual('TypeError', results.fatal_invalid);
+ testing.expectEqual('TypeError', results.fatal_truncated);
+ testing.expectEqual(['\u20AC'], results.fatal_split);
+ testing.expectEqual('TypeError', results.string_chunk);
+ testing.expectEqual('TypeError', results.number_chunk);
+ testing.expectEqual('TypeError', first.name);
+ testing.expectEqual(true, first === second);
+ });
+</script>
diff --git a/src/browser/webapi/encoding/TextDecoder.zig b/src/browser/webapi/encoding/TextDecoder.zig
index a99800077..e052bdff5 100644
--- a/src/browser/webapi/encoding/TextDecoder.zig
+++ b/src/browser/webapi/encoding/TextDecoder.zig
@@ -41,19 +41,19 @@ const InitOpts = struct {
ignoreBOM: bool = false,
};
-pub fn init(label_: ?[]const u8, opts_: ?InitOpts, page: *Page) !*TextDecoder {
- const label = label_ orelse "utf-8";
-
- const info = html5ever.encoding_for_label(label.ptr, label.len);
- if (!info.isValid()) {
+/// The encoding a TextDecoder or TextDecoderStream label names: a RangeError
+/// for an unknown label, and for "replacement", which can't decode.
+pub fn encodingForLabel(label_: ?[]const u8) !html5ever.EncodingInfo {
+ const info = html5ever.EncodingInfo.forLabel(label_ orelse "utf-8") orelse return error.RangeError;
+ if (std.mem.eql(u8, info.name(), "replacement")) {
return error.RangeError;
}
+ return info;
+}
- // Check for "replacement" encoding - it's not usable for decoding per spec
+pub fn init(label_: ?[]const u8, opts_: ?InitOpts, page: *Page) !*TextDecoder {
+ const info = try encodingForLabel(label_);
const enc_name = info.name();
- if (std.mem.eql(u8, enc_name, "replacement")) {
- return error.RangeError;
- }
// Only ever holds the decoder itself and the lazily-lowercased encoding
// name; decode output comes from the caller's call_arena.
@@ -139,15 +139,39 @@ pub fn decode(self: *TextDecoder, input_: ?[]const u8, opts_: ?DecodeOpts, exec:
}
fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, decoder: *anyopaque, is_last: bool) ![]const u8 {
+ return decodeChunk(arena, .{
+ .encoding_name = self._encoding_name,
+ .fatal = self._fatal,
+ .ignore_bom = self._ignore_bom,
+ .bom_seen = &self._bom_seen,
+ }, decoder, input, is_last) catch |err| {
+ if (err == error.TypeError) {
+ // Reset decoder on error
+ html5ever.encoding_decoder_free(decoder);
+ self._decoder = null;
+ self._bom_seen = false;
+ }
+ return err;
+ };
+}
+
+pub const ChunkOpts = struct {
+ encoding_name: []const u8,
+ fatal: bool,
+ ignore_bom: bool,
+ // Whether output has come out yet, for the BOM.
+ bom_seen: *bool,
+};
+
+/// Decodes `input` with the streaming `decoder` into `arena`, as TextDecoder
+/// and TextDecoderStream do. A decoding error in fatal mode is a TypeError.
+pub fn decodeChunk(arena: std.mem.Allocator, opts: ChunkOpts, decoder: *anyopaque, input: []const u8, is_last: bool) ![]const u8 {
if (input.len == 0 and !is_last) {
return "";
}
- // Calculate max output size (add extra for potential buffered bytes when finishing)
- const max_out = html5ever.encoding_max_utf8_buffer_length(
- self._encoding_handle,
- if (input.len == 0) 4 else input.len,
- );
+ // Counts the bytes the decoder holds from earlier input too.
+ const max_out = html5ever.encoding_decoder_max_utf8_buffer_length(decoder, input.len);
if (max_out == 0) {
return "";
@@ -165,12 +189,7 @@ fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, deco
@intFromBool(is_last),
);
- // Handle errors in fatal mode
- if (self._fatal and result.hadErrors()) {
- // Reset decoder on error
- html5ever.encoding_decoder_free(decoder);
- self._decoder = null;
- self._bom_seen = false;
+ if (opts.fatal and result.hadErrors()) {
return error.TypeError;
}
@@ -179,11 +198,11 @@ fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, deco
// The decoder does no BOM sniffing, so a BOM of this encoding is a
// leading U+FEFF in the first output; other bytes stay as decoded.
// Only UTF-8 and UTF-16 have a BOM: in gb18030, U+FEFF is a character.
- if (!self._bom_seen and decoded.len > 0) {
- if (self._ignore_bom == false and hasBom(self._encoding_name)) {
+ if (!opts.bom_seen.* and decoded.len > 0) {
+ if (opts.ignore_bom == false and hasBom(opts.encoding_name)) {
decoded = stripBom(decoded);
}
- self._bom_seen = true;
+ opts.bom_seen.* = true;
}
return decoded;
diff --git a/src/browser/webapi/encoding/TextDecoderStream.zig b/src/browser/webapi/encoding/TextDecoderStream.zig
index e71567768..fe2eeb118 100644
--- a/src/browser/webapi/encoding/TextDecoderStream.zig
+++ b/src/browser/webapi/encoding/TextDecoderStream.zig
@@ -19,59 +19,106 @@
const std = @import("std");
const js = @import("../../js/js.zig");
+const html5ever = @import("../../parser/html5ever.zig");
+
+const TextDecoder = @import("TextDecoder.zig");
const ReadableStream = @import("../streams/ReadableStream.zig");
const WritableStream = @import("../streams/WritableStream.zig");
const TransformStream = @import("../streams/TransformStream.zig");
const Execution = js.Execution;
+// https://encoding.spec.whatwg.org/#interface-textdecoderstream
const TextDecoderStream = @This();
_transform: *TransformStream,
-_fatal: bool,
-_ignore_bom: bool,
-
-const Label = enum {
- utf8,
- @"utf-8",
- @"unicode-1-1-utf-8",
-};
+_decoder: *Decoder,
+_encoding: []const u8,
const InitOpts = struct {
fatal: bool = false,
ignoreBOM: bool = false,
};
-pub fn init(label_: ?[]const u8, opts_: ?InitOpts, exec: *const Execution) !TextDecoderStream {
- if (label_) |label| {
- _ = std.meta.stringToEnum(Label, label) orelse return error.RangeError;
- }
-
+pub fn init(label: ?[]const u8, opts_: ?InitOpts, exec: *const Execution) !TextDecoderStream {
+ const info = try TextDecoder.encodingForLabel(label);
const opts = opts_ orelse InitOpts{};
- const transform = try TransformStream.initWithZigTransformer(.{ .text_decoder = .{ .ignore_bom = opts.ignoreBOM } }, exec);
+
+ const decoder = try Decoder.init(info, opts.fatal, opts.ignoreBOM, exec);
+ const transform = try TransformStream.initWithZigTransformer(.{ .text_decoder = decoder }, exec);
return .{
._transform = transform,
- ._fatal = opts.fatal,
- ._ignore_bom = opts.ignoreBOM,
+ ._decoder = decoder,
+ ._encoding = try std.ascii.allocLowerString(exec.page_arena, info.name()),
};
}
-pub fn decodeTransform(controller: *TransformStream.DefaultController, chunk: js.Value, ignoreBOM: bool) !void {
- // chunk should be a Uint8Array; decode it as UTF-8 string
- const typed_array = try chunk.toZig(js.TypedArray(u8));
- var input = typed_array.values;
+/// The stream's decoder, in the frame's memory like the TransformStream that
+/// uses it: encoding_rs's state is built in place and holds no heap memory,
+/// so a stream that's never closed leaks nothing.
+pub const Decoder = struct {
+ exec: *const Execution,
+ encoding_name: []const u8,
+ fatal: bool,
+ ignore_bom: bool,
+ bom_seen: bool = false,
+ state: *anyopaque,
+
+ fn init(info: html5ever.EncodingInfo, fatal: bool, ignore_bom: bool, exec: *const Execution) !*Decoder {
+ const size = html5ever.encoding_decoder_size();
+ const alignment = html5ever.encoding_decoder_align();
+ std.debug.assert(alignment <= 16);
+ const state = try exec.page_arena.alignedAlloc(u8, .@"16", size);
+ // Only fails for a null pointer.
+ const ok = html5ever.encoding_decoder_init(info.handle.?, state.ptr);
+ std.debug.assert(ok == 1);
+ return exec._factory.create(Decoder{
+ .exec = exec,
+ .encoding_name = info.name(),
+ .fatal = fatal,
+ .ignore_bom = ignore_bom,
+ .state = state.ptr,
+ });
+ }
- // Strip UTF-8 BOM if present
- if (ignoreBOM == false and std.mem.startsWith(u8, input, &.{ 0xEF, 0xBB, 0xBF })) {
- input = input[3..];
+ /// Each chunk must be a BufferSource, shared or not; its bytes decode with what the last
+ /// chunk left of a split sequence.
+ pub fn transform(self: *Decoder, controller: *TransformStream.DefaultController, chunk: js.Value) !void {
+ const input = chunk.toZig(js.AllowSharedBufferSource) catch {
+ return fail(controller, "The chunk is not an ArrayBuffer or ArrayBufferView");
+ };
+ return self.decode(controller, input.bytes, false);
}
- // Per spec, empty chunks produce no output
- if (input.len == 0) return;
+ /// What a split sequence left at the end decodes now: U+FFFD, or a
+ /// TypeError in fatal mode.
+ pub fn flush(self: *Decoder, controller: *TransformStream.DefaultController) !void {
+ return self.decode(controller, "", true);
+ }
- try controller.enqueue(.{ .string = input });
-}
+ // The output only lives for the call: enqueue copies it into a JS string.
+ fn decode(self: *Decoder, controller: *TransformStream.DefaultController, input: []const u8, is_last: bool) !void {
+ const decoded = TextDecoder.decodeChunk(self.exec.call_arena, .{
+ .encoding_name = self.encoding_name,
+ .fatal = self.fatal,
+ .ignore_bom = self.ignore_bom,
+ .bom_seen = &self.bom_seen,
+ }, self.state, input, is_last) catch |err| switch (err) {
+ error.TypeError => return fail(controller, "The encoded data is not valid"),
+ else => return err,
+ };
+ // Per spec, a chunk that decodes to nothing produces no output.
+ if (decoded.len > 0) {
+ try controller.enqueue(.{ .string = decoded });
+ }
+ }
+
+ fn fail(controller: *TransformStream.DefaultController, message: []const u8) !void {
+ try controller.typeError(message);
+ return error.StreamError;
+ }
+};
pub fn getReadable(self: *const TextDecoderStream) *ReadableStream {
return self._transform.getReadable();
@@ -81,12 +128,16 @@ pub fn getWritable(self: *const TextDecoderStream) *WritableStream {
return self._transform.getWritable();
}
+fn getEncoding(self: *const TextDecoderStream) []const u8 {
+ return self._encoding;
+}
+
fn getFatal(self: *const TextDecoderStream) bool {
- return self._fatal;
+ return self._decoder.fatal;
}
fn getIgnoreBOM(self: *const TextDecoderStream) bool {
- return self._ignore_bom;
+ return self._decoder.ignore_bom;
}
pub const JsApi = struct {
@@ -98,7 +149,7 @@ pub const JsApi = struct {
};
pub const constructor = bridge.constructor(TextDecoderStream.init, .{});
- pub const encoding = bridge.property("utf-8", .{ .template = false });
+ pub const encoding = bridge.accessor(TextDecoderStream.getEncoding, null, .{});
pub const readable = bridge.accessor(TextDecoderStream.getReadable, null, .{});
pub const writable = bridge.accessor(TextDecoderStream.getWritable, null, .{});
pub const fatal = bridge.accessor(TextDecoderStream.getFatal, null, .{});
diff --git a/src/browser/webapi/streams/TransformStream.zig b/src/browser/webapi/streams/TransformStream.zig
index cc895e957..53165eaa5 100644
--- a/src/browser/webapi/streams/TransformStream.zig
+++ b/src/browser/webapi/streams/TransformStream.zig
@@ -34,22 +34,21 @@ pub const DefaultController = TransformStreamDefaultController;
/// The transformers implemented in Zig.
pub const ZigTransformer = union(enum) {
text_encoder,
- text_decoder: struct { ignore_bom: bool },
+ text_decoder: *TextDecoderStream.Decoder,
compressor: *compress.Compressor,
decompressor: *compress.Decompressor,
fn transform(self: ZigTransformer, controller: *TransformStreamDefaultController, chunk: js.Value) !void {
return switch (self) {
.text_encoder => TextEncoderStream.encodeTransform(controller, chunk),
- .text_decoder => |opts| TextDecoderStream.decodeTransform(controller, chunk, opts.ignore_bom),
- inline .compressor, .decompressor => |t| t.transform(controller, chunk),
+ inline .text_decoder, .compressor, .decompressor => |t| t.transform(controller, chunk),
};
}
fn flush(self: ZigTransformer, controller: *TransformStreamDefaultController) !void {
return switch (self) {
- .text_encoder, .text_decoder => {},
- inline .compressor, .decompressor => |t| t.flush(controller),
+ .text_encoder => {},
+ inline .text_decoder, .compressor, .decompressor => |t| t.flush(controller),
};
}
};
diff --git a/src/rust/html5ever/lib.rs b/src/rust/html5ever/lib.rs
index 1340b4894..ce259ac4a 100644
--- a/src/rust/html5ever/lib.rs
+++ b/src/rust/html5ever/lib.rs
@@ -289,6 +289,44 @@ pub extern "C" fn encoding_decoder_new(handle: *const c_void) -> *mut c_void {
Box::into_raw(decoder) as *mut c_void
}
+/// The size and alignment of a streaming decoder, for `encoding_decoder_init`.
+#[no_mangle]
+pub extern "C" fn encoding_decoder_size() -> usize {
+ std::mem::size_of::<Decoder>()
+}
+
+#[no_mangle]
+pub extern "C" fn encoding_decoder_align() -> usize {
+ std::mem::align_of::<Decoder>()
+}
+
+/// Like `encoding_decoder_new`, but in `out`, which the caller owns: at least
+/// `encoding_decoder_size()` bytes aligned to `encoding_decoder_align()`. A
+/// Decoder holds no heap memory, so it's never freed.
+#[no_mangle]
+pub extern "C" fn encoding_decoder_init(handle: *const c_void, out: *mut c_void) -> u8 {
+ if handle.is_null() || out.is_null() {
+ return 0;
+ }
+ let encoding: &'static Encoding = unsafe { &*(handle as *const Encoding) };
+ unsafe { std::ptr::write(out as *mut Decoder, encoding.new_decoder_without_bom_handling()) };
+ 1
+}
+
+/// The most UTF-8 a streaming decoder can write for `input_len` more bytes,
+/// counting what it holds from earlier input.
+#[no_mangle]
+pub extern "C" fn encoding_decoder_max_utf8_buffer_length(
+ decoder_ptr: *const c_void,
+ input_len: usize,
+) -> usize {
+ if decoder_ptr.is_null() {
+ return 0;
+ }
+ let decoder: &Decoder = unsafe { &*(decoder_ptr as *const Decoder) };
+ decoder.max_utf8_buffer_length(input_len).unwrap_or(0)
+}
+
/// Decode using a streaming decoder (maintains state for incomplete sequences)
#[no_mangle]
pub extern "C" fn encoding_decoder_decode(