Lightpanda: the headless browser designed for AI and automation
webapi: stop TextDecoder from sniffing a BOM of another encoding
Jay Hemnani38703cacc055
diff --git a/src/browser/tests/encoding/text_decoder.html b/src/browser/tests/encoding/text_decoder.htmlindex 4f5574f40..219765bb4 100644--- a/src/browser/tests/encoding/text_decoder.html+++ b/src/browser/tests/encoding/text_decoder.html@@ -63,6 +63,20 @@testing.expectEqual('', d5.decode());</script>+<script id=bom>+ // Only a BOM of the decoder's own encoding is stripped; nothing is sniffed.+ testing.expectEqual('\ufffd\ufffdA\u0000', new TextDecoder().decode(new Uint8Array([0xFF, 0xFE, 0x41, 0x00])));+ testing.expectEqual('\u00ef\u00bb\u00bf\u00e9', new TextDecoder('windows-1252').decode(new Uint8Array([0xEF, 0xBB, 0xBF, 0xE9])));+ testing.expectEqual('A', new TextDecoder('utf-16le').decode(new Uint8Array([0xFF, 0xFE, 0x41, 0x00])));+ testing.expectEqual('\ufeffA', new TextDecoder('utf-8', {ignoreBOM: true}).decode(new Uint8Array([0xEF, 0xBB, 0xBF, 0x41])));++ // A BOM split across chunks is still stripped, and again after a flush.+ let d6 = new TextDecoder();+ testing.expectEqual('', d6.decode(new Uint8Array([0xEF, 0xBB]), { stream: true }));+ testing.expectEqual('A', d6.decode(new Uint8Array([0xBF, 0x41])));+ testing.expectEqual('B', d6.decode(new Uint8Array([0xEF, 0xBB, 0xBF, 0x42])));+</script>+<script id=slice>const buf1 = new ArrayBuffer(7);const arr1 = new Uint8Array(buf1)diff --git a/src/browser/webapi/encoding/TextDecoder.zig b/src/browser/webapi/encoding/TextDecoder.zigindex 521cba303..46d357512 100644--- a/src/browser/webapi/encoding/TextDecoder.zig+++ b/src/browser/webapi/encoding/TextDecoder.zig@@ -124,24 +124,21 @@ pub fn decode(self: *TextDecoder, input_: ?[]const u8, opts_: ?DecodeOpts, exec:return error.OutOfMemory;}}- return self._decode(exec.call_arena, input, self._decoder, false);+ return self._decode(exec.call_arena, input, self._decoder.?, false);}- if (self._decoder) |decoder| {- // Non-streaming with existing decoder: flush with is_last=true, then free- const result = try self._decode(exec.call_arena, input, decoder, true);-- // on error, _decode will free the decoder. So we only free it on non-error- html5ever.encoding_decoder_free(decoder);- self._decoder = null;- return result;- }-- // non-streaming, no existing decoder- return self._decode(exec.call_arena, input, null, true);+ // Non-streaming: flush the decoder from earlier streaming calls (or a new+ // one) with is_last=true, then free it. On error, _decode frees it.+ const decoder = self._decoder orelse html5ever.encoding_decoder_new(self._encoding_handle) orelse return error.OutOfMemory;+ self._decoder = decoder;+ const result = try self._decode(exec.call_arena, input, decoder, true);+ html5ever.encoding_decoder_free(decoder);+ self._decoder = null;+ self._bom_seen = false;+ return result;}-fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, streaming_decoder: ?*anyopaque, is_last: bool) ![]const u8 {+fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, decoder: *anyopaque, is_last: bool) ![]const u8 {if (input.len == 0 and !is_last) {return "";}@@ -159,44 +156,30 @@ fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, stre// Allocate output bufferconst output = try arena.alloc(u8, max_out);- // Decode using either streaming or one-shot decoder- const result = if (streaming_decoder) |decoder|- html5ever.encoding_decoder_decode(- decoder,- input.ptr,- input.len,- output.ptr,- output.len,- @intFromBool(is_last),- )- else- html5ever.encoding_decode(- self._encoding_handle,- input.ptr,- input.len,- output.ptr,- output.len,- 1, // is_last = true for one-shot- );+ const result = html5ever.encoding_decoder_decode(+ decoder,+ input.ptr,+ input.len,+ output.ptr,+ output.len,+ @intFromBool(is_last),+ );// Handle errors in fatal modeif (self._fatal and result.hadErrors()) {- if (streaming_decoder != null) {- // Reset decoder on error- if (self._decoder) |decoder| {- html5ever.encoding_decoder_free(decoder);- self._decoder = null;- }- }+ // Reset decoder on error+ html5ever.encoding_decoder_free(decoder);+ self._decoder = null;self._bom_seen = false;return error.TypeError;}var decoded: []const u8 = output[0..result.bytes_written];- // Handle BOM stripping- if (!self._bom_seen and !self._ignore_bom) {- decoded = stripBom(decoded);+ // The decoder does no BOM sniffing, so a BOM of this encoding is a+ // leading U+FEFF in the first output; other bytes stay as decoded.+ if (!self._bom_seen and decoded.len > 0) {+ if (!self._ignore_bom) decoded = stripBom(decoded);self._bom_seen = true;}diff --git a/src/rust/html5ever/lib.rs b/src/rust/html5ever/lib.rsindex a42d8edf2..1340b4894 100644--- a/src/rust/html5ever/lib.rs+++ b/src/rust/html5ever/lib.rs@@ -277,14 +277,15 @@ pub extern "C" fn encoding_decode(use encoding_rs::Decoder;-/// Create a streaming decoder that maintains state across calls+/// Create a streaming decoder that maintains state across calls.+/// It does no BOM sniffing: a BOM decodes to U+FEFF, as TextDecoder needs.#[no_mangle]pub extern "C" fn encoding_decoder_new(handle: *const c_void) -> *mut c_void {if handle.is_null() {return std::ptr::null_mut();}let encoding: &'static Encoding = unsafe { &*(handle as *const Encoding) };- let decoder = Box::new(encoding.new_decoder());+ let decoder = Box::new(encoding.new_decoder_without_bom_handling());Box::into_raw(decoder) as *mut c_void}