Skip to content

Lightpanda: the headless browser designed for AI and automation

webapi: stop TextDecoder from sniffing a BOM of another encoding

Jay Hemnani38703cacc055
diff --git a/src/browser/tests/encoding/text_decoder.html b/src/browser/tests/encoding/text_decoder.html
index 4f5574f40..219765bb4 100644
--- a/src/browser/tests/encoding/text_decoder.html
+++ b/src/browser/tests/encoding/text_decoder.html
@@ -63,6 +63,20 @@
testing.expectEqual('', d5.decode());
</script>
+<script id=bom>
+ // Only a BOM of the decoder's own encoding is stripped; nothing is sniffed.
+ testing.expectEqual('\ufffd\ufffdA\u0000', new TextDecoder().decode(new Uint8Array([0xFF, 0xFE, 0x41, 0x00])));
+ testing.expectEqual('\u00ef\u00bb\u00bf\u00e9', new TextDecoder('windows-1252').decode(new Uint8Array([0xEF, 0xBB, 0xBF, 0xE9])));
+ testing.expectEqual('A', new TextDecoder('utf-16le').decode(new Uint8Array([0xFF, 0xFE, 0x41, 0x00])));
+ testing.expectEqual('\ufeffA', new TextDecoder('utf-8', {ignoreBOM: true}).decode(new Uint8Array([0xEF, 0xBB, 0xBF, 0x41])));
+
+ // A BOM split across chunks is still stripped, and again after a flush.
+ let d6 = new TextDecoder();
+ testing.expectEqual('', d6.decode(new Uint8Array([0xEF, 0xBB]), { stream: true }));
+ testing.expectEqual('A', d6.decode(new Uint8Array([0xBF, 0x41])));
+ testing.expectEqual('B', d6.decode(new Uint8Array([0xEF, 0xBB, 0xBF, 0x42])));
+</script>
+
<script id=slice>
const buf1 = new ArrayBuffer(7);
const arr1 = new Uint8Array(buf1)
diff --git a/src/browser/webapi/encoding/TextDecoder.zig b/src/browser/webapi/encoding/TextDecoder.zig
index 521cba303..46d357512 100644
--- a/src/browser/webapi/encoding/TextDecoder.zig
+++ b/src/browser/webapi/encoding/TextDecoder.zig
@@ -124,24 +124,21 @@ pub fn decode(self: *TextDecoder, input_: ?[]const u8, opts_: ?DecodeOpts, exec:
return error.OutOfMemory;
}
}
- return self._decode(exec.call_arena, input, self._decoder, false);
+ return self._decode(exec.call_arena, input, self._decoder.?, false);
}
- if (self._decoder) |decoder| {
- // Non-streaming with existing decoder: flush with is_last=true, then free
- const result = try self._decode(exec.call_arena, input, decoder, true);
-
- // on error, _decode will free the decoder. So we only free it on non-error
- html5ever.encoding_decoder_free(decoder);
- self._decoder = null;
- return result;
- }
-
- // non-streaming, no existing decoder
- return self._decode(exec.call_arena, input, null, true);
+ // Non-streaming: flush the decoder from earlier streaming calls (or a new
+ // one) with is_last=true, then free it. On error, _decode frees it.
+ const decoder = self._decoder orelse html5ever.encoding_decoder_new(self._encoding_handle) orelse return error.OutOfMemory;
+ self._decoder = decoder;
+ const result = try self._decode(exec.call_arena, input, decoder, true);
+ html5ever.encoding_decoder_free(decoder);
+ self._decoder = null;
+ self._bom_seen = false;
+ return result;
}
-fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, streaming_decoder: ?*anyopaque, is_last: bool) ![]const u8 {
+fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, decoder: *anyopaque, is_last: bool) ![]const u8 {
if (input.len == 0 and !is_last) {
return "";
}
@@ -159,44 +156,30 @@ fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, stre
// Allocate output buffer
const output = try arena.alloc(u8, max_out);
- // Decode using either streaming or one-shot decoder
- const result = if (streaming_decoder) |decoder|
- html5ever.encoding_decoder_decode(
- decoder,
- input.ptr,
- input.len,
- output.ptr,
- output.len,
- @intFromBool(is_last),
- )
- else
- html5ever.encoding_decode(
- self._encoding_handle,
- input.ptr,
- input.len,
- output.ptr,
- output.len,
- 1, // is_last = true for one-shot
- );
+ const result = html5ever.encoding_decoder_decode(
+ decoder,
+ input.ptr,
+ input.len,
+ output.ptr,
+ output.len,
+ @intFromBool(is_last),
+ );
// Handle errors in fatal mode
if (self._fatal and result.hadErrors()) {
- if (streaming_decoder != null) {
- // Reset decoder on error
- if (self._decoder) |decoder| {
- html5ever.encoding_decoder_free(decoder);
- self._decoder = null;
- }
- }
+ // Reset decoder on error
+ html5ever.encoding_decoder_free(decoder);
+ self._decoder = null;
self._bom_seen = false;
return error.TypeError;
}
var decoded: []const u8 = output[0..result.bytes_written];
- // Handle BOM stripping
- if (!self._bom_seen and !self._ignore_bom) {
- decoded = stripBom(decoded);
+ // The decoder does no BOM sniffing, so a BOM of this encoding is a
+ // leading U+FEFF in the first output; other bytes stay as decoded.
+ if (!self._bom_seen and decoded.len > 0) {
+ if (!self._ignore_bom) decoded = stripBom(decoded);
self._bom_seen = true;
}
diff --git a/src/rust/html5ever/lib.rs b/src/rust/html5ever/lib.rs
index a42d8edf2..1340b4894 100644
--- a/src/rust/html5ever/lib.rs
+++ b/src/rust/html5ever/lib.rs
@@ -277,14 +277,15 @@ pub extern "C" fn encoding_decode(
use encoding_rs::Decoder;
-/// Create a streaming decoder that maintains state across calls
+/// Create a streaming decoder that maintains state across calls.
+/// It does no BOM sniffing: a BOM decodes to U+FEFF, as TextDecoder needs.
#[no_mangle]
pub extern "C" fn encoding_decoder_new(handle: *const c_void) -> *mut c_void {
if handle.is_null() {
return std::ptr::null_mut();
}
let encoding: &'static Encoding = unsafe { &*(handle as *const Encoding) };
- let decoder = Box::new(encoding.new_decoder());
+ let decoder = Box::new(encoding.new_decoder_without_bom_handling());
Box::into_raw(decoder) as *mut c_void
}