Skip to content

Lightpanda: the headless browser designed for AI and automation

Only UTF-8 and UTF-16 have a BOM, so it can only be stripped from them

Karl Seguin926e490d9bee
diff --git a/src/browser/tests/encoding/text_decoder.html b/src/browser/tests/encoding/text_decoder.html
index 219765bb4..914f26b3d 100644
--- a/src/browser/tests/encoding/text_decoder.html
+++ b/src/browser/tests/encoding/text_decoder.html
@@ -69,6 +69,9 @@
testing.expectEqual('\u00ef\u00bb\u00bf\u00e9', new TextDecoder('windows-1252').decode(new Uint8Array([0xEF, 0xBB, 0xBF, 0xE9])));
testing.expectEqual('A', new TextDecoder('utf-16le').decode(new Uint8Array([0xFF, 0xFE, 0x41, 0x00])));
testing.expectEqual('\ufeffA', new TextDecoder('utf-8', {ignoreBOM: true}).decode(new Uint8Array([0xEF, 0xBB, 0xBF, 0x41])));
+ testing.expectEqual('A', new TextDecoder('utf-16be').decode(new Uint8Array([0xFE, 0xFF, 0x00, 0x41])));
+ // gb18030 has no BOM: a leading U+FEFF is kept.
+ testing.expectEqual('\ufeffA', new TextDecoder('gb18030').decode(new Uint8Array([0x84, 0x31, 0x95, 0x33, 0x41])));
// A BOM split across chunks is still stripped, and again after a flush.
let d6 = new TextDecoder();
diff --git a/src/browser/webapi/encoding/TextDecoder.zig b/src/browser/webapi/encoding/TextDecoder.zig
index 46d357512..a99800077 100644
--- a/src/browser/webapi/encoding/TextDecoder.zig
+++ b/src/browser/webapi/encoding/TextDecoder.zig
@@ -178,14 +178,23 @@ fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, deco
// The decoder does no BOM sniffing, so a BOM of this encoding is a
// leading U+FEFF in the first output; other bytes stay as decoded.
+ // Only UTF-8 and UTF-16 have a BOM: in gb18030, U+FEFF is a character.
if (!self._bom_seen and decoded.len > 0) {
- if (!self._ignore_bom) decoded = stripBom(decoded);
+ if (self._ignore_bom == false and hasBom(self._encoding_name)) {
+ decoded = stripBom(decoded);
+ }
self._bom_seen = true;
}
return decoded;
}
+fn hasBom(encoding_name: []const u8) bool {
+ return std.mem.eql(u8, encoding_name, "UTF-8") or
+ std.mem.eql(u8, encoding_name, "UTF-16LE") or
+ std.mem.eql(u8, encoding_name, "UTF-16BE");
+}
+
fn stripBom(data: []const u8) []const u8 {
// UTF-8 BOM in decoded output appears as U+FEFF (EF BB BF in UTF-8)
const bom = "\u{FEFF}";