Lightpanda: the headless browser designed for AI and automation
Only UTF-8 and UTF-16 have a BOM, so it can only be stripped from them
Karl Seguin926e490d9bee
diff --git a/src/browser/tests/encoding/text_decoder.html b/src/browser/tests/encoding/text_decoder.htmlindex 219765bb4..914f26b3d 100644--- a/src/browser/tests/encoding/text_decoder.html+++ b/src/browser/tests/encoding/text_decoder.html@@ -69,6 +69,9 @@testing.expectEqual('\u00ef\u00bb\u00bf\u00e9', new TextDecoder('windows-1252').decode(new Uint8Array([0xEF, 0xBB, 0xBF, 0xE9])));testing.expectEqual('A', new TextDecoder('utf-16le').decode(new Uint8Array([0xFF, 0xFE, 0x41, 0x00])));testing.expectEqual('\ufeffA', new TextDecoder('utf-8', {ignoreBOM: true}).decode(new Uint8Array([0xEF, 0xBB, 0xBF, 0x41])));+ testing.expectEqual('A', new TextDecoder('utf-16be').decode(new Uint8Array([0xFE, 0xFF, 0x00, 0x41])));+ // gb18030 has no BOM: a leading U+FEFF is kept.+ testing.expectEqual('\ufeffA', new TextDecoder('gb18030').decode(new Uint8Array([0x84, 0x31, 0x95, 0x33, 0x41])));// A BOM split across chunks is still stripped, and again after a flush.let d6 = new TextDecoder();diff --git a/src/browser/webapi/encoding/TextDecoder.zig b/src/browser/webapi/encoding/TextDecoder.zigindex 46d357512..a99800077 100644--- a/src/browser/webapi/encoding/TextDecoder.zig+++ b/src/browser/webapi/encoding/TextDecoder.zig@@ -178,14 +178,23 @@ fn _decode(self: *TextDecoder, arena: std.mem.Allocator, input: []const u8, deco// The decoder does no BOM sniffing, so a BOM of this encoding is a// leading U+FEFF in the first output; other bytes stay as decoded.+ // Only UTF-8 and UTF-16 have a BOM: in gb18030, U+FEFF is a character.if (!self._bom_seen and decoded.len > 0) {- if (!self._ignore_bom) decoded = stripBom(decoded);+ if (self._ignore_bom == false and hasBom(self._encoding_name)) {+ decoded = stripBom(decoded);+ }self._bom_seen = true;}return decoded;}+fn hasBom(encoding_name: []const u8) bool {+ return std.mem.eql(u8, encoding_name, "UTF-8") or+ std.mem.eql(u8, encoding_name, "UTF-16LE") or+ std.mem.eql(u8, encoding_name, "UTF-16BE");+}+fn stripBom(data: []const u8) []const u8 {// UTF-8 BOM in decoded output appears as U+FEFF (EF BB BF in UTF-8)const bom = "\u{FEFF}";