Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

BOM-less UTF-16 XML detection

BibekPathakeaafee04fe8c
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs
index fc065ec5b..e19586380 100644
--- a/moli-encoding/src/document.rs
+++ b/moli-encoding/src/document.rs
@@ -119,6 +119,9 @@ impl HtmlDocumentStreamingDecoder {
if !finishing && bytes_could_still_be_bom_prefix(&self.sniff_buffer) {
return None;
}
+ if let Some(encoding) = encoding_for_document_xml_declaration(&self.sniff_buffer) {
+ return Some(encoding);
+ }
if let Some(encoding) = self.transport_encoding {
return Some(encoding);
}
@@ -230,3 +233,31 @@ fn encoding_for_document_bom(bytes: &[u8]) -> Option<&'static Encoding> {
}
None
}
+
+/// Detect BOM-less UTF-16LE/BE documents that begin with an XML declaration.
+///
+/// The WHATWG encoding standard's sniffing algorithm requires detecting UTF-16LE
+/// and UTF-16BE byte patterns for `<?` (the start of an XML declaration) even
+/// without BOM or transport charset, so that the document's encoding can be
+/// determined correctly.
+fn encoding_for_document_xml_declaration(bytes: &[u8]) -> Option<&'static Encoding> {
+ // UTF-16LE: 0x3C 0x00 0x3F 0x00 = "<?" in UTF-16LE
+ if bytes.len() >= 4
+ && bytes[0] == 0x3C
+ && bytes[1] == 0x00
+ && bytes[2] == 0x3F
+ && bytes[3] == 0x00
+ {
+ return Some(encoding_rs::UTF_16LE);
+ }
+ // UTF-16BE: 0x00 0x3C 0x00 0x3F = "<?" in UTF-16BE
+ if bytes.len() >= 4
+ && bytes[0] == 0x00
+ && bytes[1] == 0x3C
+ && bytes[2] == 0x00
+ && bytes[3] == 0x3F
+ {
+ return Some(encoding_rs::UTF_16BE);
+ }
+ None
+}
diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index ce7b35fd3..bb47b731c 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -492,3 +492,34 @@ fn transport_utf16_still_wins_over_the_meta_rewrite() {
assert_eq!(encoding, "UTF-16LE");
assert!(text.ends_with("hi"), "decoded as {text:?}");
}
+
+#[test]
+fn bom_less_utf16le_xml_declaration_is_detected() {
+ let input = utf16le_bytes(
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<html><head></head><body>hi</body></html>",
+ );
+
+ let (text, encoding) = decode_html_document(&input, &[]);
+
+ assert_eq!(encoding, "UTF-16LE");
+ assert!(text.contains("hi"), "decoded as {text:?}");
+}
+
+fn utf16be_bytes(input: &str) -> Vec<u8> {
+ input
+ .encode_utf16()
+ .flat_map(|unit| unit.to_be_bytes())
+ .collect()
+}
+
+#[test]
+fn bom_less_utf16be_xml_declaration_is_detected() {
+ let input = utf16be_bytes(
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<html><head></head><body>hi</body></html>",
+ );
+
+ let (text, encoding) = decode_html_document(&input, &[]);
+
+ assert_eq!(encoding, "UTF-16BE");
+ assert!(text.contains("hi"), "decoded as {text:?}");
+}