Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
BOM-less UTF-16 XML detection
BibekPathakeaafee04fe8c
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rsindex fc065ec5b..e19586380 100644--- a/moli-encoding/src/document.rs+++ b/moli-encoding/src/document.rs@@ -119,6 +119,9 @@ impl HtmlDocumentStreamingDecoder {if !finishing && bytes_could_still_be_bom_prefix(&self.sniff_buffer) {return None;}+ if let Some(encoding) = encoding_for_document_xml_declaration(&self.sniff_buffer) {+ return Some(encoding);+ }if let Some(encoding) = self.transport_encoding {return Some(encoding);}@@ -230,3 +233,31 @@ fn encoding_for_document_bom(bytes: &[u8]) -> Option<&'static Encoding> {}None}++/// Detect BOM-less UTF-16LE/BE documents that begin with an XML declaration.+///+/// The WHATWG encoding standard's sniffing algorithm requires detecting UTF-16LE+/// and UTF-16BE byte patterns for `<?` (the start of an XML declaration) even+/// without BOM or transport charset, so that the document's encoding can be+/// determined correctly.+fn encoding_for_document_xml_declaration(bytes: &[u8]) -> Option<&'static Encoding> {+ // UTF-16LE: 0x3C 0x00 0x3F 0x00 = "<?" in UTF-16LE+ if bytes.len() >= 4+ && bytes[0] == 0x3C+ && bytes[1] == 0x00+ && bytes[2] == 0x3F+ && bytes[3] == 0x00+ {+ return Some(encoding_rs::UTF_16LE);+ }+ // UTF-16BE: 0x00 0x3C 0x00 0x3F = "<?" in UTF-16BE+ if bytes.len() >= 4+ && bytes[0] == 0x00+ && bytes[1] == 0x3C+ && bytes[2] == 0x00+ && bytes[3] == 0x3F+ {+ return Some(encoding_rs::UTF_16BE);+ }+ None+}diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex ce7b35fd3..bb47b731c 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -492,3 +492,34 @@ fn transport_utf16_still_wins_over_the_meta_rewrite() {assert_eq!(encoding, "UTF-16LE");assert!(text.ends_with("hi"), "decoded as {text:?}");}++#[test]+fn bom_less_utf16le_xml_declaration_is_detected() {+ let input = utf16le_bytes(+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<html><head></head><body>hi</body></html>",+ );++ let (text, encoding) = decode_html_document(&input, &[]);++ assert_eq!(encoding, "UTF-16LE");+ assert!(text.contains("hi"), "decoded as {text:?}");+}++fn utf16be_bytes(input: &str) -> Vec<u8> {+ input+ .encode_utf16()+ .flat_map(|unit| unit.to_be_bytes())+ .collect()+}++#[test]+fn bom_less_utf16be_xml_declaration_is_detected() {+ let input = utf16be_bytes(+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<html><head></head><body>hi</body></html>",+ );++ let (text, encoding) = decode_html_document(&input, &[]);++ assert_eq!(encoding, "UTF-16BE");+ assert!(text.contains("hi"), "decoded as {text:?}");+}