Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(encoding): preserve split UTF-16 XML signatures
ldm06fa68cadbed0
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rsindex f6ea1e237..b7e3da79b 100644--- a/moli-encoding/src/document.rs+++ b/moli-encoding/src/document.rs@@ -4,6 +4,8 @@ use moli_charset_parser::{HtmlMetaCharsetParser, HtmlMetaCharsetScanResult};use crate::{charset_from_headers, encoding_for_label};const DEFAULT_HTML_DOCUMENT_ENCODING: &str = "windows-1252";+const UTF16LE_XML_PREFIX: &[u8; 6] = b"<\0?\0x\0";+const UTF16BE_XML_PREFIX: &[u8; 6] = b"\0<\0?\0x";pub fn decode_html_document(bytes: &[u8], headers: &[(String, String)]) -> (String, &'static str) {decode_html_document_with_fallback(bytes, headers, None)@@ -189,6 +191,7 @@ impl HtmlDocumentStreamingDecoder {fn take_safe_ascii_sniff_prefix(&mut self) -> String {if self.emitted_sniff_len >= self.sniff_buffer.len()|| bytes_could_still_be_bom_prefix(&self.sniff_buffer)+ || bytes_could_still_be_utf16_xml_prefix(&self.sniff_buffer){return String::new();}@@ -221,6 +224,11 @@ fn bytes_could_still_be_bom_prefix(bytes: &[u8]) -> bool {matches!(bytes, [] | [0xEF] | [0xEF, 0xBB] | [0xFF] | [0xFE])}+fn bytes_could_still_be_utf16_xml_prefix(bytes: &[u8]) -> bool {+ bytes.len() < UTF16LE_XML_PREFIX.len()+ && (UTF16LE_XML_PREFIX.starts_with(bytes) || UTF16BE_XML_PREFIX.starts_with(bytes))+}+fn encoding_for_document_bom(bytes: &[u8]) -> Option<&'static Encoding> {if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {return Some(encoding_rs::UTF_8);@@ -242,24 +250,10 @@ fn encoding_for_document_bom(bytes: &[u8]) -> Option<&'static Encoding> {////// These match the start of `<?xml` without requiring a BOM or transport charset.fn encoding_for_document_xml_declaration(bytes: &[u8]) -> Option<&'static Encoding> {- if bytes.len() >= 6- && bytes[0] == 0x3C- && bytes[1] == 0x00- && bytes[2] == 0x3F- && bytes[3] == 0x00- && bytes[4] == 0x78- && bytes[5] == 0x00- {+ if bytes.starts_with(UTF16LE_XML_PREFIX) {return Some(encoding_rs::UTF_16LE);}- if bytes.len() >= 6- && bytes[0] == 0x00- && bytes[1] == 0x3C- && bytes[2] == 0x00- && bytes[3] == 0x3F- && bytes[4] == 0x00- && bytes[5] == 0x78- {+ if bytes.starts_with(UTF16BE_XML_PREFIX) {return Some(encoding_rs::UTF_16BE);}Nonediff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex 6efc38a7f..91c4b7bf1 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -527,6 +527,77 @@ fn utf16be_bytes(input: &str) -> Vec<u8> {.collect()}+fn assert_bom_less_utf16_xml_decodes_across_signature_splits(+ input: &[u8],+ expected_text: &str,+ expected_encoding: &str,+) {+ for split in 1..6 {+ let mut decoder = HtmlDocumentStreamingDecoder::new(&[]);+ assert_eq!(+ decoder.push(&input[..split]),+ Vec::<String>::new(),+ "split after signature byte {split} must remain buffered"+ );++ let mut decoded = decoder.push(&input[split..]).concat();+ if let Some(tail) = decoder.finish() {+ decoded.push_str(&tail);+ }++ assert_eq!(+ decoder.selected_encoding_name(),+ Some(expected_encoding),+ "wrong encoding after signature byte {split} split"+ );+ assert_eq!(+ decoded, expected_text,+ "misdecoded document after signature byte {split} split"+ );+ }+}++#[test]+fn bom_less_utf16le_xml_signature_survives_every_streaming_split() {+ let source = "<?xml version=\"1.0\"?><html><head></head><body>little endian</body></html>";+ let input = utf16le_bytes(source);++ assert_bom_less_utf16_xml_decodes_across_signature_splits(&input, source, "UTF-16LE");+}++#[test]+fn bom_less_utf16be_xml_signature_survives_every_streaming_split() {+ let source = "<?xml version=\"1.0\"?><html><head></head><body>big endian</body></html>";+ let input = utf16be_bytes(source);++ assert_bom_less_utf16_xml_decodes_across_signature_splits(&input, source, "UTF-16BE");+}++#[test]+fn diverged_utf16_xml_signature_prefix_resumes_ascii_streaming() {+ let mut decoder = HtmlDocumentStreamingDecoder::new(&[]);++ assert!(decoder.push(b"<").is_empty());+ assert_eq!(+ decoder.push(b"!doctype html><p>ordinary html"),+ vec!["<!doctype html><p>ordinary html"]+ );+ assert_eq!(decoder.finish(), None);+ assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));+}++#[test]+fn transport_charset_does_not_wait_for_a_utf16_xml_signature_prefix() {+ let headers = vec![(+ "Content-Type".to_owned(),+ "text/html; charset=windows-1252".to_owned(),+ )];+ let mut decoder = HtmlDocumentStreamingDecoder::new(&headers);++ assert_eq!(decoder.push(b"<"), vec!["<"]);+ assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));+}+#[test]fn bom_less_utf16be_xml_declaration_is_detected() {let input = utf16be_bytes(@@ -561,9 +632,10 @@ fn bom_less_utf16le_uppercase_x_not_detected() {fn bom_less_utf16le_truncated_not_detected() {let input: Vec<u8> = vec![0x3C, 0x00, 0x3F, 0x00];- let (_, encoding) = decode_html_document(&input, &[]);+ let (text, encoding) = decode_html_document(&input, &[]);assert_eq!(encoding, "windows-1252");+ assert_eq!(text, "<\0?\0");}#[test]@@ -588,7 +660,8 @@ fn bom_less_utf16be_uppercase_x_not_detected() {fn bom_less_utf16be_truncated_not_detected() {let input: Vec<u8> = vec![0x00, 0x3C, 0x00, 0x3F];- let (_, encoding) = decode_html_document(&input, &[]);+ let (text, encoding) = decode_html_document(&input, &[]);assert_eq!(encoding, "windows-1252");+ assert_eq!(text, "\0<\0?");}