Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(encoding): honor compatibility XML declarations
ldm04c00e6dab6ae
diff --git a/moli-benchmark/wpt-cross-current/failed-cases.txt b/moli-benchmark/wpt-cross-current/failed-cases.txtindex 7aeb3e201..4e2a02b22 100644--- a/moli-benchmark/wpt-cross-current/failed-cases.txt+++ b/moli-benchmark/wpt-cross-current/failed-cases.txt@@ -3299,8 +3299,6 @@ html/syntax/parsing/template/additions-to-the-in-body-insertion-mode/ignore-headhtml/syntax/parsing/template/creating-an-element-for-the-token/template-owner-document.htmlhtml/syntax/parsing/the-end.htmlhtml/syntax/serializing-xml-fragments/outerHTML.html-html/syntax/xmldecl/xmldecl-1.html-html/syntax/xmldecl/xmldecl-2.htmlhtml/webappapis/dynamic-markup-insertion/document-write/module-delayed.htmlhtml/webappapis/dynamic-markup-insertion/document-write/module-static-import-delayed.htmlhtml/webappapis/dynamic-markup-insertion/document-write/module-static-import.htmldiff --git a/moli-benchmark/wpt-cross-current/passed-cases.txt b/moli-benchmark/wpt-cross-current/passed-cases.txtindex 51428a194..449b3b377 100644--- a/moli-benchmark/wpt-cross-current/passed-cases.txt+++ b/moli-benchmark/wpt-cross-current/passed-cases.txt@@ -6464,6 +6464,8 @@ html/syntax/serializing-html-fragments/serializing-cdata-in-html-document.htmlhtml/syntax/serializing-html-fragments/serializing-lt-gt.htmlhtml/syntax/serializing-html-fragments/serializing.htmlhtml/syntax/serializing-html-fragments/template.html+html/syntax/xmldecl/xmldecl-1.html+html/syntax/xmldecl/xmldecl-2.htmlhtml/webappapis/animation-frames/callback-cross-realm-report-exception.htmlhtml/webappapis/animation-frames/callback-exception.htmlhtml/webappapis/animation-frames/callback-handle.htmldiff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rsindex d1a95a50e..88df60445 100644--- a/moli-encoding/src/document.rs+++ b/moli-encoding/src/document.rs@@ -149,7 +149,7 @@ impl HtmlDocumentStreamingDecoder {if let Some(encoding) = self.transport_encoding {return Some(encoding);}- if let Some(encoding) = encoding_for_document_xml_declaration(&self.sniff_buffer) {+ if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) {return Some(encoding);}let meta_scan = self.feed_meta_charset_prescan(finishing);@@ -157,16 +157,18 @@ impl HtmlDocumentStreamingDecoder {return Some(encoding);}match meta_scan {- HtmlMetaCharsetScanResult::NotFound => self- .detected_legacy_content_encoding()- .or(Some(self.fallback_encoding)),- HtmlMetaCharsetScanResult::Pending if finishing => self- .detected_legacy_content_encoding()- .or(Some(self.fallback_encoding)),+ HtmlMetaCharsetScanResult::NotFound => Some(self.encoding_after_prescan()),+ HtmlMetaCharsetScanResult::Pending if finishing => Some(self.encoding_after_prescan()),HtmlMetaCharsetScanResult::Pending | HtmlMetaCharsetScanResult::Found(_) => None,}}+ fn encoding_after_prescan(&self) -> &'static Encoding {+ encoding_for_document_xml_declaration(&self.sniff_buffer)+ .or_else(|| self.detected_legacy_content_encoding())+ .unwrap_or(self.fallback_encoding)+ }+fn detected_legacy_content_encoding(&self) -> Option<&'static Encoding> {self.legacy_encoding_detector.and_then(|detector| detector(&self.sniff_buffer, self.url_hint.as_deref()))@@ -283,7 +285,7 @@ fn encoding_for_document_bom(bytes: &[u8]) -> Option<&'static Encoding> {/// UTF-16BE: 00 3C 00 3F 00 78 ("\<?x")////// These match the start of `<?xml` without requiring a BOM or transport charset.-fn encoding_for_document_xml_declaration(bytes: &[u8]) -> Option<&'static Encoding> {+fn encoding_for_document_utf16_xml_prefix(bytes: &[u8]) -> Option<&'static Encoding> {if bytes.starts_with(UTF16LE_XML_PREFIX) {return Some(encoding_rs::UTF_16LE);}@@ -292,3 +294,59 @@ fn encoding_for_document_xml_declaration(bytes: &[u8]) -> Option<&'static Encodi}None}++/// Applies the HTML prescan's compatibility parser for an ASCII-compatible+/// XML encoding declaration at the very start of a `text/html` byte stream.+///+/// This deliberately implements the permissive "get an XML encoding"+/// algorithm rather than XML syntax. In particular, only `<?xml` and the+/// lowercase `encoding` keyword are fixed syntax, control bytes are accepted+/// around `=`, and the declaration ends at the first `>` byte.+///+/// <https://html.spec.whatwg.org/multipage/parsing.html#concept-get-xml-encoding>+fn encoding_for_document_xml_declaration(bytes: &[u8]) -> Option<&'static Encoding> {+ if !bytes.starts_with(b"<?xml") {+ return None;+ }++ let declaration_end = bytes.iter().position(|byte| *byte == b'>')?;+ let declaration = &bytes[..declaration_end];+ let keyword_offset = declaration+ .windows(b"encoding".len())+ .position(|window| window == b"encoding")?;+ let mut position = keyword_offset + b"encoding".len();++ while declaration.get(position).is_some_and(|byte| *byte <= 0x20) {+ position += 1;+ }+ if declaration.get(position) != Some(&b'=') {+ return None;+ }+ position += 1;+ while declaration.get(position).is_some_and(|byte| *byte <= 0x20) {+ position += 1;+ }++ let quote = *declaration.get(position)?;+ if !matches!(quote, b'\'' | b'"') {+ return None;+ }+ position += 1;+ let label_end = declaration[position..]+ .iter()+ .position(|byte| *byte == quote)+ .map(|offset| position + offset)?;+ let label = &declaration[position..label_end];+ if label.iter().any(|byte| *byte <= 0x20) {+ return None;+ }++ let encoding = Encoding::for_label(label)?;+ Some(+ if encoding == encoding_rs::UTF_16BE || encoding == encoding_rs::UTF_16LE {+ encoding_rs::UTF_8+ } else {+ encoding+ },+ )+}diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex dbec8f5fe..a6307a0e1 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -713,6 +713,89 @@ fn bom_less_utf16be_xml_declaration_is_detected() {assert!(text.contains("hi"), "decoded as {text:?}");}+#[test]+fn ascii_xml_declaration_selects_its_encoding_after_meta_prescan() {+ let mut input = br#"<?xml version="1.0" encoding="windows-1251"?><p>"#.to_vec();+ input.extend_from_slice(&encoding_rs::WINDOWS_1251.encode("ж").0);+ let (text, encoding) = decode_html_document(&input, &[]);++ assert_eq!(encoding, "windows-1251");+ assert!(text.ends_with("ж"), "decoded as {text:?}");+}++#[test]+fn ascii_xml_declaration_uses_the_compatibility_grammar() {+ for declaration in [+ b"<?xml encoding='cp1251'>".as_slice(),+ b"<?xmlencoding=\"windows-1251\">",+ b"<?xmla<encoding=\"windows-1251\">",+ b"<?xml encoding\x01=\x00\"windows-1251\">",+ ] {+ assert_eq!(+ decode_html_document(declaration, &[]).1,+ "windows-1251",+ "declaration={declaration:?}"+ );+ }+}++#[test]+fn ascii_xml_declaration_rejects_near_misses() {+ for declaration in [+ b"<?XML encoding=\"windows-1251\">".as_slice(),+ b"<?xml ENCODING=\"windows-1251\">",+ b"<?xml encodingencoding=\"windows-1251\">",+ b"<?xml encoding=encoding=\"windows-1251\">",+ b"<?xml encoding=windows-1251>",+ b"<?xml encoding=\" windows-1251\">",+ b"<?xml>encoding=\"windows-1251\">",+ b"<?xml encoding=\"windows-1251'>",+ ] {+ assert_eq!(+ decode_html_document(declaration, &[]).1,+ "windows-1252",+ "declaration={declaration:?}"+ );+ }+}++#[test]+fn meta_and_transport_encodings_precede_ascii_xml_declarations() {+ let input = br#"<?xml encoding="windows-1251"?><meta charset="windows-1253">"#;+ assert_eq!(decode_html_document(input, &[]).1, "windows-1253");++ let headers = vec![(+ "Content-Type".to_owned(),+ "text/html; charset=UTF-8".to_owned(),+ )];+ assert_eq!(decode_html_document(input, &headers).1, "UTF-8");+}++#[test]+fn ascii_xml_declaration_rewrites_utf16_and_preserves_replacement() {+ assert_eq!(+ decode_html_document(br#"<?xml encoding="UTF-16">"#, &[]).1,+ "UTF-8"+ );+ assert_eq!(+ decode_html_document(br#"<?xml encoding="ISO-2022-KR">"#, &[]).1,+ "replacement"+ );+}++#[test]+fn ascii_xml_declaration_can_finish_after_the_prescan_boundary() {+ let mut input = br#"<?xml version="1.0" "#.to_vec();+ input.resize(HTML_META_CHARSET_PRESCAN_LIMIT + 1, b' ');+ input.extend_from_slice(br#"encoding="windows-1251"><p>"#);+ input.extend_from_slice(&encoding_rs::WINDOWS_1251.encode("ж").0);++ let (text, encoding) = decode_html_document(&input, &[]);++ assert_eq!(encoding, "windows-1251");+ assert!(text.ends_with("ж"), "decoded as {text:?}");+}+#[test]fn bom_less_utf16le_non_xml_not_detected() {let input = utf16le_bytes("<?y>");