Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(encoding): honor compatibility XML declarations

ldm04c00e6dab6ae
diff --git a/moli-benchmark/wpt-cross-current/failed-cases.txt b/moli-benchmark/wpt-cross-current/failed-cases.txt
index 7aeb3e201..4e2a02b22 100644
--- a/moli-benchmark/wpt-cross-current/failed-cases.txt
+++ b/moli-benchmark/wpt-cross-current/failed-cases.txt
@@ -3299,8 +3299,6 @@ html/syntax/parsing/template/additions-to-the-in-body-insertion-mode/ignore-head
html/syntax/parsing/template/creating-an-element-for-the-token/template-owner-document.html
html/syntax/parsing/the-end.html
html/syntax/serializing-xml-fragments/outerHTML.html
-html/syntax/xmldecl/xmldecl-1.html
-html/syntax/xmldecl/xmldecl-2.html
html/webappapis/dynamic-markup-insertion/document-write/module-delayed.html
html/webappapis/dynamic-markup-insertion/document-write/module-static-import-delayed.html
html/webappapis/dynamic-markup-insertion/document-write/module-static-import.html
diff --git a/moli-benchmark/wpt-cross-current/passed-cases.txt b/moli-benchmark/wpt-cross-current/passed-cases.txt
index 51428a194..449b3b377 100644
--- a/moli-benchmark/wpt-cross-current/passed-cases.txt
+++ b/moli-benchmark/wpt-cross-current/passed-cases.txt
@@ -6464,6 +6464,8 @@ html/syntax/serializing-html-fragments/serializing-cdata-in-html-document.html
html/syntax/serializing-html-fragments/serializing-lt-gt.html
html/syntax/serializing-html-fragments/serializing.html
html/syntax/serializing-html-fragments/template.html
+html/syntax/xmldecl/xmldecl-1.html
+html/syntax/xmldecl/xmldecl-2.html
html/webappapis/animation-frames/callback-cross-realm-report-exception.html
html/webappapis/animation-frames/callback-exception.html
html/webappapis/animation-frames/callback-handle.html
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs
index d1a95a50e..88df60445 100644
--- a/moli-encoding/src/document.rs
+++ b/moli-encoding/src/document.rs
@@ -149,7 +149,7 @@ impl HtmlDocumentStreamingDecoder {
if let Some(encoding) = self.transport_encoding {
return Some(encoding);
}
- if let Some(encoding) = encoding_for_document_xml_declaration(&self.sniff_buffer) {
+ if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) {
return Some(encoding);
}
let meta_scan = self.feed_meta_charset_prescan(finishing);
@@ -157,16 +157,18 @@ impl HtmlDocumentStreamingDecoder {
return Some(encoding);
}
match meta_scan {
- HtmlMetaCharsetScanResult::NotFound => self
- .detected_legacy_content_encoding()
- .or(Some(self.fallback_encoding)),
- HtmlMetaCharsetScanResult::Pending if finishing => self
- .detected_legacy_content_encoding()
- .or(Some(self.fallback_encoding)),
+ HtmlMetaCharsetScanResult::NotFound => Some(self.encoding_after_prescan()),
+ HtmlMetaCharsetScanResult::Pending if finishing => Some(self.encoding_after_prescan()),
HtmlMetaCharsetScanResult::Pending | HtmlMetaCharsetScanResult::Found(_) => None,
}
}
+ fn encoding_after_prescan(&self) -> &'static Encoding {
+ encoding_for_document_xml_declaration(&self.sniff_buffer)
+ .or_else(|| self.detected_legacy_content_encoding())
+ .unwrap_or(self.fallback_encoding)
+ }
+
fn detected_legacy_content_encoding(&self) -> Option<&'static Encoding> {
self.legacy_encoding_detector
.and_then(|detector| detector(&self.sniff_buffer, self.url_hint.as_deref()))
@@ -283,7 +285,7 @@ fn encoding_for_document_bom(bytes: &[u8]) -> Option<&'static Encoding> {
/// UTF-16BE: 00 3C 00 3F 00 78 ("\<?x")
///
/// These match the start of `<?xml` without requiring a BOM or transport charset.
-fn encoding_for_document_xml_declaration(bytes: &[u8]) -> Option<&'static Encoding> {
+fn encoding_for_document_utf16_xml_prefix(bytes: &[u8]) -> Option<&'static Encoding> {
if bytes.starts_with(UTF16LE_XML_PREFIX) {
return Some(encoding_rs::UTF_16LE);
}
@@ -292,3 +294,59 @@ fn encoding_for_document_xml_declaration(bytes: &[u8]) -> Option<&'static Encodi
}
None
}
+
+/// Applies the HTML prescan's compatibility parser for an ASCII-compatible
+/// XML encoding declaration at the very start of a `text/html` byte stream.
+///
+/// This deliberately implements the permissive "get an XML encoding"
+/// algorithm rather than XML syntax. In particular, only `<?xml` and the
+/// lowercase `encoding` keyword are fixed syntax, control bytes are accepted
+/// around `=`, and the declaration ends at the first `>` byte.
+///
+/// <https://html.spec.whatwg.org/multipage/parsing.html#concept-get-xml-encoding>
+fn encoding_for_document_xml_declaration(bytes: &[u8]) -> Option<&'static Encoding> {
+ if !bytes.starts_with(b"<?xml") {
+ return None;
+ }
+
+ let declaration_end = bytes.iter().position(|byte| *byte == b'>')?;
+ let declaration = &bytes[..declaration_end];
+ let keyword_offset = declaration
+ .windows(b"encoding".len())
+ .position(|window| window == b"encoding")?;
+ let mut position = keyword_offset + b"encoding".len();
+
+ while declaration.get(position).is_some_and(|byte| *byte <= 0x20) {
+ position += 1;
+ }
+ if declaration.get(position) != Some(&b'=') {
+ return None;
+ }
+ position += 1;
+ while declaration.get(position).is_some_and(|byte| *byte <= 0x20) {
+ position += 1;
+ }
+
+ let quote = *declaration.get(position)?;
+ if !matches!(quote, b'\'' | b'"') {
+ return None;
+ }
+ position += 1;
+ let label_end = declaration[position..]
+ .iter()
+ .position(|byte| *byte == quote)
+ .map(|offset| position + offset)?;
+ let label = &declaration[position..label_end];
+ if label.iter().any(|byte| *byte <= 0x20) {
+ return None;
+ }
+
+ let encoding = Encoding::for_label(label)?;
+ Some(
+ if encoding == encoding_rs::UTF_16BE || encoding == encoding_rs::UTF_16LE {
+ encoding_rs::UTF_8
+ } else {
+ encoding
+ },
+ )
+}
diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index dbec8f5fe..a6307a0e1 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -713,6 +713,89 @@ fn bom_less_utf16be_xml_declaration_is_detected() {
assert!(text.contains("hi"), "decoded as {text:?}");
}
+#[test]
+fn ascii_xml_declaration_selects_its_encoding_after_meta_prescan() {
+ let mut input = br#"<?xml version="1.0" encoding="windows-1251"?><p>"#.to_vec();
+ input.extend_from_slice(&encoding_rs::WINDOWS_1251.encode("ж").0);
+ let (text, encoding) = decode_html_document(&input, &[]);
+
+ assert_eq!(encoding, "windows-1251");
+ assert!(text.ends_with("ж"), "decoded as {text:?}");
+}
+
+#[test]
+fn ascii_xml_declaration_uses_the_compatibility_grammar() {
+ for declaration in [
+ b"<?xml encoding='cp1251'>".as_slice(),
+ b"<?xmlencoding=\"windows-1251\">",
+ b"<?xmla<encoding=\"windows-1251\">",
+ b"<?xml encoding\x01=\x00\"windows-1251\">",
+ ] {
+ assert_eq!(
+ decode_html_document(declaration, &[]).1,
+ "windows-1251",
+ "declaration={declaration:?}"
+ );
+ }
+}
+
+#[test]
+fn ascii_xml_declaration_rejects_near_misses() {
+ for declaration in [
+ b"<?XML encoding=\"windows-1251\">".as_slice(),
+ b"<?xml ENCODING=\"windows-1251\">",
+ b"<?xml encodingencoding=\"windows-1251\">",
+ b"<?xml encoding=encoding=\"windows-1251\">",
+ b"<?xml encoding=windows-1251>",
+ b"<?xml encoding=\" windows-1251\">",
+ b"<?xml>encoding=\"windows-1251\">",
+ b"<?xml encoding=\"windows-1251'>",
+ ] {
+ assert_eq!(
+ decode_html_document(declaration, &[]).1,
+ "windows-1252",
+ "declaration={declaration:?}"
+ );
+ }
+}
+
+#[test]
+fn meta_and_transport_encodings_precede_ascii_xml_declarations() {
+ let input = br#"<?xml encoding="windows-1251"?><meta charset="windows-1253">"#;
+ assert_eq!(decode_html_document(input, &[]).1, "windows-1253");
+
+ let headers = vec![(
+ "Content-Type".to_owned(),
+ "text/html; charset=UTF-8".to_owned(),
+ )];
+ assert_eq!(decode_html_document(input, &headers).1, "UTF-8");
+}
+
+#[test]
+fn ascii_xml_declaration_rewrites_utf16_and_preserves_replacement() {
+ assert_eq!(
+ decode_html_document(br#"<?xml encoding="UTF-16">"#, &[]).1,
+ "UTF-8"
+ );
+ assert_eq!(
+ decode_html_document(br#"<?xml encoding="ISO-2022-KR">"#, &[]).1,
+ "replacement"
+ );
+}
+
+#[test]
+fn ascii_xml_declaration_can_finish_after_the_prescan_boundary() {
+ let mut input = br#"<?xml version="1.0" "#.to_vec();
+ input.resize(HTML_META_CHARSET_PRESCAN_LIMIT + 1, b' ');
+ input.extend_from_slice(br#"encoding="windows-1251"><p>"#);
+ input.extend_from_slice(&encoding_rs::WINDOWS_1251.encode("ж").0);
+
+ let (text, encoding) = decode_html_document(&input, &[]);
+
+ assert_eq!(encoding, "windows-1251");
+ assert!(text.ends_with("ж"), "decoded as {text:?}");
+}
+
#[test]
fn bom_less_utf16le_non_xml_not_detected() {
let input = utf16le_bytes("<?y>");