Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(encoding): exclude HTML meta from XML encoding selection
ldm014cc9792efa0
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rsindex db8e77016..188351fc6 100644--- a/moli-encoding/src/document.rs+++ b/moli-encoding/src/document.rs@@ -36,6 +36,13 @@ pub fn decode_html_document_with_fallback((output, encoding)}+#[derive(Clone, Copy)]+enum DocumentDeclarationPolicy {+ Html,+ Xml,+ Text,+}+pub struct HtmlDocumentStreamingDecoder {transport_encoding: Option<&'static Encoding>,fallback_encoding: &'static Encoding,@@ -47,7 +54,7 @@ pub struct HtmlDocumentStreamingDecoder {url_hint: Option<String>,decoder: Option<Decoder>,selected_encoding: Option<&'static Encoding>,- sniff_html_declarations: bool,+ declaration_policy: DocumentDeclarationPolicy,}impl HtmlDocumentStreamingDecoder {@@ -68,16 +75,28 @@ impl HtmlDocumentStreamingDecoder {inherited_encoding,Some(url_hint),inherited_encoding.is_none().then_some(detector),+ DocumentDeclarationPolicy::Text,);- decoder.sniff_html_declarations = false;if json && decoder.transport_encoding.is_none() {decoder.transport_encoding = Some(encoding_rs::UTF_8);}decoder}+ /// Decode XML using BOMs, transport charsets, and its initial declaration.+ /// HTML meta elements never select an encoding; undeclared XML uses UTF-8.+ pub fn new_xml_document(headers: &[(String, Vec<u8>)]) -> Self {+ Self::new_with_options(+ headers,+ Some("UTF-8"),+ None,+ None,+ DocumentDeclarationPolicy::Xml,+ )+ }+pub fn new(headers: &[(String, Vec<u8>)]) -> Self {- Self::new_with_options(headers, None, None, None)+ Self::new_with_options(headers, None, None, None, DocumentDeclarationPolicy::Html)}pub fn new_with_legacy_encoding_detector(@@ -85,14 +104,26 @@ impl HtmlDocumentStreamingDecoder {url_hint: &str,detector: LegacyEncodingDetector,) -> Self {- Self::new_with_options(headers, None, Some(url_hint), Some(detector))+ Self::new_with_options(+ headers,+ None,+ Some(url_hint),+ Some(detector),+ DocumentDeclarationPolicy::Html,+ )}pub fn new_with_fallback(headers: &[(String, Vec<u8>)],fallback_encoding: Option<&str>,) -> Self {- Self::new_with_options(headers, fallback_encoding, None, None)+ Self::new_with_options(+ headers,+ fallback_encoding,+ None,+ None,+ DocumentDeclarationPolicy::Html,+ )}fn new_with_options(@@ -100,6 +131,7 @@ impl HtmlDocumentStreamingDecoder {fallback_encoding: Option<&str>,url_hint: Option<&str>,legacy_encoding_detector: Option<LegacyEncodingDetector>,+ declaration_policy: DocumentDeclarationPolicy,) -> Self {Self {transport_encoding: encoding_from_response_headers(headers),@@ -114,7 +146,7 @@ impl HtmlDocumentStreamingDecoder {url_hint: url_hint.map(str::to_owned),decoder: None,selected_encoding: None,- sniff_html_declarations: true,+ declaration_policy,}}@@ -177,15 +209,49 @@ impl HtmlDocumentStreamingDecoder {if let Some(encoding) = self.transport_encoding {return Some(encoding);}- if !self.sniff_html_declarations {- if self.legacy_encoding_detector.is_none() {- return Some(self.fallback_encoding);+ match self.declaration_policy {+ DocumentDeclarationPolicy::Html => self.html_encoding_ready(finishing),+ DocumentDeclarationPolicy::Xml => self.xml_encoding_ready(finishing),+ DocumentDeclarationPolicy::Text => {+ if self.legacy_encoding_detector.is_none() {+ return Some(self.fallback_encoding);+ }+ (finishing || self.sniff_buffer.len() >= 1024).then(|| {+ self.detected_legacy_content_encoding()+ .unwrap_or(self.fallback_encoding)+ })+ }+ }+ }++ fn xml_encoding_ready(&self, finishing: bool) -> Option<&'static Encoding> {+ let bytes = &self.sniff_buffer;+ if let Some(encoding) = encoding_for_document_utf16_xml_prefix(bytes) {+ return Some(encoding);+ }+ if !finishing+ && (bytes_could_still_be_utf16_xml_prefix(bytes) || b"<?xml".starts_with(bytes))+ {+ return None;+ }+ // Only an initial XML declaration can select an encoding. In particular,+ // <?xml-stylesheet ...?> is an ordinary processing instruction.+ if bytes.starts_with(b"<?xml") && matches!(bytes.get(5), Some(b' ' | b'\t' | b'\r' | b'\n'))+ {+ let Some(end) = bytes.iter().position(|byte| *byte == b'>') else {+ return finishing.then_some(self.fallback_encoding);+ };+ if bytes[end - 1] == b'?' {+ return Some(+ encoding_for_document_xml_declaration(&bytes[..=end])+ .unwrap_or(self.fallback_encoding),+ );}- return (finishing || self.sniff_buffer.len() >= 1024).then(|| {- self.detected_legacy_content_encoding()- .unwrap_or(self.fallback_encoding)- });}+ Some(self.fallback_encoding)+ }++ fn html_encoding_ready(&mut self, finishing: bool) -> Option<&'static Encoding> {if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) {return Some(encoding);}diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex 1ce5be37b..b4c0104e7 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -24,6 +24,187 @@ fn unexpected_legacy_encoding_detector(panic!("declared encoding must take precedence over heuristic detection")}+#[test]+fn xml_document_ignores_html_meta_across_chunk_splits() {+ for declaration in ["", "<?xml version='1.0' encoding='UTF-8'?>"] {+ for meta in [+ "<meta charset='windows-1252'/>",+ "<meta http-equiv='content-type' content='text/html; charset=windows-1252'/>",+ ] {+ let source = format!("{declaration}<root>{meta}<value>café</value></root>");+ assert_xml_document_decodes_across_chunk_splits(+ &[],+ source.as_bytes(),+ &source,+ "UTF-8",+ );+ }+ }+}++fn assert_xml_document_decodes_across_chunk_splits(+ headers: &[(String, Vec<u8>)],+ bytes: &[u8],+ source: &str,+ encoding: &str,+) {+ for split in 0..=bytes.len() {+ let mut decoder = HtmlDocumentStreamingDecoder::new_xml_document(headers);+ let mut decoded = decoder.push(&bytes[..split]).concat();+ decoded.push_str(&decoder.push(&bytes[split..]).concat());+ decoded.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(decoded, source, "{encoding}: split {split}");+ assert_eq!(decoder.document_encoding_name(), encoding, "split {split}");+ }+ let mut decoder = HtmlDocumentStreamingDecoder::new_xml_document(headers);+ let mut decoded = String::new();+ for byte in bytes {+ decoded.push_str(&decoder.push(std::slice::from_ref(byte)).concat());+ }+ decoded.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(decoded, source, "{encoding}: single-byte chunks");+ assert_eq!(decoder.document_encoding_name(), encoding);+}++#[test]+fn xml_document_preserves_encoding_precedence_across_chunk_splits() {+ let source = "<?xml version='1.0' encoding='windows-1252'?>\+ <root><meta charset='UTF-8'/><value>café</value></root>";+ let legacy = encoding_rs::WINDOWS_1252.encode(source).0.into_owned();+ assert_xml_document_decodes_across_chunk_splits(&[], &legacy, source, "windows-1252");++ let headers = [(+ "Content-Type".to_owned(),+ b"application/xml; charset=utf-8".to_vec(),+ )];+ assert_xml_document_decodes_across_chunk_splits(&headers, source.as_bytes(), source, "UTF-8");++ let headers = [(+ "Content-Type".to_owned(),+ b"application/xml; charset=windows-1252".to_vec(),+ )];+ let bom_body = [b"\xef\xbb\xbf".as_slice(), source.as_bytes()].concat();+ assert_xml_document_decodes_across_chunk_splits(&headers, &bom_body, source, "UTF-8");++ let source = "<?xml version='1.0' encoding='UTF-16'?>\+ <root><meta charset='windows-1252'/><value>café</value></root>";+ for (bytes, bom, encoding) in [+ (utf16le_bytes(source), [0xff, 0xfe], "UTF-16LE"),+ (utf16be_bytes(source), [0xfe, 0xff], "UTF-16BE"),+ ] {+ assert_xml_document_decodes_across_chunk_splits(&[], &bytes, source, encoding);+ let bom_body = [bom.as_slice(), &bytes].concat();+ assert_xml_document_decodes_across_chunk_splits(&headers, &bom_body, source, encoding);+ }+}++#[test]+fn xml_document_ignores_encoding_attributes_in_processing_instructions() {+ let source = "<?xml-stylesheet encoding='windows-1252'?>\+ <root><value>café</value></root>";+ assert_xml_document_decodes_across_chunk_splits(&[], source.as_bytes(), source, "UTF-8");+}++#[test]+fn xml_document_selects_encoding_at_the_end_of_its_declaration() {+ let mut decoder = HtmlDocumentStreamingDecoder::new_xml_document(&[]);+ let mut decoded = decoder+ .push(b"<?xml version='1.0' encoding='windows-")+ .concat();+ assert_eq!(decoder.selected_encoding_name(), None);+ decoded.push_str(&decoder.push(b"1252'?>").concat());+ assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));+ decoded.push_str(&decoder.push(b"<root>caf\xe9</root>").concat());+ decoded.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(+ decoded,+ "<?xml version='1.0' encoding='windows-1252'?><root>café</root>"+ );+}++#[test]+fn xml_document_finishes_empty_or_incomplete_declarations_with_utf8() {+ for source in [+ "",+ "<",+ "<?xml",+ "<?xml version='1.0' encoding='windows-1252'",+ ] {+ assert_xml_document_decodes_across_chunk_splits(&[], source.as_bytes(), source, "UTF-8");+ }+}++// Chromium a03603fe9af6230a12f1b2fb2c18a7d003a0d937:+// third_party/blink/renderer/core/html/parser/text_resource_decoder_test.cc,+// XMLDeclPieces. Also exercise the XML/Text policies with the same byte stream.+#[test]+fn chromium_xml_declaration_pieces_follow_the_document_policy() {+ let source = b"<?xml encoding='utf-8'?>foo";+ for (mut decoder, expected_encoding) in [+ (HtmlDocumentStreamingDecoder::new(&[]), "UTF-8"),+ (HtmlDocumentStreamingDecoder::new_xml_document(&[]), "UTF-8"),+ (+ HtmlDocumentStreamingDecoder::new_text_document(+ &[],+ "https://example.test/",+ test_legacy_encoding_detector,+ false,+ None,+ ),+ "windows-1252",+ ),+ ] {+ let mut decoded = String::new();+ for byte in source {+ decoded.push_str(&decoder.push(std::slice::from_ref(byte)).concat());+ }+ decoded.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(decoded.as_bytes(), source);+ assert_eq!(decoder.document_encoding_name(), expected_encoding);+ }+}++// Chromium text_resource_decoder_test.cc, BrokenBOMs (revision above).+#[test]+fn chromium_plain_text_broken_boms_are_flushed_with_the_default_encoding() {+ for (bytes, expected) in [+ (b"\xef\xbb".as_slice(), "ï»"),+ (b"\xff", "ÿ"),+ (b"\xfe", "þ"),+ ] {+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(+ &[],+ "https://example.test/",+ test_legacy_encoding_detector,+ false,+ None,+ );+ for byte in bytes {+ assert!(decoder.push(std::slice::from_ref(byte)).is_empty());+ }+ assert_eq!(decoder.finish().as_deref(), Some(expected));+ assert_eq!(decoder.document_encoding_name(), "windows-1252");+ }+}++// Adapt Chromium fast/encoding/bom-in-content{,-utf16}.html to the XML policy.+#[test]+fn chromium_bom_in_content_is_preserved_by_xml_decoding() {+ let source = "<?xml version='1.0'?><root>\u{feff}</root>";+ for (bytes, bom, encoding) in [+ (+ source.as_bytes().to_vec(),+ b"\xef\xbb\xbf".as_slice(),+ "UTF-8",+ ),+ (utf16le_bytes(source), b"\xff\xfe", "UTF-16LE"),+ (utf16be_bytes(source), b"\xfe\xff", "UTF-16BE"),+ ] {+ let bytes = [bom, &bytes].concat();+ assert_xml_document_decodes_across_chunk_splits(&[], &bytes, source, encoding);+ }+}+#[test]fn json_text_document_defaults_to_utf8_across_every_chunk_split() {let input = "{\"name\":\"Gülçek\"}";diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rsindex ab41e0223..3b712d0e0 100644--- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs+++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs@@ -160,9 +160,22 @@ async fn webdriver_classic_xml_suffix_navigation_preserves_namespaces_and_dom_pa#[tokio::test]async fn webdriver_classic_xml_main_and_child_documents_decode_response_bytes() {- const SOURCE: &str = "<feed xmlns='http://www.w3.org/2005/Atom'><title>café</title></feed>";- let legacy_body = b"<feed xmlns='http://www.w3.org/2005/Atom'><title>caf\xe9</title></feed>";+ const SOURCE: &str = "<feed xmlns='http://www.w3.org/2005/Atom'>\+ <meta charset='windows-1252'/><title>café</title></feed>";+ let legacy_body = b"<feed xmlns='http://www.w3.org/2005/Atom'>\+ <meta charset='UTF-8'/><title>caf\xe9</title></feed>";let cases = [+ ("meta", "", SOURCE.as_bytes().to_vec(), "UTF-8"),+ (+ "declared-utf8-meta",+ "",+ [+ b"<?xml version='1.0' encoding='UTF-8'?>".as_slice(),+ SOURCE.as_bytes(),+ ]+ .concat(),+ "UTF-8",+ ),("charset","; charset=windows-1252",@@ -273,6 +286,156 @@ async fn webdriver_classic_xml_main_and_child_documents_decode_response_bytes()classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;}+#[tokio::test]+async fn webdriver_classic_chromium_xml_encoding_scenarios() {+ // Adapted from Chromium a03603fe9af6230a12f1b2fb2c18a7d003a0d937,+ // third_party/blink/web_tests/fast/encoding/{meta-in-xhtml.xhtml,+ // default-xhtml-encoding.xhtml, xml-utf-8-default.xml,+ // external-script-charset.xhtml, dumpAsText/utf-16-no-bom.xml} and+ // http/tests/xmlviewer/dumpAsText/xmlviewer-charset-{cp1251,utf8}.xml.+ const XHTML_NAMESPACE: &str = "http://www.w3.org/1999/xhtml";+ let report_charset = "<body><p id='result'/><script>\+ document.getElementById('result').textContent = document.characterSet;\+ </script></body>";+ let unicode = "—∆∏π𝓐𝖚";+ let cases = [+ (+ "meta-in-xhtml",+ "application/xhtml+xml",+ format!(+ "<?xml version='1.0'?><html xmlns='{XHTML_NAMESPACE}'>\+ <head><meta content='text/html; charset=windows-1251' http-equiv='Content-Type'/></head>\+ {report_charset}</html>"+ ).into_bytes(),+ "UTF-8",+ "UTF-8",+ Some(XHTML_NAMESPACE),+ ),+ (+ "default-xhtml-encoding",+ "application/xhtml+xml",+ format!("<html xmlns='{XHTML_NAMESPACE}'>{report_charset}</html>").into_bytes(),+ "UTF-8",+ "UTF-8",+ Some(XHTML_NAMESPACE),+ ),+ (+ "xml-utf-8-default",+ "application/xml",+ format!(+ "<?xml version='1.0'?><html xmlns='{XHTML_NAMESPACE}'>\+ <body><p id='result'>{unicode}</p></body></html>"+ ).into_bytes(),+ "UTF-8",+ unicode,+ Some(XHTML_NAMESPACE),+ ),+ (+ "external-script-charset",+ "application/xhtml+xml",+ format!(+ "<?xml version='1.0' encoding='koi8-r'?><html xmlns='{XHTML_NAMESPACE}'>\+ <body><p id='result'/><script type='text/javascript' src='/external-script-charset-koi-8.js'>\+ </script></body></html>"+ ).into_bytes(),+ "KOI8-R",+ "PASS",+ Some(XHTML_NAMESPACE),+ ),+ (+ "xmlviewer-charset-cp1251",+ "application/xml",+ b"<?xml version='1.0' encoding='cp1251'?><root>\+ <cp1251 id='result'>SU\xd1\xd1\xc5SS</cp1251></root>".to_vec(),+ "windows-1251",+ "SUССЕSS",+ None,+ ),+ (+ "xmlviewer-charset-utf8",+ "application/xml",+ "<?xml version='1.0' encoding='utf-8'?><root>\+ <utf8 id='result'>SUССЕSS</utf8></root>".as_bytes().to_vec(),+ "UTF-8",+ "SUССЕSS",+ None,+ ),+ (+ "utf-16-no-bom",+ "application/xml",+ "<?xml version='1.0' encoding='UTF-16'?><TEST id='result'>ё</TEST>"+ .encode_utf16().flat_map(u16::to_be_bytes).collect(),+ "UTF-16BE",+ "ё",+ None,+ ),+ ];+ let mut fixture = axum::Router::new();+ for (name, mime, body, _, _, _) in &cases {+ let mime = *mime;+ let body = body.clone();+ fixture = fixture.route(+ &format!("/{name}"),+ axum::routing::get(move || {+ let body = body.clone();+ async move { ([(header::CONTENT_TYPE, mime)], body) }+ }),+ );+ }+ fixture = fixture+ .route(+ "/external-script-charset-koi-8.js",+ axum::routing::get(|| async {+ (+ [(header::CONTENT_TYPE, "text/javascript")],+ b"document.getElementById('result').textContent = ('\xf3' == '\\u0421') ? 'PASS' : 'FAIL';".as_slice(),+ )+ }),+ )+ .route(+ "/parent/{kind}",+ axum::routing::get(+ |axum::extract::Path(kind): axum::extract::Path<String>| async move {+ (+ [(header::CONTENT_TYPE, "text/html; charset=windows-1252")],+ format!("<iframe src='/{kind}'></iframe>"),+ )+ },+ ),+ );+ let (addr, _server) = spawn_dedicated_fixture_server(fixture, "chromium-xml-encoding");+ let app = build_router(test_state());+ let session = classic_request_json(app.clone(), Method::POST, "/session").await;+ let session_id = session["value"]["sessionId"].as_str().unwrap();+ for (name, mime, _, encoding, expected, namespace) in cases {+ for prefix in ["", "parent/"] {+ let navigated = classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/url"),+ json!({"url":format!("http://{addr}/{prefix}{name}")}),+ )+ .await;+ assert_eq!(navigated, json!({"value":null}), "{prefix}{name}");+ let observed = classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/execute/sync"),+ json!({+ "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; const result=d.getElementById('result'); return [result.textContent,d.characterSet,d.contentType,result.namespaceURI];",+ "args": [],+ }),+ ).await;+ assert_eq!(+ observed["value"],+ json!([expected, encoding, mime, namespace]),+ "{prefix}{name}"+ );+ }+ }+ classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;+}+#[tokio::test]async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {const PAYLOAD: &str =diff --git a/moli-renderer-v8/src/document_response_decoder.rs b/moli-renderer-v8/src/document_response_decoder.rsindex 5d291b90e..4acc309b9 100644--- a/moli-renderer-v8/src/document_response_decoder.rs+++ b/moli-renderer-v8/src/document_response_decoder.rs@@ -18,9 +18,7 @@ pub(crate) fn new_document_response_decoder(inherited_encoding,)} else if content_type.is_some_and(is_xml_document_mime) {- // XML declarations still apply; neither parent encoding nor heuristics- // can replace XML's UTF-8 default.- HtmlDocumentStreamingDecoder::new_with_fallback(headers, Some("UTF-8"))+ HtmlDocumentStreamingDecoder::new_xml_document(headers)} else if inherited_encoding.is_some() {HtmlDocumentStreamingDecoder::new_with_fallback(headers, inherited_encoding)} else {@@ -52,6 +50,34 @@ pub(crate) fn decode_document_response(mod tests {use super::*;+ #[test]+ fn xml_responses_ignore_html_meta_encoding_declarations() {+ for mime in [+ "application/xml",+ "text/xml",+ "application/atom+xml",+ "text/example+xml",+ "application/xhtml+xml",+ ] {+ let headers = [("Content-Type".to_owned(), mime.as_bytes().to_vec())];+ for declaration in ["", "<?xml version='1.0' encoding='UTF-8'?>"] {+ let source = format!(+ "{declaration}<html xmlns='http://www.w3.org/1999/xhtml'>\+ <head><meta charset='windows-1252'/></head><body>café</body></html>"+ );+ let (decoded, encoding) = decode_document_response(+ source.as_bytes(),+ &headers,+ Some(mime),+ &Url::parse("https://example.test/document").unwrap(),+ Some("GBK"),+ );+ assert_eq!(decoded, source, "{mime}: {declaration}");+ assert_eq!(encoding, "UTF-8", "{mime}: {declaration}");+ }+ }+ }+#[test]fn json_and_xml_responses_keep_utf8_defaults_with_a_legacy_parent() {for (mime, source) in [diff --git a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rsindex 51ad69720..cffd37554 100644--- a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs+++ b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs@@ -2,8 +2,10 @@ use super::*;#[tokio::test(flavor = "multi_thread")]async fn external_raw_xml_document_decodes_response_bytes() {- const SOURCE: &str = "<feed xmlns='http://www.w3.org/2005/Atom'><title>café</title></feed>";- let legacy_body = b"<feed xmlns='http://www.w3.org/2005/Atom'><title>caf\xe9</title></feed>";+ const SOURCE: &str = "<feed xmlns='http://www.w3.org/2005/Atom'>\+ <meta charset='windows-1252'/><title>café</title></feed>";+ let legacy_body = b"<feed xmlns='http://www.w3.org/2005/Atom'>\+ <meta charset='UTF-8'/><title>caf\xe9</title></feed>";let cases = [("application/atom+xml; charset=windows-1252",