Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(encoding): rewrite meta-declared UTF-16 to UTF-8
Athul Nambiar787fde9723fd
diff --git a/moli-charset-parser/src/prescan.rs b/moli-charset-parser/src/prescan.rsindex 4f9405268..91f32b537 100644--- a/moli-charset-parser/src/prescan.rs+++ b/moli-charset-parser/src/prescan.rs@@ -186,7 +186,27 @@ fn html_token_attr_value(tag: &Tag, name: &str) -> Option<String> {}fn encoding_for_label(label: String) -> Option<&'static Encoding> {- Encoding::for_label(label.trim().as_bytes())+ Encoding::for_label(label.trim().as_bytes()).map(prescan_charset_override)+}++/// Applies the two charset rewrites the prescan performs before returning a+/// label it found on a `meta` element.+///+/// A prescan only ever reaches a `meta` element through ASCII-compatible+/// bytes, so a document that declares a UTF-16 encoding there has necessarily+/// mislabeled itself, and the standard decodes it as UTF-8 instead.+/// `x-user-defined` is likewise rewritten to windows-1252, leaving its+/// private-use mapping to apply only when the transport layer asked for it.+///+/// <https://html.spec.whatwg.org/multipage/parsing.html#prescan-a-byte-stream-to-determine-its-encoding>+fn prescan_charset_override(encoding: &'static Encoding) -> &'static Encoding {+ if encoding == encoding_rs::UTF_16BE || encoding == encoding_rs::UTF_16LE {+ encoding_rs::UTF_8+ } else if encoding == encoding_rs::X_USER_DEFINED {+ encoding_rs::WINDOWS_1252+ } else {+ encoding+ }}fn charset_from_content_type(value: &str) -> Option<String> {diff --git a/moli-charset-parser/src/tests.rs b/moli-charset-parser/src/tests.rsindex 327e3ce52..a1f3047b6 100644--- a/moli-charset-parser/src/tests.rs+++ b/moli-charset-parser/src/tests.rs@@ -135,3 +135,62 @@ fn finish_reports_not_found() {);assert_eq!(parser.finish(), HtmlMetaCharsetScanResult::NotFound);}++#[test]+fn meta_declared_utf16_is_rewritten_to_utf8() {+ for label in ["utf-16", "utf-16le", "utf-16be", "UTF-16BE", " utf-16 "] {+ let input = format!(r#"<meta charset="{label}">"#);+ assert_eq!(+ sniff_html_meta_charset(input.as_bytes()).map(Encoding::name),+ Some("UTF-8"),+ "charset={label}"+ );+ }+}++#[test]+fn meta_declared_x_user_defined_is_rewritten_to_windows1252() {+ assert_eq!(+ sniff_html_meta_charset(br#"<meta charset="x-user-defined">"#).map(Encoding::name),+ Some("windows-1252")+ );+}++#[test]+fn content_type_pragma_charset_is_rewritten_too() {+ assert_eq!(+ sniff_html_meta_charset(+ br#"<meta http-equiv="content-type" content="text/html; charset=utf-16">"#+ )+ .map(Encoding::name),+ Some("UTF-8")+ );+ assert_eq!(+ sniff_html_meta_charset(+ br#"<meta http-equiv="content-type" content="text/html; charset=x-user-defined">"#+ )+ .map(Encoding::name),+ Some("windows-1252")+ );+}++#[test]+fn other_meta_charset_labels_are_left_alone() {+ for (label, expected) in [+ ("utf-8", "UTF-8"),+ ("gbk", "GBK"),+ ("shift_jis", "Shift_JIS"),+ ("iso-8859-1", "windows-1252"),+ ] {+ let input = format!(r#"<meta charset="{label}">"#);+ assert_eq!(+ sniff_html_meta_charset(input.as_bytes()).map(Encoding::name),+ Some(expected),+ "charset={label}"+ );+ }++ // A label outside the Encoding Standard is still no match at all, rather+ // than being rewritten to one of the two replacements above.+ assert_eq!(sniff_html_meta_charset(br#"<meta charset="utf-32">"#), None);+}diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex 1f52e8460..ce7b35fd3 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -447,3 +447,48 @@ fn url_query_encoder_leaves_utf8_and_queryless_inputs_borrowed() {Cow::Borrowed(_)));}++#[test]+fn meta_declared_utf16_document_decodes_as_utf8() {+ let (text, encoding) = decode_html_document(+ br#"<!DOCTYPE html><meta charset="utf-16"><p>Hello, world!</p>"#,+ &[],+ );++ assert_eq!(encoding, "UTF-8");+ assert!(text.contains("Hello, world!"), "decoded as {text:?}");+}++/// `encoding_rs` has no UTF-16 encoder — the Encoding Standard replaces it+/// with UTF-8 for output — so UTF-16LE input has to be built by hand.+fn utf16le_bytes(input: &str) -> Vec<u8> {+ input+ .encode_utf16()+ .flat_map(|unit| unit.to_le_bytes())+ .collect()+}++#[test]+fn meta_declared_utf16_does_not_override_a_bom() {+ let mut input = vec![0xFF, 0xFE];+ input.extend_from_slice(&utf16le_bytes("<meta charset=\"utf-16\">hi"));++ let (text, encoding) = decode_html_document(&input, &[]);++ assert_eq!(encoding, "UTF-16LE");+ assert!(text.ends_with("hi"), "decoded as {text:?}");+}++#[test]+fn transport_utf16_still_wins_over_the_meta_rewrite() {+ let headers = vec![(+ "Content-Type".to_owned(),+ "text/html; charset=utf-16le".to_owned(),+ )];+ let input = utf16le_bytes("<meta charset=\"utf-16\">hi");++ let (text, encoding) = decode_html_document(&input, &headers);++ assert_eq!(encoding, "UTF-16LE");+ assert!(text.ends_with("hi"), "decoded as {text:?}");+}