Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(encoding): rewrite meta-declared UTF-16 to UTF-8

Athul Nambiar787fde9723fd
diff --git a/moli-charset-parser/src/prescan.rs b/moli-charset-parser/src/prescan.rs
index 4f9405268..91f32b537 100644
--- a/moli-charset-parser/src/prescan.rs
+++ b/moli-charset-parser/src/prescan.rs
@@ -186,7 +186,27 @@ fn html_token_attr_value(tag: &Tag, name: &str) -> Option<String> {
}
fn encoding_for_label(label: String) -> Option<&'static Encoding> {
- Encoding::for_label(label.trim().as_bytes())
+ Encoding::for_label(label.trim().as_bytes()).map(prescan_charset_override)
+}
+
+/// Applies the two charset rewrites the prescan performs before returning a
+/// label it found on a `meta` element.
+///
+/// A prescan only ever reaches a `meta` element through ASCII-compatible
+/// bytes, so a document that declares a UTF-16 encoding there has necessarily
+/// mislabeled itself, and the standard decodes it as UTF-8 instead.
+/// `x-user-defined` is likewise rewritten to windows-1252, leaving its
+/// private-use mapping to apply only when the transport layer asked for it.
+///
+/// <https://html.spec.whatwg.org/multipage/parsing.html#prescan-a-byte-stream-to-determine-its-encoding>
+fn prescan_charset_override(encoding: &'static Encoding) -> &'static Encoding {
+ if encoding == encoding_rs::UTF_16BE || encoding == encoding_rs::UTF_16LE {
+ encoding_rs::UTF_8
+ } else if encoding == encoding_rs::X_USER_DEFINED {
+ encoding_rs::WINDOWS_1252
+ } else {
+ encoding
+ }
}
fn charset_from_content_type(value: &str) -> Option<String> {
diff --git a/moli-charset-parser/src/tests.rs b/moli-charset-parser/src/tests.rs
index 327e3ce52..a1f3047b6 100644
--- a/moli-charset-parser/src/tests.rs
+++ b/moli-charset-parser/src/tests.rs
@@ -135,3 +135,62 @@ fn finish_reports_not_found() {
);
assert_eq!(parser.finish(), HtmlMetaCharsetScanResult::NotFound);
}
+
+#[test]
+fn meta_declared_utf16_is_rewritten_to_utf8() {
+ for label in ["utf-16", "utf-16le", "utf-16be", "UTF-16BE", " utf-16 "] {
+ let input = format!(r#"<meta charset="{label}">"#);
+ assert_eq!(
+ sniff_html_meta_charset(input.as_bytes()).map(Encoding::name),
+ Some("UTF-8"),
+ "charset={label}"
+ );
+ }
+}
+
+#[test]
+fn meta_declared_x_user_defined_is_rewritten_to_windows1252() {
+ assert_eq!(
+ sniff_html_meta_charset(br#"<meta charset="x-user-defined">"#).map(Encoding::name),
+ Some("windows-1252")
+ );
+}
+
+#[test]
+fn content_type_pragma_charset_is_rewritten_too() {
+ assert_eq!(
+ sniff_html_meta_charset(
+ br#"<meta http-equiv="content-type" content="text/html; charset=utf-16">"#
+ )
+ .map(Encoding::name),
+ Some("UTF-8")
+ );
+ assert_eq!(
+ sniff_html_meta_charset(
+ br#"<meta http-equiv="content-type" content="text/html; charset=x-user-defined">"#
+ )
+ .map(Encoding::name),
+ Some("windows-1252")
+ );
+}
+
+#[test]
+fn other_meta_charset_labels_are_left_alone() {
+ for (label, expected) in [
+ ("utf-8", "UTF-8"),
+ ("gbk", "GBK"),
+ ("shift_jis", "Shift_JIS"),
+ ("iso-8859-1", "windows-1252"),
+ ] {
+ let input = format!(r#"<meta charset="{label}">"#);
+ assert_eq!(
+ sniff_html_meta_charset(input.as_bytes()).map(Encoding::name),
+ Some(expected),
+ "charset={label}"
+ );
+ }
+
+ // A label outside the Encoding Standard is still no match at all, rather
+ // than being rewritten to one of the two replacements above.
+ assert_eq!(sniff_html_meta_charset(br#"<meta charset="utf-32">"#), None);
+}
diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index 1f52e8460..ce7b35fd3 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -447,3 +447,48 @@ fn url_query_encoder_leaves_utf8_and_queryless_inputs_borrowed() {
Cow::Borrowed(_)
));
}
+
+#[test]
+fn meta_declared_utf16_document_decodes_as_utf8() {
+ let (text, encoding) = decode_html_document(
+ br#"<!DOCTYPE html><meta charset="utf-16"><p>Hello, world!</p>"#,
+ &[],
+ );
+
+ assert_eq!(encoding, "UTF-8");
+ assert!(text.contains("Hello, world!"), "decoded as {text:?}");
+}
+
+/// `encoding_rs` has no UTF-16 encoder — the Encoding Standard replaces it
+/// with UTF-8 for output — so UTF-16LE input has to be built by hand.
+fn utf16le_bytes(input: &str) -> Vec<u8> {
+ input
+ .encode_utf16()
+ .flat_map(|unit| unit.to_le_bytes())
+ .collect()
+}
+
+#[test]
+fn meta_declared_utf16_does_not_override_a_bom() {
+ let mut input = vec![0xFF, 0xFE];
+ input.extend_from_slice(&utf16le_bytes("<meta charset=\"utf-16\">hi"));
+
+ let (text, encoding) = decode_html_document(&input, &[]);
+
+ assert_eq!(encoding, "UTF-16LE");
+ assert!(text.ends_with("hi"), "decoded as {text:?}");
+}
+
+#[test]
+fn transport_utf16_still_wins_over_the_meta_rewrite() {
+ let headers = vec![(
+ "Content-Type".to_owned(),
+ "text/html; charset=utf-16le".to_owned(),
+ )];
+ let input = utf16le_bytes("<meta charset=\"utf-16\">hi");
+
+ let (text, encoding) = decode_html_document(&input, &headers);
+
+ assert_eq!(encoding, "UTF-16LE");
+ assert!(text.ends_with("hi"), "decoded as {text:?}");
+}