Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(encoding): extract meta content-type charset per the standard

Athul Nambiar86839b630cc1
diff --git a/moli-charset-parser/src/prescan.rs b/moli-charset-parser/src/prescan.rs
index 91f32b537..f86f7114c 100644
--- a/moli-charset-parser/src/prescan.rs
+++ b/moli-charset-parser/src/prescan.rs
@@ -209,24 +209,68 @@ fn prescan_charset_override(encoding: &'static Encoding) -> &'static Encoding {
}
}
+const CHARSET_KEYWORD: &[u8] = b"charset";
+
+/// Extracts a character encoding label from a `meta` element's `content`
+/// attribute.
+///
+/// This is the HTML Standard's own algorithm rather than a MIME parameter
+/// parse: it searches for the literal `charset` anywhere in the value, so a
+/// `content` attribute carrying no media type still yields its label, and it
+/// terminates an unquoted label at the first ASCII whitespace or `;`. A label
+/// opened with a quote must be closed by the same quote, otherwise the
+/// attribute contributes nothing.
+///
+/// <https://html.spec.whatwg.org/multipage/parsing.html#algorithm-for-extracting-a-character-encoding-from-a-meta-element>
fn charset_from_content_type(value: &str) -> Option<String> {
- for parameter in value.split(';').skip(1) {
- let parameter = parameter.trim();
- let Some((name, value)) = parameter.split_once('=') else {
- continue;
- };
- if !name.trim().eq_ignore_ascii_case("charset") {
+ let bytes = value.as_bytes();
+ let mut position = 0;
+
+ loop {
+ let offset = bytes[position..]
+ .windows(CHARSET_KEYWORD.len())
+ .position(|window| window.eq_ignore_ascii_case(CHARSET_KEYWORD))?;
+ let keyword_end = position + offset + CHARSET_KEYWORD.len();
+ let separator = keyword_end + leading_ascii_whitespace(&bytes[keyword_end..]);
+
+ if bytes.get(separator) != Some(&b'=') {
+ // The keyword was not a label assignment, so resume the search
+ // just past it rather than rescanning the same match forever.
+ position = separator.max(keyword_end);
continue;
}
- let value = value
- .trim()
- .trim_matches(|ch| ch == '"' || ch == '\'')
- .trim();
- if !value.is_empty() {
- return Some(value.to_owned());
+
+ let label_start = separator + 1;
+ let label_start = label_start + leading_ascii_whitespace(&bytes[label_start..]);
+ return charset_label_at(value, label_start).map(str::to_owned);
+ }
+}
+
+fn charset_label_at(value: &str, start: usize) -> Option<&str> {
+ let bytes = value.as_bytes();
+ match bytes.get(start) {
+ Some(&quote) if quote == b'"' || quote == b'\'' => {
+ let quoted_start = start + 1;
+ let quoted_len = bytes[quoted_start..]
+ .iter()
+ .position(|byte| *byte == quote)?;
+ Some(&value[quoted_start..quoted_start + quoted_len])
+ }
+ _ => {
+ let end = bytes[start..]
+ .iter()
+ .position(|byte| byte.is_ascii_whitespace() || *byte == b';')
+ .map_or(bytes.len(), |offset| start + offset);
+ Some(&value[start..end])
}
}
- None
+}
+
+fn leading_ascii_whitespace(bytes: &[u8]) -> usize {
+ bytes
+ .iter()
+ .take_while(|byte| byte.is_ascii_whitespace())
+ .count()
}
fn byte_prescan_input_as_latin1(bytes: &[u8]) -> String {
diff --git a/moli-charset-parser/src/tests.rs b/moli-charset-parser/src/tests.rs
index a1f3047b6..9a6a70efb 100644
--- a/moli-charset-parser/src/tests.rs
+++ b/moli-charset-parser/src/tests.rs
@@ -194,3 +194,103 @@ fn other_meta_charset_labels_are_left_alone() {
// than being rewritten to one of the two replacements above.
assert_eq!(sniff_html_meta_charset(br#"<meta charset="utf-32">"#), None);
}
+
+#[test]
+fn content_type_charset_is_extracted_by_keyword_not_by_mime_parameter() {
+ // The algorithm searches for the keyword; it does not require a media
+ // type, and it does not parse MIME parameters.
+ assert_eq!(
+ sniff_html_meta_charset(br#"<meta http-equiv="content-type" content="charset=utf-8">"#)
+ .map(Encoding::name),
+ Some("UTF-8")
+ );
+ // The keyword is matched literally, including inside a longer word.
+ assert_eq!(
+ sniff_html_meta_charset(
+ br#"<meta http-equiv="content-type" content="text/html; xcharset=utf-8">"#
+ )
+ .map(Encoding::name),
+ Some("UTF-8")
+ );
+ // Whitespace is allowed on either side of the equals sign.
+ assert_eq!(
+ sniff_html_meta_charset(
+ br#"<meta http-equiv="content-type" content="text/html; charset = gbk">"#
+ )
+ .map(Encoding::name),
+ Some("GBK")
+ );
+}
+
+#[test]
+fn unquoted_content_type_charset_ends_at_whitespace_or_semicolon() {
+ assert_eq!(
+ sniff_html_meta_charset(
+ br#"<meta http-equiv="content-type" content="text/html; charset=utf-8 profile=x">"#
+ )
+ .map(Encoding::name),
+ Some("UTF-8")
+ );
+ assert_eq!(
+ sniff_html_meta_charset(
+ br#"<meta http-equiv="content-type" content="text/html; charset=utf-8;q=1">"#
+ )
+ .map(Encoding::name),
+ Some("UTF-8")
+ );
+}
+
+#[test]
+fn content_type_charset_requires_matching_quotes() {
+ for quoted in [
+ &br#"<meta http-equiv="content-type" content="text/html; charset='utf-8'">"#[..],
+ &br#"<meta http-equiv="content-type" content='text/html; charset="utf-8"'>"#[..],
+ ] {
+ assert_eq!(
+ sniff_html_meta_charset(quoted).map(Encoding::name),
+ Some("UTF-8")
+ );
+ }
+ // Opened but never closed contributes nothing, rather than being read to
+ // the end of the value.
+ assert_eq!(
+ sniff_html_meta_charset(
+ br#"<meta http-equiv="content-type" content='text/html; charset="utf-8'>"#
+ ),
+ None
+ );
+ // Opened with one quote and closed with the other is equally unmatched.
+ assert_eq!(
+ sniff_html_meta_charset(
+ br#"<meta http-equiv="content-type" content="text/html; charset=&quot;utf-8'">"#
+ ),
+ None
+ );
+}
+
+#[test]
+fn content_type_charset_search_resumes_after_a_bare_keyword() {
+ assert_eq!(
+ sniff_html_meta_charset(
+ br#"<meta http-equiv="content-type" content="charset; charset=gbk">"#
+ )
+ .map(Encoding::name),
+ Some("GBK")
+ );
+ assert_eq!(
+ sniff_html_meta_charset(br#"<meta http-equiv="content-type" content="charset">"#),
+ None
+ );
+}
+
+#[test]
+fn empty_content_type_charset_ends_the_extraction() {
+ // The algorithm returns the empty label it found; it does not keep
+ // searching the same value for a second assignment.
+ assert_eq!(
+ sniff_html_meta_charset(
+ br#"<meta http-equiv="content-type" content="text/html; charset=; charset=gbk">"#
+ ),
+ None
+ );
+}