Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(encoding): extract meta content-type charset per the standard
Athul Nambiar86839b630cc1
diff --git a/moli-charset-parser/src/prescan.rs b/moli-charset-parser/src/prescan.rsindex 91f32b537..f86f7114c 100644--- a/moli-charset-parser/src/prescan.rs+++ b/moli-charset-parser/src/prescan.rs@@ -209,24 +209,68 @@ fn prescan_charset_override(encoding: &'static Encoding) -> &'static Encoding {}}+const CHARSET_KEYWORD: &[u8] = b"charset";++/// Extracts a character encoding label from a `meta` element's `content`+/// attribute.+///+/// This is the HTML Standard's own algorithm rather than a MIME parameter+/// parse: it searches for the literal `charset` anywhere in the value, so a+/// `content` attribute carrying no media type still yields its label, and it+/// terminates an unquoted label at the first ASCII whitespace or `;`. A label+/// opened with a quote must be closed by the same quote, otherwise the+/// attribute contributes nothing.+///+/// <https://html.spec.whatwg.org/multipage/parsing.html#algorithm-for-extracting-a-character-encoding-from-a-meta-element>fn charset_from_content_type(value: &str) -> Option<String> {- for parameter in value.split(';').skip(1) {- let parameter = parameter.trim();- let Some((name, value)) = parameter.split_once('=') else {- continue;- };- if !name.trim().eq_ignore_ascii_case("charset") {+ let bytes = value.as_bytes();+ let mut position = 0;++ loop {+ let offset = bytes[position..]+ .windows(CHARSET_KEYWORD.len())+ .position(|window| window.eq_ignore_ascii_case(CHARSET_KEYWORD))?;+ let keyword_end = position + offset + CHARSET_KEYWORD.len();+ let separator = keyword_end + leading_ascii_whitespace(&bytes[keyword_end..]);++ if bytes.get(separator) != Some(&b'=') {+ // The keyword was not a label assignment, so resume the search+ // just past it rather than rescanning the same match forever.+ position = separator.max(keyword_end);continue;}- let value = value- .trim()- .trim_matches(|ch| ch == '"' || ch == '\'')- .trim();- if !value.is_empty() {- return Some(value.to_owned());++ let label_start = separator + 1;+ let label_start = label_start + leading_ascii_whitespace(&bytes[label_start..]);+ return charset_label_at(value, label_start).map(str::to_owned);+ }+}++fn charset_label_at(value: &str, start: usize) -> Option<&str> {+ let bytes = value.as_bytes();+ match bytes.get(start) {+ Some("e) if quote == b'"' || quote == b'\'' => {+ let quoted_start = start + 1;+ let quoted_len = bytes[quoted_start..]+ .iter()+ .position(|byte| *byte == quote)?;+ Some(&value[quoted_start..quoted_start + quoted_len])+ }+ _ => {+ let end = bytes[start..]+ .iter()+ .position(|byte| byte.is_ascii_whitespace() || *byte == b';')+ .map_or(bytes.len(), |offset| start + offset);+ Some(&value[start..end])}}- None+}++fn leading_ascii_whitespace(bytes: &[u8]) -> usize {+ bytes+ .iter()+ .take_while(|byte| byte.is_ascii_whitespace())+ .count()}fn byte_prescan_input_as_latin1(bytes: &[u8]) -> String {diff --git a/moli-charset-parser/src/tests.rs b/moli-charset-parser/src/tests.rsindex a1f3047b6..9a6a70efb 100644--- a/moli-charset-parser/src/tests.rs+++ b/moli-charset-parser/src/tests.rs@@ -194,3 +194,103 @@ fn other_meta_charset_labels_are_left_alone() {// than being rewritten to one of the two replacements above.assert_eq!(sniff_html_meta_charset(br#"<meta charset="utf-32">"#), None);}++#[test]+fn content_type_charset_is_extracted_by_keyword_not_by_mime_parameter() {+ // The algorithm searches for the keyword; it does not require a media+ // type, and it does not parse MIME parameters.+ assert_eq!(+ sniff_html_meta_charset(br#"<meta http-equiv="content-type" content="charset=utf-8">"#)+ .map(Encoding::name),+ Some("UTF-8")+ );+ // The keyword is matched literally, including inside a longer word.+ assert_eq!(+ sniff_html_meta_charset(+ br#"<meta http-equiv="content-type" content="text/html; xcharset=utf-8">"#+ )+ .map(Encoding::name),+ Some("UTF-8")+ );+ // Whitespace is allowed on either side of the equals sign.+ assert_eq!(+ sniff_html_meta_charset(+ br#"<meta http-equiv="content-type" content="text/html; charset = gbk">"#+ )+ .map(Encoding::name),+ Some("GBK")+ );+}++#[test]+fn unquoted_content_type_charset_ends_at_whitespace_or_semicolon() {+ assert_eq!(+ sniff_html_meta_charset(+ br#"<meta http-equiv="content-type" content="text/html; charset=utf-8 profile=x">"#+ )+ .map(Encoding::name),+ Some("UTF-8")+ );+ assert_eq!(+ sniff_html_meta_charset(+ br#"<meta http-equiv="content-type" content="text/html; charset=utf-8;q=1">"#+ )+ .map(Encoding::name),+ Some("UTF-8")+ );+}++#[test]+fn content_type_charset_requires_matching_quotes() {+ for quoted in [+ &br#"<meta http-equiv="content-type" content="text/html; charset='utf-8'">"#[..],+ &br#"<meta http-equiv="content-type" content='text/html; charset="utf-8"'>"#[..],+ ] {+ assert_eq!(+ sniff_html_meta_charset(quoted).map(Encoding::name),+ Some("UTF-8")+ );+ }+ // Opened but never closed contributes nothing, rather than being read to+ // the end of the value.+ assert_eq!(+ sniff_html_meta_charset(+ br#"<meta http-equiv="content-type" content='text/html; charset="utf-8'>"#+ ),+ None+ );+ // Opened with one quote and closed with the other is equally unmatched.+ assert_eq!(+ sniff_html_meta_charset(+ br#"<meta http-equiv="content-type" content="text/html; charset="utf-8'">"#+ ),+ None+ );+}++#[test]+fn content_type_charset_search_resumes_after_a_bare_keyword() {+ assert_eq!(+ sniff_html_meta_charset(+ br#"<meta http-equiv="content-type" content="charset; charset=gbk">"#+ )+ .map(Encoding::name),+ Some("GBK")+ );+ assert_eq!(+ sniff_html_meta_charset(br#"<meta http-equiv="content-type" content="charset">"#),+ None+ );+}++#[test]+fn empty_content_type_charset_ends_the_extraction() {+ // The algorithm returns the empty label it found; it does not keep+ // searching the same value for a second assignment.+ assert_eq!(+ sniff_html_meta_charset(+ br#"<meta http-equiv="content-type" content="text/html; charset=; charset=gbk">"#+ ),+ None+ );+}