Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(encoding): separate response charset lookup
ldm09b5961086ca2
diff --git a/moli-content-type/src/response.rs b/moli-content-type/src/response.rsindex 1fc52be40..c9a1269bf 100644--- a/moli-content-type/src/response.rs+++ b/moli-content-type/src/response.rs@@ -142,9 +142,11 @@ fn trim_trailing_http_lws(bytes: &[u8], start: usize, mut end: usize) -> usize {}fn trim_http_lws(value: &str) -> &str {- value.trim_matches(|character: char| matches!(character, ' ' | '\t' | '\r' | '\n'))+ value.trim_matches(|character: char| matches!(character, ' ' | '\t'))}fn is_http_lws(byte: u8) -> bool {- matches!(byte, b' ' | b'\t' | b'\r' | b'\n')+ // Chromium's HTTP_LWS is exactly SP | HT and deliberately excludes+ // newlines.+ matches!(byte, b' ' | b'\t')}diff --git a/moli-content-type/src/tests.rs b/moli-content-type/src/tests.rsindex 929bfa561..fc9b5dd4d 100644--- a/moli-content-type/src/tests.rs+++ b/moli-content-type/src/tests.rs@@ -126,6 +126,31 @@ fn response_parser_preserves_parameter_name_and_quote_semantics() {);}+#[test]+fn response_parser_http_lws_is_only_space_and_tab() {+ for newline in ['\r', '\n'] {+ let unquoted = format!("text/html; charset={newline}gbk");+ let quoted = format!("text/html; charset=\"{newline}gbk\"");+ let prefixed_name = format!("text/html;{newline}charset=gbk");+ let expected = format!("{newline}gbk");++ assert_eq!(+ response_charset(&unquoted).as_deref(),+ Some(expected.as_str())+ );+ assert_eq!(+ response_charset("ed).as_deref(),+ Some(expected.as_str())+ );+ assert_eq!(response_charset(&prefixed_name), None);+ }++ assert_eq!(+ response_charset("text/html; charset= \tgbk\t ").as_deref(),+ Some("gbk")+ );+}+#[test]fn response_parser_has_a_distinct_network_level_validity_boundary() {assert!(parse_response_content_type("garbage; charset=utf-8").is_none());diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rsindex b7e3da79b..09098542c 100644--- a/moli-encoding/src/document.rs+++ b/moli-encoding/src/document.rs@@ -1,7 +1,7 @@use encoding_rs::{CoderResult, Decoder, Encoding};use moli_charset_parser::{HtmlMetaCharsetParser, HtmlMetaCharsetScanResult};-use crate::{charset_from_headers, encoding_for_label};+use crate::{encoding_for_label, encoding_from_response_headers};const DEFAULT_HTML_DOCUMENT_ENCODING: &str = "windows-1252";const UTF16LE_XML_PREFIX: &[u8; 6] = b"<\0?\0x\0";@@ -51,9 +51,7 @@ impl HtmlDocumentStreamingDecoder {fallback_encoding: Option<&str>,) -> Self {Self {- transport_encoding: charset_from_headers(headers)- .as_deref()- .and_then(encoding_for_label),+ transport_encoding: encoding_from_response_headers(headers),fallback_encoding: fallback_encoding.and_then(encoding_for_label).unwrap_or(encoding_rs::WINDOWS_1252),diff --git a/moli-encoding/src/labels.rs b/moli-encoding/src/labels.rsindex d59538d82..edc3bd0bf 100644--- a/moli-encoding/src/labels.rs+++ b/moli-encoding/src/labels.rs@@ -1,14 +1,23 @@use encoding_rs::Encoding;use moli_content_type::parse_response_content_type;-/// Looks up an encoding label after the caller has applied its own whitespace-/// rules, matching Blink's exact `TextEncoding` registry lookup.+/// Gets an encoding from a label using the Encoding Standard's preprocessing.+///+/// `encoding_rs` removes leading and trailing ASCII whitespace before matching+/// the label. Use this for web-platform label sources such as a classic+/// script's `charset` attribute.pub fn encoding_for_label(label: &str) -> Option<&'static Encoding> {+ Encoding::for_label(label.as_bytes())+}++/// Looks up a charset extracted from Chromium response transport metadata.+///+/// Chromium's network parser has already removed its HTTP LWS set (SP and HT)+/// before Blink performs an exact encoding-name lookup. `encoding_rs` would+/// apply additional Encoding Standard whitespace preprocessing, so reject any+/// remaining leading or trailing ASCII whitespace first.+pub fn encoding_for_response_charset(label: &str) -> Option<&'static Encoding> {let bytes = label.as_bytes();- // `encoding_rs` implements the Encoding Standard's preprocessing and- // trims ASCII whitespace itself. Blink's response path has already- // removed HTTP LWS, so accepting anything still present here (notably VT- // and FF) would turn an invalid response charset into a valid label.if bytes.first().is_some_and(|byte| byte.is_ascii_whitespace())|| bytes.last().is_some_and(|byte| byte.is_ascii_whitespace()){@@ -35,3 +44,10 @@ pub fn charset_from_headers(headers: &[(String, String)]) -> Option<String> {.find(|(name, _)| name.eq_ignore_ascii_case("content-type")).and_then(|(_, value)| charset_from_content_type(value))}++/// Selects Chromium's transport encoding from HTTP response headers.+pub fn encoding_from_response_headers(headers: &[(String, String)]) -> Option<&'static Encoding> {+ charset_from_headers(headers)+ .as_deref()+ .and_then(encoding_for_response_charset)+}diff --git a/moli-encoding/src/lib.rs b/moli-encoding/src/lib.rsindex 985471af4..b623fb353 100644--- a/moli-encoding/src/lib.rs+++ b/moli-encoding/src/lib.rs@@ -15,7 +15,10 @@ pub use form::{encode_text_for_legacy_web, form_output_encoding, form_output_encoding_for_label,form_submission_encoding, form_urlencoded_serialize_pairs, is_charset_sentinel_name,};-pub use labels::{charset_from_content_type, charset_from_headers, encoding_for_label};+pub use labels::{+ charset_from_content_type, charset_from_headers, encoding_for_label,+ encoding_for_response_charset, encoding_from_response_headers,+};pub use legacy_text::decode_text_for_legacy_web;pub use query::encode_url_query_for_legacy_web;pub use script::decode_classic_script_source;diff --git a/moli-encoding/src/script.rs b/moli-encoding/src/script.rsindex 5c6c58ceb..0dee5e296 100644--- a/moli-encoding/src/script.rs+++ b/moli-encoding/src/script.rs@@ -1,6 +1,6 @@use encoding_rs::Encoding;-use crate::{charset_from_headers, encoding_for_label};+use crate::{encoding_for_label, encoding_from_response_headers};pub fn decode_classic_script_source(bytes: &[u8],@@ -8,10 +8,9 @@ pub fn decode_classic_script_source(script_charset: Option<&str>,document_character_set: Option<&str>,) -> String {- let header_charset = charset_from_headers(headers);let encoding = Encoding::for_bom(bytes).map(|(encoding, _)| encoding)- .or_else(|| header_charset.as_deref().and_then(encoding_for_label))+ .or_else(|| encoding_from_response_headers(headers)).or_else(|| script_charset.and_then(encoding_for_label)).or_else(|| document_character_set.and_then(encoding_for_label)).unwrap_or(encoding_rs::UTF_8);diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex 4b639069e..a8696f378 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -359,6 +359,17 @@ fn classic_script_charset_attribute_is_fallback_before_document_character_set());}+#[test]+fn classic_script_charset_uses_encoding_standard_label_preprocessing() {+ let script = r#"document.body.textContent = "目次";"#;+ let bytes = encoding_rs::SHIFT_JIS.encode(script).0.into_owned();++ assert_eq!(+ decode_classic_script_source(&bytes, &[], Some(" shift_jis "), Some("utf-8")),+ script+ );+}+#[test]fn classic_script_bom_wins_over_labels() {let script = r#"document.body.textContent = "目次";"#;@@ -717,7 +728,7 @@ fn header_charset_matches_chromium_network_tolerances() {assert_eq!(charset_from_content_type(header).as_deref()- .and_then(encoding_for_label)+ .and_then(encoding_for_response_charset).map(Encoding::name),Some("UTF-8"),"header={header}"@@ -737,7 +748,7 @@ fn header_charset_matches_chromium_network_tolerances() {assert!(charset_from_content_type("text/html; charset='utf-8'").as_deref()- .and_then(encoding_for_label)+ .and_then(encoding_for_response_charset).is_none());assert_eq!(@@ -747,8 +758,9 @@ fn header_charset_matches_chromium_network_tolerances() {}#[test]-fn header_charset_does_not_trim_non_http_ascii_whitespace() {- for whitespace in ['\u{000b}', '\u{000c}'] {+fn header_charset_exact_lookup_rejects_non_http_whitespace() {+ for whitespace in ['\u{000b}', '\u{000c}', '\r', '\n'] {+ let expected_label = format!("{whitespace}gbk");let headers = vec![("Content-Type".to_owned(),format!("text/html; charset={whitespace}gbk"),@@ -756,8 +768,9 @@ fn header_charset_does_not_trim_non_http_ascii_whitespace() {assert_eq!(charset_from_headers(&headers).as_deref(),- Some(format!("{whitespace}gbk").as_str())+ Some(expected_label.as_str()));+ assert_eq!(encoding_from_response_headers(&headers), None);assert_eq!(decode_html_document(&gbk_bytes("太平洋"), &headers).1,"windows-1252"@@ -766,8 +779,21 @@ fn header_charset_does_not_trim_non_http_ascii_whitespace() {}#[test]-fn encoding_label_lookup_requires_callers_to_preprocess_whitespace() {+fn encoding_label_lookup_applies_encoding_standard_whitespace_preprocessing() {assert_eq!(encoding_for_label("gbk"), Some(encoding_rs::GBK));+ for label in [" gbk", "gbk ", "\tgbk", "gbk\n", "\u{000c}gbk", "gbk\r"] {+ assert_eq!(+ encoding_for_label(label),+ Some(encoding_rs::GBK),+ "label={label:?}"+ );+ }+ assert_eq!(encoding_for_label("\u{000b}gbk"), None);+}++#[test]+fn response_charset_lookup_is_exact_after_http_lws_preprocessing() {+ assert_eq!(encoding_for_response_charset("gbk"), Some(encoding_rs::GBK));for label in [" gbk","gbk ",@@ -776,7 +802,11 @@ fn encoding_label_lookup_requires_callers_to_preprocess_whitespace() {"\u{000b}gbk","gbk\u{000c}",] {- assert_eq!(encoding_for_label(label), None, "label={label:?}");+ assert_eq!(+ encoding_for_response_charset(label),+ None,+ "label={label:?}"+ );}}diff --git a/moli-protocol/src/domains/page/resource_search.rs b/moli-protocol/src/domains/page/resource_search.rsindex 7d53b0b87..1ec7fb612 100644--- a/moli-protocol/src/domains/page/resource_search.rs+++ b/moli-protocol/src/domains/page/resource_search.rs@@ -5,8 +5,8 @@ use moli_core::page::{SubresourceResourceType,};use moli_encoding::{- charset_from_headers, decode_classic_script_source, decode_html_document_with_fallback,- decode_text_for_legacy_web,+ decode_classic_script_source, decode_html_document_with_fallback, decode_text_for_legacy_web,+ encoding_from_response_headers,};use moli_web_mime::{effective_response_mime_essence, is_dom_parser_xml_mime, is_html_document_mime,@@ -384,18 +384,18 @@ fn decode_resource_content({return decode_classic_script_source(bytes, headers, None, None);}+ let response_charset = encoding_from_response_headers(headers).map(|encoding| encoding.name());if matches!(kind,ResourceContentKind::Subresource(SubresourceResourceType::Stylesheet)) {- return decode_text_for_legacy_web(bytes, charset_from_headers(headers).as_deref());+ return decode_text_for_legacy_web(bytes, response_charset);}if is_dom_parser_xml_mime(&mime) || is_json_module_mime(&mime) {- return decode_text_for_legacy_web(bytes, charset_from_headers(headers).as_deref());+ return decode_text_for_legacy_web(bytes, response_charset);}if is_text_mime_essence(&mime) {- let charset = charset_from_headers(headers);- return decode_text_for_legacy_web(bytes, charset.as_deref().or(Some("windows-1252")));+ return decode_text_for_legacy_web(bytes, response_charset.or(Some("windows-1252")));}BASE64_STANDARD.encode(bytes)}@@ -442,6 +442,23 @@ mod tests {);}+ #[test]+ fn stylesheet_resource_rejects_invalid_response_charset_whitespace() {+ let headers = vec![(+ "content-type".to_owned(),+ "text/css; charset=\nshift_jis".to_owned(),+ )];++ assert_eq!(+ decode_resource_content(+ "body::after { content: '目次'; }".as_bytes(),+ &headers,+ ResourceContentKind::Subresource(SubresourceResourceType::Stylesheet),+ ),+ "body::after { content: '目次'; }"+ );+ }+#[test]fn binary_resource_searches_chromium_style_base64_content() {assert_eq!(