Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(encoding): separate response charset lookup

ldm09b5961086ca2
diff --git a/moli-content-type/src/response.rs b/moli-content-type/src/response.rs
index 1fc52be40..c9a1269bf 100644
--- a/moli-content-type/src/response.rs
+++ b/moli-content-type/src/response.rs
@@ -142,9 +142,11 @@ fn trim_trailing_http_lws(bytes: &[u8], start: usize, mut end: usize) -> usize {
}
fn trim_http_lws(value: &str) -> &str {
- value.trim_matches(|character: char| matches!(character, ' ' | '\t' | '\r' | '\n'))
+ value.trim_matches(|character: char| matches!(character, ' ' | '\t'))
}
fn is_http_lws(byte: u8) -> bool {
- matches!(byte, b' ' | b'\t' | b'\r' | b'\n')
+ // Chromium's HTTP_LWS is exactly SP | HT and deliberately excludes
+ // newlines.
+ matches!(byte, b' ' | b'\t')
}
diff --git a/moli-content-type/src/tests.rs b/moli-content-type/src/tests.rs
index 929bfa561..fc9b5dd4d 100644
--- a/moli-content-type/src/tests.rs
+++ b/moli-content-type/src/tests.rs
@@ -126,6 +126,31 @@ fn response_parser_preserves_parameter_name_and_quote_semantics() {
);
}
+#[test]
+fn response_parser_http_lws_is_only_space_and_tab() {
+ for newline in ['\r', '\n'] {
+ let unquoted = format!("text/html; charset={newline}gbk");
+ let quoted = format!("text/html; charset=\"{newline}gbk\"");
+ let prefixed_name = format!("text/html;{newline}charset=gbk");
+ let expected = format!("{newline}gbk");
+
+ assert_eq!(
+ response_charset(&unquoted).as_deref(),
+ Some(expected.as_str())
+ );
+ assert_eq!(
+ response_charset(&quoted).as_deref(),
+ Some(expected.as_str())
+ );
+ assert_eq!(response_charset(&prefixed_name), None);
+ }
+
+ assert_eq!(
+ response_charset("text/html; charset= \tgbk\t ").as_deref(),
+ Some("gbk")
+ );
+}
+
#[test]
fn response_parser_has_a_distinct_network_level_validity_boundary() {
assert!(parse_response_content_type("garbage; charset=utf-8").is_none());
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs
index b7e3da79b..09098542c 100644
--- a/moli-encoding/src/document.rs
+++ b/moli-encoding/src/document.rs
@@ -1,7 +1,7 @@
use encoding_rs::{CoderResult, Decoder, Encoding};
use moli_charset_parser::{HtmlMetaCharsetParser, HtmlMetaCharsetScanResult};
-use crate::{charset_from_headers, encoding_for_label};
+use crate::{encoding_for_label, encoding_from_response_headers};
const DEFAULT_HTML_DOCUMENT_ENCODING: &str = "windows-1252";
const UTF16LE_XML_PREFIX: &[u8; 6] = b"<\0?\0x\0";
@@ -51,9 +51,7 @@ impl HtmlDocumentStreamingDecoder {
fallback_encoding: Option<&str>,
) -> Self {
Self {
- transport_encoding: charset_from_headers(headers)
- .as_deref()
- .and_then(encoding_for_label),
+ transport_encoding: encoding_from_response_headers(headers),
fallback_encoding: fallback_encoding
.and_then(encoding_for_label)
.unwrap_or(encoding_rs::WINDOWS_1252),
diff --git a/moli-encoding/src/labels.rs b/moli-encoding/src/labels.rs
index d59538d82..edc3bd0bf 100644
--- a/moli-encoding/src/labels.rs
+++ b/moli-encoding/src/labels.rs
@@ -1,14 +1,23 @@
use encoding_rs::Encoding;
use moli_content_type::parse_response_content_type;
-/// Looks up an encoding label after the caller has applied its own whitespace
-/// rules, matching Blink's exact `TextEncoding` registry lookup.
+/// Gets an encoding from a label using the Encoding Standard's preprocessing.
+///
+/// `encoding_rs` removes leading and trailing ASCII whitespace before matching
+/// the label. Use this for web-platform label sources such as a classic
+/// script's `charset` attribute.
pub fn encoding_for_label(label: &str) -> Option<&'static Encoding> {
+ Encoding::for_label(label.as_bytes())
+}
+
+/// Looks up a charset extracted from Chromium response transport metadata.
+///
+/// Chromium's network parser has already removed its HTTP LWS set (SP and HT)
+/// before Blink performs an exact encoding-name lookup. `encoding_rs` would
+/// apply additional Encoding Standard whitespace preprocessing, so reject any
+/// remaining leading or trailing ASCII whitespace first.
+pub fn encoding_for_response_charset(label: &str) -> Option<&'static Encoding> {
let bytes = label.as_bytes();
- // `encoding_rs` implements the Encoding Standard's preprocessing and
- // trims ASCII whitespace itself. Blink's response path has already
- // removed HTTP LWS, so accepting anything still present here (notably VT
- // and FF) would turn an invalid response charset into a valid label.
if bytes.first().is_some_and(|byte| byte.is_ascii_whitespace())
|| bytes.last().is_some_and(|byte| byte.is_ascii_whitespace())
{
@@ -35,3 +44,10 @@ pub fn charset_from_headers(headers: &[(String, String)]) -> Option<String> {
.find(|(name, _)| name.eq_ignore_ascii_case("content-type"))
.and_then(|(_, value)| charset_from_content_type(value))
}
+
+/// Selects Chromium's transport encoding from HTTP response headers.
+pub fn encoding_from_response_headers(headers: &[(String, String)]) -> Option<&'static Encoding> {
+ charset_from_headers(headers)
+ .as_deref()
+ .and_then(encoding_for_response_charset)
+}
diff --git a/moli-encoding/src/lib.rs b/moli-encoding/src/lib.rs
index 985471af4..b623fb353 100644
--- a/moli-encoding/src/lib.rs
+++ b/moli-encoding/src/lib.rs
@@ -15,7 +15,10 @@ pub use form::{
encode_text_for_legacy_web, form_output_encoding, form_output_encoding_for_label,
form_submission_encoding, form_urlencoded_serialize_pairs, is_charset_sentinel_name,
};
-pub use labels::{charset_from_content_type, charset_from_headers, encoding_for_label};
+pub use labels::{
+ charset_from_content_type, charset_from_headers, encoding_for_label,
+ encoding_for_response_charset, encoding_from_response_headers,
+};
pub use legacy_text::decode_text_for_legacy_web;
pub use query::encode_url_query_for_legacy_web;
pub use script::decode_classic_script_source;
diff --git a/moli-encoding/src/script.rs b/moli-encoding/src/script.rs
index 5c6c58ceb..0dee5e296 100644
--- a/moli-encoding/src/script.rs
+++ b/moli-encoding/src/script.rs
@@ -1,6 +1,6 @@
use encoding_rs::Encoding;
-use crate::{charset_from_headers, encoding_for_label};
+use crate::{encoding_for_label, encoding_from_response_headers};
pub fn decode_classic_script_source(
bytes: &[u8],
@@ -8,10 +8,9 @@ pub fn decode_classic_script_source(
script_charset: Option<&str>,
document_character_set: Option<&str>,
) -> String {
- let header_charset = charset_from_headers(headers);
let encoding = Encoding::for_bom(bytes)
.map(|(encoding, _)| encoding)
- .or_else(|| header_charset.as_deref().and_then(encoding_for_label))
+ .or_else(|| encoding_from_response_headers(headers))
.or_else(|| script_charset.and_then(encoding_for_label))
.or_else(|| document_character_set.and_then(encoding_for_label))
.unwrap_or(encoding_rs::UTF_8);
diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index 4b639069e..a8696f378 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -359,6 +359,17 @@ fn classic_script_charset_attribute_is_fallback_before_document_character_set()
);
}
+#[test]
+fn classic_script_charset_uses_encoding_standard_label_preprocessing() {
+ let script = r#"document.body.textContent = "目次";"#;
+ let bytes = encoding_rs::SHIFT_JIS.encode(script).0.into_owned();
+
+ assert_eq!(
+ decode_classic_script_source(&bytes, &[], Some(" shift_jis "), Some("utf-8")),
+ script
+ );
+}
+
#[test]
fn classic_script_bom_wins_over_labels() {
let script = r#"document.body.textContent = "目次";"#;
@@ -717,7 +728,7 @@ fn header_charset_matches_chromium_network_tolerances() {
assert_eq!(
charset_from_content_type(header)
.as_deref()
- .and_then(encoding_for_label)
+ .and_then(encoding_for_response_charset)
.map(Encoding::name),
Some("UTF-8"),
"header={header}"
@@ -737,7 +748,7 @@ fn header_charset_matches_chromium_network_tolerances() {
assert!(
charset_from_content_type("text/html; charset='utf-8'")
.as_deref()
- .and_then(encoding_for_label)
+ .and_then(encoding_for_response_charset)
.is_none()
);
assert_eq!(
@@ -747,8 +758,9 @@ fn header_charset_matches_chromium_network_tolerances() {
}
#[test]
-fn header_charset_does_not_trim_non_http_ascii_whitespace() {
- for whitespace in ['\u{000b}', '\u{000c}'] {
+fn header_charset_exact_lookup_rejects_non_http_whitespace() {
+ for whitespace in ['\u{000b}', '\u{000c}', '\r', '\n'] {
+ let expected_label = format!("{whitespace}gbk");
let headers = vec![(
"Content-Type".to_owned(),
format!("text/html; charset={whitespace}gbk"),
@@ -756,8 +768,9 @@ fn header_charset_does_not_trim_non_http_ascii_whitespace() {
assert_eq!(
charset_from_headers(&headers).as_deref(),
- Some(format!("{whitespace}gbk").as_str())
+ Some(expected_label.as_str())
);
+ assert_eq!(encoding_from_response_headers(&headers), None);
assert_eq!(
decode_html_document(&gbk_bytes("太平洋"), &headers).1,
"windows-1252"
@@ -766,8 +779,21 @@ fn header_charset_does_not_trim_non_http_ascii_whitespace() {
}
#[test]
-fn encoding_label_lookup_requires_callers_to_preprocess_whitespace() {
+fn encoding_label_lookup_applies_encoding_standard_whitespace_preprocessing() {
assert_eq!(encoding_for_label("gbk"), Some(encoding_rs::GBK));
+ for label in [" gbk", "gbk ", "\tgbk", "gbk\n", "\u{000c}gbk", "gbk\r"] {
+ assert_eq!(
+ encoding_for_label(label),
+ Some(encoding_rs::GBK),
+ "label={label:?}"
+ );
+ }
+ assert_eq!(encoding_for_label("\u{000b}gbk"), None);
+}
+
+#[test]
+fn response_charset_lookup_is_exact_after_http_lws_preprocessing() {
+ assert_eq!(encoding_for_response_charset("gbk"), Some(encoding_rs::GBK));
for label in [
" gbk",
"gbk ",
@@ -776,7 +802,11 @@ fn encoding_label_lookup_requires_callers_to_preprocess_whitespace() {
"\u{000b}gbk",
"gbk\u{000c}",
] {
- assert_eq!(encoding_for_label(label), None, "label={label:?}");
+ assert_eq!(
+ encoding_for_response_charset(label),
+ None,
+ "label={label:?}"
+ );
}
}
diff --git a/moli-protocol/src/domains/page/resource_search.rs b/moli-protocol/src/domains/page/resource_search.rs
index 7d53b0b87..1ec7fb612 100644
--- a/moli-protocol/src/domains/page/resource_search.rs
+++ b/moli-protocol/src/domains/page/resource_search.rs
@@ -5,8 +5,8 @@ use moli_core::page::{
SubresourceResourceType,
};
use moli_encoding::{
- charset_from_headers, decode_classic_script_source, decode_html_document_with_fallback,
- decode_text_for_legacy_web,
+ decode_classic_script_source, decode_html_document_with_fallback, decode_text_for_legacy_web,
+ encoding_from_response_headers,
};
use moli_web_mime::{
effective_response_mime_essence, is_dom_parser_xml_mime, is_html_document_mime,
@@ -384,18 +384,18 @@ fn decode_resource_content(
{
return decode_classic_script_source(bytes, headers, None, None);
}
+ let response_charset = encoding_from_response_headers(headers).map(|encoding| encoding.name());
if matches!(
kind,
ResourceContentKind::Subresource(SubresourceResourceType::Stylesheet)
) {
- return decode_text_for_legacy_web(bytes, charset_from_headers(headers).as_deref());
+ return decode_text_for_legacy_web(bytes, response_charset);
}
if is_dom_parser_xml_mime(&mime) || is_json_module_mime(&mime) {
- return decode_text_for_legacy_web(bytes, charset_from_headers(headers).as_deref());
+ return decode_text_for_legacy_web(bytes, response_charset);
}
if is_text_mime_essence(&mime) {
- let charset = charset_from_headers(headers);
- return decode_text_for_legacy_web(bytes, charset.as_deref().or(Some("windows-1252")));
+ return decode_text_for_legacy_web(bytes, response_charset.or(Some("windows-1252")));
}
BASE64_STANDARD.encode(bytes)
}
@@ -442,6 +442,23 @@ mod tests {
);
}
+ #[test]
+ fn stylesheet_resource_rejects_invalid_response_charset_whitespace() {
+ let headers = vec![(
+ "content-type".to_owned(),
+ "text/css; charset=\nshift_jis".to_owned(),
+ )];
+
+ assert_eq!(
+ decode_resource_content(
+ "body::after { content: '目次'; }".as_bytes(),
+ &headers,
+ ResourceContentKind::Subresource(SubresourceResourceType::Stylesheet),
+ ),
+ "body::after { content: '目次'; }"
+ );
+ }
+
#[test]
fn binary_resource_searches_chromium_style_base64_content() {
assert_eq!(