Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(mime): unify JSON/font types and response header extraction
ldm0ce2edbb9e152
diff --git a/Cargo.lock b/Cargo.lockindex 85928ba0c..79dcf5c65 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2647,6 +2647,7 @@ dependencies = ["moli-charset-parser","moli-content-type","moli-header-field",+ "moli-web-mime",][[package]]diff --git a/moli-encoding/Cargo.toml b/moli-encoding/Cargo.tomlindex 4882e295d..c3c7dcf9a 100644--- a/moli-encoding/Cargo.toml+++ b/moli-encoding/Cargo.toml@@ -9,6 +9,7 @@ moli-header-field = { path = "../moli-header-field" }encoding_rs = "0.8"moli-charset-parser = { path = "../moli-charset-parser" }moli-content-type = { path = "../moli-content-type" }+moli-web-mime = { path = "../moli-web-mime" }[lints]workspace = truediff --git a/moli-encoding/src/script.rs b/moli-encoding/src/script.rsindex 406cc748b..d3bd7889d 100644--- a/moli-encoding/src/script.rs+++ b/moli-encoding/src/script.rs@@ -1,6 +1,6 @@use encoding_rs::Encoding;-use crate::{encoding_for_label, encoding_from_response_headers};+use crate::encoding_for_label;pub fn decode_utf8(bytes: &[u8]) -> String {encoding_rs::UTF_8@@ -17,7 +17,10 @@ pub fn decode_classic_script_source() -> String {let encoding = Encoding::for_bom(bytes).map(|(encoding, _)| encoding)- .or_else(|| encoding_from_response_headers(headers))+ .or_else(|| {+ moli_web_mime::extract_response_mime_type(headers)+ .and_then(|mime| mime.parameter("charset").and_then(encoding_for_label))+ }).or_else(|| script_charset.and_then(encoding_for_label)).or_else(|| document_character_set.and_then(encoding_for_label)).unwrap_or(encoding_rs::UTF_8);diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex b4c0104e7..d2621452e 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -763,6 +763,64 @@ fn classic_script_header_charset_wins_over_document_character_set() {);}+#[test]+fn classic_script_charset_uses_the_extracted_mime_record() {+ for (values, expected) in [+ (+ vec!["text/plain; charset=windows-1252", "text/javascript"],+ "€",+ ),+ (+ vec!["text/javascript; charset=windows-1252", "text/javascript"],+ "€",+ ),+ (+ vec![+ "text/javascript; charset=windows-1252",+ "invalid",+ "text/javascript",+ ],+ "€",+ ),+ (+ vec![+ "text/javascript; charset=windows-1252",+ "text/plain",+ "text/javascript",+ ],+ "€",+ ),+ (+ vec![+ "text/javascript; charset=shift_jis",+ "text/javascript; charset=windows-1252",+ ],+ "€",+ ),+ (+ vec!["text/javascript; charset=bogus", "text/javascript"],+ "€",+ ),+ (vec!["text/javascript; charset=\" windows-1252 \""], "€"),+ ] {+ for combined in [false, true] {+ let headers: Vec<_> = if combined {+ vec![("Content-Type".to_owned(), values.join(", ").into_bytes())]+ } else {+ values+ .iter()+ .map(|value| ("Content-Type".to_owned(), value.as_bytes().to_vec()))+ .collect()+ };+ assert_eq!(+ decode_classic_script_source("€".as_bytes(), &headers, None, Some("utf-8")),+ expected,+ "combined={combined}: {values:?}"+ );+ }+ }+}+#[test]fn classic_script_charset_attribute_is_fallback_before_document_character_set() {let script = r#"document.body.textContent = "目次";"#;diff --git a/moli-page-types/src/lib.rs b/moli-page-types/src/lib.rsindex 035247ba0..1b0451822 100644--- a/moli-page-types/src/lib.rs+++ b/moli-page-types/src/lib.rs@@ -36,7 +36,7 @@ use moli_fetch::{NegotiatedHttpVersion, NetworkRequestExtraInfo, NetworkResponseExtraInfo, RedirectInfo,RequestAuth, RequestAuthScheme, RequestAuthTarget, Response, ResponseBody, ResponseHead,};-use moli_web_mime::is_json_module_mime;+use moli_web_mime::{extract_response_mime_essence, is_json_mime};const SUBRESOURCE_RESPONSE_BODY_MEMORY_LIMIT: usize = 1024 * 1024;@@ -1993,10 +1993,10 @@ fn json_path_satisfies(path: &[String],predicate: impl FnOnce(&Value) -> bool,) -> bool {- let Some(content_type) = header_value(headers, "content-type") else {+ let Some(content_type) = extract_response_mime_essence(headers) else {return false;};- if !is_json_module_mime(&content_type) {+ if !is_json_mime(&content_type) {return false;}@@ -2040,16 +2040,6 @@ fn json_value_matches_regex(value: &Value, regex: &Regex) -> bool {}}-fn header_value<'a>(- headers: &'a [(String, Vec<u8>)],- name: &str,-) -> Option<std::borrow::Cow<'a, str>> {- headers- .iter()- .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name))- .map(|(_, value)| moli_fetch::decode_header_value(value))-}-impl SubresourceNetworkRecord {fn from_staged_lifecycle(request: &SubresourceRequestStarted,@@ -4343,11 +4333,32 @@ mod tests {r#"{"ok":true}"#,&expectation,));- assert!(!json_path_equals(+ assert!(json_path_equals(&[("content-type".to_owned(), b"text/json".to_vec())],r#"{"ok":true}"#,&expectation,));+ assert!(!json_path_equals(+ &[("content-type".to_owned(), b"text/plain".to_vec())],+ r#"{"ok":true}"#,+ &expectation,+ ));+ for (values, matches) in [+ (vec!["text/plain", "text/json"], true),+ (vec!["text/plain, text/json"], true),+ (vec!["text/json", "invalid", "*/*"], true),+ (vec!["text/json", "text/plain"], false),+ ] {+ let headers: Vec<_> = values+ .iter()+ .map(|value| ("Content-Type".to_owned(), value.as_bytes().to_vec()))+ .collect();+ assert_eq!(+ json_path_equals(&headers, r#"{"ok":true}"#, &expectation),+ matches,+ "{values:?}"+ );+ }let regex_expectation = SubresourceJsonPathRegex {path: vec!["data".to_owned(), "url".to_owned()],@@ -4361,6 +4372,14 @@ mod tests {r#"{"data":{"url":"/item/42"}}"#,®ex_expectation,));+ assert!(json_path_matches_regex(+ &[(+ "content-type".to_owned(),+ b"Text/JSON; charset=utf-8".to_vec(),+ )],+ r#"{"data":{"url":"/item/42"}}"#,+ ®ex_expectation,+ ));assert!(!json_path_matches_regex(&[("content-type".to_owned(), b"application/json".to_vec())],r#"{"data":{"url":"/orders/42"}}"#,diff --git a/moli-protocol/src/domains/page/resource_search.rs b/moli-protocol/src/domains/page/resource_search.rsindex 7d87eaf53..767fc2fae 100644--- a/moli-protocol/src/domains/page/resource_search.rs+++ b/moli-protocol/src/domains/page/resource_search.rs@@ -10,7 +10,7 @@ use moli_encoding::{};use moli_web_mime::{effective_response_mime_essence, is_dom_parser_xml_mime, is_html_document_mime,- is_javascript_mime_essence, is_json_module_mime, is_text_mime_essence,+ is_javascript_mime_essence, is_json_mime, is_text_mime_essence,};use serde::Deserialize;use serde_json::json;@@ -395,7 +395,7 @@ fn decode_resource_content() {return decode_text_for_legacy_web(bytes, response_charset);}- if is_dom_parser_xml_mime(&mime) || is_json_module_mime(&mime) {+ if is_dom_parser_xml_mime(&mime) || is_json_mime(&mime) {return decode_text_for_legacy_web(bytes, response_charset);}if is_text_mime_essence(&mime) {diff --git a/moli-renderer-v8/src/document_response_decoder.rs b/moli-renderer-v8/src/document_response_decoder.rsindex 4acc309b9..82cc35102 100644--- a/moli-renderer-v8/src/document_response_decoder.rs+++ b/moli-renderer-v8/src/document_response_decoder.rs@@ -1,6 +1,6 @@use moli_encoding::HtmlDocumentStreamingDecoder;use moli_encoding_detector::detect_legacy_html_encoding;-use moli_web_mime::{is_json_document_mime, is_text_document_mime, is_xml_document_mime};+use moli_web_mime::{is_json_mime, is_text_document_mime, is_xml_document_mime};use url::Url;pub(crate) fn new_document_response_decoder(@@ -14,7 +14,7 @@ pub(crate) fn new_document_response_decoder(headers,final_url.as_str(),detect_legacy_html_encoding,- is_json_document_mime(mime),+ is_json_mime(mime),inherited_encoding,)} else if content_type.is_some_and(is_xml_document_mime) {diff --git a/moli-renderer-v8/src/module_runtime/graph.rs b/moli-renderer-v8/src/module_runtime/graph.rsindex 0e57c1fab..e376f1702 100644--- a/moli-renderer-v8/src/module_runtime/graph.rs+++ b/moli-renderer-v8/src/module_runtime/graph.rs@@ -3957,10 +3957,21 @@ import "./c.mjs";}#[tokio::test(flavor = "multi_thread")]- async fn module_graph_fetch_enforces_json_and_css_response_mime() -> anyhow::Result<()> {+ async fn module_graph_fetch_enforces_response_mime_for_module_kinds() -> anyhow::Result<()> {for (kind, mime, body) in [+ (+ ModuleKind::JavaScript,+ "text/javascript",+ "export const answer = 42;",+ ),(ModuleKind::Json, "application/json", r#"{"answer":42}"#),+ (ModuleKind::Json, "text/json", r#"{"answer":42}"#),(ModuleKind::Css, "text/css", "#test { color: red; }"),+ (+ ModuleKind::WebAssembly,+ "application/wasm",+ "\0asm\x01\0\0\0",+ ),] {let cases = [(vec![mime.to_owned()], true),@@ -4018,10 +4029,15 @@ import "./c.mjs";server.await?;assert_eq!(result.is_ok(), accepts, "{kind:?}: {values:?}: {result:?}");if let Err(error) = result {- let expected = if kind == ModuleKind::Json {- "non-JSON module response"- } else {- "non-CSS module response"+ let expected = match kind {+ ModuleKind::JavaScript | ModuleKind::WebAssembly => {+ "unsupported script MIME type"+ }+ ModuleKind::Json => "non-JSON module response",+ ModuleKind::Css => "non-CSS module response",+ ModuleKind::ModulePreloadText => {+ unreachable!("text preloads do not enforce a response MIME type")+ }};assert!(error.to_string().contains(expected), "{error:#}");}diff --git a/moli-renderer-v8/src/module_runtime/response_mime.rs b/moli-renderer-v8/src/module_runtime/response_mime.rsindex ab3422e4d..5d78fffce 100644--- a/moli-renderer-v8/src/module_runtime/response_mime.rs+++ b/moli-renderer-v8/src/module_runtime/response_mime.rs@@ -1,9 +1,9 @@-use moli_web_mime::{extract_response_mime_essence, is_css_mime, is_json_module_mime};+use moli_web_mime::{extract_response_mime_essence, is_css_mime, is_json_mime};pub(crate) fn validate_json_module_response_mime(headers: &[(String, Vec<u8>)],) -> Result<(), String> {- validate_module_response_mime(headers, "JSON", is_json_module_mime)+ validate_module_response_mime(headers, "JSON", is_json_mime)}pub(crate) fn validate_css_module_response_mime(diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs b/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rsindex 1a84f1a04..a53f44811 100644--- a/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs@@ -240,7 +240,7 @@ impl JsContextHost {moli_web_mime::is_html_document_mime(&mime)|| moli_web_mime::is_xml_document_mime(&mime)|| moli_web_mime::is_text_mime(&mime)- || moli_web_mime::is_json_module_mime(&mime)+ || moli_web_mime::is_json_mime(&mime)|| moli_web_mime::is_javascript_mime(&mime)}diff --git a/moli-renderer-v8/src/stylesheet_blocking.rs b/moli-renderer-v8/src/stylesheet_blocking.rsindex 60dca0533..808265df2 100644--- a/moli-renderer-v8/src/stylesheet_blocking.rs+++ b/moli-renderer-v8/src/stylesheet_blocking.rs@@ -421,6 +421,37 @@ mod tests {assert_eq!(response.body_text(), "body { color: red; }");}+ #[test]+ fn validates_stylesheet_response_using_complete_content_type_header_list() {+ let url = Url::parse("https://example.com/app.css").unwrap();+ for (values, accepts) in [+ (vec!["text/plain", "text/css"], true),+ (vec!["text/plain, text/css"], true),+ (vec!["TEXT/CSS; charset=utf-8", "invalid", "*/*"], true),+ (vec!["text/css", "text/plain"], false),+ (vec!["text/css, text/plain"], false),+ (vec![r#"text/plain; a=",text/css""#], false),+ ] {+ for nosniff in [false, true] {+ let mut response = stylesheet_response(&url, None, "body { color: red; }");+ response.headers = values+ .iter()+ .map(|value| ("Content-Type".to_owned(), value.as_bytes().to_vec()))+ .collect();+ if nosniff {+ response+ .headers+ .push(("X-Content-Type-Options".to_owned(), b"nosniff".to_vec()));+ }+ assert_eq!(+ validate_stylesheet_response(&url, response).is_ok(),+ accepts,+ "nosniff={nosniff}: {values:?}"+ );+ }+ }+ }+#[test]fn linked_stylesheet_request_uses_captured_processing_attributes() {let document_url = Url::parse("https://example.com/page").unwrap();diff --git a/moli-renderer-v8/src/worker/module_mime.rs b/moli-renderer-v8/src/worker/module_mime.rsindex 210e72d1f..dbf735670 100644--- a/moli-renderer-v8/src/worker/module_mime.rs+++ b/moli-renderer-v8/src/worker/module_mime.rs@@ -1,4 +1,4 @@-use moli_web_mime::{is_text_mime, response_header_values};+use moli_web_mime::is_text_mime;pub(crate) fn ensure_worker_wasm_module_mime(response: &moli_fetch::Response,@@ -28,7 +28,7 @@ pub(crate) fn ensure_worker_css_module_mime(response: &moli_fetch::Response) ->pub(crate) fn ensure_worker_text_module_mime(response: &moli_fetch::Response,) -> Result<(), String> {- let content_type = worker_module_response_content_type(&response.headers);+ let content_type = super::script_mime::worker_response_content_type(&response.headers);let Some(content_type) = content_type else {return Err("non-text module response for text import attribute: missing Content-Type".to_owned(),@@ -41,9 +41,3 @@ pub(crate) fn ensure_worker_text_module_mime("non-text module response for text import attribute: `{content_type}`"))}--fn worker_module_response_content_type(headers: &[(String, Vec<u8>)]) -> Option<String> {- response_header_values(headers, "content-type")- .into_iter()- .next_back()-}diff --git a/moli-renderer-v8/src/worker/script_mime.rs b/moli-renderer-v8/src/worker/script_mime.rsindex 40080d0c6..b42bf28d1 100644--- a/moli-renderer-v8/src/worker/script_mime.rs+++ b/moli-renderer-v8/src/worker/script_mime.rs@@ -1,5 +1,6 @@use moli_web_mime::{- FetchDestination, ScriptResponseMimeError, check_script_response_mime, response_header_values,+ FetchDestination, ScriptResponseMimeError, check_script_response_mime,+ extract_response_mime_type,};use url::Url;@@ -13,9 +14,7 @@ pub(crate) fn ensure_worker_script_mime_acceptable(}pub(crate) fn worker_response_content_type(headers: &[(String, Vec<u8>)]) -> Option<String> {- response_header_values(headers, "content-type")- .into_iter()- .next_back()+ extract_response_mime_type(headers).map(|mime| mime.to_string())}pub(crate) fn worker_response_has_webassembly_mime(headers: &[(String, Vec<u8>)]) -> bool {@@ -87,4 +86,34 @@ mod tests {assert!(ensure_worker_script_mime_acceptable(&url, &headers, b"").is_err());}++ #[test]+ fn worker_javascript_and_wasm_mime_use_complete_header_lists() {+ let url = Url::parse("https://example.test/worker").unwrap();+ for mime in ["text/javascript", "application/wasm"] {+ for (values, accepts) in [+ (vec!["text/plain".to_owned(), mime.to_owned()], true),+ (vec![format!("text/plain, {mime}")], true),+ (+ vec![mime.to_owned(), "invalid".to_owned(), "*/*".to_owned()],+ true,+ ),+ (vec![mime.to_owned(), "text/plain".to_owned()], false),+ (vec![format!("{mime}, text/plain")], false),+ (vec![format!(r#"text/plain; a=",{mime}""#)], false),+ (vec![], false),+ ] {+ let headers: Vec<_> = values+ .iter()+ .map(|value| ("Content-Type".to_owned(), value.as_bytes().to_vec()))+ .collect();+ let accepted = if mime == "application/wasm" {+ worker_response_has_webassembly_mime(&headers)+ } else {+ ensure_worker_script_mime_acceptable(&url, &headers, b"").is_ok()+ };+ assert_eq!(accepted, accepts, "{mime}: {values:?}");+ }+ }+ }}diff --git a/moli-renderer-v8/src/worker/thread/tests/modules.rs b/moli-renderer-v8/src/worker/thread/tests/modules.rsindex 13ef8b160..ed4ea98b4 100644--- a/moli-renderer-v8/src/worker/thread/tests/modules.rs+++ b/moli-renderer-v8/src/worker/thread/tests/modules.rs@@ -5667,6 +5667,10 @@ async fn worker_importscripts_cross_origin_failure_reports_helper_callsite_from_async fn worker_json_imports_use_shared_response_mime_extraction() {ensure_v8();for (mime, accepts) in [+ ("text/json", true),+ ("TeXt/JsOn; charset=windows-1250", true),+ ("text/plain, text/json", true),+ ("text/json, text/plain", false),("text/plain, application/json", true),("application/json, invalid, */*", true),("application/json, text/plain", false),diff --git a/moli-web-mime/src/classification.rs b/moli-web-mime/src/classification.rsindex 9574cdde2..a50fca152 100644--- a/moli-web-mime/src/classification.rs+++ b/moli-web-mime/src/classification.rs@@ -64,7 +64,7 @@ pub fn is_text_mime_essence(input: &str) -> bool {pub fn is_text_document_mime(input: &str) -> bool {!is_html_document_mime(input)&& !is_xml_document_mime(input)- && (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input))+ && (is_text_mime(input) || is_json_mime(input) || is_javascript_mime(input))}/// Whether a `style` element's raw `type` attribute selects classic CSS.@@ -131,6 +131,16 @@ pub fn is_font_mime(input: &str) -> bool {pub fn is_font_mime_essence(input: &str) -> bool {input.starts_with("font/")+ || matches!(+ input,+ "application/font-cff"+ | "application/font-otf"+ | "application/font-sfnt"+ | "application/font-ttf"+ | "application/font-woff"+ | "application/vnd.ms-fontobject"+ | "application/vnd.ms-opentype"+ )}pub fn is_form_urlencoded_mime(input: &str) -> bool {@@ -147,14 +157,13 @@ pub fn multipart_form_data_boundary(input: &str) -> Option<String> {.flatten()}-pub fn is_json_module_mime(input: &str) -> bool {- mime_essence(input).is_some_and(|mime| mime == "application/json" || mime.ends_with("+json"))+/// The JSON MIME type group shared by JSON modules and documents.+pub fn is_json_mime(input: &str) -> bool {+ mime_essence(input).is_some_and(|mime| is_json_mime_essence(&mime))}-pub fn is_json_document_mime(input: &str) -> bool {- mime_essence(input).is_some_and(|mime| {- matches!(mime.as_str(), "application/json" | "text/json") || mime.ends_with("+json")- })+pub fn is_json_mime_essence(input: &str) -> bool {+ matches!(input, "application/json" | "text/json") || input.ends_with("+json")}pub fn is_webassembly_mime(input: &str) -> bool {@@ -174,8 +183,7 @@ pub fn is_supported_document_mime_type(input: &str) -> bool {|| is_audio_mime_essence(&essence)|| is_video_mime_essence(&essence)|| is_javascript_mime_essence(&essence)- || essence == "application/json"- || essence.ends_with("+json")+ || is_json_mime_essence(&essence)|| matches!(essence.as_str(),"application/xml"diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rsindex bcae3a9eb..4669dfb1b 100644--- a/moli-web-mime/src/lib.rs+++ b/moli-web-mime/src/lib.rs@@ -13,7 +13,7 @@ pub use classification::{is_css_style_element_type_attribute, is_css_stylesheet_type_hint, is_dom_parser_xml_mime,is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime,is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence,- is_json_document_mime, is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime,+ is_json_mime, is_json_mime_essence, is_multipart_form_data_mime, is_png_image_mime,is_png_image_mime_essence, is_supported_document_mime_type, is_svg_image_mime,is_svg_image_mime_essence, is_text_document_mime, is_text_mime, is_text_mime_essence,is_video_mime, is_video_mime_essence, is_webassembly_mime, is_xml_document_mime,diff --git a/moli-web-mime/src/response_policy.rs b/moli-web-mime/src/response_policy.rsindex 6c560dd7b..6dcdcb926 100644--- a/moli-web-mime/src/response_policy.rs+++ b/moli-web-mime/src/response_policy.rs@@ -1,10 +1,10 @@use crate::classification::{is_audio_mime_essence, is_css_mime, is_font_mime_essence, is_image_mime_essence,- is_javascript_mime, is_video_mime_essence,+ is_javascript_mime, is_json_mime_essence, is_video_mime_essence,};use crate::destination::FetchDestination;-use crate::headers::response_header_value;-use crate::parse::{mime_charset, mime_essence};+use crate::headers::{extract_response_mime_essence, extract_response_mime_type};+use crate::parse::mime_charset;use crate::sniffing::{MimeSniffingContext, computed_mime_type, sniff_image_mime_type};pub fn determine_nosniff(headers: &[(String, Vec<u8>)]) -> bool {@@ -27,7 +27,7 @@ pub fn should_response_be_blocked_due_to_nosniff(return false;}- let content_type = response_header_value(headers, "content-type");+ let content_type = extract_response_mime_essence(headers);if destination.is_script_like() {return content_type.as_deref()@@ -42,23 +42,10 @@ pub fn should_response_be_blocked_due_to_nosniff(}pub fn should_opaque_response_be_blocked_by_orb(headers: &[(String, Vec<u8>)]) -> bool {- let content_type = response_header_value(headers, "content-type");- if determine_nosniff(headers)- && content_type- .as_deref()- .is_none_or(|content_type| content_type.trim().is_empty())- {- return true;- }-- let Some(content_type) = content_type else {- return false;- };- let Some(essence) = mime_essence(&content_type) else {+ let Some(essence) = extract_response_mime_essence(headers) else {return determine_nosniff(headers);};- if is_javascript_mime(&content_type) || is_css_mime(&content_type) || essence == "image/svg+xml"- {+ if is_javascript_mime(&essence) || is_css_mime(&essence) || essence == "image/svg+xml" {return false;}@@ -67,9 +54,7 @@ pub fn should_opaque_response_be_blocked_by_orb(headers: &[(String, Vec<u8>)]) -|| essence == "text/xml"|| essence == "application/xml"|| essence == "application/xhtml+xml"- || essence == "application/json"- || essence == "text/json"- || essence.ends_with("+json")+ || is_json_mime_essence(&essence)|| essence == "application/dash+xml"|| essence == "application/gzip"|| essence == "application/x-gzip"@@ -127,14 +112,12 @@ pub fn should_opaque_response_be_blocked_by_orb_with_body(return false;}- let Some(content_type) = response_header_value(headers, "content-type") else {+ let Some(mime) = extract_response_mime_type(headers) else {return true;};- let Some(essence) = mime_essence(&content_type) else {- return true;- };- if is_json_like_mime_essence(&essence)- && response_body_looks_like_orb_allowed_javascript(&content_type, body)+ let essence = mime.essence();+ if is_json_mime_essence(&essence)+ && response_body_looks_like_orb_allowed_javascript(&mime.to_string(), body){return false;}@@ -147,7 +130,7 @@ pub fn computed_response_mime_type(context: MimeSniffingContext,body: &[u8],) -> String {- let content_type = response_header_value(headers, "content-type");+ let content_type = extract_response_mime_type(headers).map(|mime| mime.to_string());computed_mime_type(content_type.as_deref(),determine_nosniff(headers),@@ -176,10 +159,7 @@ pub fn check_script_response_mime(// A script-context sniffing default cannot satisfy an explicit// JavaScript MIME requirement, including when Content-Type is absent// or cannot be parsed.- let supplied_mime_type = response_header_value(headers, "content-type")- .as_deref()- .and_then(mime_essence)- .unwrap_or_default();+ let supplied_mime_type = extract_response_mime_essence(headers).unwrap_or_default();return is_javascript_mime(&supplied_mime_type).then_some(()).ok_or(ScriptResponseMimeError::Unsupported(supplied_mime_type));@@ -196,10 +176,7 @@ pub fn check_script_response_mime(pub fn should_script_like_response_be_blocked_due_to_mime_type(headers: &[(String, Vec<u8>)],) -> bool {- let Some(content_type) = response_header_value(headers, "content-type") else {- return false;- };- let Some(essence) = mime_essence(&content_type) else {+ let Some(essence) = extract_response_mime_essence(headers) else {return false;};is_audio_mime_essence(&essence)@@ -208,10 +185,6 @@ pub fn should_script_like_response_be_blocked_due_to_mime_type(|| essence == "text/csv"}-fn is_json_like_mime_essence(essence: &str) -> bool {- essence == "application/json" || essence == "text/json" || essence.ends_with("+json")-}-fn response_body_looks_like_orb_allowed_javascript(content_type: &str, body: &[u8]) -> bool {let text = decode_orb_script_candidate(content_type, body);let trimmed = text.trim_start_matches(|ch: char| ch.is_whitespace());diff --git a/moli-web-mime/src/tests.rs b/moli-web-mime/src/tests.rsindex d361b05e0..711db5aaa 100644--- a/moli-web-mime/src/tests.rs+++ b/moli-web-mime/src/tests.rs@@ -290,7 +290,7 @@ fn matches_script_and_form_content_types() {assert!(is_video_mime_essence("video/webm"));assert!(is_font_mime("font/woff2"));assert!(is_font_mime_essence("font/ttf"));- assert!(!is_font_mime("application/font-woff"));+ assert!(is_font_mime("application/font-woff"));assert_eq!(media_mime_support("Video/MP4; codecs=\"avc1.42E01E\""),MediaMimeSupport::Probably@@ -314,9 +314,9 @@ fn matches_script_and_form_content_types() {);assert_eq!(multipart_form_data_boundary("text/plain"), None);- assert!(is_json_module_mime("Application/JSON; charset=utf-8"));- assert!(is_json_module_mime("application/manifest+json"));- assert!(!is_json_module_mime("text/json"));+ assert!(is_json_mime("Application/JSON; charset=utf-8"));+ assert!(is_json_mime("application/manifest+json"));+ assert!(is_json_mime("text/json"));assert!(is_media_source_type_supported("video/mp4; codecs=\"avc1.42E01E\""@@ -709,6 +709,45 @@ fn orb_body_sniffing_decodes_utf16_javascript_candidates() {));}+#[test]+fn orb_mime_checks_and_body_decoding_use_the_extracted_response_type() {+ for (values, blocked) in [+ (vec!["text/javascript", "text/json"], true),+ (vec!["text/javascript, text/json"], true),+ (vec!["text/json", "text/javascript"], false),+ (vec!["text/json", "invalid", "*/*"], true),+ ] {+ let headers: Vec<_> = values+ .iter()+ .map(|value| ("Content-Type".to_owned(), value.as_bytes().to_vec()))+ .collect();+ assert_eq!(+ should_opaque_response_be_blocked_by_orb(&headers),+ blocked,+ "{values:?}"+ );+ assert_eq!(+ should_opaque_response_be_blocked_by_orb_with_body(&headers, br#"{"ok":true}"#),+ blocked,+ "{values:?}"+ );+ }+ let headers = [+ ("Content-Type".to_owned(), b"text/plain".to_vec()),+ (+ "Content-Type".to_owned(),+ b"text/json; charset=utf-16, text/json".to_vec(),+ ),+ ];+ let body: Vec<u8> = "\"use strict\";"+ .encode_utf16()+ .flat_map(u16::to_le_bytes)+ .collect();+ assert!(!should_opaque_response_be_blocked_by_orb_with_body(+ &headers, &body+ ));+}+#[test]fn computes_response_mime_type_from_headers_and_body() {let image_without_type =@@ -927,12 +966,157 @@ fn navigation_xml_mime_classification_keeps_dom_parser_allowlist_separate() {}#[test]-fn navigation_json_mime_includes_text_json_without_changing_module_mime() {- for mime in ["application/json", "application/problem+json"] {- assert!(is_json_document_mime(mime), "{mime}");- assert!(is_json_module_mime(mime), "{mime}");+fn json_mime_group_includes_text_json_and_valid_suffixes() {+ for mime in [+ "application/json",+ "application/problem+json",+ "text/html+json",+ "image/svg+json",+ "text/json",+ "Text/JSON; charset=utf-8",+ "text/json; charset=windows-1250",+ "text/json; boundary=something",+ "text/json; foo=bar",+ "text/json; +json",+ ] {+ assert!(is_json_mime(mime), "{mime}");+ }+ for mime in [+ "text/plain",+ "application/javascript",+ "application/jsonp",+ "text/jsonp",+ "application/json+xml",+ "text/plain; json=application/json",+ "invalid+json",+ "text /json",+ "text/ json",+ ] {+ assert!(!is_json_mime(mime), "{mime}");}- assert!(is_json_document_mime("Text/JSON; charset=utf-8"));- assert!(!is_json_module_mime("Text/JSON; charset=utf-8"));- assert!(!is_json_document_mime("text/plain"));+}++#[test]+fn font_mime_group_includes_registered_application_aliases() {+ for mime in [+ "font/woff2",+ "application/font-cff",+ "application/font-otf",+ "application/font-sfnt",+ "application/font-ttf",+ "application/font-woff",+ "application/vnd.ms-fontobject",+ "application/vnd.ms-opentype",+ ] {+ assert!(is_font_mime_essence(mime), "{mime}");+ assert!(is_font_mime(mime), "{mime}");+ assert!(is_font_mime(&format!(+ "{}; version=1",+ mime.to_ascii_uppercase()+ )));+ assert!(is_binary_document_mime_type(mime), "{mime}");+ assert!(+ should_opaque_response_be_blocked_by_orb(&[(+ "Content-Type".to_owned(),+ mime.as_bytes().to_vec(),+ )]),+ "{mime}"+ );+ }+ for mime in [+ "application/font-woff2",+ "application/x-font-ttf",+ "fontish/woff",+ "font/",+ ] {+ assert!(!is_font_mime(mime), "{mime}");+ }+}++#[test]+fn script_and_style_policies_extract_mime_from_the_complete_header_list() {+ for (mime, destination) in [+ ("text/javascript", FetchDestination::Script),+ ("text/javascript", FetchDestination::Worker),+ ("text/css", FetchDestination::Style),+ ] {+ for (values, accepts) in [+ (vec!["text/plain".to_owned(), mime.to_owned()], true),+ (vec![format!("text/plain, {mime}")], true),+ (+ vec![mime.to_owned(), "invalid".to_owned(), "*/*".to_owned()],+ true,+ ),+ (vec![mime.to_owned(), "text/plain".to_owned()], false),+ (vec![format!("{mime}, text/plain")], false),+ (vec![format!(r#"text/plain; a=",{mime}""#)], false),+ (+ vec![r#"text/plain; a=""#.to_owned(), mime.to_owned()],+ false,+ ),+ (vec!["invalid".to_owned(), "*/*".to_owned()], false),+ (vec![], false),+ ] {+ let mut headers: Vec<_> = values+ .iter()+ .map(|value| ("Content-Type".to_owned(), value.as_bytes().to_vec()))+ .collect();+ if destination.is_script_like() {+ assert_eq!(+ check_script_response_mime(&headers, b"", destination, true).is_ok(),+ accepts,+ "{destination:?}: {values:?}"+ );+ }+ headers.push(("X-Content-Type-Options".to_owned(), b"nosniff".to_vec()));+ assert_eq!(+ should_response_be_blocked_due_to_nosniff(&headers, destination),+ !accepts,+ "{destination:?}: {values:?}"+ );+ if destination.is_script_like() {+ assert_eq!(+ check_script_response_mime(&headers, b"", destination, true).is_ok(),+ accepts,+ "{destination:?}: {values:?}"+ );+ }+ }+ }+}++#[test]+fn classic_script_mime_block_uses_extracted_type_and_keeps_fetch_prefix_rules() {+ for blocked in ["audio/mpeg", "image/png", "video/mp4", "text/csv"] {+ for values in [+ vec!["text/javascript".to_owned(), blocked.to_owned()],+ vec![format!("text/javascript, {blocked}")],+ ] {+ let headers: Vec<_> = values+ .iter()+ .map(|value| ("Content-Type".to_owned(), value.as_bytes().to_vec()))+ .collect();+ assert!(+ should_script_like_response_be_blocked_due_to_mime_type(&headers),+ "{values:?}"+ );+ assert_eq!(+ check_script_response_mime(&headers, b"", FetchDestination::Script, false),+ Err(ScriptResponseMimeError::Unsupported(blocked.to_owned())),+ "{values:?}"+ );+ }+ let headers = [(+ "Content-Type".to_owned(),+ format!("{blocked}, text/javascript").into_bytes(),+ )];+ assert!(!should_script_like_response_be_blocked_due_to_mime_type(+ &headers+ ));+ assert!(check_script_response_mime(&headers, b"", FetchDestination::Script, false).is_ok());+ }+ let ogg = [("Content-Type".to_owned(), b"application/ogg".to_vec())];+ assert!(!should_script_like_response_be_blocked_due_to_mime_type(+ &ogg+ ));}