Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(modules): enforce import-map integrity
ldm0416936f73282
diff --git a/moli-parser/src/script_planning.rs b/moli-parser/src/script_planning.rsindex 1c5cbc46c..240373158 100644--- a/moli-parser/src/script_planning.rs+++ b/moli-parser/src/script_planning.rs@@ -291,7 +291,9 @@ impl ScriptFetchMetadata {cross_origin: normalize_cross_origin(cross_origin),referrer_policy: normalize_referrer_policy(referrer_policy),charset: normalize_attr_token(charset),- integrity: normalize_non_empty_attr(integrity),+ // Attribute presence is significant for module scripts: an explicit+ // empty value suppresses integrity metadata supplied by an import map.+ integrity: integrity.map(str::trim).map(str::to_owned),nonce: normalize_non_empty_attr(nonce),fetch_priority: FetchPriorityHint::from_attribute(fetch_priority),parser_inserted: false,@@ -589,6 +591,14 @@ mod tests {assert_eq!(metadata.referrer_policy, None);}+ #[test]+ fn script_fetch_metadata_preserves_explicit_empty_integrity() {+ let metadata =+ ScriptFetchMetadata::from_script_attributes(None, None, None, Some(" "), None, None);++ assert_eq!(metadata.integrity.as_deref(), Some(""));+ }+#[test]fn parser_script_classification_uses_language_when_type_is_absent() {let document = HtmlParser::SCRIPTING_ENABLED.parse(diff --git a/moli-renderer-v8/src/host/scripts.rs b/moli-renderer-v8/src/host/scripts.rsindex 1fdd687a8..f090a27ea 100644--- a/moli-renderer-v8/src/host/scripts.rs+++ b/moli-renderer-v8/src/host/scripts.rs@@ -1518,7 +1518,7 @@ impl HostScriptScheduler {mode: ScriptMode,) -> std::result::Result<PreparedScript, String> {let position = self.next_dynamic_position();- let script = build_runtime_prepared_script(+ let mut script = build_runtime_prepared_script(preparation,NativeNodeId::new(node_id.index()),position,@@ -1528,6 +1528,12 @@ impl HostScriptScheduler {kind,mode,)?;+ if script.kind == ScriptKind::Module+ && script.source_kind == ScriptSourceKind::External+ && script.fetch_metadata.integrity.is_none()+ {+ script.fetch_metadata.integrity = self.resolve_module_integrity(&script.url);+ }if let Some(state) = self.script_handles.get_mut(host_script_handle) {state.followup_lane = Self::followup_lane_for_script(state.source, script.mode);if Self::queued_script_waits_until_dom_content_loaded(diff --git a/moli-renderer-v8/src/host/scripts/tests.rs b/moli-renderer-v8/src/host/scripts/tests.rsindex c0184552a..666da3cea 100644--- a/moli-renderer-v8/src/host/scripts/tests.rs+++ b/moli-renderer-v8/src/host/scripts/tests.rs@@ -237,6 +237,59 @@ fn draining_dynamic_module_scripts_keeps_distinct_module_lane() {assert_eq!(batch.module_in_order[0].mode, ScriptMode::ModuleInOrder);}+#[test]+fn dynamic_external_module_uses_import_map_integrity_when_attribute_is_absent() {+ let preparation = preparation("https://example.test/", NodeId::new(0));+ let mut scheduler = HostScriptScheduler::default();+ scheduler.register_dynamic_import_map(+ &preparation,+ r#"{"integrity":{"/mod.js":"sha384-from-map"}}"#,+ );+ scheduler+ .queue_dynamic_script(+ &preparation,+ "module",+ "/mod.js",+ ScriptSourceKind::External,+ ScriptKind::Module,+ ScriptMode::ModuleInOrder,+ )+ .expect("module script should queue");++ let batch = scheduler.drain_dynamic_scripts();+ assert_eq!(+ batch.module_in_order[0].fetch_metadata.integrity.as_deref(),+ Some("sha384-from-map")+ );+}++#[test]+fn dynamic_external_module_empty_integrity_overrides_import_map() {+ let mut preparation = preparation("https://example.test/", NodeId::new(0));+ preparation.fetch_metadata.integrity = Some(String::new());+ let mut scheduler = HostScriptScheduler::default();+ scheduler.register_dynamic_import_map(+ &preparation,+ r#"{"integrity":{"/mod.js":"sha384-from-map"}}"#,+ );+ scheduler+ .queue_dynamic_script(+ &preparation,+ "module",+ "/mod.js",+ ScriptSourceKind::External,+ ScriptKind::Module,+ ScriptMode::ModuleInOrder,+ )+ .expect("module script should queue");++ let batch = scheduler.drain_dynamic_scripts();+ assert_eq!(+ batch.module_in_order[0].fetch_metadata.integrity.as_deref(),+ Some("")+ );+}+#[test]fn queueing_dynamic_module_does_not_block_later_import_map_merge() {let preparation = preparation("https://example.test/", NodeId::new(0));diff --git a/moli-renderer-v8/src/module_runtime/graph.rs b/moli-renderer-v8/src/module_runtime/graph.rsindex 6a28a08c4..198a297e0 100644--- a/moli-renderer-v8/src/module_runtime/graph.rs+++ b/moli-renderer-v8/src/module_runtime/graph.rs@@ -460,14 +460,24 @@ impl NativeModuleGraphFetchRequest {.message().to_owned());}- crate::subresource_integrity::observe_subresource_integrity_metadata(- integrity.as_deref(),- );let response_referrer_policy =crate::referrer_policy::response_referrer_policy_from_headers(&response.headers,);- let (head, _body, body_bytes) = response.into_parts();+ let (head, _, body_bytes) = response.into_parts();+ if !crate::subresource_integrity::response_body_matches_subresource_integrity_metadata(+ &body_bytes,+ integrity.as_deref(),+ ) {+ return Err(ModuleLoadError::new(+ ModuleLoadStage::Fetch,+ format!(+ "module request `{source_url}` failed its integrity check (FailedToLoad)"+ ),+ )+ .message()+ .to_owned());+ }Ok(match kind {ModuleKind::WebAssembly => ModuleGraphFetchedSource::new(head.final_url,@@ -989,6 +999,7 @@ fn trace_module_graph_job_created(kind: &'static str, root: &ModuleRootInput) {initiator_url = %root.initiator_url,phase = ?root.phase,parser_owned = root.parser_owned,+ integrity = ?root.fetch_metadata.request_metadata.integrity,);}@@ -3788,7 +3799,7 @@ import "./c.mjs";}#[tokio::test(flavor = "multi_thread")]- async fn module_graph_fetch_allows_invalid_integrity() -> anyhow::Result<()> {+ async fn module_graph_fetch_rejects_integrity_mismatch() -> anyhow::Result<()> {let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;let addr = listener.local_addr()?;let server = tokio::spawn(async move {@@ -3809,22 +3820,26 @@ import "./c.mjs";let module_url = Url::parse(&format!("http://{addr}/bad-integrity.js"))?;let metadata = ModuleFetchMetadata {request_metadata: ScriptFetchRequestMetadata {- integrity: Some("sha384-invalid".to_owned()),+ integrity: Some(+ "sha384-Li9vy3DqF8tnTXuiaAJuML3ky+er10rcgNR/VqsVpcw+ThHmYcwiB1pbOxEbzJr7"+ .to_owned(),+ ),..ScriptFetchRequestMetadata::default()},..ModuleFetchMetadata::default()};- let response = fetch_module_for_test(+ let error = fetch_module_for_test(&loader,module_url,Url::parse(&format!("http://{addr}/page.html"))?,metadata,)- .await?;+ .await+ .expect_err("an integrity mismatch must reject the module fetch");assert!(- !response.from_cache,- "module graph fetch should complete normally even with invalid integrity"+ error.to_string().contains("failed its integrity check"),+ "module graph fetch should report an integrity mismatch: {error:#}");server.await?;Ok(())diff --git a/moli-renderer-v8/src/planning.rs b/moli-renderer-v8/src/planning.rsindex 9cba635eb..11ebd32eb 100644--- a/moli-renderer-v8/src/planning.rs+++ b/moli-renderer-v8/src/planning.rs@@ -542,10 +542,15 @@ fn external_script_source_load_outcome_from_response_inner(validate_external_script_response_mime(&script.url, script.kind, &response){Err(error)+ } else if !crate::subresource_integrity::response_body_matches_subresource_integrity_metadata(+ &response_bytes,+ script.fetch_metadata.integrity.as_deref(),+ ) {+ Err(format!(+ "script request `{}` failed its integrity check",+ script.url+ ))} else {- crate::subresource_integrity::observe_subresource_integrity_metadata(- script.fetch_metadata.integrity.as_deref(),- );Ok(decode_external_script_source(script,&response,@@ -1224,6 +1229,49 @@ mod tests {);}+ #[test]+ fn external_script_source_load_enforces_integrity() {+ use base64::Engine as _;++ let body = b"export const loaded = true;";+ let digest = base64::engine::general_purpose::STANDARD+ .encode(moli_crypto::DigestAlgorithm::Sha384.digest_bytes(body));+ let mut script =+ prepared_external_script("https://example.test/module.mjs", ScriptKind::Module);+ script.fetch_metadata.integrity = Some(format!("sha384-{digest}"));++ let matching = external_script_source_load_outcome_from_response(+ &script,+ script_response_with_body_bytes(+ &script.url,+ vec![("Content-Type", "text/javascript")],+ body.to_vec(),+ ),+ None,+ );+ assert_eq!(+ matching.source_result.expect("matching integrity"),+ String::from_utf8_lossy(body)+ );++ script.fetch_metadata.integrity = Some("sha384-foobar".to_owned());+ let mismatch = external_script_source_load_outcome_from_response(+ &script,+ script_response_with_body_bytes(+ &script.url,+ vec![("Content-Type", "text/javascript")],+ body.to_vec(),+ ),+ None,+ );+ assert!(+ mismatch+ .source_result+ .expect_err("mismatching integrity")+ .contains("failed its integrity check")+ );+ }+#[test]fn service_worker_opaque_status_zero_external_script_response_uses_internal_body() {let script =diff --git a/moli-renderer-v8/src/runtime/phase_one/mod.rs b/moli-renderer-v8/src/runtime/phase_one/mod.rsindex 95950f3de..eb81afe46 100644--- a/moli-renderer-v8/src/runtime/phase_one/mod.rs+++ b/moli-renderer-v8/src/runtime/phase_one/mod.rs@@ -2827,6 +2827,46 @@ document.body.setAttribute('data-error-state', [assert!(scanner.finish_script_scan().is_empty());}+ #[test]+ fn html_preload_scanner_stops_module_preloads_after_import_map() {+ let final_url = Url::parse("https://example.test/docs/page.html").expect("test url");+ let requests = collect_preloadable_external_script_requests_from_html(+ &final_url,+ r#"+ <script type="module" src="/before.mjs"></script>+ <script type="importmap">{"integrity": {}}</script>+ <script type="module" src="/after.mjs"></script>+ <script src="/classic.js"></script>+ "#,+ );++ assert_eq!(+ preload_request_urls(requests),+ vec![+ Url::parse("https://example.test/before.mjs").expect("before module url"),+ Url::parse("https://example.test/classic.js").expect("classic url"),+ ]+ );+ }++ #[test]+ fn html_preload_scanner_remembers_import_map_across_chunks() {+ let final_url = Url::parse("https://example.test/docs/page.html").expect("test url");+ let mut scanner = IncrementalHtmlPreloadScanner::new(final_url);++ assert!(+ scanner+ .scan_script_chunk(r#"<script type="importmap">{}</script>"#)+ .is_empty()+ );+ assert!(+ scanner+ .scan_script_chunk(r#"<script type="module" src="/after.mjs"></script>"#)+ .is_empty()+ );+ assert!(scanner.finish_script_scan().is_empty());+ }+#[test]fn incremental_html_preload_scanner_handles_split_stylesheet_tag_boundaries() {let final_url = Url::parse("https://example.test/docs/page.html").expect("test url");diff --git a/moli-renderer-v8/src/runtime/script_preloads.rs b/moli-renderer-v8/src/runtime/script_preloads.rsindex d7b19e3d8..d3bffeacd 100644--- a/moli-renderer-v8/src/runtime/script_preloads.rs+++ b/moli-renderer-v8/src/runtime/script_preloads.rs@@ -1253,6 +1253,7 @@ struct HtmlPreloadScannerSink {meta_csp_mode: MetaCspScannerMode,seen_meta_csp_count: Cell<usize>,reported_meta_csp_count: Cell<usize>,+ seen_import_map: Cell<bool>,template_depth: Cell<usize>,picture_depth: Cell<usize>,}@@ -1268,6 +1269,7 @@ impl HtmlPreloadScannerSink {meta_csp_mode,seen_meta_csp_count: Cell::new(0),reported_meta_csp_count: Cell::new(0),+ seen_import_map: Cell::new(false),template_depth: Cell::new(0),picture_depth: Cell::new(0),}@@ -1302,6 +1304,9 @@ impl HtmlPreloadScannerSink {let kind_hint =buffered_script_kind_from_type(html_attr_value(&tag.attrs, "type").as_deref());+ if matches!(kind_hint, crate::types::ScriptKind::Module) && self.seen_import_map.get() {+ return;+ }if html_attr_present(&tag.attrs, "nomodule")&& matches!(kind_hint, crate::types::ScriptKind::Classic){@@ -1354,6 +1359,15 @@ impl HtmlPreloadScannerSink {}}+ fn maybe_note_import_map(&self, tag: &Tag) {+ if matches!(+ buffered_script_kind_from_type(html_attr_value(&tag.attrs, "type").as_deref()),+ crate::types::ScriptKind::ImportMap+ ) {+ self.seen_import_map.set(true);+ }+ }+fn maybe_update_base_url(&self, tag: &Tag) {if self.has_valid_base.get() {return;@@ -1569,6 +1583,7 @@ impl TokenSink for HtmlPreloadScannerSink {TokenSinkResult::Continue}"script" => {+ self.maybe_note_import_map(&tag);self.maybe_collect_script_preload(&tag);TokenSinkResult::RawData(ScriptData)}diff --git a/moli-renderer-v8/src/subresource_integrity.rs b/moli-renderer-v8/src/subresource_integrity.rsindex 80969fc0a..128eb15dd 100644--- a/moli-renderer-v8/src/subresource_integrity.rs+++ b/moli-renderer-v8/src/subresource_integrity.rs@@ -24,71 +24,85 @@ impl SubresourceIntegrityAlgorithm {struct ParsedIntegrityMetadata {tokens: Vec<ParsedIntegrityToken>,- saw_supported_algorithm: bool,}struct ParsedIntegrityToken {algorithm: SubresourceIntegrityAlgorithm,- expected_digest: Vec<u8>,+ expected_digest: Option<Vec<u8>>,}-pub(crate) fn observe_subresource_integrity_metadata(integrity: Option<&str>) {+pub(crate) fn response_body_matches_subresource_integrity_metadata(+ body: &[u8],+ integrity: Option<&str>,+) -> bool {let Some(integrity) = integrity.map(str::trim).filter(|integrity| !integrity.is_empty())else {- return;+ return true;};let metadata = parse_integrity_metadata(integrity);- let _strongest = metadata+ let Some(strongest_algorithm) = metadata.tokens.iter().map(|token| token.algorithm)- .max_by_key(|algorithm| algorithm.output_len_bytes());- let _saw_supported_algorithm = metadata.saw_supported_algorithm;- let _has_decoded_digest = metadata- .tokens- .iter()- .any(|token| !token.expected_digest.is_empty());+ .max_by_key(|algorithm| algorithm.output_len_bytes())+ else {+ return true;+ };+ let actual_digest = strongest_algorithm.digest_algorithm().digest_bytes(body);+ metadata.tokens.iter().any(|token| {+ token.algorithm == strongest_algorithm+ && token+ .expected_digest+ .as_deref()+ .is_some_and(|expected_digest| expected_digest == actual_digest)+ })}fn parse_integrity_metadata(integrity: &str) -> ParsedIntegrityMetadata {- let mut saw_supported_algorithm = false;let tokens = integrity.split_ascii_whitespace()- .filter_map(|token| {- let parsed = parse_integrity_token(token);- saw_supported_algorithm |= parsed.saw_supported_algorithm;- parsed.token- })+ .filter_map(parse_integrity_token).collect();- ParsedIntegrityMetadata {- tokens,- saw_supported_algorithm,- }+ ParsedIntegrityMetadata { tokens }}-struct ParsedIntegrityTokenResult {- token: Option<ParsedIntegrityToken>,- saw_supported_algorithm: bool,-}--fn parse_integrity_token(token: &str) -> ParsedIntegrityTokenResult {- let Some((algorithm, digest)) = parse_integrity_algorithm_and_digest(token) else {- return ParsedIntegrityTokenResult {- token: None,- saw_supported_algorithm: false,- };- };+fn parse_integrity_token(token: &str) -> Option<ParsedIntegrityToken> {+ let (algorithm, digest) = parse_integrity_algorithm_and_digest(token)?;let digest = digest.split_once('?').map_or(digest, |(digest, _)| digest);+ if !is_integrity_digest_syntax(digest) {+ return None;+ }let expected_digest = decode_integrity_digest(digest);- ParsedIntegrityTokenResult {- token: expected_digest.map(|expected_digest| ParsedIntegrityToken {- algorithm,- expected_digest,- }),- saw_supported_algorithm: true,+ Some(ParsedIntegrityToken {+ algorithm,+ expected_digest,+ })+}++fn is_integrity_digest_syntax(digest: &str) -> bool {+ if digest.is_empty() {+ return false;}+ let mut padding = 0_u8;+ let mut data_characters = 0_usize;+ for byte in digest.bytes() {+ if byte == b'=' {+ padding = padding.saturating_add(1);+ if padding > 2 {+ return false;+ }+ continue;+ }+ if padding != 0+ || !(byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'-' | b'_'))+ {+ return false;+ }+ data_characters += 1;+ }+ data_characters != 0}fn parse_integrity_algorithm_and_digest(@@ -129,10 +143,6 @@ fn decode_integrity_digest(digest: &str) -> Option<Vec<u8>> {mod tests {use super::*;- fn observe(integrity: &str) {- observe_subresource_integrity_metadata(Some(integrity));- }-#[test]fn script_integrity_metadata_parses_matching_supported_hash() {let body = b"console.log('integrity ok')";@@ -141,14 +151,19 @@ mod tests {let integrity = format!("sha384-{digest}");let metadata = parse_integrity_metadata(&integrity);- assert!(metadata.saw_supported_algorithm);assert_eq!(metadata.tokens.len(), 1);assert_eq!(metadata.tokens[0].algorithm,SubresourceIntegrityAlgorithm::Sha384);- assert_eq!(metadata.tokens[0].expected_digest.len(), 48);- observe(&integrity);+ assert_eq!(+ metadata.tokens[0].expected_digest.as_ref().map(Vec::len),+ Some(48)+ );+ assert!(response_body_matches_subresource_integrity_metadata(+ body,+ Some(&integrity)+ ));}#[test]@@ -159,8 +174,14 @@ mod tests {let sha512 = base64::engine::general_purpose::STANDARD_NO_PAD.encode(DigestAlgorithm::Sha512.digest_bytes(body));- observe(&format!("sha256-{sha256}"));- observe(&format!("sha512-{sha512}"));+ assert!(response_body_matches_subresource_integrity_metadata(+ body,+ Some(&format!("sha256-{sha256}"))+ ));+ assert!(response_body_matches_subresource_integrity_metadata(+ body,+ Some(&format!("sha512-{sha512}"))+ ));}#[test]@@ -170,7 +191,6 @@ mod tests {.encode(DigestAlgorithm::Sha384.digest_bytes(body));let metadata = parse_integrity_metadata(&format!("sha-384-{digest}"));- assert!(metadata.saw_supported_algorithm);assert_eq!(metadata.tokens.len(), 1);}@@ -180,22 +200,32 @@ mod tests {let digest = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(DigestAlgorithm::Sha384.digest_bytes(body));- observe(&format!("sha384-{digest}"));+ assert!(response_body_matches_subresource_integrity_metadata(+ body,+ Some(&format!("sha384-{digest}"))+ ));}#[test]fn script_integrity_metadata_ignores_unrecognized_algorithms() {let metadata = parse_integrity_metadata("sha1-this-is-ignored");- assert!(!metadata.saw_supported_algorithm);assert!(metadata.tokens.is_empty());-- observe("sha1-this-is-ignored");}#[test]- fn script_integrity_mismatch_is_non_blocking() {- observe("sha384-doesnotmatch");- observe("sha384-invalid");+ fn script_integrity_matching_response_uses_raw_body_bytes() {+ let body = b"console.log('integrity ok')";+ let digest = base64::engine::general_purpose::STANDARD+ .encode(DigestAlgorithm::Sha384.digest_bytes(body));++ assert!(response_body_matches_subresource_integrity_metadata(+ body,+ Some(&format!("sha384-{digest}"))+ ));+ assert!(!response_body_matches_subresource_integrity_metadata(+ b"console.log('different')",+ Some(&format!("sha384-{digest}"))+ ));}#[test]@@ -213,16 +243,64 @@ mod tests {.map(|token| token.algorithm).max_by_key(|algorithm| algorithm.output_len_bytes());assert_eq!(strongest, Some(SubresourceIntegrityAlgorithm::Sha384));- observe(&integrity);+ assert!(response_body_matches_subresource_integrity_metadata(+ body,+ Some(&integrity)+ ));}#[test]- fn script_integrity_wrong_length_stronger_hash_is_non_blocking() {+ fn script_integrity_rejects_when_strongest_supported_hash_does_not_match() {let body = b"console.log('strongest wrong length')";let weak = base64::engine::general_purpose::STANDARD.encode(DigestAlgorithm::Sha384.digest_bytes(body));let integrity = format!("sha384-{weak} sha512-aW52YWxpZA==");- observe(&integrity);+ assert!(!response_body_matches_subresource_integrity_metadata(+ body,+ Some(&integrity)+ ));+ }++ #[test]+ fn script_integrity_treats_syntactic_but_noncanonical_digest_as_mismatch() {+ assert!(!response_body_matches_subresource_integrity_metadata(+ b"console.log('body')",+ Some("sha384-foobar")+ ));+ }++ #[test]+ fn script_integrity_allows_empty_invalid_or_unsupported_metadata() {+ let body = b"console.log('no supported metadata')";++ assert!(response_body_matches_subresource_integrity_metadata(+ body, None+ ));+ assert!(response_body_matches_subresource_integrity_metadata(+ body,+ Some("")+ ));+ assert!(response_body_matches_subresource_integrity_metadata(+ body,+ Some("sha384-*** sha1-ignored")+ ));+ }++ #[test]+ fn script_integrity_accepts_any_matching_digest_at_strongest_level() {+ let body = b"console.log('one of two')";+ let wrong = base64::engine::general_purpose::STANDARD+ .encode(DigestAlgorithm::Sha512.digest_bytes(b"wrong"));+ let matching = base64::engine::general_purpose::STANDARD+ .encode(DigestAlgorithm::Sha512.digest_bytes(body));+ let weaker = base64::engine::general_purpose::STANDARD+ .encode(DigestAlgorithm::Sha384.digest_bytes(b"wrong"));+ let integrity = format!("sha384-{weaker} sha512-{wrong} sha512-{matching}?ignored");++ assert!(response_body_matches_subresource_integrity_metadata(+ body,+ Some(&integrity)+ ));}}