Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
feat: implement Web Bot Auth signing
ldm0ce7fa9cd172e
diff --git a/Cargo.lock b/Cargo.lockindex 376d8fd34..972d6cb58 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2221,6 +2221,7 @@ dependencies = ["moli-trace","moli-url","moli-url-policy",+ "moli-web-bot-auth","moli-web-mime","openssl-sys","parking_lot",@@ -2825,6 +2826,17 @@ dependencies = ["v8",]+[[package]]+name = "moli-web-bot-auth"+version = "0.1.0"+dependencies = [+ "anyhow",+ "aws-lc-rs",+ "base64 0.22.1",+ "moli-crypto",+ "url",+]+[[package]]name = "moli-web-errors"version = "0.1.0"diff --git a/Cargo.toml b/Cargo.tomlindex f77ec5928..9e3855530 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -55,6 +55,7 @@ members = ["moli-v8-test-util","moli-v8-util","moli-web-mime",+ "moli-web-bot-auth","moli-web-errors","moli-protocol-webdriver-bidi","moli-protocol-webdriver-classic",diff --git a/moli-crypto/src/ed25519.rs b/moli-crypto/src/ed25519.rsnew file mode 100644index 000000000..0736c0de9--- /dev/null+++ b/moli-crypto/src/ed25519.rs@@ -0,0 +1,98 @@+use std::fmt;++use aws_lc_rs::signature::{Ed25519KeyPair, KeyPair};++const ED25519_PUBLIC_KEY_LENGTH: usize = 32;+const ED25519_SIGNATURE_LENGTH: usize = 64;++pub struct Ed25519SigningKey {+ key_pair: Ed25519KeyPair,+ public_key: [u8; ED25519_PUBLIC_KEY_LENGTH],+}++impl Ed25519SigningKey {+ pub fn from_pkcs8(der: &[u8]) -> Result<Self, Ed25519Error> {+ let key_pair = Ed25519KeyPair::from_pkcs8(der).map_err(|_| Ed25519Error)?;+ let public_key = key_pair+ .public_key()+ .as_ref()+ .try_into()+ .map_err(|_| Ed25519Error)?;+ Ok(Self {+ key_pair,+ public_key,+ })+ }++ pub fn public_key(&self) -> &[u8; ED25519_PUBLIC_KEY_LENGTH] {+ &self.public_key+ }++ pub fn sign(&self, message: &[u8]) -> Result<[u8; ED25519_SIGNATURE_LENGTH], Ed25519Error> {+ self.key_pair+ .try_sign(message)+ .map_err(|_| Ed25519Error)?+ .as_ref()+ .try_into()+ .map_err(|_| Ed25519Error)+ }+}++impl fmt::Debug for Ed25519SigningKey {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter+ .debug_struct("Ed25519SigningKey")+ .field("public_key", &self.public_key)+ .finish_non_exhaustive()+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct Ed25519Error;++impl fmt::Display for Ed25519Error {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter.write_str("invalid Ed25519 key or signing operation failed")+ }+}++impl std::error::Error for Ed25519Error {}++#[cfg(test)]+mod tests {+ use super::*;++ const RFC_9421_PRIVATE_KEY_DER: &[u8] = &[+ 0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04,+ 0x20, 0x9f, 0x83, 0x62, 0xf8, 0x7a, 0x48, 0x4a, 0x95, 0x4e, 0x6e, 0x74, 0x0c, 0x5b, 0x4c,+ 0x0e, 0x84, 0x22, 0x91, 0x39, 0xa2, 0x0a, 0xa8, 0xab, 0x56, 0xff, 0x66, 0x58, 0x6f, 0x6a,+ 0x7d, 0x29, 0xc5,+ ];++ #[test]+ fn imports_pkcs8_and_signs_deterministically() {+ let key = Ed25519SigningKey::from_pkcs8(RFC_9421_PRIVATE_KEY_DER).unwrap();+ assert_eq!(+ key.public_key(),+ &[+ 0x26, 0xb4, 0x0b, 0x8f, 0x93, 0xff, 0xf3, 0xd8, 0x97, 0x11, 0x2f, 0x7e, 0xbc, 0x58,+ 0x2b, 0x23, 0x2d, 0xbd, 0x72, 0x51, 0x7d, 0x08, 0x2f, 0xe8, 0x3c, 0xfb, 0x30, 0xdd,+ 0xce, 0x43, 0xd1, 0xbb,+ ]+ );++ let first = key.sign(b"web bot auth").unwrap();+ let second = key.sign(b"web bot auth").unwrap();+ assert_eq!(first.len(), ED25519_SIGNATURE_LENGTH);+ assert_eq!(first, second);+ assert_ne!(first, key.sign(b"different message").unwrap());+ }++ #[test]+ fn rejects_non_ed25519_pkcs8() {+ assert_eq!(+ Ed25519SigningKey::from_pkcs8(b"not a key").unwrap_err(),+ Ed25519Error+ );+ }+}diff --git a/moli-crypto/src/lib.rs b/moli-crypto/src/lib.rsindex 891b8fae3..c52fdb80c 100644--- a/moli-crypto/src/lib.rs+++ b/moli-crypto/src/lib.rs@@ -5,7 +5,9 @@//! owner (`moli-webcrypto` or the renderer subsystem using the primitive).mod digest;+mod ed25519;mod random;pub use digest::{DigestAlgorithm, Sha256Context, sha1_digest, sha256_digest, sha256_hex};+pub use ed25519::{Ed25519Error, Ed25519SigningKey};pub use random::fill_secure_random;diff --git a/moli-fetch/Cargo.toml b/moli-fetch/Cargo.tomlindex b4f57e7f7..87542af5c 100644--- a/moli-fetch/Cargo.toml+++ b/moli-fetch/Cargo.toml@@ -23,6 +23,7 @@ moli-http-cache = { path = "../moli-http-cache" }moli-trace = { path = "../moli-trace" }moli-url = { path = "../moli-url" }moli-url-policy = { path = "../moli-url-policy" }+moli-web-bot-auth = { path = "../moli-web-bot-auth" }moli-web-mime = { path = "../moli-web-mime" }parking_lot = "0.12"strum = { version = "0.28.0", features = ["derive"] }diff --git a/moli-fetch/src/blocking/cache/key.rs b/moli-fetch/src/blocking/cache/key.rsindex a4801c2a5..00c847732 100644--- a/moli-fetch/src/blocking/cache/key.rs+++ b/moli-fetch/src/blocking/cache/key.rs@@ -19,6 +19,7 @@ pub(super) fn cache_store_and_key_for_request(|| !request.method.eq_ignore_ascii_case("GET")|| request.body.is_some()|| request.auth().is_some()+ || config.web_bot_auth().is_some()|| !config.default_request_headers().is_empty()|| !subresource_validation_allows_http_cache(request)|| cookie_header.is_some()diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rsindex ffd176d00..42a5748d7 100644--- a/moli-fetch/src/blocking/mod.rs+++ b/moli-fetch/src/blocking/mod.rs@@ -244,21 +244,12 @@ pub(crate) fn outgoing_request_headers_for_url(outgoing}-pub(crate) fn network_request_extra_info_for_url(+pub(crate) fn network_request_extra_info_from_headers(config: &FetchConfig,- request: &Request,- request_url: &Url,- redirect_chain: &[RedirectInfo],- cookie_header: Option<&str>,+ outgoing_headers: &[(String, String)],cookie_report: Option<&StoredCookieQueryReport>,) -> NetworkRequestExtraInfo {- let mut headers = outgoing_request_headers_for_url(- config,- request,- request_url,- redirect_chain,- cookie_header,- );+ let mut headers = outgoing_headers.to_vec();append_header_if_missing(&mut headers, "User-Agent", config.user_agent().to_owned());NetworkRequestExtraInfo {headers,@@ -599,7 +590,7 @@ pub(crate) fn configure_easy<H: Handler>(cookie_header: Option<&str>,http_version: RequestHttpVersion,validation_headers: Option<Vec<(String, String)>>,-) -> Result<()> {+) -> Result<Vec<(String, String)>> {ensure_http_network_transport_url(request_url)?;enforce_request_target_policy(config, request_url)?;configure_curl_http_protocol_allowlist(easy)?;@@ -712,17 +703,22 @@ pub(crate) fn configure_easy<H: Handler>(}let mut headers = List::new();- let outgoing_headers = outgoing_request_headers_for_url(+ let mut outgoing_headers = outgoing_request_headers_for_url(config,request,request_url,redirect_chain,cookie_header,);+ if let Some(web_bot_auth) = config.web_bot_auth() {+ web_bot_auth+ .append_request_headers(&mut outgoing_headers, &request.method, request_url)+ .with_context(|| anyhow!("failed to sign web bot auth request for {request_url}"))?;+ }let mut has_headers = false;let mut has_content_type_header = false;- for (name, value) in outgoing_headers {+ for (name, value) in &outgoing_headers {has_content_type_header |= name.eq_ignore_ascii_case("content-type");let header_line = if value.is_empty() {format!("{name}:")@@ -797,7 +793,7 @@ pub(crate) fn configure_easy<H: Handler>(}}- Ok(())+ Ok(outgoing_headers)}pub(crate) fn configure_openssl_tls_context(diff --git a/moli-fetch/src/config.rs b/moli-fetch/src/config.rsindex 179140ef4..523568466 100644--- a/moli-fetch/src/config.rs+++ b/moli-fetch/src/config.rs@@ -3,6 +3,8 @@ use std::num::NonZeroU32;use cidr::AnyIpCidr;use moli_browser_profile::{BrowserIdentityProfile, DEFAULT_ACCEPT_LANGUAGE};+use crate::WebBotAuthSigner;+#[derive(Debug, Clone, PartialEq)]pub struct FetchConfig {browser_identity: BrowserIdentityProfile,@@ -34,6 +36,7 @@ pub struct FetchConfig {block_private_networks: bool,block_cidrs: Vec<AnyIpCidr>,tls_verify_host: bool,+ web_bot_auth: Option<WebBotAuthSigner>,}impl FetchConfig {@@ -144,6 +147,14 @@ impl FetchConfig {self.tls_verify_host = tls_verify_host;}+ pub fn web_bot_auth(&self) -> Option<&WebBotAuthSigner> {+ self.web_bot_auth.as_ref()+ }++ pub fn set_web_bot_auth(&mut self, web_bot_auth: Option<WebBotAuthSigner>) {+ self.web_bot_auth = web_bot_auth;+ }+pub fn set_http_proxy(&mut self, http_proxy: Option<String>) {self.http_proxy = http_proxy;}@@ -274,6 +285,7 @@ impl Default for FetchConfig {block_private_networks: false,block_cidrs: Vec::new(),tls_verify_host: true,+ web_bot_auth: None,}}}diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rsindex 34247e14b..95b055bfe 100644--- a/moli-fetch/src/lib.rs+++ b/moli-fetch/src/lib.rs@@ -45,6 +45,7 @@ pub use headers::{is_forbidden_response_header_name, is_no_cors_safelisted_request_header,};pub use moli_cookie_jar::SharedBrowserCookieStore as SharedCookieStore;+pub use moli_web_bot_auth::{WebBotAuthProfile, WebBotAuthSigner};pub use network_fetch_result::{NetworkExchangeObservation, NetworkFetchFailure, NetworkFetchFailureRequestContext,NetworkFetchResult, NetworkObservationJournal, NetworkRequestObservation,diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex de50acd9d..771312bf3 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -45,7 +45,7 @@ use crate::{cookie_access_report_for_request, cookie_header_from_report,finish_streaming_cached_response, load_cached_streaming_response_lookup,log_request_completion, merge_cached_not_modified_streaming_response_lookup,- network_request_extra_info_for_url, next_followed_redirect_url_from_parts,+ network_request_extra_info_from_headers, next_followed_redirect_url_from_parts,remove_cached_response, response_headers_forbid_cache_storage, store_response_cookies,transfer_metrics_from_easy, validation_headers_for_cached_streaming_response_lookup,},@@ -863,22 +863,6 @@ impl RuntimeOwner {&job.request,&job.current_url,);- let request_extra_info = job.request.is_top_level_navigation_request().then(|| {- network_request_extra_info_for_url(- &self.config,- &prepared_request.request,- &job.current_url,- &job.redirect_chain,- cookie_header.as_deref(),- request_cookie_report.as_ref(),- )- });- attach_next_request_extra_info(- &mut job.redirect_chain,- request_cookie_report.clone(),- request_extra_info.as_ref(),- );-let mut easy = Easy2::new(FetchTransferHandler::new_buffered(ResponseCollector::new(Some(job.cancel_handle.clone()),)));@@ -894,7 +878,7 @@ impl RuntimeOwner {) {return Err((job.response_tx, error));}- if let Err(error) = configure_easy(+ let outgoing_headers = match configure_easy(&mut easy,&self.config,&prepared_request.request,@@ -909,8 +893,21 @@ impl RuntimeOwner {).with_context(|| anyhow!("failed to configure curl request for {}", job.current_url)){- return Err((job.response_tx, error));- }+ Ok(headers) => headers,+ Err(error) => return Err((job.response_tx, error)),+ };+ let request_extra_info = job.request.is_top_level_navigation_request().then(|| {+ network_request_extra_info_from_headers(+ &self.config,+ &outgoing_headers,+ request_cookie_report.as_ref(),+ )+ });+ attach_next_request_extra_info(+ &mut job.redirect_chain,+ request_cookie_report.clone(),+ request_extra_info.as_ref(),+ );let label = job.current_url.to_string();let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);@@ -995,16 +992,6 @@ impl RuntimeOwner {&job.request,&job.current_url,);- let request_extra_info = job.request.is_top_level_navigation_request().then(|| {- network_request_extra_info_for_url(- &self.config,- &prepared_request.request,- &job.current_url,- &job.redirect_chain,- cookie_header.as_deref(),- request_cookie_report.as_ref(),- )- });match load_cached_streaming_response_lookup(&self.config,&prepared_request.request,@@ -1033,12 +1020,6 @@ impl RuntimeOwner {Err(error) => return Err((Box::new(job), None, error)),}- attach_next_request_extra_info(- &mut job.redirect_chain,- request_cookie_report.clone(),- request_extra_info.as_ref(),- );-let mut easy = job.easy.take().unwrap_or_else(|| {Easy2::new(FetchTransferHandler::new_streaming(StreamingResponseCollector::new(@@ -1061,21 +1042,6 @@ impl RuntimeOwner {cookie_header.clone(),));- let collector = easy- .get_mut()- .streaming_mut()- .expect("streaming request should use streaming collector");- collector.begin_request_with_cache_plan(- self.config.http_max_response_size(),- job.current_url.clone(),- job.current_cookie_context.clone(),- request_cookie_report.clone(),- credentials_allowed,- job.redirect_chain.clone(),- request_extra_info.clone(),- cache_plan,- );- collector.set_client_hint_response_policy(prepared_request.response_policy);if let Err(error) = configure_network_observation(&mut easy,&job.request,@@ -1084,7 +1050,7 @@ impl RuntimeOwner {) {return Err((Box::new(job), Some(easy), error));}- if let Err(error) = configure_easy(+ let outgoing_headers = match configure_easy(&mut easy,&self.config,&prepared_request.request,@@ -1096,8 +1062,36 @@ impl RuntimeOwner {).with_context(|| anyhow!("failed to configure curl request for {}", job.current_url)){- return Err((Box::new(job), Some(easy), error));- }+ Ok(headers) => headers,+ Err(error) => return Err((Box::new(job), Some(easy), error)),+ };+ let request_extra_info = job.request.is_top_level_navigation_request().then(|| {+ network_request_extra_info_from_headers(+ &self.config,+ &outgoing_headers,+ request_cookie_report.as_ref(),+ )+ });+ attach_next_request_extra_info(+ &mut job.redirect_chain,+ request_cookie_report.clone(),+ request_extra_info.as_ref(),+ );+ let collector = easy+ .get_mut()+ .streaming_mut()+ .expect("streaming request should use streaming collector");+ collector.begin_request_with_cache_plan(+ self.config.http_max_response_size(),+ job.current_url.clone(),+ job.current_cookie_context.clone(),+ request_cookie_report.clone(),+ credentials_allowed,+ job.redirect_chain.clone(),+ request_extra_info.clone(),+ cache_plan,+ );+ collector.set_client_hint_response_policy(prepared_request.response_policy);let label = job.current_url.to_string();let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);@@ -1189,16 +1183,6 @@ impl RuntimeOwner {&job.request,&job.current_url,);- let request_extra_info = job.request.is_top_level_navigation_request().then(|| {- network_request_extra_info_for_url(- &self.config,- &prepared_request.request,- &job.current_url,- &job.redirect_chain,- cookie_header.as_deref(),- request_cookie_report.as_ref(),- )- });let mut stale_cached_lookup = None;match load_cached_streaming_response_lookup(&self.config,@@ -1231,12 +1215,6 @@ impl RuntimeOwner {Err(error) => return Err((Box::new(job), None, error)),}- attach_next_request_extra_info(- &mut job.redirect_chain,- request_cookie_report.clone(),- request_extra_info.as_ref(),- );-let mut easy = job.easy.take().unwrap_or_else(|| {Easy2::new(FetchTransferHandler::new_raw_streaming(RawStreamingResponseCollector::new(@@ -1258,22 +1236,6 @@ impl RuntimeOwner {job.current_url.clone(),cookie_header.clone(),));- let collector = easy- .get_mut()- .raw_streaming_mut()- .expect("raw streaming request should use raw streaming collector");- collector.begin_request_with_cache_plan(- self.config.http_max_response_size(),- job.current_url.clone(),- job.current_cookie_context.clone(),- request_cookie_report.clone(),- credentials_allowed,- job.redirect_chain.clone(),- request_extra_info.clone(),- cache_plan,- stale_cached_lookup.is_some(),- );- collector.set_client_hint_response_policy(prepared_request.response_policy);if let Err(error) = configure_network_observation(&mut easy,&job.request,@@ -1282,7 +1244,7 @@ impl RuntimeOwner {) {return Err((Box::new(job), Some(easy), error));}- if let Err(error) = configure_easy(+ let outgoing_headers = match configure_easy(&mut easy,&self.config,&prepared_request.request,@@ -1296,8 +1258,37 @@ impl RuntimeOwner {).with_context(|| anyhow!("failed to configure curl request for {}", job.current_url)){- return Err((Box::new(job), Some(easy), error));- }+ Ok(headers) => headers,+ Err(error) => return Err((Box::new(job), Some(easy), error)),+ };+ let request_extra_info = job.request.is_top_level_navigation_request().then(|| {+ network_request_extra_info_from_headers(+ &self.config,+ &outgoing_headers,+ request_cookie_report.as_ref(),+ )+ });+ attach_next_request_extra_info(+ &mut job.redirect_chain,+ request_cookie_report.clone(),+ request_extra_info.as_ref(),+ );+ let collector = easy+ .get_mut()+ .raw_streaming_mut()+ .expect("raw streaming request should use raw streaming collector");+ collector.begin_request_with_cache_plan(+ self.config.http_max_response_size(),+ job.current_url.clone(),+ job.current_cookie_context.clone(),+ request_cookie_report.clone(),+ credentials_allowed,+ job.redirect_chain.clone(),+ request_extra_info.clone(),+ cache_plan,+ stale_cached_lookup.is_some(),+ );+ collector.set_client_hint_response_policy(prepared_request.response_policy);let label = job.current_url.to_string();let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rsindex 7e8512765..e6d7b41e1 100644--- a/moli-fetch/src/tests/mod.rs+++ b/moli-fetch/src/tests/mod.rs@@ -8,6 +8,7 @@ use moli_browser_profile::DEFAULT_ACCEPT_LANGUAGE;use moli_cookie_jar::{NetworkCookieRequestContext, new_shared_browser_cookie_store};use moli_http_cache::{HttpCacheEntryMetadata, HttpCacheStore};use std::{+ collections::BTreeSet,fs,io::Read,num::NonZeroU32,@@ -28,7 +29,8 @@ use crate::{RequestAuthScheme, RequestAuthTarget, RequestCacheMode, RequestCredentialsMode, RequestMode,RequestRedirectMode, RequestResourceType, Response, ResponseBody, ResponseHead,ScriptFetchRequestMetadata, ScriptFetchSchedulerPriority, StreamingResponseCollector,- SubresourceRequestMetadata, http_cache_stats, runtime::FetchRuntimeOwner,+ SubresourceRequestMetadata, WebBotAuthProfile, WebBotAuthSigner, http_cache_stats,+ runtime::FetchRuntimeOwner,};use self::support::{@@ -39,6 +41,19 @@ use self::support::{const ENV_PROXY_CHILD_TEST: &str = "MOLI_FETCH_ENV_PROXY_CHILD";const ENV_PROXY_URL: &str = "MOLI_FETCH_ENV_PROXY_URL";const TEST_HIGH_ENTROPY_CLIENT_HINTS: &str = "Sec-CH-UA-Full-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Arch, Sec-CH-UA-Bitness, Sec-CH-UA-Platform-Version, Sec-CH-UA-Model, Sec-CH-UA-WoW64";+const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\+MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\+-----END PRIVATE KEY-----\n";++fn test_web_bot_auth_signer() -> WebBotAuthSigner {+ WebBotAuthSigner::from_pem(+ RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),+ "bot.example",+ Some("poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U"),+ WebBotAuthProfile::Cloudflare,+ )+ .unwrap()+}fn sample_response_head() -> ResponseHead {ResponseHead {@@ -1191,6 +1206,172 @@ async fn fetch_raw_stream_treats_https11_switching_protocols_as_final_response()Ok(())}+#[test]+fn web_bot_auth_resigns_restarts_redirects_and_subresources() -> Result<()> {+ let server = ScriptedHttps11Server::spawn(vec![+ ScriptedResponse::status(403, "Challenge")+ .with_header("Accept-CH", "Sec-CH-UA-Arch")+ .with_header("Critical-CH", "Sec-CH-UA-Arch"),+ ScriptedResponse::status(302, "Found").with_header("Location", "/final"),+ ScriptedResponse::ok("final"),+ ScriptedResponse::ok("asset"),+ ]);+ let mut config = FetchConfig::default();+ config.set_tls_verify_host(false);+ config.set_web_bot_auth(Some(test_web_bot_auth_signer()));+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());++ let response = fetch_response_for_test(&client, Request::get(&server.url_path("/start"))?)?;+ assert_eq!(response.body_text(), "final");+ assert_eq!(response.redirect_chain.len(), 2);+ assert_eq!(response.redirect_chain[0].status, 307);+ assert_eq!(response.redirect_chain[1].status, 302);++ let final_url = Url::parse(&server.url_path("/final"))?;+ let subresource = Request::new(+ "POST",+ &server.url_path("/asset"),+ Some("probe".to_owned()),+ Vec::new(),+ )?+ .with_browser_request_metadata(BrowserRequestMetadata::Fetch)+ .with_initiator_url(&final_url);+ assert_eq!(+ fetch_response_for_test(&client, subresource)?.body_text(),+ "asset"+ );++ assert_eq!(+ server.requests(),+ [+ "/start".to_owned(),+ "/start".to_owned(),+ "/final".to_owned(),+ "/asset".to_owned(),+ ]+ );+ let request_heads = server.request_heads();+ assert_eq!(request_heads.len(), 4);+ for request_head in &request_heads {+ assert_eq!(+ request_head_header_value(request_head, "Signature-Agent"),+ Some("\"https://bot.example\"")+ );+ let signature_input = request_head_header_value(request_head, "Signature-Input")+ .expect("signed request should include Signature-Input");+ assert!(+ signature_input.contains("(\"@authority\" \"@method\" \"@path\" \"signature-agent\")")+ );+ assert!(request_head_header_value(request_head, "Signature").is_some());+ }+ assert!(request_heads[3].starts_with("POST /asset HTTP/1.1\r\n"));++ let nonces = request_heads+ .iter()+ .map(|request_head| {+ nonce_from_signature_input(+ request_head_header_value(request_head, "Signature-Input").unwrap(),+ )+ .to_owned()+ })+ .collect::<BTreeSet<_>>();+ assert_eq!(nonces.len(), request_heads.len());++ let restart = &response.redirect_chain[0];+ assert_request_extra_signature_matches_wire(+ &restart+ .response_extra_info+ .as_ref()+ .expect("restart response extra info")+ .request_extra_info,+ &request_heads[0],+ );+ assert_request_extra_signature_matches_wire(+ restart+ .request_extra_info+ .as_ref()+ .expect("restarted request extra info"),+ &request_heads[1],+ );+ let redirect = &response.redirect_chain[1];+ assert_request_extra_signature_matches_wire(+ &redirect+ .response_extra_info+ .as_ref()+ .expect("redirect response extra info")+ .request_extra_info,+ &request_heads[1],+ );+ assert_request_extra_signature_matches_wire(+ redirect+ .request_extra_info+ .as_ref()+ .expect("redirect target request extra info"),+ &request_heads[2],+ );+ assert_request_extra_signature_matches_wire(+ response+ .network_request_extra_info()+ .expect("final request extra info"),+ &request_heads[2],+ );++ server.shutdown();+ Ok(())+}++#[test]+fn web_bot_auth_is_not_sent_over_plain_http() -> Result<()> {+ let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("plain")]);+ let mut config = FetchConfig::default();+ config.set_web_bot_auth(Some(test_web_bot_auth_signer()));++ let response = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;+ assert_eq!(response.body_text(), "plain");+ let requests = server.requests();+ assert_eq!(requests.len(), 1);+ for name in ["Signature-Agent", "Signature-Input", "Signature"] {+ assert_eq!(request_head_header_value(&requests[0], name), None);+ }++ server.shutdown();+ Ok(())+}++#[test]+fn web_bot_auth_bypasses_shared_http_cache() -> Result<()> {+ let cache_dir = unique_test_cache_dir();+ let server = ScriptedHttps11Server::spawn(vec![+ ScriptedResponse::ok("first").with_header("Cache-Control", "max-age=60"),+ ScriptedResponse::ok("second").with_header("Cache-Control", "max-age=60"),+ ]);+ let mut config = FetchConfig::default();+ config.set_tls_verify_host(false);+ config.set_http_cache_dir(Some(cache_dir.display().to_string()));+ config.set_web_bot_auth(Some(test_web_bot_auth_signer()));++ let first = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;+ let second = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;+ assert_eq!(first.body_text(), "first");+ assert_eq!(second.body_text(), "second");+ assert!(!first.from_cache);+ assert!(!second.from_cache);+ assert_eq!(server.hits(), 2);+ let request_heads = server.request_heads();+ assert_ne!(+ request_head_header_value(&request_heads[0], "Signature-Input"),+ request_head_header_value(&request_heads[1], "Signature-Input")+ );+ assert!(+ !cache_dir.exists() || fs::read_dir(&cache_dir)?.next().is_none(),+ "authenticated responses must not enter the shared disk cache"+ );++ server.shutdown();+ let _ = fs::remove_dir_all(cache_dir);+ Ok(())+}+#[tokio::test]async fn fetch_raw_stream_finishes_switching_protocols_body_without_connection_close() -> Result<()>{@@ -3616,6 +3797,34 @@ fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a s.map(|(_, value)| value.as_str())}+fn request_head_header_value<'a>(request_head: &'a str, name: &str) -> Option<&'a str> {+ request_head.lines().skip(1).find_map(|line| {+ let (header_name, value) = line.split_once(':')?;+ header_name.eq_ignore_ascii_case(name).then(|| value.trim())+ })+}++fn nonce_from_signature_input(signature_input: &str) -> &str {+ signature_input+ .split(";nonce=\"")+ .nth(1)+ .and_then(|value| value.split('"').next())+ .expect("Signature-Input should contain a nonce parameter")+}++fn assert_request_extra_signature_matches_wire(+ extra_info: &crate::NetworkRequestExtraInfo,+ request_head: &str,+) {+ for name in ["Signature-Agent", "Signature-Input", "Signature"] {+ assert_eq!(+ header_value(&extra_info.headers, name),+ request_head_header_value(request_head, name),+ "network extra info and wire request differ for {name}"+ );+ }+}+#[test]fn fetch_client_cache_respects_pragma_no_cache_response() {let cache_dir = unique_test_cache_dir();diff --git a/moli-fetch/src/tests/support.rs b/moli-fetch/src/tests/support.rsindex d7b86d8bc..5d3b8f06b 100644--- a/moli-fetch/src/tests/support.rs+++ b/moli-fetch/src/tests/support.rs@@ -43,6 +43,7 @@ pub(super) struct ScriptedHttps11Server {addr: std::net::SocketAddr,hits: Arc<AtomicUsize>,requests: Arc<Mutex<Vec<String>>>,+ request_heads: Arc<Mutex<Vec<String>>>,shutdown_tx: std_mpsc::Sender<()>,join_handle: Option<thread::JoinHandle<()>>,}@@ -56,16 +57,19 @@ impl ScriptedHttps11Server {let addr = listener.local_addr().unwrap();let hits = Arc::new(AtomicUsize::new(0));let requests = Arc::new(Mutex::new(Vec::new()));+ let request_heads = Arc::new(Mutex::new(Vec::new()));let (shutdown_tx, shutdown_rx) = std_mpsc::channel();let responses = Arc::new(Mutex::new(VecDeque::from(responses)));let hits_for_thread = Arc::clone(&hits);let requests_for_thread = Arc::clone(&requests);+ let request_heads_for_thread = Arc::clone(&request_heads);let join_handle = thread::spawn(move || {run_https11_server(listener,shutdown_rx,hits_for_thread,requests_for_thread,+ request_heads_for_thread,responses,);});@@ -74,6 +78,7 @@ impl ScriptedHttps11Server {addr,hits,requests,+ request_heads,shutdown_tx,join_handle: Some(join_handle),}@@ -83,6 +88,11 @@ impl ScriptedHttps11Server {format!("https://{}/cache", self.addr)}+ pub(super) fn url_path(&self, path: &str) -> String {+ let path = path.strip_prefix('/').unwrap_or(path);+ format!("https://{}/{}", self.addr, path)+ }+pub(super) fn hits(&self) -> usize {self.hits.load(Ordering::SeqCst)}@@ -91,6 +101,10 @@ impl ScriptedHttps11Server {self.requests.lock().clone()}+ pub(super) fn request_heads(&self) -> Vec<String> {+ self.request_heads.lock().clone()+ }+pub(super) fn shutdown(mut self) {let _ = self.shutdown_tx.send(());if let Some(join_handle) = self.join_handle.take() {@@ -710,6 +724,7 @@ fn run_https11_server(shutdown_rx: std_mpsc::Receiver<()>,hits: Arc<AtomicUsize>,requests: Arc<Mutex<Vec<String>>>,+ request_heads: Arc<Mutex<Vec<String>>>,responses: Arc<Mutex<VecDeque<ScriptedResponse>>>,) {let rt = tokio::runtime::Builder::new_current_thread()@@ -731,6 +746,7 @@ fn run_https11_server(let tls_acceptor = tls_acceptor.clone();let hits = Arc::clone(&hits);let requests = Arc::clone(&requests);+ let request_heads = Arc::clone(&request_heads);let responses = Arc::clone(&responses);tokio::spawn(async move {let Ok(mut stream) = tls_acceptor.accept(stream).await else {@@ -751,6 +767,9 @@ fn run_https11_server(if let Some(path) = request_path_from_head(&request) {requests.lock().push(path);}+ request_heads+ .lock()+ .push(String::from_utf8_lossy(&request).into_owned());let _ = hits.fetch_add(1, Ordering::SeqCst) + 1;let response_spec = {let mut responses = responses.lock();diff --git a/moli-web-bot-auth/Cargo.toml b/moli-web-bot-auth/Cargo.tomlnew file mode 100644index 000000000..5ce2292f6--- /dev/null+++ b/moli-web-bot-auth/Cargo.toml@@ -0,0 +1,18 @@+[package]+license.workspace = true+name = "moli-web-bot-auth"+version = "0.1.0"+edition = "2024"+description = "Web Bot Auth HTTP message signatures for Moli"++[dependencies]+anyhow = "1.0.100"+base64 = "0.22"+moli-crypto = { path = "../moli-crypto" }+url = "2.5.7"++[dev-dependencies]+aws-lc-rs = { version = "1.16.2", default-features = false, features = ["aws-lc-sys"] }++[lints]+workspace = truediff --git a/moli-web-bot-auth/src/key.rs b/moli-web-bot-auth/src/key.rsnew file mode 100644index 000000000..d9f7ea81b--- /dev/null+++ b/moli-web-bot-auth/src/key.rs@@ -0,0 +1,133 @@+use std::str;++use anyhow::{Context, Result, anyhow, bail};+use base64::{Engine as _, engine::general_purpose};+use moli_crypto::{Ed25519SigningKey, sha256_digest};++pub(crate) struct WebBotAuthKey {+ signing_key: Ed25519SigningKey,+ public_key: [u8; 32],+ keyid: String,+}++impl WebBotAuthKey {+ pub(crate) fn from_pem(private_key_pem: &[u8], expected_keyid: Option<&str>) -> Result<Self> {+ let private_key_der = decode_pkcs8_private_key_pem(private_key_pem)?;+ let signing_key = Ed25519SigningKey::from_pkcs8(&private_key_der).map_err(|_| {+ anyhow!("web bot auth key must be an unencrypted PKCS#8 Ed25519 private key")+ })?;+ let public_key = *signing_key.public_key();+ let keyid = jwk_thumbprint(&public_key);+ if let Some(expected_keyid) = expected_keyid+ && expected_keyid != keyid+ {+ bail!(+ "--web-bot-auth-keyid does not match the private key; expected `{expected_keyid}`, computed `{keyid}`"+ );+ }++ Ok(Self {+ signing_key,+ public_key,+ keyid,+ })+ }++ pub(crate) fn keyid(&self) -> &str {+ &self.keyid+ }++ pub(crate) fn public_key(&self) -> &[u8; 32] {+ &self.public_key+ }++ pub(crate) fn sign(&self, message: &[u8]) -> Result<[u8; 64]> {+ self.signing_key+ .sign(message)+ .context("failed to create web bot auth Ed25519 signature")+ }+}++fn decode_pkcs8_private_key_pem(private_key_pem: &[u8]) -> Result<Vec<u8>> {+ const BEGIN: &str = "-----BEGIN PRIVATE KEY-----";+ const END: &str = "-----END PRIVATE KEY-----";++ let pem = str::from_utf8(private_key_pem)+ .context("web bot auth private key PEM is not valid UTF-8")?;+ if pem.contains("-----BEGIN ENCRYPTED PRIVATE KEY-----") {+ bail!("encrypted web bot auth private keys are not supported");+ }+ let pem = pem.trim();+ let body = pem+ .strip_prefix(BEGIN)+ .and_then(|pem| pem.strip_suffix(END))+ .ok_or_else(|| {+ anyhow!("web bot auth key must use BEGIN PRIVATE KEY PKCS#8 PEM encoding")+ })?;+ if body.contains("-----") {+ bail!("web bot auth key PEM must contain exactly one private key");+ }+ let encoded = body+ .chars()+ .filter(|character| !character.is_ascii_whitespace())+ .collect::<String>();+ if encoded.is_empty() {+ bail!("web bot auth private key PEM body is empty");+ }+ general_purpose::STANDARD+ .decode(encoded)+ .context("web bot auth private key PEM contains invalid base64")+}++fn jwk_thumbprint(public_key: &[u8; 32]) -> String {+ let x = general_purpose::URL_SAFE_NO_PAD.encode(public_key);+ let canonical_jwk = format!(r#"{{"crv":"Ed25519","kty":"OKP","x":"{x}"}}"#);+ general_purpose::URL_SAFE_NO_PAD.encode(sha256_digest(canonical_jwk))+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::test_support::{EXPECTED_KEYID, RFC_9421_ED25519_PRIVATE_KEY};++ #[test]+ fn derives_rfc_jwk_thumbprint_and_public_key() {+ let key = WebBotAuthKey::from_pem(+ RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),+ Some(EXPECTED_KEYID),+ )+ .unwrap();++ assert_eq!(key.keyid(), EXPECTED_KEYID);+ assert_eq!(+ general_purpose::URL_SAFE_NO_PAD.encode(key.public_key()),+ "JrQLj5P_89iXES9-vFgrIy29clF9CC_oPPsw3c5D0bs"+ );+ }++ #[test]+ fn rejects_mismatched_keyid_and_invalid_pem() {+ let wrong_keyid =+ WebBotAuthKey::from_pem(RFC_9421_ED25519_PRIVATE_KEY.as_bytes(), Some("wrong"))+ .err()+ .expect("mismatched keyid should fail")+ .to_string();+ assert!(wrong_keyid.contains("does not match"));+ assert!(wrong_keyid.contains(EXPECTED_KEYID));++ let invalid_pem = WebBotAuthKey::from_pem(b"not a private key", None)+ .err()+ .expect("invalid PEM should fail")+ .to_string();+ assert!(invalid_pem.contains("BEGIN PRIVATE KEY"));++ let encrypted = WebBotAuthKey::from_pem(+ b"-----BEGIN ENCRYPTED PRIVATE KEY-----\nAA==\n-----END ENCRYPTED PRIVATE KEY-----",+ None,+ )+ .err()+ .expect("encrypted PEM should fail")+ .to_string();+ assert!(encrypted.contains("encrypted"));+ }+}diff --git a/moli-web-bot-auth/src/lib.rs b/moli-web-bot-auth/src/lib.rsnew file mode 100644index 000000000..2c13d9d0c--- /dev/null+++ b/moli-web-bot-auth/src/lib.rs@@ -0,0 +1,12 @@+//! Web Bot Auth request signing built on RFC 9421 HTTP message signatures.++mod key;+mod profile;+mod signer;+mod wire;++#[cfg(test)]+mod test_support;++pub use profile::WebBotAuthProfile;+pub use signer::WebBotAuthSigner;diff --git a/moli-web-bot-auth/src/profile.rs b/moli-web-bot-auth/src/profile.rsnew file mode 100644index 000000000..122ed672c--- /dev/null+++ b/moli-web-bot-auth/src/profile.rs@@ -0,0 +1,15 @@+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub enum WebBotAuthProfile {+ #[default]+ Cloudflare,+ IetfDraft01,+}++impl WebBotAuthProfile {+ pub const fn as_str(self) -> &'static str {+ match self {+ Self::Cloudflare => "cloudflare",+ Self::IetfDraft01 => "ietf-01",+ }+ }+}diff --git a/moli-web-bot-auth/src/signer.rs b/moli-web-bot-auth/src/signer.rsnew file mode 100644index 000000000..3e811aa85--- /dev/null+++ b/moli-web-bot-auth/src/signer.rs@@ -0,0 +1,291 @@+use std::{+ fmt,+ sync::{+ Arc,+ atomic::{AtomicU64, Ordering},+ },+ time::{SystemTime, UNIX_EPOCH},+};++use anyhow::{Context, Result, anyhow, bail};+use base64::{Engine as _, engine::general_purpose};+use moli_crypto::{DigestAlgorithm, fill_secure_random};+use url::Url;++use crate::{+ key::WebBotAuthKey,+ profile::WebBotAuthProfile,+ wire::{SignatureOptions, sign_request},+};++const SIGNATURE_LABEL: &str = "sig1";+const SIGNATURE_TTL_SECS: u64 = 60;+const NONCE_BYTES: usize = 64;++#[derive(Clone)]+pub struct WebBotAuthSigner {+ inner: Arc<WebBotAuthSignerInner>,+}++struct WebBotAuthSignerInner {+ key: WebBotAuthKey,+ signature_agent_origin: String,+ profile: WebBotAuthProfile,+ nonce_seed: [u8; 32],+ nonce_counter: AtomicU64,+}++impl WebBotAuthSigner {+ pub fn from_pem(+ private_key_pem: &[u8],+ domain: &str,+ expected_keyid: Option<&str>,+ profile: WebBotAuthProfile,+ ) -> Result<Self> {+ let key = WebBotAuthKey::from_pem(private_key_pem, expected_keyid)?;+ let signature_agent_origin = normalize_signature_agent_origin(domain)?;+ let mut nonce_seed = [0_u8; 32];+ fill_secure_random(&mut nonce_seed)+ .map_err(|_| anyhow!("failed to initialize web bot auth nonce generation"))?;++ Ok(Self {+ inner: Arc::new(WebBotAuthSignerInner {+ key,+ signature_agent_origin,+ profile,+ nonce_seed,+ nonce_counter: AtomicU64::new(0),+ }),+ })+ }++ pub fn keyid(&self) -> &str {+ self.inner.key.keyid()+ }++ pub fn public_key(&self) -> &[u8; 32] {+ self.inner.key.public_key()+ }++ pub fn signature_agent_origin(&self) -> &str {+ &self.inner.signature_agent_origin+ }++ pub fn profile(&self) -> WebBotAuthProfile {+ self.inner.profile+ }++ pub fn append_request_headers(+ &self,+ headers: &mut Vec<(String, String)>,+ method: &str,+ request_url: &Url,+ ) -> Result<()> {+ if request_url.scheme() != "https" {+ return Ok(());+ }++ headers.retain(|(name, _)| !is_web_bot_auth_header(name));+ let created = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap_or_default()+ .as_secs();+ let signed = sign_request(+ &self.inner.key,+ &self.inner.signature_agent_origin,+ self.inner.profile,+ method,+ request_url,+ &SignatureOptions {+ signature_label: SIGNATURE_LABEL,+ signature_agent_label: SIGNATURE_LABEL,+ created,+ expires: created.saturating_add(SIGNATURE_TTL_SECS),+ nonce: self.next_nonce(),+ cover_request_target: true,+ },+ )?;+ headers.extend(signed.headers.into_pairs());+ Ok(())+ }++ fn next_nonce(&self) -> String {+ let counter = self.inner.nonce_counter.fetch_add(1, Ordering::Relaxed);+ let mut material = Vec::with_capacity(32 + 8 + 24);+ material.extend_from_slice(b"moli-web-bot-auth-nonce\0");+ material.extend_from_slice(&self.inner.nonce_seed);+ material.extend_from_slice(&counter.to_be_bytes());+ let nonce = DigestAlgorithm::Sha512.digest_bytes(&material);+ debug_assert_eq!(nonce.len(), NONCE_BYTES);+ general_purpose::STANDARD.encode(nonce)+ }+}++impl fmt::Debug for WebBotAuthSigner {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter+ .debug_struct("WebBotAuthSigner")+ .field("keyid", &self.inner.key.keyid())+ .field("signature_agent_origin", &self.inner.signature_agent_origin)+ .field("profile", &self.inner.profile)+ .finish_non_exhaustive()+ }+}++impl PartialEq for WebBotAuthSigner {+ fn eq(&self, other: &Self) -> bool {+ Arc::ptr_eq(&self.inner, &other.inner)+ || (self.inner.key.public_key() == other.inner.key.public_key()+ && self.inner.signature_agent_origin == other.inner.signature_agent_origin+ && self.inner.profile == other.inner.profile)+ }+}++impl Eq for WebBotAuthSigner {}++fn normalize_signature_agent_origin(domain: &str) -> Result<String> {+ if domain.is_empty() || domain.trim() != domain {+ bail!("--web-bot-auth-domain must be a non-empty domain without surrounding whitespace");+ }+ if domain.contains(['/', '?', '#', '@']) {+ bail!(+ "--web-bot-auth-domain must contain only a host and optional port, without a scheme, credentials, path, query, or fragment"+ );+ }++ let url = Url::parse(&format!("https://{domain}"))+ .context("failed to parse --web-bot-auth-domain as an HTTPS origin")?;+ if url.host().is_none()+ || !url.username().is_empty()+ || url.password().is_some()+ || url.path() != "/"+ || url.query().is_some()+ || url.fragment().is_some()+ {+ bail!("--web-bot-auth-domain must identify a single HTTPS origin");+ }+ Ok(url.origin().ascii_serialization())+}++fn is_web_bot_auth_header(name: &str) -> bool {+ name.eq_ignore_ascii_case("signature-agent")+ || name.eq_ignore_ascii_case("signature-input")+ || name.eq_ignore_ascii_case("signature")+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::test_support::{EXPECTED_KEYID, RFC_9421_ED25519_PRIVATE_KEY, signer};++ #[test]+ fn normalizes_signature_agent_origin() {+ let signer = signer(WebBotAuthProfile::Cloudflare);++ assert_eq!(signer.keyid(), EXPECTED_KEYID);+ assert_eq!(+ signer.signature_agent_origin(),+ "https://signature-agent.test"+ );+ }++ #[test]+ fn production_requests_use_unique_nonces() {+ let signer = signer(WebBotAuthProfile::Cloudflare);+ let request_url = Url::parse("https://example.com/path").unwrap();+ let mut first_headers = Vec::new();+ signer+ .append_request_headers(&mut first_headers, "post", &request_url)+ .unwrap();+ let mut second_headers = Vec::new();+ signer+ .append_request_headers(&mut second_headers, "post", &request_url)+ .unwrap();++ let first_input = header_value(&first_headers, "signature-input");+ assert!(first_input.contains("(\"@authority\" \"@method\" \"@path\" \"signature-agent\")"));+ assert!(first_input.contains(";expires="));+ assert!(first_input.contains(";tag=\"web-bot-auth\""));+ assert_ne!(+ nonce_from_signature_input(first_input),+ nonce_from_signature_input(header_value(&second_headers, "signature-input"))+ );+ }++ #[test]+ fn omits_signatures_on_insecure_requests_and_replaces_spoofed_https_headers() {+ let signer = signer(WebBotAuthProfile::Cloudflare);+ let spoofed = vec![+ ("Signature-Agent".to_owned(), "spoofed".to_owned()),+ ("Signature-Input".to_owned(), "spoofed".to_owned()),+ ("Signature".to_owned(), "spoofed".to_owned()),+ ];+ let mut http_headers = spoofed.clone();+ signer+ .append_request_headers(+ &mut http_headers,+ "GET",+ &Url::parse("http://example.com/").unwrap(),+ )+ .unwrap();+ assert_eq!(http_headers, spoofed);++ let mut https_headers = spoofed;+ signer+ .append_request_headers(+ &mut https_headers,+ "GET",+ &Url::parse("https://example.com/").unwrap(),+ )+ .unwrap();+ assert_eq!(https_headers.len(), 3);+ assert!(https_headers.iter().all(|(_, value)| value != "spoofed"));+ }++ #[test]+ fn validates_domain() {+ for domain in [+ "",+ " example.com",+ "https://example.com",+ "[email protected]",+ "example.com/path",+ "example.com?query",+ ] {+ let error = WebBotAuthSigner::from_pem(+ RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),+ domain,+ None,+ WebBotAuthProfile::Cloudflare,+ )+ .unwrap_err();+ assert!(error.to_string().contains("--web-bot-auth-domain"));+ }+ }++ #[test]+ fn debug_output_never_contains_private_key_material() {+ let signer = signer(WebBotAuthProfile::Cloudflare);+ let rendered = format!("{signer:?}");++ assert!(rendered.contains(EXPECTED_KEYID));+ assert!(!rendered.contains("IJ+DYvh6"));+ assert!(!rendered.contains("PRIVATE KEY"));+ }++ fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {+ headers+ .iter()+ .find(|(candidate, _)| candidate.eq_ignore_ascii_case(name))+ .map(|(_, value)| value.as_str())+ .unwrap()+ }++ fn nonce_from_signature_input(input: &str) -> &str {+ input+ .split(";nonce=\"")+ .nth(1)+ .and_then(|value| value.split('"').next())+ .unwrap()+ }+}diff --git a/moli-web-bot-auth/src/test_support.rs b/moli-web-bot-auth/src/test_support.rsnew file mode 100644index 000000000..d156f1081--- /dev/null+++ b/moli-web-bot-auth/src/test_support.rs@@ -0,0 +1,24 @@+use crate::{WebBotAuthProfile, WebBotAuthSigner, key::WebBotAuthKey};++pub(crate) const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\+MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\+-----END PRIVATE KEY-----\n";+pub(crate) const EXPECTED_KEYID: &str = "poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U";++pub(crate) fn key() -> WebBotAuthKey {+ WebBotAuthKey::from_pem(+ RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),+ Some(EXPECTED_KEYID),+ )+ .unwrap()+}++pub(crate) fn signer(profile: WebBotAuthProfile) -> WebBotAuthSigner {+ WebBotAuthSigner::from_pem(+ RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),+ "signature-agent.test",+ Some(EXPECTED_KEYID),+ profile,+ )+ .unwrap()+}diff --git a/moli-web-bot-auth/src/wire.rs b/moli-web-bot-auth/src/wire.rsnew file mode 100644index 000000000..7b8e0cc98--- /dev/null+++ b/moli-web-bot-auth/src/wire.rs@@ -0,0 +1,279 @@+use anyhow::{Context, Result};+use base64::{Engine as _, engine::general_purpose};+use url::{Host, Url};++use crate::{key::WebBotAuthKey, profile::WebBotAuthProfile};++pub(crate) struct SignatureOptions<'a> {+ pub(crate) signature_label: &'a str,+ pub(crate) signature_agent_label: &'a str,+ pub(crate) created: u64,+ pub(crate) expires: u64,+ pub(crate) nonce: String,+ pub(crate) cover_request_target: bool,+}++pub(crate) struct SignedRequest {+ pub(crate) headers: WebBotAuthHeaders,+ #[cfg(test)]+ pub(crate) signature_base: String,+}++pub(crate) struct WebBotAuthHeaders {+ pub(crate) signature_agent: String,+ pub(crate) signature_input: String,+ pub(crate) signature: String,+}++impl WebBotAuthHeaders {+ pub(crate) fn into_pairs(self) -> [(String, String); 3] {+ [+ ("Signature-Agent".to_owned(), self.signature_agent),+ ("Signature-Input".to_owned(), self.signature_input),+ ("Signature".to_owned(), self.signature),+ ]+ }+}++pub(crate) fn sign_request(+ key: &WebBotAuthKey,+ signature_agent_origin: &str,+ profile: WebBotAuthProfile,+ method: &str,+ request_url: &Url,+ options: &SignatureOptions<'_>,+) -> Result<SignedRequest> {+ let authority = request_authority(request_url)?;+ let method = method.to_ascii_uppercase();+ let path = request_url.path();+ let signature_agent = signature_agent_header(+ profile,+ signature_agent_origin,+ options.signature_agent_label,+ );+ let signature_agent_component =+ signature_agent_component_identifier(profile, options.signature_agent_label);++ let mut covered_components = vec!["\"@authority\"".to_owned()];+ if options.cover_request_target {+ covered_components.push("\"@method\"".to_owned());+ covered_components.push("\"@path\"".to_owned());+ }+ covered_components.push(signature_agent_component.clone());++ let signature_params = format!(+ "({});created={};keyid=\"{}\";alg=\"ed25519\";expires={};nonce=\"{}\";tag=\"web-bot-auth\"",+ covered_components.join(" "),+ options.created,+ key.keyid(),+ options.expires,+ options.nonce,+ );++ let mut signature_base_lines = vec![format!("\"@authority\": {authority}")];+ if options.cover_request_target {+ signature_base_lines.push(format!("\"@method\": {method}"));+ signature_base_lines.push(format!("\"@path\": {path}"));+ }+ signature_base_lines.push(format!(+ "{signature_agent_component}: \"{signature_agent_origin}\""+ ));+ signature_base_lines.push(format!("\"@signature-params\": {signature_params}"));+ let signature_base = signature_base_lines.join("\n");+ let signature = key.sign(signature_base.as_bytes())?;++ Ok(SignedRequest {+ headers: WebBotAuthHeaders {+ signature_agent,+ signature_input: format!("{}={signature_params}", options.signature_label),+ signature: format!(+ "{}=:{}:",+ options.signature_label,+ general_purpose::STANDARD.encode(signature)+ ),+ },+ #[cfg(test)]+ signature_base,+ })+}++fn signature_agent_header(+ profile: WebBotAuthProfile,+ signature_agent_origin: &str,+ label: &str,+) -> String {+ match profile {+ WebBotAuthProfile::Cloudflare => format!("\"{signature_agent_origin}\""),+ WebBotAuthProfile::IetfDraft01 => {+ format!("{label}=\"{signature_agent_origin}\"")+ }+ }+}++fn signature_agent_component_identifier(profile: WebBotAuthProfile, label: &str) -> String {+ match profile {+ WebBotAuthProfile::Cloudflare => "\"signature-agent\"".to_owned(),+ WebBotAuthProfile::IetfDraft01 => {+ format!("\"signature-agent\";key=\"{label}\"")+ }+ }+}++fn request_authority(request_url: &Url) -> Result<String> {+ let host = match request_url+ .host()+ .context("Web Bot Auth request URL must have an authority")?+ {+ Host::Domain(domain) => domain.to_owned(),+ Host::Ipv4(address) => address.to_string(),+ Host::Ipv6(address) => format!("[{address}]"),+ };+ Ok(match request_url.port() {+ Some(port) => format!("{host}:{port}"),+ None => host,+ })+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::test_support::{EXPECTED_KEYID, key};+ use aws_lc_rs::signature::{ED25519, UnparsedPublicKey};++ #[test]+ fn matches_ietf_draft_01_ed25519_test_vector() {+ let key = key();+ let request_url = Url::parse("https://example.com/foo?param=Value&Pet=dog").unwrap();+ let signed = sign_request(+ &key,+ "https://signature-agent.test",+ WebBotAuthProfile::IetfDraft01,+ "POST",+ &request_url,+ &SignatureOptions {+ signature_label: "sig2",+ signature_agent_label: "agent2",+ created: 1_735_689_600,+ expires: 4_889_289_600,+ nonce: "n9p433xm+NJ3ph3upfBIGmsuwHw387YV7Q/F+6BSpGCVjYCqQw6rznNA8PVVLySrAWsv0hQtFioQb6E1YsauiA==".to_owned(),+ cover_request_target: false,+ },+ )+ .unwrap();++ assert_eq!(key.keyid(), EXPECTED_KEYID);+ assert_eq!(+ signed.headers.signature_agent,+ "agent2=\"https://signature-agent.test\""+ );+ assert_eq!(+ signed.headers.signature_input,+ "sig2=(\"@authority\" \"signature-agent\";key=\"agent2\");created=1735689600;keyid=\"poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U\";alg=\"ed25519\";expires=4889289600;nonce=\"n9p433xm+NJ3ph3upfBIGmsuwHw387YV7Q/F+6BSpGCVjYCqQw6rznNA8PVVLySrAWsv0hQtFioQb6E1YsauiA==\";tag=\"web-bot-auth\""+ );+ assert_eq!(+ signed.headers.signature,+ "sig2=:RdNFx5Bj6au3YgAMQL/RzmUlZE8QZLIaXGRpw985hWnwPfMxT228NMk6ehRS1PSl4e8PhbNZACSanGdhEwYCCg==:"+ );+ }++ #[test]+ fn matches_cloudflare_legacy_ed25519_test_vector() {+ let key = key();+ let request_url = Url::parse("https://example.com/foo?param=Value&Pet=dog").unwrap();+ let signed = sign_request(+ &key,+ "https://signature-agent.test",+ WebBotAuthProfile::Cloudflare,+ "POST",+ &request_url,+ &SignatureOptions {+ signature_label: "sig2",+ signature_agent_label: "agent2",+ created: 1_735_689_600,+ expires: 1_735_693_200,+ nonce: "e8N7S2MFd/qrd6T2R3tdfAuuANngKI7LFtKYI/vowzk4lAZYadIX6wW25MwG7DCT9RUKAJ0qVkU0mEeLElW1qg==".to_owned(),+ cover_request_target: false,+ },+ )+ .unwrap();++ assert_eq!(+ signed.headers.signature_agent,+ "\"https://signature-agent.test\""+ );+ assert_eq!(+ signed.headers.signature,+ "sig2=:jdq0SqOwHdyHr9+r5jw3iYZH6aNGKijYp/EstF4RQTQdi5N5YYKrD+mCT1HA1nZDsi6nJKuHxUi/5Syp3rLWBA==:"+ );+ }++ #[test]+ fn production_components_cover_authority_method_and_path() {+ let key = key();+ let request_url = Url::parse("https://example.com:8443/a%20path?query=yes").unwrap();+ let signed = sign_request(+ &key,+ "https://signature-agent.test",+ WebBotAuthProfile::Cloudflare,+ "post",+ &request_url,+ &SignatureOptions {+ signature_label: "sig1",+ signature_agent_label: "sig1",+ created: 1_735_689_600,+ expires: 1_735_689_660,+ nonce: general_purpose::STANDARD.encode([7_u8; 64]),+ cover_request_target: true,+ },+ )+ .unwrap();++ assert!(+ signed+ .signature_base+ .contains("\"@authority\": example.com:8443")+ );+ assert!(signed.signature_base.contains("\"@method\": POST"));+ assert!(signed.signature_base.contains("\"@path\": /a%20path"));+ let signature = signed+ .headers+ .signature+ .strip_prefix("sig1=:")+ .and_then(|value| value.strip_suffix(':'))+ .and_then(|value| general_purpose::STANDARD.decode(value).ok())+ .unwrap();+ UnparsedPublicKey::new(&ED25519, key.public_key())+ .verify(signed.signature_base.as_bytes(), &signature)+ .unwrap();+ }++ #[test]+ fn signing_rejects_a_request_url_without_an_authority() {+ let key = key();+ let request_url = Url::parse("data:text/plain,no-authority").unwrap();+ let result = sign_request(+ &key,+ "https://signature-agent.test",+ WebBotAuthProfile::Cloudflare,+ "GET",+ &request_url,+ &SignatureOptions {+ signature_label: "sig1",+ signature_agent_label: "sig1",+ created: 1_735_689_600,+ expires: 1_735_689_660,+ nonce: general_purpose::STANDARD.encode([7_u8; 64]),+ cover_request_target: true,+ },+ );++ let error = match result {+ Ok(_) => panic!("signing a URL without an authority must fail"),+ Err(error) => error,+ };+ assert_eq!(+ error.to_string(),+ "Web Bot Auth request URL must have an authority"+ );+ }+}diff --git a/moli/src/cli.rs b/moli/src/cli.rsindex ede40b461..e35e9eaa4 100644--- a/moli/src/cli.rs+++ b/moli/src/cli.rs@@ -32,6 +32,10 @@ const FETCH_INFER_FLAGS: &[&str] = &["--disable-subframes","--wait-ms","--profile-dir",+ "--web-bot-auth-key-file",+ "--web-bot-auth-keyid",+ "--web-bot-auth-domain",+ "--web-bot-auth-profile",];const SERVE_INFER_FLAGS: &[&str] = &["--host", "--port", "--timeout", "--layout"];const EXPLICIT_COMMANDS: &[&str] = &["fetch", "serve", "help", "version"];@@ -367,6 +371,35 @@ pub struct CommonArgs {#[arg(long)]pub user_agent_suffix: Option<String>,++ /// Unencrypted PKCS#8 Ed25519 private key used for Web Bot Auth signatures.+ #[arg(long, value_name = "PATH", requires = "web_bot_auth_domain")]+ pub web_bot_auth_key_file: Option<String>,++ /// Assert the RFC 7638 JWK thumbprint derived from the Web Bot Auth key.+ #[arg(long, value_name = "THUMBPRINT", requires = "web_bot_auth_key_file")]+ pub web_bot_auth_keyid: Option<String>,++ /// Operator domain publishing /.well-known/http-message-signatures-directory.+ #[arg(long, value_name = "DOMAIN", requires = "web_bot_auth_key_file")]+ pub web_bot_auth_domain: Option<String>,++ /// Signature-Agent wire format. Cloudflare compatibility is the default.+ #[arg(+ long,+ value_enum,+ default_value = "cloudflare",+ requires = "web_bot_auth_key_file"+ )]+ pub web_bot_auth_profile: WebBotAuthProfileChoice,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, ValueEnum)]+pub enum WebBotAuthProfileChoice {+ #[default]+ Cloudflare,+ #[value(name = "ietf-01")]+ IetfDraft01,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]diff --git a/moli/src/config.rs b/moli/src/config.rsindex 4cd98d9f9..28e979438 100644--- a/moli/src/config.rs+++ b/moli/src/config.rs@@ -6,10 +6,13 @@ use moli_core::{page::{SubresourceJsonPathEquals, SubresourceResponseWaitCriteria},runtime::BrowserConfig,};+use moli_fetch::{FetchConfig, WebBotAuthProfile, WebBotAuthSigner};use std::path::PathBuf;use std::str::FromStr;-use crate::cli::{Cli, Commands, CommonArgs, DumpFormat, LogFormat, StripOptions};+use crate::cli::{+ Cli, Commands, CommonArgs, DumpFormat, LogFormat, StripOptions, WebBotAuthProfileChoice,+};use crate::network_trace::NetworkTraceConfigSummary;pub use moli_protocol_server::ServerConfig;@@ -183,6 +186,7 @@ fn apply_common_args(config: &mut AppConfig, common: &CommonArgs) -> Result<()>.browser.fetch_mut().set_tls_verify_host(!common.insecure_disable_tls_host_verification);+ configure_web_bot_auth(config.browser.fetch_mut(), common)?;for source in &common.document_start_script {config.add_document_start_script(source.clone());@@ -199,6 +203,37 @@ fn apply_common_args(config: &mut AppConfig, common: &CommonArgs) -> Result<()>Ok(())}+fn configure_web_bot_auth(fetch: &mut FetchConfig, common: &CommonArgs) -> Result<()> {+ let (key_file, domain) = match (+ common.web_bot_auth_key_file.as_deref(),+ common.web_bot_auth_domain.as_deref(),+ ) {+ (None, None) => {+ if common.web_bot_auth_keyid.is_some() {+ bail!("--web-bot-auth-keyid requires --web-bot-auth-key-file");+ }+ return Ok(());+ }+ (Some(_), None) => bail!("--web-bot-auth-key-file requires --web-bot-auth-domain"),+ (None, Some(_)) => bail!("--web-bot-auth-domain requires --web-bot-auth-key-file"),+ (Some(key_file), Some(domain)) => (key_file, domain),+ };+ let private_key_pem = std::fs::read(key_file)+ .with_context(|| format!("failed to read Web Bot Auth private key `{key_file}`"))?;+ let profile = match common.web_bot_auth_profile {+ WebBotAuthProfileChoice::Cloudflare => WebBotAuthProfile::Cloudflare,+ WebBotAuthProfileChoice::IetfDraft01 => WebBotAuthProfile::IetfDraft01,+ };+ let signer = WebBotAuthSigner::from_pem(+ &private_key_pem,+ domain,+ common.web_bot_auth_keyid.as_deref(),+ profile,+ )?;+ fetch.set_web_bot_auth(Some(signer));+ Ok(())+}+fn validate_http_host_resolve_entries(entries: &[String]) -> Result<()> {for entry in entries {validate_http_host_resolve_entry(entry)?;diff --git a/moli/tests/cli.rs b/moli/tests/cli.rsindex fb46bdd75..bde0fa79f 100644--- a/moli/tests/cli.rs+++ b/moli/tests/cli.rs@@ -1,15 +1,27 @@use clap::Parser;-use std::{num::NonZeroU32, process::Command};+use std::{+ fs,+ num::NonZeroU32,+ path::PathBuf,+ process::Command,+ sync::atomic::{AtomicU64, Ordering},+};use moli::cli::{Cli, Commands, CommonArgs, DumpFormat, FetchArgs, FetchWaitUntil, LogFormat, LogLevel,RequestHeaderArg, ResponseJsonPathArg, ServeArgs, StripModeChoice, StripOptions,- normalize_args_for_compat,+ WebBotAuthProfileChoice, normalize_args_for_compat,};use moli::config::AppConfig;use moli_browser_profile::BrowserProfilePaths;use moli_core::OptionalResourceFetchMask;-use moli_fetch::FetchConfig;+use moli_fetch::{FetchConfig, WebBotAuthProfile};++const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\+MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\+-----END PRIVATE KEY-----\n";+const RFC_9421_ED25519_KEYID: &str = "poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U";+static NEXT_TEMP_KEY_FILE: AtomicU64 = AtomicU64::new(0);#[test]fn parses_explicit_fetch_command_with_compatibility_flags() {@@ -125,6 +137,10 @@ fn parses_explicit_fetch_command_with_compatibility_flags() {log_filter_scopes: Some("http,event".to_owned()),user_agent: None,user_agent_suffix: Some("internal-tester".to_owned()),+ web_bot_auth_key_file: None,+ web_bot_auth_keyid: None,+ web_bot_auth_domain: None,+ web_bot_auth_profile: WebBotAuthProfileChoice::Cloudflare,},url: "https://example.com".to_owned(),}))@@ -200,22 +216,79 @@ fn parses_binary_dump_modes_with_inferred_fetch_command() {}#[test]-fn rejects_removed_web_bot_auth_flags() {- for flag in [+fn parses_web_bot_auth_flags() {+ let cli = Cli::try_parse_from(normalize_args_for_compat([+ "moli",+ "fetch","--web-bot-auth-key-file",+ "bot-key.pem","--web-bot-auth-keyid",+ RFC_9421_ED25519_KEYID,"--web-bot-auth-domain",+ "bot.example",+ "--web-bot-auth-profile",+ "ietf-01",+ "https://example.com",+ ]))+ .unwrap();++ let Commands::Fetch(args) = cli.command else {+ panic!("expected fetch command");+ };+ assert_eq!(+ args.common.web_bot_auth_key_file.as_deref(),+ Some("bot-key.pem")+ );+ assert_eq!(+ args.common.web_bot_auth_keyid.as_deref(),+ Some(RFC_9421_ED25519_KEYID)+ );+ assert_eq!(+ args.common.web_bot_auth_domain.as_deref(),+ Some("bot.example")+ );+ assert_eq!(+ args.common.web_bot_auth_profile,+ WebBotAuthProfileChoice::IetfDraft01+ );+}++#[test]+fn web_bot_auth_profile_default_does_not_enable_signing() {+ let cli = Cli::try_parse_from(normalize_args_for_compat([+ "moli",+ "fetch",+ "https://example.com",+ ]))+ .unwrap();++ let Commands::Fetch(args) = cli.command else {+ panic!("expected fetch command");+ };+ assert_eq!(+ args.common.web_bot_auth_profile,+ WebBotAuthProfileChoice::Cloudflare+ );+ assert!(args.common.web_bot_auth_key_file.is_none());+}++#[test]+fn web_bot_auth_flags_require_key_and_domain_together() {+ for args in [+ vec!["--web-bot-auth-key-file", "bot-key.pem"],+ vec!["--web-bot-auth-domain", "bot.example"],+ vec!["--web-bot-auth-keyid", RFC_9421_ED25519_KEYID],+ vec!["--web-bot-auth-profile", "ietf-01"],] {- let error = Cli::try_parse_from(normalize_args_for_compat([- "moli",- "fetch",- flag,- "unused",- "https://example.com",- ]))- .unwrap_err();+ let mut command = vec!["moli", "fetch"];+ command.extend(args);+ command.push("https://example.com");+ let error = Cli::try_parse_from(normalize_args_for_compat(command)).unwrap_err();- assert_eq!(error.kind(), clap::error::ErrorKind::UnknownArgument);+ assert_eq!(+ error.kind(),+ clap::error::ErrorKind::MissingRequiredArgument+ );}}@@ -550,6 +623,54 @@ fn app_config_uses_moli_user_agent_defaults() {assert_eq!(config.browser.fetch().user_agent(), "ExampleBrowser/1.0");}+#[test]+fn app_config_loads_and_validates_web_bot_auth_key() {+ let key_file = write_temp_web_bot_auth_key();+ let key_path_arg = key_file.path.to_string_lossy().into_owned();+ let cli = Cli::try_parse_from(normalize_args_for_compat([+ "moli",+ "fetch",+ "--web-bot-auth-key-file",+ key_path_arg.as_str(),+ "--web-bot-auth-keyid",+ RFC_9421_ED25519_KEYID,+ "--web-bot-auth-domain",+ "bot.example:8443",+ "--web-bot-auth-profile",+ "ietf-01",+ "https://example.com",+ ]))+ .unwrap();++ let config = AppConfig::from_cli(&cli).unwrap();+ let signer = config.browser.fetch().web_bot_auth().unwrap();+ assert_eq!(signer.keyid(), RFC_9421_ED25519_KEYID);+ assert_eq!(signer.signature_agent_origin(), "https://bot.example:8443");+ assert_eq!(signer.profile(), WebBotAuthProfile::IetfDraft01);+}++#[test]+fn app_config_rejects_mismatched_web_bot_auth_keyid() {+ let key_file = write_temp_web_bot_auth_key();+ let key_path_arg = key_file.path.to_string_lossy().into_owned();+ let cli = Cli::try_parse_from(normalize_args_for_compat([+ "moli",+ "fetch",+ "--web-bot-auth-key-file",+ key_path_arg.as_str(),+ "--web-bot-auth-keyid",+ "wrong-thumbprint",+ "--web-bot-auth-domain",+ "bot.example",+ "https://example.com",+ ]))+ .unwrap();++ let error = AppConfig::from_cli(&cli).unwrap_err().to_string();+ assert!(error.contains("does not match the private key"));+ assert!(error.contains(RFC_9421_ED25519_KEYID));+}+#[test]fn app_config_preserves_repeatable_request_headers() {let cli = Cli::try_parse_from(normalize_args_for_compat([@@ -574,6 +695,26 @@ fn app_config_preserves_repeatable_request_headers() {assert!(config.browser.fetch().default_request_headers().is_empty());}+struct TempWebBotAuthKeyFile {+ path: PathBuf,+}++impl Drop for TempWebBotAuthKeyFile {+ fn drop(&mut self) {+ let _ = fs::remove_file(&self.path);+ }+}++fn write_temp_web_bot_auth_key() -> TempWebBotAuthKeyFile {+ let sequence = NEXT_TEMP_KEY_FILE.fetch_add(1, Ordering::Relaxed);+ let path = std::env::temp_dir().join(format!(+ "moli-web-bot-auth-{}-{sequence}.pem",+ std::process::id()+ ));+ fs::write(&path, RFC_9421_ED25519_PRIVATE_KEY).unwrap();+ TempWebBotAuthKeyFile { path }+}+#[test]fn parses_request_headers_with_embedded_colons_and_empty_values() {let cli = Cli::try_parse_from(normalize_args_for_compat([diff --git a/skills/moli-webfetch/references/fetch-recipes.md b/skills/moli-webfetch/references/fetch-recipes.mdindex e122efa68..e2deba9a1 100644--- a/skills/moli-webfetch/references/fetch-recipes.md+++ b/skills/moli-webfetch/references/fetch-recipes.md@@ -135,6 +135,12 @@ For a crawl rather than a single lookup:- Use `--http-proxy`, `--http-no-proxy`, or`--http-host-resolve HOST:PORT:ADDR` when required by the environment.- Use either `--user-agent` or `--user-agent-suffix`, not both.+- Use `--web-bot-auth-key-file <PKCS8-PEM>` together with+ `--web-bot-auth-domain <DOMAIN>` only when the user supplied an authorized+ bot identity. The key remains local, and Moli signs HTTPS requests only.+- Add `--web-bot-auth-keyid <THUMBPRINT>` to assert the derived JWK thumbprint.+ Keep the default Cloudflare profile unless the receiver explicitly supports+ `--web-bot-auth-profile ietf-01`.- Use `--document-start-script` or `--document-start-script-file` only when thetask explicitly requires pre-navigation instrumentation.- Combine `--block-private-networks` with `--block-cidrs` for untrusted URL