Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

feat: implement Web Bot Auth signing

ldm0ce7fa9cd172e
diff --git a/Cargo.lock b/Cargo.lock
index 376d8fd34..972d6cb58 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2221,6 +2221,7 @@ dependencies = [
"moli-trace",
"moli-url",
"moli-url-policy",
+ "moli-web-bot-auth",
"moli-web-mime",
"openssl-sys",
"parking_lot",
@@ -2825,6 +2826,17 @@ dependencies = [
"v8",
]
+[[package]]
+name = "moli-web-bot-auth"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "aws-lc-rs",
+ "base64 0.22.1",
+ "moli-crypto",
+ "url",
+]
+
[[package]]
name = "moli-web-errors"
version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index f77ec5928..9e3855530 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -55,6 +55,7 @@ members = [
"moli-v8-test-util",
"moli-v8-util",
"moli-web-mime",
+ "moli-web-bot-auth",
"moli-web-errors",
"moli-protocol-webdriver-bidi",
"moli-protocol-webdriver-classic",
diff --git a/moli-crypto/src/ed25519.rs b/moli-crypto/src/ed25519.rs
new file mode 100644
index 000000000..0736c0de9
--- /dev/null
+++ b/moli-crypto/src/ed25519.rs
@@ -0,0 +1,98 @@
+use std::fmt;
+
+use aws_lc_rs::signature::{Ed25519KeyPair, KeyPair};
+
+const ED25519_PUBLIC_KEY_LENGTH: usize = 32;
+const ED25519_SIGNATURE_LENGTH: usize = 64;
+
+pub struct Ed25519SigningKey {
+ key_pair: Ed25519KeyPair,
+ public_key: [u8; ED25519_PUBLIC_KEY_LENGTH],
+}
+
+impl Ed25519SigningKey {
+ pub fn from_pkcs8(der: &[u8]) -> Result<Self, Ed25519Error> {
+ let key_pair = Ed25519KeyPair::from_pkcs8(der).map_err(|_| Ed25519Error)?;
+ let public_key = key_pair
+ .public_key()
+ .as_ref()
+ .try_into()
+ .map_err(|_| Ed25519Error)?;
+ Ok(Self {
+ key_pair,
+ public_key,
+ })
+ }
+
+ pub fn public_key(&self) -> &[u8; ED25519_PUBLIC_KEY_LENGTH] {
+ &self.public_key
+ }
+
+ pub fn sign(&self, message: &[u8]) -> Result<[u8; ED25519_SIGNATURE_LENGTH], Ed25519Error> {
+ self.key_pair
+ .try_sign(message)
+ .map_err(|_| Ed25519Error)?
+ .as_ref()
+ .try_into()
+ .map_err(|_| Ed25519Error)
+ }
+}
+
+impl fmt::Debug for Ed25519SigningKey {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter
+ .debug_struct("Ed25519SigningKey")
+ .field("public_key", &self.public_key)
+ .finish_non_exhaustive()
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Ed25519Error;
+
+impl fmt::Display for Ed25519Error {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str("invalid Ed25519 key or signing operation failed")
+ }
+}
+
+impl std::error::Error for Ed25519Error {}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const RFC_9421_PRIVATE_KEY_DER: &[u8] = &[
+ 0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04,
+ 0x20, 0x9f, 0x83, 0x62, 0xf8, 0x7a, 0x48, 0x4a, 0x95, 0x4e, 0x6e, 0x74, 0x0c, 0x5b, 0x4c,
+ 0x0e, 0x84, 0x22, 0x91, 0x39, 0xa2, 0x0a, 0xa8, 0xab, 0x56, 0xff, 0x66, 0x58, 0x6f, 0x6a,
+ 0x7d, 0x29, 0xc5,
+ ];
+
+ #[test]
+ fn imports_pkcs8_and_signs_deterministically() {
+ let key = Ed25519SigningKey::from_pkcs8(RFC_9421_PRIVATE_KEY_DER).unwrap();
+ assert_eq!(
+ key.public_key(),
+ &[
+ 0x26, 0xb4, 0x0b, 0x8f, 0x93, 0xff, 0xf3, 0xd8, 0x97, 0x11, 0x2f, 0x7e, 0xbc, 0x58,
+ 0x2b, 0x23, 0x2d, 0xbd, 0x72, 0x51, 0x7d, 0x08, 0x2f, 0xe8, 0x3c, 0xfb, 0x30, 0xdd,
+ 0xce, 0x43, 0xd1, 0xbb,
+ ]
+ );
+
+ let first = key.sign(b"web bot auth").unwrap();
+ let second = key.sign(b"web bot auth").unwrap();
+ assert_eq!(first.len(), ED25519_SIGNATURE_LENGTH);
+ assert_eq!(first, second);
+ assert_ne!(first, key.sign(b"different message").unwrap());
+ }
+
+ #[test]
+ fn rejects_non_ed25519_pkcs8() {
+ assert_eq!(
+ Ed25519SigningKey::from_pkcs8(b"not a key").unwrap_err(),
+ Ed25519Error
+ );
+ }
+}
diff --git a/moli-crypto/src/lib.rs b/moli-crypto/src/lib.rs
index 891b8fae3..c52fdb80c 100644
--- a/moli-crypto/src/lib.rs
+++ b/moli-crypto/src/lib.rs
@@ -5,7 +5,9 @@
//! owner (`moli-webcrypto` or the renderer subsystem using the primitive).
mod digest;
+mod ed25519;
mod random;
pub use digest::{DigestAlgorithm, Sha256Context, sha1_digest, sha256_digest, sha256_hex};
+pub use ed25519::{Ed25519Error, Ed25519SigningKey};
pub use random::fill_secure_random;
diff --git a/moli-fetch/Cargo.toml b/moli-fetch/Cargo.toml
index b4f57e7f7..87542af5c 100644
--- a/moli-fetch/Cargo.toml
+++ b/moli-fetch/Cargo.toml
@@ -23,6 +23,7 @@ moli-http-cache = { path = "../moli-http-cache" }
moli-trace = { path = "../moli-trace" }
moli-url = { path = "../moli-url" }
moli-url-policy = { path = "../moli-url-policy" }
+moli-web-bot-auth = { path = "../moli-web-bot-auth" }
moli-web-mime = { path = "../moli-web-mime" }
parking_lot = "0.12"
strum = { version = "0.28.0", features = ["derive"] }
diff --git a/moli-fetch/src/blocking/cache/key.rs b/moli-fetch/src/blocking/cache/key.rs
index a4801c2a5..00c847732 100644
--- a/moli-fetch/src/blocking/cache/key.rs
+++ b/moli-fetch/src/blocking/cache/key.rs
@@ -19,6 +19,7 @@ pub(super) fn cache_store_and_key_for_request(
|| !request.method.eq_ignore_ascii_case("GET")
|| request.body.is_some()
|| request.auth().is_some()
+ || config.web_bot_auth().is_some()
|| !config.default_request_headers().is_empty()
|| !subresource_validation_allows_http_cache(request)
|| cookie_header.is_some()
diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs
index ffd176d00..42a5748d7 100644
--- a/moli-fetch/src/blocking/mod.rs
+++ b/moli-fetch/src/blocking/mod.rs
@@ -244,21 +244,12 @@ pub(crate) fn outgoing_request_headers_for_url(
outgoing
}
-pub(crate) fn network_request_extra_info_for_url(
+pub(crate) fn network_request_extra_info_from_headers(
config: &FetchConfig,
- request: &Request,
- request_url: &Url,
- redirect_chain: &[RedirectInfo],
- cookie_header: Option<&str>,
+ outgoing_headers: &[(String, String)],
cookie_report: Option<&StoredCookieQueryReport>,
) -> NetworkRequestExtraInfo {
- let mut headers = outgoing_request_headers_for_url(
- config,
- request,
- request_url,
- redirect_chain,
- cookie_header,
- );
+ let mut headers = outgoing_headers.to_vec();
append_header_if_missing(&mut headers, "User-Agent", config.user_agent().to_owned());
NetworkRequestExtraInfo {
headers,
@@ -599,7 +590,7 @@ pub(crate) fn configure_easy<H: Handler>(
cookie_header: Option<&str>,
http_version: RequestHttpVersion,
validation_headers: Option<Vec<(String, String)>>,
-) -> Result<()> {
+) -> Result<Vec<(String, String)>> {
ensure_http_network_transport_url(request_url)?;
enforce_request_target_policy(config, request_url)?;
configure_curl_http_protocol_allowlist(easy)?;
@@ -712,17 +703,22 @@ pub(crate) fn configure_easy<H: Handler>(
}
let mut headers = List::new();
- let outgoing_headers = outgoing_request_headers_for_url(
+ let mut outgoing_headers = outgoing_request_headers_for_url(
config,
request,
request_url,
redirect_chain,
cookie_header,
);
+ if let Some(web_bot_auth) = config.web_bot_auth() {
+ web_bot_auth
+ .append_request_headers(&mut outgoing_headers, &request.method, request_url)
+ .with_context(|| anyhow!("failed to sign web bot auth request for {request_url}"))?;
+ }
let mut has_headers = false;
let mut has_content_type_header = false;
- for (name, value) in outgoing_headers {
+ for (name, value) in &outgoing_headers {
has_content_type_header |= name.eq_ignore_ascii_case("content-type");
let header_line = if value.is_empty() {
format!("{name}:")
@@ -797,7 +793,7 @@ pub(crate) fn configure_easy<H: Handler>(
}
}
- Ok(())
+ Ok(outgoing_headers)
}
pub(crate) fn configure_openssl_tls_context(
diff --git a/moli-fetch/src/config.rs b/moli-fetch/src/config.rs
index 179140ef4..523568466 100644
--- a/moli-fetch/src/config.rs
+++ b/moli-fetch/src/config.rs
@@ -3,6 +3,8 @@ use std::num::NonZeroU32;
use cidr::AnyIpCidr;
use moli_browser_profile::{BrowserIdentityProfile, DEFAULT_ACCEPT_LANGUAGE};
+use crate::WebBotAuthSigner;
+
#[derive(Debug, Clone, PartialEq)]
pub struct FetchConfig {
browser_identity: BrowserIdentityProfile,
@@ -34,6 +36,7 @@ pub struct FetchConfig {
block_private_networks: bool,
block_cidrs: Vec<AnyIpCidr>,
tls_verify_host: bool,
+ web_bot_auth: Option<WebBotAuthSigner>,
}
impl FetchConfig {
@@ -144,6 +147,14 @@ impl FetchConfig {
self.tls_verify_host = tls_verify_host;
}
+ pub fn web_bot_auth(&self) -> Option<&WebBotAuthSigner> {
+ self.web_bot_auth.as_ref()
+ }
+
+ pub fn set_web_bot_auth(&mut self, web_bot_auth: Option<WebBotAuthSigner>) {
+ self.web_bot_auth = web_bot_auth;
+ }
+
pub fn set_http_proxy(&mut self, http_proxy: Option<String>) {
self.http_proxy = http_proxy;
}
@@ -274,6 +285,7 @@ impl Default for FetchConfig {
block_private_networks: false,
block_cidrs: Vec::new(),
tls_verify_host: true,
+ web_bot_auth: None,
}
}
}
diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rs
index 34247e14b..95b055bfe 100644
--- a/moli-fetch/src/lib.rs
+++ b/moli-fetch/src/lib.rs
@@ -45,6 +45,7 @@ pub use headers::{
is_forbidden_response_header_name, is_no_cors_safelisted_request_header,
};
pub use moli_cookie_jar::SharedBrowserCookieStore as SharedCookieStore;
+pub use moli_web_bot_auth::{WebBotAuthProfile, WebBotAuthSigner};
pub use network_fetch_result::{
NetworkExchangeObservation, NetworkFetchFailure, NetworkFetchFailureRequestContext,
NetworkFetchResult, NetworkObservationJournal, NetworkRequestObservation,
diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs
index de50acd9d..771312bf3 100644
--- a/moli-fetch/src/runtime.rs
+++ b/moli-fetch/src/runtime.rs
@@ -45,7 +45,7 @@ use crate::{
cookie_access_report_for_request, cookie_header_from_report,
finish_streaming_cached_response, load_cached_streaming_response_lookup,
log_request_completion, merge_cached_not_modified_streaming_response_lookup,
- network_request_extra_info_for_url, next_followed_redirect_url_from_parts,
+ network_request_extra_info_from_headers, next_followed_redirect_url_from_parts,
remove_cached_response, response_headers_forbid_cache_storage, store_response_cookies,
transfer_metrics_from_easy, validation_headers_for_cached_streaming_response_lookup,
},
@@ -863,22 +863,6 @@ impl RuntimeOwner {
&job.request,
&job.current_url,
);
- let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
- network_request_extra_info_for_url(
- &self.config,
- &prepared_request.request,
- &job.current_url,
- &job.redirect_chain,
- cookie_header.as_deref(),
- request_cookie_report.as_ref(),
- )
- });
- attach_next_request_extra_info(
- &mut job.redirect_chain,
- request_cookie_report.clone(),
- request_extra_info.as_ref(),
- );
-
let mut easy = Easy2::new(FetchTransferHandler::new_buffered(ResponseCollector::new(
Some(job.cancel_handle.clone()),
)));
@@ -894,7 +878,7 @@ impl RuntimeOwner {
) {
return Err((job.response_tx, error));
}
- if let Err(error) = configure_easy(
+ let outgoing_headers = match configure_easy(
&mut easy,
&self.config,
&prepared_request.request,
@@ -909,8 +893,21 @@ impl RuntimeOwner {
)
.with_context(|| anyhow!("failed to configure curl request for {}", job.current_url))
{
- return Err((job.response_tx, error));
- }
+ Ok(headers) => headers,
+ Err(error) => return Err((job.response_tx, error)),
+ };
+ let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
+ network_request_extra_info_from_headers(
+ &self.config,
+ &outgoing_headers,
+ request_cookie_report.as_ref(),
+ )
+ });
+ attach_next_request_extra_info(
+ &mut job.redirect_chain,
+ request_cookie_report.clone(),
+ request_extra_info.as_ref(),
+ );
let label = job.current_url.to_string();
let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);
@@ -995,16 +992,6 @@ impl RuntimeOwner {
&job.request,
&job.current_url,
);
- let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
- network_request_extra_info_for_url(
- &self.config,
- &prepared_request.request,
- &job.current_url,
- &job.redirect_chain,
- cookie_header.as_deref(),
- request_cookie_report.as_ref(),
- )
- });
match load_cached_streaming_response_lookup(
&self.config,
&prepared_request.request,
@@ -1033,12 +1020,6 @@ impl RuntimeOwner {
Err(error) => return Err((Box::new(job), None, error)),
}
- attach_next_request_extra_info(
- &mut job.redirect_chain,
- request_cookie_report.clone(),
- request_extra_info.as_ref(),
- );
-
let mut easy = job.easy.take().unwrap_or_else(|| {
Easy2::new(FetchTransferHandler::new_streaming(
StreamingResponseCollector::new(
@@ -1061,21 +1042,6 @@ impl RuntimeOwner {
cookie_header.clone(),
));
- let collector = easy
- .get_mut()
- .streaming_mut()
- .expect("streaming request should use streaming collector");
- collector.begin_request_with_cache_plan(
- self.config.http_max_response_size(),
- job.current_url.clone(),
- job.current_cookie_context.clone(),
- request_cookie_report.clone(),
- credentials_allowed,
- job.redirect_chain.clone(),
- request_extra_info.clone(),
- cache_plan,
- );
- collector.set_client_hint_response_policy(prepared_request.response_policy);
if let Err(error) = configure_network_observation(
&mut easy,
&job.request,
@@ -1084,7 +1050,7 @@ impl RuntimeOwner {
) {
return Err((Box::new(job), Some(easy), error));
}
- if let Err(error) = configure_easy(
+ let outgoing_headers = match configure_easy(
&mut easy,
&self.config,
&prepared_request.request,
@@ -1096,8 +1062,36 @@ impl RuntimeOwner {
)
.with_context(|| anyhow!("failed to configure curl request for {}", job.current_url))
{
- return Err((Box::new(job), Some(easy), error));
- }
+ Ok(headers) => headers,
+ Err(error) => return Err((Box::new(job), Some(easy), error)),
+ };
+ let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
+ network_request_extra_info_from_headers(
+ &self.config,
+ &outgoing_headers,
+ request_cookie_report.as_ref(),
+ )
+ });
+ attach_next_request_extra_info(
+ &mut job.redirect_chain,
+ request_cookie_report.clone(),
+ request_extra_info.as_ref(),
+ );
+ let collector = easy
+ .get_mut()
+ .streaming_mut()
+ .expect("streaming request should use streaming collector");
+ collector.begin_request_with_cache_plan(
+ self.config.http_max_response_size(),
+ job.current_url.clone(),
+ job.current_cookie_context.clone(),
+ request_cookie_report.clone(),
+ credentials_allowed,
+ job.redirect_chain.clone(),
+ request_extra_info.clone(),
+ cache_plan,
+ );
+ collector.set_client_hint_response_policy(prepared_request.response_policy);
let label = job.current_url.to_string();
let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);
@@ -1189,16 +1183,6 @@ impl RuntimeOwner {
&job.request,
&job.current_url,
);
- let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
- network_request_extra_info_for_url(
- &self.config,
- &prepared_request.request,
- &job.current_url,
- &job.redirect_chain,
- cookie_header.as_deref(),
- request_cookie_report.as_ref(),
- )
- });
let mut stale_cached_lookup = None;
match load_cached_streaming_response_lookup(
&self.config,
@@ -1231,12 +1215,6 @@ impl RuntimeOwner {
Err(error) => return Err((Box::new(job), None, error)),
}
- attach_next_request_extra_info(
- &mut job.redirect_chain,
- request_cookie_report.clone(),
- request_extra_info.as_ref(),
- );
-
let mut easy = job.easy.take().unwrap_or_else(|| {
Easy2::new(FetchTransferHandler::new_raw_streaming(
RawStreamingResponseCollector::new(
@@ -1258,22 +1236,6 @@ impl RuntimeOwner {
job.current_url.clone(),
cookie_header.clone(),
));
- let collector = easy
- .get_mut()
- .raw_streaming_mut()
- .expect("raw streaming request should use raw streaming collector");
- collector.begin_request_with_cache_plan(
- self.config.http_max_response_size(),
- job.current_url.clone(),
- job.current_cookie_context.clone(),
- request_cookie_report.clone(),
- credentials_allowed,
- job.redirect_chain.clone(),
- request_extra_info.clone(),
- cache_plan,
- stale_cached_lookup.is_some(),
- );
- collector.set_client_hint_response_policy(prepared_request.response_policy);
if let Err(error) = configure_network_observation(
&mut easy,
&job.request,
@@ -1282,7 +1244,7 @@ impl RuntimeOwner {
) {
return Err((Box::new(job), Some(easy), error));
}
- if let Err(error) = configure_easy(
+ let outgoing_headers = match configure_easy(
&mut easy,
&self.config,
&prepared_request.request,
@@ -1296,8 +1258,37 @@ impl RuntimeOwner {
)
.with_context(|| anyhow!("failed to configure curl request for {}", job.current_url))
{
- return Err((Box::new(job), Some(easy), error));
- }
+ Ok(headers) => headers,
+ Err(error) => return Err((Box::new(job), Some(easy), error)),
+ };
+ let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
+ network_request_extra_info_from_headers(
+ &self.config,
+ &outgoing_headers,
+ request_cookie_report.as_ref(),
+ )
+ });
+ attach_next_request_extra_info(
+ &mut job.redirect_chain,
+ request_cookie_report.clone(),
+ request_extra_info.as_ref(),
+ );
+ let collector = easy
+ .get_mut()
+ .raw_streaming_mut()
+ .expect("raw streaming request should use raw streaming collector");
+ collector.begin_request_with_cache_plan(
+ self.config.http_max_response_size(),
+ job.current_url.clone(),
+ job.current_cookie_context.clone(),
+ request_cookie_report.clone(),
+ credentials_allowed,
+ job.redirect_chain.clone(),
+ request_extra_info.clone(),
+ cache_plan,
+ stale_cached_lookup.is_some(),
+ );
+ collector.set_client_hint_response_policy(prepared_request.response_policy);
let label = job.current_url.to_string();
let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);
diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rs
index 7e8512765..e6d7b41e1 100644
--- a/moli-fetch/src/tests/mod.rs
+++ b/moli-fetch/src/tests/mod.rs
@@ -8,6 +8,7 @@ use moli_browser_profile::DEFAULT_ACCEPT_LANGUAGE;
use moli_cookie_jar::{NetworkCookieRequestContext, new_shared_browser_cookie_store};
use moli_http_cache::{HttpCacheEntryMetadata, HttpCacheStore};
use std::{
+ collections::BTreeSet,
fs,
io::Read,
num::NonZeroU32,
@@ -28,7 +29,8 @@ use crate::{
RequestAuthScheme, RequestAuthTarget, RequestCacheMode, RequestCredentialsMode, RequestMode,
RequestRedirectMode, RequestResourceType, Response, ResponseBody, ResponseHead,
ScriptFetchRequestMetadata, ScriptFetchSchedulerPriority, StreamingResponseCollector,
- SubresourceRequestMetadata, http_cache_stats, runtime::FetchRuntimeOwner,
+ SubresourceRequestMetadata, WebBotAuthProfile, WebBotAuthSigner, http_cache_stats,
+ runtime::FetchRuntimeOwner,
};
use self::support::{
@@ -39,6 +41,19 @@ use self::support::{
const ENV_PROXY_CHILD_TEST: &str = "MOLI_FETCH_ENV_PROXY_CHILD";
const ENV_PROXY_URL: &str = "MOLI_FETCH_ENV_PROXY_URL";
const TEST_HIGH_ENTROPY_CLIENT_HINTS: &str = "Sec-CH-UA-Full-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Arch, Sec-CH-UA-Bitness, Sec-CH-UA-Platform-Version, Sec-CH-UA-Model, Sec-CH-UA-WoW64";
+const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\
+MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\
+-----END PRIVATE KEY-----\n";
+
+fn test_web_bot_auth_signer() -> WebBotAuthSigner {
+ WebBotAuthSigner::from_pem(
+ RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
+ "bot.example",
+ Some("poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U"),
+ WebBotAuthProfile::Cloudflare,
+ )
+ .unwrap()
+}
fn sample_response_head() -> ResponseHead {
ResponseHead {
@@ -1191,6 +1206,172 @@ async fn fetch_raw_stream_treats_https11_switching_protocols_as_final_response()
Ok(())
}
+#[test]
+fn web_bot_auth_resigns_restarts_redirects_and_subresources() -> Result<()> {
+ let server = ScriptedHttps11Server::spawn(vec![
+ ScriptedResponse::status(403, "Challenge")
+ .with_header("Accept-CH", "Sec-CH-UA-Arch")
+ .with_header("Critical-CH", "Sec-CH-UA-Arch"),
+ ScriptedResponse::status(302, "Found").with_header("Location", "/final"),
+ ScriptedResponse::ok("final"),
+ ScriptedResponse::ok("asset"),
+ ]);
+ let mut config = FetchConfig::default();
+ config.set_tls_verify_host(false);
+ config.set_web_bot_auth(Some(test_web_bot_auth_signer()));
+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());
+
+ let response = fetch_response_for_test(&client, Request::get(&server.url_path("/start"))?)?;
+ assert_eq!(response.body_text(), "final");
+ assert_eq!(response.redirect_chain.len(), 2);
+ assert_eq!(response.redirect_chain[0].status, 307);
+ assert_eq!(response.redirect_chain[1].status, 302);
+
+ let final_url = Url::parse(&server.url_path("/final"))?;
+ let subresource = Request::new(
+ "POST",
+ &server.url_path("/asset"),
+ Some("probe".to_owned()),
+ Vec::new(),
+ )?
+ .with_browser_request_metadata(BrowserRequestMetadata::Fetch)
+ .with_initiator_url(&final_url);
+ assert_eq!(
+ fetch_response_for_test(&client, subresource)?.body_text(),
+ "asset"
+ );
+
+ assert_eq!(
+ server.requests(),
+ [
+ "/start".to_owned(),
+ "/start".to_owned(),
+ "/final".to_owned(),
+ "/asset".to_owned(),
+ ]
+ );
+ let request_heads = server.request_heads();
+ assert_eq!(request_heads.len(), 4);
+ for request_head in &request_heads {
+ assert_eq!(
+ request_head_header_value(request_head, "Signature-Agent"),
+ Some("\"https://bot.example\"")
+ );
+ let signature_input = request_head_header_value(request_head, "Signature-Input")
+ .expect("signed request should include Signature-Input");
+ assert!(
+ signature_input.contains("(\"@authority\" \"@method\" \"@path\" \"signature-agent\")")
+ );
+ assert!(request_head_header_value(request_head, "Signature").is_some());
+ }
+ assert!(request_heads[3].starts_with("POST /asset HTTP/1.1\r\n"));
+
+ let nonces = request_heads
+ .iter()
+ .map(|request_head| {
+ nonce_from_signature_input(
+ request_head_header_value(request_head, "Signature-Input").unwrap(),
+ )
+ .to_owned()
+ })
+ .collect::<BTreeSet<_>>();
+ assert_eq!(nonces.len(), request_heads.len());
+
+ let restart = &response.redirect_chain[0];
+ assert_request_extra_signature_matches_wire(
+ &restart
+ .response_extra_info
+ .as_ref()
+ .expect("restart response extra info")
+ .request_extra_info,
+ &request_heads[0],
+ );
+ assert_request_extra_signature_matches_wire(
+ restart
+ .request_extra_info
+ .as_ref()
+ .expect("restarted request extra info"),
+ &request_heads[1],
+ );
+ let redirect = &response.redirect_chain[1];
+ assert_request_extra_signature_matches_wire(
+ &redirect
+ .response_extra_info
+ .as_ref()
+ .expect("redirect response extra info")
+ .request_extra_info,
+ &request_heads[1],
+ );
+ assert_request_extra_signature_matches_wire(
+ redirect
+ .request_extra_info
+ .as_ref()
+ .expect("redirect target request extra info"),
+ &request_heads[2],
+ );
+ assert_request_extra_signature_matches_wire(
+ response
+ .network_request_extra_info()
+ .expect("final request extra info"),
+ &request_heads[2],
+ );
+
+ server.shutdown();
+ Ok(())
+}
+
+#[test]
+fn web_bot_auth_is_not_sent_over_plain_http() -> Result<()> {
+ let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("plain")]);
+ let mut config = FetchConfig::default();
+ config.set_web_bot_auth(Some(test_web_bot_auth_signer()));
+
+ let response = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;
+ assert_eq!(response.body_text(), "plain");
+ let requests = server.requests();
+ assert_eq!(requests.len(), 1);
+ for name in ["Signature-Agent", "Signature-Input", "Signature"] {
+ assert_eq!(request_head_header_value(&requests[0], name), None);
+ }
+
+ server.shutdown();
+ Ok(())
+}
+
+#[test]
+fn web_bot_auth_bypasses_shared_http_cache() -> Result<()> {
+ let cache_dir = unique_test_cache_dir();
+ let server = ScriptedHttps11Server::spawn(vec![
+ ScriptedResponse::ok("first").with_header("Cache-Control", "max-age=60"),
+ ScriptedResponse::ok("second").with_header("Cache-Control", "max-age=60"),
+ ]);
+ let mut config = FetchConfig::default();
+ config.set_tls_verify_host(false);
+ config.set_http_cache_dir(Some(cache_dir.display().to_string()));
+ config.set_web_bot_auth(Some(test_web_bot_auth_signer()));
+
+ let first = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;
+ let second = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;
+ assert_eq!(first.body_text(), "first");
+ assert_eq!(second.body_text(), "second");
+ assert!(!first.from_cache);
+ assert!(!second.from_cache);
+ assert_eq!(server.hits(), 2);
+ let request_heads = server.request_heads();
+ assert_ne!(
+ request_head_header_value(&request_heads[0], "Signature-Input"),
+ request_head_header_value(&request_heads[1], "Signature-Input")
+ );
+ assert!(
+ !cache_dir.exists() || fs::read_dir(&cache_dir)?.next().is_none(),
+ "authenticated responses must not enter the shared disk cache"
+ );
+
+ server.shutdown();
+ let _ = fs::remove_dir_all(cache_dir);
+ Ok(())
+}
+
#[tokio::test]
async fn fetch_raw_stream_finishes_switching_protocols_body_without_connection_close() -> Result<()>
{
@@ -3616,6 +3797,34 @@ fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a s
.map(|(_, value)| value.as_str())
}
+fn request_head_header_value<'a>(request_head: &'a str, name: &str) -> Option<&'a str> {
+ request_head.lines().skip(1).find_map(|line| {
+ let (header_name, value) = line.split_once(':')?;
+ header_name.eq_ignore_ascii_case(name).then(|| value.trim())
+ })
+}
+
+fn nonce_from_signature_input(signature_input: &str) -> &str {
+ signature_input
+ .split(";nonce=\"")
+ .nth(1)
+ .and_then(|value| value.split('"').next())
+ .expect("Signature-Input should contain a nonce parameter")
+}
+
+fn assert_request_extra_signature_matches_wire(
+ extra_info: &crate::NetworkRequestExtraInfo,
+ request_head: &str,
+) {
+ for name in ["Signature-Agent", "Signature-Input", "Signature"] {
+ assert_eq!(
+ header_value(&extra_info.headers, name),
+ request_head_header_value(request_head, name),
+ "network extra info and wire request differ for {name}"
+ );
+ }
+}
+
#[test]
fn fetch_client_cache_respects_pragma_no_cache_response() {
let cache_dir = unique_test_cache_dir();
diff --git a/moli-fetch/src/tests/support.rs b/moli-fetch/src/tests/support.rs
index d7b86d8bc..5d3b8f06b 100644
--- a/moli-fetch/src/tests/support.rs
+++ b/moli-fetch/src/tests/support.rs
@@ -43,6 +43,7 @@ pub(super) struct ScriptedHttps11Server {
addr: std::net::SocketAddr,
hits: Arc<AtomicUsize>,
requests: Arc<Mutex<Vec<String>>>,
+ request_heads: Arc<Mutex<Vec<String>>>,
shutdown_tx: std_mpsc::Sender<()>,
join_handle: Option<thread::JoinHandle<()>>,
}
@@ -56,16 +57,19 @@ impl ScriptedHttps11Server {
let addr = listener.local_addr().unwrap();
let hits = Arc::new(AtomicUsize::new(0));
let requests = Arc::new(Mutex::new(Vec::new()));
+ let request_heads = Arc::new(Mutex::new(Vec::new()));
let (shutdown_tx, shutdown_rx) = std_mpsc::channel();
let responses = Arc::new(Mutex::new(VecDeque::from(responses)));
let hits_for_thread = Arc::clone(&hits);
let requests_for_thread = Arc::clone(&requests);
+ let request_heads_for_thread = Arc::clone(&request_heads);
let join_handle = thread::spawn(move || {
run_https11_server(
listener,
shutdown_rx,
hits_for_thread,
requests_for_thread,
+ request_heads_for_thread,
responses,
);
});
@@ -74,6 +78,7 @@ impl ScriptedHttps11Server {
addr,
hits,
requests,
+ request_heads,
shutdown_tx,
join_handle: Some(join_handle),
}
@@ -83,6 +88,11 @@ impl ScriptedHttps11Server {
format!("https://{}/cache", self.addr)
}
+ pub(super) fn url_path(&self, path: &str) -> String {
+ let path = path.strip_prefix('/').unwrap_or(path);
+ format!("https://{}/{}", self.addr, path)
+ }
+
pub(super) fn hits(&self) -> usize {
self.hits.load(Ordering::SeqCst)
}
@@ -91,6 +101,10 @@ impl ScriptedHttps11Server {
self.requests.lock().clone()
}
+ pub(super) fn request_heads(&self) -> Vec<String> {
+ self.request_heads.lock().clone()
+ }
+
pub(super) fn shutdown(mut self) {
let _ = self.shutdown_tx.send(());
if let Some(join_handle) = self.join_handle.take() {
@@ -710,6 +724,7 @@ fn run_https11_server(
shutdown_rx: std_mpsc::Receiver<()>,
hits: Arc<AtomicUsize>,
requests: Arc<Mutex<Vec<String>>>,
+ request_heads: Arc<Mutex<Vec<String>>>,
responses: Arc<Mutex<VecDeque<ScriptedResponse>>>,
) {
let rt = tokio::runtime::Builder::new_current_thread()
@@ -731,6 +746,7 @@ fn run_https11_server(
let tls_acceptor = tls_acceptor.clone();
let hits = Arc::clone(&hits);
let requests = Arc::clone(&requests);
+ let request_heads = Arc::clone(&request_heads);
let responses = Arc::clone(&responses);
tokio::spawn(async move {
let Ok(mut stream) = tls_acceptor.accept(stream).await else {
@@ -751,6 +767,9 @@ fn run_https11_server(
if let Some(path) = request_path_from_head(&request) {
requests.lock().push(path);
}
+ request_heads
+ .lock()
+ .push(String::from_utf8_lossy(&request).into_owned());
let _ = hits.fetch_add(1, Ordering::SeqCst) + 1;
let response_spec = {
let mut responses = responses.lock();
diff --git a/moli-web-bot-auth/Cargo.toml b/moli-web-bot-auth/Cargo.toml
new file mode 100644
index 000000000..5ce2292f6
--- /dev/null
+++ b/moli-web-bot-auth/Cargo.toml
@@ -0,0 +1,18 @@
+[package]
+license.workspace = true
+name = "moli-web-bot-auth"
+version = "0.1.0"
+edition = "2024"
+description = "Web Bot Auth HTTP message signatures for Moli"
+
+[dependencies]
+anyhow = "1.0.100"
+base64 = "0.22"
+moli-crypto = { path = "../moli-crypto" }
+url = "2.5.7"
+
+[dev-dependencies]
+aws-lc-rs = { version = "1.16.2", default-features = false, features = ["aws-lc-sys"] }
+
+[lints]
+workspace = true
diff --git a/moli-web-bot-auth/src/key.rs b/moli-web-bot-auth/src/key.rs
new file mode 100644
index 000000000..d9f7ea81b
--- /dev/null
+++ b/moli-web-bot-auth/src/key.rs
@@ -0,0 +1,133 @@
+use std::str;
+
+use anyhow::{Context, Result, anyhow, bail};
+use base64::{Engine as _, engine::general_purpose};
+use moli_crypto::{Ed25519SigningKey, sha256_digest};
+
+pub(crate) struct WebBotAuthKey {
+ signing_key: Ed25519SigningKey,
+ public_key: [u8; 32],
+ keyid: String,
+}
+
+impl WebBotAuthKey {
+ pub(crate) fn from_pem(private_key_pem: &[u8], expected_keyid: Option<&str>) -> Result<Self> {
+ let private_key_der = decode_pkcs8_private_key_pem(private_key_pem)?;
+ let signing_key = Ed25519SigningKey::from_pkcs8(&private_key_der).map_err(|_| {
+ anyhow!("web bot auth key must be an unencrypted PKCS#8 Ed25519 private key")
+ })?;
+ let public_key = *signing_key.public_key();
+ let keyid = jwk_thumbprint(&public_key);
+ if let Some(expected_keyid) = expected_keyid
+ && expected_keyid != keyid
+ {
+ bail!(
+ "--web-bot-auth-keyid does not match the private key; expected `{expected_keyid}`, computed `{keyid}`"
+ );
+ }
+
+ Ok(Self {
+ signing_key,
+ public_key,
+ keyid,
+ })
+ }
+
+ pub(crate) fn keyid(&self) -> &str {
+ &self.keyid
+ }
+
+ pub(crate) fn public_key(&self) -> &[u8; 32] {
+ &self.public_key
+ }
+
+ pub(crate) fn sign(&self, message: &[u8]) -> Result<[u8; 64]> {
+ self.signing_key
+ .sign(message)
+ .context("failed to create web bot auth Ed25519 signature")
+ }
+}
+
+fn decode_pkcs8_private_key_pem(private_key_pem: &[u8]) -> Result<Vec<u8>> {
+ const BEGIN: &str = "-----BEGIN PRIVATE KEY-----";
+ const END: &str = "-----END PRIVATE KEY-----";
+
+ let pem = str::from_utf8(private_key_pem)
+ .context("web bot auth private key PEM is not valid UTF-8")?;
+ if pem.contains("-----BEGIN ENCRYPTED PRIVATE KEY-----") {
+ bail!("encrypted web bot auth private keys are not supported");
+ }
+ let pem = pem.trim();
+ let body = pem
+ .strip_prefix(BEGIN)
+ .and_then(|pem| pem.strip_suffix(END))
+ .ok_or_else(|| {
+ anyhow!("web bot auth key must use BEGIN PRIVATE KEY PKCS#8 PEM encoding")
+ })?;
+ if body.contains("-----") {
+ bail!("web bot auth key PEM must contain exactly one private key");
+ }
+ let encoded = body
+ .chars()
+ .filter(|character| !character.is_ascii_whitespace())
+ .collect::<String>();
+ if encoded.is_empty() {
+ bail!("web bot auth private key PEM body is empty");
+ }
+ general_purpose::STANDARD
+ .decode(encoded)
+ .context("web bot auth private key PEM contains invalid base64")
+}
+
+fn jwk_thumbprint(public_key: &[u8; 32]) -> String {
+ let x = general_purpose::URL_SAFE_NO_PAD.encode(public_key);
+ let canonical_jwk = format!(r#"{{"crv":"Ed25519","kty":"OKP","x":"{x}"}}"#);
+ general_purpose::URL_SAFE_NO_PAD.encode(sha256_digest(canonical_jwk))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::test_support::{EXPECTED_KEYID, RFC_9421_ED25519_PRIVATE_KEY};
+
+ #[test]
+ fn derives_rfc_jwk_thumbprint_and_public_key() {
+ let key = WebBotAuthKey::from_pem(
+ RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
+ Some(EXPECTED_KEYID),
+ )
+ .unwrap();
+
+ assert_eq!(key.keyid(), EXPECTED_KEYID);
+ assert_eq!(
+ general_purpose::URL_SAFE_NO_PAD.encode(key.public_key()),
+ "JrQLj5P_89iXES9-vFgrIy29clF9CC_oPPsw3c5D0bs"
+ );
+ }
+
+ #[test]
+ fn rejects_mismatched_keyid_and_invalid_pem() {
+ let wrong_keyid =
+ WebBotAuthKey::from_pem(RFC_9421_ED25519_PRIVATE_KEY.as_bytes(), Some("wrong"))
+ .err()
+ .expect("mismatched keyid should fail")
+ .to_string();
+ assert!(wrong_keyid.contains("does not match"));
+ assert!(wrong_keyid.contains(EXPECTED_KEYID));
+
+ let invalid_pem = WebBotAuthKey::from_pem(b"not a private key", None)
+ .err()
+ .expect("invalid PEM should fail")
+ .to_string();
+ assert!(invalid_pem.contains("BEGIN PRIVATE KEY"));
+
+ let encrypted = WebBotAuthKey::from_pem(
+ b"-----BEGIN ENCRYPTED PRIVATE KEY-----\nAA==\n-----END ENCRYPTED PRIVATE KEY-----",
+ None,
+ )
+ .err()
+ .expect("encrypted PEM should fail")
+ .to_string();
+ assert!(encrypted.contains("encrypted"));
+ }
+}
diff --git a/moli-web-bot-auth/src/lib.rs b/moli-web-bot-auth/src/lib.rs
new file mode 100644
index 000000000..2c13d9d0c
--- /dev/null
+++ b/moli-web-bot-auth/src/lib.rs
@@ -0,0 +1,12 @@
+//! Web Bot Auth request signing built on RFC 9421 HTTP message signatures.
+
+mod key;
+mod profile;
+mod signer;
+mod wire;
+
+#[cfg(test)]
+mod test_support;
+
+pub use profile::WebBotAuthProfile;
+pub use signer::WebBotAuthSigner;
diff --git a/moli-web-bot-auth/src/profile.rs b/moli-web-bot-auth/src/profile.rs
new file mode 100644
index 000000000..122ed672c
--- /dev/null
+++ b/moli-web-bot-auth/src/profile.rs
@@ -0,0 +1,15 @@
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub enum WebBotAuthProfile {
+ #[default]
+ Cloudflare,
+ IetfDraft01,
+}
+
+impl WebBotAuthProfile {
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::Cloudflare => "cloudflare",
+ Self::IetfDraft01 => "ietf-01",
+ }
+ }
+}
diff --git a/moli-web-bot-auth/src/signer.rs b/moli-web-bot-auth/src/signer.rs
new file mode 100644
index 000000000..3e811aa85
--- /dev/null
+++ b/moli-web-bot-auth/src/signer.rs
@@ -0,0 +1,291 @@
+use std::{
+ fmt,
+ sync::{
+ Arc,
+ atomic::{AtomicU64, Ordering},
+ },
+ time::{SystemTime, UNIX_EPOCH},
+};
+
+use anyhow::{Context, Result, anyhow, bail};
+use base64::{Engine as _, engine::general_purpose};
+use moli_crypto::{DigestAlgorithm, fill_secure_random};
+use url::Url;
+
+use crate::{
+ key::WebBotAuthKey,
+ profile::WebBotAuthProfile,
+ wire::{SignatureOptions, sign_request},
+};
+
+const SIGNATURE_LABEL: &str = "sig1";
+const SIGNATURE_TTL_SECS: u64 = 60;
+const NONCE_BYTES: usize = 64;
+
+#[derive(Clone)]
+pub struct WebBotAuthSigner {
+ inner: Arc<WebBotAuthSignerInner>,
+}
+
+struct WebBotAuthSignerInner {
+ key: WebBotAuthKey,
+ signature_agent_origin: String,
+ profile: WebBotAuthProfile,
+ nonce_seed: [u8; 32],
+ nonce_counter: AtomicU64,
+}
+
+impl WebBotAuthSigner {
+ pub fn from_pem(
+ private_key_pem: &[u8],
+ domain: &str,
+ expected_keyid: Option<&str>,
+ profile: WebBotAuthProfile,
+ ) -> Result<Self> {
+ let key = WebBotAuthKey::from_pem(private_key_pem, expected_keyid)?;
+ let signature_agent_origin = normalize_signature_agent_origin(domain)?;
+ let mut nonce_seed = [0_u8; 32];
+ fill_secure_random(&mut nonce_seed)
+ .map_err(|_| anyhow!("failed to initialize web bot auth nonce generation"))?;
+
+ Ok(Self {
+ inner: Arc::new(WebBotAuthSignerInner {
+ key,
+ signature_agent_origin,
+ profile,
+ nonce_seed,
+ nonce_counter: AtomicU64::new(0),
+ }),
+ })
+ }
+
+ pub fn keyid(&self) -> &str {
+ self.inner.key.keyid()
+ }
+
+ pub fn public_key(&self) -> &[u8; 32] {
+ self.inner.key.public_key()
+ }
+
+ pub fn signature_agent_origin(&self) -> &str {
+ &self.inner.signature_agent_origin
+ }
+
+ pub fn profile(&self) -> WebBotAuthProfile {
+ self.inner.profile
+ }
+
+ pub fn append_request_headers(
+ &self,
+ headers: &mut Vec<(String, String)>,
+ method: &str,
+ request_url: &Url,
+ ) -> Result<()> {
+ if request_url.scheme() != "https" {
+ return Ok(());
+ }
+
+ headers.retain(|(name, _)| !is_web_bot_auth_header(name));
+ let created = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_secs();
+ let signed = sign_request(
+ &self.inner.key,
+ &self.inner.signature_agent_origin,
+ self.inner.profile,
+ method,
+ request_url,
+ &SignatureOptions {
+ signature_label: SIGNATURE_LABEL,
+ signature_agent_label: SIGNATURE_LABEL,
+ created,
+ expires: created.saturating_add(SIGNATURE_TTL_SECS),
+ nonce: self.next_nonce(),
+ cover_request_target: true,
+ },
+ )?;
+ headers.extend(signed.headers.into_pairs());
+ Ok(())
+ }
+
+ fn next_nonce(&self) -> String {
+ let counter = self.inner.nonce_counter.fetch_add(1, Ordering::Relaxed);
+ let mut material = Vec::with_capacity(32 + 8 + 24);
+ material.extend_from_slice(b"moli-web-bot-auth-nonce\0");
+ material.extend_from_slice(&self.inner.nonce_seed);
+ material.extend_from_slice(&counter.to_be_bytes());
+ let nonce = DigestAlgorithm::Sha512.digest_bytes(&material);
+ debug_assert_eq!(nonce.len(), NONCE_BYTES);
+ general_purpose::STANDARD.encode(nonce)
+ }
+}
+
+impl fmt::Debug for WebBotAuthSigner {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter
+ .debug_struct("WebBotAuthSigner")
+ .field("keyid", &self.inner.key.keyid())
+ .field("signature_agent_origin", &self.inner.signature_agent_origin)
+ .field("profile", &self.inner.profile)
+ .finish_non_exhaustive()
+ }
+}
+
+impl PartialEq for WebBotAuthSigner {
+ fn eq(&self, other: &Self) -> bool {
+ Arc::ptr_eq(&self.inner, &other.inner)
+ || (self.inner.key.public_key() == other.inner.key.public_key()
+ && self.inner.signature_agent_origin == other.inner.signature_agent_origin
+ && self.inner.profile == other.inner.profile)
+ }
+}
+
+impl Eq for WebBotAuthSigner {}
+
+fn normalize_signature_agent_origin(domain: &str) -> Result<String> {
+ if domain.is_empty() || domain.trim() != domain {
+ bail!("--web-bot-auth-domain must be a non-empty domain without surrounding whitespace");
+ }
+ if domain.contains(['/', '?', '#', '@']) {
+ bail!(
+ "--web-bot-auth-domain must contain only a host and optional port, without a scheme, credentials, path, query, or fragment"
+ );
+ }
+
+ let url = Url::parse(&format!("https://{domain}"))
+ .context("failed to parse --web-bot-auth-domain as an HTTPS origin")?;
+ if url.host().is_none()
+ || !url.username().is_empty()
+ || url.password().is_some()
+ || url.path() != "/"
+ || url.query().is_some()
+ || url.fragment().is_some()
+ {
+ bail!("--web-bot-auth-domain must identify a single HTTPS origin");
+ }
+ Ok(url.origin().ascii_serialization())
+}
+
+fn is_web_bot_auth_header(name: &str) -> bool {
+ name.eq_ignore_ascii_case("signature-agent")
+ || name.eq_ignore_ascii_case("signature-input")
+ || name.eq_ignore_ascii_case("signature")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::test_support::{EXPECTED_KEYID, RFC_9421_ED25519_PRIVATE_KEY, signer};
+
+ #[test]
+ fn normalizes_signature_agent_origin() {
+ let signer = signer(WebBotAuthProfile::Cloudflare);
+
+ assert_eq!(signer.keyid(), EXPECTED_KEYID);
+ assert_eq!(
+ signer.signature_agent_origin(),
+ "https://signature-agent.test"
+ );
+ }
+
+ #[test]
+ fn production_requests_use_unique_nonces() {
+ let signer = signer(WebBotAuthProfile::Cloudflare);
+ let request_url = Url::parse("https://example.com/path").unwrap();
+ let mut first_headers = Vec::new();
+ signer
+ .append_request_headers(&mut first_headers, "post", &request_url)
+ .unwrap();
+ let mut second_headers = Vec::new();
+ signer
+ .append_request_headers(&mut second_headers, "post", &request_url)
+ .unwrap();
+
+ let first_input = header_value(&first_headers, "signature-input");
+ assert!(first_input.contains("(\"@authority\" \"@method\" \"@path\" \"signature-agent\")"));
+ assert!(first_input.contains(";expires="));
+ assert!(first_input.contains(";tag=\"web-bot-auth\""));
+ assert_ne!(
+ nonce_from_signature_input(first_input),
+ nonce_from_signature_input(header_value(&second_headers, "signature-input"))
+ );
+ }
+
+ #[test]
+ fn omits_signatures_on_insecure_requests_and_replaces_spoofed_https_headers() {
+ let signer = signer(WebBotAuthProfile::Cloudflare);
+ let spoofed = vec![
+ ("Signature-Agent".to_owned(), "spoofed".to_owned()),
+ ("Signature-Input".to_owned(), "spoofed".to_owned()),
+ ("Signature".to_owned(), "spoofed".to_owned()),
+ ];
+ let mut http_headers = spoofed.clone();
+ signer
+ .append_request_headers(
+ &mut http_headers,
+ "GET",
+ &Url::parse("http://example.com/").unwrap(),
+ )
+ .unwrap();
+ assert_eq!(http_headers, spoofed);
+
+ let mut https_headers = spoofed;
+ signer
+ .append_request_headers(
+ &mut https_headers,
+ "GET",
+ &Url::parse("https://example.com/").unwrap(),
+ )
+ .unwrap();
+ assert_eq!(https_headers.len(), 3);
+ assert!(https_headers.iter().all(|(_, value)| value != "spoofed"));
+ }
+
+ #[test]
+ fn validates_domain() {
+ for domain in [
+ "",
+ " example.com",
+ "https://example.com",
+ "example.com/path",
+ "example.com?query",
+ ] {
+ let error = WebBotAuthSigner::from_pem(
+ RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
+ domain,
+ None,
+ WebBotAuthProfile::Cloudflare,
+ )
+ .unwrap_err();
+ assert!(error.to_string().contains("--web-bot-auth-domain"));
+ }
+ }
+
+ #[test]
+ fn debug_output_never_contains_private_key_material() {
+ let signer = signer(WebBotAuthProfile::Cloudflare);
+ let rendered = format!("{signer:?}");
+
+ assert!(rendered.contains(EXPECTED_KEYID));
+ assert!(!rendered.contains("IJ+DYvh6"));
+ assert!(!rendered.contains("PRIVATE KEY"));
+ }
+
+ fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {
+ headers
+ .iter()
+ .find(|(candidate, _)| candidate.eq_ignore_ascii_case(name))
+ .map(|(_, value)| value.as_str())
+ .unwrap()
+ }
+
+ fn nonce_from_signature_input(input: &str) -> &str {
+ input
+ .split(";nonce=\"")
+ .nth(1)
+ .and_then(|value| value.split('"').next())
+ .unwrap()
+ }
+}
diff --git a/moli-web-bot-auth/src/test_support.rs b/moli-web-bot-auth/src/test_support.rs
new file mode 100644
index 000000000..d156f1081
--- /dev/null
+++ b/moli-web-bot-auth/src/test_support.rs
@@ -0,0 +1,24 @@
+use crate::{WebBotAuthProfile, WebBotAuthSigner, key::WebBotAuthKey};
+
+pub(crate) const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\
+MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\
+-----END PRIVATE KEY-----\n";
+pub(crate) const EXPECTED_KEYID: &str = "poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U";
+
+pub(crate) fn key() -> WebBotAuthKey {
+ WebBotAuthKey::from_pem(
+ RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
+ Some(EXPECTED_KEYID),
+ )
+ .unwrap()
+}
+
+pub(crate) fn signer(profile: WebBotAuthProfile) -> WebBotAuthSigner {
+ WebBotAuthSigner::from_pem(
+ RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
+ "signature-agent.test",
+ Some(EXPECTED_KEYID),
+ profile,
+ )
+ .unwrap()
+}
diff --git a/moli-web-bot-auth/src/wire.rs b/moli-web-bot-auth/src/wire.rs
new file mode 100644
index 000000000..7b8e0cc98
--- /dev/null
+++ b/moli-web-bot-auth/src/wire.rs
@@ -0,0 +1,279 @@
+use anyhow::{Context, Result};
+use base64::{Engine as _, engine::general_purpose};
+use url::{Host, Url};
+
+use crate::{key::WebBotAuthKey, profile::WebBotAuthProfile};
+
+pub(crate) struct SignatureOptions<'a> {
+ pub(crate) signature_label: &'a str,
+ pub(crate) signature_agent_label: &'a str,
+ pub(crate) created: u64,
+ pub(crate) expires: u64,
+ pub(crate) nonce: String,
+ pub(crate) cover_request_target: bool,
+}
+
+pub(crate) struct SignedRequest {
+ pub(crate) headers: WebBotAuthHeaders,
+ #[cfg(test)]
+ pub(crate) signature_base: String,
+}
+
+pub(crate) struct WebBotAuthHeaders {
+ pub(crate) signature_agent: String,
+ pub(crate) signature_input: String,
+ pub(crate) signature: String,
+}
+
+impl WebBotAuthHeaders {
+ pub(crate) fn into_pairs(self) -> [(String, String); 3] {
+ [
+ ("Signature-Agent".to_owned(), self.signature_agent),
+ ("Signature-Input".to_owned(), self.signature_input),
+ ("Signature".to_owned(), self.signature),
+ ]
+ }
+}
+
+pub(crate) fn sign_request(
+ key: &WebBotAuthKey,
+ signature_agent_origin: &str,
+ profile: WebBotAuthProfile,
+ method: &str,
+ request_url: &Url,
+ options: &SignatureOptions<'_>,
+) -> Result<SignedRequest> {
+ let authority = request_authority(request_url)?;
+ let method = method.to_ascii_uppercase();
+ let path = request_url.path();
+ let signature_agent = signature_agent_header(
+ profile,
+ signature_agent_origin,
+ options.signature_agent_label,
+ );
+ let signature_agent_component =
+ signature_agent_component_identifier(profile, options.signature_agent_label);
+
+ let mut covered_components = vec!["\"@authority\"".to_owned()];
+ if options.cover_request_target {
+ covered_components.push("\"@method\"".to_owned());
+ covered_components.push("\"@path\"".to_owned());
+ }
+ covered_components.push(signature_agent_component.clone());
+
+ let signature_params = format!(
+ "({});created={};keyid=\"{}\";alg=\"ed25519\";expires={};nonce=\"{}\";tag=\"web-bot-auth\"",
+ covered_components.join(" "),
+ options.created,
+ key.keyid(),
+ options.expires,
+ options.nonce,
+ );
+
+ let mut signature_base_lines = vec![format!("\"@authority\": {authority}")];
+ if options.cover_request_target {
+ signature_base_lines.push(format!("\"@method\": {method}"));
+ signature_base_lines.push(format!("\"@path\": {path}"));
+ }
+ signature_base_lines.push(format!(
+ "{signature_agent_component}: \"{signature_agent_origin}\""
+ ));
+ signature_base_lines.push(format!("\"@signature-params\": {signature_params}"));
+ let signature_base = signature_base_lines.join("\n");
+ let signature = key.sign(signature_base.as_bytes())?;
+
+ Ok(SignedRequest {
+ headers: WebBotAuthHeaders {
+ signature_agent,
+ signature_input: format!("{}={signature_params}", options.signature_label),
+ signature: format!(
+ "{}=:{}:",
+ options.signature_label,
+ general_purpose::STANDARD.encode(signature)
+ ),
+ },
+ #[cfg(test)]
+ signature_base,
+ })
+}
+
+fn signature_agent_header(
+ profile: WebBotAuthProfile,
+ signature_agent_origin: &str,
+ label: &str,
+) -> String {
+ match profile {
+ WebBotAuthProfile::Cloudflare => format!("\"{signature_agent_origin}\""),
+ WebBotAuthProfile::IetfDraft01 => {
+ format!("{label}=\"{signature_agent_origin}\"")
+ }
+ }
+}
+
+fn signature_agent_component_identifier(profile: WebBotAuthProfile, label: &str) -> String {
+ match profile {
+ WebBotAuthProfile::Cloudflare => "\"signature-agent\"".to_owned(),
+ WebBotAuthProfile::IetfDraft01 => {
+ format!("\"signature-agent\";key=\"{label}\"")
+ }
+ }
+}
+
+fn request_authority(request_url: &Url) -> Result<String> {
+ let host = match request_url
+ .host()
+ .context("Web Bot Auth request URL must have an authority")?
+ {
+ Host::Domain(domain) => domain.to_owned(),
+ Host::Ipv4(address) => address.to_string(),
+ Host::Ipv6(address) => format!("[{address}]"),
+ };
+ Ok(match request_url.port() {
+ Some(port) => format!("{host}:{port}"),
+ None => host,
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::test_support::{EXPECTED_KEYID, key};
+ use aws_lc_rs::signature::{ED25519, UnparsedPublicKey};
+
+ #[test]
+ fn matches_ietf_draft_01_ed25519_test_vector() {
+ let key = key();
+ let request_url = Url::parse("https://example.com/foo?param=Value&Pet=dog").unwrap();
+ let signed = sign_request(
+ &key,
+ "https://signature-agent.test",
+ WebBotAuthProfile::IetfDraft01,
+ "POST",
+ &request_url,
+ &SignatureOptions {
+ signature_label: "sig2",
+ signature_agent_label: "agent2",
+ created: 1_735_689_600,
+ expires: 4_889_289_600,
+ nonce: "n9p433xm+NJ3ph3upfBIGmsuwHw387YV7Q/F+6BSpGCVjYCqQw6rznNA8PVVLySrAWsv0hQtFioQb6E1YsauiA==".to_owned(),
+ cover_request_target: false,
+ },
+ )
+ .unwrap();
+
+ assert_eq!(key.keyid(), EXPECTED_KEYID);
+ assert_eq!(
+ signed.headers.signature_agent,
+ "agent2=\"https://signature-agent.test\""
+ );
+ assert_eq!(
+ signed.headers.signature_input,
+ "sig2=(\"@authority\" \"signature-agent\";key=\"agent2\");created=1735689600;keyid=\"poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U\";alg=\"ed25519\";expires=4889289600;nonce=\"n9p433xm+NJ3ph3upfBIGmsuwHw387YV7Q/F+6BSpGCVjYCqQw6rznNA8PVVLySrAWsv0hQtFioQb6E1YsauiA==\";tag=\"web-bot-auth\""
+ );
+ assert_eq!(
+ signed.headers.signature,
+ "sig2=:RdNFx5Bj6au3YgAMQL/RzmUlZE8QZLIaXGRpw985hWnwPfMxT228NMk6ehRS1PSl4e8PhbNZACSanGdhEwYCCg==:"
+ );
+ }
+
+ #[test]
+ fn matches_cloudflare_legacy_ed25519_test_vector() {
+ let key = key();
+ let request_url = Url::parse("https://example.com/foo?param=Value&Pet=dog").unwrap();
+ let signed = sign_request(
+ &key,
+ "https://signature-agent.test",
+ WebBotAuthProfile::Cloudflare,
+ "POST",
+ &request_url,
+ &SignatureOptions {
+ signature_label: "sig2",
+ signature_agent_label: "agent2",
+ created: 1_735_689_600,
+ expires: 1_735_693_200,
+ nonce: "e8N7S2MFd/qrd6T2R3tdfAuuANngKI7LFtKYI/vowzk4lAZYadIX6wW25MwG7DCT9RUKAJ0qVkU0mEeLElW1qg==".to_owned(),
+ cover_request_target: false,
+ },
+ )
+ .unwrap();
+
+ assert_eq!(
+ signed.headers.signature_agent,
+ "\"https://signature-agent.test\""
+ );
+ assert_eq!(
+ signed.headers.signature,
+ "sig2=:jdq0SqOwHdyHr9+r5jw3iYZH6aNGKijYp/EstF4RQTQdi5N5YYKrD+mCT1HA1nZDsi6nJKuHxUi/5Syp3rLWBA==:"
+ );
+ }
+
+ #[test]
+ fn production_components_cover_authority_method_and_path() {
+ let key = key();
+ let request_url = Url::parse("https://example.com:8443/a%20path?query=yes").unwrap();
+ let signed = sign_request(
+ &key,
+ "https://signature-agent.test",
+ WebBotAuthProfile::Cloudflare,
+ "post",
+ &request_url,
+ &SignatureOptions {
+ signature_label: "sig1",
+ signature_agent_label: "sig1",
+ created: 1_735_689_600,
+ expires: 1_735_689_660,
+ nonce: general_purpose::STANDARD.encode([7_u8; 64]),
+ cover_request_target: true,
+ },
+ )
+ .unwrap();
+
+ assert!(
+ signed
+ .signature_base
+ .contains("\"@authority\": example.com:8443")
+ );
+ assert!(signed.signature_base.contains("\"@method\": POST"));
+ assert!(signed.signature_base.contains("\"@path\": /a%20path"));
+ let signature = signed
+ .headers
+ .signature
+ .strip_prefix("sig1=:")
+ .and_then(|value| value.strip_suffix(':'))
+ .and_then(|value| general_purpose::STANDARD.decode(value).ok())
+ .unwrap();
+ UnparsedPublicKey::new(&ED25519, key.public_key())
+ .verify(signed.signature_base.as_bytes(), &signature)
+ .unwrap();
+ }
+
+ #[test]
+ fn signing_rejects_a_request_url_without_an_authority() {
+ let key = key();
+ let request_url = Url::parse("data:text/plain,no-authority").unwrap();
+ let result = sign_request(
+ &key,
+ "https://signature-agent.test",
+ WebBotAuthProfile::Cloudflare,
+ "GET",
+ &request_url,
+ &SignatureOptions {
+ signature_label: "sig1",
+ signature_agent_label: "sig1",
+ created: 1_735_689_600,
+ expires: 1_735_689_660,
+ nonce: general_purpose::STANDARD.encode([7_u8; 64]),
+ cover_request_target: true,
+ },
+ );
+
+ let error = match result {
+ Ok(_) => panic!("signing a URL without an authority must fail"),
+ Err(error) => error,
+ };
+ assert_eq!(
+ error.to_string(),
+ "Web Bot Auth request URL must have an authority"
+ );
+ }
+}
diff --git a/moli/src/cli.rs b/moli/src/cli.rs
index ede40b461..e35e9eaa4 100644
--- a/moli/src/cli.rs
+++ b/moli/src/cli.rs
@@ -32,6 +32,10 @@ const FETCH_INFER_FLAGS: &[&str] = &[
"--disable-subframes",
"--wait-ms",
"--profile-dir",
+ "--web-bot-auth-key-file",
+ "--web-bot-auth-keyid",
+ "--web-bot-auth-domain",
+ "--web-bot-auth-profile",
];
const SERVE_INFER_FLAGS: &[&str] = &["--host", "--port", "--timeout", "--layout"];
const EXPLICIT_COMMANDS: &[&str] = &["fetch", "serve", "help", "version"];
@@ -367,6 +371,35 @@ pub struct CommonArgs {
#[arg(long)]
pub user_agent_suffix: Option<String>,
+
+ /// Unencrypted PKCS#8 Ed25519 private key used for Web Bot Auth signatures.
+ #[arg(long, value_name = "PATH", requires = "web_bot_auth_domain")]
+ pub web_bot_auth_key_file: Option<String>,
+
+ /// Assert the RFC 7638 JWK thumbprint derived from the Web Bot Auth key.
+ #[arg(long, value_name = "THUMBPRINT", requires = "web_bot_auth_key_file")]
+ pub web_bot_auth_keyid: Option<String>,
+
+ /// Operator domain publishing /.well-known/http-message-signatures-directory.
+ #[arg(long, value_name = "DOMAIN", requires = "web_bot_auth_key_file")]
+ pub web_bot_auth_domain: Option<String>,
+
+ /// Signature-Agent wire format. Cloudflare compatibility is the default.
+ #[arg(
+ long,
+ value_enum,
+ default_value = "cloudflare",
+ requires = "web_bot_auth_key_file"
+ )]
+ pub web_bot_auth_profile: WebBotAuthProfileChoice,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, ValueEnum)]
+pub enum WebBotAuthProfileChoice {
+ #[default]
+ Cloudflare,
+ #[value(name = "ietf-01")]
+ IetfDraft01,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
diff --git a/moli/src/config.rs b/moli/src/config.rs
index 4cd98d9f9..28e979438 100644
--- a/moli/src/config.rs
+++ b/moli/src/config.rs
@@ -6,10 +6,13 @@ use moli_core::{
page::{SubresourceJsonPathEquals, SubresourceResponseWaitCriteria},
runtime::BrowserConfig,
};
+use moli_fetch::{FetchConfig, WebBotAuthProfile, WebBotAuthSigner};
use std::path::PathBuf;
use std::str::FromStr;
-use crate::cli::{Cli, Commands, CommonArgs, DumpFormat, LogFormat, StripOptions};
+use crate::cli::{
+ Cli, Commands, CommonArgs, DumpFormat, LogFormat, StripOptions, WebBotAuthProfileChoice,
+};
use crate::network_trace::NetworkTraceConfigSummary;
pub use moli_protocol_server::ServerConfig;
@@ -183,6 +186,7 @@ fn apply_common_args(config: &mut AppConfig, common: &CommonArgs) -> Result<()>
.browser
.fetch_mut()
.set_tls_verify_host(!common.insecure_disable_tls_host_verification);
+ configure_web_bot_auth(config.browser.fetch_mut(), common)?;
for source in &common.document_start_script {
config.add_document_start_script(source.clone());
@@ -199,6 +203,37 @@ fn apply_common_args(config: &mut AppConfig, common: &CommonArgs) -> Result<()>
Ok(())
}
+fn configure_web_bot_auth(fetch: &mut FetchConfig, common: &CommonArgs) -> Result<()> {
+ let (key_file, domain) = match (
+ common.web_bot_auth_key_file.as_deref(),
+ common.web_bot_auth_domain.as_deref(),
+ ) {
+ (None, None) => {
+ if common.web_bot_auth_keyid.is_some() {
+ bail!("--web-bot-auth-keyid requires --web-bot-auth-key-file");
+ }
+ return Ok(());
+ }
+ (Some(_), None) => bail!("--web-bot-auth-key-file requires --web-bot-auth-domain"),
+ (None, Some(_)) => bail!("--web-bot-auth-domain requires --web-bot-auth-key-file"),
+ (Some(key_file), Some(domain)) => (key_file, domain),
+ };
+ let private_key_pem = std::fs::read(key_file)
+ .with_context(|| format!("failed to read Web Bot Auth private key `{key_file}`"))?;
+ let profile = match common.web_bot_auth_profile {
+ WebBotAuthProfileChoice::Cloudflare => WebBotAuthProfile::Cloudflare,
+ WebBotAuthProfileChoice::IetfDraft01 => WebBotAuthProfile::IetfDraft01,
+ };
+ let signer = WebBotAuthSigner::from_pem(
+ &private_key_pem,
+ domain,
+ common.web_bot_auth_keyid.as_deref(),
+ profile,
+ )?;
+ fetch.set_web_bot_auth(Some(signer));
+ Ok(())
+}
+
fn validate_http_host_resolve_entries(entries: &[String]) -> Result<()> {
for entry in entries {
validate_http_host_resolve_entry(entry)?;
diff --git a/moli/tests/cli.rs b/moli/tests/cli.rs
index fb46bdd75..bde0fa79f 100644
--- a/moli/tests/cli.rs
+++ b/moli/tests/cli.rs
@@ -1,15 +1,27 @@
use clap::Parser;
-use std::{num::NonZeroU32, process::Command};
+use std::{
+ fs,
+ num::NonZeroU32,
+ path::PathBuf,
+ process::Command,
+ sync::atomic::{AtomicU64, Ordering},
+};
use moli::cli::{
Cli, Commands, CommonArgs, DumpFormat, FetchArgs, FetchWaitUntil, LogFormat, LogLevel,
RequestHeaderArg, ResponseJsonPathArg, ServeArgs, StripModeChoice, StripOptions,
- normalize_args_for_compat,
+ WebBotAuthProfileChoice, normalize_args_for_compat,
};
use moli::config::AppConfig;
use moli_browser_profile::BrowserProfilePaths;
use moli_core::OptionalResourceFetchMask;
-use moli_fetch::FetchConfig;
+use moli_fetch::{FetchConfig, WebBotAuthProfile};
+
+const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\
+MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\
+-----END PRIVATE KEY-----\n";
+const RFC_9421_ED25519_KEYID: &str = "poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U";
+static NEXT_TEMP_KEY_FILE: AtomicU64 = AtomicU64::new(0);
#[test]
fn parses_explicit_fetch_command_with_compatibility_flags() {
@@ -125,6 +137,10 @@ fn parses_explicit_fetch_command_with_compatibility_flags() {
log_filter_scopes: Some("http,event".to_owned()),
user_agent: None,
user_agent_suffix: Some("internal-tester".to_owned()),
+ web_bot_auth_key_file: None,
+ web_bot_auth_keyid: None,
+ web_bot_auth_domain: None,
+ web_bot_auth_profile: WebBotAuthProfileChoice::Cloudflare,
},
url: "https://example.com".to_owned(),
}))
@@ -200,22 +216,79 @@ fn parses_binary_dump_modes_with_inferred_fetch_command() {
}
#[test]
-fn rejects_removed_web_bot_auth_flags() {
- for flag in [
+fn parses_web_bot_auth_flags() {
+ let cli = Cli::try_parse_from(normalize_args_for_compat([
+ "moli",
+ "fetch",
"--web-bot-auth-key-file",
+ "bot-key.pem",
"--web-bot-auth-keyid",
+ RFC_9421_ED25519_KEYID,
"--web-bot-auth-domain",
+ "bot.example",
+ "--web-bot-auth-profile",
+ "ietf-01",
+ "https://example.com",
+ ]))
+ .unwrap();
+
+ let Commands::Fetch(args) = cli.command else {
+ panic!("expected fetch command");
+ };
+ assert_eq!(
+ args.common.web_bot_auth_key_file.as_deref(),
+ Some("bot-key.pem")
+ );
+ assert_eq!(
+ args.common.web_bot_auth_keyid.as_deref(),
+ Some(RFC_9421_ED25519_KEYID)
+ );
+ assert_eq!(
+ args.common.web_bot_auth_domain.as_deref(),
+ Some("bot.example")
+ );
+ assert_eq!(
+ args.common.web_bot_auth_profile,
+ WebBotAuthProfileChoice::IetfDraft01
+ );
+}
+
+#[test]
+fn web_bot_auth_profile_default_does_not_enable_signing() {
+ let cli = Cli::try_parse_from(normalize_args_for_compat([
+ "moli",
+ "fetch",
+ "https://example.com",
+ ]))
+ .unwrap();
+
+ let Commands::Fetch(args) = cli.command else {
+ panic!("expected fetch command");
+ };
+ assert_eq!(
+ args.common.web_bot_auth_profile,
+ WebBotAuthProfileChoice::Cloudflare
+ );
+ assert!(args.common.web_bot_auth_key_file.is_none());
+}
+
+#[test]
+fn web_bot_auth_flags_require_key_and_domain_together() {
+ for args in [
+ vec!["--web-bot-auth-key-file", "bot-key.pem"],
+ vec!["--web-bot-auth-domain", "bot.example"],
+ vec!["--web-bot-auth-keyid", RFC_9421_ED25519_KEYID],
+ vec!["--web-bot-auth-profile", "ietf-01"],
] {
- let error = Cli::try_parse_from(normalize_args_for_compat([
- "moli",
- "fetch",
- flag,
- "unused",
- "https://example.com",
- ]))
- .unwrap_err();
+ let mut command = vec!["moli", "fetch"];
+ command.extend(args);
+ command.push("https://example.com");
+ let error = Cli::try_parse_from(normalize_args_for_compat(command)).unwrap_err();
- assert_eq!(error.kind(), clap::error::ErrorKind::UnknownArgument);
+ assert_eq!(
+ error.kind(),
+ clap::error::ErrorKind::MissingRequiredArgument
+ );
}
}
@@ -550,6 +623,54 @@ fn app_config_uses_moli_user_agent_defaults() {
assert_eq!(config.browser.fetch().user_agent(), "ExampleBrowser/1.0");
}
+#[test]
+fn app_config_loads_and_validates_web_bot_auth_key() {
+ let key_file = write_temp_web_bot_auth_key();
+ let key_path_arg = key_file.path.to_string_lossy().into_owned();
+ let cli = Cli::try_parse_from(normalize_args_for_compat([
+ "moli",
+ "fetch",
+ "--web-bot-auth-key-file",
+ key_path_arg.as_str(),
+ "--web-bot-auth-keyid",
+ RFC_9421_ED25519_KEYID,
+ "--web-bot-auth-domain",
+ "bot.example:8443",
+ "--web-bot-auth-profile",
+ "ietf-01",
+ "https://example.com",
+ ]))
+ .unwrap();
+
+ let config = AppConfig::from_cli(&cli).unwrap();
+ let signer = config.browser.fetch().web_bot_auth().unwrap();
+ assert_eq!(signer.keyid(), RFC_9421_ED25519_KEYID);
+ assert_eq!(signer.signature_agent_origin(), "https://bot.example:8443");
+ assert_eq!(signer.profile(), WebBotAuthProfile::IetfDraft01);
+}
+
+#[test]
+fn app_config_rejects_mismatched_web_bot_auth_keyid() {
+ let key_file = write_temp_web_bot_auth_key();
+ let key_path_arg = key_file.path.to_string_lossy().into_owned();
+ let cli = Cli::try_parse_from(normalize_args_for_compat([
+ "moli",
+ "fetch",
+ "--web-bot-auth-key-file",
+ key_path_arg.as_str(),
+ "--web-bot-auth-keyid",
+ "wrong-thumbprint",
+ "--web-bot-auth-domain",
+ "bot.example",
+ "https://example.com",
+ ]))
+ .unwrap();
+
+ let error = AppConfig::from_cli(&cli).unwrap_err().to_string();
+ assert!(error.contains("does not match the private key"));
+ assert!(error.contains(RFC_9421_ED25519_KEYID));
+}
+
#[test]
fn app_config_preserves_repeatable_request_headers() {
let cli = Cli::try_parse_from(normalize_args_for_compat([
@@ -574,6 +695,26 @@ fn app_config_preserves_repeatable_request_headers() {
assert!(config.browser.fetch().default_request_headers().is_empty());
}
+struct TempWebBotAuthKeyFile {
+ path: PathBuf,
+}
+
+impl Drop for TempWebBotAuthKeyFile {
+ fn drop(&mut self) {
+ let _ = fs::remove_file(&self.path);
+ }
+}
+
+fn write_temp_web_bot_auth_key() -> TempWebBotAuthKeyFile {
+ let sequence = NEXT_TEMP_KEY_FILE.fetch_add(1, Ordering::Relaxed);
+ let path = std::env::temp_dir().join(format!(
+ "moli-web-bot-auth-{}-{sequence}.pem",
+ std::process::id()
+ ));
+ fs::write(&path, RFC_9421_ED25519_PRIVATE_KEY).unwrap();
+ TempWebBotAuthKeyFile { path }
+}
+
#[test]
fn parses_request_headers_with_embedded_colons_and_empty_values() {
let cli = Cli::try_parse_from(normalize_args_for_compat([
diff --git a/skills/moli-webfetch/references/fetch-recipes.md b/skills/moli-webfetch/references/fetch-recipes.md
index e122efa68..e2deba9a1 100644
--- a/skills/moli-webfetch/references/fetch-recipes.md
+++ b/skills/moli-webfetch/references/fetch-recipes.md
@@ -135,6 +135,12 @@ For a crawl rather than a single lookup:
- Use `--http-proxy`, `--http-no-proxy`, or
`--http-host-resolve HOST:PORT:ADDR` when required by the environment.
- Use either `--user-agent` or `--user-agent-suffix`, not both.
+- Use `--web-bot-auth-key-file <PKCS8-PEM>` together with
+ `--web-bot-auth-domain <DOMAIN>` only when the user supplied an authorized
+ bot identity. The key remains local, and Moli signs HTTPS requests only.
+- Add `--web-bot-auth-keyid <THUMBPRINT>` to assert the derived JWK thumbprint.
+ Keep the default Cloudflare profile unless the receiver explicitly supports
+ `--web-bot-auth-profile ietf-01`.
- Use `--document-start-script` or `--document-start-script-file` only when the
task explicitly requires pre-navigation instrumentation.
- Combine `--block-private-networks` with `--block-cidrs` for untrusted URL