Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): honor credentials mode for TLS client identities
ldm0114508118aa6
diff --git a/Cargo.lock b/Cargo.lockindex c72c23141..3aa78f743 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3780,6 +3780,16 @@ version = "1.0.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"+[[package]]+name = "pem"+version = "3.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"+dependencies = [+ "base64 0.22.1",+ "serde_core",+]+[[package]]name = "peniko"version = "0.6.1"@@ -4060,6 +4070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2"dependencies = ["aws-lc-rs",+ "pem","rustls-pki-types","time","yasna",diff --git a/moli-fetch/Cargo.toml b/moli-fetch/Cargo.tomlindex 7e4cb83c2..bca36392e 100644--- a/moli-fetch/Cargo.toml+++ b/moli-fetch/Cargo.toml@@ -35,7 +35,7 @@ wildcard = "0.3.0"[dev-dependencies]h2 = "0.4"moli-cookie-jar = { path = "../moli-cookie-jar", features = ["test-support"] }-rcgen = { version = "0.13", default-features = false, features = ["aws_lc_rs"] }+rcgen = { version = "0.13", default-features = false, features = ["aws_lc_rs", "pem"] }tokio = { version = "1.51.0", features = ["io-util", "macros", "net", "rt", "sync", "time"] }tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "logging", "tls12"] }diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rsindex a8ff8c8a5..d60bbd931 100644--- a/moli-fetch/src/blocking/mod.rs+++ b/moli-fetch/src/blocking/mod.rs@@ -664,39 +664,44 @@ pub(crate) fn configure_easy<H: Handler>()})?;}- if let Some(client_cert) = config.client_cert() {- easy.ssl_cert(client_cert).with_context(|| {- format!(- "failed to configure curl client certificate `{}`",- client_cert.display()- )- })?;- let cert_type = match client_cert- .extension()- .and_then(|extension| extension.to_str())- .map(str::to_ascii_lowercase)- .as_deref()- {- Some("p12" | "pfx") => "P12",- Some("cer" | "der") => "DER",- _ => "PEM",- };- easy.ssl_cert_type(cert_type)- .context("failed to configure curl client certificate type")?;- }- if let Some(client_key) = config.client_key() {- easy.ssl_key(client_key).with_context(|| {- format!(- "failed to configure curl client private key `{}`",- client_key.display()- )- })?;- easy.ssl_key_type("PEM")- .context("failed to configure curl client private key type")?;- }- if let Some(password) = config.client_cert_password() {- easy.key_password(password)- .context("failed to configure curl client certificate password")?;+ // TLS client identities are Fetch credentials. Check each redirect target;+ // curl also uses the client certificate configuration to match reusable+ // connections, keeping authenticated and anonymous requests separate.+ if request.allows_credentials_for_url(request_url) {+ if let Some(client_cert) = config.client_cert() {+ easy.ssl_cert(client_cert).with_context(|| {+ format!(+ "failed to configure curl client certificate `{}`",+ client_cert.display()+ )+ })?;+ let cert_type = match client_cert+ .extension()+ .and_then(|extension| extension.to_str())+ .map(str::to_ascii_lowercase)+ .as_deref()+ {+ Some("p12" | "pfx") => "P12",+ Some("cer" | "der") => "DER",+ _ => "PEM",+ };+ easy.ssl_cert_type(cert_type)+ .context("failed to configure curl client certificate type")?;+ }+ if let Some(client_key) = config.client_key() {+ easy.ssl_key(client_key).with_context(|| {+ format!(+ "failed to configure curl client private key `{}`",+ client_key.display()+ )+ })?;+ easy.ssl_key_type("PEM")+ .context("failed to configure curl client private key type")?;+ }+ if let Some(password) = config.client_cert_password() {+ easy.key_password(password)+ .context("failed to configure curl client certificate password")?;+ }}easy.useragent(config.user_agent()).context("failed to set curl user-agent")?;diff --git a/moli-fetch/src/config.rs b/moli-fetch/src/config.rsindex b34e5db72..83d7f396d 100644--- a/moli-fetch/src/config.rs+++ b/moli-fetch/src/config.rs@@ -151,8 +151,10 @@ impl FetchConfig {self.tls_verify_host = tls_verify_host;}- /// Configure TLS trust and mutual-TLS credentials for every request made- /// by this fetch runtime, including navigations and subresources.+ /// Configure TLS trust and mutual-TLS credentials for this fetch runtime,+ /// including navigations and subresources. Client identities are used only+ /// when the request's credentials mode allows them for the current URL;+ /// CA trust applies regardless of credentials mode.pub fn set_tls_credentials(&mut self,ca_cert: Option<PathBuf>,diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rsindex 9724c1bcb..7be8072e6 100644--- a/moli-fetch/src/tests/mod.rs+++ b/moli-fetch/src/tests/mod.rs@@ -1,6 +1,7 @@mod cookie_context;mod cookie_store;mod support;+mod tls_credentials;use anyhow::{Context, Result};use curl::easy::Handler;diff --git a/moli-fetch/src/tests/tls_credentials.rs b/moli-fetch/src/tests/tls_credentials.rsnew file mode 100644index 000000000..19a6ca7a5--- /dev/null+++ b/moli-fetch/src/tests/tls_credentials.rs@@ -0,0 +1,353 @@+use std::{fs, path::PathBuf, sync::Arc};++use anyhow::Result;+use moli_cookie_jar::new_shared_browser_cookie_store;+use parking_lot::Mutex;+use rcgen::{+ BasicConstraints, CertificateParams, DnType, ExtendedKeyUsagePurpose, IsCa, KeyPair,+ KeyUsagePurpose,+};+use tokio::{+ io::{AsyncReadExt, AsyncWriteExt},+ net::TcpListener,+ sync::oneshot,+ task::{JoinHandle, JoinSet},+};+use tokio_rustls::{+ TlsAcceptor,+ rustls::{+ RootCertStore, ServerConfig,+ pki_types::{CertificateDer, PrivatePkcs8KeyDer},+ server::WebPkiClientVerifier,+ },+};+use url::Url;++use crate::{+ FetchCancelHandle, FetchClient, FetchClientHandle, FetchConfig, Request, RequestCredentialsMode,+};++use super::support::unique_test_cache_dir;++struct TlsCredentials {+ dir: PathBuf,+ acceptor: TlsAcceptor,+ client_certificate: CertificateDer<'static>,+}++impl TlsCredentials {+ fn new() -> Result<Self> {+ let ca_key = KeyPair::generate()?;+ let mut ca_params = CertificateParams::new(Vec::new())?;+ ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);+ ca_params.key_usages = vec![KeyUsagePurpose::KeyCertSign];+ ca_params+ .distinguished_name+ .push(DnType::CommonName, "Moli fetch test CA");+ let ca = ca_params.self_signed(&ca_key)?;++ let server_key = KeyPair::generate()?;+ let mut server_params =+ CertificateParams::new(vec!["localhost".to_owned(), "127.0.0.1".to_owned()])?;+ server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];+ let server_cert = server_params.signed_by(&server_key, &ca, &ca_key)?;++ let client_key = KeyPair::generate()?;+ let mut client_params = CertificateParams::new(Vec::new())?;+ client_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];+ let client_cert = client_params.signed_by(&client_key, &ca, &ca_key)?;++ let mut roots = RootCertStore::empty();+ roots.add(ca.der().clone())?;+ // Request a client certificate but allow anonymous handshakes so the+ // tests can inspect what the server actually received in either mode.+ let verifier = WebPkiClientVerifier::builder(Arc::new(roots))+ .allow_unauthenticated()+ .build()?;+ let mut config = ServerConfig::builder()+ .with_client_cert_verifier(verifier)+ .with_single_cert(+ vec![server_cert.der().clone()],+ PrivatePkcs8KeyDer::from(server_key.serialize_der()).into(),+ )?;+ config.alpn_protocols = vec![b"http/1.1".to_vec()];++ let credentials = Self {+ dir: unique_test_cache_dir(),+ acceptor: TlsAcceptor::from(Arc::new(config)),+ client_certificate: client_cert.der().clone(),+ };+ fs::create_dir(&credentials.dir)?;+ fs::write(credentials.dir.join("ca.pem"), ca.pem())?;+ fs::write(credentials.dir.join("client.pem"), client_cert.pem())?;+ fs::write(+ credentials.dir.join("client-key.pem"),+ client_key.serialize_pem(),+ )?;+ Ok(credentials)+ }++ fn fetch_config(&self) -> FetchConfig {+ let mut config = FetchConfig::default();+ config.set_http_proxy(Some(String::new()));+ config.set_request_timeout_ms(5_000);+ // Keep verification enabled: even anonymous requests need this CA.+ config.set_tls_credentials(+ Some(self.dir.join("ca.pem")),+ Some(self.dir.join("client.pem")),+ Some(self.dir.join("client-key.pem")),+ Some("test-password".to_owned()),+ );+ config+ }+}++impl Drop for TlsCredentials {+ fn drop(&mut self) {+ let _ = fs::remove_dir_all(&self.dir);+ }+}++#[derive(Clone, Debug)]+struct ObservedRequest {+ connection_id: usize,+ path: String,+ client_certificates: Vec<CertificateDer<'static>>,+}++struct TlsServer {+ url: Url,+ requests: Arc<Mutex<Vec<ObservedRequest>>>,+ task: JoinHandle<()>,+}++impl TlsServer {+ async fn spawn(credentials: &TlsCredentials) -> Result<Self> {+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let port = listener.local_addr()?.port();+ let url = Url::parse(&format!("https://127.0.0.1:{port}/whoami"))?;+ let requests = Arc::new(Mutex::new(Vec::new()));+ let observed = Arc::clone(&requests);+ let acceptor = credentials.acceptor.clone();+ let task = tokio::spawn(async move {+ let mut connections = JoinSet::new();+ let mut connection_id = 0;+ while let Ok((stream, _)) = listener.accept().await {+ let acceptor = acceptor.clone();+ let observed = Arc::clone(&observed);+ connections.spawn(async move {+ let mut stream = acceptor.accept(stream).await?;+ let client_certificates = stream+ .get_ref()+ .1+ .peer_certificates()+ .unwrap_or_default()+ .to_vec();+ loop {+ let mut head = Vec::new();+ while !head.ends_with(b"\r\n\r\n") {+ let mut byte = [0];+ if stream.read(&mut byte).await? == 0 {+ return Ok::<(), anyhow::Error>(());+ }+ head.push(byte[0]);+ }+ let head = String::from_utf8(head)?;+ let path = head.split_whitespace().nth(1).unwrap().to_owned();+ observed.lock().push(ObservedRequest {+ connection_id,+ path: path.clone(),+ client_certificates: client_certificates.clone(),+ });+ let response = match path.as_str() {+ "/redirect-cross" => format!(+ "HTTP/1.1 302 Found\r\nLocation: https://localhost:{port}/redirect-back\r\nContent-Length: 0\r\n\r\n"+ ),+ "/redirect-back" => format!(+ "HTTP/1.1 302 Found\r\nLocation: https://127.0.0.1:{port}/whoami\r\nContent-Length: 0\r\n\r\n"+ ),+ _ => "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\r\nok".to_owned(),+ };+ // HTTP/1.1 stays open for further requests, including+ // any incorrect reuse of an authenticated connection.+ stream.write_all(response.as_bytes()).await?;+ stream.flush().await?;+ }+ });+ connection_id += 1;+ }+ });+ Ok(Self {+ url,+ requests,+ task,+ })+ }+}++impl Drop for TlsServer {+ fn drop(&mut self) {+ self.task.abort();+ }+}++#[derive(Clone, Copy, Debug)]+enum Transport {+ Buffered,+ Html,+ Raw,+}++impl Transport {+ const ALL: [Self; 3] = [Self::Buffered, Self::Html, Self::Raw];++ async fn fetch(self, client: &FetchClientHandle, request: Request) -> Result<()> {+ let body = match self {+ Self::Buffered => {+ let (tx, rx) = oneshot::channel();+ client.fetch_with_cancel_callback(+ request,+ FetchCancelHandle::new(),+ move |response| {+ let _ = tx.send(response);+ },+ )?;+ rx.await??.body_text().to_owned()+ }+ Self::Html => {+ let mut response = client.fetch_html_stream(request).await?;+ let mut body = String::new();+ while let Some(chunk) = response.next_chunk().await {+ body.push_str(&chunk);+ }+ response.finish().await?;+ body+ }+ Self::Raw => client.fetch(request).await?.body_text().to_owned(),+ };+ assert_eq!(body, "ok");+ Ok(())+ }+}++#[tokio::test]+async fn tls_client_certificate_respects_credentials_in_every_transport() -> Result<()> {+ let credentials = TlsCredentials::new()?;+ let server = TlsServer::spawn(&credentials).await?;+ let cross_origin = Url::parse("https://other.example.test/")?;+ for transport in Transport::ALL {+ for (mode, initiator, authenticated) in [+ (RequestCredentialsMode::Include, &cross_origin, true),+ (RequestCredentialsMode::Omit, &server.url, false),+ (RequestCredentialsMode::SameOrigin, &server.url, true),+ (RequestCredentialsMode::SameOrigin, &cross_origin, false),+ ] {+ // Each case uses a fresh client to exercise a new TLS handshake.+ let client = FetchClient::new(+ &credentials.fetch_config(),+ new_shared_browser_cookie_store(),+ );+ let request = Request::get(server.url.as_str())?+ .with_initiator_url(initiator)+ .with_credentials_mode(mode);+ transport.fetch(&client, request).await?;+ let observed = server.requests.lock().pop().unwrap();+ let expected: Vec<_> = authenticated+ .then(|| credentials.client_certificate.clone())+ .into_iter()+ .collect();+ assert_eq!(+ observed.client_certificates, expected,+ "{transport:?} {mode:?}"+ );+ assert!(client.shutdown().is_clean());+ }+ }+ Ok(())+}++#[tokio::test]+async fn tls_client_certificate_isolated_on_reused_connections() -> Result<()> {+ let credentials = TlsCredentials::new()?;+ let server = TlsServer::spawn(&credentials).await?;+ let cross_origin = Url::parse("https://other.example.test/")?;+ for transport in Transport::ALL {+ let client = FetchClient::new(+ &credentials.fetch_config(),+ new_shared_browser_cookie_store(),+ );+ for (mode, initiator) in [+ (RequestCredentialsMode::Include, &server.url),+ (RequestCredentialsMode::Include, &server.url),+ (RequestCredentialsMode::Omit, &server.url),+ (RequestCredentialsMode::Omit, &server.url),+ (RequestCredentialsMode::SameOrigin, &cross_origin),+ (RequestCredentialsMode::SameOrigin, &server.url),+ ] {+ let request = Request::get(server.url.as_str())?+ .with_initiator_url(initiator)+ .with_credentials_mode(mode);+ transport.fetch(&client, request).await?;+ }+ let observed = std::mem::take(&mut *server.requests.lock());+ assert_eq!(observed.len(), 6);+ for (request, authenticated) in observed.iter().zip([true, true, false, false, false, true])+ {+ let expected: Vec<_> = authenticated+ .then(|| credentials.client_certificate.clone())+ .into_iter()+ .collect();+ assert_eq!(request.client_certificates, expected, "{transport:?}");+ }+ assert_eq!(observed[0].connection_id, observed[1].connection_id);+ assert_ne!(observed[1].connection_id, observed[2].connection_id);+ assert_eq!(observed[2].connection_id, observed[3].connection_id);+ assert_eq!(observed[3].connection_id, observed[4].connection_id);+ assert_eq!(observed[0].connection_id, observed[5].connection_id);+ assert!(client.shutdown().is_clean());+ }+ Ok(())+}++#[tokio::test]+async fn tls_client_certificate_uses_current_redirect_origin() -> Result<()> {+ let credentials = TlsCredentials::new()?;+ let server = TlsServer::spawn(&credentials).await?;+ for transport in Transport::ALL {+ for (mode, expected_authentication) in [+ (RequestCredentialsMode::Include, [true, true, true]),+ (RequestCredentialsMode::Omit, [false, false, false]),+ (RequestCredentialsMode::SameOrigin, [true, false, true]),+ ] {+ let client = FetchClient::new(+ &credentials.fetch_config(),+ new_shared_browser_cookie_store(),+ );+ let request = Request::get(server.url.join("/redirect-cross")?.as_str())?+ .with_initiator_url(&server.url)+ .with_credentials_mode(mode);+ transport.fetch(&client, request).await?;+ let observed = std::mem::take(&mut *server.requests.lock());+ assert_eq!(+ observed+ .iter()+ .map(|request| request.path.as_str())+ .collect::<Vec<_>>(),+ ["/redirect-cross", "/redirect-back", "/whoami"]+ );+ for (request, authenticated) in observed.iter().zip(expected_authentication) {+ let expected: Vec<_> = authenticated+ .then(|| credentials.client_certificate.clone())+ .into_iter()+ .collect();+ assert_eq!(+ request.client_certificates, expected,+ "{transport:?} {mode:?} {}",+ request.path+ );+ }+ assert!(client.shutdown().is_clean());+ }+ }+ Ok(())+}