Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix: verify HTTPS proxy connections
ldm090c0ede82930
diff --git a/moli-curl/src/tls.rs b/moli-curl/src/tls.rsindex ca17241c1..92bf4e85f 100644--- a/moli-curl/src/tls.rs+++ b/moli-curl/src/tls.rs@@ -102,6 +102,33 @@ impl CurlTlsConfig {}Ok(())}++ /// Install certificate-chain and hostname verification for an HTTPS proxy.+ ///+ /// libcurl keeps proxy TLS settings separate from origin TLS settings. The+ /// configured CA is shared, but client identities remain origin-only unless+ /// a dedicated proxy identity is added in the future.+ pub fn configure_https_proxy<H: Handler>(&self, easy: &mut Easy2<H>) -> Result<()> {+ easy.proxy_ssl_verify_peer(self.verify)+ .context("failed to configure curl HTTPS proxy peer verification")?;+ easy.proxy_ssl_verify_host(self.verify)+ .context("failed to configure curl HTTPS proxy host verification")?;+ if let Some(ca_cert) = &self.ca_cert {+ let ca_cert_text = ca_cert.to_str().with_context(|| {+ format!(+ "curl HTTPS proxy CA certificate path is not valid UTF-8: `{}`",+ ca_cert.display()+ )+ })?;+ easy.proxy_cainfo(ca_cert_text).with_context(|| {+ format!(+ "failed to configure curl HTTPS proxy CA certificate `{}`",+ ca_cert.display()+ )+ })?;+ }+ Ok(())+ }}#[cfg(test)]diff --git a/moli-curl/src/websocket/request.rs b/moli-curl/src/websocket/request.rsindex 488974503..7c57c19f2 100644--- a/moli-curl/src/websocket/request.rs+++ b/moli-curl/src/websocket/request.rs@@ -82,6 +82,9 @@ pub(super) fn configure(request: &CurlWebSocketRequest) -> Result<Easy2<Handshak// The caller selects the client identity carried by this connection request.request.tls.configure(&mut easy, true)?;easy.proxy(request.proxy.as_deref().unwrap_or(""))?;+ if request.proxy.as_deref().is_some_and(proxy_uses_https_tls) {+ request.tls.configure_https_proxy(&mut easy)?;+ }// The browser has already applied no_proxy; do not evaluate environment policy twice.easy.noproxy("")?;easy.http_proxy_tunnel(request.proxy.as_deref().is_some_and(proxy_uses_http_tunnel))?;@@ -110,6 +113,12 @@ fn proxy_uses_http_tunnel(proxy: &str) -> bool {scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https")}+fn proxy_uses_https_tls(proxy: &str) -> bool {+ proxy+ .split_once("://")+ .is_some_and(|(scheme, _)| scheme.eq_ignore_ascii_case("https"))+}+fn headers(entries: &[(String, String)]) -> Result<List> {let mut list = List::new();let mut size = 0usize;diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rsindex 56801a3bd..eb3aef09e 100644--- a/moli-fetch/src/blocking/mod.rs+++ b/moli-fetch/src/blocking/mod.rs@@ -14,7 +14,7 @@ use moli_cookie_jar::{NetworkCookieRequestContext, SharedBrowserCookieStore, StoredCookieQueryReport,StoredCookieSetReport, same_site_urls,};-use moli_curl::HostResolveOverrides;+use moli_curl::{HostResolveOverrides, ProxyScheme};use moli_url::is_potentially_trustworthy_url;use moli_url_policy::ensure_http_network_transport_url;use tracing::debug;@@ -627,6 +627,9 @@ pub(crate) fn configure_easy<H: Handler>(HttpProxyRoute::Proxy(proxy) => {easy.proxy(proxy.curl_url()).with_context(|| anyhow!("failed to configure HTTP proxy `{}`", proxy.url()))?;+ if proxy.scheme() == ProxyScheme::Https {+ config.tls_config().configure_https_proxy(easy)?;+ }configure_proxy_headers(easy, config, request, proxy.scheme().uses_http_headers())?;}}diff --git a/moli-fetch/src/dns.rs b/moli-fetch/src/dns.rsindex 4fa2b645d..b08784615 100644--- a/moli-fetch/src/dns.rs+++ b/moli-fetch/src/dns.rs@@ -31,6 +31,9 @@ pub(crate) fn curl_dns_resolution(.with_network_address_policy(policy, url.to_string()))}FetchCurlDnsAdmission::ProxyEndpoint(target) => {+ // Pin the locally connected proxy, but do not apply origin address+ // policy to it. Configured enterprise proxies commonly live on+ // private networks, and the proxy owns target DNS/admission.Ok(CurlDnsResolution::resolve_endpoint(target, static_entries))}}@@ -142,14 +145,19 @@ mod tests {fn selected_proxy_domain_uses_shared_endpoint_resolution() {let config = FetchConfig::default();- assert_eq!(- admission(- &config,- "https://api.example.test/path",- &proxy_route("http://proxy.test:8080"),- ),- proxy_endpoint("proxy.test", 8080)- );+ for (proxy, port) in [+ ("http://proxy.test:8080", 8080),+ ("https://proxy.test:8443", 8443),+ ] {+ assert_eq!(+ admission(+ &config,+ "https://api.example.test/path",+ &proxy_route(proxy),+ ),+ proxy_endpoint("proxy.test", port)+ );+ }}#[test]diff --git a/moli-fetch/src/tests/tls_credentials.rs b/moli-fetch/src/tests/tls_credentials.rsindex fe7e5553c..9f1477139 100644--- a/moli-fetch/src/tests/tls_credentials.rs+++ b/moli-fetch/src/tests/tls_credentials.rs@@ -1,6 +1,6 @@use std::{fs, path::PathBuf, sync::Arc};-use anyhow::Result;+use anyhow::{Result, bail};use moli_cookie_jar::new_shared_browser_cookie_store;use parking_lot::Mutex;use rcgen::{@@ -12,6 +12,7 @@ use tokio::{net::TcpListener,sync::oneshot,task::{JoinHandle, JoinSet},+ time::{Duration, timeout},};use tokio_rustls::{TlsAcceptor,@@ -122,6 +123,85 @@ struct TlsServer {task: JoinHandle<()>,}+#[derive(Debug)]+struct ObservedHttpsProxyRequest {+ request_head: String,+ server_name: Option<String>,+ client_certificates: Vec<CertificateDer<'static>>,+}++struct HttpsProxy {+ url: String,+ task: Option<JoinHandle<Result<ObservedHttpsProxyRequest>>>,+}++impl HttpsProxy {+ async fn spawn(credentials: &TlsCredentials) -> Result<Self> {+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let port = listener.local_addr()?.port();+ let acceptor = credentials.acceptor.clone();+ let task = tokio::spawn(async move {+ let (stream, _) = listener.accept().await?;+ let mut stream = acceptor.accept(stream).await?;+ let server_name = stream.get_ref().1.server_name().map(str::to_owned);+ let client_certificates = stream+ .get_ref()+ .1+ .peer_certificates()+ .unwrap_or_default()+ .to_vec();+ let mut request_head = Vec::new();+ while !request_head.ends_with(b"\r\n\r\n") {+ let mut byte = [0];+ if stream.read(&mut byte).await? == 0 {+ bail!("HTTPS proxy client closed before completing its request");+ }+ request_head.push(byte[0]);+ if request_head.len() > 64 * 1024 {+ bail!("HTTPS proxy request headers are too large");+ }+ }+ let body = b"through-https-proxy";+ stream+ .write_all(+ format!(+ "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",+ body.len()+ )+ .as_bytes(),+ )+ .await?;+ stream.write_all(body).await?;+ stream.flush().await?;+ Ok(ObservedHttpsProxyRequest {+ request_head: String::from_utf8(request_head)?,+ server_name,+ client_certificates,+ })+ });+ Ok(Self {+ url: format!("https://localhost:{port}"),+ task: Some(task),+ })+ }++ async fn finish(mut self) -> Result<ObservedHttpsProxyRequest> {+ let task = self+ .task+ .take()+ .expect("HTTPS proxy task should be present");+ timeout(Duration::from_secs(5), task).await??+ }+}++impl Drop for HttpsProxy {+ fn drop(&mut self) {+ if let Some(task) = self.task.take() {+ task.abort();+ }+ }+}+impl TlsServer {async fn spawn(credentials: &TlsCredentials) -> Result<Self> {let listener = TcpListener::bind("127.0.0.1:0").await?;@@ -235,6 +315,39 @@ impl Transport {}}+#[tokio::test]+async fn https_proxy_uses_shared_dns_and_proxy_hostname_tls() -> Result<()> {+ let credentials = TlsCredentials::new()?;+ let proxy = HttpsProxy::spawn(&credentials).await?;+ let mut config = credentials.fetch_config();+ config.set_http_proxy(Some(proxy.url.clone()));+ config.set_http_no_proxy(Some(String::new()));+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());++ let response = client+ .fetch(Request::get(+ "http://fetch-target.invalid/through-https-proxy",+ )?)+ .await?;+ assert_eq!(response.body_text(), "through-https-proxy");+ assert!(client.shutdown().is_clean());++ let observed = proxy.finish().await?;+ assert_eq!(observed.server_name.as_deref(), Some("localhost"));+ assert!(+ observed+ .request_head+ .starts_with("GET http://fetch-target.invalid/through-https-proxy HTTP/1.1\r\n"),+ "unexpected HTTPS proxy request: {:?}",+ observed.request_head+ );+ assert!(+ observed.client_certificates.is_empty(),+ "origin client identity must not be sent to the HTTPS proxy"+ );+ Ok(())+}+#[tokio::test]async fn tls_client_certificate_respects_credentials_in_every_transport() -> Result<()> {let credentials = TlsCredentials::new()?;diff --git a/moli-websocket/src/test_support.rs b/moli-websocket/src/test_support.rsindex f84dda8c9..40ebce97a 100644--- a/moli-websocket/src/test_support.rs+++ b/moli-websocket/src/test_support.rs@@ -504,6 +504,70 @@ pub async fn spawn_http_connect_proxy() -> ((format!("http://{addr}"), request_rx, handle)}+#[derive(Debug)]+pub struct HttpsProxyObservation {+ pub request: String,+ pub server_name: Option<String>,+ pub client_certificates: Vec<Vec<u8>>,+}++pub async fn spawn_https_connect_proxy(+ acceptor: tokio_rustls::TlsAcceptor,+) -> (+ String,+ oneshot::Receiver<HttpsProxyObservation>,+ tokio::task::JoinHandle<Result<(), String>>,+) {+ let listener = TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind HTTPS WebSocket proxy");+ let port = listener.local_addr().expect("HTTPS proxy addr").port();+ let (observation_tx, observation_rx) = oneshot::channel();+ let handle = tokio::spawn(async move {+ let (client, _) = listener+ .accept()+ .await+ .map_err(|error| format!("accept HTTPS proxy client: {error}"))?;+ let mut client = acceptor+ .accept(client)+ .await+ .map_err(|error| format!("accept HTTPS proxy TLS: {error}"))?;+ let server_name = client.get_ref().1.server_name().map(str::to_owned);+ let client_certificates = client+ .get_ref()+ .1+ .peer_certificates()+ .unwrap_or_default()+ .iter()+ .map(|certificate| certificate.to_vec())+ .collect();+ let request = try_read_http_headers(&mut client)+ .await+ .map_err(|error| format!("read HTTPS proxy request: {error}"))?;+ let target = request+ .lines()+ .next()+ .and_then(|line| line.split_whitespace().nth(1))+ .expect("HTTPS proxy CONNECT target")+ .to_owned();+ let _ = observation_tx.send(HttpsProxyObservation {+ request,+ server_name,+ client_certificates,+ });+ let mut upstream = TcpStream::connect(&target)+ .await+ .map_err(|error| format!("connect HTTPS proxy upstream: {error}"))?;+ client+ .write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n")+ .await+ .map_err(|error| format!("write HTTPS proxy CONNECT response: {error}"))?;+ let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await;+ Ok(())+ });+ (format!("https://localhost:{port}"), observation_rx, handle)+}+pub async fn spawn_http_connect_proxy_response(response: &'static [u8],) -> (@@ -1085,18 +1149,31 @@ pub async fn spawn_dropping_websocket_server() -> (String, tokio::task::JoinHand(format!("ws://{addr}/drop"), handle)}-async fn read_http_headers(stream: &mut TcpStream) -> String {+async fn read_http_headers(stream: &mut (impl tokio::io::AsyncRead + Unpin)) -> String {+ try_read_http_headers(stream)+ .await+ .expect("read HTTP request")+}++async fn try_read_http_headers(+ stream: &mut (impl tokio::io::AsyncRead + Unpin),+) -> Result<String, String> {let mut request_bytes = Vec::new();let mut chunk = [0_u8; 512];loop {- let count = stream.read(&mut chunk).await.expect("read HTTP request");- assert!(count > 0, "client closed before HTTP headers completed");+ let count = stream+ .read(&mut chunk)+ .await+ .map_err(|error| error.to_string())?;+ if count == 0 {+ return Err("client closed before HTTP headers completed".to_owned());+ }request_bytes.extend_from_slice(&chunk[..count]);if request_bytes.windows(4).any(|window| window == b"\r\n\r\n") {break;}}- String::from_utf8_lossy(&request_bytes).into_owned()+ Ok(String::from_utf8_lossy(&request_bytes).into_owned())}fn websocket_request_header(request: &str, name: &str) -> Option<String> {diff --git a/moli-websocket/src/test_support/tls.rs b/moli-websocket/src/test_support/tls.rsindex 78e50a02e..e6002befe 100644--- a/moli-websocket/src/test_support/tls.rs+++ b/moli-websocket/src/test_support/tls.rs@@ -135,11 +135,7 @@ impl TlsWebSocketFixture {}}- /// Echoes until a clean WebSocket close and returns the actual TLS peer chain.- pub async fn spawn(- &self,- require_identity: bool,- ) -> (String, JoinHandle<Result<Vec<Vec<u8>>, String>>) {+ pub fn server_acceptor(&self, require_identity: bool) -> tokio_rustls::TlsAcceptor {let verifier = WebPkiClientVerifier::builder(Arc::new(self.roots.clone()));let verifier = if require_identity {verifier@@ -153,7 +149,15 @@ impl TlsWebSocketFixture {self.server_key.clone_key().into(),).unwrap();- let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config));+ tokio_rustls::TlsAcceptor::from(Arc::new(config))+ }++ /// Echoes until a clean WebSocket close and returns the actual TLS peer chain.+ pub async fn spawn(+ &self,+ require_identity: bool,+ ) -> (String, JoinHandle<Result<Vec<Vec<u8>>, String>>) {+ let acceptor = self.server_acceptor(require_identity);let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();let port = listener.local_addr().unwrap().port();let task = tokio::spawn(async move {diff --git a/moli-websocket/src/tests.rs b/moli-websocket/src/tests.rsindex 7c336b248..7726b536d 100644--- a/moli-websocket/src/tests.rs+++ b/moli-websocket/src/tests.rs@@ -308,6 +308,22 @@ fn websocket_proxy_route_normalizes_chromium_socks_schemes() {}}+#[test]+fn websocket_https_proxy_domain_selects_shared_endpoint_dns() {+ let uri = Url::parse("wss://target.test/socket").unwrap();+ let mut context = test_websocket_context();+ context.http_proxy = Some("https://proxy.test:8443".to_owned());+ let route = websocket_proxy_route_with_env(&uri, &context, |_| None).unwrap();+ let endpoint = route+ .connection_dns_endpoint(&uri, &moli_curl::HostResolveOverrides::default())+ .unwrap()+ .expect("proxy domain must use shared endpoint DNS");++ assert_eq!(endpoint.role(), moli_curl::ConnectionEndpointRole::Proxy);+ assert_eq!(endpoint.target().host(), "proxy.test");+ assert_eq!(endpoint.target().port(), 8443);+}+#[test]fn websocket_request_preparation_applies_context_protocols_and_cookie() {let mut context = test_websocket_context();diff --git a/moli-websocket/src/tests/tls.rs b/moli-websocket/src/tests/tls.rsindex d60e368c1..d96929750 100644--- a/moli-websocket/src/tests/tls.rs+++ b/moli-websocket/src/tests/tls.rs@@ -186,3 +186,62 @@ async fn websocket_tls_credentials_apply_inside_http_connect_tunnel() {)));proxy.await.unwrap();}++#[tokio::test]+async fn websocket_uses_https_proxy_with_shared_dns_and_proxy_hostname_tls() {+ let fixture = TlsWebSocketFixture::new();+ let (url, server) = fixture.spawn(false).await;+ let (proxy_url, observation, proxy) =+ spawn_https_connect_proxy(fixture.server_acceptor(false)).await;+ let mut context = tls_context(fixture.tls_config());+ context.http_proxy = Some(proxy_url);++ exercise_tls_connection(url.clone(), context, true).await;+ assert_eq!(+ server.await.unwrap().unwrap(),+ std::slice::from_ref(&fixture.client_certificate)+ );+ let observed = timeout(Duration::from_secs(5), observation)+ .await+ .unwrap()+ .unwrap();+ let target = Url::parse(&url).unwrap();+ assert!(observed.request.starts_with(&format!(+ "CONNECT localhost:{} HTTP/1.1\r\n",+ target.port().unwrap()+ )));+ assert_eq!(observed.server_name.as_deref(), Some("localhost"));+ assert!(+ observed.client_certificates.is_empty(),+ "origin client identity must not be sent to the HTTPS proxy"+ );+ proxy.await.unwrap().unwrap();+}++#[tokio::test]+async fn websocket_https_proxy_rejects_the_wrong_proxy_hostname() {+ let fixture = TlsWebSocketFixture::new();+ let (proxy_url, _observation, proxy) =+ spawn_https_connect_proxy(fixture.server_acceptor(false)).await;+ let mut proxy_url = Url::parse(&proxy_url).unwrap();+ proxy_url.set_host(Some("127.0.0.1")).unwrap();+ let mut context = tls_context(fixture.tls_config());+ context.http_proxy = Some(proxy_url.to_string());+ let (events, mut receiver) = mpsc::channel(32);++ let _connection = spawn_standalone_connection(+ 502,+ "ws://websocket-target.invalid/socket".to_owned(),+ Vec::new(),+ context,+ events,+ );+ let message = recv_handshake_failure_events(&mut receiver).await;+ let message = message.to_ascii_lowercase();+ assert!(+ message.contains("certificate") || message.contains("ssl"),+ "unexpected HTTPS proxy TLS error: {message}"+ );+ let proxy_error = proxy.await.unwrap().unwrap_err();+ assert!(proxy_error.contains("HTTPS proxy"), "{proxy_error}");+}