Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix: verify HTTPS proxy connections

ldm090c0ede82930
diff --git a/moli-curl/src/tls.rs b/moli-curl/src/tls.rs
index ca17241c1..92bf4e85f 100644
--- a/moli-curl/src/tls.rs
+++ b/moli-curl/src/tls.rs
@@ -102,6 +102,33 @@ impl CurlTlsConfig {
}
Ok(())
}
+
+ /// Install certificate-chain and hostname verification for an HTTPS proxy.
+ ///
+ /// libcurl keeps proxy TLS settings separate from origin TLS settings. The
+ /// configured CA is shared, but client identities remain origin-only unless
+ /// a dedicated proxy identity is added in the future.
+ pub fn configure_https_proxy<H: Handler>(&self, easy: &mut Easy2<H>) -> Result<()> {
+ easy.proxy_ssl_verify_peer(self.verify)
+ .context("failed to configure curl HTTPS proxy peer verification")?;
+ easy.proxy_ssl_verify_host(self.verify)
+ .context("failed to configure curl HTTPS proxy host verification")?;
+ if let Some(ca_cert) = &self.ca_cert {
+ let ca_cert_text = ca_cert.to_str().with_context(|| {
+ format!(
+ "curl HTTPS proxy CA certificate path is not valid UTF-8: `{}`",
+ ca_cert.display()
+ )
+ })?;
+ easy.proxy_cainfo(ca_cert_text).with_context(|| {
+ format!(
+ "failed to configure curl HTTPS proxy CA certificate `{}`",
+ ca_cert.display()
+ )
+ })?;
+ }
+ Ok(())
+ }
}
#[cfg(test)]
diff --git a/moli-curl/src/websocket/request.rs b/moli-curl/src/websocket/request.rs
index 488974503..7c57c19f2 100644
--- a/moli-curl/src/websocket/request.rs
+++ b/moli-curl/src/websocket/request.rs
@@ -82,6 +82,9 @@ pub(super) fn configure(request: &CurlWebSocketRequest) -> Result<Easy2<Handshak
// The caller selects the client identity carried by this connection request.
request.tls.configure(&mut easy, true)?;
easy.proxy(request.proxy.as_deref().unwrap_or(""))?;
+ if request.proxy.as_deref().is_some_and(proxy_uses_https_tls) {
+ request.tls.configure_https_proxy(&mut easy)?;
+ }
// The browser has already applied no_proxy; do not evaluate environment policy twice.
easy.noproxy("")?;
easy.http_proxy_tunnel(request.proxy.as_deref().is_some_and(proxy_uses_http_tunnel))?;
@@ -110,6 +113,12 @@ fn proxy_uses_http_tunnel(proxy: &str) -> bool {
scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https")
}
+fn proxy_uses_https_tls(proxy: &str) -> bool {
+ proxy
+ .split_once("://")
+ .is_some_and(|(scheme, _)| scheme.eq_ignore_ascii_case("https"))
+}
+
fn headers(entries: &[(String, String)]) -> Result<List> {
let mut list = List::new();
let mut size = 0usize;
diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs
index 56801a3bd..eb3aef09e 100644
--- a/moli-fetch/src/blocking/mod.rs
+++ b/moli-fetch/src/blocking/mod.rs
@@ -14,7 +14,7 @@ use moli_cookie_jar::{
NetworkCookieRequestContext, SharedBrowserCookieStore, StoredCookieQueryReport,
StoredCookieSetReport, same_site_urls,
};
-use moli_curl::HostResolveOverrides;
+use moli_curl::{HostResolveOverrides, ProxyScheme};
use moli_url::is_potentially_trustworthy_url;
use moli_url_policy::ensure_http_network_transport_url;
use tracing::debug;
@@ -627,6 +627,9 @@ pub(crate) fn configure_easy<H: Handler>(
HttpProxyRoute::Proxy(proxy) => {
easy.proxy(proxy.curl_url())
.with_context(|| anyhow!("failed to configure HTTP proxy `{}`", proxy.url()))?;
+ if proxy.scheme() == ProxyScheme::Https {
+ config.tls_config().configure_https_proxy(easy)?;
+ }
configure_proxy_headers(easy, config, request, proxy.scheme().uses_http_headers())?;
}
}
diff --git a/moli-fetch/src/dns.rs b/moli-fetch/src/dns.rs
index 4fa2b645d..b08784615 100644
--- a/moli-fetch/src/dns.rs
+++ b/moli-fetch/src/dns.rs
@@ -31,6 +31,9 @@ pub(crate) fn curl_dns_resolution(
.with_network_address_policy(policy, url.to_string()))
}
FetchCurlDnsAdmission::ProxyEndpoint(target) => {
+ // Pin the locally connected proxy, but do not apply origin address
+ // policy to it. Configured enterprise proxies commonly live on
+ // private networks, and the proxy owns target DNS/admission.
Ok(CurlDnsResolution::resolve_endpoint(target, static_entries))
}
}
@@ -142,14 +145,19 @@ mod tests {
fn selected_proxy_domain_uses_shared_endpoint_resolution() {
let config = FetchConfig::default();
- assert_eq!(
- admission(
- &config,
- "https://api.example.test/path",
- &proxy_route("http://proxy.test:8080"),
- ),
- proxy_endpoint("proxy.test", 8080)
- );
+ for (proxy, port) in [
+ ("http://proxy.test:8080", 8080),
+ ("https://proxy.test:8443", 8443),
+ ] {
+ assert_eq!(
+ admission(
+ &config,
+ "https://api.example.test/path",
+ &proxy_route(proxy),
+ ),
+ proxy_endpoint("proxy.test", port)
+ );
+ }
}
#[test]
diff --git a/moli-fetch/src/tests/tls_credentials.rs b/moli-fetch/src/tests/tls_credentials.rs
index fe7e5553c..9f1477139 100644
--- a/moli-fetch/src/tests/tls_credentials.rs
+++ b/moli-fetch/src/tests/tls_credentials.rs
@@ -1,6 +1,6 @@
use std::{fs, path::PathBuf, sync::Arc};
-use anyhow::Result;
+use anyhow::{Result, bail};
use moli_cookie_jar::new_shared_browser_cookie_store;
use parking_lot::Mutex;
use rcgen::{
@@ -12,6 +12,7 @@ use tokio::{
net::TcpListener,
sync::oneshot,
task::{JoinHandle, JoinSet},
+ time::{Duration, timeout},
};
use tokio_rustls::{
TlsAcceptor,
@@ -122,6 +123,85 @@ struct TlsServer {
task: JoinHandle<()>,
}
+#[derive(Debug)]
+struct ObservedHttpsProxyRequest {
+ request_head: String,
+ server_name: Option<String>,
+ client_certificates: Vec<CertificateDer<'static>>,
+}
+
+struct HttpsProxy {
+ url: String,
+ task: Option<JoinHandle<Result<ObservedHttpsProxyRequest>>>,
+}
+
+impl HttpsProxy {
+ async fn spawn(credentials: &TlsCredentials) -> Result<Self> {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let port = listener.local_addr()?.port();
+ let acceptor = credentials.acceptor.clone();
+ let task = tokio::spawn(async move {
+ let (stream, _) = listener.accept().await?;
+ let mut stream = acceptor.accept(stream).await?;
+ let server_name = stream.get_ref().1.server_name().map(str::to_owned);
+ let client_certificates = stream
+ .get_ref()
+ .1
+ .peer_certificates()
+ .unwrap_or_default()
+ .to_vec();
+ let mut request_head = Vec::new();
+ while !request_head.ends_with(b"\r\n\r\n") {
+ let mut byte = [0];
+ if stream.read(&mut byte).await? == 0 {
+ bail!("HTTPS proxy client closed before completing its request");
+ }
+ request_head.push(byte[0]);
+ if request_head.len() > 64 * 1024 {
+ bail!("HTTPS proxy request headers are too large");
+ }
+ }
+ let body = b"through-https-proxy";
+ stream
+ .write_all(
+ format!(
+ "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+ body.len()
+ )
+ .as_bytes(),
+ )
+ .await?;
+ stream.write_all(body).await?;
+ stream.flush().await?;
+ Ok(ObservedHttpsProxyRequest {
+ request_head: String::from_utf8(request_head)?,
+ server_name,
+ client_certificates,
+ })
+ });
+ Ok(Self {
+ url: format!("https://localhost:{port}"),
+ task: Some(task),
+ })
+ }
+
+ async fn finish(mut self) -> Result<ObservedHttpsProxyRequest> {
+ let task = self
+ .task
+ .take()
+ .expect("HTTPS proxy task should be present");
+ timeout(Duration::from_secs(5), task).await??
+ }
+}
+
+impl Drop for HttpsProxy {
+ fn drop(&mut self) {
+ if let Some(task) = self.task.take() {
+ task.abort();
+ }
+ }
+}
+
impl TlsServer {
async fn spawn(credentials: &TlsCredentials) -> Result<Self> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
@@ -235,6 +315,39 @@ impl Transport {
}
}
+#[tokio::test]
+async fn https_proxy_uses_shared_dns_and_proxy_hostname_tls() -> Result<()> {
+ let credentials = TlsCredentials::new()?;
+ let proxy = HttpsProxy::spawn(&credentials).await?;
+ let mut config = credentials.fetch_config();
+ config.set_http_proxy(Some(proxy.url.clone()));
+ config.set_http_no_proxy(Some(String::new()));
+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());
+
+ let response = client
+ .fetch(Request::get(
+ "http://fetch-target.invalid/through-https-proxy",
+ )?)
+ .await?;
+ assert_eq!(response.body_text(), "through-https-proxy");
+ assert!(client.shutdown().is_clean());
+
+ let observed = proxy.finish().await?;
+ assert_eq!(observed.server_name.as_deref(), Some("localhost"));
+ assert!(
+ observed
+ .request_head
+ .starts_with("GET http://fetch-target.invalid/through-https-proxy HTTP/1.1\r\n"),
+ "unexpected HTTPS proxy request: {:?}",
+ observed.request_head
+ );
+ assert!(
+ observed.client_certificates.is_empty(),
+ "origin client identity must not be sent to the HTTPS proxy"
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn tls_client_certificate_respects_credentials_in_every_transport() -> Result<()> {
let credentials = TlsCredentials::new()?;
diff --git a/moli-websocket/src/test_support.rs b/moli-websocket/src/test_support.rs
index f84dda8c9..40ebce97a 100644
--- a/moli-websocket/src/test_support.rs
+++ b/moli-websocket/src/test_support.rs
@@ -504,6 +504,70 @@ pub async fn spawn_http_connect_proxy() -> (
(format!("http://{addr}"), request_rx, handle)
}
+#[derive(Debug)]
+pub struct HttpsProxyObservation {
+ pub request: String,
+ pub server_name: Option<String>,
+ pub client_certificates: Vec<Vec<u8>>,
+}
+
+pub async fn spawn_https_connect_proxy(
+ acceptor: tokio_rustls::TlsAcceptor,
+) -> (
+ String,
+ oneshot::Receiver<HttpsProxyObservation>,
+ tokio::task::JoinHandle<Result<(), String>>,
+) {
+ let listener = TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("bind HTTPS WebSocket proxy");
+ let port = listener.local_addr().expect("HTTPS proxy addr").port();
+ let (observation_tx, observation_rx) = oneshot::channel();
+ let handle = tokio::spawn(async move {
+ let (client, _) = listener
+ .accept()
+ .await
+ .map_err(|error| format!("accept HTTPS proxy client: {error}"))?;
+ let mut client = acceptor
+ .accept(client)
+ .await
+ .map_err(|error| format!("accept HTTPS proxy TLS: {error}"))?;
+ let server_name = client.get_ref().1.server_name().map(str::to_owned);
+ let client_certificates = client
+ .get_ref()
+ .1
+ .peer_certificates()
+ .unwrap_or_default()
+ .iter()
+ .map(|certificate| certificate.to_vec())
+ .collect();
+ let request = try_read_http_headers(&mut client)
+ .await
+ .map_err(|error| format!("read HTTPS proxy request: {error}"))?;
+ let target = request
+ .lines()
+ .next()
+ .and_then(|line| line.split_whitespace().nth(1))
+ .expect("HTTPS proxy CONNECT target")
+ .to_owned();
+ let _ = observation_tx.send(HttpsProxyObservation {
+ request,
+ server_name,
+ client_certificates,
+ });
+ let mut upstream = TcpStream::connect(&target)
+ .await
+ .map_err(|error| format!("connect HTTPS proxy upstream: {error}"))?;
+ client
+ .write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n")
+ .await
+ .map_err(|error| format!("write HTTPS proxy CONNECT response: {error}"))?;
+ let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await;
+ Ok(())
+ });
+ (format!("https://localhost:{port}"), observation_rx, handle)
+}
+
pub async fn spawn_http_connect_proxy_response(
response: &'static [u8],
) -> (
@@ -1085,18 +1149,31 @@ pub async fn spawn_dropping_websocket_server() -> (String, tokio::task::JoinHand
(format!("ws://{addr}/drop"), handle)
}
-async fn read_http_headers(stream: &mut TcpStream) -> String {
+async fn read_http_headers(stream: &mut (impl tokio::io::AsyncRead + Unpin)) -> String {
+ try_read_http_headers(stream)
+ .await
+ .expect("read HTTP request")
+}
+
+async fn try_read_http_headers(
+ stream: &mut (impl tokio::io::AsyncRead + Unpin),
+) -> Result<String, String> {
let mut request_bytes = Vec::new();
let mut chunk = [0_u8; 512];
loop {
- let count = stream.read(&mut chunk).await.expect("read HTTP request");
- assert!(count > 0, "client closed before HTTP headers completed");
+ let count = stream
+ .read(&mut chunk)
+ .await
+ .map_err(|error| error.to_string())?;
+ if count == 0 {
+ return Err("client closed before HTTP headers completed".to_owned());
+ }
request_bytes.extend_from_slice(&chunk[..count]);
if request_bytes.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
- String::from_utf8_lossy(&request_bytes).into_owned()
+ Ok(String::from_utf8_lossy(&request_bytes).into_owned())
}
fn websocket_request_header(request: &str, name: &str) -> Option<String> {
diff --git a/moli-websocket/src/test_support/tls.rs b/moli-websocket/src/test_support/tls.rs
index 78e50a02e..e6002befe 100644
--- a/moli-websocket/src/test_support/tls.rs
+++ b/moli-websocket/src/test_support/tls.rs
@@ -135,11 +135,7 @@ impl TlsWebSocketFixture {
}
}
- /// Echoes until a clean WebSocket close and returns the actual TLS peer chain.
- pub async fn spawn(
- &self,
- require_identity: bool,
- ) -> (String, JoinHandle<Result<Vec<Vec<u8>>, String>>) {
+ pub fn server_acceptor(&self, require_identity: bool) -> tokio_rustls::TlsAcceptor {
let verifier = WebPkiClientVerifier::builder(Arc::new(self.roots.clone()));
let verifier = if require_identity {
verifier
@@ -153,7 +149,15 @@ impl TlsWebSocketFixture {
self.server_key.clone_key().into(),
)
.unwrap();
- let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config));
+ tokio_rustls::TlsAcceptor::from(Arc::new(config))
+ }
+
+ /// Echoes until a clean WebSocket close and returns the actual TLS peer chain.
+ pub async fn spawn(
+ &self,
+ require_identity: bool,
+ ) -> (String, JoinHandle<Result<Vec<Vec<u8>>, String>>) {
+ let acceptor = self.server_acceptor(require_identity);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let task = tokio::spawn(async move {
diff --git a/moli-websocket/src/tests.rs b/moli-websocket/src/tests.rs
index 7c336b248..7726b536d 100644
--- a/moli-websocket/src/tests.rs
+++ b/moli-websocket/src/tests.rs
@@ -308,6 +308,22 @@ fn websocket_proxy_route_normalizes_chromium_socks_schemes() {
}
}
+#[test]
+fn websocket_https_proxy_domain_selects_shared_endpoint_dns() {
+ let uri = Url::parse("wss://target.test/socket").unwrap();
+ let mut context = test_websocket_context();
+ context.http_proxy = Some("https://proxy.test:8443".to_owned());
+ let route = websocket_proxy_route_with_env(&uri, &context, |_| None).unwrap();
+ let endpoint = route
+ .connection_dns_endpoint(&uri, &moli_curl::HostResolveOverrides::default())
+ .unwrap()
+ .expect("proxy domain must use shared endpoint DNS");
+
+ assert_eq!(endpoint.role(), moli_curl::ConnectionEndpointRole::Proxy);
+ assert_eq!(endpoint.target().host(), "proxy.test");
+ assert_eq!(endpoint.target().port(), 8443);
+}
+
#[test]
fn websocket_request_preparation_applies_context_protocols_and_cookie() {
let mut context = test_websocket_context();
diff --git a/moli-websocket/src/tests/tls.rs b/moli-websocket/src/tests/tls.rs
index d60e368c1..d96929750 100644
--- a/moli-websocket/src/tests/tls.rs
+++ b/moli-websocket/src/tests/tls.rs
@@ -186,3 +186,62 @@ async fn websocket_tls_credentials_apply_inside_http_connect_tunnel() {
)));
proxy.await.unwrap();
}
+
+#[tokio::test]
+async fn websocket_uses_https_proxy_with_shared_dns_and_proxy_hostname_tls() {
+ let fixture = TlsWebSocketFixture::new();
+ let (url, server) = fixture.spawn(false).await;
+ let (proxy_url, observation, proxy) =
+ spawn_https_connect_proxy(fixture.server_acceptor(false)).await;
+ let mut context = tls_context(fixture.tls_config());
+ context.http_proxy = Some(proxy_url);
+
+ exercise_tls_connection(url.clone(), context, true).await;
+ assert_eq!(
+ server.await.unwrap().unwrap(),
+ std::slice::from_ref(&fixture.client_certificate)
+ );
+ let observed = timeout(Duration::from_secs(5), observation)
+ .await
+ .unwrap()
+ .unwrap();
+ let target = Url::parse(&url).unwrap();
+ assert!(observed.request.starts_with(&format!(
+ "CONNECT localhost:{} HTTP/1.1\r\n",
+ target.port().unwrap()
+ )));
+ assert_eq!(observed.server_name.as_deref(), Some("localhost"));
+ assert!(
+ observed.client_certificates.is_empty(),
+ "origin client identity must not be sent to the HTTPS proxy"
+ );
+ proxy.await.unwrap().unwrap();
+}
+
+#[tokio::test]
+async fn websocket_https_proxy_rejects_the_wrong_proxy_hostname() {
+ let fixture = TlsWebSocketFixture::new();
+ let (proxy_url, _observation, proxy) =
+ spawn_https_connect_proxy(fixture.server_acceptor(false)).await;
+ let mut proxy_url = Url::parse(&proxy_url).unwrap();
+ proxy_url.set_host(Some("127.0.0.1")).unwrap();
+ let mut context = tls_context(fixture.tls_config());
+ context.http_proxy = Some(proxy_url.to_string());
+ let (events, mut receiver) = mpsc::channel(32);
+
+ let _connection = spawn_standalone_connection(
+ 502,
+ "ws://websocket-target.invalid/socket".to_owned(),
+ Vec::new(),
+ context,
+ events,
+ );
+ let message = recv_handshake_failure_events(&mut receiver).await;
+ let message = message.to_ascii_lowercase();
+ assert!(
+ message.contains("certificate") || message.contains("ssl"),
+ "unexpected HTTPS proxy TLS error: {message}"
+ );
+ let proxy_error = proxy.await.unwrap().unwrap_err();
+ assert!(proxy_error.contains("HTTPS proxy"), "{proxy_error}");
+}