Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
refactor: share proxy route selection
ldm05417fe675e1e
diff --git a/moli-curl/src/lib.rs b/moli-curl/src/lib.rsindex 337b3431c..b2a246eaa 100644--- a/moli-curl/src/lib.rs+++ b/moli-curl/src/lib.rs@@ -3,6 +3,7 @@mod dns_adapter;mod http;mod network_policy;+mod proxy;mod runtime;mod tls;pub mod websocket;@@ -10,5 +11,9 @@ pub mod websocket;pub use dns_adapter::CurlDnsResolution;pub use http::{CurlHttpSender, CurlMultiCompletion, CurlMultiJob, CurlOriginKey, CurlSubmitError};pub use network_policy::NetworkAddressPolicy;+pub use proxy::{+ ProxyRoute, ProxyScheme, ProxyTargetResolution, SelectedProxy, select_proxy_route,+ select_proxy_route_with_env,+};pub use runtime::{CurlMultiRuntime, CurlMultiRuntimeConfig, CurlTransferId};pub use tls::CurlTlsConfig;diff --git a/moli-curl/src/proxy.rs b/moli-curl/src/proxy.rsnew file mode 100644index 000000000..22045b575--- /dev/null+++ b/moli-curl/src/proxy.rs@@ -0,0 +1,418 @@+use std::net::IpAddr;++use anyhow::{Context, Result, bail};+use cidr::AnyIpCidr;+use url::{Host, Url};++/// How an accepted proxy resolves the request target hostname.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ProxyTargetResolution {+ /// The target hostname is sent to the proxy without a local DNS lookup.+ Proxy,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ProxyScheme {+ Http,+ Https,+ Socks5h,+ Socks4a,+}++impl ProxyScheme {+ pub fn uses_http_headers(self) -> bool {+ matches!(self, Self::Http | Self::Https)+ }++ fn default_port(self) -> u16 {+ match self {+ Self::Http => 80,+ Self::Https => 443,+ Self::Socks5h | Self::Socks4a => 1080,+ }+ }+}++/// Parsed proxy selected for a single request target.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct SelectedProxy {+ url: Url,+ scheme: ProxyScheme,+ endpoint_port: u16,+}++impl SelectedProxy {+ pub fn parse(raw: &str) -> Result<Self> {+ let normalized = if raw.contains("://") {+ raw.to_owned()+ } else {+ format!("http://{raw}")+ };+ let url = Url::parse(&normalized)+ .with_context(|| format!("failed to parse proxy URL `{raw}`"))?;+ let scheme = match url.scheme() {+ "http" => ProxyScheme::Http,+ "https" => ProxyScheme::Https,+ "socks5h" => ProxyScheme::Socks5h,+ "socks4a" => ProxyScheme::Socks4a,+ "socks5" | "socks4" => bail!(+ "unsupported proxy scheme `{}`; use a remote-DNS proxy scheme such as `socks5h://`",+ url.scheme()+ ),+ scheme => bail!(+ "unsupported proxy scheme `{scheme}`; expected http, https, socks5h, or socks4a"+ ),+ };+ if url.host().is_none() {+ bail!("proxy URL `{raw}` is missing a host");+ }+ let endpoint_port = url.port().unwrap_or_else(|| scheme.default_port());+ Ok(Self {+ url,+ scheme,+ endpoint_port,+ })+ }++ pub fn url(&self) -> &str {+ self.url.as_str()+ }++ pub fn scheme(&self) -> ProxyScheme {+ self.scheme+ }++ pub fn endpoint_host(&self) -> &str {+ self.url+ .host_str()+ .expect("selected proxy was validated with a host")+ }++ pub fn endpoint_ip(&self) -> Option<IpAddr> {+ match self+ .url+ .host()+ .expect("selected proxy was validated with a host")+ {+ Host::Ipv4(address) => Some(address.into()),+ Host::Ipv6(address) => Some(address.into()),+ Host::Domain(_) => None,+ }+ }++ pub fn endpoint_port(&self) -> u16 {+ self.endpoint_port+ }++ pub fn target_resolution(&self) -> ProxyTargetResolution {+ ProxyTargetResolution::Proxy+ }+}++/// Concrete route selected exactly once for a request target.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum ProxyRoute {+ Direct,+ Proxy(SelectedProxy),+}++impl ProxyRoute {+ pub fn from_proxy_url(raw: &str) -> Result<Self> {+ Ok(Self::Proxy(SelectedProxy::parse(raw)?))+ }++ pub fn is_proxy(&self) -> bool {+ matches!(self, Self::Proxy(_))+ }++ pub fn proxy(&self) -> Option<&SelectedProxy> {+ match self {+ Self::Direct => None,+ Self::Proxy(proxy) => Some(proxy),+ }+ }+}++pub fn select_proxy_route(+ target: &Url,+ configured_proxy: Option<&str>,+ configured_no_proxy: Option<&str>,+) -> Result<ProxyRoute> {+ select_proxy_route_with_env(target, configured_proxy, configured_no_proxy, |name| {+ std::env::var(name).ok()+ })+}++pub fn select_proxy_route_with_env(+ target: &Url,+ configured_proxy: Option<&str>,+ configured_no_proxy: Option<&str>,+ mut env: impl FnMut(&str) -> Option<String>,+) -> Result<ProxyRoute> {+ let proxy = match configured_proxy {+ Some("") => return Ok(ProxyRoute::Direct),+ Some(proxy) => Some(proxy.to_owned()),+ None => env_proxy_for_scheme(target.scheme(), &mut env),+ };+ let Some(proxy) = proxy.filter(|proxy| !proxy.is_empty()) else {+ return Ok(ProxyRoute::Direct);+ };++ let no_proxy = match configured_no_proxy {+ Some(no_proxy) => Some(no_proxy.to_owned()),+ None => env_no_proxy(&mut env),+ };+ if let (Some(host), Some(port)) = (target.host_str(), target.port_or_known_default())+ && no_proxy_matches(host, port, no_proxy.as_deref())+ {+ return Ok(ProxyRoute::Direct);+ }++ ProxyRoute::from_proxy_url(&proxy)+}++fn env_proxy_for_scheme(+ scheme: &str,+ env: &mut impl FnMut(&str) -> Option<String>,+) -> Option<String> {+ let names: &[&str] = match scheme {+ // curl deliberately ignores uppercase HTTP_PROXY because CGI servers+ // commonly expose an attacker-controlled Proxy header under that name.+ "http" | "ws" => &["http_proxy"],+ "https" | "wss" => &["https_proxy", "HTTPS_PROXY"],+ _ => &[],+ };+ for name in names {+ if let Some(value) = env(name).filter(|value| !value.is_empty()) {+ return Some(value);+ }+ }+ for name in ["all_proxy", "ALL_PROXY"] {+ if let Some(value) = env(name).filter(|value| !value.is_empty()) {+ return Some(value);+ }+ }+ None+}++fn env_no_proxy(env: &mut impl FnMut(&str) -> Option<String>) -> Option<String> {+ env("no_proxy")+ .filter(|value| !value.is_empty())+ .or_else(|| env("NO_PROXY").filter(|value| !value.is_empty()))+}++fn no_proxy_matches(host: &str, port: u16, no_proxy: Option<&str>) -> bool {+ let Some(no_proxy) = no_proxy else {+ return false;+ };+ let host = host+ .trim_matches(['[', ']'])+ .trim_end_matches('.')+ .to_ascii_lowercase();+ let host_ip = host.parse::<IpAddr>().ok();+ no_proxy.split(',').any(|token| {+ let token = token.trim();+ if token.is_empty() {+ return false;+ }+ if token == "*" {+ return true;+ }+ let (token_host, token_port) = split_no_proxy_host_port(token);+ if let Some(token_port) = token_port+ && token_port != port+ {+ return false;+ }+ let token_host = token_host+ .trim_matches(['[', ']'])+ .trim_start_matches('.')+ .trim_end_matches('.')+ .to_ascii_lowercase();+ if token_host.is_empty() {+ return false;+ }+ if let Some(host_ip) = host_ip {+ return token_host.parse::<IpAddr>().is_ok_and(|ip| ip == host_ip)+ || token_host+ .parse::<AnyIpCidr>()+ .is_ok_and(|cidr| cidr.contains(&host_ip));+ }+ host == token_host+ || host+ .strip_suffix(&token_host)+ .is_some_and(|prefix| prefix.ends_with('.'))+ })+}++fn split_no_proxy_host_port(token: &str) -> (&str, Option<u16>) {+ if let Some(bracketed) = token.strip_prefix('[')+ && let Some((host, port)) = bracketed.rsplit_once("]:")+ && let Ok(port) = port.parse::<u16>()+ {+ return (host, Some(port));+ }+ let Some((host, port)) = token.rsplit_once(':') else {+ return (token, None);+ };+ match port.parse::<u16>() {+ Ok(port) if !host.contains(':') => (host, Some(port)),+ _ => (token, None),+ }+}++#[cfg(test)]+mod tests {+ use std::collections::HashMap;++ use super::*;++ fn route(+ raw_target: &str,+ configured_proxy: Option<&str>,+ configured_no_proxy: Option<&str>,+ env: &[(&str, &str)],+ ) -> Result<ProxyRoute> {+ let env = env+ .iter()+ .map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))+ .collect::<HashMap<_, _>>();+ select_proxy_route_with_env(+ &Url::parse(raw_target).expect("test URL should parse"),+ configured_proxy,+ configured_no_proxy,+ |name| env.get(name).cloned(),+ )+ }++ #[test]+ fn explicit_proxy_and_no_proxy_select_one_route() {+ assert_eq!(+ route(+ "https://api.example.test/path",+ Some("http://proxy.test:8080"),+ None,+ &[],+ )+ .unwrap(),+ ProxyRoute::from_proxy_url("http://proxy.test:8080").unwrap()+ );+ assert_eq!(+ route(+ "https://api.example.test/path",+ Some("http://proxy.test:8080"),+ Some(".example.test"),+ &[],+ )+ .unwrap(),+ ProxyRoute::Direct+ );+ }++ #[test]+ fn environment_proxy_selection_is_shared_by_http_and_websocket_schemes() {+ assert!(+ route(+ "http://example.test/path",+ None,+ None,+ &[("http_proxy", "http://proxy.test:8080")],+ )+ .unwrap()+ .is_proxy()+ );+ assert!(+ route(+ "ws://example.test/socket",+ None,+ None,+ &[("http_proxy", "http://proxy.test:8080")],+ )+ .unwrap()+ .is_proxy()+ );+ assert!(+ route(+ "wss://example.test/socket",+ None,+ None,+ &[("HTTPS_PROXY", "http://proxy.test:8080")],+ )+ .unwrap()+ .is_proxy()+ );+ assert_eq!(+ route(+ "ws://example.test/socket",+ None,+ None,+ &[("HTTP_PROXY", "http://proxy.test:8080")],+ )+ .unwrap(),+ ProxyRoute::Direct+ );+ }++ #[test]+ fn empty_explicit_proxy_disables_environment_proxy_fallback() {+ assert_eq!(+ route(+ "http://example.test/path",+ Some(""),+ None,+ &[("http_proxy", "http://proxy.test:8080")],+ )+ .unwrap(),+ ProxyRoute::Direct+ );+ }++ #[test]+ fn no_proxy_handles_domains_ports_ip_cidrs_and_trailing_dots() {+ assert!(no_proxy_matches(+ "api.example.test",+ 8443,+ Some("example.test:8443")+ ));+ assert!(!no_proxy_matches(+ "api.example.test",+ 443,+ Some("example.test:8443")+ ));+ assert!(!no_proxy_matches(+ "notexample.test",+ 443,+ Some("example.test")+ ));+ assert!(no_proxy_matches("anything.test", 443, Some("*")));+ assert!(no_proxy_matches(+ "api.example.test.",+ 443,+ Some("example.test.")+ ));+ assert!(no_proxy_matches("192.0.2.42", 80, Some("192.0.2.0/24")));+ assert!(!no_proxy_matches("198.51.100.42", 80, Some("192.0.2.0/24")));+ assert!(no_proxy_matches("[::1]", 8443, Some("[::1]:8443")));+ }++ #[test]+ fn proxy_parser_accepts_only_remote_dns_schemes() {+ for (raw, scheme, port) in [+ ("proxy.test:8080", ProxyScheme::Http, 8080),+ ("https://proxy.test", ProxyScheme::Https, 443),+ ("socks5h://proxy.test", ProxyScheme::Socks5h, 1080),+ ("socks4a://proxy.test", ProxyScheme::Socks4a, 1080),+ ] {+ let proxy = SelectedProxy::parse(raw).unwrap();+ assert_eq!(proxy.scheme(), scheme);+ assert_eq!(proxy.endpoint_host(), "proxy.test");+ assert_eq!(proxy.endpoint_port(), port);+ assert_eq!(proxy.target_resolution(), ProxyTargetResolution::Proxy);+ }++ for scheme in ["socks5", "socks4"] {+ let error = SelectedProxy::parse(&format!("{scheme}://proxy.test:1080")).unwrap_err();+ assert!(error.to_string().contains("remote-DNS proxy scheme"));+ }+ }+}diff --git a/moli-curl/src/websocket/request.rs b/moli-curl/src/websocket/request.rsindex 4157a8b54..6b42bb7bb 100644--- a/moli-curl/src/websocket/request.rs+++ b/moli-curl/src/websocket/request.rs@@ -84,7 +84,7 @@ pub(super) fn configure(request: &CurlWebSocketRequest) -> Result<Easy2<Handshakeasy.proxy(request.proxy.as_deref().unwrap_or(""))?;// The browser has already applied no_proxy; do not evaluate environment policy twice.easy.noproxy("")?;- easy.http_proxy_tunnel(request.proxy.is_some())?;+ easy.http_proxy_tunnel(request.proxy.as_deref().is_some_and(proxy_uses_http_tunnel))?;easy.separate_proxy_headers(true)?;easy.http_headers(headers(&request.headers)?)?;easy.proxy_headers(headers(&request.proxy_headers)?)?;@@ -93,6 +93,13 @@ pub(super) fn configure(request: &CurlWebSocketRequest) -> Result<Easy2<HandshakOk(easy)}+fn proxy_uses_http_tunnel(proxy: &str) -> bool {+ let Some((scheme, _)) = proxy.split_once("://") else {+ return true;+ };+ scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https")+}+fn headers(entries: &[(String, String)]) -> Result<List> {let mut list = List::new();let mut size = 0usize;diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rsindex bd26c5b29..7235084cd 100644--- a/moli-fetch/src/blocking/mod.rs+++ b/moli-fetch/src/blocking/mod.rs@@ -639,8 +639,8 @@ pub(crate) fn configure_easy<H: Handler>(.proxy("").context("failed to disable the HTTP proxy for a direct request")?,HttpProxyRoute::Proxy(proxy) => easy- .proxy(proxy)- .with_context(|| anyhow!("failed to configure HTTP proxy `{proxy}`"))?,+ .proxy(proxy.url())+ .with_context(|| anyhow!("failed to configure HTTP proxy `{}`", proxy.url()))?,}// Proxy bypass has already been resolved into `proxy_route`. Keep curl// from re-evaluating environment `no_proxy` after that security decision.diff --git a/moli-fetch/src/dns.rs b/moli-fetch/src/dns.rsindex 94f00b403..39dadcde5 100644--- a/moli-fetch/src/dns.rs+++ b/moli-fetch/src/dns.rs@@ -99,6 +99,10 @@ mod tests {FetchCurlDnsAdmission::SharedResolver(DnsTarget::new(host, port))}+ fn proxy_route(raw: &str) -> HttpProxyRoute {+ HttpProxyRoute::from_proxy_url(raw).expect("test proxy URL should parse")+ }+#[test]fn direct_http_and_https_domains_use_shared_resolution() {let config = FetchConfig::default();@@ -149,7 +153,7 @@ mod tests {admission(&config,"https://api.example.test/path",- &HttpProxyRoute::Proxy("http://proxy.test:8080".to_owned()),+ &proxy_route("http://proxy.test:8080"),),FetchCurlDnsAdmission::NoSharedResolution);@@ -163,7 +167,7 @@ mod tests {let error = admission_result(&config,"https://api.example.test/path",- &HttpProxyRoute::Proxy("http://proxy.test:8080".to_owned()),+ &proxy_route("http://proxy.test:8080"),).expect_err("a strict policy cannot verify proxy-side DNS");diff --git a/moli-fetch/src/proxy.rs b/moli-fetch/src/proxy.rsindex 06bbccba2..d623387ae 100644--- a/moli-fetch/src/proxy.rs+++ b/moli-fetch/src/proxy.rs@@ -1,254 +1,10 @@-use std::net::IpAddr;--use cidr::AnyIpCidr;+use anyhow::Result;use url::Url;use crate::FetchConfig;-/// Concrete proxy route selected once for one HTTP request target.-///-/// The same value must drive both DNS admission and curl configuration so-/// environment variables and `no_proxy` cannot be interpreted twice.-#[derive(Debug, Clone, PartialEq, Eq)]-pub(crate) enum HttpProxyRoute {- Direct,- Proxy(String),-}--impl HttpProxyRoute {- pub(crate) fn is_proxy(&self) -> bool {- matches!(self, Self::Proxy(_))- }-}--pub(crate) fn resolve_http_proxy_route(config: &FetchConfig, url: &Url) -> HttpProxyRoute {- resolve_http_proxy_route_with_env(config, url, |name| std::env::var(name).ok())-}--fn resolve_http_proxy_route_with_env(- config: &FetchConfig,- url: &Url,- mut env: impl FnMut(&str) -> Option<String>,-) -> HttpProxyRoute {- let proxy = match config.http_proxy() {- Some("") => return HttpProxyRoute::Direct,- Some(proxy) => Some(proxy.to_owned()),- None => env_proxy_for_scheme(url.scheme(), &mut env),- };- let Some(proxy) = proxy.filter(|proxy| !proxy.is_empty()) else {- return HttpProxyRoute::Direct;- };-- let no_proxy = match config.http_no_proxy() {- Some(no_proxy) => Some(no_proxy.to_owned()),- None => env_no_proxy(&mut env),- };- if let (Some(host), Some(port)) = (url.host_str(), url.port_or_known_default())- && no_proxy_matches(host, port, no_proxy.as_deref())- {- return HttpProxyRoute::Direct;- }-- HttpProxyRoute::Proxy(proxy)-}--fn env_proxy_for_scheme(- scheme: &str,- env: &mut impl FnMut(&str) -> Option<String>,-) -> Option<String> {- let names: &[&str] = match scheme {- // curl deliberately ignores uppercase HTTP_PROXY because CGI servers- // commonly expose an attacker-controlled Proxy header under that name.- "http" => &["http_proxy"],- "https" => &["https_proxy", "HTTPS_PROXY"],- _ => &[],- };- for name in names {- if let Some(value) = env(name).filter(|value| !value.is_empty()) {- return Some(value);- }- }- for name in ["all_proxy", "ALL_PROXY"] {- if let Some(value) = env(name).filter(|value| !value.is_empty()) {- return Some(value);- }- }- None-}--fn env_no_proxy(env: &mut impl FnMut(&str) -> Option<String>) -> Option<String> {- env("no_proxy")- .filter(|value| !value.is_empty())- .or_else(|| env("NO_PROXY").filter(|value| !value.is_empty()))-}--fn no_proxy_matches(host: &str, port: u16, no_proxy: Option<&str>) -> bool {- let Some(no_proxy) = no_proxy else {- return false;- };- let host = host- .trim_matches(['[', ']'])- .trim_end_matches('.')- .to_ascii_lowercase();- let host_ip = host.parse::<IpAddr>().ok();- no_proxy.split(',').any(|token| {- let token = token.trim();- if token.is_empty() {- return false;- }- if token == "*" {- return true;- }- let (token_host, token_port) = split_no_proxy_host_port(token);- if let Some(token_port) = token_port- && token_port != port- {- return false;- }- let token_host = token_host- .trim_matches(['[', ']'])- .trim_start_matches('.')- .trim_end_matches('.')- .to_ascii_lowercase();- if token_host.is_empty() {- return false;- }- if let Some(host_ip) = host_ip {- return token_host.parse::<IpAddr>().is_ok_and(|ip| ip == host_ip)- || token_host- .parse::<AnyIpCidr>()- .is_ok_and(|cidr| cidr.contains(&host_ip));- }- host == token_host- || host- .strip_suffix(&token_host)- .is_some_and(|prefix| prefix.ends_with('.'))- })-}--fn split_no_proxy_host_port(token: &str) -> (&str, Option<u16>) {- if let Some(bracketed) = token.strip_prefix('[')- && let Some((host, port)) = bracketed.rsplit_once("]:")- && let Ok(port) = port.parse::<u16>()- {- return (host, Some(port));- }- let Some((host, port)) = token.rsplit_once(':') else {- return (token, None);- };- match port.parse::<u16>() {- Ok(port) if !host.contains(':') => (host, Some(port)),- _ => (token, None),- }-}--#[cfg(test)]-mod tests {- use std::collections::HashMap;-- use super::*;-- fn route(config: &FetchConfig, raw_url: &str, env: &[(&str, &str)]) -> HttpProxyRoute {- let env = env- .iter()- .map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))- .collect::<HashMap<_, _>>();- resolve_http_proxy_route_with_env(- config,- &Url::parse(raw_url).expect("test URL should parse"),- |name| env.get(name).cloned(),- )- }-- #[test]- fn explicit_proxy_and_no_proxy_select_one_route() {- let mut config = FetchConfig::default();- config.set_http_proxy(Some("http://proxy.test:8080".to_owned()));-- assert_eq!(- route(&config, "https://api.example.test/path", &[]),- HttpProxyRoute::Proxy("http://proxy.test:8080".to_owned())- );- config.set_http_no_proxy(Some(".example.test".to_owned()));- assert_eq!(- route(&config, "https://api.example.test/path", &[]),- HttpProxyRoute::Direct- );- }-- #[test]- fn empty_explicit_proxy_disables_environment_proxy_fallback() {- let mut config = FetchConfig::default();- config.set_http_proxy(Some(String::new()));-- assert_eq!(- route(- &config,- "http://example.test/path",- &[("http_proxy", "http://proxy.test:8080")],- ),- HttpProxyRoute::Direct- );- }-- #[test]- fn environment_proxy_and_no_proxy_follow_expected_precedence() {- let config = FetchConfig::default();-- assert_eq!(- route(- &config,- "http://example.test/path",- &[("http_proxy", "http://proxy.test:8080")],- ),- HttpProxyRoute::Proxy("http://proxy.test:8080".to_owned())- );- assert_eq!(- route(- &config,- "https://example.test/path",- &[("HTTPS_PROXY", "http://proxy.test:8080")],- ),- HttpProxyRoute::Proxy("http://proxy.test:8080".to_owned())- );- assert_eq!(- route(- &config,- "https://api.example.test/path",- &[- ("all_proxy", "http://proxy.test:8080"),- ("NO_PROXY", "example.test"),- ],- ),- HttpProxyRoute::Direct- );- }+pub(crate) use moli_curl::ProxyRoute as HttpProxyRoute;- #[test]- fn no_proxy_port_and_domain_boundaries_are_exact() {- assert!(no_proxy_matches(- "api.example.test",- 8443,- Some("example.test:8443")- ));- assert!(!no_proxy_matches(- "api.example.test",- 443,- Some("example.test:8443")- ));- assert!(!no_proxy_matches(- "notexample.test",- 443,- Some("example.test")- ));- assert!(no_proxy_matches("anything.test", 443, Some("*")));- assert!(no_proxy_matches(- "api.example.test.",- 443,- Some("example.test.")- ));- assert!(no_proxy_matches("192.0.2.42", 80, Some("192.0.2.0/24")));- assert!(!no_proxy_matches("198.51.100.42", 80, Some("192.0.2.0/24")));- assert!(no_proxy_matches("[::1]", 8443, Some("[::1]:8443")));- }+pub(crate) fn resolve_http_proxy_route(config: &FetchConfig, url: &Url) -> Result<HttpProxyRoute> {+ moli_curl::select_proxy_route(url, config.http_proxy(), config.http_no_proxy())}diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex 5611feb51..e7c27dd3b 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -896,7 +896,10 @@ impl RuntimeOwner {.buffered_mut().expect("buffered request should use buffered collector").begin_request(self.config.http_max_response_size());- let proxy_route = resolve_http_proxy_route(&self.config, &job.current_url);+ let proxy_route = match resolve_http_proxy_route(&self.config, &job.current_url) {+ Ok(route) => route,+ Err(error) => return Err((job.response_tx, error)),+ };if let Err(error) = configure_network_observation(&mut easy,&job.request,@@ -1077,7 +1080,10 @@ impl RuntimeOwner {cookie_header.clone(),));- let proxy_route = resolve_http_proxy_route(&self.config, &job.current_url);+ let proxy_route = match resolve_http_proxy_route(&self.config, &job.current_url) {+ Ok(route) => route,+ Err(error) => return Err((Box::new(job), Some(easy), error)),+ };if let Err(error) = configure_network_observation(&mut easy,&job.request,@@ -1280,7 +1286,10 @@ impl RuntimeOwner {job.current_url.clone(),cookie_header.clone(),));- let proxy_route = resolve_http_proxy_route(&self.config, &job.current_url);+ let proxy_route = match resolve_http_proxy_route(&self.config, &job.current_url) {+ Ok(route) => route,+ Err(error) => return Err((Box::new(job), Some(easy), error)),+ };if let Err(error) = configure_network_observation(&mut easy,&job.request,diff --git a/moli-websocket/src/proxy.rs b/moli-websocket/src/proxy.rsindex cdad371f6..1fde6f7c1 100644--- a/moli-websocket/src/proxy.rs+++ b/moli-websocket/src/proxy.rs@@ -1,117 +1,30 @@+use moli_curl::ProxyRoute;use url::Url;use crate::ConnectOptions;-pub(crate) fn websocket_proxy_url(+pub(crate) fn websocket_proxy_route(url: &Url,context: &ConnectOptions,-) -> Result<Option<Url>, String> {- websocket_proxy_url_with_env(url, context, |name| std::env::var(name).ok())+) -> Result<ProxyRoute, String> {+ moli_curl::select_proxy_route(+ url,+ context.http_proxy.as_deref(),+ context.http_no_proxy.as_deref(),+ )+ .map_err(|error| error.to_string())}-pub(crate) fn websocket_proxy_url_with_env(+pub(crate) fn websocket_proxy_route_with_env(url: &Url,context: &ConnectOptions,- mut env: impl FnMut(&str) -> Option<String>,-) -> Result<Option<Url>, String> {- let proxy = match context.http_proxy.as_deref() {- Some("") => return Ok(None),- Some(proxy) => Some(proxy.to_owned()),- None => websocket_env_proxy_for_scheme(url.scheme(), &mut env),- };- let Some(proxy) = proxy.filter(|proxy| !proxy.is_empty()) else {- return Ok(None);- };- let host = url- .host_str()- .ok_or_else(|| "WebSocket URL is missing host".to_owned())?;- let no_proxy = match context.http_no_proxy.as_deref() {- Some(no_proxy) => Some(no_proxy.to_owned()),- None => websocket_env_no_proxy(&mut env),- };- if no_proxy_matches(host, url.port(), no_proxy.as_deref()) {- return Ok(None);- }- let proxy_url = Url::parse(&proxy)- .map_err(|error| format!("failed to parse WebSocket proxy URL `{proxy}`: {error}"))?;- if proxy_url.scheme() != "http" {- return Err(format!(- "unsupported WebSocket proxy scheme `{}`; only http proxies are supported",- proxy_url.scheme()- ));- }- if proxy_url.host_str().is_none() {- return Err("WebSocket proxy URL is missing host".to_owned());- }- Ok(Some(proxy_url))-}--fn websocket_env_proxy_for_scheme(- scheme: &str,- env: &mut impl FnMut(&str) -> Option<String>,-) -> Option<String> {- let mut names: &[&str] = match scheme {- "ws" => &["http_proxy"],- "wss" => &["https_proxy", "HTTPS_PROXY"],- _ => &[],- };- for name in names {- if let Some(value) = env(name).filter(|value| !value.is_empty()) {- return Some(value);- }- }- names = &["all_proxy", "ALL_PROXY"];- for name in names {- if let Some(value) = env(name).filter(|value| !value.is_empty()) {- return Some(value);- }- }- None-}--fn websocket_env_no_proxy(env: &mut impl FnMut(&str) -> Option<String>) -> Option<String> {- env("no_proxy")- .filter(|value| !value.is_empty())- .or_else(|| env("NO_PROXY").filter(|value| !value.is_empty()))-}--pub(crate) fn no_proxy_matches(host: &str, port: Option<u16>, no_proxy: Option<&str>) -> bool {- let Some(no_proxy) = no_proxy else {- return false;- };- let host = host.trim_matches(['[', ']']).to_ascii_lowercase();- no_proxy.split(',').any(|token| {- let token = token.trim();- if token.is_empty() {- return false;- }- if token == "*" {- return true;- }- let (token_host, token_port) = split_no_proxy_host_port(token);- if let Some(token_port) = token_port- && Some(token_port) != port- {- return false;- }- let token_host = token_host- .trim_matches(['[', ']'])- .trim_start_matches('.')- .to_ascii_lowercase();- !token_host.is_empty()- && (host == token_host- || host- .strip_suffix(&token_host)- .is_some_and(|prefix| prefix.ends_with('.')))- })-}--fn split_no_proxy_host_port(token: &str) -> (&str, Option<u16>) {- let Some((host, port)) = token.rsplit_once(':') else {- return (token, None);- };- match port.parse::<u16>() {- Ok(port) if !host.contains(':') => (host, Some(port)),- _ => (token, None),- }+ env: impl FnMut(&str) -> Option<String>,+) -> Result<ProxyRoute, String> {+ moli_curl::select_proxy_route_with_env(+ url,+ context.http_proxy.as_deref(),+ context.http_no_proxy.as_deref(),+ env,+ )+ .map_err(|error| error.to_string())}diff --git a/moli-websocket/src/test_support.rs b/moli-websocket/src/test_support.rsindex b341d34de..30eb2d31d 100644--- a/moli-websocket/src/test_support.rs+++ b/moli-websocket/src/test_support.rs@@ -16,7 +16,7 @@ use tokio_tungstenite::tungstenite::{};use crate::{- ConnectOptions, Event, FrameOpcode, proxy::websocket_proxy_url_with_env,+ ConnectOptions, Event, FrameOpcode, proxy::websocket_proxy_route_with_env,spawn_standalone_connection,};@@ -49,12 +49,13 @@ pub fn test_websocket_proxy_url_with_env(context: &ConnectOptions,env: &[(&str, &str)],) -> Option<String> {- websocket_proxy_url_with_env(url, context, |name| {+ websocket_proxy_route_with_env(url, context, |name| {env.iter().find_map(|(env_name, value)| (*env_name == name).then(|| (*value).to_owned()))}).expect("websocket proxy url should resolve")- .map(|url| url.to_string())+ .proxy()+ .map(|proxy| proxy.url().to_owned())}pub async fn websocket_raw_handshake_failure_message(diff --git a/moli-websocket/src/tests.rs b/moli-websocket/src/tests.rsindex cc0ebd854..6004befe0 100644--- a/moli-websocket/src/tests.rs+++ b/moli-websocket/src/tests.rs@@ -6,7 +6,7 @@ use crate::{MAX_PENDING_WEBSOCKET_HANDSHAKES, MAX_WEBSOCKET_CONNECTIONS_PER_RUNTIME,acquire_limited_websocket_slot,},- proxy::no_proxy_matches,+ proxy::websocket_proxy_route_with_env,request::prepare_websocket_request,test_support::*,};@@ -206,32 +206,6 @@ fn websocket_request_preparation_rejects_blocked_ports() {assert!(prepare_websocket_request("ws://127.0.0.1:43210/socket", &[], &context).is_ok());}-#[test]-fn websocket_no_proxy_matches_hosts_domains_ports_and_wildcard() {- assert!(no_proxy_matches("example.com", None, Some("example.com")));- assert!(no_proxy_matches(- "api.example.com",- None,- Some(".example.com")- ));- assert!(no_proxy_matches(- "api.example.com",- Some(8080),- Some("example.com:8080")- ));- assert!(no_proxy_matches("anything.test", None, Some("*")));- assert!(!no_proxy_matches(- "api.example.com",- Some(8081),- Some("example.com:8080")- ));- assert!(!no_proxy_matches(- "notexample.com",- None,- Some("example.com")- ));-}-#[test]fn websocket_proxy_url_uses_env_http_proxy_for_ws_when_unset() {let context = test_websocket_context();@@ -314,6 +288,17 @@ fn websocket_proxy_url_explicit_empty_proxy_disables_env_fallback() {assert_eq!(proxy, None);}+#[test]+fn websocket_proxy_route_rejects_local_dns_socks_schemes() {+ let uri = "wss://target.test/socket".parse().unwrap();+ for scheme in ["socks5", "socks4"] {+ let mut context = test_websocket_context();+ context.http_proxy = Some(format!("{scheme}://proxy.test:1080"));+ let error = websocket_proxy_route_with_env(&uri, &context, |_| None).unwrap_err();+ assert!(error.contains("remote-DNS proxy scheme"), "{error}");+ }+}+#[test]fn websocket_request_preparation_applies_context_protocols_and_cookie() {let mut context = test_websocket_context();diff --git a/moli-websocket/src/transport.rs b/moli-websocket/src/transport.rsindex 2e209cf73..545b164ba 100644--- a/moli-websocket/src/transport.rs+++ b/moli-websocket/src/transport.rs@@ -2,7 +2,7 @@ use crate::{ConnectOptions,handshake::{HandshakeResponse, parse_handshake_response, validate_handshake_response},headers::header_map_entries,- proxy::websocket_proxy_url,+ proxy::websocket_proxy_route,request::PreparedWebSocketRequest,};use moli_curl::{@@ -28,21 +28,25 @@ pub(crate) async fn open_websocket_connection(request: PreparedWebSocketRequest,context: &ConnectOptions,) -> Result<OpenedConnection, String> {- let proxy = websocket_proxy_url(&request.url, context)?;+ let proxy_route = websocket_proxy_route(&request.url, context)?;let mut native = CurlWebSocketRequest::new(request.url.to_string());native.headers = header_map_entries(&request.headers);- native.proxy = proxy.map(|url| url.to_string());+ native.proxy = proxy_route.proxy().map(|proxy| proxy.url().to_owned());// WebSocket opening handshakes use credentials=include, including across// origins: https://websockets.spec.whatwg.org/#opening-handshakenative.tls = context.tls.clone();- if native.proxy.is_some() {- native- .proxy_headers- .push(("User-Agent".to_owned(), context.user_agent.clone()));- if let Some(token) = &context.proxy_bearer_token {+ if let Some(proxy) = proxy_route.proxy() {+ if proxy.scheme().uses_http_headers() {native.proxy_headers- .push(("Proxy-Authorization".to_owned(), format!("Bearer {token}")));+ .push(("User-Agent".to_owned(), context.user_agent.clone()));+ if let Some(token) = &context.proxy_bearer_token {+ native+ .proxy_headers+ .push(("Proxy-Authorization".to_owned(), format!("Bearer {token}")));+ }+ } else if context.proxy_bearer_token.is_some() {+ return Err("proxy bearer authentication requires an HTTP(S) proxy".to_owned());}} else {let url = &request.url;