Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

refactor: share proxy route selection

ldm05417fe675e1e
diff --git a/moli-curl/src/lib.rs b/moli-curl/src/lib.rs
index 337b3431c..b2a246eaa 100644
--- a/moli-curl/src/lib.rs
+++ b/moli-curl/src/lib.rs
@@ -3,6 +3,7 @@
mod dns_adapter;
mod http;
mod network_policy;
+mod proxy;
mod runtime;
mod tls;
pub mod websocket;
@@ -10,5 +11,9 @@ pub mod websocket;
pub use dns_adapter::CurlDnsResolution;
pub use http::{CurlHttpSender, CurlMultiCompletion, CurlMultiJob, CurlOriginKey, CurlSubmitError};
pub use network_policy::NetworkAddressPolicy;
+pub use proxy::{
+ ProxyRoute, ProxyScheme, ProxyTargetResolution, SelectedProxy, select_proxy_route,
+ select_proxy_route_with_env,
+};
pub use runtime::{CurlMultiRuntime, CurlMultiRuntimeConfig, CurlTransferId};
pub use tls::CurlTlsConfig;
diff --git a/moli-curl/src/proxy.rs b/moli-curl/src/proxy.rs
new file mode 100644
index 000000000..22045b575
--- /dev/null
+++ b/moli-curl/src/proxy.rs
@@ -0,0 +1,418 @@
+use std::net::IpAddr;
+
+use anyhow::{Context, Result, bail};
+use cidr::AnyIpCidr;
+use url::{Host, Url};
+
+/// How an accepted proxy resolves the request target hostname.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ProxyTargetResolution {
+ /// The target hostname is sent to the proxy without a local DNS lookup.
+ Proxy,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ProxyScheme {
+ Http,
+ Https,
+ Socks5h,
+ Socks4a,
+}
+
+impl ProxyScheme {
+ pub fn uses_http_headers(self) -> bool {
+ matches!(self, Self::Http | Self::Https)
+ }
+
+ fn default_port(self) -> u16 {
+ match self {
+ Self::Http => 80,
+ Self::Https => 443,
+ Self::Socks5h | Self::Socks4a => 1080,
+ }
+ }
+}
+
+/// Parsed proxy selected for a single request target.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SelectedProxy {
+ url: Url,
+ scheme: ProxyScheme,
+ endpoint_port: u16,
+}
+
+impl SelectedProxy {
+ pub fn parse(raw: &str) -> Result<Self> {
+ let normalized = if raw.contains("://") {
+ raw.to_owned()
+ } else {
+ format!("http://{raw}")
+ };
+ let url = Url::parse(&normalized)
+ .with_context(|| format!("failed to parse proxy URL `{raw}`"))?;
+ let scheme = match url.scheme() {
+ "http" => ProxyScheme::Http,
+ "https" => ProxyScheme::Https,
+ "socks5h" => ProxyScheme::Socks5h,
+ "socks4a" => ProxyScheme::Socks4a,
+ "socks5" | "socks4" => bail!(
+ "unsupported proxy scheme `{}`; use a remote-DNS proxy scheme such as `socks5h://`",
+ url.scheme()
+ ),
+ scheme => bail!(
+ "unsupported proxy scheme `{scheme}`; expected http, https, socks5h, or socks4a"
+ ),
+ };
+ if url.host().is_none() {
+ bail!("proxy URL `{raw}` is missing a host");
+ }
+ let endpoint_port = url.port().unwrap_or_else(|| scheme.default_port());
+ Ok(Self {
+ url,
+ scheme,
+ endpoint_port,
+ })
+ }
+
+ pub fn url(&self) -> &str {
+ self.url.as_str()
+ }
+
+ pub fn scheme(&self) -> ProxyScheme {
+ self.scheme
+ }
+
+ pub fn endpoint_host(&self) -> &str {
+ self.url
+ .host_str()
+ .expect("selected proxy was validated with a host")
+ }
+
+ pub fn endpoint_ip(&self) -> Option<IpAddr> {
+ match self
+ .url
+ .host()
+ .expect("selected proxy was validated with a host")
+ {
+ Host::Ipv4(address) => Some(address.into()),
+ Host::Ipv6(address) => Some(address.into()),
+ Host::Domain(_) => None,
+ }
+ }
+
+ pub fn endpoint_port(&self) -> u16 {
+ self.endpoint_port
+ }
+
+ pub fn target_resolution(&self) -> ProxyTargetResolution {
+ ProxyTargetResolution::Proxy
+ }
+}
+
+/// Concrete route selected exactly once for a request target.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ProxyRoute {
+ Direct,
+ Proxy(SelectedProxy),
+}
+
+impl ProxyRoute {
+ pub fn from_proxy_url(raw: &str) -> Result<Self> {
+ Ok(Self::Proxy(SelectedProxy::parse(raw)?))
+ }
+
+ pub fn is_proxy(&self) -> bool {
+ matches!(self, Self::Proxy(_))
+ }
+
+ pub fn proxy(&self) -> Option<&SelectedProxy> {
+ match self {
+ Self::Direct => None,
+ Self::Proxy(proxy) => Some(proxy),
+ }
+ }
+}
+
+pub fn select_proxy_route(
+ target: &Url,
+ configured_proxy: Option<&str>,
+ configured_no_proxy: Option<&str>,
+) -> Result<ProxyRoute> {
+ select_proxy_route_with_env(target, configured_proxy, configured_no_proxy, |name| {
+ std::env::var(name).ok()
+ })
+}
+
+pub fn select_proxy_route_with_env(
+ target: &Url,
+ configured_proxy: Option<&str>,
+ configured_no_proxy: Option<&str>,
+ mut env: impl FnMut(&str) -> Option<String>,
+) -> Result<ProxyRoute> {
+ let proxy = match configured_proxy {
+ Some("") => return Ok(ProxyRoute::Direct),
+ Some(proxy) => Some(proxy.to_owned()),
+ None => env_proxy_for_scheme(target.scheme(), &mut env),
+ };
+ let Some(proxy) = proxy.filter(|proxy| !proxy.is_empty()) else {
+ return Ok(ProxyRoute::Direct);
+ };
+
+ let no_proxy = match configured_no_proxy {
+ Some(no_proxy) => Some(no_proxy.to_owned()),
+ None => env_no_proxy(&mut env),
+ };
+ if let (Some(host), Some(port)) = (target.host_str(), target.port_or_known_default())
+ && no_proxy_matches(host, port, no_proxy.as_deref())
+ {
+ return Ok(ProxyRoute::Direct);
+ }
+
+ ProxyRoute::from_proxy_url(&proxy)
+}
+
+fn env_proxy_for_scheme(
+ scheme: &str,
+ env: &mut impl FnMut(&str) -> Option<String>,
+) -> Option<String> {
+ let names: &[&str] = match scheme {
+ // curl deliberately ignores uppercase HTTP_PROXY because CGI servers
+ // commonly expose an attacker-controlled Proxy header under that name.
+ "http" | "ws" => &["http_proxy"],
+ "https" | "wss" => &["https_proxy", "HTTPS_PROXY"],
+ _ => &[],
+ };
+ for name in names {
+ if let Some(value) = env(name).filter(|value| !value.is_empty()) {
+ return Some(value);
+ }
+ }
+ for name in ["all_proxy", "ALL_PROXY"] {
+ if let Some(value) = env(name).filter(|value| !value.is_empty()) {
+ return Some(value);
+ }
+ }
+ None
+}
+
+fn env_no_proxy(env: &mut impl FnMut(&str) -> Option<String>) -> Option<String> {
+ env("no_proxy")
+ .filter(|value| !value.is_empty())
+ .or_else(|| env("NO_PROXY").filter(|value| !value.is_empty()))
+}
+
+fn no_proxy_matches(host: &str, port: u16, no_proxy: Option<&str>) -> bool {
+ let Some(no_proxy) = no_proxy else {
+ return false;
+ };
+ let host = host
+ .trim_matches(['[', ']'])
+ .trim_end_matches('.')
+ .to_ascii_lowercase();
+ let host_ip = host.parse::<IpAddr>().ok();
+ no_proxy.split(',').any(|token| {
+ let token = token.trim();
+ if token.is_empty() {
+ return false;
+ }
+ if token == "*" {
+ return true;
+ }
+ let (token_host, token_port) = split_no_proxy_host_port(token);
+ if let Some(token_port) = token_port
+ && token_port != port
+ {
+ return false;
+ }
+ let token_host = token_host
+ .trim_matches(['[', ']'])
+ .trim_start_matches('.')
+ .trim_end_matches('.')
+ .to_ascii_lowercase();
+ if token_host.is_empty() {
+ return false;
+ }
+ if let Some(host_ip) = host_ip {
+ return token_host.parse::<IpAddr>().is_ok_and(|ip| ip == host_ip)
+ || token_host
+ .parse::<AnyIpCidr>()
+ .is_ok_and(|cidr| cidr.contains(&host_ip));
+ }
+ host == token_host
+ || host
+ .strip_suffix(&token_host)
+ .is_some_and(|prefix| prefix.ends_with('.'))
+ })
+}
+
+fn split_no_proxy_host_port(token: &str) -> (&str, Option<u16>) {
+ if let Some(bracketed) = token.strip_prefix('[')
+ && let Some((host, port)) = bracketed.rsplit_once("]:")
+ && let Ok(port) = port.parse::<u16>()
+ {
+ return (host, Some(port));
+ }
+ let Some((host, port)) = token.rsplit_once(':') else {
+ return (token, None);
+ };
+ match port.parse::<u16>() {
+ Ok(port) if !host.contains(':') => (host, Some(port)),
+ _ => (token, None),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::HashMap;
+
+ use super::*;
+
+ fn route(
+ raw_target: &str,
+ configured_proxy: Option<&str>,
+ configured_no_proxy: Option<&str>,
+ env: &[(&str, &str)],
+ ) -> Result<ProxyRoute> {
+ let env = env
+ .iter()
+ .map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))
+ .collect::<HashMap<_, _>>();
+ select_proxy_route_with_env(
+ &Url::parse(raw_target).expect("test URL should parse"),
+ configured_proxy,
+ configured_no_proxy,
+ |name| env.get(name).cloned(),
+ )
+ }
+
+ #[test]
+ fn explicit_proxy_and_no_proxy_select_one_route() {
+ assert_eq!(
+ route(
+ "https://api.example.test/path",
+ Some("http://proxy.test:8080"),
+ None,
+ &[],
+ )
+ .unwrap(),
+ ProxyRoute::from_proxy_url("http://proxy.test:8080").unwrap()
+ );
+ assert_eq!(
+ route(
+ "https://api.example.test/path",
+ Some("http://proxy.test:8080"),
+ Some(".example.test"),
+ &[],
+ )
+ .unwrap(),
+ ProxyRoute::Direct
+ );
+ }
+
+ #[test]
+ fn environment_proxy_selection_is_shared_by_http_and_websocket_schemes() {
+ assert!(
+ route(
+ "http://example.test/path",
+ None,
+ None,
+ &[("http_proxy", "http://proxy.test:8080")],
+ )
+ .unwrap()
+ .is_proxy()
+ );
+ assert!(
+ route(
+ "ws://example.test/socket",
+ None,
+ None,
+ &[("http_proxy", "http://proxy.test:8080")],
+ )
+ .unwrap()
+ .is_proxy()
+ );
+ assert!(
+ route(
+ "wss://example.test/socket",
+ None,
+ None,
+ &[("HTTPS_PROXY", "http://proxy.test:8080")],
+ )
+ .unwrap()
+ .is_proxy()
+ );
+ assert_eq!(
+ route(
+ "ws://example.test/socket",
+ None,
+ None,
+ &[("HTTP_PROXY", "http://proxy.test:8080")],
+ )
+ .unwrap(),
+ ProxyRoute::Direct
+ );
+ }
+
+ #[test]
+ fn empty_explicit_proxy_disables_environment_proxy_fallback() {
+ assert_eq!(
+ route(
+ "http://example.test/path",
+ Some(""),
+ None,
+ &[("http_proxy", "http://proxy.test:8080")],
+ )
+ .unwrap(),
+ ProxyRoute::Direct
+ );
+ }
+
+ #[test]
+ fn no_proxy_handles_domains_ports_ip_cidrs_and_trailing_dots() {
+ assert!(no_proxy_matches(
+ "api.example.test",
+ 8443,
+ Some("example.test:8443")
+ ));
+ assert!(!no_proxy_matches(
+ "api.example.test",
+ 443,
+ Some("example.test:8443")
+ ));
+ assert!(!no_proxy_matches(
+ "notexample.test",
+ 443,
+ Some("example.test")
+ ));
+ assert!(no_proxy_matches("anything.test", 443, Some("*")));
+ assert!(no_proxy_matches(
+ "api.example.test.",
+ 443,
+ Some("example.test.")
+ ));
+ assert!(no_proxy_matches("192.0.2.42", 80, Some("192.0.2.0/24")));
+ assert!(!no_proxy_matches("198.51.100.42", 80, Some("192.0.2.0/24")));
+ assert!(no_proxy_matches("[::1]", 8443, Some("[::1]:8443")));
+ }
+
+ #[test]
+ fn proxy_parser_accepts_only_remote_dns_schemes() {
+ for (raw, scheme, port) in [
+ ("proxy.test:8080", ProxyScheme::Http, 8080),
+ ("https://proxy.test", ProxyScheme::Https, 443),
+ ("socks5h://proxy.test", ProxyScheme::Socks5h, 1080),
+ ("socks4a://proxy.test", ProxyScheme::Socks4a, 1080),
+ ] {
+ let proxy = SelectedProxy::parse(raw).unwrap();
+ assert_eq!(proxy.scheme(), scheme);
+ assert_eq!(proxy.endpoint_host(), "proxy.test");
+ assert_eq!(proxy.endpoint_port(), port);
+ assert_eq!(proxy.target_resolution(), ProxyTargetResolution::Proxy);
+ }
+
+ for scheme in ["socks5", "socks4"] {
+ let error = SelectedProxy::parse(&format!("{scheme}://proxy.test:1080")).unwrap_err();
+ assert!(error.to_string().contains("remote-DNS proxy scheme"));
+ }
+ }
+}
diff --git a/moli-curl/src/websocket/request.rs b/moli-curl/src/websocket/request.rs
index 4157a8b54..6b42bb7bb 100644
--- a/moli-curl/src/websocket/request.rs
+++ b/moli-curl/src/websocket/request.rs
@@ -84,7 +84,7 @@ pub(super) fn configure(request: &CurlWebSocketRequest) -> Result<Easy2<Handshak
easy.proxy(request.proxy.as_deref().unwrap_or(""))?;
// The browser has already applied no_proxy; do not evaluate environment policy twice.
easy.noproxy("")?;
- easy.http_proxy_tunnel(request.proxy.is_some())?;
+ easy.http_proxy_tunnel(request.proxy.as_deref().is_some_and(proxy_uses_http_tunnel))?;
easy.separate_proxy_headers(true)?;
easy.http_headers(headers(&request.headers)?)?;
easy.proxy_headers(headers(&request.proxy_headers)?)?;
@@ -93,6 +93,13 @@ pub(super) fn configure(request: &CurlWebSocketRequest) -> Result<Easy2<Handshak
Ok(easy)
}
+fn proxy_uses_http_tunnel(proxy: &str) -> bool {
+ let Some((scheme, _)) = proxy.split_once("://") else {
+ return true;
+ };
+ scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https")
+}
+
fn headers(entries: &[(String, String)]) -> Result<List> {
let mut list = List::new();
let mut size = 0usize;
diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs
index bd26c5b29..7235084cd 100644
--- a/moli-fetch/src/blocking/mod.rs
+++ b/moli-fetch/src/blocking/mod.rs
@@ -639,8 +639,8 @@ pub(crate) fn configure_easy<H: Handler>(
.proxy("")
.context("failed to disable the HTTP proxy for a direct request")?,
HttpProxyRoute::Proxy(proxy) => easy
- .proxy(proxy)
- .with_context(|| anyhow!("failed to configure HTTP proxy `{proxy}`"))?,
+ .proxy(proxy.url())
+ .with_context(|| anyhow!("failed to configure HTTP proxy `{}`", proxy.url()))?,
}
// Proxy bypass has already been resolved into `proxy_route`. Keep curl
// from re-evaluating environment `no_proxy` after that security decision.
diff --git a/moli-fetch/src/dns.rs b/moli-fetch/src/dns.rs
index 94f00b403..39dadcde5 100644
--- a/moli-fetch/src/dns.rs
+++ b/moli-fetch/src/dns.rs
@@ -99,6 +99,10 @@ mod tests {
FetchCurlDnsAdmission::SharedResolver(DnsTarget::new(host, port))
}
+ fn proxy_route(raw: &str) -> HttpProxyRoute {
+ HttpProxyRoute::from_proxy_url(raw).expect("test proxy URL should parse")
+ }
+
#[test]
fn direct_http_and_https_domains_use_shared_resolution() {
let config = FetchConfig::default();
@@ -149,7 +153,7 @@ mod tests {
admission(
&config,
"https://api.example.test/path",
- &HttpProxyRoute::Proxy("http://proxy.test:8080".to_owned()),
+ &proxy_route("http://proxy.test:8080"),
),
FetchCurlDnsAdmission::NoSharedResolution
);
@@ -163,7 +167,7 @@ mod tests {
let error = admission_result(
&config,
"https://api.example.test/path",
- &HttpProxyRoute::Proxy("http://proxy.test:8080".to_owned()),
+ &proxy_route("http://proxy.test:8080"),
)
.expect_err("a strict policy cannot verify proxy-side DNS");
diff --git a/moli-fetch/src/proxy.rs b/moli-fetch/src/proxy.rs
index 06bbccba2..d623387ae 100644
--- a/moli-fetch/src/proxy.rs
+++ b/moli-fetch/src/proxy.rs
@@ -1,254 +1,10 @@
-use std::net::IpAddr;
-
-use cidr::AnyIpCidr;
+use anyhow::Result;
use url::Url;
use crate::FetchConfig;
-/// Concrete proxy route selected once for one HTTP request target.
-///
-/// The same value must drive both DNS admission and curl configuration so
-/// environment variables and `no_proxy` cannot be interpreted twice.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) enum HttpProxyRoute {
- Direct,
- Proxy(String),
-}
-
-impl HttpProxyRoute {
- pub(crate) fn is_proxy(&self) -> bool {
- matches!(self, Self::Proxy(_))
- }
-}
-
-pub(crate) fn resolve_http_proxy_route(config: &FetchConfig, url: &Url) -> HttpProxyRoute {
- resolve_http_proxy_route_with_env(config, url, |name| std::env::var(name).ok())
-}
-
-fn resolve_http_proxy_route_with_env(
- config: &FetchConfig,
- url: &Url,
- mut env: impl FnMut(&str) -> Option<String>,
-) -> HttpProxyRoute {
- let proxy = match config.http_proxy() {
- Some("") => return HttpProxyRoute::Direct,
- Some(proxy) => Some(proxy.to_owned()),
- None => env_proxy_for_scheme(url.scheme(), &mut env),
- };
- let Some(proxy) = proxy.filter(|proxy| !proxy.is_empty()) else {
- return HttpProxyRoute::Direct;
- };
-
- let no_proxy = match config.http_no_proxy() {
- Some(no_proxy) => Some(no_proxy.to_owned()),
- None => env_no_proxy(&mut env),
- };
- if let (Some(host), Some(port)) = (url.host_str(), url.port_or_known_default())
- && no_proxy_matches(host, port, no_proxy.as_deref())
- {
- return HttpProxyRoute::Direct;
- }
-
- HttpProxyRoute::Proxy(proxy)
-}
-
-fn env_proxy_for_scheme(
- scheme: &str,
- env: &mut impl FnMut(&str) -> Option<String>,
-) -> Option<String> {
- let names: &[&str] = match scheme {
- // curl deliberately ignores uppercase HTTP_PROXY because CGI servers
- // commonly expose an attacker-controlled Proxy header under that name.
- "http" => &["http_proxy"],
- "https" => &["https_proxy", "HTTPS_PROXY"],
- _ => &[],
- };
- for name in names {
- if let Some(value) = env(name).filter(|value| !value.is_empty()) {
- return Some(value);
- }
- }
- for name in ["all_proxy", "ALL_PROXY"] {
- if let Some(value) = env(name).filter(|value| !value.is_empty()) {
- return Some(value);
- }
- }
- None
-}
-
-fn env_no_proxy(env: &mut impl FnMut(&str) -> Option<String>) -> Option<String> {
- env("no_proxy")
- .filter(|value| !value.is_empty())
- .or_else(|| env("NO_PROXY").filter(|value| !value.is_empty()))
-}
-
-fn no_proxy_matches(host: &str, port: u16, no_proxy: Option<&str>) -> bool {
- let Some(no_proxy) = no_proxy else {
- return false;
- };
- let host = host
- .trim_matches(['[', ']'])
- .trim_end_matches('.')
- .to_ascii_lowercase();
- let host_ip = host.parse::<IpAddr>().ok();
- no_proxy.split(',').any(|token| {
- let token = token.trim();
- if token.is_empty() {
- return false;
- }
- if token == "*" {
- return true;
- }
- let (token_host, token_port) = split_no_proxy_host_port(token);
- if let Some(token_port) = token_port
- && token_port != port
- {
- return false;
- }
- let token_host = token_host
- .trim_matches(['[', ']'])
- .trim_start_matches('.')
- .trim_end_matches('.')
- .to_ascii_lowercase();
- if token_host.is_empty() {
- return false;
- }
- if let Some(host_ip) = host_ip {
- return token_host.parse::<IpAddr>().is_ok_and(|ip| ip == host_ip)
- || token_host
- .parse::<AnyIpCidr>()
- .is_ok_and(|cidr| cidr.contains(&host_ip));
- }
- host == token_host
- || host
- .strip_suffix(&token_host)
- .is_some_and(|prefix| prefix.ends_with('.'))
- })
-}
-
-fn split_no_proxy_host_port(token: &str) -> (&str, Option<u16>) {
- if let Some(bracketed) = token.strip_prefix('[')
- && let Some((host, port)) = bracketed.rsplit_once("]:")
- && let Ok(port) = port.parse::<u16>()
- {
- return (host, Some(port));
- }
- let Some((host, port)) = token.rsplit_once(':') else {
- return (token, None);
- };
- match port.parse::<u16>() {
- Ok(port) if !host.contains(':') => (host, Some(port)),
- _ => (token, None),
- }
-}
-
-#[cfg(test)]
-mod tests {
- use std::collections::HashMap;
-
- use super::*;
-
- fn route(config: &FetchConfig, raw_url: &str, env: &[(&str, &str)]) -> HttpProxyRoute {
- let env = env
- .iter()
- .map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))
- .collect::<HashMap<_, _>>();
- resolve_http_proxy_route_with_env(
- config,
- &Url::parse(raw_url).expect("test URL should parse"),
- |name| env.get(name).cloned(),
- )
- }
-
- #[test]
- fn explicit_proxy_and_no_proxy_select_one_route() {
- let mut config = FetchConfig::default();
- config.set_http_proxy(Some("http://proxy.test:8080".to_owned()));
-
- assert_eq!(
- route(&config, "https://api.example.test/path", &[]),
- HttpProxyRoute::Proxy("http://proxy.test:8080".to_owned())
- );
- config.set_http_no_proxy(Some(".example.test".to_owned()));
- assert_eq!(
- route(&config, "https://api.example.test/path", &[]),
- HttpProxyRoute::Direct
- );
- }
-
- #[test]
- fn empty_explicit_proxy_disables_environment_proxy_fallback() {
- let mut config = FetchConfig::default();
- config.set_http_proxy(Some(String::new()));
-
- assert_eq!(
- route(
- &config,
- "http://example.test/path",
- &[("http_proxy", "http://proxy.test:8080")],
- ),
- HttpProxyRoute::Direct
- );
- }
-
- #[test]
- fn environment_proxy_and_no_proxy_follow_expected_precedence() {
- let config = FetchConfig::default();
-
- assert_eq!(
- route(
- &config,
- "http://example.test/path",
- &[("http_proxy", "http://proxy.test:8080")],
- ),
- HttpProxyRoute::Proxy("http://proxy.test:8080".to_owned())
- );
- assert_eq!(
- route(
- &config,
- "https://example.test/path",
- &[("HTTPS_PROXY", "http://proxy.test:8080")],
- ),
- HttpProxyRoute::Proxy("http://proxy.test:8080".to_owned())
- );
- assert_eq!(
- route(
- &config,
- "https://api.example.test/path",
- &[
- ("all_proxy", "http://proxy.test:8080"),
- ("NO_PROXY", "example.test"),
- ],
- ),
- HttpProxyRoute::Direct
- );
- }
+pub(crate) use moli_curl::ProxyRoute as HttpProxyRoute;
- #[test]
- fn no_proxy_port_and_domain_boundaries_are_exact() {
- assert!(no_proxy_matches(
- "api.example.test",
- 8443,
- Some("example.test:8443")
- ));
- assert!(!no_proxy_matches(
- "api.example.test",
- 443,
- Some("example.test:8443")
- ));
- assert!(!no_proxy_matches(
- "notexample.test",
- 443,
- Some("example.test")
- ));
- assert!(no_proxy_matches("anything.test", 443, Some("*")));
- assert!(no_proxy_matches(
- "api.example.test.",
- 443,
- Some("example.test.")
- ));
- assert!(no_proxy_matches("192.0.2.42", 80, Some("192.0.2.0/24")));
- assert!(!no_proxy_matches("198.51.100.42", 80, Some("192.0.2.0/24")));
- assert!(no_proxy_matches("[::1]", 8443, Some("[::1]:8443")));
- }
+pub(crate) fn resolve_http_proxy_route(config: &FetchConfig, url: &Url) -> Result<HttpProxyRoute> {
+ moli_curl::select_proxy_route(url, config.http_proxy(), config.http_no_proxy())
}
diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs
index 5611feb51..e7c27dd3b 100644
--- a/moli-fetch/src/runtime.rs
+++ b/moli-fetch/src/runtime.rs
@@ -896,7 +896,10 @@ impl RuntimeOwner {
.buffered_mut()
.expect("buffered request should use buffered collector")
.begin_request(self.config.http_max_response_size());
- let proxy_route = resolve_http_proxy_route(&self.config, &job.current_url);
+ let proxy_route = match resolve_http_proxy_route(&self.config, &job.current_url) {
+ Ok(route) => route,
+ Err(error) => return Err((job.response_tx, error)),
+ };
if let Err(error) = configure_network_observation(
&mut easy,
&job.request,
@@ -1077,7 +1080,10 @@ impl RuntimeOwner {
cookie_header.clone(),
));
- let proxy_route = resolve_http_proxy_route(&self.config, &job.current_url);
+ let proxy_route = match resolve_http_proxy_route(&self.config, &job.current_url) {
+ Ok(route) => route,
+ Err(error) => return Err((Box::new(job), Some(easy), error)),
+ };
if let Err(error) = configure_network_observation(
&mut easy,
&job.request,
@@ -1280,7 +1286,10 @@ impl RuntimeOwner {
job.current_url.clone(),
cookie_header.clone(),
));
- let proxy_route = resolve_http_proxy_route(&self.config, &job.current_url);
+ let proxy_route = match resolve_http_proxy_route(&self.config, &job.current_url) {
+ Ok(route) => route,
+ Err(error) => return Err((Box::new(job), Some(easy), error)),
+ };
if let Err(error) = configure_network_observation(
&mut easy,
&job.request,
diff --git a/moli-websocket/src/proxy.rs b/moli-websocket/src/proxy.rs
index cdad371f6..1fde6f7c1 100644
--- a/moli-websocket/src/proxy.rs
+++ b/moli-websocket/src/proxy.rs
@@ -1,117 +1,30 @@
+use moli_curl::ProxyRoute;
use url::Url;
use crate::ConnectOptions;
-pub(crate) fn websocket_proxy_url(
+pub(crate) fn websocket_proxy_route(
url: &Url,
context: &ConnectOptions,
-) -> Result<Option<Url>, String> {
- websocket_proxy_url_with_env(url, context, |name| std::env::var(name).ok())
+) -> Result<ProxyRoute, String> {
+ moli_curl::select_proxy_route(
+ url,
+ context.http_proxy.as_deref(),
+ context.http_no_proxy.as_deref(),
+ )
+ .map_err(|error| error.to_string())
}
-pub(crate) fn websocket_proxy_url_with_env(
+pub(crate) fn websocket_proxy_route_with_env(
url: &Url,
context: &ConnectOptions,
- mut env: impl FnMut(&str) -> Option<String>,
-) -> Result<Option<Url>, String> {
- let proxy = match context.http_proxy.as_deref() {
- Some("") => return Ok(None),
- Some(proxy) => Some(proxy.to_owned()),
- None => websocket_env_proxy_for_scheme(url.scheme(), &mut env),
- };
- let Some(proxy) = proxy.filter(|proxy| !proxy.is_empty()) else {
- return Ok(None);
- };
- let host = url
- .host_str()
- .ok_or_else(|| "WebSocket URL is missing host".to_owned())?;
- let no_proxy = match context.http_no_proxy.as_deref() {
- Some(no_proxy) => Some(no_proxy.to_owned()),
- None => websocket_env_no_proxy(&mut env),
- };
- if no_proxy_matches(host, url.port(), no_proxy.as_deref()) {
- return Ok(None);
- }
- let proxy_url = Url::parse(&proxy)
- .map_err(|error| format!("failed to parse WebSocket proxy URL `{proxy}`: {error}"))?;
- if proxy_url.scheme() != "http" {
- return Err(format!(
- "unsupported WebSocket proxy scheme `{}`; only http proxies are supported",
- proxy_url.scheme()
- ));
- }
- if proxy_url.host_str().is_none() {
- return Err("WebSocket proxy URL is missing host".to_owned());
- }
- Ok(Some(proxy_url))
-}
-
-fn websocket_env_proxy_for_scheme(
- scheme: &str,
- env: &mut impl FnMut(&str) -> Option<String>,
-) -> Option<String> {
- let mut names: &[&str] = match scheme {
- "ws" => &["http_proxy"],
- "wss" => &["https_proxy", "HTTPS_PROXY"],
- _ => &[],
- };
- for name in names {
- if let Some(value) = env(name).filter(|value| !value.is_empty()) {
- return Some(value);
- }
- }
- names = &["all_proxy", "ALL_PROXY"];
- for name in names {
- if let Some(value) = env(name).filter(|value| !value.is_empty()) {
- return Some(value);
- }
- }
- None
-}
-
-fn websocket_env_no_proxy(env: &mut impl FnMut(&str) -> Option<String>) -> Option<String> {
- env("no_proxy")
- .filter(|value| !value.is_empty())
- .or_else(|| env("NO_PROXY").filter(|value| !value.is_empty()))
-}
-
-pub(crate) fn no_proxy_matches(host: &str, port: Option<u16>, no_proxy: Option<&str>) -> bool {
- let Some(no_proxy) = no_proxy else {
- return false;
- };
- let host = host.trim_matches(['[', ']']).to_ascii_lowercase();
- no_proxy.split(',').any(|token| {
- let token = token.trim();
- if token.is_empty() {
- return false;
- }
- if token == "*" {
- return true;
- }
- let (token_host, token_port) = split_no_proxy_host_port(token);
- if let Some(token_port) = token_port
- && Some(token_port) != port
- {
- return false;
- }
- let token_host = token_host
- .trim_matches(['[', ']'])
- .trim_start_matches('.')
- .to_ascii_lowercase();
- !token_host.is_empty()
- && (host == token_host
- || host
- .strip_suffix(&token_host)
- .is_some_and(|prefix| prefix.ends_with('.')))
- })
-}
-
-fn split_no_proxy_host_port(token: &str) -> (&str, Option<u16>) {
- let Some((host, port)) = token.rsplit_once(':') else {
- return (token, None);
- };
- match port.parse::<u16>() {
- Ok(port) if !host.contains(':') => (host, Some(port)),
- _ => (token, None),
- }
+ env: impl FnMut(&str) -> Option<String>,
+) -> Result<ProxyRoute, String> {
+ moli_curl::select_proxy_route_with_env(
+ url,
+ context.http_proxy.as_deref(),
+ context.http_no_proxy.as_deref(),
+ env,
+ )
+ .map_err(|error| error.to_string())
}
diff --git a/moli-websocket/src/test_support.rs b/moli-websocket/src/test_support.rs
index b341d34de..30eb2d31d 100644
--- a/moli-websocket/src/test_support.rs
+++ b/moli-websocket/src/test_support.rs
@@ -16,7 +16,7 @@ use tokio_tungstenite::tungstenite::{
};
use crate::{
- ConnectOptions, Event, FrameOpcode, proxy::websocket_proxy_url_with_env,
+ ConnectOptions, Event, FrameOpcode, proxy::websocket_proxy_route_with_env,
spawn_standalone_connection,
};
@@ -49,12 +49,13 @@ pub fn test_websocket_proxy_url_with_env(
context: &ConnectOptions,
env: &[(&str, &str)],
) -> Option<String> {
- websocket_proxy_url_with_env(url, context, |name| {
+ websocket_proxy_route_with_env(url, context, |name| {
env.iter()
.find_map(|(env_name, value)| (*env_name == name).then(|| (*value).to_owned()))
})
.expect("websocket proxy url should resolve")
- .map(|url| url.to_string())
+ .proxy()
+ .map(|proxy| proxy.url().to_owned())
}
pub async fn websocket_raw_handshake_failure_message(
diff --git a/moli-websocket/src/tests.rs b/moli-websocket/src/tests.rs
index cc0ebd854..6004befe0 100644
--- a/moli-websocket/src/tests.rs
+++ b/moli-websocket/src/tests.rs
@@ -6,7 +6,7 @@ use crate::{
MAX_PENDING_WEBSOCKET_HANDSHAKES, MAX_WEBSOCKET_CONNECTIONS_PER_RUNTIME,
acquire_limited_websocket_slot,
},
- proxy::no_proxy_matches,
+ proxy::websocket_proxy_route_with_env,
request::prepare_websocket_request,
test_support::*,
};
@@ -206,32 +206,6 @@ fn websocket_request_preparation_rejects_blocked_ports() {
assert!(prepare_websocket_request("ws://127.0.0.1:43210/socket", &[], &context).is_ok());
}
-#[test]
-fn websocket_no_proxy_matches_hosts_domains_ports_and_wildcard() {
- assert!(no_proxy_matches("example.com", None, Some("example.com")));
- assert!(no_proxy_matches(
- "api.example.com",
- None,
- Some(".example.com")
- ));
- assert!(no_proxy_matches(
- "api.example.com",
- Some(8080),
- Some("example.com:8080")
- ));
- assert!(no_proxy_matches("anything.test", None, Some("*")));
- assert!(!no_proxy_matches(
- "api.example.com",
- Some(8081),
- Some("example.com:8080")
- ));
- assert!(!no_proxy_matches(
- "notexample.com",
- None,
- Some("example.com")
- ));
-}
-
#[test]
fn websocket_proxy_url_uses_env_http_proxy_for_ws_when_unset() {
let context = test_websocket_context();
@@ -314,6 +288,17 @@ fn websocket_proxy_url_explicit_empty_proxy_disables_env_fallback() {
assert_eq!(proxy, None);
}
+#[test]
+fn websocket_proxy_route_rejects_local_dns_socks_schemes() {
+ let uri = "wss://target.test/socket".parse().unwrap();
+ for scheme in ["socks5", "socks4"] {
+ let mut context = test_websocket_context();
+ context.http_proxy = Some(format!("{scheme}://proxy.test:1080"));
+ let error = websocket_proxy_route_with_env(&uri, &context, |_| None).unwrap_err();
+ assert!(error.contains("remote-DNS proxy scheme"), "{error}");
+ }
+}
+
#[test]
fn websocket_request_preparation_applies_context_protocols_and_cookie() {
let mut context = test_websocket_context();
diff --git a/moli-websocket/src/transport.rs b/moli-websocket/src/transport.rs
index 2e209cf73..545b164ba 100644
--- a/moli-websocket/src/transport.rs
+++ b/moli-websocket/src/transport.rs
@@ -2,7 +2,7 @@ use crate::{
ConnectOptions,
handshake::{HandshakeResponse, parse_handshake_response, validate_handshake_response},
headers::header_map_entries,
- proxy::websocket_proxy_url,
+ proxy::websocket_proxy_route,
request::PreparedWebSocketRequest,
};
use moli_curl::{
@@ -28,21 +28,25 @@ pub(crate) async fn open_websocket_connection(
request: PreparedWebSocketRequest,
context: &ConnectOptions,
) -> Result<OpenedConnection, String> {
- let proxy = websocket_proxy_url(&request.url, context)?;
+ let proxy_route = websocket_proxy_route(&request.url, context)?;
let mut native = CurlWebSocketRequest::new(request.url.to_string());
native.headers = header_map_entries(&request.headers);
- native.proxy = proxy.map(|url| url.to_string());
+ native.proxy = proxy_route.proxy().map(|proxy| proxy.url().to_owned());
// WebSocket opening handshakes use credentials=include, including across
// origins: https://websockets.spec.whatwg.org/#opening-handshake
native.tls = context.tls.clone();
- if native.proxy.is_some() {
- native
- .proxy_headers
- .push(("User-Agent".to_owned(), context.user_agent.clone()));
- if let Some(token) = &context.proxy_bearer_token {
+ if let Some(proxy) = proxy_route.proxy() {
+ if proxy.scheme().uses_http_headers() {
native
.proxy_headers
- .push(("Proxy-Authorization".to_owned(), format!("Bearer {token}")));
+ .push(("User-Agent".to_owned(), context.user_agent.clone()));
+ if let Some(token) = &context.proxy_bearer_token {
+ native
+ .proxy_headers
+ .push(("Proxy-Authorization".to_owned(), format!("Bearer {token}")));
+ }
+ } else if context.proxy_bearer_token.is_some() {
+ return Err("proxy bearer authentication requires an HTTP(S) proxy".to_owned());
}
} else {
let url = &request.url;