Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix: match Chromium SOCKS proxy semantics

ldm07184290d9f7c
diff --git a/moli-curl/src/proxy.rs b/moli-curl/src/proxy.rs
index 196382cdf..97b057e96 100644
--- a/moli-curl/src/proxy.rs
+++ b/moli-curl/src/proxy.rs
@@ -61,7 +61,10 @@ impl ProxyScheme {
/// Parsed proxy selected for a single request target.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SelectedProxy {
+ /// Proxy URL as configured by the caller, retained for diagnostics.
url: Url,
+ /// Equivalent remote-DNS URL understood by libcurl.
+ curl_url: Url,
scheme: ProxyScheme,
endpoint_port: u16,
}
@@ -75,25 +78,26 @@ impl SelectedProxy {
};
let url = Url::parse(&normalized)
.with_context(|| format!("failed to parse proxy URL `{raw}`"))?;
- let scheme = match url.scheme() {
- "http" => ProxyScheme::Http,
- "https" => ProxyScheme::Https,
- "socks5h" => ProxyScheme::Socks5h,
- "socks4a" => ProxyScheme::Socks4a,
- "socks5" | "socks4" => bail!(
- "unsupported proxy scheme `{}`; use a remote-DNS proxy scheme such as `socks5h://`",
- url.scheme()
- ),
+ let (scheme, curl_scheme) = match url.scheme() {
+ "http" => (ProxyScheme::Http, "http"),
+ "https" => (ProxyScheme::Https, "https"),
+ "socks" | "socks5" | "socks5h" => (ProxyScheme::Socks5h, "socks5h"),
+ "socks4" | "socks4a" => (ProxyScheme::Socks4a, "socks4a"),
scheme => bail!(
- "unsupported proxy scheme `{scheme}`; expected http, https, socks5h, or socks4a"
+ "unsupported proxy scheme `{scheme}`; expected http, https, socks, socks5, socks5h, socks4, or socks4a"
),
};
if url.host().is_none() {
bail!("proxy URL `{raw}` is missing a host");
}
let endpoint_port = url.port().unwrap_or_else(|| scheme.default_port());
+ let mut curl_url = url.clone();
+ curl_url
+ .set_scheme(curl_scheme)
+ .map_err(|()| anyhow::anyhow!("failed to normalize proxy URL `{raw}` for curl"))?;
Ok(Self {
url,
+ curl_url,
scheme,
endpoint_port,
})
@@ -103,6 +107,11 @@ impl SelectedProxy {
self.url.as_str()
}
+ /// Proxy URL passed to libcurl after applying Moli's remote-DNS semantics.
+ pub fn curl_url(&self) -> &str {
+ self.curl_url.as_str()
+ }
+
pub fn scheme(&self) -> ProxyScheme {
self.scheme
}
@@ -483,23 +492,38 @@ mod tests {
}
#[test]
- fn proxy_parser_accepts_only_remote_dns_schemes() {
- for (raw, scheme, port) in [
- ("proxy.test:8080", ProxyScheme::Http, 8080),
- ("https://proxy.test", ProxyScheme::Https, 443),
- ("socks5h://proxy.test", ProxyScheme::Socks5h, 1080),
- ("socks4a://proxy.test", ProxyScheme::Socks4a, 1080),
+ fn proxy_parser_normalizes_socks_schemes_to_remote_dns_for_curl() {
+ for (raw, scheme, port, curl_scheme) in [
+ ("proxy.test:8080", ProxyScheme::Http, 8080, "http"),
+ ("https://proxy.test", ProxyScheme::Https, 443, "https"),
+ ("socks://proxy.test", ProxyScheme::Socks5h, 1080, "socks5h"),
+ ("socks5://proxy.test", ProxyScheme::Socks5h, 1080, "socks5h"),
+ (
+ "socks5h://proxy.test",
+ ProxyScheme::Socks5h,
+ 1080,
+ "socks5h",
+ ),
+ ("socks4://proxy.test", ProxyScheme::Socks4a, 1080, "socks4a"),
+ (
+ "socks4a://proxy.test",
+ ProxyScheme::Socks4a,
+ 1080,
+ "socks4a",
+ ),
] {
let proxy = SelectedProxy::parse(raw).unwrap();
assert_eq!(proxy.scheme(), scheme);
assert_eq!(proxy.endpoint_host(), "proxy.test");
assert_eq!(proxy.endpoint_port(), port);
assert_eq!(proxy.target_resolution(), ProxyTargetResolution::Proxy);
- }
-
- for scheme in ["socks5", "socks4"] {
- let error = SelectedProxy::parse(&format!("{scheme}://proxy.test:1080")).unwrap_err();
- assert!(error.to_string().contains("remote-DNS proxy scheme"));
+ assert_eq!(Url::parse(proxy.curl_url()).unwrap().scheme(), curl_scheme);
+ if raw.contains("://") {
+ assert_eq!(
+ Url::parse(proxy.url()).unwrap().scheme(),
+ raw.split_once("://").unwrap().0
+ );
+ }
}
}
diff --git a/moli-curl/src/websocket/connector.rs b/moli-curl/src/websocket/connector.rs
index 298a56bb4..5900af35e 100644
--- a/moli-curl/src/websocket/connector.rs
+++ b/moli-curl/src/websocket/connector.rs
@@ -104,14 +104,15 @@ impl CurlWebSocketConnector {
let url = Url::parse(&request.url)
.with_context(|| format!("failed to parse WebSocket URL `{}`", request.url))?;
// A validated remote-DNS proxy owns request-target admission. An empty
- // proxy string disables proxying in curl, while socks4/socks5 resolve
- // the target locally; neither may bypass direct-target admission.
+ // proxy string disables proxying in curl. Chromium-style socks/socks5
+ // and socks4 URLs are normalized to curl's remote-DNS variants here.
if let Some(proxy) = request.proxy.as_deref().filter(|proxy| !proxy.is_empty()) {
- SelectedProxy::parse(proxy).with_context(|| {
+ let selected = SelectedProxy::parse(proxy).with_context(|| {
format!(
"network address policy requires a supported remote-DNS WebSocket proxy, got `{proxy}`"
)
})?;
+ request.proxy = Some(selected.curl_url().to_owned());
return Ok(());
}
let host = url
diff --git a/moli-curl/src/websocket/tests.rs b/moli-curl/src/websocket/tests.rs
index 0ca049981..8534a349b 100644
--- a/moli-curl/src/websocket/tests.rs
+++ b/moli-curl/src/websocket/tests.rs
@@ -53,21 +53,18 @@ fn address_policy_does_not_treat_an_empty_proxy_as_remote_dns() {
}
#[test]
-fn address_policy_rejects_a_local_dns_websocket_proxy() {
+fn address_policy_accepts_chromium_socks5_semantics() {
let runtime = CurlWebSocketRuntime::new().unwrap();
let connector = runtime
.connector()
.with_network_address_policy(NetworkAddressPolicy::new(true, Vec::new()));
let mut request = CurlWebSocketRequest::new("ws://example.test/socket".to_owned());
- request.proxy = Some("socks5://127.0.0.1:1080".to_owned());
+ request.proxy = Some("socks5://127.0.0.1:1".to_owned());
- let error = connector
+ let connection = connector
.connect(request)
- .expect_err("a local-DNS proxy cannot satisfy request-target admission");
- assert!(
- format!("{error:#}").contains("supported remote-DNS WebSocket proxy"),
- "{error:#}"
- );
+ .expect("socks5 must be normalized to remote-DNS socks5h before curl");
+ drop(connection);
}
#[test]
diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs
index f4f52acd7..56801a3bd 100644
--- a/moli-fetch/src/blocking/mod.rs
+++ b/moli-fetch/src/blocking/mod.rs
@@ -625,7 +625,7 @@ pub(crate) fn configure_easy<H: Handler>(
.proxy("")
.context("failed to disable the HTTP proxy for a direct request")?,
HttpProxyRoute::Proxy(proxy) => {
- easy.proxy(proxy.url())
+ easy.proxy(proxy.curl_url())
.with_context(|| anyhow!("failed to configure HTTP proxy `{}`", proxy.url()))?;
configure_proxy_headers(easy, config, request, proxy.scheme().uses_http_headers())?;
}
diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rs
index 6ceff38ba..23da1018c 100644
--- a/moli-fetch/src/tests/mod.rs
+++ b/moli-fetch/src/tests/mod.rs
@@ -39,8 +39,8 @@ use crate::{
use self::support::{
EmptyHttpHttpsUpgradeServer, Http2ProtocolFallbackServer, ScriptedH2Server, ScriptedHttpServer,
- ScriptedHttps11Server, ScriptedResponse, spawn_socks5h_proxy, unique_test_cache_dir,
- wait_for_runtime_owner_count,
+ ScriptedHttps11Server, ScriptedResponse, spawn_socks4_proxy, spawn_socks5_proxy,
+ unique_test_cache_dir, wait_for_runtime_owner_count,
};
const ENV_PROXY_CHILD_TEST: &str = "MOLI_FETCH_ENV_PROXY_CHILD";
@@ -2007,7 +2007,7 @@ fn fetch_client_rejects_host_resolve_for_remote_proxy_target() {
fn fetch_client_rejects_http_header_auth_for_socks_proxy() {
let mut config = FetchConfig::default();
config.set_proxy_options(
- Some("socks5h://127.0.0.1:1".to_owned()),
+ Some("socks5://127.0.0.1:1".to_owned()),
Some("proxy-token".to_owned()),
);
@@ -2025,26 +2025,7 @@ fn fetch_client_rejects_http_header_auth_for_socks_proxy() {
}
#[test]
-fn fetch_client_rejects_local_dns_socks_proxy_schemes() {
- for scheme in ["socks5", "socks4"] {
- let mut config = FetchConfig::default();
- config.set_http_proxy(Some(format!("{scheme}://127.0.0.1:1080")));
-
- let error = fetch_with_config_for_test(
- &config,
- Request::get("http://target.invalid/through-socks").unwrap(),
- )
- .expect_err("high-level fetch must reject local-DNS SOCKS schemes");
- let error_chain = format!("{error:#}");
- assert!(
- error_chain.contains("remote-DNS proxy scheme"),
- "{error_chain}"
- );
- }
-}
-
-#[test]
-fn fetch_client_socks5h_sends_target_hostname_to_proxy() {
+fn fetch_client_socks5_sends_target_hostname_to_proxy() {
let upstream = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("through-socks")]);
let upstream_url = Url::parse(&upstream.origin()).unwrap();
let upstream_port = upstream_url
@@ -2053,7 +2034,7 @@ fn fetch_client_socks5h_sends_target_hostname_to_proxy() {
let upstream_addr = format!("127.0.0.1:{upstream_port}")
.parse()
.expect("upstream socket address");
- let (proxy_url, request_rx, proxy) = spawn_socks5h_proxy(upstream_addr);
+ let (proxy_url, request_rx, proxy) = spawn_socks5_proxy(upstream_addr);
let mut config = FetchConfig::default();
config.set_http_proxy(Some(proxy_url));
config.set_http_no_proxy(Some(String::new()));
@@ -2065,7 +2046,7 @@ fn fetch_client_socks5h_sends_target_hostname_to_proxy() {
))
.unwrap(),
)
- .expect("socks5h proxy should relay the HTTP request");
+ .expect("socks5 must use remote DNS and relay the HTTP request");
let (requested_host, requested_port) = request_rx
.recv_timeout(Duration::from_secs(3))
.expect("SOCKS request should arrive");
@@ -2077,6 +2058,40 @@ fn fetch_client_socks5h_sends_target_hostname_to_proxy() {
upstream.shutdown();
}
+#[test]
+fn fetch_client_socks4_sends_target_hostname_to_proxy() {
+ let upstream = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("through-socks4")]);
+ let upstream_url = Url::parse(&upstream.origin()).unwrap();
+ let upstream_port = upstream_url
+ .port()
+ .expect("upstream URL should include a port");
+ let upstream_addr = format!("127.0.0.1:{upstream_port}")
+ .parse()
+ .expect("upstream socket address");
+ let (proxy_url, request_rx, proxy) = spawn_socks4_proxy(upstream_addr);
+ let mut config = FetchConfig::default();
+ config.set_http_proxy(Some(proxy_url));
+ config.set_http_no_proxy(Some(String::new()));
+
+ let response = fetch_with_config_for_test(
+ &config,
+ Request::get(&format!(
+ "http://http-socks4-target.invalid:{upstream_port}/through-socks4"
+ ))
+ .unwrap(),
+ )
+ .expect("socks4 must use remote DNS and relay the HTTP request");
+ let (requested_host, requested_port) = request_rx
+ .recv_timeout(Duration::from_secs(3))
+ .expect("SOCKS4 request should arrive");
+ proxy.join().expect("fetch SOCKS4 proxy should finish");
+
+ assert_eq!(response.body_text(), "through-socks4");
+ assert_eq!(requested_host, "http-socks4-target.invalid");
+ assert_eq!(requested_port, upstream_port);
+ upstream.shutdown();
+}
+
#[test]
fn fetch_redirect_reselects_direct_or_proxy_endpoint_per_hop() {
let proxy = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("redirected-through-proxy")]);
diff --git a/moli-fetch/src/tests/support.rs b/moli-fetch/src/tests/support.rs
index 79a00751f..4f7d02611 100644
--- a/moli-fetch/src/tests/support.rs
+++ b/moli-fetch/src/tests/support.rs
@@ -14,7 +14,7 @@ use std::{
use crate::FetchClientHandle;
use parking_lot::Mutex;
-pub(super) fn spawn_socks5h_proxy(
+pub(super) fn spawn_socks5_proxy(
upstream_addr: std::net::SocketAddr,
) -> (
String,
@@ -57,7 +57,7 @@ pub(super) fn spawn_socks5h_proxy(
.expect("read fetch SOCKS domain");
String::from_utf8(domain).expect("fetch SOCKS domain should be UTF-8")
}
- atyp => panic!("socks5h must send a domain target, got address type {atyp}"),
+ atyp => panic!("SOCKS5 must send a domain target, got address type {atyp}"),
};
let mut port = [0u8; 2];
client
@@ -66,21 +66,74 @@ pub(super) fn spawn_socks5h_proxy(
let port = u16::from_be_bytes(port);
let _ = request_tx.send((host, port));
- let mut upstream = TcpStream::connect(upstream_addr).expect("connect fetch SOCKS upstream");
+ let upstream = TcpStream::connect(upstream_addr).expect("connect fetch SOCKS upstream");
client
.write_all(&[5, 0, 0, 1, 127, 0, 0, 1, 0, 0])
.expect("write fetch SOCKS success response");
- let mut client_read = client.try_clone().expect("clone fetch SOCKS client");
- let mut upstream_write = upstream.try_clone().expect("clone fetch SOCKS upstream");
- let upload = thread::spawn(move || {
- let _ = std::io::copy(&mut client_read, &mut upstream_write);
- });
- let _ = std::io::copy(&mut upstream, &mut client);
- let _ = client.shutdown(std::net::Shutdown::Both);
- let _ = upstream.shutdown(std::net::Shutdown::Both);
- let _ = upload.join();
+ relay_proxy_streams(client, upstream);
+ });
+ (format!("socks5://{addr}"), request_rx, handle)
+}
+
+pub(super) fn spawn_socks4_proxy(
+ upstream_addr: std::net::SocketAddr,
+) -> (
+ String,
+ std_mpsc::Receiver<(String, u16)>,
+ thread::JoinHandle<()>,
+) {
+ let listener = TcpListener::bind("127.0.0.1:0").expect("bind fetch SOCKS4 proxy");
+ let addr = listener.local_addr().expect("fetch SOCKS4 proxy address");
+ let (request_tx, request_rx) = std_mpsc::channel();
+ let handle = thread::spawn(move || {
+ let (mut client, _) = listener.accept().expect("accept fetch SOCKS4 client");
+ let mut request_head = [0u8; 8];
+ client
+ .read_exact(&mut request_head)
+ .expect("read fetch SOCKS4 request head");
+ assert_eq!(&request_head[..2], &[4, 1]);
+ assert_eq!(&request_head[4..7], &[0, 0, 0]);
+ assert_ne!(request_head[7], 0, "SOCKS4a must use a synthetic IP");
+ let port = u16::from_be_bytes([request_head[2], request_head[3]]);
+ let _user_id = read_socks4_string(&mut client, "user id");
+ let domain = read_socks4_string(&mut client, "domain");
+ let host = String::from_utf8(domain).expect("fetch SOCKS4 domain should be UTF-8");
+ let _ = request_tx.send((host, port));
+
+ let upstream = TcpStream::connect(upstream_addr).expect("connect fetch SOCKS4 upstream");
+ client
+ .write_all(&[0, 0x5a, 0, 0, 0, 0, 0, 0])
+ .expect("write fetch SOCKS4 success response");
+ relay_proxy_streams(client, upstream);
+ });
+ (format!("socks4://{addr}"), request_rx, handle)
+}
+
+fn read_socks4_string(stream: &mut TcpStream, label: &str) -> Vec<u8> {
+ let mut value = Vec::new();
+ loop {
+ let mut byte = [0u8; 1];
+ stream
+ .read_exact(&mut byte)
+ .unwrap_or_else(|error| panic!("read fetch SOCKS4 {label}: {error}"));
+ if byte[0] == 0 {
+ return value;
+ }
+ assert!(value.len() < 255, "fetch SOCKS4 {label} is too long");
+ value.push(byte[0]);
+ }
+}
+
+fn relay_proxy_streams(mut client: TcpStream, mut upstream: TcpStream) {
+ let mut client_read = client.try_clone().expect("clone fetch proxy client");
+ let mut upstream_write = upstream.try_clone().expect("clone fetch proxy upstream");
+ let upload = thread::spawn(move || {
+ let _ = std::io::copy(&mut client_read, &mut upstream_write);
});
- (format!("socks5h://{addr}"), request_rx, handle)
+ let _ = std::io::copy(&mut upstream, &mut client);
+ let _ = client.shutdown(std::net::Shutdown::Both);
+ let _ = upstream.shutdown(std::net::Shutdown::Both);
+ let _ = upload.join();
}
pub(super) struct ScriptedH2Server {
diff --git a/moli-websocket/src/test_support.rs b/moli-websocket/src/test_support.rs
index d5b518499..f84dda8c9 100644
--- a/moli-websocket/src/test_support.rs
+++ b/moli-websocket/src/test_support.rs
@@ -533,7 +533,7 @@ pub async fn spawn_http_connect_proxy_response(
(format!("http://{addr}"), request_rx, handle)
}
-pub async fn spawn_socks5h_proxy(
+pub async fn spawn_socks5_proxy(
upstream_addr: std::net::SocketAddr,
) -> (
String,
@@ -584,7 +584,7 @@ pub async fn spawn_socks5h_proxy(
.expect("read SOCKS domain");
String::from_utf8(domain).expect("SOCKS domain should be UTF-8")
}
- atyp => panic!("socks5h must send a domain target, got address type {atyp}"),
+ atyp => panic!("SOCKS5 must send a domain target, got address type {atyp}"),
};
let mut port = [0u8; 2];
client
@@ -603,7 +603,7 @@ pub async fn spawn_socks5h_proxy(
.expect("write SOCKS success response");
let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await;
});
- (format!("socks5h://{addr}"), request_rx, handle)
+ (format!("socks5://{addr}"), request_rx, handle)
}
pub async fn spawn_text_echo_websocket_server() -> (String, tokio::task::JoinHandle<()>) {
diff --git a/moli-websocket/src/tests.rs b/moli-websocket/src/tests.rs
index 6e9b1c7fc..7c336b248 100644
--- a/moli-websocket/src/tests.rs
+++ b/moli-websocket/src/tests.rs
@@ -289,13 +289,22 @@ fn websocket_proxy_url_explicit_empty_proxy_disables_env_fallback() {
}
#[test]
-fn websocket_proxy_route_rejects_local_dns_socks_schemes() {
+fn websocket_proxy_route_normalizes_chromium_socks_schemes() {
let uri = "wss://target.test/socket".parse().unwrap();
- for scheme in ["socks5", "socks4"] {
+ for (scheme, curl_scheme) in [
+ ("socks", "socks5h"),
+ ("socks5", "socks5h"),
+ ("socks4", "socks4a"),
+ ] {
let mut context = test_websocket_context();
context.http_proxy = Some(format!("{scheme}://proxy.test:1080"));
- let error = websocket_proxy_route_with_env(&uri, &context, |_| None).unwrap_err();
- assert!(error.contains("remote-DNS proxy scheme"), "{error}");
+ let route = websocket_proxy_route_with_env(&uri, &context, |_| None).unwrap();
+ let proxy = route.proxy().expect("SOCKS route should use a proxy");
+ assert_eq!(url::Url::parse(proxy.url()).unwrap().scheme(), scheme);
+ assert_eq!(
+ url::Url::parse(proxy.curl_url()).unwrap().scheme(),
+ curl_scheme
+ );
}
}
@@ -971,7 +980,7 @@ async fn websocket_transport_uses_explicit_http_proxy_connect_without_forwarding
}
#[tokio::test]
-async fn websocket_transport_socks5h_sends_target_hostname_to_proxy() {
+async fn websocket_transport_socks5_sends_target_hostname_to_proxy() {
let (server_url, headers_rx, server) = spawn_header_capture_websocket_server().await;
let mut target = Url::parse(&server_url).expect("WebSocket target URL");
let target_port = target
@@ -983,7 +992,7 @@ async fn websocket_transport_socks5h_sends_target_hostname_to_proxy() {
target
.set_host(Some("websocket-target.invalid"))
.expect("WebSocket target hostname should be replaceable");
- let (proxy_url, proxy_request_rx, proxy) = spawn_socks5h_proxy(upstream_addr).await;
+ let (proxy_url, proxy_request_rx, proxy) = spawn_socks5_proxy(upstream_addr).await;
let (event_tx, mut event_rx) = mpsc::channel(32);
let mut context = test_websocket_context();
context.http_proxy = Some(proxy_url);
diff --git a/moli-websocket/src/transport.rs b/moli-websocket/src/transport.rs
index 4919e46f3..7733ec4f8 100644
--- a/moli-websocket/src/transport.rs
+++ b/moli-websocket/src/transport.rs
@@ -36,7 +36,7 @@ pub(crate) async fn open_websocket_connection(
let resolve_entries = host_resolve.normalized_entries();
let mut native = CurlWebSocketRequest::new(request.url.to_string());
native.headers = header_map_entries(&request.headers);
- native.proxy = proxy_route.proxy().map(|proxy| proxy.url().to_owned());
+ native.proxy = proxy_route.proxy().map(|proxy| proxy.curl_url().to_owned());
native.resolve_entries = resolve_entries.clone();
// WebSocket opening handshakes use credentials=include, including across
// origins: https://websockets.spec.whatwg.org/#opening-handshake
diff --git a/moli/src/cli.rs b/moli/src/cli.rs
index 7550dc7fc..c605d13f1 100644
--- a/moli/src/cli.rs
+++ b/moli/src/cli.rs
@@ -484,9 +484,11 @@ pub struct CommonArgs {
#[arg(long)]
pub obey_robots: bool,
- /// Route HTTP(S) and WebSocket traffic through an HTTP, HTTPS, socks5h, or
- /// socks4a proxy. Moli resolves and pins the proxy endpoint; the proxy is
- /// responsible for resolving and filtering request target hostnames.
+ /// Route HTTP(S) and WebSocket traffic through an HTTP, HTTPS, SOCKS5, or
+ /// SOCKS4 proxy. SOCKS targets always use proxy-side DNS; `socks` is a
+ /// SOCKS5 alias, while `socks5h` and `socks4a` remain accepted. Moli
+ /// resolves and pins the proxy endpoint; the proxy is responsible for
+ /// resolving and filtering request target hostnames.
#[arg(long, value_name = "URL")]
pub http_proxy: Option<String>,