Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix: match Chromium SOCKS proxy semantics
ldm07184290d9f7c
diff --git a/moli-curl/src/proxy.rs b/moli-curl/src/proxy.rsindex 196382cdf..97b057e96 100644--- a/moli-curl/src/proxy.rs+++ b/moli-curl/src/proxy.rs@@ -61,7 +61,10 @@ impl ProxyScheme {/// Parsed proxy selected for a single request target.#[derive(Debug, Clone, PartialEq, Eq)]pub struct SelectedProxy {+ /// Proxy URL as configured by the caller, retained for diagnostics.url: Url,+ /// Equivalent remote-DNS URL understood by libcurl.+ curl_url: Url,scheme: ProxyScheme,endpoint_port: u16,}@@ -75,25 +78,26 @@ impl SelectedProxy {};let url = Url::parse(&normalized).with_context(|| format!("failed to parse proxy URL `{raw}`"))?;- let scheme = match url.scheme() {- "http" => ProxyScheme::Http,- "https" => ProxyScheme::Https,- "socks5h" => ProxyScheme::Socks5h,- "socks4a" => ProxyScheme::Socks4a,- "socks5" | "socks4" => bail!(- "unsupported proxy scheme `{}`; use a remote-DNS proxy scheme such as `socks5h://`",- url.scheme()- ),+ let (scheme, curl_scheme) = match url.scheme() {+ "http" => (ProxyScheme::Http, "http"),+ "https" => (ProxyScheme::Https, "https"),+ "socks" | "socks5" | "socks5h" => (ProxyScheme::Socks5h, "socks5h"),+ "socks4" | "socks4a" => (ProxyScheme::Socks4a, "socks4a"),scheme => bail!(- "unsupported proxy scheme `{scheme}`; expected http, https, socks5h, or socks4a"+ "unsupported proxy scheme `{scheme}`; expected http, https, socks, socks5, socks5h, socks4, or socks4a"),};if url.host().is_none() {bail!("proxy URL `{raw}` is missing a host");}let endpoint_port = url.port().unwrap_or_else(|| scheme.default_port());+ let mut curl_url = url.clone();+ curl_url+ .set_scheme(curl_scheme)+ .map_err(|()| anyhow::anyhow!("failed to normalize proxy URL `{raw}` for curl"))?;Ok(Self {url,+ curl_url,scheme,endpoint_port,})@@ -103,6 +107,11 @@ impl SelectedProxy {self.url.as_str()}+ /// Proxy URL passed to libcurl after applying Moli's remote-DNS semantics.+ pub fn curl_url(&self) -> &str {+ self.curl_url.as_str()+ }+pub fn scheme(&self) -> ProxyScheme {self.scheme}@@ -483,23 +492,38 @@ mod tests {}#[test]- fn proxy_parser_accepts_only_remote_dns_schemes() {- for (raw, scheme, port) in [- ("proxy.test:8080", ProxyScheme::Http, 8080),- ("https://proxy.test", ProxyScheme::Https, 443),- ("socks5h://proxy.test", ProxyScheme::Socks5h, 1080),- ("socks4a://proxy.test", ProxyScheme::Socks4a, 1080),+ fn proxy_parser_normalizes_socks_schemes_to_remote_dns_for_curl() {+ for (raw, scheme, port, curl_scheme) in [+ ("proxy.test:8080", ProxyScheme::Http, 8080, "http"),+ ("https://proxy.test", ProxyScheme::Https, 443, "https"),+ ("socks://proxy.test", ProxyScheme::Socks5h, 1080, "socks5h"),+ ("socks5://proxy.test", ProxyScheme::Socks5h, 1080, "socks5h"),+ (+ "socks5h://proxy.test",+ ProxyScheme::Socks5h,+ 1080,+ "socks5h",+ ),+ ("socks4://proxy.test", ProxyScheme::Socks4a, 1080, "socks4a"),+ (+ "socks4a://proxy.test",+ ProxyScheme::Socks4a,+ 1080,+ "socks4a",+ ),] {let proxy = SelectedProxy::parse(raw).unwrap();assert_eq!(proxy.scheme(), scheme);assert_eq!(proxy.endpoint_host(), "proxy.test");assert_eq!(proxy.endpoint_port(), port);assert_eq!(proxy.target_resolution(), ProxyTargetResolution::Proxy);- }-- for scheme in ["socks5", "socks4"] {- let error = SelectedProxy::parse(&format!("{scheme}://proxy.test:1080")).unwrap_err();- assert!(error.to_string().contains("remote-DNS proxy scheme"));+ assert_eq!(Url::parse(proxy.curl_url()).unwrap().scheme(), curl_scheme);+ if raw.contains("://") {+ assert_eq!(+ Url::parse(proxy.url()).unwrap().scheme(),+ raw.split_once("://").unwrap().0+ );+ }}}diff --git a/moli-curl/src/websocket/connector.rs b/moli-curl/src/websocket/connector.rsindex 298a56bb4..5900af35e 100644--- a/moli-curl/src/websocket/connector.rs+++ b/moli-curl/src/websocket/connector.rs@@ -104,14 +104,15 @@ impl CurlWebSocketConnector {let url = Url::parse(&request.url).with_context(|| format!("failed to parse WebSocket URL `{}`", request.url))?;// A validated remote-DNS proxy owns request-target admission. An empty- // proxy string disables proxying in curl, while socks4/socks5 resolve- // the target locally; neither may bypass direct-target admission.+ // proxy string disables proxying in curl. Chromium-style socks/socks5+ // and socks4 URLs are normalized to curl's remote-DNS variants here.if let Some(proxy) = request.proxy.as_deref().filter(|proxy| !proxy.is_empty()) {- SelectedProxy::parse(proxy).with_context(|| {+ let selected = SelectedProxy::parse(proxy).with_context(|| {format!("network address policy requires a supported remote-DNS WebSocket proxy, got `{proxy}`")})?;+ request.proxy = Some(selected.curl_url().to_owned());return Ok(());}let host = urldiff --git a/moli-curl/src/websocket/tests.rs b/moli-curl/src/websocket/tests.rsindex 0ca049981..8534a349b 100644--- a/moli-curl/src/websocket/tests.rs+++ b/moli-curl/src/websocket/tests.rs@@ -53,21 +53,18 @@ fn address_policy_does_not_treat_an_empty_proxy_as_remote_dns() {}#[test]-fn address_policy_rejects_a_local_dns_websocket_proxy() {+fn address_policy_accepts_chromium_socks5_semantics() {let runtime = CurlWebSocketRuntime::new().unwrap();let connector = runtime.connector().with_network_address_policy(NetworkAddressPolicy::new(true, Vec::new()));let mut request = CurlWebSocketRequest::new("ws://example.test/socket".to_owned());- request.proxy = Some("socks5://127.0.0.1:1080".to_owned());+ request.proxy = Some("socks5://127.0.0.1:1".to_owned());- let error = connector+ let connection = connector.connect(request)- .expect_err("a local-DNS proxy cannot satisfy request-target admission");- assert!(- format!("{error:#}").contains("supported remote-DNS WebSocket proxy"),- "{error:#}"- );+ .expect("socks5 must be normalized to remote-DNS socks5h before curl");+ drop(connection);}#[test]diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rsindex f4f52acd7..56801a3bd 100644--- a/moli-fetch/src/blocking/mod.rs+++ b/moli-fetch/src/blocking/mod.rs@@ -625,7 +625,7 @@ pub(crate) fn configure_easy<H: Handler>(.proxy("").context("failed to disable the HTTP proxy for a direct request")?,HttpProxyRoute::Proxy(proxy) => {- easy.proxy(proxy.url())+ easy.proxy(proxy.curl_url()).with_context(|| anyhow!("failed to configure HTTP proxy `{}`", proxy.url()))?;configure_proxy_headers(easy, config, request, proxy.scheme().uses_http_headers())?;}diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rsindex 6ceff38ba..23da1018c 100644--- a/moli-fetch/src/tests/mod.rs+++ b/moli-fetch/src/tests/mod.rs@@ -39,8 +39,8 @@ use crate::{use self::support::{EmptyHttpHttpsUpgradeServer, Http2ProtocolFallbackServer, ScriptedH2Server, ScriptedHttpServer,- ScriptedHttps11Server, ScriptedResponse, spawn_socks5h_proxy, unique_test_cache_dir,- wait_for_runtime_owner_count,+ ScriptedHttps11Server, ScriptedResponse, spawn_socks4_proxy, spawn_socks5_proxy,+ unique_test_cache_dir, wait_for_runtime_owner_count,};const ENV_PROXY_CHILD_TEST: &str = "MOLI_FETCH_ENV_PROXY_CHILD";@@ -2007,7 +2007,7 @@ fn fetch_client_rejects_host_resolve_for_remote_proxy_target() {fn fetch_client_rejects_http_header_auth_for_socks_proxy() {let mut config = FetchConfig::default();config.set_proxy_options(- Some("socks5h://127.0.0.1:1".to_owned()),+ Some("socks5://127.0.0.1:1".to_owned()),Some("proxy-token".to_owned()),);@@ -2025,26 +2025,7 @@ fn fetch_client_rejects_http_header_auth_for_socks_proxy() {}#[test]-fn fetch_client_rejects_local_dns_socks_proxy_schemes() {- for scheme in ["socks5", "socks4"] {- let mut config = FetchConfig::default();- config.set_http_proxy(Some(format!("{scheme}://127.0.0.1:1080")));-- let error = fetch_with_config_for_test(- &config,- Request::get("http://target.invalid/through-socks").unwrap(),- )- .expect_err("high-level fetch must reject local-DNS SOCKS schemes");- let error_chain = format!("{error:#}");- assert!(- error_chain.contains("remote-DNS proxy scheme"),- "{error_chain}"- );- }-}--#[test]-fn fetch_client_socks5h_sends_target_hostname_to_proxy() {+fn fetch_client_socks5_sends_target_hostname_to_proxy() {let upstream = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("through-socks")]);let upstream_url = Url::parse(&upstream.origin()).unwrap();let upstream_port = upstream_url@@ -2053,7 +2034,7 @@ fn fetch_client_socks5h_sends_target_hostname_to_proxy() {let upstream_addr = format!("127.0.0.1:{upstream_port}").parse().expect("upstream socket address");- let (proxy_url, request_rx, proxy) = spawn_socks5h_proxy(upstream_addr);+ let (proxy_url, request_rx, proxy) = spawn_socks5_proxy(upstream_addr);let mut config = FetchConfig::default();config.set_http_proxy(Some(proxy_url));config.set_http_no_proxy(Some(String::new()));@@ -2065,7 +2046,7 @@ fn fetch_client_socks5h_sends_target_hostname_to_proxy() {)).unwrap(),)- .expect("socks5h proxy should relay the HTTP request");+ .expect("socks5 must use remote DNS and relay the HTTP request");let (requested_host, requested_port) = request_rx.recv_timeout(Duration::from_secs(3)).expect("SOCKS request should arrive");@@ -2077,6 +2058,40 @@ fn fetch_client_socks5h_sends_target_hostname_to_proxy() {upstream.shutdown();}+#[test]+fn fetch_client_socks4_sends_target_hostname_to_proxy() {+ let upstream = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("through-socks4")]);+ let upstream_url = Url::parse(&upstream.origin()).unwrap();+ let upstream_port = upstream_url+ .port()+ .expect("upstream URL should include a port");+ let upstream_addr = format!("127.0.0.1:{upstream_port}")+ .parse()+ .expect("upstream socket address");+ let (proxy_url, request_rx, proxy) = spawn_socks4_proxy(upstream_addr);+ let mut config = FetchConfig::default();+ config.set_http_proxy(Some(proxy_url));+ config.set_http_no_proxy(Some(String::new()));++ let response = fetch_with_config_for_test(+ &config,+ Request::get(&format!(+ "http://http-socks4-target.invalid:{upstream_port}/through-socks4"+ ))+ .unwrap(),+ )+ .expect("socks4 must use remote DNS and relay the HTTP request");+ let (requested_host, requested_port) = request_rx+ .recv_timeout(Duration::from_secs(3))+ .expect("SOCKS4 request should arrive");+ proxy.join().expect("fetch SOCKS4 proxy should finish");++ assert_eq!(response.body_text(), "through-socks4");+ assert_eq!(requested_host, "http-socks4-target.invalid");+ assert_eq!(requested_port, upstream_port);+ upstream.shutdown();+}+#[test]fn fetch_redirect_reselects_direct_or_proxy_endpoint_per_hop() {let proxy = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("redirected-through-proxy")]);diff --git a/moli-fetch/src/tests/support.rs b/moli-fetch/src/tests/support.rsindex 79a00751f..4f7d02611 100644--- a/moli-fetch/src/tests/support.rs+++ b/moli-fetch/src/tests/support.rs@@ -14,7 +14,7 @@ use std::{use crate::FetchClientHandle;use parking_lot::Mutex;-pub(super) fn spawn_socks5h_proxy(+pub(super) fn spawn_socks5_proxy(upstream_addr: std::net::SocketAddr,) -> (String,@@ -57,7 +57,7 @@ pub(super) fn spawn_socks5h_proxy(.expect("read fetch SOCKS domain");String::from_utf8(domain).expect("fetch SOCKS domain should be UTF-8")}- atyp => panic!("socks5h must send a domain target, got address type {atyp}"),+ atyp => panic!("SOCKS5 must send a domain target, got address type {atyp}"),};let mut port = [0u8; 2];client@@ -66,21 +66,74 @@ pub(super) fn spawn_socks5h_proxy(let port = u16::from_be_bytes(port);let _ = request_tx.send((host, port));- let mut upstream = TcpStream::connect(upstream_addr).expect("connect fetch SOCKS upstream");+ let upstream = TcpStream::connect(upstream_addr).expect("connect fetch SOCKS upstream");client.write_all(&[5, 0, 0, 1, 127, 0, 0, 1, 0, 0]).expect("write fetch SOCKS success response");- let mut client_read = client.try_clone().expect("clone fetch SOCKS client");- let mut upstream_write = upstream.try_clone().expect("clone fetch SOCKS upstream");- let upload = thread::spawn(move || {- let _ = std::io::copy(&mut client_read, &mut upstream_write);- });- let _ = std::io::copy(&mut upstream, &mut client);- let _ = client.shutdown(std::net::Shutdown::Both);- let _ = upstream.shutdown(std::net::Shutdown::Both);- let _ = upload.join();+ relay_proxy_streams(client, upstream);+ });+ (format!("socks5://{addr}"), request_rx, handle)+}++pub(super) fn spawn_socks4_proxy(+ upstream_addr: std::net::SocketAddr,+) -> (+ String,+ std_mpsc::Receiver<(String, u16)>,+ thread::JoinHandle<()>,+) {+ let listener = TcpListener::bind("127.0.0.1:0").expect("bind fetch SOCKS4 proxy");+ let addr = listener.local_addr().expect("fetch SOCKS4 proxy address");+ let (request_tx, request_rx) = std_mpsc::channel();+ let handle = thread::spawn(move || {+ let (mut client, _) = listener.accept().expect("accept fetch SOCKS4 client");+ let mut request_head = [0u8; 8];+ client+ .read_exact(&mut request_head)+ .expect("read fetch SOCKS4 request head");+ assert_eq!(&request_head[..2], &[4, 1]);+ assert_eq!(&request_head[4..7], &[0, 0, 0]);+ assert_ne!(request_head[7], 0, "SOCKS4a must use a synthetic IP");+ let port = u16::from_be_bytes([request_head[2], request_head[3]]);+ let _user_id = read_socks4_string(&mut client, "user id");+ let domain = read_socks4_string(&mut client, "domain");+ let host = String::from_utf8(domain).expect("fetch SOCKS4 domain should be UTF-8");+ let _ = request_tx.send((host, port));++ let upstream = TcpStream::connect(upstream_addr).expect("connect fetch SOCKS4 upstream");+ client+ .write_all(&[0, 0x5a, 0, 0, 0, 0, 0, 0])+ .expect("write fetch SOCKS4 success response");+ relay_proxy_streams(client, upstream);+ });+ (format!("socks4://{addr}"), request_rx, handle)+}++fn read_socks4_string(stream: &mut TcpStream, label: &str) -> Vec<u8> {+ let mut value = Vec::new();+ loop {+ let mut byte = [0u8; 1];+ stream+ .read_exact(&mut byte)+ .unwrap_or_else(|error| panic!("read fetch SOCKS4 {label}: {error}"));+ if byte[0] == 0 {+ return value;+ }+ assert!(value.len() < 255, "fetch SOCKS4 {label} is too long");+ value.push(byte[0]);+ }+}++fn relay_proxy_streams(mut client: TcpStream, mut upstream: TcpStream) {+ let mut client_read = client.try_clone().expect("clone fetch proxy client");+ let mut upstream_write = upstream.try_clone().expect("clone fetch proxy upstream");+ let upload = thread::spawn(move || {+ let _ = std::io::copy(&mut client_read, &mut upstream_write);});- (format!("socks5h://{addr}"), request_rx, handle)+ let _ = std::io::copy(&mut upstream, &mut client);+ let _ = client.shutdown(std::net::Shutdown::Both);+ let _ = upstream.shutdown(std::net::Shutdown::Both);+ let _ = upload.join();}pub(super) struct ScriptedH2Server {diff --git a/moli-websocket/src/test_support.rs b/moli-websocket/src/test_support.rsindex d5b518499..f84dda8c9 100644--- a/moli-websocket/src/test_support.rs+++ b/moli-websocket/src/test_support.rs@@ -533,7 +533,7 @@ pub async fn spawn_http_connect_proxy_response((format!("http://{addr}"), request_rx, handle)}-pub async fn spawn_socks5h_proxy(+pub async fn spawn_socks5_proxy(upstream_addr: std::net::SocketAddr,) -> (String,@@ -584,7 +584,7 @@ pub async fn spawn_socks5h_proxy(.expect("read SOCKS domain");String::from_utf8(domain).expect("SOCKS domain should be UTF-8")}- atyp => panic!("socks5h must send a domain target, got address type {atyp}"),+ atyp => panic!("SOCKS5 must send a domain target, got address type {atyp}"),};let mut port = [0u8; 2];client@@ -603,7 +603,7 @@ pub async fn spawn_socks5h_proxy(.expect("write SOCKS success response");let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await;});- (format!("socks5h://{addr}"), request_rx, handle)+ (format!("socks5://{addr}"), request_rx, handle)}pub async fn spawn_text_echo_websocket_server() -> (String, tokio::task::JoinHandle<()>) {diff --git a/moli-websocket/src/tests.rs b/moli-websocket/src/tests.rsindex 6e9b1c7fc..7c336b248 100644--- a/moli-websocket/src/tests.rs+++ b/moli-websocket/src/tests.rs@@ -289,13 +289,22 @@ fn websocket_proxy_url_explicit_empty_proxy_disables_env_fallback() {}#[test]-fn websocket_proxy_route_rejects_local_dns_socks_schemes() {+fn websocket_proxy_route_normalizes_chromium_socks_schemes() {let uri = "wss://target.test/socket".parse().unwrap();- for scheme in ["socks5", "socks4"] {+ for (scheme, curl_scheme) in [+ ("socks", "socks5h"),+ ("socks5", "socks5h"),+ ("socks4", "socks4a"),+ ] {let mut context = test_websocket_context();context.http_proxy = Some(format!("{scheme}://proxy.test:1080"));- let error = websocket_proxy_route_with_env(&uri, &context, |_| None).unwrap_err();- assert!(error.contains("remote-DNS proxy scheme"), "{error}");+ let route = websocket_proxy_route_with_env(&uri, &context, |_| None).unwrap();+ let proxy = route.proxy().expect("SOCKS route should use a proxy");+ assert_eq!(url::Url::parse(proxy.url()).unwrap().scheme(), scheme);+ assert_eq!(+ url::Url::parse(proxy.curl_url()).unwrap().scheme(),+ curl_scheme+ );}}@@ -971,7 +980,7 @@ async fn websocket_transport_uses_explicit_http_proxy_connect_without_forwarding}#[tokio::test]-async fn websocket_transport_socks5h_sends_target_hostname_to_proxy() {+async fn websocket_transport_socks5_sends_target_hostname_to_proxy() {let (server_url, headers_rx, server) = spawn_header_capture_websocket_server().await;let mut target = Url::parse(&server_url).expect("WebSocket target URL");let target_port = target@@ -983,7 +992,7 @@ async fn websocket_transport_socks5h_sends_target_hostname_to_proxy() {target.set_host(Some("websocket-target.invalid")).expect("WebSocket target hostname should be replaceable");- let (proxy_url, proxy_request_rx, proxy) = spawn_socks5h_proxy(upstream_addr).await;+ let (proxy_url, proxy_request_rx, proxy) = spawn_socks5_proxy(upstream_addr).await;let (event_tx, mut event_rx) = mpsc::channel(32);let mut context = test_websocket_context();context.http_proxy = Some(proxy_url);diff --git a/moli-websocket/src/transport.rs b/moli-websocket/src/transport.rsindex 4919e46f3..7733ec4f8 100644--- a/moli-websocket/src/transport.rs+++ b/moli-websocket/src/transport.rs@@ -36,7 +36,7 @@ pub(crate) async fn open_websocket_connection(let resolve_entries = host_resolve.normalized_entries();let mut native = CurlWebSocketRequest::new(request.url.to_string());native.headers = header_map_entries(&request.headers);- native.proxy = proxy_route.proxy().map(|proxy| proxy.url().to_owned());+ native.proxy = proxy_route.proxy().map(|proxy| proxy.curl_url().to_owned());native.resolve_entries = resolve_entries.clone();// WebSocket opening handshakes use credentials=include, including across// origins: https://websockets.spec.whatwg.org/#opening-handshakediff --git a/moli/src/cli.rs b/moli/src/cli.rsindex 7550dc7fc..c605d13f1 100644--- a/moli/src/cli.rs+++ b/moli/src/cli.rs@@ -484,9 +484,11 @@ pub struct CommonArgs {#[arg(long)]pub obey_robots: bool,- /// Route HTTP(S) and WebSocket traffic through an HTTP, HTTPS, socks5h, or- /// socks4a proxy. Moli resolves and pins the proxy endpoint; the proxy is- /// responsible for resolving and filtering request target hostnames.+ /// Route HTTP(S) and WebSocket traffic through an HTTP, HTTPS, SOCKS5, or+ /// SOCKS4 proxy. SOCKS targets always use proxy-side DNS; `socks` is a+ /// SOCKS5 alias, while `socks5h` and `socks4a` remain accepted. Moli+ /// resolves and pins the proxy endpoint; the proxy is responsible for+ /// resolving and filtering request target hostnames.#[arg(long, value_name = "URL")]pub http_proxy: Option<String>,