Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): honor host-resolve overrides in network blocking
Duang7773cbc3c2b495a
diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rsindex 42a5748d7..002b11e46 100644--- a/moli-fetch/src/blocking/mod.rs+++ b/moli-fetch/src/blocking/mod.rs@@ -693,9 +693,9 @@ pub(crate) fn configure_easy<H: Handler>(}if !config.http_host_resolve().is_empty() {let mut resolve = List::new();- for entry in config.http_host_resolve() {+ for entry in normalized_http_host_resolve_entries(config.http_host_resolve())? {resolve- .append(entry)+ .append(&entry).with_context(|| anyhow!("failed to build curl host resolve entry `{entry}`"))?;}easy.resolve(resolve)@@ -827,7 +827,7 @@ fn enforce_request_target_policy(config: &FetchConfig, request_url: &Url) -> Res.port_or_known_default().ok_or_else(|| anyhow!("could not determine port for request url `{request_url}`"))?;- let resolved_ips = resolve_target_ips(host, port)+ let resolved_ips = resolve_target_ips(config, host, port).with_context(|| anyhow!("failed to resolve request host `{host}` for `{request_url}`"))?;for ip in resolved_ips {if config.block_private_networks() && is_private_or_internal_ip(ip) {@@ -841,11 +841,59 @@ fn enforce_request_target_policy(config: &FetchConfig, request_url: &Url) -> ResOk(())}-fn resolve_target_ips(host: &str, port: u16) -> Result<Vec<IpAddr>> {+fn resolve_target_ips(config: &FetchConfig, host: &str, port: u16) -> Result<Vec<IpAddr>> {+ if let Some(resolved_ips) =+ resolve_host_resolve_override_ips(config.http_host_resolve(), host, port)?+ {+ return Ok(resolved_ips);+ }+if let Ok(ip) = host.parse::<IpAddr>() {return Ok(vec![ip]);}+ resolve_system_target_ips(host, port)+}++fn resolve_host_resolve_override_ips(+ entries: &[String],+ host: &str,+ port: u16,+) -> Result<Option<Vec<IpAddr>>> {+ let mut exact_match: Option<Vec<IpAddr>> = None;+ let mut wildcard_match: Option<Vec<IpAddr>> = None;+ for entry in entries {+ match parse_http_host_resolve_entry(entry)? {+ HttpHostResolveEntry::Add {+ host: entry_host,+ port: entry_port,+ addresses,+ ..+ } if entry_port == port => {+ if entry_host == "*" {+ wildcard_match = Some(addresses);+ } else if entry_host.eq_ignore_ascii_case(host) {+ exact_match = Some(addresses);+ }+ }+ HttpHostResolveEntry::Remove {+ host: entry_host,+ port: entry_port,+ } if entry_port == port => {+ if entry_host == "*" {+ wildcard_match = None;+ } else if entry_host.eq_ignore_ascii_case(host) {+ exact_match = None;+ }+ }+ _ => {}+ }+ }++ Ok(exact_match.or(wildcard_match))+}++fn resolve_system_target_ips(host: &str, port: u16) -> Result<Vec<IpAddr>> {let mut resolved = Vec::new();for addr in (host, port).to_socket_addrs()? {let ip = addr.ip();@@ -859,6 +907,172 @@ fn resolve_target_ips(host: &str, port: u16) -> Result<Vec<IpAddr>> {Ok(resolved)}+enum HttpHostResolveEntry {+ Add {+ plus_prefix: bool,+ host: String,+ port: u16,+ addresses: Vec<IpAddr>,+ },+ Remove {+ host: String,+ port: u16,+ },+}++impl HttpHostResolveEntry {+ fn curl_entry(&self) -> String {+ match self {+ Self::Add {+ plus_prefix,+ host,+ port,+ addresses,+ } => {+ let prefix = if *plus_prefix { "+" } else { "" };+ let addresses = addresses+ .iter()+ .map(format_http_host_resolve_ip)+ .collect::<Vec<_>>()+ .join(",");+ format!(+ "{prefix}{}:{port}:{addresses}",+ format_http_host_resolve_host(host)+ )+ }+ Self::Remove { host, port } => {+ format!("-{}:{port}", format_http_host_resolve_host(host))+ }+ }+ }+}++fn normalized_http_host_resolve_entries(entries: &[String]) -> Result<Vec<String>> {+ entries+ .iter()+ .map(|entry| parse_http_host_resolve_entry(entry).map(|entry| entry.curl_entry()))+ .collect()+}++fn parse_http_host_resolve_entry(entry: &str) -> Result<HttpHostResolveEntry> {+ let entry = entry.trim();+ if let Some(entry) = entry.strip_prefix('-') {+ let (host, port) = split_http_host_resolve_remove_entry(entry)?;+ return Ok(HttpHostResolveEntry::Remove {+ host: normalize_http_host_resolve_host(host),+ port: parse_http_host_resolve_port(port, entry)?,+ });+ }++ let (plus_prefix, entry) = if let Some(entry) = entry.strip_prefix('+') {+ (true, entry)+ } else {+ (false, entry)+ };+ let (host, port, address) = split_http_host_resolve_add_entry(entry)?;+ let port = parse_http_host_resolve_port(port, entry)?;+ let mut addresses = Vec::new();+ for address in address.split(',').map(str::trim) {+ if address.is_empty() {+ bail!("--http-host-resolve entries must not contain empty addresses");+ }+ let address = address.trim_matches(['[', ']']);+ let address = address+ .parse::<IpAddr>()+ .with_context(|| format!("invalid --http-host-resolve address in `{entry}`"))?;+ if !addresses.contains(&address) {+ addresses.push(address);+ }+ }++ Ok(HttpHostResolveEntry::Add {+ plus_prefix,+ host: normalize_http_host_resolve_host(host),+ port,+ addresses,+ })+}++fn split_http_host_resolve_add_entry(entry: &str) -> Result<(&str, &str, &str)> {+ if let Some(entry) = entry.strip_prefix('[') {+ let Some(host_end) = entry.find(']') else {+ bail!("--http-host-resolve must be in HOST:PORT:ADDR form");+ };+ let host = &entry[..host_end];+ let remainder = &entry[host_end + 1..];+ let Some(remainder) = remainder.strip_prefix(':') else {+ bail!("--http-host-resolve must be in HOST:PORT:ADDR form");+ };+ let Some((port, address)) = remainder.split_once(':') else {+ bail!("--http-host-resolve must be in HOST:PORT:ADDR form");+ };+ if host.trim().is_empty() || port.trim().is_empty() || address.trim().is_empty() {+ bail!("--http-host-resolve must be in HOST:PORT:ADDR form");+ }+ return Ok((host.trim(), port.trim(), address.trim()));+ }++ let mut parts = entry.splitn(3, ':');+ let host = parts.next().unwrap_or_default().trim();+ let port = parts.next().unwrap_or_default().trim();+ let address = parts.next().unwrap_or_default().trim();++ if host.is_empty() || port.is_empty() || address.is_empty() {+ bail!("--http-host-resolve must be in HOST:PORT:ADDR form");+ }++ Ok((host, port, address))+}++fn split_http_host_resolve_remove_entry(entry: &str) -> Result<(&str, &str)> {+ if let Some(entry) = entry.strip_prefix('[') {+ let Some(host_end) = entry.find(']') else {+ bail!("--http-host-resolve removal must be in HOST:PORT form");+ };+ let host = &entry[..host_end];+ let remainder = &entry[host_end + 1..];+ let Some(port) = remainder.strip_prefix(':') else {+ bail!("--http-host-resolve removal must be in HOST:PORT form");+ };+ if host.trim().is_empty() || port.trim().is_empty() {+ bail!("--http-host-resolve removal must be in HOST:PORT form");+ }+ return Ok((host.trim(), port.trim()));+ }++ let Some((host, port)) = entry.split_once(':') else {+ bail!("--http-host-resolve removal must be in HOST:PORT form");+ };+ if host.trim().is_empty() || port.trim().is_empty() {+ bail!("--http-host-resolve removal must be in HOST:PORT form");+ }+ Ok((host.trim(), port.trim()))+}++fn parse_http_host_resolve_port(port: &str, entry: &str) -> Result<u16> {+ port.parse::<u16>()+ .with_context(|| format!("invalid --http-host-resolve port in `{entry}`"))+}++fn normalize_http_host_resolve_host(host: &str) -> String {+ host.trim_matches(['[', ']']).to_owned()+}++fn format_http_host_resolve_host(host: &str) -> String {+ if host != "*" && host.contains(':') {+ format!("[{host}]")+ } else {+ host.to_owned()+ }+}++fn format_http_host_resolve_ip(ip: &IpAddr) -> String {+ match ip {+ IpAddr::V4(ip) => ip.to_string(),+ IpAddr::V6(ip) => format!("[{ip}]"),+ }+}+fn is_private_or_internal_ip(ip: IpAddr) -> bool {match ip {IpAddr::V4(ipv4) => {@@ -1392,6 +1606,23 @@ mod tests {assert!(header_value(&headers, "sec-ch-ua").is_some());}+ #[test]+ fn http_host_resolve_entries_are_normalized_before_curl_configuration() {+ let entries = normalized_http_host_resolve_entries(&[+ " localhost:80: 1.1.1.1 , [2001:db8::1] ".to_owned(),+ " - localhost:80 ".to_owned(),+ ])+ .unwrap();++ assert_eq!(+ entries,+ vec![+ "localhost:80:1.1.1.1,[2001:db8::1]".to_owned(),+ "-localhost:80".to_owned()+ ]+ );+ }+#[test]fn third_party_credentialed_request_reports_active_storage_access() {let config = FetchConfig::default();diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rsindex 35bc0a6f3..c469fdeee 100644--- a/moli-fetch/src/tests/mod.rs+++ b/moli-fetch/src/tests/mod.rs@@ -1786,6 +1786,62 @@ fn fetch_client_rejects_private_network_targets() {);}+#[test]+fn fetch_client_rejects_private_network_targets_via_host_resolve_override() {+ let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("should-not-be-reached")]);+ let port = Url::parse(&server.url())+ .unwrap()+ .port()+ .expect("scripted server URL should include a port");+ let mut config = FetchConfig::default();+ config.set_http_proxy(Some(String::new()));+ config.set_network_blocking(true, vec![]);+ config.set_http_host_resolve(vec![format!("example.com:{port}:127.0.0.1")]);++ let error = fetch_with_config_for_test(+ &config,+ Request::get(&format!("http://example.com:{port}/")).unwrap(),+ )+ .unwrap_err();+ let error_chain = format!("{error:#}");++ assert!(+ error_chain.contains("blocked private network address `127.0.0.1`"),+ "unexpected error: {error_chain}"+ );+ assert_eq!(server.hits(), 0);++ server.shutdown();+}++#[test]+fn fetch_client_rejects_ip_literal_private_target_via_host_resolve_override() {+ let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("should-not-be-reached")]);+ let port = Url::parse(&server.url())+ .unwrap()+ .port()+ .expect("scripted server URL should include a port");+ let mut config = FetchConfig::default();+ config.set_http_proxy(Some(String::new()));+ config.set_network_blocking(true, vec![]);+ config.set_http_host_resolve(vec![format!("1.1.1.1:{port}:127.0.0.1")]);++ let error = fetch_with_config_for_test(+ &config,+ Request::get(&format!("http://1.1.1.1:{port}/")).unwrap(),+ )+ .unwrap_err();+ let error_chain = format!("{error:#}");++ assert!(+ error_chain.contains("blocked private network address `127.0.0.1`"),+ "unexpected error: {error_chain}"+ );+ assert_eq!(server.hits(), 0);++ server.shutdown();+}+#[test]fn fetch_client_rejects_configured_cidrs() {let mut config = FetchConfig::default();