Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): honor host-resolve overrides in network blocking

Duang7773cbc3c2b495a
diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs
index 42a5748d7..002b11e46 100644
--- a/moli-fetch/src/blocking/mod.rs
+++ b/moli-fetch/src/blocking/mod.rs
@@ -693,9 +693,9 @@ pub(crate) fn configure_easy<H: Handler>(
}
if !config.http_host_resolve().is_empty() {
let mut resolve = List::new();
- for entry in config.http_host_resolve() {
+ for entry in normalized_http_host_resolve_entries(config.http_host_resolve())? {
resolve
- .append(entry)
+ .append(&entry)
.with_context(|| anyhow!("failed to build curl host resolve entry `{entry}`"))?;
}
easy.resolve(resolve)
@@ -827,7 +827,7 @@ fn enforce_request_target_policy(config: &FetchConfig, request_url: &Url) -> Res
.port_or_known_default()
.ok_or_else(|| anyhow!("could not determine port for request url `{request_url}`"))?;
- let resolved_ips = resolve_target_ips(host, port)
+ let resolved_ips = resolve_target_ips(config, host, port)
.with_context(|| anyhow!("failed to resolve request host `{host}` for `{request_url}`"))?;
for ip in resolved_ips {
if config.block_private_networks() && is_private_or_internal_ip(ip) {
@@ -841,11 +841,59 @@ fn enforce_request_target_policy(config: &FetchConfig, request_url: &Url) -> Res
Ok(())
}
-fn resolve_target_ips(host: &str, port: u16) -> Result<Vec<IpAddr>> {
+fn resolve_target_ips(config: &FetchConfig, host: &str, port: u16) -> Result<Vec<IpAddr>> {
+ if let Some(resolved_ips) =
+ resolve_host_resolve_override_ips(config.http_host_resolve(), host, port)?
+ {
+ return Ok(resolved_ips);
+ }
+
if let Ok(ip) = host.parse::<IpAddr>() {
return Ok(vec![ip]);
}
+ resolve_system_target_ips(host, port)
+}
+
+fn resolve_host_resolve_override_ips(
+ entries: &[String],
+ host: &str,
+ port: u16,
+) -> Result<Option<Vec<IpAddr>>> {
+ let mut exact_match: Option<Vec<IpAddr>> = None;
+ let mut wildcard_match: Option<Vec<IpAddr>> = None;
+ for entry in entries {
+ match parse_http_host_resolve_entry(entry)? {
+ HttpHostResolveEntry::Add {
+ host: entry_host,
+ port: entry_port,
+ addresses,
+ ..
+ } if entry_port == port => {
+ if entry_host == "*" {
+ wildcard_match = Some(addresses);
+ } else if entry_host.eq_ignore_ascii_case(host) {
+ exact_match = Some(addresses);
+ }
+ }
+ HttpHostResolveEntry::Remove {
+ host: entry_host,
+ port: entry_port,
+ } if entry_port == port => {
+ if entry_host == "*" {
+ wildcard_match = None;
+ } else if entry_host.eq_ignore_ascii_case(host) {
+ exact_match = None;
+ }
+ }
+ _ => {}
+ }
+ }
+
+ Ok(exact_match.or(wildcard_match))
+}
+
+fn resolve_system_target_ips(host: &str, port: u16) -> Result<Vec<IpAddr>> {
let mut resolved = Vec::new();
for addr in (host, port).to_socket_addrs()? {
let ip = addr.ip();
@@ -859,6 +907,172 @@ fn resolve_target_ips(host: &str, port: u16) -> Result<Vec<IpAddr>> {
Ok(resolved)
}
+enum HttpHostResolveEntry {
+ Add {
+ plus_prefix: bool,
+ host: String,
+ port: u16,
+ addresses: Vec<IpAddr>,
+ },
+ Remove {
+ host: String,
+ port: u16,
+ },
+}
+
+impl HttpHostResolveEntry {
+ fn curl_entry(&self) -> String {
+ match self {
+ Self::Add {
+ plus_prefix,
+ host,
+ port,
+ addresses,
+ } => {
+ let prefix = if *plus_prefix { "+" } else { "" };
+ let addresses = addresses
+ .iter()
+ .map(format_http_host_resolve_ip)
+ .collect::<Vec<_>>()
+ .join(",");
+ format!(
+ "{prefix}{}:{port}:{addresses}",
+ format_http_host_resolve_host(host)
+ )
+ }
+ Self::Remove { host, port } => {
+ format!("-{}:{port}", format_http_host_resolve_host(host))
+ }
+ }
+ }
+}
+
+fn normalized_http_host_resolve_entries(entries: &[String]) -> Result<Vec<String>> {
+ entries
+ .iter()
+ .map(|entry| parse_http_host_resolve_entry(entry).map(|entry| entry.curl_entry()))
+ .collect()
+}
+
+fn parse_http_host_resolve_entry(entry: &str) -> Result<HttpHostResolveEntry> {
+ let entry = entry.trim();
+ if let Some(entry) = entry.strip_prefix('-') {
+ let (host, port) = split_http_host_resolve_remove_entry(entry)?;
+ return Ok(HttpHostResolveEntry::Remove {
+ host: normalize_http_host_resolve_host(host),
+ port: parse_http_host_resolve_port(port, entry)?,
+ });
+ }
+
+ let (plus_prefix, entry) = if let Some(entry) = entry.strip_prefix('+') {
+ (true, entry)
+ } else {
+ (false, entry)
+ };
+ let (host, port, address) = split_http_host_resolve_add_entry(entry)?;
+ let port = parse_http_host_resolve_port(port, entry)?;
+ let mut addresses = Vec::new();
+ for address in address.split(',').map(str::trim) {
+ if address.is_empty() {
+ bail!("--http-host-resolve entries must not contain empty addresses");
+ }
+ let address = address.trim_matches(['[', ']']);
+ let address = address
+ .parse::<IpAddr>()
+ .with_context(|| format!("invalid --http-host-resolve address in `{entry}`"))?;
+ if !addresses.contains(&address) {
+ addresses.push(address);
+ }
+ }
+
+ Ok(HttpHostResolveEntry::Add {
+ plus_prefix,
+ host: normalize_http_host_resolve_host(host),
+ port,
+ addresses,
+ })
+}
+
+fn split_http_host_resolve_add_entry(entry: &str) -> Result<(&str, &str, &str)> {
+ if let Some(entry) = entry.strip_prefix('[') {
+ let Some(host_end) = entry.find(']') else {
+ bail!("--http-host-resolve must be in HOST:PORT:ADDR form");
+ };
+ let host = &entry[..host_end];
+ let remainder = &entry[host_end + 1..];
+ let Some(remainder) = remainder.strip_prefix(':') else {
+ bail!("--http-host-resolve must be in HOST:PORT:ADDR form");
+ };
+ let Some((port, address)) = remainder.split_once(':') else {
+ bail!("--http-host-resolve must be in HOST:PORT:ADDR form");
+ };
+ if host.trim().is_empty() || port.trim().is_empty() || address.trim().is_empty() {
+ bail!("--http-host-resolve must be in HOST:PORT:ADDR form");
+ }
+ return Ok((host.trim(), port.trim(), address.trim()));
+ }
+
+ let mut parts = entry.splitn(3, ':');
+ let host = parts.next().unwrap_or_default().trim();
+ let port = parts.next().unwrap_or_default().trim();
+ let address = parts.next().unwrap_or_default().trim();
+
+ if host.is_empty() || port.is_empty() || address.is_empty() {
+ bail!("--http-host-resolve must be in HOST:PORT:ADDR form");
+ }
+
+ Ok((host, port, address))
+}
+
+fn split_http_host_resolve_remove_entry(entry: &str) -> Result<(&str, &str)> {
+ if let Some(entry) = entry.strip_prefix('[') {
+ let Some(host_end) = entry.find(']') else {
+ bail!("--http-host-resolve removal must be in HOST:PORT form");
+ };
+ let host = &entry[..host_end];
+ let remainder = &entry[host_end + 1..];
+ let Some(port) = remainder.strip_prefix(':') else {
+ bail!("--http-host-resolve removal must be in HOST:PORT form");
+ };
+ if host.trim().is_empty() || port.trim().is_empty() {
+ bail!("--http-host-resolve removal must be in HOST:PORT form");
+ }
+ return Ok((host.trim(), port.trim()));
+ }
+
+ let Some((host, port)) = entry.split_once(':') else {
+ bail!("--http-host-resolve removal must be in HOST:PORT form");
+ };
+ if host.trim().is_empty() || port.trim().is_empty() {
+ bail!("--http-host-resolve removal must be in HOST:PORT form");
+ }
+ Ok((host.trim(), port.trim()))
+}
+
+fn parse_http_host_resolve_port(port: &str, entry: &str) -> Result<u16> {
+ port.parse::<u16>()
+ .with_context(|| format!("invalid --http-host-resolve port in `{entry}`"))
+}
+
+fn normalize_http_host_resolve_host(host: &str) -> String {
+ host.trim_matches(['[', ']']).to_owned()
+}
+
+fn format_http_host_resolve_host(host: &str) -> String {
+ if host != "*" && host.contains(':') {
+ format!("[{host}]")
+ } else {
+ host.to_owned()
+ }
+}
+
+fn format_http_host_resolve_ip(ip: &IpAddr) -> String {
+ match ip {
+ IpAddr::V4(ip) => ip.to_string(),
+ IpAddr::V6(ip) => format!("[{ip}]"),
+ }
+}
+
fn is_private_or_internal_ip(ip: IpAddr) -> bool {
match ip {
IpAddr::V4(ipv4) => {
@@ -1392,6 +1606,23 @@ mod tests {
assert!(header_value(&headers, "sec-ch-ua").is_some());
}
+ #[test]
+ fn http_host_resolve_entries_are_normalized_before_curl_configuration() {
+ let entries = normalized_http_host_resolve_entries(&[
+ " localhost:80: 1.1.1.1 , [2001:db8::1] ".to_owned(),
+ " - localhost:80 ".to_owned(),
+ ])
+ .unwrap();
+
+ assert_eq!(
+ entries,
+ vec![
+ "localhost:80:1.1.1.1,[2001:db8::1]".to_owned(),
+ "-localhost:80".to_owned()
+ ]
+ );
+ }
+
#[test]
fn third_party_credentialed_request_reports_active_storage_access() {
let config = FetchConfig::default();
diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rs
index 35bc0a6f3..c469fdeee 100644
--- a/moli-fetch/src/tests/mod.rs
+++ b/moli-fetch/src/tests/mod.rs
@@ -1786,6 +1786,62 @@ fn fetch_client_rejects_private_network_targets() {
);
}
+#[test]
+fn fetch_client_rejects_private_network_targets_via_host_resolve_override() {
+ let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("should-not-be-reached")]);
+ let port = Url::parse(&server.url())
+ .unwrap()
+ .port()
+ .expect("scripted server URL should include a port");
+ let mut config = FetchConfig::default();
+ config.set_http_proxy(Some(String::new()));
+ config.set_network_blocking(true, vec![]);
+ config.set_http_host_resolve(vec![format!("example.com:{port}:127.0.0.1")]);
+
+ let error = fetch_with_config_for_test(
+ &config,
+ Request::get(&format!("http://example.com:{port}/")).unwrap(),
+ )
+ .unwrap_err();
+ let error_chain = format!("{error:#}");
+
+ assert!(
+ error_chain.contains("blocked private network address `127.0.0.1`"),
+ "unexpected error: {error_chain}"
+ );
+ assert_eq!(server.hits(), 0);
+
+ server.shutdown();
+}
+
+#[test]
+fn fetch_client_rejects_ip_literal_private_target_via_host_resolve_override() {
+ let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("should-not-be-reached")]);
+ let port = Url::parse(&server.url())
+ .unwrap()
+ .port()
+ .expect("scripted server URL should include a port");
+ let mut config = FetchConfig::default();
+ config.set_http_proxy(Some(String::new()));
+ config.set_network_blocking(true, vec![]);
+ config.set_http_host_resolve(vec![format!("1.1.1.1:{port}:127.0.0.1")]);
+
+ let error = fetch_with_config_for_test(
+ &config,
+ Request::get(&format!("http://1.1.1.1:{port}/")).unwrap(),
+ )
+ .unwrap_err();
+ let error_chain = format!("{error:#}");
+
+ assert!(
+ error_chain.contains("blocked private network address `127.0.0.1`"),
+ "unexpected error: {error_chain}"
+ );
+ assert_eq!(server.hits(), 0);
+
+ server.shutdown();
+}
+
#[test]
fn fetch_client_rejects_configured_cidrs() {
let mut config = FetchConfig::default();