Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix: prevent DNS rebinding in network policy
ldm0fcc716892589
diff --git a/Cargo.lock b/Cargo.lockindex 5bc31ffc4..a94737256 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2587,6 +2587,7 @@ name = "moli-curl"version = "0.1.0"dependencies = ["anyhow",+ "cidr","crossbeam-channel","curl","moli-dns-resolver",diff --git a/moli-curl/Cargo.toml b/moli-curl/Cargo.tomlindex 4bed4413c..748b0ddc8 100644--- a/moli-curl/Cargo.toml+++ b/moli-curl/Cargo.toml@@ -6,6 +6,7 @@ edition = "2024"[dependencies]anyhow = "1.0.100"+cidr = "0.3.2"crossbeam-channel = "0.5.15"curl = { version = "0.4.49", default-features = false, features = ["brotli", "http2", "poll_7_68_0", "ssl", "static-curl", "websocket"] }moli-dns-resolver = { path = "../moli-dns-resolver" }diff --git a/moli-curl/src/dns_adapter.rs b/moli-curl/src/dns_adapter.rsindex 057034471..e41341ffc 100644--- a/moli-curl/src/dns_adapter.rs+++ b/moli-curl/src/dns_adapter.rs@@ -8,6 +8,8 @@ use curl::{};use moli_dns_resolver::{DnsCachePartition, DnsLookupResult, DnsResolverService, DnsTarget};+use crate::NetworkAddressPolicy;+/// Curl-side policy for DNS ownership before a transfer enters the multi set.////// `origin == None` means curl owns name resolution. `Some` means the transfer@@ -25,6 +27,8 @@ struct CurlDnsOriginResolution {/// Existing caller-provided `CURLOPT_RESOLVE` entries that must remain/// installed when the generated origin answer is added.static_entries: Vec<String>,+ address_policy: NetworkAddressPolicy,+ policy_target: String,}impl CurlDnsResolution {@@ -33,14 +37,39 @@ impl CurlDnsResolution {}pub fn resolve_origin(target: DnsTarget, static_entries: Vec<String>) -> Self {+ let policy_target = format!("{}:{}", target.host(), target.port());Self {origin: Some(Box::new(CurlDnsOriginResolution {target,static_entries,+ address_policy: NetworkAddressPolicy::default(),+ policy_target,})),}}+ /// Applies address admission to the shared-resolver result before it is+ /// installed on curl. The target text is retained for actionable errors.+ pub fn with_network_address_policy(+ mut self,+ address_policy: NetworkAddressPolicy,+ policy_target: impl Into<String>,+ ) -> Self {+ self.set_network_address_policy(address_policy, policy_target);+ self+ }++ pub(crate) fn set_network_address_policy(+ &mut self,+ address_policy: NetworkAddressPolicy,+ policy_target: impl Into<String>,+ ) {+ if let Some(resolution) = self.origin.as_mut() {+ resolution.address_policy = address_policy;+ resolution.policy_target = policy_target.into();+ }+ }+pub(crate) fn target(&self) -> Option<&DnsTarget> {self.origin.as_ref().map(|resolution| &resolution.target)}@@ -54,6 +83,9 @@ impl CurlDnsResolution {let Some(resolution) = self.origin.as_ref() else {return Ok(());};+ resolution+ .address_policy+ .check_addresses(addresses, &resolution.policy_target)?;let mut resolve = List::new();for entry in &resolution.static_entries {resolve@@ -259,6 +291,34 @@ mod tests {assert_eq!(policy.target(), None);}+ #[test]+ fn blocked_address_prevents_installing_the_complete_dns_answer() {+ let target = DnsTarget::new("example.test", 443);+ let mut resolution = CurlDnsResolution::resolve_origin(target.clone(), Vec::new())+ .with_network_address_policy(+ NetworkAddressPolicy::new(true, Vec::new()),+ "https://example.test/",+ );+ let mut easy = Easy2::new(TestHandler);++ let error = resolution+ .install(+ &mut easy,+ &[+ IpAddr::from([93, 184, 216, 34]),+ IpAddr::from([127, 0, 0, 1]),+ ],+ )+ .expect_err("one private answer must reject the complete DNS result");++ assert!(error.to_string().contains("127.0.0.1"));+ assert_eq!(+ resolution.target(),+ Some(&target),+ "a rejected answer must never transition to an installed state"+ );+ }+#[test]fn exact_completion_can_be_claimed_only_once() {let mut residence = CurlDnsOwnerResidence::<u64, _>::default();diff --git a/moli-curl/src/lib.rs b/moli-curl/src/lib.rsindex 02c10e1ba..337b3431c 100644--- a/moli-curl/src/lib.rs+++ b/moli-curl/src/lib.rs@@ -2,11 +2,13 @@mod dns_adapter;mod http;+mod network_policy;mod runtime;mod tls;pub mod websocket;pub use dns_adapter::CurlDnsResolution;pub use http::{CurlHttpSender, CurlMultiCompletion, CurlMultiJob, CurlOriginKey, CurlSubmitError};+pub use network_policy::NetworkAddressPolicy;pub use runtime::{CurlMultiRuntime, CurlMultiRuntimeConfig, CurlTransferId};pub use tls::CurlTlsConfig;diff --git a/moli-curl/src/network_policy.rs b/moli-curl/src/network_policy.rsnew file mode 100644index 000000000..7bf30f349--- /dev/null+++ b/moli-curl/src/network_policy.rs@@ -0,0 +1,137 @@+use std::net::IpAddr;++use anyhow::{Result, bail};+use cidr::AnyIpCidr;++/// Admission policy for the concrete addresses a network transport may use.+///+/// Callers must apply this policy to every address returned by DNS before+/// installing that same, complete answer on the transport. Filtering a DNS+/// answer is deliberately not supported because doing so changes address+/// selection and fallback semantics.+#[derive(Debug, Clone, Default)]+pub struct NetworkAddressPolicy {+ block_private_networks: bool,+ block_cidrs: Vec<AnyIpCidr>,+}++impl NetworkAddressPolicy {+ pub fn new(block_private_networks: bool, block_cidrs: Vec<AnyIpCidr>) -> Self {+ Self {+ block_private_networks,+ block_cidrs,+ }+ }++ pub fn is_enforced(&self) -> bool {+ self.block_private_networks || !self.block_cidrs.is_empty()+ }++ pub fn check_addresses(&self, addresses: &[IpAddr], target: &str) -> Result<()> {+ for &address in addresses {+ self.check_address(address, target)?;+ }+ Ok(())+ }++ pub fn check_address(&self, address: IpAddr, target: &str) -> Result<()> {+ if self.block_private_networks && is_private_or_internal_ip(address) {+ bail!("blocked private network address `{address}` for `{target}`");+ }+ let mapped_ipv4 = match address {+ IpAddr::V6(address) => address.to_ipv4_mapped().map(IpAddr::V4),+ IpAddr::V4(_) => None,+ };+ if let Some(cidr) = self.block_cidrs.iter().find(|cidr| {+ cidr.contains(&address)+ || mapped_ipv4+ .as_ref()+ .is_some_and(|mapped| cidr.contains(mapped))+ }) {+ bail!("blocked address `{address}` for `{target}` because it matches `{cidr}`");+ }+ Ok(())+ }+}++fn is_private_or_internal_ip(ip: IpAddr) -> bool {+ match ip {+ IpAddr::V4(ipv4) => {+ ipv4.is_private()+ || ipv4.is_loopback()+ || ipv4.is_link_local()+ || ipv4.is_broadcast()+ || ipv4.is_documentation()+ || ipv4.is_unspecified()+ || ipv4.is_multicast()+ || matches!(ipv4.octets(), [100, second, ..] if (64..=127).contains(&second))+ || matches!(ipv4.octets(), [198, 18 | 19, ..])+ || matches!(ipv4.octets(), [240..=255, ..])+ }+ IpAddr::V6(ipv6) => {+ if let Some(ipv4) = ipv6.to_ipv4_mapped() {+ return is_private_or_internal_ip(IpAddr::V4(ipv4));+ }+ ipv6.is_loopback()+ || ipv6.is_unspecified()+ || ipv6.is_multicast()+ || ipv6.is_unicast_link_local()+ || ipv6.is_unique_local()+ || (ipv6.segments()[0] == 0x2001 && ipv6.segments()[1] == 0x0db8)+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn private_policy_rejects_ipv4_mapped_loopback() {+ let policy = NetworkAddressPolicy::new(true, Vec::new());++ let error = policy+ .check_address("::ffff:127.0.0.1".parse().unwrap(), "wss://example.test/")+ .expect_err("mapped loopback must be private");++ assert!(+ error+ .to_string()+ .contains("blocked private network address")+ );+ }++ #[test]+ fn every_address_must_pass_policy() {+ let policy = NetworkAddressPolicy::new(true, Vec::new());++ let error = policy+ .check_addresses(+ &[+ "93.184.216.34".parse().unwrap(),+ "127.0.0.1".parse().unwrap(),+ ],+ "https://example.test/",+ )+ .expect_err("one private answer must reject the complete DNS result");++ assert!(error.to_string().contains("127.0.0.1"));+ }++ #[test]+ fn ipv4_cidr_rejects_an_ipv4_mapped_ipv6_address() {+ let policy = NetworkAddressPolicy::new(+ false,+ vec!["198.18.0.0/15".parse().expect("test CIDR should parse")],+ );++ let error = policy+ .check_address(+ "::ffff:198.18.0.1".parse().unwrap(),+ "https://example.test/",+ )+ .expect_err("mapped IPv4 must be checked against IPv4 CIDRs");++ assert!(error.to_string().contains("198.18.0.0/15"));+ }+}diff --git a/moli-curl/src/websocket/connector.rs b/moli-curl/src/websocket/connector.rsindex 5445a740a..adc331e18 100644--- a/moli-curl/src/websocket/connector.rs+++ b/moli-curl/src/websocket/connector.rs@@ -3,7 +3,7 @@use super::{CurlWebSocketConnection, CurlWebSocketRequest, SESSION_CAPACITY, connection::SessionIo,};-use crate::{CurlTransferId, runtime::identity::next_transfer_id};+use crate::{CurlTransferId, NetworkAddressPolicy, runtime::identity::next_transfer_id};use anyhow::{Context, Result, bail};use curl::multi::MultiWaker;use std::sync::{@@ -11,6 +11,7 @@ use std::sync::{atomic::{AtomicBool, Ordering},};use tokio::sync::Semaphore;+use url::{Host, Url};pub(crate) struct Submission {pub(super) id: CurlTransferId,@@ -23,6 +24,7 @@ pub(crate) struct Submission {#[derive(Clone, Debug)]pub struct CurlWebSocketConnector {inner: Arc<ConnectorInner>,+ network_address_policy: NetworkAddressPolicy,}#[derive(Debug)]@@ -58,13 +60,23 @@ impl CurlWebSocketConnector {waker,shutdown,}),+ network_address_policy: NetworkAddressPolicy::default(),}}- pub fn connect(&self, request: CurlWebSocketRequest) -> Result<CurlWebSocketConnection> {+ pub fn with_network_address_policy(+ mut self,+ network_address_policy: NetworkAddressPolicy,+ ) -> Self {+ self.network_address_policy = network_address_policy;+ self+ }++ pub fn connect(&self, mut request: CurlWebSocketRequest) -> Result<CurlWebSocketConnection> {if self.inner.shutdown.load(Ordering::Acquire) {bail!("curl WebSocket runtime is closed");}+ self.apply_network_address_policy(&mut request)?;let slot = self.inner.slots@@ -80,4 +92,52 @@ impl CurlWebSocketConnector {let _ = self.inner.waker.wakeup();Ok(connection)}++ fn apply_network_address_policy(&self, request: &mut CurlWebSocketRequest) -> Result<()> {+ if !self.network_address_policy.is_enforced() {+ return Ok(());+ }++ let url = Url::parse(&request.url)+ .with_context(|| format!("failed to parse WebSocket URL `{}`", request.url))?;+ let host = url+ .host()+ .ok_or_else(|| anyhow::anyhow!("WebSocket URL `{url}` is missing a host"))?;++ match host {+ Host::Domain(host) => {+ if request.proxy.is_some() {+ bail!(+ "cannot enforce network address policy for proxied WebSocket hostname `{host}` in `{url}`; the proxy must not resolve an unchecked target hostname"+ );+ }+ let port = url+ .port_or_known_default()+ .ok_or_else(|| anyhow::anyhow!("WebSocket URL `{url}` has no port"))?;+ let target = request.dns_resolution.target().ok_or_else(|| {+ anyhow::anyhow!(+ "network address policy requires shared DNS resolution for WebSocket hostname `{host}` in `{url}`"+ )+ })?;+ if !target.host().eq_ignore_ascii_case(host) || target.port() != port {+ bail!(+ "WebSocket DNS target `{}:{}` does not match policy target `{host}:{port}`",+ target.host(),+ target.port()+ );+ }+ request.dns_resolution.set_network_address_policy(+ self.network_address_policy.clone(),+ url.to_string(),+ );+ }+ Host::Ipv4(address) => self+ .network_address_policy+ .check_address(address.into(), url.as_str())?,+ Host::Ipv6(address) => self+ .network_address_policy+ .check_address(address.into(), url.as_str())?,+ }+ Ok(())+ }}diff --git a/moli-curl/src/websocket/tests.rs b/moli-curl/src/websocket/tests.rsindex 7f0e41a8a..8f8e1c0a4 100644--- a/moli-curl/src/websocket/tests.rs+++ b/moli-curl/src/websocket/tests.rs@@ -9,6 +9,8 @@ use tokio::{sync::oneshot, time::timeout};use tokio_tungstenite::tungstenite::{self, Message, handshake::derive_accept_key};use super::*;+use crate::NetworkAddressPolicy;+use moli_dns_resolver::DnsTarget;mod readiness;mod shared;@@ -16,6 +18,46 @@ mod storage;const DEADLINE: Duration = Duration::from_secs(10);+#[test]+fn strict_connector_rejects_proxy_resolved_websocket_hostname() {+ let runtime = CurlWebSocketRuntime::new().unwrap();+ let connector = runtime+ .connector()+ .with_network_address_policy(NetworkAddressPolicy::new(true, Vec::new()));+ let mut request = CurlWebSocketRequest::new("wss://example.test/socket".to_owned());+ request.proxy = Some("http://proxy.test:8080".to_owned());++ let error = connector+ .connect(request)+ .expect_err("a strict connector cannot verify proxy-side DNS");++ assert!(error.to_string().contains("proxied WebSocket hostname"));+}++#[tokio::test]+async fn strict_connector_checks_shared_websocket_dns_before_connecting() {+ let runtime = CurlWebSocketRuntime::new().unwrap();+ let connector = runtime+ .connector()+ .with_network_address_policy(NetworkAddressPolicy::new(true, Vec::new()));+ let mut request = CurlWebSocketRequest::new("ws://localhost:65534/socket".to_owned());+ request.dns_resolution =+ CurlDnsResolution::resolve_origin(DnsTarget::new("localhost", 65534), Vec::new());++ let mut connection = connector.connect(request).unwrap();+ let event = timeout(DEADLINE, connection.recv())+ .await+ .expect("WebSocket DNS policy deadline")+ .expect("WebSocket DNS policy event");++ match event {+ CurlWebSocketEvent::Closed { result: Err(error) } => {+ assert!(error.contains("blocked private network address"))+ }+ unexpected => panic!("expected a DNS policy failure, got {unexpected:?}"),+ }+}+fn server(handler: impl FnOnce(TcpStream) + Send + 'static) -> (String, thread::JoinHandle<()>) {let listener = TcpListener::bind("127.0.0.1:0").unwrap();let url = format!("ws://{}/native", listener.local_addr().unwrap());diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rsindex 848bc6d32..520f2ffbc 100644--- a/moli-fetch/src/blocking/mod.rs+++ b/moli-fetch/src/blocking/mod.rs@@ -3,7 +3,7 @@ mod collectors;use std::{ffi::{c_char, c_long},- net::{IpAddr, ToSocketAddrs},+ net::IpAddr,str,time::Duration,};@@ -17,7 +17,7 @@ use moli_cookie_jar::{use moli_url::is_potentially_trustworthy_url;use moli_url_policy::ensure_http_network_transport_url;use tracing::debug;-use url::Url;+use url::{Host, Url};pub(crate) use self::cache::{CachedStreamingResponseLookup, cached_streaming_response_body_exceeds_response_limit,@@ -645,7 +645,10 @@ pub(crate) fn configure_easy<H: Handler>(}if !config.http_host_resolve().is_empty() {let mut resolve = List::new();- for entry in normalized_http_host_resolve_entries(config.http_host_resolve())? {+ let force_permanent = config.network_address_policy().is_enforced();+ for entry in+ normalized_http_host_resolve_entries(config.http_host_resolve(), force_permanent)?+ {resolve.append(&entry).with_context(|| anyhow!("failed to build curl host resolve entry `{entry}`"))?;@@ -755,42 +758,31 @@ fn enforce_request_target_policy(config: &FetchConfig, request_url: &Url) -> Resreturn Ok(());}- let Some(host) = request_url.host_str() else {+ let Some(host) = request_url.host() else {return Ok(());};+ let host_text = request_url+ .host_str()+ .expect("a parsed URL host must have text");let port = request_url.port_or_known_default().ok_or_else(|| anyhow!("could not determine port for request url `{request_url}`"))?;- let resolved_ips = resolve_target_ips(config, host, port)- .with_context(|| anyhow!("failed to resolve request host `{host}` for `{request_url}`"))?;- for ip in resolved_ips {- if config.block_private_networks() && is_private_or_internal_ip(ip) {- bail!("blocked private network address `{ip}` for `{request_url}`");- }- if let Some(cidr) = config.block_cidrs().iter().find(|cidr| cidr.contains(&ip)) {- bail!("blocked address `{ip}` for `{request_url}` because it matches `{cidr}`");- }- }-- Ok(())-}--fn resolve_target_ips(config: &FetchConfig, host: &str, port: u16) -> Result<Vec<IpAddr>> {- if let Some(resolved_ips) =- resolve_host_resolve_override_ips(config.http_host_resolve(), host, port)?+ let policy = config.network_address_policy();+ if let Some(addresses) =+ resolve_host_resolve_override_ips(config.http_host_resolve(), host_text, port)?{- return Ok(resolved_ips);+ return policy.check_addresses(&addresses, request_url.as_str());}- if let Ok(ip) = host.parse::<IpAddr>() {- return Ok(vec![ip]);+ match host {+ Host::Ipv4(address) => policy.check_address(IpAddr::V4(address), request_url.as_str()),+ Host::Ipv6(address) => policy.check_address(IpAddr::V6(address), request_url.as_str()),+ Host::Domain(_) => Ok(()),}-- resolve_system_target_ips(host, port)}-fn resolve_host_resolve_override_ips(+pub(crate) fn resolve_host_resolve_override_ips(entries: &[String],host: &str,port: u16,@@ -827,21 +819,6 @@ fn resolve_host_resolve_override_ips(Ok(exact_match.or(wildcard_match))}--fn resolve_system_target_ips(host: &str, port: u16) -> Result<Vec<IpAddr>> {- let mut resolved = Vec::new();- for addr in (host, port).to_socket_addrs()? {- let ip = addr.ip();- if !resolved.contains(&ip) {- resolved.push(ip);- }- }- if resolved.is_empty() {- bail!("no addresses resolved");- }- Ok(resolved)-}-enum HttpHostResolveEntry {Add {plus_prefix: bool,@@ -856,7 +833,7 @@ enum HttpHostResolveEntry {}impl HttpHostResolveEntry {- fn curl_entry(&self) -> String {+ fn curl_entry(&self, force_permanent: bool) -> String {match self {Self::Add {plus_prefix,@@ -864,7 +841,11 @@ impl HttpHostResolveEntry {port,addresses,} => {- let prefix = if *plus_prefix { "+" } else { "" };+ let prefix = if *plus_prefix && !force_permanent {+ "+"+ } else {+ ""+ };let addresses = addresses.iter().map(format_http_host_resolve_ip)@@ -882,10 +863,15 @@ impl HttpHostResolveEntry {}}-fn normalized_http_host_resolve_entries(entries: &[String]) -> Result<Vec<String>> {+pub(crate) fn normalized_http_host_resolve_entries(+ entries: &[String],+ force_permanent: bool,+) -> Result<Vec<String>> {entries.iter()- .map(|entry| parse_http_host_resolve_entry(entry).map(|entry| entry.curl_entry()))+ .map(|entry| {+ parse_http_host_resolve_entry(entry).map(|entry| entry.curl_entry(force_permanent))+ }).collect()}@@ -1008,31 +994,6 @@ fn format_http_host_resolve_ip(ip: &IpAddr) -> String {}}-fn is_private_or_internal_ip(ip: IpAddr) -> bool {- match ip {- IpAddr::V4(ipv4) => {- ipv4.is_private()- || ipv4.is_loopback()- || ipv4.is_link_local()- || ipv4.is_broadcast()- || ipv4.is_documentation()- || ipv4.is_unspecified()- || ipv4.is_multicast()- || matches!(ipv4.octets(), [100, second, ..] if (64..=127).contains(&second))- || matches!(ipv4.octets(), [198, 18 | 19, ..])- || matches!(ipv4.octets(), [240..=255, ..])- }- IpAddr::V6(ipv6) => {- ipv6.is_loopback()- || ipv6.is_unspecified()- || ipv6.is_multicast()- || ipv6.is_unicast_link_local()- || ipv6.is_unique_local()- || (ipv6.segments()[0] == 0x2001 && ipv6.segments()[1] == 0x0db8)- }- }-}-fn configure_curl_http_protocol_allowlist<H: Handler>(easy: &mut Easy2<H>) -> Result<()> {for (option, description) in [(CURL_OPTION_PROTOCOLS_STR, "request protocols"),@@ -1586,10 +1547,13 @@ mod tests {#[test]fn http_host_resolve_entries_are_normalized_before_curl_configuration() {- let entries = normalized_http_host_resolve_entries(&[- " localhost:80: 1.1.1.1 , [2001:db8::1] ".to_owned(),- " - localhost:80 ".to_owned(),- ])+ let entries = normalized_http_host_resolve_entries(+ &[+ " localhost:80: 1.1.1.1 , [2001:db8::1] ".to_owned(),+ " - localhost:80 ".to_owned(),+ ],+ false,+ ).unwrap();assert_eq!(@@ -1601,6 +1565,20 @@ mod tests {);}+ #[test]+ fn address_policy_makes_temporary_host_resolve_entries_permanent() {+ let entries = vec!["+example.test:443:93.184.216.34".to_owned()];++ assert_eq!(+ normalized_http_host_resolve_entries(&entries, false).unwrap(),+ ["+example.test:443:93.184.216.34"]+ );+ assert_eq!(+ normalized_http_host_resolve_entries(&entries, true).unwrap(),+ ["example.test:443:93.184.216.34"]+ );+ }+#[test]fn third_party_credentialed_request_reports_active_storage_access() {let config = FetchConfig::default();diff --git a/moli-fetch/src/config.rs b/moli-fetch/src/config.rsindex b47397f48..a586eccf9 100644--- a/moli-fetch/src/config.rs+++ b/moli-fetch/src/config.rs@@ -2,7 +2,7 @@ use std::{num::NonZeroU32, path::PathBuf};use cidr::AnyIpCidr;use moli_browser_profile::{BrowserIdentityProfile, DEFAULT_ACCEPT_LANGUAGE};-use moli_curl::CurlTlsConfig;+use moli_curl::{CurlTlsConfig, NetworkAddressPolicy};use crate::WebBotAuthSigner;@@ -296,6 +296,10 @@ impl FetchConfig {pub fn block_cidrs(&self) -> &[AnyIpCidr] {&self.block_cidrs}++ pub(crate) fn network_address_policy(&self) -> NetworkAddressPolicy {+ NetworkAddressPolicy::new(self.block_private_networks, self.block_cidrs.clone())+ }}impl Default for FetchConfig {diff --git a/moli-fetch/src/dns.rs b/moli-fetch/src/dns.rsindex 81fe87a85..3a18842be 100644--- a/moli-fetch/src/dns.rs+++ b/moli-fetch/src/dns.rs@@ -1,27 +1,40 @@+use anyhow::{Result, bail};use moli_curl::CurlDnsResolution;use moli_dns_resolver::DnsTarget;use url::{Host, Url};-use crate::FetchConfig;+use crate::{+ FetchConfig,+ blocking::{normalized_http_host_resolve_entries, resolve_host_resolve_override_ips},+};/// Fetch-side DNS admission decision.////// The shared resolver is used only when Fetch can prove that curl will-/// connect directly to an HTTP(S) origin. Proxy traffic stays curl-managed:-/// the proxy, rather than the local process, may be responsible for resolving-/// the origin hostname. IP literals and explicit host-resolve configuration-/// already have exact routing and must not be resolved again.+/// connect directly to an HTTP(S) origin. Proxy traffic stays curl-managed+/// only when no address policy is active; otherwise a proxy-resolved hostname+/// cannot be verified locally and is rejected. IP literals and matching+/// explicit host-resolve entries already have exact routing and are checked+/// synchronously before this decision.#[derive(Debug, Clone, PartialEq, Eq)]enum FetchCurlDnsAdmission {CurlManaged,SharedResolver(DnsTarget),}-pub(crate) fn curl_dns_resolution(config: &FetchConfig, url: &Url) -> CurlDnsResolution {- match curl_dns_admission_with_env(config, url, |name| std::env::var(name).ok()) {- FetchCurlDnsAdmission::CurlManaged => CurlDnsResolution::curl_managed(),+pub(crate) fn curl_dns_resolution(config: &FetchConfig, url: &Url) -> Result<CurlDnsResolution> {+ match curl_dns_admission_with_env(config, url, |name| std::env::var(name).ok())? {+ FetchCurlDnsAdmission::CurlManaged => Ok(CurlDnsResolution::curl_managed()),FetchCurlDnsAdmission::SharedResolver(target) => {- CurlDnsResolution::resolve_origin(target, config.http_host_resolve().to_vec())+ let policy = config.network_address_policy();+ Ok(CurlDnsResolution::resolve_origin(+ target,+ normalized_http_host_resolve_entries(+ config.http_host_resolve(),+ policy.is_enforced(),+ )?,+ )+ .with_network_address_policy(policy, url.to_string()))}}}@@ -30,20 +43,30 @@ fn curl_dns_admission_with_env(config: &FetchConfig,url: &Url,mut env: impl FnMut(&str) -> Option<String>,-) -> FetchCurlDnsAdmission {- if !matches!(url.scheme(), "http" | "https") || !config.http_host_resolve().is_empty() {- return FetchCurlDnsAdmission::CurlManaged;+) -> Result<FetchCurlDnsAdmission> {+ if !matches!(url.scheme(), "http" | "https") {+ return Ok(FetchCurlDnsAdmission::CurlManaged);}let Some(Host::Domain(host)) = url.host() else {- return FetchCurlDnsAdmission::CurlManaged;+ return Ok(FetchCurlDnsAdmission::CurlManaged);};let Some(port) = url.port_or_known_default() else {- return FetchCurlDnsAdmission::CurlManaged;+ return Ok(FetchCurlDnsAdmission::CurlManaged);};if request_uses_proxy(config, url, host, port, &mut env) {- return FetchCurlDnsAdmission::CurlManaged;+ if config.network_address_policy().is_enforced() {+ bail!(+ "cannot enforce network address policy for proxied hostname `{host}` in `{url}`; the proxy must not resolve an unchecked target hostname"+ );+ }+ return Ok(FetchCurlDnsAdmission::CurlManaged);+ }+ if resolve_host_resolve_override_ips(config.http_host_resolve(), host, port)?.is_some() {+ return Ok(FetchCurlDnsAdmission::CurlManaged);}- FetchCurlDnsAdmission::SharedResolver(DnsTarget::new(host, port))+ Ok(FetchCurlDnsAdmission::SharedResolver(DnsTarget::new(+ host, port,+ )))}fn request_uses_proxy(@@ -150,6 +173,14 @@ mod tests {raw_url: &str,env: &[(&str, &str)],) -> FetchCurlDnsAdmission {+ admission_result(config, raw_url, env).expect("test DNS admission should succeed")+ }++ fn admission_result(+ config: &FetchConfig,+ raw_url: &str,+ env: &[(&str, &str)],+ ) -> Result<FetchCurlDnsAdmission> {let env = env.iter().map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))@@ -180,7 +211,7 @@ mod tests {}#[test]- fn ip_literals_and_host_resolve_configuration_stay_curl_managed() {+ fn ip_literals_and_matching_host_resolve_entries_stay_curl_managed() {let mut config = FetchConfig::default();assert_eq!(@@ -196,6 +227,11 @@ mod tests {admission(&config, "http://example.test/path", &[]),FetchCurlDnsAdmission::CurlManaged);+ assert_eq!(+ admission(&config, "http://other.test/path", &[]),+ shared_target("other.test", 80),+ "an unrelated host-resolve entry must not return DNS ownership to curl"+ );}#[test]@@ -214,6 +250,18 @@ mod tests {);}+ #[test]+ fn enforced_policy_rejects_proxy_resolved_hostname() {+ let mut config = FetchConfig::default();+ config.set_http_proxy(Some("http://proxy.test:8080".to_owned()));+ config.set_network_blocking(true, Vec::new());++ let error = admission_result(&config, "https://api.example.test/path", &[])+ .expect_err("a strict policy cannot verify proxy-side DNS");++ assert!(error.to_string().contains("proxied hostname"));+ }+#[test]fn empty_explicit_proxy_disables_environment_proxy_fallback() {let mut config = FetchConfig::default();diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex a44ea631c..be1824d79 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -390,7 +390,9 @@ impl FetchRuntimeOwner {let owner_started = Arc::new(AtomicBool::new(false));let (curl_runtime, curl_completion_rx) = CurlMultiRuntime::new(curl_runtime_config(config)).expect("failed to start fetch curl multi runtime");- let websocket_connector = curl_runtime.websocket_connector();+ let websocket_connector = curl_runtime+ .websocket_connector()+ .with_network_address_policy(config.network_address_policy());let curl_http = curl_runtime.http_sender();let owner = RuntimeOwner {config: config.clone(),@@ -932,7 +934,10 @@ impl RuntimeOwner {);let label = job.current_url.to_string();- let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);+ let dns_resolution = match curl_dns_resolution(&self.config, &job.current_url) {+ Ok(resolution) => resolution,+ Err(error) => return Err((job.response_tx, error)),+ };let context = ActiveBufferedTransferContext {job,request_cookie_report,@@ -1119,7 +1124,10 @@ impl RuntimeOwner {collector.set_client_hint_response_policy(prepared_request.response_policy);let label = job.current_url.to_string();- let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);+ let dns_resolution = match curl_dns_resolution(&self.config, &job.current_url) {+ Ok(resolution) => resolution,+ Err(error) => return Err((Box::new(job), Some(easy), error)),+ };let context = ActiveStreamingTransferContext {job,request_cookie_report,@@ -1319,7 +1327,10 @@ impl RuntimeOwner {collector.set_client_hint_response_policy(prepared_request.response_policy);let label = job.current_url.to_string();- let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);+ let dns_resolution = match curl_dns_resolution(&self.config, &job.current_url) {+ Ok(resolution) => resolution,+ Err(error) => return Err((Box::new(job), Some(easy), error)),+ };let context = ActiveRawStreamingTransferContext {job,request_cookie_report,diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rsindex 471b84541..774378cd1 100644--- a/moli-fetch/src/tests/mod.rs+++ b/moli-fetch/src/tests/mod.rs@@ -1899,6 +1899,41 @@ fn fetch_client_rejects_private_network_targets() {);}+#[test]+fn fetch_client_checks_the_shared_dns_answer_for_domains() {+ let mut config = FetchConfig::default();+ config.set_http_proxy(Some(String::new()));+ config.set_network_blocking(true, vec![]);++ let error = fetch_with_config_for_test(&config, Request::get("http://localhost/").unwrap())+ .expect_err("the shared localhost DNS answer must be rejected before curl connects");+ let error_chain = format!("{error:#}");++ assert!(+ error_chain.contains("blocked private network address"),+ "unexpected error: {error_chain}"+ );+}++#[test]+fn fetch_client_rejects_proxy_side_dns_under_address_policy() {+ let mut config = FetchConfig::default();+ config.set_http_proxy(Some("http://proxy.test:8080".to_owned()));+ config.set_network_blocking(true, vec![]);++ let error = fetch_with_config_for_test(+ &config,+ Request::get("https://proxy-resolved.example.test/").unwrap(),+ )+ .expect_err("a strict client cannot verify proxy-side DNS");+ let error_chain = format!("{error:#}");++ assert!(+ error_chain.contains("cannot enforce network address policy for proxied hostname"),+ "unexpected error: {error_chain}"+ );+}+#[test]fn fetch_client_rejects_private_network_targets_via_host_resolve_override() {let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("should-not-be-reached")]);diff --git a/moli-fetch/src/tests/websocket_transport.rs b/moli-fetch/src/tests/websocket_transport.rsindex 67dc3f7d6..0968991c5 100644--- a/moli-fetch/src/tests/websocket_transport.rs+++ b/moli-fetch/src/tests/websocket_transport.rs@@ -31,3 +31,24 @@ async fn websocket_connector_follows_fetch_owner_lifetime() -> Result<()> {server.shutdown();Ok(())}++#[test]+fn websocket_connector_inherits_private_network_policy() {+ let mut config = FetchConfig::default();+ config.set_network_blocking(true, Vec::new());+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());++ let error = client+ .handle()+ .websocket_connector()+ .connect(CurlWebSocketRequest::new(+ "ws://127.0.0.1/private".to_owned(),+ ))+ .expect_err("a WebSocket IP literal must use the fetch address policy");++ assert!(+ error+ .to_string()+ .contains("blocked private network address `127.0.0.1`")+ );+}