Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix: prevent DNS rebinding in network policy

ldm0fcc716892589
diff --git a/Cargo.lock b/Cargo.lock
index 5bc31ffc4..a94737256 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2587,6 +2587,7 @@ name = "moli-curl"
version = "0.1.0"
dependencies = [
"anyhow",
+ "cidr",
"crossbeam-channel",
"curl",
"moli-dns-resolver",
diff --git a/moli-curl/Cargo.toml b/moli-curl/Cargo.toml
index 4bed4413c..748b0ddc8 100644
--- a/moli-curl/Cargo.toml
+++ b/moli-curl/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2024"
[dependencies]
anyhow = "1.0.100"
+cidr = "0.3.2"
crossbeam-channel = "0.5.15"
curl = { version = "0.4.49", default-features = false, features = ["brotli", "http2", "poll_7_68_0", "ssl", "static-curl", "websocket"] }
moli-dns-resolver = { path = "../moli-dns-resolver" }
diff --git a/moli-curl/src/dns_adapter.rs b/moli-curl/src/dns_adapter.rs
index 057034471..e41341ffc 100644
--- a/moli-curl/src/dns_adapter.rs
+++ b/moli-curl/src/dns_adapter.rs
@@ -8,6 +8,8 @@ use curl::{
};
use moli_dns_resolver::{DnsCachePartition, DnsLookupResult, DnsResolverService, DnsTarget};
+use crate::NetworkAddressPolicy;
+
/// Curl-side policy for DNS ownership before a transfer enters the multi set.
///
/// `origin == None` means curl owns name resolution. `Some` means the transfer
@@ -25,6 +27,8 @@ struct CurlDnsOriginResolution {
/// Existing caller-provided `CURLOPT_RESOLVE` entries that must remain
/// installed when the generated origin answer is added.
static_entries: Vec<String>,
+ address_policy: NetworkAddressPolicy,
+ policy_target: String,
}
impl CurlDnsResolution {
@@ -33,14 +37,39 @@ impl CurlDnsResolution {
}
pub fn resolve_origin(target: DnsTarget, static_entries: Vec<String>) -> Self {
+ let policy_target = format!("{}:{}", target.host(), target.port());
Self {
origin: Some(Box::new(CurlDnsOriginResolution {
target,
static_entries,
+ address_policy: NetworkAddressPolicy::default(),
+ policy_target,
})),
}
}
+ /// Applies address admission to the shared-resolver result before it is
+ /// installed on curl. The target text is retained for actionable errors.
+ pub fn with_network_address_policy(
+ mut self,
+ address_policy: NetworkAddressPolicy,
+ policy_target: impl Into<String>,
+ ) -> Self {
+ self.set_network_address_policy(address_policy, policy_target);
+ self
+ }
+
+ pub(crate) fn set_network_address_policy(
+ &mut self,
+ address_policy: NetworkAddressPolicy,
+ policy_target: impl Into<String>,
+ ) {
+ if let Some(resolution) = self.origin.as_mut() {
+ resolution.address_policy = address_policy;
+ resolution.policy_target = policy_target.into();
+ }
+ }
+
pub(crate) fn target(&self) -> Option<&DnsTarget> {
self.origin.as_ref().map(|resolution| &resolution.target)
}
@@ -54,6 +83,9 @@ impl CurlDnsResolution {
let Some(resolution) = self.origin.as_ref() else {
return Ok(());
};
+ resolution
+ .address_policy
+ .check_addresses(addresses, &resolution.policy_target)?;
let mut resolve = List::new();
for entry in &resolution.static_entries {
resolve
@@ -259,6 +291,34 @@ mod tests {
assert_eq!(policy.target(), None);
}
+ #[test]
+ fn blocked_address_prevents_installing_the_complete_dns_answer() {
+ let target = DnsTarget::new("example.test", 443);
+ let mut resolution = CurlDnsResolution::resolve_origin(target.clone(), Vec::new())
+ .with_network_address_policy(
+ NetworkAddressPolicy::new(true, Vec::new()),
+ "https://example.test/",
+ );
+ let mut easy = Easy2::new(TestHandler);
+
+ let error = resolution
+ .install(
+ &mut easy,
+ &[
+ IpAddr::from([93, 184, 216, 34]),
+ IpAddr::from([127, 0, 0, 1]),
+ ],
+ )
+ .expect_err("one private answer must reject the complete DNS result");
+
+ assert!(error.to_string().contains("127.0.0.1"));
+ assert_eq!(
+ resolution.target(),
+ Some(&target),
+ "a rejected answer must never transition to an installed state"
+ );
+ }
+
#[test]
fn exact_completion_can_be_claimed_only_once() {
let mut residence = CurlDnsOwnerResidence::<u64, _>::default();
diff --git a/moli-curl/src/lib.rs b/moli-curl/src/lib.rs
index 02c10e1ba..337b3431c 100644
--- a/moli-curl/src/lib.rs
+++ b/moli-curl/src/lib.rs
@@ -2,11 +2,13 @@
mod dns_adapter;
mod http;
+mod network_policy;
mod runtime;
mod tls;
pub mod websocket;
pub use dns_adapter::CurlDnsResolution;
pub use http::{CurlHttpSender, CurlMultiCompletion, CurlMultiJob, CurlOriginKey, CurlSubmitError};
+pub use network_policy::NetworkAddressPolicy;
pub use runtime::{CurlMultiRuntime, CurlMultiRuntimeConfig, CurlTransferId};
pub use tls::CurlTlsConfig;
diff --git a/moli-curl/src/network_policy.rs b/moli-curl/src/network_policy.rs
new file mode 100644
index 000000000..7bf30f349
--- /dev/null
+++ b/moli-curl/src/network_policy.rs
@@ -0,0 +1,137 @@
+use std::net::IpAddr;
+
+use anyhow::{Result, bail};
+use cidr::AnyIpCidr;
+
+/// Admission policy for the concrete addresses a network transport may use.
+///
+/// Callers must apply this policy to every address returned by DNS before
+/// installing that same, complete answer on the transport. Filtering a DNS
+/// answer is deliberately not supported because doing so changes address
+/// selection and fallback semantics.
+#[derive(Debug, Clone, Default)]
+pub struct NetworkAddressPolicy {
+ block_private_networks: bool,
+ block_cidrs: Vec<AnyIpCidr>,
+}
+
+impl NetworkAddressPolicy {
+ pub fn new(block_private_networks: bool, block_cidrs: Vec<AnyIpCidr>) -> Self {
+ Self {
+ block_private_networks,
+ block_cidrs,
+ }
+ }
+
+ pub fn is_enforced(&self) -> bool {
+ self.block_private_networks || !self.block_cidrs.is_empty()
+ }
+
+ pub fn check_addresses(&self, addresses: &[IpAddr], target: &str) -> Result<()> {
+ for &address in addresses {
+ self.check_address(address, target)?;
+ }
+ Ok(())
+ }
+
+ pub fn check_address(&self, address: IpAddr, target: &str) -> Result<()> {
+ if self.block_private_networks && is_private_or_internal_ip(address) {
+ bail!("blocked private network address `{address}` for `{target}`");
+ }
+ let mapped_ipv4 = match address {
+ IpAddr::V6(address) => address.to_ipv4_mapped().map(IpAddr::V4),
+ IpAddr::V4(_) => None,
+ };
+ if let Some(cidr) = self.block_cidrs.iter().find(|cidr| {
+ cidr.contains(&address)
+ || mapped_ipv4
+ .as_ref()
+ .is_some_and(|mapped| cidr.contains(mapped))
+ }) {
+ bail!("blocked address `{address}` for `{target}` because it matches `{cidr}`");
+ }
+ Ok(())
+ }
+}
+
+fn is_private_or_internal_ip(ip: IpAddr) -> bool {
+ match ip {
+ IpAddr::V4(ipv4) => {
+ ipv4.is_private()
+ || ipv4.is_loopback()
+ || ipv4.is_link_local()
+ || ipv4.is_broadcast()
+ || ipv4.is_documentation()
+ || ipv4.is_unspecified()
+ || ipv4.is_multicast()
+ || matches!(ipv4.octets(), [100, second, ..] if (64..=127).contains(&second))
+ || matches!(ipv4.octets(), [198, 18 | 19, ..])
+ || matches!(ipv4.octets(), [240..=255, ..])
+ }
+ IpAddr::V6(ipv6) => {
+ if let Some(ipv4) = ipv6.to_ipv4_mapped() {
+ return is_private_or_internal_ip(IpAddr::V4(ipv4));
+ }
+ ipv6.is_loopback()
+ || ipv6.is_unspecified()
+ || ipv6.is_multicast()
+ || ipv6.is_unicast_link_local()
+ || ipv6.is_unique_local()
+ || (ipv6.segments()[0] == 0x2001 && ipv6.segments()[1] == 0x0db8)
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn private_policy_rejects_ipv4_mapped_loopback() {
+ let policy = NetworkAddressPolicy::new(true, Vec::new());
+
+ let error = policy
+ .check_address("::ffff:127.0.0.1".parse().unwrap(), "wss://example.test/")
+ .expect_err("mapped loopback must be private");
+
+ assert!(
+ error
+ .to_string()
+ .contains("blocked private network address")
+ );
+ }
+
+ #[test]
+ fn every_address_must_pass_policy() {
+ let policy = NetworkAddressPolicy::new(true, Vec::new());
+
+ let error = policy
+ .check_addresses(
+ &[
+ "93.184.216.34".parse().unwrap(),
+ "127.0.0.1".parse().unwrap(),
+ ],
+ "https://example.test/",
+ )
+ .expect_err("one private answer must reject the complete DNS result");
+
+ assert!(error.to_string().contains("127.0.0.1"));
+ }
+
+ #[test]
+ fn ipv4_cidr_rejects_an_ipv4_mapped_ipv6_address() {
+ let policy = NetworkAddressPolicy::new(
+ false,
+ vec!["198.18.0.0/15".parse().expect("test CIDR should parse")],
+ );
+
+ let error = policy
+ .check_address(
+ "::ffff:198.18.0.1".parse().unwrap(),
+ "https://example.test/",
+ )
+ .expect_err("mapped IPv4 must be checked against IPv4 CIDRs");
+
+ assert!(error.to_string().contains("198.18.0.0/15"));
+ }
+}
diff --git a/moli-curl/src/websocket/connector.rs b/moli-curl/src/websocket/connector.rs
index 5445a740a..adc331e18 100644
--- a/moli-curl/src/websocket/connector.rs
+++ b/moli-curl/src/websocket/connector.rs
@@ -3,7 +3,7 @@
use super::{
CurlWebSocketConnection, CurlWebSocketRequest, SESSION_CAPACITY, connection::SessionIo,
};
-use crate::{CurlTransferId, runtime::identity::next_transfer_id};
+use crate::{CurlTransferId, NetworkAddressPolicy, runtime::identity::next_transfer_id};
use anyhow::{Context, Result, bail};
use curl::multi::MultiWaker;
use std::sync::{
@@ -11,6 +11,7 @@ use std::sync::{
atomic::{AtomicBool, Ordering},
};
use tokio::sync::Semaphore;
+use url::{Host, Url};
pub(crate) struct Submission {
pub(super) id: CurlTransferId,
@@ -23,6 +24,7 @@ pub(crate) struct Submission {
#[derive(Clone, Debug)]
pub struct CurlWebSocketConnector {
inner: Arc<ConnectorInner>,
+ network_address_policy: NetworkAddressPolicy,
}
#[derive(Debug)]
@@ -58,13 +60,23 @@ impl CurlWebSocketConnector {
waker,
shutdown,
}),
+ network_address_policy: NetworkAddressPolicy::default(),
}
}
- pub fn connect(&self, request: CurlWebSocketRequest) -> Result<CurlWebSocketConnection> {
+ pub fn with_network_address_policy(
+ mut self,
+ network_address_policy: NetworkAddressPolicy,
+ ) -> Self {
+ self.network_address_policy = network_address_policy;
+ self
+ }
+
+ pub fn connect(&self, mut request: CurlWebSocketRequest) -> Result<CurlWebSocketConnection> {
if self.inner.shutdown.load(Ordering::Acquire) {
bail!("curl WebSocket runtime is closed");
}
+ self.apply_network_address_policy(&mut request)?;
let slot = self
.inner
.slots
@@ -80,4 +92,52 @@ impl CurlWebSocketConnector {
let _ = self.inner.waker.wakeup();
Ok(connection)
}
+
+ fn apply_network_address_policy(&self, request: &mut CurlWebSocketRequest) -> Result<()> {
+ if !self.network_address_policy.is_enforced() {
+ return Ok(());
+ }
+
+ let url = Url::parse(&request.url)
+ .with_context(|| format!("failed to parse WebSocket URL `{}`", request.url))?;
+ let host = url
+ .host()
+ .ok_or_else(|| anyhow::anyhow!("WebSocket URL `{url}` is missing a host"))?;
+
+ match host {
+ Host::Domain(host) => {
+ if request.proxy.is_some() {
+ bail!(
+ "cannot enforce network address policy for proxied WebSocket hostname `{host}` in `{url}`; the proxy must not resolve an unchecked target hostname"
+ );
+ }
+ let port = url
+ .port_or_known_default()
+ .ok_or_else(|| anyhow::anyhow!("WebSocket URL `{url}` has no port"))?;
+ let target = request.dns_resolution.target().ok_or_else(|| {
+ anyhow::anyhow!(
+ "network address policy requires shared DNS resolution for WebSocket hostname `{host}` in `{url}`"
+ )
+ })?;
+ if !target.host().eq_ignore_ascii_case(host) || target.port() != port {
+ bail!(
+ "WebSocket DNS target `{}:{}` does not match policy target `{host}:{port}`",
+ target.host(),
+ target.port()
+ );
+ }
+ request.dns_resolution.set_network_address_policy(
+ self.network_address_policy.clone(),
+ url.to_string(),
+ );
+ }
+ Host::Ipv4(address) => self
+ .network_address_policy
+ .check_address(address.into(), url.as_str())?,
+ Host::Ipv6(address) => self
+ .network_address_policy
+ .check_address(address.into(), url.as_str())?,
+ }
+ Ok(())
+ }
}
diff --git a/moli-curl/src/websocket/tests.rs b/moli-curl/src/websocket/tests.rs
index 7f0e41a8a..8f8e1c0a4 100644
--- a/moli-curl/src/websocket/tests.rs
+++ b/moli-curl/src/websocket/tests.rs
@@ -9,6 +9,8 @@ use tokio::{sync::oneshot, time::timeout};
use tokio_tungstenite::tungstenite::{self, Message, handshake::derive_accept_key};
use super::*;
+use crate::NetworkAddressPolicy;
+use moli_dns_resolver::DnsTarget;
mod readiness;
mod shared;
@@ -16,6 +18,46 @@ mod storage;
const DEADLINE: Duration = Duration::from_secs(10);
+#[test]
+fn strict_connector_rejects_proxy_resolved_websocket_hostname() {
+ let runtime = CurlWebSocketRuntime::new().unwrap();
+ let connector = runtime
+ .connector()
+ .with_network_address_policy(NetworkAddressPolicy::new(true, Vec::new()));
+ let mut request = CurlWebSocketRequest::new("wss://example.test/socket".to_owned());
+ request.proxy = Some("http://proxy.test:8080".to_owned());
+
+ let error = connector
+ .connect(request)
+ .expect_err("a strict connector cannot verify proxy-side DNS");
+
+ assert!(error.to_string().contains("proxied WebSocket hostname"));
+}
+
+#[tokio::test]
+async fn strict_connector_checks_shared_websocket_dns_before_connecting() {
+ let runtime = CurlWebSocketRuntime::new().unwrap();
+ let connector = runtime
+ .connector()
+ .with_network_address_policy(NetworkAddressPolicy::new(true, Vec::new()));
+ let mut request = CurlWebSocketRequest::new("ws://localhost:65534/socket".to_owned());
+ request.dns_resolution =
+ CurlDnsResolution::resolve_origin(DnsTarget::new("localhost", 65534), Vec::new());
+
+ let mut connection = connector.connect(request).unwrap();
+ let event = timeout(DEADLINE, connection.recv())
+ .await
+ .expect("WebSocket DNS policy deadline")
+ .expect("WebSocket DNS policy event");
+
+ match event {
+ CurlWebSocketEvent::Closed { result: Err(error) } => {
+ assert!(error.contains("blocked private network address"))
+ }
+ unexpected => panic!("expected a DNS policy failure, got {unexpected:?}"),
+ }
+}
+
fn server(handler: impl FnOnce(TcpStream) + Send + 'static) -> (String, thread::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("ws://{}/native", listener.local_addr().unwrap());
diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs
index 848bc6d32..520f2ffbc 100644
--- a/moli-fetch/src/blocking/mod.rs
+++ b/moli-fetch/src/blocking/mod.rs
@@ -3,7 +3,7 @@ mod collectors;
use std::{
ffi::{c_char, c_long},
- net::{IpAddr, ToSocketAddrs},
+ net::IpAddr,
str,
time::Duration,
};
@@ -17,7 +17,7 @@ use moli_cookie_jar::{
use moli_url::is_potentially_trustworthy_url;
use moli_url_policy::ensure_http_network_transport_url;
use tracing::debug;
-use url::Url;
+use url::{Host, Url};
pub(crate) use self::cache::{
CachedStreamingResponseLookup, cached_streaming_response_body_exceeds_response_limit,
@@ -645,7 +645,10 @@ pub(crate) fn configure_easy<H: Handler>(
}
if !config.http_host_resolve().is_empty() {
let mut resolve = List::new();
- for entry in normalized_http_host_resolve_entries(config.http_host_resolve())? {
+ let force_permanent = config.network_address_policy().is_enforced();
+ for entry in
+ normalized_http_host_resolve_entries(config.http_host_resolve(), force_permanent)?
+ {
resolve
.append(&entry)
.with_context(|| anyhow!("failed to build curl host resolve entry `{entry}`"))?;
@@ -755,42 +758,31 @@ fn enforce_request_target_policy(config: &FetchConfig, request_url: &Url) -> Res
return Ok(());
}
- let Some(host) = request_url.host_str() else {
+ let Some(host) = request_url.host() else {
return Ok(());
};
+ let host_text = request_url
+ .host_str()
+ .expect("a parsed URL host must have text");
let port = request_url
.port_or_known_default()
.ok_or_else(|| anyhow!("could not determine port for request url `{request_url}`"))?;
- let resolved_ips = resolve_target_ips(config, host, port)
- .with_context(|| anyhow!("failed to resolve request host `{host}` for `{request_url}`"))?;
- for ip in resolved_ips {
- if config.block_private_networks() && is_private_or_internal_ip(ip) {
- bail!("blocked private network address `{ip}` for `{request_url}`");
- }
- if let Some(cidr) = config.block_cidrs().iter().find(|cidr| cidr.contains(&ip)) {
- bail!("blocked address `{ip}` for `{request_url}` because it matches `{cidr}`");
- }
- }
-
- Ok(())
-}
-
-fn resolve_target_ips(config: &FetchConfig, host: &str, port: u16) -> Result<Vec<IpAddr>> {
- if let Some(resolved_ips) =
- resolve_host_resolve_override_ips(config.http_host_resolve(), host, port)?
+ let policy = config.network_address_policy();
+ if let Some(addresses) =
+ resolve_host_resolve_override_ips(config.http_host_resolve(), host_text, port)?
{
- return Ok(resolved_ips);
+ return policy.check_addresses(&addresses, request_url.as_str());
}
- if let Ok(ip) = host.parse::<IpAddr>() {
- return Ok(vec![ip]);
+ match host {
+ Host::Ipv4(address) => policy.check_address(IpAddr::V4(address), request_url.as_str()),
+ Host::Ipv6(address) => policy.check_address(IpAddr::V6(address), request_url.as_str()),
+ Host::Domain(_) => Ok(()),
}
-
- resolve_system_target_ips(host, port)
}
-fn resolve_host_resolve_override_ips(
+pub(crate) fn resolve_host_resolve_override_ips(
entries: &[String],
host: &str,
port: u16,
@@ -827,21 +819,6 @@ fn resolve_host_resolve_override_ips(
Ok(exact_match.or(wildcard_match))
}
-
-fn resolve_system_target_ips(host: &str, port: u16) -> Result<Vec<IpAddr>> {
- let mut resolved = Vec::new();
- for addr in (host, port).to_socket_addrs()? {
- let ip = addr.ip();
- if !resolved.contains(&ip) {
- resolved.push(ip);
- }
- }
- if resolved.is_empty() {
- bail!("no addresses resolved");
- }
- Ok(resolved)
-}
-
enum HttpHostResolveEntry {
Add {
plus_prefix: bool,
@@ -856,7 +833,7 @@ enum HttpHostResolveEntry {
}
impl HttpHostResolveEntry {
- fn curl_entry(&self) -> String {
+ fn curl_entry(&self, force_permanent: bool) -> String {
match self {
Self::Add {
plus_prefix,
@@ -864,7 +841,11 @@ impl HttpHostResolveEntry {
port,
addresses,
} => {
- let prefix = if *plus_prefix { "+" } else { "" };
+ let prefix = if *plus_prefix && !force_permanent {
+ "+"
+ } else {
+ ""
+ };
let addresses = addresses
.iter()
.map(format_http_host_resolve_ip)
@@ -882,10 +863,15 @@ impl HttpHostResolveEntry {
}
}
-fn normalized_http_host_resolve_entries(entries: &[String]) -> Result<Vec<String>> {
+pub(crate) fn normalized_http_host_resolve_entries(
+ entries: &[String],
+ force_permanent: bool,
+) -> Result<Vec<String>> {
entries
.iter()
- .map(|entry| parse_http_host_resolve_entry(entry).map(|entry| entry.curl_entry()))
+ .map(|entry| {
+ parse_http_host_resolve_entry(entry).map(|entry| entry.curl_entry(force_permanent))
+ })
.collect()
}
@@ -1008,31 +994,6 @@ fn format_http_host_resolve_ip(ip: &IpAddr) -> String {
}
}
-fn is_private_or_internal_ip(ip: IpAddr) -> bool {
- match ip {
- IpAddr::V4(ipv4) => {
- ipv4.is_private()
- || ipv4.is_loopback()
- || ipv4.is_link_local()
- || ipv4.is_broadcast()
- || ipv4.is_documentation()
- || ipv4.is_unspecified()
- || ipv4.is_multicast()
- || matches!(ipv4.octets(), [100, second, ..] if (64..=127).contains(&second))
- || matches!(ipv4.octets(), [198, 18 | 19, ..])
- || matches!(ipv4.octets(), [240..=255, ..])
- }
- IpAddr::V6(ipv6) => {
- ipv6.is_loopback()
- || ipv6.is_unspecified()
- || ipv6.is_multicast()
- || ipv6.is_unicast_link_local()
- || ipv6.is_unique_local()
- || (ipv6.segments()[0] == 0x2001 && ipv6.segments()[1] == 0x0db8)
- }
- }
-}
-
fn configure_curl_http_protocol_allowlist<H: Handler>(easy: &mut Easy2<H>) -> Result<()> {
for (option, description) in [
(CURL_OPTION_PROTOCOLS_STR, "request protocols"),
@@ -1586,10 +1547,13 @@ mod tests {
#[test]
fn http_host_resolve_entries_are_normalized_before_curl_configuration() {
- let entries = normalized_http_host_resolve_entries(&[
- " localhost:80: 1.1.1.1 , [2001:db8::1] ".to_owned(),
- " - localhost:80 ".to_owned(),
- ])
+ let entries = normalized_http_host_resolve_entries(
+ &[
+ " localhost:80: 1.1.1.1 , [2001:db8::1] ".to_owned(),
+ " - localhost:80 ".to_owned(),
+ ],
+ false,
+ )
.unwrap();
assert_eq!(
@@ -1601,6 +1565,20 @@ mod tests {
);
}
+ #[test]
+ fn address_policy_makes_temporary_host_resolve_entries_permanent() {
+ let entries = vec!["+example.test:443:93.184.216.34".to_owned()];
+
+ assert_eq!(
+ normalized_http_host_resolve_entries(&entries, false).unwrap(),
+ ["+example.test:443:93.184.216.34"]
+ );
+ assert_eq!(
+ normalized_http_host_resolve_entries(&entries, true).unwrap(),
+ ["example.test:443:93.184.216.34"]
+ );
+ }
+
#[test]
fn third_party_credentialed_request_reports_active_storage_access() {
let config = FetchConfig::default();
diff --git a/moli-fetch/src/config.rs b/moli-fetch/src/config.rs
index b47397f48..a586eccf9 100644
--- a/moli-fetch/src/config.rs
+++ b/moli-fetch/src/config.rs
@@ -2,7 +2,7 @@ use std::{num::NonZeroU32, path::PathBuf};
use cidr::AnyIpCidr;
use moli_browser_profile::{BrowserIdentityProfile, DEFAULT_ACCEPT_LANGUAGE};
-use moli_curl::CurlTlsConfig;
+use moli_curl::{CurlTlsConfig, NetworkAddressPolicy};
use crate::WebBotAuthSigner;
@@ -296,6 +296,10 @@ impl FetchConfig {
pub fn block_cidrs(&self) -> &[AnyIpCidr] {
&self.block_cidrs
}
+
+ pub(crate) fn network_address_policy(&self) -> NetworkAddressPolicy {
+ NetworkAddressPolicy::new(self.block_private_networks, self.block_cidrs.clone())
+ }
}
impl Default for FetchConfig {
diff --git a/moli-fetch/src/dns.rs b/moli-fetch/src/dns.rs
index 81fe87a85..3a18842be 100644
--- a/moli-fetch/src/dns.rs
+++ b/moli-fetch/src/dns.rs
@@ -1,27 +1,40 @@
+use anyhow::{Result, bail};
use moli_curl::CurlDnsResolution;
use moli_dns_resolver::DnsTarget;
use url::{Host, Url};
-use crate::FetchConfig;
+use crate::{
+ FetchConfig,
+ blocking::{normalized_http_host_resolve_entries, resolve_host_resolve_override_ips},
+};
/// Fetch-side DNS admission decision.
///
/// The shared resolver is used only when Fetch can prove that curl will
-/// connect directly to an HTTP(S) origin. Proxy traffic stays curl-managed:
-/// the proxy, rather than the local process, may be responsible for resolving
-/// the origin hostname. IP literals and explicit host-resolve configuration
-/// already have exact routing and must not be resolved again.
+/// connect directly to an HTTP(S) origin. Proxy traffic stays curl-managed
+/// only when no address policy is active; otherwise a proxy-resolved hostname
+/// cannot be verified locally and is rejected. IP literals and matching
+/// explicit host-resolve entries already have exact routing and are checked
+/// synchronously before this decision.
#[derive(Debug, Clone, PartialEq, Eq)]
enum FetchCurlDnsAdmission {
CurlManaged,
SharedResolver(DnsTarget),
}
-pub(crate) fn curl_dns_resolution(config: &FetchConfig, url: &Url) -> CurlDnsResolution {
- match curl_dns_admission_with_env(config, url, |name| std::env::var(name).ok()) {
- FetchCurlDnsAdmission::CurlManaged => CurlDnsResolution::curl_managed(),
+pub(crate) fn curl_dns_resolution(config: &FetchConfig, url: &Url) -> Result<CurlDnsResolution> {
+ match curl_dns_admission_with_env(config, url, |name| std::env::var(name).ok())? {
+ FetchCurlDnsAdmission::CurlManaged => Ok(CurlDnsResolution::curl_managed()),
FetchCurlDnsAdmission::SharedResolver(target) => {
- CurlDnsResolution::resolve_origin(target, config.http_host_resolve().to_vec())
+ let policy = config.network_address_policy();
+ Ok(CurlDnsResolution::resolve_origin(
+ target,
+ normalized_http_host_resolve_entries(
+ config.http_host_resolve(),
+ policy.is_enforced(),
+ )?,
+ )
+ .with_network_address_policy(policy, url.to_string()))
}
}
}
@@ -30,20 +43,30 @@ fn curl_dns_admission_with_env(
config: &FetchConfig,
url: &Url,
mut env: impl FnMut(&str) -> Option<String>,
-) -> FetchCurlDnsAdmission {
- if !matches!(url.scheme(), "http" | "https") || !config.http_host_resolve().is_empty() {
- return FetchCurlDnsAdmission::CurlManaged;
+) -> Result<FetchCurlDnsAdmission> {
+ if !matches!(url.scheme(), "http" | "https") {
+ return Ok(FetchCurlDnsAdmission::CurlManaged);
}
let Some(Host::Domain(host)) = url.host() else {
- return FetchCurlDnsAdmission::CurlManaged;
+ return Ok(FetchCurlDnsAdmission::CurlManaged);
};
let Some(port) = url.port_or_known_default() else {
- return FetchCurlDnsAdmission::CurlManaged;
+ return Ok(FetchCurlDnsAdmission::CurlManaged);
};
if request_uses_proxy(config, url, host, port, &mut env) {
- return FetchCurlDnsAdmission::CurlManaged;
+ if config.network_address_policy().is_enforced() {
+ bail!(
+ "cannot enforce network address policy for proxied hostname `{host}` in `{url}`; the proxy must not resolve an unchecked target hostname"
+ );
+ }
+ return Ok(FetchCurlDnsAdmission::CurlManaged);
+ }
+ if resolve_host_resolve_override_ips(config.http_host_resolve(), host, port)?.is_some() {
+ return Ok(FetchCurlDnsAdmission::CurlManaged);
}
- FetchCurlDnsAdmission::SharedResolver(DnsTarget::new(host, port))
+ Ok(FetchCurlDnsAdmission::SharedResolver(DnsTarget::new(
+ host, port,
+ )))
}
fn request_uses_proxy(
@@ -150,6 +173,14 @@ mod tests {
raw_url: &str,
env: &[(&str, &str)],
) -> FetchCurlDnsAdmission {
+ admission_result(config, raw_url, env).expect("test DNS admission should succeed")
+ }
+
+ fn admission_result(
+ config: &FetchConfig,
+ raw_url: &str,
+ env: &[(&str, &str)],
+ ) -> Result<FetchCurlDnsAdmission> {
let env = env
.iter()
.map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))
@@ -180,7 +211,7 @@ mod tests {
}
#[test]
- fn ip_literals_and_host_resolve_configuration_stay_curl_managed() {
+ fn ip_literals_and_matching_host_resolve_entries_stay_curl_managed() {
let mut config = FetchConfig::default();
assert_eq!(
@@ -196,6 +227,11 @@ mod tests {
admission(&config, "http://example.test/path", &[]),
FetchCurlDnsAdmission::CurlManaged
);
+ assert_eq!(
+ admission(&config, "http://other.test/path", &[]),
+ shared_target("other.test", 80),
+ "an unrelated host-resolve entry must not return DNS ownership to curl"
+ );
}
#[test]
@@ -214,6 +250,18 @@ mod tests {
);
}
+ #[test]
+ fn enforced_policy_rejects_proxy_resolved_hostname() {
+ let mut config = FetchConfig::default();
+ config.set_http_proxy(Some("http://proxy.test:8080".to_owned()));
+ config.set_network_blocking(true, Vec::new());
+
+ let error = admission_result(&config, "https://api.example.test/path", &[])
+ .expect_err("a strict policy cannot verify proxy-side DNS");
+
+ assert!(error.to_string().contains("proxied hostname"));
+ }
+
#[test]
fn empty_explicit_proxy_disables_environment_proxy_fallback() {
let mut config = FetchConfig::default();
diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs
index a44ea631c..be1824d79 100644
--- a/moli-fetch/src/runtime.rs
+++ b/moli-fetch/src/runtime.rs
@@ -390,7 +390,9 @@ impl FetchRuntimeOwner {
let owner_started = Arc::new(AtomicBool::new(false));
let (curl_runtime, curl_completion_rx) = CurlMultiRuntime::new(curl_runtime_config(config))
.expect("failed to start fetch curl multi runtime");
- let websocket_connector = curl_runtime.websocket_connector();
+ let websocket_connector = curl_runtime
+ .websocket_connector()
+ .with_network_address_policy(config.network_address_policy());
let curl_http = curl_runtime.http_sender();
let owner = RuntimeOwner {
config: config.clone(),
@@ -932,7 +934,10 @@ impl RuntimeOwner {
);
let label = job.current_url.to_string();
- let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);
+ let dns_resolution = match curl_dns_resolution(&self.config, &job.current_url) {
+ Ok(resolution) => resolution,
+ Err(error) => return Err((job.response_tx, error)),
+ };
let context = ActiveBufferedTransferContext {
job,
request_cookie_report,
@@ -1119,7 +1124,10 @@ impl RuntimeOwner {
collector.set_client_hint_response_policy(prepared_request.response_policy);
let label = job.current_url.to_string();
- let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);
+ let dns_resolution = match curl_dns_resolution(&self.config, &job.current_url) {
+ Ok(resolution) => resolution,
+ Err(error) => return Err((Box::new(job), Some(easy), error)),
+ };
let context = ActiveStreamingTransferContext {
job,
request_cookie_report,
@@ -1319,7 +1327,10 @@ impl RuntimeOwner {
collector.set_client_hint_response_policy(prepared_request.response_policy);
let label = job.current_url.to_string();
- let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);
+ let dns_resolution = match curl_dns_resolution(&self.config, &job.current_url) {
+ Ok(resolution) => resolution,
+ Err(error) => return Err((Box::new(job), Some(easy), error)),
+ };
let context = ActiveRawStreamingTransferContext {
job,
request_cookie_report,
diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rs
index 471b84541..774378cd1 100644
--- a/moli-fetch/src/tests/mod.rs
+++ b/moli-fetch/src/tests/mod.rs
@@ -1899,6 +1899,41 @@ fn fetch_client_rejects_private_network_targets() {
);
}
+#[test]
+fn fetch_client_checks_the_shared_dns_answer_for_domains() {
+ let mut config = FetchConfig::default();
+ config.set_http_proxy(Some(String::new()));
+ config.set_network_blocking(true, vec![]);
+
+ let error = fetch_with_config_for_test(&config, Request::get("http://localhost/").unwrap())
+ .expect_err("the shared localhost DNS answer must be rejected before curl connects");
+ let error_chain = format!("{error:#}");
+
+ assert!(
+ error_chain.contains("blocked private network address"),
+ "unexpected error: {error_chain}"
+ );
+}
+
+#[test]
+fn fetch_client_rejects_proxy_side_dns_under_address_policy() {
+ let mut config = FetchConfig::default();
+ config.set_http_proxy(Some("http://proxy.test:8080".to_owned()));
+ config.set_network_blocking(true, vec![]);
+
+ let error = fetch_with_config_for_test(
+ &config,
+ Request::get("https://proxy-resolved.example.test/").unwrap(),
+ )
+ .expect_err("a strict client cannot verify proxy-side DNS");
+ let error_chain = format!("{error:#}");
+
+ assert!(
+ error_chain.contains("cannot enforce network address policy for proxied hostname"),
+ "unexpected error: {error_chain}"
+ );
+}
+
#[test]
fn fetch_client_rejects_private_network_targets_via_host_resolve_override() {
let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("should-not-be-reached")]);
diff --git a/moli-fetch/src/tests/websocket_transport.rs b/moli-fetch/src/tests/websocket_transport.rs
index 67dc3f7d6..0968991c5 100644
--- a/moli-fetch/src/tests/websocket_transport.rs
+++ b/moli-fetch/src/tests/websocket_transport.rs
@@ -31,3 +31,24 @@ async fn websocket_connector_follows_fetch_owner_lifetime() -> Result<()> {
server.shutdown();
Ok(())
}
+
+#[test]
+fn websocket_connector_inherits_private_network_policy() {
+ let mut config = FetchConfig::default();
+ config.set_network_blocking(true, Vec::new());
+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());
+
+ let error = client
+ .handle()
+ .websocket_connector()
+ .connect(CurlWebSocketRequest::new(
+ "ws://127.0.0.1/private".to_owned(),
+ ))
+ .expect_err("a WebSocket IP literal must use the fetch address policy");
+
+ assert!(
+ error
+ .to_string()
+ .contains("blocked private network address `127.0.0.1`")
+ );
+}