Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
use std::{num::NonZeroU32, path::PathBuf};
use cidr::AnyIpCidr;
use moli_browser_profile::{BrowserIdentityProfile, DEFAULT_ACCEPT_LANGUAGE};
use moli_curl::{CurlTlsConfig, NetworkAddressPolicy};
use crate::WebBotAuthSigner;
#[derive(Debug, Clone, PartialEq)]
pub struct FetchConfig {
browser_identity: BrowserIdentityProfile,
default_request_headers: Vec<(String, String)>,
request_timeout_ms: u64,
http_connect_timeout_ms: Option<u64>,
obey_robots: bool,
http_proxy: Option<String>,
http_no_proxy: Option<String>,
http_cache_dir: Option<String>,
http_cache_max_bytes: Option<u64>,
http_host_resolve: Vec<String>,
proxy_bearer_token: Option<String>,
http_max_concurrent: Option<NonZeroU32>,
// Scheduler cap for simultaneously active transfers to one origin. This is
// intentionally separate from the libcurl per-host connection-pool cap so
// HTTP/2 can use multiple streams without being limited by the HTTP/1
// socket default.
http_max_host_open: Option<NonZeroU32>,
// Shared HTTP/WS transport cap passed to libcurl's per-host connection
// pool. When unset, use six connections per host/group.
http_max_host_connections: Option<u8>,
// Shared HTTP/WS transport cap for cached/open connections across hosts.
http_max_total_connections: Option<u16>,
// HTTP/2 stream cap; this is separate from HTTP/1 connection count.
http2_max_concurrent_streams: Option<u16>,
http_max_response_size: Option<usize>,
block_private_networks: bool,
block_cidrs: Vec<AnyIpCidr>,
tls: CurlTlsConfig,
web_bot_auth: Option<WebBotAuthSigner>,
}
impl FetchConfig {
pub const DEFAULT_USER_AGENT: &str = moli_browser_profile::DEFAULT_USER_AGENT;
/// Chromium's normal socket pool uses six connections per host/group for
/// HTTP/1-style connection pooling. This is a transport connection cap, not
/// a runtime active-transfer cap and not an HTTP/2 stream cap.
pub const DEFAULT_HTTP_MAX_HOST_CONNECTIONS: u8 = 6;
pub fn user_agent(&self) -> &str {
self.browser_identity.user_agent()
}
pub fn set_user_agent(&mut self, user_agent: impl Into<String>) {
self.browser_identity =
BrowserIdentityProfile::new(user_agent, self.browser_identity.accept_language());
}
pub fn set_browser_identity(&mut self, browser_identity: BrowserIdentityProfile) {
self.browser_identity = browser_identity;
}
pub fn set_user_agent_suffix(&mut self, suffix: impl AsRef<str>) {
self.set_user_agent(format!("{} {}", Self::DEFAULT_USER_AGENT, suffix.as_ref()));
}
pub fn browser_identity(&self) -> &BrowserIdentityProfile {
&self.browser_identity
}
pub fn default_request_headers(&self) -> &[(String, String)] {
&self.default_request_headers
}
pub fn set_default_request_headers(&mut self, headers: Vec<(String, String)>) {
self.default_request_headers = headers;
self.refresh_identity_accept_language();
}
pub fn push_default_request_header(
&mut self,
name: impl Into<String>,
value: impl Into<String>,
) {
self.default_request_headers
.push((name.into(), value.into()));
self.refresh_identity_accept_language();
}
pub fn request_timeout_ms(&self) -> u64 {
self.request_timeout_ms
}
pub fn set_request_timeout_ms(&mut self, request_timeout_ms: u64) {
self.request_timeout_ms = request_timeout_ms;
}
pub fn set_connect_timeout_ms(&mut self, http_connect_timeout_ms: Option<u64>) {
self.http_connect_timeout_ms = http_connect_timeout_ms;
}
pub fn set_obey_robots(&mut self, obey_robots: bool) {
self.obey_robots = obey_robots;
}
pub fn set_proxy_options(
&mut self,
http_proxy: Option<String>,
proxy_bearer_token: Option<String>,
) {
self.http_proxy = http_proxy;
self.proxy_bearer_token = proxy_bearer_token;
}
pub fn set_connection_limits(
&mut self,
http_max_concurrent: Option<NonZeroU32>,
http_max_host_open: Option<NonZeroU32>,
http_max_response_size: Option<usize>,
) {
// These are fetch-runtime scheduler limits. In particular,
// `http_max_host_open` limits active work per origin and does not
// configure libcurl's connection pool.
self.http_max_concurrent = http_max_concurrent;
self.http_max_host_open = http_max_host_open;
self.http_max_response_size = http_max_response_size;
}
pub fn set_transport_connection_limits(
&mut self,
http_max_host_connections: Option<u8>,
http_max_total_connections: Option<u16>,
http2_max_concurrent_streams: Option<u16>,
) {
// Host/total limits count HTTP and WebSocket sockets together. The
// runtime scheduler and HTTP/2 stream limits count work separately.
self.http_max_host_connections = http_max_host_connections;
self.http_max_total_connections = http_max_total_connections;
self.http2_max_concurrent_streams = http2_max_concurrent_streams;
}
pub fn tls_verify_host(&self) -> bool {
self.tls.verify
}
pub fn set_tls_verify_host(&mut self, tls_verify_host: bool) {
self.tls.verify = tls_verify_host;
}
pub fn tls_config(&self) -> &CurlTlsConfig {
&self.tls
}
/// Configure TLS trust and mutual-TLS credentials for this fetch runtime,
/// including navigations and subresources. Client identities are used only
/// when the request's credentials mode allows them for the current URL;
/// CA trust applies regardless of credentials mode.
pub fn set_tls_credentials(
&mut self,
ca_cert: Option<PathBuf>,
client_cert: Option<PathBuf>,
client_key: Option<PathBuf>,
client_cert_password: Option<String>,
) {
self.tls.ca_cert = ca_cert;
self.tls.client_cert = client_cert;
self.tls.client_key = client_key;
self.tls.client_cert_password = client_cert_password;
}
pub fn ca_cert(&self) -> Option<&std::path::Path> {
self.tls.ca_cert.as_deref()
}
pub fn client_cert(&self) -> Option<&std::path::Path> {
self.tls.client_cert.as_deref()
}
pub fn client_key(&self) -> Option<&std::path::Path> {
self.tls.client_key.as_deref()
}
pub fn client_cert_password(&self) -> Option<&str> {
self.tls.client_cert_password.as_deref()
}
pub fn web_bot_auth(&self) -> Option<&WebBotAuthSigner> {
self.web_bot_auth.as_ref()
}
pub fn set_web_bot_auth(&mut self, web_bot_auth: Option<WebBotAuthSigner>) {
self.web_bot_auth = web_bot_auth;
}
pub fn set_http_proxy(&mut self, http_proxy: Option<String>) {
self.http_proxy = http_proxy;
}
pub fn set_http_no_proxy(&mut self, http_no_proxy: Option<String>) {
self.http_no_proxy = http_no_proxy;
}
pub fn set_http_host_resolve(&mut self, entries: Vec<String>) {
self.http_host_resolve = entries;
}
pub fn set_http_cache_dir(&mut self, http_cache_dir: Option<String>) {
self.http_cache_dir = http_cache_dir;
}
pub fn set_http_cache_max_bytes(&mut self, http_cache_max_bytes: Option<u64>) {
self.http_cache_max_bytes = http_cache_max_bytes;
}
pub fn set_network_blocking(
&mut self,
block_private_networks: bool,
block_cidrs: Vec<AnyIpCidr>,
) {
self.block_private_networks = block_private_networks;
self.block_cidrs = block_cidrs;
}
pub fn http_proxy(&self) -> Option<&str> {
self.http_proxy.as_deref()
}
pub fn http_no_proxy(&self) -> Option<&str> {
self.http_no_proxy.as_deref()
}
pub fn http_host_resolve(&self) -> &[String] {
&self.http_host_resolve
}
pub fn http_cache_dir(&self) -> Option<&str> {
self.http_cache_dir.as_deref()
}
pub fn http_cache_max_bytes(&self) -> Option<u64> {
self.http_cache_max_bytes
}
pub fn proxy_bearer_token(&self) -> Option<&str> {
self.proxy_bearer_token.as_deref()
}
pub fn http_connect_timeout_ms(&self) -> Option<u64> {
self.http_connect_timeout_ms
}
pub fn obey_robots(&self) -> bool {
self.obey_robots
}
pub fn http_max_concurrent(&self) -> Option<NonZeroU32> {
self.http_max_concurrent
}
pub fn http_max_host_open(&self) -> Option<NonZeroU32> {
self.http_max_host_open
}
/// Effective libcurl per-host connection cap.
///
/// The default only applies to the transport connection pool. It must not be
/// reused as a scheduler per-origin active-transfer cap, because that would
/// also throttle HTTP/2 streams to the HTTP/1 connection default.
pub fn effective_http_max_host_connections(&self) -> Option<u8> {
match self.http_max_host_connections {
Some(0) => None,
Some(value) => Some(value),
None => Some(Self::DEFAULT_HTTP_MAX_HOST_CONNECTIONS),
}
}
pub fn http_max_host_connections(&self) -> Option<u8> {
self.http_max_host_connections
}
pub fn http_max_total_connections(&self) -> Option<u16> {
self.http_max_total_connections
}
pub fn http2_max_concurrent_streams(&self) -> Option<u16> {
self.http2_max_concurrent_streams
}
pub fn http_max_response_size(&self) -> Option<usize> {
self.http_max_response_size
}
pub fn block_private_networks(&self) -> bool {
self.block_private_networks
}
pub fn block_cidrs(&self) -> &[AnyIpCidr] {
&self.block_cidrs
}
pub(crate) fn network_address_policy(&self) -> NetworkAddressPolicy {
NetworkAddressPolicy::new(self.block_private_networks, self.block_cidrs.clone())
}
}
impl Default for FetchConfig {
fn default() -> Self {
Self {
browser_identity: BrowserIdentityProfile::default(),
default_request_headers: Vec::new(),
request_timeout_ms: 30_000,
http_connect_timeout_ms: None,
obey_robots: false,
http_proxy: None,
http_no_proxy: None,
http_cache_dir: None,
http_cache_max_bytes: Some(100 * 1024 * 1024),
http_host_resolve: Vec::new(),
proxy_bearer_token: None,
http_max_concurrent: None,
http_max_host_open: None,
http_max_host_connections: None,
http_max_total_connections: None,
http2_max_concurrent_streams: None,
http_max_response_size: None,
block_private_networks: false,
block_cidrs: Vec::new(),
tls: CurlTlsConfig::default(),
web_bot_auth: None,
}
}
}
impl FetchConfig {
fn refresh_identity_accept_language(&mut self) {
let accept_language = self
.default_request_headers
.iter()
.find(|(name, _)| name.eq_ignore_ascii_case("accept-language"))
.map(|(_, value)| value.as_str())
.unwrap_or(DEFAULT_ACCEPT_LANGUAGE)
.to_owned();
self.browser_identity = self.browser_identity.with_accept_language(accept_language);
}
}