Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
use anyhow::Context;
use moli_core::network::{
BrowserResourceRuntime, BrowserResourceRuntimeOwner, ResourceRequestClient,
};
use moli_core::page::{CompletedPageCommand, PendingPageCommand};
use super::{
BrowserContext, CdpConnection, TargetNavigationLoadInputs,
state::BrowserContextResourceStorageHandles,
};
impl CdpConnection {
pub(crate) fn invalidate_resource_runtime(&mut self) {
self.active_navigation_engine_mut()
.reset_resource_runtime_without_loaded_page();
}
pub(crate) async fn invalidate_resource_runtime_async(&mut self) {
self.active_navigation_engine_mut()
.reset_resource_runtime_async(None)
.await;
}
pub(crate) fn resource_storage_handles(&self) -> BrowserContextResourceStorageHandles {
self.browser_context
.as_ref()
.map(BrowserContext::resource_storage_handles)
.unwrap_or_else(|| self.initial_storage_partition.resource_storage_handles())
}
pub(crate) fn ensure_resource_request_client(
&mut self,
) -> anyhow::Result<ResourceRequestClient> {
self.apply_active_engine_fetch_overrides();
let storage = self.resource_storage_handles();
let engine = self.active_navigation_engine_mut();
engine
.ensure_resource_runtime_ready_for_navigation_storage(storage.into_navigation_storage())
.context("failed to initialize resource runtime")?;
engine
.resource_request_client()
.context("resource request client unavailable")
}
pub(crate) fn ensure_resource_request_client_for_navigation_load_inputs(
&mut self,
load_inputs: &TargetNavigationLoadInputs,
) -> anyhow::Result<ResourceRequestClient> {
let storage = load_inputs.resource_storage_handles();
let engine = self
.configured_navigation_engine_for_load_inputs_mut(load_inputs)
.context("navigation Page engine unavailable")?;
engine
.ensure_resource_runtime_ready_for_navigation_storage(storage.into_navigation_storage())
.context("failed to initialize resource runtime")?;
engine
.resource_request_client()
.context("resource request client unavailable")
}
pub(super) fn configured_navigation_engine_for_load_inputs_mut(
&mut self,
load_inputs: &TargetNavigationLoadInputs,
) -> Option<&mut moli_core::runtime::NavigationEngine> {
let browser_identity = load_inputs
.browser_identity_override
.clone()
.or_else(|| self.global_browser_identity_override.clone())
.unwrap_or_else(|| self.base_browser_identity.clone());
let http_proxy = load_inputs
.http_proxy_override
.clone()
.or_else(|| self.base_http_proxy.clone());
let http_no_proxy = load_inputs
.http_no_proxy_override
.clone()
.or_else(|| self.base_http_no_proxy.clone());
let tls_verify_host = load_inputs
.tls_verify_host_override
.unwrap_or(self.base_tls_verify_host);
let engine = self.navigation_engine_for_load_inputs_mut(load_inputs)?;
engine.set_browser_identity_override(browser_identity);
engine.set_http_proxy_override(http_proxy);
engine.set_http_no_proxy_override(http_no_proxy);
engine.set_tls_verify_host(tls_verify_host);
engine.set_extra_http_headers(&load_inputs.extra_http_headers);
engine.set_network_offline(load_inputs.network_offline);
engine.set_blocked_url_patterns(&load_inputs.blocked_url_patterns);
engine.set_bypass_service_worker(load_inputs.bypass_service_worker);
engine.set_cache_disabled(load_inputs.cache_disabled);
Some(engine)
}
pub(super) fn navigation_engine_for_load_inputs_mut(
&mut self,
load_inputs: &TargetNavigationLoadInputs,
) -> Option<&mut moli_core::runtime::NavigationEngine> {
let mut engine = match (
load_inputs.browser_context_id.as_deref(),
load_inputs.root_frame_id.as_deref(),
) {
(Some(browser_context_id), Some(target_id)) => {
self.ensure_page_navigation_engine_for_target(browser_context_id, target_id)
}
(Some(browser_context_id), None) => {
let active_target_id = self
.browser_context_by_id(browser_context_id)
.and_then(|context| context.page_targets.active_target_id())
.map(str::to_owned);
if let Some(target_id) = active_target_id {
self.ensure_page_navigation_engine_for_target(browser_context_id, &target_id)
} else {
Some(self.standalone_navigation_engine.ensure_mut())
}
}
(None, _) => Some(self.standalone_navigation_engine.ensure_mut()),
};
if let Some(engine) = engine.as_mut() {
engine.set_document_activity(load_inputs.document_activity);
engine.set_emulated_scrollbars_hidden(load_inputs.scrollbars_hidden);
}
engine
}
pub(crate) fn build_registered_browser_resource_runtime_for_navigation_load_inputs(
&self,
load_inputs: &TargetNavigationLoadInputs,
) -> Result<BrowserResourceRuntime, String> {
let mut fetch_config = self.fetch_config().clone();
let browser_identity = load_inputs
.browser_identity_override
.clone()
.or_else(|| self.global_browser_identity_override.clone())
.unwrap_or_else(|| self.base_browser_identity.clone());
fetch_config.set_browser_identity(browser_identity);
fetch_config.set_http_proxy(
load_inputs
.http_proxy_override
.clone()
.or_else(|| self.base_http_proxy.clone()),
);
fetch_config.set_http_no_proxy(
load_inputs
.http_no_proxy_override
.clone()
.or_else(|| self.base_http_no_proxy.clone()),
);
fetch_config.set_tls_verify_host(
load_inputs
.tls_verify_host_override
.unwrap_or(self.base_tls_verify_host),
);
let storage = load_inputs.resource_storage_handles();
load_inputs
.renderer_runtime
.replace_owned(BrowserResourceRuntimeOwner::new(
&fetch_config,
storage.cookie_store,
))
.map_err(|error| format!("browser context resource owner unavailable: {error}"))
}
#[cfg(test)]
pub(crate) fn ensure_cookie_store(
&mut self,
) -> Result<moli_cookie_jar::SharedBrowserCookieStore, String> {
self.apply_active_engine_fetch_overrides();
let storage = self.resource_storage_handles();
self.active_navigation_engine_mut()
.ensure_cookie_store_for_navigation_storage(storage.into_navigation_storage())
.map_err(|error| format!("failed to initialize loader: {error}"))
}
#[cfg(test)]
pub(crate) async fn reset_resource_runtime_async(&mut self) {
if let Some((engine, slot)) = self
.browser_context
.as_mut()
.and_then(|context| context.page_targets.active_mut())
.and_then(|host| host.navigation_engine_and_runtime_slot_mut())
{
engine
.reset_resource_runtime_async(slot.loaded_page_mut())
.await;
return;
}
self.standalone_navigation_engine
.ensure_mut()
.reset_resource_runtime_async(None)
.await;
}
pub(crate) async fn rebuild_resource_runtime_for_loaded_page_async(&mut self) {
let storage = self.resource_storage_handles();
let rebuild_result = if let Some((engine, slot)) = self
.browser_context
.as_mut()
.and_then(|context| context.page_targets.active_mut())
.and_then(|host| host.navigation_engine_and_runtime_slot_mut())
{
engine
.rebuild_resource_runtime_for_page_with_storage_async(
storage.into_navigation_storage(),
slot.loaded_page_mut(),
)
.await
} else {
self.standalone_navigation_engine
.ensure_mut()
.rebuild_resource_runtime_for_page_with_storage_async(
storage.into_navigation_storage(),
None,
)
.await
};
if rebuild_result.is_err() {
if let Some((engine, slot)) = self
.browser_context
.as_mut()
.and_then(|context| context.page_targets.active_mut())
.and_then(|host| host.navigation_engine_and_runtime_slot_mut())
{
engine
.reset_resource_runtime_async(slot.loaded_page_mut())
.await;
} else {
self.standalone_navigation_engine
.ensure_mut()
.reset_resource_runtime_async(None)
.await;
}
}
}
pub(crate) fn start_rebuild_resource_runtime_for_session_owner(
&mut self,
session_id: Option<&str>,
) -> Result<Option<PendingPageCommand>, String> {
let owner = super::CommandOwnerScope::capture(self, session_id);
self.start_rebuild_resource_runtime_for_owner(&owner)
}
pub(crate) fn start_rebuild_resource_runtime_for_owner(
&mut self,
owner: &super::CommandOwnerScope,
) -> Result<Option<PendingPageCommand>, String> {
let load_inputs = self.navigation_load_inputs_for_owner(owner);
let navigator_identity = load_inputs
.navigator_identity_override
.clone()
.or_else(|| self.global_browser_identity_override.clone())
.unwrap_or_else(|| self.base_browser_identity.clone());
let storage = load_inputs.resource_storage_handles();
let request_client = self
.configured_navigation_engine_for_load_inputs_mut(&load_inputs)
.ok_or_else(|| "navigation Page engine unavailable".to_owned())?
.rebuild_resource_request_client_for_navigation_storage(
storage.into_navigation_storage(),
)
.map_err(|error| format!("failed to rebuild resource runtime: {error}"))?;
let Some(page) = self.resource_runtime_apply_page_for_owner(owner) else {
return Ok(None);
};
page.start_replace_browser_resource_runtime_with_navigator_identity(
&request_client.browser_resource_runtime(),
navigator_identity,
)
.map(Some)
.map_err(|error| format!("failed to update page resource runtime: {error}"))
}
pub(crate) fn finish_rebuild_resource_runtime_for_session_owner(
&mut self,
session_id: Option<&str>,
completion: CompletedPageCommand,
) -> Result<(), String> {
finish_resource_runtime_update_on_current_attachment(
self.resource_runtime_apply_page_for_session_owner(session_id),
completion,
)
}
pub(crate) fn finish_rebuild_resource_runtime_for_owner(
&mut self,
owner: &super::CommandOwnerScope,
completion: CompletedPageCommand,
) -> Result<(), String> {
finish_resource_runtime_update_on_current_attachment(
self.resource_runtime_apply_page_for_owner(owner),
completion,
)
}
fn resource_runtime_apply_page_for_session_owner(
&mut self,
session_id: Option<&str>,
) -> Option<&mut moli_core::page::Page> {
if matches!(
self.session_route(session_id),
Some(super::CdpSessionRoute::Browser)
) {
return self
.browser_context
.as_mut()
.and_then(|bc| bc.active_page_target_mut().runtime_slot.loaded_page_mut());
}
self.loaded_page_mut_for_session_owner(session_id)
}
fn resource_runtime_apply_page_for_owner(
&mut self,
owner: &super::CommandOwnerScope,
) -> Option<&mut moli_core::page::Page> {
if matches!(
owner.resolve_route(self),
Some(super::CdpSessionRoute::Browser)
) {
return self
.browser_context
.as_mut()
.and_then(|bc| bc.active_page_target_mut().runtime_slot.loaded_page_mut());
}
self.loaded_page_mut_for_owner(owner)
}
}
fn finish_resource_runtime_update_on_current_attachment(
page: Option<&mut moli_core::page::Page>,
completion: CompletedPageCommand,
) -> Result<(), String> {
let completion_attachment = completion.renderer_agent_attachment_id();
if let Some(page) = page
&& page.renderer_agent_attachment_id() == completion_attachment
{
return page
.finish_replace_browser_resource_runtime(completion)
.map_err(|error| format!("failed to update page resource runtime: {error}"));
}
completion
.into_unit_page_command_turn()
.map(drop)
.map_err(|error| {
format!("stale resource-runtime update returned an unexpected reply: {error}")
})
}