Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
Refactor network fetch errors as anyhow context
ldm0595a538bcf57
diff --git a/moli-core/src/runtime/lifecycle_fetch.rs b/moli-core/src/runtime/lifecycle_fetch.rsindex 4ec3db70b..b3a178574 100644--- a/moli-core/src/runtime/lifecycle_fetch.rs+++ b/moli-core/src/runtime/lifecycle_fetch.rs@@ -8,8 +8,8 @@ use super::{Browser, FetchDeadline, FetchedDocument, RenderedDomWaitUntil, RendererLifecycleDecider,RendererLifecycleDecision, RendererLifecycleSnapshot, RendererReplyBoundary,};-use anyhow::{Context, Result, anyhow};-use moli_fetch::Request;+use anyhow::{Result, anyhow};+use moli_fetch::{NetworkFetchFailureContext, Request};use std::time::Duration;impl Browser {@@ -67,10 +67,17 @@ impl Browser {Some(decider),).await- .with_context(|| {- anyhow!(- "failed while applying the {wait_until:?} lifecycle-target decision or following its successor navigation"- )+ .map_err(|error| {+ // This typed context already supplies the caller-facing fetch+ // message. Keep it outermost instead of obscuring it with a+ // lifecycle layer that adds no actionable network information.+ if error.is::<NetworkFetchFailureContext>() {+ error+ } else {+ error.context(anyhow!(+ "failed while applying the {wait_until:?} lifecycle-target decision or following its successor navigation"+ ))+ }})}}diff --git a/moli-fetch/src/client.rs b/moli-fetch/src/client.rsindex a6374875c..928815881 100644--- a/moli-fetch/src/client.rs+++ b/moli-fetch/src/client.rs@@ -7,7 +7,7 @@ use crate::{FetchCancelHandle, FetchConfig, NetworkFetchResult, RawResponse, Request, Response,StreamingHtmlResponse, StreamingRawResponse,client_hints::ClientHintPreferences,- network_fetch_result::{NetworkFetchFailure, NetworkObservationRecorder},+ network_fetch_result::{NetworkFetchFailureContext, NetworkObservationRecorder},runtime::{FetchRuntimeHandle, FetchRuntimeJoinReport, FetchRuntimeOwner},};@@ -268,10 +268,11 @@ fn network_fetch_result_from_result<R>(response,observation_journal,)),- Err(source) => match source.downcast::<NetworkFetchFailure>() {- Ok(failure) => Err(failure.into()),- Err(source) => Err(NetworkFetchFailure::new(source, observation_journal).into()),- },+ Err(source) if source.is::<NetworkFetchFailureContext>() => Err(source),+ Err(source) => Err(NetworkFetchFailureContext::attach(+ source,+ observation_journal,+ )),}}diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rsindex 95b055bfe..0914dd20f 100644--- a/moli-fetch/src/lib.rs+++ b/moli-fetch/src/lib.rs@@ -47,7 +47,7 @@ pub use headers::{pub use moli_cookie_jar::SharedBrowserCookieStore as SharedCookieStore;pub use moli_web_bot_auth::{WebBotAuthProfile, WebBotAuthSigner};pub use network_fetch_result::{- NetworkExchangeObservation, NetworkFetchFailure, NetworkFetchFailureRequestContext,+ NetworkExchangeObservation, NetworkFetchFailureContext, NetworkFetchFailureRequestContext,NetworkFetchResult, NetworkObservationJournal, NetworkRequestObservation,NetworkResponseObservation,};diff --git a/moli-fetch/src/network_fetch_result.rs b/moli-fetch/src/network_fetch_result.rsindex 228969e9f..a21627f1f 100644--- a/moli-fetch/src/network_fetch_result.rs+++ b/moli-fetch/src/network_fetch_result.rs@@ -1,4 +1,4 @@-use std::{error::Error as StdError, fmt, sync::Arc};+use std::{fmt, sync::Arc};use moli_cookie_jar::StoredCookieQueryReport;use parking_lot::Mutex;@@ -368,10 +368,12 @@ fn parse_status_line(line: &str) -> Option<u16> {.ok()}-/// A fetch failure plus transport observations completed before that failure.-#[derive(Debug)]-pub struct NetworkFetchFailure {- source: anyhow::Error,+/// Machine-readable context attached to a network fetch error.+///+/// The underlying cause remains owned by [`anyhow::Error`]. This context only+/// carries the transport and request state needed by browser-facing protocol+/// consumers when a transfer fails before response metadata is available.+pub struct NetworkFetchFailureContext {observation_journal: NetworkObservationJournal,network_error_text: &'static str,request_context: Option<NetworkFetchFailureRequestContext>,@@ -429,32 +431,30 @@ impl NetworkFetchFailureRequestContext {}}-impl NetworkFetchFailure {- pub(crate) fn new(+impl NetworkFetchFailureContext {+ pub(crate) fn attach(source: anyhow::Error,observation_journal: NetworkObservationJournal,- ) -> Self {+ ) -> anyhow::Error {let network_error_text = crate::error::browser_network_error_text(&source);- Self {- source,+ source.context(Self {observation_journal,network_error_text,request_context: None,- }+ })}- pub(crate) fn with_request_context(+ pub(crate) fn attach_with_request_context(source: anyhow::Error,observation_journal: NetworkObservationJournal,request_context: NetworkFetchFailureRequestContext,- ) -> Self {+ ) -> anyhow::Error {let network_error_text = crate::error::browser_network_error_text(&source);- Self {- source,+ source.context(Self {observation_journal,network_error_text,request_context: Some(request_context),- }+ })}pub fn observation_journal(&self) -> &NetworkObservationJournal {@@ -468,21 +468,44 @@ impl NetworkFetchFailure {pub fn request_context(&self) -> Option<&NetworkFetchFailureRequestContext> {self.request_context.as_ref()}-- pub fn into_parts(self) -> (anyhow::Error, NetworkObservationJournal) {- (self.source, self.observation_journal)- }}-impl fmt::Display for NetworkFetchFailure {+impl fmt::Display for NetworkFetchFailureContext {fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {- self.source.fmt(formatter)+ match &self.request_context {+ Some(request_context) => {+ write!(+ formatter,+ "failed to fetch `{}`",+ request_context.current_url+ )+ }+ None => formatter.write_str("network fetch failed"),+ }}}-impl StdError for NetworkFetchFailure {- fn source(&self) -> Option<&(dyn StdError + 'static)> {- Some(self.source.as_ref())+impl fmt::Debug for NetworkFetchFailureContext {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ let mut debug = formatter.debug_struct("NetworkFetchFailureContext");+ debug+ .field(+ "observed_exchange_count",+ &self.observation_journal.exchanges().len(),+ )+ .field("network_error_text", &self.network_error_text);+ if let Some(request_context) = &self.request_context {+ debug+ .field("current_url", &request_context.current_url)+ .field("request_method", &request_context.request_method)+ .field("has_request_body", &request_context.request_body.is_some())+ .field(+ "request_header_count",+ &request_context.request_headers.len(),+ )+ .field("redirect_count", &request_context.redirect_chain.len());+ }+ debug.finish()}}@@ -557,6 +580,55 @@ impl<R> NetworkFetchResult<R> {mod tests {use super::*;+ #[test]+ fn network_fetch_failure_context_preserves_source_without_repeating_it() {+ let source = std::io::Error::other("transport failure sentinel");+ let error = NetworkFetchFailureContext::attach_with_request_context(+ anyhow::Error::new(source),+ NetworkObservationJournal::default(),+ NetworkFetchFailureRequestContext::new(+ Url::parse("http://localhost/").expect("test URL should parse"),+ "GET".to_owned(),+ None,+ Vec::new(),+ Vec::new(),+ ),+ )+ .context("outer failure sentinel");++ let report = format!("{error:?}");++ assert!(+ report.contains("failed to fetch `http://localhost/`"),+ "{report}"+ );+ assert_eq!(report.matches("transport failure sentinel").count(), 1);+ assert!(error.downcast_ref::<NetworkFetchFailureContext>().is_some());+ assert!(error.downcast_ref::<std::io::Error>().is_some());+ }++ #[test]+ fn network_fetch_failure_debug_redacts_request_values() {+ let failure = NetworkFetchFailureContext {+ observation_journal: NetworkObservationJournal::default(),+ network_error_text: "net::ERR_FAILED",+ request_context: Some(NetworkFetchFailureRequestContext::new(+ Url::parse("https://example.test/").expect("test URL should parse"),+ "POST".to_owned(),+ Some(b"secret request body".to_vec()),+ vec![("Authorization".to_owned(), "secret token".to_owned())],+ Vec::new(),+ )),+ };++ let report = format!("{failure:?}");++ assert!(!report.contains("secret request body"), "{report}");+ assert!(!report.contains("secret token"), "{report}");+ assert!(report.contains("has_request_body: true"), "{report}");+ assert!(report.contains("request_header_count: 1"), "{report}");+ }+#[test]fn recorder_preserves_redirect_exchange_order_and_raw_response_status() {let recorder = NetworkObservationRecorder::default();diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex 771312bf3..dc54bc03c 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -33,7 +33,7 @@ use tokio::sync::{mpsc, oneshot};use url::{Host, Url};use crate::{- FetchCancelHandle, FetchConfig, NegotiatedHttpVersion, NetworkFetchFailure,+ FetchCancelHandle, FetchConfig, NegotiatedHttpVersion, NetworkFetchFailureContext,NetworkFetchFailureRequestContext, NetworkRequestExtraInfo, NetworkResponseExtraInfo,RawResponse, RedirectInfo, Request, Response, ResponseHead, StreamingHtmlResponse,StreamingRawResponse,@@ -3615,13 +3615,13 @@ fn network_fetch_failure_for_request(redirect_chain: &[RedirectInfo],error: anyhow::Error,) -> anyhow::Error {- if error.downcast_ref::<NetworkFetchFailure>().is_some() {+ if error.is::<NetworkFetchFailureContext>() {return error;}let Some(recorder) = request.network_observation_recorder() else {return error;};- NetworkFetchFailure::with_request_context(+ NetworkFetchFailureContext::attach_with_request_context(error,recorder.snapshot(),NetworkFetchFailureRequestContext::new(@@ -3632,7 +3632,6 @@ fn network_fetch_failure_for_request(redirect_chain.to_vec(),),)- .into()}#[cfg(test)]diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rsindex e6d7b41e1..35bc0a6f3 100644--- a/moli-fetch/src/tests/mod.rs+++ b/moli-fetch/src/tests/mod.rs@@ -25,7 +25,7 @@ use url::Url;use crate::{BrowserNavigationRequestKind, BrowserRequestMetadata, FetchCancelHandle, FetchClient,FetchClientHandle, FetchConfig, FetchPriorityHint, FetchRuntimeJoinStatus,- NegotiatedHttpVersion, NetworkFetchFailure, RawResponse, Request, RequestAuth,+ NegotiatedHttpVersion, NetworkFetchFailureContext, RawResponse, Request, RequestAuth,RequestAuthScheme, RequestAuthTarget, RequestCacheMode, RequestCredentialsMode, RequestMode,RequestRedirectMode, RequestResourceType, Response, ResponseBody, ResponseHead,ScriptFetchRequestMetadata, ScriptFetchSchedulerPriority, StreamingResponseCollector,@@ -3608,7 +3608,7 @@ async fn network_metadata_failure_preserves_request_sent_before_empty_response().await.expect_err("empty response should fail");let failure = error- .downcast_ref::<NetworkFetchFailure>()+ .downcast_ref::<NetworkFetchFailureContext>().expect("metadata fetch failure should retain its typed observation journal");let [exchange] = failure.observation_journal().exchanges() else {panic!("expected exactly one observed exchange");@@ -3668,7 +3668,7 @@ async fn network_metadata_failure_preserves_redirect_chain_and_final_request() -.await.expect_err("redirect final request should fail without a response");let failure = error- .downcast_ref::<NetworkFetchFailure>()+ .downcast_ref::<NetworkFetchFailureContext>().expect("metadata fetch failure should retain typed request context");let context = failure.request_context()diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rsindex 3210f3227..ee618eaa7 100644--- a/moli-protocol/src/conn/runtime_load.rs+++ b/moli-protocol/src/conn/runtime_load.rs@@ -9,7 +9,7 @@ use moli_core::{},};use moli_fetch::{- BrowserNavigationRequestKind, FetchCancelHandle, FetchConfig, NetworkFetchFailure,+ BrowserNavigationRequestKind, FetchCancelHandle, FetchConfig, NetworkFetchFailureContext,NetworkFetchResult, NetworkObservationJournal, RawResponse, Request, ResponseHead,StreamingRawResponse, url_pattern_matches,};@@ -906,7 +906,7 @@ impl BackgroundNavigationLoadJob {let navigation_response = match navigation_response {Ok(response) => response,Err(error) => {- if let Some(failure) = error.downcast_ref::<NetworkFetchFailure>() {+ if let Some(failure) = error.downcast_ref::<NetworkFetchFailureContext>() {let unreachable_url = failure.request_context().map(|request_context| request_context.current_url().clone())@@ -929,7 +929,7 @@ impl BackgroundNavigationLoadJob {}tracing::debug!(url = %self.raw_url,- error = %error,+ error = ?error,network_error_text = failure.network_error_text(),"main document transport failed before response metadata");diff --git a/moli/src/app.rs b/moli/src/app.rsindex 78edd42a9..d1f42fb04 100644--- a/moli/src/app.rs+++ b/moli/src/app.rs@@ -16,7 +16,7 @@ use clap::Parser;use moli_core::runtime::{Browser, FetchedDocument, NavigationRuntimeConfig, storage_partition::StoragePartitionState,};-use moli_fetch::{Request, ensure_http_status_success};+use moli_fetch::{NetworkFetchFailureContext, Request, ensure_http_status_success};use moli_protocol_server::ProtocolServer;use self::{http_error_navigation::is_http_error_status, readiness::ReadinessPlan};@@ -52,7 +52,7 @@ pub async fn run_cli_with_config<W: Write>(Ok(document) => document,Err(error) => {finalize_fetch_browser(browser);- return Err(error).with_context(|| anyhow!("failed to fetch `{}`", args.url));+ return Err(with_fetch_context(error, &args.url));}};@@ -109,7 +109,7 @@ pub async fn run_cli_with_config<W: Write>();}finalize_fetch_browser(browser);- return Err(error).with_context(|| anyhow!("failed to fetch `{}`", args.url));+ return Err(with_fetch_context(error, &args.url));}if let Err(error) = readiness.wait_for_page(&browser, &mut page).await {@@ -120,7 +120,7 @@ pub async fn run_cli_with_config<W: Write>();}finalize_fetch_browser(browser);- return Err(error).with_context(|| anyhow!("failed to fetch `{}`", args.url));+ return Err(with_fetch_context(error, &args.url));}if args.delay_ms > 0 {@@ -165,6 +165,16 @@ fn build_fetch_request(url: &str, config: &AppConfig) -> Result<Request> {Ok(request)}+fn with_fetch_context(error: anyhow::Error, url: &str) -> anyhow::Error {+ // Network failures already carry a URL-aware typed context. Adding the+ // same CLI context again would print two "failed to fetch" chain entries.+ if error.is::<NetworkFetchFailureContext>() {+ error+ } else {+ error.context(anyhow!("failed to fetch `{url}`"))+ }+}+fn load_cookie_state(browser: &Browser, config: &AppConfig) -> Result<()> {browser.import_cookies(load_cookie_state_cookies(config)?)?;Ok(())