Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): preserve readiness deadlines and diagnostics
ldm054f453cdd0b0
diff --git a/moli-browser-profile/src/profile.rs b/moli-browser-profile/src/profile.rsindex 519ab2a02..1e75711fb 100644--- a/moli-browser-profile/src/profile.rs+++ b/moli-browser-profile/src/profile.rs@@ -20,12 +20,7 @@ impl BrowserProfile {}pub fn open_paths(paths: BrowserProfilePaths) -> Result<Self> {- let profile_lock = BrowserProfileLock::acquire(&paths).with_context(|| {- anyhow!(- "failed to acquire browser profile lock `{}`",- paths.lock_path.display()- )- })?;+ let profile_lock = BrowserProfileLock::acquire(&paths)?;let manifest = ensure_profile_manifest(&paths).with_context(|| {anyhow!("failed to initialize browser profile manifest `{}`",diff --git a/moli-core/src/runtime/fetch_deadline.rs b/moli-core/src/runtime/fetch_deadline.rsindex 96b23956d..628142308 100644--- a/moli-core/src/runtime/fetch_deadline.rs+++ b/moli-core/src/runtime/fetch_deadline.rs@@ -1,7 +1,96 @@//! One absolute timeout budget shared by every fetch-readiness phase.-use anyhow::{Context, Result, anyhow, bail};-use std::{future::Future, time::Duration};+use anyhow::{Context, Result, anyhow};+use std::{fmt, future::Future, time::Duration};++/// The concrete operation consuming the shared fetch-readiness deadline.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum FetchTimeoutPhase {+ WaitingForResponseHeaders,+ StreamingMainBody,+ ProcessingMainDocument,+ WaitingForParserBlockingScript,+ WaitingForParserBlockingStylesheet,+ WaitingForDomContentLoaded,+ WaitingForLoad,+ WaitingForSubresourceResponse,+ WaitingForSelector,+ WaitingForScript,+}++impl FetchTimeoutPhase {+ pub(super) fn from_renderer_phase(phase: moli_renderer_v8::RendererPageCreationPhase) -> Self {+ match phase {+ moli_renderer_v8::RendererPageCreationPhase::StreamingMainBody => {+ Self::StreamingMainBody+ }+ moli_renderer_v8::RendererPageCreationPhase::ProcessingMainDocument => {+ Self::ProcessingMainDocument+ }+ moli_renderer_v8::RendererPageCreationPhase::WaitingForParserBlockingScript => {+ Self::WaitingForParserBlockingScript+ }+ moli_renderer_v8::RendererPageCreationPhase::WaitingForParserBlockingStylesheet => {+ Self::WaitingForParserBlockingStylesheet+ }+ moli_renderer_v8::RendererPageCreationPhase::WaitingForDomContentLoaded => {+ Self::WaitingForDomContentLoaded+ }+ moli_renderer_v8::RendererPageCreationPhase::WaitingForLoad => Self::WaitingForLoad,+ }+ }+}++impl fmt::Display for FetchTimeoutPhase {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter.write_str(match self {+ Self::WaitingForResponseHeaders => "waiting for response headers",+ Self::StreamingMainBody => "streaming main body",+ Self::ProcessingMainDocument => "processing main document",+ Self::WaitingForParserBlockingScript => "waiting for parser-blocking script",+ Self::WaitingForParserBlockingStylesheet => "waiting for parser-blocking stylesheet",+ Self::WaitingForDomContentLoaded => "waiting for DOMContentLoaded",+ Self::WaitingForLoad => "waiting for load",+ Self::WaitingForSubresourceResponse => "waiting for a subresource response",+ Self::WaitingForSelector => "waiting for a selector",+ Self::WaitingForScript => "waiting for a script to become truthy",+ })+ }+}++/// A deadline failure that retains the operation active when the budget won.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub struct FetchReadinessTimeout {+ timeout: Duration,+ phase: FetchTimeoutPhase,+}++impl FetchReadinessTimeout {+ pub fn new(timeout: Duration, phase: FetchTimeoutPhase) -> Self {+ Self { timeout, phase }+ }++ pub fn timeout(self) -> Duration {+ self.timeout+ }++ pub fn phase(self) -> FetchTimeoutPhase {+ self.phase+ }+}++impl fmt::Display for FetchReadinessTimeout {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ write!(+ formatter,+ "fetch readiness timed out after {} ms while {}",+ self.timeout.as_millis(),+ self.phase+ )+ }+}++impl std::error::Error for FetchReadinessTimeout {}/// An absolute deadline for a complete fetch-readiness plan.///@@ -44,23 +133,23 @@ impl FetchDeadline {self.at}- pub(super) async fn wait<T, F>(self, phase: &'static str, future: F) -> Result<T>+ pub(super) async fn wait<T, F>(self, phase: FetchTimeoutPhase, future: F) -> Result<T>whereF: Future<Output = Result<T>>,{match tokio::time::timeout_at(self.at, future).await {Ok(result) => result,- Err(_) => bail!(- "fetch readiness timed out after {} ms while {phase}",- self.timeout.as_millis()- ),+ Err(_) => Err(anyhow::Error::new(FetchReadinessTimeout::new(+ self.timeout,+ phase,+ ))),}}}#[cfg(test)]mod tests {- use super::FetchDeadline;+ use super::{FetchDeadline, FetchReadinessTimeout, FetchTimeoutPhase};use anyhow::Result;use std::time::Duration;@@ -84,7 +173,7 @@ mod tests {async fn wait_reports_total_budget_and_active_phase() -> Result<()> {let deadline = FetchDeadline::new(Duration::from_millis(25))?;let error = deadline- .wait("waiting for a selector", async {+ .wait(FetchTimeoutPhase::WaitingForSelector, async {tokio::time::sleep(Duration::from_secs(1)).await;Ok(())})@@ -95,6 +184,12 @@ mod tests {error.to_string(),"fetch readiness timed out after 25 ms while waiting for a selector");+ assert_eq!(+ error+ .downcast_ref::<FetchReadinessTimeout>()+ .map(|timeout| timeout.phase()),+ Some(FetchTimeoutPhase::WaitingForSelector)+ );Ok(())}diff --git a/moli-core/src/runtime/lifecycle_fetch.rs b/moli-core/src/runtime/lifecycle_fetch.rsindex b3a178574..b8483d5a8 100644--- a/moli-core/src/runtime/lifecycle_fetch.rs+++ b/moli-core/src/runtime/lifecycle_fetch.rs@@ -5,11 +5,12 @@//! lifecycle decision cannot reset or extend the caller's timeout budget.use super::{- Browser, FetchDeadline, FetchedDocument, RenderedDomWaitUntil, RendererLifecycleDecider,- RendererLifecycleDecision, RendererLifecycleSnapshot, RendererReplyBoundary,+ Browser, FetchDeadline, FetchedDocument, RawDocumentDisposition, RenderedDomWaitUntil,+ RendererLifecycleDecider, RendererLifecycleDecision, RendererLifecycleSnapshot,+ RendererReplyBoundary,};-use anyhow::{Result, anyhow};-use moli_fetch::{NetworkFetchFailureContext, Request};+use anyhow::Result;+use moli_fetch::Request;use std::time::Duration;impl Browser {@@ -65,19 +66,8 @@ impl Browser {deadline,RendererReplyBoundary::Stage,Some(decider),+ RawDocumentDisposition::Materialize,).await- .map_err(|error| {- // This typed context already supplies the caller-facing fetch- // message. Keep it outermost instead of obscuring it with a- // lifecycle layer that adds no actionable network information.- if error.is::<NetworkFetchFailureContext>() {- error- } else {- error.context(anyhow!(- "failed while applying the {wait_until:?} lifecycle-target decision or following its successor navigation"- ))- }- })}}diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rsindex 05f73d8d3..2bfaa1a40 100644--- a/moli-core/src/runtime/mod.rs+++ b/moli-core/src/runtime/mod.rs@@ -12,7 +12,7 @@ use crate::{selector::QueryEngine,};use anyhow::Result;-use anyhow::{Context, anyhow, bail};+use anyhow::{Context, anyhow};use moli_cookie_jar::StoredCookie;use moli_fetch::{FetchCancelHandle, RawResponse, Request, StreamingRawResponse, ensure_http_status_success,@@ -35,7 +35,7 @@ use url::Url;pub use crate::renderer::ExternalRawDocumentBodyStream;pub use crate::renderer::PageVmInitStage;pub use crate::renderer::RendererReplyBoundary;-pub use fetch_deadline::FetchDeadline;+pub use fetch_deadline::{FetchDeadline, FetchReadinessTimeout, FetchTimeoutPhase};pub use moli_renderer_v8::{DetachedParserScriptFetchContinuation, RendererBrowserContextRuntime,RendererBrowserContextRuntimeOwner, RendererBrowserContextRuntimeOwnerAccess,@@ -52,6 +52,16 @@ pub use navigation_engine::{static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+enum RawDocumentDisposition {+ Materialize,+ RejectForPageApi,+}++fn raw_document_page_api_error() -> anyhow::Error {+ anyhow!("raw non-HTML document cannot be returned through the Page fetch API")+}+fn is_fetch_readiness_timeout(error: &anyhow::Error, wait_until: RenderedDomWaitUntil) -> bool {let expected = match wait_until {RenderedDomWaitUntil::NetworkIdle => "timed out waiting for networkidle",@@ -156,6 +166,13 @@ impl RenderedDomWaitUntil {}}+fn fetch_timeout_phase_for_stage(stage: PageVmInitStage) -> FetchTimeoutPhase {+ match stage {+ PageVmInitStage::DomContentLoaded => FetchTimeoutPhase::WaitingForDomContentLoaded,+ PageVmInitStage::Load => FetchTimeoutPhase::WaitingForLoad,+ }+}+#[derive(Debug)]pub struct RawDocument {response: RawResponse,@@ -354,48 +371,21 @@ impl Browser {deadline: FetchDeadline,allow_http_error: bool,) -> Result<Page> {- let raw_url = request.url.as_str().to_owned();- let stage = wait_until.base_stage();- debug!(- url = %raw_url,- wait_until = ?wait_until,- timeout_ms = deadline.timeout().as_millis(),- stage = ?stage,- "starting fetch_with_wait_until deadline"- );-- // Materialization only reaches the concrete DCL/Load base stage. The- // selected stability observation is a separate Page operation so it- // can consume the remaining budget and soften only its own timeout.- let mut page = match tokio::time::timeout_at(- deadline.at(),- self.fetch_allow_http_error_internal(request, stage),- )- .await+ let mut page = match self+ .fetch_document_to_base_stage(+ request,+ wait_until,+ deadline,+ RendererReplyBoundary::Stage,+ None,+ RawDocumentDisposition::RejectForPageApi,+ )+ .await?{- // Preserve transport and policy errors from the fetch itself. The- // outer deadline should add an error only when it actually wins.- Ok(result) => result?,- Err(_) => {- warn!(- url = %raw_url,- wait_until = ?wait_until,- timeout_ms = deadline.timeout().as_millis(),- stage = ?stage,- allow_http_error,- "fetch_with_wait_until timed out"- );- if allow_http_error {- bail!(- "fetch allow-http-error wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",- deadline.timeout().as_millis()- );- }- bail!(- "fetch wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",- deadline.timeout().as_millis()- );- }+ FetchedDocument::Page(page) => page,+ FetchedDocument::Raw(_) => unreachable!(+ "the Page fetch path rejects raw documents before body materialization"+ ),};self.wait_for_page_readiness_with_deadline(&mut page, wait_until, deadline)@@ -434,6 +424,7 @@ impl Browser {deadline,RendererReplyBoundary::Stage,None,+ RawDocumentDisposition::Materialize,).await?;@@ -457,6 +448,7 @@ impl Browser {deadline: FetchDeadline,reply_boundary: RendererReplyBoundary,lifecycle_decider: Option<RendererLifecycleDecider>,+ raw_document_disposition: RawDocumentDisposition,) -> Result<FetchedDocument> {let timeout = deadline.timeout();let stage = wait_until.base_stage();@@ -483,6 +475,7 @@ impl Browser {timeout,stage,outer_deadline,+ || fetch_timeout_phase_for_stage(stage),self.materialize_static_html_page(&raw_url,requested_url,@@ -507,6 +500,7 @@ impl Browser {timeout,stage,outer_deadline,+ || FetchTimeoutPhase::WaitingForResponseHeaders,self.fetch_service_worker_main_resource_for_navigation(&request,&navigation_loader,@@ -519,15 +513,20 @@ impl Browser {} = service_worker_fetch;if let Some(response) = response {navigation_loader.note_service_worker_response_ready()?;- let document_fetch_context_seed =- navigation_loader.commit(response.final_url.clone())?;if response_headers_indicate_raw_document(&response.headers) {+ if raw_document_disposition == RawDocumentDisposition::RejectForPageApi {+ return Err(raw_document_page_api_error());+ }let raw_response =RawResponse::from_head_and_body(response.head(), response.clone_body_bytes());return Ok(FetchedDocument::Raw(Box::new(RawDocument::from_response(raw_response,))));}+ let document_fetch_context_seed =+ navigation_loader.commit(response.final_url.clone())?;+ let page_creation_progress = moli_renderer_v8::RendererPageCreationProgress::new();+ let timeout_progress = page_creation_progress.clone();return self.fetch_document_wait_timeout(&raw_url,@@ -535,7 +534,8 @@ impl Browser {timeout,stage,outer_deadline,- self.materialize_streaming_raw_response_page(+ move || FetchTimeoutPhase::from_renderer_phase(timeout_progress.phase()),+ self.materialize_streaming_raw_response_page_with_page_creation_progress(&raw_url,requested_url,stage,@@ -544,6 +544,7 @@ impl Browser {streaming_raw_response_from_navigation_response(response)?,document_fetch_context_seed,reserved_client,+ page_creation_progress,),).await@@ -556,28 +557,42 @@ impl Browser {timeout,stage,outer_deadline,+ || FetchTimeoutPhase::WaitingForResponseHeaders,navigation_loader.fetch_raw_stream(request),).await?;if response_headers_indicate_raw_document(&response.headers) {- // `wait_until` is a DOM lifecycle deadline. Binary/download-like main- // resources do not have DCL/load milestones, so only the initial- // document response acquisition is gated above. The body dump is plain- // network output and should not be reported as a DCL timeout.+ if raw_document_disposition == RawDocumentDisposition::RejectForPageApi {+ return Err(raw_document_page_api_error());+ }+ // Binary/download-like main resources have no DOM lifecycle+ // milestones, but materializing a RawDocument still consumes the+ // caller-owned absolute fetch-readiness budget.return self- .materialize_streaming_raw_response_raw_document(&raw_url, response)+ .fetch_document_wait_timeout(+ &raw_url,+ wait_until,+ timeout,+ stage,+ outer_deadline,+ || FetchTimeoutPhase::StreamingMainBody,+ self.materialize_streaming_raw_response_raw_document(response),+ ).await.map(|raw| FetchedDocument::Raw(Box::new(raw)));}let document_fetch_context_seed = navigation_loader.commit(response.final_url.clone())?;+ let page_creation_progress = moli_renderer_v8::RendererPageCreationProgress::new();+ let timeout_progress = page_creation_progress.clone();self.fetch_document_wait_timeout(&raw_url,wait_until,timeout,stage,outer_deadline,- self.materialize_streaming_raw_response_page(+ move || FetchTimeoutPhase::from_renderer_phase(timeout_progress.phase()),+ self.materialize_streaming_raw_response_page_with_page_creation_progress(&raw_url,requested_url,stage,@@ -586,6 +601,7 @@ impl Browser {response,document_fetch_context_seed,reserved_client,+ page_creation_progress,),).await@@ -658,7 +674,7 @@ impl Browser {let remaining = deadline.remaining();deadline.wait(- "waiting for a selector",+ FetchTimeoutPhase::WaitingForSelector,page.wait_for_selector(&loader, selector, remaining),).await@@ -686,7 +702,7 @@ impl Browser {let remaining = deadline.remaining();deadline.wait(- "waiting for a script to become truthy",+ FetchTimeoutPhase::WaitingForScript,page.wait_for_script_truthy(&loader, expression, remaining),).await@@ -713,7 +729,7 @@ impl Browser {let remaining = deadline.remaining();deadline.wait(- "waiting for a subresource response",+ FetchTimeoutPhase::WaitingForSubresourceResponse,page.wait_for_subresource_response(&loader, criteria, remaining),).await@@ -970,7 +986,7 @@ impl Browser {let document_fetch_context_seed = navigation_loader.commit(response.final_url.clone())?;if response_headers_indicate_raw_document(&response.headers) {return self- .materialize_streaming_raw_response_raw_document(&raw_url, response)+ .materialize_streaming_raw_response_raw_document(response).await.map(|raw| FetchedDocument::Raw(Box::new(raw)));}@@ -999,23 +1015,29 @@ impl Browser {.await}- async fn fetch_document_wait_timeout<T, F>(+ async fn fetch_document_wait_timeout<T, F, P>(&self,raw_url: &str,wait_until: RenderedDomWaitUntil,timeout: Duration,stage: PageVmInitStage,deadline: tokio::time::Instant,+ timeout_phase: P,future: F,) -> Result<T>whereF: std::future::Future<Output = Result<T>>,+ P: FnOnce() -> FetchTimeoutPhase,{match tokio::time::timeout_at(deadline, future).await {Ok(result) => result,- Err(_) => {- Err(self.fetch_document_wait_timeout_error(raw_url, wait_until, timeout, stage))- }+ Err(_) => Err(self.fetch_document_wait_timeout_error(+ raw_url,+ wait_until,+ timeout,+ stage,+ timeout_phase(),+ )),}}@@ -1025,18 +1047,17 @@ impl Browser {wait_until: RenderedDomWaitUntil,timeout: Duration,stage: PageVmInitStage,+ phase: FetchTimeoutPhase,) -> anyhow::Error {warn!(url = %raw_url,wait_until = ?wait_until,timeout_ms = timeout.as_millis(),stage = ?stage,+ timeout_phase = %phase,"fetch_document_allow_http_error_with_wait_until timed out");- anyhow!(- "fetch document allow-http-error wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",- timeout.as_millis()- )+ anyhow::Error::new(FetchReadinessTimeout::new(timeout, phase))}async fn materialize_static_html_page(@@ -1100,8 +1121,7 @@ impl Browser {create_page_request.storage_bucket_store = Some(self.partition.storage_bucket_store());let reply = renderer_owner.dispatch_command(RendererOwnerCommand::CreateHtmlPage(create_page_request))- .await- .with_context(|| anyhow!("failed to execute scripts for page `{raw_url}`"))?;+ .await?;let page = materialize_page_created_reply(&renderer_owner, reply)?;info!(page_id = page.page_id(),@@ -1122,6 +1142,32 @@ impl Browser {response: StreamingRawResponse,document_fetch_context_seed: DocumentFetchContextSeed,reserved_service_worker_client: Option<RendererReservedServiceWorkerClient>,+ ) -> Result<Page> {+ self.materialize_streaming_raw_response_page_with_page_creation_progress(+ raw_url,+ requested_url,+ stage,+ reply_boundary,+ lifecycle_decider,+ response,+ document_fetch_context_seed,+ reserved_service_worker_client,+ moli_renderer_v8::RendererPageCreationProgress::new(),+ )+ .await+ }++ async fn materialize_streaming_raw_response_page_with_page_creation_progress(+ &self,+ raw_url: &str,+ requested_url: Url,+ stage: PageVmInitStage,+ reply_boundary: RendererReplyBoundary,+ lifecycle_decider: Option<RendererLifecycleDecider>,+ response: StreamingRawResponse,+ document_fetch_context_seed: DocumentFetchContextSeed,+ reserved_service_worker_client: Option<RendererReservedServiceWorkerClient>,+ page_creation_progress: moli_renderer_v8::RendererPageCreationProgress,) -> Result<Page> {let started = Instant::now();debug!(@@ -1146,7 +1192,11 @@ impl Browser {let response_headers = response.headers.clone();let redirected = response.redirected;let redirect_count = response.redirect_chain.len();- let raw_body = external_raw_document_body_from_streaming_response(response);+ let raw_body =+ external_raw_document_body_from_streaming_response_with_page_creation_progress(+ response,+ page_creation_progress,+ );let document_start_scripts = self.config.document_start_scripts()@@ -1210,8 +1260,7 @@ impl Browser {None,lifecycle_decider,)- .await- .with_context(|| anyhow!("failed to execute scripts for page `{raw_url}`"))?;+ .await?;if pending_download.is_some() {return Err(anyhow!("raw streaming page creation produced a pending download for `{raw_url}`"@@ -1233,16 +1282,12 @@ impl Browser {async fn materialize_streaming_raw_response_raw_document(&self,- raw_url: &str,response: StreamingRawResponse,) -> Result<RawDocument> {let started = Instant::now();let final_url = response.final_url.clone();let status = response.status;- let raw_response = response- .into_materialized_raw_response()- .await- .with_context(|| anyhow!("failed to read raw document body for `{raw_url}`"))?;+ let raw_response = response.into_materialized_raw_response().await?;info!(url = %final_url,status,@@ -1258,18 +1303,42 @@ impl Browser {}}-fn external_raw_document_body_from_streaming_response(+fn external_raw_document_body_from_streaming_response_with_page_creation_progress(response: StreamingRawResponse,+ page_creation_progress: moli_renderer_v8::RendererPageCreationProgress,) -> ExternalRawDocumentBodyStream {- external_raw_document_body_from_streaming_response_with_body_eof_observer(response, None)+ external_raw_document_body_from_streaming_response_with_observer_and_progress(+ response,+ None,+ Some(page_creation_progress),+ )}+#[cfg(test)]fn external_raw_document_body_from_streaming_response_with_body_eof_observer(+ response: StreamingRawResponse,+ body_eof_observer: Option<oneshot::Sender<()>>,+) -> ExternalRawDocumentBodyStream {+ external_raw_document_body_from_streaming_response_with_observer_and_progress(+ response,+ body_eof_observer,+ None,+ )+}++fn external_raw_document_body_from_streaming_response_with_observer_and_progress(mut response: StreamingRawResponse,mut body_eof_observer: Option<oneshot::Sender<()>>,+ page_creation_progress: Option<moli_renderer_v8::RendererPageCreationProgress>,) -> ExternalRawDocumentBodyStream {let (completion_tx, completion_rx) = oneshot::channel();- let (body_tx, body_stream) = ExternalRawDocumentBodyStream::channel(completion_rx);+ let (body_tx, body_stream) = match page_creation_progress {+ Some(progress) => ExternalRawDocumentBodyStream::channel_with_page_creation_progress(+ completion_rx,+ progress,+ ),+ None => ExternalRawDocumentBodyStream::channel(completion_rx),+ };tokio::spawn(async move {let result = async {loop {diff --git a/moli-core/src/runtime/storage_partition.rs b/moli-core/src/runtime/storage_partition.rsindex f2cdbe489..5bc1da92f 100644--- a/moli-core/src/runtime/storage_partition.rs+++ b/moli-core/src/runtime/storage_partition.rs@@ -98,8 +98,7 @@ impl StoragePartitionState {pub fn open(profile_dir: Option<&Path>) -> Result<Self> {let profile = profile_dir.map(BrowserProfile::open)- .transpose()- .context("failed to open browser profile")?+ .transpose()?.map(Arc::new);Self::from_profile(profile)}diff --git a/moli-core/tests/fetch.rs b/moli-core/tests/fetch.rsindex 2cd702382..5c59a66ed 100644--- a/moli-core/tests/fetch.rs+++ b/moli-core/tests/fetch.rs@@ -1,11 +1,46 @@use moli_test_support as support;use anyhow::Result;-use moli_core::runtime::{Browser, BrowserConfig as AppConfig, RenderedDomWaitUntil};-use std::time::Duration;+use moli_core::runtime::{+ Browser, BrowserConfig as AppConfig, FetchReadinessTimeout, FetchTimeoutPhase,+ RenderedDomWaitUntil,+};+use moli_fetch::Request;+use std::time::{Duration, Instant};use support::FixtureServer;+use tokio::io::{AsyncReadExt, AsyncWriteExt};+use tokio::net::TcpListener;+use tokio::task::JoinHandle;use url::Url;+async fn spawn_stalled_binary_response_server() -> Result<(String, JoinHandle<()>)> {+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let address = listener.local_addr()?;+ let server = tokio::spawn(async move {+ let Ok((mut stream, _)) = listener.accept().await else {+ return;+ };+ let mut request = [0_u8; 4096];+ let Ok(read) = stream.read(&mut request).await else {+ return;+ };+ if read == 0 {+ return;+ }+ if stream+ .write_all(+ b"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: 1048576\r\nConnection: keep-alive\r\n\r\n",+ )+ .await+ .is_err()+ {+ return;+ }+ std::future::pending::<()>().await;+ });+ Ok((format!("http://{address}/stalled.bin"), server))+}+#[tokio::test]async fn fetches_static_fixture() -> Result<()> {let server = FixtureServer::spawn().await?;@@ -122,6 +157,80 @@ async fn fetch_allow_http_error_returns_page_for_success() -> Result<()> {Ok(())}+#[tokio::test]+async fn fetch_with_wait_until_rejects_raw_document_through_page_api() -> Result<()> {+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;++ let error = browser+ .fetch_with_wait_until(+ &server.url("/net/upstream/xhr/binary"),+ RenderedDomWaitUntil::Load,+ Duration::from_secs(1),+ )+ .await+ .expect_err("the Page fetch API must reject a raw non-HTML document");++ assert_eq!(+ error.to_string(),+ "raw non-HTML document cannot be returned through the Page fetch API"+ );++ server.shutdown().await;+ Ok(())+}++#[tokio::test]+async fn page_fetch_rejects_raw_response_without_waiting_for_its_stalled_body() -> Result<()> {+ let (url, server) = spawn_stalled_binary_response_server().await?;+ let browser = Browser::new(AppConfig::default())?;+ let started = Instant::now();++ let error = tokio::time::timeout(+ Duration::from_secs(1),+ browser.fetch_with_wait_until(&url, RenderedDomWaitUntil::Load, Duration::from_millis(100)),+ )+ .await+ .expect("the Page API must not wait for a raw response body")+ .expect_err("the Page API must reject a raw non-HTML document");++ assert_eq!(+ error.to_string(),+ "raw non-HTML document cannot be returned through the Page fetch API"+ );+ assert!(+ started.elapsed() < Duration::from_secs(1),+ "raw Page rejection must not fall through to the HTTP transport timeout"+ );+ server.abort();+ Ok(())+}++#[tokio::test]+async fn raw_document_body_materialization_uses_the_fetch_deadline() -> Result<()> {+ let (url, server) = spawn_stalled_binary_response_server().await?;+ let browser = Browser::new(AppConfig::default())?;++ let error = tokio::time::timeout(+ Duration::from_secs(1),+ browser.fetch_request_document_allow_http_error_with_wait_until(+ Request::get(&url)?,+ RenderedDomWaitUntil::Load,+ Duration::from_millis(100),+ ),+ )+ .await+ .expect("raw body materialization must honor the fetch deadline")+ .expect_err("the stalled raw body must time out");+ let timeout = error+ .downcast_ref::<FetchReadinessTimeout>()+ .expect("the raw body timeout must retain its typed phase");+ assert_eq!(timeout.phase(), FetchTimeoutPhase::StreamingMainBody);++ server.abort();+ Ok(())+}+#[tokio::test]async fn fetch_allow_http_error_renders_404_response() -> Result<()> {let server = FixtureServer::spawn().await?;diff --git a/moli-fetch/src/network_fetch_result.rs b/moli-fetch/src/network_fetch_result.rsindex a21627f1f..e692bf04f 100644--- a/moli-fetch/src/network_fetch_result.rs+++ b/moli-fetch/src/network_fetch_result.rs@@ -376,6 +376,7 @@ fn parse_status_line(line: &str) -> Option<u16> {pub struct NetworkFetchFailureContext {observation_journal: NetworkObservationJournal,network_error_text: &'static str,+ reason: String,request_context: Option<NetworkFetchFailureRequestContext>,}@@ -437,9 +438,11 @@ impl NetworkFetchFailureContext {observation_journal: NetworkObservationJournal,) -> anyhow::Error {let network_error_text = crate::error::browser_network_error_text(&source);+ let reason = source.to_string();source.context(Self {observation_journal,network_error_text,+ reason,request_context: None,})}@@ -450,9 +453,11 @@ impl NetworkFetchFailureContext {request_context: NetworkFetchFailureRequestContext,) -> anyhow::Error {let network_error_text = crate::error::browser_network_error_text(&source);+ let reason = source.to_string();source.context(Self {observation_journal,network_error_text,+ reason,request_context: Some(request_context),})}@@ -465,6 +470,12 @@ impl NetworkFetchFailureContext {self.network_error_text}+ /// Returns the single human-readable transport or policy reason captured+ /// before this machine-readable context was attached.+ pub fn reason(&self) -> &str {+ &self.reason+ }+pub fn request_context(&self) -> Option<&NetworkFetchFailureRequestContext> {self.request_context.as_ref()}@@ -604,6 +615,12 @@ mod tests {);assert_eq!(report.matches("transport failure sentinel").count(), 1);assert!(error.downcast_ref::<NetworkFetchFailureContext>().is_some());+ assert_eq!(+ error+ .downcast_ref::<NetworkFetchFailureContext>()+ .map(NetworkFetchFailureContext::reason),+ Some("transport failure sentinel")+ );assert!(error.downcast_ref::<std::io::Error>().is_some());}@@ -612,6 +629,7 @@ mod tests {let failure = NetworkFetchFailureContext {observation_journal: NetworkObservationJournal::default(),network_error_text: "net::ERR_FAILED",+ reason: "transport rejected secret values".to_owned(),request_context: Some(NetworkFetchFailureRequestContext::new(Url::parse("https://example.test/").expect("test URL should parse"),"POST".to_owned(),@@ -625,6 +643,10 @@ mod tests {assert!(!report.contains("secret request body"), "{report}");assert!(!report.contains("secret token"), "{report}");+ assert!(+ !report.contains("transport rejected secret values"),+ "{report}"+ );assert!(report.contains("has_request_body: true"), "{report}");assert!(report.contains("request_header_count: 1"), "{report}");}diff --git a/moli-renderer-v8/src/lib.rs b/moli-renderer-v8/src/lib.rsindex d26b83fd8..06dadaa99 100644--- a/moli-renderer-v8/src/lib.rs+++ b/moli-renderer-v8/src/lib.rs@@ -272,8 +272,9 @@ pub use runtime::{RendererOwnerResourceActivitySource, RendererOwnerRuntimeActivitySource, RendererPageCommand,RendererPageCommandPending, RendererPageCommandPostResponseContinuation,RendererPageCookieFacadeSnapshotReply, RendererPageCreationArtifacts,- RendererPageCreationDiagnostics, RendererPageDiagnosticsSnapshot, RendererPageDumpFormat,- RendererPageDumpOptions, RendererPageDumpStripOptions, RendererPageHandle, RendererPageReply,+ RendererPageCreationDiagnostics, RendererPageCreationPhase, RendererPageCreationProgress,+ RendererPageDiagnosticsSnapshot, RendererPageDumpFormat, RendererPageDumpOptions,+ RendererPageDumpStripOptions, RendererPageHandle, RendererPageReply,RendererPageReservationToken, RendererPageState, RendererPageTestingHandle, RendererPageView,RendererPendingDownloadActivation, RendererPendingDownloadResponse,RendererPendingFileChooserActivation, RendererPendingJavaScriptDialog,diff --git a/moli-renderer-v8/src/runtime/mod.rs b/moli-renderer-v8/src/runtime/mod.rsindex c88c61b99..85d98bd0a 100644--- a/moli-renderer-v8/src/runtime/mod.rs+++ b/moli-renderer-v8/src/runtime/mod.rs@@ -57,6 +57,7 @@ mod owner_maintenance;mod page;mod page_commands;mod page_context_cancel;+mod page_creation_progress;mod page_css;pub(crate) mod page_dom;mod page_dom_snapshot;@@ -81,6 +82,8 @@ pub(crate) use self::script_preloads::{IncrementalBufferedScriptPreloadScanner,};+pub use self::page_creation_progress::{RendererPageCreationPhase, RendererPageCreationProgress};+pub(crate) use self::browser_context_runtime::RendererOutputTransportSenderSlot;pub(in crate::runtime) use self::document_lifecycle_turn::PendingDocumentLifecycleTurn;pub(crate) use self::page_turn_scheduler::{diff --git a/moli-renderer-v8/src/runtime/page_creation_progress.rs b/moli-renderer-v8/src/runtime/page_creation_progress.rsnew file mode 100644index 000000000..243bf7745--- /dev/null+++ b/moli-renderer-v8/src/runtime/page_creation_progress.rs@@ -0,0 +1,84 @@+use std::sync::{+ Arc,+ atomic::{AtomicU8, Ordering},+};++/// The renderer-side condition currently preventing an initial page from+/// reaching its requested lifecycle boundary.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+#[repr(u8)]+pub enum RendererPageCreationPhase {+ StreamingMainBody = 0,+ ProcessingMainDocument = 1,+ WaitingForParserBlockingScript = 2,+ WaitingForParserBlockingStylesheet = 3,+ WaitingForDomContentLoaded = 4,+ WaitingForLoad = 5,+}++impl RendererPageCreationPhase {+ fn from_u8(value: u8) -> Self {+ match value {+ 0 => Self::StreamingMainBody,+ 1 => Self::ProcessingMainDocument,+ 2 => Self::WaitingForParserBlockingScript,+ 3 => Self::WaitingForParserBlockingStylesheet,+ 4 => Self::WaitingForDomContentLoaded,+ 5 => Self::WaitingForLoad,+ _ => Self::ProcessingMainDocument,+ }+ }+}++/// Shared progress reported by the renderer while a streamed main Document is+/// being created.+///+/// The browser-side deadline owns a clone and reads it only if that deadline+/// expires. Renderer owner turns update the same value immediately before+/// parking on body input, parser-blocking resources, or a lifecycle target.+#[derive(Clone, Debug)]+pub struct RendererPageCreationProgress {+ phase: Arc<AtomicU8>,+}++impl RendererPageCreationProgress {+ pub fn new() -> Self {+ Self {+ phase: Arc::new(AtomicU8::new(+ RendererPageCreationPhase::StreamingMainBody as u8,+ )),+ }+ }++ pub fn phase(&self) -> RendererPageCreationPhase {+ RendererPageCreationPhase::from_u8(self.phase.load(Ordering::Relaxed))+ }++ pub(crate) fn set_phase(&self, phase: RendererPageCreationPhase) {+ self.phase.store(phase as u8, Ordering::Relaxed);+ }+}++impl Default for RendererPageCreationProgress {+ fn default() -> Self {+ Self::new()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn clones_observe_the_latest_phase() {+ let progress = RendererPageCreationProgress::new();+ let observer = progress.clone();++ progress.set_phase(RendererPageCreationPhase::WaitingForParserBlockingStylesheet);++ assert_eq!(+ observer.phase(),+ RendererPageCreationPhase::WaitingForParserBlockingStylesheet+ );+ }+}diff --git a/moli-renderer-v8/src/runtime/page_vm/mod.rs b/moli-renderer-v8/src/runtime/page_vm/mod.rsindex 8a87ebc6f..873b4255e 100644--- a/moli-renderer-v8/src/runtime/page_vm/mod.rs+++ b/moli-renderer-v8/src/runtime/page_vm/mod.rs@@ -1114,6 +1114,7 @@ pub(crate) struct PageVmEnvConfig {#[derive(Clone)]pub(crate) struct PageVmRuntimeHooks {owner_wake: Option<RendererOwnerWakeSender>,+ page_creation_progress: Option<super::RendererPageCreationProgress>,javascript_dialog_runtime: RendererJavaScriptDialogRuntime,resource_task_runner: Option<crate::network::RendererResourceTaskRunner>,pub(crate) browser_context_runtime: super::RendererBrowserContextRuntime,@@ -1175,6 +1176,7 @@ impl PageVmRuntimeHooks {Self::bind_standalone_browser_context_owner_for_test(&browser_context_runtime);Self {owner_wake: None,+ page_creation_progress: None,javascript_dialog_runtime: RendererJavaScriptDialogRuntime::default(),resource_task_runner: None,browser_context_runtime,@@ -1246,6 +1248,7 @@ impl PageVmRuntimeHooks {Self {javascript_dialog_runtime: RendererJavaScriptDialogRuntime::default(),owner_wake: Some(owner_wake),+ page_creation_progress: None,resource_task_runner: Some(residence.resource_task_runner()),browser_context_runtime,renderer_document_isolate_mode:@@ -1284,6 +1287,7 @@ impl PageVmRuntimeHooks {Self {javascript_dialog_runtime: RendererJavaScriptDialogRuntime::default(),owner_wake: Some(owner_wake),+ page_creation_progress: None,resource_task_runner: Some(crate::network::RendererResourceTaskRunner::from_current_tokio().expect("renderer owner must install its resource task runner"),@@ -1304,6 +1308,20 @@ impl PageVmRuntimeHooks {}}+ pub(crate) fn with_page_creation_progress(+ mut self,+ page_creation_progress: super::RendererPageCreationProgress,+ ) -> Self {+ self.page_creation_progress = Some(page_creation_progress);+ self+ }++ pub(crate) fn set_page_creation_phase(&self, phase: super::RendererPageCreationPhase) {+ if let Some(progress) = &self.page_creation_progress {+ progress.set_phase(phase);+ }+ }+pub(crate) fn with_renderer_document_isolate_allocator(mut self,allocator: RendererDocumentIsolateAllocator,diff --git a/moli-renderer-v8/src/runtime/phase_one/pending_residence.rs b/moli-renderer-v8/src/runtime/phase_one/pending_residence.rsindex 96c1be038..d72eb80bb 100644--- a/moli-renderer-v8/src/runtime/phase_one/pending_residence.rs+++ b/moli-renderer-v8/src/runtime/phase_one/pending_residence.rs@@ -73,6 +73,7 @@ impl PendingPhaseOneResidence {runtime: Box<ConcurrentParseTimeRuntime>,started: Instant,) -> Self {+ runtime.publish_pending_page_creation_phase();Self::ParserBlockingSourceLoad { runtime, started }}@@ -80,12 +81,14 @@ impl PendingPhaseOneResidence {runtime: Box<ConcurrentParseTimeRuntime>,started: Instant,) -> Self {+ runtime.publish_pending_page_creation_phase();Self::ClosedInputPageWork { runtime, started }}pub(in crate::runtime) fn open_streaming(continuation: Box<PendingStreamingPhaseOneContinuation>,) -> Self {+ continuation.publish_pending_page_creation_phase();Self::OpenStreaming(continuation)}@@ -134,6 +137,7 @@ impl PendingPhaseOneResidence {match self {Self::ParserBlockingSourceLoad { runtime, started }| Self::ClosedInputPageWork { runtime, started } => {+ runtime.publish_processing_main_document_phase();let outcome = (*runtime).continue_creation_from_phase_one_runtime(started).await?;diff --git a/moli-renderer-v8/src/runtime/phase_one/phase_two.rs b/moli-renderer-v8/src/runtime/phase_one/phase_two.rsindex cd6c143e0..9965da692 100644--- a/moli-renderer-v8/src/runtime/phase_one/phase_two.rs+++ b/moli-renderer-v8/src/runtime/phase_one/phase_two.rs@@ -17,6 +17,7 @@ impl ConcurrentParseTimeRuntime {&self.page_vm.local_executor,"parse-time phase transition handoff",);+ self.publish_target_lifecycle_phase();let (stranded_parse_time_document_script_events, stranded_parse_time_lifecycle_work) =self.retire_main_parser_continuation();self.page_vmdiff --git a/moli-renderer-v8/src/runtime/phase_one/state.rs b/moli-renderer-v8/src/runtime/phase_one/state.rsindex e66bed246..c70916171 100644--- a/moli-renderer-v8/src/runtime/phase_one/state.rs+++ b/moli-renderer-v8/src/runtime/phase_one/state.rs@@ -84,6 +84,46 @@ pub(in crate::runtime) struct ConcurrentParseTimeRuntime {}impl ConcurrentParseTimeRuntime {+ pub(in crate::runtime) fn publish_processing_main_document_phase(&self) {+ self.page_vm.runtime_hooks.set_page_creation_phase(+ crate::runtime::RendererPageCreationPhase::ProcessingMainDocument,+ );+ }++ pub(in crate::runtime) fn publish_pending_page_creation_phase(&self) {+ let phase = match self.pending_parsing_blocking_wait {+ PendingParsingBlockingWait::PageTaskBlockingStylesheet => {+ crate::runtime::RendererPageCreationPhase::WaitingForParserBlockingStylesheet+ }+ PendingParsingBlockingWait::LegacyDocumentProcessing+ | PendingParsingBlockingWait::PageNetworkingDocumentWriteExternalScript => {+ crate::runtime::RendererPageCreationPhase::WaitingForParserBlockingScript+ }+ PendingParsingBlockingWait::None+ if self.has_unready_pending_parser_blocking_source_load() =>+ {+ crate::runtime::RendererPageCreationPhase::WaitingForParserBlockingScript+ }+ PendingParsingBlockingWait::None if !self.state.input_closed => {+ crate::runtime::RendererPageCreationPhase::StreamingMainBody+ }+ PendingParsingBlockingWait::None => {+ crate::runtime::RendererPageCreationPhase::ProcessingMainDocument+ }+ };+ self.page_vm.runtime_hooks.set_page_creation_phase(phase);+ }++ pub(super) fn publish_target_lifecycle_phase(&self) {+ let phase = match self.stage {+ PageVmInitStage::DomContentLoaded => {+ crate::runtime::RendererPageCreationPhase::WaitingForDomContentLoaded+ }+ PageVmInitStage::Load => crate::runtime::RendererPageCreationPhase::WaitingForLoad,+ };+ self.page_vm.runtime_hooks.set_page_creation_phase(phase);+ }+pub(in crate::runtime) fn page_vm(&self) -> &PageVm {&self.page_vm}diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rsindex 7c4d0746a..edb798a3b 100644--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs@@ -341,6 +341,10 @@ impl ConcurrentParseTimeRuntime {boundary: CommittedNavigationBootstrapBoundary,) -> Result<StreamingNavigationPageCreationResult> {debug_assert!(!response_headers_indicate_download(&response_headers));+ let runtime_hooks = match raw_body.page_creation_progress() {+ Some(progress) => runtime_hooks.with_page_creation_progress(progress),+ None => runtime_hooks,+ };let mut env = env.clone();env.document_content_security_policies = if env.bypass_content_security_policy {Vec::new()@@ -568,6 +572,7 @@ const EXTERNAL_RAW_DOCUMENT_BODY_BUFFERED_CHUNKS: usize = 8;pub struct ExternalRawDocumentBodyStream {body_chunks: mpsc::Receiver<Vec<u8>>,completion: Option<oneshot::Receiver<Result<()>>>,+ page_creation_progress: Option<crate::runtime::RendererPageCreationProgress>,}impl ExternalRawDocumentBodyStream {@@ -581,6 +586,23 @@ impl ExternalRawDocumentBodyStream {(body_tx, Self::new(body_chunks, completion))}+ /// Creates a bounded stream whose renderer progress remains observable by+ /// the browser-side navigation deadline.+ pub fn channel_with_page_creation_progress(+ completion: oneshot::Receiver<Result<()>>,+ page_creation_progress: crate::runtime::RendererPageCreationProgress,+ ) -> (mpsc::Sender<Vec<u8>>, Self) {+ let (body_tx, body_chunks) = mpsc::channel(EXTERNAL_RAW_DOCUMENT_BODY_BUFFERED_CHUNKS);+ (+ body_tx,+ Self {+ body_chunks,+ completion: Some(completion),+ page_creation_progress: Some(page_creation_progress),+ },+ )+ }+pub fn new(body_chunks: mpsc::Receiver<Vec<u8>>,completion: oneshot::Receiver<Result<()>>,@@ -588,6 +610,7 @@ impl ExternalRawDocumentBodyStream {Self {body_chunks,completion: Some(completion),+ page_creation_progress: None,}}@@ -603,6 +626,10 @@ impl ExternalRawDocumentBodyStream {fn body_chunk_stream_is_exhausted(&self) -> bool {self.body_chunks.is_closed() && self.body_chunks.is_empty()}++ fn page_creation_progress(&self) -> Option<crate::runtime::RendererPageCreationProgress> {+ self.page_creation_progress.clone()+ }}pub(super) enum RawDocumentBodySource {diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming_residence.rs b/moli-renderer-v8/src/runtime/phase_one/streaming_residence.rsindex 96557ba94..714542f0f 100644--- a/moli-renderer-v8/src/runtime/phase_one/streaming_residence.rs+++ b/moli-renderer-v8/src/runtime/phase_one/streaming_residence.rs@@ -28,6 +28,10 @@ pub(in crate::runtime) struct PendingStreamingPhaseOneContinuation {}impl PendingStreamingPhaseOneContinuation {+ pub(super) fn publish_pending_page_creation_phase(&self) {+ self.runtime.publish_pending_page_creation_phase();+ }+pub(super) fn bridge(runtime: ConcurrentParseTimeRuntime,raw_body: RawDocumentBodySource,@@ -71,6 +75,7 @@ impl PendingStreamingPhaseOneContinuation {}pub(in crate::runtime) async fn resume(self) -> Result<PendingPhaseOneResumeOutcome> {+ self.runtime.publish_processing_main_document_phase();let Self {runtime,mut input,diff --git a/moli/src/app.rs b/moli/src/app.rsindex 094a774d3..180a0c2dd 100644--- a/moli/src/app.rs+++ b/moli/src/app.rs@@ -3,7 +3,7 @@mod readiness;mod redirect_navigation;-use std::{io::Write, sync::Arc};+use std::{fmt, io::Write, sync::Arc};use crate::{cli::{Cli, Commands, normalize_args_for_compat},@@ -49,8 +49,7 @@ pub async fn run_cli_with_config<W: Write>(.await.map_err(|error| with_fetch_context(error, &args.url))?;}- let browser = Browser::new(config.browser.clone())- .context("failed to initialize browser runtime")?;+ let browser = Browser::new(config.browser.clone())?;load_cookie_state(&browser, &config)?;let readiness =ReadinessPlan::from_fetch_args(&args, config.fetch.response_wait.clone())?;@@ -68,16 +67,20 @@ pub async fn run_cli_with_config<W: Write>(FetchedDocument::Raw(raw_document) => {if readiness.has_page_waits() || args.delay_ms > 0 {finalize_fetch_browser(browser);- return Err(anyhow!(- "raw non-HTML document fetch does not support page wait options"+ return Err(with_fetch_context(+ anyhow!(+ "raw non-HTML document fetch does not support page wait options"+ ),+ &args.url,));}let rendered =fetch_dump::render_raw_document_dump(&raw_document, &config.fetch)- .context("failed to render raw fetch output")?;+ .map_err(|error| with_fetch_context(error, &args.url))?;stdout.write_all(&rendered)- .context("failed to write raw fetch output")?;+ .context("failed to write raw fetch output")+ .map_err(|error| with_fetch_context(error, &args.url))?;let _ = stdout.flush();finalize_fetch_browser(browser);return Ok(());@@ -99,15 +102,17 @@ pub async fn run_cli_with_config<W: Write>(browser.wait_for_page_delay(&mut page, std::time::Duration::from_millis(args.delay_ms)).await- .context("failed while waiting for page delay")?;+ .context("failed while waiting for page delay")+ .map_err(|error| with_fetch_context(error, &args.url))?;}let rendered = fetch_dump::render_page_output_async(&mut page, &config.fetch).await- .context("failed to render fetch output")?;+ .map_err(|error| with_fetch_context(error, &args.url))?;stdout.write_all(&rendered)- .context("failed to write fetch output")?;+ .context("failed to write fetch output")+ .map_err(|error| with_fetch_context(error, &args.url))?;let _ = stdout.flush();if let Err(error) = page.close_async().await {tracing::warn!(error = %error, "failed to close fetched page before browser shutdown");@@ -146,14 +151,66 @@ fn build_fetch_request(url: &str, config: &AppConfig) -> Result<Request> {Ok(request)}+struct CliFetchFailureContext {+ url: String,+ reason: String,+}++impl fmt::Display for CliFetchFailureContext {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ write!(formatter, "failed to fetch `{}`", self.url)+ }+}++impl fmt::Debug for CliFetchFailureContext {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter+ .debug_struct("CliFetchFailureContext")+ .field("url", &self.url)+ .field("has_reason", &!self.reason.is_empty())+ .finish()+ }+}+fn with_fetch_context(error: anyhow::Error, url: &str) -> anyhow::Error {- // Network failures already carry a URL-aware typed context. Adding the- // same CLI context again would print two "failed to fetch" chain entries.- if error.is::<NetworkFetchFailureContext>() {- error- } else {- error.context(anyhow!("failed to fetch `{url}`"))+ if error.is::<CliFetchFailureContext>() {+ return error;+ }+ let reason = error+ .downcast_ref::<NetworkFetchFailureContext>()+ .map(|failure| failure.reason().to_owned())+ .unwrap_or_else(|| error.to_string());+ with_fetch_context_reason(error, url, reason)+}++fn with_fetch_context_reason(+ error: anyhow::Error,+ url: &str,+ reason: impl Into<String>,+) -> anyhow::Error {+ error.context(CliFetchFailureContext {+ url: url.to_owned(),+ reason: reason.into(),+ })+}++/// Writes the stable, concise command-line error presentation.+///+/// Fetch errors carry their selected user-facing reason in a typed outer+/// context and deliberately render only that two-line presentation. Other CLI+/// failures retain their anyhow context chain so startup and configuration+/// diagnostics do not lose their actionable inner cause.+pub fn write_error_report<W: Write>(writer: &mut W, error: &anyhow::Error) -> std::io::Result<()> {+ if let Some(fetch) = error.downcast_ref::<CliFetchFailureContext>() {+ writeln!(writer, "Error: {fetch}")?;+ writeln!(writer, "Reason: {}", one_line_reason(&fetch.reason))?;+ return Ok(());}+ writeln!(writer, "Error: {error:#}")+}++fn one_line_reason(reason: &str) -> String {+ reason.split_whitespace().collect::<Vec<_>>().join(" ")}fn load_cookie_state(browser: &Browser, config: &AppConfig) -> Result<()> {@@ -178,3 +235,38 @@ fn finalize_fetch_browser(browser: Browser) {// Browser::drop owns profile cookie writeback when --profile-dir is set.drop(browser);}++#[cfg(test)]+mod tests {+ use super::{with_fetch_context, write_error_report};++ #[test]+ fn fetch_report_has_one_reason_line_without_rendering_the_source_chain() {+ let error = with_fetch_context(+ anyhow::anyhow!("first failure line\nsecond failure line"),+ "https://example.test/",+ );+ let mut report = Vec::new();++ write_error_report(&mut report, &error).expect("report should write");+ let report = String::from_utf8(report).expect("report should be UTF-8");++ assert_eq!(+ report,+ "Error: failed to fetch `https://example.test/`\nReason: first failure line second failure line\n"+ );+ assert!(!report.contains("Caused by:"));+ }++ #[test]+ fn non_fetch_report_retains_the_anyhow_context_chain() {+ let error = anyhow::anyhow!("inner cause").context("outer context");+ let mut report = Vec::new();++ write_error_report(&mut report, &error).expect("report should write");+ let report = String::from_utf8(report).expect("report should be UTF-8");++ assert!(report.contains("outer context"));+ assert!(report.contains("inner cause"));+ }+}diff --git a/moli/src/main.rs b/moli/src/main.rsindex 28918ecb7..8d6768bd3 100644--- a/moli/src/main.rs+++ b/moli/src/main.rs@@ -5,11 +5,12 @@//! protocol server.use anyhow::Result;+use std::process::ExitCode;#[cfg(all(feature = "jemalloc", not(target_os = "windows")))]mod allocator;-fn main() -> Result<()> {+fn run() -> Result<()> {moli_process_signal::install_immediate_exit_handlers()?;let runtime_thread_budget = moli::runtime_thread_budget::tokio_runtime_thread_budget();tokio::runtime::Builder::new_multi_thread()@@ -19,3 +20,14 @@ fn main() -> Result<()> {.build()?.block_on(moli::app::run_from_env())}++fn main() -> ExitCode {+ match run() {+ Ok(()) => ExitCode::SUCCESS,+ Err(error) => {+ let stderr = std::io::stderr();+ let _ = moli::app::write_error_report(&mut stderr.lock(), &error);+ ExitCode::FAILURE+ }+ }+}diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex 99b14b27b..eb83b4a3d 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -31,7 +31,7 @@ use std::{};use strip_ansi_escapes::strip;use support::FixtureServer;-use tokio::{net::TcpListener, task::JoinHandle};+use tokio::{io::AsyncReadExt, net::TcpListener, task::JoinHandle};use tracing_subscriber::fmt::MakeWriter;struct Output {@@ -169,7 +169,7 @@ wherelet result = runtime.block_on(app::run_cli_with_config(cli, config, &mut stdout));if let Err(error) = &result {let mut stderr = stderr.lock();- writeln!(&mut *stderr, "{error:?}")?;+ app::write_error_report(&mut *stderr, error)?;}let stderr = stderr.lock().clone();Ok(Output {@@ -247,6 +247,11 @@ struct BinaryDocumentFixtureServer {task: JoinHandle<()>,}+struct BlockingStylesheetTimeoutFixtureServer {+ base_url: String,+ task: JoinHandle<()>,+}+const OPTIONAL_RESOURCE_PATHS: [&str; 5] = ["/optional-audio.mp3","/optional-font.woff2",@@ -463,6 +468,52 @@ impl BinaryDocumentFixtureServer {}}+impl BlockingStylesheetTimeoutFixtureServer {+ async fn spawn() -> Result<Self> {+ async fn page() -> Html<&'static str> {+ Html(+ r#"<!doctype html>+<html>+ <head>+ <link rel="stylesheet" href="/slow.css">+ <script>document.documentElement.dataset.parserResumed = "true";</script>+ </head>+ <body></body>+</html>"#,+ )+ }++ async fn slow_css() -> impl IntoResponse {+ tokio::time::sleep(std::time::Duration::from_secs(2)).await;+ ([("content-type", "text/css; charset=utf-8")], "html {}")+ }++ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let addr = listener.local_addr()?;+ let app = Router::new()+ .route("/", get(page))+ .route("/slow.css", get(slow_css));+ let task = tokio::spawn(async move {+ axum::serve(listener, app)+ .await+ .expect("blocking stylesheet timeout fixture should serve");+ });+ Ok(Self {+ base_url: format!("http://{addr}"),+ task,+ })+ }++ fn url(&self) -> String {+ format!("{}/", self.base_url)+ }++ async fn shutdown(self) {+ self.task.abort();+ let _ = self.task.await;+ }+}+impl CacheableFixtureServer {async fn spawn() -> Result<Self> {async fn cacheable(@@ -542,6 +593,19 @@ fn assert_cli_dump_html_callback_error_contract(output: &Output, expected_messag);}+fn assert_single_fetch_failure_reason(stderr: &str, url: &str, reason: &str) {+ assert!(+ stderr.contains(&format!("Error: failed to fetch `{url}`")),+ "stderr={stderr}"+ );+ assert!(+ stderr.contains(&format!("Reason: {reason}")),+ "stderr={stderr}"+ );+ assert_eq!(stderr.matches("Reason:").count(), 1, "stderr={stderr}");+ assert!(!stderr.contains("Caused by:"), "stderr={stderr}");+}+fn assert_cli_dump_html_debug_callback_error_contract(output: &Output, expected_message: &str) {assert!(output.status.success(),@@ -947,6 +1011,26 @@ fn cli_fetch_attachment_dump_html_bypasses_dcl_page_lifecycle() -> Result<()> {Ok(())}+#[test]+fn cli_fetch_raw_document_with_page_wait_uses_fetch_failure_report() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(BinaryDocumentFixtureServer::spawn())?;+ let url = server.url("/inline.pdf");+ let output = run_fetch_cli_with_dump_and_args(&url, "html", &["--wait-script", "true"])?;+ runtime.block_on(server.shutdown());++ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");+ assert!(stdout.is_empty(), "stdout={stdout}");+ assert_single_fetch_failure_reason(+ &stderr,+ &url,+ "raw non-HTML document fetch does not support page wait options",+ );+ Ok(())+}+#[test]fn cli_dump_html_keeps_uncaught_script_errors_out_of_stdout() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;@@ -2738,7 +2822,7 @@ fn cli_load_uses_one_timeout_across_403_replacement_navigation() -> Result<()> {let stderr = clean_output(&output.stderr);assert!(stdout.is_empty(), "stdout={stdout}");assert!(- stderr.contains("allow-http-error wait_until Load timed out after 250 ms"),+ stderr.contains("fetch readiness timed out after 250 ms while waiting for load"),"stderr={stderr}");Ok(())@@ -3445,6 +3529,54 @@ fn cli_dump_json_keeps_transport_failures_as_process_errors() -> Result<()> {Ok(())}+#[test]+fn cli_timeout_reports_waiting_for_response_headers_as_one_reason() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let listener = runtime.block_on(TcpListener::bind("127.0.0.1:0"))?;+ let addr = listener.local_addr()?;+ let hanging_server = runtime.spawn(async move {+ let (mut stream, _) = listener+ .accept()+ .await+ .expect("header timeout fixture should accept");+ let mut request = [0_u8; 1024];+ let _ = stream.read(&mut request).await;+ std::future::pending::<()>().await;+ });+ let url = format!("http://{addr}/headers-never-arrive");++ let output = run_moli([+ "moli",+ "fetch",+ "--log-level",+ "error",+ "--http-no-proxy",+ "*",+ "--http-timeout",+ "5000",+ "--wait-until",+ "domcontentloaded",+ "--timeout",+ "300",+ "--dump",+ "html",+ &url,+ ])?;+ hanging_server.abort();+ let _ = runtime.block_on(hanging_server);++ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");+ assert!(stdout.is_empty(), "stdout={stdout}");+ assert_single_fetch_failure_reason(+ &stderr,+ &url,+ "fetch readiness timed out after 300 ms while waiting for response headers",+ );+ Ok(())+}+#[test]fn cli_streaming_page_creation_timeout_exits_cleanly_for_every_wait_mode() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;@@ -3491,6 +3623,11 @@ fn cli_streaming_page_creation_timeout_exits_cleanly_for_every_wait_mode() -> Restderr.contains("timed out after 400 ms"),"wait_until={wait_until} stderr={stderr}");+ assert_single_fetch_failure_reason(+ &stderr,+ &url,+ "fetch readiness timed out after 400 ms while streaming main body",+ );assert!(!stderr.contains("resident renderer page entry must retain an active PageVm"),"wait_until={wait_until} leaked the renderer teardown invariant: stderr={stderr}"@@ -3534,6 +3671,42 @@ fn cli_meta_refresh_shutdown_exits_without_renderer_abort() -> Result<()> {Ok(())}+#[test]+fn cli_timeout_reports_parser_blocking_stylesheet_as_one_reason() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(BlockingStylesheetTimeoutFixtureServer::spawn())?;+ let url = server.url();+ let output = run_moli([+ "moli",+ "fetch",+ "--log-level",+ "error",+ "--http-no-proxy",+ "*",+ "--http-timeout",+ "5000",+ "--wait-until",+ "domcontentloaded",+ "--timeout",+ "500",+ "--dump",+ "html",+ &url,+ ])?;+ runtime.block_on(server.shutdown());++ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");+ assert!(stdout.is_empty(), "stdout={stdout}");+ assert_single_fetch_failure_reason(+ &stderr,+ &url,+ "fetch readiness timed out after 500 ms while waiting for parser-blocking stylesheet",+ );+ Ok(())+}+#[test]fn cli_domcontentloaded_exit_stays_stable_with_inflight_background_fetch_tail() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;