Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): preserve readiness deadlines and diagnostics

ldm054f453cdd0b0
diff --git a/moli-browser-profile/src/profile.rs b/moli-browser-profile/src/profile.rs
index 519ab2a02..1e75711fb 100644
--- a/moli-browser-profile/src/profile.rs
+++ b/moli-browser-profile/src/profile.rs
@@ -20,12 +20,7 @@ impl BrowserProfile {
}
pub fn open_paths(paths: BrowserProfilePaths) -> Result<Self> {
- let profile_lock = BrowserProfileLock::acquire(&paths).with_context(|| {
- anyhow!(
- "failed to acquire browser profile lock `{}`",
- paths.lock_path.display()
- )
- })?;
+ let profile_lock = BrowserProfileLock::acquire(&paths)?;
let manifest = ensure_profile_manifest(&paths).with_context(|| {
anyhow!(
"failed to initialize browser profile manifest `{}`",
diff --git a/moli-core/src/runtime/fetch_deadline.rs b/moli-core/src/runtime/fetch_deadline.rs
index 96b23956d..628142308 100644
--- a/moli-core/src/runtime/fetch_deadline.rs
+++ b/moli-core/src/runtime/fetch_deadline.rs
@@ -1,7 +1,96 @@
//! One absolute timeout budget shared by every fetch-readiness phase.
-use anyhow::{Context, Result, anyhow, bail};
-use std::{future::Future, time::Duration};
+use anyhow::{Context, Result, anyhow};
+use std::{fmt, future::Future, time::Duration};
+
+/// The concrete operation consuming the shared fetch-readiness deadline.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum FetchTimeoutPhase {
+ WaitingForResponseHeaders,
+ StreamingMainBody,
+ ProcessingMainDocument,
+ WaitingForParserBlockingScript,
+ WaitingForParserBlockingStylesheet,
+ WaitingForDomContentLoaded,
+ WaitingForLoad,
+ WaitingForSubresourceResponse,
+ WaitingForSelector,
+ WaitingForScript,
+}
+
+impl FetchTimeoutPhase {
+ pub(super) fn from_renderer_phase(phase: moli_renderer_v8::RendererPageCreationPhase) -> Self {
+ match phase {
+ moli_renderer_v8::RendererPageCreationPhase::StreamingMainBody => {
+ Self::StreamingMainBody
+ }
+ moli_renderer_v8::RendererPageCreationPhase::ProcessingMainDocument => {
+ Self::ProcessingMainDocument
+ }
+ moli_renderer_v8::RendererPageCreationPhase::WaitingForParserBlockingScript => {
+ Self::WaitingForParserBlockingScript
+ }
+ moli_renderer_v8::RendererPageCreationPhase::WaitingForParserBlockingStylesheet => {
+ Self::WaitingForParserBlockingStylesheet
+ }
+ moli_renderer_v8::RendererPageCreationPhase::WaitingForDomContentLoaded => {
+ Self::WaitingForDomContentLoaded
+ }
+ moli_renderer_v8::RendererPageCreationPhase::WaitingForLoad => Self::WaitingForLoad,
+ }
+ }
+}
+
+impl fmt::Display for FetchTimeoutPhase {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str(match self {
+ Self::WaitingForResponseHeaders => "waiting for response headers",
+ Self::StreamingMainBody => "streaming main body",
+ Self::ProcessingMainDocument => "processing main document",
+ Self::WaitingForParserBlockingScript => "waiting for parser-blocking script",
+ Self::WaitingForParserBlockingStylesheet => "waiting for parser-blocking stylesheet",
+ Self::WaitingForDomContentLoaded => "waiting for DOMContentLoaded",
+ Self::WaitingForLoad => "waiting for load",
+ Self::WaitingForSubresourceResponse => "waiting for a subresource response",
+ Self::WaitingForSelector => "waiting for a selector",
+ Self::WaitingForScript => "waiting for a script to become truthy",
+ })
+ }
+}
+
+/// A deadline failure that retains the operation active when the budget won.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct FetchReadinessTimeout {
+ timeout: Duration,
+ phase: FetchTimeoutPhase,
+}
+
+impl FetchReadinessTimeout {
+ pub fn new(timeout: Duration, phase: FetchTimeoutPhase) -> Self {
+ Self { timeout, phase }
+ }
+
+ pub fn timeout(self) -> Duration {
+ self.timeout
+ }
+
+ pub fn phase(self) -> FetchTimeoutPhase {
+ self.phase
+ }
+}
+
+impl fmt::Display for FetchReadinessTimeout {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(
+ formatter,
+ "fetch readiness timed out after {} ms while {}",
+ self.timeout.as_millis(),
+ self.phase
+ )
+ }
+}
+
+impl std::error::Error for FetchReadinessTimeout {}
/// An absolute deadline for a complete fetch-readiness plan.
///
@@ -44,23 +133,23 @@ impl FetchDeadline {
self.at
}
- pub(super) async fn wait<T, F>(self, phase: &'static str, future: F) -> Result<T>
+ pub(super) async fn wait<T, F>(self, phase: FetchTimeoutPhase, future: F) -> Result<T>
where
F: Future<Output = Result<T>>,
{
match tokio::time::timeout_at(self.at, future).await {
Ok(result) => result,
- Err(_) => bail!(
- "fetch readiness timed out after {} ms while {phase}",
- self.timeout.as_millis()
- ),
+ Err(_) => Err(anyhow::Error::new(FetchReadinessTimeout::new(
+ self.timeout,
+ phase,
+ ))),
}
}
}
#[cfg(test)]
mod tests {
- use super::FetchDeadline;
+ use super::{FetchDeadline, FetchReadinessTimeout, FetchTimeoutPhase};
use anyhow::Result;
use std::time::Duration;
@@ -84,7 +173,7 @@ mod tests {
async fn wait_reports_total_budget_and_active_phase() -> Result<()> {
let deadline = FetchDeadline::new(Duration::from_millis(25))?;
let error = deadline
- .wait("waiting for a selector", async {
+ .wait(FetchTimeoutPhase::WaitingForSelector, async {
tokio::time::sleep(Duration::from_secs(1)).await;
Ok(())
})
@@ -95,6 +184,12 @@ mod tests {
error.to_string(),
"fetch readiness timed out after 25 ms while waiting for a selector"
);
+ assert_eq!(
+ error
+ .downcast_ref::<FetchReadinessTimeout>()
+ .map(|timeout| timeout.phase()),
+ Some(FetchTimeoutPhase::WaitingForSelector)
+ );
Ok(())
}
diff --git a/moli-core/src/runtime/lifecycle_fetch.rs b/moli-core/src/runtime/lifecycle_fetch.rs
index b3a178574..b8483d5a8 100644
--- a/moli-core/src/runtime/lifecycle_fetch.rs
+++ b/moli-core/src/runtime/lifecycle_fetch.rs
@@ -5,11 +5,12 @@
//! lifecycle decision cannot reset or extend the caller's timeout budget.
use super::{
- Browser, FetchDeadline, FetchedDocument, RenderedDomWaitUntil, RendererLifecycleDecider,
- RendererLifecycleDecision, RendererLifecycleSnapshot, RendererReplyBoundary,
+ Browser, FetchDeadline, FetchedDocument, RawDocumentDisposition, RenderedDomWaitUntil,
+ RendererLifecycleDecider, RendererLifecycleDecision, RendererLifecycleSnapshot,
+ RendererReplyBoundary,
};
-use anyhow::{Result, anyhow};
-use moli_fetch::{NetworkFetchFailureContext, Request};
+use anyhow::Result;
+use moli_fetch::Request;
use std::time::Duration;
impl Browser {
@@ -65,19 +66,8 @@ impl Browser {
deadline,
RendererReplyBoundary::Stage,
Some(decider),
+ RawDocumentDisposition::Materialize,
)
.await
- .map_err(|error| {
- // This typed context already supplies the caller-facing fetch
- // message. Keep it outermost instead of obscuring it with a
- // lifecycle layer that adds no actionable network information.
- if error.is::<NetworkFetchFailureContext>() {
- error
- } else {
- error.context(anyhow!(
- "failed while applying the {wait_until:?} lifecycle-target decision or following its successor navigation"
- ))
- }
- })
}
}
diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rs
index 05f73d8d3..2bfaa1a40 100644
--- a/moli-core/src/runtime/mod.rs
+++ b/moli-core/src/runtime/mod.rs
@@ -12,7 +12,7 @@ use crate::{
selector::QueryEngine,
};
use anyhow::Result;
-use anyhow::{Context, anyhow, bail};
+use anyhow::{Context, anyhow};
use moli_cookie_jar::StoredCookie;
use moli_fetch::{
FetchCancelHandle, RawResponse, Request, StreamingRawResponse, ensure_http_status_success,
@@ -35,7 +35,7 @@ use url::Url;
pub use crate::renderer::ExternalRawDocumentBodyStream;
pub use crate::renderer::PageVmInitStage;
pub use crate::renderer::RendererReplyBoundary;
-pub use fetch_deadline::FetchDeadline;
+pub use fetch_deadline::{FetchDeadline, FetchReadinessTimeout, FetchTimeoutPhase};
pub use moli_renderer_v8::{
DetachedParserScriptFetchContinuation, RendererBrowserContextRuntime,
RendererBrowserContextRuntimeOwner, RendererBrowserContextRuntimeOwnerAccess,
@@ -52,6 +52,16 @@ pub use navigation_engine::{
static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum RawDocumentDisposition {
+ Materialize,
+ RejectForPageApi,
+}
+
+fn raw_document_page_api_error() -> anyhow::Error {
+ anyhow!("raw non-HTML document cannot be returned through the Page fetch API")
+}
+
fn is_fetch_readiness_timeout(error: &anyhow::Error, wait_until: RenderedDomWaitUntil) -> bool {
let expected = match wait_until {
RenderedDomWaitUntil::NetworkIdle => "timed out waiting for networkidle",
@@ -156,6 +166,13 @@ impl RenderedDomWaitUntil {
}
}
+fn fetch_timeout_phase_for_stage(stage: PageVmInitStage) -> FetchTimeoutPhase {
+ match stage {
+ PageVmInitStage::DomContentLoaded => FetchTimeoutPhase::WaitingForDomContentLoaded,
+ PageVmInitStage::Load => FetchTimeoutPhase::WaitingForLoad,
+ }
+}
+
#[derive(Debug)]
pub struct RawDocument {
response: RawResponse,
@@ -354,48 +371,21 @@ impl Browser {
deadline: FetchDeadline,
allow_http_error: bool,
) -> Result<Page> {
- let raw_url = request.url.as_str().to_owned();
- let stage = wait_until.base_stage();
- debug!(
- url = %raw_url,
- wait_until = ?wait_until,
- timeout_ms = deadline.timeout().as_millis(),
- stage = ?stage,
- "starting fetch_with_wait_until deadline"
- );
-
- // Materialization only reaches the concrete DCL/Load base stage. The
- // selected stability observation is a separate Page operation so it
- // can consume the remaining budget and soften only its own timeout.
- let mut page = match tokio::time::timeout_at(
- deadline.at(),
- self.fetch_allow_http_error_internal(request, stage),
- )
- .await
+ let mut page = match self
+ .fetch_document_to_base_stage(
+ request,
+ wait_until,
+ deadline,
+ RendererReplyBoundary::Stage,
+ None,
+ RawDocumentDisposition::RejectForPageApi,
+ )
+ .await?
{
- // Preserve transport and policy errors from the fetch itself. The
- // outer deadline should add an error only when it actually wins.
- Ok(result) => result?,
- Err(_) => {
- warn!(
- url = %raw_url,
- wait_until = ?wait_until,
- timeout_ms = deadline.timeout().as_millis(),
- stage = ?stage,
- allow_http_error,
- "fetch_with_wait_until timed out"
- );
- if allow_http_error {
- bail!(
- "fetch allow-http-error wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",
- deadline.timeout().as_millis()
- );
- }
- bail!(
- "fetch wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",
- deadline.timeout().as_millis()
- );
- }
+ FetchedDocument::Page(page) => page,
+ FetchedDocument::Raw(_) => unreachable!(
+ "the Page fetch path rejects raw documents before body materialization"
+ ),
};
self.wait_for_page_readiness_with_deadline(&mut page, wait_until, deadline)
@@ -434,6 +424,7 @@ impl Browser {
deadline,
RendererReplyBoundary::Stage,
None,
+ RawDocumentDisposition::Materialize,
)
.await?;
@@ -457,6 +448,7 @@ impl Browser {
deadline: FetchDeadline,
reply_boundary: RendererReplyBoundary,
lifecycle_decider: Option<RendererLifecycleDecider>,
+ raw_document_disposition: RawDocumentDisposition,
) -> Result<FetchedDocument> {
let timeout = deadline.timeout();
let stage = wait_until.base_stage();
@@ -483,6 +475,7 @@ impl Browser {
timeout,
stage,
outer_deadline,
+ || fetch_timeout_phase_for_stage(stage),
self.materialize_static_html_page(
&raw_url,
requested_url,
@@ -507,6 +500,7 @@ impl Browser {
timeout,
stage,
outer_deadline,
+ || FetchTimeoutPhase::WaitingForResponseHeaders,
self.fetch_service_worker_main_resource_for_navigation(
&request,
&navigation_loader,
@@ -519,15 +513,20 @@ impl Browser {
} = service_worker_fetch;
if let Some(response) = response {
navigation_loader.note_service_worker_response_ready()?;
- let document_fetch_context_seed =
- navigation_loader.commit(response.final_url.clone())?;
if response_headers_indicate_raw_document(&response.headers) {
+ if raw_document_disposition == RawDocumentDisposition::RejectForPageApi {
+ return Err(raw_document_page_api_error());
+ }
let raw_response =
RawResponse::from_head_and_body(response.head(), response.clone_body_bytes());
return Ok(FetchedDocument::Raw(Box::new(RawDocument::from_response(
raw_response,
))));
}
+ let document_fetch_context_seed =
+ navigation_loader.commit(response.final_url.clone())?;
+ let page_creation_progress = moli_renderer_v8::RendererPageCreationProgress::new();
+ let timeout_progress = page_creation_progress.clone();
return self
.fetch_document_wait_timeout(
&raw_url,
@@ -535,7 +534,8 @@ impl Browser {
timeout,
stage,
outer_deadline,
- self.materialize_streaming_raw_response_page(
+ move || FetchTimeoutPhase::from_renderer_phase(timeout_progress.phase()),
+ self.materialize_streaming_raw_response_page_with_page_creation_progress(
&raw_url,
requested_url,
stage,
@@ -544,6 +544,7 @@ impl Browser {
streaming_raw_response_from_navigation_response(response)?,
document_fetch_context_seed,
reserved_client,
+ page_creation_progress,
),
)
.await
@@ -556,28 +557,42 @@ impl Browser {
timeout,
stage,
outer_deadline,
+ || FetchTimeoutPhase::WaitingForResponseHeaders,
navigation_loader.fetch_raw_stream(request),
)
.await?;
if response_headers_indicate_raw_document(&response.headers) {
- // `wait_until` is a DOM lifecycle deadline. Binary/download-like main
- // resources do not have DCL/load milestones, so only the initial
- // document response acquisition is gated above. The body dump is plain
- // network output and should not be reported as a DCL timeout.
+ if raw_document_disposition == RawDocumentDisposition::RejectForPageApi {
+ return Err(raw_document_page_api_error());
+ }
+ // Binary/download-like main resources have no DOM lifecycle
+ // milestones, but materializing a RawDocument still consumes the
+ // caller-owned absolute fetch-readiness budget.
return self
- .materialize_streaming_raw_response_raw_document(&raw_url, response)
+ .fetch_document_wait_timeout(
+ &raw_url,
+ wait_until,
+ timeout,
+ stage,
+ outer_deadline,
+ || FetchTimeoutPhase::StreamingMainBody,
+ self.materialize_streaming_raw_response_raw_document(response),
+ )
.await
.map(|raw| FetchedDocument::Raw(Box::new(raw)));
}
let document_fetch_context_seed = navigation_loader.commit(response.final_url.clone())?;
+ let page_creation_progress = moli_renderer_v8::RendererPageCreationProgress::new();
+ let timeout_progress = page_creation_progress.clone();
self.fetch_document_wait_timeout(
&raw_url,
wait_until,
timeout,
stage,
outer_deadline,
- self.materialize_streaming_raw_response_page(
+ move || FetchTimeoutPhase::from_renderer_phase(timeout_progress.phase()),
+ self.materialize_streaming_raw_response_page_with_page_creation_progress(
&raw_url,
requested_url,
stage,
@@ -586,6 +601,7 @@ impl Browser {
response,
document_fetch_context_seed,
reserved_client,
+ page_creation_progress,
),
)
.await
@@ -658,7 +674,7 @@ impl Browser {
let remaining = deadline.remaining();
deadline
.wait(
- "waiting for a selector",
+ FetchTimeoutPhase::WaitingForSelector,
page.wait_for_selector(&loader, selector, remaining),
)
.await
@@ -686,7 +702,7 @@ impl Browser {
let remaining = deadline.remaining();
deadline
.wait(
- "waiting for a script to become truthy",
+ FetchTimeoutPhase::WaitingForScript,
page.wait_for_script_truthy(&loader, expression, remaining),
)
.await
@@ -713,7 +729,7 @@ impl Browser {
let remaining = deadline.remaining();
deadline
.wait(
- "waiting for a subresource response",
+ FetchTimeoutPhase::WaitingForSubresourceResponse,
page.wait_for_subresource_response(&loader, criteria, remaining),
)
.await
@@ -970,7 +986,7 @@ impl Browser {
let document_fetch_context_seed = navigation_loader.commit(response.final_url.clone())?;
if response_headers_indicate_raw_document(&response.headers) {
return self
- .materialize_streaming_raw_response_raw_document(&raw_url, response)
+ .materialize_streaming_raw_response_raw_document(response)
.await
.map(|raw| FetchedDocument::Raw(Box::new(raw)));
}
@@ -999,23 +1015,29 @@ impl Browser {
.await
}
- async fn fetch_document_wait_timeout<T, F>(
+ async fn fetch_document_wait_timeout<T, F, P>(
&self,
raw_url: &str,
wait_until: RenderedDomWaitUntil,
timeout: Duration,
stage: PageVmInitStage,
deadline: tokio::time::Instant,
+ timeout_phase: P,
future: F,
) -> Result<T>
where
F: std::future::Future<Output = Result<T>>,
+ P: FnOnce() -> FetchTimeoutPhase,
{
match tokio::time::timeout_at(deadline, future).await {
Ok(result) => result,
- Err(_) => {
- Err(self.fetch_document_wait_timeout_error(raw_url, wait_until, timeout, stage))
- }
+ Err(_) => Err(self.fetch_document_wait_timeout_error(
+ raw_url,
+ wait_until,
+ timeout,
+ stage,
+ timeout_phase(),
+ )),
}
}
@@ -1025,18 +1047,17 @@ impl Browser {
wait_until: RenderedDomWaitUntil,
timeout: Duration,
stage: PageVmInitStage,
+ phase: FetchTimeoutPhase,
) -> anyhow::Error {
warn!(
url = %raw_url,
wait_until = ?wait_until,
timeout_ms = timeout.as_millis(),
stage = ?stage,
+ timeout_phase = %phase,
"fetch_document_allow_http_error_with_wait_until timed out"
);
- anyhow!(
- "fetch document allow-http-error wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",
- timeout.as_millis()
- )
+ anyhow::Error::new(FetchReadinessTimeout::new(timeout, phase))
}
async fn materialize_static_html_page(
@@ -1100,8 +1121,7 @@ impl Browser {
create_page_request.storage_bucket_store = Some(self.partition.storage_bucket_store());
let reply = renderer_owner
.dispatch_command(RendererOwnerCommand::CreateHtmlPage(create_page_request))
- .await
- .with_context(|| anyhow!("failed to execute scripts for page `{raw_url}`"))?;
+ .await?;
let page = materialize_page_created_reply(&renderer_owner, reply)?;
info!(
page_id = page.page_id(),
@@ -1122,6 +1142,32 @@ impl Browser {
response: StreamingRawResponse,
document_fetch_context_seed: DocumentFetchContextSeed,
reserved_service_worker_client: Option<RendererReservedServiceWorkerClient>,
+ ) -> Result<Page> {
+ self.materialize_streaming_raw_response_page_with_page_creation_progress(
+ raw_url,
+ requested_url,
+ stage,
+ reply_boundary,
+ lifecycle_decider,
+ response,
+ document_fetch_context_seed,
+ reserved_service_worker_client,
+ moli_renderer_v8::RendererPageCreationProgress::new(),
+ )
+ .await
+ }
+
+ async fn materialize_streaming_raw_response_page_with_page_creation_progress(
+ &self,
+ raw_url: &str,
+ requested_url: Url,
+ stage: PageVmInitStage,
+ reply_boundary: RendererReplyBoundary,
+ lifecycle_decider: Option<RendererLifecycleDecider>,
+ response: StreamingRawResponse,
+ document_fetch_context_seed: DocumentFetchContextSeed,
+ reserved_service_worker_client: Option<RendererReservedServiceWorkerClient>,
+ page_creation_progress: moli_renderer_v8::RendererPageCreationProgress,
) -> Result<Page> {
let started = Instant::now();
debug!(
@@ -1146,7 +1192,11 @@ impl Browser {
let response_headers = response.headers.clone();
let redirected = response.redirected;
let redirect_count = response.redirect_chain.len();
- let raw_body = external_raw_document_body_from_streaming_response(response);
+ let raw_body =
+ external_raw_document_body_from_streaming_response_with_page_creation_progress(
+ response,
+ page_creation_progress,
+ );
let document_start_scripts = self
.config
.document_start_scripts()
@@ -1210,8 +1260,7 @@ impl Browser {
None,
lifecycle_decider,
)
- .await
- .with_context(|| anyhow!("failed to execute scripts for page `{raw_url}`"))?;
+ .await?;
if pending_download.is_some() {
return Err(anyhow!(
"raw streaming page creation produced a pending download for `{raw_url}`"
@@ -1233,16 +1282,12 @@ impl Browser {
async fn materialize_streaming_raw_response_raw_document(
&self,
- raw_url: &str,
response: StreamingRawResponse,
) -> Result<RawDocument> {
let started = Instant::now();
let final_url = response.final_url.clone();
let status = response.status;
- let raw_response = response
- .into_materialized_raw_response()
- .await
- .with_context(|| anyhow!("failed to read raw document body for `{raw_url}`"))?;
+ let raw_response = response.into_materialized_raw_response().await?;
info!(
url = %final_url,
status,
@@ -1258,18 +1303,42 @@ impl Browser {
}
}
-fn external_raw_document_body_from_streaming_response(
+fn external_raw_document_body_from_streaming_response_with_page_creation_progress(
response: StreamingRawResponse,
+ page_creation_progress: moli_renderer_v8::RendererPageCreationProgress,
) -> ExternalRawDocumentBodyStream {
- external_raw_document_body_from_streaming_response_with_body_eof_observer(response, None)
+ external_raw_document_body_from_streaming_response_with_observer_and_progress(
+ response,
+ None,
+ Some(page_creation_progress),
+ )
}
+#[cfg(test)]
fn external_raw_document_body_from_streaming_response_with_body_eof_observer(
+ response: StreamingRawResponse,
+ body_eof_observer: Option<oneshot::Sender<()>>,
+) -> ExternalRawDocumentBodyStream {
+ external_raw_document_body_from_streaming_response_with_observer_and_progress(
+ response,
+ body_eof_observer,
+ None,
+ )
+}
+
+fn external_raw_document_body_from_streaming_response_with_observer_and_progress(
mut response: StreamingRawResponse,
mut body_eof_observer: Option<oneshot::Sender<()>>,
+ page_creation_progress: Option<moli_renderer_v8::RendererPageCreationProgress>,
) -> ExternalRawDocumentBodyStream {
let (completion_tx, completion_rx) = oneshot::channel();
- let (body_tx, body_stream) = ExternalRawDocumentBodyStream::channel(completion_rx);
+ let (body_tx, body_stream) = match page_creation_progress {
+ Some(progress) => ExternalRawDocumentBodyStream::channel_with_page_creation_progress(
+ completion_rx,
+ progress,
+ ),
+ None => ExternalRawDocumentBodyStream::channel(completion_rx),
+ };
tokio::spawn(async move {
let result = async {
loop {
diff --git a/moli-core/src/runtime/storage_partition.rs b/moli-core/src/runtime/storage_partition.rs
index f2cdbe489..5bc1da92f 100644
--- a/moli-core/src/runtime/storage_partition.rs
+++ b/moli-core/src/runtime/storage_partition.rs
@@ -98,8 +98,7 @@ impl StoragePartitionState {
pub fn open(profile_dir: Option<&Path>) -> Result<Self> {
let profile = profile_dir
.map(BrowserProfile::open)
- .transpose()
- .context("failed to open browser profile")?
+ .transpose()?
.map(Arc::new);
Self::from_profile(profile)
}
diff --git a/moli-core/tests/fetch.rs b/moli-core/tests/fetch.rs
index 2cd702382..5c59a66ed 100644
--- a/moli-core/tests/fetch.rs
+++ b/moli-core/tests/fetch.rs
@@ -1,11 +1,46 @@
use moli_test_support as support;
use anyhow::Result;
-use moli_core::runtime::{Browser, BrowserConfig as AppConfig, RenderedDomWaitUntil};
-use std::time::Duration;
+use moli_core::runtime::{
+ Browser, BrowserConfig as AppConfig, FetchReadinessTimeout, FetchTimeoutPhase,
+ RenderedDomWaitUntil,
+};
+use moli_fetch::Request;
+use std::time::{Duration, Instant};
use support::FixtureServer;
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::net::TcpListener;
+use tokio::task::JoinHandle;
use url::Url;
+async fn spawn_stalled_binary_response_server() -> Result<(String, JoinHandle<()>)> {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let address = listener.local_addr()?;
+ let server = tokio::spawn(async move {
+ let Ok((mut stream, _)) = listener.accept().await else {
+ return;
+ };
+ let mut request = [0_u8; 4096];
+ let Ok(read) = stream.read(&mut request).await else {
+ return;
+ };
+ if read == 0 {
+ return;
+ }
+ if stream
+ .write_all(
+ b"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: 1048576\r\nConnection: keep-alive\r\n\r\n",
+ )
+ .await
+ .is_err()
+ {
+ return;
+ }
+ std::future::pending::<()>().await;
+ });
+ Ok((format!("http://{address}/stalled.bin"), server))
+}
+
#[tokio::test]
async fn fetches_static_fixture() -> Result<()> {
let server = FixtureServer::spawn().await?;
@@ -122,6 +157,80 @@ async fn fetch_allow_http_error_returns_page_for_success() -> Result<()> {
Ok(())
}
+#[tokio::test]
+async fn fetch_with_wait_until_rejects_raw_document_through_page_api() -> Result<()> {
+ let server = FixtureServer::spawn().await?;
+ let browser = Browser::new(AppConfig::default())?;
+
+ let error = browser
+ .fetch_with_wait_until(
+ &server.url("/net/upstream/xhr/binary"),
+ RenderedDomWaitUntil::Load,
+ Duration::from_secs(1),
+ )
+ .await
+ .expect_err("the Page fetch API must reject a raw non-HTML document");
+
+ assert_eq!(
+ error.to_string(),
+ "raw non-HTML document cannot be returned through the Page fetch API"
+ );
+
+ server.shutdown().await;
+ Ok(())
+}
+
+#[tokio::test]
+async fn page_fetch_rejects_raw_response_without_waiting_for_its_stalled_body() -> Result<()> {
+ let (url, server) = spawn_stalled_binary_response_server().await?;
+ let browser = Browser::new(AppConfig::default())?;
+ let started = Instant::now();
+
+ let error = tokio::time::timeout(
+ Duration::from_secs(1),
+ browser.fetch_with_wait_until(&url, RenderedDomWaitUntil::Load, Duration::from_millis(100)),
+ )
+ .await
+ .expect("the Page API must not wait for a raw response body")
+ .expect_err("the Page API must reject a raw non-HTML document");
+
+ assert_eq!(
+ error.to_string(),
+ "raw non-HTML document cannot be returned through the Page fetch API"
+ );
+ assert!(
+ started.elapsed() < Duration::from_secs(1),
+ "raw Page rejection must not fall through to the HTTP transport timeout"
+ );
+ server.abort();
+ Ok(())
+}
+
+#[tokio::test]
+async fn raw_document_body_materialization_uses_the_fetch_deadline() -> Result<()> {
+ let (url, server) = spawn_stalled_binary_response_server().await?;
+ let browser = Browser::new(AppConfig::default())?;
+
+ let error = tokio::time::timeout(
+ Duration::from_secs(1),
+ browser.fetch_request_document_allow_http_error_with_wait_until(
+ Request::get(&url)?,
+ RenderedDomWaitUntil::Load,
+ Duration::from_millis(100),
+ ),
+ )
+ .await
+ .expect("raw body materialization must honor the fetch deadline")
+ .expect_err("the stalled raw body must time out");
+ let timeout = error
+ .downcast_ref::<FetchReadinessTimeout>()
+ .expect("the raw body timeout must retain its typed phase");
+ assert_eq!(timeout.phase(), FetchTimeoutPhase::StreamingMainBody);
+
+ server.abort();
+ Ok(())
+}
+
#[tokio::test]
async fn fetch_allow_http_error_renders_404_response() -> Result<()> {
let server = FixtureServer::spawn().await?;
diff --git a/moli-fetch/src/network_fetch_result.rs b/moli-fetch/src/network_fetch_result.rs
index a21627f1f..e692bf04f 100644
--- a/moli-fetch/src/network_fetch_result.rs
+++ b/moli-fetch/src/network_fetch_result.rs
@@ -376,6 +376,7 @@ fn parse_status_line(line: &str) -> Option<u16> {
pub struct NetworkFetchFailureContext {
observation_journal: NetworkObservationJournal,
network_error_text: &'static str,
+ reason: String,
request_context: Option<NetworkFetchFailureRequestContext>,
}
@@ -437,9 +438,11 @@ impl NetworkFetchFailureContext {
observation_journal: NetworkObservationJournal,
) -> anyhow::Error {
let network_error_text = crate::error::browser_network_error_text(&source);
+ let reason = source.to_string();
source.context(Self {
observation_journal,
network_error_text,
+ reason,
request_context: None,
})
}
@@ -450,9 +453,11 @@ impl NetworkFetchFailureContext {
request_context: NetworkFetchFailureRequestContext,
) -> anyhow::Error {
let network_error_text = crate::error::browser_network_error_text(&source);
+ let reason = source.to_string();
source.context(Self {
observation_journal,
network_error_text,
+ reason,
request_context: Some(request_context),
})
}
@@ -465,6 +470,12 @@ impl NetworkFetchFailureContext {
self.network_error_text
}
+ /// Returns the single human-readable transport or policy reason captured
+ /// before this machine-readable context was attached.
+ pub fn reason(&self) -> &str {
+ &self.reason
+ }
+
pub fn request_context(&self) -> Option<&NetworkFetchFailureRequestContext> {
self.request_context.as_ref()
}
@@ -604,6 +615,12 @@ mod tests {
);
assert_eq!(report.matches("transport failure sentinel").count(), 1);
assert!(error.downcast_ref::<NetworkFetchFailureContext>().is_some());
+ assert_eq!(
+ error
+ .downcast_ref::<NetworkFetchFailureContext>()
+ .map(NetworkFetchFailureContext::reason),
+ Some("transport failure sentinel")
+ );
assert!(error.downcast_ref::<std::io::Error>().is_some());
}
@@ -612,6 +629,7 @@ mod tests {
let failure = NetworkFetchFailureContext {
observation_journal: NetworkObservationJournal::default(),
network_error_text: "net::ERR_FAILED",
+ reason: "transport rejected secret values".to_owned(),
request_context: Some(NetworkFetchFailureRequestContext::new(
Url::parse("https://example.test/").expect("test URL should parse"),
"POST".to_owned(),
@@ -625,6 +643,10 @@ mod tests {
assert!(!report.contains("secret request body"), "{report}");
assert!(!report.contains("secret token"), "{report}");
+ assert!(
+ !report.contains("transport rejected secret values"),
+ "{report}"
+ );
assert!(report.contains("has_request_body: true"), "{report}");
assert!(report.contains("request_header_count: 1"), "{report}");
}
diff --git a/moli-renderer-v8/src/lib.rs b/moli-renderer-v8/src/lib.rs
index d26b83fd8..06dadaa99 100644
--- a/moli-renderer-v8/src/lib.rs
+++ b/moli-renderer-v8/src/lib.rs
@@ -272,8 +272,9 @@ pub use runtime::{
RendererOwnerResourceActivitySource, RendererOwnerRuntimeActivitySource, RendererPageCommand,
RendererPageCommandPending, RendererPageCommandPostResponseContinuation,
RendererPageCookieFacadeSnapshotReply, RendererPageCreationArtifacts,
- RendererPageCreationDiagnostics, RendererPageDiagnosticsSnapshot, RendererPageDumpFormat,
- RendererPageDumpOptions, RendererPageDumpStripOptions, RendererPageHandle, RendererPageReply,
+ RendererPageCreationDiagnostics, RendererPageCreationPhase, RendererPageCreationProgress,
+ RendererPageDiagnosticsSnapshot, RendererPageDumpFormat, RendererPageDumpOptions,
+ RendererPageDumpStripOptions, RendererPageHandle, RendererPageReply,
RendererPageReservationToken, RendererPageState, RendererPageTestingHandle, RendererPageView,
RendererPendingDownloadActivation, RendererPendingDownloadResponse,
RendererPendingFileChooserActivation, RendererPendingJavaScriptDialog,
diff --git a/moli-renderer-v8/src/runtime/mod.rs b/moli-renderer-v8/src/runtime/mod.rs
index c88c61b99..85d98bd0a 100644
--- a/moli-renderer-v8/src/runtime/mod.rs
+++ b/moli-renderer-v8/src/runtime/mod.rs
@@ -57,6 +57,7 @@ mod owner_maintenance;
mod page;
mod page_commands;
mod page_context_cancel;
+mod page_creation_progress;
mod page_css;
pub(crate) mod page_dom;
mod page_dom_snapshot;
@@ -81,6 +82,8 @@ pub(crate) use self::script_preloads::{
IncrementalBufferedScriptPreloadScanner,
};
+pub use self::page_creation_progress::{RendererPageCreationPhase, RendererPageCreationProgress};
+
pub(crate) use self::browser_context_runtime::RendererOutputTransportSenderSlot;
pub(in crate::runtime) use self::document_lifecycle_turn::PendingDocumentLifecycleTurn;
pub(crate) use self::page_turn_scheduler::{
diff --git a/moli-renderer-v8/src/runtime/page_creation_progress.rs b/moli-renderer-v8/src/runtime/page_creation_progress.rs
new file mode 100644
index 000000000..243bf7745
--- /dev/null
+++ b/moli-renderer-v8/src/runtime/page_creation_progress.rs
@@ -0,0 +1,84 @@
+use std::sync::{
+ Arc,
+ atomic::{AtomicU8, Ordering},
+};
+
+/// The renderer-side condition currently preventing an initial page from
+/// reaching its requested lifecycle boundary.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[repr(u8)]
+pub enum RendererPageCreationPhase {
+ StreamingMainBody = 0,
+ ProcessingMainDocument = 1,
+ WaitingForParserBlockingScript = 2,
+ WaitingForParserBlockingStylesheet = 3,
+ WaitingForDomContentLoaded = 4,
+ WaitingForLoad = 5,
+}
+
+impl RendererPageCreationPhase {
+ fn from_u8(value: u8) -> Self {
+ match value {
+ 0 => Self::StreamingMainBody,
+ 1 => Self::ProcessingMainDocument,
+ 2 => Self::WaitingForParserBlockingScript,
+ 3 => Self::WaitingForParserBlockingStylesheet,
+ 4 => Self::WaitingForDomContentLoaded,
+ 5 => Self::WaitingForLoad,
+ _ => Self::ProcessingMainDocument,
+ }
+ }
+}
+
+/// Shared progress reported by the renderer while a streamed main Document is
+/// being created.
+///
+/// The browser-side deadline owns a clone and reads it only if that deadline
+/// expires. Renderer owner turns update the same value immediately before
+/// parking on body input, parser-blocking resources, or a lifecycle target.
+#[derive(Clone, Debug)]
+pub struct RendererPageCreationProgress {
+ phase: Arc<AtomicU8>,
+}
+
+impl RendererPageCreationProgress {
+ pub fn new() -> Self {
+ Self {
+ phase: Arc::new(AtomicU8::new(
+ RendererPageCreationPhase::StreamingMainBody as u8,
+ )),
+ }
+ }
+
+ pub fn phase(&self) -> RendererPageCreationPhase {
+ RendererPageCreationPhase::from_u8(self.phase.load(Ordering::Relaxed))
+ }
+
+ pub(crate) fn set_phase(&self, phase: RendererPageCreationPhase) {
+ self.phase.store(phase as u8, Ordering::Relaxed);
+ }
+}
+
+impl Default for RendererPageCreationProgress {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn clones_observe_the_latest_phase() {
+ let progress = RendererPageCreationProgress::new();
+ let observer = progress.clone();
+
+ progress.set_phase(RendererPageCreationPhase::WaitingForParserBlockingStylesheet);
+
+ assert_eq!(
+ observer.phase(),
+ RendererPageCreationPhase::WaitingForParserBlockingStylesheet
+ );
+ }
+}
diff --git a/moli-renderer-v8/src/runtime/page_vm/mod.rs b/moli-renderer-v8/src/runtime/page_vm/mod.rs
index 8a87ebc6f..873b4255e 100644
--- a/moli-renderer-v8/src/runtime/page_vm/mod.rs
+++ b/moli-renderer-v8/src/runtime/page_vm/mod.rs
@@ -1114,6 +1114,7 @@ pub(crate) struct PageVmEnvConfig {
#[derive(Clone)]
pub(crate) struct PageVmRuntimeHooks {
owner_wake: Option<RendererOwnerWakeSender>,
+ page_creation_progress: Option<super::RendererPageCreationProgress>,
javascript_dialog_runtime: RendererJavaScriptDialogRuntime,
resource_task_runner: Option<crate::network::RendererResourceTaskRunner>,
pub(crate) browser_context_runtime: super::RendererBrowserContextRuntime,
@@ -1175,6 +1176,7 @@ impl PageVmRuntimeHooks {
Self::bind_standalone_browser_context_owner_for_test(&browser_context_runtime);
Self {
owner_wake: None,
+ page_creation_progress: None,
javascript_dialog_runtime: RendererJavaScriptDialogRuntime::default(),
resource_task_runner: None,
browser_context_runtime,
@@ -1246,6 +1248,7 @@ impl PageVmRuntimeHooks {
Self {
javascript_dialog_runtime: RendererJavaScriptDialogRuntime::default(),
owner_wake: Some(owner_wake),
+ page_creation_progress: None,
resource_task_runner: Some(residence.resource_task_runner()),
browser_context_runtime,
renderer_document_isolate_mode:
@@ -1284,6 +1287,7 @@ impl PageVmRuntimeHooks {
Self {
javascript_dialog_runtime: RendererJavaScriptDialogRuntime::default(),
owner_wake: Some(owner_wake),
+ page_creation_progress: None,
resource_task_runner: Some(
crate::network::RendererResourceTaskRunner::from_current_tokio()
.expect("renderer owner must install its resource task runner"),
@@ -1304,6 +1308,20 @@ impl PageVmRuntimeHooks {
}
}
+ pub(crate) fn with_page_creation_progress(
+ mut self,
+ page_creation_progress: super::RendererPageCreationProgress,
+ ) -> Self {
+ self.page_creation_progress = Some(page_creation_progress);
+ self
+ }
+
+ pub(crate) fn set_page_creation_phase(&self, phase: super::RendererPageCreationPhase) {
+ if let Some(progress) = &self.page_creation_progress {
+ progress.set_phase(phase);
+ }
+ }
+
pub(crate) fn with_renderer_document_isolate_allocator(
mut self,
allocator: RendererDocumentIsolateAllocator,
diff --git a/moli-renderer-v8/src/runtime/phase_one/pending_residence.rs b/moli-renderer-v8/src/runtime/phase_one/pending_residence.rs
index 96c1be038..d72eb80bb 100644
--- a/moli-renderer-v8/src/runtime/phase_one/pending_residence.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/pending_residence.rs
@@ -73,6 +73,7 @@ impl PendingPhaseOneResidence {
runtime: Box<ConcurrentParseTimeRuntime>,
started: Instant,
) -> Self {
+ runtime.publish_pending_page_creation_phase();
Self::ParserBlockingSourceLoad { runtime, started }
}
@@ -80,12 +81,14 @@ impl PendingPhaseOneResidence {
runtime: Box<ConcurrentParseTimeRuntime>,
started: Instant,
) -> Self {
+ runtime.publish_pending_page_creation_phase();
Self::ClosedInputPageWork { runtime, started }
}
pub(in crate::runtime) fn open_streaming(
continuation: Box<PendingStreamingPhaseOneContinuation>,
) -> Self {
+ continuation.publish_pending_page_creation_phase();
Self::OpenStreaming(continuation)
}
@@ -134,6 +137,7 @@ impl PendingPhaseOneResidence {
match self {
Self::ParserBlockingSourceLoad { runtime, started }
| Self::ClosedInputPageWork { runtime, started } => {
+ runtime.publish_processing_main_document_phase();
let outcome = (*runtime)
.continue_creation_from_phase_one_runtime(started)
.await?;
diff --git a/moli-renderer-v8/src/runtime/phase_one/phase_two.rs b/moli-renderer-v8/src/runtime/phase_one/phase_two.rs
index cd6c143e0..9965da692 100644
--- a/moli-renderer-v8/src/runtime/phase_one/phase_two.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/phase_two.rs
@@ -17,6 +17,7 @@ impl ConcurrentParseTimeRuntime {
&self.page_vm.local_executor,
"parse-time phase transition handoff",
);
+ self.publish_target_lifecycle_phase();
let (stranded_parse_time_document_script_events, stranded_parse_time_lifecycle_work) =
self.retire_main_parser_continuation();
self.page_vm
diff --git a/moli-renderer-v8/src/runtime/phase_one/state.rs b/moli-renderer-v8/src/runtime/phase_one/state.rs
index e66bed246..c70916171 100644
--- a/moli-renderer-v8/src/runtime/phase_one/state.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/state.rs
@@ -84,6 +84,46 @@ pub(in crate::runtime) struct ConcurrentParseTimeRuntime {
}
impl ConcurrentParseTimeRuntime {
+ pub(in crate::runtime) fn publish_processing_main_document_phase(&self) {
+ self.page_vm.runtime_hooks.set_page_creation_phase(
+ crate::runtime::RendererPageCreationPhase::ProcessingMainDocument,
+ );
+ }
+
+ pub(in crate::runtime) fn publish_pending_page_creation_phase(&self) {
+ let phase = match self.pending_parsing_blocking_wait {
+ PendingParsingBlockingWait::PageTaskBlockingStylesheet => {
+ crate::runtime::RendererPageCreationPhase::WaitingForParserBlockingStylesheet
+ }
+ PendingParsingBlockingWait::LegacyDocumentProcessing
+ | PendingParsingBlockingWait::PageNetworkingDocumentWriteExternalScript => {
+ crate::runtime::RendererPageCreationPhase::WaitingForParserBlockingScript
+ }
+ PendingParsingBlockingWait::None
+ if self.has_unready_pending_parser_blocking_source_load() =>
+ {
+ crate::runtime::RendererPageCreationPhase::WaitingForParserBlockingScript
+ }
+ PendingParsingBlockingWait::None if !self.state.input_closed => {
+ crate::runtime::RendererPageCreationPhase::StreamingMainBody
+ }
+ PendingParsingBlockingWait::None => {
+ crate::runtime::RendererPageCreationPhase::ProcessingMainDocument
+ }
+ };
+ self.page_vm.runtime_hooks.set_page_creation_phase(phase);
+ }
+
+ pub(super) fn publish_target_lifecycle_phase(&self) {
+ let phase = match self.stage {
+ PageVmInitStage::DomContentLoaded => {
+ crate::runtime::RendererPageCreationPhase::WaitingForDomContentLoaded
+ }
+ PageVmInitStage::Load => crate::runtime::RendererPageCreationPhase::WaitingForLoad,
+ };
+ self.page_vm.runtime_hooks.set_page_creation_phase(phase);
+ }
+
pub(in crate::runtime) fn page_vm(&self) -> &PageVm {
&self.page_vm
}
diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
index 7c4d0746a..edb798a3b 100644
--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
@@ -341,6 +341,10 @@ impl ConcurrentParseTimeRuntime {
boundary: CommittedNavigationBootstrapBoundary,
) -> Result<StreamingNavigationPageCreationResult> {
debug_assert!(!response_headers_indicate_download(&response_headers));
+ let runtime_hooks = match raw_body.page_creation_progress() {
+ Some(progress) => runtime_hooks.with_page_creation_progress(progress),
+ None => runtime_hooks,
+ };
let mut env = env.clone();
env.document_content_security_policies = if env.bypass_content_security_policy {
Vec::new()
@@ -568,6 +572,7 @@ const EXTERNAL_RAW_DOCUMENT_BODY_BUFFERED_CHUNKS: usize = 8;
pub struct ExternalRawDocumentBodyStream {
body_chunks: mpsc::Receiver<Vec<u8>>,
completion: Option<oneshot::Receiver<Result<()>>>,
+ page_creation_progress: Option<crate::runtime::RendererPageCreationProgress>,
}
impl ExternalRawDocumentBodyStream {
@@ -581,6 +586,23 @@ impl ExternalRawDocumentBodyStream {
(body_tx, Self::new(body_chunks, completion))
}
+ /// Creates a bounded stream whose renderer progress remains observable by
+ /// the browser-side navigation deadline.
+ pub fn channel_with_page_creation_progress(
+ completion: oneshot::Receiver<Result<()>>,
+ page_creation_progress: crate::runtime::RendererPageCreationProgress,
+ ) -> (mpsc::Sender<Vec<u8>>, Self) {
+ let (body_tx, body_chunks) = mpsc::channel(EXTERNAL_RAW_DOCUMENT_BODY_BUFFERED_CHUNKS);
+ (
+ body_tx,
+ Self {
+ body_chunks,
+ completion: Some(completion),
+ page_creation_progress: Some(page_creation_progress),
+ },
+ )
+ }
+
pub fn new(
body_chunks: mpsc::Receiver<Vec<u8>>,
completion: oneshot::Receiver<Result<()>>,
@@ -588,6 +610,7 @@ impl ExternalRawDocumentBodyStream {
Self {
body_chunks,
completion: Some(completion),
+ page_creation_progress: None,
}
}
@@ -603,6 +626,10 @@ impl ExternalRawDocumentBodyStream {
fn body_chunk_stream_is_exhausted(&self) -> bool {
self.body_chunks.is_closed() && self.body_chunks.is_empty()
}
+
+ fn page_creation_progress(&self) -> Option<crate::runtime::RendererPageCreationProgress> {
+ self.page_creation_progress.clone()
+ }
}
pub(super) enum RawDocumentBodySource {
diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming_residence.rs b/moli-renderer-v8/src/runtime/phase_one/streaming_residence.rs
index 96557ba94..714542f0f 100644
--- a/moli-renderer-v8/src/runtime/phase_one/streaming_residence.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/streaming_residence.rs
@@ -28,6 +28,10 @@ pub(in crate::runtime) struct PendingStreamingPhaseOneContinuation {
}
impl PendingStreamingPhaseOneContinuation {
+ pub(super) fn publish_pending_page_creation_phase(&self) {
+ self.runtime.publish_pending_page_creation_phase();
+ }
+
pub(super) fn bridge(
runtime: ConcurrentParseTimeRuntime,
raw_body: RawDocumentBodySource,
@@ -71,6 +75,7 @@ impl PendingStreamingPhaseOneContinuation {
}
pub(in crate::runtime) async fn resume(self) -> Result<PendingPhaseOneResumeOutcome> {
+ self.runtime.publish_processing_main_document_phase();
let Self {
runtime,
mut input,
diff --git a/moli/src/app.rs b/moli/src/app.rs
index 094a774d3..180a0c2dd 100644
--- a/moli/src/app.rs
+++ b/moli/src/app.rs
@@ -3,7 +3,7 @@
mod readiness;
mod redirect_navigation;
-use std::{io::Write, sync::Arc};
+use std::{fmt, io::Write, sync::Arc};
use crate::{
cli::{Cli, Commands, normalize_args_for_compat},
@@ -49,8 +49,7 @@ pub async fn run_cli_with_config<W: Write>(
.await
.map_err(|error| with_fetch_context(error, &args.url))?;
}
- let browser = Browser::new(config.browser.clone())
- .context("failed to initialize browser runtime")?;
+ let browser = Browser::new(config.browser.clone())?;
load_cookie_state(&browser, &config)?;
let readiness =
ReadinessPlan::from_fetch_args(&args, config.fetch.response_wait.clone())?;
@@ -68,16 +67,20 @@ pub async fn run_cli_with_config<W: Write>(
FetchedDocument::Raw(raw_document) => {
if readiness.has_page_waits() || args.delay_ms > 0 {
finalize_fetch_browser(browser);
- return Err(anyhow!(
- "raw non-HTML document fetch does not support page wait options"
+ return Err(with_fetch_context(
+ anyhow!(
+ "raw non-HTML document fetch does not support page wait options"
+ ),
+ &args.url,
));
}
let rendered =
fetch_dump::render_raw_document_dump(&raw_document, &config.fetch)
- .context("failed to render raw fetch output")?;
+ .map_err(|error| with_fetch_context(error, &args.url))?;
stdout
.write_all(&rendered)
- .context("failed to write raw fetch output")?;
+ .context("failed to write raw fetch output")
+ .map_err(|error| with_fetch_context(error, &args.url))?;
let _ = stdout.flush();
finalize_fetch_browser(browser);
return Ok(());
@@ -99,15 +102,17 @@ pub async fn run_cli_with_config<W: Write>(
browser
.wait_for_page_delay(&mut page, std::time::Duration::from_millis(args.delay_ms))
.await
- .context("failed while waiting for page delay")?;
+ .context("failed while waiting for page delay")
+ .map_err(|error| with_fetch_context(error, &args.url))?;
}
let rendered = fetch_dump::render_page_output_async(&mut page, &config.fetch)
.await
- .context("failed to render fetch output")?;
+ .map_err(|error| with_fetch_context(error, &args.url))?;
stdout
.write_all(&rendered)
- .context("failed to write fetch output")?;
+ .context("failed to write fetch output")
+ .map_err(|error| with_fetch_context(error, &args.url))?;
let _ = stdout.flush();
if let Err(error) = page.close_async().await {
tracing::warn!(error = %error, "failed to close fetched page before browser shutdown");
@@ -146,14 +151,66 @@ fn build_fetch_request(url: &str, config: &AppConfig) -> Result<Request> {
Ok(request)
}
+struct CliFetchFailureContext {
+ url: String,
+ reason: String,
+}
+
+impl fmt::Display for CliFetchFailureContext {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(formatter, "failed to fetch `{}`", self.url)
+ }
+}
+
+impl fmt::Debug for CliFetchFailureContext {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter
+ .debug_struct("CliFetchFailureContext")
+ .field("url", &self.url)
+ .field("has_reason", &!self.reason.is_empty())
+ .finish()
+ }
+}
+
fn with_fetch_context(error: anyhow::Error, url: &str) -> anyhow::Error {
- // Network failures already carry a URL-aware typed context. Adding the
- // same CLI context again would print two "failed to fetch" chain entries.
- if error.is::<NetworkFetchFailureContext>() {
- error
- } else {
- error.context(anyhow!("failed to fetch `{url}`"))
+ if error.is::<CliFetchFailureContext>() {
+ return error;
+ }
+ let reason = error
+ .downcast_ref::<NetworkFetchFailureContext>()
+ .map(|failure| failure.reason().to_owned())
+ .unwrap_or_else(|| error.to_string());
+ with_fetch_context_reason(error, url, reason)
+}
+
+fn with_fetch_context_reason(
+ error: anyhow::Error,
+ url: &str,
+ reason: impl Into<String>,
+) -> anyhow::Error {
+ error.context(CliFetchFailureContext {
+ url: url.to_owned(),
+ reason: reason.into(),
+ })
+}
+
+/// Writes the stable, concise command-line error presentation.
+///
+/// Fetch errors carry their selected user-facing reason in a typed outer
+/// context and deliberately render only that two-line presentation. Other CLI
+/// failures retain their anyhow context chain so startup and configuration
+/// diagnostics do not lose their actionable inner cause.
+pub fn write_error_report<W: Write>(writer: &mut W, error: &anyhow::Error) -> std::io::Result<()> {
+ if let Some(fetch) = error.downcast_ref::<CliFetchFailureContext>() {
+ writeln!(writer, "Error: {fetch}")?;
+ writeln!(writer, "Reason: {}", one_line_reason(&fetch.reason))?;
+ return Ok(());
}
+ writeln!(writer, "Error: {error:#}")
+}
+
+fn one_line_reason(reason: &str) -> String {
+ reason.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn load_cookie_state(browser: &Browser, config: &AppConfig) -> Result<()> {
@@ -178,3 +235,38 @@ fn finalize_fetch_browser(browser: Browser) {
// Browser::drop owns profile cookie writeback when --profile-dir is set.
drop(browser);
}
+
+#[cfg(test)]
+mod tests {
+ use super::{with_fetch_context, write_error_report};
+
+ #[test]
+ fn fetch_report_has_one_reason_line_without_rendering_the_source_chain() {
+ let error = with_fetch_context(
+ anyhow::anyhow!("first failure line\nsecond failure line"),
+ "https://example.test/",
+ );
+ let mut report = Vec::new();
+
+ write_error_report(&mut report, &error).expect("report should write");
+ let report = String::from_utf8(report).expect("report should be UTF-8");
+
+ assert_eq!(
+ report,
+ "Error: failed to fetch `https://example.test/`\nReason: first failure line second failure line\n"
+ );
+ assert!(!report.contains("Caused by:"));
+ }
+
+ #[test]
+ fn non_fetch_report_retains_the_anyhow_context_chain() {
+ let error = anyhow::anyhow!("inner cause").context("outer context");
+ let mut report = Vec::new();
+
+ write_error_report(&mut report, &error).expect("report should write");
+ let report = String::from_utf8(report).expect("report should be UTF-8");
+
+ assert!(report.contains("outer context"));
+ assert!(report.contains("inner cause"));
+ }
+}
diff --git a/moli/src/main.rs b/moli/src/main.rs
index 28918ecb7..8d6768bd3 100644
--- a/moli/src/main.rs
+++ b/moli/src/main.rs
@@ -5,11 +5,12 @@
//! protocol server.
use anyhow::Result;
+use std::process::ExitCode;
#[cfg(all(feature = "jemalloc", not(target_os = "windows")))]
mod allocator;
-fn main() -> Result<()> {
+fn run() -> Result<()> {
moli_process_signal::install_immediate_exit_handlers()?;
let runtime_thread_budget = moli::runtime_thread_budget::tokio_runtime_thread_budget();
tokio::runtime::Builder::new_multi_thread()
@@ -19,3 +20,14 @@ fn main() -> Result<()> {
.build()?
.block_on(moli::app::run_from_env())
}
+
+fn main() -> ExitCode {
+ match run() {
+ Ok(()) => ExitCode::SUCCESS,
+ Err(error) => {
+ let stderr = std::io::stderr();
+ let _ = moli::app::write_error_report(&mut stderr.lock(), &error);
+ ExitCode::FAILURE
+ }
+ }
+}
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index 99b14b27b..eb83b4a3d 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -31,7 +31,7 @@ use std::{
};
use strip_ansi_escapes::strip;
use support::FixtureServer;
-use tokio::{net::TcpListener, task::JoinHandle};
+use tokio::{io::AsyncReadExt, net::TcpListener, task::JoinHandle};
use tracing_subscriber::fmt::MakeWriter;
struct Output {
@@ -169,7 +169,7 @@ where
let result = runtime.block_on(app::run_cli_with_config(cli, config, &mut stdout));
if let Err(error) = &result {
let mut stderr = stderr.lock();
- writeln!(&mut *stderr, "{error:?}")?;
+ app::write_error_report(&mut *stderr, error)?;
}
let stderr = stderr.lock().clone();
Ok(Output {
@@ -247,6 +247,11 @@ struct BinaryDocumentFixtureServer {
task: JoinHandle<()>,
}
+struct BlockingStylesheetTimeoutFixtureServer {
+ base_url: String,
+ task: JoinHandle<()>,
+}
+
const OPTIONAL_RESOURCE_PATHS: [&str; 5] = [
"/optional-audio.mp3",
"/optional-font.woff2",
@@ -463,6 +468,52 @@ impl BinaryDocumentFixtureServer {
}
}
+impl BlockingStylesheetTimeoutFixtureServer {
+ async fn spawn() -> Result<Self> {
+ async fn page() -> Html<&'static str> {
+ Html(
+ r#"<!doctype html>
+<html>
+ <head>
+ <link rel="stylesheet" href="/slow.css">
+ <script>document.documentElement.dataset.parserResumed = "true";</script>
+ </head>
+ <body></body>
+</html>"#,
+ )
+ }
+
+ async fn slow_css() -> impl IntoResponse {
+ tokio::time::sleep(std::time::Duration::from_secs(2)).await;
+ ([("content-type", "text/css; charset=utf-8")], "html {}")
+ }
+
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let addr = listener.local_addr()?;
+ let app = Router::new()
+ .route("/", get(page))
+ .route("/slow.css", get(slow_css));
+ let task = tokio::spawn(async move {
+ axum::serve(listener, app)
+ .await
+ .expect("blocking stylesheet timeout fixture should serve");
+ });
+ Ok(Self {
+ base_url: format!("http://{addr}"),
+ task,
+ })
+ }
+
+ fn url(&self) -> String {
+ format!("{}/", self.base_url)
+ }
+
+ async fn shutdown(self) {
+ self.task.abort();
+ let _ = self.task.await;
+ }
+}
+
impl CacheableFixtureServer {
async fn spawn() -> Result<Self> {
async fn cacheable(
@@ -542,6 +593,19 @@ fn assert_cli_dump_html_callback_error_contract(output: &Output, expected_messag
);
}
+fn assert_single_fetch_failure_reason(stderr: &str, url: &str, reason: &str) {
+ assert!(
+ stderr.contains(&format!("Error: failed to fetch `{url}`")),
+ "stderr={stderr}"
+ );
+ assert!(
+ stderr.contains(&format!("Reason: {reason}")),
+ "stderr={stderr}"
+ );
+ assert_eq!(stderr.matches("Reason:").count(), 1, "stderr={stderr}");
+ assert!(!stderr.contains("Caused by:"), "stderr={stderr}");
+}
+
fn assert_cli_dump_html_debug_callback_error_contract(output: &Output, expected_message: &str) {
assert!(
output.status.success(),
@@ -947,6 +1011,26 @@ fn cli_fetch_attachment_dump_html_bypasses_dcl_page_lifecycle() -> Result<()> {
Ok(())
}
+#[test]
+fn cli_fetch_raw_document_with_page_wait_uses_fetch_failure_report() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(BinaryDocumentFixtureServer::spawn())?;
+ let url = server.url("/inline.pdf");
+ let output = run_fetch_cli_with_dump_and_args(&url, "html", &["--wait-script", "true"])?;
+ runtime.block_on(server.shutdown());
+
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");
+ assert!(stdout.is_empty(), "stdout={stdout}");
+ assert_single_fetch_failure_reason(
+ &stderr,
+ &url,
+ "raw non-HTML document fetch does not support page wait options",
+ );
+ Ok(())
+}
+
#[test]
fn cli_dump_html_keeps_uncaught_script_errors_out_of_stdout() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
@@ -2738,7 +2822,7 @@ fn cli_load_uses_one_timeout_across_403_replacement_navigation() -> Result<()> {
let stderr = clean_output(&output.stderr);
assert!(stdout.is_empty(), "stdout={stdout}");
assert!(
- stderr.contains("allow-http-error wait_until Load timed out after 250 ms"),
+ stderr.contains("fetch readiness timed out after 250 ms while waiting for load"),
"stderr={stderr}"
);
Ok(())
@@ -3445,6 +3529,54 @@ fn cli_dump_json_keeps_transport_failures_as_process_errors() -> Result<()> {
Ok(())
}
+#[test]
+fn cli_timeout_reports_waiting_for_response_headers_as_one_reason() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let listener = runtime.block_on(TcpListener::bind("127.0.0.1:0"))?;
+ let addr = listener.local_addr()?;
+ let hanging_server = runtime.spawn(async move {
+ let (mut stream, _) = listener
+ .accept()
+ .await
+ .expect("header timeout fixture should accept");
+ let mut request = [0_u8; 1024];
+ let _ = stream.read(&mut request).await;
+ std::future::pending::<()>().await;
+ });
+ let url = format!("http://{addr}/headers-never-arrive");
+
+ let output = run_moli([
+ "moli",
+ "fetch",
+ "--log-level",
+ "error",
+ "--http-no-proxy",
+ "*",
+ "--http-timeout",
+ "5000",
+ "--wait-until",
+ "domcontentloaded",
+ "--timeout",
+ "300",
+ "--dump",
+ "html",
+ &url,
+ ])?;
+ hanging_server.abort();
+ let _ = runtime.block_on(hanging_server);
+
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");
+ assert!(stdout.is_empty(), "stdout={stdout}");
+ assert_single_fetch_failure_reason(
+ &stderr,
+ &url,
+ "fetch readiness timed out after 300 ms while waiting for response headers",
+ );
+ Ok(())
+}
+
#[test]
fn cli_streaming_page_creation_timeout_exits_cleanly_for_every_wait_mode() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
@@ -3491,6 +3623,11 @@ fn cli_streaming_page_creation_timeout_exits_cleanly_for_every_wait_mode() -> Re
stderr.contains("timed out after 400 ms"),
"wait_until={wait_until} stderr={stderr}"
);
+ assert_single_fetch_failure_reason(
+ &stderr,
+ &url,
+ "fetch readiness timed out after 400 ms while streaming main body",
+ );
assert!(
!stderr.contains("resident renderer page entry must retain an active PageVm"),
"wait_until={wait_until} leaked the renderer teardown invariant: stderr={stderr}"
@@ -3534,6 +3671,42 @@ fn cli_meta_refresh_shutdown_exits_without_renderer_abort() -> Result<()> {
Ok(())
}
+#[test]
+fn cli_timeout_reports_parser_blocking_stylesheet_as_one_reason() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(BlockingStylesheetTimeoutFixtureServer::spawn())?;
+ let url = server.url();
+ let output = run_moli([
+ "moli",
+ "fetch",
+ "--log-level",
+ "error",
+ "--http-no-proxy",
+ "*",
+ "--http-timeout",
+ "5000",
+ "--wait-until",
+ "domcontentloaded",
+ "--timeout",
+ "500",
+ "--dump",
+ "html",
+ &url,
+ ])?;
+ runtime.block_on(server.shutdown());
+
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");
+ assert!(stdout.is_empty(), "stdout={stdout}");
+ assert_single_fetch_failure_reason(
+ &stderr,
+ &url,
+ "fetch readiness timed out after 500 ms while waiting for parser-blocking stylesheet",
+ );
+ Ok(())
+}
+
#[test]
fn cli_domcontentloaded_exit_stays_stable_with_inflight_background_fetch_tail() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;