Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): share deadline with best-effort readiness

ldm0aefa01568d52
diff --git a/moli-core/src/runtime/fetch_deadline.rs b/moli-core/src/runtime/fetch_deadline.rs
index c48b50f82..96b23956d 100644
--- a/moli-core/src/runtime/fetch_deadline.rs
+++ b/moli-core/src/runtime/fetch_deadline.rs
@@ -44,15 +44,6 @@ impl FetchDeadline {
self.at
}
- pub(super) fn at_with_cushion(self, cushion: Duration) -> Result<tokio::time::Instant> {
- self.at.checked_add(cushion).with_context(|| {
- anyhow!(
- "fetch readiness deadline with {} ms cleanup cushion exceeds the supported range",
- cushion.as_millis()
- )
- })
- }
-
pub(super) async fn wait<T, F>(self, phase: &'static str, future: F) -> Result<T>
where
F: Future<Output = Result<T>>,
diff --git a/moli-core/src/runtime/lifecycle_fetch.rs b/moli-core/src/runtime/lifecycle_fetch.rs
index bb9a4cf1d..4ec3db70b 100644
--- a/moli-core/src/runtime/lifecycle_fetch.rs
+++ b/moli-core/src/runtime/lifecycle_fetch.rs
@@ -59,7 +59,7 @@ impl Browser {
"a lifecycle decider requires DCL, load, or done"
);
let decider = RendererLifecycleDecider::new(decider);
- self.fetch_document_with_wait(
+ self.fetch_document_to_base_stage(
request,
wait_until,
deadline,
diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rs
index 83356d2a3..05f73d8d3 100644
--- a/moli-core/src/runtime/mod.rs
+++ b/moli-core/src/runtime/mod.rs
@@ -12,7 +12,7 @@ use crate::{
selector::QueryEngine,
};
use anyhow::Result;
-use anyhow::{Context, anyhow};
+use anyhow::{Context, anyhow, bail};
use moli_cookie_jar::StoredCookie;
use moli_fetch::{
FetchCancelHandle, RawResponse, Request, StreamingRawResponse, ensure_http_status_success,
@@ -52,17 +52,6 @@ pub use navigation_engine::{
static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);
-fn wait_until_outer_timeout(wait_until: RenderedDomWaitUntil, timeout: Duration) -> Duration {
- match wait_until {
- RenderedDomWaitUntil::NetworkIdle | RenderedDomWaitUntil::DomStable => {
- timeout.saturating_add(Duration::from_secs(1))
- }
- RenderedDomWaitUntil::DomContentLoaded
- | RenderedDomWaitUntil::Load
- | RenderedDomWaitUntil::Done => timeout,
- }
-}
-
fn is_fetch_readiness_timeout(error: &anyhow::Error, wait_until: RenderedDomWaitUntil) -> bool {
let expected = match wait_until {
RenderedDomWaitUntil::NetworkIdle => "timed out waiting for networkidle",
@@ -151,6 +140,22 @@ pub enum RenderedDomWaitUntil {
Done,
}
+impl RenderedDomWaitUntil {
+ /// The concrete lifecycle boundary at which a live `Page` can first be
+ /// handed back to the host. Network-idle and DOM-stable are observations
+ /// made on that live Page, after Load and DCL respectively.
+ fn base_stage(self) -> PageVmInitStage {
+ match self {
+ Self::DomContentLoaded | Self::DomStable => PageVmInitStage::DomContentLoaded,
+ Self::Load | Self::NetworkIdle | Self::Done => PageVmInitStage::Load,
+ }
+ }
+
+ fn has_best_effort_page_wait(self) -> bool {
+ matches!(self, Self::NetworkIdle | Self::DomStable)
+ }
+}
+
#[derive(Debug)]
pub struct RawDocument {
response: RawResponse,
@@ -273,8 +278,7 @@ impl Browser {
}
pub async fn fetch_request(&self, request: Request) -> Result<Page> {
- self.fetch_internal(request, PageVmInitStage::Load, None)
- .await
+ self.fetch_internal(request, PageVmInitStage::Load).await
}
pub async fn fetch_request_with_wait_until(
@@ -283,44 +287,13 @@ impl Browser {
wait_until: RenderedDomWaitUntil,
timeout: Duration,
) -> Result<Page> {
- let stage = match wait_until {
- RenderedDomWaitUntil::DomContentLoaded => PageVmInitStage::DomContentLoaded,
- RenderedDomWaitUntil::Load => PageVmInitStage::Load,
- RenderedDomWaitUntil::NetworkIdle => PageVmInitStage::Load,
- RenderedDomWaitUntil::DomStable => PageVmInitStage::DomContentLoaded,
- RenderedDomWaitUntil::Done => PageVmInitStage::Load,
- };
-
- let raw_url = request.url.as_str().to_owned();
- debug!(
- url = %raw_url,
- wait_until = ?wait_until,
- timeout_ms = timeout.as_millis(),
- stage = ?stage,
- "starting fetch_with_wait_until deadline"
- );
- let outer_timeout = wait_until_outer_timeout(wait_until, timeout);
- match tokio::time::timeout(
- outer_timeout,
- self.fetch_internal(request, stage, Some((wait_until, timeout))),
+ self.fetch_page_with_wait_until_deadline(
+ request,
+ wait_until,
+ FetchDeadline::new(timeout)?,
+ false,
)
.await
- {
- Ok(result) => result,
- Err(_) => {
- warn!(
- url = %raw_url,
- wait_until = ?wait_until,
- timeout_ms = timeout.as_millis(),
- stage = ?stage,
- "fetch_with_wait_until timed out"
- );
- Err(anyhow!(
- "fetch wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",
- timeout.as_millis()
- ))
- }
- }
}
pub async fn fetch_allow_http_error(&self, raw_url: &str) -> Result<Page> {
@@ -343,7 +316,7 @@ impl Browser {
}
pub async fn fetch_request_allow_http_error(&self, request: Request) -> Result<Page> {
- self.fetch_allow_http_error_internal(request, PageVmInitStage::Load, None)
+ self.fetch_allow_http_error_internal(request, PageVmInitStage::Load)
.await
}
@@ -355,7 +328,6 @@ impl Browser {
request,
PageVmInitStage::Load,
RendererReplyBoundary::Stage,
- None,
)
.await
}
@@ -366,44 +338,72 @@ impl Browser {
wait_until: RenderedDomWaitUntil,
timeout: Duration,
) -> Result<Page> {
- let stage = match wait_until {
- RenderedDomWaitUntil::DomContentLoaded => PageVmInitStage::DomContentLoaded,
- RenderedDomWaitUntil::Load => PageVmInitStage::Load,
- RenderedDomWaitUntil::NetworkIdle => PageVmInitStage::Load,
- RenderedDomWaitUntil::DomStable => PageVmInitStage::DomContentLoaded,
- RenderedDomWaitUntil::Done => PageVmInitStage::Load,
- };
+ self.fetch_page_with_wait_until_deadline(
+ request,
+ wait_until,
+ FetchDeadline::new(timeout)?,
+ true,
+ )
+ .await
+ }
+ async fn fetch_page_with_wait_until_deadline(
+ &self,
+ request: Request,
+ wait_until: RenderedDomWaitUntil,
+ deadline: FetchDeadline,
+ allow_http_error: bool,
+ ) -> Result<Page> {
let raw_url = request.url.as_str().to_owned();
+ let stage = wait_until.base_stage();
debug!(
url = %raw_url,
wait_until = ?wait_until,
- timeout_ms = timeout.as_millis(),
+ timeout_ms = deadline.timeout().as_millis(),
stage = ?stage,
- "starting fetch_allow_http_error_with_wait_until deadline"
+ "starting fetch_with_wait_until deadline"
);
- let outer_timeout = wait_until_outer_timeout(wait_until, timeout);
- match tokio::time::timeout(
- outer_timeout,
- self.fetch_allow_http_error_internal(request, stage, Some((wait_until, timeout))),
+
+ // Materialization only reaches the concrete DCL/Load base stage. The
+ // selected stability observation is a separate Page operation so it
+ // can consume the remaining budget and soften only its own timeout.
+ let mut page = match tokio::time::timeout_at(
+ deadline.at(),
+ self.fetch_allow_http_error_internal(request, stage),
)
.await
{
- Ok(result) => result,
+ // Preserve transport and policy errors from the fetch itself. The
+ // outer deadline should add an error only when it actually wins.
+ Ok(result) => result?,
Err(_) => {
warn!(
url = %raw_url,
wait_until = ?wait_until,
- timeout_ms = timeout.as_millis(),
+ timeout_ms = deadline.timeout().as_millis(),
stage = ?stage,
- "fetch_allow_http_error_with_wait_until timed out"
+ allow_http_error,
+ "fetch_with_wait_until timed out"
+ );
+ if allow_http_error {
+ bail!(
+ "fetch allow-http-error wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",
+ deadline.timeout().as_millis()
+ );
+ }
+ bail!(
+ "fetch wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",
+ deadline.timeout().as_millis()
);
- Err(anyhow!(
- "fetch allow-http-error wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",
- timeout.as_millis()
- ))
}
+ };
+
+ self.wait_for_page_readiness_with_deadline(&mut page, wait_until, deadline)
+ .await?;
+ if !allow_http_error {
+ ensure_http_status_success(page.final_url().as_str(), page.status(), false)?;
}
+ Ok(page)
}
pub async fn fetch_request_document_allow_http_error_with_wait_until(
@@ -427,17 +427,30 @@ impl Browser {
wait_until: RenderedDomWaitUntil,
deadline: FetchDeadline,
) -> Result<FetchedDocument> {
- self.fetch_document_with_wait(
- request,
- wait_until,
- deadline,
- RendererReplyBoundary::Stage,
- None,
- )
- .await
+ let fetched = self
+ .fetch_document_to_base_stage(
+ request,
+ wait_until,
+ deadline,
+ RendererReplyBoundary::Stage,
+ None,
+ )
+ .await?;
+
+ match fetched {
+ FetchedDocument::Page(mut page) => {
+ self.wait_for_page_readiness_with_deadline(&mut page, wait_until, deadline)
+ .await?;
+ Ok(FetchedDocument::Page(page))
+ }
+ FetchedDocument::Raw(raw) => Ok(FetchedDocument::Raw(raw)),
+ }
}
- async fn fetch_document_with_wait(
+ /// Reaches the concrete lifecycle boundary that makes a live Page
+ /// available. For NetworkIdle and DomStable this is only the Load or DCL
+ /// base stage; their best-effort observation runs outside materialization.
+ async fn fetch_document_to_base_stage(
&self,
request: Request,
wait_until: RenderedDomWaitUntil,
@@ -446,13 +459,7 @@ impl Browser {
lifecycle_decider: Option<RendererLifecycleDecider>,
) -> Result<FetchedDocument> {
let timeout = deadline.timeout();
- let stage = match wait_until {
- RenderedDomWaitUntil::DomContentLoaded => PageVmInitStage::DomContentLoaded,
- RenderedDomWaitUntil::Load => PageVmInitStage::Load,
- RenderedDomWaitUntil::NetworkIdle => PageVmInitStage::Load,
- RenderedDomWaitUntil::DomStable => PageVmInitStage::DomContentLoaded,
- RenderedDomWaitUntil::Done => PageVmInitStage::Load,
- };
+ let stage = wait_until.base_stage();
let raw_url = request.url.as_str().to_owned();
debug!(
@@ -462,18 +469,11 @@ impl Browser {
stage = ?stage,
"starting fetch_document_allow_http_error_with_wait_until deadline"
);
- // Network-idle and DOM-stable return a best-effort Page after their
- // renderer-side readiness timer expires. Keep the historical one-second
- // transport cushion so that Page can cross the owner boundary, while
- // the logical FetchDeadline remains expired for every later plan step.
- let outer_deadline = match wait_until {
- RenderedDomWaitUntil::NetworkIdle | RenderedDomWaitUntil::DomStable => {
- deadline.at_with_cushion(Duration::from_secs(1))?
- }
- RenderedDomWaitUntil::DomContentLoaded
- | RenderedDomWaitUntil::Load
- | RenderedDomWaitUntil::Done => deadline.at(),
- };
+ // This deadline is strict: if the response or base DCL/Load stage does
+ // not arrive in time, there is no Page on which best-effort readiness
+ // can operate. Only the later NetworkIdle/DomStable observation may
+ // soften an expired deadline.
+ let outer_deadline = deadline.at();
let requested_url = request.url.clone();
if is_about_blank_url(&requested_url) {
return self
@@ -489,7 +489,6 @@ impl Browser {
stage,
reply_boundary,
lifecycle_decider,
- Some((wait_until, timeout)),
String::new(),
),
)
@@ -542,7 +541,6 @@ impl Browser {
stage,
reply_boundary,
lifecycle_decider,
- Some((wait_until, timeout)),
streaming_raw_response_from_navigation_response(response)?,
document_fetch_context_seed,
reserved_client,
@@ -585,7 +583,6 @@ impl Browser {
stage,
reply_boundary,
lifecycle_decider,
- Some((wait_until, timeout)),
response,
document_fetch_context_seed,
reserved_client,
@@ -722,53 +719,68 @@ impl Browser {
.await
}
- async fn wait_for_fetch_readiness(
+ /// Observes NetworkIdle or DomStable with the unspent part of `deadline`.
+ ///
+ /// A timeout in this observation is deliberately best-effort: the Page
+ /// has already reached its required Load/DCL base stage, so the current
+ /// snapshot is returned with a warning. Errors unrelated to timeout still
+ /// fail the fetch. Concrete lifecycle modes are no-ops here.
+ pub async fn wait_for_page_readiness_with_deadline(
&self,
page: &mut Page,
- raw_url: &str,
- wait: Option<(RenderedDomWaitUntil, Duration)>,
+ wait_until: RenderedDomWaitUntil,
+ deadline: FetchDeadline,
) -> Result<()> {
- let Some((wait_until, timeout)) = wait else {
+ if !wait_until.has_best_effort_page_wait() {
return Ok(());
- };
+ }
+
+ let loader = self.resource_request_client();
+ let remaining = deadline.remaining();
let result = match wait_until {
RenderedDomWaitUntil::NetworkIdle => {
- page.wait_for_network_idle(&self.resource_request_client(), timeout)
- .await
+ tokio::time::timeout_at(
+ deadline.at(),
+ page.wait_for_network_idle(&loader, remaining),
+ )
+ .await
}
RenderedDomWaitUntil::DomStable => {
- page.wait_for_dom_stable(&self.resource_request_client(), timeout)
+ tokio::time::timeout_at(deadline.at(), page.wait_for_dom_stable(&loader, remaining))
.await
}
RenderedDomWaitUntil::DomContentLoaded
| RenderedDomWaitUntil::Load
- | RenderedDomWaitUntil::Done => return Ok(()),
+ | RenderedDomWaitUntil::Done => unreachable!(
+ "concrete lifecycle modes returned before starting a best-effort Page wait"
+ ),
};
- match result {
- Ok(()) => Ok(()),
- Err(error) if is_fetch_readiness_timeout(&error, wait_until) => {
- warn!(
- page_id = page.page_id(),
- url = %raw_url,
- final_url = %page.final_url(),
- wait_until = ?wait_until,
- timeout_ms = timeout.as_millis(),
- error = %error,
- "fetch readiness wait timed out; returning best-effort page"
- );
- Ok(())
+ let timeout_error = match result {
+ Ok(Ok(())) => return Ok(()),
+ Ok(Err(error)) if is_fetch_readiness_timeout(&error, wait_until) => error,
+ Ok(Err(error)) => {
+ return Err(error).with_context(|| {
+ anyhow!("failed while waiting for page readiness {wait_until:?}")
+ });
}
- Err(error) => Err(error),
- }
+ Err(error) => anyhow::Error::new(error),
+ };
+
+ warn!(
+ page_id = page.page_id(),
+ url = %page.requested_url(),
+ final_url = %page.final_url(),
+ wait_until = ?wait_until,
+ timeout_ms = deadline.timeout().as_millis(),
+ remaining_ms = remaining.as_millis(),
+ error = %timeout_error,
+ "fetch readiness wait timed out; returning best-effort page"
+ );
+ Ok(())
}
- async fn fetch_internal(
- &self,
- request: Request,
- stage: PageVmInitStage,
- wait: Option<(RenderedDomWaitUntil, Duration)>,
- ) -> Result<Page> {
+ async fn fetch_internal(&self, request: Request, stage: PageVmInitStage) -> Result<Page> {
let raw_url = request.url.as_str().to_owned();
let requested_url = request.url.clone();
if is_about_blank_url(&requested_url) {
@@ -779,7 +791,6 @@ impl Browser {
stage,
RendererReplyBoundary::Stage,
None,
- wait,
String::new(),
)
.await;
@@ -808,7 +819,6 @@ impl Browser {
stage,
RendererReplyBoundary::Stage,
None,
- wait,
response,
document_fetch_context_seed,
reserved_client,
@@ -826,7 +836,6 @@ impl Browser {
stage,
RendererReplyBoundary::Stage,
None,
- wait,
response,
document_fetch_context_seed,
reserved_client,
@@ -840,7 +849,6 @@ impl Browser {
&self,
request: Request,
stage: PageVmInitStage,
- wait: Option<(RenderedDomWaitUntil, Duration)>,
) -> Result<Page> {
let raw_url = request.url.as_str().to_owned();
let requested_url = request.url.clone();
@@ -852,7 +860,6 @@ impl Browser {
stage,
RendererReplyBoundary::Stage,
None,
- wait,
String::new(),
)
.await;
@@ -880,7 +887,6 @@ impl Browser {
stage,
RendererReplyBoundary::Stage,
None,
- wait,
streaming_raw_response_from_navigation_response(response)?,
document_fetch_context_seed,
reserved_client,
@@ -895,7 +901,6 @@ impl Browser {
stage,
RendererReplyBoundary::Stage,
None,
- wait,
response,
document_fetch_context_seed,
reserved_client,
@@ -908,7 +913,6 @@ impl Browser {
request: Request,
stage: PageVmInitStage,
reply_boundary: RendererReplyBoundary,
- wait: Option<(RenderedDomWaitUntil, Duration)>,
) -> Result<FetchedDocument> {
let raw_url = request.url.as_str().to_owned();
let requested_url = request.url.clone();
@@ -920,7 +924,6 @@ impl Browser {
stage,
reply_boundary,
None,
- wait,
String::new(),
)
.await
@@ -956,7 +959,6 @@ impl Browser {
stage,
reply_boundary,
None,
- wait,
streaming_raw_response_from_navigation_response(response)?,
document_fetch_context_seed,
reserved_client,
@@ -978,7 +980,6 @@ impl Browser {
stage,
reply_boundary,
None,
- wait,
response,
document_fetch_context_seed,
reserved_client,
@@ -1045,14 +1046,12 @@ impl Browser {
stage: PageVmInitStage,
reply_boundary: RendererReplyBoundary,
lifecycle_decider: Option<RendererLifecycleDecider>,
- wait: Option<(RenderedDomWaitUntil, Duration)>,
response_body: String,
) -> Result<Page> {
let started = Instant::now();
debug!(
url = %raw_url,
stage = ?stage,
- wait = ?wait.map(|(wait_until, timeout)| (wait_until, timeout.as_millis())),
"starting static html fetch_internal"
);
@@ -1103,13 +1102,10 @@ impl Browser {
.dispatch_command(RendererOwnerCommand::CreateHtmlPage(create_page_request))
.await
.with_context(|| anyhow!("failed to execute scripts for page `{raw_url}`"))?;
- let mut page = materialize_page_created_reply(&renderer_owner, reply)?;
- self.wait_for_fetch_readiness(&mut page, raw_url, wait)
- .await?;
+ let page = materialize_page_created_reply(&renderer_owner, reply)?;
info!(
page_id = page.page_id(),
url = %page.requested_url(),
- wait_until = ?wait.map(|(until, timeout)| (until, timeout.as_millis())),
elapsed_ms = started.elapsed().as_millis(),
"static html fetch_internal completed"
);
@@ -1123,7 +1119,6 @@ impl Browser {
stage: PageVmInitStage,
reply_boundary: RendererReplyBoundary,
lifecycle_decider: Option<RendererLifecycleDecider>,
- wait: Option<(RenderedDomWaitUntil, Duration)>,
response: StreamingRawResponse,
document_fetch_context_seed: DocumentFetchContextSeed,
reserved_service_worker_client: Option<RendererReservedServiceWorkerClient>,
@@ -1132,7 +1127,6 @@ impl Browser {
debug!(
url = %raw_url,
stage = ?stage,
- wait = ?wait.map(|(wait_until, timeout)| (wait_until, timeout.as_millis())),
"starting raw streaming fetch_internal"
);
@@ -1223,17 +1217,14 @@ impl Browser {
"raw streaming page creation produced a pending download for `{raw_url}`"
));
}
- let mut page = Page::from_attached_handle_with_creation_artifacts(
+ let page = Page::from_attached_handle_with_creation_artifacts(
handle,
page_state,
page_creation_artifacts,
);
- self.wait_for_fetch_readiness(&mut page, raw_url, wait)
- .await?;
info!(
page_id = page.page_id(),
url = %page.requested_url(),
- wait_until = ?wait.map(|(until, timeout)| (until, timeout.as_millis())),
elapsed_ms = started.elapsed().as_millis(),
"raw streaming fetch_internal completed"
);
diff --git a/moli-core/src/runtime/tests.rs b/moli-core/src/runtime/tests.rs
index cc3c7e76d..4ff4f66c0 100644
--- a/moli-core/src/runtime/tests.rs
+++ b/moli-core/src/runtime/tests.rs
@@ -2126,6 +2126,145 @@ async fn fetch_deadline_spans_lifecycle_selector_and_script_without_reset() -> R
Ok(())
}
+#[tokio::test(flavor = "multi_thread")]
+async fn best_effort_page_readiness_consumes_only_the_remaining_fetch_deadline() -> Result<()> {
+ let server = FixtureServer::spawn().await?;
+ let browser = Browser::new(AppConfig::default())?;
+ let renderer_owner = browser.js_runtime.renderer_owner_handle();
+
+ for (wait_until, path, expected_html) in [
+ (
+ RenderedDomWaitUntil::NetworkIdle,
+ "/wait-until-slow-interval-fetch",
+ "data-state=\"init\"",
+ ),
+ (
+ RenderedDomWaitUntil::DomStable,
+ "/wait-until-slow-interval-dom-mutation",
+ "id=\"mutation-count\"",
+ ),
+ ] {
+ // The main response costs 500 ms. Readiness must receive only the
+ // roughly 500 ms left in this one-second plan, not a fresh second.
+ let deadline = FetchDeadline::new(Duration::from_secs(1))?;
+ let started = std::time::Instant::now();
+ let fetched = browser
+ .fetch_request_document_allow_http_error_with_wait_until_deadline(
+ Request::get(&server.url(path))?,
+ wait_until,
+ deadline,
+ )
+ .await?;
+ let elapsed = started.elapsed();
+ let FetchedDocument::Page(mut page) = fetched else {
+ panic!("HTML fixture unexpectedly produced a raw document");
+ };
+ let page_id = page.renderer_page_id();
+
+ assert!(
+ elapsed < Duration::from_millis(1_300),
+ "{wait_until:?} appears to have restarted its timeout after the slow main response: {elapsed:?}"
+ );
+ assert!(
+ page.serialize_html_async().await?.contains(expected_html),
+ "the best-effort Page must remain usable after {wait_until:?} expires"
+ );
+
+ let post_wait_started = std::time::Instant::now();
+ let error = browser
+ .wait_for_selector_with_deadline(&mut page, "#never-appears", deadline)
+ .await
+ .unwrap_err();
+ assert!(
+ error
+ .chain()
+ .any(|cause| cause.to_string().contains("timed out")),
+ "unexpected exhausted-deadline error: {error:?}"
+ );
+ assert!(
+ post_wait_started.elapsed() < Duration::from_millis(200),
+ "a post-readiness selector must not receive a fresh timeout"
+ );
+
+ tokio::time::timeout(Duration::from_secs(1), page.close_async()).await??;
+ assert!(
+ renderer_owner.record(page_id).is_none(),
+ "best-effort deadline cancellation must return the Page entry for close"
+ );
+ }
+
+ server.shutdown().await;
+ Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn page_fetch_wait_until_apis_do_not_restart_best_effort_timeout() -> Result<()> {
+ let server = FixtureServer::spawn().await?;
+ let browser = Browser::new(AppConfig::default())?;
+
+ for (wait_until, path) in [
+ (
+ RenderedDomWaitUntil::NetworkIdle,
+ "/wait-until-slow-interval-fetch",
+ ),
+ (
+ RenderedDomWaitUntil::DomStable,
+ "/wait-until-slow-interval-dom-mutation",
+ ),
+ ] {
+ let started = std::time::Instant::now();
+ let page = browser
+ .fetch_with_wait_until(&server.url(path), wait_until, Duration::from_secs(1))
+ .await?;
+
+ assert!(
+ started.elapsed() < Duration::from_millis(1_300),
+ "the Page-returning {wait_until:?} API appears to have started a fresh timeout"
+ );
+ page.close_async().await?;
+ }
+
+ server.shutdown().await;
+ Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn best_effort_readiness_does_not_soften_a_base_lifecycle_timeout() -> Result<()> {
+ let server = FixtureServer::spawn().await?;
+ let browser = Browser::new(AppConfig::default())?;
+
+ for wait_until in [
+ RenderedDomWaitUntil::NetworkIdle,
+ RenderedDomWaitUntil::DomStable,
+ ] {
+ // No Page exists at 200 ms because the main response itself takes
+ // 500 ms. Best effort applies only after the Load/DCL base boundary.
+ let started = std::time::Instant::now();
+ let error = browser
+ .fetch_request_document_allow_http_error_with_wait_until(
+ Request::get(&server.url("/wait-until-slow-static"))?,
+ wait_until,
+ Duration::from_millis(200),
+ )
+ .await
+ .unwrap_err();
+
+ assert!(
+ error
+ .chain()
+ .any(|cause| cause.to_string().contains("timed out")),
+ "unexpected base lifecycle error: {error:?}"
+ );
+ assert!(
+ started.elapsed() < Duration::from_millis(400),
+ "{wait_until:?} incorrectly softened or extended the base lifecycle timeout"
+ );
+ }
+
+ server.shutdown().await;
+ Ok(())
+}
+
#[tokio::test(flavor = "multi_thread")]
async fn live_page_selector_wait_allows_unrelated_evaluate_while_pending() -> Result<()> {
let server = FixtureServer::spawn().await?;
diff --git a/moli-test-support/src/routes_wait.rs b/moli-test-support/src/routes_wait.rs
index 3f8f96d82..7eaec156e 100644
--- a/moli-test-support/src/routes_wait.rs
+++ b/moli-test-support/src/routes_wait.rs
@@ -69,6 +69,8 @@ const WAIT_UNTIL_OUTER_HTML_TAMPER_HTML: &str = "<!doctype html><html><body data
// is around 500 ms; a 50 ms interval leaves enough margin under nextest load.
const WAIT_UNTIL_INTERVAL_FETCH_HTML: &str = "<!doctype html><html><body data-state=\"init\"><script>window.addEventListener('load', () => { setInterval(() => { fetch('/wait-until-data').then(() => { document.body.setAttribute('data-ping', String((Number(document.body.getAttribute('data-ping') || '0') + 1))); }); }, 50); });</script></body></html>";
const WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML: &str = "<!doctype html><html><body data-state=\"init\"><main id=\"mutation-count\">0</main><script>window.addEventListener('load', () => { setInterval(() => { const count = Number(document.body.getAttribute('data-mutation-count') || '0') + 1; document.body.setAttribute('data-mutation-count', String(count)); document.getElementById('mutation-count').textContent = String(count); }, 50); });</script></body></html>";
+const WAIT_UNTIL_SLOW_STATIC_HTML: &str =
+ "<!doctype html><html><body><main id=\"slow-main\">slow-main=ready</main></body></html>";
const WAIT_UNTIL_HTTP_ERROR_NAVIGATION_CHALLENGE_HTML: &str = "<!doctype html><html><head><title>403 challenge</title></head><body><main id=\"challenge\">http-error-navigation=challenge</main><script>window.addEventListener('load', () => { setTimeout(() => { document.cookie = 'moli-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax'; location.reload(); }, 75); });</script></body></html>";
const WAIT_UNTIL_HTTP_ERROR_LATE_NAVIGATION_CHALLENGE_HTML: &str = "<!doctype html><html><head><title>late 403 challenge</title></head><body><main id=\"challenge\">http-error-navigation=late-challenge</main><script>window.addEventListener('load', () => { setTimeout(() => { document.cookie = 'moli-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax'; location.reload(); }, 1200); });</script></body></html>";
const WAIT_UNTIL_READINESS_HTTP_ERROR_CHALLENGE_HTML: &str = r#"<!doctype html>
@@ -255,6 +257,15 @@ pub(super) fn add_wait_routes(router: Router) -> Router {
"/wait-until-interval-dom-mutation",
get(wait_until_interval_dom_mutation_page),
)
+ .route("/wait-until-slow-static", get(wait_until_slow_static_page))
+ .route(
+ "/wait-until-slow-interval-fetch",
+ get(wait_until_slow_interval_fetch_page),
+ )
+ .route(
+ "/wait-until-slow-interval-dom-mutation",
+ get(wait_until_slow_interval_dom_mutation_page),
+ )
.route(
"/wait-until-http-error-navigation",
get(wait_until_http_error_navigation_page),
@@ -503,6 +514,24 @@ async fn wait_until_interval_dom_mutation_page() -> Html<&'static str> {
Html(WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML)
}
+// These delayed main-resource fixtures make timeout accounting observable:
+// the initial 500 ms must be deducted before NetworkIdle/DomStable starts.
+// A fresh readiness timeout would incorrectly extend the total by 500 ms.
+async fn wait_until_slow_static_page() -> Html<&'static str> {
+ sleep(Duration::from_millis(500)).await;
+ Html(WAIT_UNTIL_SLOW_STATIC_HTML)
+}
+
+async fn wait_until_slow_interval_fetch_page() -> Html<&'static str> {
+ sleep(Duration::from_millis(500)).await;
+ Html(WAIT_UNTIL_INTERVAL_FETCH_HTML)
+}
+
+async fn wait_until_slow_interval_dom_mutation_page() -> Html<&'static str> {
+ sleep(Duration::from_millis(500)).await;
+ Html(WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML)
+}
+
async fn wait_until_data_page() -> Response {
(
[(
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index 517c67672..38b82955a 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -2098,6 +2098,261 @@ fn cli_networkidle_timeout_logs_warning_and_returns_best_effort_output() -> Resu
Ok(())
}
+fn assert_cli_best_effort_stage_uses_remaining_deadline(
+ wait_until: &str,
+ path: &str,
+ expected_html: &str,
+) -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ // The main response consumes 500 ms. A fresh one-second stability timer
+ // would make this take roughly 1.5 s; one shared deadline must return a
+ // usable best-effort Page at roughly the one-second mark.
+ let started = std::time::Instant::now();
+ let output = run_moli([
+ "moli",
+ "fetch",
+ "--log-level",
+ "warn",
+ "--http-no-proxy",
+ "*",
+ "--wait-until",
+ wait_until,
+ "--timeout",
+ "1000",
+ "--dump",
+ "html",
+ &server.url(path),
+ ])?;
+ let elapsed = started.elapsed();
+
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "wait_until={wait_until} stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(stdout.contains(expected_html), "stdout={stdout}");
+ assert!(
+ stderr.contains("fetch readiness wait timed out; returning best-effort page"),
+ "stderr={stderr}"
+ );
+ assert!(stderr.contains("timeout_ms=1000"), "stderr={stderr}");
+ assert!(stderr.contains("remaining_ms="), "stderr={stderr}");
+ assert!(
+ elapsed < std::time::Duration::from_millis(1_300),
+ "wait_until={wait_until} appears to have restarted its timeout: {elapsed:?}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_networkidle_uses_only_the_deadline_left_after_a_slow_main_response() -> Result<()> {
+ assert_cli_best_effort_stage_uses_remaining_deadline(
+ "networkidle",
+ "/wait-until-slow-interval-fetch",
+ "data-state=\"init\"",
+ )
+}
+
+#[test]
+fn cli_domstable_uses_only_the_deadline_left_after_a_slow_main_response() -> Result<()> {
+ assert_cli_best_effort_stage_uses_remaining_deadline(
+ "domstable",
+ "/wait-until-slow-interval-dom-mutation",
+ "id=\"mutation-count\"",
+ )
+}
+
+#[test]
+fn cli_best_effort_stage_does_not_soften_a_slow_base_lifecycle_timeout() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-slow-static");
+
+ for wait_until in ["networkidle", "domstable"] {
+ // The response takes 500 ms, so at 200 ms no live Page exists yet.
+ // This is a hard base-stage failure, not a best-effort stability exit.
+ let started = std::time::Instant::now();
+ let output = run_moli([
+ "moli",
+ "fetch",
+ "--log-level",
+ "warn",
+ "--http-no-proxy",
+ "*",
+ "--wait-until",
+ wait_until,
+ "--timeout",
+ "200",
+ "--dump",
+ "html",
+ &url,
+ ])?;
+ let elapsed = started.elapsed();
+
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");
+ assert!(stdout.is_empty(), "stdout={stdout}");
+ assert!(stderr.contains("timed out after 200 ms"), "stderr={stderr}");
+ assert!(
+ !stderr.contains("returning best-effort page"),
+ "a base lifecycle timeout must not be softened: stderr={stderr}"
+ );
+ assert!(
+ elapsed < std::time::Duration::from_millis(400),
+ "wait_until={wait_until} extended the hard base-stage timeout: {elapsed:?}"
+ );
+ }
+
+ runtime.block_on(server.shutdown());
+ Ok(())
+}
+
+#[test]
+fn cli_zero_deadline_cannot_enter_a_best_effort_page_stage() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-slow-static");
+
+ for wait_until in ["networkidle", "domstable"] {
+ let started = std::time::Instant::now();
+ let output = run_moli([
+ "moli",
+ "fetch",
+ "--log-level",
+ "warn",
+ "--http-no-proxy",
+ "*",
+ "--wait-until",
+ wait_until,
+ "--timeout",
+ "0",
+ "--dump",
+ "html",
+ &url,
+ ])?;
+
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");
+ assert!(stdout.is_empty(), "stdout={stdout}");
+ assert!(stderr.contains("timed out after 0 ms"), "stderr={stderr}");
+ assert!(
+ !stderr.contains("returning best-effort page"),
+ "zero budget cannot produce a Page to return: stderr={stderr}"
+ );
+ assert!(
+ started.elapsed() < std::time::Duration::from_millis(200),
+ "wait_until={wait_until} did not fail its zero deadline immediately"
+ );
+ }
+
+ runtime.block_on(server.shutdown());
+ Ok(())
+}
+
+fn assert_cli_quiet_page_uses_remaining_best_effort_budget(wait_until: &str) -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-slow-static");
+ // After the 500 ms response, 250 ms remains. That is shorter than either
+ // stability window, although a freshly started timer would let this
+ // otherwise quiet page settle successfully.
+ let output = run_moli([
+ "moli",
+ "fetch",
+ "--log-level",
+ "warn",
+ "--http-no-proxy",
+ "*",
+ "--wait-until",
+ wait_until,
+ "--timeout",
+ "750",
+ "--dump",
+ "html",
+ &url,
+ ])?;
+
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "wait_until={wait_until} stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(stdout.contains("slow-main=ready"), "stdout={stdout}");
+ assert!(
+ stderr.contains("fetch readiness wait timed out; returning best-effort page"),
+ "wait_until={wait_until} should use only the remaining budget: stderr={stderr}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_quiet_page_networkidle_uses_remaining_best_effort_budget() -> Result<()> {
+ assert_cli_quiet_page_uses_remaining_best_effort_budget("networkidle")
+}
+
+#[test]
+fn cli_quiet_page_domstable_uses_remaining_best_effort_budget() -> Result<()> {
+ assert_cli_quiet_page_uses_remaining_best_effort_budget("domstable")
+}
+
+#[test]
+fn cli_post_readiness_wait_cannot_restart_an_exhausted_best_effort_deadline() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-slow-interval-fetch");
+ let started = std::time::Instant::now();
+ let output = run_moli([
+ "moli",
+ "fetch",
+ "--log-level",
+ "warn",
+ "--http-no-proxy",
+ "*",
+ "--wait-until",
+ "networkidle",
+ "--wait-selector",
+ "#never-appears",
+ "--timeout",
+ "1000",
+ "--dump",
+ "html",
+ &url,
+ ])?;
+ let elapsed = started.elapsed();
+ runtime.block_on(server.shutdown());
+
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");
+ assert!(stdout.is_empty(), "stdout={stdout}");
+ assert!(
+ stderr.contains("returning best-effort page"),
+ "stderr={stderr}"
+ );
+ assert!(
+ stderr.contains("failed while waiting for selector `#never-appears`"),
+ "stderr={stderr}"
+ );
+ assert!(
+ elapsed < std::time::Duration::from_millis(1_300),
+ "the selector appears to have received a fresh timeout: {elapsed:?}"
+ );
+ Ok(())
+}
+
#[test]
fn cli_domstable_waits_for_inflight_slow_fetch_content() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;