Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): share deadline with best-effort readiness
ldm0aefa01568d52
diff --git a/moli-core/src/runtime/fetch_deadline.rs b/moli-core/src/runtime/fetch_deadline.rsindex c48b50f82..96b23956d 100644--- a/moli-core/src/runtime/fetch_deadline.rs+++ b/moli-core/src/runtime/fetch_deadline.rs@@ -44,15 +44,6 @@ impl FetchDeadline {self.at}- pub(super) fn at_with_cushion(self, cushion: Duration) -> Result<tokio::time::Instant> {- self.at.checked_add(cushion).with_context(|| {- anyhow!(- "fetch readiness deadline with {} ms cleanup cushion exceeds the supported range",- cushion.as_millis()- )- })- }-pub(super) async fn wait<T, F>(self, phase: &'static str, future: F) -> Result<T>whereF: Future<Output = Result<T>>,diff --git a/moli-core/src/runtime/lifecycle_fetch.rs b/moli-core/src/runtime/lifecycle_fetch.rsindex bb9a4cf1d..4ec3db70b 100644--- a/moli-core/src/runtime/lifecycle_fetch.rs+++ b/moli-core/src/runtime/lifecycle_fetch.rs@@ -59,7 +59,7 @@ impl Browser {"a lifecycle decider requires DCL, load, or done");let decider = RendererLifecycleDecider::new(decider);- self.fetch_document_with_wait(+ self.fetch_document_to_base_stage(request,wait_until,deadline,diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rsindex 83356d2a3..05f73d8d3 100644--- a/moli-core/src/runtime/mod.rs+++ b/moli-core/src/runtime/mod.rs@@ -12,7 +12,7 @@ use crate::{selector::QueryEngine,};use anyhow::Result;-use anyhow::{Context, anyhow};+use anyhow::{Context, anyhow, bail};use moli_cookie_jar::StoredCookie;use moli_fetch::{FetchCancelHandle, RawResponse, Request, StreamingRawResponse, ensure_http_status_success,@@ -52,17 +52,6 @@ pub use navigation_engine::{static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);-fn wait_until_outer_timeout(wait_until: RenderedDomWaitUntil, timeout: Duration) -> Duration {- match wait_until {- RenderedDomWaitUntil::NetworkIdle | RenderedDomWaitUntil::DomStable => {- timeout.saturating_add(Duration::from_secs(1))- }- RenderedDomWaitUntil::DomContentLoaded- | RenderedDomWaitUntil::Load- | RenderedDomWaitUntil::Done => timeout,- }-}-fn is_fetch_readiness_timeout(error: &anyhow::Error, wait_until: RenderedDomWaitUntil) -> bool {let expected = match wait_until {RenderedDomWaitUntil::NetworkIdle => "timed out waiting for networkidle",@@ -151,6 +140,22 @@ pub enum RenderedDomWaitUntil {Done,}+impl RenderedDomWaitUntil {+ /// The concrete lifecycle boundary at which a live `Page` can first be+ /// handed back to the host. Network-idle and DOM-stable are observations+ /// made on that live Page, after Load and DCL respectively.+ fn base_stage(self) -> PageVmInitStage {+ match self {+ Self::DomContentLoaded | Self::DomStable => PageVmInitStage::DomContentLoaded,+ Self::Load | Self::NetworkIdle | Self::Done => PageVmInitStage::Load,+ }+ }++ fn has_best_effort_page_wait(self) -> bool {+ matches!(self, Self::NetworkIdle | Self::DomStable)+ }+}+#[derive(Debug)]pub struct RawDocument {response: RawResponse,@@ -273,8 +278,7 @@ impl Browser {}pub async fn fetch_request(&self, request: Request) -> Result<Page> {- self.fetch_internal(request, PageVmInitStage::Load, None)- .await+ self.fetch_internal(request, PageVmInitStage::Load).await}pub async fn fetch_request_with_wait_until(@@ -283,44 +287,13 @@ impl Browser {wait_until: RenderedDomWaitUntil,timeout: Duration,) -> Result<Page> {- let stage = match wait_until {- RenderedDomWaitUntil::DomContentLoaded => PageVmInitStage::DomContentLoaded,- RenderedDomWaitUntil::Load => PageVmInitStage::Load,- RenderedDomWaitUntil::NetworkIdle => PageVmInitStage::Load,- RenderedDomWaitUntil::DomStable => PageVmInitStage::DomContentLoaded,- RenderedDomWaitUntil::Done => PageVmInitStage::Load,- };-- let raw_url = request.url.as_str().to_owned();- debug!(- url = %raw_url,- wait_until = ?wait_until,- timeout_ms = timeout.as_millis(),- stage = ?stage,- "starting fetch_with_wait_until deadline"- );- let outer_timeout = wait_until_outer_timeout(wait_until, timeout);- match tokio::time::timeout(- outer_timeout,- self.fetch_internal(request, stage, Some((wait_until, timeout))),+ self.fetch_page_with_wait_until_deadline(+ request,+ wait_until,+ FetchDeadline::new(timeout)?,+ false,).await- {- Ok(result) => result,- Err(_) => {- warn!(- url = %raw_url,- wait_until = ?wait_until,- timeout_ms = timeout.as_millis(),- stage = ?stage,- "fetch_with_wait_until timed out"- );- Err(anyhow!(- "fetch wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",- timeout.as_millis()- ))- }- }}pub async fn fetch_allow_http_error(&self, raw_url: &str) -> Result<Page> {@@ -343,7 +316,7 @@ impl Browser {}pub async fn fetch_request_allow_http_error(&self, request: Request) -> Result<Page> {- self.fetch_allow_http_error_internal(request, PageVmInitStage::Load, None)+ self.fetch_allow_http_error_internal(request, PageVmInitStage::Load).await}@@ -355,7 +328,6 @@ impl Browser {request,PageVmInitStage::Load,RendererReplyBoundary::Stage,- None,).await}@@ -366,44 +338,72 @@ impl Browser {wait_until: RenderedDomWaitUntil,timeout: Duration,) -> Result<Page> {- let stage = match wait_until {- RenderedDomWaitUntil::DomContentLoaded => PageVmInitStage::DomContentLoaded,- RenderedDomWaitUntil::Load => PageVmInitStage::Load,- RenderedDomWaitUntil::NetworkIdle => PageVmInitStage::Load,- RenderedDomWaitUntil::DomStable => PageVmInitStage::DomContentLoaded,- RenderedDomWaitUntil::Done => PageVmInitStage::Load,- };+ self.fetch_page_with_wait_until_deadline(+ request,+ wait_until,+ FetchDeadline::new(timeout)?,+ true,+ )+ .await+ }+ async fn fetch_page_with_wait_until_deadline(+ &self,+ request: Request,+ wait_until: RenderedDomWaitUntil,+ deadline: FetchDeadline,+ allow_http_error: bool,+ ) -> Result<Page> {let raw_url = request.url.as_str().to_owned();+ let stage = wait_until.base_stage();debug!(url = %raw_url,wait_until = ?wait_until,- timeout_ms = timeout.as_millis(),+ timeout_ms = deadline.timeout().as_millis(),stage = ?stage,- "starting fetch_allow_http_error_with_wait_until deadline"+ "starting fetch_with_wait_until deadline");- let outer_timeout = wait_until_outer_timeout(wait_until, timeout);- match tokio::time::timeout(- outer_timeout,- self.fetch_allow_http_error_internal(request, stage, Some((wait_until, timeout))),++ // Materialization only reaches the concrete DCL/Load base stage. The+ // selected stability observation is a separate Page operation so it+ // can consume the remaining budget and soften only its own timeout.+ let mut page = match tokio::time::timeout_at(+ deadline.at(),+ self.fetch_allow_http_error_internal(request, stage),).await{- Ok(result) => result,+ // Preserve transport and policy errors from the fetch itself. The+ // outer deadline should add an error only when it actually wins.+ Ok(result) => result?,Err(_) => {warn!(url = %raw_url,wait_until = ?wait_until,- timeout_ms = timeout.as_millis(),+ timeout_ms = deadline.timeout().as_millis(),stage = ?stage,- "fetch_allow_http_error_with_wait_until timed out"+ allow_http_error,+ "fetch_with_wait_until timed out"+ );+ if allow_http_error {+ bail!(+ "fetch allow-http-error wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",+ deadline.timeout().as_millis()+ );+ }+ bail!(+ "fetch wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",+ deadline.timeout().as_millis());- Err(anyhow!(- "fetch allow-http-error wait_until {wait_until:?} timed out after {} ms for `{raw_url}`",- timeout.as_millis()- ))}+ };++ self.wait_for_page_readiness_with_deadline(&mut page, wait_until, deadline)+ .await?;+ if !allow_http_error {+ ensure_http_status_success(page.final_url().as_str(), page.status(), false)?;}+ Ok(page)}pub async fn fetch_request_document_allow_http_error_with_wait_until(@@ -427,17 +427,30 @@ impl Browser {wait_until: RenderedDomWaitUntil,deadline: FetchDeadline,) -> Result<FetchedDocument> {- self.fetch_document_with_wait(- request,- wait_until,- deadline,- RendererReplyBoundary::Stage,- None,- )- .await+ let fetched = self+ .fetch_document_to_base_stage(+ request,+ wait_until,+ deadline,+ RendererReplyBoundary::Stage,+ None,+ )+ .await?;++ match fetched {+ FetchedDocument::Page(mut page) => {+ self.wait_for_page_readiness_with_deadline(&mut page, wait_until, deadline)+ .await?;+ Ok(FetchedDocument::Page(page))+ }+ FetchedDocument::Raw(raw) => Ok(FetchedDocument::Raw(raw)),+ }}- async fn fetch_document_with_wait(+ /// Reaches the concrete lifecycle boundary that makes a live Page+ /// available. For NetworkIdle and DomStable this is only the Load or DCL+ /// base stage; their best-effort observation runs outside materialization.+ async fn fetch_document_to_base_stage(&self,request: Request,wait_until: RenderedDomWaitUntil,@@ -446,13 +459,7 @@ impl Browser {lifecycle_decider: Option<RendererLifecycleDecider>,) -> Result<FetchedDocument> {let timeout = deadline.timeout();- let stage = match wait_until {- RenderedDomWaitUntil::DomContentLoaded => PageVmInitStage::DomContentLoaded,- RenderedDomWaitUntil::Load => PageVmInitStage::Load,- RenderedDomWaitUntil::NetworkIdle => PageVmInitStage::Load,- RenderedDomWaitUntil::DomStable => PageVmInitStage::DomContentLoaded,- RenderedDomWaitUntil::Done => PageVmInitStage::Load,- };+ let stage = wait_until.base_stage();let raw_url = request.url.as_str().to_owned();debug!(@@ -462,18 +469,11 @@ impl Browser {stage = ?stage,"starting fetch_document_allow_http_error_with_wait_until deadline");- // Network-idle and DOM-stable return a best-effort Page after their- // renderer-side readiness timer expires. Keep the historical one-second- // transport cushion so that Page can cross the owner boundary, while- // the logical FetchDeadline remains expired for every later plan step.- let outer_deadline = match wait_until {- RenderedDomWaitUntil::NetworkIdle | RenderedDomWaitUntil::DomStable => {- deadline.at_with_cushion(Duration::from_secs(1))?- }- RenderedDomWaitUntil::DomContentLoaded- | RenderedDomWaitUntil::Load- | RenderedDomWaitUntil::Done => deadline.at(),- };+ // This deadline is strict: if the response or base DCL/Load stage does+ // not arrive in time, there is no Page on which best-effort readiness+ // can operate. Only the later NetworkIdle/DomStable observation may+ // soften an expired deadline.+ let outer_deadline = deadline.at();let requested_url = request.url.clone();if is_about_blank_url(&requested_url) {return self@@ -489,7 +489,6 @@ impl Browser {stage,reply_boundary,lifecycle_decider,- Some((wait_until, timeout)),String::new(),),)@@ -542,7 +541,6 @@ impl Browser {stage,reply_boundary,lifecycle_decider,- Some((wait_until, timeout)),streaming_raw_response_from_navigation_response(response)?,document_fetch_context_seed,reserved_client,@@ -585,7 +583,6 @@ impl Browser {stage,reply_boundary,lifecycle_decider,- Some((wait_until, timeout)),response,document_fetch_context_seed,reserved_client,@@ -722,53 +719,68 @@ impl Browser {.await}- async fn wait_for_fetch_readiness(+ /// Observes NetworkIdle or DomStable with the unspent part of `deadline`.+ ///+ /// A timeout in this observation is deliberately best-effort: the Page+ /// has already reached its required Load/DCL base stage, so the current+ /// snapshot is returned with a warning. Errors unrelated to timeout still+ /// fail the fetch. Concrete lifecycle modes are no-ops here.+ pub async fn wait_for_page_readiness_with_deadline(&self,page: &mut Page,- raw_url: &str,- wait: Option<(RenderedDomWaitUntil, Duration)>,+ wait_until: RenderedDomWaitUntil,+ deadline: FetchDeadline,) -> Result<()> {- let Some((wait_until, timeout)) = wait else {+ if !wait_until.has_best_effort_page_wait() {return Ok(());- };+ }++ let loader = self.resource_request_client();+ let remaining = deadline.remaining();let result = match wait_until {RenderedDomWaitUntil::NetworkIdle => {- page.wait_for_network_idle(&self.resource_request_client(), timeout)- .await+ tokio::time::timeout_at(+ deadline.at(),+ page.wait_for_network_idle(&loader, remaining),+ )+ .await}RenderedDomWaitUntil::DomStable => {- page.wait_for_dom_stable(&self.resource_request_client(), timeout)+ tokio::time::timeout_at(deadline.at(), page.wait_for_dom_stable(&loader, remaining)).await}RenderedDomWaitUntil::DomContentLoaded| RenderedDomWaitUntil::Load- | RenderedDomWaitUntil::Done => return Ok(()),+ | RenderedDomWaitUntil::Done => unreachable!(+ "concrete lifecycle modes returned before starting a best-effort Page wait"+ ),};- match result {- Ok(()) => Ok(()),- Err(error) if is_fetch_readiness_timeout(&error, wait_until) => {- warn!(- page_id = page.page_id(),- url = %raw_url,- final_url = %page.final_url(),- wait_until = ?wait_until,- timeout_ms = timeout.as_millis(),- error = %error,- "fetch readiness wait timed out; returning best-effort page"- );- Ok(())+ let timeout_error = match result {+ Ok(Ok(())) => return Ok(()),+ Ok(Err(error)) if is_fetch_readiness_timeout(&error, wait_until) => error,+ Ok(Err(error)) => {+ return Err(error).with_context(|| {+ anyhow!("failed while waiting for page readiness {wait_until:?}")+ });}- Err(error) => Err(error),- }+ Err(error) => anyhow::Error::new(error),+ };++ warn!(+ page_id = page.page_id(),+ url = %page.requested_url(),+ final_url = %page.final_url(),+ wait_until = ?wait_until,+ timeout_ms = deadline.timeout().as_millis(),+ remaining_ms = remaining.as_millis(),+ error = %timeout_error,+ "fetch readiness wait timed out; returning best-effort page"+ );+ Ok(())}- async fn fetch_internal(- &self,- request: Request,- stage: PageVmInitStage,- wait: Option<(RenderedDomWaitUntil, Duration)>,- ) -> Result<Page> {+ async fn fetch_internal(&self, request: Request, stage: PageVmInitStage) -> Result<Page> {let raw_url = request.url.as_str().to_owned();let requested_url = request.url.clone();if is_about_blank_url(&requested_url) {@@ -779,7 +791,6 @@ impl Browser {stage,RendererReplyBoundary::Stage,None,- wait,String::new(),).await;@@ -808,7 +819,6 @@ impl Browser {stage,RendererReplyBoundary::Stage,None,- wait,response,document_fetch_context_seed,reserved_client,@@ -826,7 +836,6 @@ impl Browser {stage,RendererReplyBoundary::Stage,None,- wait,response,document_fetch_context_seed,reserved_client,@@ -840,7 +849,6 @@ impl Browser {&self,request: Request,stage: PageVmInitStage,- wait: Option<(RenderedDomWaitUntil, Duration)>,) -> Result<Page> {let raw_url = request.url.as_str().to_owned();let requested_url = request.url.clone();@@ -852,7 +860,6 @@ impl Browser {stage,RendererReplyBoundary::Stage,None,- wait,String::new(),).await;@@ -880,7 +887,6 @@ impl Browser {stage,RendererReplyBoundary::Stage,None,- wait,streaming_raw_response_from_navigation_response(response)?,document_fetch_context_seed,reserved_client,@@ -895,7 +901,6 @@ impl Browser {stage,RendererReplyBoundary::Stage,None,- wait,response,document_fetch_context_seed,reserved_client,@@ -908,7 +913,6 @@ impl Browser {request: Request,stage: PageVmInitStage,reply_boundary: RendererReplyBoundary,- wait: Option<(RenderedDomWaitUntil, Duration)>,) -> Result<FetchedDocument> {let raw_url = request.url.as_str().to_owned();let requested_url = request.url.clone();@@ -920,7 +924,6 @@ impl Browser {stage,reply_boundary,None,- wait,String::new(),).await@@ -956,7 +959,6 @@ impl Browser {stage,reply_boundary,None,- wait,streaming_raw_response_from_navigation_response(response)?,document_fetch_context_seed,reserved_client,@@ -978,7 +980,6 @@ impl Browser {stage,reply_boundary,None,- wait,response,document_fetch_context_seed,reserved_client,@@ -1045,14 +1046,12 @@ impl Browser {stage: PageVmInitStage,reply_boundary: RendererReplyBoundary,lifecycle_decider: Option<RendererLifecycleDecider>,- wait: Option<(RenderedDomWaitUntil, Duration)>,response_body: String,) -> Result<Page> {let started = Instant::now();debug!(url = %raw_url,stage = ?stage,- wait = ?wait.map(|(wait_until, timeout)| (wait_until, timeout.as_millis())),"starting static html fetch_internal");@@ -1103,13 +1102,10 @@ impl Browser {.dispatch_command(RendererOwnerCommand::CreateHtmlPage(create_page_request)).await.with_context(|| anyhow!("failed to execute scripts for page `{raw_url}`"))?;- let mut page = materialize_page_created_reply(&renderer_owner, reply)?;- self.wait_for_fetch_readiness(&mut page, raw_url, wait)- .await?;+ let page = materialize_page_created_reply(&renderer_owner, reply)?;info!(page_id = page.page_id(),url = %page.requested_url(),- wait_until = ?wait.map(|(until, timeout)| (until, timeout.as_millis())),elapsed_ms = started.elapsed().as_millis(),"static html fetch_internal completed");@@ -1123,7 +1119,6 @@ impl Browser {stage: PageVmInitStage,reply_boundary: RendererReplyBoundary,lifecycle_decider: Option<RendererLifecycleDecider>,- wait: Option<(RenderedDomWaitUntil, Duration)>,response: StreamingRawResponse,document_fetch_context_seed: DocumentFetchContextSeed,reserved_service_worker_client: Option<RendererReservedServiceWorkerClient>,@@ -1132,7 +1127,6 @@ impl Browser {debug!(url = %raw_url,stage = ?stage,- wait = ?wait.map(|(wait_until, timeout)| (wait_until, timeout.as_millis())),"starting raw streaming fetch_internal");@@ -1223,17 +1217,14 @@ impl Browser {"raw streaming page creation produced a pending download for `{raw_url}`"));}- let mut page = Page::from_attached_handle_with_creation_artifacts(+ let page = Page::from_attached_handle_with_creation_artifacts(handle,page_state,page_creation_artifacts,);- self.wait_for_fetch_readiness(&mut page, raw_url, wait)- .await?;info!(page_id = page.page_id(),url = %page.requested_url(),- wait_until = ?wait.map(|(until, timeout)| (until, timeout.as_millis())),elapsed_ms = started.elapsed().as_millis(),"raw streaming fetch_internal completed");diff --git a/moli-core/src/runtime/tests.rs b/moli-core/src/runtime/tests.rsindex cc3c7e76d..4ff4f66c0 100644--- a/moli-core/src/runtime/tests.rs+++ b/moli-core/src/runtime/tests.rs@@ -2126,6 +2126,145 @@ async fn fetch_deadline_spans_lifecycle_selector_and_script_without_reset() -> ROk(())}+#[tokio::test(flavor = "multi_thread")]+async fn best_effort_page_readiness_consumes_only_the_remaining_fetch_deadline() -> Result<()> {+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;+ let renderer_owner = browser.js_runtime.renderer_owner_handle();++ for (wait_until, path, expected_html) in [+ (+ RenderedDomWaitUntil::NetworkIdle,+ "/wait-until-slow-interval-fetch",+ "data-state=\"init\"",+ ),+ (+ RenderedDomWaitUntil::DomStable,+ "/wait-until-slow-interval-dom-mutation",+ "id=\"mutation-count\"",+ ),+ ] {+ // The main response costs 500 ms. Readiness must receive only the+ // roughly 500 ms left in this one-second plan, not a fresh second.+ let deadline = FetchDeadline::new(Duration::from_secs(1))?;+ let started = std::time::Instant::now();+ let fetched = browser+ .fetch_request_document_allow_http_error_with_wait_until_deadline(+ Request::get(&server.url(path))?,+ wait_until,+ deadline,+ )+ .await?;+ let elapsed = started.elapsed();+ let FetchedDocument::Page(mut page) = fetched else {+ panic!("HTML fixture unexpectedly produced a raw document");+ };+ let page_id = page.renderer_page_id();++ assert!(+ elapsed < Duration::from_millis(1_300),+ "{wait_until:?} appears to have restarted its timeout after the slow main response: {elapsed:?}"+ );+ assert!(+ page.serialize_html_async().await?.contains(expected_html),+ "the best-effort Page must remain usable after {wait_until:?} expires"+ );++ let post_wait_started = std::time::Instant::now();+ let error = browser+ .wait_for_selector_with_deadline(&mut page, "#never-appears", deadline)+ .await+ .unwrap_err();+ assert!(+ error+ .chain()+ .any(|cause| cause.to_string().contains("timed out")),+ "unexpected exhausted-deadline error: {error:?}"+ );+ assert!(+ post_wait_started.elapsed() < Duration::from_millis(200),+ "a post-readiness selector must not receive a fresh timeout"+ );++ tokio::time::timeout(Duration::from_secs(1), page.close_async()).await??;+ assert!(+ renderer_owner.record(page_id).is_none(),+ "best-effort deadline cancellation must return the Page entry for close"+ );+ }++ server.shutdown().await;+ Ok(())+}++#[tokio::test(flavor = "multi_thread")]+async fn page_fetch_wait_until_apis_do_not_restart_best_effort_timeout() -> Result<()> {+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;++ for (wait_until, path) in [+ (+ RenderedDomWaitUntil::NetworkIdle,+ "/wait-until-slow-interval-fetch",+ ),+ (+ RenderedDomWaitUntil::DomStable,+ "/wait-until-slow-interval-dom-mutation",+ ),+ ] {+ let started = std::time::Instant::now();+ let page = browser+ .fetch_with_wait_until(&server.url(path), wait_until, Duration::from_secs(1))+ .await?;++ assert!(+ started.elapsed() < Duration::from_millis(1_300),+ "the Page-returning {wait_until:?} API appears to have started a fresh timeout"+ );+ page.close_async().await?;+ }++ server.shutdown().await;+ Ok(())+}++#[tokio::test(flavor = "multi_thread")]+async fn best_effort_readiness_does_not_soften_a_base_lifecycle_timeout() -> Result<()> {+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;++ for wait_until in [+ RenderedDomWaitUntil::NetworkIdle,+ RenderedDomWaitUntil::DomStable,+ ] {+ // No Page exists at 200 ms because the main response itself takes+ // 500 ms. Best effort applies only after the Load/DCL base boundary.+ let started = std::time::Instant::now();+ let error = browser+ .fetch_request_document_allow_http_error_with_wait_until(+ Request::get(&server.url("/wait-until-slow-static"))?,+ wait_until,+ Duration::from_millis(200),+ )+ .await+ .unwrap_err();++ assert!(+ error+ .chain()+ .any(|cause| cause.to_string().contains("timed out")),+ "unexpected base lifecycle error: {error:?}"+ );+ assert!(+ started.elapsed() < Duration::from_millis(400),+ "{wait_until:?} incorrectly softened or extended the base lifecycle timeout"+ );+ }++ server.shutdown().await;+ Ok(())+}+#[tokio::test(flavor = "multi_thread")]async fn live_page_selector_wait_allows_unrelated_evaluate_while_pending() -> Result<()> {let server = FixtureServer::spawn().await?;diff --git a/moli-test-support/src/routes_wait.rs b/moli-test-support/src/routes_wait.rsindex 3f8f96d82..7eaec156e 100644--- a/moli-test-support/src/routes_wait.rs+++ b/moli-test-support/src/routes_wait.rs@@ -69,6 +69,8 @@ const WAIT_UNTIL_OUTER_HTML_TAMPER_HTML: &str = "<!doctype html><html><body data// is around 500 ms; a 50 ms interval leaves enough margin under nextest load.const WAIT_UNTIL_INTERVAL_FETCH_HTML: &str = "<!doctype html><html><body data-state=\"init\"><script>window.addEventListener('load', () => { setInterval(() => { fetch('/wait-until-data').then(() => { document.body.setAttribute('data-ping', String((Number(document.body.getAttribute('data-ping') || '0') + 1))); }); }, 50); });</script></body></html>";const WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML: &str = "<!doctype html><html><body data-state=\"init\"><main id=\"mutation-count\">0</main><script>window.addEventListener('load', () => { setInterval(() => { const count = Number(document.body.getAttribute('data-mutation-count') || '0') + 1; document.body.setAttribute('data-mutation-count', String(count)); document.getElementById('mutation-count').textContent = String(count); }, 50); });</script></body></html>";+const WAIT_UNTIL_SLOW_STATIC_HTML: &str =+ "<!doctype html><html><body><main id=\"slow-main\">slow-main=ready</main></body></html>";const WAIT_UNTIL_HTTP_ERROR_NAVIGATION_CHALLENGE_HTML: &str = "<!doctype html><html><head><title>403 challenge</title></head><body><main id=\"challenge\">http-error-navigation=challenge</main><script>window.addEventListener('load', () => { setTimeout(() => { document.cookie = 'moli-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax'; location.reload(); }, 75); });</script></body></html>";const WAIT_UNTIL_HTTP_ERROR_LATE_NAVIGATION_CHALLENGE_HTML: &str = "<!doctype html><html><head><title>late 403 challenge</title></head><body><main id=\"challenge\">http-error-navigation=late-challenge</main><script>window.addEventListener('load', () => { setTimeout(() => { document.cookie = 'moli-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax'; location.reload(); }, 1200); });</script></body></html>";const WAIT_UNTIL_READINESS_HTTP_ERROR_CHALLENGE_HTML: &str = r#"<!doctype html>@@ -255,6 +257,15 @@ pub(super) fn add_wait_routes(router: Router) -> Router {"/wait-until-interval-dom-mutation",get(wait_until_interval_dom_mutation_page),)+ .route("/wait-until-slow-static", get(wait_until_slow_static_page))+ .route(+ "/wait-until-slow-interval-fetch",+ get(wait_until_slow_interval_fetch_page),+ )+ .route(+ "/wait-until-slow-interval-dom-mutation",+ get(wait_until_slow_interval_dom_mutation_page),+ ).route("/wait-until-http-error-navigation",get(wait_until_http_error_navigation_page),@@ -503,6 +514,24 @@ async fn wait_until_interval_dom_mutation_page() -> Html<&'static str> {Html(WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML)}+// These delayed main-resource fixtures make timeout accounting observable:+// the initial 500 ms must be deducted before NetworkIdle/DomStable starts.+// A fresh readiness timeout would incorrectly extend the total by 500 ms.+async fn wait_until_slow_static_page() -> Html<&'static str> {+ sleep(Duration::from_millis(500)).await;+ Html(WAIT_UNTIL_SLOW_STATIC_HTML)+}++async fn wait_until_slow_interval_fetch_page() -> Html<&'static str> {+ sleep(Duration::from_millis(500)).await;+ Html(WAIT_UNTIL_INTERVAL_FETCH_HTML)+}++async fn wait_until_slow_interval_dom_mutation_page() -> Html<&'static str> {+ sleep(Duration::from_millis(500)).await;+ Html(WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML)+}+async fn wait_until_data_page() -> Response {([(diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex 517c67672..38b82955a 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -2098,6 +2098,261 @@ fn cli_networkidle_timeout_logs_warning_and_returns_best_effort_output() -> ResuOk(())}+fn assert_cli_best_effort_stage_uses_remaining_deadline(+ wait_until: &str,+ path: &str,+ expected_html: &str,+) -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ // The main response consumes 500 ms. A fresh one-second stability timer+ // would make this take roughly 1.5 s; one shared deadline must return a+ // usable best-effort Page at roughly the one-second mark.+ let started = std::time::Instant::now();+ let output = run_moli([+ "moli",+ "fetch",+ "--log-level",+ "warn",+ "--http-no-proxy",+ "*",+ "--wait-until",+ wait_until,+ "--timeout",+ "1000",+ "--dump",+ "html",+ &server.url(path),+ ])?;+ let elapsed = started.elapsed();++ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "wait_until={wait_until} stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(stdout.contains(expected_html), "stdout={stdout}");+ assert!(+ stderr.contains("fetch readiness wait timed out; returning best-effort page"),+ "stderr={stderr}"+ );+ assert!(stderr.contains("timeout_ms=1000"), "stderr={stderr}");+ assert!(stderr.contains("remaining_ms="), "stderr={stderr}");+ assert!(+ elapsed < std::time::Duration::from_millis(1_300),+ "wait_until={wait_until} appears to have restarted its timeout: {elapsed:?}"+ );+ Ok(())+}++#[test]+fn cli_networkidle_uses_only_the_deadline_left_after_a_slow_main_response() -> Result<()> {+ assert_cli_best_effort_stage_uses_remaining_deadline(+ "networkidle",+ "/wait-until-slow-interval-fetch",+ "data-state=\"init\"",+ )+}++#[test]+fn cli_domstable_uses_only_the_deadline_left_after_a_slow_main_response() -> Result<()> {+ assert_cli_best_effort_stage_uses_remaining_deadline(+ "domstable",+ "/wait-until-slow-interval-dom-mutation",+ "id=\"mutation-count\"",+ )+}++#[test]+fn cli_best_effort_stage_does_not_soften_a_slow_base_lifecycle_timeout() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-slow-static");++ for wait_until in ["networkidle", "domstable"] {+ // The response takes 500 ms, so at 200 ms no live Page exists yet.+ // This is a hard base-stage failure, not a best-effort stability exit.+ let started = std::time::Instant::now();+ let output = run_moli([+ "moli",+ "fetch",+ "--log-level",+ "warn",+ "--http-no-proxy",+ "*",+ "--wait-until",+ wait_until,+ "--timeout",+ "200",+ "--dump",+ "html",+ &url,+ ])?;+ let elapsed = started.elapsed();++ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");+ assert!(stdout.is_empty(), "stdout={stdout}");+ assert!(stderr.contains("timed out after 200 ms"), "stderr={stderr}");+ assert!(+ !stderr.contains("returning best-effort page"),+ "a base lifecycle timeout must not be softened: stderr={stderr}"+ );+ assert!(+ elapsed < std::time::Duration::from_millis(400),+ "wait_until={wait_until} extended the hard base-stage timeout: {elapsed:?}"+ );+ }++ runtime.block_on(server.shutdown());+ Ok(())+}++#[test]+fn cli_zero_deadline_cannot_enter_a_best_effort_page_stage() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-slow-static");++ for wait_until in ["networkidle", "domstable"] {+ let started = std::time::Instant::now();+ let output = run_moli([+ "moli",+ "fetch",+ "--log-level",+ "warn",+ "--http-no-proxy",+ "*",+ "--wait-until",+ wait_until,+ "--timeout",+ "0",+ "--dump",+ "html",+ &url,+ ])?;++ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");+ assert!(stdout.is_empty(), "stdout={stdout}");+ assert!(stderr.contains("timed out after 0 ms"), "stderr={stderr}");+ assert!(+ !stderr.contains("returning best-effort page"),+ "zero budget cannot produce a Page to return: stderr={stderr}"+ );+ assert!(+ started.elapsed() < std::time::Duration::from_millis(200),+ "wait_until={wait_until} did not fail its zero deadline immediately"+ );+ }++ runtime.block_on(server.shutdown());+ Ok(())+}++fn assert_cli_quiet_page_uses_remaining_best_effort_budget(wait_until: &str) -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-slow-static");+ // After the 500 ms response, 250 ms remains. That is shorter than either+ // stability window, although a freshly started timer would let this+ // otherwise quiet page settle successfully.+ let output = run_moli([+ "moli",+ "fetch",+ "--log-level",+ "warn",+ "--http-no-proxy",+ "*",+ "--wait-until",+ wait_until,+ "--timeout",+ "750",+ "--dump",+ "html",+ &url,+ ])?;++ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "wait_until={wait_until} stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(stdout.contains("slow-main=ready"), "stdout={stdout}");+ assert!(+ stderr.contains("fetch readiness wait timed out; returning best-effort page"),+ "wait_until={wait_until} should use only the remaining budget: stderr={stderr}"+ );+ Ok(())+}++#[test]+fn cli_quiet_page_networkidle_uses_remaining_best_effort_budget() -> Result<()> {+ assert_cli_quiet_page_uses_remaining_best_effort_budget("networkidle")+}++#[test]+fn cli_quiet_page_domstable_uses_remaining_best_effort_budget() -> Result<()> {+ assert_cli_quiet_page_uses_remaining_best_effort_budget("domstable")+}++#[test]+fn cli_post_readiness_wait_cannot_restart_an_exhausted_best_effort_deadline() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-slow-interval-fetch");+ let started = std::time::Instant::now();+ let output = run_moli([+ "moli",+ "fetch",+ "--log-level",+ "warn",+ "--http-no-proxy",+ "*",+ "--wait-until",+ "networkidle",+ "--wait-selector",+ "#never-appears",+ "--timeout",+ "1000",+ "--dump",+ "html",+ &url,+ ])?;+ let elapsed = started.elapsed();+ runtime.block_on(server.shutdown());++ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");+ assert!(stdout.is_empty(), "stdout={stdout}");+ assert!(+ stderr.contains("returning best-effort page"),+ "stderr={stderr}"+ );+ assert!(+ stderr.contains("failed while waiting for selector `#never-appears`"),+ "stderr={stderr}"+ );+ assert!(+ elapsed < std::time::Duration::from_millis(1_300),+ "the selector appears to have received a fresh timeout: {elapsed:?}"+ );+ Ok(())+}+#[test]fn cli_domstable_waits_for_inflight_slow_fetch_content() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;