Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
feat(fetch): return best-effort results on readiness timeout
ldm095a671e2e054
diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rsindex f33e336d2..67568f515 100644--- a/moli-core/src/runtime/mod.rs+++ b/moli-core/src/runtime/mod.rs@@ -59,6 +59,18 @@ fn wait_until_outer_timeout(wait_until: RenderedDomWaitUntil, timeout: Duration)}}+fn is_fetch_readiness_timeout(error: &anyhow::Error, wait_until: RenderedDomWaitUntil) -> bool {+ let expected = match wait_until {+ RenderedDomWaitUntil::NetworkIdle => "timed out waiting for networkidle",+ RenderedDomWaitUntil::DomStable => "timed out waiting for domstable",+ RenderedDomWaitUntil::DomContentLoaded+ | RenderedDomWaitUntil::Load+ | RenderedDomWaitUntil::Done => return false,+ };++ error.chain().any(|cause| cause.to_string() == expected)+}+#[derive(Debug, Clone, Default)]struct AutomationController;@@ -607,6 +619,47 @@ impl Browser {.await}+ async fn wait_for_fetch_readiness(+ &self,+ page: &mut Page,+ raw_url: &str,+ wait: Option<(RenderedDomWaitUntil, Duration)>,+ ) -> Result<()> {+ let Some((wait_until, timeout)) = wait else {+ return Ok(());+ };+ let result = match wait_until {+ RenderedDomWaitUntil::NetworkIdle => {+ page.wait_for_network_idle(&self.resource_request_client(), timeout)+ .await+ }+ RenderedDomWaitUntil::DomStable => {+ page.wait_for_dom_stable(&self.resource_request_client(), timeout)+ .await+ }+ RenderedDomWaitUntil::DomContentLoaded+ | RenderedDomWaitUntil::Load+ | RenderedDomWaitUntil::Done => return Ok(()),+ };++ match result {+ Ok(()) => Ok(()),+ Err(error) if is_fetch_readiness_timeout(&error, wait_until) => {+ warn!(+ page_id = page.page_id(),+ url = %raw_url,+ final_url = %page.final_url(),+ wait_until = ?wait_until,+ timeout_ms = timeout.as_millis(),+ error = %error,+ "fetch readiness wait timed out; returning best-effort page"+ );+ Ok(())+ }+ Err(error) => Err(error),+ }+ }+async fn fetch_internal(&self,request: Request,@@ -897,17 +950,8 @@ impl Browser {.await.with_context(|| anyhow!("failed to execute scripts for page `{raw_url}`"))?;let mut page = materialize_page_created_reply(&renderer_owner, reply)?;- match wait {- Some((RenderedDomWaitUntil::NetworkIdle, timeout)) => {- page.wait_for_network_idle(&self.resource_request_client(), timeout)- .await?;- }- Some((RenderedDomWaitUntil::DomStable, timeout)) => {- page.wait_for_dom_stable(&self.resource_request_client(), timeout)- .await?;- }- _ => {}- }+ self.wait_for_fetch_readiness(&mut page, raw_url, wait)+ .await?;info!(page_id = page.page_id(),url = %page.requested_url(),@@ -1027,17 +1071,8 @@ impl Browser {page_state,page_creation_artifacts,);- match wait {- Some((RenderedDomWaitUntil::NetworkIdle, timeout)) => {- page.wait_for_network_idle(&self.resource_request_client(), timeout)- .await?;- }- Some((RenderedDomWaitUntil::DomStable, timeout)) => {- page.wait_for_dom_stable(&self.resource_request_client(), timeout)- .await?;- }- _ => {}- }+ self.wait_for_fetch_readiness(&mut page, raw_url, wait)+ .await?;info!(page_id = page.page_id(),url = %page.requested_url(),diff --git a/moli-core/tests/fetch_behaviors.rs b/moli-core/tests/fetch_behaviors.rsindex 4126f1a64..99096c55d 100644--- a/moli-core/tests/fetch_behaviors.rs+++ b/moli-core/tests/fetch_behaviors.rs@@ -1939,20 +1939,21 @@ async fn fetch_with_domstable_ignores_page_tampered_outer_html_getter() -> Resul}#[tokio::test(flavor = "multi_thread")]-async fn fetch_with_networkidle_times_out_when_quiet_window_cannot_complete() -> Result<()> {+async fn fetch_with_networkidle_returns_best_effort_when_quiet_window_cannot_complete() -> Result<()>+{let server = FixtureServer::spawn().await?;let browser = Browser::new(AppConfig::default())?;- let error = browser+ let page = browser.fetch_with_wait_until(&server.url("/wait-until-delayed-fetch"),RenderedDomWaitUntil::NetworkIdle,Duration::from_millis(100),)- .await- .unwrap_err();+ .await?;- assert!(error.to_string().contains("timed out"));+ let html = page.serialize_html_async().await?;+ assert!(html.contains("data-state=\"init\""), "html={html}");server.shutdown().await;Ok(())@@ -2012,24 +2013,45 @@ async fn fetch_with_networkidle_waits_for_later_activity_before_quiet_window() -}#[tokio::test(flavor = "multi_thread")]-async fn fetch_with_networkidle_times_out_with_periodic_interval_fetch() -> Result<()> {+async fn fetch_with_networkidle_returns_best_effort_with_periodic_interval_fetch() -> Result<()> {let server = FixtureServer::spawn().await?;let browser = Browser::new(AppConfig::default())?;- let error = browser+ let page = browser.fetch_with_wait_until(&server.url("/wait-until-interval-fetch"),RenderedDomWaitUntil::NetworkIdle,Duration::from_millis(1200),)- .await- .unwrap_err();+ .await?;+ let html = page.serialize_html_async().await?;assert!(- error- .to_string()- .contains("timed out waiting for networkidle"),- "expected networkidle timeout, got: {error:#}"+ html.contains("data-ping=") && html.contains("data-state=\"init\""),+ "html={html}"+ );++ server.shutdown().await;+ Ok(())+}++#[tokio::test(flavor = "multi_thread")]+async fn fetch_with_domstable_returns_best_effort_with_periodic_dom_mutation() -> Result<()> {+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;++ let page = browser+ .fetch_with_wait_until(+ &server.url("/wait-until-interval-dom-mutation"),+ RenderedDomWaitUntil::DomStable,+ Duration::from_millis(700),+ )+ .await?;++ let html = page.serialize_html_async().await?;+ assert!(+ html.contains("data-mutation-count=") && html.contains("id=\"mutation-count\""),+ "html={html}");server.shutdown().await;diff --git a/moli-test-support/src/routes_wait.rs b/moli-test-support/src/routes_wait.rsindex e37ff800e..d596e166f 100644--- a/moli-test-support/src/routes_wait.rs+++ b/moli-test-support/src/routes_wait.rs@@ -32,15 +32,9 @@ const WAIT_UNTIL_TIMER_CALLBACK_ERROR_HTML: &str = "<!doctype html><html><body dconst WAIT_UNTIL_INTERVAL_CALLBACK_ERROR_HTML: &str = "<!doctype html><html><body data-state=\"init\"><script>window.addEventListener('load', () => { let count = 0; const id = setInterval(() => { count += 1; document.body.setAttribute('data-interval-count', String(count)); if (count === 1) { document.body.setAttribute('data-interval-before-error', 'yes'); throw new Error('interval boom'); } clearInterval(id); document.body.setAttribute('data-interval-after-error', 'yes'); const main = document.createElement('main'); main.id = 'after-interval-error'; main.textContent = 'after-interval-error'; document.body.appendChild(main); }, 20); });</script></body></html>";const WAIT_UNTIL_TIMER_DRIVER_WRAPPER_TAMPER_HTML: &str = "<!doctype html><html><body data-state=\"init\"><script>document.body.setAttribute('data-public-timer-driver-exposed', String('__moliRunNextTimeout' in globalThis)); document.body.setAttribute('data-host-timer-driver-exposed', String('__moliHostRunNextTimeout' in globalThis)); globalThis.__moliRunNextTimeout = () => { throw new Error('tampered timer driver wrapper'); }; window.addEventListener('load', () => { setTimeout(() => { document.body.setAttribute('data-after-tamper', 'yes'); const main = document.createElement('main'); main.id = 'after-tamper'; main.textContent = 'after-tamper'; document.body.appendChild(main); }, 20); });</script></body></html>";const WAIT_UNTIL_OUTER_HTML_TAMPER_HTML: &str = "<!doctype html><html><body data-state=\"init\"><script>Object.defineProperty(document.documentElement, 'outerHTML', { configurable: true, get() { throw new Error('domstable must not read outerHTML'); } }); window.addEventListener('load', () => { setTimeout(() => { document.body.setAttribute('data-after-outerhtml-tamper', 'yes'); const main = document.createElement('main'); main.id = 'after-outerhtml-tamper'; main.textContent = 'after-outerhtml-tamper'; document.body.appendChild(main); }, 20); });</script></body></html>";-// 50 ms interval (was 200 ms): the only caller of this fixture-// (fetch_with_networkidle_times_out_with_periodic_interval_fetch) asserts-// that the page *never* reaches network idle inside its 1200 ms test-// deadline by keeping a periodic fetch in flight. The fetcher's idle-// threshold is in the ~500 ms range, so a 200 ms interval has only a-// ~2.5x safety margin — when the renderer's tokio thread is contended-// the timer can easily stretch past 500 ms, the wait declares idle, and-// the test sees `Ok` instead of the expected timeout error. 50 ms gives-// a ~10x margin while still landing many fetches inside 1200 ms.+// Keep activity frequent enough that network-idle cannot complete before the+// best-effort timeout tests reach their deadline. The fetcher's idle threshold+// is around 500 ms; a 50 ms interval leaves enough margin under nextest load.const WAIT_UNTIL_INTERVAL_FETCH_HTML: &str = "<!doctype html><html><body data-state=\"init\"><script>window.addEventListener('load', () => { setInterval(() => { fetch('/wait-until-data').then(() => { document.body.setAttribute('data-ping', String((Number(document.body.getAttribute('data-ping') || '0') + 1))); }); }, 50); });</script></body></html>";const WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML: &str = "<!doctype html><html><body data-state=\"init\"><main id=\"mutation-count\">0</main><script>window.addEventListener('load', () => { setInterval(() => { const count = Number(document.body.getAttribute('data-mutation-count') || '0') + 1; document.body.setAttribute('data-mutation-count', String(count)); document.getElementById('mutation-count').textContent = String(count); }, 50); });</script></body></html>";diff --git a/moli/src/cli.rs b/moli/src/cli.rsindex 7b2bf94a1..2977d4866 100644--- a/moli/src/cli.rs+++ b/moli/src/cli.rs@@ -119,7 +119,9 @@ pub struct FetchArgs {#[arg(long, value_parser = parse_response_json_path_arg)]pub wait_response_json: Option<ResponseJsonPathArg>,- #[arg(long, alias = "wait-ms", default_value_t = 30_000)]+ /// Maximum wait time in milliseconds. Network-idle and DOM-stable fetches+ /// return the current page with a warning when this deadline expires.+ #[arg(long, alias = "wait-ms", default_value_t = 10_000)]pub timeout: u64,#[command(flatten)]diff --git a/moli/tests/cli.rs b/moli/tests/cli.rsindex e9830b590..237dae7ee 100644--- a/moli/tests/cli.rs+++ b/moli/tests/cli.rs@@ -94,7 +94,7 @@ fn parses_explicit_fetch_command_with_compatibility_flags() {wait_response_url: None,wait_response_body: None,wait_response_json: None,- timeout: 30_000,+ timeout: 10_000,common: CommonArgs {insecure_disable_tls_host_verification: false,obey_robots: true,@@ -323,7 +323,7 @@ fn infers_fetch_mode_from_bare_url() {wait_response_url: None,wait_response_body: None,wait_response_json: None,- timeout: 30_000,+ timeout: 10_000,common: CommonArgs::default(),url: "https://example.com".to_owned(),}))@@ -358,7 +358,7 @@ fn infers_fetch_mode_from_fetch_only_flags_and_defaults_dump_to_html() {wait_response_url: None,wait_response_body: None,wait_response_json: None,- timeout: 30_000,+ timeout: 10_000,common: CommonArgs::default(),url: "https://example.com".to_owned(),}))@@ -397,7 +397,7 @@ fn infers_fetch_mode_from_header_flag() {wait_response_url: None,wait_response_body: None,wait_response_json: None,- timeout: 30_000,+ timeout: 10_000,common: CommonArgs::default(),url: "https://example.com".to_owned(),}))diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex 2408b007b..904d26759 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -1914,6 +1914,47 @@ fn cli_domstable_waits_for_late_content() -> Result<()> {Ok(())}+#[test]+fn cli_networkidle_timeout_logs_warning_and_returns_best_effort_output() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-interval-fetch");+ let output = run_moli([+ "moli",+ "fetch",+ "--log-level",+ "warn",+ "--http-no-proxy",+ "*",+ "--wait-until",+ "networkidle",+ "--timeout",+ "1200",+ "--dump",+ "html",+ &url,+ ])?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "moli fetch failed: stdout={}\nstderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );++ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(stdout.contains("data-ping="), "stdout={stdout}");+ assert!(+ stderr.contains("fetch readiness wait timed out; returning best-effort page"),+ "stderr={stderr}"+ );+ assert!(stderr.contains("wait_until=NetworkIdle"), "stderr={stderr}");+ assert!(stderr.contains("timeout_ms=1200"), "stderr={stderr}");+ Ok(())+}+#[test]fn cli_domstable_waits_for_inflight_slow_fetch_content() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;