Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

feat(fetch): return best-effort results on readiness timeout

ldm095a671e2e054
diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rs
index f33e336d2..67568f515 100644
--- a/moli-core/src/runtime/mod.rs
+++ b/moli-core/src/runtime/mod.rs
@@ -59,6 +59,18 @@ fn wait_until_outer_timeout(wait_until: RenderedDomWaitUntil, timeout: Duration)
}
}
+fn is_fetch_readiness_timeout(error: &anyhow::Error, wait_until: RenderedDomWaitUntil) -> bool {
+ let expected = match wait_until {
+ RenderedDomWaitUntil::NetworkIdle => "timed out waiting for networkidle",
+ RenderedDomWaitUntil::DomStable => "timed out waiting for domstable",
+ RenderedDomWaitUntil::DomContentLoaded
+ | RenderedDomWaitUntil::Load
+ | RenderedDomWaitUntil::Done => return false,
+ };
+
+ error.chain().any(|cause| cause.to_string() == expected)
+}
+
#[derive(Debug, Clone, Default)]
struct AutomationController;
@@ -607,6 +619,47 @@ impl Browser {
.await
}
+ async fn wait_for_fetch_readiness(
+ &self,
+ page: &mut Page,
+ raw_url: &str,
+ wait: Option<(RenderedDomWaitUntil, Duration)>,
+ ) -> Result<()> {
+ let Some((wait_until, timeout)) = wait else {
+ return Ok(());
+ };
+ let result = match wait_until {
+ RenderedDomWaitUntil::NetworkIdle => {
+ page.wait_for_network_idle(&self.resource_request_client(), timeout)
+ .await
+ }
+ RenderedDomWaitUntil::DomStable => {
+ page.wait_for_dom_stable(&self.resource_request_client(), timeout)
+ .await
+ }
+ RenderedDomWaitUntil::DomContentLoaded
+ | RenderedDomWaitUntil::Load
+ | RenderedDomWaitUntil::Done => return Ok(()),
+ };
+
+ match result {
+ Ok(()) => Ok(()),
+ Err(error) if is_fetch_readiness_timeout(&error, wait_until) => {
+ warn!(
+ page_id = page.page_id(),
+ url = %raw_url,
+ final_url = %page.final_url(),
+ wait_until = ?wait_until,
+ timeout_ms = timeout.as_millis(),
+ error = %error,
+ "fetch readiness wait timed out; returning best-effort page"
+ );
+ Ok(())
+ }
+ Err(error) => Err(error),
+ }
+ }
+
async fn fetch_internal(
&self,
request: Request,
@@ -897,17 +950,8 @@ impl Browser {
.await
.with_context(|| anyhow!("failed to execute scripts for page `{raw_url}`"))?;
let mut page = materialize_page_created_reply(&renderer_owner, reply)?;
- match wait {
- Some((RenderedDomWaitUntil::NetworkIdle, timeout)) => {
- page.wait_for_network_idle(&self.resource_request_client(), timeout)
- .await?;
- }
- Some((RenderedDomWaitUntil::DomStable, timeout)) => {
- page.wait_for_dom_stable(&self.resource_request_client(), timeout)
- .await?;
- }
- _ => {}
- }
+ self.wait_for_fetch_readiness(&mut page, raw_url, wait)
+ .await?;
info!(
page_id = page.page_id(),
url = %page.requested_url(),
@@ -1027,17 +1071,8 @@ impl Browser {
page_state,
page_creation_artifacts,
);
- match wait {
- Some((RenderedDomWaitUntil::NetworkIdle, timeout)) => {
- page.wait_for_network_idle(&self.resource_request_client(), timeout)
- .await?;
- }
- Some((RenderedDomWaitUntil::DomStable, timeout)) => {
- page.wait_for_dom_stable(&self.resource_request_client(), timeout)
- .await?;
- }
- _ => {}
- }
+ self.wait_for_fetch_readiness(&mut page, raw_url, wait)
+ .await?;
info!(
page_id = page.page_id(),
url = %page.requested_url(),
diff --git a/moli-core/tests/fetch_behaviors.rs b/moli-core/tests/fetch_behaviors.rs
index 4126f1a64..99096c55d 100644
--- a/moli-core/tests/fetch_behaviors.rs
+++ b/moli-core/tests/fetch_behaviors.rs
@@ -1939,20 +1939,21 @@ async fn fetch_with_domstable_ignores_page_tampered_outer_html_getter() -> Resul
}
#[tokio::test(flavor = "multi_thread")]
-async fn fetch_with_networkidle_times_out_when_quiet_window_cannot_complete() -> Result<()> {
+async fn fetch_with_networkidle_returns_best_effort_when_quiet_window_cannot_complete() -> Result<()>
+{
let server = FixtureServer::spawn().await?;
let browser = Browser::new(AppConfig::default())?;
- let error = browser
+ let page = browser
.fetch_with_wait_until(
&server.url("/wait-until-delayed-fetch"),
RenderedDomWaitUntil::NetworkIdle,
Duration::from_millis(100),
)
- .await
- .unwrap_err();
+ .await?;
- assert!(error.to_string().contains("timed out"));
+ let html = page.serialize_html_async().await?;
+ assert!(html.contains("data-state=\"init\""), "html={html}");
server.shutdown().await;
Ok(())
@@ -2012,24 +2013,45 @@ async fn fetch_with_networkidle_waits_for_later_activity_before_quiet_window() -
}
#[tokio::test(flavor = "multi_thread")]
-async fn fetch_with_networkidle_times_out_with_periodic_interval_fetch() -> Result<()> {
+async fn fetch_with_networkidle_returns_best_effort_with_periodic_interval_fetch() -> Result<()> {
let server = FixtureServer::spawn().await?;
let browser = Browser::new(AppConfig::default())?;
- let error = browser
+ let page = browser
.fetch_with_wait_until(
&server.url("/wait-until-interval-fetch"),
RenderedDomWaitUntil::NetworkIdle,
Duration::from_millis(1200),
)
- .await
- .unwrap_err();
+ .await?;
+ let html = page.serialize_html_async().await?;
assert!(
- error
- .to_string()
- .contains("timed out waiting for networkidle"),
- "expected networkidle timeout, got: {error:#}"
+ html.contains("data-ping=") && html.contains("data-state=\"init\""),
+ "html={html}"
+ );
+
+ server.shutdown().await;
+ Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn fetch_with_domstable_returns_best_effort_with_periodic_dom_mutation() -> Result<()> {
+ let server = FixtureServer::spawn().await?;
+ let browser = Browser::new(AppConfig::default())?;
+
+ let page = browser
+ .fetch_with_wait_until(
+ &server.url("/wait-until-interval-dom-mutation"),
+ RenderedDomWaitUntil::DomStable,
+ Duration::from_millis(700),
+ )
+ .await?;
+
+ let html = page.serialize_html_async().await?;
+ assert!(
+ html.contains("data-mutation-count=") && html.contains("id=\"mutation-count\""),
+ "html={html}"
);
server.shutdown().await;
diff --git a/moli-test-support/src/routes_wait.rs b/moli-test-support/src/routes_wait.rs
index e37ff800e..d596e166f 100644
--- a/moli-test-support/src/routes_wait.rs
+++ b/moli-test-support/src/routes_wait.rs
@@ -32,15 +32,9 @@ const WAIT_UNTIL_TIMER_CALLBACK_ERROR_HTML: &str = "<!doctype html><html><body d
const WAIT_UNTIL_INTERVAL_CALLBACK_ERROR_HTML: &str = "<!doctype html><html><body data-state=\"init\"><script>window.addEventListener('load', () => { let count = 0; const id = setInterval(() => { count += 1; document.body.setAttribute('data-interval-count', String(count)); if (count === 1) { document.body.setAttribute('data-interval-before-error', 'yes'); throw new Error('interval boom'); } clearInterval(id); document.body.setAttribute('data-interval-after-error', 'yes'); const main = document.createElement('main'); main.id = 'after-interval-error'; main.textContent = 'after-interval-error'; document.body.appendChild(main); }, 20); });</script></body></html>";
const WAIT_UNTIL_TIMER_DRIVER_WRAPPER_TAMPER_HTML: &str = "<!doctype html><html><body data-state=\"init\"><script>document.body.setAttribute('data-public-timer-driver-exposed', String('__moliRunNextTimeout' in globalThis)); document.body.setAttribute('data-host-timer-driver-exposed', String('__moliHostRunNextTimeout' in globalThis)); globalThis.__moliRunNextTimeout = () => { throw new Error('tampered timer driver wrapper'); }; window.addEventListener('load', () => { setTimeout(() => { document.body.setAttribute('data-after-tamper', 'yes'); const main = document.createElement('main'); main.id = 'after-tamper'; main.textContent = 'after-tamper'; document.body.appendChild(main); }, 20); });</script></body></html>";
const WAIT_UNTIL_OUTER_HTML_TAMPER_HTML: &str = "<!doctype html><html><body data-state=\"init\"><script>Object.defineProperty(document.documentElement, 'outerHTML', { configurable: true, get() { throw new Error('domstable must not read outerHTML'); } }); window.addEventListener('load', () => { setTimeout(() => { document.body.setAttribute('data-after-outerhtml-tamper', 'yes'); const main = document.createElement('main'); main.id = 'after-outerhtml-tamper'; main.textContent = 'after-outerhtml-tamper'; document.body.appendChild(main); }, 20); });</script></body></html>";
-// 50 ms interval (was 200 ms): the only caller of this fixture
-// (fetch_with_networkidle_times_out_with_periodic_interval_fetch) asserts
-// that the page *never* reaches network idle inside its 1200 ms test
-// deadline by keeping a periodic fetch in flight. The fetcher's idle
-// threshold is in the ~500 ms range, so a 200 ms interval has only a
-// ~2.5x safety margin — when the renderer's tokio thread is contended
-// the timer can easily stretch past 500 ms, the wait declares idle, and
-// the test sees `Ok` instead of the expected timeout error. 50 ms gives
-// a ~10x margin while still landing many fetches inside 1200 ms.
+// Keep activity frequent enough that network-idle cannot complete before the
+// best-effort timeout tests reach their deadline. The fetcher's idle threshold
+// is around 500 ms; a 50 ms interval leaves enough margin under nextest load.
const WAIT_UNTIL_INTERVAL_FETCH_HTML: &str = "<!doctype html><html><body data-state=\"init\"><script>window.addEventListener('load', () => { setInterval(() => { fetch('/wait-until-data').then(() => { document.body.setAttribute('data-ping', String((Number(document.body.getAttribute('data-ping') || '0') + 1))); }); }, 50); });</script></body></html>";
const WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML: &str = "<!doctype html><html><body data-state=\"init\"><main id=\"mutation-count\">0</main><script>window.addEventListener('load', () => { setInterval(() => { const count = Number(document.body.getAttribute('data-mutation-count') || '0') + 1; document.body.setAttribute('data-mutation-count', String(count)); document.getElementById('mutation-count').textContent = String(count); }, 50); });</script></body></html>";
diff --git a/moli/src/cli.rs b/moli/src/cli.rs
index 7b2bf94a1..2977d4866 100644
--- a/moli/src/cli.rs
+++ b/moli/src/cli.rs
@@ -119,7 +119,9 @@ pub struct FetchArgs {
#[arg(long, value_parser = parse_response_json_path_arg)]
pub wait_response_json: Option<ResponseJsonPathArg>,
- #[arg(long, alias = "wait-ms", default_value_t = 30_000)]
+ /// Maximum wait time in milliseconds. Network-idle and DOM-stable fetches
+ /// return the current page with a warning when this deadline expires.
+ #[arg(long, alias = "wait-ms", default_value_t = 10_000)]
pub timeout: u64,
#[command(flatten)]
diff --git a/moli/tests/cli.rs b/moli/tests/cli.rs
index e9830b590..237dae7ee 100644
--- a/moli/tests/cli.rs
+++ b/moli/tests/cli.rs
@@ -94,7 +94,7 @@ fn parses_explicit_fetch_command_with_compatibility_flags() {
wait_response_url: None,
wait_response_body: None,
wait_response_json: None,
- timeout: 30_000,
+ timeout: 10_000,
common: CommonArgs {
insecure_disable_tls_host_verification: false,
obey_robots: true,
@@ -323,7 +323,7 @@ fn infers_fetch_mode_from_bare_url() {
wait_response_url: None,
wait_response_body: None,
wait_response_json: None,
- timeout: 30_000,
+ timeout: 10_000,
common: CommonArgs::default(),
url: "https://example.com".to_owned(),
}))
@@ -358,7 +358,7 @@ fn infers_fetch_mode_from_fetch_only_flags_and_defaults_dump_to_html() {
wait_response_url: None,
wait_response_body: None,
wait_response_json: None,
- timeout: 30_000,
+ timeout: 10_000,
common: CommonArgs::default(),
url: "https://example.com".to_owned(),
}))
@@ -397,7 +397,7 @@ fn infers_fetch_mode_from_header_flag() {
wait_response_url: None,
wait_response_body: None,
wait_response_json: None,
- timeout: 30_000,
+ timeout: 10_000,
common: CommonArgs::default(),
url: "https://example.com".to_owned(),
}))
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index 2408b007b..904d26759 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -1914,6 +1914,47 @@ fn cli_domstable_waits_for_late_content() -> Result<()> {
Ok(())
}
+#[test]
+fn cli_networkidle_timeout_logs_warning_and_returns_best_effort_output() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-interval-fetch");
+ let output = run_moli([
+ "moli",
+ "fetch",
+ "--log-level",
+ "warn",
+ "--http-no-proxy",
+ "*",
+ "--wait-until",
+ "networkidle",
+ "--timeout",
+ "1200",
+ "--dump",
+ "html",
+ &url,
+ ])?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "moli fetch failed: stdout={}\nstderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(stdout.contains("data-ping="), "stdout={stdout}");
+ assert!(
+ stderr.contains("fetch readiness wait timed out; returning best-effort page"),
+ "stderr={stderr}"
+ );
+ assert!(stderr.contains("wait_until=NetworkIdle"), "stderr={stderr}");
+ assert!(stderr.contains("timeout_ms=1200"), "stderr={stderr}");
+ Ok(())
+}
+
#[test]
fn cli_domstable_waits_for_inflight_slow_fetch_content() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;