Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
test: cover HTTP error navigation edge cases
ldm0bea0f130b554
diff --git a/moli-core/tests/lifecycle_decision.rs b/moli-core/tests/lifecycle_decision.rsindex b5c2a2beb..56903348c 100644--- a/moli-core/tests/lifecycle_decision.rs+++ b/moli-core/tests/lifecycle_decision.rs@@ -1,6 +1,6 @@use moli_test_support as support;-use anyhow::{Result, anyhow};+use anyhow::{Context, Result, anyhow, bail, ensure};use moli_core::{page::Page,runtime::{@@ -13,7 +13,7 @@ use moli_fetch::Request;use parking_lot::Mutex;use std::sync::Arc;use support::FixtureServer;-use tokio::time::Duration;+use tokio::{sync::oneshot, time::Duration};fn executable_page(document: FetchedDocument) -> Result<Page> {match document {@@ -29,6 +29,35 @@ fn diagnostic_global<'a>(page: &'a Page, name: &str) -> Option<&'a JsValueSnapshpage.script_execution().global(name)}+async fn follow_http_error_navigation(+ browser: &Browser,+ url: &str,+ wait_until: RenderedDomWaitUntil,+ navigation_grace_ms: u64,+ successor_timeout: Duration,+) -> Result<Page> {+ executable_page(+ browser+ .fetch_document_with_lifecycle_decider(+ Request::get(url)?,+ wait_until,+ Duration::from_secs(5),+ successor_timeout,+ move |target| {+ ensure!(+ (400..=599).contains(&target.status),+ "expected an HTTP error lifecycle target, got status {}",+ target.status+ );+ Ok(RendererLifecycleDecision::FollowNextDocument {+ navigation_grace_ms,+ })+ },+ )+ .await?,+ )+}+#[tokio::test(flavor = "multi_thread")]async fn lifecycle_decider_finishes_without_extra_owner_command() -> Result<()> {let server = FixtureServer::spawn().await?;@@ -221,6 +250,93 @@ async fn http_error_navigation_wait_follows_same_url_reload_to_domcontentloaded(diagnostic_global(&page, "httpErrorNavigationDcl"),Some(&JsValueSnapshot::Bool(true)));+ assert_eq!(+ diagnostic_global(&page, "httpErrorNavigationLoad"),+ Some(&JsValueSnapshot::Bool(false)),+ "the successor snapshot must stop at DCL rather than drifting to Load"+ );+ assert_eq!(+ diagnostic_global(&page, "httpErrorNavigationSlowScript"),+ None,+ "the DCL-inserted slow script must not complete before the DCL reply"+ );++ server.shutdown().await;+ Ok(())+}++#[tokio::test(flavor = "multi_thread")]+async fn http_error_navigation_wait_follows_five_same_url_reloads_to_domcontentloaded() -> Result<()>+{+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;+ let url = server.url("/wait-until-http-error-five-navigations");++ let page = follow_http_error_navigation(+ &browser,+ &url,+ RenderedDomWaitUntil::DomContentLoaded,+ 1_000,+ Duration::from_secs(10),+ )+ .await?;++ assert_eq!(page.status(), 200);+ assert_eq!(page.final_url().as_str(), url);+ assert_eq!(+ diagnostic_global(&page, "httpErrorNavigationChainStep"),+ Some(&JsValueSnapshot::Number(5.0))+ );+ assert_eq!(+ diagnostic_global(&page, "httpErrorNavigationDcl"),+ Some(&JsValueSnapshot::Bool(true))+ );+ assert_eq!(+ diagnostic_global(&page, "httpErrorNavigationLoad"),+ Some(&JsValueSnapshot::Bool(false))+ );+ assert_eq!(+ diagnostic_global(&page, "httpErrorNavigationSlowScript"),+ None+ );++ server.shutdown().await;+ Ok(())+}++#[tokio::test(flavor = "multi_thread")]+async fn http_error_navigation_wait_follows_five_same_url_reloads_to_load() -> Result<()> {+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;+ let url = server.url("/wait-until-http-error-five-navigations");++ let page = follow_http_error_navigation(+ &browser,+ &url,+ RenderedDomWaitUntil::Load,+ 1_000,+ Duration::from_secs(10),+ )+ .await?;++ assert_eq!(page.status(), 200);+ assert_eq!(page.final_url().as_str(), url);+ assert_eq!(+ diagnostic_global(&page, "httpErrorNavigationChainStep"),+ Some(&JsValueSnapshot::Number(5.0))+ );+ assert_eq!(+ diagnostic_global(&page, "httpErrorNavigationDcl"),+ Some(&JsValueSnapshot::Bool(true))+ );+ assert_eq!(+ diagnostic_global(&page, "httpErrorNavigationLoad"),+ Some(&JsValueSnapshot::Bool(true))+ );+ assert_eq!(+ diagnostic_global(&page, "httpErrorNavigationSlowScript"),+ Some(&JsValueSnapshot::Bool(true))+ );server.shutdown().await;Ok(())@@ -304,3 +420,149 @@ async fn http_error_navigation_wait_reports_no_navigation_without_refetching() -server.shutdown().await;Ok(())}++#[tokio::test(flavor = "multi_thread")]+async fn same_document_navigation_does_not_satisfy_http_error_replacement_wait() -> Result<()> {+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;+ let url = server.url("/wait-until-http-error-same-document-navigation");++ let error = browser+ .fetch_document_with_lifecycle_decider(+ Request::get(&url)?,+ RenderedDomWaitUntil::Load,+ Duration::from_secs(5),+ Duration::from_secs(1),+ |target| {+ ensure!(target.status == 403, "expected 403, got {}", target.status);+ Ok(RendererLifecycleDecision::FollowNextDocument {+ navigation_grace_ms: 150,+ })+ },+ )+ .await+ .expect_err("a fragment-only navigation must not replace the HTTP error Document");+ let error = format!("{error:#}");+ assert!(error.contains("403 Forbidden"), "error={error}");+ assert!(error.contains("150 ms grace period"), "error={error}");++ server.shutdown().await;+ Ok(())+}++#[tokio::test(flavor = "multi_thread")]+async fn dropping_http_error_navigation_wait_keeps_renderer_owner_usable() -> Result<()> {+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;+ let url = server.url("/net/upstream/xhr/404");+ let (decision_tx, decision_rx) = oneshot::channel();++ let mut waiting_fetch = Box::pin(browser.fetch_document_with_lifecycle_decider(+ Request::get(&url)?,+ RenderedDomWaitUntil::Load,+ Duration::from_secs(5),+ Duration::from_secs(11),+ move |target| {+ ensure!(target.status == 404, "expected 404, got {}", target.status);+ decision_tx+ .send(())+ .map_err(|_| anyhow!("decision observer was dropped"))?;+ Ok(RendererLifecycleDecision::FollowNextDocument {+ navigation_grace_ms: 10_000,+ })+ },+ ));+ tokio::select! {+ decision = decision_rx => decision.context("lifecycle decider did not signal")?,+ result = &mut waiting_fetch => {+ result?;+ bail!("HTTP error wait completed before it could be cancelled");+ }+ }+ drop(waiting_fetch);++ let page = tokio::time::timeout(+ Duration::from_secs(2),+ browser.fetch_request_document_allow_http_error(Request::get("about:blank")?),+ )+ .await+ .context("renderer owner stayed blocked after cancelling the lifecycle wait")??;+ assert_eq!(executable_page(page)?.status(), 200);++ server.shutdown().await;+ Ok(())+}++#[tokio::test(flavor = "multi_thread")]+async fn parked_http_error_navigation_wait_does_not_block_another_page() -> Result<()> {+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;+ let url = server.url("/net/upstream/xhr/404");+ let (decision_tx, decision_rx) = oneshot::channel();++ let mut waiting_fetch = Box::pin(browser.fetch_document_with_lifecycle_decider(+ Request::get(&url)?,+ RenderedDomWaitUntil::Load,+ Duration::from_secs(5),+ Duration::from_secs(11),+ move |target| {+ ensure!(target.status == 404, "expected 404, got {}", target.status);+ decision_tx+ .send(())+ .map_err(|_| anyhow!("decision observer was dropped"))?;+ Ok(RendererLifecycleDecision::FollowNextDocument {+ navigation_grace_ms: 10_000,+ })+ },+ ));+ tokio::select! {+ decision = decision_rx => decision.context("lifecycle decider did not signal")?,+ result = &mut waiting_fetch => {+ result?;+ bail!("HTTP error wait completed before the concurrency check");+ }+ }++ let quick_page = tokio::time::timeout(+ Duration::from_secs(2),+ browser.fetch_request_document_allow_http_error(Request::get("about:blank")?),+ )+ .await+ .context("a parked lifecycle wait blocked an unrelated Page creation")??;+ assert_eq!(executable_page(quick_page)?.status(), 200);+ drop(waiting_fetch);++ server.shutdown().await;+ Ok(())+}++#[tokio::test(flavor = "multi_thread")]+async fn http_error_replacement_wait_keeps_chained_navigation_limit() -> Result<()> {+ let server = FixtureServer::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;+ let url = server.url("/wait-until-http-error-navigation-loop");++ let error = browser+ .fetch_document_with_lifecycle_decider(+ Request::get(&url)?,+ RenderedDomWaitUntil::Load,+ Duration::from_secs(5),+ Duration::from_secs(20),+ |target| {+ ensure!(target.status == 403, "expected 403, got {}", target.status);+ Ok(RendererLifecycleDecision::FollowNextDocument {+ navigation_grace_ms: 1_000,+ })+ },+ )+ .await+ .expect_err("the replacement navigation loop must hit the owner chain limit");+ let error = format!("{error:#}");+ assert!(+ error.contains("too many chained location navigations"),+ "error={error}"+ );++ server.shutdown().await;+ Ok(())+}diff --git a/moli-test-support/src/routes_wait.rs b/moli-test-support/src/routes_wait.rsindex a604e1450..04185d402 100644--- a/moli-test-support/src/routes_wait.rs+++ b/moli-test-support/src/routes_wait.rs@@ -40,6 +40,73 @@ const WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML: &str = "<!doctype html><html><bodyconst WAIT_UNTIL_HTTP_ERROR_NAVIGATION_CHALLENGE_HTML: &str = "<!doctype html><html><head><title>403 challenge</title></head><body><main id=\"challenge\">http-error-navigation=challenge</main><script>window.addEventListener('load', () => { setTimeout(() => { document.cookie = 'moli-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax'; location.reload(); }, 75); });</script></body></html>";const WAIT_UNTIL_HTTP_ERROR_LATE_NAVIGATION_CHALLENGE_HTML: &str = "<!doctype html><html><head><title>late 403 challenge</title></head><body><main id=\"challenge\">http-error-navigation=late-challenge</main><script>window.addEventListener('load', () => { setTimeout(() => { document.cookie = 'moli-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax'; location.reload(); }, 1200); });</script></body></html>";const WAIT_UNTIL_HTTP_ERROR_NAVIGATION_FINAL_HTML: &str = "<!doctype html><html><head><title>challenge passed</title></head><body><main id=\"http-error-navigation-target\">http-error-navigation=done</main><script>window.httpErrorNavigationDcl = false; window.httpErrorNavigationLoad = false; document.addEventListener('DOMContentLoaded', () => { window.httpErrorNavigationDcl = true; document.body.setAttribute('data-reached-dcl', 'true'); const script = document.createElement('script'); script.src = '/assets/runtime_owned_in_order_load_very_slow.js'; script.onload = () => { window.httpErrorNavigationSlowScript = true; const tail = document.createElement('main'); tail.id = 'http-error-navigation-load-tail'; tail.textContent = 'http-error-navigation=load-tail'; document.body.appendChild(tail); }; document.head.appendChild(script); }); window.addEventListener('load', () => { window.httpErrorNavigationLoad = true; document.body.setAttribute('data-reached-load', 'true'); });</script></body></html>";+const WAIT_UNTIL_HTTP_ERROR_IMMEDIATE_NAVIGATION_CHALLENGE_HTML: &str = r#"<!doctype html>+<html><head><title>immediate 403 challenge</title></head><body>+<main id="challenge">http-error-navigation=immediate-challenge</main>+<script>+window.addEventListener('load', () => {+ document.cookie = 'moli-http-error-immediate-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax';+ location.reload();+});+</script>+</body></html>"#;+const WAIT_UNTIL_HTTP_ERROR_SAME_DOCUMENT_NAVIGATION_HTML: &str = r#"<!doctype html>+<html><head><title>same-document 403 challenge</title></head><body>+<main id="challenge">http-error-navigation=same-document</main>+<script>+window.addEventListener('load', () => {+ setTimeout(() => { location.hash = 'same-document-only'; }, 25);+});+</script>+</body></html>"#;+const WAIT_UNTIL_HTTP_ERROR_FIVE_NAVIGATION_CHALLENGE_HTML: &str = r#"<!doctype html>+<html><head><title>five-navigation 403 challenge</title></head><body>+<main id="challenge">http-error-navigation=five-navigation-challenge</main>+<script>+window.addEventListener('load', () => {+ setTimeout(() => {+ document.cookie = 'moli-http-error-five-navigation-step=1; Path=/; Max-Age=3600; SameSite=Lax';+ location.reload();+ }, 75);+});+</script>+</body></html>"#;+const WAIT_UNTIL_HTTP_ERROR_FIVE_NAVIGATION_FINAL_HTML: &str = r#"<!doctype html>+<html><head><title>five-navigation challenge passed</title></head><body>+<main id="http-error-five-navigation-target">http-error-navigation=five-navigation-done</main>+<script>+window.httpErrorNavigationChainStep = 5;+window.httpErrorNavigationDcl = false;+window.httpErrorNavigationLoad = false;+document.addEventListener('DOMContentLoaded', () => {+ window.httpErrorNavigationDcl = true;+ document.body.setAttribute('data-reached-dcl', 'true');+ const script = document.createElement('script');+ script.src = '/assets/runtime_owned_in_order_load_very_slow.js';+ script.onload = () => {+ window.httpErrorNavigationSlowScript = true;+ const tail = document.createElement('main');+ tail.id = 'http-error-five-navigation-load-tail';+ tail.textContent = 'http-error-navigation=five-navigation-load-tail';+ document.body.appendChild(tail);+ };+ document.head.appendChild(script);+});+window.addEventListener('load', () => {+ window.httpErrorNavigationLoad = true;+ document.body.setAttribute('data-reached-load', 'true');+});+</script>+</body></html>"#;+const WAIT_UNTIL_HTTP_ERROR_NAVIGATION_LOOP_HTML: &str = r#"<!doctype html>+<html><head><title>navigation-loop 403 challenge</title></head><body>+<main id="challenge">http-error-navigation=loop-challenge</main>+<script>+window.addEventListener('load', () => {+ setTimeout(() => { location.replace('/location-nav/loop-a'); }, 25);+});+</script>+</body></html>"#;pub(super) fn add_wait_routes(router: Router) -> Router {router@@ -125,6 +192,22 @@ pub(super) fn add_wait_routes(router: Router) -> Router {"/wait-until-http-error-navigation-to-error",get(wait_until_http_error_navigation_to_error_page),)+ .route(+ "/wait-until-http-error-immediate-navigation",+ get(wait_until_http_error_immediate_navigation_page),+ )+ .route(+ "/wait-until-http-error-same-document-navigation",+ get(wait_until_http_error_same_document_navigation_page),+ )+ .route(+ "/wait-until-http-error-five-navigations",+ get(wait_until_http_error_five_navigations_page),+ )+ .route(+ "/wait-until-http-error-navigation-loop",+ get(wait_until_http_error_navigation_loop_page),+ ).route("/wait-until-http-error-late-navigation",get(wait_until_http_error_late_navigation_page),@@ -172,6 +255,68 @@ async fn wait_until_http_error_navigation_to_error_page(headers: HeaderMap) -> R.into_response()}+async fn wait_until_http_error_immediate_navigation_page(headers: HeaderMap) -> Response {+ if has_cookie(&headers, "moli-http-error-immediate-navigation=passed") {+ return Html(WAIT_UNTIL_HTTP_ERROR_NAVIGATION_FINAL_HTML).into_response();+ }++ (+ StatusCode::FORBIDDEN,+ Html(WAIT_UNTIL_HTTP_ERROR_IMMEDIATE_NAVIGATION_CHALLENGE_HTML),+ )+ .into_response()+}++async fn wait_until_http_error_same_document_navigation_page() -> Response {+ (+ StatusCode::FORBIDDEN,+ Html(WAIT_UNTIL_HTTP_ERROR_SAME_DOCUMENT_NAVIGATION_HTML),+ )+ .into_response()+}++async fn wait_until_http_error_five_navigations_page(headers: HeaderMap) -> Response {+ let step = (1_u8..=5)+ .find(|step| {+ has_cookie(+ &headers,+ &format!("moli-http-error-five-navigation-step={step}"),+ )+ })+ .unwrap_or(0);+ if step == 0 {+ return (+ StatusCode::FORBIDDEN,+ Html(WAIT_UNTIL_HTTP_ERROR_FIVE_NAVIGATION_CHALLENGE_HTML),+ )+ .into_response();+ }+ if step == 5 {+ return Html(WAIT_UNTIL_HTTP_ERROR_FIVE_NAVIGATION_FINAL_HTML).into_response();+ }++ let next_step = step + 1;+ Html(format!(+ r#"<!doctype html>+<html><head><title>five-navigation step {step}</title></head><body>+<main id="http-error-five-navigation-step-{step}">step={step}</main>+<script>+document.cookie = 'moli-http-error-five-navigation-step={next_step}; Path=/; Max-Age=3600; SameSite=Lax';+location.reload();+</script>+</body></html>"#+ ))+ .into_response()+}++async fn wait_until_http_error_navigation_loop_page() -> Response {+ (+ StatusCode::FORBIDDEN,+ Html(WAIT_UNTIL_HTTP_ERROR_NAVIGATION_LOOP_HTML),+ )+ .into_response()+}+async fn wait_until_http_error_late_navigation_page(headers: HeaderMap) -> Response {if has_cookie(&headers, "moli-http-error-navigation=passed") {return Html(WAIT_UNTIL_HTTP_ERROR_NAVIGATION_FINAL_HTML).into_response();diff --git a/moli/src/app/http_error_navigation.rs b/moli/src/app/http_error_navigation.rsindex cf5810f29..65dab03f3 100644--- a/moli/src/app/http_error_navigation.rs+++ b/moli/src/app/http_error_navigation.rs@@ -43,3 +43,18 @@ pub(super) async fn fetch_with_http_error_navigation().await}++#[cfg(test)]+mod tests {+ use super::is_http_error_status;++ #[test]+ fn http_error_status_covers_only_four_hundred_and_five_hundred_ranges() {+ assert!(!is_http_error_status(399));+ assert!(is_http_error_status(400));+ assert!(is_http_error_status(499));+ assert!(is_http_error_status(500));+ assert!(is_http_error_status(599));+ assert!(!is_http_error_status(600));+ }+}diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex 1af0c5f73..29930d14b 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -2098,6 +2098,33 @@ fn cli_load_recovers_from_delayed_403_navigation_at_same_stage() -> Result<()> {Ok(())}+#[test]+fn cli_load_follows_five_same_url_replacements_after_403() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-five-navigations");+ let output = run_fetch_cli_with_wait_until(&url, "load")?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "moli fetch failed: stdout={}\nstderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ assert!(+ stdout.contains("id=\"http-error-five-navigation-target\""),+ "stdout={stdout}"+ );+ assert!(+ stdout.contains("id=\"http-error-five-navigation-load-tail\""),+ "the fifth replacement must still reach Load: stdout={stdout}"+ );+ assert!(!stdout.contains("id=\"challenge\""), "stdout={stdout}");+ Ok(())+}+#[test]fn cli_load_keeps_the_successor_stage_timeout_after_403_navigation() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;@@ -2233,6 +2260,49 @@ fn cli_redirect_time_can_extend_http_error_navigation_grace() -> Result<()> {Ok(())}+#[test]+fn cli_zero_redirect_time_accepts_navigation_already_pending_at_stage() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-immediate-navigation");+ let output = run_fetch_cli_with_dump_and_args(&url, "html", &["--redirect-time", "0"])?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "a replacement already pending at the 403 Load stage must not need a grace timer: stdout={}\nstderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ assert!(+ stdout.contains("id=\"http-error-navigation-target\""),+ "stdout={stdout}"+ );+ assert!(+ stdout.contains("id=\"http-error-navigation-load-tail\""),+ "stdout={stdout}"+ );+ Ok(())+}++#[test]+fn cli_zero_redirect_time_rejects_navigation_that_starts_later() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-navigation");+ let output = run_fetch_cli_with_dump_and_args(&url, "html", &["--redirect-time", "0"])?;+ runtime.block_on(server.shutdown());++ assert!(!output.status.success());+ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(stdout.is_empty(), "stdout={stdout}");+ assert!(stderr.contains("403 Forbidden"), "stderr={stderr}");+ assert!(stderr.contains("0 ms grace period"), "stderr={stderr}");+ Ok(())+}+#[test]fn cli_explicit_load_rejects_http_error_navigation_to_another_error() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;