Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(cli): apply redirect wait to terminal 3xx pages
ldm0cb287fb651f8
diff --git a/moli-test-support/src/routes_wait.rs b/moli-test-support/src/routes_wait.rsindex b3d08a3c7..f0bffdf24 100644--- a/moli-test-support/src/routes_wait.rs+++ b/moli-test-support/src/routes_wait.rs@@ -71,6 +71,36 @@ const WAIT_UNTIL_INTERVAL_FETCH_HTML: &str = "<!doctype html><html><body data-stconst WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML: &str = "<!doctype html><html><body data-state=\"init\"><main id=\"mutation-count\">0</main><script>window.addEventListener('load', () => { setInterval(() => { const count = Number(document.body.getAttribute('data-mutation-count') || '0') + 1; document.body.setAttribute('data-mutation-count', String(count)); document.getElementById('mutation-count').textContent = String(count); }, 50); });</script></body></html>";const WAIT_UNTIL_SLOW_STATIC_HTML: &str ="<!doctype html><html><body><main id=\"slow-main\">slow-main=ready</main></body></html>";+const WAIT_UNTIL_REDIRECT_STATUS_NAVIGATION_HTML: &str = r#"<!doctype html>+<html><head><title>terminal 302 navigation</title></head><body>+<main id="redirect-status-challenge">redirect-status=challenge</main>+<script>+window.addEventListener('load', () => {+ setTimeout(() => {+ document.cookie = 'moli-redirect-status-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax';+ location.reload();+ }, 75);+});+</script>+</body></html>"#;+const WAIT_UNTIL_REDIRECT_STATUS_NAVIGATION_FINAL_HTML: &str = r#"<!doctype html>+<html><head><title>terminal 302 navigation passed</title></head><body>+<main id="redirect-status-navigation-target">redirect-status=navigation-done</main>+</body></html>"#;+const WAIT_UNTIL_REDIRECT_STATUS_NO_NAVIGATION_HTML: &str = r#"<!doctype html>+<html><head><title>terminal 302</title></head><body>+<main id="terminal-redirect-status">redirect-status=terminal</main>+<script>+window.addEventListener('load', () => {+ setTimeout(() => {+ const marker = document.createElement('main');+ marker.id = 'redirect-status-wait-marker';+ marker.textContent = 'redirect-status=waited';+ document.body.appendChild(marker);+ }, 300);+});+</script>+</body></html>"#;const WAIT_UNTIL_HTTP_ERROR_NAVIGATION_CHALLENGE_HTML: &str = "<!doctype html><html><head><title>403 challenge</title></head><body><main id=\"challenge\">http-error-navigation=challenge</main><script>window.addEventListener('load', () => { setTimeout(() => { document.cookie = 'moli-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax'; location.reload(); }, 75); });</script></body></html>";const WAIT_UNTIL_HTTP_ERROR_LATE_NAVIGATION_CHALLENGE_HTML: &str = "<!doctype html><html><head><title>late 403 challenge</title></head><body><main id=\"challenge\">http-error-navigation=late-challenge</main><script>window.addEventListener('load', () => { setTimeout(() => { document.cookie = 'moli-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax'; location.reload(); }, 1200); });</script></body></html>";const WAIT_UNTIL_READINESS_HTTP_ERROR_CHALLENGE_HTML: &str = r#"<!doctype html>@@ -192,6 +222,19 @@ window.addEventListener('load', () => {</script></head><body><main id="slow-http-error-head">slow-500-head</main>"#;const WAIT_UNTIL_SLOW_STREAMING_500_TAIL: &[u8] =br#"<main id="slow-http-error-tail">slow-500-tail</main></body></html>"#;+const WAIT_UNTIL_SLOW_STREAMING_302_HEAD: &[u8] = br#"<!doctype html>+<html><head><title>slow streaming 302</title><script>+window.addEventListener('load', () => {+ setTimeout(() => {+ const marker = document.createElement('main');+ marker.id = 'slow-redirect-status-post-load';+ marker.textContent = 'slow-redirect-status=post-load';+ document.body.appendChild(marker);+ }, 400);+});+</script></head><body><main id="slow-redirect-status-head">slow-302-head</main>"#;+const WAIT_UNTIL_SLOW_STREAMING_302_TAIL: &[u8] =+ br#"<main id="slow-redirect-status-tail">slow-302-tail</main></body></html>"#;const WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_HEAD: &[u8] = br#"<!doctype html><html><head><title>slow streaming 403 navigation</title><script>window.addEventListener('load', () => {@@ -208,7 +251,7 @@ const WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_FINAL_HTML: &str = r#"<!doctype h<main id="slow-http-error-navigation-target">slow-http-error-navigation=done</main></body></html>"#;-const SLOW_HTTP_ERROR_STREAM_TAIL_DELAY: Duration = Duration::from_millis(1_250);+const SLOW_STATUS_STREAM_TAIL_DELAY: Duration = Duration::from_millis(1_250);pub(super) fn add_wait_routes(router: Router) -> Router {router@@ -299,6 +342,18 @@ pub(super) fn add_wait_routes(router: Router) -> Router {"/wait-until-slow-interval-dom-mutation",get(wait_until_slow_interval_dom_mutation_page),)+ .route(+ "/wait-until-redirect-status-navigation",+ get(wait_until_redirect_status_navigation_page),+ )+ .route(+ "/wait-until-redirect-status-no-navigation",+ get(wait_until_redirect_status_no_navigation_page),+ )+ .route(+ "/wait-until-redirect-status-slow-streaming-302",+ get(wait_until_redirect_status_slow_streaming_302_page),+ ).route("/wait-until-http-error-navigation",get(wait_until_http_error_navigation_page),@@ -362,6 +417,35 @@ async fn wait_until_lifecycle_page() -> Html<&'static str> {Html(WAIT_UNTIL_LIFECYCLE_HTML)}+async fn wait_until_redirect_status_navigation_page(headers: HeaderMap) -> Response {+ if has_cookie(&headers, "moli-redirect-status-navigation=passed") {+ return Html(WAIT_UNTIL_REDIRECT_STATUS_NAVIGATION_FINAL_HTML).into_response();+ }++ (+ StatusCode::FOUND,+ Html(WAIT_UNTIL_REDIRECT_STATUS_NAVIGATION_HTML),+ )+ .into_response()+}++async fn wait_until_redirect_status_no_navigation_page() -> Response {+ (+ StatusCode::FOUND,+ Html(WAIT_UNTIL_REDIRECT_STATUS_NO_NAVIGATION_HTML),+ )+ .into_response()+}++async fn wait_until_redirect_status_slow_streaming_302_page() -> Response {+ slow_status_streaming_response(+ StatusCode::FOUND,+ "text/html; charset=utf-8",+ WAIT_UNTIL_SLOW_STREAMING_302_HEAD,+ WAIT_UNTIL_SLOW_STREAMING_302_TAIL,+ )+}+async fn wait_until_http_error_navigation_page(headers: HeaderMap) -> Response {if has_cookie(&headers, "moli-http-error-navigation=passed") {return Html(WAIT_UNTIL_HTTP_ERROR_NAVIGATION_FINAL_HTML).into_response();@@ -394,7 +478,7 @@ async fn wait_until_http_error_navigation_to_slow_streaming_500_page(headers: HeaderMap,) -> Response {if has_cookie(&headers, "moli-http-error-navigation=passed") {- return slow_http_error_streaming_response(+ return slow_status_streaming_response(StatusCode::INTERNAL_SERVER_ERROR,"text/html; charset=utf-8",WAIT_UNTIL_SLOW_STREAMING_500_HEAD,@@ -483,7 +567,7 @@ async fn wait_until_http_error_late_navigation_page(headers: HeaderMap) -> Respo.into_response()}-fn slow_http_error_streaming_response(+fn slow_status_streaming_response(status: StatusCode,content_type: &'static str,head: &'static [u8],@@ -500,7 +584,7 @@ fn slow_http_error_streaming_response({return;}- sleep(SLOW_HTTP_ERROR_STREAM_TAIL_DELAY).await;+ sleep(SLOW_STATUS_STREAM_TAIL_DELAY).await;let _ = tx.send(Ok::<Bytes, std::convert::Infallible>(Bytes::from_static(tail,@@ -514,11 +598,11 @@ fn slow_http_error_streaming_response(.body(Body::from_stream(tokio_stream::wrappers::ReceiverStream::new(rx),))- .expect("slow HTTP error streaming response should build")+ .expect("slow status streaming response should build")}async fn wait_until_http_error_slow_streaming_500_page() -> Response {- slow_http_error_streaming_response(+ slow_status_streaming_response(StatusCode::INTERNAL_SERVER_ERROR,"text/html; charset=utf-8",WAIT_UNTIL_SLOW_STREAMING_500_HEAD,@@ -531,7 +615,7 @@ async fn wait_until_http_error_slow_streaming_navigation_page(headers: HeaderMapreturn Html(WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_FINAL_HTML).into_response();}- slow_http_error_streaming_response(+ slow_status_streaming_response(StatusCode::FORBIDDEN,"text/html; charset=utf-8",WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_HEAD,@@ -540,7 +624,7 @@ async fn wait_until_http_error_slow_streaming_navigation_page(headers: HeaderMap}async fn wait_until_http_error_slow_streaming_raw_404_page() -> Response {- slow_http_error_streaming_response(+ slow_status_streaming_response(StatusCode::NOT_FOUND,"application/octet-stream",b"slow-raw-404-head|",diff --git a/moli/src/app.rs b/moli/src/app.rsindex 6a327a4ef..094a774d3 100644--- a/moli/src/app.rs+++ b/moli/src/app.rs@@ -1,7 +1,7 @@//! Callable command runner for the Moli CLI.-mod http_error_navigation;mod readiness;+mod redirect_navigation;use std::{io::Write, sync::Arc};diff --git a/moli/src/app/readiness.rs b/moli/src/app/readiness.rsindex 47fee6549..167e0729f 100644--- a/moli/src/app/readiness.rs+++ b/moli/src/app/readiness.rs@@ -7,7 +7,7 @@//! retained by the Page, so starting that wait after lifecycle completion does//! not lose an early matching response.-use super::http_error_navigation::fetch_with_http_error_navigation;+use super::redirect_navigation::fetch_with_redirect_wait;use crate::cli::{FetchArgs, FetchWaitUntil};use anyhow::{Context, Result, anyhow, bail};use moli_core::{@@ -57,7 +57,7 @@ impl ReadinessPlan {RenderedDomWaitUntil::DomContentLoaded| RenderedDomWaitUntil::Load| RenderedDomWaitUntil::Done => {- fetch_with_http_error_navigation(+ fetch_with_redirect_wait(browser,request,self.wait_until,diff --git a/moli/src/app/http_error_navigation.rs b/moli/src/app/redirect_navigation.rssimilarity index 67%rename from moli/src/app/http_error_navigation.rsrename to moli/src/app/redirect_navigation.rsindex efbf75c90..73326a6c4 100644--- a/moli/src/app/http_error_navigation.rs+++ b/moli/src/app/redirect_navigation.rs@@ -1,4 +1,4 @@-//! CLI policy for replacing an HTTP error Document with its next navigation.+//! CLI policy for replacing a non-success Document with its next navigation.//!//! HTTP status interpretation and timeout configuration belong to the CLI//! layer. The renderer receives only the generic lifecycle-target decision.@@ -10,11 +10,11 @@ use moli_core::runtime::{use moli_fetch::Request;use std::time::{Duration, Instant};-pub(super) fn is_http_error_status(status: u16) -> bool {- (400..=599).contains(&status)+pub(super) fn uses_redirect_wait(status: u16) -> bool {+ (300..=599).contains(&status)}-pub(super) async fn fetch_with_http_error_navigation(+pub(super) async fn fetch_with_redirect_wait(browser: &Browser,request: Request,wait_until: RenderedDomWaitUntil,@@ -23,20 +23,20 @@ pub(super) async fn fetch_with_http_error_navigation() -> Result<FetchedDocument> {let minimum_navigation_deadline = Instant::now().checked_add(minimum_navigation_wait)- .context("HTTP error replacement-navigation wait exceeds the supported range")?;+ .context("response replacement-navigation wait exceeds the supported range")?;// The first DCL/load is delivered to this synchronous decision normally.- // A 4xx/5xx keeps running until the configured minimum time from fetch- // start has elapsed, giving client-side challenges a chance to replace the- // error Document. The initial lifecycle load therefore consumes this- // window instead of receiving a fresh grace period afterward. The outer- // readiness deadline still caps this wait and any successor lifecycle.+ // A 3xx/4xx/5xx Document keeps running until the configured minimum time+ // from fetch start has elapsed, giving client-side responses a chance to+ // replace it. The initial lifecycle load therefore consumes this window+ // instead of receiving a fresh grace period afterward. The outer readiness+ // deadline still caps this wait and any successor lifecycle.browser.fetch_document_with_lifecycle_decider_and_deadline(request,wait_until,deadline,move |target| {- Ok(if is_http_error_status(target.status) {+ Ok(if uses_redirect_wait(target.status) {RendererLifecycleDecision::FollowNextDocumentOrFinish {navigation_grace_ms: remaining_wait_milliseconds(minimum_navigation_deadline,@@ -61,17 +61,20 @@ fn remaining_wait_milliseconds(deadline: Instant, now: Instant) -> u64 {#[cfg(test)]mod tests {- use super::{is_http_error_status, remaining_wait_milliseconds};+ use super::{remaining_wait_milliseconds, uses_redirect_wait};use std::time::{Duration, Instant};#[test]- fn http_error_status_covers_only_four_hundred_and_five_hundred_ranges() {- assert!(!is_http_error_status(399));- assert!(is_http_error_status(400));- assert!(is_http_error_status(499));- assert!(is_http_error_status(500));- assert!(is_http_error_status(599));- assert!(!is_http_error_status(600));+ fn redirect_wait_status_covers_three_hundred_through_five_hundred_ranges() {+ assert!(!uses_redirect_wait(299));+ assert!(uses_redirect_wait(300));+ assert!(uses_redirect_wait(304));+ assert!(uses_redirect_wait(399));+ assert!(uses_redirect_wait(400));+ assert!(uses_redirect_wait(499));+ assert!(uses_redirect_wait(500));+ assert!(uses_redirect_wait(599));+ assert!(!uses_redirect_wait(600));}#[test]diff --git a/moli/src/cli.rs b/moli/src/cli.rsindex d5d93d16f..a5d5ae3a8 100644--- a/moli/src/cli.rs+++ b/moli/src/cli.rs@@ -68,12 +68,12 @@ pub struct FetchArgs {#[arg(long, value_enum, default_value = "done")]pub wait_until: FetchWaitUntil,- /// Minimum milliseconds from fetch start to allow an executable 4xx/5xx+ /// Minimum milliseconds from fetch start to allow an executable 3xx-5xx/// Document to begin a client-side replacement navigation. Time spent/// reaching the selected lifecycle stage counts toward this minimum. A/// value of 0 disables additional waiting but still accepts a navigation/// already pending when the stage result is inspected. If no replacement- /// starts in time, the current error response is returned with its body.+ /// starts in time, the current response is returned with its body./// This only applies to `domcontentloaded`, `load`, and `done` waits.#[arg(long,@@ -681,6 +681,7 @@ mod tests {assert!(help.contains("--redirect-wait-ms <MILLISECONDS>"));assert!(!help.contains("--redirect-time"));assert!(help.contains("Minimum milliseconds from fetch start"));+ assert!(help.contains("executable 3xx-5xx"));assert!(help.contains("Maximum total readiness time in milliseconds"));assert!(help.contains("response match, selector, and script waits share one absolute"));}diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex 92d9c9f01..99b14b27b 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -2493,6 +2493,154 @@ fn cli_load_waits_for_slow_post_domcontentloaded_runtime_script() -> Result<()>Ok(())}+#[test]+fn cli_default_redirect_wait_follows_client_navigation_from_terminal_302() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-redirect-status-navigation");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 200);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"redirect-status-navigation-target\""),+ "stdout={stdout}"+ );+ assert!(+ !html.contains("id=\"redirect-status-challenge\""),+ "the successor Document must replace the terminal 302 body: stdout={stdout}"+ );+ Ok(())+}++#[test]+fn cli_terminal_302_without_navigation_waits_then_returns_its_body() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-redirect-status-no-navigation");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 302);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"terminal-redirect-status\""),+ "stdout={stdout}"+ );+ assert!(+ html.contains("id=\"redirect-status-wait-marker\""),+ "the default redirect wait must keep the terminal 302 Page live: stdout={stdout}"+ );+ Ok(())+}++#[test]+fn cli_zero_redirect_wait_returns_terminal_302_before_delayed_page_work() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-redirect-status-no-navigation");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &["--redirect-wait-ms", "0"])?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 302);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"terminal-redirect-status\""),+ "stdout={stdout}"+ );+ assert!(+ !html.contains("id=\"redirect-status-wait-marker\""),+ "a zero redirect wait must not run delayed Page work: stdout={stdout}"+ );+ Ok(())+}++#[test]+fn cli_slow_streaming_terminal_302_does_not_add_a_post_load_wait() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-redirect-status-slow-streaming-302");++ for wait_until in ["domcontentloaded", "load", "done"] {+ let output = run_fetch_cli_with_wait_until_and_dump(&url, wait_until, "json")?;+ assert!(+ output.status.success(),+ "wait_until={wait_until} stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 302);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"slow-redirect-status-head\""),+ "wait_until={wait_until} stdout={stdout}"+ );+ assert!(+ html.contains("id=\"slow-redirect-status-tail\""),+ "the delayed terminal 302 body must be complete: wait_until={wait_until} stdout={stdout}"+ );+ assert!(+ !html.contains("id=\"slow-redirect-status-post-load\""),+ "time spent loading the body must consume the minimum redirect wait: wait_until={wait_until} stdout={stdout}"+ );+ }++ runtime.block_on(server.shutdown());+ Ok(())+}++#[test]+fn cli_http_redirect_with_location_does_not_consume_redirect_wait() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/redirect");+ let final_url = server.url("/static");+ let output = run_fetch_cli_with_dump_and_args(+ &url,+ "json",+ &["--timeout", "4000", "--redirect-wait-ms", "10000"],+ )?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "an HTTP redirect with Location must finish at the network destination without a renderer grace: stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &final_url, 200);+ Ok(())+}+#[test]fn cli_domcontentloaded_recovers_from_delayed_403_navigation_at_same_stage() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;