Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(cli): apply redirect wait to terminal 3xx pages

ldm0cb287fb651f8
diff --git a/moli-test-support/src/routes_wait.rs b/moli-test-support/src/routes_wait.rs
index b3d08a3c7..f0bffdf24 100644
--- a/moli-test-support/src/routes_wait.rs
+++ b/moli-test-support/src/routes_wait.rs
@@ -71,6 +71,36 @@ const WAIT_UNTIL_INTERVAL_FETCH_HTML: &str = "<!doctype html><html><body data-st
const WAIT_UNTIL_INTERVAL_DOM_MUTATION_HTML: &str = "<!doctype html><html><body data-state=\"init\"><main id=\"mutation-count\">0</main><script>window.addEventListener('load', () => { setInterval(() => { const count = Number(document.body.getAttribute('data-mutation-count') || '0') + 1; document.body.setAttribute('data-mutation-count', String(count)); document.getElementById('mutation-count').textContent = String(count); }, 50); });</script></body></html>";
const WAIT_UNTIL_SLOW_STATIC_HTML: &str =
"<!doctype html><html><body><main id=\"slow-main\">slow-main=ready</main></body></html>";
+const WAIT_UNTIL_REDIRECT_STATUS_NAVIGATION_HTML: &str = r#"<!doctype html>
+<html><head><title>terminal 302 navigation</title></head><body>
+<main id="redirect-status-challenge">redirect-status=challenge</main>
+<script>
+window.addEventListener('load', () => {
+ setTimeout(() => {
+ document.cookie = 'moli-redirect-status-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax';
+ location.reload();
+ }, 75);
+});
+</script>
+</body></html>"#;
+const WAIT_UNTIL_REDIRECT_STATUS_NAVIGATION_FINAL_HTML: &str = r#"<!doctype html>
+<html><head><title>terminal 302 navigation passed</title></head><body>
+<main id="redirect-status-navigation-target">redirect-status=navigation-done</main>
+</body></html>"#;
+const WAIT_UNTIL_REDIRECT_STATUS_NO_NAVIGATION_HTML: &str = r#"<!doctype html>
+<html><head><title>terminal 302</title></head><body>
+<main id="terminal-redirect-status">redirect-status=terminal</main>
+<script>
+window.addEventListener('load', () => {
+ setTimeout(() => {
+ const marker = document.createElement('main');
+ marker.id = 'redirect-status-wait-marker';
+ marker.textContent = 'redirect-status=waited';
+ document.body.appendChild(marker);
+ }, 300);
+});
+</script>
+</body></html>"#;
const WAIT_UNTIL_HTTP_ERROR_NAVIGATION_CHALLENGE_HTML: &str = "<!doctype html><html><head><title>403 challenge</title></head><body><main id=\"challenge\">http-error-navigation=challenge</main><script>window.addEventListener('load', () => { setTimeout(() => { document.cookie = 'moli-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax'; location.reload(); }, 75); });</script></body></html>";
const WAIT_UNTIL_HTTP_ERROR_LATE_NAVIGATION_CHALLENGE_HTML: &str = "<!doctype html><html><head><title>late 403 challenge</title></head><body><main id=\"challenge\">http-error-navigation=late-challenge</main><script>window.addEventListener('load', () => { setTimeout(() => { document.cookie = 'moli-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax'; location.reload(); }, 1200); });</script></body></html>";
const WAIT_UNTIL_READINESS_HTTP_ERROR_CHALLENGE_HTML: &str = r#"<!doctype html>
@@ -192,6 +222,19 @@ window.addEventListener('load', () => {
</script></head><body><main id="slow-http-error-head">slow-500-head</main>"#;
const WAIT_UNTIL_SLOW_STREAMING_500_TAIL: &[u8] =
br#"<main id="slow-http-error-tail">slow-500-tail</main></body></html>"#;
+const WAIT_UNTIL_SLOW_STREAMING_302_HEAD: &[u8] = br#"<!doctype html>
+<html><head><title>slow streaming 302</title><script>
+window.addEventListener('load', () => {
+ setTimeout(() => {
+ const marker = document.createElement('main');
+ marker.id = 'slow-redirect-status-post-load';
+ marker.textContent = 'slow-redirect-status=post-load';
+ document.body.appendChild(marker);
+ }, 400);
+});
+</script></head><body><main id="slow-redirect-status-head">slow-302-head</main>"#;
+const WAIT_UNTIL_SLOW_STREAMING_302_TAIL: &[u8] =
+ br#"<main id="slow-redirect-status-tail">slow-302-tail</main></body></html>"#;
const WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_HEAD: &[u8] = br#"<!doctype html>
<html><head><title>slow streaming 403 navigation</title><script>
window.addEventListener('load', () => {
@@ -208,7 +251,7 @@ const WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_FINAL_HTML: &str = r#"<!doctype h
<main id="slow-http-error-navigation-target">slow-http-error-navigation=done</main>
</body></html>"#;
-const SLOW_HTTP_ERROR_STREAM_TAIL_DELAY: Duration = Duration::from_millis(1_250);
+const SLOW_STATUS_STREAM_TAIL_DELAY: Duration = Duration::from_millis(1_250);
pub(super) fn add_wait_routes(router: Router) -> Router {
router
@@ -299,6 +342,18 @@ pub(super) fn add_wait_routes(router: Router) -> Router {
"/wait-until-slow-interval-dom-mutation",
get(wait_until_slow_interval_dom_mutation_page),
)
+ .route(
+ "/wait-until-redirect-status-navigation",
+ get(wait_until_redirect_status_navigation_page),
+ )
+ .route(
+ "/wait-until-redirect-status-no-navigation",
+ get(wait_until_redirect_status_no_navigation_page),
+ )
+ .route(
+ "/wait-until-redirect-status-slow-streaming-302",
+ get(wait_until_redirect_status_slow_streaming_302_page),
+ )
.route(
"/wait-until-http-error-navigation",
get(wait_until_http_error_navigation_page),
@@ -362,6 +417,35 @@ async fn wait_until_lifecycle_page() -> Html<&'static str> {
Html(WAIT_UNTIL_LIFECYCLE_HTML)
}
+async fn wait_until_redirect_status_navigation_page(headers: HeaderMap) -> Response {
+ if has_cookie(&headers, "moli-redirect-status-navigation=passed") {
+ return Html(WAIT_UNTIL_REDIRECT_STATUS_NAVIGATION_FINAL_HTML).into_response();
+ }
+
+ (
+ StatusCode::FOUND,
+ Html(WAIT_UNTIL_REDIRECT_STATUS_NAVIGATION_HTML),
+ )
+ .into_response()
+}
+
+async fn wait_until_redirect_status_no_navigation_page() -> Response {
+ (
+ StatusCode::FOUND,
+ Html(WAIT_UNTIL_REDIRECT_STATUS_NO_NAVIGATION_HTML),
+ )
+ .into_response()
+}
+
+async fn wait_until_redirect_status_slow_streaming_302_page() -> Response {
+ slow_status_streaming_response(
+ StatusCode::FOUND,
+ "text/html; charset=utf-8",
+ WAIT_UNTIL_SLOW_STREAMING_302_HEAD,
+ WAIT_UNTIL_SLOW_STREAMING_302_TAIL,
+ )
+}
+
async fn wait_until_http_error_navigation_page(headers: HeaderMap) -> Response {
if has_cookie(&headers, "moli-http-error-navigation=passed") {
return Html(WAIT_UNTIL_HTTP_ERROR_NAVIGATION_FINAL_HTML).into_response();
@@ -394,7 +478,7 @@ async fn wait_until_http_error_navigation_to_slow_streaming_500_page(
headers: HeaderMap,
) -> Response {
if has_cookie(&headers, "moli-http-error-navigation=passed") {
- return slow_http_error_streaming_response(
+ return slow_status_streaming_response(
StatusCode::INTERNAL_SERVER_ERROR,
"text/html; charset=utf-8",
WAIT_UNTIL_SLOW_STREAMING_500_HEAD,
@@ -483,7 +567,7 @@ async fn wait_until_http_error_late_navigation_page(headers: HeaderMap) -> Respo
.into_response()
}
-fn slow_http_error_streaming_response(
+fn slow_status_streaming_response(
status: StatusCode,
content_type: &'static str,
head: &'static [u8],
@@ -500,7 +584,7 @@ fn slow_http_error_streaming_response(
{
return;
}
- sleep(SLOW_HTTP_ERROR_STREAM_TAIL_DELAY).await;
+ sleep(SLOW_STATUS_STREAM_TAIL_DELAY).await;
let _ = tx
.send(Ok::<Bytes, std::convert::Infallible>(Bytes::from_static(
tail,
@@ -514,11 +598,11 @@ fn slow_http_error_streaming_response(
.body(Body::from_stream(
tokio_stream::wrappers::ReceiverStream::new(rx),
))
- .expect("slow HTTP error streaming response should build")
+ .expect("slow status streaming response should build")
}
async fn wait_until_http_error_slow_streaming_500_page() -> Response {
- slow_http_error_streaming_response(
+ slow_status_streaming_response(
StatusCode::INTERNAL_SERVER_ERROR,
"text/html; charset=utf-8",
WAIT_UNTIL_SLOW_STREAMING_500_HEAD,
@@ -531,7 +615,7 @@ async fn wait_until_http_error_slow_streaming_navigation_page(headers: HeaderMap
return Html(WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_FINAL_HTML).into_response();
}
- slow_http_error_streaming_response(
+ slow_status_streaming_response(
StatusCode::FORBIDDEN,
"text/html; charset=utf-8",
WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_HEAD,
@@ -540,7 +624,7 @@ async fn wait_until_http_error_slow_streaming_navigation_page(headers: HeaderMap
}
async fn wait_until_http_error_slow_streaming_raw_404_page() -> Response {
- slow_http_error_streaming_response(
+ slow_status_streaming_response(
StatusCode::NOT_FOUND,
"application/octet-stream",
b"slow-raw-404-head|",
diff --git a/moli/src/app.rs b/moli/src/app.rs
index 6a327a4ef..094a774d3 100644
--- a/moli/src/app.rs
+++ b/moli/src/app.rs
@@ -1,7 +1,7 @@
//! Callable command runner for the Moli CLI.
-mod http_error_navigation;
mod readiness;
+mod redirect_navigation;
use std::{io::Write, sync::Arc};
diff --git a/moli/src/app/readiness.rs b/moli/src/app/readiness.rs
index 47fee6549..167e0729f 100644
--- a/moli/src/app/readiness.rs
+++ b/moli/src/app/readiness.rs
@@ -7,7 +7,7 @@
//! retained by the Page, so starting that wait after lifecycle completion does
//! not lose an early matching response.
-use super::http_error_navigation::fetch_with_http_error_navigation;
+use super::redirect_navigation::fetch_with_redirect_wait;
use crate::cli::{FetchArgs, FetchWaitUntil};
use anyhow::{Context, Result, anyhow, bail};
use moli_core::{
@@ -57,7 +57,7 @@ impl ReadinessPlan {
RenderedDomWaitUntil::DomContentLoaded
| RenderedDomWaitUntil::Load
| RenderedDomWaitUntil::Done => {
- fetch_with_http_error_navigation(
+ fetch_with_redirect_wait(
browser,
request,
self.wait_until,
diff --git a/moli/src/app/http_error_navigation.rs b/moli/src/app/redirect_navigation.rs
similarity index 67%
rename from moli/src/app/http_error_navigation.rs
rename to moli/src/app/redirect_navigation.rs
index efbf75c90..73326a6c4 100644
--- a/moli/src/app/http_error_navigation.rs
+++ b/moli/src/app/redirect_navigation.rs
@@ -1,4 +1,4 @@
-//! CLI policy for replacing an HTTP error Document with its next navigation.
+//! CLI policy for replacing a non-success Document with its next navigation.
//!
//! HTTP status interpretation and timeout configuration belong to the CLI
//! layer. The renderer receives only the generic lifecycle-target decision.
@@ -10,11 +10,11 @@ use moli_core::runtime::{
use moli_fetch::Request;
use std::time::{Duration, Instant};
-pub(super) fn is_http_error_status(status: u16) -> bool {
- (400..=599).contains(&status)
+pub(super) fn uses_redirect_wait(status: u16) -> bool {
+ (300..=599).contains(&status)
}
-pub(super) async fn fetch_with_http_error_navigation(
+pub(super) async fn fetch_with_redirect_wait(
browser: &Browser,
request: Request,
wait_until: RenderedDomWaitUntil,
@@ -23,20 +23,20 @@ pub(super) async fn fetch_with_http_error_navigation(
) -> Result<FetchedDocument> {
let minimum_navigation_deadline = Instant::now()
.checked_add(minimum_navigation_wait)
- .context("HTTP error replacement-navigation wait exceeds the supported range")?;
+ .context("response replacement-navigation wait exceeds the supported range")?;
// The first DCL/load is delivered to this synchronous decision normally.
- // A 4xx/5xx keeps running until the configured minimum time from fetch
- // start has elapsed, giving client-side challenges a chance to replace the
- // error Document. The initial lifecycle load therefore consumes this
- // window instead of receiving a fresh grace period afterward. The outer
- // readiness deadline still caps this wait and any successor lifecycle.
+ // A 3xx/4xx/5xx Document keeps running until the configured minimum time
+ // from fetch start has elapsed, giving client-side responses a chance to
+ // replace it. The initial lifecycle load therefore consumes this window
+ // instead of receiving a fresh grace period afterward. The outer readiness
+ // deadline still caps this wait and any successor lifecycle.
browser
.fetch_document_with_lifecycle_decider_and_deadline(
request,
wait_until,
deadline,
move |target| {
- Ok(if is_http_error_status(target.status) {
+ Ok(if uses_redirect_wait(target.status) {
RendererLifecycleDecision::FollowNextDocumentOrFinish {
navigation_grace_ms: remaining_wait_milliseconds(
minimum_navigation_deadline,
@@ -61,17 +61,20 @@ fn remaining_wait_milliseconds(deadline: Instant, now: Instant) -> u64 {
#[cfg(test)]
mod tests {
- use super::{is_http_error_status, remaining_wait_milliseconds};
+ use super::{remaining_wait_milliseconds, uses_redirect_wait};
use std::time::{Duration, Instant};
#[test]
- fn http_error_status_covers_only_four_hundred_and_five_hundred_ranges() {
- assert!(!is_http_error_status(399));
- assert!(is_http_error_status(400));
- assert!(is_http_error_status(499));
- assert!(is_http_error_status(500));
- assert!(is_http_error_status(599));
- assert!(!is_http_error_status(600));
+ fn redirect_wait_status_covers_three_hundred_through_five_hundred_ranges() {
+ assert!(!uses_redirect_wait(299));
+ assert!(uses_redirect_wait(300));
+ assert!(uses_redirect_wait(304));
+ assert!(uses_redirect_wait(399));
+ assert!(uses_redirect_wait(400));
+ assert!(uses_redirect_wait(499));
+ assert!(uses_redirect_wait(500));
+ assert!(uses_redirect_wait(599));
+ assert!(!uses_redirect_wait(600));
}
#[test]
diff --git a/moli/src/cli.rs b/moli/src/cli.rs
index d5d93d16f..a5d5ae3a8 100644
--- a/moli/src/cli.rs
+++ b/moli/src/cli.rs
@@ -68,12 +68,12 @@ pub struct FetchArgs {
#[arg(long, value_enum, default_value = "done")]
pub wait_until: FetchWaitUntil,
- /// Minimum milliseconds from fetch start to allow an executable 4xx/5xx
+ /// Minimum milliseconds from fetch start to allow an executable 3xx-5xx
/// Document to begin a client-side replacement navigation. Time spent
/// reaching the selected lifecycle stage counts toward this minimum. A
/// value of 0 disables additional waiting but still accepts a navigation
/// already pending when the stage result is inspected. If no replacement
- /// starts in time, the current error response is returned with its body.
+ /// starts in time, the current response is returned with its body.
/// This only applies to `domcontentloaded`, `load`, and `done` waits.
#[arg(
long,
@@ -681,6 +681,7 @@ mod tests {
assert!(help.contains("--redirect-wait-ms <MILLISECONDS>"));
assert!(!help.contains("--redirect-time"));
assert!(help.contains("Minimum milliseconds from fetch start"));
+ assert!(help.contains("executable 3xx-5xx"));
assert!(help.contains("Maximum total readiness time in milliseconds"));
assert!(help.contains("response match, selector, and script waits share one absolute"));
}
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index 92d9c9f01..99b14b27b 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -2493,6 +2493,154 @@ fn cli_load_waits_for_slow_post_domcontentloaded_runtime_script() -> Result<()>
Ok(())
}
+#[test]
+fn cli_default_redirect_wait_follows_client_navigation_from_terminal_302() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-redirect-status-navigation");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 200);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"redirect-status-navigation-target\""),
+ "stdout={stdout}"
+ );
+ assert!(
+ !html.contains("id=\"redirect-status-challenge\""),
+ "the successor Document must replace the terminal 302 body: stdout={stdout}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_terminal_302_without_navigation_waits_then_returns_its_body() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-redirect-status-no-navigation");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 302);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"terminal-redirect-status\""),
+ "stdout={stdout}"
+ );
+ assert!(
+ html.contains("id=\"redirect-status-wait-marker\""),
+ "the default redirect wait must keep the terminal 302 Page live: stdout={stdout}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_zero_redirect_wait_returns_terminal_302_before_delayed_page_work() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-redirect-status-no-navigation");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &["--redirect-wait-ms", "0"])?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 302);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"terminal-redirect-status\""),
+ "stdout={stdout}"
+ );
+ assert!(
+ !html.contains("id=\"redirect-status-wait-marker\""),
+ "a zero redirect wait must not run delayed Page work: stdout={stdout}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_slow_streaming_terminal_302_does_not_add_a_post_load_wait() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-redirect-status-slow-streaming-302");
+
+ for wait_until in ["domcontentloaded", "load", "done"] {
+ let output = run_fetch_cli_with_wait_until_and_dump(&url, wait_until, "json")?;
+ assert!(
+ output.status.success(),
+ "wait_until={wait_until} stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 302);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"slow-redirect-status-head\""),
+ "wait_until={wait_until} stdout={stdout}"
+ );
+ assert!(
+ html.contains("id=\"slow-redirect-status-tail\""),
+ "the delayed terminal 302 body must be complete: wait_until={wait_until} stdout={stdout}"
+ );
+ assert!(
+ !html.contains("id=\"slow-redirect-status-post-load\""),
+ "time spent loading the body must consume the minimum redirect wait: wait_until={wait_until} stdout={stdout}"
+ );
+ }
+
+ runtime.block_on(server.shutdown());
+ Ok(())
+}
+
+#[test]
+fn cli_http_redirect_with_location_does_not_consume_redirect_wait() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/redirect");
+ let final_url = server.url("/static");
+ let output = run_fetch_cli_with_dump_and_args(
+ &url,
+ "json",
+ &["--timeout", "4000", "--redirect-wait-ms", "10000"],
+ )?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "an HTTP redirect with Location must finish at the network destination without a renderer grace: stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &final_url, 200);
+ Ok(())
+}
+
#[test]
fn cli_domcontentloaded_recovers_from_delayed_403_navigation_at_same_stage() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;