Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(cli): preserve HTTP error bodies after readiness

ldm0c86a373bcc77
diff --git a/moli-test-support/src/routes_core.rs b/moli-test-support/src/routes_core.rs
index ce04e46b7..06420e23d 100644
--- a/moli-test-support/src/routes_core.rs
+++ b/moli-test-support/src/routes_core.rs
@@ -5803,6 +5803,15 @@ pub(super) async fn net_upstream_xhr_500_page() -> Response {
(StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response()
}
+pub(super) async fn net_upstream_xhr_binary_404_page() -> Response {
+ (
+ StatusCode::NOT_FOUND,
+ [(CONTENT_TYPE, "application/octet-stream")],
+ b"raw-error-body".to_vec(),
+ )
+ .into_response()
+}
+
pub(super) async fn net_upstream_xhr_binary_page() -> Response {
let bytes = [0u8, 0, 1, 2, 0, 0, 9];
([(CONTENT_TYPE, "application/octet-stream")], bytes.to_vec()).into_response()
diff --git a/moli-test-support/src/routes_wait.rs b/moli-test-support/src/routes_wait.rs
index 7eaec156e..b3d08a3c7 100644
--- a/moli-test-support/src/routes_wait.rs
+++ b/moli-test-support/src/routes_wait.rs
@@ -176,6 +176,39 @@ window.addEventListener('load', () => {
});
</script>
</body></html>"#;
+const WAIT_UNTIL_SLOW_STREAMING_500_HEAD: &[u8] = br#"<!doctype html>
+<html><head><title>slow streaming 500</title><script>
+const appendSlowErrorMarker = (id, text) => {
+ const marker = document.createElement('main');
+ marker.id = id;
+ marker.textContent = text;
+ document.body.appendChild(marker);
+};
+window.addEventListener('load', () => {
+ setTimeout(() => appendSlowErrorMarker('slow-http-error-post-load-early', 'post-load-early'), 400);
+ setTimeout(() => appendSlowErrorMarker('slow-http-error-post-load-late', 'post-load-late'), 1400);
+ setTimeout(() => appendSlowErrorMarker('slow-http-error-post-load-after-total-deadline', 'post-load-after-total-deadline'), 3000);
+});
+</script></head><body><main id="slow-http-error-head">slow-500-head</main>"#;
+const WAIT_UNTIL_SLOW_STREAMING_500_TAIL: &[u8] =
+ br#"<main id="slow-http-error-tail">slow-500-tail</main></body></html>"#;
+const WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_HEAD: &[u8] = br#"<!doctype html>
+<html><head><title>slow streaming 403 navigation</title><script>
+window.addEventListener('load', () => {
+ setTimeout(() => {
+ document.cookie = 'moli-slow-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax';
+ location.reload();
+ }, 300);
+});
+</script></head><body><main id="slow-http-error-navigation-head">slow-403-head</main>"#;
+const WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_TAIL: &[u8] =
+ br#"<main id="slow-http-error-navigation-tail">slow-403-tail</main></body></html>"#;
+const WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_FINAL_HTML: &str = r#"<!doctype html>
+<html><head><title>slow streaming challenge passed</title></head><body>
+<main id="slow-http-error-navigation-target">slow-http-error-navigation=done</main>
+</body></html>"#;
+
+const SLOW_HTTP_ERROR_STREAM_TAIL_DELAY: Duration = Duration::from_millis(1_250);
pub(super) fn add_wait_routes(router: Router) -> Router {
router
@@ -274,6 +307,10 @@ pub(super) fn add_wait_routes(router: Router) -> Router {
"/wait-until-http-error-navigation-to-error",
get(wait_until_http_error_navigation_to_error_page),
)
+ .route(
+ "/wait-until-http-error-navigation-to-slow-streaming-500",
+ get(wait_until_http_error_navigation_to_slow_streaming_500_page),
+ )
.route(
"/wait-until-http-error-immediate-navigation",
get(wait_until_http_error_immediate_navigation_page),
@@ -294,6 +331,18 @@ pub(super) fn add_wait_routes(router: Router) -> Router {
"/wait-until-http-error-late-navigation",
get(wait_until_http_error_late_navigation_page),
)
+ .route(
+ "/wait-until-http-error-slow-streaming-500",
+ get(wait_until_http_error_slow_streaming_500_page),
+ )
+ .route(
+ "/wait-until-http-error-slow-streaming-navigation",
+ get(wait_until_http_error_slow_streaming_navigation_page),
+ )
+ .route(
+ "/wait-until-http-error-slow-streaming-raw-404",
+ get(wait_until_http_error_slow_streaming_raw_404_page),
+ )
.route(
"/wait-until-readiness-http-error-navigation",
get(wait_until_readiness_http_error_navigation_page),
@@ -341,6 +390,25 @@ async fn wait_until_http_error_navigation_to_error_page(headers: HeaderMap) -> R
.into_response()
}
+async fn wait_until_http_error_navigation_to_slow_streaming_500_page(
+ headers: HeaderMap,
+) -> Response {
+ if has_cookie(&headers, "moli-http-error-navigation=passed") {
+ return slow_http_error_streaming_response(
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "text/html; charset=utf-8",
+ WAIT_UNTIL_SLOW_STREAMING_500_HEAD,
+ WAIT_UNTIL_SLOW_STREAMING_500_TAIL,
+ );
+ }
+
+ (
+ StatusCode::FORBIDDEN,
+ Html(WAIT_UNTIL_HTTP_ERROR_NAVIGATION_CHALLENGE_HTML),
+ )
+ .into_response()
+}
+
async fn wait_until_http_error_immediate_navigation_page(headers: HeaderMap) -> Response {
if has_cookie(&headers, "moli-http-error-immediate-navigation=passed") {
return Html(WAIT_UNTIL_HTTP_ERROR_NAVIGATION_FINAL_HTML).into_response();
@@ -415,6 +483,71 @@ async fn wait_until_http_error_late_navigation_page(headers: HeaderMap) -> Respo
.into_response()
}
+fn slow_http_error_streaming_response(
+ status: StatusCode,
+ content_type: &'static str,
+ head: &'static [u8],
+ tail: &'static [u8],
+) -> Response {
+ let (tx, rx) = tokio::sync::mpsc::channel(2);
+ tokio::spawn(async move {
+ if tx
+ .send(Ok::<Bytes, std::convert::Infallible>(Bytes::from_static(
+ head,
+ )))
+ .await
+ .is_err()
+ {
+ return;
+ }
+ sleep(SLOW_HTTP_ERROR_STREAM_TAIL_DELAY).await;
+ let _ = tx
+ .send(Ok::<Bytes, std::convert::Infallible>(Bytes::from_static(
+ tail,
+ )))
+ .await;
+ });
+
+ Response::builder()
+ .status(status)
+ .header(CONTENT_TYPE, content_type)
+ .body(Body::from_stream(
+ tokio_stream::wrappers::ReceiverStream::new(rx),
+ ))
+ .expect("slow HTTP error streaming response should build")
+}
+
+async fn wait_until_http_error_slow_streaming_500_page() -> Response {
+ slow_http_error_streaming_response(
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "text/html; charset=utf-8",
+ WAIT_UNTIL_SLOW_STREAMING_500_HEAD,
+ WAIT_UNTIL_SLOW_STREAMING_500_TAIL,
+ )
+}
+
+async fn wait_until_http_error_slow_streaming_navigation_page(headers: HeaderMap) -> Response {
+ if has_cookie(&headers, "moli-slow-http-error-navigation=passed") {
+ return Html(WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_FINAL_HTML).into_response();
+ }
+
+ slow_http_error_streaming_response(
+ StatusCode::FORBIDDEN,
+ "text/html; charset=utf-8",
+ WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_HEAD,
+ WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_TAIL,
+ )
+}
+
+async fn wait_until_http_error_slow_streaming_raw_404_page() -> Response {
+ slow_http_error_streaming_response(
+ StatusCode::NOT_FOUND,
+ "application/octet-stream",
+ b"slow-raw-404-head|",
+ b"slow-raw-404-tail",
+ )
+}
+
async fn wait_until_readiness_http_error_navigation_page(headers: HeaderMap) -> Response {
if has_cookie(&headers, "moli-readiness-navigation=passed") {
return Html(WAIT_UNTIL_READINESS_HTTP_ERROR_FINAL_HTML).into_response();
diff --git a/moli-test-support/src/server_routes.rs b/moli-test-support/src/server_routes.rs
index 8d0778d0d..bf52c9738 100644
--- a/moli-test-support/src/server_routes.rs
+++ b/moli-test-support/src/server_routes.rs
@@ -2515,6 +2515,10 @@ pub(super) fn build_router() -> Router {
any(net_upstream_xhr_404_then_200_page),
)
.route("/net/upstream/xhr/500", any(net_upstream_xhr_500_page))
+ .route(
+ "/net/upstream/xhr/binary-404",
+ any(net_upstream_xhr_binary_404_page),
+ )
.route(
"/net/upstream/xhr/binary",
any(net_upstream_xhr_binary_page),
diff --git a/moli/src/app.rs b/moli/src/app.rs
index bd85f980a..6a327a4ef 100644
--- a/moli/src/app.rs
+++ b/moli/src/app.rs
@@ -16,10 +16,10 @@ use clap::Parser;
use moli_core::runtime::{
Browser, FetchedDocument, NavigationRuntimeConfig, storage_partition::StoragePartitionState,
};
-use moli_fetch::{NetworkFetchFailureContext, Request, ensure_http_status_success};
+use moli_fetch::{NetworkFetchFailureContext, Request};
use moli_protocol_server::ProtocolServer;
-use self::{http_error_navigation::is_http_error_status, readiness::ReadinessPlan};
+use self::readiness::ReadinessPlan;
pub async fn run_from_env() -> Result<()> {
let cli = Cli::parse_from(normalize_args_for_compat(std::env::args_os()));
@@ -66,21 +66,6 @@ pub async fn run_cli_with_config<W: Write>(
let mut page = match fetched_document {
FetchedDocument::Page(page) => page,
FetchedDocument::Raw(raw_document) => {
- if readiness.lifecycle_stage().is_some()
- && is_http_error_status(raw_document.status())
- {
- let status_error = ensure_http_status_success(
- raw_document.final_url().as_str(),
- raw_document.status(),
- false,
- );
- finalize_fetch_browser(browser);
- return status_error
- .context(
- "HTTP error response is not an executable document and cannot navigate",
- )
- .with_context(|| anyhow!("failed to fetch `{}`", args.url));
- }
if readiness.has_page_waits() || args.delay_ms > 0 {
finalize_fetch_browser(browser);
return Err(anyhow!(
@@ -99,26 +84,6 @@ pub async fn run_cli_with_config<W: Write>(
}
};
- if readiness.lifecycle_stage().is_some() && is_http_error_status(page.status()) {
- let error = ensure_http_status_success(
- page.final_url().as_str(),
- page.status(),
- false,
- )
- .context(
- "navigation from the HTTP error document reached another HTTP error document",
- )
- .expect_err("HTTP error status must fail success validation");
- if let Err(close_error) = page.close_async().await {
- tracing::warn!(
- error = %close_error,
- "failed to close fetched page after HTTP error navigation failure"
- );
- }
- finalize_fetch_browser(browser);
- return Err(with_fetch_context(error, &args.url));
- }
-
if let Err(error) = readiness.wait_for_page(&browser, &mut page).await {
if let Err(close_error) = page.close_async().await {
tracing::warn!(
diff --git a/moli/src/app/http_error_navigation.rs b/moli/src/app/http_error_navigation.rs
index afcbff42a..efbf75c90 100644
--- a/moli/src/app/http_error_navigation.rs
+++ b/moli/src/app/http_error_navigation.rs
@@ -3,12 +3,12 @@
//! HTTP status interpretation and timeout configuration belong to the CLI
//! layer. The renderer receives only the generic lifecycle-target decision.
-use anyhow::Result;
+use anyhow::{Context, Result};
use moli_core::runtime::{
Browser, FetchDeadline, FetchedDocument, RenderedDomWaitUntil, RendererLifecycleDecision,
};
use moli_fetch::Request;
-use std::time::Duration;
+use std::time::{Duration, Instant};
pub(super) fn is_http_error_status(status: u16) -> bool {
(400..=599).contains(&status)
@@ -19,13 +19,17 @@ pub(super) async fn fetch_with_http_error_navigation(
request: Request,
wait_until: RenderedDomWaitUntil,
deadline: FetchDeadline,
- navigation_grace: Duration,
+ minimum_navigation_wait: Duration,
) -> Result<FetchedDocument> {
- let navigation_grace_ms = navigation_grace.as_millis().min(u128::from(u64::MAX)) as u64;
+ let minimum_navigation_deadline = Instant::now()
+ .checked_add(minimum_navigation_wait)
+ .context("HTTP error replacement-navigation wait exceeds the supported range")?;
// The first DCL/load is delivered to this synchronous decision normally.
- // A 4xx/5xx may spend up to the configured grace waiting for a replacement
- // navigation. That wait and the successor's matching lifecycle stage both
- // consume the plan's absolute deadline; neither starts a fresh budget.
+ // A 4xx/5xx keeps running until the configured minimum time from fetch
+ // start has elapsed, giving client-side challenges a chance to replace the
+ // error Document. The initial lifecycle load therefore consumes this
+ // window instead of receiving a fresh grace period afterward. The outer
+ // readiness deadline still caps this wait and any successor lifecycle.
browser
.fetch_document_with_lifecycle_decider_and_deadline(
request,
@@ -33,8 +37,11 @@ pub(super) async fn fetch_with_http_error_navigation(
deadline,
move |target| {
Ok(if is_http_error_status(target.status) {
- RendererLifecycleDecision::FollowNextDocument {
- navigation_grace_ms,
+ RendererLifecycleDecision::FollowNextDocumentOrFinish {
+ navigation_grace_ms: remaining_wait_milliseconds(
+ minimum_navigation_deadline,
+ Instant::now(),
+ ),
}
} else {
RendererLifecycleDecision::Finish
@@ -44,9 +51,18 @@ pub(super) async fn fetch_with_http_error_navigation(
.await
}
+fn remaining_wait_milliseconds(deadline: Instant, now: Instant) -> u64 {
+ deadline
+ .saturating_duration_since(now)
+ .as_nanos()
+ .div_ceil(1_000_000)
+ .min(u128::from(u64::MAX)) as u64
+}
+
#[cfg(test)]
mod tests {
- use super::is_http_error_status;
+ use super::{is_http_error_status, remaining_wait_milliseconds};
+ use std::time::{Duration, Instant};
#[test]
fn http_error_status_covers_only_four_hundred_and_five_hundred_ranges() {
@@ -57,4 +73,32 @@ mod tests {
assert!(is_http_error_status(599));
assert!(!is_http_error_status(600));
}
+
+ #[test]
+ fn replacement_wait_is_the_unconsumed_minimum_from_fetch_start() {
+ let started = Instant::now();
+ let deadline = started + Duration::from_millis(1_000);
+
+ assert_eq!(remaining_wait_milliseconds(deadline, started), 1_000);
+ assert_eq!(
+ remaining_wait_milliseconds(deadline, started + Duration::from_millis(275)),
+ 725
+ );
+ assert_eq!(
+ remaining_wait_milliseconds(deadline, started + Duration::from_millis(1_000)),
+ 0
+ );
+ assert_eq!(
+ remaining_wait_milliseconds(deadline, started + Duration::from_millis(1_500)),
+ 0
+ );
+ }
+
+ #[test]
+ fn replacement_wait_rounds_up_a_partial_millisecond() {
+ let now = Instant::now();
+ let deadline = now + Duration::from_nanos(1_000_001);
+
+ assert_eq!(remaining_wait_milliseconds(deadline, now), 2);
+ }
}
diff --git a/moli/src/app/readiness.rs b/moli/src/app/readiness.rs
index a1c11fe1b..47fee6549 100644
--- a/moli/src/app/readiness.rs
+++ b/moli/src/app/readiness.rs
@@ -12,7 +12,7 @@ use crate::cli::{FetchArgs, FetchWaitUntil};
use anyhow::{Context, Result, anyhow, bail};
use moli_core::{
page::{Page, SubresourceResponseWaitCriteria},
- runtime::{Browser, FetchDeadline, FetchedDocument, PageVmInitStage, RenderedDomWaitUntil},
+ runtime::{Browser, FetchDeadline, FetchedDocument, RenderedDomWaitUntil},
};
use moli_fetch::Request;
use std::time::Duration;
@@ -21,7 +21,7 @@ use std::time::Duration;
pub(super) struct ReadinessPlan {
wait_until: RenderedDomWaitUntil,
deadline: FetchDeadline,
- redirect_wait: Duration,
+ minimum_navigation_wait: Duration,
response: Option<SubresourceResponseWaitCriteria>,
selector: Option<String>,
script: Option<String>,
@@ -37,24 +37,13 @@ impl ReadinessPlan {
wait_until: rendered_wait_until(args.wait_until),
deadline: FetchDeadline::new(Duration::from_millis(args.timeout))
.context("failed to create fetch readiness deadline")?,
- redirect_wait: Duration::from_millis(args.redirect_wait_ms),
+ minimum_navigation_wait: Duration::from_millis(args.redirect_wait_ms),
response,
selector: args.wait_selector.clone(),
script,
})
}
- pub(super) fn lifecycle_stage(&self) -> Option<PageVmInitStage> {
- // Only concrete Document lifecycle milestones opt into HTTP-error
- // navigation recovery. `done` retains the historical load boundary;
- // network-idle and DOM-stable retain best-effort HTTP-error dumps.
- match self.wait_until {
- RenderedDomWaitUntil::DomContentLoaded => Some(PageVmInitStage::DomContentLoaded),
- RenderedDomWaitUntil::Load | RenderedDomWaitUntil::Done => Some(PageVmInitStage::Load),
- RenderedDomWaitUntil::NetworkIdle | RenderedDomWaitUntil::DomStable => None,
- }
- }
-
pub(super) fn has_page_waits(&self) -> bool {
self.response.is_some() || self.selector.is_some() || self.script.is_some()
}
@@ -73,7 +62,7 @@ impl ReadinessPlan {
request,
self.wait_until,
self.deadline,
- self.redirect_wait,
+ self.minimum_navigation_wait,
)
.await
}
diff --git a/moli/src/cli.rs b/moli/src/cli.rs
index 57006a565..d5d93d16f 100644
--- a/moli/src/cli.rs
+++ b/moli/src/cli.rs
@@ -68,11 +68,13 @@ pub struct FetchArgs {
#[arg(long, value_enum, default_value = "done")]
pub wait_until: FetchWaitUntil,
- /// Milliseconds to wait for a client-side replacement navigation after an
- /// executable 4xx/5xx Document reaches the selected lifecycle stage. A
+ /// Minimum milliseconds from fetch start to allow an executable 4xx/5xx
+ /// Document to begin a client-side replacement navigation. Time spent
+ /// reaching the selected lifecycle stage counts toward this minimum. A
/// value of 0 disables additional waiting but still accepts a navigation
- /// that is already pending when the stage result is inspected. This only
- /// applies to `domcontentloaded`, `load`, and `done` waits.
+ /// already pending when the stage result is inspected. If no replacement
+ /// starts in time, the current error response is returned with its body.
+ /// This only applies to `domcontentloaded`, `load`, and `done` waits.
#[arg(
long,
value_name = "MILLISECONDS",
@@ -678,6 +680,7 @@ mod tests {
assert!(help.contains("--redirect-wait-ms <MILLISECONDS>"));
assert!(!help.contains("--redirect-time"));
+ assert!(help.contains("Minimum milliseconds from fetch start"));
assert!(help.contains("Maximum total readiness time in milliseconds"));
assert!(help.contains("response match, selector, and script waits share one absolute"));
}
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index c638d0ffa..92d9c9f01 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -182,6 +182,14 @@ where
}
fn run_fetch_cli_with_wait_until(url: &str, wait_until: &str) -> Result<Output> {
+ run_fetch_cli_with_wait_until_and_dump(url, wait_until, "html")
+}
+
+fn run_fetch_cli_with_wait_until_and_dump(
+ url: &str,
+ wait_until: &str,
+ dump: &str,
+) -> Result<Output> {
run_moli([
"moli",
"fetch",
@@ -192,7 +200,7 @@ fn run_fetch_cli_with_wait_until(url: &str, wait_until: &str) -> Result<Output>
"--wait-until",
wait_until,
"--dump",
- "html",
+ dump,
url,
])
}
@@ -2618,7 +2626,7 @@ fn cli_default_done_recovers_from_delayed_403_navigation_at_load() -> Result<()>
}
#[test]
-fn cli_explicit_load_fails_when_404_does_not_navigate_and_does_not_retry() -> Result<()> {
+fn cli_explicit_load_returns_404_body_without_retrying() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
let server = runtime.block_on(FixtureServer::spawn())?;
let url = server.url("/net/upstream/xhr/404-then-200");
@@ -2626,53 +2634,57 @@ fn cli_explicit_load_fails_when_404_does_not_navigate_and_does_not_retry() -> Re
runtime.block_on(server.shutdown());
assert!(
- !output.status.success(),
- "an HTTP error document without navigation must fail: stdout={}\nstderr={}",
+ output.status.success(),
+ "an HTTP error document with a body must be returned: stdout={}\nstderr={}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let stdout = clean_output(&output.stdout);
- let stderr = clean_output(&output.stderr);
- assert!(stdout.is_empty(), "stdout={stdout}");
- assert!(stderr.contains("404 Not Found"), "stderr={stderr}");
- assert!(stderr.contains("1000 ms grace period"), "stderr={stderr}");
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 404);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(html.contains("first-hit-404"), "stdout={stdout}");
+ assert!(!html.contains("second-hit-200"), "stdout={stdout}");
Ok(())
}
#[test]
-fn cli_explicit_domcontentloaded_fails_when_500_does_not_navigate() -> Result<()> {
+fn cli_explicit_domcontentloaded_returns_500_body() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
let server = runtime.block_on(FixtureServer::spawn())?;
let url = server.url("/net/upstream/xhr/500");
let output = run_fetch_cli_with_wait_until(&url, "domcontentloaded")?;
runtime.block_on(server.shutdown());
- assert!(!output.status.success());
- let stdout = clean_output(&output.stdout);
- let stderr = clean_output(&output.stderr);
- assert!(stdout.is_empty(), "stdout={stdout}");
assert!(
- stderr.contains("500 Internal Server Error"),
- "stderr={stderr}"
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
);
- assert!(stderr.contains("1000 ms grace period"), "stderr={stderr}");
+ let stdout = clean_output(&output.stdout);
+ assert!(stdout.contains("Internal Server Error"), "stdout={stdout}");
Ok(())
}
#[test]
-fn cli_explicit_load_rejects_navigation_after_the_one_second_grace() -> Result<()> {
+fn cli_explicit_load_returns_error_page_when_navigation_starts_after_the_minimum() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
let server = runtime.block_on(FixtureServer::spawn())?;
let url = server.url("/wait-until-http-error-late-navigation");
let output = run_fetch_cli_with_wait_until(&url, "load")?;
runtime.block_on(server.shutdown());
- assert!(!output.status.success());
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
let stdout = clean_output(&output.stdout);
- let stderr = clean_output(&output.stderr);
- assert!(stdout.is_empty(), "stdout={stdout}");
- assert!(stderr.contains("403 Forbidden"), "stderr={stderr}");
- assert!(stderr.contains("1000 ms grace period"), "stderr={stderr}");
+ assert!(stdout.contains("id=\"challenge\""), "stdout={stdout}");
+ assert!(stdout.contains("late-challenge"), "stdout={stdout}");
+ assert!(!stdout.contains("http-error-navigation-target"));
Ok(())
}
@@ -2816,41 +2828,72 @@ fn cli_zero_redirect_wait_accepts_navigation_already_pending_at_stage() -> Resul
}
#[test]
-fn cli_zero_redirect_wait_rejects_navigation_that_starts_later() -> Result<()> {
+fn cli_zero_redirect_wait_returns_the_current_error_page() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
let server = runtime.block_on(FixtureServer::spawn())?;
let url = server.url("/wait-until-http-error-navigation");
let output = run_fetch_cli_with_dump_and_args(&url, "html", &["--redirect-wait-ms", "0"])?;
runtime.block_on(server.shutdown());
- assert!(!output.status.success());
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
let stdout = clean_output(&output.stdout);
- let stderr = clean_output(&output.stderr);
- assert!(stdout.is_empty(), "stdout={stdout}");
- assert!(stderr.contains("403 Forbidden"), "stderr={stderr}");
- assert!(stderr.contains("0 ms grace period"), "stderr={stderr}");
+ assert!(stdout.contains("id=\"challenge\""), "stdout={stdout}");
+ assert!(!stdout.contains("http-error-navigation-target"));
Ok(())
}
#[test]
-fn cli_explicit_load_rejects_http_error_navigation_to_another_error() -> Result<()> {
+fn cli_same_document_navigation_returns_the_current_http_error_page() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
let server = runtime.block_on(FixtureServer::spawn())?;
- let url = server.url("/wait-until-http-error-navigation-to-error");
- let output = run_fetch_cli_with_wait_until(&url, "load")?;
+ let url = server.url("/wait-until-http-error-same-document-navigation");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;
runtime.block_on(server.shutdown());
- assert!(!output.status.success());
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
let stdout = clean_output(&output.stdout);
- let stderr = clean_output(&output.stderr);
- assert!(stdout.is_empty(), "stdout={stdout}");
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_eq!(payload["status"], 403);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(html.contains("id=\"challenge\""), "stdout={stdout}");
assert!(
- stderr.contains("500 Internal Server Error"),
- "stderr={stderr}"
+ html.contains("http-error-navigation=same-document"),
+ "stdout={stdout}"
);
+ Ok(())
+}
+
+#[test]
+fn cli_explicit_load_returns_a_replacement_error_document() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-http-error-navigation-to-error");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;
+ runtime.block_on(server.shutdown());
+
assert!(
- stderr.contains("reached another HTTP error document"),
- "stderr={stderr}"
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 500);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"http-error-navigation-target\""),
+ "stdout={stdout}"
);
Ok(())
}
@@ -2881,7 +2924,7 @@ fn cli_non_lifecycle_wait_modes_keep_http_error_dump_behavior() -> Result<()> {
}
#[test]
-fn cli_default_done_fails_when_http_error_document_does_not_navigate() -> Result<()> {
+fn cli_default_done_returns_http_error_status_and_body() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
let server = runtime.block_on(FixtureServer::spawn())?;
let url = server.url("/net/upstream/xhr/404");
@@ -2890,13 +2933,19 @@ fn cli_default_done_fails_when_http_error_document_does_not_navigate() -> Result
let stdout = clean_output(&output.stdout);
assert!(
- !output.status.success(),
- "default done must fail an HTTP error document without navigation: stdout={stdout}"
+ output.status.success(),
+ "default done must return an HTTP response with its body: stdout={stdout} stderr={}",
+ clean_output(&output.stderr)
+ );
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 404);
+ assert!(
+ payload["html"]
+ .as_str()
+ .unwrap_or_default()
+ .contains("Not Found"),
+ "stdout={stdout}"
);
- let stderr = clean_output(&output.stderr);
- assert!(stdout.is_empty(), "stdout={stdout}");
- assert!(stderr.contains("404 Not Found"), "stderr={stderr}");
- assert!(stderr.contains("1000 ms grace period"), "stderr={stderr}");
Ok(())
}
@@ -2909,13 +2958,311 @@ fn cli_default_done_http_error_wait_does_not_retry_request() -> Result<()> {
runtime.block_on(server.shutdown());
let stdout = clean_output(&output.stdout);
+ assert!(
+ output.status.success(),
+ "stderr={}",
+ clean_output(&output.stderr)
+ );
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 404);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(html.contains("first-hit-404"), "stdout={stdout}");
+ assert!(
+ !html.contains("second-hit-200"),
+ "a hidden HTTP retry would incorrectly receive the fixture's second 200 response: stdout={stdout}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_default_done_returns_raw_http_error_status_and_body() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/net/upstream/xhr/binary-404");
+ let output = run_fetch_cli_with_default_wait_and_dump(&url, "json")?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 404);
+ assert_eq!(payload["html"], "raw-error-body");
+ Ok(())
+}
+
+#[test]
+fn cli_slow_streaming_500_returns_complete_body_without_an_additive_wait() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-http-error-slow-streaming-500");
+
+ for wait_until in ["domcontentloaded", "load", "done"] {
+ let output = run_fetch_cli_with_wait_until_and_dump(&url, wait_until, "json")?;
+ assert!(
+ output.status.success(),
+ "wait_until={wait_until} stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 500);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"slow-http-error-head\""),
+ "wait_until={wait_until} stdout={stdout}"
+ );
+ assert!(
+ html.contains("id=\"slow-http-error-tail\""),
+ "the delayed main-response tail must be present: wait_until={wait_until} stdout={stdout}"
+ );
+ assert!(
+ !html.contains("id=\"slow-http-error-post-load-early\""),
+ "a fresh post-Load wait would run the 400 ms marker: wait_until={wait_until} stdout={stdout}"
+ );
+ assert!(
+ !html.contains("id=\"slow-http-error-post-load-late\""),
+ "wait_until={wait_until} stdout={stdout}"
+ );
+ }
+
+ runtime.block_on(server.shutdown());
+ Ok(())
+}
+
+#[test]
+fn cli_slow_streaming_500_fallback_remains_live_for_page_waits() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-http-error-slow-streaming-500");
+ let output = run_fetch_cli_with_dump_and_args(
+ &url,
+ "json",
+ &["--wait-selector", "#slow-http-error-post-load-early"],
+ )?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 500);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"slow-http-error-tail\""),
+ "stdout={stdout}"
+ );
+ assert!(
+ html.contains("id=\"slow-http-error-post-load-early\""),
+ "the fallback Page must keep running for explicit page waits: stdout={stdout}"
+ );
+ assert!(
+ !html.contains("id=\"slow-http-error-post-load-late\""),
+ "the selector wait should not turn into an open-ended delay: stdout={stdout}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_slow_streaming_500_and_page_wait_share_one_total_timeout() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-http-error-slow-streaming-500");
+ let output = run_fetch_cli_with_dump_and_args(
+ &url,
+ "json",
+ &[
+ "--timeout",
+ "4000",
+ "--wait-selector",
+ "#slow-http-error-post-load-after-total-deadline",
+ ],
+ )?;
+ runtime.block_on(server.shutdown());
+
assert!(
!output.status.success(),
- "a hidden HTTP retry would incorrectly receive the fixture's second 200 response"
+ "a fresh selector timeout would incorrectly succeed: stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
);
+ let stdout = clean_output(&output.stdout);
let stderr = clean_output(&output.stderr);
assert!(stdout.is_empty(), "stdout={stdout}");
- assert!(stderr.contains("404 Not Found"), "stderr={stderr}");
+ assert!(
+ stderr.contains("failed while waiting for selector"),
+ "stderr={stderr}"
+ );
+ assert!(stderr.contains("timed out"), "stderr={stderr}");
+ Ok(())
+}
+
+#[test]
+fn cli_slow_streaming_500_returns_complete_body_for_best_effort_modes() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-http-error-slow-streaming-500");
+
+ for wait_until in ["networkidle", "domstable"] {
+ let output = run_fetch_cli_with_wait_until_and_dump(&url, wait_until, "json")?;
+ assert!(
+ output.status.success(),
+ "wait_until={wait_until} stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 500);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"slow-http-error-head\""),
+ "wait_until={wait_until} stdout={stdout}"
+ );
+ assert!(
+ html.contains("id=\"slow-http-error-tail\""),
+ "wait_until={wait_until} stdout={stdout}"
+ );
+ }
+
+ runtime.block_on(server.shutdown());
+ Ok(())
+}
+
+#[test]
+fn cli_slow_streaming_403_returns_before_a_post_load_navigation_by_default() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-http-error-slow-streaming-navigation");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 403);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"slow-http-error-navigation-tail\""),
+ "the 1.25 s main-response tail must be returned: stdout={stdout}"
+ );
+ assert!(
+ !html.contains("slow-http-error-navigation-target"),
+ "the default minimum elapsed before Load and must not become an extra post-Load grace: stdout={stdout}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_slow_streaming_403_uses_remaining_configured_wait_for_navigation() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-http-error-slow-streaming-navigation");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &["--redirect-wait-ms", "3000"])?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 200);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"slow-http-error-navigation-target\""),
+ "the unconsumed minimum must catch the post-Load navigation: stdout={stdout}"
+ );
+ assert!(
+ !html.contains("slow-http-error-navigation-tail"),
+ "the replacement Document must supersede the original 403 body: stdout={stdout}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_http_error_replacement_to_slow_streaming_500_returns_complete_body() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-http-error-navigation-to-slow-streaming-500");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 500);
+ let html = payload["html"].as_str().unwrap_or_default();
+ assert!(
+ html.contains("id=\"slow-http-error-head\""),
+ "stdout={stdout}"
+ );
+ assert!(
+ html.contains("id=\"slow-http-error-tail\""),
+ "the replacement 500 main-response tail must be complete: stdout={stdout}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_slow_streaming_http_error_still_obeys_the_total_timeout() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-http-error-slow-streaming-500");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &["--timeout", "400"])?;
+ runtime.block_on(server.shutdown());
+
+ assert!(!output.status.success());
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(stdout.is_empty(), "stdout={stdout}");
+ assert!(stderr.contains("timed out after 400 ms"), "stderr={stderr}");
+ assert!(
+ !stderr.contains("resident renderer page entry must retain an active PageVm"),
+ "the partial error Page must tear down cleanly: stderr={stderr}"
+ );
+ Ok(())
+}
+
+#[test]
+fn cli_slow_streaming_raw_404_returns_complete_status_and_body() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/wait-until-http-error-slow-streaming-raw-404");
+ let output = run_fetch_cli_with_default_wait_and_dump(&url, "json")?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &url, 404);
+ assert_eq!(payload["html"], "slow-raw-404-head|slow-raw-404-tail");
Ok(())
}