Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(cli): preserve HTTP error bodies after readiness
ldm0c86a373bcc77
diff --git a/moli-test-support/src/routes_core.rs b/moli-test-support/src/routes_core.rsindex ce04e46b7..06420e23d 100644--- a/moli-test-support/src/routes_core.rs+++ b/moli-test-support/src/routes_core.rs@@ -5803,6 +5803,15 @@ pub(super) async fn net_upstream_xhr_500_page() -> Response {(StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response()}+pub(super) async fn net_upstream_xhr_binary_404_page() -> Response {+ (+ StatusCode::NOT_FOUND,+ [(CONTENT_TYPE, "application/octet-stream")],+ b"raw-error-body".to_vec(),+ )+ .into_response()+}+pub(super) async fn net_upstream_xhr_binary_page() -> Response {let bytes = [0u8, 0, 1, 2, 0, 0, 9];([(CONTENT_TYPE, "application/octet-stream")], bytes.to_vec()).into_response()diff --git a/moli-test-support/src/routes_wait.rs b/moli-test-support/src/routes_wait.rsindex 7eaec156e..b3d08a3c7 100644--- a/moli-test-support/src/routes_wait.rs+++ b/moli-test-support/src/routes_wait.rs@@ -176,6 +176,39 @@ window.addEventListener('load', () => {});</script></body></html>"#;+const WAIT_UNTIL_SLOW_STREAMING_500_HEAD: &[u8] = br#"<!doctype html>+<html><head><title>slow streaming 500</title><script>+const appendSlowErrorMarker = (id, text) => {+ const marker = document.createElement('main');+ marker.id = id;+ marker.textContent = text;+ document.body.appendChild(marker);+};+window.addEventListener('load', () => {+ setTimeout(() => appendSlowErrorMarker('slow-http-error-post-load-early', 'post-load-early'), 400);+ setTimeout(() => appendSlowErrorMarker('slow-http-error-post-load-late', 'post-load-late'), 1400);+ setTimeout(() => appendSlowErrorMarker('slow-http-error-post-load-after-total-deadline', 'post-load-after-total-deadline'), 3000);+});+</script></head><body><main id="slow-http-error-head">slow-500-head</main>"#;+const WAIT_UNTIL_SLOW_STREAMING_500_TAIL: &[u8] =+ br#"<main id="slow-http-error-tail">slow-500-tail</main></body></html>"#;+const WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_HEAD: &[u8] = br#"<!doctype html>+<html><head><title>slow streaming 403 navigation</title><script>+window.addEventListener('load', () => {+ setTimeout(() => {+ document.cookie = 'moli-slow-http-error-navigation=passed; Path=/; Max-Age=3600; SameSite=Lax';+ location.reload();+ }, 300);+});+</script></head><body><main id="slow-http-error-navigation-head">slow-403-head</main>"#;+const WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_TAIL: &[u8] =+ br#"<main id="slow-http-error-navigation-tail">slow-403-tail</main></body></html>"#;+const WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_FINAL_HTML: &str = r#"<!doctype html>+<html><head><title>slow streaming challenge passed</title></head><body>+<main id="slow-http-error-navigation-target">slow-http-error-navigation=done</main>+</body></html>"#;++const SLOW_HTTP_ERROR_STREAM_TAIL_DELAY: Duration = Duration::from_millis(1_250);pub(super) fn add_wait_routes(router: Router) -> Router {router@@ -274,6 +307,10 @@ pub(super) fn add_wait_routes(router: Router) -> Router {"/wait-until-http-error-navigation-to-error",get(wait_until_http_error_navigation_to_error_page),)+ .route(+ "/wait-until-http-error-navigation-to-slow-streaming-500",+ get(wait_until_http_error_navigation_to_slow_streaming_500_page),+ ).route("/wait-until-http-error-immediate-navigation",get(wait_until_http_error_immediate_navigation_page),@@ -294,6 +331,18 @@ pub(super) fn add_wait_routes(router: Router) -> Router {"/wait-until-http-error-late-navigation",get(wait_until_http_error_late_navigation_page),)+ .route(+ "/wait-until-http-error-slow-streaming-500",+ get(wait_until_http_error_slow_streaming_500_page),+ )+ .route(+ "/wait-until-http-error-slow-streaming-navigation",+ get(wait_until_http_error_slow_streaming_navigation_page),+ )+ .route(+ "/wait-until-http-error-slow-streaming-raw-404",+ get(wait_until_http_error_slow_streaming_raw_404_page),+ ).route("/wait-until-readiness-http-error-navigation",get(wait_until_readiness_http_error_navigation_page),@@ -341,6 +390,25 @@ async fn wait_until_http_error_navigation_to_error_page(headers: HeaderMap) -> R.into_response()}+async fn wait_until_http_error_navigation_to_slow_streaming_500_page(+ headers: HeaderMap,+) -> Response {+ if has_cookie(&headers, "moli-http-error-navigation=passed") {+ return slow_http_error_streaming_response(+ StatusCode::INTERNAL_SERVER_ERROR,+ "text/html; charset=utf-8",+ WAIT_UNTIL_SLOW_STREAMING_500_HEAD,+ WAIT_UNTIL_SLOW_STREAMING_500_TAIL,+ );+ }++ (+ StatusCode::FORBIDDEN,+ Html(WAIT_UNTIL_HTTP_ERROR_NAVIGATION_CHALLENGE_HTML),+ )+ .into_response()+}+async fn wait_until_http_error_immediate_navigation_page(headers: HeaderMap) -> Response {if has_cookie(&headers, "moli-http-error-immediate-navigation=passed") {return Html(WAIT_UNTIL_HTTP_ERROR_NAVIGATION_FINAL_HTML).into_response();@@ -415,6 +483,71 @@ async fn wait_until_http_error_late_navigation_page(headers: HeaderMap) -> Respo.into_response()}+fn slow_http_error_streaming_response(+ status: StatusCode,+ content_type: &'static str,+ head: &'static [u8],+ tail: &'static [u8],+) -> Response {+ let (tx, rx) = tokio::sync::mpsc::channel(2);+ tokio::spawn(async move {+ if tx+ .send(Ok::<Bytes, std::convert::Infallible>(Bytes::from_static(+ head,+ )))+ .await+ .is_err()+ {+ return;+ }+ sleep(SLOW_HTTP_ERROR_STREAM_TAIL_DELAY).await;+ let _ = tx+ .send(Ok::<Bytes, std::convert::Infallible>(Bytes::from_static(+ tail,+ )))+ .await;+ });++ Response::builder()+ .status(status)+ .header(CONTENT_TYPE, content_type)+ .body(Body::from_stream(+ tokio_stream::wrappers::ReceiverStream::new(rx),+ ))+ .expect("slow HTTP error streaming response should build")+}++async fn wait_until_http_error_slow_streaming_500_page() -> Response {+ slow_http_error_streaming_response(+ StatusCode::INTERNAL_SERVER_ERROR,+ "text/html; charset=utf-8",+ WAIT_UNTIL_SLOW_STREAMING_500_HEAD,+ WAIT_UNTIL_SLOW_STREAMING_500_TAIL,+ )+}++async fn wait_until_http_error_slow_streaming_navigation_page(headers: HeaderMap) -> Response {+ if has_cookie(&headers, "moli-slow-http-error-navigation=passed") {+ return Html(WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_FINAL_HTML).into_response();+ }++ slow_http_error_streaming_response(+ StatusCode::FORBIDDEN,+ "text/html; charset=utf-8",+ WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_HEAD,+ WAIT_UNTIL_SLOW_STREAMING_403_NAVIGATION_TAIL,+ )+}++async fn wait_until_http_error_slow_streaming_raw_404_page() -> Response {+ slow_http_error_streaming_response(+ StatusCode::NOT_FOUND,+ "application/octet-stream",+ b"slow-raw-404-head|",+ b"slow-raw-404-tail",+ )+}+async fn wait_until_readiness_http_error_navigation_page(headers: HeaderMap) -> Response {if has_cookie(&headers, "moli-readiness-navigation=passed") {return Html(WAIT_UNTIL_READINESS_HTTP_ERROR_FINAL_HTML).into_response();diff --git a/moli-test-support/src/server_routes.rs b/moli-test-support/src/server_routes.rsindex 8d0778d0d..bf52c9738 100644--- a/moli-test-support/src/server_routes.rs+++ b/moli-test-support/src/server_routes.rs@@ -2515,6 +2515,10 @@ pub(super) fn build_router() -> Router {any(net_upstream_xhr_404_then_200_page),).route("/net/upstream/xhr/500", any(net_upstream_xhr_500_page))+ .route(+ "/net/upstream/xhr/binary-404",+ any(net_upstream_xhr_binary_404_page),+ ).route("/net/upstream/xhr/binary",any(net_upstream_xhr_binary_page),diff --git a/moli/src/app.rs b/moli/src/app.rsindex bd85f980a..6a327a4ef 100644--- a/moli/src/app.rs+++ b/moli/src/app.rs@@ -16,10 +16,10 @@ use clap::Parser;use moli_core::runtime::{Browser, FetchedDocument, NavigationRuntimeConfig, storage_partition::StoragePartitionState,};-use moli_fetch::{NetworkFetchFailureContext, Request, ensure_http_status_success};+use moli_fetch::{NetworkFetchFailureContext, Request};use moli_protocol_server::ProtocolServer;-use self::{http_error_navigation::is_http_error_status, readiness::ReadinessPlan};+use self::readiness::ReadinessPlan;pub async fn run_from_env() -> Result<()> {let cli = Cli::parse_from(normalize_args_for_compat(std::env::args_os()));@@ -66,21 +66,6 @@ pub async fn run_cli_with_config<W: Write>(let mut page = match fetched_document {FetchedDocument::Page(page) => page,FetchedDocument::Raw(raw_document) => {- if readiness.lifecycle_stage().is_some()- && is_http_error_status(raw_document.status())- {- let status_error = ensure_http_status_success(- raw_document.final_url().as_str(),- raw_document.status(),- false,- );- finalize_fetch_browser(browser);- return status_error- .context(- "HTTP error response is not an executable document and cannot navigate",- )- .with_context(|| anyhow!("failed to fetch `{}`", args.url));- }if readiness.has_page_waits() || args.delay_ms > 0 {finalize_fetch_browser(browser);return Err(anyhow!(@@ -99,26 +84,6 @@ pub async fn run_cli_with_config<W: Write>(}};- if readiness.lifecycle_stage().is_some() && is_http_error_status(page.status()) {- let error = ensure_http_status_success(- page.final_url().as_str(),- page.status(),- false,- )- .context(- "navigation from the HTTP error document reached another HTTP error document",- )- .expect_err("HTTP error status must fail success validation");- if let Err(close_error) = page.close_async().await {- tracing::warn!(- error = %close_error,- "failed to close fetched page after HTTP error navigation failure"- );- }- finalize_fetch_browser(browser);- return Err(with_fetch_context(error, &args.url));- }-if let Err(error) = readiness.wait_for_page(&browser, &mut page).await {if let Err(close_error) = page.close_async().await {tracing::warn!(diff --git a/moli/src/app/http_error_navigation.rs b/moli/src/app/http_error_navigation.rsindex afcbff42a..efbf75c90 100644--- a/moli/src/app/http_error_navigation.rs+++ b/moli/src/app/http_error_navigation.rs@@ -3,12 +3,12 @@//! HTTP status interpretation and timeout configuration belong to the CLI//! layer. The renderer receives only the generic lifecycle-target decision.-use anyhow::Result;+use anyhow::{Context, Result};use moli_core::runtime::{Browser, FetchDeadline, FetchedDocument, RenderedDomWaitUntil, RendererLifecycleDecision,};use moli_fetch::Request;-use std::time::Duration;+use std::time::{Duration, Instant};pub(super) fn is_http_error_status(status: u16) -> bool {(400..=599).contains(&status)@@ -19,13 +19,17 @@ pub(super) async fn fetch_with_http_error_navigation(request: Request,wait_until: RenderedDomWaitUntil,deadline: FetchDeadline,- navigation_grace: Duration,+ minimum_navigation_wait: Duration,) -> Result<FetchedDocument> {- let navigation_grace_ms = navigation_grace.as_millis().min(u128::from(u64::MAX)) as u64;+ let minimum_navigation_deadline = Instant::now()+ .checked_add(minimum_navigation_wait)+ .context("HTTP error replacement-navigation wait exceeds the supported range")?;// The first DCL/load is delivered to this synchronous decision normally.- // A 4xx/5xx may spend up to the configured grace waiting for a replacement- // navigation. That wait and the successor's matching lifecycle stage both- // consume the plan's absolute deadline; neither starts a fresh budget.+ // A 4xx/5xx keeps running until the configured minimum time from fetch+ // start has elapsed, giving client-side challenges a chance to replace the+ // error Document. The initial lifecycle load therefore consumes this+ // window instead of receiving a fresh grace period afterward. The outer+ // readiness deadline still caps this wait and any successor lifecycle.browser.fetch_document_with_lifecycle_decider_and_deadline(request,@@ -33,8 +37,11 @@ pub(super) async fn fetch_with_http_error_navigation(deadline,move |target| {Ok(if is_http_error_status(target.status) {- RendererLifecycleDecision::FollowNextDocument {- navigation_grace_ms,+ RendererLifecycleDecision::FollowNextDocumentOrFinish {+ navigation_grace_ms: remaining_wait_milliseconds(+ minimum_navigation_deadline,+ Instant::now(),+ ),}} else {RendererLifecycleDecision::Finish@@ -44,9 +51,18 @@ pub(super) async fn fetch_with_http_error_navigation(.await}+fn remaining_wait_milliseconds(deadline: Instant, now: Instant) -> u64 {+ deadline+ .saturating_duration_since(now)+ .as_nanos()+ .div_ceil(1_000_000)+ .min(u128::from(u64::MAX)) as u64+}+#[cfg(test)]mod tests {- use super::is_http_error_status;+ use super::{is_http_error_status, remaining_wait_milliseconds};+ use std::time::{Duration, Instant};#[test]fn http_error_status_covers_only_four_hundred_and_five_hundred_ranges() {@@ -57,4 +73,32 @@ mod tests {assert!(is_http_error_status(599));assert!(!is_http_error_status(600));}++ #[test]+ fn replacement_wait_is_the_unconsumed_minimum_from_fetch_start() {+ let started = Instant::now();+ let deadline = started + Duration::from_millis(1_000);++ assert_eq!(remaining_wait_milliseconds(deadline, started), 1_000);+ assert_eq!(+ remaining_wait_milliseconds(deadline, started + Duration::from_millis(275)),+ 725+ );+ assert_eq!(+ remaining_wait_milliseconds(deadline, started + Duration::from_millis(1_000)),+ 0+ );+ assert_eq!(+ remaining_wait_milliseconds(deadline, started + Duration::from_millis(1_500)),+ 0+ );+ }++ #[test]+ fn replacement_wait_rounds_up_a_partial_millisecond() {+ let now = Instant::now();+ let deadline = now + Duration::from_nanos(1_000_001);++ assert_eq!(remaining_wait_milliseconds(deadline, now), 2);+ }}diff --git a/moli/src/app/readiness.rs b/moli/src/app/readiness.rsindex a1c11fe1b..47fee6549 100644--- a/moli/src/app/readiness.rs+++ b/moli/src/app/readiness.rs@@ -12,7 +12,7 @@ use crate::cli::{FetchArgs, FetchWaitUntil};use anyhow::{Context, Result, anyhow, bail};use moli_core::{page::{Page, SubresourceResponseWaitCriteria},- runtime::{Browser, FetchDeadline, FetchedDocument, PageVmInitStage, RenderedDomWaitUntil},+ runtime::{Browser, FetchDeadline, FetchedDocument, RenderedDomWaitUntil},};use moli_fetch::Request;use std::time::Duration;@@ -21,7 +21,7 @@ use std::time::Duration;pub(super) struct ReadinessPlan {wait_until: RenderedDomWaitUntil,deadline: FetchDeadline,- redirect_wait: Duration,+ minimum_navigation_wait: Duration,response: Option<SubresourceResponseWaitCriteria>,selector: Option<String>,script: Option<String>,@@ -37,24 +37,13 @@ impl ReadinessPlan {wait_until: rendered_wait_until(args.wait_until),deadline: FetchDeadline::new(Duration::from_millis(args.timeout)).context("failed to create fetch readiness deadline")?,- redirect_wait: Duration::from_millis(args.redirect_wait_ms),+ minimum_navigation_wait: Duration::from_millis(args.redirect_wait_ms),response,selector: args.wait_selector.clone(),script,})}- pub(super) fn lifecycle_stage(&self) -> Option<PageVmInitStage> {- // Only concrete Document lifecycle milestones opt into HTTP-error- // navigation recovery. `done` retains the historical load boundary;- // network-idle and DOM-stable retain best-effort HTTP-error dumps.- match self.wait_until {- RenderedDomWaitUntil::DomContentLoaded => Some(PageVmInitStage::DomContentLoaded),- RenderedDomWaitUntil::Load | RenderedDomWaitUntil::Done => Some(PageVmInitStage::Load),- RenderedDomWaitUntil::NetworkIdle | RenderedDomWaitUntil::DomStable => None,- }- }-pub(super) fn has_page_waits(&self) -> bool {self.response.is_some() || self.selector.is_some() || self.script.is_some()}@@ -73,7 +62,7 @@ impl ReadinessPlan {request,self.wait_until,self.deadline,- self.redirect_wait,+ self.minimum_navigation_wait,).await}diff --git a/moli/src/cli.rs b/moli/src/cli.rsindex 57006a565..d5d93d16f 100644--- a/moli/src/cli.rs+++ b/moli/src/cli.rs@@ -68,11 +68,13 @@ pub struct FetchArgs {#[arg(long, value_enum, default_value = "done")]pub wait_until: FetchWaitUntil,- /// Milliseconds to wait for a client-side replacement navigation after an- /// executable 4xx/5xx Document reaches the selected lifecycle stage. A+ /// Minimum milliseconds from fetch start to allow an executable 4xx/5xx+ /// Document to begin a client-side replacement navigation. Time spent+ /// reaching the selected lifecycle stage counts toward this minimum. A/// value of 0 disables additional waiting but still accepts a navigation- /// that is already pending when the stage result is inspected. This only- /// applies to `domcontentloaded`, `load`, and `done` waits.+ /// already pending when the stage result is inspected. If no replacement+ /// starts in time, the current error response is returned with its body.+ /// This only applies to `domcontentloaded`, `load`, and `done` waits.#[arg(long,value_name = "MILLISECONDS",@@ -678,6 +680,7 @@ mod tests {assert!(help.contains("--redirect-wait-ms <MILLISECONDS>"));assert!(!help.contains("--redirect-time"));+ assert!(help.contains("Minimum milliseconds from fetch start"));assert!(help.contains("Maximum total readiness time in milliseconds"));assert!(help.contains("response match, selector, and script waits share one absolute"));}diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex c638d0ffa..92d9c9f01 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -182,6 +182,14 @@ where}fn run_fetch_cli_with_wait_until(url: &str, wait_until: &str) -> Result<Output> {+ run_fetch_cli_with_wait_until_and_dump(url, wait_until, "html")+}++fn run_fetch_cli_with_wait_until_and_dump(+ url: &str,+ wait_until: &str,+ dump: &str,+) -> Result<Output> {run_moli(["moli","fetch",@@ -192,7 +200,7 @@ fn run_fetch_cli_with_wait_until(url: &str, wait_until: &str) -> Result<Output>"--wait-until",wait_until,"--dump",- "html",+ dump,url,])}@@ -2618,7 +2626,7 @@ fn cli_default_done_recovers_from_delayed_403_navigation_at_load() -> Result<()>}#[test]-fn cli_explicit_load_fails_when_404_does_not_navigate_and_does_not_retry() -> Result<()> {+fn cli_explicit_load_returns_404_body_without_retrying() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;let server = runtime.block_on(FixtureServer::spawn())?;let url = server.url("/net/upstream/xhr/404-then-200");@@ -2626,53 +2634,57 @@ fn cli_explicit_load_fails_when_404_does_not_navigate_and_does_not_retry() -> Reruntime.block_on(server.shutdown());assert!(- !output.status.success(),- "an HTTP error document without navigation must fail: stdout={}\nstderr={}",+ output.status.success(),+ "an HTTP error document with a body must be returned: stdout={}\nstderr={}",String::from_utf8_lossy(&output.stdout),String::from_utf8_lossy(&output.stderr));let stdout = clean_output(&output.stdout);- let stderr = clean_output(&output.stderr);- assert!(stdout.is_empty(), "stdout={stdout}");- assert!(stderr.contains("404 Not Found"), "stderr={stderr}");- assert!(stderr.contains("1000 ms grace period"), "stderr={stderr}");+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 404);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(html.contains("first-hit-404"), "stdout={stdout}");+ assert!(!html.contains("second-hit-200"), "stdout={stdout}");Ok(())}#[test]-fn cli_explicit_domcontentloaded_fails_when_500_does_not_navigate() -> Result<()> {+fn cli_explicit_domcontentloaded_returns_500_body() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;let server = runtime.block_on(FixtureServer::spawn())?;let url = server.url("/net/upstream/xhr/500");let output = run_fetch_cli_with_wait_until(&url, "domcontentloaded")?;runtime.block_on(server.shutdown());- assert!(!output.status.success());- let stdout = clean_output(&output.stdout);- let stderr = clean_output(&output.stderr);- assert!(stdout.is_empty(), "stdout={stdout}");assert!(- stderr.contains("500 Internal Server Error"),- "stderr={stderr}"+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr));- assert!(stderr.contains("1000 ms grace period"), "stderr={stderr}");+ let stdout = clean_output(&output.stdout);+ assert!(stdout.contains("Internal Server Error"), "stdout={stdout}");Ok(())}#[test]-fn cli_explicit_load_rejects_navigation_after_the_one_second_grace() -> Result<()> {+fn cli_explicit_load_returns_error_page_when_navigation_starts_after_the_minimum() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;let server = runtime.block_on(FixtureServer::spawn())?;let url = server.url("/wait-until-http-error-late-navigation");let output = run_fetch_cli_with_wait_until(&url, "load")?;runtime.block_on(server.shutdown());- assert!(!output.status.success());+ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );let stdout = clean_output(&output.stdout);- let stderr = clean_output(&output.stderr);- assert!(stdout.is_empty(), "stdout={stdout}");- assert!(stderr.contains("403 Forbidden"), "stderr={stderr}");- assert!(stderr.contains("1000 ms grace period"), "stderr={stderr}");+ assert!(stdout.contains("id=\"challenge\""), "stdout={stdout}");+ assert!(stdout.contains("late-challenge"), "stdout={stdout}");+ assert!(!stdout.contains("http-error-navigation-target"));Ok(())}@@ -2816,41 +2828,72 @@ fn cli_zero_redirect_wait_accepts_navigation_already_pending_at_stage() -> Resul}#[test]-fn cli_zero_redirect_wait_rejects_navigation_that_starts_later() -> Result<()> {+fn cli_zero_redirect_wait_returns_the_current_error_page() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;let server = runtime.block_on(FixtureServer::spawn())?;let url = server.url("/wait-until-http-error-navigation");let output = run_fetch_cli_with_dump_and_args(&url, "html", &["--redirect-wait-ms", "0"])?;runtime.block_on(server.shutdown());- assert!(!output.status.success());+ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );let stdout = clean_output(&output.stdout);- let stderr = clean_output(&output.stderr);- assert!(stdout.is_empty(), "stdout={stdout}");- assert!(stderr.contains("403 Forbidden"), "stderr={stderr}");- assert!(stderr.contains("0 ms grace period"), "stderr={stderr}");+ assert!(stdout.contains("id=\"challenge\""), "stdout={stdout}");+ assert!(!stdout.contains("http-error-navigation-target"));Ok(())}#[test]-fn cli_explicit_load_rejects_http_error_navigation_to_another_error() -> Result<()> {+fn cli_same_document_navigation_returns_the_current_http_error_page() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;let server = runtime.block_on(FixtureServer::spawn())?;- let url = server.url("/wait-until-http-error-navigation-to-error");- let output = run_fetch_cli_with_wait_until(&url, "load")?;+ let url = server.url("/wait-until-http-error-same-document-navigation");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;runtime.block_on(server.shutdown());- assert!(!output.status.success());+ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );let stdout = clean_output(&output.stdout);- let stderr = clean_output(&output.stderr);- assert!(stdout.is_empty(), "stdout={stdout}");+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_eq!(payload["status"], 403);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(html.contains("id=\"challenge\""), "stdout={stdout}");assert!(- stderr.contains("500 Internal Server Error"),- "stderr={stderr}"+ html.contains("http-error-navigation=same-document"),+ "stdout={stdout}");+ Ok(())+}++#[test]+fn cli_explicit_load_returns_a_replacement_error_document() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-navigation-to-error");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;+ runtime.block_on(server.shutdown());+assert!(- stderr.contains("reached another HTTP error document"),- "stderr={stderr}"+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 500);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"http-error-navigation-target\""),+ "stdout={stdout}");Ok(())}@@ -2881,7 +2924,7 @@ fn cli_non_lifecycle_wait_modes_keep_http_error_dump_behavior() -> Result<()> {}#[test]-fn cli_default_done_fails_when_http_error_document_does_not_navigate() -> Result<()> {+fn cli_default_done_returns_http_error_status_and_body() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;let server = runtime.block_on(FixtureServer::spawn())?;let url = server.url("/net/upstream/xhr/404");@@ -2890,13 +2933,19 @@ fn cli_default_done_fails_when_http_error_document_does_not_navigate() -> Resultlet stdout = clean_output(&output.stdout);assert!(- !output.status.success(),- "default done must fail an HTTP error document without navigation: stdout={stdout}"+ output.status.success(),+ "default done must return an HTTP response with its body: stdout={stdout} stderr={}",+ clean_output(&output.stderr)+ );+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 404);+ assert!(+ payload["html"]+ .as_str()+ .unwrap_or_default()+ .contains("Not Found"),+ "stdout={stdout}");- let stderr = clean_output(&output.stderr);- assert!(stdout.is_empty(), "stdout={stdout}");- assert!(stderr.contains("404 Not Found"), "stderr={stderr}");- assert!(stderr.contains("1000 ms grace period"), "stderr={stderr}");Ok(())}@@ -2909,13 +2958,311 @@ fn cli_default_done_http_error_wait_does_not_retry_request() -> Result<()> {runtime.block_on(server.shutdown());let stdout = clean_output(&output.stdout);+ assert!(+ output.status.success(),+ "stderr={}",+ clean_output(&output.stderr)+ );+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 404);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(html.contains("first-hit-404"), "stdout={stdout}");+ assert!(+ !html.contains("second-hit-200"),+ "a hidden HTTP retry would incorrectly receive the fixture's second 200 response: stdout={stdout}"+ );+ Ok(())+}++#[test]+fn cli_default_done_returns_raw_http_error_status_and_body() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/net/upstream/xhr/binary-404");+ let output = run_fetch_cli_with_default_wait_and_dump(&url, "json")?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 404);+ assert_eq!(payload["html"], "raw-error-body");+ Ok(())+}++#[test]+fn cli_slow_streaming_500_returns_complete_body_without_an_additive_wait() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-slow-streaming-500");++ for wait_until in ["domcontentloaded", "load", "done"] {+ let output = run_fetch_cli_with_wait_until_and_dump(&url, wait_until, "json")?;+ assert!(+ output.status.success(),+ "wait_until={wait_until} stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 500);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"slow-http-error-head\""),+ "wait_until={wait_until} stdout={stdout}"+ );+ assert!(+ html.contains("id=\"slow-http-error-tail\""),+ "the delayed main-response tail must be present: wait_until={wait_until} stdout={stdout}"+ );+ assert!(+ !html.contains("id=\"slow-http-error-post-load-early\""),+ "a fresh post-Load wait would run the 400 ms marker: wait_until={wait_until} stdout={stdout}"+ );+ assert!(+ !html.contains("id=\"slow-http-error-post-load-late\""),+ "wait_until={wait_until} stdout={stdout}"+ );+ }++ runtime.block_on(server.shutdown());+ Ok(())+}++#[test]+fn cli_slow_streaming_500_fallback_remains_live_for_page_waits() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-slow-streaming-500");+ let output = run_fetch_cli_with_dump_and_args(+ &url,+ "json",+ &["--wait-selector", "#slow-http-error-post-load-early"],+ )?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 500);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"slow-http-error-tail\""),+ "stdout={stdout}"+ );+ assert!(+ html.contains("id=\"slow-http-error-post-load-early\""),+ "the fallback Page must keep running for explicit page waits: stdout={stdout}"+ );+ assert!(+ !html.contains("id=\"slow-http-error-post-load-late\""),+ "the selector wait should not turn into an open-ended delay: stdout={stdout}"+ );+ Ok(())+}++#[test]+fn cli_slow_streaming_500_and_page_wait_share_one_total_timeout() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-slow-streaming-500");+ let output = run_fetch_cli_with_dump_and_args(+ &url,+ "json",+ &[+ "--timeout",+ "4000",+ "--wait-selector",+ "#slow-http-error-post-load-after-total-deadline",+ ],+ )?;+ runtime.block_on(server.shutdown());+assert!(!output.status.success(),- "a hidden HTTP retry would incorrectly receive the fixture's second 200 response"+ "a fresh selector timeout would incorrectly succeed: stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr));+ let stdout = clean_output(&output.stdout);let stderr = clean_output(&output.stderr);assert!(stdout.is_empty(), "stdout={stdout}");- assert!(stderr.contains("404 Not Found"), "stderr={stderr}");+ assert!(+ stderr.contains("failed while waiting for selector"),+ "stderr={stderr}"+ );+ assert!(stderr.contains("timed out"), "stderr={stderr}");+ Ok(())+}++#[test]+fn cli_slow_streaming_500_returns_complete_body_for_best_effort_modes() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-slow-streaming-500");++ for wait_until in ["networkidle", "domstable"] {+ let output = run_fetch_cli_with_wait_until_and_dump(&url, wait_until, "json")?;+ assert!(+ output.status.success(),+ "wait_until={wait_until} stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 500);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"slow-http-error-head\""),+ "wait_until={wait_until} stdout={stdout}"+ );+ assert!(+ html.contains("id=\"slow-http-error-tail\""),+ "wait_until={wait_until} stdout={stdout}"+ );+ }++ runtime.block_on(server.shutdown());+ Ok(())+}++#[test]+fn cli_slow_streaming_403_returns_before_a_post_load_navigation_by_default() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-slow-streaming-navigation");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 403);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"slow-http-error-navigation-tail\""),+ "the 1.25 s main-response tail must be returned: stdout={stdout}"+ );+ assert!(+ !html.contains("slow-http-error-navigation-target"),+ "the default minimum elapsed before Load and must not become an extra post-Load grace: stdout={stdout}"+ );+ Ok(())+}++#[test]+fn cli_slow_streaming_403_uses_remaining_configured_wait_for_navigation() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-slow-streaming-navigation");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &["--redirect-wait-ms", "3000"])?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 200);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"slow-http-error-navigation-target\""),+ "the unconsumed minimum must catch the post-Load navigation: stdout={stdout}"+ );+ assert!(+ !html.contains("slow-http-error-navigation-tail"),+ "the replacement Document must supersede the original 403 body: stdout={stdout}"+ );+ Ok(())+}++#[test]+fn cli_http_error_replacement_to_slow_streaming_500_returns_complete_body() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-navigation-to-slow-streaming-500");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 500);+ let html = payload["html"].as_str().unwrap_or_default();+ assert!(+ html.contains("id=\"slow-http-error-head\""),+ "stdout={stdout}"+ );+ assert!(+ html.contains("id=\"slow-http-error-tail\""),+ "the replacement 500 main-response tail must be complete: stdout={stdout}"+ );+ Ok(())+}++#[test]+fn cli_slow_streaming_http_error_still_obeys_the_total_timeout() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-slow-streaming-500");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &["--timeout", "400"])?;+ runtime.block_on(server.shutdown());++ assert!(!output.status.success());+ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(stdout.is_empty(), "stdout={stdout}");+ assert!(stderr.contains("timed out after 400 ms"), "stderr={stderr}");+ assert!(+ !stderr.contains("resident renderer page entry must retain an active PageVm"),+ "the partial error Page must tear down cleanly: stderr={stderr}"+ );+ Ok(())+}++#[test]+fn cli_slow_streaming_raw_404_returns_complete_status_and_body() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/wait-until-http-error-slow-streaming-raw-404");+ let output = run_fetch_cli_with_default_wait_and_dump(&url, "json")?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &url, 404);+ assert_eq!(payload["html"], "slow-raw-404-head|slow-raw-404-tail");Ok(())}