Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): cancel raw downloads for unsupported output
ldm0a3784688b753
diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rsindex f5f924f7d..cf35a9878 100644--- a/moli-core/src/runtime/mod.rs+++ b/moli-core/src/runtime/mod.rs@@ -59,6 +59,10 @@ static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);pub enum RawDocumentFetchPolicy {/// Read the response body and return [`FetchedDocument::Raw`].Materialize,+ /// Return response metadata with an empty body and cancel the download.+ /// Callers can use the metadata to select an error without reading a body+ /// that their output format cannot consume.+ MetadataOnly,/// Read the body only for statuses below the threshold. Otherwise return/// response metadata with an empty body and cancel the download.MaterializeBelowStatus(u16),@@ -82,6 +86,7 @@ impl RawDocumentFetchPolicy {fn should_materialize_body(self, status: u16) -> Result<bool> {match self {Self::Materialize => Ok(true),+ Self::MetadataOnly => Ok(false),Self::MaterializeBelowStatus(threshold) => Ok(status < threshold),Self::RequirePage => Err(anyhow::Error::new(RawDocumentPageRequired)),}diff --git a/moli/src/app.rs b/moli/src/app.rsindex 0a381f91b..0fadeadb5 100644--- a/moli/src/app.rs+++ b/moli/src/app.rs@@ -58,14 +58,14 @@ pub async fn run_cli_with_config<W: Write>(ReadinessPlan::from_fetch_args(&args, config.fetch.response_wait.clone())?;let raw_document_output =fetch_dump::RawDocumentOutputPolicy::from_command(&config.fetch);- // HTTP failure modes need the response status before deciding- // whether the selected output format accepts a raw download.- let raw_document_fetch_policy = if args.fail {- RawDocumentFetchPolicy::MaterializeBelowStatus(400)- } else if args.fail_with_body {- RawDocumentFetchPolicy::Materialize- } else {- raw_document_output.fetch_policy()+ // Preserve status metadata for HTTP error precedence, but never+ // consume a raw body that the selected output format cannot use.+ let raw_document_fetch_policy = match raw_document_output.fetch_policy() {+ RawDocumentFetchPolicy::RequirePage if args.fail || args.fail_with_body => {+ RawDocumentFetchPolicy::MetadataOnly+ }+ _ if args.fail => RawDocumentFetchPolicy::MaterializeBelowStatus(400),+ policy => policy,};let request = build_fetch_request(&args.url, &config)?;if config.browser.fetch().obey_robots() {diff --git a/moli/tests/fetch_cli/http_fail.rs b/moli/tests/fetch_cli/http_fail.rsindex f4dfdeb0d..179a5f265 100644--- a/moli/tests/fetch_cli/http_fail.rs+++ b/moli/tests/fetch_cli/http_fail.rs@@ -473,38 +473,67 @@ async fn fail_checks_raw_http_status_before_output_format_and_page_options() ->}#[tokio::test(flavor = "multi_thread", worker_threads = 2)]-async fn fail_cancels_raw_http_errors_without_waiting_for_the_body() -> Result<()> {- for (status, content_type) in [(403, "application/octet-stream"), (503, "application/pdf")] {+async fn unsupported_raw_output_cancels_without_waiting_for_the_body() -> Result<()> {+ for (status, content_type) in [+ (200, "application/octet-stream"),+ (403, "application/octet-stream"),+ (503, "application/pdf"),+ ] {for wait_until in ["done", "networkidle"] {- let mut server = HttpFailureFixture::spawn_stalled_raw(status, content_type).await?;- let mut command = Command::new(env!("CARGO_BIN_EXE_moli"));- command- .args([- "fetch",- "--fail",- "--dump",- "markdown",- "--http-no-proxy",- "*",- "--redirect-wait-ms",- "0",- "--timeout",- "5000",- "--wait-until",- wait_until,- ])- .arg(format!("{}/raw", server.base_url));- let output = tokio::time::timeout(- Duration::from_secs(2),- tokio::task::spawn_blocking(move || command.output()),- )- .await- .expect("--fail must return before the raw body timeout")??;- server.assert_http_failure(&output, status, "/raw");- assert!(output.stdout.is_empty(), "{output:?}");- tokio::time::timeout(Duration::from_secs(2), &mut server.task)- .await- .expect("the HTTP download must be cancelled")?;+ for flag in [None, Some("--fail"), Some("--fail-with-body")] {+ for dump in ["html", "markdown"] {+ let mut server =+ HttpFailureFixture::spawn_stalled_raw(status, content_type).await?;+ let mut command = Command::new(env!("CARGO_BIN_EXE_moli"));+ command+ .env("MOLI_LAYOUT", "false")+ .args([+ "fetch",+ "--dump",+ dump,+ "--http-no-proxy",+ "*",+ "--redirect-wait-ms",+ "0",+ "--timeout",+ "5000",+ "--wait-until",+ wait_until,+ ])+ .args(flag)+ .arg(format!("{}/raw", server.base_url));+ let output = tokio::time::timeout(+ Duration::from_secs(2),+ tokio::task::spawn_blocking(move || command.output()),+ )+ .await+ .expect("unsupported raw output must return before the body timeout")??;+ assert!(output.stdout.is_empty(), "{output:?}");+ let stderr = String::from_utf8_lossy(&output.stderr);+ if flag == Some("--fail") && status >= 400 {+ server.assert_http_failure(&output, status, "/raw");+ assert!(!stderr.contains("raw download output"), "{stderr}");+ } else {+ assert_eq!(output.status.code(), Some(1), "{output:?}");+ let reason = if dump == "html" {+ "--dump html requires a renderable HTML document"+ } else {+ "raw download output only supports"+ };+ assert!(stderr.contains(reason), "{stderr}");+ assert_eq!(+ stderr.contains(&format!("HTTP status {status}")),+ flag == Some("--fail-with-body") && status >= 400,+ "{stderr}"+ );+ assert_eq!(stderr.lines().count(), 2, "{stderr}");+ }+ assert!(!stderr.contains("timed out"), "{stderr}");+ tokio::time::timeout(Duration::from_secs(2), &mut server.task)+ .await+ .expect("the HTTP download must be cancelled")?;+ }+ }}}Ok(())