Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): cancel raw downloads for unsupported output

ldm0a3784688b753
diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rs
index f5f924f7d..cf35a9878 100644
--- a/moli-core/src/runtime/mod.rs
+++ b/moli-core/src/runtime/mod.rs
@@ -59,6 +59,10 @@ static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);
pub enum RawDocumentFetchPolicy {
/// Read the response body and return [`FetchedDocument::Raw`].
Materialize,
+ /// Return response metadata with an empty body and cancel the download.
+ /// Callers can use the metadata to select an error without reading a body
+ /// that their output format cannot consume.
+ MetadataOnly,
/// Read the body only for statuses below the threshold. Otherwise return
/// response metadata with an empty body and cancel the download.
MaterializeBelowStatus(u16),
@@ -82,6 +86,7 @@ impl RawDocumentFetchPolicy {
fn should_materialize_body(self, status: u16) -> Result<bool> {
match self {
Self::Materialize => Ok(true),
+ Self::MetadataOnly => Ok(false),
Self::MaterializeBelowStatus(threshold) => Ok(status < threshold),
Self::RequirePage => Err(anyhow::Error::new(RawDocumentPageRequired)),
}
diff --git a/moli/src/app.rs b/moli/src/app.rs
index 0a381f91b..0fadeadb5 100644
--- a/moli/src/app.rs
+++ b/moli/src/app.rs
@@ -58,14 +58,14 @@ pub async fn run_cli_with_config<W: Write>(
ReadinessPlan::from_fetch_args(&args, config.fetch.response_wait.clone())?;
let raw_document_output =
fetch_dump::RawDocumentOutputPolicy::from_command(&config.fetch);
- // HTTP failure modes need the response status before deciding
- // whether the selected output format accepts a raw download.
- let raw_document_fetch_policy = if args.fail {
- RawDocumentFetchPolicy::MaterializeBelowStatus(400)
- } else if args.fail_with_body {
- RawDocumentFetchPolicy::Materialize
- } else {
- raw_document_output.fetch_policy()
+ // Preserve status metadata for HTTP error precedence, but never
+ // consume a raw body that the selected output format cannot use.
+ let raw_document_fetch_policy = match raw_document_output.fetch_policy() {
+ RawDocumentFetchPolicy::RequirePage if args.fail || args.fail_with_body => {
+ RawDocumentFetchPolicy::MetadataOnly
+ }
+ _ if args.fail => RawDocumentFetchPolicy::MaterializeBelowStatus(400),
+ policy => policy,
};
let request = build_fetch_request(&args.url, &config)?;
if config.browser.fetch().obey_robots() {
diff --git a/moli/tests/fetch_cli/http_fail.rs b/moli/tests/fetch_cli/http_fail.rs
index f4dfdeb0d..179a5f265 100644
--- a/moli/tests/fetch_cli/http_fail.rs
+++ b/moli/tests/fetch_cli/http_fail.rs
@@ -473,38 +473,67 @@ async fn fail_checks_raw_http_status_before_output_format_and_page_options() ->
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn fail_cancels_raw_http_errors_without_waiting_for_the_body() -> Result<()> {
- for (status, content_type) in [(403, "application/octet-stream"), (503, "application/pdf")] {
+async fn unsupported_raw_output_cancels_without_waiting_for_the_body() -> Result<()> {
+ for (status, content_type) in [
+ (200, "application/octet-stream"),
+ (403, "application/octet-stream"),
+ (503, "application/pdf"),
+ ] {
for wait_until in ["done", "networkidle"] {
- let mut server = HttpFailureFixture::spawn_stalled_raw(status, content_type).await?;
- let mut command = Command::new(env!("CARGO_BIN_EXE_moli"));
- command
- .args([
- "fetch",
- "--fail",
- "--dump",
- "markdown",
- "--http-no-proxy",
- "*",
- "--redirect-wait-ms",
- "0",
- "--timeout",
- "5000",
- "--wait-until",
- wait_until,
- ])
- .arg(format!("{}/raw", server.base_url));
- let output = tokio::time::timeout(
- Duration::from_secs(2),
- tokio::task::spawn_blocking(move || command.output()),
- )
- .await
- .expect("--fail must return before the raw body timeout")??;
- server.assert_http_failure(&output, status, "/raw");
- assert!(output.stdout.is_empty(), "{output:?}");
- tokio::time::timeout(Duration::from_secs(2), &mut server.task)
- .await
- .expect("the HTTP download must be cancelled")?;
+ for flag in [None, Some("--fail"), Some("--fail-with-body")] {
+ for dump in ["html", "markdown"] {
+ let mut server =
+ HttpFailureFixture::spawn_stalled_raw(status, content_type).await?;
+ let mut command = Command::new(env!("CARGO_BIN_EXE_moli"));
+ command
+ .env("MOLI_LAYOUT", "false")
+ .args([
+ "fetch",
+ "--dump",
+ dump,
+ "--http-no-proxy",
+ "*",
+ "--redirect-wait-ms",
+ "0",
+ "--timeout",
+ "5000",
+ "--wait-until",
+ wait_until,
+ ])
+ .args(flag)
+ .arg(format!("{}/raw", server.base_url));
+ let output = tokio::time::timeout(
+ Duration::from_secs(2),
+ tokio::task::spawn_blocking(move || command.output()),
+ )
+ .await
+ .expect("unsupported raw output must return before the body timeout")??;
+ assert!(output.stdout.is_empty(), "{output:?}");
+ let stderr = String::from_utf8_lossy(&output.stderr);
+ if flag == Some("--fail") && status >= 400 {
+ server.assert_http_failure(&output, status, "/raw");
+ assert!(!stderr.contains("raw download output"), "{stderr}");
+ } else {
+ assert_eq!(output.status.code(), Some(1), "{output:?}");
+ let reason = if dump == "html" {
+ "--dump html requires a renderable HTML document"
+ } else {
+ "raw download output only supports"
+ };
+ assert!(stderr.contains(reason), "{stderr}");
+ assert_eq!(
+ stderr.contains(&format!("HTTP status {status}")),
+ flag == Some("--fail-with-body") && status >= 400,
+ "{stderr}"
+ );
+ assert_eq!(stderr.lines().count(), 2, "{stderr}");
+ }
+ assert!(!stderr.contains("timed out"), "{stderr}");
+ tokio::time::timeout(Duration::from_secs(2), &mut server.task)
+ .await
+ .expect("the HTTP download must be cancelled")?;
+ }
+ }
}
}
Ok(())