Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
Preserve raw bodies in fetch JSON output
ldm08c34e035d3b2
diff --git a/Cargo.lock b/Cargo.lockindex 0eb0e30c9..16c2026d3 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2278,6 +2278,7 @@ version = "1.1.2"dependencies = ["anyhow","axum",+ "base64 0.22.1","cidr","clap","libc",diff --git a/moli/Cargo.toml b/moli/Cargo.tomlindex 7ac825d96..e18ed56d8 100644--- a/moli/Cargo.toml+++ b/moli/Cargo.toml@@ -15,6 +15,7 @@ jemalloc = ["dep:tikv-jemalloc-sys", "dep:tikv-jemallocator"][dependencies]anyhow = "1.0.100"axum = { version = "0.8.6", features = ["ws"] }+base64 = "0.22.1"cidr = "0.3.2"clap = { version = "4.5.39", features = ["derive", "env"] }moli-browser-profile = { path = "../moli-browser-profile" }diff --git a/moli/src/fetch_dump/dom.rs b/moli/src/fetch_dump/dom.rsindex 0a0064d9c..69bba7c3d 100644--- a/moli/src/fetch_dump/dom.rs+++ b/moli/src/fetch_dump/dom.rs@@ -1,4 +1,5 @@use anyhow::Result;+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};use moli_core::page::{Page, RendererPageDumpFormat, RendererPageDumpOptions, RendererPageDumpStripOptions,SubresourceResponseWaitCriteria,@@ -59,6 +60,39 @@ pub(super) fn render_json_payload(network: Option<Value>,) -> Result<String> {// Keep --dump json as a stable machine interface for scrapling-style callers.+ let mut payload = response_metadata(final_url, status, title, headers, redirect_chain);+ payload.insert("html".to_owned(), json!(html));+ if let Some(network) = network {+ payload.insert("network".to_owned(), network);+ }+ Ok(serde_json::to_string_pretty(&Value::Object(payload))?)+}++pub(super) fn render_raw_json_payload(+ final_url: &str,+ status: u16,+ headers: &[(String, String)],+ redirect_chain: &[moli_core::page::NavigationRedirect],+ body: &[u8],+) -> Result<String> {+ let mut payload = response_metadata(final_url, status, None, headers, redirect_chain);+ payload.insert("html".to_owned(), Value::Null);+ // Raw responses may contain arbitrary bytes. Always use one lossless+ // representation instead of changing the schema based on UTF-8 validity.+ payload.insert(+ "body_base64".to_owned(),+ Value::String(BASE64_STANDARD.encode(body)),+ );+ Ok(serde_json::to_string_pretty(&Value::Object(payload))?)+}++fn response_metadata(+ final_url: &str,+ status: u16,+ title: Option<&str>,+ headers: &[(String, String)],+ redirect_chain: &[moli_core::page::NavigationRedirect],+) -> Map<String, Value> {let mut payload = Map::new();payload.insert("final_url".to_owned(), json!(final_url));payload.insert("status".to_owned(), json!(status));@@ -73,11 +107,7 @@ pub(super) fn render_json_payload(.collect(),),);- payload.insert("html".to_owned(), json!(html));- if let Some(network) = network {- payload.insert("network".to_owned(), network);- }- Ok(serde_json::to_string_pretty(&Value::Object(payload))?)+ payload}fn render_headers(headers: &[(String, String)]) -> Value {diff --git a/moli/src/fetch_dump/mod.rs b/moli/src/fetch_dump/mod.rsindex 244ba325b..aee350e32 100644--- a/moli/src/fetch_dump/mod.rs+++ b/moli/src/fetch_dump/mod.rs@@ -104,21 +104,18 @@ pub(crate) fn render_raw_document_output(match policy {RawDocumentOutputPolicy::Passthrough => Ok(raw.body_bytes().to_vec()),RawDocumentOutputPolicy::Json => {- let html = String::from_utf8_lossy(raw.body_bytes());let redirect_chain = raw.navigation_redirect_chain().iter().cloned().map(Into::into).collect::<Vec<_>>();- let payload = dom::render_json_payload(+ let payload = dom::render_raw_json_payload(raw.final_url().as_str(),raw.status(),- None,raw.headers(),&redirect_chain,- &html,- None,+ raw.body_bytes(),)?;Ok(payload.into_bytes())}diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex c504a8304..925053342 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -8,6 +8,7 @@ use axum::{response::{Html, IntoResponse, Response},routing::get,};+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};use clap::Parser;use moli::{app,@@ -273,6 +274,47 @@ fn assert_json_dump_shape(payload: &Value, expected_url: &str, expected_status:assert!(payload["html"].is_string(), "payload={payload}");}+fn assert_raw_json_dump_shape(+ payload: &Value,+ expected_url: &str,+ expected_status: u64,+ expected_body: &[u8],+) {+ let Some(object) = payload.as_object() else {+ panic!("expected top-level JSON object, got {payload}");+ };++ let mut keys = object.keys().cloned().collect::<Vec<_>>();+ keys.sort();+ assert_eq!(+ keys,+ vec![+ "body_base64",+ "final_url",+ "headers",+ "html",+ "redirect_chain",+ "status",+ "title",+ ]+ );+ assert_eq!(payload["final_url"], expected_url);+ assert_eq!(payload["status"], expected_status);+ assert!(payload["title"].is_null(), "payload={payload}");+ assert!(payload["headers"].is_array(), "payload={payload}");+ assert!(payload["redirect_chain"].is_array(), "payload={payload}");+ assert!(payload["html"].is_null(), "payload={payload}");+ let encoded = payload["body_base64"]+ .as_str()+ .expect("raw JSON body must be a base64 string");+ assert_eq!(+ BASE64_STANDARD+ .decode(encoded)+ .expect("raw JSON body must contain valid base64"),+ expected_body+ );+}+fn unique_temp_dir(name: &str) -> Result<PathBuf> {let nonce = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();let dir = std::env::temp_dir().join(format!("moli-{name}-{}-{nonce}", std::process::id()));@@ -293,6 +335,13 @@ struct CacheableFixtureServer {const PDF_FIXTURE_BODY: &[u8] =b"%PDF-1.7\n% moli raw document fixture\n1 0 obj\n<<>>\nendobj\n%%EOF\n";const VIDEO_FIXTURE_BODY: &[u8] = b"\0\0\0\x18ftypmp42\xff\0moli-video-fixture";+const PNG_FIXTURE_BODY: &[u8] = &[+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,+ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4,+ 0x89, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x44, 0x41, 0x54, 0x08, 0x1d, 0x63, 0xf8, 0xcf, 0xc0, 0xf0,+ 0x1f, 0x00, 0x05, 0x80, 0x02, 0x3f, 0x49, 0xc2, 0xfa, 0x39, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,+ 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,+];struct BinaryDocumentFixtureServer {base_url: String,@@ -529,11 +578,16 @@ impl BinaryDocumentFixtureServer {)}+ async fn inline_png() -> impl IntoResponse {+ ([("content-type", "image/png")], PNG_FIXTURE_BODY.to_vec())+ }+let listener = TcpListener::bind("127.0.0.1:0").await?;let addr = listener.local_addr()?;let app = Router::new().route("/inline.pdf", get(inline_pdf)).route("/clip.mp4", get(inline_video))+ .route("/pixel.png", get(inline_png)).route("/attachment.html", get(attachment_pdf));let task = tokio::spawn(async move {axum::serve(listener, app)@@ -1117,6 +1171,33 @@ fn cli_fetch_without_dump_writes_attachment_verbatim() -> Result<()> {Ok(())}+#[test]+fn cli_dump_json_encodes_raw_documents_losslessly() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(BinaryDocumentFixtureServer::spawn())?;++ for (path, expected_body) in [+ ("/pixel.png", PNG_FIXTURE_BODY),+ ("/inline.pdf", PDF_FIXTURE_BODY),+ ("/clip.mp4", VIDEO_FIXTURE_BODY),+ ("/attachment.html", PDF_FIXTURE_BODY),+ ] {+ let url = server.url(path);+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;+ assert!(+ output.status.success(),+ "path={path} stdout={} stderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let payload: Value = serde_json::from_slice(&output.stdout)?;+ assert_raw_json_dump_shape(&payload, &url, 200, expected_body);+ }++ runtime.block_on(server.shutdown());+ Ok(())+}+#[test]fn cli_dump_html_rejects_raw_documents_without_partial_output() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;@@ -3476,10 +3557,8 @@ fn cli_default_done_returns_raw_http_error_status_and_body() -> Result<()> {String::from_utf8_lossy(&output.stdout),String::from_utf8_lossy(&output.stderr));- let stdout = clean_output(&output.stdout);- let payload: Value = serde_json::from_str(&stdout)?;- assert_json_dump_shape(&payload, &url, 404);- assert_eq!(payload["html"], "raw-error-body");+ let payload: Value = serde_json::from_slice(&output.stdout)?;+ assert_raw_json_dump_shape(&payload, &url, 404, b"raw-error-body");Ok(())}@@ -3746,10 +3825,8 @@ fn cli_slow_streaming_raw_404_returns_complete_status_and_body() -> Result<()> {String::from_utf8_lossy(&output.stdout),String::from_utf8_lossy(&output.stderr));- let stdout = clean_output(&output.stdout);- let payload: Value = serde_json::from_str(&stdout)?;- assert_json_dump_shape(&payload, &url, 404);- assert_eq!(payload["html"], "slow-raw-404-head|slow-raw-404-tail");+ let payload: Value = serde_json::from_slice(&output.stdout)?;+ assert_raw_json_dump_shape(&payload, &url, 404, b"slow-raw-404-head|slow-raw-404-tail");Ok(())}diff --git a/skills/moli-webfetch/SKILL.md b/skills/moli-webfetch/SKILL.mdindex 5d7c54465..6ca16e2a4 100644--- a/skills/moli-webfetch/SKILL.md+++ b/skills/moli-webfetch/SKILL.md@@ -83,7 +83,8 @@ structure-first; enable layout only when the result needs pixels or pagination.when roles and accessible names matter.- Use `json` for automation that needs `final_url`, HTTP `status`, `title`,duplicate-safe response `headers`, the main-navigation `redirect_chain`,- serialized `html`, or network trace data.+ serialized `html`, or network trace data. For a raw download, `html` is null+ and `body_base64` contains the exact response bytes.- Use `html` to diagnose DOM serialization or preserve exact markup.- Use `--eval` for a focused value or structured extraction from the live pagewithout dumping the full DOM.diff --git a/skills/moli-webfetch/references/fetch-recipes.md b/skills/moli-webfetch/references/fetch-recipes.mdindex c593c4aeb..6e135d19f 100644--- a/skills/moli-webfetch/references/fetch-recipes.md+++ b/skills/moli-webfetch/references/fetch-recipes.md@@ -18,14 +18,17 @@| Read page content | `--dump markdown` | Default for research and summarization || Inspect semantic structure | `--dump semantic_tree_text` | Compact roles, labels, text, and backend node IDs || Process semantic structure | `--dump semantic_tree` | Structured accessibility-oriented payload |-| Process stable fields | `--dump json` | Returns `final_url`, `status`, `title`, `headers`, `redirect_chain`, and `html` |+| Process stable fields | `--dump json` | Returns response metadata plus rendered `html` or a raw `body_base64` || Inspect exact DOM | `--dump html` | Useful when Markdown loses important structure || Diagnose requests | `--dump json --trace-network` | Adds the `network` object || Capture the viewport | `--layout --dump screenshot` | Writes PNG bytes to stdout || Capture the full document | `--layout --dump screenshot_full` | Writes one full-page PNG to stdout || Capture a paginated document | `--layout --dump pdf` | Writes PDF bytes to stdout |-Raw non-HTML responses support only `html` and `json`.+When `--dump` is omitted, raw downloads are written byte-for-byte to stdout.+With `--dump json`, their `html` and `title` fields are null and the exact body+is encoded in `body_base64`. Other explicit dump formats require a renderable+HTML document.JSON `headers` is an ordered list of `{name, value}` records so duplicateheaders are preserved. `redirect_chain` contains every main-navigation HTTP