Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

Preserve raw bodies in fetch JSON output

ldm08c34e035d3b2
diff --git a/Cargo.lock b/Cargo.lock
index 0eb0e30c9..16c2026d3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2278,6 +2278,7 @@ version = "1.1.2"
dependencies = [
"anyhow",
"axum",
+ "base64 0.22.1",
"cidr",
"clap",
"libc",
diff --git a/moli/Cargo.toml b/moli/Cargo.toml
index 7ac825d96..e18ed56d8 100644
--- a/moli/Cargo.toml
+++ b/moli/Cargo.toml
@@ -15,6 +15,7 @@ jemalloc = ["dep:tikv-jemalloc-sys", "dep:tikv-jemallocator"]
[dependencies]
anyhow = "1.0.100"
axum = { version = "0.8.6", features = ["ws"] }
+base64 = "0.22.1"
cidr = "0.3.2"
clap = { version = "4.5.39", features = ["derive", "env"] }
moli-browser-profile = { path = "../moli-browser-profile" }
diff --git a/moli/src/fetch_dump/dom.rs b/moli/src/fetch_dump/dom.rs
index 0a0064d9c..69bba7c3d 100644
--- a/moli/src/fetch_dump/dom.rs
+++ b/moli/src/fetch_dump/dom.rs
@@ -1,4 +1,5 @@
use anyhow::Result;
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use moli_core::page::{
Page, RendererPageDumpFormat, RendererPageDumpOptions, RendererPageDumpStripOptions,
SubresourceResponseWaitCriteria,
@@ -59,6 +60,39 @@ pub(super) fn render_json_payload(
network: Option<Value>,
) -> Result<String> {
// Keep --dump json as a stable machine interface for scrapling-style callers.
+ let mut payload = response_metadata(final_url, status, title, headers, redirect_chain);
+ payload.insert("html".to_owned(), json!(html));
+ if let Some(network) = network {
+ payload.insert("network".to_owned(), network);
+ }
+ Ok(serde_json::to_string_pretty(&Value::Object(payload))?)
+}
+
+pub(super) fn render_raw_json_payload(
+ final_url: &str,
+ status: u16,
+ headers: &[(String, String)],
+ redirect_chain: &[moli_core::page::NavigationRedirect],
+ body: &[u8],
+) -> Result<String> {
+ let mut payload = response_metadata(final_url, status, None, headers, redirect_chain);
+ payload.insert("html".to_owned(), Value::Null);
+ // Raw responses may contain arbitrary bytes. Always use one lossless
+ // representation instead of changing the schema based on UTF-8 validity.
+ payload.insert(
+ "body_base64".to_owned(),
+ Value::String(BASE64_STANDARD.encode(body)),
+ );
+ Ok(serde_json::to_string_pretty(&Value::Object(payload))?)
+}
+
+fn response_metadata(
+ final_url: &str,
+ status: u16,
+ title: Option<&str>,
+ headers: &[(String, String)],
+ redirect_chain: &[moli_core::page::NavigationRedirect],
+) -> Map<String, Value> {
let mut payload = Map::new();
payload.insert("final_url".to_owned(), json!(final_url));
payload.insert("status".to_owned(), json!(status));
@@ -73,11 +107,7 @@ pub(super) fn render_json_payload(
.collect(),
),
);
- payload.insert("html".to_owned(), json!(html));
- if let Some(network) = network {
- payload.insert("network".to_owned(), network);
- }
- Ok(serde_json::to_string_pretty(&Value::Object(payload))?)
+ payload
}
fn render_headers(headers: &[(String, String)]) -> Value {
diff --git a/moli/src/fetch_dump/mod.rs b/moli/src/fetch_dump/mod.rs
index 244ba325b..aee350e32 100644
--- a/moli/src/fetch_dump/mod.rs
+++ b/moli/src/fetch_dump/mod.rs
@@ -104,21 +104,18 @@ pub(crate) fn render_raw_document_output(
match policy {
RawDocumentOutputPolicy::Passthrough => Ok(raw.body_bytes().to_vec()),
RawDocumentOutputPolicy::Json => {
- let html = String::from_utf8_lossy(raw.body_bytes());
let redirect_chain = raw
.navigation_redirect_chain()
.iter()
.cloned()
.map(Into::into)
.collect::<Vec<_>>();
- let payload = dom::render_json_payload(
+ let payload = dom::render_raw_json_payload(
raw.final_url().as_str(),
raw.status(),
- None,
raw.headers(),
&redirect_chain,
- &html,
- None,
+ raw.body_bytes(),
)?;
Ok(payload.into_bytes())
}
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index c504a8304..925053342 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -8,6 +8,7 @@ use axum::{
response::{Html, IntoResponse, Response},
routing::get,
};
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use clap::Parser;
use moli::{
app,
@@ -273,6 +274,47 @@ fn assert_json_dump_shape(payload: &Value, expected_url: &str, expected_status:
assert!(payload["html"].is_string(), "payload={payload}");
}
+fn assert_raw_json_dump_shape(
+ payload: &Value,
+ expected_url: &str,
+ expected_status: u64,
+ expected_body: &[u8],
+) {
+ let Some(object) = payload.as_object() else {
+ panic!("expected top-level JSON object, got {payload}");
+ };
+
+ let mut keys = object.keys().cloned().collect::<Vec<_>>();
+ keys.sort();
+ assert_eq!(
+ keys,
+ vec![
+ "body_base64",
+ "final_url",
+ "headers",
+ "html",
+ "redirect_chain",
+ "status",
+ "title",
+ ]
+ );
+ assert_eq!(payload["final_url"], expected_url);
+ assert_eq!(payload["status"], expected_status);
+ assert!(payload["title"].is_null(), "payload={payload}");
+ assert!(payload["headers"].is_array(), "payload={payload}");
+ assert!(payload["redirect_chain"].is_array(), "payload={payload}");
+ assert!(payload["html"].is_null(), "payload={payload}");
+ let encoded = payload["body_base64"]
+ .as_str()
+ .expect("raw JSON body must be a base64 string");
+ assert_eq!(
+ BASE64_STANDARD
+ .decode(encoded)
+ .expect("raw JSON body must contain valid base64"),
+ expected_body
+ );
+}
+
fn unique_temp_dir(name: &str) -> Result<PathBuf> {
let nonce = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();
let dir = std::env::temp_dir().join(format!("moli-{name}-{}-{nonce}", std::process::id()));
@@ -293,6 +335,13 @@ struct CacheableFixtureServer {
const PDF_FIXTURE_BODY: &[u8] =
b"%PDF-1.7\n% moli raw document fixture\n1 0 obj\n<<>>\nendobj\n%%EOF\n";
const VIDEO_FIXTURE_BODY: &[u8] = b"\0\0\0\x18ftypmp42\xff\0moli-video-fixture";
+const PNG_FIXTURE_BODY: &[u8] = &[
+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
+ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4,
+ 0x89, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x44, 0x41, 0x54, 0x08, 0x1d, 0x63, 0xf8, 0xcf, 0xc0, 0xf0,
+ 0x1f, 0x00, 0x05, 0x80, 0x02, 0x3f, 0x49, 0xc2, 0xfa, 0x39, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,
+ 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
+];
struct BinaryDocumentFixtureServer {
base_url: String,
@@ -529,11 +578,16 @@ impl BinaryDocumentFixtureServer {
)
}
+ async fn inline_png() -> impl IntoResponse {
+ ([("content-type", "image/png")], PNG_FIXTURE_BODY.to_vec())
+ }
+
let listener = TcpListener::bind("127.0.0.1:0").await?;
let addr = listener.local_addr()?;
let app = Router::new()
.route("/inline.pdf", get(inline_pdf))
.route("/clip.mp4", get(inline_video))
+ .route("/pixel.png", get(inline_png))
.route("/attachment.html", get(attachment_pdf));
let task = tokio::spawn(async move {
axum::serve(listener, app)
@@ -1117,6 +1171,33 @@ fn cli_fetch_without_dump_writes_attachment_verbatim() -> Result<()> {
Ok(())
}
+#[test]
+fn cli_dump_json_encodes_raw_documents_losslessly() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(BinaryDocumentFixtureServer::spawn())?;
+
+ for (path, expected_body) in [
+ ("/pixel.png", PNG_FIXTURE_BODY),
+ ("/inline.pdf", PDF_FIXTURE_BODY),
+ ("/clip.mp4", VIDEO_FIXTURE_BODY),
+ ("/attachment.html", PDF_FIXTURE_BODY),
+ ] {
+ let url = server.url(path);
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;
+ assert!(
+ output.status.success(),
+ "path={path} stdout={} stderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let payload: Value = serde_json::from_slice(&output.stdout)?;
+ assert_raw_json_dump_shape(&payload, &url, 200, expected_body);
+ }
+
+ runtime.block_on(server.shutdown());
+ Ok(())
+}
+
#[test]
fn cli_dump_html_rejects_raw_documents_without_partial_output() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
@@ -3476,10 +3557,8 @@ fn cli_default_done_returns_raw_http_error_status_and_body() -> Result<()> {
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
- let stdout = clean_output(&output.stdout);
- let payload: Value = serde_json::from_str(&stdout)?;
- assert_json_dump_shape(&payload, &url, 404);
- assert_eq!(payload["html"], "raw-error-body");
+ let payload: Value = serde_json::from_slice(&output.stdout)?;
+ assert_raw_json_dump_shape(&payload, &url, 404, b"raw-error-body");
Ok(())
}
@@ -3746,10 +3825,8 @@ fn cli_slow_streaming_raw_404_returns_complete_status_and_body() -> Result<()> {
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
- let stdout = clean_output(&output.stdout);
- let payload: Value = serde_json::from_str(&stdout)?;
- assert_json_dump_shape(&payload, &url, 404);
- assert_eq!(payload["html"], "slow-raw-404-head|slow-raw-404-tail");
+ let payload: Value = serde_json::from_slice(&output.stdout)?;
+ assert_raw_json_dump_shape(&payload, &url, 404, b"slow-raw-404-head|slow-raw-404-tail");
Ok(())
}
diff --git a/skills/moli-webfetch/SKILL.md b/skills/moli-webfetch/SKILL.md
index 5d7c54465..6ca16e2a4 100644
--- a/skills/moli-webfetch/SKILL.md
+++ b/skills/moli-webfetch/SKILL.md
@@ -83,7 +83,8 @@ structure-first; enable layout only when the result needs pixels or pagination.
when roles and accessible names matter.
- Use `json` for automation that needs `final_url`, HTTP `status`, `title`,
duplicate-safe response `headers`, the main-navigation `redirect_chain`,
- serialized `html`, or network trace data.
+ serialized `html`, or network trace data. For a raw download, `html` is null
+ and `body_base64` contains the exact response bytes.
- Use `html` to diagnose DOM serialization or preserve exact markup.
- Use `--eval` for a focused value or structured extraction from the live page
without dumping the full DOM.
diff --git a/skills/moli-webfetch/references/fetch-recipes.md b/skills/moli-webfetch/references/fetch-recipes.md
index c593c4aeb..6e135d19f 100644
--- a/skills/moli-webfetch/references/fetch-recipes.md
+++ b/skills/moli-webfetch/references/fetch-recipes.md
@@ -18,14 +18,17 @@
| Read page content | `--dump markdown` | Default for research and summarization |
| Inspect semantic structure | `--dump semantic_tree_text` | Compact roles, labels, text, and backend node IDs |
| Process semantic structure | `--dump semantic_tree` | Structured accessibility-oriented payload |
-| Process stable fields | `--dump json` | Returns `final_url`, `status`, `title`, `headers`, `redirect_chain`, and `html` |
+| Process stable fields | `--dump json` | Returns response metadata plus rendered `html` or a raw `body_base64` |
| Inspect exact DOM | `--dump html` | Useful when Markdown loses important structure |
| Diagnose requests | `--dump json --trace-network` | Adds the `network` object |
| Capture the viewport | `--layout --dump screenshot` | Writes PNG bytes to stdout |
| Capture the full document | `--layout --dump screenshot_full` | Writes one full-page PNG to stdout |
| Capture a paginated document | `--layout --dump pdf` | Writes PDF bytes to stdout |
-Raw non-HTML responses support only `html` and `json`.
+When `--dump` is omitted, raw downloads are written byte-for-byte to stdout.
+With `--dump json`, their `html` and `title` fields are null and the exact body
+is encoded in `body_base64`. Other explicit dump formats require a renderable
+HTML document.
JSON `headers` is an ordered list of `{name, value}` records so duplicate
headers are preserved. `redirect_chain` contains every main-navigation HTTP