Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

feat(fetch): allow explicit interaction before page dump

lanyue-llk96218551c4ea
diff --git a/README.md b/README.md
index c1f0eee57..8dfd19478 100644
--- a/README.md
+++ b/README.md
@@ -139,6 +139,19 @@ moli fetch --layout --dump screenshot_full https://example.com > full-page.png
moli fetch --layout --dump pdf https://example.com > page.pdf
```
+To export a page after an explicit interaction, combine `--eval` (or
+`--eval-file`) with `--dump`. The expression runs after page readiness;
+promises are awaited, its return value is discarded, and the resulting page
+state is exported. An exception fails the command without emitting a dump.
+
+```bash
+moli fetch --eval 'document.querySelector("button.show-details").click()' \
+ --dump markdown https://example.com
+```
+
+Only the supplied expression performs interactions. Without `--dump`,
+`--eval` retains its expression-value output behavior.
+
Run `fetch --help` for the complete option list, including output formats,
page-load/response waits, profiles, proxy settings, resource policies, and
tracing options.
diff --git a/moli/src/app.rs b/moli/src/app.rs
index 0fadeadb5..54ca9a0b5 100644
--- a/moli/src/app.rs
+++ b/moli/src/app.rs
@@ -191,7 +191,14 @@ pub async fn run_cli_with_config<W: Write>(
.context("failed to publish layout before evaluating JavaScript")
.map_err(with_output_context)?;
}
- eval_output::evaluate(&mut page, expression).await
+ if args.dump.is_some() {
+ eval_output::execute(&mut page, expression)
+ .await
+ .map_err(with_output_context)?;
+ fetch_dump::render_page_output_async(&mut page, &config.fetch).await
+ } else {
+ eval_output::evaluate(&mut page, expression).await
+ }
} else {
fetch_dump::render_page_output_async(&mut page, &config.fetch).await
}
diff --git a/moli/src/cli.rs b/moli/src/cli.rs
index da4a11ce6..4dc5b4e5e 100644
--- a/moli/src/cli.rs
+++ b/moli/src/cli.rs
@@ -128,30 +128,17 @@ pub struct FetchArgs {
#[arg(long, conflicts_with = "fail")]
pub fail_with_body: bool,
- /// Evaluate one JavaScript expression after page readiness and write its
- /// value to stdout. Promises are awaited. Strings are written as text;
- /// other serializable values are written as compact JSON.
- #[arg(
- long,
- value_name = "EXPRESSION",
- conflicts_with_all = [
- "eval_file",
- "dump",
- "with_base",
- "with_frames",
- "strip_mode"
- ]
- )]
+ /// Evaluate one expression after page readiness. Promises are awaited.
+ /// With --dump, discard its value and dump the resulting page state.
+ /// Otherwise write strings as text and other serializable values as JSON.
+ #[arg(long, value_name = "EXPRESSION", conflicts_with = "eval_file")]
pub eval: Option<String>,
/// Read the JavaScript expression from a UTF-8 file, or from stdin when PATH
/// is `-`. This avoids command-line argument length limits for large or
- /// multiline scripts. Promises are awaited.
- #[arg(
- long,
- value_name = "PATH",
- conflicts_with_all = ["eval", "dump", "with_base", "with_frames", "strip_mode"]
- )]
+ /// multiline scripts. Promises are awaited. With --dump, export the
+ /// resulting page state instead of the expression value.
+ #[arg(long, value_name = "PATH", conflicts_with = "eval")]
pub eval_file: Option<PathBuf>,
#[arg(short = 'H', long = "header", value_name = "HEADER", value_parser = parse_request_header_arg)]
diff --git a/moli/src/config.rs b/moli/src/config.rs
index ae0067665..a99a263ce 100644
--- a/moli/src/config.rs
+++ b/moli/src/config.rs
@@ -31,6 +31,12 @@ impl AppConfig {
match &cli.command {
Commands::Fetch(args) => {
+ if (args.eval.is_some() || args.eval_file.is_some())
+ && args.dump.is_none()
+ && (args.with_base || args.with_frames || !args.strip_mode.is_empty())
+ {
+ bail!("page output options with --eval or --eval-file require --dump");
+ }
if args.trace_network && args.dump != Some(DumpFormat::Json) {
bail!("--trace-network requires --dump json");
}
diff --git a/moli/src/eval_output.rs b/moli/src/eval_output.rs
index 1b8cb949b..18385c176 100644
--- a/moli/src/eval_output.rs
+++ b/moli/src/eval_output.rs
@@ -10,10 +10,23 @@ pub async fn evaluate(page: &mut Page, expression: &str) -> Result<Vec<u8>> {
render_result(&result)
}
-fn render_result(result: &Value) -> Result<Vec<u8>> {
+pub async fn execute(page: &mut Page, expression: &str) -> Result<()> {
+ let result = page
+ .evaluate_runtime_expression_by_value_async(expression)
+ .await
+ .context("failed to evaluate JavaScript expression")?;
+ reject_exception(&result)
+}
+
+fn reject_exception(result: &Value) -> Result<()> {
if let Some(exception) = result.get("exception").and_then(Value::as_str) {
bail!("JavaScript evaluation failed: {exception}");
}
+ Ok(())
+}
+
+fn render_result(result: &Value) -> Result<Vec<u8>> {
+ reject_exception(result)?;
if result.get("type").and_then(Value::as_str) == Some("undefined") {
return Ok(b"undefined\n".to_vec());
diff --git a/moli/tests/cli.rs b/moli/tests/cli.rs
index df4e38ce5..1c5052f3b 100644
--- a/moli/tests/cli.rs
+++ b/moli/tests/cli.rs
@@ -1029,23 +1029,42 @@ fn fetch_eval_sources_are_mutually_exclusive() {
}
#[test]
-fn fetch_eval_rejects_page_dump_output_options() {
+fn fetch_eval_can_prepare_page_dump_output() {
for eval_args in [
&["--eval", "document.title"][..],
&["--eval-file", "extract.js"][..],
] {
- for conflicting_args in [
- &["--dump", "json"][..],
+ for dump in ["html", "markdown", "json"] {
+ let mut args = vec!["moli", "fetch"];
+ args.extend_from_slice(eval_args);
+ args.extend_from_slice(&[
+ "--dump",
+ dump,
+ "--with-base",
+ "--with-frames",
+ "--strip-mode",
+ "js",
+ ]);
+ args.push("https://example.com");
+ let cli = Cli::try_parse_from(normalize_args_for_compat(args)).unwrap();
+ AppConfig::from_cli(&cli).unwrap();
+ }
+ }
+}
+
+#[test]
+fn eval_value_output_rejects_unused_page_options() {
+ for eval in ["--eval", "--eval-file"] {
+ for option in [
&["--with-base"][..],
&["--with-frames"][..],
&["--strip-mode", "js"][..],
] {
- let mut args = vec!["moli", "fetch"];
- args.extend_from_slice(eval_args);
- args.extend_from_slice(conflicting_args);
+ let mut args = vec!["moli", "fetch", eval, "prepare.js"];
+ args.extend_from_slice(option);
args.push("https://example.com");
- let error = Cli::try_parse_from(normalize_args_for_compat(args)).unwrap_err();
- assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict);
+ let cli = Cli::try_parse_from(normalize_args_for_compat(args)).unwrap();
+ assert!(AppConfig::from_cli(&cli).is_err());
}
}
}
diff --git a/moli/tests/fetch_cli/eval.rs b/moli/tests/fetch_cli/eval.rs
index 5051e9775..53870ae1e 100644
--- a/moli/tests/fetch_cli/eval.rs
+++ b/moli/tests/fetch_cli/eval.rs
@@ -65,6 +65,62 @@ fn run_eval_source(
run_moli(args)
}
+#[test]
+fn eval_prepares_live_page_before_markdown_dump() -> Result<()> {
+ // Exercise a named handler and an asynchronous listener, rather than
+ // modifying visibility from the extraction expression itself.
+ let url = "data:text/html,<body><button id='open' onclick='reveal()'>Open</button><button id='async'>Async</button><button onclick='document.body.append(\"Unrequested content\")'>Other</button><div id='detail'></div><div id='later'></div><script>function reveal(){document.getElementById('detail').textContent='Match detail'};document.getElementById('async').addEventListener('click',()=>setTimeout(()=>document.getElementById('later').textContent='Delayed detail',0))</script>";
+ let script = "(async()=>{document.getElementById('open').click();document.getElementById('async').click();await new Promise(r=>setTimeout(r,20));return 'Do not print this value'})()";
+ for source in ["--eval", "--eval-file"] {
+ let path = unique_temp_file_path("prepare-dump", "prepare.js")?;
+ std::fs::write(&path, script)?;
+ let expression = if source == "--eval" {
+ OsStr::new(script)
+ } else {
+ path.as_os_str()
+ };
+ let output = run_eval_source(url, source, expression, &["--dump", "markdown"])?;
+ std::fs::remove_file(path)?;
+ assert!(output.status.success(), "{}", clean_output(&output.stderr));
+ let text = clean_output(&output.stdout);
+ assert!(text.contains("Match detail"), "{text}");
+ assert!(text.contains("Delayed detail"), "{text}");
+ assert!(!text.contains("Unrequested content"), "{text}");
+ assert!(!text.contains("Do not print this value"), "{text}");
+ }
+ Ok(())
+}
+
+#[test]
+fn eval_failure_does_not_emit_a_page_dump() -> Result<()> {
+ for script in [
+ "throw new Error('preparation failed')",
+ "Promise.reject(new Error('preparation failed'))",
+ ] {
+ let output = run_eval(
+ "data:text/html,<p>Do not emit this page</p>",
+ script,
+ &["--dump", "markdown"],
+ )?;
+ assert!(!output.status.success());
+ assert!(output.stdout.is_empty());
+ assert!(clean_output(&output.stderr).contains("preparation failed"));
+ }
+ Ok(())
+}
+
+#[test]
+fn eval_can_dump_html_without_a_serializable_return_value() -> Result<()> {
+ let output = run_eval(
+ "data:text/html,<p>Initial</p>",
+ "document.querySelector('p').textContent='Edited'; document.body",
+ &["--dump", "html"],
+ )?;
+ assert!(output.status.success(), "{}", clean_output(&output.stderr));
+ assert!(clean_output(&output.stdout).contains("Edited"));
+ Ok(())
+}
+
#[test]
fn eval_can_use_realtime_audio_oscillator_and_analyser_shims() -> Result<()> {
let output = run_eval(