Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
feat(fetch): allow explicit interaction before page dump
lanyue-llk96218551c4ea
diff --git a/README.md b/README.mdindex c1f0eee57..8dfd19478 100644--- a/README.md+++ b/README.md@@ -139,6 +139,19 @@ moli fetch --layout --dump screenshot_full https://example.com > full-page.pngmoli fetch --layout --dump pdf https://example.com > page.pdf```+To export a page after an explicit interaction, combine `--eval` (or+`--eval-file`) with `--dump`. The expression runs after page readiness;+promises are awaited, its return value is discarded, and the resulting page+state is exported. An exception fails the command without emitting a dump.++```bash+moli fetch --eval 'document.querySelector("button.show-details").click()' \+ --dump markdown https://example.com+```++Only the supplied expression performs interactions. Without `--dump`,+`--eval` retains its expression-value output behavior.+Run `fetch --help` for the complete option list, including output formats,page-load/response waits, profiles, proxy settings, resource policies, andtracing options.diff --git a/moli/src/app.rs b/moli/src/app.rsindex 0fadeadb5..54ca9a0b5 100644--- a/moli/src/app.rs+++ b/moli/src/app.rs@@ -191,7 +191,14 @@ pub async fn run_cli_with_config<W: Write>(.context("failed to publish layout before evaluating JavaScript").map_err(with_output_context)?;}- eval_output::evaluate(&mut page, expression).await+ if args.dump.is_some() {+ eval_output::execute(&mut page, expression)+ .await+ .map_err(with_output_context)?;+ fetch_dump::render_page_output_async(&mut page, &config.fetch).await+ } else {+ eval_output::evaluate(&mut page, expression).await+ }} else {fetch_dump::render_page_output_async(&mut page, &config.fetch).await}diff --git a/moli/src/cli.rs b/moli/src/cli.rsindex da4a11ce6..4dc5b4e5e 100644--- a/moli/src/cli.rs+++ b/moli/src/cli.rs@@ -128,30 +128,17 @@ pub struct FetchArgs {#[arg(long, conflicts_with = "fail")]pub fail_with_body: bool,- /// Evaluate one JavaScript expression after page readiness and write its- /// value to stdout. Promises are awaited. Strings are written as text;- /// other serializable values are written as compact JSON.- #[arg(- long,- value_name = "EXPRESSION",- conflicts_with_all = [- "eval_file",- "dump",- "with_base",- "with_frames",- "strip_mode"- ]- )]+ /// Evaluate one expression after page readiness. Promises are awaited.+ /// With --dump, discard its value and dump the resulting page state.+ /// Otherwise write strings as text and other serializable values as JSON.+ #[arg(long, value_name = "EXPRESSION", conflicts_with = "eval_file")]pub eval: Option<String>,/// Read the JavaScript expression from a UTF-8 file, or from stdin when PATH/// is `-`. This avoids command-line argument length limits for large or- /// multiline scripts. Promises are awaited.- #[arg(- long,- value_name = "PATH",- conflicts_with_all = ["eval", "dump", "with_base", "with_frames", "strip_mode"]- )]+ /// multiline scripts. Promises are awaited. With --dump, export the+ /// resulting page state instead of the expression value.+ #[arg(long, value_name = "PATH", conflicts_with = "eval")]pub eval_file: Option<PathBuf>,#[arg(short = 'H', long = "header", value_name = "HEADER", value_parser = parse_request_header_arg)]diff --git a/moli/src/config.rs b/moli/src/config.rsindex ae0067665..a99a263ce 100644--- a/moli/src/config.rs+++ b/moli/src/config.rs@@ -31,6 +31,12 @@ impl AppConfig {match &cli.command {Commands::Fetch(args) => {+ if (args.eval.is_some() || args.eval_file.is_some())+ && args.dump.is_none()+ && (args.with_base || args.with_frames || !args.strip_mode.is_empty())+ {+ bail!("page output options with --eval or --eval-file require --dump");+ }if args.trace_network && args.dump != Some(DumpFormat::Json) {bail!("--trace-network requires --dump json");}diff --git a/moli/src/eval_output.rs b/moli/src/eval_output.rsindex 1b8cb949b..18385c176 100644--- a/moli/src/eval_output.rs+++ b/moli/src/eval_output.rs@@ -10,10 +10,23 @@ pub async fn evaluate(page: &mut Page, expression: &str) -> Result<Vec<u8>> {render_result(&result)}-fn render_result(result: &Value) -> Result<Vec<u8>> {+pub async fn execute(page: &mut Page, expression: &str) -> Result<()> {+ let result = page+ .evaluate_runtime_expression_by_value_async(expression)+ .await+ .context("failed to evaluate JavaScript expression")?;+ reject_exception(&result)+}++fn reject_exception(result: &Value) -> Result<()> {if let Some(exception) = result.get("exception").and_then(Value::as_str) {bail!("JavaScript evaluation failed: {exception}");}+ Ok(())+}++fn render_result(result: &Value) -> Result<Vec<u8>> {+ reject_exception(result)?;if result.get("type").and_then(Value::as_str) == Some("undefined") {return Ok(b"undefined\n".to_vec());diff --git a/moli/tests/cli.rs b/moli/tests/cli.rsindex df4e38ce5..1c5052f3b 100644--- a/moli/tests/cli.rs+++ b/moli/tests/cli.rs@@ -1029,23 +1029,42 @@ fn fetch_eval_sources_are_mutually_exclusive() {}#[test]-fn fetch_eval_rejects_page_dump_output_options() {+fn fetch_eval_can_prepare_page_dump_output() {for eval_args in [&["--eval", "document.title"][..],&["--eval-file", "extract.js"][..],] {- for conflicting_args in [- &["--dump", "json"][..],+ for dump in ["html", "markdown", "json"] {+ let mut args = vec!["moli", "fetch"];+ args.extend_from_slice(eval_args);+ args.extend_from_slice(&[+ "--dump",+ dump,+ "--with-base",+ "--with-frames",+ "--strip-mode",+ "js",+ ]);+ args.push("https://example.com");+ let cli = Cli::try_parse_from(normalize_args_for_compat(args)).unwrap();+ AppConfig::from_cli(&cli).unwrap();+ }+ }+}++#[test]+fn eval_value_output_rejects_unused_page_options() {+ for eval in ["--eval", "--eval-file"] {+ for option in [&["--with-base"][..],&["--with-frames"][..],&["--strip-mode", "js"][..],] {- let mut args = vec!["moli", "fetch"];- args.extend_from_slice(eval_args);- args.extend_from_slice(conflicting_args);+ let mut args = vec!["moli", "fetch", eval, "prepare.js"];+ args.extend_from_slice(option);args.push("https://example.com");- let error = Cli::try_parse_from(normalize_args_for_compat(args)).unwrap_err();- assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict);+ let cli = Cli::try_parse_from(normalize_args_for_compat(args)).unwrap();+ assert!(AppConfig::from_cli(&cli).is_err());}}}diff --git a/moli/tests/fetch_cli/eval.rs b/moli/tests/fetch_cli/eval.rsindex 5051e9775..53870ae1e 100644--- a/moli/tests/fetch_cli/eval.rs+++ b/moli/tests/fetch_cli/eval.rs@@ -65,6 +65,62 @@ fn run_eval_source(run_moli(args)}+#[test]+fn eval_prepares_live_page_before_markdown_dump() -> Result<()> {+ // Exercise a named handler and an asynchronous listener, rather than+ // modifying visibility from the extraction expression itself.+ let url = "data:text/html,<body><button id='open' onclick='reveal()'>Open</button><button id='async'>Async</button><button onclick='document.body.append(\"Unrequested content\")'>Other</button><div id='detail'></div><div id='later'></div><script>function reveal(){document.getElementById('detail').textContent='Match detail'};document.getElementById('async').addEventListener('click',()=>setTimeout(()=>document.getElementById('later').textContent='Delayed detail',0))</script>";+ let script = "(async()=>{document.getElementById('open').click();document.getElementById('async').click();await new Promise(r=>setTimeout(r,20));return 'Do not print this value'})()";+ for source in ["--eval", "--eval-file"] {+ let path = unique_temp_file_path("prepare-dump", "prepare.js")?;+ std::fs::write(&path, script)?;+ let expression = if source == "--eval" {+ OsStr::new(script)+ } else {+ path.as_os_str()+ };+ let output = run_eval_source(url, source, expression, &["--dump", "markdown"])?;+ std::fs::remove_file(path)?;+ assert!(output.status.success(), "{}", clean_output(&output.stderr));+ let text = clean_output(&output.stdout);+ assert!(text.contains("Match detail"), "{text}");+ assert!(text.contains("Delayed detail"), "{text}");+ assert!(!text.contains("Unrequested content"), "{text}");+ assert!(!text.contains("Do not print this value"), "{text}");+ }+ Ok(())+}++#[test]+fn eval_failure_does_not_emit_a_page_dump() -> Result<()> {+ for script in [+ "throw new Error('preparation failed')",+ "Promise.reject(new Error('preparation failed'))",+ ] {+ let output = run_eval(+ "data:text/html,<p>Do not emit this page</p>",+ script,+ &["--dump", "markdown"],+ )?;+ assert!(!output.status.success());+ assert!(output.stdout.is_empty());+ assert!(clean_output(&output.stderr).contains("preparation failed"));+ }+ Ok(())+}++#[test]+fn eval_can_dump_html_without_a_serializable_return_value() -> Result<()> {+ let output = run_eval(+ "data:text/html,<p>Initial</p>",+ "document.querySelector('p').textContent='Edited'; document.body",+ &["--dump", "html"],+ )?;+ assert!(output.status.success(), "{}", clean_output(&output.stderr));+ assert!(clean_output(&output.stdout).contains("Edited"));+ Ok(())+}+#[test]fn eval_can_use_realtime_audio_oscillator_and_analyser_shims() -> Result<()> {let output = run_eval(