Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(cli): read wait scripts from stdin
ldm0437a82abc807
diff --git a/moli/src/app.rs b/moli/src/app.rsindex bad461f61..49252a740 100644--- a/moli/src/app.rs+++ b/moli/src/app.rs@@ -11,7 +11,7 @@ use std::{};use crate::{- cli::{Cli, Commands, normalize_args_for_compat},+ cli::{Cli, Commands, FetchArgs, normalize_args_for_compat},config::AppConfig,cookie_cache, eval_output, fetch_dump, robots,};@@ -47,21 +47,14 @@ pub async fn run_cli_with_config<W: Write>() -> Result<()> {match cli.command {Commands::Fetch(mut args) => {+ reject_multiple_stdin_script_sources(&args)?;let eval_expression = if let Some(path) = args.eval_file.as_deref() {- if path == Path::new("-") {- let mut expression = String::new();- std::io::stdin()- .read_to_string(&mut expression)- .context("failed to read --eval-file `-` from stdin")?;- Some(expression)- } else {- Some(std::fs::read_to_string(path).with_context(|| {- format!("failed to read --eval-file `{}`", path.display())- })?)- }+ Some(read_script_file_arg("--eval-file", path)?)} else {args.eval.take()};+ let readiness =+ ReadinessPlan::from_fetch_args(&args, config.fetch.response_wait.clone())?;let request = build_fetch_request(&args.url, &config)?;if config.browser.fetch().obey_robots() {// Checked before the browser starts so a refused fetch costs@@ -72,8 +65,6 @@ pub async fn run_cli_with_config<W: Write>(}let browser = Browser::new(config.browser.clone())?;load_cookie_state(&browser, &config)?;- let readiness =- ReadinessPlan::from_fetch_args(&args, config.fetch.response_wait.clone())?;let fetch_result = readiness.fetch_document(&browser, request).await;let fetched_document = match fetch_result {Ok(document) => document,@@ -177,6 +168,30 @@ pub async fn run_cli_with_config<W: Write>(Ok(())}+fn reject_multiple_stdin_script_sources(args: &FetchArgs) -> Result<()> {+ if args.eval_file.as_deref() == Some(Path::new("-"))+ && args.wait_script_file.as_deref() == Some("-")+ {+ return Err(anyhow!(+ "`--eval-file -` and `--wait-script-file -` cannot both read from stdin"+ ));+ }+ Ok(())+}++fn read_script_file_arg(option: &str, path: &Path) -> Result<String> {+ if path == Path::new("-") {+ let mut source = String::new();+ std::io::stdin()+ .read_to_string(&mut source)+ .with_context(|| format!("failed to read {option} `-` from stdin"))?;+ Ok(source)+ } else {+ std::fs::read_to_string(path)+ .with_context(|| format!("failed to read {option} `{}`", path.display()))+ }+}+fn build_fetch_request(url: &str, config: &AppConfig) -> Result<Request> {let mut request = Request::get(url)?;// Keep CLI-provided headers scoped to the initial document navigation.diff --git a/moli/src/app/readiness.rs b/moli/src/app/readiness.rsindex 167e0729f..b61162e39 100644--- a/moli/src/app/readiness.rs+++ b/moli/src/app/readiness.rs@@ -15,7 +15,7 @@ use moli_core::{runtime::{Browser, FetchDeadline, FetchedDocument, RenderedDomWaitUntil},};use moli_fetch::Request;-use std::time::Duration;+use std::{path::Path, time::Duration};#[derive(Debug)]pub(super) struct ReadinessPlan {@@ -123,9 +123,9 @@ fn resolve_wait_script(args: &FetchArgs) -> Result<Option<String>> {bail!("`--wait-script` and `--wait-script-file` are mutually exclusive")}(Some(script), None) => Ok(Some(script.to_owned())),- (None, Some(path)) => std::fs::read_to_string(path)- .map(Some)- .with_context(|| anyhow!("failed to read wait script file `{path}`")),+ (None, Some(path)) => {+ super::read_script_file_arg("--wait-script-file", Path::new(path)).map(Some)+ }(None, None) => Ok(None),}}diff --git a/moli/src/cli.rs b/moli/src/cli.rsindex acbc0fa1e..4e1b7cac2 100644--- a/moli/src/cli.rs+++ b/moli/src/cli.rs@@ -123,8 +123,9 @@ pub struct FetchArgs {#[arg(long)]pub wait_script: Option<String>,- /// Read the JavaScript wait expression or function predicate from a file.- #[arg(long)]+ /// Read the JavaScript wait expression or function predicate from a UTF-8+ /// file, or from stdin when PATH is `-`.+ #[arg(long, value_name = "PATH")]pub wait_script_file: Option<String>,#[arg(long, default_value_t = 0)]diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex aab4e1410..8d68632c6 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -40,6 +40,8 @@ mod anubis_deferred_module;mod disable_js;#[path = "fetch_cli/eval.rs"]mod eval;+#[path = "fetch_cli/readiness.rs"]+mod readiness;struct Output {status: OutputStatus,diff --git a/moli/tests/fetch_cli/readiness.rs b/moli/tests/fetch_cli/readiness.rsnew file mode 100644index 000000000..b588f1fee--- /dev/null+++ b/moli/tests/fetch_cli/readiness.rs@@ -0,0 +1,104 @@+use super::clean_output;+use anyhow::Result;+use std::{+ io::Write,+ process::{Command, Stdio},+};++fn run_script_stdin(+ url: &str,+ script: &[u8],+ source_args: &[&str],+) -> Result<std::process::Output> {+ let mut command = Command::new(env!("CARGO_BIN_EXE_moli"));+ command+ .args([+ "fetch",+ "--log-level",+ "error",+ "--http-no-proxy",+ "*",+ "--wait-until",+ "load",+ ])+ .args(source_args)+ .arg(url);+ let mut child = command+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::piped())+ .spawn()?;+ child+ .stdin+ .take()+ .expect("piped stdin must be available")+ .write_all(script)?;+ Ok(child.wait_with_output()?)+}++#[test]+fn wait_script_file_dash_reads_predicate_from_stdin() -> Result<()> {+ let output = run_script_stdin(+ "data:text/html,<main>pending</main><script>setTimeout(()=>document.querySelector('main').textContent='ready',20)</script>",+ b"() => document.querySelector('main').textContent === 'ready'",+ &["--wait-script-file", "-", "--dump", "html"],+ )?;++ assert!(+ output.status.success(),+ "stderr={}",+ clean_output(&output.stderr)+ );+ assert!(+ clean_output(&output.stdout).contains("<main>ready</main>"),+ "stdout={}",+ clean_output(&output.stdout)+ );+ Ok(())+}++#[test]+fn stdin_cannot_supply_both_eval_and_wait_scripts() -> Result<()> {+ let output = run_script_stdin(+ "https://stdin-script-conflict-must-not-fetch.invalid/",+ b"() => true",+ &["--eval-file", "-", "--wait-script-file", "-"],+ )?;++ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");+ assert!(stdout.is_empty(), "stdout={stdout}");+ assert!(+ stderr.contains("`--eval-file -` and `--wait-script-file -` cannot both read from stdin"),+ "stderr={stderr}"+ );+ assert!(+ !stderr.contains("failed to fetch"),+ "the ambiguous stdin sources must be rejected before fetching: stderr={stderr}"+ );+ Ok(())+}++#[test]+fn wait_script_file_dash_rejects_invalid_utf8_before_fetching() -> Result<()> {+ let output = run_script_stdin(+ "https://wait-script-file-stdin-must-not-fetch.invalid/",+ b"() => true\xff",+ &["--wait-script-file", "-", "--dump", "html"],+ )?;++ let stdout = clean_output(&output.stdout);+ let stderr = clean_output(&output.stderr);+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");+ assert!(stdout.is_empty(), "stdout={stdout}");+ assert!(+ stderr.contains("failed to read --wait-script-file `-` from stdin"),+ "stderr={stderr}"+ );+ assert!(+ !stderr.contains("failed to fetch"),+ "stdin must be read before starting a fetch: stderr={stderr}"+ );+ Ok(())+}