Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(cli): read wait scripts from stdin

ldm0437a82abc807
diff --git a/moli/src/app.rs b/moli/src/app.rs
index bad461f61..49252a740 100644
--- a/moli/src/app.rs
+++ b/moli/src/app.rs
@@ -11,7 +11,7 @@ use std::{
};
use crate::{
- cli::{Cli, Commands, normalize_args_for_compat},
+ cli::{Cli, Commands, FetchArgs, normalize_args_for_compat},
config::AppConfig,
cookie_cache, eval_output, fetch_dump, robots,
};
@@ -47,21 +47,14 @@ pub async fn run_cli_with_config<W: Write>(
) -> Result<()> {
match cli.command {
Commands::Fetch(mut args) => {
+ reject_multiple_stdin_script_sources(&args)?;
let eval_expression = if let Some(path) = args.eval_file.as_deref() {
- if path == Path::new("-") {
- let mut expression = String::new();
- std::io::stdin()
- .read_to_string(&mut expression)
- .context("failed to read --eval-file `-` from stdin")?;
- Some(expression)
- } else {
- Some(std::fs::read_to_string(path).with_context(|| {
- format!("failed to read --eval-file `{}`", path.display())
- })?)
- }
+ Some(read_script_file_arg("--eval-file", path)?)
} else {
args.eval.take()
};
+ let readiness =
+ ReadinessPlan::from_fetch_args(&args, config.fetch.response_wait.clone())?;
let request = build_fetch_request(&args.url, &config)?;
if config.browser.fetch().obey_robots() {
// Checked before the browser starts so a refused fetch costs
@@ -72,8 +65,6 @@ pub async fn run_cli_with_config<W: Write>(
}
let browser = Browser::new(config.browser.clone())?;
load_cookie_state(&browser, &config)?;
- let readiness =
- ReadinessPlan::from_fetch_args(&args, config.fetch.response_wait.clone())?;
let fetch_result = readiness.fetch_document(&browser, request).await;
let fetched_document = match fetch_result {
Ok(document) => document,
@@ -177,6 +168,30 @@ pub async fn run_cli_with_config<W: Write>(
Ok(())
}
+fn reject_multiple_stdin_script_sources(args: &FetchArgs) -> Result<()> {
+ if args.eval_file.as_deref() == Some(Path::new("-"))
+ && args.wait_script_file.as_deref() == Some("-")
+ {
+ return Err(anyhow!(
+ "`--eval-file -` and `--wait-script-file -` cannot both read from stdin"
+ ));
+ }
+ Ok(())
+}
+
+fn read_script_file_arg(option: &str, path: &Path) -> Result<String> {
+ if path == Path::new("-") {
+ let mut source = String::new();
+ std::io::stdin()
+ .read_to_string(&mut source)
+ .with_context(|| format!("failed to read {option} `-` from stdin"))?;
+ Ok(source)
+ } else {
+ std::fs::read_to_string(path)
+ .with_context(|| format!("failed to read {option} `{}`", path.display()))
+ }
+}
+
fn build_fetch_request(url: &str, config: &AppConfig) -> Result<Request> {
let mut request = Request::get(url)?;
// Keep CLI-provided headers scoped to the initial document navigation.
diff --git a/moli/src/app/readiness.rs b/moli/src/app/readiness.rs
index 167e0729f..b61162e39 100644
--- a/moli/src/app/readiness.rs
+++ b/moli/src/app/readiness.rs
@@ -15,7 +15,7 @@ use moli_core::{
runtime::{Browser, FetchDeadline, FetchedDocument, RenderedDomWaitUntil},
};
use moli_fetch::Request;
-use std::time::Duration;
+use std::{path::Path, time::Duration};
#[derive(Debug)]
pub(super) struct ReadinessPlan {
@@ -123,9 +123,9 @@ fn resolve_wait_script(args: &FetchArgs) -> Result<Option<String>> {
bail!("`--wait-script` and `--wait-script-file` are mutually exclusive")
}
(Some(script), None) => Ok(Some(script.to_owned())),
- (None, Some(path)) => std::fs::read_to_string(path)
- .map(Some)
- .with_context(|| anyhow!("failed to read wait script file `{path}`")),
+ (None, Some(path)) => {
+ super::read_script_file_arg("--wait-script-file", Path::new(path)).map(Some)
+ }
(None, None) => Ok(None),
}
}
diff --git a/moli/src/cli.rs b/moli/src/cli.rs
index acbc0fa1e..4e1b7cac2 100644
--- a/moli/src/cli.rs
+++ b/moli/src/cli.rs
@@ -123,8 +123,9 @@ pub struct FetchArgs {
#[arg(long)]
pub wait_script: Option<String>,
- /// Read the JavaScript wait expression or function predicate from a file.
- #[arg(long)]
+ /// Read the JavaScript wait expression or function predicate from a UTF-8
+ /// file, or from stdin when PATH is `-`.
+ #[arg(long, value_name = "PATH")]
pub wait_script_file: Option<String>,
#[arg(long, default_value_t = 0)]
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index aab4e1410..8d68632c6 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -40,6 +40,8 @@ mod anubis_deferred_module;
mod disable_js;
#[path = "fetch_cli/eval.rs"]
mod eval;
+#[path = "fetch_cli/readiness.rs"]
+mod readiness;
struct Output {
status: OutputStatus,
diff --git a/moli/tests/fetch_cli/readiness.rs b/moli/tests/fetch_cli/readiness.rs
new file mode 100644
index 000000000..b588f1fee
--- /dev/null
+++ b/moli/tests/fetch_cli/readiness.rs
@@ -0,0 +1,104 @@
+use super::clean_output;
+use anyhow::Result;
+use std::{
+ io::Write,
+ process::{Command, Stdio},
+};
+
+fn run_script_stdin(
+ url: &str,
+ script: &[u8],
+ source_args: &[&str],
+) -> Result<std::process::Output> {
+ let mut command = Command::new(env!("CARGO_BIN_EXE_moli"));
+ command
+ .args([
+ "fetch",
+ "--log-level",
+ "error",
+ "--http-no-proxy",
+ "*",
+ "--wait-until",
+ "load",
+ ])
+ .args(source_args)
+ .arg(url);
+ let mut child = command
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()?;
+ child
+ .stdin
+ .take()
+ .expect("piped stdin must be available")
+ .write_all(script)?;
+ Ok(child.wait_with_output()?)
+}
+
+#[test]
+fn wait_script_file_dash_reads_predicate_from_stdin() -> Result<()> {
+ let output = run_script_stdin(
+ "data:text/html,<main>pending</main><script>setTimeout(()=>document.querySelector('main').textContent='ready',20)</script>",
+ b"() => document.querySelector('main').textContent === 'ready'",
+ &["--wait-script-file", "-", "--dump", "html"],
+ )?;
+
+ assert!(
+ output.status.success(),
+ "stderr={}",
+ clean_output(&output.stderr)
+ );
+ assert!(
+ clean_output(&output.stdout).contains("<main>ready</main>"),
+ "stdout={}",
+ clean_output(&output.stdout)
+ );
+ Ok(())
+}
+
+#[test]
+fn stdin_cannot_supply_both_eval_and_wait_scripts() -> Result<()> {
+ let output = run_script_stdin(
+ "https://stdin-script-conflict-must-not-fetch.invalid/",
+ b"() => true",
+ &["--eval-file", "-", "--wait-script-file", "-"],
+ )?;
+
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");
+ assert!(stdout.is_empty(), "stdout={stdout}");
+ assert!(
+ stderr.contains("`--eval-file -` and `--wait-script-file -` cannot both read from stdin"),
+ "stderr={stderr}"
+ );
+ assert!(
+ !stderr.contains("failed to fetch"),
+ "the ambiguous stdin sources must be rejected before fetching: stderr={stderr}"
+ );
+ Ok(())
+}
+
+#[test]
+fn wait_script_file_dash_rejects_invalid_utf8_before_fetching() -> Result<()> {
+ let output = run_script_stdin(
+ "https://wait-script-file-stdin-must-not-fetch.invalid/",
+ b"() => true\xff",
+ &["--wait-script-file", "-", "--dump", "html"],
+ )?;
+
+ let stdout = clean_output(&output.stdout);
+ let stderr = clean_output(&output.stderr);
+ assert!(!output.status.success(), "stdout={stdout}\nstderr={stderr}");
+ assert!(stdout.is_empty(), "stdout={stdout}");
+ assert!(
+ stderr.contains("failed to read --wait-script-file `-` from stdin"),
+ "stderr={stderr}"
+ );
+ assert!(
+ !stderr.contains("failed to fetch"),
+ "stdin must be read before starting a fetch: stderr={stderr}"
+ );
+ Ok(())
+}