Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(cli): error on invalid --wait-response-body regex
BibekPathakcf78d1490903
diff --git a/Cargo.lock b/Cargo.lockindex c39f7f520..3935f6464 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1969,6 +1969,7 @@ dependencies = ["moli-robots","moli-test-support","parking_lot",+ "regex","serde_json","strip-ansi-escapes","tikv-jemalloc-sys",@@ -2358,6 +2359,7 @@ dependencies = ["moli-fetch","moli-web-mime","parking_lot",+ "regex","serde","serde_json","url",diff --git a/moli-page-types/Cargo.toml b/moli-page-types/Cargo.tomlindex 9ec05fcd1..e38b087fd 100644--- a/moli-page-types/Cargo.toml+++ b/moli-page-types/Cargo.toml@@ -15,6 +15,7 @@ moli-fetch = { path = "../moli-fetch" }moli-web-mime = { path = "../moli-web-mime" }http-auth = { version = "0.1.10", default-features = false }parking_lot = "0.12"+regex = "1.12.3"serde = { version = "1.0.228", features = ["derive"] }serde_json = "1.0.145"url = "2.5.7"diff --git a/moli-page-types/src/lib.rs b/moli-page-types/src/lib.rsindex 45c402c15..e5d1180f3 100644--- a/moli-page-types/src/lib.rs+++ b/moli-page-types/src/lib.rs@@ -27,6 +27,7 @@ use std::{use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};use parking_lot::Mutex;+use regex::Regex;use serde::{Deserialize, Serialize};use serde_json::Value;use url::Url;@@ -1923,13 +1924,24 @@ fn configure_secure_subresource_response_body_spool_file_options(options: &mut O#[cfg(not(unix))]fn configure_secure_subresource_response_body_spool_file_options(_options: &mut OpenOptions) {}-#[derive(Debug, Clone, Default, PartialEq, Eq)]+#[derive(Debug, Clone, Default)]pub struct SubresourceResponseWaitCriteria {pub url_contains: Option<String>,- pub body_contains: Option<String>,+ pub body_contains: Option<Regex>,pub json_path_equals: Option<SubresourceJsonPathEquals>,}+impl PartialEq for SubresourceResponseWaitCriteria {+ fn eq(&self, other: &Self) -> bool {+ self.url_contains == other.url_contains+ && self.body_contains.as_ref().map(Regex::as_str)+ == other.body_contains.as_ref().map(Regex::as_str)+ && self.json_path_equals == other.json_path_equals+ }+}++impl Eq for SubresourceResponseWaitCriteria {}+impl SubresourceResponseWaitCriteria {pub fn is_empty(&self) -> bool {self.url_contains.is_none()@@ -1979,11 +1991,8 @@ impl SubresourceResponseWaitCriteria {None};- if let Some(needle) = self.body_contains.as_deref()- && !response_body_text- .as_deref()- .unwrap_or_default()- .contains(needle)+ if let Some(needle) = self.body_contains.as_ref()+ && !needle.is_match(response_body_text.as_deref().unwrap_or_default()){return Ok(false);}@@ -4069,7 +4078,7 @@ mod tests {);let criteria = SubresourceResponseWaitCriteria {url_contains: None,- body_contains: Some("missing".to_owned()),+ body_contains: Some(Regex::new("missing").unwrap()),json_path_equals: None,};assert!(@@ -4082,6 +4091,47 @@ mod tests {);}+ #[test]+ fn body_wait_criteria_matches_response_body_as_regex() {+ let record = SubresourceNetworkRecord::success_with_body(+ None,+ Url::parse("https://example.test/page").unwrap(),+ Url::parse("https://example.test/api").unwrap(),+ "GET".to_owned(),+ Vec::new(),+ None,+ SubresourceResourceType::Fetch,+ None,+ Vec::new(),+ Url::parse("https://example.test/api").unwrap(),+ 200,+ vec![("content-type".to_owned(), "text/plain".to_owned())],+ SubresourceResponseBody::from_text_and_bytes(+ "order #42 ready".to_owned(),+ b"order #42 ready".to_vec(),+ ),+ Vec::new(),+ );++ let criteria = SubresourceResponseWaitCriteria {+ url_contains: None,+ body_contains: Some(Regex::new(r"order #\d+ ready").unwrap()),+ json_path_equals: None,+ };+ assert!(criteria.try_matches(&record).is_ok_and(|matches| matches));++ let non_matching = SubresourceResponseWaitCriteria {+ url_contains: None,+ body_contains: Some(Regex::new(r"^order #\d{3} ready$").unwrap()),+ json_path_equals: None,+ };+ assert!(+ non_matching+ .try_matches(&record)+ .is_ok_and(|matches| !matches)+ );+ }+#[test]fn json_path_wait_criteria_uses_shared_json_mime_parser() {let expectation = SubresourceJsonPathEquals {diff --git a/moli/Cargo.toml b/moli/Cargo.tomlindex 61c03f77c..8e19144c7 100644--- a/moli/Cargo.toml+++ b/moli/Cargo.toml@@ -27,6 +27,7 @@ moli-process-signal = { path = "../moli-process-signal" }moli-protocol-server = { path = "../moli-protocol-server" }moli-robots = { path = "../moli-robots" }parking_lot = "0.12"+regex = "1.12.3"serde_json = "1.0.145"tokio = { version = "1.51.0", features = ["macros", "net", "rt-multi-thread", "signal", "sync"] }tracing = "0.1.41"diff --git a/moli/src/cli.rs b/moli/src/cli.rsindex 7782b0440..099722c9b 100644--- a/moli/src/cli.rs+++ b/moli/src/cli.rs@@ -1,6 +1,7 @@use std::{ffi::OsString, num::NonZeroU32};use clap::{Args, Parser, Subcommand, ValueEnum};+use regex::Regex;const FETCH_INFER_FLAGS: &[&str] = &["--dump",@@ -151,8 +152,10 @@ pub struct FetchArgs {#[arg(long)]pub wait_response_url: Option<String>,- #[arg(long)]- pub wait_response_body: Option<String>,+ /// Wait for a response whose body matches this regex. An invalid regex+ /// fails the fetch command immediately.+ #[arg(long, value_parser = parse_response_body_regex)]+ pub wait_response_body: Option<ResponseBodyRegexArg>,#[arg(long, value_parser = parse_response_json_path_arg)]pub wait_response_json: Option<ResponseJsonPathArg>,@@ -224,6 +227,31 @@ fn parse_response_json_path_arg(raw: &str) -> Result<ResponseJsonPathArg, String})}+#[derive(Debug, Clone)]+pub struct ResponseBodyRegexArg {+ pub regex: Regex,+}++impl ResponseBodyRegexArg {+ pub fn pattern(&self) -> &str {+ self.regex.as_str()+ }+}++impl PartialEq for ResponseBodyRegexArg {+ fn eq(&self, other: &Self) -> bool {+ self.regex.as_str() == other.regex.as_str()+ }+}++impl Eq for ResponseBodyRegexArg {}++fn parse_response_body_regex(raw: &str) -> Result<ResponseBodyRegexArg, String> {+ Regex::new(raw)+ .map(|regex| ResponseBodyRegexArg { regex })+ .map_err(|error| format!("invalid --wait-response-body regex: {error}"))+}+#[derive(Debug, Clone, PartialEq, Eq)]pub struct RequestHeaderArg {pub name: String,diff --git a/moli/src/config.rs b/moli/src/config.rsindex b6a8aef87..7972dd434 100644--- a/moli/src/config.rs+++ b/moli/src/config.rs@@ -299,7 +299,10 @@ pub fn response_wait_criteria_from_args() -> SubresourceResponseWaitCriteria {SubresourceResponseWaitCriteria {url_contains: args.wait_response_url.clone(),- body_contains: args.wait_response_body.clone(),+ body_contains: args+ .wait_response_body+ .as_ref()+ .map(|arg| arg.regex.clone()),json_path_equals: args.wait_response_json.as_ref()diff --git a/moli/tests/cli.rs b/moli/tests/cli.rsindex 056e625c5..a353ecb99 100644--- a/moli/tests/cli.rs+++ b/moli/tests/cli.rs@@ -9,8 +9,8 @@ use std::{use moli::cli::{Cli, Commands, CommonArgs, DumpFormat, FetchArgs, FetchWaitUntil, LogFormat, LogLevel,- RequestHeaderArg, ResponseJsonPathArg, ServeArgs, StripModeChoice, StripOptions,- WebBotAuthProfileChoice, normalize_args_for_compat,+ RequestHeaderArg, ResponseBodyRegexArg, ResponseJsonPathArg, ServeArgs, StripModeChoice,+ StripOptions, WebBotAuthProfileChoice, normalize_args_for_compat,};use moli::config::AppConfig;use moli_browser_profile::BrowserProfilePaths;@@ -843,7 +843,12 @@ fn parses_fetch_response_wait_flags() {args.wait_response_url.as_deref(),Some("mtop.taobao.idle.pc.detail"));- assert_eq!(args.wait_response_body.as_deref(), Some("SUCCESS"));+ assert_eq!(+ args.wait_response_body+ .as_ref()+ .map(ResponseBodyRegexArg::pattern),+ Some("SUCCESS")+ );assert_eq!(args.wait_response_json,Some(ResponseJsonPathArg {@@ -853,6 +858,25 @@ fn parses_fetch_response_wait_flags() {);}+#[test]+fn rejects_invalid_wait_response_body_regex() {+ let error = Cli::try_parse_from(normalize_args_for_compat([+ "moli",+ "fetch",+ "--wait-response-body",+ "[",+ "https://example.com",+ ]))+ .unwrap_err();++ assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation);+ assert!(+ error+ .to_string()+ .contains("invalid --wait-response-body regex")+ );+}+#[test]fn parses_fetch_domstable_wait_until() {let cli = Cli::try_parse_from(normalize_args_for_compat([diff --git a/skills/moli-webfetch/references/fetch-recipes.md b/skills/moli-webfetch/references/fetch-recipes.mdindex 745e23f6b..32340878e 100644--- a/skills/moli-webfetch/references/fetch-recipes.md+++ b/skills/moli-webfetch/references/fetch-recipes.md@@ -46,7 +46,7 @@ a better completion signal:- `--wait-script-file <path>`: use a reusable or multiline condition. It ismutually exclusive with `--wait-script`.- `--wait-response-url <substring>`: wait for an application request by URL.-- `--wait-response-body <substring>`: require text in that response.+- `--wait-response-body <regex>`: require text matching the regex in that response.- `--wait-response-json <path=value>`: require a JSON field value. All suppliedresponse criteria must match one response.- `--delay-ms <ms>`: use only when the site has no observable readiness signal.