Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(cli): error on invalid --wait-response-body regex

BibekPathakcf78d1490903
diff --git a/Cargo.lock b/Cargo.lock
index c39f7f520..3935f6464 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1969,6 +1969,7 @@ dependencies = [
"moli-robots",
"moli-test-support",
"parking_lot",
+ "regex",
"serde_json",
"strip-ansi-escapes",
"tikv-jemalloc-sys",
@@ -2358,6 +2359,7 @@ dependencies = [
"moli-fetch",
"moli-web-mime",
"parking_lot",
+ "regex",
"serde",
"serde_json",
"url",
diff --git a/moli-page-types/Cargo.toml b/moli-page-types/Cargo.toml
index 9ec05fcd1..e38b087fd 100644
--- a/moli-page-types/Cargo.toml
+++ b/moli-page-types/Cargo.toml
@@ -15,6 +15,7 @@ moli-fetch = { path = "../moli-fetch" }
moli-web-mime = { path = "../moli-web-mime" }
http-auth = { version = "0.1.10", default-features = false }
parking_lot = "0.12"
+regex = "1.12.3"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.145"
url = "2.5.7"
diff --git a/moli-page-types/src/lib.rs b/moli-page-types/src/lib.rs
index 45c402c15..e5d1180f3 100644
--- a/moli-page-types/src/lib.rs
+++ b/moli-page-types/src/lib.rs
@@ -27,6 +27,7 @@ use std::{
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};
use parking_lot::Mutex;
+use regex::Regex;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use url::Url;
@@ -1923,13 +1924,24 @@ fn configure_secure_subresource_response_body_spool_file_options(options: &mut O
#[cfg(not(unix))]
fn configure_secure_subresource_response_body_spool_file_options(_options: &mut OpenOptions) {}
-#[derive(Debug, Clone, Default, PartialEq, Eq)]
+#[derive(Debug, Clone, Default)]
pub struct SubresourceResponseWaitCriteria {
pub url_contains: Option<String>,
- pub body_contains: Option<String>,
+ pub body_contains: Option<Regex>,
pub json_path_equals: Option<SubresourceJsonPathEquals>,
}
+impl PartialEq for SubresourceResponseWaitCriteria {
+ fn eq(&self, other: &Self) -> bool {
+ self.url_contains == other.url_contains
+ && self.body_contains.as_ref().map(Regex::as_str)
+ == other.body_contains.as_ref().map(Regex::as_str)
+ && self.json_path_equals == other.json_path_equals
+ }
+}
+
+impl Eq for SubresourceResponseWaitCriteria {}
+
impl SubresourceResponseWaitCriteria {
pub fn is_empty(&self) -> bool {
self.url_contains.is_none()
@@ -1979,11 +1991,8 @@ impl SubresourceResponseWaitCriteria {
None
};
- if let Some(needle) = self.body_contains.as_deref()
- && !response_body_text
- .as_deref()
- .unwrap_or_default()
- .contains(needle)
+ if let Some(needle) = self.body_contains.as_ref()
+ && !needle.is_match(response_body_text.as_deref().unwrap_or_default())
{
return Ok(false);
}
@@ -4069,7 +4078,7 @@ mod tests {
);
let criteria = SubresourceResponseWaitCriteria {
url_contains: None,
- body_contains: Some("missing".to_owned()),
+ body_contains: Some(Regex::new("missing").unwrap()),
json_path_equals: None,
};
assert!(
@@ -4082,6 +4091,47 @@ mod tests {
);
}
+ #[test]
+ fn body_wait_criteria_matches_response_body_as_regex() {
+ let record = SubresourceNetworkRecord::success_with_body(
+ None,
+ Url::parse("https://example.test/page").unwrap(),
+ Url::parse("https://example.test/api").unwrap(),
+ "GET".to_owned(),
+ Vec::new(),
+ None,
+ SubresourceResourceType::Fetch,
+ None,
+ Vec::new(),
+ Url::parse("https://example.test/api").unwrap(),
+ 200,
+ vec![("content-type".to_owned(), "text/plain".to_owned())],
+ SubresourceResponseBody::from_text_and_bytes(
+ "order #42 ready".to_owned(),
+ b"order #42 ready".to_vec(),
+ ),
+ Vec::new(),
+ );
+
+ let criteria = SubresourceResponseWaitCriteria {
+ url_contains: None,
+ body_contains: Some(Regex::new(r"order #\d+ ready").unwrap()),
+ json_path_equals: None,
+ };
+ assert!(criteria.try_matches(&record).is_ok_and(|matches| matches));
+
+ let non_matching = SubresourceResponseWaitCriteria {
+ url_contains: None,
+ body_contains: Some(Regex::new(r"^order #\d{3} ready$").unwrap()),
+ json_path_equals: None,
+ };
+ assert!(
+ non_matching
+ .try_matches(&record)
+ .is_ok_and(|matches| !matches)
+ );
+ }
+
#[test]
fn json_path_wait_criteria_uses_shared_json_mime_parser() {
let expectation = SubresourceJsonPathEquals {
diff --git a/moli/Cargo.toml b/moli/Cargo.toml
index 61c03f77c..8e19144c7 100644
--- a/moli/Cargo.toml
+++ b/moli/Cargo.toml
@@ -27,6 +27,7 @@ moli-process-signal = { path = "../moli-process-signal" }
moli-protocol-server = { path = "../moli-protocol-server" }
moli-robots = { path = "../moli-robots" }
parking_lot = "0.12"
+regex = "1.12.3"
serde_json = "1.0.145"
tokio = { version = "1.51.0", features = ["macros", "net", "rt-multi-thread", "signal", "sync"] }
tracing = "0.1.41"
diff --git a/moli/src/cli.rs b/moli/src/cli.rs
index 7782b0440..099722c9b 100644
--- a/moli/src/cli.rs
+++ b/moli/src/cli.rs
@@ -1,6 +1,7 @@
use std::{ffi::OsString, num::NonZeroU32};
use clap::{Args, Parser, Subcommand, ValueEnum};
+use regex::Regex;
const FETCH_INFER_FLAGS: &[&str] = &[
"--dump",
@@ -151,8 +152,10 @@ pub struct FetchArgs {
#[arg(long)]
pub wait_response_url: Option<String>,
- #[arg(long)]
- pub wait_response_body: Option<String>,
+ /// Wait for a response whose body matches this regex. An invalid regex
+ /// fails the fetch command immediately.
+ #[arg(long, value_parser = parse_response_body_regex)]
+ pub wait_response_body: Option<ResponseBodyRegexArg>,
#[arg(long, value_parser = parse_response_json_path_arg)]
pub wait_response_json: Option<ResponseJsonPathArg>,
@@ -224,6 +227,31 @@ fn parse_response_json_path_arg(raw: &str) -> Result<ResponseJsonPathArg, String
})
}
+#[derive(Debug, Clone)]
+pub struct ResponseBodyRegexArg {
+ pub regex: Regex,
+}
+
+impl ResponseBodyRegexArg {
+ pub fn pattern(&self) -> &str {
+ self.regex.as_str()
+ }
+}
+
+impl PartialEq for ResponseBodyRegexArg {
+ fn eq(&self, other: &Self) -> bool {
+ self.regex.as_str() == other.regex.as_str()
+ }
+}
+
+impl Eq for ResponseBodyRegexArg {}
+
+fn parse_response_body_regex(raw: &str) -> Result<ResponseBodyRegexArg, String> {
+ Regex::new(raw)
+ .map(|regex| ResponseBodyRegexArg { regex })
+ .map_err(|error| format!("invalid --wait-response-body regex: {error}"))
+}
+
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RequestHeaderArg {
pub name: String,
diff --git a/moli/src/config.rs b/moli/src/config.rs
index b6a8aef87..7972dd434 100644
--- a/moli/src/config.rs
+++ b/moli/src/config.rs
@@ -299,7 +299,10 @@ pub fn response_wait_criteria_from_args(
) -> SubresourceResponseWaitCriteria {
SubresourceResponseWaitCriteria {
url_contains: args.wait_response_url.clone(),
- body_contains: args.wait_response_body.clone(),
+ body_contains: args
+ .wait_response_body
+ .as_ref()
+ .map(|arg| arg.regex.clone()),
json_path_equals: args
.wait_response_json
.as_ref()
diff --git a/moli/tests/cli.rs b/moli/tests/cli.rs
index 056e625c5..a353ecb99 100644
--- a/moli/tests/cli.rs
+++ b/moli/tests/cli.rs
@@ -9,8 +9,8 @@ use std::{
use moli::cli::{
Cli, Commands, CommonArgs, DumpFormat, FetchArgs, FetchWaitUntil, LogFormat, LogLevel,
- RequestHeaderArg, ResponseJsonPathArg, ServeArgs, StripModeChoice, StripOptions,
- WebBotAuthProfileChoice, normalize_args_for_compat,
+ RequestHeaderArg, ResponseBodyRegexArg, ResponseJsonPathArg, ServeArgs, StripModeChoice,
+ StripOptions, WebBotAuthProfileChoice, normalize_args_for_compat,
};
use moli::config::AppConfig;
use moli_browser_profile::BrowserProfilePaths;
@@ -843,7 +843,12 @@ fn parses_fetch_response_wait_flags() {
args.wait_response_url.as_deref(),
Some("mtop.taobao.idle.pc.detail")
);
- assert_eq!(args.wait_response_body.as_deref(), Some("SUCCESS"));
+ assert_eq!(
+ args.wait_response_body
+ .as_ref()
+ .map(ResponseBodyRegexArg::pattern),
+ Some("SUCCESS")
+ );
assert_eq!(
args.wait_response_json,
Some(ResponseJsonPathArg {
@@ -853,6 +858,25 @@ fn parses_fetch_response_wait_flags() {
);
}
+#[test]
+fn rejects_invalid_wait_response_body_regex() {
+ let error = Cli::try_parse_from(normalize_args_for_compat([
+ "moli",
+ "fetch",
+ "--wait-response-body",
+ "[",
+ "https://example.com",
+ ]))
+ .unwrap_err();
+
+ assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation);
+ assert!(
+ error
+ .to_string()
+ .contains("invalid --wait-response-body regex")
+ );
+}
+
#[test]
fn parses_fetch_domstable_wait_until() {
let cli = Cli::try_parse_from(normalize_args_for_compat([
diff --git a/skills/moli-webfetch/references/fetch-recipes.md b/skills/moli-webfetch/references/fetch-recipes.md
index 745e23f6b..32340878e 100644
--- a/skills/moli-webfetch/references/fetch-recipes.md
+++ b/skills/moli-webfetch/references/fetch-recipes.md
@@ -46,7 +46,7 @@ a better completion signal:
- `--wait-script-file <path>`: use a reusable or multiline condition. It is
mutually exclusive with `--wait-script`.
- `--wait-response-url <substring>`: wait for an application request by URL.
-- `--wait-response-body <substring>`: require text in that response.
+- `--wait-response-body <regex>`: require text matching the regex in that response.
- `--wait-response-json <path=value>`: require a JSON field value. All supplied
response criteria must match one response.
- `--delay-ms <ms>`: use only when the site has no observable readiness signal.