Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
refactor(cli): remove unused WPT and web bot auth options
ldm0515df47845c5
diff --git a/moli/src/cli.rs b/moli/src/cli.rsindex 2977d4866..ede40b461 100644--- a/moli/src/cli.rs+++ b/moli/src/cli.rs@@ -41,7 +41,6 @@ const DUMP_MODES: &[&str] = &["markdown","screenshot","pdf",- "wpt","semantic_tree","semantic_tree_text",];@@ -368,15 +367,6 @@ pub struct CommonArgs {#[arg(long)]pub user_agent_suffix: Option<String>,-- #[arg(long)]- pub web_bot_auth_key_file: Option<String>,-- #[arg(long)]- pub web_bot_auth_keyid: Option<String>,-- #[arg(long)]- pub web_bot_auth_domain: Option<String>,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]@@ -423,7 +413,6 @@ pub enum DumpFormat {Markdown,Screenshot,Pdf,- Wpt,SemanticTree,SemanticTreeText,}diff --git a/moli/src/config.rs b/moli/src/config.rsindex 7dda317ea..4cd98d9f9 100644--- a/moli/src/config.rs+++ b/moli/src/config.rs@@ -183,7 +183,6 @@ fn apply_common_args(config: &mut AppConfig, common: &CommonArgs) -> Result<()>.browser.fetch_mut().set_tls_verify_host(!common.insecure_disable_tls_host_verification);- reject_unsupported_web_bot_auth(common)?;for source in &common.document_start_script {config.add_document_start_script(source.clone());@@ -221,28 +220,6 @@ fn validate_http_host_resolve_entry(entry: &str) -> Result<()> {Ok(())}-fn reject_unsupported_web_bot_auth(common: &CommonArgs) -> Result<()> {- let mut flags = Vec::new();- if common.web_bot_auth_key_file.is_some() {- flags.push("--web-bot-auth-key-file");- }- if common.web_bot_auth_keyid.is_some() {- flags.push("--web-bot-auth-keyid");- }- if common.web_bot_auth_domain.is_some() {- flags.push("--web-bot-auth-domain");- }- if flags.is_empty() {- return Ok(());- }-- bail!(- "web bot auth is not implemented yet; unsupported flag(s): {}. \- No request signing would be performed, so these flags now fail explicitly.",- flags.join(", ")- )-}-fn parse_block_cidrs(raw: Option<&str>) -> Vec<AnyIpCidr> {raw.into_iter().flat_map(|value| value.split(','))diff --git a/moli/src/fetch_dump.rs b/moli/src/fetch_dump.rsindex 65fb35006..6fd4af832 100644--- a/moli/src/fetch_dump.rs+++ b/moli/src/fetch_dump.rs@@ -16,87 +16,6 @@ use crate::{network_trace::{NetworkTraceConfigSummary, NetworkTraceOptions, render_network_trace},};-const WPT_DUMP_SCRIPT: &str = r#"-(() => {- const unavailable = (message) => JSON.stringify({- url: window.location.href,- status: 'Unavailable',- message,- summary: { total: 0, passed: 0, failed: 0, timeout: 0, notrun: 0, unsupported: 0 },- cases: []- }, null, 2);- const summarize = (cases) => ({- total: cases.length,- passed: cases.filter(c => c.status === 'Pass').length,- failed: cases.filter(c => c.status === 'Fail').length,- timeout: cases.filter(c => c.status === 'Timeout').length,- notrun: cases.filter(c => c.status === 'Not Run').length,- unsupported: cases.filter(c => c.status === 'Optional Feature Unsupported').length- });- const normalizeWptStatus = (status) => {- switch (status) {- case 'PASS':- case 'Pass':- return 'Pass';- case 'FAIL':- case 'Fail':- return 'Fail';- case 'TIMEOUT':- case 'Timeout':- return 'Timeout';- case 'NOTRUN':- case 'Not Run':- return 'Not Run';- case 'Optional Feature Unsupported':- return 'Optional Feature Unsupported';- default:- return status || 'Unknown';- }- };- const moliReport = window.__moliWptReport;- if (moliReport && typeof moliReport === 'object' && Array.isArray(moliReport.tests)) {- const cases = moliReport.tests.map(test => ({- name: String(test.name ?? ''),- status: normalizeWptStatus(test.status),- message: test.message || null- }));- return JSON.stringify({- url: window.location.href,- status: moliReport.status?.status ?? (moliReport.complete ? 'OK' : 'PENDING'),- message: moliReport.status?.message ?? null,- complete: moliReport.complete === true,- summary: summarize(cases),- cases- }, null, 2);- }- const statuses = ['Pass', 'Fail', 'Timeout', 'Not Run', 'Optional Feature Unsupported'];- const reportRef = globalThis.report;- if (!reportRef || typeof reportRef !== 'object' || !reportRef.cases) {- return unavailable('WPT report is unavailable');- }- const parse = (raw) => {- for (const status of statuses) {- const idx = raw.indexOf('|' + status);- if (idx !== -1) {- const name = raw.slice(0, idx);- const rest = raw.slice(idx + status.length + 1);- const message = rest.length > 0 && rest[0] === '|' ? rest.slice(1) : null;- return { name, status, message };- }- }- return { name: raw, status: 'Unknown', message: null };- };- const cases = Object.values(reportRef.cases).map(parse);- return JSON.stringify({- url: window.location.href,- status: reportRef.status ?? 'Unknown',- message: reportRef.message ?? null,- summary: summarize(cases),- cases- }, null, 2);-})()-"#;-struct CapturedRaster {mime_type: String,width: u32,@@ -152,7 +71,6 @@ pub fn render_raw_document_dump(DumpFormat::Markdown| DumpFormat::Screenshot| DumpFormat::Pdf- | DumpFormat::Wpt| DumpFormat::SemanticTree| DumpFormat::SemanticTreeText => {anyhow::bail!("raw non-HTML document output only supports --dump html or --dump json")@@ -215,7 +133,6 @@ async fn render_page_dump_with_trace_config_async(}DumpFormat::SemanticTree => render_semantic_tree_dump_async(page).await,DumpFormat::SemanticTreeText => render_semantic_tree_text_dump_async(page).await,- DumpFormat::Wpt => render_wpt_dump_async(page).await,}}@@ -415,17 +332,6 @@ async fn render_semantic_tree_text_dump_async(page: &mut Page) -> Result<String>Ok(out.trim_end().to_owned())}-async fn render_wpt_dump_async(page: &mut Page) -> Result<String> {- let payload = page- .evaluate_runtime_expression_async(WPT_DUMP_SCRIPT)- .await?;- let rendered = payload- .get("value")- .and_then(Value::as_str)- .unwrap_or_default();- Ok(format!("== WPT Results ==\n{rendered}"))-}-fn render_semantic_node_text(node_id: &str,by_id: &std::collections::HashMap<String, &Value>,@@ -760,67 +666,6 @@ mod tests {Ok(())}- #[tokio::test]- async fn render_page_dump_async_renders_legacy_wpt_payload() -> Result<()> {- let (_browser, mut page, http_server) = load_page(- r#"<!doctype html><html><body><script>globalThis.report={status:"OK",cases:{caseOne:"alpha|Pass"}};</script></body></html>"#,- )- .await?;-- let rendered = render_page_dump_async(- &mut page,- &FetchCommandConfig {- dump_mode: Some(DumpFormat::Wpt),- ..FetchCommandConfig::default()- },- )- .await?;-- assert!(rendered.starts_with("== WPT Results ==\n"));- assert!(rendered.contains(r#""status": "OK""#));- assert!(rendered.contains(r#""name": "alpha""#));- http_server.abort();- Ok(())- }-- #[tokio::test]- async fn render_page_dump_async_prefers_moli_wpt_report_payload() -> Result<()> {- let (_browser, mut page, http_server) = load_page(- r#"<!doctype html><html><body><script>- window.__moliWptReport={- complete:true,- status:{status:"OK",message:""},- tests:[- {name:"modern pass",status:"PASS",message:""},- {name:"modern fail",status:"FAIL",message:"expected true"}- ]- };- globalThis.report={status:"LEGACY",cases:{caseOne:"legacy|Pass"}};- </script></body></html>"#,- )- .await?;-- let rendered = render_page_dump_async(- &mut page,- &FetchCommandConfig {- dump_mode: Some(DumpFormat::Wpt),- ..FetchCommandConfig::default()- },- )- .await?;-- assert!(rendered.starts_with("== WPT Results ==\n"));- assert!(rendered.contains(r#""status": "OK""#));- assert!(rendered.contains(r#""complete": true"#));- assert!(rendered.contains(r#""name": "modern pass""#));- assert!(rendered.contains(r#""name": "modern fail""#));- assert!(rendered.contains(r#""passed": 1"#));- assert!(rendered.contains(r#""failed": 1"#));- assert!(!rendered.contains("legacy"));- http_server.abort();- Ok(())- }-#[tokio::test]async fn render_page_dump_async_includes_network_trace_config_summary() -> Result<()> {let (_browser, mut page, http_server) =diff --git a/moli/tests/cli.rs b/moli/tests/cli.rsindex 237dae7ee..fb46bdd75 100644--- a/moli/tests/cli.rs+++ b/moli/tests/cli.rs@@ -56,12 +56,6 @@ fn parses_explicit_fetch_command_with_compatibility_flags() {"http,event","--user-agent-suffix","internal-tester",- "--web-bot-auth-key-file",- "/tmp/key.pem",- "--web-bot-auth-keyid",- "kid",- "--web-bot-auth-domain",- "example.com","https://example.com",])).unwrap();@@ -131,9 +125,6 @@ fn parses_explicit_fetch_command_with_compatibility_flags() {log_filter_scopes: Some("http,event".to_owned()),user_agent: None,user_agent_suffix: Some("internal-tester".to_owned()),- web_bot_auth_key_file: Some("/tmp/key.pem".to_owned()),- web_bot_auth_keyid: Some("kid".to_owned()),- web_bot_auth_domain: Some("example.com".to_owned()),},url: "https://example.com".to_owned(),}))@@ -174,6 +165,19 @@ fn parses_json_dump_mode() {assert_eq!(args.dump, Some(DumpFormat::Json));}+#[test]+fn rejects_removed_wpt_dump_mode() {+ let result = Cli::try_parse_from(normalize_args_for_compat([+ "moli",+ "fetch",+ "--dump",+ "wpt",+ "https://example.com",+ ]));++ assert!(result.is_err());+}+#[test]fn parses_binary_dump_modes_with_inferred_fetch_command() {for (value, expected) in [@@ -196,32 +200,23 @@ fn parses_binary_dump_modes_with_inferred_fetch_command() {}#[test]-fn app_config_rejects_unimplemented_web_bot_auth_flags() {- let cli = Cli::try_parse_from(normalize_args_for_compat([- "moli",- "fetch",+fn rejects_removed_web_bot_auth_flags() {+ for flag in ["--web-bot-auth-key-file",- "/tmp/key.pem","--web-bot-auth-keyid",- "kid","--web-bot-auth-domain",- "example.com",- "https://example.com",- ]))- .unwrap();+ ] {+ let error = Cli::try_parse_from(normalize_args_for_compat([+ "moli",+ "fetch",+ flag,+ "unused",+ "https://example.com",+ ]))+ .unwrap_err();- let error = AppConfig::from_cli(&cli).unwrap_err().to_string();- assert!(- error.contains("web bot auth is not implemented yet"),- "error={error}"- );- assert!(error.contains("--web-bot-auth-key-file"), "error={error}");- assert!(error.contains("--web-bot-auth-keyid"), "error={error}");- assert!(error.contains("--web-bot-auth-domain"), "error={error}");- assert!(- error.contains("No request signing would be performed"),- "error={error}"- );+ assert_eq!(error.kind(), clap::error::ErrorKind::UnknownArgument);+ }}#[test]diff --git a/skills/moli-webfetch/references/fetch-recipes.md b/skills/moli-webfetch/references/fetch-recipes.mdindex 3566ca4c7..e122efa68 100644--- a/skills/moli-webfetch/references/fetch-recipes.md+++ b/skills/moli-webfetch/references/fetch-recipes.md@@ -22,7 +22,6 @@| Diagnose requests | `--dump json --trace-network` | Adds the `network` object || Capture the viewport | `--layout --dump screenshot` | Writes PNG bytes to stdout || Capture a paginated document | `--layout --dump pdf` | Writes PDF bytes to stdout |-| Run repository WPT workflows | `--dump wpt` | Emits Moli's WPT-oriented report |Raw non-HTML responses support only `html` and `json`.@@ -50,7 +49,7 @@ a better completion signal:- `--wait-response-json <path=value>`: require a JSON field value. All suppliedresponse criteria must match one response.- `--delay-ms <ms>`: use only when the site has no observable readiness signal.-- `--timeout <ms>`: bound navigation and explicit waits; the default is 30000.+- `--timeout <ms>`: bound navigation and explicit waits; the default is 10000.Examples: