Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver-testkit): collapse daemon-proxy spawn and test the macOS proxy path (#5812)

injaneity9f21ecda6889
diff --git a/.github/workflows/ci-rust-macos.yml b/.github/workflows/ci-rust-macos.yml
index 0bc1282d89..b77a26b6dc 100644
--- a/.github/workflows/ci-rust-macos.yml
+++ b/.github/workflows/ci-rust-macos.yml
@@ -78,6 +78,9 @@ jobs:
- name: Run cua-driver-core unit tests on macOS
working-directory: libs/cua-driver/rust
run: cargo test --locked -p cua-driver-core --lib
+ - name: Run cua-driver-testkit unit tests on macOS
+ working-directory: libs/cua-driver/rust
+ run: cargo test --locked -p cua-driver-testkit --lib
- name: Run macOS protocol schema and status pid-file contracts
working-directory: libs/cua-driver/rust
env:
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/bin/cua-e2e-report.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/bin/cua-e2e-report.rs
index a4e57ec53a..a2191fec87 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/bin/cua-e2e-report.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/bin/cua-e2e-report.rs
@@ -110,7 +110,7 @@ fn main() {
std::process::exit(1);
}
};
- let markdown = summary.markdown_with_declarations_source_and_environment(
+ let markdown = summary.markdown(
&declarations,
&results,
source_sha.as_deref(),
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
index 2a7d93e651..dcd486dbcf 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
@@ -872,37 +872,7 @@ pub struct ValidationSummary {
}
impl ValidationSummary {
- pub fn markdown(&self, results: &[CaseResult]) -> String {
- let declarations = results
- .iter()
- .map(|result| result.case.clone())
- .collect::<Vec<_>>();
- self.markdown_with_declarations(&declarations, results)
- }
-
- pub fn markdown_with_declarations(
- &self,
- declarations: &[CaseSpec],
- results: &[CaseResult],
- ) -> String {
- self.markdown_with_declarations_and_source(declarations, results, None)
- }
-
- pub fn markdown_with_declarations_and_source(
- &self,
- declarations: &[CaseSpec],
- results: &[CaseResult],
- source_sha: Option<&str>,
- ) -> String {
- self.markdown_with_declarations_source_and_environment(
- declarations,
- results,
- source_sha,
- None,
- )
- }
-
- pub fn markdown_with_declarations_source_and_environment(
+ pub fn markdown(
&self,
declarations: &[CaseSpec],
results: &[CaseResult],
@@ -1218,22 +1188,6 @@ pub fn validate_catalog(
results: &[CaseResult],
artifact_root: Option<&Path>,
require_video: bool,
-) -> Result<ValidationSummary, Vec<String>> {
- validate_catalog_with_evidence(
- declarations,
- results,
- artifact_root,
- require_video,
- require_video,
- )
-}
-
-pub fn validate_catalog_with_evidence(
- declarations: &[CaseSpec],
- results: &[CaseResult],
- artifact_root: Option<&Path>,
- require_video: bool,
- require_turn_evidence: bool,
) -> Result<ValidationSummary, Vec<String>> {
let mut errors = Vec::new();
let policy = CatalogPolicy::from_environment(&mut errors);
@@ -1327,7 +1281,7 @@ pub fn validate_catalog_with_evidence(
}
}
}
- if require_turn_evidence {
+ if require_video {
match (artifact_root, recording_directory(&result.evidence)) {
(Some(root), Some(relative_dir)) => {
validate_turn_evidence(
@@ -3008,7 +2962,12 @@ mod tests {
.expect("valid catalog");
assert_eq!(summary.delivered, 1);
assert!(summary
- .markdown_with_declarations(std::slice::from_ref(&case), std::slice::from_ref(&result),)
+ .markdown(
+ std::slice::from_ref(&case),
+ std::slice::from_ref(&result),
+ None,
+ None
+ )
.contains("1 delivered"));
let value = serde_json::to_value(result).expect("serialize result");
@@ -3090,7 +3049,7 @@ mod tests {
let declarations = vec![delivered, refused, failed, grouped];
let summary =
validate_catalog(&declarations, &results, None, false).expect("valid catalog");
- let markdown = summary.markdown_with_declarations(&declarations, &results);
+ let markdown = summary.markdown(&declarations, &results, None, None);
assert!(markdown.contains("| electron | click | PASS | - | REFUSED | FAIL | - | - |"));
assert!(markdown.contains("| tauri | type_text | PASS | - | - | - | - | - |"));
@@ -3112,10 +3071,11 @@ mod tests {
)
.expect("valid catalog");
let sha = "0123456789abcdef0123456789abcdef01234567";
- let markdown = summary.markdown_with_declarations_and_source(
+ let markdown = summary.markdown(
std::slice::from_ref(&case),
std::slice::from_ref(&result),
Some(sha),
+ None,
);
assert!(markdown.contains(&format!("**Source SHA:** `{sha}`")));
@@ -3150,7 +3110,7 @@ mod tests {
let declarations = vec![page, not_applicable];
let summary =
validate_catalog(&declarations, &results, None, false).expect("valid catalog");
- let markdown = summary.markdown_with_declarations(&declarations, &results);
+ let markdown = summary.markdown(&declarations, &results, None, None);
assert!(markdown.contains(
"| web | evaluate | - | - | - | - | Background: PASS | NotApplicable/Background: PASS |"
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs
index c988e0554e..87e9eb583a 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs
@@ -101,7 +101,12 @@ impl McpDriver {
/// source-tree daemon.
pub fn spawn_daemon_proxy_named(socket: &str, recording_label: &str) -> Option<Self> {
if !daemon_socket_is_reachable(socket) {
- eprintln!("[testkit] CuaDriver daemon not listening at {socket} — skipping");
+ let hint = if cfg!(target_os = "macos") {
+ " — run `./scripts/install-local.sh` and `cua-driver permissions grant`;"
+ } else {
+ " —"
+ };
+ eprintln!("[testkit] CuaDriver daemon not listening at {socket}{hint} skipping");
return None;
}
Self::spawn_internal(
@@ -270,35 +275,11 @@ impl McpDriver {
/// daemon is the product path agents use, so ignored GUI tests proxy through an
/// already-running daemon and skip with a clear note when it is absent.
#[cfg(target_os = "macos")]
- pub fn spawn_macos_daemon_proxy() -> Option<Self> {
- Self::spawn_macos_daemon_proxy_internal(None)
- }
-
- /// macOS daemon-proxy variant with a stable recording artifact label.
- #[cfg(target_os = "macos")]
pub fn spawn_macos_daemon_proxy_named(recording_label: &str) -> Option<Self> {
- Self::spawn_macos_daemon_proxy_internal(Some(recording_label))
- }
-
- #[cfg(target_os = "macos")]
- fn spawn_macos_daemon_proxy_internal(recording_label: Option<&str>) -> Option<Self> {
let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());
let socket = std::env::var("CUA_E2E_MACOS_DAEMON_SOCKET")
.unwrap_or_else(|_| format!("{home}/Library/Caches/cua-driver/cua-driver.sock"));
- if !daemon_socket_is_reachable(&socket) {
- eprintln!(
- "[testkit] CuaDriver daemon not listening at {socket} — \
- run `./scripts/install-local.sh` and `cua-driver permissions grant`; skipping"
- );
- return None;
- }
- Self::spawn_internal(
- &[],
- &["mcp", "--socket", &socket],
- recording_label,
- false,
- true,
- )
+ Self::spawn_daemon_proxy_named(&socket, recording_label)
}
pub fn daemon_socket(&self) -> &str {
@@ -787,4 +768,66 @@ mod tests {
"TIMEOUT (>75s) on click"
);
}
+
+ #[cfg(target_os = "macos")]
+ const PROXY_CHILD_ENV: &str = "CUA_TESTKIT_PROXY_CHILD";
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ #[ignore]
+ fn macos_proxy_child() {
+ if std::env::var_os(PROXY_CHILD_ENV).is_some() {
+ assert!(super::McpDriver::spawn_macos_daemon_proxy_named("child").is_none());
+ }
+ }
+
+ #[cfg(target_os = "macos")]
+ fn run_proxy_child(socket: &std::path::Path) -> String {
+ let out = std::process::Command::new(std::env::current_exe().unwrap())
+ .args([
+ "mcp::tests::macos_proxy_child",
+ "--exact",
+ "--ignored",
+ "--nocapture",
+ ])
+ .env(PROXY_CHILD_ENV, "1")
+ .env("CUA_E2E_MACOS_DAEMON_SOCKET", socket)
+ .env("CUA_TEST_DRIVER_BIN", socket.with_extension("missing-bin"))
+ .output()
+ .unwrap();
+ assert!(out.status.success());
+ assert!(String::from_utf8_lossy(&out.stdout).contains("running 1 test"));
+ String::from_utf8_lossy(&out.stderr).into_owned()
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn macos_proxy_with_missing_socket_prints_install_hint() {
+ let dir = std::env::temp_dir().join(format!("cua-proxy-miss-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let socket = dir.join("d.sock");
+ let stderr = run_proxy_child(&socket);
+ let _ = std::fs::remove_dir_all(&dir);
+ assert!(stderr.contains(&format!("not listening at {}", socket.display())));
+ assert!(stderr.contains("./scripts/install-local.sh"));
+ assert!(stderr.contains("cua-driver permissions grant"));
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn macos_proxy_with_reachable_socket_probes_once() {
+ let dir = std::env::temp_dir().join(format!("cua-proxy-ok-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let socket = dir.join("d.sock");
+ let listener = std::os::unix::net::UnixListener::bind(&socket).unwrap();
+ listener.set_nonblocking(true).unwrap();
+ let stderr = run_proxy_child(&socket);
+ let mut probes = 0;
+ while listener.accept().is_ok() {
+ probes += 1;
+ }
+ let _ = std::fs::remove_dir_all(&dir);
+ assert_eq!(probes, 1);
+ assert!(!stderr.contains("not listening"));
+ }
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/raw.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/raw.rs
index be8c76c0ab..d6c96c0bb1 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/raw.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/raw.rs
@@ -69,13 +69,9 @@ impl RawDriver {
}
/// Spawn a daemon-backed raw driver with the certified platform overlay
- /// host enabled. Cursor protocol tests use this deliberately; ordinary
- /// protocol tests keep the no-overlay daemon so they remain headless.
- pub fn spawn_with_overlay() -> Option<Self> {
- Self::spawn_daemon_backed(driver_binary(), true, &[])
- }
-
- /// Spawn an overlay-enabled daemon with explicit trusted launch settings.
+ /// host enabled, with explicit trusted launch settings. Cursor protocol
+ /// tests use this deliberately; ordinary protocol tests keep the
+ /// no-overlay daemon so they remain headless.
pub fn spawn_with_overlay_and_env(env: &[(&str, &str)]) -> Option<Self> {
Self::spawn_daemon_backed(driver_binary(), true, env)
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs
index 91bcb06888..e287f231ba 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs
@@ -95,15 +95,6 @@ impl ToolResponse {
.and_then(Value::as_str)
}
- /// `get_window_state` degraded flag: an AX walk that ran but returned zero
- /// actionable elements (non-AX surface / tree not ready). `false` when absent.
- pub fn degraded(&self) -> bool {
- self.structured
- .get("degraded")
- .and_then(Value::as_bool)
- .unwrap_or(false)
- }
-
pub fn snapshot_id(&self) -> &str {
self.structured
.get("snapshot_id")
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs
index a7633bb55a..1716795edd 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs
@@ -443,7 +443,7 @@ fn cursor_motion_knobs_are_applied() {
// macOS serves cursor-overlay controls only when the daemon hosts the
// overlay; Linux applies the knobs without one.
#[cfg(target_os = "macos")]
- let driver = RawDriver::spawn_with_overlay();
+ let driver = RawDriver::spawn_with_overlay_and_env(&[]);
#[cfg(not(target_os = "macos"))]
let driver = RawDriver::spawn();
let mut driver = driver.expect("spawn source-built driver");