Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): rewrite test_webkit_js with correct DriverClient API

Francesco Bonacci6c3d8576feef
diff --git a/libs/cua-driver/Tests/integration/test_webkit_js.py b/libs/cua-driver/Tests/integration/test_webkit_js.py
index 2f5a417a1d..a3d46f697c 100644
--- a/libs/cua-driver/Tests/integration/test_webkit_js.py
+++ b/libs/cua-driver/Tests/integration/test_webkit_js.py
@@ -1,14 +1,14 @@
-"""Integration tests: WKWebView/Tauri AX fallback path (page tool).
+"""Integration tests: WKWebView/Tauri AX fallback (page tool).
-Tests the AX-based fallback for apps where the WebKit remote inspector is
-blocked — specifically:
- - page(action=get_text) routes through AXPageReader when the target is
- a WKWebView app (WebInspectorXPC.isWKWebViewApp returns true)
- - page(action=query_dom) same AX fallback with CSS selector → AX role
+Tests AX-based get_text and query_dom for Tauri/WKWebView apps where the
+WebKit remote inspector is blocked, and verifies that:
+ - execute_javascript returns a clear error pointing to AX alternatives
+ - get_text returns content via the AX tree
+ - query_dom returns elements via the AX tree
+ - Chrome is NOT misdetected as a WKWebView app
-Requires Conductor.app (a Tauri-based app shipped with cua-driver) to be
-installed in /Applications. Tests that skip the inspector path skip when
-Conductor is not installed or when TAURI_WEBVIEW_AUTOMATION is not set.
+Requires Conductor.app (/Applications/Conductor.app) — a Tauri app.
+Tests that need Conductor skip when it is not installed.
Run:
scripts/test.sh test_webkit_js
@@ -16,8 +16,8 @@ Run:
from __future__ import annotations
-import json
import os
+import re
import subprocess
import sys
import time
@@ -26,8 +26,8 @@ import unittest
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from driver_client import DriverClient, default_binary_path
-CONDUCTOR_BUNDLE = "com.trycua.Conductor"
-CONDUCTOR_APP_PATH = "/Applications/Conductor.app"
+CONDUCTOR_BUNDLE = "com.conductor.app"
+CONDUCTOR_PATH = "/Applications/Conductor.app"
# ---------------------------------------------------------------------------
@@ -35,245 +35,184 @@ CONDUCTOR_APP_PATH = "/Applications/Conductor.app"
# ---------------------------------------------------------------------------
def _tool_text(result: dict) -> str:
- """Extract the text string from a tool call result."""
for item in result.get("content", []):
if item.get("type") == "text":
return item.get("text", "")
return ""
-def _is_error(result: dict) -> bool:
- return result.get("isError", False)
-
-
def _conductor_installed() -> bool:
- return os.path.isdir(CONDUCTOR_APP_PATH)
-
-
-def _find_conductor_pid(client: DriverClient) -> int | None:
- """Return the pid of a running Conductor instance, or None."""
- result = client.call("list_apps", {})
- text = _tool_text(result)
- for line in text.splitlines():
- if CONDUCTOR_BUNDLE in line or "Conductor" in line:
- # Line format: "Conductor (com.trycua.Conductor) pid=1234"
- for part in line.split():
- if part.startswith("pid="):
- try:
- return int(part[4:])
- except ValueError:
- pass
- return None
+ return os.path.isdir(CONDUCTOR_PATH)
-# ---------------------------------------------------------------------------
-# AXPageReader unit-level tests (no app required)
-# ---------------------------------------------------------------------------
+def _kill_conductor() -> None:
+ subprocess.run(
+ ["osascript", "-e", 'tell application "Conductor" to quit'],
+ check=False, timeout=5,
+ )
+ time.sleep(1)
-class TestAXPageReaderExtractText(unittest.TestCase):
- """Test that AXPageReader.extractText produces the right output."""
- # We test this indirectly via page(get_text) on a synthetic AX tree
- # produced by get_window_state — so we still need the driver running.
+def _find_pid(text: str) -> int | None:
+ m = re.search(r'pid[=:\s]+(\d+)', text, re.IGNORECASE)
+ return int(m.group(1)) if m else None
- @classmethod
- def setUpClass(cls):
- cls.client = DriverClient(binary_path=default_binary_path())
- cls.client.start()
-
- @classmethod
- def tearDownClass(cls):
- cls.client.stop()
- def test_driver_responds(self):
- """Smoke test: driver is reachable."""
- result = self.client.call("get_screen_size", {})
- self.assertFalse(_is_error(result), _tool_text(result))
- text = _tool_text(result)
- self.assertIn("width", text.lower())
+def _main_window(client: DriverClient, pid: int) -> int | None:
+ """Return the on-screen Conductor window_id, or None if not found."""
+ text = _tool_text(client.call_tool("list_windows", {"pid": pid}))
+ for line in text.splitlines():
+ if "on_screen=True" in line or "is_on_screen=True" in line:
+ m = re.search(r'\[window_id:\s*(\d+)\]', line)
+ if m:
+ return int(m.group(1))
+ # Fall back to any titled window
+ m = re.search(r'"[^"]+"\s+\[window_id:\s*(\d+)\]', text)
+ if m:
+ return int(m.group(1))
+ m = re.search(r'\[window_id:\s*(\d+)\]', text)
+ return int(m.group(1)) if m else None
# ---------------------------------------------------------------------------
-# WebInspectorXPC detection tests
+# Conductor / Tauri AX fallback tests
# ---------------------------------------------------------------------------
-class TestWebInspectorXPCDetection(unittest.TestCase):
- """Test isWKWebViewApp detection heuristic via execute_javascript errors."""
[email protected](_conductor_installed(), "Conductor not installed")
+class WebKitJSTests(unittest.TestCase):
+ """AX-based page primitives work end-to-end against a live Conductor window."""
- @classmethod
- def setUpClass(cls):
- cls.client = DriverClient(binary_path=default_binary_path())
- cls.client.start()
- # List apps to find Chrome pid (should NOT be detected as WKWebView app).
- cls.chrome_pid = None
- result = cls.client.call("list_apps", {})
- text = _tool_text(result)
- for line in text.splitlines():
- if "com.google.Chrome" in line:
- for part in line.split():
- if part.startswith("pid="):
- try:
- cls.chrome_pid = int(part[4:])
- except ValueError:
- pass
+ _pid: int
+ _window_id: int
@classmethod
- def tearDownClass(cls):
- cls.client.stop()
+ def setUpClass(cls) -> None:
+ cls.binary = default_binary_path()
+ _kill_conductor()
- @unittest.skipUnless(
- os.path.exists("/Applications/Google Chrome.app"),
- "Chrome not installed"
- )
- def test_chrome_not_detected_as_wkwebview(self):
- """Chrome must NOT be classified as a WKWebView app."""
- if self.chrome_pid is None:
- self.skipTest("Chrome not running")
- # list_windows to get a window_id for Chrome.
- win_result = self.client.call("list_windows", {"pid": self.chrome_pid})
- text = _tool_text(win_result)
- # Extract first window_id from output.
- window_id = None
- for line in text.splitlines():
- for part in line.split():
- if part.startswith("id="):
- try:
- window_id = int(part[3:])
- break
- except ValueError:
- pass
- if window_id:
- break
- if window_id is None:
- self.skipTest("No Chrome window found")
- # execute_javascript on Chrome should succeed or fail with an Apple Events
- # error — NOT with the WKWebView-specific "use get_text or query_dom" message.
- result = self.client.call("page", {
- "pid": self.chrome_pid,
- "window_id": window_id,
- "action": "execute_javascript",
- "javascript": "1+1",
- })
- text = _tool_text(result)
- self.assertNotIn(
- "WKWebView",
- text,
- f"Chrome was incorrectly detected as a WKWebView app: {text}",
- )
+ with DriverClient(cls.binary) as c:
+ result = c.call_tool("launch_app", {"bundle_id": CONDUCTOR_BUNDLE})
+ text = _tool_text(result)
+ cls._pid = _find_pid(text) or 0
+ if not cls._pid:
+ raise RuntimeError("Could not determine Conductor pid from launch_app result")
-# ---------------------------------------------------------------------------
-# Conductor / Tauri AX fallback tests
-# ---------------------------------------------------------------------------
+ deadline = time.monotonic() + 10.0
+ cls._window_id = None
+ while time.monotonic() < deadline:
+ with DriverClient(cls.binary) as c:
+ wid = _main_window(c, cls._pid)
+ if wid:
+ cls._window_id = wid
+ break
+ time.sleep(0.5)
-class TestConductorAXFallback(unittest.TestCase):
- """Test AX-based get_text / query_dom for Conductor (Tauri app)."""
+ if not cls._window_id:
+ raise RuntimeError(f"No window found for Conductor pid {cls._pid}")
- @classmethod
- def setUpClass(cls):
- if not _conductor_installed():
- return
- cls.client = DriverClient(binary_path=default_binary_path())
- cls.client.start()
- # Launch Conductor if not already running.
- launch_result = cls.client.call("launch_app", {
- "bundle_id": CONDUCTOR_BUNDLE,
- })
- time.sleep(3) # Wait for app to be ready.
- cls.pid = _find_conductor_pid(cls.client)
+ print(f"\n Conductor pid={cls._pid} window_id={cls._window_id}")
@classmethod
- def tearDownClass(cls):
- if not _conductor_installed():
- return
- if hasattr(cls, "client"):
- cls.client.stop()
-
- def _get_window_id(self) -> int | None:
- if self.pid is None:
- return None
- result = self.client.call("list_windows", {"pid": self.pid})
- text = _tool_text(result)
- for line in text.splitlines():
- for part in line.split():
- if part.startswith("id="):
- try:
- return int(part[3:])
- except ValueError:
- pass
- return None
-
- @unittest.skipUnless(_conductor_installed(), "Conductor not installed")
- def test_execute_javascript_returns_wkwebview_error(self):
- """execute_javascript on Conductor must return the WKWebView error."""
- window_id = self._get_window_id()
- if window_id is None:
- self.skipTest("Conductor window not found")
- result = self.client.call("page", {
- "pid": self.pid,
- "window_id": window_id,
- "action": "execute_javascript",
- "javascript": "document.title",
- })
- self.assertTrue(_is_error(result), "Expected error for WKWebView JS")
+ def tearDownClass(cls) -> None:
+ _kill_conductor()
+
+ # -----------------------------------------------------------------------
+ # execute_javascript — must return WKWebView error, not crash
+ # -----------------------------------------------------------------------
+
+ def test_13_execute_javascript_returns_clear_error_for_wkwebview(self) -> None:
+ """execute_javascript on a WKWebView app returns a clear error."""
+ with DriverClient(self.binary) as c:
+ result = c.call_tool("page", {
+ "pid": self._pid,
+ "window_id": self._window_id,
+ "action": "execute_javascript",
+ "javascript": "document.title",
+ })
+ self.assertTrue(result.get("isError"), "Expected isError=True for WKWebView JS")
text = _tool_text(result)
self.assertIn("WKWebView", text)
self.assertIn("get_text", text)
- @unittest.skipUnless(_conductor_installed(), "Conductor not installed")
- def test_get_text_via_ax_tree(self):
- """get_text on Conductor must return content via AX tree."""
- window_id = self._get_window_id()
- if window_id is None:
- self.skipTest("Conductor window not found")
- result = self.client.call("page", {
- "pid": self.pid,
- "window_id": window_id,
- "action": "get_text",
- })
- self.assertFalse(_is_error(result), _tool_text(result))
+ # -----------------------------------------------------------------------
+ # get_text — must return content via AX tree
+ # -----------------------------------------------------------------------
+
+ def test_11_ax_get_text_works_without_inspector(self) -> None:
+ """get_text returns page text via AX tree (no inspector needed)."""
+ with DriverClient(self.binary) as c:
+ result = c.call_tool("page", {
+ "pid": self._pid,
+ "window_id": self._window_id,
+ "action": "get_text",
+ })
+ self.assertFalse(result.get("isError"), _tool_text(result))
text = _tool_text(result)
- # AX tree result should mention the fallback path.
- self.assertTrue(
- len(text.strip()) > 0,
- "get_text returned empty content"
- )
-
- @unittest.skipUnless(_conductor_installed(), "Conductor not installed")
- def test_query_dom_buttons_via_ax_tree(self):
- """query_dom(button) on Conductor returns AX buttons."""
- window_id = self._get_window_id()
- if window_id is None:
- self.skipTest("Conductor window not found")
- result = self.client.call("page", {
- "pid": self.pid,
- "window_id": window_id,
- "action": "query_dom",
- "css_selector": "button",
- })
+ self.assertTrue(len(text.strip()) > 0, "get_text returned empty content")
+
+ # -----------------------------------------------------------------------
+ # query_dom — must return AX elements via AX tree
+ # -----------------------------------------------------------------------
+
+ def test_12_ax_query_dom_buttons(self) -> None:
+ """query_dom for button elements returns AX tree results."""
+ with DriverClient(self.binary) as c:
+ result = c.call_tool("page", {
+ "pid": self._pid,
+ "window_id": self._window_id,
+ "action": "query_dom",
+ "css_selector": "button",
+ })
text = _tool_text(result)
- # Either returns buttons or returns a well-formed error — not a crash.
- self.assertFalse(
- "Traceback" in text or "fatal" in text.lower(),
- f"Unexpected crash output: {text}"
- )
+ self.assertNotIn("Traceback", text)
+ # Either found buttons (role key) or returned empty — both are valid
+ if not result.get("isError"):
+ self.assertTrue(
+ "AXButton" in text or "role" in text or "[]" in text,
+ f"Unexpected query_dom output: {text[:200]}"
+ )
+
+ # -----------------------------------------------------------------------
+ # Chrome must NOT be misdetected as WKWebView
+ # -----------------------------------------------------------------------
+
+ def test_chrome_not_detected_as_wkwebview(self) -> None:
+ """Chrome execute_javascript must not return the WKWebView error."""
+ with DriverClient(self.binary) as c:
+ result = c.call_tool("list_apps")
+ text = _tool_text(result)
+
+ chrome_pid = None
+ for line in text.splitlines():
+ if "com.google.Chrome" in line:
+ m = re.search(r'pid[=:\s]+(\d+)', line, re.IGNORECASE)
+ if m:
+ chrome_pid = int(m.group(1))
+ break
- @unittest.skipUnless(_conductor_installed(), "Conductor not installed")
- def test_query_dom_links_via_ax_tree(self):
- """query_dom(a) on Conductor returns AX links."""
- window_id = self._get_window_id()
- if window_id is None:
- self.skipTest("Conductor window not found")
- result = self.client.call("page", {
- "pid": self.pid,
- "window_id": window_id,
- "action": "query_dom",
- "css_selector": "a",
- })
+ if chrome_pid is None:
+ self.skipTest("Chrome not running")
+
+ with DriverClient(self.binary) as c:
+ windows_text = _tool_text(c.call_tool("list_windows", {"pid": chrome_pid}))
+
+ m = re.search(r'\[window_id:\s*(\d+)\]', windows_text)
+ if not m:
+ self.skipTest("No Chrome window found")
+ window_id = int(m.group(1))
+
+ with DriverClient(self.binary) as c:
+ result = c.call_tool("page", {
+ "pid": chrome_pid,
+ "window_id": window_id,
+ "action": "execute_javascript",
+ "javascript": "1+1",
+ })
text = _tool_text(result)
- self.assertFalse(
- "Traceback" in text or "fatal" in text.lower(),
- f"Unexpected crash output: {text}"
+ self.assertNotIn(
+ "WKWebView", text,
+ f"Chrome was incorrectly detected as WKWebView app: {text[:200]}"
)