Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): rewrite test_webkit_js with correct DriverClient API
Francesco Bonacci6c3d8576feef
diff --git a/libs/cua-driver/Tests/integration/test_webkit_js.py b/libs/cua-driver/Tests/integration/test_webkit_js.pyindex 2f5a417a1d..a3d46f697c 100644--- a/libs/cua-driver/Tests/integration/test_webkit_js.py+++ b/libs/cua-driver/Tests/integration/test_webkit_js.py@@ -1,14 +1,14 @@-"""Integration tests: WKWebView/Tauri AX fallback path (page tool).+"""Integration tests: WKWebView/Tauri AX fallback (page tool).-Tests the AX-based fallback for apps where the WebKit remote inspector is-blocked — specifically:- - page(action=get_text) routes through AXPageReader when the target is- a WKWebView app (WebInspectorXPC.isWKWebViewApp returns true)- - page(action=query_dom) same AX fallback with CSS selector → AX role+Tests AX-based get_text and query_dom for Tauri/WKWebView apps where the+WebKit remote inspector is blocked, and verifies that:+ - execute_javascript returns a clear error pointing to AX alternatives+ - get_text returns content via the AX tree+ - query_dom returns elements via the AX tree+ - Chrome is NOT misdetected as a WKWebView app-Requires Conductor.app (a Tauri-based app shipped with cua-driver) to be-installed in /Applications. Tests that skip the inspector path skip when-Conductor is not installed or when TAURI_WEBVIEW_AUTOMATION is not set.+Requires Conductor.app (/Applications/Conductor.app) — a Tauri app.+Tests that need Conductor skip when it is not installed.Run:scripts/test.sh test_webkit_js@@ -16,8 +16,8 @@ Run:from __future__ import annotations-import jsonimport os+import reimport subprocessimport sysimport time@@ -26,8 +26,8 @@ import unittestsys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))from driver_client import DriverClient, default_binary_path-CONDUCTOR_BUNDLE = "com.trycua.Conductor"-CONDUCTOR_APP_PATH = "/Applications/Conductor.app"+CONDUCTOR_BUNDLE = "com.conductor.app"+CONDUCTOR_PATH = "/Applications/Conductor.app"# ---------------------------------------------------------------------------@@ -35,245 +35,184 @@ CONDUCTOR_APP_PATH = "/Applications/Conductor.app"# ---------------------------------------------------------------------------def _tool_text(result: dict) -> str:- """Extract the text string from a tool call result."""for item in result.get("content", []):if item.get("type") == "text":return item.get("text", "")return ""-def _is_error(result: dict) -> bool:- return result.get("isError", False)--def _conductor_installed() -> bool:- return os.path.isdir(CONDUCTOR_APP_PATH)---def _find_conductor_pid(client: DriverClient) -> int | None:- """Return the pid of a running Conductor instance, or None."""- result = client.call("list_apps", {})- text = _tool_text(result)- for line in text.splitlines():- if CONDUCTOR_BUNDLE in line or "Conductor" in line:- # Line format: "Conductor (com.trycua.Conductor) pid=1234"- for part in line.split():- if part.startswith("pid="):- try:- return int(part[4:])- except ValueError:- pass- return None+ return os.path.isdir(CONDUCTOR_PATH)-# ----------------------------------------------------------------------------# AXPageReader unit-level tests (no app required)-# ---------------------------------------------------------------------------+def _kill_conductor() -> None:+ subprocess.run(+ ["osascript", "-e", 'tell application "Conductor" to quit'],+ check=False, timeout=5,+ )+ time.sleep(1)-class TestAXPageReaderExtractText(unittest.TestCase):- """Test that AXPageReader.extractText produces the right output."""- # We test this indirectly via page(get_text) on a synthetic AX tree- # produced by get_window_state — so we still need the driver running.+def _find_pid(text: str) -> int | None:+ m = re.search(r'pid[=:\s]+(\d+)', text, re.IGNORECASE)+ return int(m.group(1)) if m else None- @classmethod- def setUpClass(cls):- cls.client = DriverClient(binary_path=default_binary_path())- cls.client.start()-- @classmethod- def tearDownClass(cls):- cls.client.stop()- def test_driver_responds(self):- """Smoke test: driver is reachable."""- result = self.client.call("get_screen_size", {})- self.assertFalse(_is_error(result), _tool_text(result))- text = _tool_text(result)- self.assertIn("width", text.lower())+def _main_window(client: DriverClient, pid: int) -> int | None:+ """Return the on-screen Conductor window_id, or None if not found."""+ text = _tool_text(client.call_tool("list_windows", {"pid": pid}))+ for line in text.splitlines():+ if "on_screen=True" in line or "is_on_screen=True" in line:+ m = re.search(r'\[window_id:\s*(\d+)\]', line)+ if m:+ return int(m.group(1))+ # Fall back to any titled window+ m = re.search(r'"[^"]+"\s+\[window_id:\s*(\d+)\]', text)+ if m:+ return int(m.group(1))+ m = re.search(r'\[window_id:\s*(\d+)\]', text)+ return int(m.group(1)) if m else None# ----------------------------------------------------------------------------# WebInspectorXPC detection tests+# Conductor / Tauri AX fallback tests# ----------------------------------------------------------------------------class TestWebInspectorXPCDetection(unittest.TestCase):- """Test isWKWebViewApp detection heuristic via execute_javascript errors."""[email protected](_conductor_installed(), "Conductor not installed")+class WebKitJSTests(unittest.TestCase):+ """AX-based page primitives work end-to-end against a live Conductor window."""- @classmethod- def setUpClass(cls):- cls.client = DriverClient(binary_path=default_binary_path())- cls.client.start()- # List apps to find Chrome pid (should NOT be detected as WKWebView app).- cls.chrome_pid = None- result = cls.client.call("list_apps", {})- text = _tool_text(result)- for line in text.splitlines():- if "com.google.Chrome" in line:- for part in line.split():- if part.startswith("pid="):- try:- cls.chrome_pid = int(part[4:])- except ValueError:- pass+ _pid: int+ _window_id: int@classmethod- def tearDownClass(cls):- cls.client.stop()+ def setUpClass(cls) -> None:+ cls.binary = default_binary_path()+ _kill_conductor()- @unittest.skipUnless(- os.path.exists("/Applications/Google Chrome.app"),- "Chrome not installed"- )- def test_chrome_not_detected_as_wkwebview(self):- """Chrome must NOT be classified as a WKWebView app."""- if self.chrome_pid is None:- self.skipTest("Chrome not running")- # list_windows to get a window_id for Chrome.- win_result = self.client.call("list_windows", {"pid": self.chrome_pid})- text = _tool_text(win_result)- # Extract first window_id from output.- window_id = None- for line in text.splitlines():- for part in line.split():- if part.startswith("id="):- try:- window_id = int(part[3:])- break- except ValueError:- pass- if window_id:- break- if window_id is None:- self.skipTest("No Chrome window found")- # execute_javascript on Chrome should succeed or fail with an Apple Events- # error — NOT with the WKWebView-specific "use get_text or query_dom" message.- result = self.client.call("page", {- "pid": self.chrome_pid,- "window_id": window_id,- "action": "execute_javascript",- "javascript": "1+1",- })- text = _tool_text(result)- self.assertNotIn(- "WKWebView",- text,- f"Chrome was incorrectly detected as a WKWebView app: {text}",- )+ with DriverClient(cls.binary) as c:+ result = c.call_tool("launch_app", {"bundle_id": CONDUCTOR_BUNDLE})+ text = _tool_text(result)+ cls._pid = _find_pid(text) or 0+ if not cls._pid:+ raise RuntimeError("Could not determine Conductor pid from launch_app result")-# ----------------------------------------------------------------------------# Conductor / Tauri AX fallback tests-# ---------------------------------------------------------------------------+ deadline = time.monotonic() + 10.0+ cls._window_id = None+ while time.monotonic() < deadline:+ with DriverClient(cls.binary) as c:+ wid = _main_window(c, cls._pid)+ if wid:+ cls._window_id = wid+ break+ time.sleep(0.5)-class TestConductorAXFallback(unittest.TestCase):- """Test AX-based get_text / query_dom for Conductor (Tauri app)."""+ if not cls._window_id:+ raise RuntimeError(f"No window found for Conductor pid {cls._pid}")- @classmethod- def setUpClass(cls):- if not _conductor_installed():- return- cls.client = DriverClient(binary_path=default_binary_path())- cls.client.start()- # Launch Conductor if not already running.- launch_result = cls.client.call("launch_app", {- "bundle_id": CONDUCTOR_BUNDLE,- })- time.sleep(3) # Wait for app to be ready.- cls.pid = _find_conductor_pid(cls.client)+ print(f"\n Conductor pid={cls._pid} window_id={cls._window_id}")@classmethod- def tearDownClass(cls):- if not _conductor_installed():- return- if hasattr(cls, "client"):- cls.client.stop()-- def _get_window_id(self) -> int | None:- if self.pid is None:- return None- result = self.client.call("list_windows", {"pid": self.pid})- text = _tool_text(result)- for line in text.splitlines():- for part in line.split():- if part.startswith("id="):- try:- return int(part[3:])- except ValueError:- pass- return None-- @unittest.skipUnless(_conductor_installed(), "Conductor not installed")- def test_execute_javascript_returns_wkwebview_error(self):- """execute_javascript on Conductor must return the WKWebView error."""- window_id = self._get_window_id()- if window_id is None:- self.skipTest("Conductor window not found")- result = self.client.call("page", {- "pid": self.pid,- "window_id": window_id,- "action": "execute_javascript",- "javascript": "document.title",- })- self.assertTrue(_is_error(result), "Expected error for WKWebView JS")+ def tearDownClass(cls) -> None:+ _kill_conductor()++ # -----------------------------------------------------------------------+ # execute_javascript — must return WKWebView error, not crash+ # -----------------------------------------------------------------------++ def test_13_execute_javascript_returns_clear_error_for_wkwebview(self) -> None:+ """execute_javascript on a WKWebView app returns a clear error."""+ with DriverClient(self.binary) as c:+ result = c.call_tool("page", {+ "pid": self._pid,+ "window_id": self._window_id,+ "action": "execute_javascript",+ "javascript": "document.title",+ })+ self.assertTrue(result.get("isError"), "Expected isError=True for WKWebView JS")text = _tool_text(result)self.assertIn("WKWebView", text)self.assertIn("get_text", text)- @unittest.skipUnless(_conductor_installed(), "Conductor not installed")- def test_get_text_via_ax_tree(self):- """get_text on Conductor must return content via AX tree."""- window_id = self._get_window_id()- if window_id is None:- self.skipTest("Conductor window not found")- result = self.client.call("page", {- "pid": self.pid,- "window_id": window_id,- "action": "get_text",- })- self.assertFalse(_is_error(result), _tool_text(result))+ # -----------------------------------------------------------------------+ # get_text — must return content via AX tree+ # -----------------------------------------------------------------------++ def test_11_ax_get_text_works_without_inspector(self) -> None:+ """get_text returns page text via AX tree (no inspector needed)."""+ with DriverClient(self.binary) as c:+ result = c.call_tool("page", {+ "pid": self._pid,+ "window_id": self._window_id,+ "action": "get_text",+ })+ self.assertFalse(result.get("isError"), _tool_text(result))text = _tool_text(result)- # AX tree result should mention the fallback path.- self.assertTrue(- len(text.strip()) > 0,- "get_text returned empty content"- )-- @unittest.skipUnless(_conductor_installed(), "Conductor not installed")- def test_query_dom_buttons_via_ax_tree(self):- """query_dom(button) on Conductor returns AX buttons."""- window_id = self._get_window_id()- if window_id is None:- self.skipTest("Conductor window not found")- result = self.client.call("page", {- "pid": self.pid,- "window_id": window_id,- "action": "query_dom",- "css_selector": "button",- })+ self.assertTrue(len(text.strip()) > 0, "get_text returned empty content")++ # -----------------------------------------------------------------------+ # query_dom — must return AX elements via AX tree+ # -----------------------------------------------------------------------++ def test_12_ax_query_dom_buttons(self) -> None:+ """query_dom for button elements returns AX tree results."""+ with DriverClient(self.binary) as c:+ result = c.call_tool("page", {+ "pid": self._pid,+ "window_id": self._window_id,+ "action": "query_dom",+ "css_selector": "button",+ })text = _tool_text(result)- # Either returns buttons or returns a well-formed error — not a crash.- self.assertFalse(- "Traceback" in text or "fatal" in text.lower(),- f"Unexpected crash output: {text}"- )+ self.assertNotIn("Traceback", text)+ # Either found buttons (role key) or returned empty — both are valid+ if not result.get("isError"):+ self.assertTrue(+ "AXButton" in text or "role" in text or "[]" in text,+ f"Unexpected query_dom output: {text[:200]}"+ )++ # -----------------------------------------------------------------------+ # Chrome must NOT be misdetected as WKWebView+ # -----------------------------------------------------------------------++ def test_chrome_not_detected_as_wkwebview(self) -> None:+ """Chrome execute_javascript must not return the WKWebView error."""+ with DriverClient(self.binary) as c:+ result = c.call_tool("list_apps")+ text = _tool_text(result)++ chrome_pid = None+ for line in text.splitlines():+ if "com.google.Chrome" in line:+ m = re.search(r'pid[=:\s]+(\d+)', line, re.IGNORECASE)+ if m:+ chrome_pid = int(m.group(1))+ break- @unittest.skipUnless(_conductor_installed(), "Conductor not installed")- def test_query_dom_links_via_ax_tree(self):- """query_dom(a) on Conductor returns AX links."""- window_id = self._get_window_id()- if window_id is None:- self.skipTest("Conductor window not found")- result = self.client.call("page", {- "pid": self.pid,- "window_id": window_id,- "action": "query_dom",- "css_selector": "a",- })+ if chrome_pid is None:+ self.skipTest("Chrome not running")++ with DriverClient(self.binary) as c:+ windows_text = _tool_text(c.call_tool("list_windows", {"pid": chrome_pid}))++ m = re.search(r'\[window_id:\s*(\d+)\]', windows_text)+ if not m:+ self.skipTest("No Chrome window found")+ window_id = int(m.group(1))++ with DriverClient(self.binary) as c:+ result = c.call_tool("page", {+ "pid": chrome_pid,+ "window_id": window_id,+ "action": "execute_javascript",+ "javascript": "1+1",+ })text = _tool_text(result)- self.assertFalse(- "Traceback" in text or "fatal" in text.lower(),- f"Unexpected crash output: {text}"+ self.assertNotIn(+ "WKWebView", text,+ f"Chrome was incorrectly detected as WKWebView app: {text[:200]}")