Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

11.6 KB
Raw Blame
"""Native tasks on the WPF and WinUI3 (Windows UIA) and GTK3 (Linux AT-SPI) harnesses (RFC #4268).

The fixtures are real ``get_window_state`` results from each harness in task
mode, recorded by ``verify_native.py --capture-dir``.
"""

from __future__ import annotations

import json
import sys
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from choose_action import validate_request
from jev_adapter import choose_mock_for_task
from native import NativeObservation, eligible_controls
from native_roles import RAW_ROLES, role_class
from native_tasks import (
    HARNESSES,
    NATIVE_TASK_IDS,
    appkit_task,
    native_choice_request,
    native_task,
    split_task_id,
)
from sources import NativeAccessibilitySource
from tasks import TaskSources

FIXTURES = Path(__file__).resolve().parents[2] / "fixtures/native"


def window_state(harness: str, name: str) -> dict:
    path = FIXTURES / f"{harness}-window-state-{name}-v1.json"
    return json.loads(path.read_text(encoding="utf-8"))


def sources(task, harness: str, name: str) -> TaskSources:
    payload = window_state(harness, name)
    observation = NativeObservation.from_window_state(
        payload, expected_pid=payload["pid"], expected_window_id=payload["window_id"]
    )
    ax = NativeAccessibilitySource.from_observation(
        observation, HARNESSES[harness].platform, redact=task.redact_text, text_method=task.text_method
    )
    return TaskSources(ax=ax)


class HarnessRegistryTest(unittest.TestCase):
    def test_every_harness_has_the_same_three_tasks(self) -> None:
        self.assertEqual(
            NATIVE_TASK_IDS,
            (
                *(f"{h}-{k}" for h in ("appkit", "wpf", "winui3", "gtk3") for k in ("counter", "save-note", "choose-size")),
                "canvas-cancel",
            ),
        )
        for task_id in NATIVE_TASK_IDS:
            if task_id == "canvas-cancel":
                continue  # the visual-only canvas is not a form harness (see test_native_canvas.py)
            harness, kind = split_task_id(task_id)
            task = native_task(task_id, Path("/tmp/none.json"), pid=7)
            self.assertEqual(task.scope.window_title, harness.window_title)
            self.assertEqual(task.oracle.schema, harness.state_schema)
            self.assertEqual(task.oracle.expected_pid, 7)
            # Same semantics everywhere: the mock takes the same IDs on every platform.
            self.assertEqual(
                task.mock_preferences,
                native_task(f"appkit-{kind}", Path("/tmp/none.json")).mock_preferences,
            )

    def test_platforms_and_windows(self) -> None:
        self.assertEqual({h.name: h.platform for h in HARNESSES.values()},
                         {"appkit": "macos", "wpf": "windows", "winui3": "windows", "gtk3": "linux"})
        self.assertEqual(HARNESSES["wpf"].window_title, "CuaTestHarness WPF Tasks")
        self.assertEqual(HARNESSES["winui3"].window_title, "CuaTestHarness WinUI3 Tasks")
        self.assertEqual(HARNESSES["winui3"].state_env, "CUA_WINUI3_TASK_STATE")
        self.assertEqual(HARNESSES["winui3"].process_name, "CuaTestHarness.WinUI3")
        self.assertEqual(HARNESSES["gtk3"].window_title, "CuaTestHarness GTK3 Tasks")

    def test_unknown_tasks_are_rejected(self) -> None:
        for task_id in ("wpf-reset", "winui3-exit", "uwp-counter", "counter", ""):
            with self.assertRaises(ValueError):
                split_task_id(task_id)
        with self.assertRaises(ValueError):
            appkit_task("wpf-counter", Path("/tmp/none.json"))


class RecordedHarnessFixtureTest(unittest.TestCase):
    """Role table and element rules against real WPF, WinUI3, and GTK3 observations."""

    HARNESSES = ("wpf", "winui3", "gtk3")

    def test_task_controls_map_to_the_same_candidate_ids(self) -> None:
        for harness in self.HARNESSES:
            with self.subTest(harness=harness):
                note = native_task(f"{harness}-save-note", Path("/tmp/none.json"), note_text="secret note")
                ids = [c.id for c in note.plan(sources(note, harness, "initial")).candidates]
                self.assertEqual(ids[-2:], ["reobserve", "abstain"])
                self.assertIn("ax:text_input:note:set:note", ids)
                self.assertIn("ax:button:save-note", ids)
                self.assertIn("ax:button:increment", ids)
                # Reset (destructive) and Exit (close_unsaved) never reach the chooser.
                self.assertNotIn("ax:button:reset", ids)
                self.assertNotIn("ax:button:exit", ids)
                size = native_task(f"{harness}-choose-size", Path("/tmp/none.json"))
                size_ids = [c.id for c in size.plan(sources(size, harness, "initial")).candidates]
                for expected in ("ax:radio:small", "ax:radio:medium", "ax:radio:large", "ax:checkbox:i-agree"):
                    self.assertIn(expected, size_ids)

    def test_requests_are_valid_and_leak_no_secret_or_token(self) -> None:
        for harness in self.HARNESSES:
            with self.subTest(harness=harness):
                task = native_task(f"{harness}-save-note", Path("/tmp/none.json"), note_text="secret note")
                task_sources = sources(task, harness, "initial")
                step = task.plan(task_sources)
                setter = {c.id: c for c in step.candidates}["ax:text_input:note:set:note"]
                self.assertEqual(setter.tool, "set_value")
                self.assertEqual(setter.arguments["value"], "secret note")
                request = native_choice_request(task, task_sources, step, [])
                validate_request(request)
                wire = json.dumps(request)
                self.assertNotIn("secret note", wire)
                self.assertNotIn("element_token", wire)
                self.assertNotIn(f"{task_sources.ax.observation.snapshot_id}:", wire)

    def test_selected_state_is_read_from_the_platform(self) -> None:
        for harness in self.HARNESSES:
            with self.subTest(harness=harness):
                size = native_task(f"{harness}-choose-size", Path("/tmp/none.json"))
                initial = sources(size, harness, "initial")
                choice, _, _ = choose_mock_for_task(size, initial, size.candidates(initial), [])
                self.assertEqual(choice, "ax:radio:large")
                after = sources(size, harness, "after-choose-size")
                controls = {c.id: c for c in after.ax.controls}
                self.assertIs(controls["ax:radio:large"].selected, True)
                self.assertIs(controls["ax:checkbox:i-agree"].selected, True)
                ids = [c.id for c in size.candidates(after)]
                self.assertNotIn("ax:radio:large", ids)  # already selected
                self.assertIn("ax:radio:small", ids)

    def test_text_entry_value_is_never_a_label(self) -> None:
        for harness in self.HARNESSES:
            with self.subTest(harness=harness):
                note = native_task(f"{harness}-save-note", Path("/tmp/none.json"))
                after = sources(note, harness, "after-save-note")
                field = next(c for c in after.ax.controls if c.role_class == "text_input")
                self.assertEqual(field.label, "Note")
                ids = [c.id for c in note.candidates(after)]
                if harness == "gtk3":
                    # Known limitation of Cua Driver 0.30.2 (#4291): a named
                    # AT-SPI text field reports no value, so the written note
                    # is not observable and the write stays offered.
                    self.assertIn("limitation", window_state(harness, "after-save-note")["_fixture"])
                    self.assertIsNone(field.value)
                    self.assertIn("ax:text_input:note:set:note", ids)
                    continue
                self.assertEqual(field.value, "jev-use native note")
                self.assertNotIn("ax:text_input:note:set:note", ids)  # already holds the parameter

    def test_windows_title_bar_is_window_chrome(self) -> None:
        for harness in ("wpf", "winui3"):
            with self.subTest(harness=harness):
                payload = window_state(harness, "initial")
                observation = NativeObservation.from_window_state(
                    payload, expected_pid=payload["pid"], expected_window_id=payload["window_id"]
                )
                native = eligible_controls(observation, "windows")
                labels = [c.label for c in native.controls]
                for chrome in ("System", "Minimize", "Maximize", "Close"):
                    self.assertNotIn(chrome, labels)
                self.assertEqual(native.excluded.get("window_chrome"), 4)
                self.assertEqual(
                    labels,
                    ["Increment", "Reset", "I agree", "Small", "Medium", "Large", "Note", "Save note", "Exit"],
                )

    def test_winui3_roles_map_through_the_windows_table(self) -> None:
        """WinUI3's automation peers report the same UIA control types as WPF.

        No WinUI3-specific row is needed: every raw role in the recorded
        WinUI3 trees is either a Windows role-table row or deliberately
        excluded. WinUI3 exposes its TextBlock as static ``Text`` (WPF's task
        window does not), which is an unknown role and never a candidate.
        """
        windows_rows = {raw for rows in RAW_ROLES["windows"].values() for raw in rows}
        for name in ("initial", "after-save-note", "after-choose-size"):
            with self.subTest(fixture=name):
                roles = {e["role"] for e in window_state("winui3", name)["elements"]}
                self.assertEqual(roles - windows_rows, {"Text", "TitleBar"})
                self.assertIsNone(role_class("Text", "windows"))
        payload = window_state("winui3", "initial")
        observation = NativeObservation.from_window_state(
            payload, expected_pid=payload["pid"], expected_window_id=payload["window_id"]
        )
        native = eligible_controls(observation, "windows")
        # The static counter label and the title bar container are unknown
        # roles; the title bar's four buttons are window chrome.
        self.assertEqual(native.excluded, {"unknown_role": 2, "window_chrome": 4})
        self.assertEqual(
            [(c.role_class, c.label) for c in native.controls],
            [("button", "Increment"), ("button", "Reset"), ("checkbox", "I agree"),
             ("radio", "Small"), ("radio", "Medium"), ("radio", "Large"),
             ("text_input", "Note"), ("button", "Save note"), ("button", "Exit")],
        )
        self.assertEqual(
            [c.id for c in native.controls],
            [c.id for c in eligible_controls(
                NativeObservation.from_window_state(
                    window_state("wpf", "initial"),
                    expected_pid=window_state("wpf", "initial")["pid"],
                    expected_window_id=window_state("wpf", "initial")["window_id"],
                ),
                "windows",
            ).controls],
        )

    def test_linux_roles_map_to_role_classes(self) -> None:
        payload = window_state("gtk3", "initial")
        observation = NativeObservation.from_window_state(
            payload, expected_pid=payload["pid"], expected_window_id=payload["window_id"]
        )
        native = eligible_controls(observation, "linux")
        self.assertEqual(native.excluded, {})
        self.assertEqual(
            [(c.role_class, c.label) for c in native.controls],
            [("button", "Increment"), ("button", "Reset"), ("checkbox", "I agree"),
             ("radio", "Small"), ("radio", "Medium"), ("radio", "Large"),
             ("text_input", "Note"), ("button", "Save note"), ("button", "Exit")],
        )

if __name__ == "__main__":
    unittest.main()