Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

17.5 KB
Raw Blame
"""Larger candidate sets, relevance capping, and accuracy measurement (#4312)."""

from __future__ import annotations

import ast
import contextlib
import io
import json
import re
import sys
import tempfile
import unittest
from dataclasses import replace
from pathlib import Path

HERE = Path(__file__).resolve().parent
EXAMPLE = HERE.parents[1]
sys.path.insert(0, str(HERE.parent))
sys.path.insert(0, str(EXAMPLE))

import measure_native  # noqa: E402
from native import NativeObservation, risk_categories  # noqa: E402
from native_tasks import (  # noqa: E402
    HARNESSES,
    MAX_EXECUTABLE_CANDIDATES,
    TASK_KINDS,
    compose,
    native_task,
)
from sources import Candidate, NativeAccessibilitySource  # noqa: E402
from tasks import TaskSources  # noqa: E402

FIXTURES = EXAMPLE / "fixtures" / "native"
APPS = EXAMPLE.parents[1] / "tests" / "fixtures" / "apps"
DENSITY_HARNESSES = ("gtk3", "appkit", "wpf", "winui3")
TARGETS = {
    "counter": {"ax:button:increment"},
    "save-note": {"ax:text_input:note:set:note", "ax:button:save-note"},
    "choose-size": {"ax:radio:large", "ax:checkbox:i-agree"},
}


def load(name: str) -> dict:
    return json.loads((FIXTURES / name).read_text(encoding="utf-8"))


def plan_for(harness: str, kind: str, payload: dict, cap_order: str = "relevance"):
    task = replace(native_task(f"{harness}-{kind}", Path("/nonexistent")), cap_order=cap_order)
    observation = NativeObservation.from_window_state(
        payload, expected_pid=payload["pid"], expected_window_id=payload["window_id"]
    )
    ax = NativeAccessibilitySource.from_observation(
        observation, HARNESSES[harness].platform, redact=task.redact_text, text_method=task.text_method
    )
    return task.plan(TaskSources(ax=ax))


class ComposeRelevanceTest(unittest.TestCase):
    @staticmethod
    def cand(name: str) -> Candidate:
        return Candidate(name, "d", "click", {}, source="ax")

    def test_within_cap_is_unchanged(self) -> None:
        group = [self.cand(f"ax:button:b{n}") for n in range(MAX_EXECUTABLE_CANDIDATES)]
        plain, _ = compose({"ax": group}, allowed_risks=frozenset())
        ranked, stats = compose(
            {"ax": group}, allowed_risks=frozenset(), relevance=lambda c: 0 if c.id.endswith("23") else 2
        )
        self.assertEqual([c.id for c in plain], [c.id for c in ranked])
        self.assertEqual(stats.dropped, 0)

    def test_over_cap_keeps_relevant_in_element_order(self) -> None:
        group = [self.cand(f"ax:button:b{n}") for n in range(30)]
        relevant = {"ax:button:b29", "ax:button:b27"}
        candidates, stats = compose(
            {"ax": group}, allowed_risks=frozenset(), relevance=lambda c: 0 if c.id in relevant else 2
        )
        ids = [c.id for c in candidates]
        self.assertEqual(len(ids), MAX_EXECUTABLE_CANDIDATES + 2)
        self.assertEqual(stats.dropped, 6)
        self.assertTrue(relevant <= set(ids))
        # Ranking decides only which survive; the order stays depth-first.
        executable = ids[:-2]
        self.assertEqual(executable, sorted(executable, key=lambda i: int(i.rsplit("b", 1)[1])))
        self.assertEqual(executable[:22], [f"ax:button:b{n}" for n in range(22)])
        self.assertEqual(ids[-2:], ["reobserve", "abstain"])
        # Deterministic: the same input gives the same set.
        again, _ = compose(
            {"ax": group}, allowed_risks=frozenset(), relevance=lambda c: 0 if c.id in relevant else 2
        )
        self.assertEqual(ids, [c.id for c in again])


class DensityFixtureTest(unittest.TestCase):
    def test_sizes_and_cap(self) -> None:
        expected_sizes = {
            ("gtk3", 12): {"counter": 12, "save-note": 14, "choose-size": 12},
            ("appkit", 12): {"counter": 14, "save-note": 16, "choose-size": 12},
            ("wpf", 12): {"counter": 12, "save-note": 14, "choose-size": 12},
            ("winui3", 12): {"counter": 12, "save-note": 14, "choose-size": 12},
            **{(harness, 24): dict.fromkeys(TASK_KINDS, 26) for harness in DENSITY_HARNESSES},
        }
        for (harness, density), sizes in expected_sizes.items():
            payload = load(f"{harness}-window-state-density-{density}-v1.json")
            for kind in TASK_KINDS:
                with self.subTest(harness=harness, density=density, kind=kind):
                    plan = plan_for(harness, kind, payload)
                    self.assertEqual(len(plan.candidates), sizes[kind])
                    self.assertTrue(TARGETS[kind] <= {c.id for c in plan.candidates})
                    self.assertEqual(plan.stats.dropped > 0, density == 24)

    def test_golden_sets(self) -> None:
        """The presented IDs match the golden file that the TypeScript test also checks."""
        golden = load("native-density-candidates-v1.json")["sets"]
        self.assertEqual(len(golden), len(DENSITY_HARNESSES) * 2 * 3 * 2)
        for harness in DENSITY_HARNESSES:
            for density in (12, 24):
                payload = load(f"{harness}-window-state-density-{density}-v1.json")
                for kind in TASK_KINDS:
                    for cap_order in ("relevance", "depth_first"):
                        plan = plan_for(harness, kind, payload, cap_order=cap_order)
                        self.assertEqual(
                            {"ids": [c.id for c in plan.candidates], "dropped": plan.stats.dropped},
                            golden[f"{harness}-{kind}-d{density}-{cap_order}"],
                        )

    def test_depth_first_drops_targets_behind_distractors(self) -> None:
        for harness in DENSITY_HARNESSES:
            payload = load(f"{harness}-window-state-density-24-v1.json")
            for kind in TASK_KINDS:
                with self.subTest(harness=harness, kind=kind):
                    plan = plan_for(harness, kind, payload, cap_order="depth_first")
                    self.assertEqual(len(plan.candidates), MAX_EXECUTABLE_CANDIDATES + 2)
                    self.assertFalse(TARGETS[kind] & {c.id for c in plan.candidates})

    def test_cap_order_is_irrelevant_within_the_cap(self) -> None:
        fixtures = {
            "appkit": "appkit-window-state-initial-v1.json",
            "wpf": "wpf-window-state-initial-v1.json",
            "winui3": "winui3-window-state-initial-v1.json",
            "gtk3": "gtk3-window-state-initial-v1.json",
        }
        for harness, name in fixtures.items():
            payload = load(name)
            for kind in TASK_KINDS:
                with self.subTest(harness=harness, kind=kind):
                    ranked = plan_for(harness, kind, payload)
                    plain = plan_for(harness, kind, payload, cap_order="depth_first")
                    self.assertEqual(
                        [(c.id, c.description, dict(c.arguments)) for c in ranked.candidates],
                        [(c.id, c.description, dict(c.arguments)) for c in plain.candidates],
                    )
        for harness in ("gtk3", "wpf", "winui3"):
            payload = load(f"{harness}-window-state-density-12-v1.json")
            for kind in TASK_KINDS:
                self.assertEqual(
                    [c.id for c in plan_for(harness, kind, payload).candidates],
                    [c.id for c in plan_for(harness, kind, payload, cap_order="depth_first").candidates],
                )

    def test_relevance_uses_no_values(self) -> None:
        payload = load("gtk3-window-state-density-24-v1.json")
        for element in payload["elements"]:
            if element.get("label") == "Search":
                element["value"] = "Increment Save note Large I agree"
        plan = plan_for("gtk3", "save-note", payload)
        self.assertNotIn("ax:text_input:search:set:note", [c.id for c in plan.candidates][:1])
        baseline = plan_for("gtk3", "save-note", load("gtk3-window-state-density-24-v1.json"))
        self.assertEqual([c.id for c in plan.candidates], [c.id for c in baseline.candidates])

    def test_invalid_cap_order(self) -> None:
        with self.assertRaises(ValueError):
            replace(native_task("gtk3-counter", Path("/nonexistent")), cap_order="random")  # type: ignore[arg-type]


class ExpectedNextTest(unittest.TestCase):
    def entry(self, task, candidate_id: str) -> dict:
        return task.history_entry(1, candidate_id, outcome="done")

    def test_counter(self) -> None:
        task = native_task("gtk3-counter", Path("/nonexistent"))
        history: list[dict] = []
        for _ in range(3):
            self.assertEqual(task.expected_next(history), ["ax:button:increment"])
            history.append(self.entry(task, "ax:button:increment"))
        self.assertEqual(task.expected_next(history), [])

    def test_ordered_and_unordered_steps(self) -> None:
        note = native_task("gtk3-save-note", Path("/nonexistent"))
        self.assertEqual(note.expected_next([]), ["ax:text_input:note:set:note"])
        self.assertEqual(
            note.expected_next([self.entry(note, "ax:text_input:note:set:note")]), ["ax:button:save-note"]
        )
        size = native_task("gtk3-choose-size", Path("/nonexistent"))
        self.assertEqual(size.expected_next([]), ["ax:radio:large", "ax:checkbox:i-agree"])
        # A refused or reobserve step performed nothing.
        refused = size.history_entry(1, "ax:radio:large", refusal="background_denied")
        self.assertEqual(size.expected_next([refused]), ["ax:radio:large", "ax:checkbox:i-agree"])
        self.assertEqual(
            size.expected_next([self.entry(size, "ax:radio:large:foreground")]), ["ax:checkbox:i-agree"]
        )


class DistractorLabelTest(unittest.TestCase):
    """The harnesses' distractor labels are benign and identical across platforms."""

    @staticmethod
    def gtk3_lists() -> dict[str, list]:
        tree = ast.parse((APPS / "linux/gtk3/main.py").read_text(encoding="utf-8"))
        values = {}
        for node in tree.body:
            if isinstance(node, ast.Assign) and isinstance(node.targets[0], ast.Name):
                name = node.targets[0].id
                if name.startswith("DISTRACTOR_") or name == "DENSITY_COUNTS":
                    values[name] = ast.literal_eval(node.value)
        return values

    @staticmethod
    def swift_list(source: str, name: str) -> list[str]:
        match = re.search(rf"let {name}\s*(?::[^=]+)?=\s*\[(.*?)\n\]", source, re.S) or re.search(
            rf"let {name}\s*=\s*\[([^\n]*)\]", source
        )
        assert match, name
        return re.findall(r'"([^"]+)"', match.group(1))

    @staticmethod
    def csharp_list(source: str, name: str) -> list[str]:
        match = re.search(rf"{name}\s*=\s*\{{(.*?)\}};", source, re.S)
        assert match, name
        return re.findall(r'"([^"]+)"', match.group(1))

    def test_labels(self) -> None:
        gtk3 = self.gtk3_lists()
        swift = (APPS / "macos/appkit/main.swift").read_text(encoding="utf-8")
        self.assertEqual(list(gtk3["DISTRACTOR_BUTTONS"]), self.swift_list(swift, "kDistractorButtons"))
        self.assertEqual(list(gtk3["DISTRACTOR_CHECKBOXES"]), self.swift_list(swift, "kDistractorCheckboxes"))
        self.assertEqual(
            [label for group in gtk3["DISTRACTOR_RADIO_GROUPS"] for label in group],
            self.swift_list(swift, "kDistractorRadioGroups"),
        )
        self.assertEqual(list(gtk3["DISTRACTOR_FIELDS"]), self.swift_list(swift, "kDistractorFields"))
        self.assertIn("12: (8, 3, 1, 1), 24: (26, 12, 4, 2)", swift)
        self.assertEqual(gtk3["DENSITY_COUNTS"], {12: (8, 3, 1, 1), 24: (26, 12, 4, 2)})
        for harness in ("wpf", "winui3"):
            with self.subTest(harness=harness):
                source = (APPS / f"windows/{harness}/TaskWindow.cs").read_text(encoding="utf-8")
                self.assertEqual(list(gtk3["DISTRACTOR_BUTTONS"]), self.csharp_list(source, "DistractorButtons"))
                self.assertEqual(list(gtk3["DISTRACTOR_CHECKBOXES"]), self.csharp_list(source, "DistractorCheckboxes"))
                self.assertEqual(
                    [label for group in gtk3["DISTRACTOR_RADIO_GROUPS"] for label in group],
                    self.csharp_list(source, "DistractorRadioGroups"),
                )
                self.assertEqual(list(gtk3["DISTRACTOR_FIELDS"]), self.csharp_list(source, "DistractorFields"))
                self.assertIn("[12] = (8, 3, 1, 1), [24] = (26, 12, 4, 2)", source)
                self.assertIn(f'"CUA_{harness.upper()}_TASK_DENSITY"', source)
        labels = [
            *gtk3["DISTRACTOR_BUTTONS"], *gtk3["DISTRACTOR_CHECKBOXES"], *gtk3["DISTRACTOR_FIELDS"],
            *(label for group in gtk3["DISTRACTOR_RADIO_GROUPS"] for label in group),
        ]
        self.assertEqual(len(labels), len(set(labels)))
        task_labels = {"Increment", "Reset", "I agree", "Small", "Medium", "Large", "Note", "Save note", "Exit"}
        for label in labels:
            with self.subTest(label=label):
                self.assertEqual(risk_categories(label), frozenset())
                self.assertNotIn(label, task_labels)


class MeasureTest(unittest.TestCase):
    def test_bucket_and_classify(self) -> None:
        self.assertEqual([measure_native.bucket(n) for n in (4, 8, 9, 14, 18, 19, 26)],
                         ["~4", "~4", "~12", "~12", "~12", "~24", "~24"])
        self.assertEqual(measure_native.classify("ax:a:foreground", ["ax:a"]), "correct")
        self.assertEqual(measure_native.classify("ax:b", ["ax:a"]), "wrong_action")
        self.assertEqual(measure_native.classify("reobserve", ["ax:a"]), "reobserve")
        self.assertEqual(measure_native.classify(None, ["ax:a"]), "abstain")
        self.assertEqual(measure_native.classify("ax:a", ["ax:a"], error=True), "error")

    def test_mock_replay(self) -> None:
        with tempfile.TemporaryDirectory() as out:
            specs = [
                f"gtk3:0:{FIXTURES / 'gtk3-window-state-initial-v1.json'}",
                f"gtk3:12:{FIXTURES / 'gtk3-window-state-density-12-v1.json'}",
                f"gtk3:24:{FIXTURES / 'gtk3-window-state-density-24-v1.json'}",
            ]
            args = ["replay", "--provider", "mock", "--reps", "2", "--cap-order", "depth_first",
                    "--cap-order", "relevance", "--order", "element", "--order", "shuffled", "--out", out,
                    "--group-by", "density,cap_order,order,bucket"]
            for spec in specs:
                args += ["--fixture", spec]
            with contextlib.redirect_stdout(io.StringIO()):
                measure_native.main(args)
            rows = json.loads((Path(out) / "table.json").read_text())
            decisions = [json.loads(line) for line in (Path(out) / "decisions.jsonl").read_text().splitlines()]
        # Seven decision points per repetition: counter 3, save-note 2, choose-size 2.
        self.assertEqual(len(decisions), 3 * 2 * 2 * 2 * 7)
        for row in rows:
            with self.subTest(row=row):
                if row["density"] == 24 and row["cap_order"] == "depth_first":
                    self.assertEqual((row["correct"], row["target_missing"]), (0, row["decisions"]))
                    self.assertEqual(row["reobserve"], row["decisions"])
                else:
                    self.assertEqual(row["accuracy"], 1.0)
                    self.assertEqual(row["target_missing"], 0)
        self.assertEqual({row["bucket"] for row in rows}, {"~4", "~12", "~24"})
        shuffled = [d for d in decisions if d["order"] == "shuffled" and d["density"] == 24]
        element = [d for d in decisions if d["order"] == "element" and d["density"] == 24]
        self.assertNotEqual({d["request_sha256"] for d in shuffled}, {d["request_sha256"] for d in element})

    def test_runner_logs(self) -> None:
        events = [
            {"event": "start", "task": "gtk3-counter", "language": "python", "provider": "s1", "platform": "linux"},
            {"event": "step", "step": 1, "candidate": "ax:button:increment", "candidate_count": 26,
             "expected_ids": ["ax:button:increment"], "expected_offered": True, "confidence": 0.9,
             "decide_ms": 2000.0, "compose": {"dropped": 4}},
            {"event": "step", "step": 2, "candidate": "ax:button:refresh", "candidate_count": 26,
             "expected_ids": ["ax:button:increment"], "expected_offered": True, "confidence": 0.4,
             "decide_ms": 2100.0},
            {"event": "step", "step": 3, "candidate": "reobserve", "candidate_count": 26,
             "expected_ids": ["ax:button:increment"], "expected_offered": True, "confidence": 0.5,
             "decide_ms": 2050.0},
            {"event": "outcome", "outcome": "abstained", "step": 4, "candidate_count": 26,
             "expected_ids": ["ax:button:increment"], "confidence": 0.6, "decide_ms": 1990.0},
            {"event": "outcome", "outcome": "unknown", "phase": "decide", "step": 5, "candidate_count": 26,
             "expected_ids": ["ax:button:increment"], "error": "S1ServiceError"},
            {"event": "outcome", "outcome": "verified", "step": 5},
        ]
        with tempfile.TemporaryDirectory() as root:
            log = Path(root) / "python-s1-gtk3-counter-d24.jsonl"
            log.write_text("".join(json.dumps(event) + "\n" for event in events))
            records = measure_native.decisions_from_log(log)
        self.assertEqual([r["result"] for r in records], ["correct", "wrong_action", "reobserve", "abstain", "error"])
        self.assertEqual({r["density"] for r in records}, {24})
        self.assertEqual({r["bucket"] for r in records}, {"~24"})
        rows = measure_native.aggregate(records, ("platform", "provider", "bucket"))
        self.assertEqual(len(rows), 1)
        self.assertEqual(rows[0]["accuracy"], 0.2)
        self.assertIn("| linux | s1 | ~24 | 26 | 5 | 1 | 20.0% |", measure_native.markdown(rows, ("platform", "provider", "bucket")))


if __name__ == "__main__":
    unittest.main()