Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

11.6 KB
Raw Blame
from __future__ import annotations

import copy
import sys
import unittest
from pathlib import Path

BASE = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(BASE / "python"))

from action_policy import (
    ActionAuthorizationError,
    ExactRegionTextAction,
    authorize_exact_region_text_action,
)
from choose_decision import choose_request
from decision_models import ModelScores


def inputs(
    text: str = "Send", required: str = "Send"
) -> tuple[dict, dict, dict, ExactRegionTextAction]:
    region = {
        "id": "text-60",
        "kind": "text",
        "text": text,
        "bounds": {"x": 12, "y": 20, "width": 72, "height": 32},
        "confidence": 0.98,
        "interactive": False,
    }
    action = ExactRegionTextAction("region:text-60", "text-60", required)
    request = {
        "capture_id": "capture-1",
        "regions": [copy.deepcopy(region)],
        "candidates": [
            action.wire_candidate(),
            {"id": "reobserve", "description": "Observe again."},
            {"id": "abstain", "description": "Stop."},
        ],
    }
    decision = {
        "schema": "cua.decision_choice_v1",
        "kind": "selected",
        "capture_id": "capture-1",
        "selected_id": action.candidate_id,
        "probabilities": {action.candidate_id: 0.777, "reobserve": 0.123, "abstain": 0.1},
    }
    parsed = {
        "schema": "cua.visual_regions_v1",
        "capture": {
            "capture_id": "capture-1",
            "source": {"kind": "window", "pid": 10, "window_id": 20},
            "screenshot": {
                "reference": "png-sha256:synthetic",
                "width": 100,
                "height": 100,
                "mime_type": "image/png",
            },
            "action_coordinate_space": {"kind": "screenshot_pixels"},
        },
        "regions": [region],
    }
    return request, decision, parsed, action


class ActionPolicyTest(unittest.TestCase):
    def authorize(self, request, decision, parsed, action, capture_id="capture-1", source=None):
        source = source or {"kind": "window", "pid": 10, "window_id": 20}
        return authorize_exact_region_text_action(
            request,
            decision,
            parsed,
            action,
            current_capture_id=capture_id,
            expected_source=source,
        )

    def test_matching_text_authorizes(self) -> None:
        click = self.authorize(*inputs())
        self.assertEqual((click.capture_id, click.x, click.y), ("capture-1", 48.0, 36.0))

    def test_selected_wrong_text_never_reaches_dispatch(self) -> None:
        request, decision, parsed, action = inputs("Send", "Save")
        request.update(
            {"schema": "cua.jev_choice_request_v1", "goal": "Choose one.", "history": []}
        )

        class WrongSelectionModel:
            name = "synthetic-unsafe-model"

            def score(self, _request):
                return ModelScores(
                    {action.candidate_id: 0.777, "reobserve": 0.123, "abstain": 0.1},
                    self.name,
                )

        decision = choose_request(request, WrongSelectionModel())
        self.assertEqual(
            (decision["kind"], decision["selected_id"]), ("selected", action.candidate_id)
        )
        dispatched: list[str] = []
        with self.assertRaisesRegex(ActionAuthorizationError, "exact-text condition"):
            self.authorize(request, decision, parsed, action)
            dispatched.append("click")
        self.assertEqual(dispatched, [])

    def test_stale_or_changed_capture_refuses(self) -> None:
        for field in ("request", "decision", "parse", "current"):
            with self.subTest(field=field):
                request, decision, parsed, action = inputs()
                current = "capture-1"
                if field == "request":
                    request["capture_id"] = "old"
                elif field == "decision":
                    decision["capture_id"] = "old"
                elif field == "parse":
                    parsed["capture"]["capture_id"] = "old"
                else:
                    current = "new"
                with self.assertRaisesRegex(ActionAuthorizationError, "capture identity"):
                    self.authorize(request, decision, parsed, action, current)

    def test_action_and_offered_condition_must_match(self) -> None:
        request, decision, parsed, action = inputs()
        request["candidates"][0]["description"] = "An unrelated condition."
        with self.assertRaisesRegex(ActionAuthorizationError, "action condition"):
            self.authorize(request, decision, parsed, action)

        request, decision, parsed, action = inputs()
        decision["selected_id"] = "abstain"
        with self.assertRaisesRegex(ActionAuthorizationError, "did not select"):
            self.authorize(request, decision, parsed, action)

    def test_nonunique_low_confidence_and_offered_region_refuse(self) -> None:
        for field in ("duplicate", "low_confidence", "offered", "bounds"):
            with self.subTest(field=field):
                request, decision, parsed, action = inputs()
                if field == "duplicate":
                    other = copy.deepcopy(parsed["regions"][0])
                    other["id"] = "text-61"
                    parsed["regions"].append(other)
                elif field == "low_confidence":
                    parsed["regions"][0]["confidence"] = 0.7999
                    request["regions"][0]["confidence"] = 0.7999
                elif field == "offered":
                    request["regions"][0]["text"] = "Save"
                else:
                    parsed["regions"][0]["bounds"]["width"] = 0
                    request["regions"][0]["bounds"]["width"] = 0
                with self.assertRaises(ActionAuthorizationError):
                    self.authorize(request, decision, parsed, action)

    def test_case_is_exact_and_reserved_outcomes_are_not_actions(self) -> None:
        request, decision, parsed, action = inputs("send", "Send")
        with self.assertRaisesRegex(ActionAuthorizationError, "exact-text condition"):
            self.authorize(request, decision, parsed, action)
        request, decision, parsed, action = inputs()
        decision.update({"kind": "abstain", "selected_id": "abstain"})
        with self.assertRaisesRegex(ActionAuthorizationError, "did not select"):
            self.authorize(request, decision, parsed, action)

    def test_unsupported_action_and_malformed_confidence_refuse(self) -> None:
        request, decision, parsed, action = inputs()
        with self.assertRaisesRegex(ActionAuthorizationError, "unsupported action"):
            self.authorize(request, decision, parsed, object())
        parsed["regions"][0]["confidence"] = float("nan")
        with self.assertRaisesRegex(ActionAuthorizationError, "malformed visual confidence"):
            self.authorize(request, decision, parsed, action)

    def test_decision_scores_must_match_offered_candidates(self) -> None:
        for case in ("missing", "wrong_set", "non_argmax"):
            with self.subTest(case=case):
                request, decision, parsed, action = inputs()
                if case == "missing":
                    del decision["probabilities"]
                elif case == "wrong_set":
                    decision["probabilities"]["invented"] = 0.0
                else:
                    decision["probabilities"] = {
                        action.candidate_id: 0.2,
                        "reobserve": 0.7,
                        "abstain": 0.1,
                    }
                with self.assertRaisesRegex(ActionAuthorizationError, "decision scores"):
                    self.authorize(request, decision, parsed, action)

    def test_wire_description_preserves_the_enforced_threshold(self) -> None:
        with self.assertRaisesRegex(ValueError, "two decimal places"):
            ExactRegionTextAction("region:one", "one", "Send", 0.805)

    def test_source_geometry_and_provenance_are_checked(self) -> None:
        for case in ("source", "screenshot", "outside", "mapping"):
            with self.subTest(case=case):
                request, decision, parsed, action = inputs()
                if case == "source":
                    parsed["capture"]["source"]["window_id"] = 21
                elif case == "screenshot":
                    parsed["capture"]["screenshot"]["mime_type"] = "image/jpeg"
                elif case == "outside":
                    parsed["regions"][0]["bounds"]["x"] = 90
                    request["regions"][0]["bounds"]["x"] = 90
                else:
                    parsed["capture"]["action_coordinate_space"] = {
                        "kind": "affine",
                        "m11": 0,
                        "m12": 0,
                        "m21": 0,
                        "m22": 0,
                        "tx": 0,
                        "ty": 0,
                    }
                with self.assertRaises(ActionAuthorizationError):
                    self.authorize(request, decision, parsed, action)

    def test_primary_desktop_and_affine_coordinates(self) -> None:
        request, decision, parsed, action = inputs()
        source = {"kind": "primary_desktop", "display_id": "primary"}
        parsed["capture"]["source"] = source
        parsed["capture"]["action_coordinate_space"] = {
            "kind": "affine",
            "m11": 2,
            "m12": 0,
            "m21": 0,
            "m22": 2,
            "tx": 10,
            "ty": 20,
        }
        click = self.authorize(request, decision, parsed, action, source=source)
        self.assertEqual((click.x, click.y), (48.0, 36.0))

    def test_affine_coordinates_are_not_mapped_twice(self) -> None:
        request, decision, parsed, action = inputs()
        parsed["capture"]["action_coordinate_space"] = {
            "kind": "affine",
            "m11": 0.5,
            "m12": 0,
            "m21": 0,
            "m22": 0.5,
            "tx": 0,
            "ty": 0,
        }
        click = self.authorize(request, decision, parsed, action)
        self.assertEqual((click.x, click.y), (48.0, 36.0))

    def test_affine_mapping_that_overflows_the_point_refuses(self) -> None:
        request, decision, parsed, action = inputs()
        parsed["capture"]["action_coordinate_space"] = {
            "kind": "affine",
            "m11": 1e308,
            "m12": 0,
            "m21": 0,
            "m22": 1e-308,
            "tx": 0,
            "ty": 0,
        }
        with self.assertRaisesRegex(ActionAuthorizationError, "non-finite point"):
            self.authorize(request, decision, parsed, action)

    def test_tied_scores_and_duplicate_offered_regions_refuse(self) -> None:
        request, decision, parsed, action = inputs()
        decision["probabilities"] = {
            action.candidate_id: 0.5,
            "reobserve": 0.5,
            "abstain": 0.0,
        }
        with self.assertRaisesRegex(ActionAuthorizationError, "decision scores"):
            self.authorize(request, decision, parsed, action)
        request, decision, parsed, action = inputs()
        request["regions"].append(copy.deepcopy(request["regions"][0]))
        with self.assertRaisesRegex(ActionAuthorizationError, "offered visual regions"):
            self.authorize(request, decision, parsed, action)

    def test_driver_optional_region_metadata_does_not_block_authorization(self) -> None:
        request, decision, parsed, action = inputs()
        parsed["regions"][0].update({"reading_order": 1, "parent_id": None})
        request["regions"][0]["label"] = None
        click = self.authorize(request, decision, parsed, action)
        self.assertEqual((click.x, click.y), (48.0, 36.0))


if __name__ == "__main__":
    unittest.main()