Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

9.8 KB
Raw Blame
from __future__ import annotations

import json
import sys
import unittest
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from core import (
    SUBMIT_IDS,
    build_candidates,
    choose_mock,
    form_state,
    has_executable_candidate,
    history_entry,
    parse_visual_regions,
    validate_choice,
    visual_submit_region,
)
from jev_adapter import choose_with_typesafe, decision_state

FIXTURES = Path(__file__).resolve().parents[2] / "fixtures"
VISUAL = json.loads((FIXTURES / "jev-visual-replay-v1.json").read_text(encoding="utf-8"))
PAGE = json.loads((FIXTURES / "jev-page-structure-replay-v1.json").read_text(encoding="utf-8"))
TOKEN = VISUAL["token"]
BEFORE = VISUAL["snapshots"]["before_typing"]
AFTER = VISUAL["snapshots"]["after_typing"]
PAYLOAD = VISUAL["visual_regions"]


def observation():
    source = PAYLOAD["capture"]["source"]
    return parse_visual_regions(
        PAYLOAD,
        expected_capture_id=PAYLOAD["capture"]["capture_id"],
        expected_pid=source["pid"],
        expected_window_id=source["window_id"],
    )


class ReplayClient:
    """Return recorded live choices in order and keep every request."""

    def __init__(self, recorded: list[dict]) -> None:
        self.recorded = list(recorded)
        self.requests: list[dict] = []

    def system_one(self, **request):
        self.requests.append(request)
        step = self.recorded.pop(0)
        answer = SimpleNamespace(
            choice=step["selected_id"],
            confidence=max(step["probabilities"].values()),
            probabilities=step["probabilities"],
        )
        return SimpleNamespace(choices={"driver_action": answer})


class VisualFormStateTest(unittest.TestCase):
    def test_fixture_exposes_submit_only_as_text_and_as_one_visual_region(self) -> None:
        for snapshot in (BEFORE, AFTER):
            self.assertFalse(any(ref["role"] == "button" for ref in snapshot["refs"]))
            self.assertIn('statictext "Submit"', snapshot["outline"])
        region = visual_submit_region(observation())
        self.assertIsNotNone(region)
        self.assertEqual(region.text, "Submit")

    def test_submit_state_names_the_visual_path_instead_of_a_missing_button(self) -> None:
        visual = observation()
        self.assertEqual(form_state(BEFORE, TOKEN)["submit_button"], "not_in_page_structure")
        self.assertEqual(
            form_state(BEFORE, TOKEN, visual_path=True),
            {"verification_field": "empty", "submit_button": "visual_check_pending"},
        )
        self.assertEqual(
            form_state(AFTER, TOKEN, visual, visual_path=True),
            {"verification_field": "contains_required_token", "submit_button": "visual_only"},
        )
        without_submit = replace(
            visual, regions=tuple(r for r in visual.regions if r.text != "Submit")
        )
        self.assertEqual(
            form_state(AFTER, TOKEN, without_submit, visual_path=True)["submit_button"],
            "not_found_visually",
        )
        submit = visual_submit_region(visual)
        duplicated = replace(visual, regions=visual.regions + (replace(submit, id="dup"),))
        self.assertEqual(
            form_state(AFTER, TOKEN, duplicated, visual_path=True)["submit_button"],
            "not_found_visually",
        )
        # A page-structure Submit ref always wins, with or without a visual path.
        page_after = PAGE["snapshots"]["after_typing"]
        self.assertEqual(
            form_state(page_after, PAGE["token"], visual, visual_path=True)["submit_button"],
            "available",
        )

    def test_pre_fix_step_one_state_contradicted_the_outline(self) -> None:
        # Pins the regression: the outline shows Submit text, the form said the
        # button was missing, and reobserve invited "incomplete" observations.
        old = VISUAL["pre_fix_step1"]
        self.assertEqual(
            old["state"]["observation"]["form"]["submit_button"], "not_in_page_structure"
        )
        self.assertIn('statictext "Submit"', old["state"]["observation"]["outline"])
        self.assertIn("incomplete", old["criteria"]["reobserve"])

        new_state = decision_state(BEFORE, None, [], TOKEN, visual_path=True)
        self.assertEqual(new_state["observation"]["form"]["submit_button"], "visual_check_pending")
        expected = json.loads(json.dumps(old["state"]))
        expected["observation"]["form"]["submit_button"] = "visual_check_pending"
        self.assertEqual(new_state, expected)

        criteria = {c.id: c.description for c in build_candidates(BEFORE, TOKEN)}
        self.assertEqual(list(criteria), list(old["criteria"]))
        self.assertNotIn("incomplete", criteria["reobserve"])
        self.assertIn("visual-only", criteria["reobserve"])
        self.assertIn("still pending", criteria["reobserve"])


class DeterministicPathsTest(unittest.TestCase):
    def test_page_structure_fixture_types_then_submits_with_browser_click(self) -> None:
        token = PAGE["token"]
        first = build_candidates(PAGE["snapshots"]["before_typing"], token)
        self.assertEqual(choose_mock(first)[0], "type-verification-value")
        second = build_candidates(PAGE["snapshots"]["after_typing"], token)
        choice = validate_choice(choose_mock(second)[0], second)
        self.assertEqual((choice.id, choice.tool), ("submit-form", "browser_click"))

    def test_visual_fixture_types_then_submits_with_a_capture_bound_click(self) -> None:
        first = build_candidates(BEFORE, TOKEN, capture_bound_click=True)
        choice = validate_choice(choose_mock(first)[0], first)
        self.assertEqual((choice.id, choice.tool), ("type-verification-value", "browser_type"))

        # After typing the page structure offers no action, so the runner parses.
        self.assertFalse(
            has_executable_candidate(build_candidates(AFTER, TOKEN, capture_bound_click=True))
        )
        visual = observation()
        second = build_candidates(AFTER, TOKEN, visual, capture_bound_click=True)
        choice = validate_choice(
            choose_mock(second)[0], second, current_capture_id=visual.capture_id
        )
        submit = visual_submit_region(visual)
        self.assertEqual((choice.id, choice.tool), ("submit-form", "click"))
        self.assertEqual(choice.arguments["capture_id"], visual.capture_id)
        self.assertEqual(choice.arguments["delivery_mode"], "background")
        self.assertEqual(
            (choice.arguments["x"], choice.arguments["y"]), visual.screenshot_center(submit)
        )

        third = build_candidates(
            AFTER, TOKEN, visual, capture_bound_click=True, visual_delivery="foreground"
        )
        choice = validate_choice(choose_mock(third)[0], third, current_capture_id=visual.capture_id)
        self.assertEqual(
            (choice.id, choice.arguments["delivery_mode"]), ("submit-form-foreground", "foreground")
        )


class VisualReplayTest(unittest.TestCase):
    def test_recorded_visual_run_choices_replay_through_the_new_state(self) -> None:
        visual = observation()
        for name, run in VISUAL["recorded_live_runs"].items():
            with self.subTest(run=name):
                client = ReplayClient(run["steps"])
                history: list[dict] = []
                typed = False
                delivery = "background"
                outcome = "budget_exhausted"
                for recorded in run["steps"]:
                    step = recorded["step"]
                    snapshot = AFTER if typed else BEFORE
                    current = visual if typed else None
                    candidates = build_candidates(
                        snapshot,
                        TOKEN,
                        current,
                        capture_bound_click=True,
                        visual_delivery=delivery,
                    )
                    self.assertEqual(set(recorded["probabilities"]), {c.id for c in candidates})
                    choice, _, _ = choose_with_typesafe(
                        client, candidates, snapshot, current, history, TOKEN, True
                    )
                    candidate = validate_choice(
                        choice,
                        candidates,
                        current_capture_id=current.capture_id if current else None,
                    )
                    request = client.requests[-1]
                    form = request["state"]["observation"]["form"]
                    self.assertNotIn(TOKEN, json.dumps(request["state"], sort_keys=True))
                    self.assertEqual(
                        form,
                        {
                            "verification_field": "contains_required_token" if typed else "empty",
                            "submit_button": "visual_only" if typed else "visual_check_pending",
                        },
                    )
                    criteria = request["questions"]["driver_action"].criteria
                    self.assertNotIn("incomplete", criteria["reobserve"])
                    for item in request["state"]["history"]:
                        self.assertEqual(set(item), {"step", "selected_id", "outcome"})
                    refusal = recorded.get("action_error")
                    history.append(history_entry(step, candidate.id, refusal=refusal))
                    if candidate.id == "type-verification-value":
                        typed = True
                    elif candidate.id in SUBMIT_IDS:
                        self.assertEqual(candidate.tool, "click")
                        if refusal:
                            delivery = "foreground"
                        else:
                            outcome = "verified"
                            break
                self.assertEqual(outcome, run["outcome"])


if __name__ == "__main__":
    unittest.main()