Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

20.6 KB
Raw Blame
"""Tests for the `cua_bench_basic` family: the conversion logic, and -- on
request -- two real live task envs end to end.

The live tests run the envs the way cua-bench 0.3 runs every task: in a real
sandbox (a local Linux container by default), so they start containers and
are opt-in: set `CUA_BENCH_S1_LIVE=1` with cua-bench installed. The pages use
`session.launch_window`, so the image needs bench-ui. (cua-bench's Playwright
`simulated` provider, which these used before, was removed in 0.3.)
"""
from __future__ import annotations

import asyncio
import random
from pathlib import Path

import pytest

from cua_bench_s1.datagen.cua_bench_basic import (
    ENV_NAMES,
    FAMILY,
    ActionRecord,
    DomElement,
    RecordedEpisode,
    RecordedStep,
    assign_splits,
    build_candidates,
    convert_episode,
    dom_elements_from_query,
    render_ax_tree,
    role_for,
    target_element,
)
from cua_bench_s1.task import FAMILIES, dataset_hash


def _dataset_dir() -> Path | None:
    here = Path(__file__).resolve()
    for parent in here.parents:
        candidate = parent / "libs" / "cua-bench" / "datasets" / "cua-bench-basic"
        if candidate.is_dir():
            return candidate
    return None


DATASET_DIR = _dataset_dir()

try:  # pragma: no cover - environment-dependent
    import os

    import cua_bench  # noqa: F401

    _LIVE = DATASET_DIR is not None and os.environ.get("CUA_BENCH_S1_LIVE") == "1"
except Exception:  # pragma: no cover
    _LIVE = False

live_only = pytest.mark.skipif(
    not _LIVE,
    reason="opt-in: CUA_BENCH_S1_LIVE=1 with cua_bench and datasets/cua-bench-basic (starts containers)",
)


# --------------------------------------------------------------------------
# Fixtures: a small, realistic hand-built recorded episode (same shape the
# live recorder produces) so the conversion logic is testable without a
# browser.
# --------------------------------------------------------------------------


def _elements() -> list[DomElement]:
    return [
        DomElement(id="dom_0", role="Button", label="Submit", frame=[24, 124, 120, 166], tag="BUTTON"),
        DomElement(id="dom_1", role="Button", label="Cancel", frame=[136, 124, 232, 166], tag="BUTTON"),
        DomElement(id="dom_2", role="Edit", label="Username", frame=[24, 60, 232, 100], tag="INPUT"),
    ]


def _episode(task_index: int = 0, text: str = "Hello World") -> RecordedEpisode:
    els = _elements()
    return RecordedEpisode(
        env_name="typing-input",
        task_index=task_index,
        instruction=f'Type "{text}" into the Username field.',
        metadata={"text": text, "field_label": "Username"},
        steps=[
            RecordedStep(0, els, ActionRecord("ClickAction", {"x": 128, "y": 80}), None),
            RecordedStep(1, els, ActionRecord("TypeAction", {"text": text}), None),
        ],
        oracle_reward=1.0,
        provider="native",
    )


# --------------------------------------------------------------------------
# Pure conversion logic
# --------------------------------------------------------------------------


class TestPrimitives:
    def test_family_is_registered(self):
        assert FAMILY == "cua_bench_basic"
        assert FAMILY in FAMILIES

    def test_thirteen_envs_named(self):
        assert len(ENV_NAMES) == 13

    def test_role_mapping(self):
        assert role_for("BUTTON", None, None) == "Button"
        assert role_for("INPUT", "checkbox", None) == "CheckBox"
        assert role_for("INPUT", "range", None) == "Slider"
        assert role_for("SELECT", None, None) == "Select"
        assert role_for("DIV", None, "button") == "Button"

    def test_dom_query_translated_to_screen_space(self):
        raw = [{"tag": "BUTTON", "type": None, "dom_id": "submit", "role": None,
                "label": "Submit", "x": 24, "y": 76, "width": 96, "height": 42}]
        els = dom_elements_from_query(raw, offset_x=0, offset_y=48)
        assert els[0].frame == [24, 124, 120, 166]
        assert els[0].center == (72, 145)
        assert els[0].contains(72, 145)
        assert not els[0].contains(72, 90)

    def test_target_element_picks_smallest_containing_box(self):
        container = DomElement("dom_c", "Generic", "container", [0, 0, 300, 300], "DIV")
        button = DomElement("dom_b", "Button", "Submit", [24, 124, 120, 166], "BUTTON")
        hit = target_element(ActionRecord("ClickAction", {"x": 72, "y": 145}), [container, button])
        assert hit is button

    def test_action_describe_is_grounded_in_a_real_element(self):
        desc = ActionRecord("ClickAction", {"x": 72, "y": 145}).describe(_elements())
        assert 'Button "Submit"' in desc and "(72, 145)" in desc
        assert ActionRecord("TypeAction", {"text": "abc"}).describe([]) == "type 'abc'"


class TestAxTree:
    def test_renders_instruction_history_and_real_boxes(self):
        text = render_ax_tree(
            instruction="Click the Submit button.",
            step_index=2,
            prior_actions=["click at (200, 165) on Edit \"Username\"", "type 'abc'"],
            elements=_elements(),
        )
        assert text.splitlines()[0] == "Task: Click the Submit button."
        assert "Step: 2" in text
        assert "  1. click at (200, 165) on Edit \"Username\"" in text
        assert "  2. type 'abc'" in text
        assert '  - Button "Cancel" [136,124,232,166]' in text

    def test_leaks_no_gold_marker(self):
        text = render_ax_tree(instruction="i", step_index=0, prior_actions=[],
                              elements=_elements())
        assert "gold" not in text.lower() and "expected" not in text.lower()


class TestCandidates:
    def test_gold_present_exactly_once_and_capped(self):
        gold = ActionRecord("ClickAction", {"x": 72, "y": 145})
        cands = build_candidates(
            gold=gold, elements=_elements(), peer_actions=[],
            max_options=4, rng=random.Random(0),
        )
        assert len(cands) == 4
        assert sum(1 for c in cands if c.key() == gold.key()) == 1

    def test_peer_actions_are_preferred_distractors(self):
        gold = ActionRecord("TypeAction", {"text": "Hello World"})
        peer = ActionRecord("TypeAction", {"text": "Testing 123"})
        cands = build_candidates(
            gold=gold, elements=_elements(), peer_actions=[peer],
            max_options=3, rng=random.Random(1),
        )
        assert peer.key() in {c.key() for c in cands}

    def test_same_target_type_confusions_are_offered(self):
        gold = ActionRecord("ClickAction", {"x": 72, "y": 145})
        cands = build_candidates(
            gold=gold, elements=_elements(), peer_actions=[],
            max_options=10, rng=random.Random(2),
        )
        kinds = {c.kind for c in cands}
        assert "DoubleClickAction" in kinds and "RightClickAction" in kinds

    def test_every_distractor_is_grounded(self):
        """No fabricated noise: every non-gold candidate either reuses the
        gold's own real coordinates, or lands on a real element of the real
        page, or is a real peer-episode oracle action."""
        gold = ActionRecord("ClickAction", {"x": 72, "y": 145})
        els = _elements()
        cands = build_candidates(
            gold=gold, elements=els, peer_actions=[], max_options=10, rng=random.Random(3)
        )
        for c in cands:
            pt = c.point()
            assert pt is not None
            assert pt == (72.0, 145.0) or target_element(c, els) is not None

    def test_determinism(self):
        args = dict(gold=ActionRecord("ClickAction", {"x": 72, "y": 145}),
                    elements=_elements(), peer_actions=[], max_options=5)
        a = [c.key() for c in build_candidates(**args, rng=random.Random(7))]
        b = [c.key() for c in build_candidates(**args, rng=random.Random(7))]
        assert a == b


class TestConvertEpisode:
    def test_one_task_per_real_step(self):
        tasks = convert_episode(_episode(), split_name="train", modality_available=("text",))
        assert len(tasks) == 2
        assert [t.id for t in tasks] == [
            "cbb_typing-input_0_0",
            "cbb_typing-input_0_1",
        ]

    def test_exactly_one_gold_click_per_task(self):
        for t in convert_episode(_episode(), split_name="train", modality_available=("text",)):
            assert sorted(t.expected.values()).count("click") == 1
            assert set(t.expected) == {e["id"] for e in t.elements}

    def test_every_element_gets_a_real_click_skip_pair(self):
        t = convert_episode(_episode(), split_name="train", modality_available=("text",))[0]
        for el in t.elements:
            acts = {o.action for o in t.options if o.element_id == el["id"]}
            assert acts == {"click", "skip"}

    def test_ax_tree_carries_instruction_history_and_real_boxes(self):
        tasks = convert_episode(_episode(), split_name="train", modality_available=("text",))
        first, second = tasks[0].ax_tree, tasks[1].ax_tree
        assert "Type \"Hello World\"" in first
        assert "(none -- this is the first step)" in first
        # Step 1's state must include step 0's real action as history.
        assert "click at (128, 80)" in second
        assert 'Button "Submit" [24,124,120,166]' in first

    def test_history_dependence_makes_the_two_steps_distinguishable(self):
        tasks = convert_episode(_episode(), split_name="train", modality_available=("text",))
        assert tasks[0].ax_tree != tasks[1].ax_tree

    def test_peer_episode_supplies_a_real_hard_distractor(self):
        ep, peer = _episode(0, "Hello World"), _episode(1, "Testing 123")
        tasks = convert_episode(ep, split_name="train", peer_episodes=[ep, peer],
                                modality_available=("text",))
        type_labels = {o.label for o in tasks[1].options}
        assert "type 'Hello World'" in type_labels
        assert "type 'Testing 123'" in type_labels

    def test_provenance_discloses_gold_source_and_oracle_reward(self):
        p = convert_episode(_episode(), split_name="val", modality_available=("text",))[0].provenance
        assert p["gold_label_method"].startswith("cua_bench_reference_solution")
        assert p["oracle_reward"] == 1.0 and p["oracle_verified"] is True
        assert p["dataset_split"] == "val"
        assert p["env_name"] == "typing-input"
        assert "live_dom_query" in p["ax_tree_method"]

    def test_require_oracle_success_drops_unverified_episodes(self):
        ep = _episode()
        ep.oracle_reward = 0.0
        assert convert_episode(ep, split_name="train", modality_available=("text",),
                               require_oracle_success=True) == []
        # The default keeps the episode, but discloses the unverified oracle.
        kept = convert_episode(ep, split_name="train", modality_available=("text",))
        assert kept and kept[0].provenance["oracle_verified"] is False

    def test_multimodal_is_dropped_when_no_screenshot_was_captured(self):
        t = convert_episode(_episode(), split_name="train")[0]
        assert t.modality_available == ["text"]
        assert t.screenshot is None

    def test_tasks_are_hashable_as_a_dataset(self):
        tasks = convert_episode(_episode(), split_name="train", modality_available=("text",))
        assert len(dataset_hash(tasks)) == 64


class TestViewportVisibility:
    """The bundled pages really do overflow their windows under the
    `simulated` provider, so off-screen elements must never be offered and
    off-screen gold steps must never be emitted."""

    def _clipped(self) -> RecordedEpisode:
        ep = _episode()
        # A real window content rect that cuts the page off at y=198.
        ep.viewport = [0, 48, 256, 198]
        ep.steps[0].elements = ep.steps[0].elements + [
            DomElement("dom_9", "Button", "Below the fold", [24, 300, 120, 340], "BUTTON")
        ]
        return ep

    def test_offscreen_elements_are_not_offered_as_options(self):
        ep = self._clipped()
        t = convert_episode(ep, split_name="train", modality_available=("text",))[0]
        assert "Below the fold" not in t.ax_tree
        assert all("Below the fold" not in o.label for o in t.options)

    def test_step_with_offscreen_gold_is_dropped_but_stays_in_history(self):
        ep = self._clipped()
        ep.steps[0].gold_action = ActionRecord("ClickAction", {"x": 72, "y": 320})
        tasks = convert_episode(ep, split_name="train", modality_available=("text",))
        assert [t.id for t in tasks] == ["cbb_typing-input_0_1"]
        assert "click at (72, 320)" in tasks[0].ax_tree  # still real history

    def test_offscreen_gold_can_be_kept_explicitly(self):
        ep = self._clipped()
        ep.steps[0].gold_action = ActionRecord("ClickAction", {"x": 72, "y": 320})
        tasks = convert_episode(ep, split_name="train", modality_available=("text",),
                                drop_offscreen_gold=False)
        assert len(tasks) == 2

    def test_no_viewport_means_no_filtering(self):
        ep = self._clipped()
        ep.viewport = None
        t = convert_episode(ep, split_name="train", modality_available=("text",))[0]
        assert "Below the fold" in t.ax_tree

    def test_provenance_reports_exclusions_and_layout_fit(self):
        ep = self._clipped()
        ep.window_layout_fitted = True
        p = convert_episode(ep, split_name="train", modality_available=("text",))[0].provenance
        assert p["n_offscreen_elements_excluded"] == 1
        assert p["viewport"] == [0, 48, 256, 198]
        assert p["window_layout_fitted"] is True

    def test_peer_action_landing_offscreen_is_not_offered(self):
        ep, peer = self._clipped(), _episode(1, "Testing 123")
        peer.steps[0].gold_action = ActionRecord("ClickAction", {"x": 72, "y": 320})
        t = convert_episode(ep, split_name="train", peer_episodes=[ep, peer],
                            modality_available=("text",))[0]
        assert all("(72, 320)" not in o.label for o in t.options)


class TestSplits:
    def test_majority_in_train_and_val(self):
        for n in (5, 7):
            s = assign_splits(n)
            assert len(s["train"]) + len(s["val"]) > len(s["test"])

    def test_test_holds_out_the_last_fresh_parameterizations(self):
        s = assign_splits(7)
        assert s == {"train": [0, 1, 2, 3], "val": [4], "test": [5, 6]}

    def test_no_overlap_and_full_coverage(self):
        for n in range(1, 10):
            s = assign_splits(n)
            allidx = s["train"] + s["val"] + s["test"]
            assert sorted(allidx) == list(range(n))
            assert len(set(allidx)) == n

    def test_tiny_parameter_counts_degrade_honestly(self):
        assert assign_splits(1) == {"train": [0], "val": [], "test": []}
        assert assign_splits(2) == {"train": [0], "val": [], "test": [1]}
        assert assign_splits(3) == {"train": [0], "val": [1], "test": [2]}


# --------------------------------------------------------------------------
# Live end-to-end against two real cua-bench envs (skipped without cua_bench)
# --------------------------------------------------------------------------


@live_only
@pytest.mark.parametrize("env_name", ["click-button", "typing-input"])
def test_live_record_episode_produces_real_steps(env_name, tmp_path):
    from cua_bench_s1.datagen.cua_bench_basic import record_episode

    ep = asyncio.run(
        record_episode(env_dir=DATASET_DIR / env_name, task_index=0, out_dir=tmp_path)
    )
    assert ep.env_name == env_name
    assert ep.steps, "the reference solution took no actions"
    assert ep.oracle_reward == 1.0 and ep.oracle_verified
    for step in ep.steps:
        assert step.elements, "no real interactive elements found in the live DOM"
        assert Path(step.screenshot).exists()
    # Real gold coordinates must land on a real element of the real page.
    first = ep.steps[0]
    assert target_element(first.gold_action, first.elements) is not None


@live_only
def test_live_window_layout_fit_gives_the_page_its_requested_size(tmp_path):
    """Without the fit, the `simulated` provider clips these pages to ~150px
    and the real gold targets fall off-screen; with it, the window is the size
    the env's own `launch_window(...)` asked for."""
    from cua_bench_s1.datagen.cua_bench_basic import record_episode

    unfitted = asyncio.run(
        record_episode(env_dir=DATASET_DIR / "click-button", task_index=0,
                       out_dir=tmp_path, fit_layout=False, capture_screenshots=False)
    )
    fitted = asyncio.run(
        record_episode(env_dir=DATASET_DIR / "click-button", task_index=0,
                       out_dir=tmp_path, capture_screenshots=False)
    )
    assert unfitted.window_layout_fitted is False
    assert fitted.window_layout_fitted is True
    # click-button launches a 256x256 window.
    assert fitted.viewport[3] - fitted.viewport[1] == 256
    assert unfitted.viewport[3] - unfitted.viewport[1] < 200


@live_only
def test_live_conversion_end_to_end(tmp_path):
    from cua_bench_s1.datagen.cua_bench_basic import record_episode

    ep = asyncio.run(
        record_episode(env_dir=DATASET_DIR / "click-button", task_index=0, out_dir=tmp_path)
    )
    tasks = convert_episode(ep, split_name="train")
    assert tasks
    t = tasks[0]
    assert t.family == FAMILY and t.app == "click-button"
    assert "multimodal" in t.modality_available and "text" in t.modality_available
    assert sorted(t.expected.values()).count("click") == 1
    assert len(t.elements) >= 3, "too few real distractors extracted from the live page"


@live_only
def test_live_agentic_env_rollout_and_truncation():
    from cua_bench_s1.agentic import MAX_STEPS, CuaBenchBasicEnv

    async def run():
        async with CuaBenchBasicEnv(
            "click-button", dataset_dir=DATASET_DIR, task_index=0, max_steps=3
        ) as env:
            obs = await env.reset()
            assert obs.screenshot and obs.instruction
            assert obs.reward is None  # no fabricated per-step shaping reward
            from cua_bench.types import ClickAction

            # Deliberately useless clicks: the episode must truncate cleanly
            # at max_steps with a real failure signal, not raise.
            for _ in range(3):
                obs = await env.step(ClickAction(x=2, y=2))
            assert obs.done and obs.truncated
            assert obs.success is False
            assert env.step_count == 3
        assert MAX_STEPS == 20

    asyncio.run(run())


@live_only
def test_live_agentic_env_oracle_succeeds():
    from cua_bench_s1.agentic import CuaBenchBasicEnv

    async def run():
        async with CuaBenchBasicEnv("click-button", dataset_dir=DATASET_DIR) as env:
            await env.reset()
            assert await env.solve() == 1.0

    asyncio.run(run())


@live_only
def test_live_rollout_helper_scores_a_real_failing_policy():
    from cua_bench_s1.agentic import rollout

    async def policy(obs):
        # Deliberately off-target, and passed as a string to exercise
        # cua-bench's own action-string parsing.
        return "click(2, 2)"

    result = asyncio.run(
        rollout("click-button", policy=policy, dataset_dir=DATASET_DIR,
                task_index=0, max_steps=2)
    )
    assert result.env_name == "click-button"
    assert result.n_steps == 2 and result.truncated
    assert result.success is False and result.error is None


@live_only
def test_live_element_grounded_policy_really_solves_the_task():
    """An end-to-end RL-shaped check: a policy that reads the live element
    list, picks the element the real instruction names, and clicks its real
    center earns the environment's own real reward through `step()` -- not
    via `solve()`."""
    from cua_bench_s1.agentic import CuaBenchBasicEnv

    async def run():
        async with CuaBenchBasicEnv(
            "color-picker", dataset_dir=DATASET_DIR, task_index=0, max_steps=5
        ) as env:
            obs = await env.reset()
            els = await env.elements()
            assert els and all("frame" in e and "label" in e for e in els)

            target = obs.metadata["name"]  # e.g. "red"
            match = next(e for e in els if target.lower() in e["label"].lower())
            x0, y0, x1, y1 = match["frame"]

            from cua_bench.types import ClickAction, DoneAction

            await env.step(ClickAction(x=(x0 + x1) // 2, y=(y0 + y1) // 2))
            final = await env.step(DoneAction())
            assert final.done and not final.truncated
            assert final.success is True and final.reward == 1.0

    asyncio.run(run())


@live_only
def test_live_rollout_reports_a_policy_error_without_crashing():
    from cua_bench_s1.agentic import rollout

    async def policy(obs):
        raise ValueError("policy blew up")

    result = asyncio.run(
        rollout("click-button", policy=policy, dataset_dir=DATASET_DIR, task_index=0)
    )
    assert result.success is False
    assert result.error is not None and "policy blew up" in result.error