Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
from __future__ import annotations
import sys
from pathlib import Path
import pytest
import torch
TRAINING_ROOT = Path(__file__).resolve().parents[2] / "training"
sys.path.insert(0, str(TRAINING_ROOT))
try:
from cua_bench_s1.task import CuaTask, OptionSpec # noqa: E402
except ImportError:
BENCH_S1_SRC = Path(__file__).resolve().parents[3] / "cua-bench-s1" / "python" / "src"
sys.path.insert(0, str(BENCH_S1_SRC))
from cua_bench_s1.task import CuaTask, OptionSpec # noqa: E402
from cua_bench_s1.eval.adapter import option_key # noqa: E402
from cua_bench_s1.eval.scoring import score_task # noqa: E402
from train_4b_v2 import ( # noqa: E402
_task_options,
build_parser,
check_val_split,
element_groups,
evaluate_val,
example_loss,
group_log_probs,
lr_schedule,
plan_example,
)
from cua_s1.four_b import assign_letters # noqa: E402
def _task(**overrides) -> CuaTask:
defaults = dict(
id="t1",
family="form_filling",
app="TestApp",
modality_available=["text"],
screenshot=None,
ax_tree="<tree/>",
ax_tree_source="synthetic",
elements=[],
elements_source="synthetic_spec",
entities=[{"id": "e1", "label": "Email", "value": "[email protected]"}],
options=[
# "submit" is uncontested: one option, so it cannot be got wrong.
OptionSpec(element_id="submit", role="Button", label="Submit", action="click"),
OptionSpec(
element_id="email", role="Edit", label="Email", action="fill", entity_id="e1"
),
OptionSpec(element_id="email", role="Edit", label="Email", action="skip"),
],
expected={"submit": "click", "email": "fill"},
)
defaults.update(overrides)
return CuaTask(**defaults)
class _FakeTokenizer:
"""Maps each option letter to a distinct single token id; anything else
(i.e. the chat-templated prompt text) to a fixed stand-in id sequence."""
def __init__(self):
self._letter_ids = {letter: 10 + i for i, letter in enumerate("ABCDEFGH")}
def encode(self, text, add_special_tokens=False):
if text in self._letter_ids:
return [self._letter_ids[text]]
return [1, 2, 3]
def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=True):
return "<chat-text>"
def __call__(self, text, return_tensors="pt"):
return _FakeBatchOutput(torch.tensor([[1, 2, 3, 4]]))
class _FakeBatchOutput:
def __init__(self, input_ids):
self.input_ids = input_ids
class _FakeOutput:
def __init__(self, logits):
self.logits = logits
class _FakeModel:
"""Returns a fixed vocabulary logit row at the final position, so the
per-element readout can be checked against a known distribution."""
device = torch.device("cpu")
dtype = torch.float32
def __init__(self, logits_by_token_id: dict[int, float]):
vocab = torch.full((1, 4, 64), -10.0)
for token_id, value in logits_by_token_id.items():
vocab[0, -1, token_id] = value
self._out = _FakeOutput(vocab)
def __call__(self, **kwargs):
return self._out
def eval(self):
return self
def train(self):
return self
def test_element_groups_drops_uncontested_elements():
task = _task()
assignment = assign_letters(_task_options(task))
groups = element_groups(task, assignment)
# Only "email" is contested; "submit"'s single option is zero-gradient and
# the scorer marks it correct unconditionally.
assert len(groups) == 1
idxs, gold_positions = groups[0]
assert [assignment.options[i].element_id for i in idxs] == ["email", "email"]
assert gold_positions == [0] # the "fill" option
def test_element_groups_puts_uniform_mass_over_several_gold_actions():
task = _task(
options=[
OptionSpec("box", "CheckBox", "Agree", "check"),
OptionSpec("box", "CheckBox", "Agree", "click"),
OptionSpec("box", "CheckBox", "Agree", "skip"),
],
expected={"box": "check"},
)
assignment = assign_letters(_task_options(task))
((idxs, gold_positions),) = element_groups(task, assignment)
assert len(idxs) == 3
assert gold_positions == [0]
def test_element_groups_skips_an_element_whose_gold_is_not_among_its_options():
task = _task(expected={"submit": "click", "email": "scroll"})
assignment = assign_letters(_task_options(task))
assert element_groups(task, assignment) == []
def test_plan_example_returns_none_when_nothing_is_contested():
task = _task(
options=[OptionSpec("submit", "Button", "Submit", "click")],
expected={"submit": "click"},
)
assert plan_example(task, _FakeTokenizer()) is None
def test_plan_example_carries_letter_ids_groups_and_a_tokenized_prompt():
plan = plan_example(_task(), _FakeTokenizer())
assert plan is not None
assert plan["task_id"] == "t1"
assert plan["letter_token_ids"] == [10, 11, 12] # A, B, C
assert len(plan["groups"]) == 1
assert plan["input_ids"].tolist() == [1, 2, 3, 4]
def test_plan_example_multimodal_needs_the_screenshot_on_disk(tmp_path):
task = _task(modality_available=["multimodal"], screenshot="shot.png", ax_tree=None)
assert plan_example(task, _FakeTokenizer(), modality="multimodal", data_root=tmp_path) is None
(tmp_path / "shot.png").write_bytes(b"")
plan = plan_example(task, _FakeTokenizer(), modality="multimodal", data_root=tmp_path)
assert plan is not None
assert plan["image_path"] == tmp_path / "shot.png"
assert "input_ids" not in plan # built lazily, per step
def test_per_element_readout_matches_eval_scoring_on_the_same_logits():
"""The loss and the readout are scoped per element because that is what
`eval.scoring` thresholds. Feeding one fixed logit row through both must
give the same verdict."""
from torch.nn import functional as F
task = _task()
plan = plan_example(task, _FakeTokenizer())
# A: submit/click, B: email/fill, C: email/skip. "skip" wins its element,
# so the task is WRONG even though the globally-largest logit is gold.
model = _FakeModel({10: 5.0, 11: 0.0, 12: 1.0})
log_probs = group_log_probs(
model, plan, {"input_ids": plan["input_ids"].unsqueeze(0)}, torch, F
)
readout_correct = all(
int(lp.argmax().item()) in set(gold)
for lp, (_, gold) in zip(log_probs, plan["groups"], strict=False)
)
# The same distribution, expressed the way the scorer consumes it: each
# element's own options renormalized among themselves.
probs = {option_key("submit", "click"): 1.0}
email = torch.softmax(torch.tensor([0.0, 1.0]), dim=-1).tolist()
probs[option_key("email", "fill")] = email[0]
probs[option_key("email", "skip")] = email[1]
assert readout_correct is score_task(task, probs).correct is False
def test_example_loss_is_zero_when_every_contested_element_is_certain():
from torch.nn import functional as F
plan = plan_example(_task(), _FakeTokenizer())
# Overwhelming mass on B (email/fill), the gold option for its element.
model = _FakeModel({10: 0.0, 11: 60.0, 12: 0.0})
loss = example_loss(model, plan, {"input_ids": plan["input_ids"].unsqueeze(0)}, torch, F)
assert float(loss) == pytest.approx(0.0, abs=1e-6)
def test_evaluate_val_counts_a_task_only_when_every_element_is_right():
from torch.nn import functional as F
plan = plan_example(_task(), _FakeTokenizer())
assert evaluate_val(_FakeModel({10: 0.0, 11: 5.0, 12: 0.0}), [plan], torch, F) == 1.0
assert evaluate_val(_FakeModel({10: 0.0, 11: 0.0, 12: 5.0}), [plan], torch, F) == 0.0
assert evaluate_val(_FakeModel({}), [], torch, F) == 0.0
def test_lr_schedule_warms_up_then_decays_to_min_lr():
kwargs = dict(base_lr=1e-4, min_lr=1e-6, warmup=10, total_steps=100)
assert lr_schedule(0, **kwargs) == pytest.approx(1e-5)
assert lr_schedule(9, **kwargs) == pytest.approx(1e-4) # end of warmup
assert lr_schedule(99, **kwargs) == pytest.approx(
1e-6, rel=0.05
) # approaching the cosine floor
assert lr_schedule(100, **kwargs) == pytest.approx(1e-6) # the floor itself
# Monotonic decay after warmup.
post = [lr_schedule(s, **kwargs) for s in range(10, 100)]
assert all(a >= b for a, b in zip(post, post[1:], strict=False))
def test_check_val_split_refuses_a_test_split():
check_val_split("runs/split/validation.jsonl") # fine
with pytest.raises(SystemExit):
check_val_split("runs/split/test.jsonl")
def test_parser_defaults_and_modality_choices():
args = build_parser().parse_args(["--train", "t.jsonl", "--val", "v.jsonl", "--out", "o"])
assert args.modality == "text"
assert args.batch_size == 8 # gradient accumulation, not per-example steps
assert args.warmup_frac == 0.05
assert args.augment is True
mm = build_parser().parse_args(
[
"--train",
"t.jsonl",
"--val",
"v.jsonl",
"--out",
"o",
"--modality",
"multimodal",
"--no-augment",
]
)
assert (mm.modality, mm.augment) == ("multimodal", False)
with pytest.raises(SystemExit):
build_parser().parse_args(
["--train", "t.jsonl", "--val", "v.jsonl", "--out", "o", "--modality", "audio"]
)