Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

15.4 KB
Raw Blame
"""Closed-candidate decision models for the jev-use example.

Models only select caller-supplied IDs. Candidate construction, capture
freshness, Driver actions, and independent verification stay with the caller.
"""

from __future__ import annotations

import math
import json
import re
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Any, Literal, Mapping, Protocol

from choose_action import REQUEST_SCHEMA, REQUEST_SCHEMA_V2, provider_observation
from jev_adapter import choose_bounded_with_typesafe

DecisionKind = Literal["selected", "reobserve", "abstain", "error"]


@dataclass(frozen=True)
class DecisionRequest:
    goal: str
    capture_id: str
    regions: tuple[Mapping[str, Any], ...]
    history: tuple[Mapping[str, str], ...]
    candidates: tuple[Mapping[str, str], ...]
    # v2 only (RFC #4268): native snapshot and compact accessibility elements,
    # and task steps counted from the runner's own performed actions (#4313).
    schema: str = REQUEST_SCHEMA
    snapshot_id: str | None = None
    elements: tuple[Mapping[str, str], ...] = ()
    progress: tuple[Mapping[str, Any], ...] = ()

    @classmethod
    def from_validated(cls, request: Mapping[str, Any]) -> "DecisionRequest":
        return cls(
            goal=request["goal"],
            capture_id=request["capture_id"],
            regions=tuple(request["regions"]),
            history=tuple(request["history"]),
            candidates=tuple(MappingProxyType(dict(item)) for item in request["candidates"]),
            schema=request.get("schema", REQUEST_SCHEMA),
            snapshot_id=request.get("snapshot_id"),
            elements=tuple(MappingProxyType(dict(item)) for item in request.get("elements", ())),
            progress=tuple(MappingProxyType(dict(item)) for item in request.get("progress", ())),
        )

    @property
    def criteria(self) -> dict[str, str]:
        return {item["id"]: item["description"] for item in self.candidates}

    def provider_observation(self) -> dict[str, Any]:
        """The provider observation; byte-identical to v1 for a v1 request."""
        validated: dict[str, Any] = {
            "schema": self.schema,
            "capture_id": self.capture_id,
            "regions": list(self.regions),
            "history": list(self.history),
            "candidates": [dict(item) for item in self.candidates],
        }
        if self.schema == REQUEST_SCHEMA_V2:
            validated["snapshot_id"] = self.snapshot_id
            validated["elements"] = [dict(item) for item in self.elements]
            validated["progress"] = [dict(item) for item in self.progress]
        return provider_observation(validated)


@dataclass(frozen=True)
class ModelScores:
    probabilities: Mapping[str, float]
    model: str
    selected_id: str | None = None
    confidence: float | None = None


@dataclass(frozen=True)
class DecisionResult:
    kind: DecisionKind
    capture_id: str
    selected_id: str | None
    model: str
    confidence: float | None
    probabilities: Mapping[str, float]
    reason: str | None = None

    def to_wire(self) -> dict[str, Any]:
        return {
            "schema": "cua.decision_choice_v1",
            "kind": self.kind,
            "capture_id": self.capture_id,
            "selected_id": self.selected_id,
            "model": self.model,
            "confidence": self.confidence,
            "probabilities": dict(self.probabilities),
            "reason": self.reason,
        }


class DecisionModel(Protocol):
    name: str

    def score(self, request: DecisionRequest) -> ModelScores: ...


def choose(model: DecisionModel, request: DecisionRequest) -> DecisionResult:
    """Reject malformed scores without converting them into an action."""
    try:
        allowed = request.criteria
        scores = model.score(request)
        if not isinstance(scores.model, str) or not 0 < len(scores.model) <= 128:
            raise ValueError("model identity must be a bounded string")
        if set(scores.probabilities) != set(allowed):
            raise ValueError("probability IDs do not match the candidate IDs")
        probabilities = dict(scores.probabilities)
        if any(
            not isinstance(value, (int, float))
            or isinstance(value, bool)
            or not math.isfinite(value)
            or not 0 <= value <= 1
            for value in probabilities.values()
        ):
            raise ValueError("probabilities must be finite numbers in [0, 1]")
        if abs(sum(probabilities.values()) - 1) > 0.02:
            raise ValueError("probability mass must be approximately one")
        top = max(probabilities.values())
        winners = [candidate_id for candidate_id in allowed if probabilities[candidate_id] == top]
        if len(winners) != 1:
            raise ValueError("a tied top score is not actionable")
        selected = scores.selected_id or winners[0]
        if selected != winners[0]:
            raise ValueError("selection must be an allowed argmax")
        confidence = scores.confidence if scores.confidence is not None else probabilities[selected]
        if (
            not isinstance(confidence, (int, float))
            or isinstance(confidence, bool)
            or not math.isfinite(confidence)
            or not 0 <= confidence <= 1
        ):
            raise ValueError("confidence must be a finite number in [0, 1]")
        kind: DecisionKind = (
            "reobserve"
            if selected == "reobserve"
            else "abstain"
            if selected == "abstain"
            else "selected"
        )
        return DecisionResult(
            kind=kind,
            capture_id=request.capture_id,
            selected_id=selected,
            model=scores.model,
            confidence=confidence,
            probabilities=probabilities,
        )
    except Exception as error:
        reason = "option_limit" if isinstance(error, OptionLimitError) else "model_error"
        model_name = getattr(model, "name", "unknown")
        if not isinstance(model_name, str) or not 0 < len(model_name) <= 128:
            model_name = "unknown"
        return DecisionResult("error", request.capture_id, None, model_name, None, {}, reason)


class OptionLimitError(ValueError):
    pass


class TypeSafeDecisionModel:
    name = "typesafe-jev"

    def __init__(self, client: Any) -> None:
        self.client = client

    def score(self, request: DecisionRequest) -> ModelScores:
        choice = choose_bounded_with_typesafe(
            self.client,
            goal=request.goal,
            observation=request.provider_observation(),
            criteria=request.criteria,
        )
        if choice.selected_id not in request.criteria:
            raise ValueError("Jev selected an unknown candidate")
        if choice.probabilities.get(choice.selected_id) != max(choice.probabilities.values()):
            raise ValueError("Jev selection disagrees with its scores")
        return ModelScores(
            choice.probabilities,
            choice.model or self.name,
            selected_id=choice.selected_id,
            confidence=choice.confidence,
        )


@dataclass(frozen=True)
class _S1Option:
    element_id: str
    role: str
    label: str
    action: str
    entity_id: str | None = None


def visual_regions_as_text(request: DecisionRequest) -> str:
    """Describe OmniParser regions; this is not an accessibility tree."""
    lines = [f"Visual-region-derived observation for capture {json.dumps(request.capture_id)}:"]
    for region in request.regions:
        bounds = region["bounds"]
        label = region.get("text") or region.get("label")
        lines.append(
            f"{json.dumps(region['id'])}: {region['kind']} {label!r} at "
            f"({bounds['x']},{bounds['y']},{bounds['width']},{bounds['height']}) "
            f"confidence={json.dumps(region['confidence'])} "
            f"interactive={json.dumps(region['interactive'])}"
        )
    if request.history:
        lines.append("Prior bounded decisions:")
        lines.extend(json.dumps(item, ensure_ascii=False) for item in request.history)
    return "\n".join(lines)


def native_elements_as_text(request: DecisionRequest) -> str:
    """Render a v2 request's native elements as an accessibility tree for S1.

    Each line is one supplied native control: its role class, label, and state.
    Visual regions from the same capture, if any, follow as a separate section,
    then the prior decisions and, when supplied, the task progress.
    """
    lines = [
        f"Accessibility tree for snapshot {json.dumps(request.snapshot_id)} "
        f"(capture {json.dumps(request.capture_id)}):"
    ]
    for element in request.elements:
        lines.append(
            f"- {element['role_class']} {json.dumps(element['label'], ensure_ascii=False)} "
            f"({element['state']})"
        )
    if request.regions:
        lines.append(visual_regions_as_text(DecisionRequest(
            request.goal, request.capture_id, request.regions, (), request.candidates
        )))
    if request.history:
        lines.append("Prior bounded decisions:")
        lines.extend(json.dumps(item, ensure_ascii=False) for item in request.history)
    if request.progress:
        lines.append(progress_as_text(request.progress))
    return "\n".join(lines)


def progress_as_text(progress: tuple[Mapping[str, Any], ...]) -> str:
    """Render task progress; counts come from the run's actions, not the app."""
    lines = [
        "Task progress (counted from this run's own completed actions, not read from the app):"
    ]
    for item in progress:
        done, required = item["done"], item["required"]
        status = f"{required - done} remaining" if done < required else "complete"
        lines.append(f"- {item['step']}: done {done} of {required} times; {status}")
    return "\n".join(lines)


S1_MODALITIES = ("text", "multimodal")
_HF_SNAPSHOT_PARENT = "snapshots"
_REVISION_PATTERN = re.compile(r"[0-9a-f]{40}")
_UNSAFE_IDENTITY_CHARS = re.compile(r"[^A-Za-z0-9._/-]")


def _hf_local_dir_revision(root: Path, modality: str) -> str | None:
    """Read the commit that `hf download --local-dir` recorded for the adapter.

    `huggingface_hub` writes `<local-dir>/.cache/huggingface/download/<file>.metadata`
    whose first line is the resolved commit hash. Only the selected modality's
    adapter files are consulted, and they must agree.
    """
    metadata_dir = root / ".cache" / "huggingface" / "download"
    revisions: set[str] = set()
    for prefix in (Path(modality), Path()):
        for name in ("adapter_config.json", "adapter_model.safetensors"):
            candidate = metadata_dir / prefix / f"{name}.metadata"
            try:
                first_line = candidate.read_text(encoding="utf-8").splitlines()[0].strip()
            except (OSError, IndexError, UnicodeDecodeError):
                continue
            if _REVISION_PATTERN.fullmatch(first_line):
                revisions.add(first_line)
        if revisions:
            break
    return revisions.pop() if len(revisions) == 1 else None


def s1_model_identity(
    adapter_path: Path,
    modality: str,
    *,
    repo_id: str | None = None,
    revision: str | None = None,
) -> str:
    """Name the S1 checkpoint that produced a decision, for evidence logs.

    The result is `<adapter>[@<revision>]:<modality>`, for example
    `cua-ai/cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79:text`.
    `repo_id` and `revision` override detection. Otherwise the adapter name
    and revision come from a Hugging Face cache snapshot path
    (`models--<org>--<name>/snapshots/<commit>`), from the metadata that
    `hf download --local-dir` leaves in the adapter directory, or, failing
    both, from the adapter directory name with no revision. Detection never
    claims a Hub repository or revision it cannot read locally.
    """
    if modality not in S1_MODALITIES:
        raise ValueError(f"unknown S1 modality: {modality!r}")
    root = Path(adapter_path).expanduser()
    if root.name in S1_MODALITIES:
        root = root.parent
    label = repo_id.strip() if repo_id and repo_id.strip() else None
    detected_revision: str | None = None
    parts = root.parts
    if len(parts) >= 3 and parts[-2] == _HF_SNAPSHOT_PARENT and parts[-3].startswith("models--"):
        if label is None:
            label = parts[-3].removeprefix("models--").replace("--", "/")
        if _REVISION_PATTERN.fullmatch(parts[-1]):
            detected_revision = parts[-1]
    else:
        detected_revision = _hf_local_dir_revision(root, modality)
    if label is None:
        label = root.name or "cua-s1-adapter"
    chosen_revision = revision.strip() if revision and revision.strip() else detected_revision
    label = _UNSAFE_IDENTITY_CHARS.sub("_", label)[:64]
    identity = label
    if chosen_revision:
        identity += "@" + _UNSAFE_IDENTITY_CHARS.sub("_", chosen_revision)[:40]
    return f"{identity}:{modality}"


class S1DecisionModel:
    # Default identity for callers that construct the adapter directly without
    # naming the checkpoint. `choose_decision.py` passes a derived identity.
    name = "cua-s1-4b-local"

    def __init__(
        self,
        scorer: Any,
        *,
        modality: Literal["text", "multimodal"] = "text",
        screenshot_path: Path | None = None,
        screenshot_capture_id: str | None = None,
        name: str | None = None,
    ) -> None:
        if name is not None:
            if not isinstance(name, str) or not 0 < len(name) <= 128:
                raise ValueError("S1 model identity must be a bounded string")
            self.name = name
        self.scorer = scorer
        self.modality = modality
        self.screenshot_path = screenshot_path
        self.screenshot_capture_id = screenshot_capture_id

    def score(self, request: DecisionRequest) -> ModelScores:
        if len(request.candidates) > 26:
            raise OptionLimitError("S1 uses one letter per option, at most 26")
        if self.modality == "multimodal" and (
            self.screenshot_path is None
            or not self.screenshot_path.is_file()
            or self.screenshot_capture_id != request.capture_id
        ):
            raise ValueError("multimodal S1 requires a screenshot bound to the request capture")
        scorer_modality = getattr(self.scorer, "modality", self.modality)
        if scorer_modality != self.modality:
            raise ValueError("S1 scorer modality does not match the decision adapter")
        options = [
            _S1Option(
                item["id"],
                "Decision",
                json.dumps(item["description"], ensure_ascii=False)[1:-1],
                "select",
            )
            for item in request.candidates
        ]
        kwargs: dict[str, Any] = {
            "app": "Cua Driver",
            "task_family": "closed-candidate decision",
            "goal": request.goal,
            "modality": self.modality,
        }
        if self.modality == "text":
            kwargs["ax_tree"] = (
                native_elements_as_text(request)
                if request.schema == REQUEST_SCHEMA_V2 and request.elements
                else visual_regions_as_text(request)
            )
        else:
            kwargs["screenshot"] = self.screenshot_path
        scores = self.scorer.forward(options, **kwargs)
        probabilities: dict[str, float] = {}
        for score in scores:
            if score.element_id in probabilities:
                raise ValueError("S1 returned a duplicate candidate ID")
            probabilities[score.element_id] = score.probability
        return ModelScores(probabilities, self.name)