Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

15.7 KB
Raw Blame
"""Model-agnostic planning and explicitly authorized form execution."""

from __future__ import annotations

import math
import re
import time
from collections.abc import Callable, Sequence
from dataclasses import asdict, is_dataclass
from typing import Any, Protocol

from .driver import BaseDriver, DeliveryMode, DriverError, WindowSnapshot, WindowTarget
from .schema import ACTION_ORDER, Decision, Element, Entity

ACTIONABLE_ROLES = {
    "button",
    "checkbox",
    "combobox",
    "edit",
    "textfield",
    "axbutton",
    "axcheckbox",
    "axcombobox",
    "axtextfield",
}
ALLOWED_ACTIONS = {"fill", "check", "click", "skip"}
SUBMIT_CONTROL_ROLES = {"button", "axbutton"}
SUBMIT_CONTROL_LABELS = {"submit", "submit form"}
APP_SUFFIXES = (
    " - Google Chrome",
    " - Microsoft Edge",
    " - Mozilla Firefox",
    " - Brave",
    " - Safari",
)


class PlannerError(RuntimeError):
    def __init__(self, code: str, message: str, **details: Any) -> None:
        super().__init__(message)
        self.code = code
        self.message = message
        self.details = details

    def to_dict(self) -> dict[str, Any]:
        return {"code": self.code, "message": self.message, "details": self.details}


class PlanningBackend(Protocol):
    """Minimal interface implemented by local, remote, or test scorers."""

    def plan(
        self,
        form_title: str,
        elements: list[Element],
        entities: list[Entity],
    ) -> Sequence[Decision]: ...


class Planner:
    """Validate decisions from an injected backend without owning a model."""

    def __init__(
        self,
        backend: PlanningBackend | Callable[[str, list[Element], list[Entity]], Sequence[Decision]],
    ) -> None:
        self.backend = backend

    def plan(
        self,
        form_title: str,
        elements: list[Element],
        entities: list[Entity],
    ) -> list[Decision]:
        normalized = normalize_title(form_title)
        if hasattr(self.backend, "plan"):
            decisions = self.backend.plan(normalized, elements, entities)  # type: ignore[union-attr]
        else:
            decisions = self.backend(normalized, elements, entities)  # type: ignore[operator]
        supplied = {_element_identity(element): element for element in elements}
        if len(supplied) != len(elements):
            raise PlannerError(
                "duplicate_element_identity",
                "Supplied actionable elements must have unique identities",
                elements=len(elements),
                identities=len(supplied),
            )

        output = []
        decided: set[tuple[str, str | int]] = set()
        for index, decision in enumerate(decisions):
            identity = _element_identity(decision.element)
            element = supplied.get(identity)
            if element is None:
                raise PlannerError(
                    "foreign_element",
                    "Decision refers to an element outside the supplied snapshot",
                    decision_index=index,
                )
            if identity in decided:
                raise PlannerError(
                    "duplicate_decision",
                    "Planning backend returned more than one decision for an element",
                    decision_index=index,
                    element_index=element.index,
                )
            if decision.action not in ALLOWED_ACTIONS:
                raise PlannerError(
                    "invalid_action",
                    "Planning backend returned an unsupported action",
                    decision_index=index,
                    action=decision.action,
                )
            if (
                isinstance(decision.probability, bool)
                or not isinstance(decision.probability, (int, float))
                or not math.isfinite(decision.probability)
                or not 0 <= decision.probability <= 1
            ):
                raise PlannerError(
                    "invalid_probability",
                    "Decision probability must be finite and between zero and one",
                    decision_index=index,
                    probability=decision.probability,
                )
            if decision.action == "fill":
                if decision.entity_index is None or not 0 <= decision.entity_index < len(entities):
                    raise PlannerError(
                        "invalid_entity_index",
                        "Fill decision does not identify a supplied entity",
                        decision_index=index,
                        entity_index=decision.entity_index,
                    )
            decided.add(identity)
            output.append(
                Decision(
                    element=element,
                    action=decision.action,
                    entity_index=decision.entity_index,
                    probability=decision.probability,
                    distribution=decision.distribution,
                )
            )
        missing = [
            element.index for identity, element in supplied.items() if identity not in decided
        ]
        if missing:
            raise PlannerError(
                "missing_decisions",
                "Planning backend omitted decisions for supplied actionable elements",
                missing_element_indices=missing,
                elements=len(elements),
                decisions=len(output),
            )
        return output


def normalize_title(title: str) -> str:
    for suffix in APP_SUFFIXES:
        if title.endswith(suffix):
            return title[: -len(suffix)]
    return title


def filter_elements(elements: list[Element]) -> list[Element]:
    return [element for element in elements if _normalized_role(element.role) in ACTIONABLE_ROLES]


def order_decisions(
    decisions: list[Decision],
    min_confidence: float,
    *,
    allow_submit: bool = False,
) -> list[Decision]:
    """Order bounded actions and retain at most one recognized submit click."""

    if (
        isinstance(min_confidence, bool)
        or not isinstance(min_confidence, (int, float))
        or not math.isfinite(min_confidence)
        or not 0 <= min_confidence <= 1
    ):
        raise PlannerError(
            "invalid_min_confidence",
            "min_confidence must be finite and between zero and one",
            min_confidence=min_confidence,
        )
    keep = [
        decision
        for decision in decisions
        if decision.action != "skip" and decision.probability >= min_confidence
    ]
    submit_clicks = sorted(
        (
            decision
            for decision in keep
            if decision.action == "click" and _is_submit_control(decision.element)
        ),
        key=lambda decision: -decision.probability,
    )
    keep = [decision for decision in keep if decision.action != "click"]
    if allow_submit and submit_clicks:
        keep.append(submit_clicks[0])
    return sorted(
        keep,
        key=lambda decision: (ACTION_ORDER[decision.action], decision.element.index),
    )


def run_form(
    planner: Planner,
    driver: BaseDriver,
    entities: list[Entity],
    *,
    target: WindowTarget,
    form_title: str,
    min_confidence: float = 0.5,
    execute: bool = False,
    submit: bool = False,
    delivery_mode: DeliveryMode = "background",
    log: Callable[[dict[str, Any]], None] | None = None,
) -> dict[str, Any]:
    """Plan by default; mutate only with explicit execute and submit flags.

    Every action uses an opaque token from the latest exact-window snapshot.
    The window is re-observed after every mutation before another token is used.
    """

    if not isinstance(execute, bool) or not isinstance(submit, bool):
        raise PlannerError(
            "invalid_authorization",
            "execute and submit authorization flags must be booleans",
            execute_type=type(execute).__name__,
            submit_type=type(submit).__name__,
        )
    if submit and not execute:
        # This is a valid preview of a submit-authorized run, but make the lack
        # of execution explicit in the report rather than silently mutating.
        authorization_note = "submit requested, but execute is false; no actions ran"
    else:
        authorization_note = None
    if delivery_mode not in ("background", "foreground"):
        raise PlannerError(
            "invalid_delivery_mode",
            "delivery_mode must be 'background' or 'foreground'",
            delivery_mode=delivery_mode,
        )

    started = time.perf_counter()
    snapshot = driver.window_state(target)
    actionable = filter_elements(snapshot.elements)
    observed_at = time.perf_counter()
    decisions = planner.plan(form_title, actionable, entities)
    planned_at = time.perf_counter()
    ordered = order_decisions(decisions, min_confidence, allow_submit=submit)
    if execute and any(decision.action == "fill" for decision in ordered):
        supports_value_mutation = getattr(driver, "supports_value_mutation", None)
        if not callable(supports_value_mutation) or not supports_value_mutation():
            raise PlannerError(
                "unsupported_driver_capability",
                "The connected driver does not advertise token-based value mutation",
                capability="set_value",
            )

    executed: list[dict[str, Any]] = []
    current_snapshot = snapshot
    stopped_after_error = False
    for decision in ordered:
        entry = _describe_decision(decision, entities)
        entry.update(
            {
                "executed": False,
                "target": target.to_driver_target(),
                "delivery_mode": "background" if decision.action == "fill" else delivery_mode,
                "snapshot_id": current_snapshot.snapshot_id,
            }
        )
        if not execute:
            entry["status"] = "planned"
            executed.append(entry)
            if log:
                log(entry)
            continue

        try:
            current = current_snapshot.element_for(decision.element)
            if decision.action == "check":
                if _normalized_role(current.role) not in {"checkbox", "axcheckbox"}:
                    raise DriverError(
                        "check_requires_checkbox",
                        "A check action requires a checkbox element",
                        details={"element_index": current.index, "role": current.role},
                    )
                if current.checked is None:
                    raise DriverError(
                        "checkbox_state_unknown",
                        "The checkbox state is unknown; refusing to toggle it",
                        details={"element_index": current.index},
                    )
                if current.checked:
                    entry["status"] = "already_satisfied"
                    executed.append(entry)
                    if log:
                        log(entry)
                    continue
            token = current_snapshot.token_for(decision.element)
            if decision.action == "fill":
                assert decision.entity_index is not None
                mutation = driver.set_value(
                    target,
                    token,
                    entities[decision.entity_index].value,
                )
            else:
                mutation = driver.click(
                    target,
                    token,
                    delivery_mode=delivery_mode,
                )
            _require_confirmed_action(decision.action, mutation.action, mutation.observation)
            entry["executed"] = True
            entry["status"] = "delivered"
            entry["action_result"] = mutation.action
            entry["post_snapshot_id"] = mutation.observation.snapshot_id
            current_snapshot = mutation.observation
            if decision.action == "check":
                post_element = current_snapshot.element_for(decision.element)
                if post_element.checked is not True:
                    raise DriverError(
                        "checkbox_postcondition_failed",
                        "The checkbox was not confirmed checked after the action",
                        details={
                            "element_index": post_element.index,
                            "checked": post_element.checked,
                        },
                    )
        except DriverError as exc:
            entry["status"] = "error"
            entry["error"] = exc.to_dict()
            stopped_after_error = True
        executed.append(entry)
        if log:
            log(entry)
        if stopped_after_error:
            break

    finished = time.perf_counter()
    return {
        "form": form_title,
        "target": target.to_driver_target(),
        "authorization": {
            "execute": execute,
            "submit": submit,
            "delivery_mode": delivery_mode,
            "note": authorization_note,
        },
        "entities": [_serialize(entity) for entity in entities],
        "elements_scored": len(actionable),
        "snapshot": _snapshot_report(snapshot),
        "all_decisions": [_describe_decision(decision, entities) for decision in decisions],
        "execution_order": executed,
        "stopped_after_error": stopped_after_error,
        "final_snapshot": _snapshot_report(current_snapshot),
        "timings_ms": {
            "snapshot": round((observed_at - started) * 1000, 1),
            "plan": round((planned_at - observed_at) * 1000, 1),
            "execute": round((finished - planned_at) * 1000, 1),
            "total": round((finished - started) * 1000, 1),
        },
        "driver_calls": driver.timings[-100:],
    }


def _describe_decision(decision: Decision, entities: list[Entity]) -> dict[str, Any]:
    entity = (
        _serialize(entities[decision.entity_index])
        if decision.entity_index is not None and 0 <= decision.entity_index < len(entities)
        else None
    )
    return {
        "element_index": decision.element.index,
        "role": decision.element.role,
        "label": decision.element.label,
        "action": decision.action,
        "confidence": round(decision.probability, 4),
        "entity": entity,
    }


def _snapshot_report(snapshot: WindowSnapshot) -> dict[str, Any]:
    return {
        "snapshot_id": snapshot.snapshot_id,
        "element_count": len(snapshot.elements),
        "elements_complete": snapshot.elements_complete,
    }


def _serialize(value: Any) -> Any:
    if is_dataclass(value):
        return asdict(value)
    if hasattr(value, "__dict__"):
        return dict(value.__dict__)
    return value


def _normalized_role(role: str) -> str:
    return role.replace("_", "").replace(" ", "").casefold()


def _normalized_control_label(label: str) -> str:
    return " ".join(re.findall(r"[a-z0-9]+", label.casefold()))


def _is_submit_control(element: Element) -> bool:
    return (
        _normalized_role(element.role) in SUBMIT_CONTROL_ROLES
        and _normalized_control_label(element.label) in SUBMIT_CONTROL_LABELS
    )


def _element_identity(element: Element) -> tuple[str, str | int]:
    if element.element_token:
        return ("token", element.element_token)
    return ("object", id(element))


def _require_confirmed_action(
    action_name: str,
    action: dict[str, Any],
    observation: WindowSnapshot,
) -> None:
    effect = action.get("effect")
    if effect == "confirmed":
        return
    raise DriverError(
        "action_effect_unconfirmed",
        "Cua Driver did not confirm the action effect",
        tool="set_value" if action_name == "fill" else "click",
        details={
            "effect": effect if isinstance(effect, str) else "missing",
            "route": action.get("route"),
            "escalation": action.get("escalation"),
            "post_action_observation": _snapshot_report(observation),
        },
        outcome_unknown=effect not in {"refused", "suspected_noop"},
    )