Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

14.3 KB
Raw Blame
from __future__ import annotations

import json
import math
import re
import sys
from typing import Any, Mapping

from jev_adapter import ProviderChoice, choose_bounded_with_typesafe
from native_roles import ROLE_CLASSES

REQUEST_SCHEMA = "cua.jev_choice_request_v1"
# Additive over v1 (RFC #4268): per-candidate ``source``, an optional root
# ``snapshot_id``, an optional compact ``elements`` list of native controls, and
# an optional ``progress`` list of task steps counted from the runner's own
# performed actions (#4313).
REQUEST_SCHEMA_V2 = "cua.jev_choice_request_v2"
REQUEST_SCHEMAS = (REQUEST_SCHEMA, REQUEST_SCHEMA_V2)
RESPONSE_SCHEMA = "cua.jev_choice_v1"
MAX_INPUT_BYTES = 65_536
MAX_CANDIDATES = 32
MAX_REGIONS = 100
MAX_HISTORY = 16
MAX_ELEMENTS = 64
MAX_PROGRESS = 16
MAX_PROGRESS_COUNT = 64
ID_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,63}\Z")
RESERVED_IDS = frozenset({"reobserve", "abstain"})
CANDIDATE_SOURCES = frozenset({"page", "ax", "visual"})
ELEMENT_STATES = frozenset(
    {"enabled", "checked", "unchecked", "selected", "not_selected", "empty", "has_text"}
)
V1_ROOT_KEYS = frozenset({"schema", "goal", "capture_id", "regions", "history", "candidates"})
V2_OPTIONAL_ROOT_KEYS = frozenset({"snapshot_id", "elements", "progress"})


def _record(value: Any, message: str) -> dict[str, Any]:
    if not isinstance(value, dict):
        raise ValueError(message)
    return value


def _bounded_string(value: Any, name: str, limit: int) -> str:
    if not isinstance(value, str) or not value.strip() or len(value) > limit:
        raise ValueError(f"{name} must be a nonempty string of at most {limit} characters")
    return value


def _identifier(value: Any, name: str) -> str:
    value = _bounded_string(value, name, 64)
    if not ID_PATTERN.fullmatch(value):
        raise ValueError(f"{name} contains unsupported characters")
    return value


def _validate_elements(value: Any) -> list[dict[str, str]]:
    if not isinstance(value, list) or len(value) > MAX_ELEMENTS:
        raise ValueError(f"elements must be an array of at most {MAX_ELEMENTS} items")
    elements: list[dict[str, str]] = []
    for raw_value in value:
        raw = _record(raw_value, "element must be an object")
        if set(raw) != {"role_class", "label", "state"}:
            raise ValueError("element may contain only role_class, label, and state")
        if raw["role_class"] not in ROLE_CLASSES:
            raise ValueError("element role_class is not a supported role class")
        if raw["state"] not in ELEMENT_STATES:
            raise ValueError("element state is not a supported state")
        elements.append(
            {
                "role_class": raw["role_class"],
                "label": _bounded_string(raw["label"], "element label", 200),
                "state": raw["state"],
            }
        )
    return elements


def _count(value: Any, name: str, minimum: int) -> int:
    if (
        not isinstance(value, int)
        or isinstance(value, bool)
        or not minimum <= value <= MAX_PROGRESS_COUNT
    ):
        raise ValueError(f"{name} must be an integer from {minimum} to {MAX_PROGRESS_COUNT}")
    return value


def _validate_progress(value: Any) -> list[dict[str, Any]]:
    """Validate task steps and how often this run has performed each one.

    ``done`` is counted from the runner's own performed actions, never read
    from the application, so ``progress`` carries no application values.
    """
    if not isinstance(value, list) or len(value) > MAX_PROGRESS:
        raise ValueError(f"progress must be an array of at most {MAX_PROGRESS} items")
    progress: list[dict[str, Any]] = []
    for raw_value in value:
        raw = _record(raw_value, "progress item must be an object")
        if set(raw) != {"step", "done", "required"}:
            raise ValueError("progress item may contain only step, done, and required")
        progress.append(
            {
                "step": _bounded_string(raw["step"], "progress step", 200),
                "done": _count(raw["done"], "progress done", 0),
                "required": _count(raw["required"], "progress required", 1),
            }
        )
    return progress


def validate_request(value: Any) -> dict[str, Any]:
    """Validate a ``cua.jev_choice_request_v1`` or ``_v2`` request strictly.

    v1 is unchanged: exactly its six root keys and ``id``/``description``
    candidates, so any v2 field in a v1 request is rejected. v2 may add a root
    ``snapshot_id``, ``elements``, and ``progress`` and a per-candidate
    ``source``; reserved candidates never carry a source.
    """
    root = _record(value, "request must be a JSON object")
    schema = root.get("schema")
    if schema == REQUEST_SCHEMA:
        if set(root) != V1_ROOT_KEYS:
            raise ValueError(f"request must match {REQUEST_SCHEMA}")
    elif schema == REQUEST_SCHEMA_V2:
        if not V1_ROOT_KEYS.issubset(root) or not set(root).issubset(
            V1_ROOT_KEYS | V2_OPTIONAL_ROOT_KEYS
        ):
            raise ValueError(f"request must match {REQUEST_SCHEMA_V2}")
    else:
        raise ValueError(f"request must match {REQUEST_SCHEMA} or {REQUEST_SCHEMA_V2}")
    v2 = schema == REQUEST_SCHEMA_V2

    goal = _bounded_string(root["goal"], "goal", 4_000)
    capture_id = _bounded_string(root["capture_id"], "capture_id", 256)

    raw_regions = root["regions"]
    if not isinstance(raw_regions, list) or len(raw_regions) > MAX_REGIONS:
        raise ValueError(f"regions must be an array of at most {MAX_REGIONS} items")
    regions: list[dict[str, Any]] = []
    region_ids: set[str] = set()
    for raw_value in raw_regions:
        raw = _record(raw_value, "region must be an object")
        allowed = {"id", "kind", "bounds", "text", "label", "confidence", "interactive"}
        required = {"id", "kind", "bounds", "confidence", "interactive"}
        if not set(raw).issubset(allowed) or not required.issubset(raw):
            raise ValueError("region has unsupported or missing fields")
        region_id = _bounded_string(raw["id"], "region id", 256)
        if region_id in region_ids:
            raise ValueError("region IDs must be unique")
        region_ids.add(region_id)
        kind = raw["kind"]
        if kind not in {"text", "icon"}:
            raise ValueError("region kind must be text or icon")
        bounds = _record(raw["bounds"], "region bounds must be an object")
        if set(bounds) != {"x", "y", "width", "height"}:
            raise ValueError("region bounds have unsupported or missing fields")
        for key in ("x", "y", "width", "height"):
            number = bounds[key]
            minimum = 0 if key in {"x", "y"} else 1
            if not isinstance(number, int) or isinstance(number, bool) or number < minimum:
                raise ValueError("region bounds must contain valid integers")
        text = raw.get("text")
        label = raw.get("label")
        if text is not None:
            text = _bounded_string(text, "region text", 1_000)
        if label is not None:
            label = _bounded_string(label, "region label", 1_000)
        if (kind == "text" and text is None) or (kind == "icon" and label is None):
            raise ValueError("region is missing content required by its kind")
        confidence = raw["confidence"]
        if (
            not isinstance(confidence, (int, float))
            or isinstance(confidence, bool)
            or not math.isfinite(float(confidence))
            or not 0 <= float(confidence) <= 1
        ):
            raise ValueError("region confidence must be between zero and one")
        if not isinstance(raw["interactive"], bool):
            raise ValueError("region interactive must be boolean")
        regions.append(
            {
                "id": region_id,
                "kind": kind,
                "bounds": dict(bounds),
                "text": text,
                "label": label,
                "confidence": float(confidence),
                "interactive": raw["interactive"],
            }
        )

    history = root["history"]
    if not isinstance(history, list) or len(history) > MAX_HISTORY:
        raise ValueError(f"history must be an array of at most {MAX_HISTORY} items")
    compact_history: list[dict[str, str]] = []
    for raw_value in history:
        raw = _record(raw_value, "history item must be an object")
        if not raw or not set(raw).issubset({"selected_id", "outcome"}):
            raise ValueError("history contains a forbidden field")
        item: dict[str, str] = {}
        if "selected_id" in raw:
            item["selected_id"] = _identifier(raw["selected_id"], "history selected_id")
        if "outcome" in raw:
            item["outcome"] = _bounded_string(raw["outcome"], "history outcome", 128)
        compact_history.append(item)

    raw_candidates = root["candidates"]
    if (
        not isinstance(raw_candidates, list)
        or not 2 <= len(raw_candidates) <= MAX_CANDIDATES
    ):
        raise ValueError(f"candidates must contain between 2 and {MAX_CANDIDATES} items")
    candidates: list[dict[str, str]] = []
    candidate_ids: set[str] = set()
    for raw_value in raw_candidates:
        raw = _record(raw_value, "candidate must be an object")
        keys = set(raw)
        if v2:
            if not {"id", "description"}.issubset(keys) or not keys.issubset(
                {"id", "description", "source"}
            ):
                raise ValueError("candidate may contain only id, description, and source")
        elif keys != {"id", "description"}:
            raise ValueError("candidate may contain only id and description")
        candidate_id = _identifier(raw["id"], "candidate id")
        if candidate_id in candidate_ids:
            raise ValueError("candidate IDs must be unique")
        candidate_ids.add(candidate_id)
        candidate = {
            "id": candidate_id,
            "description": _bounded_string(raw["description"], "description", 1_000),
        }
        if "source" in raw:
            if candidate_id in RESERVED_IDS:
                raise ValueError("reserved candidates must not carry a source")
            if raw["source"] not in CANDIDATE_SOURCES:
                raise ValueError("candidate source must be page, ax, or visual")
            candidate["source"] = raw["source"]
        candidates.append(candidate)
    if not RESERVED_IDS.issubset(candidate_ids):
        raise ValueError("candidates must include reobserve and abstain")

    validated: dict[str, Any] = {
        "schema": schema,
        "goal": goal,
        "capture_id": capture_id,
        "regions": regions,
        "history": compact_history,
        "candidates": candidates,
    }
    if v2:
        snapshot_id = root.get("snapshot_id")
        validated["snapshot_id"] = (
            None if snapshot_id is None else _bounded_string(snapshot_id, "snapshot_id", 64)
        )
        validated["elements"] = _validate_elements(root.get("elements", []))
        validated["progress"] = _validate_progress(root.get("progress", []))
    return validated


def provider_observation(validated: Mapping[str, Any]) -> dict[str, Any]:
    """Build the observation a provider sees for a validated request.

    A v1 request keeps its exact v1 observation. A v2 request adds the native
    ``snapshot_id``, the compact ``elements``, and each candidate's ``source``,
    plus ``progress`` when the request carries any; none of these contain
    element tokens, values, or pixels.
    """
    observation: dict[str, Any] = {
        "capture_id": validated["capture_id"],
        "regions": validated["regions"],
        "history": validated["history"],
    }
    if validated.get("schema") == REQUEST_SCHEMA_V2:
        observation["snapshot_id"] = validated.get("snapshot_id")
        observation["elements"] = validated.get("elements", [])
        observation["candidate_sources"] = {
            item["id"]: item["source"] for item in validated["candidates"] if "source" in item
        }
        if validated.get("progress"):
            observation["progress"] = validated["progress"]
    return observation


def choose_request(
    request: Mapping[str, Any], *, client: Any | None = None, mock: bool = False
) -> dict[str, Any]:
    validated = validate_request(dict(request))
    criteria = {item["id"]: item["description"] for item in validated["candidates"]}
    if mock:
        selected_id = next(
            (
                candidate_id
                for candidate_id in criteria
                if candidate_id not in {"reobserve", "abstain"}
            ),
            "reobserve",
        )
        choice = ProviderChoice(
            selected_id=selected_id,
            confidence=1.0,
            probabilities={
                candidate_id: float(candidate_id == selected_id)
                for candidate_id in criteria
            },
            model="mock",
        )
    else:
        if client is None:
            from typesafe_sdk import TypeSafeClient

            with TypeSafeClient() as live_client:
                choice = choose_bounded_with_typesafe(
                    live_client,
                    goal=validated["goal"],
                    observation=provider_observation(validated),
                    criteria=criteria,
                )
        else:
            choice = choose_bounded_with_typesafe(
                client,
                goal=validated["goal"],
                observation=provider_observation(validated),
                criteria=criteria,
            )
    return {
        "schema": RESPONSE_SCHEMA,
        "selected_id": choice.selected_id,
        "model": choice.model,
        "confidence": choice.confidence,
        "probabilities": choice.probabilities,
    }


def main() -> None:
    if sys.argv[1:] not in ([], ["--mock"]):
        raise SystemExit("usage: choose_action.py [--mock]")
    raw = sys.stdin.buffer.read(MAX_INPUT_BYTES + 1)
    if len(raw) > MAX_INPUT_BYTES:
        raise SystemExit("request exceeds input limit")
    try:
        request = json.loads(raw)
        validate_request(request)
    except (UnicodeDecodeError, json.JSONDecodeError, ValueError, TypeError, KeyError) as error:
        raise SystemExit(f"invalid request: {error}") from None
    try:
        response = choose_request(request, mock=sys.argv[1:] == ["--mock"])
    except Exception:
        raise SystemExit("provider request failed") from None
    sys.stdout.write(json.dumps(response, separators=(",", ":")) + "\n")


if __name__ == "__main__":
    main()