Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""Cua-S1-Nano: a from-scratch, ~855K-parameter option-attention scorer for
computer-use GUI decisions.
Given a screen state (either a rendered accessibility-tree excerpt per
element, or a screenshot crop per element) and a fixed, closed set of
candidate (element, action) options, Cua-S1-Nano scores every option in one
parallel forward pass and returns a probability distribution over each
element's own options. It is a separate checkpoint family from
`cua-s1-form-v0`: it is not a form-filling specialist, and it makes no use of
`cua_s1.planner`, `cua_s1.driver`, or `cua_s1.synth`, which remain specific to
the form-filling checkpoint.
Two context modalities are supported, selected per batch via
`batch["modality"]`:
- `"text"`: context tokens come from `NanoTextContextEncoder`, a small
trainable byte-embedding + transformer encoder over a rendered
accessibility-tree excerpt for the element. This path has no dependency on
any vision backbone.
- `"multimodal"`: context tokens come from a frozen vision backbone's
per-element screenshot-crop features, projected down by a small trainable
`visual_proj` layer. The backbone is selected explicitly via the
`vision_backbone` config field (`"smolvlm"` or `"siglip"`); this library
never chooses a backbone from an environment variable, since that would be
surprising behavior for public library code. Loading a vision backbone
requires the optional `transformers` and `pillow` dependencies and
downloads pretrained weights on first use; `NanoScorer` itself never
imports them unless `encode_images` is actually called.
Options are always rendered as text and encoded by `NanoOptionEncoder`
regardless of modality. Both context encoders and the option encoder share
one hidden width, so a single `AttentionHead` (from `cua_s1.model`) serves
both modalities; only one of the two context encoders runs per forward call.
This module is inference- and scoring-only. It intentionally does not include
a training loop or the data-generation code used to produce a `cua-s1-nano`
checkpoint; see `libs/cua-s1/training/` for the package's existing training
entry points, and the module docstring there for the status of nano training
support.
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import torch
from torch import nn
from .checkpoint import load_checkpoint_files, save_checkpoint_files
from .model import AttentionHead
VISION_FEATURE_DIM = 768 # hidden size shared by both supported vision backbones
TensorBatch = dict[str, torch.Tensor]
@dataclass(frozen=True)
class NanoElement:
"""One GUI element: its rendered context and its candidate options.
`context` is either a rendered accessibility-tree excerpt (text
modality) or ignored placeholder text (multimodal modality, where the
context tokens instead come from a screenshot crop supplied out of
band via `crop_features`). `options` are always rendered as text.
"""
element_id: str
context: str
options: tuple[str, ...]
def _byte_ids(text: str, length: int) -> list[int]:
return [byte + 1 for byte in text.encode("utf-8", errors="replace")[:length]]
class NanoByteCollator:
"""Collate `NanoElement`s into padded UTF-8 byte tensors for the text
modality. Produces the same tensor shapes `NanoScorer.forward` expects
under `batch["modality"] == "text"`."""
def __init__(self, context_tokens: int, option_tokens: int) -> None:
if context_tokens <= 0 or option_tokens <= 0:
raise ValueError("token limits must be positive")
self.context_tokens = context_tokens
self.option_tokens = option_tokens
def __call__(self, elements: Sequence[NanoElement]) -> TensorBatch:
if not elements:
raise ValueError("cannot collate an empty batch")
contexts = [_byte_ids(item.context, self.context_tokens) for item in elements]
option_rows = [
[_byte_ids(option, self.option_tokens) for option in item.options] for item in elements
]
batch = _tensor_batch(contexts, option_rows)
batch["modality"] = "text"
return batch
def _tensor_batch(
contexts: Sequence[Sequence[int]],
option_rows: Sequence[Sequence[Sequence[int]]],
) -> TensorBatch:
batch = len(contexts)
max_context = max(1, max(map(len, contexts)))
max_options = max(len(row) for row in option_rows)
max_option_tokens = max(
1, max((len(tokens) for row in option_rows for tokens in row), default=1)
)
context_ids = torch.zeros((batch, max_context), dtype=torch.long)
option_ids = torch.zeros((batch, max_options, max_option_tokens), dtype=torch.long)
option_mask = torch.zeros((batch, max_options), dtype=torch.bool)
for row, tokens in enumerate(contexts):
if tokens:
context_ids[row, : len(tokens)] = torch.tensor(tokens, dtype=torch.long)
for row, options in enumerate(option_rows):
option_mask[row, : len(options)] = True
for column, tokens in enumerate(options):
if tokens:
option_ids[row, column, : len(tokens)] = torch.tensor(tokens, dtype=torch.long)
return {
"context_ids": context_ids,
"context_token_mask": context_ids.ne(0),
"option_ids": option_ids,
"option_token_mask": option_ids.ne(0),
"option_mask": option_mask,
}
class NanoOptionEncoder(nn.Module):
"""Byte embeddings + a 1-layer transformer, mean-pooled per option."""
def __init__(
self, width: int, option_tokens: int, heads: int = 4, dropout: float = 0.1
) -> None:
super().__init__()
if width % heads:
raise ValueError("model width must be divisible by attention heads")
self.embedding = nn.Embedding(257, width, padding_idx=0)
self.position = nn.Embedding(option_tokens, width)
self.encoder = nn.TransformerEncoder(
nn.TransformerEncoderLayer(
width, heads, width * 4, dropout, batch_first=True, norm_first=True
),
1,
)
def forward(self, option_ids: torch.Tensor, option_token_mask: torch.Tensor) -> torch.Tensor:
b, n, t = option_ids.shape
flat_ids = option_ids.reshape(b * n, t)
flat_mask = option_token_mask.reshape(b * n, t)
safe_mask = flat_mask.clone()
safe_mask[:, 0] = True # padded/absent options still need one valid key for the transformer
positions = torch.arange(t, device=option_ids.device)
embedded = self.embedding(flat_ids) + self.position(positions)
hidden = self.encoder(embedded, src_key_padding_mask=~safe_mask)
weights = flat_mask.unsqueeze(-1).float()
pooled = (hidden * weights).sum(1) / weights.sum(1).clamp_min(1)
return pooled.reshape(b, n, -1)
class NanoTextContextEncoder(nn.Module):
"""Small trainable byte transformer over an element's rendered context.
Produces a full token sequence (not pooled) so the attention head has a
real set of context tokens to attend over, matching the shape the
multimodal path produces (a sequence of patch tokens, not one vector).
"""
def __init__(
self, width: int, context_tokens: int, heads: int = 4, layers: int = 2, dropout: float = 0.1
) -> None:
super().__init__()
if width % heads:
raise ValueError("model width must be divisible by attention heads")
self.embedding = nn.Embedding(257, width, padding_idx=0)
self.position = nn.Embedding(context_tokens, width)
self.encoder = nn.TransformerEncoder(
nn.TransformerEncoderLayer(
width, heads, width * 4, dropout, batch_first=True, norm_first=True
),
layers,
)
def forward(self, context_ids: torch.Tensor, context_token_mask: torch.Tensor) -> torch.Tensor:
safe_mask = context_token_mask.clone()
safe_mask[:, 0] = True
positions = torch.arange(context_ids.shape[1], device=context_ids.device)
embedded = self.embedding(context_ids) + self.position(positions)
return self.encoder(embedded, src_key_padding_mask=~safe_mask)
class NanoScorer(nn.Module):
"""The Cua-S1-Nano architecture over a closed option set.
`forward` takes a collated batch whose `"modality"` key is either
`"multimodal"` or `"text"` and picks the matching context encoder; both
share one `AttentionHead` and one `NanoOptionEncoder`.
"""
def __init__(
self,
width: int = 128,
rank: int = 128,
option_tokens: int = 96,
context_tokens: int = 256,
vision_dim: int = VISION_FEATURE_DIM,
) -> None:
super().__init__()
if min(width, rank, option_tokens, context_tokens, vision_dim) <= 0:
raise ValueError("model dimensions must be positive")
self.visual_proj = nn.Sequential(nn.LayerNorm(vision_dim), nn.Linear(vision_dim, width))
self.text_context_encoder = NanoTextContextEncoder(width, context_tokens)
self.option_encoder = NanoOptionEncoder(width, option_tokens)
self.head = AttentionHead(width, rank)
self.width, self.rank = width, rank
self.option_tokens, self.context_tokens, self.vision_dim = (
option_tokens,
context_tokens,
vision_dim,
)
def forward(self, batch: TensorBatch, shuffle_context: bool = False) -> torch.Tensor:
modality = batch["modality"]
if modality == "multimodal":
context = self.visual_proj(batch["crop_features"])
context_mask = batch["crop_mask"]
elif modality == "text":
context = self.text_context_encoder(batch["context_ids"], batch["context_token_mask"])
context_mask = batch["context_token_mask"]
else:
raise ValueError(f"unknown modality: {modality!r}")
options = self.option_encoder(batch["option_ids"], batch["option_token_mask"])
return self.head(context, context_mask, options, batch["option_mask"], shuffle_context)
@torch.no_grad()
def score_elements(
self,
elements: Sequence[NanoElement],
collator: NanoByteCollator,
device: torch.device | None = None,
) -> dict[str, dict[int, float]]:
"""Score a batch of text-modality elements against their own options.
Returns, per `element_id`, a dict mapping the index of each of that
element's candidate options (in `elements[i].options` order) to a
softmax probability. Each element's own distribution sums to ~1 and
covers exactly its offered options.
"""
if not elements:
return {}
device = device or next(self.parameters()).device
batch = collator(elements)
batch = {k: (v.to(device) if isinstance(v, torch.Tensor) else v) for k, v in batch.items()}
logits = self.forward(batch)
probs = torch.softmax(logits, dim=-1)
result: dict[str, dict[int, float]] = {}
for i, element in enumerate(elements):
n = len(element.options)
result[element.element_id] = {j: float(probs[i, j]) for j in range(n)}
return result
def select_device(name: str = "auto") -> torch.device:
"""Resolve an explicit device or select the best available local device."""
if name != "auto":
return torch.device(name)
if torch.cuda.is_available():
return torch.device("cuda")
if torch.backends.mps.is_available():
return torch.device("mps")
return torch.device("cpu")
def _positive_int(config: Mapping[str, Any], key: str, default: int | None = None) -> int:
value = config.get(key, default)
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
raise ValueError(f"model config field {key!r} must be a positive integer")
return value
def make_nano_system(
config: Mapping[str, Any], device: torch.device | str
) -> tuple[NanoScorer, NanoByteCollator]:
"""Construct a `NanoScorer` and matching text collator from validated
configuration. `config["vision_backbone"]` (if present) must be
`"smolvlm"` or `"siglip"`; it is not read or used here (multimodal
context is supplied by the caller as pre-computed `crop_features`), but
it is validated so a bad config fails fast rather than silently loading
the wrong backbone later via `load_vision_backbone`.
"""
target = select_device(device) if isinstance(device, str) else device
backbone = config.get("vision_backbone")
if backbone is not None and backbone not in ("smolvlm", "siglip"):
raise ValueError("model config field 'vision_backbone' must be 'smolvlm' or 'siglip'")
model = NanoScorer(
width=_positive_int(config, "width", 128),
rank=_positive_int(config, "rank", 128),
option_tokens=_positive_int(config, "option_tokens", 96),
context_tokens=_positive_int(config, "context_tokens", 256),
vision_dim=_positive_int(config, "vision_dim", VISION_FEATURE_DIM),
)
collator = NanoByteCollator(model.context_tokens, model.option_tokens)
return model.to(target), collator
def trainable_state(model: nn.Module) -> dict[str, torch.Tensor]:
"""Copy trainable parameters to contiguous CPU tensors."""
return {
name: parameter.detach().cpu().contiguous()
for name, parameter in model.named_parameters()
if parameter.requires_grad
}
def save_nano_checkpoint(
directory_or_path: str | Path,
model: NanoScorer,
config: Mapping[str, Any],
metadata: Mapping[str, Any] | None = None,
) -> tuple[Path, Path]:
"""Save a `NanoScorer`'s trainable state as safetensors with a JSON
configuration, reusing `cua_s1.checkpoint`'s format-agnostic,
safetensors-only save path unmodified."""
return save_checkpoint_files(directory_or_path, trainable_state(model), config, metadata)
def load_nano_checkpoint(
directory_or_path: str | Path, device: torch.device | str
) -> tuple[NanoScorer, NanoByteCollator, dict[str, Any]]:
"""Load a `NanoScorer` from safetensors and JSON without executing
serialized code, reusing `cua_s1.checkpoint`'s format-agnostic,
safetensors-only load path unmodified."""
state_dict, config, _metadata = load_checkpoint_files(directory_or_path)
model, collator = make_nano_system(config, device)
try:
missing, unexpected = model.load_state_dict(state_dict, strict=False)
except RuntimeError as exc:
raise ValueError(f"checkpoint tensor mismatch: {exc}") from exc
if missing or unexpected:
raise ValueError(
f"checkpoint mismatch: missing={list(missing)}, unexpected={list(unexpected)}"
)
model.eval()
return model, collator, config
def parameter_count(model: nn.Module) -> int:
"""Return the number of trainable scalar parameters."""
return sum(parameter.numel() for parameter in model.parameters() if parameter.requires_grad)
# ---------------------------------------------------------------------------
# Optional frozen vision backbones for the multimodal modality.
#
# Neither backbone is imported, downloaded, or instantiated unless
# `load_vision_backbone` is actually called; `NanoScorer` and the rest of
# this module have no import-time dependency on `transformers` or `PIL`.
# Install the `nano-vision` extra to use either backbone.
# ---------------------------------------------------------------------------
class SmolVlmVisionBackbone:
"""Frozen SmolVLM-256M-Instruct vision tower (a SigLIP-style
Idefics3VisionTransformer, ~86.4M parameters). Only used to produce
per-element screenshot-crop features for the multimodal modality; the
backbone itself is never trained."""
MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct"
FEATURE_DIM = VISION_FEATURE_DIM
RAW_GRID = 32 # 512x512 processed image / patch_size 16 -> 32x32 raw patches
POOLED_GRID = 8 # adaptive-pooled down to 8x8 so per-crop feature sets stay small
TOKENS_PER_CROP = POOLED_GRID * POOLED_GRID
def __init__(self, device: str = "auto", dtype: "torch.dtype" = torch.float16) -> None:
from transformers import AutoProcessor, Idefics3ForConditionalGeneration
resolved = select_device(device)
self.device = resolved
self.dtype = dtype if self.device.type == "cuda" else torch.float32
processor = AutoProcessor.from_pretrained(self.MODEL_NAME)
self.image_processor = processor.image_processor
self.patch_size = 16
model = Idefics3ForConditionalGeneration.from_pretrained(
self.MODEL_NAME, dtype=self.dtype, device_map=str(self.device)
)
self.vision_model = model.model.vision_model.eval()
for parameter in self.vision_model.parameters():
parameter.requires_grad_(False)
del model
if self.device.type == "cuda":
torch.cuda.empty_cache()
def _patch_attention_mask(self, pixel_attention_mask: torch.Tensor) -> torch.Tensor:
p = self.patch_size
subgrid = pixel_attention_mask.unfold(1, p, p).unfold(2, p, p)
return (subgrid.sum(dim=(-1, -2)) > 0).bool()
@torch.no_grad()
def encode(self, images: Sequence[Any], batch_size: int = 64) -> torch.Tensor:
"""Returns `(N, TOKENS_PER_CROP, FEATURE_DIM)` float32 CPU tensor,
one row per image, adaptive-pooled down to `POOLED_GRID`x`POOLED_GRID`
patches per crop."""
all_features = []
for start in range(0, len(images), batch_size):
batch = list(images[start : start + batch_size])
out = self.image_processor(images=batch, return_tensors="pt", do_image_splitting=False)
pixel_values = out["pixel_values"].to(self.device, dtype=self.dtype)
b, n, c, h, w = pixel_values.shape
pixel_values = pixel_values.reshape(b * n, c, h, w)
pixel_attention_mask = out["pixel_attention_mask"].to(self.device)
pixel_attention_mask = pixel_attention_mask.reshape(
b * n, pixel_attention_mask.shape[-2], pixel_attention_mask.shape[-1]
)
patch_mask = self._patch_attention_mask(pixel_attention_mask)
features = self.vision_model(
pixel_values=pixel_values, patch_attention_mask=patch_mask
).last_hidden_state
features = self._pool(features)
all_features.append(features.float().cpu())
return torch.cat(all_features, dim=0)
def _pool(self, features: torch.Tensor) -> torch.Tensor:
b = features.shape[0]
grid = features.reshape(b, self.RAW_GRID, self.RAW_GRID, self.FEATURE_DIM).permute(
0, 3, 1, 2
)
pooled = torch.nn.functional.adaptive_avg_pool2d(grid, (self.POOLED_GRID, self.POOLED_GRID))
return pooled.permute(0, 2, 3, 1).reshape(
b, self.POOLED_GRID * self.POOLED_GRID, self.FEATURE_DIM
)
class SiglipVisionBackbone:
"""Alternative frozen backbone: `google/siglip-base-patch16-224`
(~93M parameters). 224x224 input with patch size 16 gives 14x14=196 raw
patches; hidden size 768 matches `VISION_FEATURE_DIM` exactly, so
`NanoScorer` needs no changes to use either backbone's output -- only the
per-crop feature tensors differ."""
MODEL_NAME = "google/siglip-base-patch16-224"
FEATURE_DIM = VISION_FEATURE_DIM
RAW_GRID = 14
POOLED_GRID = 8
TOKENS_PER_CROP = POOLED_GRID * POOLED_GRID
def __init__(self, device: str = "auto", dtype: "torch.dtype" = torch.float16) -> None:
from transformers import AutoImageProcessor, SiglipVisionModel
resolved = select_device(device)
self.device = resolved
self.dtype = dtype if self.device.type == "cuda" else torch.float32
self.image_processor = AutoImageProcessor.from_pretrained(self.MODEL_NAME)
self.vision_model = (
SiglipVisionModel.from_pretrained(self.MODEL_NAME, dtype=self.dtype)
.to(self.device)
.eval()
)
for parameter in self.vision_model.parameters():
parameter.requires_grad_(False)
if self.vision_model.config.hidden_size != self.FEATURE_DIM:
raise ValueError(
f"siglip hidden_size {self.vision_model.config.hidden_size} != "
f"expected {self.FEATURE_DIM}"
)
if self.device.type == "cuda":
torch.cuda.empty_cache()
@torch.no_grad()
def encode(self, images: Sequence[Any], batch_size: int = 64) -> torch.Tensor:
all_features = []
for start in range(0, len(images), batch_size):
batch = list(images[start : start + batch_size])
out = self.image_processor(images=batch, return_tensors="pt")
pixel_values = out["pixel_values"].to(self.device, dtype=self.dtype)
features = self.vision_model(pixel_values=pixel_values).last_hidden_state
features = self._pool(features)
all_features.append(features.float().cpu())
return torch.cat(all_features, dim=0)
def _pool(self, features: torch.Tensor) -> torch.Tensor:
b = features.shape[0]
grid = features.reshape(b, self.RAW_GRID, self.RAW_GRID, self.FEATURE_DIM).permute(
0, 3, 1, 2
)
pooled = torch.nn.functional.adaptive_avg_pool2d(grid, (self.POOLED_GRID, self.POOLED_GRID))
return pooled.permute(0, 2, 3, 1).reshape(
b, self.POOLED_GRID * self.POOLED_GRID, self.FEATURE_DIM
)
def load_vision_backbone(
name: str, device: str = "auto"
) -> SmolVlmVisionBackbone | SiglipVisionBackbone:
"""Explicitly construct one of the two supported frozen vision backbones.
Unlike the research harness this architecture was ported from,
`name` is a required, explicit argument here -- never an environment
variable -- since implicit environment-driven backbone selection is
surprising behavior for public library code. Requires the optional
`transformers` and `pillow` dependencies (the `nano-vision` extra) and
downloads pretrained weights on first use.
"""
if name == "smolvlm":
return SmolVlmVisionBackbone(device)
if name == "siglip":
return SiglipVisionBackbone(device)
raise ValueError("vision backbone name must be 'smolvlm' or 'siglip'")