Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

14.4 KB
Raw Blame
"""Deterministic synthetic training data for the Cua-S1 form specialist.

Each episode describes one form, one person, and document-style entities. Every
actionable element becomes a choice-classification row. Splits are assigned by
the form's concept signature, so the same field set cannot cross split
boundaries.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import random
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Sequence

from .concepts import (
    CHROME_ELEMENTS,
    CONCEPT_BY_KEY,
    CONCEPTS,
    DISTRACTOR_ENTITIES,
    FORM_TITLES,
    NON_SUBMIT_BUTTONS,
    OPTIONAL_CHECKBOXES,
    OPTIONAL_FIELDS,
    REQUIRED_CHECKBOXES,
    SUBMIT_LABELS,
    person,
)
from .schema import Element, Entity, encode, render_context, render_options

SPLIT_NAMES = ("train", "validation", "test")
DATASET_FORMAT = "cua-s1-choice-jsonl"
DATASET_FORMAT_VERSION = 1


@dataclass(frozen=True)
class SynthesisConfig:
    """Controls stochastic data features while preserving seeded replay."""

    missing_entity_probability: float = 0.12
    partial_form_probability: float = 0.30
    filled_field_probability: float = 0.60
    stale_value_probability: float = 0.05
    hard_negative_probability: float = 0.35
    max_extra_entities: int = 5
    max_distractor_entities: int = 3

    def validate(self) -> None:
        probabilities = (
            self.missing_entity_probability,
            self.partial_form_probability,
            self.filled_field_probability,
            self.stale_value_probability,
            self.hard_negative_probability,
        )
        if any(probability < 0.0 or probability > 1.0 for probability in probabilities):
            raise ValueError("synthesis probabilities must be between 0 and 1")
        if self.max_extra_entities < 0 or self.max_distractor_entities < 0:
            raise ValueError("entity limits must be non-negative")


def _stable_fraction(text: str) -> float:
    digest = hashlib.sha256(text.encode("utf-8")).digest()
    return int.from_bytes(digest[:8], "big") / 2**64


def _validate_ratios(ratios: Sequence[float]) -> tuple[float, float, float]:
    if len(ratios) != len(SPLIT_NAMES):
        raise ValueError("ratios must contain train, validation, and test values")
    normalized = tuple(float(ratio) for ratio in ratios)
    if any(ratio < 0 for ratio in normalized):
        raise ValueError("split ratios must be non-negative")
    total = sum(normalized)
    if total <= 0:
        raise ValueError("at least one split ratio must be positive")
    return tuple(ratio / total for ratio in normalized)  # type: ignore[return-value]


def sample_form(rng: random.Random, config: SynthesisConfig | None = None) -> dict:
    """Create a form with confusable labels, optional fields, and one submit."""

    config = config or SynthesisConfig()
    groups = sorted({concept.group for concept in CONCEPTS})
    chosen_groups = rng.sample(groups, rng.randint(2, min(5, len(groups))))
    pool = [concept for concept in CONCEPTS if concept.group in chosen_groups]
    concepts = rng.sample(pool, min(len(pool), rng.randint(4, 16)))

    # Co-locate look-alike concepts so matching requires the complete label.
    hard_negatives = (
        ("email", "street"),
        ("phone", "ec_phone"),
        ("full_name", "ec_name"),
        ("first_name", "ec_name"),
        ("phone", "work_phone"),
        ("dob", "start_date"),
        ("dob", "incident_date"),
        ("zip", "policy"),
        ("state", "university"),
        ("city", "country"),
        ("employer", "insurer"),
        ("account", "policy"),
    )
    for first, second in hard_negatives:
        if rng.random() < config.hard_negative_probability:
            for key in (first, second):
                if all(concept.key != key for concept in concepts):
                    concepts.append(CONCEPT_BY_KEY[key])

    keys = {concept.key for concept in concepts}
    if "full_name" in keys and {"first_name", "last_name"} & keys and rng.random() < 0.8:
        concepts = [concept for concept in concepts if concept.key != "full_name"]

    fields = [
        {
            "concept": concept.key,
            "label": rng.choice(concept.form_labels),
            "kind": concept.kind,
            "placeholder": (
                rng.choice(concept.placeholder)
                if concept.placeholder and rng.random() < 0.5
                else ""
            ),
        }
        for concept in concepts
    ]
    for _ in range(rng.choice((0, 0, 1, 1, 2))):
        fields.append(
            {
                "concept": None,
                "label": rng.choice(OPTIONAL_FIELDS),
                "kind": "text",
                "placeholder": "",
            }
        )
    rng.shuffle(fields)

    checkboxes = [(rng.choice(REQUIRED_CHECKBOXES), True) for _ in range(rng.choice((0, 1, 1, 2)))]
    checkboxes.extend(
        (rng.choice(OPTIONAL_CHECKBOXES), False) for _ in range(rng.choice((0, 0, 1)))
    )
    rng.shuffle(checkboxes)

    buttons = [(rng.choice(NON_SUBMIT_BUTTONS), False) for _ in range(rng.choice((0, 1, 1, 2)))]
    buttons.append((rng.choice(SUBMIT_LABELS), True))
    if rng.random() < 0.3:
        buttons.reverse()

    title = rng.choice(FORM_TITLES)
    if rng.random() < 0.5:
        title += rng.choice(
            (" - Google Chrome", " - Microsoft Edge", " - Mozilla Firefox", " - Safari", "")
        )
    if rng.random() < 0.2:
        title = rng.choice(("", "Untitled", "Form", "Page 2 of 3"))
    return {"title": title, "fields": fields, "checkboxes": checkboxes, "buttons": buttons}


def form_signature(form: dict) -> str:
    """Return the form identity used to keep train and evaluation disjoint."""

    fields = sorted(field["concept"] or f"optional:{field['label']}" for field in form["fields"])
    return "|".join(fields)


def sample_document(
    rng: random.Random,
    form: dict,
    profile: dict,
    config: SynthesisConfig | None = None,
) -> tuple[list[Entity], dict[str, int]]:
    """Generate document entities and a form-concept-to-entity index."""

    config = config or SynthesisConfig()
    entities: list[Entity] = []
    concept_to_entity: dict[str, int] = {}
    present = [field["concept"] for field in form["fields"] if field["concept"]]
    for key in present:
        if rng.random() < config.missing_entity_probability:
            continue
        concept = CONCEPT_BY_KEY[key]
        concept_to_entity[key] = len(entities)
        entities.append(Entity(rng.choice(concept.doc_labels), concept.value(rng, profile)))

    absent = [concept for concept in CONCEPTS if concept.key not in present]
    extra_count = rng.randint(0, min(config.max_extra_entities, len(absent)))
    for concept in rng.sample(absent, extra_count):
        entities.append(Entity(rng.choice(concept.doc_labels), concept.value(rng, profile)))

    distractor_count = rng.randint(0, min(config.max_distractor_entities, len(DISTRACTOR_ENTITIES)))
    for label, generate in rng.sample(DISTRACTOR_ENTITIES, distractor_count):
        entities.append(Entity(label, generate(rng)))

    order = list(range(len(entities)))
    rng.shuffle(order)
    entities = [entities[index] for index in order]
    remap = {old: new for new, old in enumerate(order)}
    concept_to_entity = {key: remap[index] for key, index in concept_to_entity.items()}
    return entities, concept_to_entity


def episode_rows(seed: int, config: SynthesisConfig | None = None) -> list[dict]:
    """Generate all element decisions for a single replayable episode."""

    config = config or SynthesisConfig()
    config.validate()
    rng = random.Random(seed)
    form = sample_form(rng, config)
    signature = form_signature(form)
    entities, concept_to_entity = sample_document(rng, form, person(rng), config)
    options = render_options(entities)
    partially_filled = rng.random() < config.partial_form_probability
    rows: list[dict] = []

    def add(
        element: Element,
        action: str,
        entity_index: int | None = None,
        placeholder: str = "",
    ) -> None:
        rows.append(
            {
                "context": render_context(form["title"], element, placeholder),
                "options": options,
                "label": encode(action, entity_index, entities),
                "meta": {
                    "seed": seed,
                    "form_signature": signature,
                    "role": element.role,
                    "action": action,
                },
            }
        )

    for field in form["fields"]:
        entity_index = concept_to_entity.get(field["concept"]) if field["concept"] else None
        filled = (
            entity_index is not None
            and partially_filled
            and rng.random() < config.filled_field_probability
        )
        value = entities[entity_index].value if filled and entity_index is not None else ""
        if filled and rng.random() < config.stale_value_probability:
            value = "stale value"
        element = Element("Edit", field["label"], value, actions=("set_value",))
        if entity_index is not None and (not value or value == "stale value"):
            add(element, "fill", entity_index, field["placeholder"])
        else:
            add(element, "skip", placeholder=field["placeholder"])

    for label, required in form["checkboxes"]:
        checked = partially_filled and rng.random() < 0.5
        element = Element("CheckBox", label, checked=checked, actions=("toggle",))
        add(element, "check" if required and not checked else "skip")

    for label, is_submit in form["buttons"]:
        add(Element("Button", label, actions=("invoke",)), "click" if is_submit else "skip")

    chrome_count = rng.randint(3, min(10, len(CHROME_ELEMENTS)))
    for role, label in rng.sample(CHROME_ELEMENTS, chrome_count):
        value = "https://example.invalid/form" if label == "Address bar" else ""
        add(Element(role, label or form["title"], value), "skip")
    return rows


def signature(rows: list[dict]) -> str:
    """Recover an episode signature from generated rows."""

    if not rows:
        raise ValueError("an episode must contain at least one row")
    return str(rows[0]["meta"]["form_signature"])


def write_splits(
    output: str | Path,
    episodes: int,
    seed: int,
    ratios: Sequence[float] = (0.8, 0.1, 0.1),
    config: SynthesisConfig | None = None,
) -> dict:
    """Write deterministic JSONL splits and return row and episode counts."""

    if episodes < 0:
        raise ValueError("episodes must be non-negative")
    config = config or SynthesisConfig()
    config.validate()
    normalized_ratios = _validate_ratios(ratios)
    train_ratio, validation_ratio, _ = normalized_ratios
    output_path = Path(output)
    output_path.mkdir(parents=True, exist_ok=True)
    handles = {
        name: (output_path / f"{name}.jsonl").open("w", encoding="utf-8") for name in SPLIT_NAMES
    }
    counts = {name: 0 for name in SPLIT_NAMES}
    episodes_per_split = {name: 0 for name in SPLIT_NAMES}
    try:
        for episode_index in range(episodes):
            rows = episode_rows(seed + episode_index * 7919, config)
            bucket = _stable_fraction(signature(rows))
            if bucket < train_ratio:
                split = "train"
            elif bucket < train_ratio + validation_ratio:
                split = "validation"
            else:
                split = "test"
            for row in rows:
                handles[split].write(
                    json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
                    + "\n"
                )
            counts[split] += len(rows)
            episodes_per_split[split] += 1
    finally:
        for handle in handles.values():
            handle.close()

    split_manifest = {}
    for name in SPLIT_NAMES:
        split_path = output_path / f"{name}.jsonl"
        split_manifest[name] = {
            "file": split_path.name,
            "sha256": hashlib.sha256(split_path.read_bytes()).hexdigest(),
            "rows": counts[name],
            "episodes": episodes_per_split[name],
        }
    manifest = {
        "format": DATASET_FORMAT,
        "format_version": DATASET_FORMAT_VERSION,
        "generator": {
            "format": "cua_s1.synth",
            "config": asdict(config),
        },
        "ratios": dict(zip(SPLIT_NAMES, normalized_ratios, strict=True)),
        "seed": seed,
        "episodes": episodes,
        "splits": split_manifest,
    }
    manifest_path = output_path / "manifest.json"
    temporary = output_path / ".manifest.json.tmp"
    temporary.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    temporary.replace(manifest_path)
    return {"rows": counts, "episodes": episodes_per_split}


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--output", type=Path, default=Path("data/cua-s1"))
    parser.add_argument("--episodes", type=int, default=6000)
    parser.add_argument("--seed", type=int, default=2026)
    parser.add_argument("--train-ratio", type=float, default=0.8)
    parser.add_argument("--validation-ratio", type=float, default=0.1)
    parser.add_argument("--test-ratio", type=float, default=0.1)
    parser.add_argument("--missing-entity-probability", type=float, default=0.12)
    parser.add_argument("--partial-form-probability", type=float, default=0.30)
    parser.add_argument("--filled-field-probability", type=float, default=0.60)
    parser.add_argument("--stale-value-probability", type=float, default=0.05)
    parser.add_argument("--hard-negative-probability", type=float, default=0.35)
    parser.add_argument("--max-extra-entities", type=int, default=5)
    parser.add_argument("--max-distractor-entities", type=int, default=3)
    args = parser.parse_args()
    config = SynthesisConfig(
        missing_entity_probability=args.missing_entity_probability,
        partial_form_probability=args.partial_form_probability,
        filled_field_probability=args.filled_field_probability,
        stale_value_probability=args.stale_value_probability,
        hard_negative_probability=args.hard_negative_probability,
        max_extra_entities=args.max_extra_entities,
        max_distractor_entities=args.max_distractor_entities,
    )
    result = write_splits(
        args.output,
        args.episodes,
        args.seed,
        (args.train_ratio, args.validation_ratio, args.test_ratio),
        config,
    )
    print(json.dumps(result, sort_keys=True))


if __name__ == "__main__":
    main()