Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

7.8 KB
Raw Blame
import base64
import json
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from io import BytesIO
from pathlib import Path

try:
    import tomllib  # Python 3.11+
except Exception:  # pragma: no cover
    import tomli as tomllib  # type: ignore


import os

# Registry now lives in the cua monorepo (the standalone `cua-bench-registry`
# repo is DEPRECATED — `cb run dataset` resolves datasets from
# `<cua-repo>/libs/cua-bench/datasets/` via cli/commands/registry.py).
# Honor CUA_REGISTRY_HOME (the same env var the cb resolver uses); otherwise
# default to this package's own libs/cua-bench directory.
#   <registry>/datasets/<dataset_id>/<environment_id>/{pyproject.toml, main.py}
#   <registry>/datasets/meta.json   (OPTIONAL — dataset-level descriptions)
task_registry = Path(os.environ.get("CUA_REGISTRY_HOME") or Path(__file__).resolve().parents[2])
task_registry_url = "https://github.com/trycua/cua/tree/main/libs/cua-bench/datasets/"

task_datasets = task_registry / "datasets"
task_metadata = task_datasets / "meta.json"
output_metadata = Path(__file__).parent / Path("./task_registry.json")


def read_meta(meta_path: Path):
    with open(meta_path, "r", encoding="utf-8") as f:
        return json.load(f)


def parse_pyproject(env_dir: Path):
    data = {
        "description": None,
        "license": None,
        "version": None,
        "authors": None,
        "difficulty": None,
        "category": None,
        "tags": None,
    }
    pp = env_dir / "pyproject.toml"
    if not pp.exists():
        return data
    with open(pp, "rb") as f:
        toml = tomllib.load(f)
    proj = toml.get("project", {})
    tool = toml.get("tool", {})
    cb = tool.get("cua-bench", {})
    data["description"] = cb.get("description") or proj.get("description")
    data["license"] = proj.get("license")
    data["version"] = proj.get("version")
    data["authors"] = proj.get("authors")
    data["difficulty"] = cb.get("difficulty")
    data["category"] = cb.get("category")
    data["tags"] = cb.get("tags")
    return data


def count_tasks_in_env(env_dir: Path) -> int:
    """Attempt to count tasks by importing the environment and calling tasks_config."""
    main_py = env_dir / "main.py"
    if not main_py.exists():
        return 0
    from cua_bench import make

    env = make(str(env_dir))
    if env.tasks_config_fn is None:
        env.close()
        return 0
    tasks = env.tasks_config_fn()
    n = len(tasks) if tasks is not None else 0
    env.close()
    return n


def generate_previews(dataset_id: str, env_dir: Path, max_previews: int = 5):
    """Run setup for first N tasks and save screenshot + task cfg.
    Returns a list of preview dicts with file paths and task info.
    """
    previews = []
    main_py = env_dir / "main.py"
    if not main_py.exists():
        return previews
    from cua_bench import make

    env = make(str(env_dir))
    if env.tasks_config_fn is None:
        env.close()
        return previews
    tasks = env.tasks_config_fn() or []
    count = min(len(tasks), max_previews)
    for i in range(count):
        screenshot_bytes, task_cfg = env.reset(task_id=i)

        # Wait a bit for the environment to stabilize
        time.sleep(5)

        # Take a fresh screenshot after waiting
        screenshot_bytes = env.provider.screenshot()

        # Convert to JPEG (quality 95) and encode as base64
        screenshot_b64 = None
        mime = "image/jpeg"
        from PIL import Image  # type: ignore

        img = Image.open(BytesIO(screenshot_bytes)).convert("RGB")
        buf = BytesIO()
        img.save(buf, format="JPEG", quality=95)
        screenshot_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")

        task_dict = {
            "description": getattr(task_cfg, "description", None),
            "task_id": getattr(task_cfg, "task_id", None) or i,
            "metadata": getattr(task_cfg, "metadata", None),
        }
        previews.append(
            {
                "index": i,
                "screenshot": f"data:{mime};base64,{screenshot_b64}",
                "task": task_dict,
            }
        )

        env.close()  # Close environment to reset before next preview
    env.close()
    return previews


def process_environment(ds_id: str, ds_github: str, env_dir: Path):
    """Process a single environment directory."""
    env_id = env_dir.name
    meta = parse_pyproject(env_dir)
    num_tasks = count_tasks_in_env(env_dir)
    env_previews = generate_previews(ds_id, env_dir, max_previews=5)
    return {
        "id": env_id,
        "github_url": f"{ds_github}{env_id}",
        "description": meta["description"],
        "num_tasks": num_tasks,
        "license": meta["license"],
        "version": meta["version"],
        "authors": meta["authors"],
        "difficulty": meta["difficulty"],
        "category": meta["category"],
        "tags": meta["tags"],
        "previews": env_previews,
    }


def build_registry(task_datasets_root: Path, meta_entries, parallelism: int = 16):
    output = []
    for entry in meta_entries:
        ds_id = entry.get("id")
        ds_desc = entry.get("description")
        ds_path = task_datasets_root / ds_id
        ds_github = f"{task_registry_url}{ds_id}/"
        if not ds_path.exists():
            output.append(
                {
                    "id": ds_id,
                    "github_url": ds_github,
                    "description": ds_desc,
                    "num_environments": 0,
                    "num_tasks": 0,
                    "environments": [],
                }
            )
            continue

        env_dirs = sorted([p for p in ds_path.iterdir() if p.is_dir()])
        envs = []
        total_tasks = 0

        # Process environments in parallel
        with ThreadPoolExecutor(max_workers=parallelism) as executor:
            future_to_env = {
                executor.submit(process_environment, ds_id, ds_github, env_dir): env_dir
                for env_dir in env_dirs
            }

            for future in as_completed(future_to_env):
                try:
                    env_data = future.result()
                    envs.append(env_data)
                    total_tasks += env_data["num_tasks"]
                except Exception as e:
                    env_dir = future_to_env[future]
                    print(f"Error processing {env_dir}: {e}", file=sys.stderr)

        output.append(
            {
                "id": ds_id,
                "github_url": ds_github,
                "description": ds_desc,
                "num_environments": len(envs),
                "num_tasks": total_tasks,
                "environments": envs,
            }
        )
    return output


def main():
    import argparse

    parser = argparse.ArgumentParser(description="Dump task registry to JSON")
    parser.add_argument(
        "--parallelism",
        type=int,
        default=16,
        help="Number of parallel workers for processing environments (default: 16)",
    )
    args = parser.parse_args()

    if not task_datasets.exists():
        print(f"datasets directory not found at {task_datasets}")
        return 1
    if task_metadata.exists():
        meta_entries = read_meta(task_metadata)
    else:
        # No meta.json (monorepo default): derive the dataset list from the
        # datasets/ subdirectories. Dataset-level descriptions stay null.
        print(f"No meta.json at {task_metadata}; deriving datasets from subdirectories.")
        meta_entries = [
            {"id": p.name, "description": None}
            for p in sorted(task_datasets.iterdir())
            if p.is_dir()
        ]
    reg = build_registry(task_datasets, meta_entries, parallelism=args.parallelism)
    output_metadata.write_text(json.dumps(reg, indent=2), encoding="utf-8")
    print(str(output_metadata))
    return 0


if __name__ == "__main__":
    sys.exit(main())