Skip to content

The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.

14.1 KB
Raw Blame
#!/usr/bin/env python3
"""What a change can affect: the scope of the checks on a pull request (CI, `run_all.sh --changed`).

A pull request that touches only one benchmark's tasks, image sources, CI hooks or docs page cannot break
another benchmark's tasks, so its CI runs the per-task checks on that benchmark's tasks (plus the repo-level checks, which are cheap and always run on everything). Anything else --
the checks themselves, the workflows, the shared modules (scripts/task_conventions.py, retry_policy,
embodied_first_exam/standard, images/monitor, ...), the shared images, the protocol documents, a path this table does not know -- runs the
FULL suite. Pushes to main always run the full suite (.github/workflows/static-checks.yml), so nothing that slips past
a scoped run stays unnoticed for longer than one merge.

A benchmark B is affected by a changed path P when P is one of B's own paths (UNITS below: its task-name prefix,
images/<b>/, its pages tasks/<folder>/*.md, ci_checks/benchmarks/<b>/, extra files) or when B's and P's benchmark refer to each other's
paths (`scripts/<x>` / `images/<x>` in a .py / .sh file under the other's own directories: shared layers that one
benchmark copies from another, e.g. MyoSuite and HumanoidBench's realtime service).

    python3 ci_checks/ci_scope.py                       # vs the merge base with origin/main
    python3 ci_checks/ci_scope.py --diff-from HEAD^1    # CI on a pull request: GitHub's merge commit vs main
    python3 ci_checks/ci_scope.py --paths a b ...       # explain the scope of the given paths
    python3 ci_checks/ci_scope.py --check               # the table against the benchmarks and images/ (run_all.sh)

Output: shell assignments (`eval "$(python3 ci_checks/ci_scope.py ...)"`): RCB_SCOPE=full|benchmarks|none,
RCB_SCOPE_REASON, RCB_SCOPE_BENCHES (labels, `;`-separated), RCB_SCOPE_PATHS (the affected benchmarks' own path
prefixes, for check-unit-tests.sh `--only`), RCB_SCOPE_TASKS (their task dirs). A new benchmark gets the conventional
paths from task_conventions.BENCHMARK_TABLE; an image directory of another name needs a row here (`--check` fails
until it has one).
"""
from __future__ import annotations

import functools
import re
import shlex
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
import task_conventions as tc  # noqa: E402  (BENCHMARK_TABLE: (task dir prefix, label, folder); bench_of)

# label -> the benchmark's own path prefixes besides its task dirs (tasks/<folder>/<task>/, the folder BENCHMARK_DIRS
# names; its pages tasks/<folder>/*.md are its own through owners()). Default for a label not listed with a key here: none
# (full suite). `d` = the directory name under images/.
def _own(d: str, *extra: str) -> list[str]:
    return [f"images/{d}/", *extra]


UNITS: dict[str, list[str]] = {
    "RoboTwin 2.0": _own("robotwin"),
    "DexToolBench": ["images/mujoco/"],   # serves server_mj.py
    "MuJoCo Playground": ["images/mujoco/", "embodied_first_exam/benchmarks/dextoolbench/standard/", "images/dextoolbench-files/"],   # ALOHA / Panda: DexToolBench's grader
    "KinDER": _own("kinder"),
    "HumanoidBench": _own("humanoidbench"),
    "MetaWorld+": ["images/metaworld/"],
    "RoboWits": _own("robowits"),
    "RoboPaint": _own("robopaint", "images/robopaint-pictures-files/"),
    "RoboLab": _own("robolab", "images/robolab-files/"),
    "BEHAVIOR-1K": _own("behavior", "images/behavior-files/"),
    "ManiGuard": _own("maniguard"),
    "MolmoSpaces": _own("molmospaces"),
    "MyoSuite": _own("myosuite"),
    "DexJoCo": _own("dexjoco"),
    "DexHand": _own("dexhand"),
    "VLABench": _own("vlabench", "images/vlabench-build-files/"),
    "RoboCasa-GR1": _own("robocasa-gr1", "embodied_first_exam/benchmarks/robocasa/", "images/robocasa-files/"),   # RoboCasa's service, tooling, graders
    "RoboCasa": _own("robocasa"),
    "RoboCasa365": _own("robocasa365", "embodied_first_exam/benchmarks/robocasa/", "images/robocasa-files/"),   # RoboCasa's service and graders
    "LIBERO-MAX": _own("libero-max"),
    "RoboARC": _own("roboarc", "images/roboarc-files/", "images/ikea-mujoco/", "images/ikea-mujoco-files/"),   # the IKEA suite's images keep their name
    "Embodied Olympics - Fun": _own("embodied-olympics-fun", "images/embodied-olympics-fun-files/"),
    "Embodied Olympics - Match": ["images/roboengineer-files/"],   # its simulator images are built from Embodied Olympics' tree, not here; its skill tasks use roboEngineer's files
    "BiGym 2.0": _own("bigym2"),
    "YAMLab": _own("yamlab"),
    "LabUtopia": _own("labutopia"),
    "AutoBio": _own("autobio"),
    "RoboDojo": _own("robodojo"),
    "Panda Dynamic Scenes": _own("pandadyn"),
    "AM-Bench": _own("ambench"),
    "iSpatialGym": _own("ispatialgym", "images/ispatialgym-goals-files/"),
    "roboEngineer": ["images/control-transfer/", "images/quadrotor-landing/"],
    "ABC Sim": _own("abc-sim"),
    "Embodied Safety Exam": _own("ese"),
    "EmbodiedSWE": _own("embodiedswe", "images/embodiedswe-standard-files/"),
    "HumanCLAW": _own("humanclaw"),
}
HAND_UNITS = {k: list(v) for k, v in UNITS.items()}   # the rows above, before the conventional paths below are added
# images/ directories every benchmark builds on: a change to one runs the full suite (no unit owns it)
SHARED_IMAGES = ("monitor", "standard-files", "agent-end-files")
# the benchmark's shared task code, embodied_first_exam/benchmarks/<folder>/ (embodied_first_exam/README.md), its CI
# hooks, ci_checks/benchmarks/<folder>/, and its files-only image, images/<folder>-files/ (task images and sidecars): the folder of its
# task dirs, tasks/<folder>/ (task_conventions.BENCHMARK_DIRS)
FILES_IMAGE = {"behavior-1k": "behavior-files", "embodied-safety-exam": "ese-files"}   # folders whose files image has another name
for _label, _folder in tc.BENCHMARK_DIRS.items():
    UNITS.setdefault(_label, []).extend([f"embodied_first_exam/benchmarks/{_folder}/", f"ci_checks/benchmarks/{_folder}/",
                                         f"images/{FILES_IMAGE.get(_folder, _folder + '-files')}/"])   # its files-only image
# changes that no check reads: they need only the repo-level checks (which always run). Everything not listed here and
# not a benchmark's own path runs the full suite.
NO_TASK_EFFECT = ("README.md", "CONTRIBUTING.md", "AGENTS.md", "CLAUDE.md", "LICENSE", "SECURITY.md", "CODE_OF_CONDUCT.md", "docs/protocol/running.md",
                  "docs/REVIEWING.md", ".gitignore",
                  "docs/core_set.toml")   # read by reports only; its own check (run_all.sh) and test (run_invariants.sh) always run
# tasks/dataset.toml: not committed (generated for a release; check-dataset-manifest.sh fails when it is tracked)
REPO_LEVEL_ONLY = ("tasks/dataset.toml",)
REF = re.compile(r"\b(scripts|images)/([A-Za-z0-9_.-]+)")


def git(*args: str) -> str:
    return subprocess.run(["git", "-C", str(ROOT), *args], check=True, capture_output=True, text=True).stdout


def changed_paths(diff_from: str | None, base: str) -> list[str]:
    rng = [diff_from, "HEAD"] if diff_from else [f"{base}...HEAD"]
    out = git("diff", "--name-only", "--no-renames", *rng)
    # uncommitted / untracked work counts too when run by hand (CI's checkout is clean)
    out += git("diff", "--name-only", "--no-renames", "HEAD")
    out += git("ls-files", "--others", "--exclude-standard")
    return sorted({p for p in out.splitlines() if p})


def label_of_task(name: str) -> str | None:
    return tc.bench_of(Path(name)) or None


def task_name_of(path: str) -> str | None:
    """The task dir a repository path is in: tasks/<folder>/<task>/... -> <task>; None for a path in no task dir (a
    file directly in tasks/ or in a benchmark folder). A path of the flat layout before the benchmark folders
    (tasks/<task>/..., the old side of the move in a diff) names its task one level up."""
    parts = path.split("/")
    if parts[0] != "tasks" or len(parts) < 3:
        return None
    if parts[1] in tc.BENCHMARK_DIRS.values():
        return parts[2] if len(parts) >= 4 else None
    return parts[1]


def owners(path: str) -> set[str]:
    """Every benchmark that owns `path` (images/mujoco/ is DexToolBench's and MuJoCo Playground's)."""
    parts = path.split("/")
    if len(parts) == 3 and parts[0] == "tasks" and parts[1] in tc.BENCHMARK_DIRS.values():   # a benchmark's page
        return {lab for lab, f in tc.BENCHMARK_DIRS.items() if f == parts[1]}
    if path.startswith("tasks/") and path.count("/") >= 2:   # a task's file: the task's name says its benchmark
        name = task_name_of(path)
        lab = label_of_task(name) if name else None
        return {lab} if lab else set()
    return {label for label, own in UNITS.items() if any(path.startswith(p) for p in own)}


def top(path: str) -> str | None:
    """`scripts/<x>` / `images/<x>` of a path (a file directly under scripts/: its stem with the dir, e.g. scripts/efe)."""
    m = REF.match(path)
    if not m:
        return None
    return f"{m.group(1)}/{m.group(2).removesuffix('.py').removesuffix('.sh')}"


@functools.cache
def refs_of(label: str) -> frozenset[str]:
    """`scripts/<x>` / `images/<x>` named in the .py / .sh files under a benchmark's own paths."""
    out: set[str] = set()
    for prefix in UNITS[label]:
        p = ROOT / prefix
        cands = [p] if p.is_file() else list(p.rglob("*")) if p.is_dir() else list(p.parent.glob(p.name + "*"))
        for f in cands:
            if f.is_dir():
                cands += [x for x in f.rglob("*") if x.is_file()]
            elif f.suffix in (".py", ".sh"):
                out |= {f"{a}/{b.removesuffix('.py').removesuffix('.sh')}" for a, b in REF.findall(f.read_text(errors="replace"))}
    return frozenset(out)


def scope(paths: list[str]) -> dict:
    if not paths:
        return {"scope": "none", "reason": "no changed paths", "benches": []}
    benches: set[str] = set()
    for p in paths:
        if p in NO_TASK_EFFECT or p in REPO_LEVEL_ONLY:
            continue
        own = owners(p)
        if not own or any(o not in UNITS for o in own):
            why = "not one benchmark's path" if not own else f"benchmark {sorted(own)} has no row in ci_checks/ci_scope.py UNITS"
            return {"scope": "full", "reason": f"{p}: {why}", "benches": []}
        benches |= own
    if not benches:
        return {"scope": "none", "reason": "only documents / tasks/dataset.toml changed", "benches": []}
    # benchmarks that share code with a changed one (either side names the other's scripts/<x> or images/<x>)
    tops = {t for p in paths if (t := top(p))}
    own_tops = {lab: {t for pre in UNITS[lab] if (t := top(pre))} for lab in UNITS}
    changed_tops = {t for lab in benches for t in own_tops[lab]} | tops
    for lab in UNITS:
        if lab in benches:
            continue
        mine = own_tops[lab]
        if refs_of(lab) & changed_tops or any(mine & refs_of(b) for b in list(benches)):
            benches.add(lab)
    return {"scope": "benchmarks", "reason": f"{len(paths)} changed path(s) in {len(benches)} benchmark(s)",
            "benches": sorted(benches)}


def check() -> list[str]:
    """The table against the benchmarks (task_conventions.BENCHMARK_TABLE) and the tree: every UNITS key is a label,
    every FILES_IMAGE key a folder, every hand-written path exists, and every images/<d>/ belongs to a benchmark or is
    one of SHARED_IMAGES (an image no unit owns would run the full suite on every change to it, silently)."""
    errs = []
    labels, folders = {lab for _, lab, _ in tc.BENCHMARK_TABLE}, {f for _, _, f in tc.BENCHMARK_TABLE}
    errs += [f"UNITS: {k!r} is not a benchmark label (task_conventions.BENCHMARK_TABLE)" for k in sorted(set(UNITS) - labels)]
    errs += [f"FILES_IMAGE: {k!r} is not a benchmark folder" for k in sorted(set(FILES_IMAGE) - folders)]
    errs += [f"UNITS[{k!r}]: {p} does not exist" for k, ps in HAND_UNITS.items() for p in ps if not (ROOT / p).exists()]
    errs += [f"RT_CORES_BY_README: {k!r} is not a benchmark label" for k in sorted(set(tc.RT_CORES_BY_README) - labels)]
    for d in sorted(x.name for x in (ROOT / "images").iterdir() if x.is_dir()):
        if d in SHARED_IMAGES:
            if owners(f"images/{d}/"):
                errs.append(f"images/{d}/ is in SHARED_IMAGES but {sorted(owners(f'images/{d}/'))} own it")
        elif not owners(f"images/{d}/"):
            errs.append(f"images/{d}/ belongs to no benchmark: add it to its UNITS row (or to SHARED_IMAGES)")
    return errs


def main(argv: list[str]) -> int:
    if argv == ["--check"]:
        errs = check()
        for e in errs:
            print(f"FAIL ci_scope: {e}")
        if not errs:
            print(f"ci_scope: {len(UNITS)} benchmarks, every images/ directory owned or shared")
        return 1 if errs else 0
    diff_from, base, paths = None, "origin/main", None
    i = 0
    while i < len(argv):
        a = argv[i]
        if a == "--diff-from":
            diff_from, i = argv[i + 1], i + 2
        elif a == "--base":
            base, i = argv[i + 1], i + 2
        elif a == "--paths":
            paths, i = argv[i + 1:], len(argv)
        else:
            raise SystemExit(f"usage: ci_scope.py [--diff-from REV | --base REF] [--paths P...]; got {a!r}")
    if paths is None:
        try:
            paths = changed_paths(diff_from, base)
        except subprocess.CalledProcessError as e:   # no history to diff against: be safe
            paths, err = None, (e.stderr or "").strip().splitlines()[-1:] or ["git diff failed"]
    if paths is None:
        s = {"scope": "full", "reason": f"cannot diff ({err[0]})", "benches": []}
    else:
        s = scope(paths)
    benches = s["benches"]
    prefixes = sorted({p for b in benches for p in UNITS[b]})
    tprefixes = tuple(pre for pre, lab in tc.BENCHMARKS if lab in benches)
    tasks = [t.relative_to(ROOT).as_posix() for t in tc.iter_tasks()
             if label_of_task(t.name) in benches] if tprefixes else []
    print(f"RCB_SCOPE={s['scope']}")
    print(f"RCB_SCOPE_REASON={shlex.quote(s['reason'])}")
    print(f"RCB_SCOPE_BENCHES={shlex.quote(';'.join(benches))}")
    print(f"RCB_SCOPE_PATHS={shlex.quote(' '.join(prefixes))}")
    print(f"RCB_SCOPE_TASKS={shlex.quote(' '.join(tasks))}")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))