Skip to content

The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.

8.7 KB
Raw Blame
#!/usr/bin/env python3
"""Task-protocol conformance report (docs/protocol/README.md): the protocol version each mode may declare, the `oracle` fields,
no placeholder solve.sh, base images from the one registry, no results/ in the repository, no personal paths, internal
names or host /data/ paths. Prints
one summary line per rule with a count and a few examples; every rule fails the check. Names, modes and the other
task.toml fields are check-task-fields.sh's; that test.sh clears /logs/verifier first is check-verifier-isolation.sh's.

Usage: check-protocol.py [task dirs...]   (default: every task under tasks/ plus repository-wide checks)
"""
import hashlib
import os
import re
import subprocess
import sys
import tomllib
from collections import defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
REGISTRY = ("ghcr.io/mll-lab-nu/efe:", "ghcr.io/mll-lab-nu/efe-private:")   # efe:<name>-<version> (public), efe-private:<name>-<version> (BUILD_ONLY)
sys.path.insert(0, str(ROOT / "scripts"))
import task_conventions as tc  # noqa: E402  (the mode registry, PROTOCOL §3)
# the `protocol` values each mode may declare: standard 2.1 / 2.2 / 2.4 (2.4: the operations capability, issue #89),
# realtime 2.3 (the realtime capability, §3), privileged none
MODE_PROTOCOLS = {m: set(d["protocols"]) for m, d in tc.MODE_REGISTRY.items()}
# Tasks a benchmark exports with its own service inside (PROTOCOL §3): each such benchmark keeps a manifest,
# ci_checks/benchmarks/<bench>/exported.toml, that lists the directories (`[[tasks]] name`) and the protocol(s) each mode serves
# (`[protocols]`, a string or a list); a listed directory may declare one of its manifest's protocols for its mode. No
# per-benchmark table here.
def _own_protocols() -> dict[str, dict[str, set[str]]]:
    out: dict[str, dict[str, set[str]]] = {}
    for man in sorted((ROOT / "ci_checks/benchmarks").glob("*/exported.toml")):
        d = tomllib.loads(man.read_text())
        protos = {m: {p} if isinstance(p, str) else set(p) for m, p in d.get("protocols", {}).items()}
        for t in d.get("tasks", []):
            # a task may name its one protocol (`protocol`): then only that one of its mode's list is accepted for it
            own = t.get("protocol")
            out[t["name"]] = {m: ps & {own} for m, ps in protos.items()} if own else protos
    return out


OWN_PROTOCOLS = _own_protocols()


def own_protocols(name: str, mode: str | None) -> set[str]:
    """The protocols a benchmark's own exported task may declare instead (empty: not such a task)."""
    return OWN_PROTOCOLS.get(name, {}).get(mode or "", set())


PERSONAL_PATH = re.compile(rb"/(home|Users)/[A-Za-z0-9_.-]+/")
# the maintainers' internal user names (in paths, logins) never reach a tracked file; the author's name (capitalised) may.
# Matched by the sha256 of each lower-case word of that length, so the check itself does not publish the names.
INTERNAL_NAME_SHA256 = {"788b431f94ca453b95744650bb8de5bba2bfb7fe7194d91289ddf4929080ee96"}
INTERNAL_NAME_WORD = re.compile(rb"(?<![a-z])[a-z]{7}(?![a-z])")


def has_internal_name(data: bytes) -> bool:
    return any(hashlib.sha256(m.group()).hexdigest() in INTERNAL_NAME_SHA256 for m in INTERNAL_NAME_WORD.finditer(data))


# an absolute /data/<dir>/ path is a host path, except the sim sidecar's own data directory inside its container
# (/data/<file>, and its subdirectories calls/, live_video/, operations/)
HOST_DATA_PATH = re.compile(rb"(?<![\w./~-])/data/(?!(?:calls|live_video|operations)/)[A-Za-z0-9_.-]+/")
TEXT_SUFFIXES = {".md", ".toml", ".py", ".sh", ".yaml", ".yml", ".json", ".jsonl", ".txt", ".TXT", ".cfg", ".xml", ".urdf", ".env",
                 ".lock", ".patch", ".in", ".example", ".c", ".svg", ".sha256", ".js", ".html", ".csv", ".ipynb",
                 ".cpp", ".h", ""}

hits: dict[str, list[str]] = defaultdict(list)


def rel(p: Path) -> str:
    return os.path.relpath(p, ROOT)


def warn(rule: str, where: str) -> None:
    hits[rule].append(where)


def commands(path: Path) -> list[str]:
    """Non-empty, non-comment lines of a shell script."""
    return [ln.strip() for ln in path.read_text().splitlines() if ln.strip() and not ln.strip().startswith("#")]


def image_refs(task: Path) -> list[str]:
    refs = []
    for f in [task / "environment/Dockerfile", task / "tests/Dockerfile",                    # and a sidecar built from its own dir
              task / "environment/sim/Dockerfile", task / "environment/scorer/Dockerfile"]:
        if f.is_file():
            refs += re.findall(r"^FROM\s+(\S+)", f.read_text(), re.M)
    for f in [task / "environment/docker-compose.yaml", task / "tests/docker-compose.yaml"]:
        if f.is_file():
            refs += re.findall(r"^\s*image:\s*(\S+)", f.read_text(), re.M)
    return refs


def check_task(task: Path) -> None:
    name = task.name
    meta = tomllib.loads((task / "task.toml").read_text()).get("metadata", {})
    # the mode is the directory's whole suffix (`-realtime` does not end in `-standard`)
    try:
        named = tc.mode_of(task)
    except ValueError:
        named = None
        warn("the directory name ends in no mode (" + " | ".join(tc.MODES) + ")", name)
    mode = named
    if name.endswith("-realtime") and not name.endswith("-realtime"):
        warn("mode name `realtime` / `-realtime` (the third mode is `realtime`, PROTOCOL §3)", name)
    proto = meta.get("protocol")
    if proto is not None and proto in own_protocols(name, mode):
        pass                                   # PROTOCOL §3: the benchmark's own protocol
    else:
        for m in {mode, named} & set(tc.MODES):
            allowed = MODE_PROTOCOLS[m]
            if allowed and proto not in allowed:
                warn(f"{m}-mode `protocol` not in {sorted(allowed)}", f"{name} ({proto})")
            if not allowed and proto is not None:
                warn(f"`protocol` set on a {m} task", name)

    oracle = meta.get("oracle")
    if oracle is None:
        warn("missing `oracle` (full | partial | none)", name)
    elif oracle not in ("full", "partial", "none"):
        warn("`oracle` must be full | partial | none", name)
    if oracle in ("partial", "none") and not meta.get("oracle_evidence"):
        warn("`oracle_evidence` missing for a non-full oracle", name)
    solve = task / "solution/solve.sh"
    if solve.is_file():
        cmds = commands(solve)
        if cmds and re.fullmatch(r"exit\s+1", cmds[-1]) and not any(re.search(r"\b(python|harbor|cp|bash)\b", c) for c in cmds):
            warn("`solve.sh` is an `exit 1` placeholder (declare `oracle = \"none\"` and remove it)", name)

    for ref in image_refs(task):
        if ref.startswith("ghcr.io/") and not ref.startswith(REGISTRY):
            warn("base image outside the registry (`ghcr.io/mll-lab-nu/efe:<name>-<version>` or `efe-private:<name>-<version>`, PROTOCOL §1)", f"{name}: {ref}")

    if (task / "results").exists():
        warn("results/ in a task directory (results are published on the results site, PROTOCOL §1)", name)


def check_repository() -> None:
    if (ROOT / "results").is_dir():
        warn("top-level results/ directory (results are published on the results site, PROTOCOL §1)", "results/")
    try:
        files = subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"], capture_output=True, text=True, check=True).stdout.split("\0")
    except (OSError, subprocess.CalledProcessError):
        files = [rel(p) for p in ROOT.rglob("*") if p.is_file() and ".git" not in p.parts]
    for f in files:
        p = ROOT / f
        if f.startswith("third_party/") or not p.is_file() or p.suffix not in TEXT_SUFFIXES or p.stat().st_size > 5_000_000:
            continue
        if f == "ci_checks/check-protocol.py":
            continue
        data = p.read_bytes()
        if PERSONAL_PATH.search(data):
            warn("personal absolute path (/home/<user>/ or /Users/<user>/)", f)
        if has_internal_name(data) or has_internal_name(str(f).encode()):
            warn("internal user name", f)
        if HOST_DATA_PATH.search(data):
            warn("host path under /data/ (inside a container only the sim sidecar's /data/ is used)", f)


def main() -> int:
    args = sys.argv[1:]
    tasks = [Path(a).resolve() for a in args] or tc.iter_tasks()
    for t in tasks:
        if (t / "task.toml").is_file():
            check_task(t)
    if not args:
        check_repository()
    if not hits:
        print(f"protocol v{tc.PROTOCOL_VERSION}: {len(tasks)} task(s) conform")
        return 0
    print(f"protocol v{tc.PROTOCOL_VERSION}: violations (docs/protocol/README.md):")
    for rule, where in sorted(hits.items(), key=lambda kv: -len(kv[1])):
        examples = ", ".join(where[:3]) + (", ..." if len(where) > 3 else "")
        print(f"  FAIL {len(where):4d}  {rule}: {examples}")
    return 1


if __name__ == "__main__":
    sys.exit(main())