Skip to content

The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.

9.9 KB
Raw Blame
#!/usr/bin/env python3
"""The task list and the release manifest, generated from the tasks; neither is committed to `main`.

Every task carries its own content digest in `tasks/<benchmark>/<task>/DIGEST` (written last by scripts/task_conventions.py
`normalize`; ci_checks/check-dataset-manifest.sh recomputes it) and its own metadata in `task.toml` (mode, family,
oracle, the agent's cutoff `[agent] timeout_sec`, resources, dynamics, and the settings results are grouped by:
reset budget, slowdown, network; the `validated` marker; and `set`, core or excluded, from docs/core_set.toml). Apart from
docs/core_set.toml (a reporting list of the task dirs the headline statistics count, not the task list), nothing in the repository lists
the tasks: pull requests that change different tasks never touch a shared file (as in Terminal-Bench, whose list is its tasks/
directory). After every merge to `main` the workflow `.github/workflows/task-list.yml` writes the list with `list` and
pushes it to the branch `task-list` (never to `main`); a release attaches the same files (`.github/workflows/release.yml`).

    python3 scripts/dataset_manifest.py write [--out FILE]   # Harbor's dataset file for `harbor publish` (default tasks/dataset.toml, ignored by git)
    python3 scripts/dataset_manifest.py list --out DIR       # DIR/tasks.json, DIR/TASKS.md and DIR/dataset.toml
Standard library only.
"""
from __future__ import annotations

import json
import re
import subprocess
import sys
import tomllib
from collections import Counter, defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
import task_conventions as tc  # noqa: E402  (benchmark labels, modes)
VERSION = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"]["version"]  # the release tag is v<VERSION>
HEADER = f'''# Dataset manifest for embodied-first-exam/embodied-first-exam (generated by scripts/dataset_manifest.py; not committed)
# Publish using: harbor publish

[dataset]
name = "embodied-first-exam/embodied-first-exam"
version = "{VERSION}"
description = "Coding agents program robots in simulation: one replayable trajectory per frozen scene, in privileged, standard and realtime mode, across the suites of the Embodied First Exam."
authors = []
keywords = [ "robotics", "manipulation", "humanoid", "code-as-policy", "harbor",]
'''
DIGEST = re.compile(r"sha256:[0-9a-f]{64}")


def render() -> str:
    out = [HEADER]
    bad = []
    for t in tc.iter_tasks():
        d = (tc.read_text(t / tc.DIGEST_FILE) or "").strip()
        if not DIGEST.fullmatch(d):
            bad.append(t.name)
            continue
        out.append(f'[[tasks]]\nname = "embodied-first-exam/{t.name}"\ndigest = "{d}"\n')
    if bad:
        raise SystemExit(f"no valid DIGEST in {len(bad)} task(s), e.g. {bad[0]} (python3 scripts/task_conventions.py tasks/<benchmark>/<task>)")
    return "\n".join(out)


def resources_row(t: Path) -> dict:
    """The task's caps per container (PROTOCOL §6): {"gpu_types": [...] or None, "containers": {service: caps}}, main's
    CPU / memory / disk from [environment], the rest from the compose file (GPU memory: x-gpu-memory-gb); a compose file
    the caps cannot be read from: "containers" None and the "error"."""
    gt = (tc.task_toml(t).get("environment") or {}).get("gpu_types")
    try:
        caps = tc.task_caps(t)
    except ValueError as e:   # a compose file the caps cannot be read from (ci_checks/check-resources.py fails it)
        return {"gpu_types": gt, "containers": None, "error": f"environment/docker-compose.yaml: {e}"}
    return {"gpu_types": gt,
            "containers": {n: {k: c[k] for k in ("cpus", "memory_gb", "pids", "gpu_memory_gb", "disk_gb") if c.get(k) is not None}
                           for n, c in caps.items()}}


def entry(t: Path) -> dict:
    """One task's row of the list: what its own files say."""
    md = tomllib.loads((t / "task.toml").read_text()).get("metadata", {})
    # the settings results are grouped by (PROTOCOL v2.1 §2, docs/protocol/running.md "Results"): rankings compare only tasks
    # and results of one configuration; `validated` is a marker, ranked together. The same fields a result records.
    cfg = tc.effective_config(t)
    return {"name": t.name, "benchmark": tc.bench_of(t), "mode": cfg["mode"], "base": tc.base_of(t),
            "family": str(md.get("family", "")), "oracle": cfg["oracle"], "timeout_sec": cfg["cutoff_s"],
            "dynamics": md.get("dynamics", ""), "protocol": cfg["service_protocol"], "simulator": md.get("simulator", ""),
            "robot": md.get("robot", ""),
            "reset_budget": cfg["reset_budget"], "slowdown": cfg["slowdown"], "network": cfg["network"],
            "validated": cfg["validated"], "set": cfg["set"], "protocol_version": cfg["protocol_version"],
            "resources": resources_row(t),
            "digest": cfg["digest"]}


def _sec(v) -> str:
    return "—" if v is None else f"{v} s"


def render_md(rows: list[dict], commit: str) -> str:
    modes = list(tc.MODES)
    short = {m: tc.MODE_REGISTRY[m]["short"] for m in modes}
    by_bench: dict[str, list[dict]] = defaultdict(list)
    for r in rows:
        by_bench[r["benchmark"] or "(no benchmark label)"].append(r)
    out = ["# Task list", "",
           f"Generated from `main` at `{commit}` by `scripts/dataset_manifest.py list` (`.github/workflows/task-list.yml`); "
           "do not edit: every number is the task's own (`task.toml`), the core set's from `docs/core_set.toml`. "
           f"{len(rows)} task directories. Modes: " + ", ".join(f"{short[m]} = {m}" for m in modes) + ".", "",
           "## Per benchmark", "",
           "| Benchmark | Tasks (" + " / ".join(short[m] for m in modes) + ") | Cutoff (" + " / ".join(short[m] for m in modes)
           + ") | Oracle (full / partial / none) | Validated | Core set (tasks) | Network (allowlist / open) |", "|---|---|---|---|---|---|---|"]
    for bench in sorted(by_bench, key=str.lower):
        rs = by_bench[bench]
        n = Counter(r["mode"] for r in rs)
        cut = []
        for m in modes:
            vals = sorted({r["timeout_sec"] for r in rs if r["mode"] == m}, key=lambda v: (v is None, v or 0))
            cut.append("—" if not vals else " / ".join(_sec(v) for v in vals) + (" (mixed)" if len(vals) > 1 else ""))
        o = Counter(r["oracle"] or "none" for r in rs)
        net = Counter(r["network"] for r in rs)
        out.append(f"| {bench} | " + " / ".join(str(n[m]) if n[m] else "—" for m in modes) + " | " + " · ".join(cut)
                   + f" | {o['full']} / {o['partial']} / {o['none']} | {sum(bool(r['validated']) for r in rs)} of {len(rs)}"
                   + f" | {sum(r['set'] == 'core' for r in rs) or '—'} | {net['allowlist']} / {net['open']} |")
    out += ["", "## Comparable configurations", "",
            "Results are ranked only within one configuration: protocol version, mode, cutoff, reset budget, slowdown S and "
            "network (`scripts/task_conventions.py` `config_key`; docs/protocol/running.md \"Results\"). Validated and unvalidated tasks "
            "(no `oracle = \"full\"` and no `[metadata] validated = true`) are ranked together; the columns below count "
            "them for reference. Each task's settings are in `tasks.json`.", "",
            "| Mode | Cutoff | Resets | S | Network | Validated tasks | Unvalidated tasks |", "|---|---|---|---|---|---|---|"]
    conf: dict[tuple, list[dict]] = defaultdict(list)
    for r in rows:
        conf[(r["mode"], r["timeout_sec"], r["reset_budget"], r["slowdown"], r["network"])].append(r)
    for (m, c, rb, S, net), rs in sorted(conf.items(), key=lambda kv: (modes.index(kv[0][0]) if kv[0][0] in modes else 99, kv[0][1] or 0,
                                                                 -1 if kv[0][2] is None else kv[0][2], kv[0][3] or 0, kv[0][4])):
        v = sum(bool(r["validated"]) for r in rs)
        out.append(f"| {m} | {_sec(c)} | {'—' if rb is None else rb} | {'—' if S is None else f'{S:g}'} | {net} | {v} | {len(rs) - v} |")
    out += ["", "## Tasks", ""]
    for bench in sorted(by_bench, key=str.lower):
        out += [f"### {bench}", "", "| Task | Mode | Family | Cutoff | Oracle | Validated | Set | Network | Dynamics |",
                "|---|---|---|---|---|---|---|---|---|"]
        out += [f"| `{r['name']}` | {r['mode']} | {r['family'] or '—'} | {_sec(r['timeout_sec'])} | {r['oracle'] or '—'} | "
                f"{'yes' if r['validated'] else 'no'} | {r['set']} | {r['network']} | {r['dynamics'] or '—'} |"
                for r in sorted(by_bench[bench], key=lambda r: r["name"])]
        out.append("")
    return "\n".join(out)


def head_commit() -> str:
    try:
        return subprocess.run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], check=True, capture_output=True,
                              text=True).stdout.strip()
    except (OSError, subprocess.CalledProcessError):
        return "unknown"


def main(argv: list[str]) -> int:
    if not argv or argv[0] not in ("write", "list"):
        raise SystemExit(__doc__)
    if argv[0] == "list":
        if "--out" not in argv:
            raise SystemExit("list needs --out DIR")
        out = Path(argv[argv.index("--out") + 1])
        out.mkdir(parents=True, exist_ok=True)
        rows, commit = [entry(t) for t in tc.iter_tasks()], head_commit()
        (out / "tasks.json").write_text(json.dumps({"commit": commit, "tasks": rows}, indent=1) + "\n", encoding="utf-8")
        (out / "TASKS.md").write_text(render_md(rows, commit), encoding="utf-8")
        (out / "dataset.toml").write_text(render(), encoding="utf-8")
        print(f"wrote {out}/tasks.json, TASKS.md, dataset.toml ({len(rows)} tasks)")
        return 0
    out = Path(argv[argv.index("--out") + 1]) if "--out" in argv else ROOT / "tasks" / "dataset.toml"
    text = render()
    out.write_text(text, encoding="utf-8")
    print(f"wrote {out} ({text.count('[[tasks]]')} tasks)")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))