The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.
Python ai-agentsbenchmarkcoding-agentsembodied-aievaluationharborllm-agentsrobot-learningroboticssimulation
#!/usr/bin/env python3
"""The task list and the release manifest, generated from the tasks; neither is committed to `main`.
Every task carries its own content digest in `tasks/<benchmark>/<task>/DIGEST` (written last by scripts/task_conventions.py
`normalize`; ci_checks/check-dataset-manifest.sh recomputes it) and its own metadata in `task.toml` (mode, family,
oracle, the agent's cutoff `[agent] timeout_sec`, resources, dynamics, and the settings results are grouped by:
reset budget, slowdown, network; the `validated` marker; and `set`, core or excluded, from docs/core_set.toml). Apart from
docs/core_set.toml (a reporting list of the task dirs the headline statistics count, not the task list), nothing in the repository lists
the tasks: pull requests that change different tasks never touch a shared file (as in Terminal-Bench, whose list is its tasks/
directory). After every merge to `main` the workflow `.github/workflows/task-list.yml` writes the list with `list` and
pushes it to the branch `task-list` (never to `main`); a release attaches the same files (`.github/workflows/release.yml`).
python3 scripts/dataset_manifest.py write [--out FILE] # Harbor's dataset file for `harbor publish` (default tasks/dataset.toml, ignored by git)
python3 scripts/dataset_manifest.py list --out DIR # DIR/tasks.json, DIR/TASKS.md and DIR/dataset.toml
Standard library only.
"""
from __future__ import annotations
import json
import re
import subprocess
import sys
import tomllib
from collections import Counter, defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
import task_conventions as tc # noqa: E402 (benchmark labels, modes)
VERSION = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"]["version"] # the release tag is v<VERSION>
HEADER = f'''# Dataset manifest for embodied-first-exam/embodied-first-exam (generated by scripts/dataset_manifest.py; not committed)
# Publish using: harbor publish
[dataset]
name = "embodied-first-exam/embodied-first-exam"
version = "{VERSION}"
description = "Coding agents program robots in simulation: one replayable trajectory per frozen scene, in privileged, standard and realtime mode, across the suites of the Embodied First Exam."
authors = []
keywords = [ "robotics", "manipulation", "humanoid", "code-as-policy", "harbor",]
'''
DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
def render() -> str:
out = [HEADER]
bad = []
for t in tc.iter_tasks():
d = (tc.read_text(t / tc.DIGEST_FILE) or "").strip()
if not DIGEST.fullmatch(d):
bad.append(t.name)
continue
out.append(f'[[tasks]]\nname = "embodied-first-exam/{t.name}"\ndigest = "{d}"\n')
if bad:
raise SystemExit(f"no valid DIGEST in {len(bad)} task(s), e.g. {bad[0]} (python3 scripts/task_conventions.py tasks/<benchmark>/<task>)")
return "\n".join(out)
def resources_row(t: Path) -> dict:
"""The task's caps per container (PROTOCOL §6): {"gpu_types": [...] or None, "containers": {service: caps}}, main's
CPU / memory / disk from [environment], the rest from the compose file (GPU memory: x-gpu-memory-gb); a compose file
the caps cannot be read from: "containers" None and the "error"."""
gt = (tc.task_toml(t).get("environment") or {}).get("gpu_types")
try:
caps = tc.task_caps(t)
except ValueError as e: # a compose file the caps cannot be read from (ci_checks/check-resources.py fails it)
return {"gpu_types": gt, "containers": None, "error": f"environment/docker-compose.yaml: {e}"}
return {"gpu_types": gt,
"containers": {n: {k: c[k] for k in ("cpus", "memory_gb", "pids", "gpu_memory_gb", "disk_gb") if c.get(k) is not None}
for n, c in caps.items()}}
def entry(t: Path) -> dict:
"""One task's row of the list: what its own files say."""
md = tomllib.loads((t / "task.toml").read_text()).get("metadata", {})
# the settings results are grouped by (PROTOCOL v2.1 §2, docs/protocol/running.md "Results"): rankings compare only tasks
# and results of one configuration; `validated` is a marker, ranked together. The same fields a result records.
cfg = tc.effective_config(t)
return {"name": t.name, "benchmark": tc.bench_of(t), "mode": cfg["mode"], "base": tc.base_of(t),
"family": str(md.get("family", "")), "oracle": cfg["oracle"], "timeout_sec": cfg["cutoff_s"],
"dynamics": md.get("dynamics", ""), "protocol": cfg["service_protocol"], "simulator": md.get("simulator", ""),
"robot": md.get("robot", ""),
"reset_budget": cfg["reset_budget"], "slowdown": cfg["slowdown"], "network": cfg["network"],
"validated": cfg["validated"], "set": cfg["set"], "protocol_version": cfg["protocol_version"],
"resources": resources_row(t),
"digest": cfg["digest"]}
def _sec(v) -> str:
return "—" if v is None else f"{v} s"
def render_md(rows: list[dict], commit: str) -> str:
modes = list(tc.MODES)
short = {m: tc.MODE_REGISTRY[m]["short"] for m in modes}
by_bench: dict[str, list[dict]] = defaultdict(list)
for r in rows:
by_bench[r["benchmark"] or "(no benchmark label)"].append(r)
out = ["# Task list", "",
f"Generated from `main` at `{commit}` by `scripts/dataset_manifest.py list` (`.github/workflows/task-list.yml`); "
"do not edit: every number is the task's own (`task.toml`), the core set's from `docs/core_set.toml`. "
f"{len(rows)} task directories. Modes: " + ", ".join(f"{short[m]} = {m}" for m in modes) + ".", "",
"## Per benchmark", "",
"| Benchmark | Tasks (" + " / ".join(short[m] for m in modes) + ") | Cutoff (" + " / ".join(short[m] for m in modes)
+ ") | Oracle (full / partial / none) | Validated | Core set (tasks) | Network (allowlist / open) |", "|---|---|---|---|---|---|---|"]
for bench in sorted(by_bench, key=str.lower):
rs = by_bench[bench]
n = Counter(r["mode"] for r in rs)
cut = []
for m in modes:
vals = sorted({r["timeout_sec"] for r in rs if r["mode"] == m}, key=lambda v: (v is None, v or 0))
cut.append("—" if not vals else " / ".join(_sec(v) for v in vals) + (" (mixed)" if len(vals) > 1 else ""))
o = Counter(r["oracle"] or "none" for r in rs)
net = Counter(r["network"] for r in rs)
out.append(f"| {bench} | " + " / ".join(str(n[m]) if n[m] else "—" for m in modes) + " | " + " · ".join(cut)
+ f" | {o['full']} / {o['partial']} / {o['none']} | {sum(bool(r['validated']) for r in rs)} of {len(rs)}"
+ f" | {sum(r['set'] == 'core' for r in rs) or '—'} | {net['allowlist']} / {net['open']} |")
out += ["", "## Comparable configurations", "",
"Results are ranked only within one configuration: protocol version, mode, cutoff, reset budget, slowdown S and "
"network (`scripts/task_conventions.py` `config_key`; docs/protocol/running.md \"Results\"). Validated and unvalidated tasks "
"(no `oracle = \"full\"` and no `[metadata] validated = true`) are ranked together; the columns below count "
"them for reference. Each task's settings are in `tasks.json`.", "",
"| Mode | Cutoff | Resets | S | Network | Validated tasks | Unvalidated tasks |", "|---|---|---|---|---|---|---|"]
conf: dict[tuple, list[dict]] = defaultdict(list)
for r in rows:
conf[(r["mode"], r["timeout_sec"], r["reset_budget"], r["slowdown"], r["network"])].append(r)
for (m, c, rb, S, net), rs in sorted(conf.items(), key=lambda kv: (modes.index(kv[0][0]) if kv[0][0] in modes else 99, kv[0][1] or 0,
-1 if kv[0][2] is None else kv[0][2], kv[0][3] or 0, kv[0][4])):
v = sum(bool(r["validated"]) for r in rs)
out.append(f"| {m} | {_sec(c)} | {'—' if rb is None else rb} | {'—' if S is None else f'{S:g}'} | {net} | {v} | {len(rs) - v} |")
out += ["", "## Tasks", ""]
for bench in sorted(by_bench, key=str.lower):
out += [f"### {bench}", "", "| Task | Mode | Family | Cutoff | Oracle | Validated | Set | Network | Dynamics |",
"|---|---|---|---|---|---|---|---|---|"]
out += [f"| `{r['name']}` | {r['mode']} | {r['family'] or '—'} | {_sec(r['timeout_sec'])} | {r['oracle'] or '—'} | "
f"{'yes' if r['validated'] else 'no'} | {r['set']} | {r['network']} | {r['dynamics'] or '—'} |"
for r in sorted(by_bench[bench], key=lambda r: r["name"])]
out.append("")
return "\n".join(out)
def head_commit() -> str:
try:
return subprocess.run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], check=True, capture_output=True,
text=True).stdout.strip()
except (OSError, subprocess.CalledProcessError):
return "unknown"
def main(argv: list[str]) -> int:
if not argv or argv[0] not in ("write", "list"):
raise SystemExit(__doc__)
if argv[0] == "list":
if "--out" not in argv:
raise SystemExit("list needs --out DIR")
out = Path(argv[argv.index("--out") + 1])
out.mkdir(parents=True, exist_ok=True)
rows, commit = [entry(t) for t in tc.iter_tasks()], head_commit()
(out / "tasks.json").write_text(json.dumps({"commit": commit, "tasks": rows}, indent=1) + "\n", encoding="utf-8")
(out / "TASKS.md").write_text(render_md(rows, commit), encoding="utf-8")
(out / "dataset.toml").write_text(render(), encoding="utf-8")
print(f"wrote {out}/tasks.json, TASKS.md, dataset.toml ({len(rows)} tasks)")
return 0
out = Path(argv[argv.index("--out") + 1]) if "--out" in argv else ROOT / "tasks" / "dataset.toml"
text = render()
out.write_text(text, encoding="utf-8")
print(f"wrote {out} ({text.count('[[tasks]]')} tasks)")
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))