The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.
Python ai-agentsbenchmarkcoding-agentsembodied-aievaluationharborllm-agentsrobot-learningroboticssimulation
#!/usr/bin/env python3
"""Run a task from one of its allowed seeds (docs/protocol/README.md §2). The runner chooses the seed, never the agent:
python3 scripts/task_seed.py list tasks/<benchmark>/<task> # default, allowed, oracle
python3 scripts/task_seed.py pick tasks/<benchmark>/<task> [--rng N] # a seed drawn uniformly from `allowed`
python3 scripts/task_seed.py materialize tasks/<benchmark>/<task> --seed S --out DIR # -> DIR/<task>, run that with Harbor
`materialize` copies the task directory and, for a seed other than the default, puts that seed's frozen instance
(`tests/seeds/<S>/instance.{json,npz}` of the task that holds the pair's instances: the privileged twin of a standard
task, else the task itself) in every place the task runs and grades from: privileged `environment/` (or a scorer
sidecar's `environment/scorer/`) and `tests/`; standard and realtime `environment/sim/` and, without a twin,
their own `tests/`; each also in an `instance/` folder one level down (ManiGuard). The copy's `instance_seed` says the
seed it runs from, so a result records it, and its `[metadata.seeds]` keeps the task's allowed and oracle seeds
explicitly (their defaults follow instance_seed); its `tests/seeds/`
is removed, and every file gets its content-derived mtime (scripts/stamp_mtimes.py, issue #258). A seed without a
committed instance, of a task with `build = "runtime"`, is written instead as a file `seed` (the integer) in each of
those places, from which the task builds the seed's instance itself. A seed that is
not allowed (a list, a range `{ from, to }` or "any") is refused. `--seed draw` draws one uniformly (with `--rng`;
"any" draws from 0 to 2**31 - 1). Standard library only.
"""
from __future__ import annotations
import argparse
import random
import re
import shutil
import sys
import tomllib
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import stamp_mtimes # noqa: E402
import task_conventions as tc # noqa: E402
def seeds(task: Path) -> dict:
return tc.seeds_of(tomllib.loads((task / "task.toml").read_text()).get("metadata", {}))
def holder(task: Path) -> Path:
twin = tc.twin_of(task) # the mode registry's twin mode (PROTOCOL §3)
return twin if twin is not None and twin.is_dir() else task
def places(task: Path) -> list[str]:
"""Where a task's instance copies are (the mapping of ci_checks/check-instance-copies.sh): the copy it runs from and the
verifier's tests/ copy (a standard task without a privileged twin carries its own), each also one level down in an
`instance/` folder (ManiGuard)."""
if tc.mode_of(task) in tc.SIDECAR_MODES:
run = "environment/sim"
else:
run = "environment/scorer" if (task / "environment" / "scorer").is_dir() else "environment"
return [d for p in (run, "tests") for d in (p, f"{p}/instance")]
ANY = 2 ** 31 # "any": a seed is drawn from 0 .. ANY - 1
def pick(task: Path, rng: int | None) -> int:
a, r = seeds(task)["allowed"], random.Random(rng)
if a == "any":
return r.randrange(ANY)
if isinstance(a, dict):
return r.randint(a["from"], a["to"])
return r.choice(a)
def materialize(task: Path, seed: int, out: Path) -> Path:
sd = seeds(task)
if not tc.seed_allowed(sd, seed):
raise SystemExit(f"{task.name}: seed {seed} is not allowed ({sd['allowed']})")
dst = out / task.name
if dst.exists():
raise SystemExit(f"{dst} exists: materialize into a new directory")
shutil.copytree(task, dst, symlinks=True)
src = holder(task) / "tests" / "seeds" / str(seed)
files = [f for f in ("instance.json", "instance.npz") if (src / f).is_file()]
if seed != sd["default"] and files: # a committed instance of that seed
copied = 0
for place in places(task):
for f in ("instance.json", "instance.npz"):
if (dst / place / f).is_file():
if f not in files:
raise SystemExit(f"{task.name}: seed {seed} has no {f} (the default instance has one)")
shutil.copyfile(src / f, dst / place / f); copied += 1
if not copied: # never a copy that says seed S and runs the default instance
raise SystemExit(f"{task.name}: no instance copy found to replace with seed {seed}'s ({', '.join(places(task))})")
elif seed != sd["default"]:
if sd["build"] != "runtime":
raise SystemExit(f"{task.name}: seed {seed} has no frozen instance under {src}")
for place in places(task): # the task builds that seed's instance itself from this file
if (dst / place).is_dir():
(dst / place / "seed").write_text(f"{seed}\n")
shutil.rmtree(dst / "tests" / "seeds", ignore_errors=True)
p = dst / "task.toml"
s = p.read_text()
s, n = re.subn(r"(?m)^instance_seed = -?\d+", f"instance_seed = {seed}", s, count=1) # the one field that names it
if n != 1:
raise SystemExit(f"{task.name}: task.toml has no `instance_seed` (run scripts/task_conventions.py on the task)")
if seed != sd["default"]: # the task's seeds stay the task's, whatever instance_seed now says
s = re.sub(r"(?ms)^\[metadata\.seeds\]\n.*?(?=^\[|\Z)", "", s)
s = s.rstrip("\n") + "\n\n[metadata.seeds]\n" + f"allowed = {tc.seeds_text(sd['allowed'])}\noracle = {tc.seeds_text(sd['oracle'])}\n" \
+ (f'build = "{sd["build"]}"\n' if sd["build"] != "frozen" else "")
p.write_text(s)
if stamp_mtimes.stamp_tree(str(dst), full=True)["err"]: # Harbor builds from this copy: content-derived mtimes (issue #258)
raise SystemExit(f"{dst}: could not stamp its files (scripts/stamp_mtimes.py)")
return dst
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
sub = ap.add_subparsers(dest="cmd", required=True)
a_list = sub.add_parser("list")
a_list.add_argument("task", type=Path)
a_pick = sub.add_parser("pick")
a_pick.add_argument("task", type=Path)
a_pick.add_argument("--rng", type=int)
a_mat = sub.add_parser("materialize")
a_mat.add_argument("task", type=Path)
a_mat.add_argument("--seed", required=True, help="an allowed seed, or `draw`")
a_mat.add_argument("--rng", type=int)
a_mat.add_argument("--out", type=Path, required=True)
a = ap.parse_args()
task = a.task.resolve()
if a.cmd == "list":
print(seeds(task))
elif a.cmd == "pick":
print(pick(task, a.rng))
else:
seed = pick(task, a.rng) if a.seed == "draw" else int(a.seed)
print(materialize(task, seed, a.out.resolve()), seed)
return 0
if __name__ == "__main__":
sys.exit(main())