The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.
Python ai-agentsbenchmarkcoding-agentsembodied-aievaluationharborllm-agentsrobot-learningroboticssimulation
#!/usr/bin/env python3
"""Write per-task agent results for the results site (not into the repository, PROTOCOL §1): <out>/<task>/<run id>/README.md
(+ replay.mp4, replay.last.png, replay.first.mp4 and replay.first.last.png when the trial made resets, config.json,
attempts.json). One folder per agent+model; the privileged and standard twins
each get their own.
usage: write_task_results.py <jobs_dir> <job_name> <run-id> "<agent label>" --out <dir>
config.json (docs/protocol/running.md "Results", PROTOCOL v2.1): the trial's effective configuration (scripts/task_conventions.py
effective_config: protocol version, mode, cutoff, reset budget, slowdown S, seed, network, validated flag, digest), its
result (`success`, `fail` or `not_a_result` with the reason), whether it is counted and, in the standard modes, the
first-attempt result `first_success` (PROTOCOL §4: the no-reset performance, reported beside the graded one). Results are ranked only within
one comparable configuration (task_conventions.config_key), validated and unvalidated tasks together.
attempts.json: every trial of this task x run id seen so far. docs/protocol/running.md "Infrastructure failures": an
infrastructure failure (not a result) is void and may be rerun with the same instance, seed and configuration; the voided
attempt is never counted, a result (the agent's failure or a success) is never rerun, and an attempt after a result is
flagged (`attempts_over_cap`)."""
import json, pathlib, shutil, sys, glob, datetime as dt
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import account_failure_check # noqa: E402
import task_conventions as tc # noqa: E402
import retry_policy as rp # noqa: E402
ROOT = pathlib.Path(__file__).resolve().parents[1]
OUTCOME = {"not_a_result": "infra_failure", "success": "success"} # anything else: the agent's failure (a result)
OVERRIDES = ("RCB_RESET_BUDGET", "RCB_SLOWDOWN") # run-time overrides of a task's declared settings (PROTOCOL §2)
def span(x):
try:
a = dt.datetime.fromisoformat(x["started_at"].replace("Z", "+00:00")); b = dt.datetime.fromisoformat(x["finished_at"].replace("Z", "+00:00")); return round((b - a).total_seconds())
except Exception: return None
def last_msgs(trial, k=3):
try:
d = json.load(open(glob.glob(str(trial / "agent/trajectory.json"))[0]))
return [s["message"].strip().replace("\n", " ") for s in d.get("steps", []) if s.get("source") == "agent" and (s.get("message") or "").strip()][-k:]
except Exception: return []
NOT_A_RESULT = "**not a result** — infrastructure error ({}): run it again; it is not the agent's 0"
def not_a_result(trial, r, rw):
"""Why a trial that is not a success is no result either (PROTOCOL §4, H.5: an infrastructure failure is neither a
success nor the agent's 0; only the agent's own end, its deliverable or the cutoff, AgentTimeoutError, is a result),
or None. A trial whose verifier never ran (no reward.json: `docker compose up` failed, e.g. a CPU
cap above the host's; the verifier timed out) was published as the agent's 0."""
if rw.get("infra_error") or rw.get("grader_error"):
return "`infra_error` / `grader_error` in the verifier line"
ex = (r.get("exception_info") or {}).get("exception_type") or ""
if not rw:
return "the verifier wrote no reward.json" + (f"; Harbor: `{ex}`" if ex else "")
if ex and ex != "AgentTimeoutError":
return f"Harbor ended the trial with `{ex}`"
try:
chk = account_failure_check.check(str(trial))
if chk.get("stop") and not any(w.startswith("gateway:") for w in chk.get("why", [])):
return "a model-account or configuration failure (scripts/account_failure_check.py)"
# the usage gateway died, or the trial's last model calls failed on the way (upstream unreachable,
# a stream broken off, a provider 5xx): the agent could not reach its model
gw = [w for w in chk.get("infra", []) if w.startswith("gateway:")]
if gw:
return "the model API could not be reached through the usage gateway (" + gw[0].removeprefix("gateway: ") + ")"
except Exception:
pass
return None
def load_json(p):
try:
d = json.load(open(p))
return d if isinstance(d, dict) else {}
except (OSError, ValueError):
return {}
def classify(trial, r, rw):
"""(status, reason, over the token budget): status `success`, `fail` (the agent's own result, counted) or
`not_a_result` (infrastructure: voided, never counted)."""
gw = load_json(trial / "gateway/totals.json")
over = bool(gw.get("over_token_budget")) # running.md "Official runs": over the token budget = failed
# PROTOCOL §4: infra_error (or a grader error) = not a result, neither a success nor the agent's 0; so is a trial
# the verifier never graded, or that Harbor or the model account ended (not_a_result). A verdict of success with no
# infrastructure flag stands; over the token budget is the agent's failure unless the verifier flagged infrastructure.
infra = bool(rw.get("infra_error") or rw.get("grader_error"))
# an agent-ended episode (PROTOCOL §3; reward.json ended_session_end) is graded on the state its session ended in, the
# token budget's end included: over the budget it is not failed on top
fails = over and "ended_session_end" not in rw
ok = rw.get("success") == 1 and not fails and not infra
why = None if ok or (over and not infra) else not_a_result(trial, r, rw)
return ("success" if ok else "not_a_result" if why else "fail"), why, over
def first_success(rw, mode, reset_budget=None):
"""The first attempt's result in the standard modes (PROTOCOL §4): reward.json's first_success; a verifier that does
not write it: the graded success when the trial's reset budget (its effective configuration) is 0, as no reset
could be made (whatever a service counts as `resets`), else the graded success when no reset was made, else 0 (a
success ends the trial). None when none of these is known (privileged mode, no verdict, no `resets` and a reset
budget other than 0 or unknown)."""
if mode == "privileged" or not rw or rw.get("infra_error") or rw.get("grader_error"):
return None
if rw.get("first_success") is not None:
return int(bool(rw["first_success"]))
if type(reset_budget) is int and reset_budget == 0:
return int(rw.get("success") == 1)
if rw.get("resets") is None:
return None
return int(rw.get("success") == 1 and int(rw.get("resets") or 0) == 0)
def trial_config(trial, task, r):
"""The trial's effective configuration (task_conventions.effective_config) with how its seed is known. The seed: the
runner's copy of the task the trial ran from (scripts/task_seed.py materialize writes the chosen seed as its
`instance_seed`), else the task's default seed (`seed_source = "default"`). Run-time overrides: the trial's
recorded environment. A trial of another version of the task (the digest in its lock.json is not the task's DIGEST now, nor that of the
runner's copy of it, <jobs>/<job>.copies.json) keeps
only what its name says; the rest is "legacy"."""
tdir = tc.task_dir(task, ROOT / "tasks")
hc = load_json(trial / "config.json")
env = {}
for src in ((hc.get("environment") or {}).get("env"), (hc.get("agent") or {}).get("env"), hc.get("env")):
if isinstance(src, dict):
env.update({k: str(v) for k, v in src.items() if k in OVERRIDES})
seed, source = None, "default"
p = ((hc.get("task") or {}).get("path") or "")
if p:
copy = pathlib.Path(p) if pathlib.Path(p).is_absolute() else ROOT / p
if copy.resolve() != tdir.resolve() and (copy / "task.toml").is_file():
try:
seed, source = tc.seeds_of(tc.metadata_of(copy))["default"], "runner copy"
except (ValueError, KeyError, TypeError):
pass
cfg = tc.effective_config(tdir, seed=seed, env=env)
cfg["seed_source"] = source
# the task's content digest as Harbor recorded it: <trial>/lock.json `task.digest` (Harbor's packager digest = DIGEST);
# result.json's `task_checksum` is a dirhash of the whole directory, another algorithm, never equal to DIGEST
want = (((load_json(trial / "lock.json").get("task") or {}).get("digest")) or r.get("task_checksum") or "").removeprefix("sha256:")
have = cfg.get("digest", "").removeprefix("sha256:")
cfg["digest_match"] = (want == have) if want and have else None
if cfg["digest_match"] is False: # the runner's own copy (run_agent_batch.sh: egress narrowed, CPU caps clamped) of this version
cp = load_json(trial.parent.parent / f"{trial.parent.name}.copies.json").get(task) or {}
if cp.get("copy", "").removeprefix("sha256:") == want and cp.get("source", "").removeprefix("sha256:") == have:
cfg["digest_match"] = True
if cfg["digest_match"] is False and source == "default": # another version of the task: only the name is known
for k in ("cutoff_s", "reset_budget", "slowdown", "seed", "network", "validated", "oracle", "service_protocol"):
cfg[k] = "legacy"
cfg["protocol_version"] = "legacy"
cfg["digest"] = "sha256:" + want
return cfg
def started(r):
return r.get("started_at") or ""
def record_attempt(out, trial, r, status):
"""attempts.json in the task's result folder: every trial of this task x run id, oldest first."""
f = out / "attempts.json"
led = load_json(f).get("attempts", [])
key = r.get("id") or trial.name
led = [a for a in led if a.get("id") != key]
led.append({"id": key, "trial": trial.name, "job": trial.parent.name, "started_at": started(r), "status": status,
"counted": status != "not_a_result"})
led.sort(key=lambda a: a.get("started_at") or "")
f.write_text(json.dumps({"attempts": led}, indent=1) + "\n")
return led
def main(argv):
jobs, job, slug, label = pathlib.Path(argv[1]), argv[2], argv[3], argv[4]
if "--out" not in argv: sys.exit("--out <dir> is required: results are published on the results site, not committed (PROTOCOL §1)")
out_root = pathlib.Path(argv[argv.index("--out") + 1]).resolve()
if out_root == ROOT or ROOT in out_root.parents: sys.exit("--out must be outside the repository (PROTOCOL §1)")
n = 0
trials = []
for trial in (jobs / job).glob("*__*"):
if (trial / "result.json").exists():
trials.append((trial, json.load(open(trial / "result.json"))))
for trial, r in sorted(trials, key=lambda tr: (started(tr[1]), tr[0].name)):
task = tc.canonical_name(r["task_name"].split("/")[-1]) # also pre-v1.0 jobs
tdir = tc.task_dir(task, ROOT / "tasks")
if not tdir.is_dir(): continue
rw = load_json(trial / "verifier/reward.json")
ar = r.get("agent_result") or {}; mode = tc.mode_of(tdir); limited = mode != "privileged"
out = out_root / task / slug; out.mkdir(parents=True, exist_ok=True)
status, why, over = classify(trial, r, rw)
led = record_attempt(out, trial, r, status)
rule = rp.check_attempts([{"outcome": OUTCOME.get(a["status"], "agent_failure")} for a in led])
over_cap = bool(rule)
if over_cap:
print(f"WARN {task} / {slug}: {len(led)} attempts break the rerun rule (PROTOCOL Appendix H.5: reruns only "
f"after an infrastructure failure): {'; '.join(rule)}")
counted = [a for a in led if a["counted"]]
if status == "not_a_result" and counted:
# a voided attempt never replaces a counted one: it is recorded in attempts.json only
print(f"{task} / {slug}: trial {trial.name} is not a result (voided); the counted attempt stays")
continue
cfg = trial_config(trial, task, r)
cfg.update({"run_id": slug, "job": job, "trial": trial.name, "result": status, "not_a_result_reason": why,
"counted": status != "not_a_result", "attempts": len(led), "attempts_over_cap": over_cap,
"first_success": first_success(rw, mode, cfg.get("reset_budget"))})
(out / "config.json").write_text(json.dumps(cfg, indent=1) + "\n")
for f in ("replay.mp4", "replay.last.png", "replay.first.mp4", "replay.first.last.png"):
if (trial / "verifier" / f).exists(): shutil.copy(trial / "verifier" / f, out / f)
ok = status == "success"
lines = [f"# {label} — {task}\n",
f"Run: Harbor job `{job}`, trial `{trial.name}`, {(r.get('started_at') or '')[:10]}. Mode: **{ {'standard': 'standard (robot as a service)', 'realtime': 'realtime (robot as a service, time does not stop)'}.get(mode, 'privileged')}**.\n",
"| | |", "|---|---|",
f"| result | {'**1 (success)**' if ok else NOT_A_RESULT.format(why) if why else '**0**' + (' — over the token budget' if over else ' — no trajectory handed in' if rw.get('missing_trajectory') else '')} |",
f"| configuration | {tc.config_label(cfg)}; seed {cfg.get('seed')} ({cfg.get('seed_source')}); "
f"{'validated' if cfg.get('validated') is True else 'unvalidated' if cfg.get('validated') is False else 'legacy'}"
f"{'; not in the core set (docs/core_set.toml)' if cfg.get('set') not in (None, '', 'core') else ''} (`config.json`) |",
f"| attempts | {len(led)}" + (" — **breaks the rerun rule (PROTOCOL H.5)**" if over_cap else "") + " |",
*([f"| first attempt (no reset) | {cfg['first_success']}; resets {rw.get('resets')} |"]
if cfg["first_success"] is not None else []),
f"| agent time | {span(r.get('agent_execution') or {})} s |",
f"| actions / physics steps | {rw.get('steps') or rw.get('n_actions') or '—'} / {rw.get('sim_steps') or '—'} |",
f"| tokens in / out | {ar.get('n_input_tokens')} / {ar.get('n_output_tokens')} |",
f"| verifier | `{json.dumps({k: v for k, v in rw.items() if k not in ('agent_sessions','agent_records','agent_replays')})}` |",
f"| verifier sandbox | {r.get('verifier_environment_mode')} |\n"]
if not limited and rw.get("agent_resets") is not None:
lines.append(f"Debugging footprint (self-reported by the agent's tooling log): {rw.get('agent_sessions')} sessions, {rw.get('agent_resets')} resets, {rw.get('agent_sim_steps')} physics steps.\n")
if (out / "replay.mp4").exists():
lines += ["## Replay (verifier's six-camera video)\n", f"\n", "[replay.mp4](replay.mp4) — top: world / observer / head; bottom: front / left wrist / right wrist.\n"]
if (out / "replay.first.mp4").exists():
lines += ["## First attempt (before the first reset; not graded)\n", "\n",
"[replay.first.mp4](replay.first.mp4)\n"]
msgs = last_msgs(trial)
if msgs:
lines += ["## Agent's last messages\n"] + [f"- {m[:600]}" for m in msgs] + [""]
(out / "README.md").write_text("\n".join(lines)); n += 1
print(f"{job}: wrote {n} result folders for {slug}")
if __name__ == "__main__":
main(sys.argv)