Skip to content

The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.

15.1 KB
Raw Blame
#!/usr/bin/env python3
"""Write per-task agent results for the results site (not into the repository, PROTOCOL §1): <out>/<task>/<run id>/README.md
(+ replay.mp4, replay.last.png, replay.first.mp4 and replay.first.last.png when the trial made resets, config.json,
attempts.json). One folder per agent+model; the privileged and standard twins
each get their own.
usage: write_task_results.py <jobs_dir> <job_name> <run-id> "<agent label>" --out <dir>

config.json (docs/protocol/running.md "Results", PROTOCOL v2.1): the trial's effective configuration (scripts/task_conventions.py
effective_config: protocol version, mode, cutoff, reset budget, slowdown S, seed, network, validated flag, digest), its
result (`success`, `fail` or `not_a_result` with the reason), whether it is counted and, in the standard modes, the
first-attempt result `first_success` (PROTOCOL §4: the no-reset performance, reported beside the graded one). Results are ranked only within
one comparable configuration (task_conventions.config_key), validated and unvalidated tasks together.
attempts.json: every trial of this task x run id seen so far. docs/protocol/running.md "Infrastructure failures": an
infrastructure failure (not a result) is void and may be rerun with the same instance, seed and configuration; the voided
attempt is never counted, a result (the agent's failure or a success) is never rerun, and an attempt after a result is
flagged (`attempts_over_cap`)."""
import json, pathlib, shutil, sys, glob, datetime as dt
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import account_failure_check  # noqa: E402
import task_conventions as tc  # noqa: E402
import retry_policy as rp  # noqa: E402

ROOT = pathlib.Path(__file__).resolve().parents[1]
OUTCOME = {"not_a_result": "infra_failure", "success": "success"}   # anything else: the agent's failure (a result)
OVERRIDES = ("RCB_RESET_BUDGET", "RCB_SLOWDOWN")   # run-time overrides of a task's declared settings (PROTOCOL §2)


def span(x):
    try:
        a = dt.datetime.fromisoformat(x["started_at"].replace("Z", "+00:00")); b = dt.datetime.fromisoformat(x["finished_at"].replace("Z", "+00:00")); return round((b - a).total_seconds())
    except Exception: return None


def last_msgs(trial, k=3):
    try:
        d = json.load(open(glob.glob(str(trial / "agent/trajectory.json"))[0]))
        return [s["message"].strip().replace("\n", " ") for s in d.get("steps", []) if s.get("source") == "agent" and (s.get("message") or "").strip()][-k:]
    except Exception: return []


NOT_A_RESULT = "**not a result** — infrastructure error ({}): run it again; it is not the agent's 0"


def not_a_result(trial, r, rw):
    """Why a trial that is not a success is no result either (PROTOCOL §4, H.5: an infrastructure failure is neither a
    success nor the agent's 0; only the agent's own end, its deliverable or the cutoff, AgentTimeoutError, is a result),
    or None. A trial whose verifier never ran (no reward.json: `docker compose up` failed, e.g. a CPU
    cap above the host's; the verifier timed out) was published as the agent's 0."""
    if rw.get("infra_error") or rw.get("grader_error"):
        return "`infra_error` / `grader_error` in the verifier line"
    ex = (r.get("exception_info") or {}).get("exception_type") or ""
    if not rw:
        return "the verifier wrote no reward.json" + (f"; Harbor: `{ex}`" if ex else "")
    if ex and ex != "AgentTimeoutError":
        return f"Harbor ended the trial with `{ex}`"
    try:
        chk = account_failure_check.check(str(trial))
        if chk.get("stop") and not any(w.startswith("gateway:") for w in chk.get("why", [])):
            return "a model-account or configuration failure (scripts/account_failure_check.py)"
        # the usage gateway died, or the trial's last model calls failed on the way (upstream unreachable,
        # a stream broken off, a provider 5xx): the agent could not reach its model
        gw = [w for w in chk.get("infra", []) if w.startswith("gateway:")]
        if gw:
            return "the model API could not be reached through the usage gateway (" + gw[0].removeprefix("gateway: ") + ")"
    except Exception:
        pass
    return None


def load_json(p):
    try:
        d = json.load(open(p))
        return d if isinstance(d, dict) else {}
    except (OSError, ValueError):
        return {}


def classify(trial, r, rw):
    """(status, reason, over the token budget): status `success`, `fail` (the agent's own result, counted) or
    `not_a_result` (infrastructure: voided, never counted)."""
    gw = load_json(trial / "gateway/totals.json")
    over = bool(gw.get("over_token_budget"))               # running.md "Official runs": over the token budget = failed
    # PROTOCOL §4: infra_error (or a grader error) = not a result, neither a success nor the agent's 0; so is a trial
    # the verifier never graded, or that Harbor or the model account ended (not_a_result). A verdict of success with no
    # infrastructure flag stands; over the token budget is the agent's failure unless the verifier flagged infrastructure.
    infra = bool(rw.get("infra_error") or rw.get("grader_error"))
    # an agent-ended episode (PROTOCOL §3; reward.json ended_session_end) is graded on the state its session ended in, the
    # token budget's end included: over the budget it is not failed on top
    fails = over and "ended_session_end" not in rw
    ok = rw.get("success") == 1 and not fails and not infra
    why = None if ok or (over and not infra) else not_a_result(trial, r, rw)
    return ("success" if ok else "not_a_result" if why else "fail"), why, over


def first_success(rw, mode, reset_budget=None):
    """The first attempt's result in the standard modes (PROTOCOL §4): reward.json's first_success; a verifier that does
    not write it: the graded success when the trial's reset budget (its effective configuration) is 0, as no reset
    could be made (whatever a service counts as `resets`), else the graded success when no reset was made, else 0 (a
    success ends the trial). None when none of these is known (privileged mode, no verdict, no `resets` and a reset
    budget other than 0 or unknown)."""
    if mode == "privileged" or not rw or rw.get("infra_error") or rw.get("grader_error"):
        return None
    if rw.get("first_success") is not None:
        return int(bool(rw["first_success"]))
    if type(reset_budget) is int and reset_budget == 0:
        return int(rw.get("success") == 1)
    if rw.get("resets") is None:
        return None
    return int(rw.get("success") == 1 and int(rw.get("resets") or 0) == 0)


def trial_config(trial, task, r):
    """The trial's effective configuration (task_conventions.effective_config) with how its seed is known. The seed: the
    runner's copy of the task the trial ran from (scripts/task_seed.py materialize writes the chosen seed as its
    `instance_seed`), else the task's default seed (`seed_source = "default"`). Run-time overrides: the trial's
    recorded environment. A trial of another version of the task (the digest in its lock.json is not the task's DIGEST now, nor that of the
    runner's copy of it, <jobs>/<job>.copies.json) keeps
    only what its name says; the rest is "legacy"."""
    tdir = tc.task_dir(task, ROOT / "tasks")
    hc = load_json(trial / "config.json")
    env = {}
    for src in ((hc.get("environment") or {}).get("env"), (hc.get("agent") or {}).get("env"), hc.get("env")):
        if isinstance(src, dict):
            env.update({k: str(v) for k, v in src.items() if k in OVERRIDES})
    seed, source = None, "default"
    p = ((hc.get("task") or {}).get("path") or "")
    if p:
        copy = pathlib.Path(p) if pathlib.Path(p).is_absolute() else ROOT / p
        if copy.resolve() != tdir.resolve() and (copy / "task.toml").is_file():
            try:
                seed, source = tc.seeds_of(tc.metadata_of(copy))["default"], "runner copy"
            except (ValueError, KeyError, TypeError):
                pass
    cfg = tc.effective_config(tdir, seed=seed, env=env)
    cfg["seed_source"] = source
    # the task's content digest as Harbor recorded it: <trial>/lock.json `task.digest` (Harbor's packager digest = DIGEST);
    # result.json's `task_checksum` is a dirhash of the whole directory, another algorithm, never equal to DIGEST
    want = (((load_json(trial / "lock.json").get("task") or {}).get("digest")) or r.get("task_checksum") or "").removeprefix("sha256:")
    have = cfg.get("digest", "").removeprefix("sha256:")
    cfg["digest_match"] = (want == have) if want and have else None
    if cfg["digest_match"] is False:   # the runner's own copy (run_agent_batch.sh: egress narrowed, CPU caps clamped) of this version
        cp = load_json(trial.parent.parent / f"{trial.parent.name}.copies.json").get(task) or {}
        if cp.get("copy", "").removeprefix("sha256:") == want and cp.get("source", "").removeprefix("sha256:") == have:
            cfg["digest_match"] = True
    if cfg["digest_match"] is False and source == "default":   # another version of the task: only the name is known
        for k in ("cutoff_s", "reset_budget", "slowdown", "seed", "network", "validated", "oracle", "service_protocol"):
            cfg[k] = "legacy"
        cfg["protocol_version"] = "legacy"
        cfg["digest"] = "sha256:" + want
    return cfg


def started(r):
    return r.get("started_at") or ""


def record_attempt(out, trial, r, status):
    """attempts.json in the task's result folder: every trial of this task x run id, oldest first."""
    f = out / "attempts.json"
    led = load_json(f).get("attempts", [])
    key = r.get("id") or trial.name
    led = [a for a in led if a.get("id") != key]
    led.append({"id": key, "trial": trial.name, "job": trial.parent.name, "started_at": started(r), "status": status,
                "counted": status != "not_a_result"})
    led.sort(key=lambda a: a.get("started_at") or "")
    f.write_text(json.dumps({"attempts": led}, indent=1) + "\n")
    return led


def main(argv):
    jobs, job, slug, label = pathlib.Path(argv[1]), argv[2], argv[3], argv[4]
    if "--out" not in argv: sys.exit("--out <dir> is required: results are published on the results site, not committed (PROTOCOL §1)")
    out_root = pathlib.Path(argv[argv.index("--out") + 1]).resolve()
    if out_root == ROOT or ROOT in out_root.parents: sys.exit("--out must be outside the repository (PROTOCOL §1)")
    n = 0
    trials = []
    for trial in (jobs / job).glob("*__*"):
        if (trial / "result.json").exists():
            trials.append((trial, json.load(open(trial / "result.json"))))
    for trial, r in sorted(trials, key=lambda tr: (started(tr[1]), tr[0].name)):
        task = tc.canonical_name(r["task_name"].split("/")[-1])   # also pre-v1.0 jobs
        tdir = tc.task_dir(task, ROOT / "tasks")
        if not tdir.is_dir(): continue
        rw = load_json(trial / "verifier/reward.json")
        ar = r.get("agent_result") or {}; mode = tc.mode_of(tdir); limited = mode != "privileged"
        out = out_root / task / slug; out.mkdir(parents=True, exist_ok=True)
        status, why, over = classify(trial, r, rw)
        led = record_attempt(out, trial, r, status)
        rule = rp.check_attempts([{"outcome": OUTCOME.get(a["status"], "agent_failure")} for a in led])
        over_cap = bool(rule)
        if over_cap:
            print(f"WARN {task} / {slug}: {len(led)} attempts break the rerun rule (PROTOCOL Appendix H.5: reruns only "
                  f"after an infrastructure failure): {'; '.join(rule)}")
        counted = [a for a in led if a["counted"]]
        if status == "not_a_result" and counted:
            # a voided attempt never replaces a counted one: it is recorded in attempts.json only
            print(f"{task} / {slug}: trial {trial.name} is not a result (voided); the counted attempt stays")
            continue
        cfg = trial_config(trial, task, r)
        cfg.update({"run_id": slug, "job": job, "trial": trial.name, "result": status, "not_a_result_reason": why,
                    "counted": status != "not_a_result", "attempts": len(led), "attempts_over_cap": over_cap,
                    "first_success": first_success(rw, mode, cfg.get("reset_budget"))})
        (out / "config.json").write_text(json.dumps(cfg, indent=1) + "\n")
        for f in ("replay.mp4", "replay.last.png", "replay.first.mp4", "replay.first.last.png"):
            if (trial / "verifier" / f).exists(): shutil.copy(trial / "verifier" / f, out / f)
        ok = status == "success"
        lines = [f"# {label} — {task}\n",
                 f"Run: Harbor job `{job}`, trial `{trial.name}`, {(r.get('started_at') or '')[:10]}. Mode: **{ {'standard': 'standard (robot as a service)', 'realtime': 'realtime (robot as a service, time does not stop)'}.get(mode, 'privileged')}**.\n",
                 "| | |", "|---|---|",
                 f"| result | {'**1 (success)**' if ok else NOT_A_RESULT.format(why) if why else '**0**' + (' — over the token budget' if over else ' — no trajectory handed in' if rw.get('missing_trajectory') else '')} |",
                 f"| configuration | {tc.config_label(cfg)}; seed {cfg.get('seed')} ({cfg.get('seed_source')}); "
                 f"{'validated' if cfg.get('validated') is True else 'unvalidated' if cfg.get('validated') is False else 'legacy'}"
                 f"{'; not in the core set (docs/core_set.toml)' if cfg.get('set') not in (None, '', 'core') else ''} (`config.json`) |",
                 f"| attempts | {len(led)}" + (" — **breaks the rerun rule (PROTOCOL H.5)**" if over_cap else "") + " |",
                 *([f"| first attempt (no reset) | {cfg['first_success']}; resets {rw.get('resets')} |"]
                   if cfg["first_success"] is not None else []),
                 f"| agent time | {span(r.get('agent_execution') or {})} s |",
                 f"| actions / physics steps | {rw.get('steps') or rw.get('n_actions') or '—'} / {rw.get('sim_steps') or '—'} |",
                 f"| tokens in / out | {ar.get('n_input_tokens')} / {ar.get('n_output_tokens')} |",
                 f"| verifier | `{json.dumps({k: v for k, v in rw.items() if k not in ('agent_sessions','agent_records','agent_replays')})}` |",
                 f"| verifier sandbox | {r.get('verifier_environment_mode')} |\n"]
        if not limited and rw.get("agent_resets") is not None:
            lines.append(f"Debugging footprint (self-reported by the agent's tooling log): {rw.get('agent_sessions')} sessions, {rw.get('agent_resets')} resets, {rw.get('agent_sim_steps')} physics steps.\n")
        if (out / "replay.mp4").exists():
            lines += ["## Replay (verifier's six-camera video)\n", f"![last frame](replay.last.png)\n", "[replay.mp4](replay.mp4) — top: world / observer / head; bottom: front / left wrist / right wrist.\n"]
        if (out / "replay.first.mp4").exists():
            lines += ["## First attempt (before the first reset; not graded)\n", "![last frame](replay.first.last.png)\n",
                      "[replay.first.mp4](replay.first.mp4)\n"]
        msgs = last_msgs(trial)
        if msgs:
            lines += ["## Agent's last messages\n"] + [f"- {m[:600]}" for m in msgs] + [""]
        (out / "README.md").write_text("\n".join(lines)); n += 1
    print(f"{job}: wrote {n} result folders for {slug}")


if __name__ == "__main__":
    main(sys.argv)