Skip to content

The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.

35.3 KB
Raw Blame
#!/usr/bin/env python3
"""Runner-side tools, standard library and bash only:

- scripts/write_task_results.py publishes a trial the verifier never graded (no reward.json), one Harbor ended with an
  exception other than the agent's cutoff (AgentTimeoutError), or one stopped by a model-account failure as "not a
  result", never as the agent's 0; a verdict, the cutoff and the token budget stay results;
- scripts/clamp_cpus.py clamps every CPU cap of a task copy to the host's count (running.md "Resources") and nothing else, and
  refuses the repository's own tasks;
- scripts/run_agent_batch.sh runs every job from its own copy of the task, every file stamped with its content's mtime
  (scripts/stamp_mtimes.py, issue #258), with a CPU cap above HOST_CPUS clamped in that copy (a stand-in `harbor` on
  PATH records what it was given), removes the copy afterwards, and leaves tasks/ unchanged;
- a trial whose last model calls failed at the usage gateway, or whose job the gateway died in, is not a
  result; with GATEWAY=1 the runner restarts a dead gateway before the next job."""
import functools
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
CLAMP = [sys.executable, str(ROOT / "scripts/clamp_cpus.py")]
sys.path.insert(0, str(ROOT / "scripts"))
import task_conventions as tc  # noqa: E402  (task_dir / iter_tasks: the task tree)
TASK = "robotwin_2_0_stack_blocks_two-standard"          # any task of the repository (write_task_results looks it up)
BIG = "behavior-boxing-food-after-dinner-i00"             # its standard twin's sidecar caps are 12 CPUs
HB = "humanoidbench-loco-walk-i00-privileged"            # [environment] cpus = 16
SMALL = "robotwin_2_0_stack_blocks_two"                  # caps of at most 8
RESETS = "dexhand-assemble-hamburger-i00-standard"       # a standard task with a reset budget (20)
fails = []


def check(cond, what):
    if not cond:
        fails.append(what)
        print("FAIL", what)


def result_line(td: Path, result: dict, reward=None, gateway=None, codex=None, down=False) -> str:
    jobs = td / "jobs"; job = f"j{len(list(jobs.glob('*'))) if jobs.exists() else 0}"
    trial = jobs / job / f"{TASK}__abc"
    trial.mkdir(parents=True)
    if down:                                   # run_agent_batch.sh found the usage gateway dead after the job
        (jobs / job / "gateway_down.json").write_text("{}")
    (trial / "result.json").write_text(json.dumps({"task_name": TASK, **result}))
    if reward is not None:
        (trial / "verifier").mkdir()
        (trial / "verifier/reward.json").write_text(reward if isinstance(reward, str) else json.dumps(reward))
    if gateway is not None:
        (trial / "gateway").mkdir()
        (trial / "gateway/totals.json").write_text(json.dumps(gateway))
    if codex is not None:
        (trial / "agent").mkdir()
        (trial / "agent/codex.txt").write_text(codex)
    out = td / "out" / job
    r = subprocess.run([sys.executable, str(ROOT / "scripts/write_task_results.py"), str(jobs), job, "codex-x-high", "label",
                        "--out", str(out)], capture_output=True, text=True)
    check(r.returncode == 0, f"write_task_results.py exited {r.returncode}: {r.stderr[-500:]}")
    readme = (out / TASK / "codex-x-high" / "README.md").read_text()
    return next(l for l in readme.splitlines() if l.startswith("| result |"))


def test_results():
    with tempfile.TemporaryDirectory() as td:
        td = Path(td)
        timeout = {"exception_info": {"exception_type": "AgentTimeoutError"}}
        cases = [
            ("a success", {}, {"success": 1}, None, None, "**1 (success)**"),
            ("the agent's 0", {}, {"success": 0}, None, None, "**0**"),
            ("0 at the cutoff", timeout, {"success": 0}, None, None, "**0**"),
            ("a success at the cutoff", timeout, {"success": 1}, None, None, "**1 (success)**"),
            ("over the token budget", {}, {"success": 1}, {"over_token_budget": True}, None, "over the token budget"),
            ("infra_error", {}, {"success": 0, "infra_error": 1}, None, None, "not a result"),
            ("grader_error", timeout, {"success": 0, "grader_error": 1}, None, None, "not a result"),
            ("compose up failed, no reward.json",
             {"exception_info": {"exception_type": "RuntimeError", "exception_message": "docker compose up failed: range of CPUs is from 0.01 to 8.00"}},
             None, None, None, "not a result"),
            ("no reward.json, no exception", {}, None, None, None, "not a result"),
            ("no reward.json at the cutoff", timeout, None, None, None, "not a result"),
            ("a corrupt reward.json", {}, '{"success": 0, "inf', None, None, "not a result"),
            ("the verifier timed out after a reward", {"exception_info": {"exception_type": "VerifierTimeoutError"}}, {"success": 0}, None, None, "not a result"),
            ("the harness died", {"exception_info": {"exception_type": "NonZeroAgentExitCodeError"}}, {"success": 0}, None, None, "not a result"),
            ("a model-account failure", {}, {"success": 0}, None,
             '{"type": "turn.failed", "error": {"message": "unexpected status 429 Too Many Requests: usage limit reached"}}\n', "not a result"),
            ("a success the account check would flag", {}, {"success": 1}, None,
             '{"type": "turn.failed", "error": {"message": "unexpected status 429 Too Many Requests"}}\n', "**1 (success)**"),
            ("over the budget without a reward", {}, None, {"over_token_budget": True}, None, "over the token budget"),
            ("over the budget, infra_error", {}, {"success": 0, "infra_error": 1}, {"over_token_budget": True}, None, "not a result"),
        ]
        # the model API unreachable through the usage gateway at the end of the trial: not the agent's 0
        gw = lambda n, last="unreachable: ConnectionRefusedError": {"requests": 3, "input": 10, "output": 2, "upstream_failures": n + 1,
                                                                    "failures_in_a_row": n, "last_upstream_failure": last}
        cases += [
            ("the gateway's upstream unreachable at the end", {}, {"success": 0}, gw(6), None, "not a result"),
            ("a stream broken off at the end, at the cutoff", timeout, {"success": 0}, gw(1, "stream broken off: TimeoutError"), None,
             "not a result"),
            ("gateway failures the agent got past", {}, {"success": 0}, gw(0), None, "**0**"),
            ("a success after gateway failures", {}, {"success": 1}, gw(2), None, "**1 (success)**"),
            ("over the budget with gateway failures", {}, {"success": 0}, {**gw(4), "over_token_budget": 1}, None, "over the token budget"),
        ]
        for name, result, reward, gateway, codex, want in cases:
            line = result_line(td, result, reward, gateway, codex)
            check(want in line, f"write_task_results: {name}: {line}")
            if want != "not a result":
                check("not a result" not in line, f"write_task_results: {name} is a result: {line}")
        line = result_line(td, {}, {"success": 0}, down=True)
        check("not a result" in line and "gateway was down" in line, f"write_task_results: the gateway down after the job: {line}")
        line = result_line(td, {"exception_info": {"exception_type": "EnvironmentStartTimeoutError"}})
        check("EnvironmentStartTimeoutError" in line and "no reward.json" in line, f"the reason names Harbor's exception: {line}")


UNVALIDATED = "humanoidbench-loco-walk-i00-realtime"   # oracle = "none": ranked with validated tasks, marked
VALIDATED_RT = "humanoidbench-loco-sit-simple-i00-realtime"   # oracle = "full", the same configuration as UNVALIDATED
WTR = [sys.executable, str(ROOT / "scripts/write_task_results.py")]


def make_trial(jobs: Path, job: str, name: str, task: str, started: str, reward=None, result=None, config=None) -> Path:
    t = jobs / job / f"{task}__{name}"
    t.mkdir(parents=True)
    (t / "result.json").write_text(json.dumps({"task_name": task, "id": name, "started_at": started, **(result or {})}))
    if reward is not None:
        (t / "verifier").mkdir()
        (t / "verifier/reward.json").write_text(json.dumps(reward))
    if config is not None:
        (t / "config.json").write_text(json.dumps(config))
    return t


def write(jobs: Path, job: str, out: Path) -> subprocess.CompletedProcess:
    r = subprocess.run(WTR + [str(jobs), job, "codex-x-high", "label", "--out", str(out)], capture_output=True, text=True)
    check(r.returncode == 0, f"write_task_results.py exited {r.returncode}: {r.stderr[-500:]}")
    return r


def test_config_and_attempts():
    """PROTOCOL v2.1 / docs/protocol/running.md: every result records its effective configuration (config.json); an infrastructure
    failure (not a result) is voided and never counted, a rerun replaces it, a voided attempt never replaces a counted
    one, an attempt after a result is flagged; a trial of another task version is legacy."""
    infra = {"success": 0, "infra_error": 1}
    with tempfile.TemporaryDirectory() as td:
        td = Path(td); jobs, out = td / "jobs", td / "out"
        dg = (tc.task_dir(TASK) / "DIGEST").read_text().strip().removeprefix("sha256:")
        make_trial(jobs, "a", "t1", TASK, "2026-10-14T01:00:00Z", infra, {"task_checksum": dg})
        make_trial(jobs, "a", "t2", TASK, "2026-10-14T02:00:00Z", {"success": 1}, {"task_checksum": dg})
        write(jobs, "a", out)
        res = out / TASK / "codex-x-high"
        cfg = json.loads((res / "config.json").read_text())
        for k, v in {"protocol_version": tc.PROTOCOL_VERSION, "mode": "standard", "network": "allowlist", "validated": True, "result": "success",
                     "counted": True, "attempts": 2, "attempts_over_cap": False, "seed_source": "default", "digest_match": True,
                     "trial": f"{TASK}__t2"}.items():
            check(cfg.get(k) == v, f"config.json {k} = {cfg.get(k)!r}, want {v!r}")
        for k in ("cutoff_s", "reset_budget", "slowdown", "seed", "digest"):
            check(k in cfg, f"config.json has no {k}")
        att = json.loads((res / "attempts.json").read_text())["attempts"]
        check([a["counted"] for a in att] == [False, True], f"attempts.json: the infra attempt is counted: {att}")
        check(f"| configuration | protocol {tc.PROTOCOL_VERSION}, standard" in (res / "README.md").read_text(), "README has no configuration row")
        # a later infrastructure failure of the same task x run id never replaces the counted result, and an attempt after a result is flagged
        make_trial(jobs, "b", "t3", TASK, "2026-10-14T03:00:00Z", infra, {"task_checksum": dg})
        r = write(jobs, "b", out)
        cfg = json.loads((res / "config.json").read_text())
        check(cfg.get("trial") == f"{TASK}__t2" and cfg.get("result") == "success", f"a voided attempt replaced the result: {cfg}")
        check("WARN" in r.stdout and "3 attempts" in r.stdout, f"no WARN for 3 attempts: {r.stdout}")
        check(len(json.loads((res / "attempts.json").read_text())["attempts"]) == 3, "attempts.json does not list the third attempt")
        # only infrastructure failures: the folder shows the latest as not a result, not counted
        make_trial(jobs, "c", "u1", f"{SMALL}-privileged", "2026-10-14T01:00:00Z", infra)
        write(jobs, "c", out)
        cfg = json.loads((out / f"{SMALL}-privileged" / "codex-x-high" / "config.json").read_text())
        check(cfg.get("result") == "not_a_result" and cfg.get("counted") is False and cfg.get("not_a_result_reason"),
              f"an infrastructure failure is counted: {cfg}")
        # a trial of another version of the task (its checksum is not today's DIGEST): legacy fields, still written
        make_trial(jobs, "d", "l1", f"{SMALL}-standard", "2026-10-14T01:00:00Z", {"success": 0}, {"task_checksum": "0" * 64})
        write(jobs, "d", out)
        cfg = json.loads((out / f"{SMALL}-standard" / "codex-x-high" / "config.json").read_text())
        check(cfg.get("cutoff_s") == "legacy" and cfg.get("digest_match") is False and cfg.get("mode") == "standard"
              and cfg.get("result") == "fail", f"legacy trial: {cfg}")
        jobs3, out3 = td / "jobs3", td / "out3"   # apart from the attempts above
        # Harbor 0.23: result.json's task_checksum is a dirhash (never DIGEST); lock.json's task.digest is DIGEST
        tl = make_trial(jobs3, "f", "k1", TASK, "2026-10-14T05:00:00Z", {"success": 0}, {"task_checksum": "1" * 64})
        (tl / "lock.json").write_text(json.dumps({"task": {"name": TASK, "digest": "sha256:" + dg}}))
        write(jobs3, "f", out3)
        cfg = json.loads((out3 / TASK / "codex-x-high" / "config.json").read_text())
        check(cfg.get("digest_match") is True and cfg.get("cutoff_s") != "legacy", f"lock.json digest = DIGEST, still legacy: {cfg}")
        # a trial of the runner's changed copy (run_agent_batch.sh records <job>.copies.json): this version, not legacy
        tl = make_trial(jobs3, "g", "k2", TASK, "2026-10-14T06:00:00Z", {"success": 0}, {"task_checksum": "1" * 64})
        (tl / "lock.json").write_text(json.dumps({"task": {"name": TASK, "digest": "sha256:" + "2" * 64}}))
        (jobs3 / "g.copies.json").write_text(json.dumps({TASK: {"copy": "sha256:" + "2" * 64, "source": "sha256:" + dg}}))
        write(jobs3, "g", out3)
        cfg = json.loads((out3 / TASK / "codex-x-high" / "config.json").read_text())
        check(cfg.get("digest_match") is True and cfg.get("cutoff_s") != "legacy", f"a runner copy's trial is legacy: {cfg}")
        # a legacy result.json without a checksum: derived from task.toml as it is
        make_trial(jobs, "e", "l2", UNVALIDATED, "2026-10-14T01:00:00Z", {"success": 0})
        write(jobs, "e", out)
        cfg = json.loads((out / UNVALIDATED / "codex-x-high" / "config.json").read_text())
        check(cfg.get("validated") is False and cfg.get("slowdown") is not None and cfg.get("digest_match") is None,
              f"an unvalidated realtime task without a checksum: {cfg}")
        # the seed the runner chose: its copy of the task says it (scripts/task_seed.py materialize)
        copy = td / "copy" / TASK
        shutil.copytree(tc.task_dir(TASK), copy)
        s = (copy / "task.toml").read_text()
        (copy / "task.toml").write_text(re.sub(r"(?m)^instance_seed = \d+", "instance_seed = 4242", s, count=1))
        make_trial(jobs, "f", "s1", TASK, "2026-10-14T01:00:00Z", {"success": 0}, None, {"task": {"path": str(copy)}})
        write(jobs, "f", td / "out2")
        cfg = json.loads((td / "out2" / TASK / "codex-x-high" / "config.json").read_text())
        check(cfg.get("seed") == 4242 and cfg.get("seed_source") == "runner copy", f"the runner's seed: {cfg}")


def test_first_attempt():
    """PROTOCOL §4: the standard modes report the first-attempt (no-reset) result beside the graded one: reward.json's
    first_success; else, on a task whose reset budget is 0, the graded success (no reset could be made, whatever the
    service counts as `resets`); else the graded success when no reset was made (no `resets`: unknown). The first
    attempt's video is published next to the graded one; privileged results and infrastructure failures carry none."""
    with tempfile.TemporaryDirectory() as td:
        td = Path(td); jobs, out = td / "jobs", td / "out"
        small, rst = f"{SMALL}-standard", RESETS                     # reset budget 0 / a task that grants resets
        cases = {"j1": (rst, {"success": 1, "resets": 0}, 1), "j2": (rst, {"success": 1, "resets": 2}, 0),
                 "j3": (rst, {"success": 1, "resets": 2, "first_success": 0}, 0), "j4": (rst, {"success": 0, "resets": 0}, 0),
                 "j5": (rst, {"success": 1}, None), "j6": (rst, {"success": 0, "infra_error": 1, "resets": 0}, None),
                 "j7": (small, {"success": 1}, 1), "j8": (small, {"success": 0}, 0), "j9": (small, {"success": 1, "resets": 1}, 1)}
        for job, (task, reward, want) in cases.items():
            t = make_trial(jobs, job, "f1", task, "2026-10-14T01:00:00Z", reward)
            if job == "j2":
                (t / "verifier/replay.first.mp4").write_bytes(b"mp4")
            res = out / job
            write(jobs, job, res)
            cfg = json.loads((res / task / "codex-x-high" / "config.json").read_text())
            check(cfg.get("first_success") == want, f"{job} {task} {reward}: first_success {cfg.get('first_success')!r}, want {want!r}")
            readme = (res / task / "codex-x-high" / "README.md").read_text()
            check(("| first attempt (no reset) |" in readme) == (want is not None), f"{job}: first-attempt row: {readme[:400]}")
        check((out / "j2" / rst / "codex-x-high" / "replay.first.mp4").exists(), "replay.first.mp4 not published")
        make_trial(jobs, "p1", "f1", f"{SMALL}-privileged", "2026-10-14T01:00:00Z", {"success": 1, "resets": 0})
        write(jobs, "p1", out / "p1")
        cfg = json.loads((out / "p1" / f"{SMALL}-privileged" / "codex-x-high" / "config.json").read_text())
        check(cfg.get("first_success") is None, f"a privileged result has first_success {cfg.get('first_success')!r}")


def summary_code() -> str:
    s = (ROOT / "scripts/run_agent_batch.sh").read_text()
    m = re.search(r"(?ms)^summary\(\) \{.*?<<'PY'\n(.*?)^PY$", s)
    check(m is not None, "run_agent_batch.sh has no summary() heredoc")
    return m.group(1) if m else ""


def test_summary_grouping():
    """run_agent_batch.sh's summary: one table per comparable configuration, unvalidated tasks ranked in it (marked),
    voided trials not counted in a success rate."""
    with tempfile.TemporaryDirectory() as td:
        td = Path(td); jobs = td / "jobs"
        make_trial(jobs, "bx-standard-a", "t1", TASK, "2026-10-14T01:00:00Z", {"success": 0, "infra_error": 1})
        make_trial(jobs, "bx-standard-a", "t2", TASK, "2026-10-14T02:00:00Z", {"success": 1})
        make_trial(jobs, "bx-privileged-a", "p1", f"{SMALL}-privileged", "2026-10-14T01:00:00Z", {"success": 0})
        make_trial(jobs, "bx-realtime-b", "r1", UNVALIDATED, "2026-10-14T01:00:00Z", {"success": 1})
        make_trial(jobs, "bx-realtime-b", "r2", VALIDATED_RT, "2026-10-14T01:00:00Z", {"success": 0})
        # a core set naming the fixed tasks here (the real list changes over time); test_core_set covers the marking
        sys.path.insert(0, str(ROOT / "scripts"))
        import task_conventions as tc
        (td / "core_set.toml").write_text("".join(
            f'[[task]]\ntask = "{tc.base_of(Path(t))}"\ndefault_mode = "{tc.mode_of(Path(t))}"\n' for t in
            (TASK, f"{SMALL}-privileged", UNVALIDATED, VALIDATED_RT)))
        r = subprocess.run([sys.executable, "-", str(jobs), "bx", str(ROOT)], input=summary_code(), capture_output=True, text=True,
                           env={**os.environ, "RCB_CORE_SET_FILE": str(td / "core_set.toml")})
        check(r.returncode == 0, f"summary failed: {r.stderr[-800:]}")
        out = r.stdout
        check("## Unvalidated" not in out and out.count(f"\n## protocol {tc.PROTOCOL_VERSION}, ") == 3, f"not one table per configuration:\n{out}")
        rt = out[out.index("realtime, "):]
        check(f"| {UNVALIDATED} (unvalidated) |" in rt and f"| {VALIDATED_RT} |" in rt,
              f"the unvalidated task is not ranked (marked) with the validated one of its configuration:\n{out}")
        check("Success (results only): 1 / 2 (validated tasks: 0 / 1); 0 voided." in rt, f"the shared table's rate:\n{out}")
        check("Success (results only): 1 / 1; 1 voided." in out, f"a voided trial counted in the success rate:\n{out}")
        check("not a result (voided" in out, f"the voided trial is not marked:\n{out}")


def test_core_set():
    """PROTOCOL §4: the core set (docs/core_set.toml: task + default_mode) is valid, marks every mode dir of its listed
    tasks `core` and every other dir `excluded` (recorded with the result as `set`, never part of config_key), and the
    batch summary marks the trials outside it and leaves them out of the rate."""
    sys.path.insert(0, str(ROOT / "scripts"))
    import task_conventions as tc
    check(not tc.check_core_set(), f"docs/core_set.toml: {tc.check_core_set()[:3]}")   # this test is in run_invariants.sh
    names = [p.name for p in tc.iter_tasks()]
    cs = tc.core_set()
    def entry(t, **k):                                   # a valid entry for task t, with the fields in k changed
        f = {"task": t, "default_mode": cs.get(t, "standard"), **k}
        return "[[task]]\n" + "".join(f'{n} = "{v}"\n' for n, v in f.items())
    ANY, ANY2 = sorted(cs)[:2]
    with tempfile.TemporaryDirectory() as td:
        bad = Path(td) / "core_set.toml"
        bad.write_text(entry("behavior-no-such-task-i00") + entry(ANY, default_mode="x") + entry(ANY2, luna="failed")
                       + entry("../tasks/" + ANY) + entry(f"{ANY}-{cs[ANY]}"))
        found = tc.check_core_set(path=bad)
        check(len(found) == 5, f"check_core_set: want a missing dir, a wrong mode, an unknown field, a path and a dir name, got {found}")
        (Path(td) / "twice.toml").write_text(entry(ANY) + entry(ANY))
        check(any("twice" in x for x in tc.check_core_set(path=Path(td) / "twice.toml")), "a task listed twice is not found")
        check(tc.check_core_set(path=Path(td) / "none.toml") != [], "a missing list passes")
        (Path(td) / "toplevel.toml").write_text('generated_from = "x"\n' + entry(ANY))
        check(any("top-level" in x for x in tc.check_core_set(path=Path(td) / "toplevel.toml")), "an unknown top-level key passes")
    core = set(tc.core_tasks())
    check(len(core) == len(cs) and all(tc.mode_of(Path(d)) == cs[tc.base_of(Path(d))] for d in core),
          "core_tasks(): not one default-mode dir per core task")
    # a listed standard dir and an unlisted one in the same configuration (one summary table); the list changes over
    # time, so both are looked up
    key = functools.lru_cache(maxsize=None)(lambda t: tc.config_key(tc.effective_config(tc.task_dir(t))))
    IN = OUT = None
    for c in (t for t in names if t.endswith("-standard") and tc.base_of(Path(t)) in cs):
        same = sorted((t for t in names if t.endswith("-standard") and tc.base_of(Path(t)) not in cs),
                      key=lambda t: t.split("-")[0] != c.split("-")[0])[:200]
        OUT = next((t for t in same if key(t) == key(c)), None)
        if OUT:
            IN = c
            break
    if IN is None:
        print("note: no core standard dir with an excluded one of the same configuration; the core-set test is skipped")
        return
    check(tc.set_of(tc.task_dir(IN)) == "core" and tc.set_of(tc.task_dir(OUT)) == "excluded",
          f"set_of: {IN} {tc.set_of(tc.task_dir(IN))}, {OUT} {tc.set_of(tc.task_dir(OUT))}")
    a, b = tc.effective_config(tc.task_dir(IN)), tc.effective_config(tc.task_dir(OUT))
    check(a.get("set") == "core" and b.get("set") == "excluded", f"effective_config set: {a.get('set')}, {b.get('set')}")
    check(tc.config_key(a) == tc.config_key(dict(a, set="excluded")), "the set is part of config_key")
    with tempfile.TemporaryDirectory() as td:
        jobs = Path(td) / "jobs"
        make_trial(jobs, "bx-standard-a", "e1", OUT, "2026-10-14T01:00:00Z", {"success": 1})
        make_trial(jobs, "bx-standard-a", "c1", IN, "2026-10-14T02:00:00Z", {"success": 0})
        r = subprocess.run([sys.executable, "-", str(jobs), "bx", str(ROOT)], input=summary_code(), capture_output=True, text=True)
        check(r.returncode == 0, f"summary failed: {r.stderr[-800:]}")
        check(re.search(rf"\| {OUT}( \(unvalidated\))? \(not core\) \|", r.stdout) is not None,
              f"the trial outside the core set is not marked:\n{r.stdout}")
        check(re.search(r"Success \(results only\): 0 / 1( \([^)]*\))?; 0 voided; 1 result\(s\) of tasks outside the core set not counted\.",
                        r.stdout) is not None,
              f"the trial outside the core set counted in the rate:\n{r.stdout}")


def test_benchmarks():
    """Every task dir has its benchmark label, the label results and reports are grouped by (dataset_manifest.py,
    the pool): the first BENCHMARKS prefix the name starts with. The real tree is clean; a name no prefix claims is
    found."""
    sys.path.insert(0, str(ROOT / "scripts"))
    import task_conventions as tc
    bad = tc.check_benchmarks()
    check(not bad, f"benchmark labels: {bad[:5]}")
    check(tc.bench_of(Path("embodied-olympics-fun-sprint-i00")) == "Embodied Olympics - Fun"
          and tc.bench_of(Path("embodied-olympics-match-tabletennis-match-i04-realtime")) == "Embodied Olympics - Match"
          and tc.bench_of(Path("roboolympics-sprint-i00-standard")) == ""
          and tc.bench_of(Path("robocasa-gr1-x-i00-standard")) == "RoboCasa-GR1", "bench_of: the first matching prefix")
    with tempfile.TemporaryDirectory() as td:
        for n in ("embodied-olympics-fun-sprint-i00-privileged", "no-such-bench-i00-standard"):
            (Path(td) / "embodied-olympics-fun" / n).mkdir(parents=True)
            (Path(td) / "embodied-olympics-fun" / n / "task.toml").write_text("")
        found = tc.check_benchmarks(Path(td))
        check(not any("sprint" in x for x in found) and any("no-such-bench" in x and "no benchmark label" in x for x in found),
              f"an unclaimed name: {found[:3]}")
    with tempfile.TemporaryDirectory() as td:            # the layout: tasks/<benchmark folder>/<task>/ and nothing else
        td = Path(td)
        for rel in ("robolab/robolab-x-i00-privileged", "robowits/robolab-y-i00-standard", "kinder-z-i00-privileged",
                    "robolab/robolab-w-i00-standard/deeper"):
            (td / rel).mkdir(parents=True)
        for rel in ("robolab/robolab-x-i00-privileged", "robowits/robolab-y-i00-standard", "kinder-z-i00-privileged",
                    "robolab/robolab-w-i00-standard/deeper"):
            (td / rel / "task.toml").write_text("")
        (td / "README.md").write_text("")
        found = tc.check_layout(td)
        check(not any("robolab-x" in x for x in found), f"a task dir in its folder was flagged: {found}")
        check(any("robolab-y" in x and "belongs in tasks/robolab/" in x for x in found), f"a task in another benchmark's folder: {found}")
        check(any("kinder-z" in x and "not a benchmark folder" in x for x in found), f"a flat task dir: {found}")
        check(any("robolab-w-i00-standard" in x and "not a task dir" in x for x in found), f"a task dir nested deeper: {found}")
        check(not any("README" in x for x in found), f"tasks/README.md was flagged: {found}")
        check(not any("no folder" in x or "not a BENCHMARKS label" in x for x in found), f"BENCHMARK_DIRS: {found}")
    check(tc.task_dir(TASK) == ROOT / "tasks" / "robotwin-2" / TASK and tc.task_relpath(TASK) == f"tasks/robotwin-2/{TASK}"
          and (tc.task_dir(TASK) / "task.toml").is_file(), f"task_dir({TASK}) = {tc.task_dir(TASK)}")
    print("ok benchmark labels: every task dir has one (its BENCHMARKS prefix), in its benchmark's folder")


def caps(task: Path) -> list[str]:
    out = re.findall(r"(?m)^cpus\s*=\s*\S+", (task / "task.toml").read_text())
    for f in sorted((task / "environment").glob("*docker-compose.y*ml")):
        out += [m.strip() for m in re.findall(r"(?m)^[ \t]*cpus:[ \t]*\S+", f.read_text())]
    return out


def num(cap: str) -> float:
    return float(re.split(r"[=:]", cap)[1])


def test_clamp():
    with tempfile.TemporaryDirectory() as td:
        td = Path(td)
        for t in (f"{BIG}-standard", HB, f"{SMALL}-standard"):
            shutil.copytree(tc.task_dir(t), td / t, symlinks=True)
        big, hb, small = td / f"{BIG}-standard", td / HB, td / f"{SMALL}-standard"
        check(max(num(c) for c in caps(big)) == 12 and max(num(c) for c in caps(hb)) == 16, "the fixtures' caps changed")
        r = subprocess.run(CLAMP + ["--cpus", "8", "--needs", str(small)], capture_output=True, text=True)
        check(r.returncode == 1 and not r.stdout, f"--needs on caps <= 8: {r.returncode} {r.stdout}")
        r = subprocess.run(CLAMP + ["--cpus", "8", "--needs", str(small), str(big)], capture_output=True, text=True)
        check(r.returncode == 0 and "12 -> 8" in r.stdout, f"--needs on a cap of 12: {r.returncode} {r.stdout}")
        check(caps(big) == caps(tc.task_dir(f"{BIG}-standard")), "--needs wrote something")
        before = {t: {p: p.read_bytes() for p in t.rglob("*") if p.is_file()} for t in (big, hb, small)}
        r = subprocess.run(CLAMP + ["--cpus", "8", str(big), str(hb), str(small)], capture_output=True, text=True)
        check(r.returncode == 0, f"clamp: {r.returncode} {r.stderr}")
        for t in (big, hb, small):
            orig = caps(tc.task_dir(t.name))
            got = caps(t)
            check(len(got) == len(orig) and all(num(g) == min(num(o), 8) for g, o in zip(got, orig)), f"{t.name}: {orig} -> {got}")
            for p, b in before[t].items():                     # only the cpus values changed, nothing else in any file
                a = p.read_bytes()
                if a != b:
                    strip = lambda x: re.sub(rb"(?m)^([ \t]*cpus[ \t]*[:=][ \t]*)\d+(?:\.\d+)?", rb"\1N", x)
                    check(strip(a) == strip(b), f"{p}: more than a cpus value changed")
        check(all(p.read_bytes() == b for p, b in before[small].items()), "a task without a cap above the host's was rewritten")
        check(subprocess.run(CLAMP + ["--cpus", "8", "--needs", str(big), str(hb)], capture_output=True).returncode == 1,
              "a clamped copy still needs clamping")
        orig = caps(tc.task_dir(HB))
        r = subprocess.run(CLAMP + ["--cpus", "8", str(tc.task_dir(HB))], capture_output=True, text=True)
        check(r.returncode != 0 and "copy" in r.stderr, f"clamping tasks/ itself was not refused: {r.returncode} {r.stderr}")
        check(caps(tc.task_dir(HB)) == orig and max(num(c) for c in orig) == 16, "tasks/ was clamped")


FAKE_HARBOR = r'''#!/usr/bin/env python3
import json, os, re, sys
a = sys.argv[1:]
tasks = [a[i + 1] for i, x in enumerate(a) if x == "-i"]
rec = {"cwd": os.getcwd(), "caps": {}, "unstamped": {}}
sys.path.insert(0, os.environ["RCB_SCRIPTS"])
import stamp_mtimes
for t in tasks:
    d = os.path.join(os.getcwd(), a[a.index("-p") + 1], t)
    rec["unstamped"][t] = stamp_mtimes.stamp_tree(d, check=True)["bad"]
    c = re.findall(r"(?m)^cpus\s*=\s*(\S+)", open(os.path.join(d, "task.toml")).read())
    c += re.findall(r"(?m)^[ \t]*cpus:[ \t]*([0-9.]+)", open(os.path.join(d, "environment", "docker-compose.yaml")).read())
    rec["caps"][t] = [float(x) for x in c]
with open(os.environ["FAKE_HARBOR_OUT"], "a") as f:
    f.write(json.dumps(rec) + "\n")
'''


def test_batch_runner():
    with tempfile.TemporaryDirectory() as td:
        td = Path(td)
        (td / "bin").mkdir()
        (td / "bin/harbor").write_text(FAKE_HARBOR); (td / "bin/harbor").chmod(0o755)
        rec = td / "harbor.jsonl"
        orig = {t: caps(tc.task_dir(t)) for t in (f"{BIG}-standard", f"{SMALL}-standard")}
        env = {**os.environ, "PATH": f"{td / 'bin'}:{os.environ['PATH']}", "FAKE_HARBOR_OUT": str(rec), "JOBS": str(td / "jobs"),
               "RESULTS": "", "LOG_MIRROR": "0", "MEM_GUARD": "0", "GATEWAY": "0", "WEB_SEARCH": "allowed", "MODES": "standard",
               "HOST_CPUS": "8", "RCB_SCRIPTS": str(ROOT / "scripts")}
        r = subprocess.run(["bash", str(ROOT / "scripts/run_agent_batch.sh"), "clamptest", BIG, SMALL], env=env,
                           capture_output=True, text=True, timeout=300)
        check(r.returncode == 0, f"run_agent_batch.sh exited {r.returncode}: {r.stderr[-800:]}")
        runs = [json.loads(l) for l in rec.read_text().splitlines()] if rec.exists() else []
        check(len(runs) == 2, f"harbor ran {len(runs)} times: {runs}")
        for run in runs:
            for t, c in run["caps"].items():
                check(c and max(c) <= 8, f"{t} ran with caps {c} on an 8-CPU host")
                # every job (GATEWAY=0 too, with or without a clamp) runs from its own copy, every file content-stamped (#258)
                base = BIG if t.startswith(BIG) else SMALL
                check(Path(run["cwd"]).resolve() == (td / f"jobs/.tasks/clamptest-standard-{base}").resolve(),
                      f"{t} ran from {run['cwd']}, not its job copy")
                check(run["unstamped"].get(t) == 0, f"{t}'s copy had {run['unstamped'].get(t)} file(s) without their content stamp")
        log = (td / "jobs/clamptest.log").read_text()
        check("clamped" in log and "12 -> 8" in log, "the clamp is not in the batch log")
        check(not any((td / f"jobs/.tasks/clamptest-standard-{b}").exists() for b in (BIG, SMALL)), "a job copy was not removed")
        for t, c in orig.items():
            check(caps(tc.task_dir(t)) == c, f"{tc.task_relpath(t)} was changed")


KILLER_HARBOR = r'''#!/usr/bin/env python3
import glob, os, signal, sys
a = sys.argv[1:]
jobs, job = a[a.index("-o") + 1], a[a.index("--job-name") + 1]
os.makedirs(os.path.join(jobs, job), exist_ok=True)
logs = sorted(glob.glob(os.path.join(jobs, ".gateway", "gateway-*.log")), key=os.path.getmtime)
with open(os.environ["FAKE_HARBOR_OUT"], "a") as f:
    f.write(job + " " + " ".join(os.path.basename(l) for l in logs) + "\n")
if not os.path.exists(os.environ["FAKE_HARBOR_OUT"] + ".killed"):     # the first job: the gateway dies under it
    open(os.environ["FAKE_HARBOR_OUT"] + ".killed", "w").close()
    os.kill(int(os.path.basename(logs[-1])[len("gateway-"):-len(".log")]), signal.SIGKILL)
'''


def test_gateway_restart():
    """GATEWAY=1, the gateway killed during the first job: that job gets gateway_down.json (its trials are
    not results), the runner restarts the gateway on its port before the next job, and the next job runs."""
    with tempfile.TemporaryDirectory() as td:
        td = Path(td)
        (td / "bin").mkdir()
        (td / "bin/harbor").write_text(KILLER_HARBOR); (td / "bin/harbor").chmod(0o755)
        rec = td / "harbor.txt"
        env = {**os.environ, "PATH": f"{td / 'bin'}:{os.environ['PATH']}", "FAKE_HARBOR_OUT": str(rec), "JOBS": str(td / "jobs"),
               "RESULTS": "", "LOG_MIRROR": "0", "MEM_GUARD": "0", "GATEWAY": "1", "GATEWAY_BIND": "127.0.0.1",
               "GATEWAY_EGRESS": "task", "WEB_SEARCH": "allowed", "MODES": "standard", "HOST_CPUS": "64"}
        r = subprocess.run(["bash", str(ROOT / "scripts/run_agent_batch.sh"), "gwtest", SMALL, "robotwin_2_0_place_a2b_left"],
                           env=env, capture_output=True, text=True, timeout=300)
        check(r.returncode == 0, f"run_agent_batch.sh (gateway) exited {r.returncode}: {r.stderr[-800:]}")
        runs = rec.read_text().splitlines() if rec.exists() else []
        check(len(runs) == 2, f"harbor ran {len(runs)} times: {runs}")
        log = (td / "jobs/gwtest.log").read_text()
        check("restarting it on port" in log and "usage gateway restarted" in log, f"no gateway restart in the log: {log[-1500:]}")
        check((td / f"jobs/gwtest-standard-{SMALL}/gateway_down.json").exists(), "the job the gateway died in has no gateway_down.json")
        check(not (td / "jobs/gwtest-standard-robotwin_2_0_place_a2b_left/gateway_down.json").exists(),
              "the job after the restart has a gateway_down.json")
        pids = {l.split()[-1] for l in runs}
        check(len(pids) == 2, f"the second job did not run behind a new gateway: {runs}")


if __name__ == "__main__":
    test_results()
    test_config_and_attempts()
    test_first_attempt()
    test_summary_grouping()
    test_core_set()
    test_benchmarks()
    test_clamp()
    test_batch_runner()
    test_gateway_restart()
    if fails:
        print(f"{len(fails)} failure(s)")
        sys.exit(1)
    print("ok: write_task_results not-a-result cases, config.json and attempts, the first-attempt result, the summary per configuration, the core set, clamp_cpus, run_agent_batch.sh stamped job copies, clamped copies and gateway restart")