The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.
Python ai-agentsbenchmarkcoding-agentsembodied-aievaluationharborllm-agentsrobot-learningroboticssimulation
#!/usr/bin/env python3
"""Resources (docs/protocol/README.md §6): every number is an upper bound (a cap) for one container, written where Harbor
and docker read it, and nothing else.
- task.toml has no `[metadata.resources]` (no minimums, no soft limits, no copy of the caps);
- main's caps in `[environment]`: `cpus` (a whole number), `memory_mb`, `storage_mb` (disk), and `gpu_types` (RTX-class
models, task_conventions.RTX_GPU_MODELS) only on a task with a GPU; never `gpus` (Harbor 0.23's local docker refuses
it: the GPU is a compose device reservation, never `gpus:` or `runtime: nvidia`);
- every other compose service (the monitor's own aside, check-monitor.py) sets `cpus`, `mem_limit` and `pids_limit`, main
sets `pids_limit` (its CPU and memory are Harbor's, from `[environment]`), none sets `deploy.resources.limits`; the
`deploy` key is written in block style (one key per line: what inject.py reads, cross-checked with a YAML parser when
PyYAML is installed); a service that reserves a GPU sets `x-gpu-memory-gb`, and only such a service
(tests/docker-compose.yaml too: written by normalize from main's cap, by hand when main holds no GPU); at most one service in the monitor's scope
(main, and in privileged mode the services the agent drives) holds a GPU, since the monitor checks one GPU-memory cap
for the scope;
- the caps of the task's rule in scripts/task_resources.toml are the ones applied (task_conventions.py writes them);
- instruction.md ends (before `## Time limit`) with the `## Resources` paragraph rendered from main's caps, after the
`## External material` paragraph every task states in the same words (§5), and the README's `## Resources` is the one
rendered from every container's caps.
Usage: check-resources.py [task dirs...] (default: every task under tasks/)
"""
import re
import sys
import tomllib
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
from task_conventions import ( # noqa: E402 (one renderer of the paragraph and the README section, one rule reader)
CUTOFF_HEADING, RESOURCES_HEADING, RTX_GPU_MODELS, iter_tasks, mode_of, readme_resources, render_external,
render_resources, rule_for, task_caps)
from rcb_monitor import inject # noqa: E402 (one reader of the compose file's services and caps)
FLOW_DEPLOY_RE = re.compile(r"^\s+(deploy|resources|reservations|devices|limits|capabilities):\s*[\[{]")
FLOW_DEPLOY_OK_RE = re.compile(r"^\s+-?\s*capabilities:\s*\[[^\]{}]*\]\s*(#.*)?$") # `capabilities: [gpu]` is fine
def yaml_view(txt: str) -> dict | None:
"""{service: (holds a GPU, sets deploy.resources.limits)} by a YAML parser, or None (no PyYAML, or no services)."""
try:
import yaml
except ImportError:
return None
try:
d = yaml.safe_load(txt) or {}
except yaml.YAMLError:
return None
out = {}
for name, svc in ((d.get("services") or {}) if isinstance(d, dict) else {}).items():
svc = svc if isinstance(svc, dict) else {}
res = ((svc.get("deploy") or {}).get("resources") or {})
devs = (res.get("reservations") or {}).get("devices") or []
gpu = any(isinstance(x, dict) and (x.get("driver") in ("nvidia", "cdi") or "gpu" in (x.get("capabilities") or []))
for x in devs) or "gpus" in svc or svc.get("runtime") == "nvidia"
out[name] = (gpu, "limits" in res)
return out
BOUNDS = {"cpus": 128, "memory_gb": 1024, "pids": 1 << 22, "gpu_memory_gb": 192, "disk_gb": 4096}
def is_num(v) -> bool:
return isinstance(v, (int, float)) and not isinstance(v, bool)
def rel(task: Path) -> str:
return str(task.relative_to(ROOT) if task.is_relative_to(ROOT) else task)
def check(task: Path) -> list[str]:
errs = []
d = tomllib.loads((task / "task.toml").read_text())
if "resources" in (d.get("metadata") or {}):
errs.append("task.toml has [metadata.resources]: resources are caps per container only, in [environment] and the "
"compose files (PROTOCOL §6; run scripts/task_conventions.py)")
env = d.get("environment") or {}
if "gpus" in env:
errs.append("[environment].gpus: Harbor's local docker refuses gpus > 0; reserve the GPU in "
"environment/docker-compose.yaml (an nvidia device) and cap its memory with x-gpu-memory-gb")
for k in ("cpus", "memory_mb", "storage_mb"):
v = env.get(k)
if not (isinstance(v, int) and not isinstance(v, bool) and v > 0):
errs.append(f"[environment].{k} must be a positive whole number (main's cap; Harbor reads it)")
comp = task / "environment/docker-compose.yaml"
try:
caps = task_caps(task)
except ValueError as e:
return errs + [f"environment/docker-compose.yaml: {e}"]
try:
own = inject.service_caps(comp.read_text()) if comp.is_file() else {}
except ValueError as e:
return errs + [f"environment/docker-compose.yaml: {e}"]
gpu = any(c.get("gpu") for c in caps.values())
gt = env.get("gpu_types")
if gt is not None and not (isinstance(gt, list) and gt and all(isinstance(x, str) and x.strip() for x in gt)):
errs.append("[environment].gpu_types must be a non-empty list of GPU model names")
elif gt and not gpu:
errs.append("[environment].gpu_types is set but no service reserves a GPU")
elif gt and set(gt) - set(RTX_GPU_MODELS):
errs.append(f"[environment].gpu_types {sorted(set(gt) - set(RTX_GPU_MODELS))}: not RTX-class models "
f"(gpu_types is for a task that needs ray-tracing cores: {', '.join(RTX_GPU_MODELS)}); a task that "
"runs on any NVIDIA GPU sets none")
for f in ("environment/docker-compose.yaml", "tests/docker-compose.yaml"):
txt = (task / f).read_text() if (task / f).is_file() else ""
for ln in txt.splitlines():
if FLOW_DEPLOY_RE.match(ln) and not FLOW_DEPLOY_OK_RE.match(ln):
errs.append(f"{f}: `{ln.strip()}`: write `deploy` in block style, one key per line (what "
"images/monitor/rcb_monitor/inject.py reads)")
break
yv = yaml_view(txt) if txt else None
if yv is not None:
try:
lv = inject.service_caps(txt)
except ValueError:
lv = {}
for n, (g, lim) in yv.items():
if lim:
errs.append(f"{f}: service {n}: deploy.resources.limits; write the caps as `cpus`, `mem_limit` and "
"`pids_limit` (docker refuses both forms together)")
if n in lv and lv[n]["gpu"] != g:
errs.append(f"{f}: service {n}: a YAML parser {'finds' if g else 'finds no'} GPU reservation, "
f"inject.py {'finds' if lv[n]['gpu'] else 'none'}: write the reservation in block style "
"(deploy.resources.reservations.devices, `driver: nvidia`, `capabilities: [gpu]`)")
for m in re.finditer(r"(?m)^ ([A-Za-z0-9_.-]+):[^\n]*\n((?: [^\n]*\n|\s*\n)*)", txt):
if any(inject.GPU_OTHER_RE.match(ln) for ln in m.group(2).splitlines()):
errs.append(f"{f}: service {m.group(1)} gets its GPU through `gpus:` or `runtime: nvidia`; reserve it as "
"a deploy device (deploy.resources.reservations.devices, driver: nvidia)")
held = [n for n, c in caps.items() if c.get("gpu") and inject.kind_of(mode_of(task), n) == "scope"]
if len(held) > 1:
errs.append(f"services {held} all hold a GPU in the monitor's scope: it checks one GPU-memory cap per scope; "
"give the agent's side one GPU service")
for svc, c in caps.items():
if svc == "main":
m = own.get("main") or {}
if m.get("cpus") is not None or m.get("memory_gb") is not None:
errs.append("service main: cpus / mem_limit in docker-compose.yaml (Harbor sets them from [environment])")
need = ("pids",)
else:
need = ("cpus", "memory_gb", "pids")
names = {"cpus": "cpus", "memory_gb": "mem_limit", "pids": "pids_limit"}
for k in need:
if c.get(k) is None:
errs.append(f"service {svc}: no `{names[k]}` (every container has its caps, PROTOCOL §6)")
if c.get("gpu") and not c.get("gpu_memory_gb"):
errs.append(f"service {svc} reserves a GPU but sets no `{inject.GPU_KEY}` (its GPU-memory cap, GiB)")
if not c.get("gpu") and c.get("gpu_memory_gb") is not None:
errs.append(f"service {svc} sets `{inject.GPU_KEY}` but reserves no GPU")
for k, hi in BOUNDS.items():
v = c.get(k)
if v is not None and not (is_num(v) and 0 < v <= hi):
errs.append(f"service {svc}: {k} = {v} outside (0, {hi}]")
tc = task / "tests/docker-compose.yaml"
if tc.is_file():
try:
for svc, c in inject.service_caps(tc.read_text()).items():
if c.get("gpu") and not c.get("gpu_memory_gb"):
errs.append(f"tests/docker-compose.yaml: service {svc} reserves a GPU but sets no `{inject.GPU_KEY}`: "
"normalize copies main's GPU-memory cap there only when main holds a GPU; otherwise write "
f"`{inject.GPU_KEY}: <GiB>` on that service yourself (normalize keeps it; PROTOCOL §6)")
except ValueError as e:
errs.append(f"tests/docker-compose.yaml: {e}")
if errs:
return errs
# the rule's caps are the applied ones
try:
r = rule_for(task)
except RuntimeError as e:
return [str(e)]
if r is not None:
want = r["services"]
if set(want) != set(caps):
errs.append(f"scripts/task_resources.toml gives services {sorted(want)}, the compose file has {sorted(caps)}")
for svc, c in want.items():
for k, v in c.items():
got = (caps.get(svc) or {}).get(k)
if got is None or abs(float(got) - float(v)) > 1e-6:
where = ("[environment]" if svc == "main" and k in ("cpus", "memory_gb", "disk_gb")
else f"docker-compose.yaml service {svc}")
errs.append(f"{svc}.{k} = {v:g} in scripts/task_resources.toml but {where} applies "
f"{'nothing' if got is None else f'{got:g}'}: declare the cap that is applied, or apply "
f"the declared one (python3 scripts/task_conventions.py {rel(task)})")
if (r.get("gpu_types") or None) != (gt or None):
errs.append(f"[environment].gpu_types {gt} differs from scripts/task_resources.toml {r.get('gpu_types')}")
if errs:
return errs
# instruction.md: `## External material`, then `## Resources`, before the closing `## Time limit`
ins = (task / "instruction.md").read_text() if (task / "instruction.md").is_file() else ""
ins = re.sub(rf"(?ms)\n^{re.escape(CUTOFF_HEADING)}[ \t]*\n.*\Z", "", ins) # with the blank line before it
want = render_resources(task)
if not ins.endswith("\n" + want):
errs.append(f"instruction.md does not end (before `## Time limit`) with the `{RESOURCES_HEADING}` paragraph "
f"rendered from the caps (run python3 scripts/task_conventions.py {rel(task)})")
elif not ins[: -len(want)].endswith("\n" + render_external(task) + "\n"):
errs.append("instruction.md does not state `## External material` (PROTOCOL §5; scripts/task_conventions.py "
f"EXTERNAL_TEXT) just before `{RESOURCES_HEADING}`")
readme = (task / "README.md").read_text() if (task / "README.md").is_file() else ""
if "\n".join(readme_resources(task, readme)) not in readme:
errs.append(f"README.md `{RESOURCES_HEADING}` does not match the caps (run python3 scripts/task_conventions.py "
f"{rel(task)})")
return errs
def main(argv: list[str]) -> int:
tasks = [Path(a).resolve() for a in argv] or iter_tasks()
rc = 0
n = 0
for t in tasks:
if not (t / "task.toml").is_file():
continue
n += 1
for e in check(t):
print(f"FAIL {t.name}: {e}")
rc = 1
if rc == 0:
print(f"resources: {n} task(s) state the caps of every container (PROTOCOL §6)")
return rc
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))