The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.
Python ai-agentsbenchmarkcoding-agentsembodied-aievaluationharborllm-agentsrobot-learningroboticssimulation
Usage gateway: skip blank lines in the container scan (#514)
Kangrui Wang8b172da7795c
diff --git a/ci_checks/check-unit-tests.sh b/ci_checks/check-unit-tests.shindex 623c8b19a6..509aa5f733 100755--- a/ci_checks/check-unit-tests.sh+++ b/ci_checks/check-unit-tests.sh@@ -33,7 +33,7 @@ run() { # run <timeout s> <test path in the repository> [arguments of the testwant "$2" && cmds+="$(lane "$2")bash $self --one $*"$'\n'}for t in scripts/usage_gateway/tests/test_rawlog.py scripts/usage_gateway/tests/test_paths.py \- scripts/usage_gateway/tests/test_budget_stop.py scripts/tests/test_codex_tools_off.py scripts/tests/test_task_seed.py \+ scripts/usage_gateway/tests/test_budget_stop.py scripts/usage_gateway/tests/test_scan.py scripts/tests/test_codex_tools_off.py scripts/tests/test_task_seed.py \scripts/tests/test_runner_results.py scripts/tests/test_retry_policy.py scripts/tests/test_stamp_mtimes.py \scripts/tests/test_efe.py; dorun 300 "$t"diff --git a/scripts/usage_gateway/gateway.py b/scripts/usage_gateway/gateway.pyindex 3ff7276480..f56d2c2354 100755--- a/scripts/usage_gateway/gateway.py+++ b/scripts/usage_gateway/gateway.py@@ -380,6 +380,21 @@ def sanitize(name: str) -> str:return re.sub(r"[^a-z0-9_-]", "-", name)+def parse_inspect(out: str) -> dict[str, str]:+ """IP -> compose project of a trial's main container, from `docker inspect` lines "<project> <ip> <ip>...". A line+ that does not fit (blank: no compose label and no network; a project but no address) is skipped, never fatal."""+ projects = {}+ for line in out.splitlines():+ fields = line.split()+ if not fields:+ continue+ proj, *ips = fields+ if proj.endswith("__env"):+ for ip in ips:+ projects[ip] = proj[: -len("__env")]+ return projects++class Trials:"""Source IP -> trial directory, from docker's compose labels. Docker reuses subnets and addresses once a trial'snetwork is gone, so a mapping is trusted for TTL seconds only and every scan replaces the whole table; a new trial@@ -397,16 +412,15 @@ class Trials:returnfmt = '{{index .Config.Labels "com.docker.compose.project"}}{{range .NetworkSettings.Networks}} {{.IPAddress}}{{end}}'out = subprocess.run(["docker", "inspect", "-f", fmt, *ids], capture_output=True, text=True, timeout=30).stdout- projects = {}- for line in out.splitlines():- proj, *ips = line.split()- if proj.endswith("__env"):- for ip in ips:- projects[ip] = proj[: -len("__env")]+ projects = parse_inspect(out)dirs = {}for job in self.jobs.iterdir():if job.is_dir() and not job.name.startswith("."):- for trial in job.iterdir():+ try:+ trials = list(job.iterdir())+ except OSError: # a job dir removed mid-scan: skip it, keep the others+ continue+ for trial in trials:if trial.is_dir() and "__" in trial.name:dirs[sanitize(trial.name)] = trialself.by_ip = {ip: dirs[proj] for ip, proj in projects.items() if proj in dirs}diff --git a/scripts/usage_gateway/tests/test_scan.py b/scripts/usage_gateway/tests/test_scan.pynew file mode 100644index 0000000000..3040fde173--- /dev/null+++ b/scripts/usage_gateway/tests/test_scan.py@@ -0,0 +1,45 @@+"""Tests for the gateway's trial attribution scan (gateway.parse_inspect, Trials._scan): a bad line never wipes the map.++ python3 scripts/usage_gateway/tests/test_scan.py # stdlib only; docker is stubbed+"""+from __future__ import annotations++import pathlib+import subprocess+import sys+import tempfile+import types+import unittest+from unittest import mock++HERE = pathlib.Path(__file__).resolve().parents[1]+sys.path.insert(0, str(HERE))+import gateway # noqa: E402+++class ParseInspect(unittest.TestCase):+ def test_blank_and_partial_lines_are_skipped(self):+ out = "a__1__env 172.18.0.2\n\n \nb__2__env\nother 172.17.0.5\nc__3__env 172.19.0.2 172.20.0.3\n"+ self.assertEqual(gateway.parse_inspect(out),+ {"172.18.0.2": "a__1", "172.19.0.2": "c__3", "172.20.0.3": "c__3"})+++class Scan(unittest.TestCase):+ def test_blank_line_keeps_every_other_trial(self):+ with tempfile.TemporaryDirectory() as td:+ jobs = pathlib.Path(td)+ for name in ("t1__aaa", "t2__bbb"):+ (jobs / "job" / name).mkdir(parents=True)+ inspect = "t1__aaa__env 172.18.0.2\n\nt2__bbb__env 172.19.0.2\n"++ def run(cmd, **_):+ return types.SimpleNamespace(stdout="id1\nid2\nid3\n" if cmd[1] == "ps" else inspect)++ trials = gateway.Trials(jobs)+ with mock.patch.object(subprocess, "run", run):+ self.assertEqual(trials.lookup("172.18.0.2"), jobs / "job" / "t1__aaa")+ self.assertEqual(trials.lookup("172.19.0.2"), jobs / "job" / "t2__bbb")+++if __name__ == "__main__":+ unittest.main()