Skip to content

The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.

Usage gateway: skip blank lines in the container scan (#514)

Kangrui Wang8b172da7795c
diff --git a/ci_checks/check-unit-tests.sh b/ci_checks/check-unit-tests.sh
index 623c8b19a6..509aa5f733 100755
--- a/ci_checks/check-unit-tests.sh
+++ b/ci_checks/check-unit-tests.sh
@@ -33,7 +33,7 @@ run() { # run <timeout s> <test path in the repository> [arguments of the test
want "$2" && cmds+="$(lane "$2")bash $self --one $*"$'\n'
}
for t in scripts/usage_gateway/tests/test_rawlog.py scripts/usage_gateway/tests/test_paths.py \
- scripts/usage_gateway/tests/test_budget_stop.py scripts/tests/test_codex_tools_off.py scripts/tests/test_task_seed.py \
+ scripts/usage_gateway/tests/test_budget_stop.py scripts/usage_gateway/tests/test_scan.py scripts/tests/test_codex_tools_off.py scripts/tests/test_task_seed.py \
scripts/tests/test_runner_results.py scripts/tests/test_retry_policy.py scripts/tests/test_stamp_mtimes.py \
scripts/tests/test_efe.py; do
run 300 "$t"
diff --git a/scripts/usage_gateway/gateway.py b/scripts/usage_gateway/gateway.py
index 3ff7276480..f56d2c2354 100755
--- a/scripts/usage_gateway/gateway.py
+++ b/scripts/usage_gateway/gateway.py
@@ -380,6 +380,21 @@ def sanitize(name: str) -> str:
return re.sub(r"[^a-z0-9_-]", "-", name)
+def parse_inspect(out: str) -> dict[str, str]:
+ """IP -> compose project of a trial's main container, from `docker inspect` lines "<project> <ip> <ip>...". A line
+ that does not fit (blank: no compose label and no network; a project but no address) is skipped, never fatal."""
+ projects = {}
+ for line in out.splitlines():
+ fields = line.split()
+ if not fields:
+ continue
+ proj, *ips = fields
+ if proj.endswith("__env"):
+ for ip in ips:
+ projects[ip] = proj[: -len("__env")]
+ return projects
+
+
class Trials:
"""Source IP -> trial directory, from docker's compose labels. Docker reuses subnets and addresses once a trial's
network is gone, so a mapping is trusted for TTL seconds only and every scan replaces the whole table; a new trial
@@ -397,16 +412,15 @@ class Trials:
return
fmt = '{{index .Config.Labels "com.docker.compose.project"}}{{range .NetworkSettings.Networks}} {{.IPAddress}}{{end}}'
out = subprocess.run(["docker", "inspect", "-f", fmt, *ids], capture_output=True, text=True, timeout=30).stdout
- projects = {}
- for line in out.splitlines():
- proj, *ips = line.split()
- if proj.endswith("__env"):
- for ip in ips:
- projects[ip] = proj[: -len("__env")]
+ projects = parse_inspect(out)
dirs = {}
for job in self.jobs.iterdir():
if job.is_dir() and not job.name.startswith("."):
- for trial in job.iterdir():
+ try:
+ trials = list(job.iterdir())
+ except OSError: # a job dir removed mid-scan: skip it, keep the others
+ continue
+ for trial in trials:
if trial.is_dir() and "__" in trial.name:
dirs[sanitize(trial.name)] = trial
self.by_ip = {ip: dirs[proj] for ip, proj in projects.items() if proj in dirs}
diff --git a/scripts/usage_gateway/tests/test_scan.py b/scripts/usage_gateway/tests/test_scan.py
new file mode 100644
index 0000000000..3040fde173
--- /dev/null
+++ b/scripts/usage_gateway/tests/test_scan.py
@@ -0,0 +1,45 @@
+"""Tests for the gateway's trial attribution scan (gateway.parse_inspect, Trials._scan): a bad line never wipes the map.
+
+ python3 scripts/usage_gateway/tests/test_scan.py # stdlib only; docker is stubbed
+"""
+from __future__ import annotations
+
+import pathlib
+import subprocess
+import sys
+import tempfile
+import types
+import unittest
+from unittest import mock
+
+HERE = pathlib.Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(HERE))
+import gateway # noqa: E402
+
+
+class ParseInspect(unittest.TestCase):
+ def test_blank_and_partial_lines_are_skipped(self):
+ out = "a__1__env 172.18.0.2\n\n \nb__2__env\nother 172.17.0.5\nc__3__env 172.19.0.2 172.20.0.3\n"
+ self.assertEqual(gateway.parse_inspect(out),
+ {"172.18.0.2": "a__1", "172.19.0.2": "c__3", "172.20.0.3": "c__3"})
+
+
+class Scan(unittest.TestCase):
+ def test_blank_line_keeps_every_other_trial(self):
+ with tempfile.TemporaryDirectory() as td:
+ jobs = pathlib.Path(td)
+ for name in ("t1__aaa", "t2__bbb"):
+ (jobs / "job" / name).mkdir(parents=True)
+ inspect = "t1__aaa__env 172.18.0.2\n\nt2__bbb__env 172.19.0.2\n"
+
+ def run(cmd, **_):
+ return types.SimpleNamespace(stdout="id1\nid2\nid3\n" if cmd[1] == "ps" else inspect)
+
+ trials = gateway.Trials(jobs)
+ with mock.patch.object(subprocess, "run", run):
+ self.assertEqual(trials.lookup("172.18.0.2"), jobs / "job" / "t1__aaa")
+ self.assertEqual(trials.lookup("172.19.0.2"), jobs / "job" / "t2__bbb")
+
+
+if __name__ == "__main__":
+ unittest.main()