Skip to content

The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.

9.0 KB
Raw Blame
#!/usr/bin/env python3
"""The agent's network egress of tasks: `allowlist` (the model APIs and the package registries; what every task in
tasks/ has, PROTOCOL Appendix H.6), or, for a run's own copy of a task, `only <hosts>` / `public`.

    scripts/set_agent_egress.py allowlist    [task dirs...]     # default: all tasks/*/*
    scripts/set_agent_egress.py only <host>[,<host>]  <task dirs...>   # a run's copy: exactly these hosts
    scripts/set_agent_egress.py package-hosts                   # prints PACKAGE_HOSTS, comma-separated (for the runner)
    scripts/set_agent_egress.py network <task dir>              # prints the task's declared egress: allowlist or open
    scripts/set_agent_egress.py public       <task dirs...>     # a run's copy with no restriction: network_mode = "public"

HOSTS below is the one list, and the one switch: every task ships with it, ci_checks/check-verifier-isolation.sh requires
exactly it of every allowlist task in tasks/ (reading it from this file's text), and the usage gateway imports it. It
is the model APIs (the hosts Codex with a ChatGPT login, Claude Code and OpenRouter need) and PACKAGE_HOSTS, the
package registries (installing packages is part of what an agent may do): PyPI (pip contacts pypi.org for the index
and files.pythonhosted.org for the files) and the apt archives the images' distributions name in their sources
(Debian: deb.debian.org; Ubuntu: archive / security / snapshot.ubuntu.com; plain HTTP, which Harbor's egress control
allows by the request's Host). GitHub, Hugging Face and the general web stay closed: that is where the benchmarks'
sources, demonstrations and trained policies are; registries mostly are not (the PyPI distributions that do carry a
solution: docs/protocol/leakage.md). To change what agents reach, edit the two lists, run `allowlist` (it rewrites the
`allowed_hosts` line of every allowlist task and leaves the other lines as they are), then
`python3 scripts/task_conventions.py tasks/*/*/` (normalize refreshes each DIGEST).

A task whose author chose open egress declares it in Harbor's own field, `[agent] network_mode = "public"` (PROTOCOL §2,
Appendix H.6): `allowlist` leaves every task that declares it alone and names it (`skip`), and a runner does not narrow
its copy to the gateway (`network` prints the declaration).

A run with no restriction at all: `public` on a copy of the task directories (never in tasks/: CI requires the list
there), and run that copy (docs/protocol/running.md "Network"). Such a copy is an open task from then on (`allowlist`
skips it too); to go back, copy the tasks again.

`only <hosts>`: Harbor can add hosts to a task's allowlist at run time (`--allow-agent-host`) but not take any off, and
the model APIs' hosts also serve what a trial does not get (account connectors, files, web tools). A run through the
usage gateway therefore runs from a copy of the task whose allowlist is the gateway's address and PACKAGE_HOSTS
(scripts/run_agent_batch.sh, GATEWAY=1; plus chatgpt.com for Codex with a ChatGPT login, whose start-up lookup needs
it); the gateway forwards the model-API paths and refuses the rest. `only` and
`public` take explicit directories and refuse one inside this repository's tasks/."""
import pathlib, re, sys
try:
    import tomllib               # Python 3.11+: the result is parsed back as a check; older runner hosts go without
except ModuleNotFoundError:
    tomllib = None

# Two literal lists (the CI check reads them from this file's text; the usage gateway imports them): HOSTS is every
# task's allowlist; PACKAGE_HOSTS, its last entries, is the part that is not a model API.
HOSTS = ["chatgpt.com", "*.chatgpt.com", "auth.openai.com", "api.openai.com", "*.openai.com",
         "api.anthropic.com", "*.anthropic.com", "openrouter.ai",
         "*.openai.azure.com", "*.cognitiveservices.azure.com",
         "pypi.org", "files.pythonhosted.org", "deb.debian.org", "security.debian.org", "archive.ubuntu.com",
         "security.ubuntu.com", "snapshot.ubuntu.com"]
PACKAGE_HOSTS = ["pypi.org", "files.pythonhosted.org", "deb.debian.org", "security.debian.org", "archive.ubuntu.com",
                 "security.ubuntu.com", "snapshot.ubuntu.com"]
assert not PACKAGE_HOSTS or HOSTS[-len(PACKAGE_HOSTS):] == PACKAGE_HOSTS
COMMENT = "# model APIs and package registries (scripts/set_agent_egress.py); PROTOCOL Appendix H.6"
root = pathlib.Path(__file__).resolve().parents[1]


def network_of(d: pathlib.Path) -> str:
    """The egress the task declares: "open" for `[agent] network_mode = "public"`, else "allowlist"
    (scripts/task_conventions.py network_of; read without a TOML parser, for older runner hosts)."""
    agent = re.split(r"(?m)^\[", (d / "task.toml").read_text().partition("\n[agent]\n")[2], maxsplit=1)[0]
    return "open" if re.search(r'(?m)^network_mode\s*=\s*"public"', agent) else "allowlist"


def main(argv: list) -> None:
    mode = argv[1] if len(argv) > 1 else ""
    if mode == "package-hosts":
        print(",".join(PACKAGE_HOSTS)); sys.exit(0)
    if mode == "network":
        if len(argv) != 3:
            sys.exit(__doc__)
        print(network_of(pathlib.Path(argv[2]))); sys.exit(0)
    if mode not in ("public", "allowlist", "only"):
        sys.exit(__doc__)
    args = argv[2:]
    hosts = HOSTS
    if mode == "only":
        if not args or not re.fullmatch(r"[A-Za-z0-9.*:-]+(,[A-Za-z0-9.*:-]+)*", args[0]):
            sys.exit(__doc__)
        hosts, args = args[0].split(","), args[1:]
    dirs = [pathlib.Path(d) for d in args]
    if mode != "allowlist":      # a run's or an experiment's copy, never the repository's tasks
        inside = [d for d in dirs if (root / "tasks").resolve() in [d.resolve(), *d.resolve().parents]]
        if not dirs or inside:
            sys.exit(f"`{mode}` takes copies of task directories, outside {root / 'tasks'}" + (f": {inside[0]}" if inside else ""))
    if not dirs:
        sys.path.insert(0, str(root / "scripts"))
        import task_conventions as tc
        dirs = tc.iter_tasks(root / "tasks")
    if mode == "allowlist":                                       # open egress (the author's choice, or a `public` copy) is kept
        for d in [d for d in dirs if network_of(d) == "open"]:
            print(f'skip {d.name}: open egress (network_mode = "public")')
        dirs = [d for d in dirs if network_of(d) != "open"]
    done = 0
    for d in dirs:
        p = d / "task.toml"; s = p.read_text()
        head, sep, rest = s.partition("[agent]\n")
        if not sep:
            if mode != "allowlist":      # a run's copy that cannot be narrowed must not run
                sys.exit(f"{d}: task.toml has no plain `[agent]` section; nothing written")
            print(f"skip {d.name}: no [agent] section"); continue
        if mode != "allowlist" and re.search(r"(?m)^\s*\[\[?\s*steps\b", s):
            # Harbor lets a step's own agent policy override [agent] (trial/network_policy.py): this script does not rewrite those
            sys.exit(f"{d}: task.toml has [[steps]]; a step's network policy would override [agent]; nothing written")
        block, sep2, tail = rest.partition("\n[")
        if re.search(r"(?m)^\s*allowed_hosts\s*=\s*\[[^\]\n]*$", block):      # a list over several lines: not rewritten line by line
            sys.exit(f"{d}: [agent] allowed_hosts spans several lines; write it on one line; nothing written")
        if mode == "allowlist":          # already on an allowlist: only its host list changes
            m = re.search(r'(?m)^allowed_hosts[ \t]*=[ \t]*\[(.*)\][ \t]*$', block)
            if re.search(r'(?m)^network_mode\s*=\s*"allowlist"', block) and m:
                if re.findall(r'"([^"]+)"', m.group(1)) != hosts:
                    block2 = block[:m.start()] + "allowed_hosts = [" + ", ".join(f'"{h}"' for h in hosts) + "]" + block[m.end():]
                    s2 = head + "[agent]\n" + block2 + (("\n[" + tail) if sep2 else "")
                    if tomllib is not None:
                        tomllib.loads(s2)
                    p.write_text(s2)
                done += 1; continue
        lines = [l for l in block.splitlines() if not re.match(r"\s*(network_mode|allowed_hosts)\s*=", l) and "set_agent_egress" not in l]
        lines = [l for l in lines if l.strip()]
        if mode == "allowlist":
            lines.append('network_mode = "allowlist"      ' + COMMENT)
            lines.append("allowed_hosts = [" + ", ".join(f'"{h}"' for h in hosts) + "]")
        elif mode == "only":
            lines.append('network_mode = "allowlist"      # this run\'s copy: the usage gateway and the package registries (scripts/set_agent_egress.py only)')
            lines.append("allowed_hosts = [" + ", ".join(f'"{h}"' for h in hosts) + "]")
        else:
            lines.append('network_mode = "public"        # this copy only: no restriction (scripts/set_agent_egress.py public); recorded as open')
        s2 = head + "[agent]\n" + "\n".join(lines) + "\n" + (("\n[" + tail) if sep2 else "")
        if tomllib is not None:
            tomllib.loads(s2)
        p.write_text(s2); done += 1
    print(f"{mode}: {done} task(s)")


if __name__ == "__main__":
    main(sys.argv)