Skip to content

The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.

7.0 KB
Raw Blame
#!/usr/bin/env python3
"""Run shell commands concurrently, print each one's output in one piece, fail if any failed.

    parallel.py [-j N] [--expect N] [--advisory] < commands   # one command per line (run with bash -c), from the repo root
    parallel.py --nproc                # the default N

--expect N (the required CI job `invariants`, ci_checks/run_invariants.sh): exit 1 unless exactly N commands were given
and all N ran to completion with exit 0 -- an incomplete run fails, never passes.
--advisory (the CI job `advisories`): a command that exits non-zero is a warning, not a failure (exit 3 is the
convention for "warnings only"; an advisory check that exits 1 is reported the same way); its `FAIL` lines are printed
as `WARN`, as GitHub annotations (`::warning ::`, when GITHUB_ACTIONS is set) and appended to the file
$RCB_ADVISORIES_OUT if set (ci_checks/pr_overview.py --advisories puts them in the pull request's overview). Exit 3
when there was any warning, else 0.

A line `@lane:<name> <command>` puts the command in a lane: the commands of one lane run one after another (in their
order, started before the commands outside lanes), each lane taking one of the N slots. For tests that cannot share
the host with each other, e.g. the service tests that pick a free port and start a server on it (two of them running
at once can pick the same port and talk to each other's server).

-j: at most N at a time (default: $RCB_CI_JOBS, else the CPUs this process may use: its CPU affinity, capped by the
cgroup's CPU quota -- a runner container started with `--cpus 4` sees all of the host's CPUs but gets four). Each command's stdout + stderr is captured and
printed when it ends, under a `== <command> (<s> s, exit <rc>)` header, so concurrent output never interleaves (a
command that succeeds silently prints nothing). The exit code is 1 when any command exited non-zero, and the failed commands are listed at the end.
"""
from __future__ import annotations

import os
import subprocess
import sys
import time
import threading
from concurrent.futures import ThreadPoolExecutor


def run(cmd: str) -> tuple[str, int, str, float]:
    t0 = time.monotonic()
    try:
        p = subprocess.run(["bash", "-c", cmd], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors="replace")
    except OSError as e:   # e.g. E2BIG, a command line longer than one argument may be: this command fails, not the pool
        return cmd, 126, f"could not start: {e}\n", time.monotonic() - t0
    return cmd, p.returncode, p.stdout, time.monotonic() - t0


def cpus() -> int:
    if os.environ.get("RCB_CI_JOBS", "").isdigit() and int(os.environ["RCB_CI_JOBS"]) > 0:
        return int(os.environ["RCB_CI_JOBS"])
    n = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1)
    try:   # cgroup v2: "<quota> <period>" or "max <period>"
        quota, period = open("/sys/fs/cgroup/cpu.max").read().split()[:2]
        if quota != "max":
            n = min(n, max(1, -(-int(quota) // int(period))))
    except (OSError, ValueError):
        try:   # cgroup v1
            quota = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read())
            period = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read())
            if quota > 0:
                n = min(n, max(1, -(-quota // period)))
        except (OSError, ValueError):
            pass
    return max(1, n)


def main(argv: list[str]) -> int:
    if argv == ["--nproc"]:
        print(cpus())
        return 0
    jobs = cpus()
    expect, advisory = None, False
    i = 0
    while i < len(argv):
        if argv[i] == "-j":
            jobs, i = int(argv[i + 1]), i + 2
        elif argv[i] == "--expect":
            expect, i = int(argv[i + 1]), i + 2
        elif argv[i] == "--advisory":
            advisory, i = True, i + 1
        else:
            raise SystemExit(f"usage: parallel.py [-j N] [--expect N] [--advisory] < commands; got {argv[i]!r}")
    lines = [c for c in (line.strip() for line in sys.stdin) if c and not c.startswith("#")]
    lanes: dict[str, list[str]] = {}
    units: list[list[str]] = []
    for c in lines:
        if c.startswith("@lane:"):
            name, _, cmd = c[len("@lane:"):].partition(" ")
            lanes.setdefault(name, []).append(cmd.strip())
        else:
            units.append([c])
    units = list(lanes.values()) + units
    cmds = [c for u in units for c in u]
    failed: list[str] = []
    done: list[str] = []
    warn_lines: list[str] = []
    lock = threading.Lock()
    if expect is not None and len(cmds) != expect:
        print(f"FAIL parallel.py: {len(cmds)} command(s) given, {expect} expected (an incomplete or padded run never passes)")
        return 1

    def report(cmd: str, rc: int, out: str, dt: float) -> None:
        with lock:
            done.append(cmd)
            if advisory and rc != 0:
                out = "".join(("WARN" + ln[4:]) if ln.startswith("FAIL") else ln for ln in out.splitlines(keepends=True))
                lines = [ln.strip() for ln in out.splitlines() if ln.strip()]
                found = [ln for ln in lines if ln.startswith(("WARN", "-"))] or lines[-5:] or [f"exit {rc}"]
                short_cmd = cmd if len(cmd) <= 100 else cmd[:97] + "..."
                warn_lines.extend(f"{short_cmd}: {ln}" for ln in found)
            if rc != 0:
                failed.append(cmd)
            if rc == 0 and not out.strip():   # silent success (e.g. task_conventions.py --check): no header
                return
            short = cmd if len(cmd) <= 160 else cmd[:157] + "..."
            sys.stdout.write(f"== {short} ({dt:.0f} s, exit {rc})\n{out}")
            if out and not out.endswith("\n"):
                sys.stdout.write("\n")
            sys.stdout.flush()

    def run_unit(unit: list[str]) -> None:
        for c in unit:
            report(*run(c))

    with ThreadPoolExecutor(max_workers=max(1, jobs)) as pool:
        for fut in [pool.submit(run_unit, u) for u in units]:
            fut.result()
    if expect is not None and len(done) != expect:
        print(f"FAIL parallel.py: {len(done)} of {expect} command(s) completed")
        return 1
    if advisory:
        if warn_lines:
            if os.environ.get("GITHUB_ACTIONS"):
                for ln in warn_lines[:200]:
                    msg = ln.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
                    print(f"::warning title=advisory::{msg}")
            if os.environ.get("RCB_ADVISORIES_OUT"):
                with open(os.environ["RCB_ADVISORIES_OUT"], "a") as f:
                    f.write("".join(ln + "\n" for ln in warn_lines))
            print(f"WARN {len(failed)} of {len(cmds)} advisory command(s) reported {len(warn_lines)} finding(s) (not blocking)")
            return 3
        print(f"ok all {len(cmds)} advisory command(s)")
        return 0
    if failed:
        print(f"FAIL {len(failed)} of {len(cmds)} command(s):")
        for c in failed:
            print(f"  - {c if len(c) <= 200 else c[:197] + '...'}")
        return 1
    print(f"ok all {len(cmds)} command(s)")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))