Skip to content

The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.

27.1 KB
Raw Blame
#!/usr/bin/env python3
"""Embodied First Exam runs on Harbor. `uv tool install --force -e . --with-executables-from harbor` (in a clone) or
`pip install embodied-first-exam` installs `harbor` (Harbor 0.23.0, plus the flags below on `harbor run`) and `efe` (the
same, `efe run ...`). Run from anywhere. The tasks: $EFE_ROOT, else the clone that contains the working directory, else
the clone of an editable install, else a shallow clone of the release tag (v<version> as pyproject.toml writes it, matched
to the installed version after PEP 440 normalization) made on first use in ~/.cache/embodied-first-exam/<version>
(find_root).

  harbor run -a codex -m openai/gpt-6-astra            # core: each core task in its default mode (the default)
  harbor run --core --privileged -a codex -m openai/gpt-6-astra   # the core tasks in another mode (also --standard, --realtime)
  harbor run --all -a codex -m openai/gpt-6-astra      # every task dir
  harbor run --all --privileged -i 'robotwin_2_0_*' -a oracle          # mode flags and -i / -x filter within the set
  harbor run --benchmark roboarc -a codex -m openai/gpt-6-astra   # one benchmark's core tasks
  harbor run -d embodied-first-exam@core -a codex -m openai/gpt-6-astra   # the same sets as a dataset: @core, @all
  c=$(python3 scripts/stamp_mtimes.py stage tasks/<benchmark>/<task> "$(mktemp -d)") && harbor run -p "$c" -a nop
                                                      # one task, from a stamped copy: Harbor's own -p, untouched
  python3 scripts/stamp_mtimes.py tasks/<benchmark> && harbor run -p tasks/<benchmark> -a nop
                                                      # every task of one benchmark, stamped in place: Harbor's own -p

Set: --core (default) or --all. core = the tasks of docs/core_set.toml, each in its default_mode; with mode flags, each
core task in those modes (a task without that mode is skipped, and counted). all = every task dir; mode flags filter it.
Modes: any of --privileged, --standard, --realtime.
-d embodied-first-exam[@core|@all] (also `efe`) is --core / --all (no version: core); any other -d is Harbor's.
--benchmark <name> keeps a benchmark: its README label or its folder under tasks/ (repeatable). With any of these flags,
-p <dir> (a task dir, a benchmark folder or tasks/) narrows the set to the task dirs under it, -l <n> keeps the first n of
them, and -c <file> is layered after the wrapper's job config. --list prints the task dirs and runs nothing. The tasks
are stamped in place (scripts/stamp_mtimes.py) and handed to Harbor as one dataset per benchmark folder (Harbor's
local datasets are not searched recursively). A
`harbor run` that names its own tasks (-p, -d, -t, -c) and none of these flags is Harbor's, unchanged; so is every other
subcommand. Before any `harbor run` on docker or podman (-e), the images Harbor's egress allowlist pulls by a short name
are pulled by their full name if missing (prepull). With -e podman, the picked tasks run from stamped copies where needed
(podman_copies): GPU requests as CDI devices (podman ignores `driver: nvidia`), and a base image's recipe stage as
`FROM` the prebuilt image when podman can pull it (podman ignores `cache_from`; EFE_PODMAN_BUILD_FROM_SOURCE=1: off)."""
import fnmatch, json, os, re, shutil, subprocess, sys, tempfile
from pathlib import Path

# --- the task tree: where the tasks and their scripts/ come from -------------------------------------------
REPO_URL = "https://github.com/mll-lab-nu/embodied-first-exam.git"
_ROOT: list[Path] = []


def is_tree(d: Path) -> bool:
    """A checkout of this repository: the core set, the tasks and the scripts that read them."""
    return (d / "docs/core_set.toml").is_file() and (d / "tasks").is_dir() and (d / "scripts/task_conventions.py").is_file()


def cache_dir() -> Path:
    from importlib.metadata import PackageNotFoundError, version
    try:
        ver = version("embodied-first-exam")
    except PackageNotFoundError:
        ver = "main"
    return Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "embodied-first-exam" / ver


_PEP440 = re.compile(r"v?(?:(?P<epoch>\d+)!)?(?P<release>\d+(?:\.\d+)*)"
                     r"(?:[-_.]?(?P<pre_l>alpha|a|beta|b|preview|pre|c|rc)[-_.]?(?P<pre_n>\d*))?"
                     r"(?P<post>-(?P<post_n1>\d+)|[-_.]?(?:post|rev|r)[-_.]?(?P<post_n2>\d*))?"
                     r"(?P<dev>[-_.]?dev[-_.]?(?P<dev_n>\d*))?(?:\+(?P<local>[a-z0-9]+(?:[-_.][a-z0-9]+)*))?")
_PRE = {"alpha": "a", "a": "a", "beta": "b", "b": "b", "c": "rc", "rc": "rc", "pre": "rc", "preview": "rc"}


def version_key(v: str):
    """A version string after PEP 440 normalization (standard library only), as a comparable key: `1.0.0-preview`,
    `v1.0.0rc0` and `1.0.0rc0` give the same key; None when v is no PEP 440 version."""
    m = _PEP440.fullmatch(v.strip().lower())
    if not m:
        return None
    g = m.groupdict()
    release = [int(x) for x in g["release"].split(".")]
    while len(release) > 1 and release[-1] == 0:          # 1.0 == 1.0.0
        release.pop()
    return (int(g["epoch"] or 0), tuple(release),
            (_PRE[g["pre_l"]], int(g["pre_n"] or 0)) if g["pre_l"] else None,
            int(g["post_n1"] or g["post_n2"] or 0) if g["post"] else None,
            int(g["dev_n"] or 0) if g["dev"] else None,
            re.sub(r"[-_]", ".", g["local"]) if g["local"] else None)


def release_tag(version: str, tags) -> str | None:
    """The tag among tags (names, without refs/tags/) whose PEP 440 normalized version equals version's: the release tag
    is v<pyproject version> as written (e.g. v1.0.0-preview), the installed version its normalized form (1.0.0rc0)."""
    key = version_key(version)
    if key is None:
        return None
    hits = sorted(t for t in tags if version_key(t) == key)
    return next((t for t in hits if t == f"v{version}"), hits[0] if hits else None)


def cached_clone(d: Path) -> Path:
    """`git clone --depth 1` of the release tag of version d.name (the tag whose PEP 440 normalized version equals it,
    release_tag; main when there is no such tag) into d, once; SystemExit(2) on failure."""
    try:
        r = subprocess.run(["git", "ls-remote", "--tags", REPO_URL], capture_output=True, text=True)
    except FileNotFoundError:
        print("efe: the tasks need git (no clone here, no EFE_ROOT)", file=sys.stderr); raise SystemExit(2)
    if r.returncode:
        print(f"efe: cannot reach {REPO_URL} (git ls-remote exit {r.returncode}); clone it and run from there, or set "
              "EFE_ROOT", file=sys.stderr); raise SystemExit(2)
    tags = {ln.split("refs/tags/", 1)[1].removesuffix("^{}") for ln in r.stdout.splitlines() if "refs/tags/" in ln}
    tag = "main" if d.name == "main" else release_tag(d.name, tags)
    if tag is None:
        print(f"efe: {REPO_URL} has no release tag for version {d.name}: using main instead (delete {d} to clone again "
              "later)", file=sys.stderr)
        tag = "main"
    print(f"efe: no clone here and no EFE_ROOT: cloning the tasks ({tag}) into {d}, once", file=sys.stderr)
    d.parent.mkdir(parents=True, exist_ok=True)
    tmp = tempfile.mkdtemp(prefix=f".{d.name}-", dir=d.parent)
    try:
        if subprocess.run(["git", "clone", "--depth", "1", "--branch", tag, REPO_URL, tmp]).returncode:
            print(f"efe: git clone of {tag} failed", file=sys.stderr); raise SystemExit(2)
        if not is_tree(d):                                  # else another run made it meanwhile
            try:
                os.rename(tmp, d)
            except OSError as e:
                print(f"efe: cannot place the clone at {d} ({e}); remove {d} and run again", file=sys.stderr)
                raise SystemExit(2)
    finally:
        shutil.rmtree(tmp, ignore_errors=True)
    return d


def find_root(clone: bool = True) -> Path | None:
    """The task tree: $EFE_ROOT; else the checkout that contains the working directory; else the checkout this file is in
    (`uv tool install -e .`); else the cached clone (made here on first use when clone; None when not)."""
    if _ROOT:
        return _ROOT[0]
    cwd = Path.cwd().resolve()
    if os.environ.get("EFE_ROOT"):
        d = Path(os.environ["EFE_ROOT"]).expanduser().resolve()
        if not is_tree(d):
            print(f"EFE_ROOT={d}: not a checkout of {REPO_URL}", file=sys.stderr); raise SystemExit(2)
    else:
        d = next((p for p in (cwd, *cwd.parents, Path(__file__).resolve().parents[1], cache_dir()) if is_tree(p)), None)
        if d is None and clone:
            d = cached_clone(cache_dir())
    if d is not None:
        _ROOT.append(d)
    return d


class _Scripts:
    """The task tree's own scripts/<name>.py (task_conventions, stamp_mtimes), imported on first use."""
    def __init__(self, name: str):
        self._name = name

    def __getattr__(self, attr: str):
        import importlib
        d = str(find_root() / "scripts")
        if d not in sys.path:
            sys.path.insert(0, d)
        return getattr(importlib.import_module(self._name), attr)


tc = _Scripts("task_conventions")
# ------------------------------------------------------------------------------------------------------------------------

MODES = {"--privileged": "privileged", "--privilege": "privileged", "--standard": "standard",
         "--realtime": "realtime"}
DATASET = ("embodied-first-exam", "efe")
SOURCES = ("-p", "--path", "-d", "--dataset", "-t", "--task", "-c", "--config", "--task-git-url")


def qualified(ref: str) -> str:
    """A Docker Hub short name with its registry: alpine:3 -> docker.io/library/alpine:3, a/b -> docker.io/a/b; a ref
    that names its registry (a first component with a dot or a port, or localhost) is unchanged."""
    name = ref.split("@", 1)[0]
    first, slash, _ = name.partition("/")
    if slash and ("." in first or ":" in first or first == "localhost"):
        return ref
    return ("docker.io/" if slash else "docker.io/library/") + ref


def engine_of(argv: list[str]) -> str | None:
    """The container engine of a `harbor run` command line: docker (no -e, or -e docker), podman (-e podman), None for
    any other environment (a cloud sandbox, a custom class)."""
    env, it = "docker", iter(argv)
    for a in it:
        if a in ("-e", "--env"):
            env = next(it, "")
        elif a.startswith("--env="):
            env = a.split("=", 1)[1]
    return env if env in ("docker", "podman") else None


def short_name_images(engine: str) -> list[str]:
    """The images Harbor 0.23 pulls by a short name before a trial whose task restricts egress (`network_mode =
    "allowlist"`, every task's default): the kernel probe's alpine (docker and podman: `container run` with a 30 s timeout,
    and a slow pull is taken for "allowlist unsupported" for the whole job) and, on podman, the egress sidecar's base
    image (its Dockerfile's FROM; podman's enforcing short-name mode refuses a short name without a terminal, unless the
    image is already in its store). Read from the installed Harbor; [] if it has neither."""
    try:
        from harbor.environments.docker import EGRESS_CONTROL_SIDECAR_CONTEXT_PATH as ctx
        from harbor.environments.docker.docker import DockerEnvironment
    except Exception:
        return []
    refs = [r for r in [getattr(DockerEnvironment, "_EGRESS_CONTROL_KERNEL_PROBE_IMAGE", None)] if r]
    if engine == "podman":
        try:
            lines = (Path(ctx) / "Dockerfile").read_text().splitlines()
        except OSError:
            lines = []
        for ln in lines:
            w = [x for x in ln.split() if not x.startswith("--")]   # FROM [--platform=...] <image> [AS <stage>]
            if len(w) > 1 and w[0].upper() == "FROM" and w[1] != "scratch" and "$" not in w[1]:
                refs.append(w[1])
    return [qualified(r) for r in refs]


def prepull(engine: str | None, refs: list[str] | None = None, run=subprocess.run) -> list[str]:
    """Pull each of `refs` (default: short_name_images(engine)) that the engine does not have, by its full name, quietly
    (one line per pull). Returns the refs it could not pull; Harbor then reports its own error."""
    if engine is None or not shutil.which(engine):
        return []
    failed = []
    for ref in short_name_images(engine) if refs is None else refs:
        try:
            if run([engine, "image", "inspect", ref], capture_output=True, timeout=60).returncode == 0:
                continue
            print(f"pulling {ref} (Harbor's egress allowlist)", file=sys.stderr)
            if run([engine, "pull", "-q", ref], capture_output=True, timeout=900, stdin=subprocess.DEVNULL).returncode:
                failed.append(ref)
        except (OSError, subprocess.SubprocessError):
            failed.append(ref)
    for ref in failed:
        print(f"could not pull {ref} (`{engine} pull {ref}` tells why)", file=sys.stderr)
    return failed


def harbor(args: list[str], app=None) -> int:
    if app is None:
        from harbor.cli import main as hm
        app = getattr(hm.app, "__wrapped_app__", hm.app)
    sys.argv = ["harbor", *args]
    return app() or 0


def bench_label(name: str) -> str | None:
    """A --benchmark value as its BENCHMARKS label: the label itself (any case) or its folder under tasks/."""
    by = {**{lab.lower(): lab for lab in tc.BENCHMARK_DIRS}, **{f: lab for lab, f in tc.BENCHMARK_DIRS.items()}}
    return by.get(name) or by.get(name.lower())


def under(paths: list[str]) -> set[str] | None:
    """The task dir names under the -p paths (a task dir, a benchmark folder, tasks/; a relative path that is not one
    here is taken in this clone, so `-p tasks/<benchmark>` works from anywhere); None: no -p given. SystemExit(2): a path
    that is neither."""
    if not paths:
        return None
    out: set[str] = set()
    for a in paths:
        d = Path(a).expanduser()
        if not d.is_dir() and not d.is_absolute() and (find_root() / d).is_dir():
            d = find_root() / d
        if not d.is_dir():
            print(f"-p {a}: no such directory (here or in {find_root()})", file=sys.stderr)
            raise SystemExit(2)
        d = d.resolve()
        if (d / "task.toml").is_file():
            out.add(d.name)
        else:
            out |= {t.parent.name for pat in ("*/task.toml", "*/*/task.toml") for t in d.glob(pat)}
    return out


def stamp(pick: list[str]) -> bool:
    """Give the picked task dirs' files their content-derived mtimes in place before Harbor builds any image from them
    (scripts/stamp_mtimes.py, issue #258: a checkout's shared mtimes let BuildKit put another task's file in an image).
    Files already stamped are skipped. False: a file could not be stamped."""
    stamp_mtimes = _Scripts("stamp_mtimes")
    err = sum(stamp_mtimes.stamp_tree(str(tc.task_dir(t)))["err"] for t in pick)
    if err:
        print(f"stamp_mtimes: {err} file(s) could not be stamped; no image is built from an unstamped task "
              "(docs/protocol/running.md, Building task images)", file=sys.stderr)
    return not err


GPU_REQUEST = re.compile(r"^(\s*)- driver: nvidia\n(\s*)count: \d+\n", re.M)


def cdi_compose(text: str, device: str) -> str:
    """A compose file's NVIDIA device requests (`- driver: nvidia` / `count: N`) as CDI device requests of `device`:
    podman ignores the first form, so the task's processes find no driver."""
    return GPU_REQUEST.sub(lambda m: f'{m[1]}- driver: cdi\n{m[2]}device_ids: ["{device}"]\n', text)


BASE_STAGE = re.compile(r"(?ms)^# >>> images/([A-Za-z0-9_.-]+)/Dockerfile\n(.*?)^# <<< images/\1/Dockerfile\n")
STAGE_FROM = re.compile(r"(?im)^\s*FROM\s+(?:--\S+\s+)*\S+\s+AS\s+(\S+)\s*$")
CACHE_FROM = re.compile(r"(?m)^\s*cache_from:\s*\[([^\]]*)\]")


def cache_refs(task: Path) -> dict[str, str]:
    """The prebuilt base images a task's compose files name as build cache (`build.cache_from`), by image name:
    {"vlabench": "ghcr.io/mll-lab-nu/efe-private:vlabench-0.2.0"} (the tag is <name>-<version>)."""
    refs = {}
    for f in task.rglob("docker-compose*.y*ml"):
        for inner in CACHE_FROM.findall(f.read_text(errors="replace")):
            for r in re.findall(r"[^\s\"',]+", inner):
                m = re.search(r":([A-Za-z0-9_.-]+?)-(\d[A-Za-z0-9_.-]*)$", r.split("@", 1)[0])
                if m:
                    refs.setdefault(m[1], r)
    return refs


def from_prebuilt(text: str, refs: dict[str, str]) -> str:
    """A Dockerfile with each base stage (`# >>> images/<name>/Dockerfile` ... `# <<< ...`, the recipe of the image
    efe-private:<name>-<version>) whose image is in `refs` replaced by `FROM <that image> AS <its last stage>`: the
    image is that stage, built. A block whose other stages are used after it is kept (it is not the image alone)."""
    def block(m: "re.Match[str]") -> str:
        ref, stages = refs.get(m[1]), STAGE_FROM.findall(m[2])
        if not ref or not stages:
            return m[0]
        rest = text[m.end():]
        for s in set(stages[:-1]) - {stages[-1]}:
            if re.search(rf"(?im)^\s*FROM\s+(?:--\S+\s+)*{re.escape(s)}(\s|$)|--from={re.escape(s)}(\s|$)", rest):
                return m[0]
        return (f"# images/{m[1]}/Dockerfile: the prebuilt image, pulled (scripts/efe.py from_prebuilt; "
                f"EFE_PODMAN_BUILD_FROM_SOURCE=1 builds the recipe instead)\nFROM {ref} AS {stages[-1]}\n")
    return BASE_STAGE.sub(block, text)


def pulled(ref: str, have: dict[str, bool], run=subprocess.run) -> bool:
    """podman has `ref` (already, or after `podman pull`; the private images need a `podman login ghcr.io`). Asked once
    per ref (`have`)."""
    if ref not in have:
        ok = False
        try:
            ok = run(["podman", "image", "inspect", ref], capture_output=True, timeout=60).returncode == 0
            if not ok:
                print(f"pulling {ref} (podman: the prebuilt base image instead of its recipe)", file=sys.stderr)
                ok = run(["podman", "pull", "-q", ref], capture_output=True, stdin=subprocess.DEVNULL).returncode == 0
                if not ok:
                    print(f"could not pull {ref}: building its recipe from source (`podman pull {ref}` tells why)",
                          file=sys.stderr)
        except (OSError, subprocess.SubprocessError):
            ok = False
        have[ref] = ok
    return have[ref]


def podman_copies(pick: list[str], out: str, run=subprocess.run) -> dict[str, str]:
    """For -e podman: a stamped copy under <out>/<benchmark folder>/ of each picked task that (a) requests an NVIDIA GPU
    in its compose files, with those requests in CDI form (cdi_compose; device EFE_PODMAN_GPUS, default
    nvidia.com/gpu=all: a CDI spec of the host's GPUs, docs/protocol/running.md), or (b) builds a base image's recipe
    (a `# >>> images/<name>/Dockerfile` stage with `build.cache_from` efe-private:<name>-<version>) that podman can pull:
    those stages become `FROM <the image>` (from_prebuilt; podman ignores cache_from and would rebuild the recipe). Not
    (b) with EFE_PODMAN_BUILD_FROM_SOURCE=1 or when the pull fails. The task dirs in the repository are not changed.
    Returns {task name: copy}."""
    stamp_mtimes = _Scripts("stamp_mtimes")
    device = os.environ.get("EFE_PODMAN_GPUS") or "nvidia.com/gpu=all"
    pull_first = os.environ.get("EFE_PODMAN_BUILD_FROM_SOURCE", "") in ("", "0") and bool(shutil.which("podman"))
    have: dict[str, bool] = {}
    copies = {}
    for t in pick:
        src = tc.task_dir(t)
        composes = [f for f in src.rglob("docker-compose*.y*ml") if "driver: nvidia" in f.read_text(errors="replace")]
        stages: dict[Path, str] = {}
        if pull_first:
            refs = cache_refs(src)
            for sub in ("environment", "tests"):
                for df in sorted((src / sub).rglob("Dockerfile")) if (src / sub).is_dir() else ():
                    text = df.read_text(errors="replace")
                    names = sorted({m[1] for m in BASE_STAGE.finditer(text)} & refs.keys())
                    usable = {n: refs[n] for n in names if pulled(refs[n], have, run)}
                    new = from_prebuilt(text, usable) if usable else text
                    if new != text:
                        stages[df] = new
        if not composes and not stages:
            continue
        dst = Path(stamp_mtimes.stage(str(src), os.path.join(out, src.parent.name)))
        for f in composes:
            c = dst / f.relative_to(src)
            c.write_text(cdi_compose(c.read_text(), device))
        for f, new in stages.items():
            (dst / f.relative_to(src)).write_text(new)
        stamp_mtimes.stamp_tree(str(dst), full=True)
        copies[t] = str(dst)
    return copies


def config_file(pick: list[str], where: dict[str, str] | None = None) -> str:
    """A Harbor job config (-c) with the picked task dirs (or their copies, `where`): one local dataset per benchmark
    folder, its task_names the picked dirs. Harbor reads it when the job is built; the caller removes it afterwards."""
    by: dict[str, list[str]] = {}
    for t in pick:
        by.setdefault(str(Path((where or {}).get(t) or tc.task_dir(t)).parent), []).append(t)
    cfg = {"datasets": [{"path": d, "task_names": names} for d, names in sorted(by.items())]}
    fd, path = tempfile.mkstemp(prefix="efe-job-", suffix=".json")
    with os.fdopen(fd, "w") as f:
        json.dump(cfg, f, indent=1)
    return path


def load_env() -> None:
    """The working directory's and the clone's .env (cp .env.example .env) into the environment: their non-empty keys
    the shell does not set (no clone is made for it)."""
    from dotenv import dotenv_values
    root = find_root(clone=False)
    for f in dict.fromkeys([Path.cwd() / ".env", *([root / ".env"] if root else [])]):
        for k, v in dotenv_values(f).items() if f.is_file() else ():
            if v and not v.startswith("#") and k not in os.environ:   # `KEY=   # note`: dotenv reads the note as the value
                os.environ[k] = v


def dataset_sets(argv: list[str]) -> list[str] | None:
    """argv with -d embodied-first-exam[@core|@all] turned into --core / --all; None: an unknown version."""
    out, it = [], iter(argv)
    for a in it:
        v = a.split("=", 1)[1] if a.startswith("--dataset=") else next(it, "") if a in ("-d", "--dataset") else None
        name, _, ver = (v or "").partition("@")
        if v is None or name not in DATASET:
            out += [a] if v is None or a.startswith("--dataset=") else [a, v]
        elif ver in ("", "core", "all"):
            out.append("--" + (ver or "core"))
        else:
            print(f"-d {v}: the versions are {name}@core and {name}@all", file=sys.stderr)
            return None
    return out


def run(argv: list[str], app=None) -> int:
    """`harbor run` with the set / mode / benchmark flags: picks the task dirs, then `harbor run -c <job config> ...`
    (config_file: one dataset per benchmark folder)."""
    argv = dataset_sets(argv)
    if argv is None:
        return 2
    load_env()
    ours = ("--core", "--all", "--benchmark", "--list")
    if any(a in SOURCES or a.startswith(tuple(s + "=" for s in SOURCES if s.startswith("--"))) for a in argv):
        if not any(a in MODES or a in ours or a.startswith("--benchmark=") for a in argv):
            if not {"-h", "--help"} & set(argv):
                prepull(engine_of(argv))
            return harbor(["run", *argv], app)                # the user's own task source: plain Harbor
    which, modes, benches, inc, exc, paths, listing, rest = "core", set(), [], [], [], [], False, []
    n_tasks = None
    it = iter(argv)
    for a in it:
        if a in ("--all", "--core"):
            which = a[2:]
        elif a in MODES:
            modes.add(MODES[a])
        elif a == "--benchmark" or a.startswith("--benchmark="):
            b = a.split("=", 1)[1] if "=" in a else next(it, "")
            lab = bench_label(b)
            if lab is None:
                print(f"--benchmark {b!r}: not a benchmark (a README label or a folder under tasks/: "
                      f"{', '.join(sorted(tc.BENCHMARK_DIRS.values()))})", file=sys.stderr)
                return 2
            benches.append(lab)
        elif a in ("-i", "--include-task-name") or a.startswith("--include-task-name="):
            inc.append(a.split("=", 1)[1] if "=" in a else next(it, ""))
        elif a in ("-x", "--exclude-task-name") or a.startswith("--exclude-task-name="):
            exc.append(a.split("=", 1)[1] if "=" in a else next(it, ""))
        elif a in ("-l", "--n-tasks") or a.startswith("--n-tasks="):   # Harbor's -l: applied here, to the picked set
            v = a.split("=", 1)[1] if "=" in a else next(it, "")
            if not v.isdigit():
                print(f"{a} {v!r}: the number of tasks to run", file=sys.stderr); return 2
            n_tasks = int(v)
        elif a in ("-p", "--path") or a.startswith("--path="):
            paths.append(a.split("=", 1)[1] if "=" in a else next(it, ""))
        elif a in ("-c", "--config"):
            rest += [a, next(it, "")]                          # the user's config: layered after ours
        elif a in SOURCES or a.startswith(tuple(s + "=" for s in SOURCES if s.startswith("--") and s != "--config")):
            print(f"{a}: names its own tasks; it does not combine with {', '.join((*ours, *MODES))}", file=sys.stderr)
            return 2
        elif a == "--list":
            listing = True
        elif a in ("-h", "--help"):
            print(__doc__); return harbor(["run", "--help"], app)
        else:
            rest.append(a)
    tasks = [t.name for t in tc.iter_tasks()]
    missing: dict[str, list[str]] = {}
    pick = tasks if which == "all" else sorted(set(tc.core_tasks(modes, missing=missing)) & set(tasks))
    for m, ts in sorted(missing.items()):
        print(f"{len(ts)} core task(s) have no {m} mode: skipped", file=sys.stderr)
    scope = under(paths)
    pick = [t for t in pick if (scope is None or t in scope)
            and (not modes or tc.mode_of(Path(t)) in modes)
            and (not benches or tc.bench_of(Path(t)) in benches)
            and (not inc or any(fnmatch.fnmatchcase(t, p) for p in inc))
            and not any(fnmatch.fnmatchcase(t, p) for p in exc)]
    pick = pick[:n_tasks] if n_tasks is not None else pick
    if not pick:
        print("no task matches", file=sys.stderr); return 2
    print(f"{len(pick)} task(s) of " + which + (f", {', '.join(sorted(modes))}" if modes else "")
          + (f", {', '.join(sorted(set(benches)))}" if benches else ""), file=sys.stderr)
    if listing:
        print("\n".join(pick)); return 0
    if not stamp(pick):
        return 2
    engine = engine_of(rest)
    prepull(engine)
    staged = tempfile.mkdtemp(prefix="efe-podman-") if engine == "podman" else None
    try:
        where = podman_copies(pick, staged) if staged else {}
        if where:
            print(f"{len(where)} task(s) run from copies (podman): GPU requests in CDI form, prebuilt base images "
                  "pulled instead of their recipes", file=sys.stderr)
        cfg = config_file(pick, where)
        try:
            return harbor(["run", "-c", cfg, *rest], app)
        finally:
            os.unlink(cfg)
    finally:
        if staged:
            shutil.rmtree(staged, ignore_errors=True)


def main() -> int:
    """The `efe` command: `efe run ...` = `harbor run ...` above, `efe` alone prints this help."""
    argv = sys.argv[1:]
    if not argv or argv[0] in ("-h", "--help"):
        print(__doc__.replace("harbor run", "efe run")); return 0
    return run(argv[1:]) if argv[0] == "run" else harbor(argv)


if __name__ == "__main__":
    sys.exit(main())