Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
26.8 KB
Raw Blame
#!/usr/bin/env python3
"""graff-evals: a self-contained eval environment for coding harnesses.

Every task is a JSON file under tasks/ that declares its fixture files, the
prompt, and a deterministic shell check. The runner materializes a sandbox,
drives any configured harness (harnesses.json) with any model, captures wall
time / first-output latency / token usage / peak RSS / CPU, verifies the
outcome, and writes JSONL results plus a summary table.

  ./run.py --harness graff --model grok-4.6              # full suite (core + rlm + swe)
  ./run.py --suite rlm --harness graff-dev-old,graff-dev # scatter-gather A/B
  ./run.py --suite swe --harness graff-dev-old,graff-dev -j 12  # DeepSWE-shaped A/B, parallel
  ./run.py --suite swe --harness graff-dev,pi-xai --model grok-4.6 -j 6  # same SuperGrok seat
  ./run.py --suite mcp --harness graff-dev-old-nolean,graff-dev-rlm-struct,graff-dev-nolean
  ./run.py --suite inhouse --harness graff-dev,grok,opencode  # distilled PR fixtures
  ./run.py --suite live --harness graff-dev --reps 3      # gated live PRs (no SPEC.md)
  ./run.py --harness grok --task fix-fib --reps 3        # one task, 3 reps
  ./run.py --interactive                                 # pick + watch live
"""
import argparse, codecs, json, os, re, resource, shutil, subprocess, sys, threading, time
from concurrent.futures import ThreadPoolExecutor, as_completed

from list_price import attach as attach_list_price
import report as eval_report
import measurement
import request_capture

ROOT = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(ROOT)
TASKS_DIR = os.path.join(ROOT, "tasks")
RESULTS_DIR = os.path.join(ROOT, "results")
SANDBOX_DIR = os.path.join(ROOT, ".sandboxes")

GRAFF_USAGE_RE = re.compile(
    r"\[usage\] (?:known subtotal: )?(\d+) api call\(s\) · (\d+) in \((\d+) cached(?:, (\d+) cache writes)?\) \+ (\d+) out tokens")


def load_tasks():
    tasks = {}
    for name in sorted(os.listdir(TASKS_DIR)):
        if not name.endswith(".json"):
            continue
        with open(os.path.join(TASKS_DIR, name)) as f:
            t = json.load(f)
        tasks[t["id"]] = t
    return tasks


def load_harnesses():
    with open(os.path.join(ROOT, "harnesses.json")) as f:
        return json.load(f)


def materialize(task, sandbox):
    if os.path.exists(sandbox):
        shutil.rmtree(sandbox)
    os.makedirs(sandbox)
    files_dir = task.get("files_dir")
    if files_dir:
        src = files_dir if os.path.isabs(files_dir) else os.path.join(ROOT, files_dir)
        if os.path.isdir(src):
            for dirpath, dirnames, filenames in os.walk(src):
                dirnames[:] = [d for d in dirnames if d != "__pycache__"]
                rel = os.path.relpath(dirpath, src)
                dest_dir = sandbox if rel == "." else os.path.join(sandbox, rel)
                os.makedirs(dest_dir, exist_ok=True)
                for name in filenames:
                    if name.endswith(".pyc") or name == ".DS_Store":
                        continue
                    shutil.copy2(os.path.join(dirpath, name), os.path.join(dest_dir, name))
    for rel, content in task.get("files", {}).items():
        path = os.path.join(sandbox, rel)
        os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
        with open(path, "w") as f:
            f.write(content)
    setup_timeout = int(task.get("setup_timeout_s", 60))
    for cmd in task.get("setup", []):
        subprocess.run(["/bin/sh", "-c", cmd], cwd=sandbox, capture_output=True, timeout=setup_timeout,
                       check=task.get("strict_setup", False), env=dict(os.environ, TASK_ROOT=ROOT))


def _resolve_model(harness, model):
    mapped = (harness.get("model_map") or {}).get(model, model)
    prefix = harness.get("model_prefix")
    if prefix and mapped and "/" not in mapped:
        return f"{prefix}/{mapped}"
    return mapped


def build_cmd(harness, task, model, sandbox=""):
    subst = {"prompt": task["prompt"], "model": _resolve_model(harness, model), "repo": REPO, "sandbox": sandbox}
    cmd = [part.format(**subst) for part in harness["cmd"]]
    if harness.get("_provider"):
        cmd = measurement.provider_command(cmd)
    if "output-schema" in task.get("requires", []):
        schema = json.dumps(task["schema"], separators=(",", ":"))
        cmd += [part.format(schema=schema, **subst) for part in harness.get("schema_args", [])]
    stdin = harness.get("stdin")
    if stdin is not None:
        stdin = stdin.format(**subst)
    return cmd, stdin


def learn_pin_info(sandbox):
    """Hardlink vs 127M copy: nlink>=2 and same inode as the live exe."""
    pin = os.path.join(sandbox, ".graff", "learn-kit", "graff-pinned")
    if not os.path.exists(pin):
        return None
    st = os.stat(pin)
    info = {"bytes": st.st_size, "nlink": st.st_nlink, "ino": st.st_ino}
    exe = os.path.join(REPO, "zig-out", "bin", "graff")
    if os.path.exists(exe):
        info["same_inode"] = os.stat(exe).st_ino == st.st_ino
    return info


def _parse_opencode_json(stdout):
    """OpenCode `run --format json` NDJSON: text / tool_use / step_finish."""
    answer, calls, tin, tread, twrite, tout, cost = "", 0, 0, 0, 0, 0, 0.0
    for line in stdout.splitlines():
        line = line.strip()
        if not line.startswith("{"):
            continue
        try:
            ev = json.loads(line)
        except json.JSONDecodeError:
            continue
        kind = ev.get("type")
        part = ev.get("part") or {}
        if kind == "text":
            text = (part.get("text") or ev.get("text") or "").strip()
            if text:
                answer = text
        elif kind == "tool_use":
            calls += 1
        elif kind == "step_finish":
            tok = part.get("tokens") or {}
            cache = tok.get("cache") or {}
            tin += int(tok.get("input") or 0)
            tout += int(tok.get("output") or 0) + int(tok.get("reasoning") or 0)
            tread += int(cache.get("read") or 0)
            twrite += int(cache.get("write") or 0)
            cost += float(part.get("cost") or 0)
            if not calls:
                calls += 1
    usage = {"calls": calls, "in": tin + tread + twrite, "cached": tread,
             "out": tout, "writes": twrite, "cost_usd": round(cost, 6)}
    return answer, usage


def _parse_gemini_json(stdout):
    answer = ""
    usage = {}
    idx = stdout.find("{")
    if idx != -1:
        try:
            data = json.loads(stdout[idx:])
            answer = (data.get("response") or "").strip()
            stats = data.get("stats") or {}
            models = stats.get("models") or {}
            tin, tread, tout, requests = 0, 0, 0, 0
            for mdata in models.values():
                tok = mdata.get("tokens") or {}
                tin += int(tok.get("input") or 0)
                tread += int(tok.get("cached") or 0)
                tout += int(tok.get("candidates") or 0) + int(tok.get("thoughts") or 0)
                api = mdata.get("api") or {}
                requests += int(api.get("totalRequests") or 0)
            tools = stats.get("tools") or {}
            calls = int(tools.get("totalCalls") or 0)
            if not calls and requests > 0:
                calls = requests
            usage = {
                "calls": calls,
                "in": tin + tread,
                "cached": tread,
                "out": tout,
            }
        except json.JSONDecodeError:
            pass
    if not answer:
        answer = stdout.strip()
    return answer, usage


def _parse_agy_json(stdout):
    """Antigravity CLI `-p --output-format json`: one object; input_tokens excludes the
    cache_read part and output_tokens includes thinking. num_turns counts conversation
    turns, not model calls, so calls stays unknown."""
    answer, usage = stdout.strip(), {}
    for line in reversed(stdout.strip().splitlines()):
        try:
            data = json.loads(line)
        except json.JSONDecodeError:
            continue
        u = data.get("usage") or {}
        answer = (data.get("response") or "").strip()
        cached = int(u.get("cache_read_tokens") or 0)
        usage = {"in": int(u.get("input_tokens") or 0) + cached, "cached": cached,
                 "out": int(u.get("output_tokens") or 0), "reasoning": int(u.get("thinking_tokens") or 0)}
        break
    return answer, usage


def parse_answer_and_usage(harness, stdout, stderr, sandbox=""):
    answer, usage = stdout.strip(), {}
    if harness["answer"] == "grok-stream":
        answer = ""
        for line in stdout.splitlines():
            line = line.strip()
            if not line.startswith("{"):
                continue
            try:
                ev = json.loads(line)
            except json.JSONDecodeError:
                continue
            if ev.get("type") == "result":
                answer = ev.get("result") or ""
                u = ev.get("usage", {})
                stu = u.get("server_tool_use") or {}
                tools = {}
                if stu.get("web_search_requests"):
                    tools["web_search"] = stu["web_search_requests"]
                if stu.get("x_search_requests"):
                    tools["x_search"] = stu["x_search_requests"]
                if stu.get("code_execution_requests"):
                    tools["code_execution"] = stu["code_execution_requests"]
                usage = {"calls": ev.get("num_turns"), "in": u.get("input_tokens"),
                         "cached": u.get("cache_read_input_tokens"),
                         "out": u.get("output_tokens"), "api_ms": ev.get("duration_api_ms"),
                         "writes": u.get("cache_creation_input_tokens") or 0}
                if tools:
                    usage["tools"] = tools
    if harness["answer"] == "pi-json":
        answer, calls, tin, tread, twrite, tout, cost = "", 0, 0, 0, 0, 0, 0.0
        for line in stdout.splitlines():
            line = line.strip()
            if not line.startswith("{"):
                continue
            try:
                ev = json.loads(line)
            except json.JSONDecodeError:
                continue
            msg = ev.get("message", {})
            if ev.get("type") == "message_end" and msg.get("role") == "assistant":
                u = msg.get("usage", {})
                calls += 1
                tin += u.get("input", 0)
                tread += u.get("cacheRead", 0)
                twrite += u.get("cacheWrite", 0)
                tout += u.get("output", 0)
                cost += u.get("cost", {}).get("total", 0.0)
                answer = "".join(c.get("text", "") for c in msg.get("content", [])
                                 if c.get("type") == "text") or answer
        usage = {"calls": calls, "in": tin + tread + twrite, "cached": tread,
                 "out": tout, "cost_usd": round(cost, 6)}
    if harness["answer"] == "opencode-json":
        answer, usage = _parse_opencode_json(stdout)
    if harness["answer"] == "gemini-json":
        answer, usage = _parse_gemini_json(stdout)
    if harness["answer"] == "agy-json":
        answer, usage = _parse_agy_json(stdout)
    if harness["answer"] == "muse" or harness.get("usage") == "muse":
        answer = stdout.strip()
        usage = {}
        # Sidecar written by graff-evals/muse.sh: .muse-usage.json + template-hashed model task log
        import glob, json as _json, os as _os, subprocess as _sp
        side = _os.path.join(sandbox, ".muse-usage.json")
        try:
            u = _json.load(open(side))
            usage = {"calls": 1, "in": int(u.get("input_tokens",0))
                     + int(u.get("cached_tokens",0)),
                     "cached": int(u.get("cached_tokens",0)),
                     "out": int(u.get("output_tokens",0) or 0) + int(u.get("reasoning_tokens",0) or 0),
                     "reasoning": int(u.get("reasoning_tokens",0) or 0)}
        except Exception:
            pass
        # Calls = model turns, counted from the stashed exec stream.
        # muse's tracer can't decode exec-stream payloads (schema drift), so count
        # tool results directly: each tool result implies a preceding model turn, so
        # turns = tool_results + 1 (final answer) is a hard LOWER bound — favorable
        # to muse on the calls axis.
        try:
            raw = _os.path.join(sandbox, ".muse-raw.jsonl")
            tools = 0
            with open(raw) as fh:
                for line in fh:
                    try:
                        if _json.loads(line).get("payload_type") == "tool.result":
                            tools += 1
                    except Exception:
                        pass
            if tools:
                usage["calls"] = tools + 1
        except Exception:
            pass
        if not usage.get("calls"):
            usage["calls"] = 1
    if harness.get("usage") == "graff-stderr":
        usage = measurement.graff_usage(stderr)
    return answer, usage


def _status_kb(pid, key):
    try:
        with open(f"/proc/{pid}/status") as f:
            for line in f:
                if line.startswith(key):
                    return int(line.split()[1])
    except (OSError, ValueError, IndexError):
        return 0
    return 0


def _children(pid):
    try:
        with open(f"/proc/{pid}/task/{pid}/children") as f:
            return [int(x) for x in f.read().split()]
    except (OSError, ValueError):
        return []


def _walk_pids(pid):
    seen, stack = set(), [pid]
    while stack:
        p = stack.pop()
        if p in seen:
            continue
        seen.add(p)
        yield p
        stack.extend(_children(p))


def tree_rss_kb(pid):
    return sum(_status_kb(p, "VmRSS:") for p in _walk_pids(pid))


def tree_cpu_s(pid):
    ticks = os.sysconf("SC_CLK_TCK") or 100
    total = 0.0
    for p in _walk_pids(pid):
        try:
            with open(f"/proc/{p}/stat") as f:
                st = f.read()
            fields = st[st.rfind(")") + 2:].split()
            total += (int(fields[11]) + int(fields[12])) / ticks
        except (OSError, ValueError, IndexError):
            pass
    return total


def dir_bytes(path):
    total = 0
    for root, dirs, files in os.walk(path):
        # The repository isolate_git made holds a second copy of the task tree.
        dirs[:] = [d for d in dirs if not (root == path and d == ".git")]
        for name in files:
            try:
                total += os.path.getsize(os.path.join(root, name))
            except OSError:
                pass
    return total


def _rusage_children():
    u = resource.getrusage(resource.RUSAGE_CHILDREN)
    return u.ru_utime, u.ru_stime, u.ru_maxrss


def one_run(hname, harness, task, model, rep, live=False):
    sandbox = os.path.join(SANDBOX_DIR, f"{hname}-{task['id']}-r{rep}")
    try:
        materialize(task, sandbox)
        measurement.isolate_git(sandbox)
    except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
        return {"harness": hname, "task": task["id"], "suite": task.get("suite", "core"),
                "rep": rep, "error": f"environment setup failed: {error}", "outcome_ok": False}
    verifier_receipt = measurement.verifier_snapshot(sandbox, task, ROOT)
    cmd, stdin_body = build_cmd(harness, task, model, sandbox)
    timeout = task.get("timeout_s", 240)
    t0 = time.monotonic()
    first_out = None
    stdout_parts, stderr_parts = [], []
    rss_peak = 0
    cpu_sample = 0.0
    ru0 = _rusage_children()
    env = ({**os.environ, "PWD": sandbox, **{k: str(v).replace("{repo}", REPO).replace("{sandbox}", sandbox)
           for k, v in harness.get("env", {}).items()}})
    if harness.get("_provider"):
        env = measurement.provider_environment(harness["_provider"], sandbox, model=model)
    if harness.get("_capture_requests"):
        request_capture.configure(env, sandbox)
    try:
        p = subprocess.Popen(cmd, cwd=sandbox, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, stdin=subprocess.PIPE if stdin_body is not None else subprocess.DEVNULL,
                             text=True, start_new_session=True,
                             env=env)
        if stdin_body is not None and p.stdin is not None:
            try:
                p.stdin.write(stdin_body)
            except BrokenPipeError:
                pass
            p.stdin.close()
        import selectors
        sel = selectors.DefaultSelector()
        sel.register(p.stdout, selectors.EVENT_READ, "out")
        sel.register(p.stderr, selectors.EVENT_READ, "err")
        decoders = {k: codecs.getincrementaldecoder("utf-8")("replace") for k in ("out", "err")}
        open_streams = 2
        while open_streams and time.monotonic() - t0 < timeout:
            rss_peak = max(rss_peak, tree_rss_kb(p.pid))
            cpu_sample = max(cpu_sample, tree_cpu_s(p.pid))
            for key, _ in sel.select(timeout=0.5):
                raw = os.read(key.fd, 65536)
                chunk = decoders[key.data].decode(raw, final=not raw)
                if not raw:
                    sel.unregister(key.fileobj)
                    open_streams -= 1
                    continue
                if first_out is None:
                    first_out = round(time.monotonic() - t0, 2)
                (stdout_parts if key.data == "out" else stderr_parts).append(chunk)
                if live:
                    sys.stdout.write(chunk if key.data == "out" else f"\x1b[2m{chunk}\x1b[0m")
                    sys.stdout.flush()
        # OpenCode's `run --format json` often closes the pipes while a local
        # server is still up. That is not a task timeout. Give writes a moment
        # to land, then reap the leftover group. A real timeout is pipes still
        # open when the budget ends.
        hit_budget = time.monotonic() - t0 >= timeout
        if p.poll() is None and not hit_budget:
            time.sleep(1.5)
            rss_peak = max(rss_peak, tree_rss_kb(p.pid))
            try:
                os.killpg(p.pid, 15)
            except OSError:
                p.terminate()
            try:
                p.wait(timeout=4)
            except subprocess.TimeoutExpired:
                try:
                    os.killpg(p.pid, 9)
                except OSError:
                    p.kill()
                p.wait(timeout=5)
            timed_out = False
        elif p.poll() is None:
            timed_out = True
            try:
                os.killpg(p.pid, 9)
            except OSError:
                p.kill()
            p.wait(timeout=10)
        else:
            timed_out = False
        # Whatever the run left in its session (background jobs keep their own
        # process groups) must not run on into the next task.
        measurement.kill_session(p.pid)
        sel.close()
        p.stdout.close()
        p.stderr.close()
        rc = p.returncode
        rss_peak = max(rss_peak, tree_rss_kb(p.pid))
    except FileNotFoundError:
        return {"harness": hname, "task": task["id"], "suite": task.get("suite", "core"),
                "rep": rep, "error": f"harness binary not found: {cmd[0]}", "outcome_ok": False}
    ru1 = _rusage_children()
    stdout, stderr = "".join(stdout_parts), "".join(stderr_parts)
    wall = round(time.monotonic() - t0, 2)
    measurement.private_logs(sandbox, stdout, stderr)
    answer, usage = parse_answer_and_usage(harness, stdout, stderr, sandbox)
    with open(os.path.join(sandbox, ".eval-answer.txt"), "w") as f:
        f.write(answer)
    check_env = dict(os.environ, ANSWER_FILE=".eval-answer.txt", TASK_ROOT=ROOT)
    check_timeout = int(task.get("check_timeout_s", 60))
    try:
        check = subprocess.run(["/bin/sh", "-c", task["check"]], cwd=sandbox,
                               capture_output=True, text=True, timeout=check_timeout, env=check_env)
        check_note = None
        if check.returncode != 0 and (check.stderr.strip() or check.stdout.strip()):
            check_note = (check.stderr.strip() or check.stdout.strip())[:200]
        check_ok = check.returncode == 0
    except subprocess.TimeoutExpired:
        check_note = f"check timed out after {check_timeout}s"
        check_ok = False
    verifiers_ok = measurement.verifiers_unchanged(verifier_receipt)
    check_ok = check_ok and verifiers_ok
    artifact_ok = check_ok
    if task.get("requires_clean_exit"):
        check_ok = check_ok and rc == 0 and not timed_out
    rec = {"harness": hname, "task": task["id"], "suite": task.get("suite", "core"),
           "artifact_ok": artifact_ok, "environment_version": task.get("environment_version"), "grader_version": task.get("grader_version"),
           "category": task.get("category", ""), "model": model, "rep": rep,
           "wall_s": wall, "first_out_s": first_out, "exit": rc, "timed_out": timed_out,
           "outcome_ok": check_ok, "answer_head": answer[:120],
           "rss_peak_kb": rss_peak, "rss_child_hwm_kb": ru1[2],
           "cpu_user_s": round(max(0.0, ru1[0] - ru0[0]), 3),
           "cpu_sys_s": round(max(0.0, ru1[1] - ru0[1]), 3),
           "cpu_sample_s": round(cpu_sample, 3),
           "sandbox_bytes": dir_bytes(sandbox)}
    rec.update({f"tok_{k}": v for k, v in usage.items()})
    rec.update(measurement.receipt(cmd, harness, task))
    if harness.get("_capture_requests"):
        rec.update(request_capture.receipt(sandbox))
    rec["verifiers_unchanged"] = verifiers_ok
    if harness.get("_provider"):
        rec.update(measurement.trace_routing(sandbox, harness["_provider"], model))
        rec["outcome_ok"] = rec["outcome_ok"] and rec["routing_ok"]
    pin = learn_pin_info(sandbox)
    if pin is not None:
        rec["learn_pin"] = pin
    attach_list_price(rec, inclusive=harness.get("usage") != "grok-stream")
    if check_note:
        rec["check_note"] = check_note
    return rec


def interactive(tasks, harnesses):
    tlist = list(tasks.values())
    print("tasks:")
    for i, t in enumerate(tlist):
        print(f"  [{i}] {t['id']:<18} {t.get('suite', 'core'):<5} {t.get('category', '')}")
    ti = int(input("task #: ").strip() or "0")
    hnames = list(harnesses)
    for i, h in enumerate(hnames):
        print(f"  [{i}] {h}")
    hi = int(input("harness #: ").strip() or "0")
    hname = hnames[hi]
    harness = harnesses[hname]
    model = input(f"model [{harness.get('default_model', '')}]: ").strip() or harness.get("default_model", "")
    task = tlist[ti]
    print(f"\n── {task['id']} on {hname} ({model}) — live output ──\n")
    rec = one_run(hname, harness, task, model, rep=1, live=True)
    print("\n── verdict ──")
    print(json.dumps(rec, indent=2, ensure_ascii=False))


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--harness", default="graff", help="comma-separated harness names (see harnesses.json)")
    ap.add_argument("--output-root", help="new private directory for receipts and isolated sandboxes")
    ap.add_argument("--capture-requests", action="store_true", help="save private wire evidence for cache analysis")
    ap.add_argument("--binary", help="explicit graff executable for this arm")
    ap.add_argument("--arm", default="baseline", help="comparison arm recorded in every result")
    ap.add_argument("--provider", help="explicit graff provider; no automatic provider selection")
    ap.add_argument("--model", default=None, help="model id (default: per-harness default_model)")
    ap.add_argument("--task", action="append", help="task id filter (repeatable)")
    ap.add_argument("--suite", default="all",
                    help="core, rlm, swe, mcp, inhouse, live, comma-mix, or all (core+rlm+swe; mcp/inhouse/live are opt-in)")
    ap.add_argument("--reps", type=int, default=1)
    ap.add_argument("--interleave", action="store_true",
                    help="order runs task by task across harnesses, so A/B arms see the same backend conditions")
    ap.add_argument("--jobs", "-j", type=int, default=1,
                    help="parallel task×harness runs (default 1; each has its own sandbox)")
    ap.add_argument("--interactive", action="store_true", help="pick a task+harness, watch it live")
    args = ap.parse_args()

    tasks = load_tasks()
    harnesses = load_harnesses()
    if args.interactive:
        interactive(tasks, harnesses)
        return

    global SANDBOX_DIR
    out_path, SANDBOX_DIR = measurement.isolated_paths(args.output_root)
    code_receipt = measurement.revision(REPO)
    suites = {s.strip() for s in args.suite.split(",") if s.strip()}
    if "all" in suites:
        suites.update({"core", "rlm", "swe"})
        suites.discard("all")
    picked = {}
    for tid, t in tasks.items():
        if args.task and tid not in args.task:
            continue
        suite = t.get("suite", "core")
        if suite not in suites:
            continue
        picked[tid] = t
    work = []
    for hname in args.harness.split(","):
        harness = dict(harnesses[hname], _arm=args.arm, _provider=args.provider, _receipt=code_receipt, _capture_requests=args.capture_requests)
        if args.provider or args.binary or args.capture_requests:
            if not hname.startswith("graff"):
                ap.error("--provider/--binary/--capture-requests require a graff harness")
            if args.provider and args.provider not in measurement.CREDENTIALS:
                ap.error("--provider requires a supported credential route")
            if args.provider and not os.environ.get(measurement.CREDENTIALS[args.provider]):
                ap.error("selected provider credential is not supplied in the environment")
            if args.binary:
                harness["cmd"] = [os.path.abspath(args.binary), *harness["cmd"][1:]]
        model = args.model or harness.get("default_model", "")
        for task in picked.values():
            missing = [c for c in task.get("requires", []) if c not in harness.get("capabilities", [])]
            if missing:
                print(f"skip {task['id']} on {hname}: needs {missing}", flush=True)
                continue
            for rep in range(1, args.reps + 1):
                work.append((hname, harness, task, model, rep))
    if args.interleave:
        arm_at = {h: i for i, h in enumerate(args.harness.split(","))}
        task_at = {tid: i for i, tid in enumerate(picked)}
        work.sort(key=lambda w: (w[4], task_at[w[2]["id"]], arm_at[w[0]]))
    jobs = max(1, args.jobs)
    print(f"{len(work)} runs · {jobs} worker{'s' if jobs != 1 else ''} · {out_path}", flush=True)
    records = []
    lock = threading.Lock()
    with open(out_path, "w") as f:
        def finish(rec):
            with lock:
                records.append(rec)
                f.write(json.dumps(rec, ensure_ascii=False) + "\n")
                f.flush()
                print(eval_report.line(rec), flush=True)

        if jobs == 1:
            for item in work:
                finish(one_run(*item))
        else:
            with ThreadPoolExecutor(max_workers=jobs) as pool:
                futs = [pool.submit(one_run, *item) for item in work]
                for fut in as_completed(futs):
                    finish(fut.result())
    eval_report.summarize(records)
    print(f"\nresults: {out_path}")


if __name__ == "__main__":
    main()