graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Background results say how to wait for them
justrachd14cb673b19e
diff --git a/graff-evals/README.md b/graff-evals/README.mdindex 1f4d245c..794ab161 100644--- a/graff-evals/README.md+++ b/graff-evals/README.md@@ -13,7 +13,7 @@ harness under test spends model calls.## Suites-`--suite` selects which tasks run (`all` is core+rlm+swe; `mcp` / `inhouse` / `live` are opt-in):+`--suite` selects which tasks run (`all` is core+rlm+swe; `mcp` / `inhouse` / `live` / `bg` are opt-in):| suite | what it measures ||---|---|@@ -23,6 +23,7 @@ harness under test spends model calls.| `mcp` | Linear-shaped fixture MCP (Blacksmith code-mode + muscle memory). Always `--no-lean` (`graff-dev-nolean`): lean is a different catalog and is not on the front. || `inhouse` | Distilled PR fixtures with SPEC.md. Cheap harness A/B — not the live badge. Opt-in. || `live` | Capped 12 gated PRs, no SPEC.md. Pass @ n=3; list$ on passing reps only. See [LIVE.md](../artifacts/graff-evals-live/LIVE.md). |+| `bg` | Commands that outlast the 15s `-p` foreground wait, so each run must collect background results. For A/Bs of how the harness words and delivers them. Opt-in. |Published live board (2026-09-09, five harnesses on grok-4.6 SuperGrok):[artifacts/graff-evals-live/RECEIPT.md](../artifacts/graff-evals-live/RECEIPT.md).diff --git a/graff-evals/tasks/59-bg-slow-tests.json b/graff-evals/tasks/59-bg-slow-tests.jsonnew file mode 100644index 00000000..8cf10227--- /dev/null+++ b/graff-evals/tasks/59-bg-slow-tests.json@@ -0,0 +1,12 @@+{+ "id": "bg-slow-tests",+ "suite": "bg",+ "category": "background",+ "prompt": "Run `python3 run_tests.py`; the suite takes about 25 seconds and is the only source of truth, so run it as given. Fix the bug in calc.py that makes it fail, then rerun the full suite and confirm it passes.",+ "files": {+ "calc.py": "def mean(xs):\n return sum(xs) / len(xs)\n\n\ndef median(xs):\n s = sorted(xs)\n mid = len(s) // 2\n return s[mid]\n",+ "run_tests.py": "import sys, time\nimport calc\n\n# The suite is slow on purpose: each group takes about eight seconds.\nGROUPS = [\n (\"mean\", lambda: calc.mean([2, 4, 6, 8]) == 5),\n (\"median of odd count\", lambda: calc.median([7, 1, 3]) == 3),\n (\"median of even count\", lambda: calc.median([4, 1, 3, 2]) == 2.5),\n]\nfailed = []\nfor name, check in GROUPS:\n time.sleep(8)\n ok = check()\n print((\"ok \" if ok else \"FAIL \") + name, flush=True)\n if not ok:\n failed.append(name)\nprint(\"ALL PASSED\" if not failed else f\"{len(failed)} FAILED\", flush=True)\nsys.exit(1 if failed else 0)\n"+ },+ "check": "python3 -c \"import calc; assert calc.mean([2,4,6,8]) == 5; assert calc.median([7,1,3]) == 3; assert calc.median([4,1,3,2]) == 2.5\"",+ "timeout_s": 420+}diff --git a/graff-evals/tasks/60-bg-build-id.json b/graff-evals/tasks/60-bg-build-id.jsonnew file mode 100644index 00000000..3723c7ed--- /dev/null+++ b/graff-evals/tasks/60-bg-build-id.json@@ -0,0 +1,11 @@+{+ "id": "bg-build-id",+ "suite": "bg",+ "category": "background",+ "prompt": "Run `sh build.sh`. It takes about 20 seconds and prints a build id at the end. Write that build id, and nothing else, to build_id.txt.",+ "files": {+ "build.sh": "#!/bin/sh\n# A slow build: it takes about 20 seconds and prints a fresh build id.\nid=$(python3 -c 'import secrets; print(secrets.token_hex(4))')\nprintf '%s\\n' \"$id\" > .build-id-truth\necho \"compiling...\"\nsleep 20\necho \"build id: $id\"\n"+ },+ "check": "test -s .build-id-truth && test \"$(tr -d ' \\n' < build_id.txt)\" = \"$(tr -d ' \\n' < .build-id-truth)\"",+ "timeout_s": 300+}diff --git a/graff-evals/tasks/61-bg-two-checks.json b/graff-evals/tasks/61-bg-two-checks.jsonnew file mode 100644index 00000000..d4fbb9e3--- /dev/null+++ b/graff-evals/tasks/61-bg-two-checks.json@@ -0,0 +1,13 @@+{+ "id": "bg-two-checks",+ "suite": "bg",+ "category": "background",+ "prompt": "Two independent checks cover app.py: `python3 lint.py` and `python3 test.py`. Each takes about 20 seconds, so run them in parallel. Fix what they report in app.py, then run both again and confirm both pass.",+ "files": {+ "app.py": "import os\nimport json\n\n\ndef greet(name):\n return \"Hello, \" + name + \"!\"\n\n\ndef total(xs):\n t = 0\n for x in xs[1:]:\n t += x\n return t\n",+ "lint.py": "import ast, sys, time\n# A slow linter: about 20 seconds. Reports imports app.py never uses.\ntime.sleep(20)\ntree = ast.parse(open(\"app.py\").read())\nimported = set()\nfor node in ast.walk(tree):\n if isinstance(node, ast.Import):\n imported.update(a.asname or a.name for a in node.names)\n elif isinstance(node, ast.ImportFrom):\n imported.update(a.asname or a.name for a in node.names)\nused = {n.id for n in ast.walk(tree) if isinstance(n, ast.Name)}\nunused = sorted(imported - used)\nfor name in unused:\n print(f\"app.py: unused import {name}\")\nprint(\"lint clean\" if not unused else f\"{len(unused)} lint error(s)\")\nsys.exit(1 if unused else 0)\n",+ "test.py": "import sys, time\nimport app\n# A slow test run: about 20 seconds.\ntime.sleep(20)\ncases = [(\"greet\", app.greet(\"Ada\") == \"Hello, Ada!\"), (\"total\", app.total([1, 2, 3]) == 6)]\nfor name, ok in cases:\n print((\"ok \" if ok else \"FAIL \") + name)\nfailed = [n for n, ok in cases if not ok]\nprint(\"tests passed\" if not failed else f\"{len(failed)} test(s) failed\")\nsys.exit(1 if failed else 0)\n"+ },+ "check": "python3 -c \"import ast; t=ast.parse(open('app.py').read()); names={a.asname or a.name for n in ast.walk(t) if isinstance(n,(ast.Import,ast.ImportFrom)) for a in n.names}; used={n.id for n in ast.walk(t) if isinstance(n,ast.Name)}; assert not (names-used), names-used\" && python3 -c \"import app; assert app.greet('Ada')=='Hello, Ada!'; assert app.total([1,2,3])==6\"",+ "timeout_s": 420+}diff --git a/src/exec_bash.zig b/src/exec_bash.zigindex 3dc28dae..e8c0a9a1 100644--- a/src/exec_bash.zig+++ b/src/exec_bash.zig@@ -80,6 +80,10 @@ fn spawnFailText(gpa: Allocator, err: anyerror) ![]u8 {/// server-like or a finite command (#1324).const Started = enum { background, background_finite, server, finite };+/// A finite job's next move (ADR 0215): its result arrives by itself, so the+/// model keeps working or ends its reply to wait. Servers never report.+const reports_back = "Its exit status and output reach you when it finishes: continue with independent work, or end your reply to wait for it.";+fn startedText(gpa: Allocator, id: u64, cmd: []const u8, ssh: bool, how: Started, wait_s: u64, partial: []const u8) ![]u8 {var aw: Io.Writer.Allocating = .init(gpa);errdefer aw.deinit();@@ -90,9 +94,9 @@ fn startedText(gpa: Allocator, id: u64, cmd: []const u8, ssh: bool, how: Started} else if (how == .server) {try w.print("Command exceeded the {d}s foreground wait and was automatically moved to the background. Process is still running as job {d}. You are notified on exit with the status and output — do not poll. action=output is a snapshot; leave it on /jobs or action=kill it.", .{ wait_s, id });} else if (how == .background_finite) {- try w.print("Running in the background as job {d}. You are notified on exit with the status and output. To block until it finishes, call action=output id {d} with wait_ms>0 once; action=kill stops it.", .{ id, id });+ try w.print("Running in the background as job {d}. {s} To block on it instead, call action=output id {d} with wait_ms>0 once; action=kill stops it.", .{ id, reports_back, id });} else if (how == .finite) {- try w.print("Command exceeded the {d}s foreground wait and was automatically moved to the background. Process is still running as job {d}. You are notified on exit with the status and output. To block until it finishes instead, call action=output id {d} with wait_ms>0 once; action=kill stops it.", .{ wait_s, id, id });+ try w.print("Command exceeded the {d}s foreground wait and was automatically moved to the background. Process is still running as job {d}. {s} To block on it instead, call action=output id {d} with wait_ms>0 once; action=kill stops it.", .{ wait_s, id, reports_back, id });} else {try w.print("This is a persistent server. It runs in the background across turns. You are notified on exit — do not poll. action=output id {d} reads unread output. action=kill stops it.", .{id});}@@ -263,6 +267,7 @@ test "an auto-parked finite command offers the wait-until-exit read (#1324)" {const text = try startedText(gpa, 8, "cargo test", false, .finite, 15, "");defer gpa.free(text);try std.testing.expect(std.mem.indexOf(u8, text, "action=output id 8 with wait_ms>0") != null);+ try std.testing.expect(std.mem.indexOf(u8, text, "end your reply to wait for it") != null);try std.testing.expect(std.mem.indexOf(u8, text, "snapshot") == null);try std.testing.expect(std.mem.indexOf(u8, text, "persistent server") == null);}@@ -272,6 +277,7 @@ test "a finite command backgrounded on purpose offers the wait-until-exit read (const text = try startedText(gpa, 9, "cargo test", false, .background_finite, 0, "");defer gpa.free(text);try std.testing.expect(std.mem.indexOf(u8, text, "action=output id 9 with wait_ms>0") != null);+ try std.testing.expect(std.mem.indexOf(u8, text, "end your reply to wait for it") != null);try std.testing.expect(std.mem.indexOf(u8, text, "persistent server") == null);try std.testing.expect(std.mem.indexOf(u8, text, "do not poll") == null);}diff --git a/src/subagent_spawn_job.zig b/src/subagent_spawn_job.zigindex e2b03442..1792b26d 100644--- a/src/subagent_spawn_job.zig+++ b/src/subagent_spawn_job.zig@@ -53,9 +53,9 @@ pub fn spawnSubBackground(ctx: ToolCtx, label: []const u8, prompt: []const u8, ssubagent.g_agent_jobs.next_id += 1;const receipt = (if (state) |saved| std.fmt.allocPrint(gpa,- "[agent {d} started: {s}] [task_id {s}]\nIt runs in the background across turns. Do not poll. agent_output(id {d}, wait_ms>0) waits for completion. agent_message queues feedback; subagent_resume continues the retained conversation after completion.",+ "[agent {d} started: {s}] [task_id {s}]\nIt runs in the background and its report reaches you when it finishes: continue with independent work, or end your reply to wait for it. Do not poll; agent_output(id {d}, wait_ms>0) blocks until it finishes. agent_message queues feedback; subagent_resume continues the retained conversation after completion.",.{ job.id, job.label, saved.record.id, job.id },- ) else std.fmt.allocPrint(gpa, "[agent {d} started: {s}]\nIt runs in the background across turns. Do not poll. agent_output(id {d}, wait_ms>0) waits for completion.", .{ job.id, job.label, job.id })) catch |err| {+ ) else std.fmt.allocPrint(gpa, "[agent {d} started: {s}]\nIt runs in the background and its report reaches you when it finishes: continue with independent work, or end your reply to wait for it. Do not poll; agent_output(id {d}, wait_ms>0) blocks until it finishes.", .{ job.id, job.label, job.id })) catch |err| {subagent.g_agent_jobs.mutex.unlock(ctx.io);gpa.destroy(job);return err;