graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Evals: mined prompts name where the test runs; new test files ship whole
justrach5ef63619ec54
diff --git a/graff-evals/design/mine_prs.py b/graff-evals/design/mine_prs.pyindex e172add7..f8ca2030 100644--- a/graff-evals/design/mine_prs.py+++ b/graff-evals/design/mine_prs.py@@ -189,7 +189,11 @@ def write(path: str, text: str, mode: int = 0o644) -> None:def prompt_for(title: str, public: dict) -> str:- return (f"{title}. Make the test named \"{public['name']}\" in {public['file']} pass. "+ # State what the grader checks: the test must run in the project's own+ # test build. "Make it pass" alone let a run prove it in a temporary+ # harness and leave it unreachable from the suite (ADR 0214).+ return (f"{title}. Make the test named \"{public['name']}\" in {public['file']} run and pass "+ "in the project's unit tests (`zig build test`), not only in isolation. ""Do not edit that test. Do not add SPEC.md.")@@ -216,7 +220,19 @@ def build(pr_number: int, public_name: str | None, hidden_name: str | None, ref:raise SystemExit(f"could not extract test {t['name']!r} from {t['file']}")blocks[role] = blocktdir = os.path.join(EVALS, "live", task_id)- write(os.path.join(tdir, "public_case.inc"), "\n" + blocks["public"])+ public_text = "\n" + blocks["public"]+ parent_has = subprocess.run(["git", "-C", REPO, "cat-file", "-e", f"{pr['parent']}:{public['file']}"],+ capture_output=True).returncode == 0+ if not parent_has and is_test_file(public["file"]):+ # A new test-only file is part of the spec: without its imports and+ # helpers the test cannot compile unless the test file is edited,+ # which the prompt forbids. Ship the whole file, minus the hidden+ # test. (A new source module with inline tests is the solution+ # itself, so there only the test block is given.)+ public_text = file_at(pr["merge"], public["file"])+ if hidden and hidden["file"] == public["file"]:+ public_text = public_text.replace(blocks["hidden"], "")+ write(os.path.join(tdir, "public_case.inc"), public_text)write(os.path.join(tdir, "check_public.sh"),"#!/bin/sh\nset -eu\nTASK_ROOT=${TASK_ROOT:?}\n"f"exec python3 \"$TASK_ROOT/named_unit_check.py\" {json.dumps(public['name'])}\n", 0o755)@@ -342,6 +358,8 @@ def self_test() -> None:assert test_block(src, "second") == 'test "second" {}\n'assert test_block(src, "missing") is Noneassert TEST_RE.match('+test "a \\"quoted\\" name" {').group(1) == 'a \\"quoted\\" name'+ prompt = prompt_for("Fix it", {"name": "alpha", "file": "src/a.zig"})+ assert "run and pass in the project's unit tests (`zig build test`)" in prompt, promptprint("mine_prs self-test ok")diff --git a/graff-evals/hillclimb/split-mined-candidate.json b/graff-evals/hillclimb/split-mined-candidate.jsonnew file mode 100644index 00000000..b1baebf7--- /dev/null+++ b/graff-evals/hillclimb/split-mined-candidate.json@@ -0,0 +1,28 @@+{+ "seed": "graff",+ "test": [+ "pr-1257",+ "pr-1309",+ "pr-1329",+ "pr-1339",+ "pr-1348",+ "pr-1364",+ "pr-1386"+ ],+ "train": [+ "pr-1253",+ "pr-1254",+ "pr-1256",+ "pr-1302",+ "pr-1306",+ "pr-1327",+ "pr-1328",+ "pr-1330",+ "pr-1336",+ "pr-1365",+ "pr-1373",+ "pr-1376",+ "pr-1385"+ ],+ "suite": "mined-candidate"+}diff --git a/graff-evals/live/mined.json b/graff-evals/live/mined.jsonindex ab7108b6..dd82a180 100644--- a/graff-evals/live/mined.json+++ b/graff-evals/live/mined.json@@ -493,33 +493,6 @@"ok": true}},- {- "id": "pr-1256",- "pr": "codegraff #1256",- "why": "Apply GRAFF_NO_PLUGINS before connecting MCP servers",- "repo": "justrach/codegraff",- "historical_parent": "4706cf974edc48b9f908d173df4acee9347fa0c0",- "historical_merge": "5d45a5fde640f75de61822a38aae44256f87cace",- "source": "local-git",- "public_filter": "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config",- "public_file": "src/session_start_plugins_test.zig",- "hidden": null,- "hidden_filter": null,- "hidden_file": null,- "check_timeout_s": 420,- "validation": {- "red": {- "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "missing"- },- "green": {- "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "pass"- },- "stable": {- "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "pass"- },- "ok": true- }- },{"id": "pr-1254","pr": "codegraff #1254",@@ -576,6 +549,33 @@},"ok": true}+ },+ {+ "id": "pr-1256",+ "pr": "codegraff #1256",+ "why": "Apply GRAFF_NO_PLUGINS before connecting MCP servers",+ "repo": "justrach/codegraff",+ "historical_parent": "4706cf974edc48b9f908d173df4acee9347fa0c0",+ "historical_merge": "5d45a5fde640f75de61822a38aae44256f87cace",+ "source": "local-git",+ "public_filter": "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config",+ "public_file": "src/session_start_plugins_test.zig",+ "hidden": null,+ "hidden_filter": null,+ "hidden_file": null,+ "check_timeout_s": 420,+ "validation": {+ "red": {+ "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "missing"+ },+ "green": {+ "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "pass"+ },+ "stable": {+ "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "pass"+ },+ "ok": true+ }}],"mined": []diff --git a/graff-evals/live/pr-1253/public_case.inc b/graff-evals/live/pr-1253/public_case.incindex 2ccd0b12..e1598051 100644--- a/graff-evals/live/pr-1253/public_case.inc+++ b/graff-evals/live/pr-1253/public_case.inc@@ -1,3 +1,22 @@+//! Codex WS turns surface deltas as they arrive, through the same per-line+//! hook (printDelta) as the SSE reader. Before, a WS turn buffered every frame+//! and only rendered the answer after response.completed: ACP clients got the+//! whole reply as one chunk at the end, and a turn that stalled mid-answer had+//! shown nothing at all.++const std = @import("std");+const builtin = @import("builtin");+const Io = std.Io;++const http = @import("http.zig");+const http_stall = @import("http_stall.zig");+const ws = @import("ws.zig");+const agent_ws = @import("agent_ws.zig");++const mock = @import("agent_ws_mock.zig");+const Mock = mock.Mock;+const nowMs = mock.nowMs;+const mockAgent = mock.mockAgent;test "a WS text delta reaches the UI before the response completes" {if (builtin.os.tag == .windows) return error.SkipZigTest;diff --git a/graff-evals/live/pr-1256/public_case.inc b/graff-evals/live/pr-1256/public_case.incindex ae4bccbf..9cb42ff2 100644--- a/graff-evals/live/pr-1256/public_case.inc+++ b/graff-evals/live/pr-1256/public_case.inc@@ -1,3 +1,8 @@+//! session_start.initRegistryConsent and the plugins switch, kept apart from+//! session_start.zig (600-line ceiling).+const std = @import("std");+const Io = std.Io;+const session_start = @import("session_start.zig");test "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config" {const plugins = @import("plugins.zig");diff --git a/graff-evals/tasks/mined-pr-1253.json b/graff-evals/tasks/mined-pr-1253.jsonindex 587e0b3f..0a2efd8c 100644--- a/graff-evals/tasks/mined-pr-1253.json+++ b/graff-evals/tasks/mined-pr-1253.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1253 — mined (ADR 0214)",- "prompt": "ws: stream codex WS deltas through printDelta like SSE. Make the test named \"a WS text delta reaches the UI before the response completes\" in src/agent_ws_stream_test.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ws: stream codex WS deltas through printDelta like SSE. Make the test named \"a WS text delta reaches the UI before the response completes\" in src/agent_ws_stream_test.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1253 ."],diff --git a/graff-evals/tasks/mined-pr-1254.json b/graff-evals/tasks/mined-pr-1254.jsonindex a56f51ef..9a390d7c 100644--- a/graff-evals/tasks/mined-pr-1254.json+++ b/graff-evals/tasks/mined-pr-1254.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1254 — mined (ADR 0214)",- "prompt": "Connect the MCP servers an ACP client passes. Make the test named \"parse: stdio with args and env, http with headers, sse and malformed left out\" in src/acp_mcp_servers.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Connect the MCP servers an ACP client passes. Make the test named \"parse: stdio with args and env, http with headers, sse and malformed left out\" in src/acp_mcp_servers.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1254 ."],diff --git a/graff-evals/tasks/mined-pr-1256.json b/graff-evals/tasks/mined-pr-1256.jsonindex 22413e22..dd2cb224 100644--- a/graff-evals/tasks/mined-pr-1256.json+++ b/graff-evals/tasks/mined-pr-1256.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1256 — mined (ADR 0214)",- "prompt": "Apply GRAFF_NO_PLUGINS before connecting MCP servers. Make the test named \"initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config\" in src/session_start_plugins_test.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Apply GRAFF_NO_PLUGINS before connecting MCP servers. Make the test named \"initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config\" in src/session_start_plugins_test.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1256 ."],diff --git a/graff-evals/tasks/mined-pr-1257.json b/graff-evals/tasks/mined-pr-1257.jsonindex 2a7cb1b7..5e3fdcdc 100644--- a/graff-evals/tasks/mined-pr-1257.json+++ b/graff-evals/tasks/mined-pr-1257.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1257 — mined (ADR 0214)",- "prompt": "stdin: read ACP and --json records past the 64 KiB buffer. Make the test named \"records larger than the reader buffer are returned whole\" in src/stdin_line.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "stdin: read ACP and --json records past the 64 KiB buffer. Make the test named \"records larger than the reader buffer are returned whole\" in src/stdin_line.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1257 ."],diff --git a/graff-evals/tasks/mined-pr-1302.json b/graff-evals/tasks/mined-pr-1302.jsonindex 089c9812..a1a1bf92 100644--- a/graff-evals/tasks/mined-pr-1302.json+++ b/graff-evals/tasks/mined-pr-1302.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1302 — mined (ADR 0214)",- "prompt": "Bound publication review input for small diffs in large files. Make the test named \"claim review uses committed context when changed head source exceeds the budget\" in src/pr_review_input.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Bound publication review input for small diffs in large files. Make the test named \"claim review uses committed context when changed head source exceeds the budget\" in src/pr_review_input.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1302 ."],diff --git a/graff-evals/tasks/mined-pr-1306.json b/graff-evals/tasks/mined-pr-1306.jsonindex 69aea126..84b9f1df 100644--- a/graff-evals/tasks/mined-pr-1306.json+++ b/graff-evals/tasks/mined-pr-1306.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1306 — mined (ADR 0214)",- "prompt": "Strip provider citation markers from ACP text. Make the test named \"a whole citation marker is removed and plain text passes through untouched\" in src/acp_citations.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Strip provider citation markers from ACP text. Make the test named \"a whole citation marker is removed and plain text passes through untouched\" in src/acp_citations.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1306 ."],diff --git a/graff-evals/tasks/mined-pr-1309.json b/graff-evals/tasks/mined-pr-1309.jsonindex 9a815856..8ef830a3 100644--- a/graff-evals/tasks/mined-pr-1309.json+++ b/graff-evals/tasks/mined-pr-1309.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1309 — mined (ADR 0214)",- "prompt": "ACP: report context left with the standard usage_update. Make the test named \"usage_update carries used and size, and cost only when known\" in src/acp_context_meter.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ACP: report context left with the standard usage_update. Make the test named \"usage_update carries used and size, and cost only when known\" in src/acp_context_meter.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1309 ."],diff --git a/graff-evals/tasks/mined-pr-1327.json b/graff-evals/tasks/mined-pr-1327.jsonindex 66e63618..a07831df 100644--- a/graff-evals/tasks/mined-pr-1327.json+++ b/graff-evals/tasks/mined-pr-1327.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1327 — mined (ADR 0214)",- "prompt": "ask_user: read the question from every argument shape models send. Make the test named \"the schema shape passes through unchanged\" in src/ask_user_args.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ask_user: read the question from every argument shape models send. Make the test named \"the schema shape passes through unchanged\" in src/ask_user_args.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1327 ."],diff --git a/graff-evals/tasks/mined-pr-1328.json b/graff-evals/tasks/mined-pr-1328.jsonindex 24a861c1..a603be54 100644--- a/graff-evals/tasks/mined-pr-1328.json+++ b/graff-evals/tasks/mined-pr-1328.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1328 — mined (ADR 0214)",- "prompt": "Free the MCP shape cache at session teardown. Make the test named \"shutdown frees a filled cache and is a no-op when empty (#1196)\" in src/mcp_shapes.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Free the MCP shape cache at session teardown. Make the test named \"shutdown frees a filled cache and is a no-op when empty (#1196)\" in src/mcp_shapes.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1328 ."],diff --git a/graff-evals/tasks/mined-pr-1329.json b/graff-evals/tasks/mined-pr-1329.jsonindex f4310b10..c410c44a 100644--- a/graff-evals/tasks/mined-pr-1329.json+++ b/graff-evals/tasks/mined-pr-1329.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1329 — mined (ADR 0214)",- "prompt": "ACP session/new stops waiting on unreachable MCP servers. Make the test named \"joinWithin (#1291): a server that fails fast ends the wait early\" in src/acp_mcp_servers.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ACP session/new stops waiting on unreachable MCP servers. Make the test named \"joinWithin (#1291): a server that fails fast ends the wait early\" in src/acp_mcp_servers.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1329 ."],diff --git a/graff-evals/tasks/mined-pr-1330.json b/graff-evals/tasks/mined-pr-1330.jsonindex 244785c9..24eb110b 100644--- a/graff-evals/tasks/mined-pr-1330.json+++ b/graff-evals/tasks/mined-pr-1330.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1330 — mined (ADR 0214)",- "prompt": "Run tool calls a server left in content as inline markup. Make the test named \"recoverInlineCalls (#1247): markup after prose becomes calls, parameters in any order\" in src/tool_call_repair_tests.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Run tool calls a server left in content as inline markup. Make the test named \"recoverInlineCalls (#1247): markup after prose becomes calls, parameters in any order\" in src/tool_call_repair_tests.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1330 ."],diff --git a/graff-evals/tasks/mined-pr-1336.json b/graff-evals/tasks/mined-pr-1336.jsonindex 8d2228b6..1703eb59 100644--- a/graff-evals/tasks/mined-pr-1336.json+++ b/graff-evals/tasks/mined-pr-1336.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1336 — mined (ADR 0214)",- "prompt": "Resolve a PR check that finished in the background. Make the test named \"a check promoted to the background resolves when its job exits 0 (#1326)\" in src/pr_local_checks.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Resolve a PR check that finished in the background. Make the test named \"a check promoted to the background resolves when its job exits 0 (#1326)\" in src/pr_local_checks.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1336 ."],diff --git a/graff-evals/tasks/mined-pr-1339.json b/graff-evals/tasks/mined-pr-1339.jsonindex a5fc5c72..6331319f 100644--- a/graff-evals/tasks/mined-pr-1339.json+++ b/graff-evals/tasks/mined-pr-1339.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1339 — mined (ADR 0214)",- "prompt": "ask_user: a question that comes back unanswered is not a cancel. Make the test named \"decline, cancel, an empty accept and an error each end the question as themselves (#1322)\" in src/acp_elicit.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ask_user: a question that comes back unanswered is not a cancel. Make the test named \"decline, cancel, an empty accept and an error each end the question as themselves (#1322)\" in src/acp_elicit.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1339 ."],diff --git a/graff-evals/tasks/mined-pr-1348.json b/graff-evals/tasks/mined-pr-1348.jsonindex fd821c26..fdbb0f1a 100644--- a/graff-evals/tasks/mined-pr-1348.json+++ b/graff-evals/tasks/mined-pr-1348.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1348 — mined (ADR 0214)",- "prompt": "PR readiness keys checks by what they run; rlm says when sleep_ms is capped. Make the test named \"one check in one directory is one entry whatever its spelling\" in src/pr_local_checks.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "PR readiness keys checks by what they run; rlm says when sleep_ms is capped. Make the test named \"one check in one directory is one entry whatever its spelling\" in src/pr_local_checks.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1348 ."],diff --git a/graff-evals/tasks/mined-pr-1364.json b/graff-evals/tasks/mined-pr-1364.jsonindex de6eaa77..2d1d5392 100644--- a/graff-evals/tasks/mined-pr-1364.json+++ b/graff-evals/tasks/mined-pr-1364.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1364 — mined (ADR 0214)",- "prompt": "Informational subagents keep read tools (#1360). Make the test named \"a read-only child keeps reads and read-only shell, not writes (#1360)\" in src/agent_tool_gate.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Informational subagents keep read tools (#1360). Make the test named \"a read-only child keeps reads and read-only shell, not writes (#1360)\" in src/agent_tool_gate.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1364 ."],diff --git a/graff-evals/tasks/mined-pr-1365.json b/graff-evals/tasks/mined-pr-1365.jsonindex 06f37fbe..430ee743 100644--- a/graff-evals/tasks/mined-pr-1365.json+++ b/graff-evals/tasks/mined-pr-1365.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1365 — mined (ADR 0214)",- "prompt": "Anthropic: refuse a cut-off tool input instead of running it as {} (refs #1218). Make the test named \"a cut-off Anthropic tool input is refused, not run as {} (#1218)\" in src/tool_call_args.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Anthropic: refuse a cut-off tool input instead of running it as {} (refs #1218). Make the test named \"a cut-off Anthropic tool input is refused, not run as {} (#1218)\" in src/tool_call_args.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1365 ."],diff --git a/graff-evals/tasks/mined-pr-1373.json b/graff-evals/tasks/mined-pr-1373.jsonindex d9ea24c1..fd6f99b4 100644--- a/graff-evals/tasks/mined-pr-1373.json+++ b/graff-evals/tasks/mined-pr-1373.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1373 — mined (ADR 0214)",- "prompt": "ACP: frame Harness room deliveries as advisory room messages. Make the test named \"an agent's room line becomes an advisory room message; a person's stays a prompt\" in src/acp_room.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ACP: frame Harness room deliveries as advisory room messages. Make the test named \"an agent's room line becomes an advisory room message; a person's stays a prompt\" in src/acp_room.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1373 ."],diff --git a/graff-evals/tasks/mined-pr-1376.json b/graff-evals/tasks/mined-pr-1376.jsonindex 93973e2c..b4d3f3ef 100644--- a/graff-evals/tasks/mined-pr-1376.json+++ b/graff-evals/tasks/mined-pr-1376.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1376 — mined (ADR 0214)",- "prompt": "ACP room deliveries: name another account's agent as such. Make the test named \"another account's agent is named as such, and framed text must say so to be kept\" in src/acp_room.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ACP room deliveries: name another account's agent as such. Make the test named \"another account's agent is named as such, and framed text must say so to be kept\" in src/acp_room.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1376 ."],diff --git a/graff-evals/tasks/mined-pr-1385.json b/graff-evals/tasks/mined-pr-1385.jsonindex 95fdc12e..94f1d7ed 100644--- a/graff-evals/tasks/mined-pr-1385.json+++ b/graff-evals/tasks/mined-pr-1385.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1385 — mined (ADR 0214)",- "prompt": "ACP: saved views carry _meta[\"graff/view\"] on the completed tool update. Make the test named \"detect: graff's own view and MCP app snapshots\" in src/acp_view_meta.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ACP: saved views carry _meta[\"graff/view\"] on the completed tool update. Make the test named \"detect: graff's own view and MCP app snapshots\" in src/acp_view_meta.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1385 ."],diff --git a/graff-evals/tasks/mined-pr-1386.json b/graff-evals/tasks/mined-pr-1386.jsonindex eba2a348..13b03649 100644--- a/graff-evals/tasks/mined-pr-1386.json+++ b/graff-evals/tasks/mined-pr-1386.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1386 — mined (ADR 0214)",- "prompt": "graff/models: each row names the model's maker. Make the test named \"maker: the gateway serves other labs' models\" in src/model_maker.zig pass. Do not edit that test. Do not add SPEC.md.",+ "prompt": "graff/models: each row names the model's maker. Make the test named \"maker: the gateway serves other labs' models\" in src/model_maker.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1386 ."],