graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Evals: sandboxes get their own repository; mined tasks protect only the named test
justrach17bd765e3752
diff --git a/docs/adr/0214-evals-are-mined-and-hillclimbed-on-a-held-out-split.md b/docs/adr/0214-evals-are-mined-and-hillclimbed-on-a-held-out-split.mdindex 7a8c768f..05b87aa3 100644--- a/docs/adr/0214-evals-are-mined-and-hillclimbed-on-a-held-out-split.md+++ b/docs/adr/0214-evals-are-mined-and-hillclimbed-on-a-held-out-split.md@@ -50,6 +50,22 @@ an unreachable test counts). On the merge tree both must pass, and a second runmust give the same verdicts. That is three filtered builds per task, onebuild at a time.+**A sandbox is its own repository, and only the named test is protected**+(amended 2026-09-29). Sandboxes sit inside this checkout. Before this change,+`git status`, `git diff` and `git show HEAD:` in a sandbox described the+checkout, not the task, and a `git stash` there would have changed the+checkout. Each sandbox now gets a one-commit repository of the task tree+(`measurement.isolate_git`).++The mined prompt names the check the grader runs, `zig build test-bin+-Dtest-filter=…`. When it named `zig build test`, runs spent most of their time+budget on that step's integration scripts, which cannot pass without+`graff-evals/`.++A mined task protects its named test's own text. Otherwise only Python test+files count as visible verifiers. In a copy of this repository,+`src/test_hooks.zig` is where a fix registers a new test file.+**A person approves each task.** `review` writes a local page, and `approve`moves a validated candidate into the `mined` suite. The loop never picks itsown tasks from one model's failures.diff --git a/graff-evals/design/mine_prs.py b/graff-evals/design/mine_prs.pyindex f8ca2030..6a1c6ce5 100644--- a/graff-evals/design/mine_prs.py+++ b/graff-evals/design/mine_prs.py@@ -123,42 +123,7 @@ def candidates(days: int, ref: str, limit: int) -> list[dict]:# ── Zig test blocks ──────────────────────────────────────────────────────-def test_block(text: str, name: str) -> str | None:- """The full `test "name" { … }` block, braces matched outside strings,- char literals, `//` comments and `\\\\` multiline string lines."""- head = f'test "{name}"'- start = text.find(head)- if start < 0:- return None- # Include the doc comments / blank line directly above? No: the block only.- i = text.find("{", start + len(head))- if i < 0:- return None- depth, n = 0, len(text)- while i < n:- c = text[i]- if c == "/" and text.startswith("//", i):- j = text.find("\n", i)- i = n if j < 0 else j- continue- if c == "\\" and text.startswith("\\\\", i):- j = text.find("\n", i)- i = n if j < 0 else j- continue- if c in "\"'":- j = i + 1- while j < n and text[j] != c:- j += 2 if text[j] == "\\" else 1- i = j + 1- continue- if c == "{":- depth += 1- elif c == "}":- depth -= 1- if depth == 0:- return text[start:i + 1] + "\n"- i += 1- return None+test_block = live_setup.test_blockdef file_at(rev: str, path: str) -> str:@@ -190,10 +155,15 @@ def write(path: str, text: str, mode: int = 0o644) -> None:def prompt_for(title: str, public: dict) -> str:# State what the grader checks: the test must run in the project's own- # test build. "Make it pass" alone let a run prove it in a temporary- # harness and leave it unreachable from the suite (ADR 0214).- return (f"{title}. Make the test named \"{public['name']}\" in {public['file']} run and pass "- "in the project's unit tests (`zig build test`), not only in isolation. "+ # unit-test binary. "Make it pass" alone let a run prove it in a temporary+ # harness and leave it unreachable from the suite. Naming only `zig build+ # test` sent runs into its integration scripts, which cannot pass without+ # graff-evals/ and ate most of the time budget (ADR 0214).+ return (f"{title}. Make the test named \"{public['name']}\" in {public['file']} pass in the "+ "project's unit-test binary, not only in isolation: "+ f"`zig build test-bin -Dtest-filter=\"{public['name']}\"` compiles it (the newest "+ "`.zig-cache/o/*/test`), and running that binary must report the test OK. The full "+ "`zig build test` also runs integration scripts this task does not need. ""Do not edit that test. Do not add SPEC.md.")@@ -359,7 +329,9 @@ def self_test() -> None:assert test_block(src, "missing") is Noneassert TEST_RE.match('+test "a \\"quoted\\" name" {').group(1) == 'a \\"quoted\\" name'prompt = prompt_for("Fix it", {"name": "alpha", "file": "src/a.zig"})- assert "run and pass in the project's unit tests (`zig build test`)" in prompt, prompt+ # The grader's own check, not the full `zig build test` (ADR 0214).+ assert "pass in the project's unit-test binary, not only in isolation" in prompt, prompt+ assert '`zig build test-bin -Dtest-filter="alpha"`' in prompt, promptprint("mine_prs self-test ok")diff --git a/graff-evals/live_setup.py b/graff-evals/live_setup.pyindex bcdf2fba..5547a093 100644--- a/graff-evals/live_setup.py+++ b/graff-evals/live_setup.py@@ -184,6 +184,56 @@ def archive_local(rev, dest, paths=None):shutil.copytree(pkg, os.path.join(dest, "zig-pkg"), symlinks=True)+def test_block(text, name):+ """The full `test "name" { … }` block, braces matched outside strings,+ char literals, `//` comments and `\\\\` multiline string lines."""+ head = f'test "{name}"'+ start = text.find(head)+ if start < 0:+ return None+ # Include the doc comments / blank line directly above? No: the block only.+ i = text.find("{", start + len(head))+ if i < 0:+ return None+ depth, n = 0, len(text)+ while i < n:+ c = text[i]+ if c == "/" and text.startswith("//", i):+ j = text.find("\n", i)+ i = n if j < 0 else j+ continue+ if c == "\\" and text.startswith("\\\\", i):+ j = text.find("\n", i)+ i = n if j < 0 else j+ continue+ if c in "\"'":+ j = i + 1+ while j < n and text[j] != c:+ j += 2 if text[j] == "\\" else 1+ i = j + 1+ continue+ if c == "{":+ depth += 1+ elif c == "}":+ depth -= 1+ if depth == 0:+ return text[start:i + 1] + "\n"+ i += 1+ return None+++def named_case(task_id):+ """(file, test name) a mined task appended and told the model not to+ edit, or None for any other task."""+ try:+ task = manifest(task_id)+ except SystemExit:+ return None+ if task.get("source") != "local-git" or not task.get("public_file") or not task.get("public_filter"):+ return None+ return task["public_file"], task["public_filter"]++def append_case(sandbox, rel, inc, name):"""Append a test block unless a test with that name is already there."""path = os.path.join(sandbox, rel)diff --git a/graff-evals/measurement.py b/graff-evals/measurement.pyindex 3f2d3bb8..213abb15 100644--- a/graff-evals/measurement.py+++ b/graff-evals/measurement.py@@ -186,20 +186,69 @@ def provider_command(command):return result+# Caches and dependency trees hold no task verifier; walking them is slow.+_NOT_VERIFIERS = {'.git', '.zig-cache', 'zig-cache', 'zig-out', 'zig-pkg', 'node_modules', '__pycache__'}+_LIVE_TASK = re.compile(r'live_setup\.py"?\s+([\w.-]+)')+++def is_visible_test(name):+ """A test or check file a task ships (`test_*.py`, `check_*.py`,+ `*_test.py`). In a copy of a Zig repository, a `test_` name is production+ code: `src/test_hooks.zig` is where a fix registers a new test file."""+ return name.endswith('.py') and (name.startswith(('test_', 'check_')) or name.endswith('_test.py'))++def verifier_snapshot(sandbox, task, task_root):- """Visible tests are immutable; held-out graders stay outside agent cwd."""- paths = {p for p in Path(sandbox).rglob('*') if p.is_file() and- (p.name.startswith('test_') or p.name.startswith('check_') or p.name.endswith('_test.py'))}+ """Visible tests are immutable; held-out graders stay outside agent cwd.+ A mined task's named test was appended to a source file the fix may also+ belong in, so that test's own text is protected, not the file."""+ snap = {}+ for root, dirs, files in os.walk(sandbox):+ dirs[:] = [d for d in dirs if d not in _NOT_VERIFIERS]+ for name in files:+ if is_visible_test(name):+ path = os.path.join(root, name)+ snap[path] = hashlib.sha256(Path(path).read_bytes()).hexdigest()for relative in re.findall(r'\$TASK_ROOT/(hidden/[^\s"\x27;]+)', task.get('check', '')):- paths.add(Path(task_root) / relative)- return {str(p): hashlib.sha256(p.read_bytes()).hexdigest() for p in paths}+ path = Path(task_root) / relative+ snap[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest()+ live = _LIVE_TASK.search(' '.join(task.get('setup', [])))+ if live:+ import live_setup+ case = live_setup.named_case(live.group(1))+ path = os.path.join(sandbox, case[0]) if case else None+ block = live_setup.test_block(Path(path).read_text(errors='replace'), case[1]) if path and os.path.exists(path) else None+ if block:+ snap[path] = {'contains': block}+ return snapdef verifiers_unchanged(snapshot):- for path, digest in snapshot.items():+ for path, want in snapshot.items():try:- if hashlib.sha256(Path(path).read_bytes()).hexdigest() != digest:- return False+ data = Path(path).read_bytes()except OSError:return False+ if isinstance(want, dict):+ if want['contains'].encode() not in data:+ return False+ elif hashlib.sha256(data).hexdigest() != want:+ return Falsereturn True+++def isolate_git(sandbox):+ """Give a sandbox its own repository, holding the task tree as one commit.+ Sandboxes sit inside this checkout. Without their own repository, `git+ status`, `git diff` and `git show HEAD:` there describe the checkout, not+ the task, and a `git stash` or `git checkout` would change the checkout. A+ task that made its own repository keeps it."""+ if os.path.exists(os.path.join(sandbox, '.git')):+ return+ git = ['git', '-c', 'user.name=eval', '-c', 'user.email=eval@localhost', '-c', 'commit.gpgsign=false',+ '-c', 'core.hooksPath=/dev/null', '-C', sandbox]+ subprocess.run(git + ['init', '-q', '-b', 'main'], check=True, capture_output=True)+ with open(os.path.join(sandbox, '.git', 'info', 'exclude'), 'a') as f:+ f.write('.zig-cache/\nzig-cache/\nzig-out/\nzig-pkg/\n.graff/\n.eval-*\n')+ subprocess.run(git + ['add', '-A'], check=True, capture_output=True)+ subprocess.run(git + ['commit', '-q', '--allow-empty', '--no-verify', '-m', 'task'], check=True, capture_output=True)diff --git a/graff-evals/run.py b/graff-evals/run.pyindex 3720900b..af88856a 100755--- a/graff-evals/run.py+++ b/graff-evals/run.py@@ -330,7 +330,9 @@ def tree_cpu_s(pid):def dir_bytes(path):total = 0- for root, _dirs, files in os.walk(path):+ for root, dirs, files in os.walk(path):+ # The repository isolate_git made holds a second copy of the task tree.+ dirs[:] = [d for d in dirs if not (root == path and d == ".git")]for name in files:try:total += os.path.getsize(os.path.join(root, name))@@ -348,6 +350,7 @@ def one_run(hname, harness, task, model, rep, live=False):sandbox = os.path.join(SANDBOX_DIR, f"{hname}-{task['id']}-r{rep}")try:materialize(task, sandbox)+ measurement.isolate_git(sandbox)except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:return {"harness": hname, "task": task["id"], "suite": task.get("suite", "core"),"rep": rep, "error": f"environment setup failed: {error}", "outcome_ok": False}diff --git a/graff-evals/tasks/mined-pr-1253.json b/graff-evals/tasks/mined-pr-1253.jsonindex 0a2efd8c..0fbcc9a0 100644--- a/graff-evals/tasks/mined-pr-1253.json+++ b/graff-evals/tasks/mined-pr-1253.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1253 — mined (ADR 0214)",- "prompt": "ws: stream codex WS deltas through printDelta like SSE. Make the test named \"a WS text delta reaches the UI before the response completes\" in src/agent_ws_stream_test.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ws: stream codex WS deltas through printDelta like SSE. Make the test named \"a WS text delta reaches the UI before the response completes\" in src/agent_ws_stream_test.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"a WS text delta reaches the UI before the response completes\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1253 ."],diff --git a/graff-evals/tasks/mined-pr-1254.json b/graff-evals/tasks/mined-pr-1254.jsonindex 9a390d7c..ae99946e 100644--- a/graff-evals/tasks/mined-pr-1254.json+++ b/graff-evals/tasks/mined-pr-1254.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1254 — mined (ADR 0214)",- "prompt": "Connect the MCP servers an ACP client passes. Make the test named \"parse: stdio with args and env, http with headers, sse and malformed left out\" in src/acp_mcp_servers.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Connect the MCP servers an ACP client passes. Make the test named \"parse: stdio with args and env, http with headers, sse and malformed left out\" in src/acp_mcp_servers.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"parse: stdio with args and env, http with headers, sse and malformed left out\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1254 ."],diff --git a/graff-evals/tasks/mined-pr-1256.json b/graff-evals/tasks/mined-pr-1256.jsonindex dd2cb224..f18c842d 100644--- a/graff-evals/tasks/mined-pr-1256.json+++ b/graff-evals/tasks/mined-pr-1256.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1256 — mined (ADR 0214)",- "prompt": "Apply GRAFF_NO_PLUGINS before connecting MCP servers. Make the test named \"initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config\" in src/session_start_plugins_test.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Apply GRAFF_NO_PLUGINS before connecting MCP servers. Make the test named \"initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config\" in src/session_start_plugins_test.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1256 ."],diff --git a/graff-evals/tasks/mined-pr-1257.json b/graff-evals/tasks/mined-pr-1257.jsonindex 5e3fdcdc..8b4e925c 100644--- a/graff-evals/tasks/mined-pr-1257.json+++ b/graff-evals/tasks/mined-pr-1257.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1257 — mined (ADR 0214)",- "prompt": "stdin: read ACP and --json records past the 64 KiB buffer. Make the test named \"records larger than the reader buffer are returned whole\" in src/stdin_line.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "stdin: read ACP and --json records past the 64 KiB buffer. Make the test named \"records larger than the reader buffer are returned whole\" in src/stdin_line.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"records larger than the reader buffer are returned whole\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1257 ."],diff --git a/graff-evals/tasks/mined-pr-1302.json b/graff-evals/tasks/mined-pr-1302.jsonindex a1a1bf92..82f2cf27 100644--- a/graff-evals/tasks/mined-pr-1302.json+++ b/graff-evals/tasks/mined-pr-1302.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1302 — mined (ADR 0214)",- "prompt": "Bound publication review input for small diffs in large files. Make the test named \"claim review uses committed context when changed head source exceeds the budget\" in src/pr_review_input.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Bound publication review input for small diffs in large files. Make the test named \"claim review uses committed context when changed head source exceeds the budget\" in src/pr_review_input.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"claim review uses committed context when changed head source exceeds the budget\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1302 ."],diff --git a/graff-evals/tasks/mined-pr-1306.json b/graff-evals/tasks/mined-pr-1306.jsonindex 84b9f1df..3bc37839 100644--- a/graff-evals/tasks/mined-pr-1306.json+++ b/graff-evals/tasks/mined-pr-1306.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1306 — mined (ADR 0214)",- "prompt": "Strip provider citation markers from ACP text. Make the test named \"a whole citation marker is removed and plain text passes through untouched\" in src/acp_citations.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Strip provider citation markers from ACP text. Make the test named \"a whole citation marker is removed and plain text passes through untouched\" in src/acp_citations.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"a whole citation marker is removed and plain text passes through untouched\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1306 ."],diff --git a/graff-evals/tasks/mined-pr-1309.json b/graff-evals/tasks/mined-pr-1309.jsonindex 8ef830a3..b2332f94 100644--- a/graff-evals/tasks/mined-pr-1309.json+++ b/graff-evals/tasks/mined-pr-1309.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1309 — mined (ADR 0214)",- "prompt": "ACP: report context left with the standard usage_update. Make the test named \"usage_update carries used and size, and cost only when known\" in src/acp_context_meter.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ACP: report context left with the standard usage_update. Make the test named \"usage_update carries used and size, and cost only when known\" in src/acp_context_meter.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"usage_update carries used and size, and cost only when known\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1309 ."],diff --git a/graff-evals/tasks/mined-pr-1327.json b/graff-evals/tasks/mined-pr-1327.jsonindex a07831df..470e49ed 100644--- a/graff-evals/tasks/mined-pr-1327.json+++ b/graff-evals/tasks/mined-pr-1327.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1327 — mined (ADR 0214)",- "prompt": "ask_user: read the question from every argument shape models send. Make the test named \"the schema shape passes through unchanged\" in src/ask_user_args.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ask_user: read the question from every argument shape models send. Make the test named \"the schema shape passes through unchanged\" in src/ask_user_args.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"the schema shape passes through unchanged\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1327 ."],diff --git a/graff-evals/tasks/mined-pr-1328.json b/graff-evals/tasks/mined-pr-1328.jsonindex a603be54..79dfe78c 100644--- a/graff-evals/tasks/mined-pr-1328.json+++ b/graff-evals/tasks/mined-pr-1328.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1328 — mined (ADR 0214)",- "prompt": "Free the MCP shape cache at session teardown. Make the test named \"shutdown frees a filled cache and is a no-op when empty (#1196)\" in src/mcp_shapes.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Free the MCP shape cache at session teardown. Make the test named \"shutdown frees a filled cache and is a no-op when empty (#1196)\" in src/mcp_shapes.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"shutdown frees a filled cache and is a no-op when empty (#1196)\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1328 ."],diff --git a/graff-evals/tasks/mined-pr-1329.json b/graff-evals/tasks/mined-pr-1329.jsonindex c410c44a..670eec14 100644--- a/graff-evals/tasks/mined-pr-1329.json+++ b/graff-evals/tasks/mined-pr-1329.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1329 — mined (ADR 0214)",- "prompt": "ACP session/new stops waiting on unreachable MCP servers. Make the test named \"joinWithin (#1291): a server that fails fast ends the wait early\" in src/acp_mcp_servers.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ACP session/new stops waiting on unreachable MCP servers. Make the test named \"joinWithin (#1291): a server that fails fast ends the wait early\" in src/acp_mcp_servers.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"joinWithin (#1291): a server that fails fast ends the wait early\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1329 ."],diff --git a/graff-evals/tasks/mined-pr-1330.json b/graff-evals/tasks/mined-pr-1330.jsonindex 24eb110b..88948736 100644--- a/graff-evals/tasks/mined-pr-1330.json+++ b/graff-evals/tasks/mined-pr-1330.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1330 — mined (ADR 0214)",- "prompt": "Run tool calls a server left in content as inline markup. Make the test named \"recoverInlineCalls (#1247): markup after prose becomes calls, parameters in any order\" in src/tool_call_repair_tests.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Run tool calls a server left in content as inline markup. Make the test named \"recoverInlineCalls (#1247): markup after prose becomes calls, parameters in any order\" in src/tool_call_repair_tests.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"recoverInlineCalls (#1247): markup after prose becomes calls, parameters in any order\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1330 ."],diff --git a/graff-evals/tasks/mined-pr-1336.json b/graff-evals/tasks/mined-pr-1336.jsonindex 1703eb59..0452808b 100644--- a/graff-evals/tasks/mined-pr-1336.json+++ b/graff-evals/tasks/mined-pr-1336.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1336 — mined (ADR 0214)",- "prompt": "Resolve a PR check that finished in the background. Make the test named \"a check promoted to the background resolves when its job exits 0 (#1326)\" in src/pr_local_checks.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Resolve a PR check that finished in the background. Make the test named \"a check promoted to the background resolves when its job exits 0 (#1326)\" in src/pr_local_checks.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"a check promoted to the background resolves when its job exits 0 (#1326)\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1336 ."],diff --git a/graff-evals/tasks/mined-pr-1339.json b/graff-evals/tasks/mined-pr-1339.jsonindex 6331319f..09ea7bd1 100644--- a/graff-evals/tasks/mined-pr-1339.json+++ b/graff-evals/tasks/mined-pr-1339.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1339 — mined (ADR 0214)",- "prompt": "ask_user: a question that comes back unanswered is not a cancel. Make the test named \"decline, cancel, an empty accept and an error each end the question as themselves (#1322)\" in src/acp_elicit.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ask_user: a question that comes back unanswered is not a cancel. Make the test named \"decline, cancel, an empty accept and an error each end the question as themselves (#1322)\" in src/acp_elicit.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"decline, cancel, an empty accept and an error each end the question as themselves (#1322)\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1339 ."],diff --git a/graff-evals/tasks/mined-pr-1348.json b/graff-evals/tasks/mined-pr-1348.jsonindex fdbb0f1a..f514b349 100644--- a/graff-evals/tasks/mined-pr-1348.json+++ b/graff-evals/tasks/mined-pr-1348.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1348 — mined (ADR 0214)",- "prompt": "PR readiness keys checks by what they run; rlm says when sleep_ms is capped. Make the test named \"one check in one directory is one entry whatever its spelling\" in src/pr_local_checks.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "PR readiness keys checks by what they run; rlm says when sleep_ms is capped. Make the test named \"one check in one directory is one entry whatever its spelling\" in src/pr_local_checks.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"one check in one directory is one entry whatever its spelling\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1348 ."],diff --git a/graff-evals/tasks/mined-pr-1364.json b/graff-evals/tasks/mined-pr-1364.jsonindex 2d1d5392..3217ed42 100644--- a/graff-evals/tasks/mined-pr-1364.json+++ b/graff-evals/tasks/mined-pr-1364.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1364 — mined (ADR 0214)",- "prompt": "Informational subagents keep read tools (#1360). Make the test named \"a read-only child keeps reads and read-only shell, not writes (#1360)\" in src/agent_tool_gate.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Informational subagents keep read tools (#1360). Make the test named \"a read-only child keeps reads and read-only shell, not writes (#1360)\" in src/agent_tool_gate.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"a read-only child keeps reads and read-only shell, not writes (#1360)\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1364 ."],diff --git a/graff-evals/tasks/mined-pr-1365.json b/graff-evals/tasks/mined-pr-1365.jsonindex 430ee743..7e6ac4a7 100644--- a/graff-evals/tasks/mined-pr-1365.json+++ b/graff-evals/tasks/mined-pr-1365.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1365 — mined (ADR 0214)",- "prompt": "Anthropic: refuse a cut-off tool input instead of running it as {} (refs #1218). Make the test named \"a cut-off Anthropic tool input is refused, not run as {} (#1218)\" in src/tool_call_args.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "Anthropic: refuse a cut-off tool input instead of running it as {} (refs #1218). Make the test named \"a cut-off Anthropic tool input is refused, not run as {} (#1218)\" in src/tool_call_args.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"a cut-off Anthropic tool input is refused, not run as {} (#1218)\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1365 ."],diff --git a/graff-evals/tasks/mined-pr-1373.json b/graff-evals/tasks/mined-pr-1373.jsonindex fd6f99b4..b3e2a3bd 100644--- a/graff-evals/tasks/mined-pr-1373.json+++ b/graff-evals/tasks/mined-pr-1373.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1373 — mined (ADR 0214)",- "prompt": "ACP: frame Harness room deliveries as advisory room messages. Make the test named \"an agent's room line becomes an advisory room message; a person's stays a prompt\" in src/acp_room.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ACP: frame Harness room deliveries as advisory room messages. Make the test named \"an agent's room line becomes an advisory room message; a person's stays a prompt\" in src/acp_room.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"an agent's room line becomes an advisory room message; a person's stays a prompt\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1373 ."],diff --git a/graff-evals/tasks/mined-pr-1376.json b/graff-evals/tasks/mined-pr-1376.jsonindex b4d3f3ef..8ae1642f 100644--- a/graff-evals/tasks/mined-pr-1376.json+++ b/graff-evals/tasks/mined-pr-1376.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1376 — mined (ADR 0214)",- "prompt": "ACP room deliveries: name another account's agent as such. Make the test named \"another account's agent is named as such, and framed text must say so to be kept\" in src/acp_room.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ACP room deliveries: name another account's agent as such. Make the test named \"another account's agent is named as such, and framed text must say so to be kept\" in src/acp_room.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"another account's agent is named as such, and framed text must say so to be kept\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1376 ."],diff --git a/graff-evals/tasks/mined-pr-1385.json b/graff-evals/tasks/mined-pr-1385.jsonindex 94f1d7ed..50ec0532 100644--- a/graff-evals/tasks/mined-pr-1385.json+++ b/graff-evals/tasks/mined-pr-1385.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1385 — mined (ADR 0214)",- "prompt": "ACP: saved views carry _meta[\"graff/view\"] on the completed tool update. Make the test named \"detect: graff's own view and MCP app snapshots\" in src/acp_view_meta.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "ACP: saved views carry _meta[\"graff/view\"] on the completed tool update. Make the test named \"detect: graff's own view and MCP app snapshots\" in src/acp_view_meta.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"detect: graff's own view and MCP app snapshots\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1385 ."],diff --git a/graff-evals/tasks/mined-pr-1386.json b/graff-evals/tasks/mined-pr-1386.jsonindex 13b03649..37bcc6ac 100644--- a/graff-evals/tasks/mined-pr-1386.json+++ b/graff-evals/tasks/mined-pr-1386.json@@ -3,7 +3,7 @@"suite": "mined-candidate","category": "live","source": "codegraff #1386 — mined (ADR 0214)",- "prompt": "graff/models: each row names the model's maker. Make the test named \"maker: the gateway serves other labs' models\" in src/model_maker.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",+ "prompt": "graff/models: each row names the model's maker. Make the test named \"maker: the gateway serves other labs' models\" in src/model_maker.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"maker: the gateway serves other labs' models\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.","setup": ["python3 \"$TASK_ROOT/live_setup.py\" pr-1386 ."],diff --git a/graff-evals/test_measurement.py b/graff-evals/test_measurement.pyindex 29a677f2..94931ffa 100644--- a/graff-evals/test_measurement.py+++ b/graff-evals/test_measurement.py@@ -256,6 +256,60 @@ class MeasurementTests(unittest.TestCase):path.write_text('pass')self.assertFalse(measurement.verifiers_unchanged(snap))+ def test_a_zig_tree_test_root_is_not_a_visible_verifier(self):+ with tempfile.TemporaryDirectory() as temp:+ (Path(temp) / 'src').mkdir()+ hooks = Path(temp) / 'src' / 'test_hooks.zig'+ hooks.write_text('test { _ = @import("a.zig"); }\n')+ public = Path(temp) / 'test_public.py'+ public.write_text('raise AssertionError()')+ snap = measurement.verifier_snapshot(temp, {}, temp)+ # Registering a new test file in the test root is part of a fix.+ hooks.write_text('test { _ = @import("a.zig"); _ = @import("b.zig"); }\n')+ self.assertTrue(measurement.verifiers_unchanged(snap))+ public.write_text('pass')+ self.assertFalse(measurement.verifiers_unchanged(snap))++ def test_a_mined_task_protects_its_named_test_not_the_file_it_sits_in(self):+ task = {'setup': ['python3 "$TASK_ROOT/live_setup.py" pr-9999 .']}+ body = 'pub fn f() u8 {\n return 1;\n}\n\ntest "f is two" {\n try std.testing.expect(f() == 2);\n}\n'+ with tempfile.TemporaryDirectory() as temp, \+ patch('live_setup.named_case', return_value=('src/f.zig', 'f is two')):+ (Path(temp) / 'src').mkdir()+ source = Path(temp) / 'src' / 'f.zig'+ source.write_text(body)+ snap = measurement.verifier_snapshot(temp, task, temp)+ source.write_text(body.replace('return 1;', 'return 2;')) # the fix, same file+ self.assertTrue(measurement.verifiers_unchanged(snap))+ source.write_text(body.replace('f() == 2', 'f() == 1')) # the test itself+ self.assertFalse(measurement.verifiers_unchanged(snap))++ def test_a_sandbox_gets_its_own_repository_holding_the_task(self):+ import subprocess+ with tempfile.TemporaryDirectory() as outer:+ subprocess.run(['git', 'init', '-q', outer], check=True)+ sandbox = Path(outer) / 'sandboxes' / 'one'+ (sandbox / '.zig-cache').mkdir(parents=True)+ (sandbox / 'note.txt').write_text('task\n')+ (sandbox / '.zig-cache' / 'x').write_text('cache')+ top = lambda: subprocess.run(['git', '-C', str(sandbox), 'rev-parse', '--show-toplevel'],+ capture_output=True, text=True).stdout.strip()+ self.assertEqual(Path(top()).resolve(), Path(outer).resolve()) # the enclosing checkout+ measurement.isolate_git(str(sandbox))+ self.assertEqual(Path(top()).resolve(), sandbox.resolve())+ status = subprocess.run(['git', '-C', str(sandbox), 'status', '--porcelain'],+ capture_output=True, text=True, check=True).stdout+ self.assertEqual(status, '')+ shown = subprocess.run(['git', '-C', str(sandbox), 'show', 'HEAD:note.txt'],+ capture_output=True, text=True, check=True).stdout+ self.assertEqual(shown, 'task\n')+ # A task that made its own repository keeps it.+ head = subprocess.run(['git', '-C', str(sandbox), 'rev-parse', 'HEAD'],+ capture_output=True, text=True, check=True).stdout+ measurement.isolate_git(str(sandbox))+ self.assertEqual(subprocess.run(['git', '-C', str(sandbox), 'rev-parse', 'HEAD'],+ capture_output=True, text=True, check=True).stdout, head)+def test_untracked_source_is_in_snapshot_receipt(self):import subprocesswith tempfile.TemporaryDirectory() as temp: