Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Evals: mined prompts name where the test runs; new test files ship whole

justrach5ef63619ec54
diff --git a/graff-evals/design/mine_prs.py b/graff-evals/design/mine_prs.py
index e172add7..f8ca2030 100644
--- a/graff-evals/design/mine_prs.py
+++ b/graff-evals/design/mine_prs.py
@@ -189,7 +189,11 @@ def write(path: str, text: str, mode: int = 0o644) -> None:
def prompt_for(title: str, public: dict) -> str:
- return (f"{title}. Make the test named \"{public['name']}\" in {public['file']} pass. "
+ # State what the grader checks: the test must run in the project's own
+ # test build. "Make it pass" alone let a run prove it in a temporary
+ # harness and leave it unreachable from the suite (ADR 0214).
+ return (f"{title}. Make the test named \"{public['name']}\" in {public['file']} run and pass "
+ "in the project's unit tests (`zig build test`), not only in isolation. "
"Do not edit that test. Do not add SPEC.md.")
@@ -216,7 +220,19 @@ def build(pr_number: int, public_name: str | None, hidden_name: str | None, ref:
raise SystemExit(f"could not extract test {t['name']!r} from {t['file']}")
blocks[role] = block
tdir = os.path.join(EVALS, "live", task_id)
- write(os.path.join(tdir, "public_case.inc"), "\n" + blocks["public"])
+ public_text = "\n" + blocks["public"]
+ parent_has = subprocess.run(["git", "-C", REPO, "cat-file", "-e", f"{pr['parent']}:{public['file']}"],
+ capture_output=True).returncode == 0
+ if not parent_has and is_test_file(public["file"]):
+ # A new test-only file is part of the spec: without its imports and
+ # helpers the test cannot compile unless the test file is edited,
+ # which the prompt forbids. Ship the whole file, minus the hidden
+ # test. (A new source module with inline tests is the solution
+ # itself, so there only the test block is given.)
+ public_text = file_at(pr["merge"], public["file"])
+ if hidden and hidden["file"] == public["file"]:
+ public_text = public_text.replace(blocks["hidden"], "")
+ write(os.path.join(tdir, "public_case.inc"), public_text)
write(os.path.join(tdir, "check_public.sh"),
"#!/bin/sh\nset -eu\nTASK_ROOT=${TASK_ROOT:?}\n"
f"exec python3 \"$TASK_ROOT/named_unit_check.py\" {json.dumps(public['name'])}\n", 0o755)
@@ -342,6 +358,8 @@ def self_test() -> None:
assert test_block(src, "second") == 'test "second" {}\n'
assert test_block(src, "missing") is None
assert TEST_RE.match('+test "a \\"quoted\\" name" {').group(1) == 'a \\"quoted\\" name'
+ prompt = prompt_for("Fix it", {"name": "alpha", "file": "src/a.zig"})
+ assert "run and pass in the project's unit tests (`zig build test`)" in prompt, prompt
print("mine_prs self-test ok")
diff --git a/graff-evals/hillclimb/split-mined-candidate.json b/graff-evals/hillclimb/split-mined-candidate.json
new file mode 100644
index 00000000..b1baebf7
--- /dev/null
+++ b/graff-evals/hillclimb/split-mined-candidate.json
@@ -0,0 +1,28 @@
+{
+ "seed": "graff",
+ "test": [
+ "pr-1257",
+ "pr-1309",
+ "pr-1329",
+ "pr-1339",
+ "pr-1348",
+ "pr-1364",
+ "pr-1386"
+ ],
+ "train": [
+ "pr-1253",
+ "pr-1254",
+ "pr-1256",
+ "pr-1302",
+ "pr-1306",
+ "pr-1327",
+ "pr-1328",
+ "pr-1330",
+ "pr-1336",
+ "pr-1365",
+ "pr-1373",
+ "pr-1376",
+ "pr-1385"
+ ],
+ "suite": "mined-candidate"
+}
diff --git a/graff-evals/live/mined.json b/graff-evals/live/mined.json
index ab7108b6..dd82a180 100644
--- a/graff-evals/live/mined.json
+++ b/graff-evals/live/mined.json
@@ -493,33 +493,6 @@
"ok": true
}
},
- {
- "id": "pr-1256",
- "pr": "codegraff #1256",
- "why": "Apply GRAFF_NO_PLUGINS before connecting MCP servers",
- "repo": "justrach/codegraff",
- "historical_parent": "4706cf974edc48b9f908d173df4acee9347fa0c0",
- "historical_merge": "5d45a5fde640f75de61822a38aae44256f87cace",
- "source": "local-git",
- "public_filter": "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config",
- "public_file": "src/session_start_plugins_test.zig",
- "hidden": null,
- "hidden_filter": null,
- "hidden_file": null,
- "check_timeout_s": 420,
- "validation": {
- "red": {
- "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "missing"
- },
- "green": {
- "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "pass"
- },
- "stable": {
- "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "pass"
- },
- "ok": true
- }
- },
{
"id": "pr-1254",
"pr": "codegraff #1254",
@@ -576,6 +549,33 @@
},
"ok": true
}
+ },
+ {
+ "id": "pr-1256",
+ "pr": "codegraff #1256",
+ "why": "Apply GRAFF_NO_PLUGINS before connecting MCP servers",
+ "repo": "justrach/codegraff",
+ "historical_parent": "4706cf974edc48b9f908d173df4acee9347fa0c0",
+ "historical_merge": "5d45a5fde640f75de61822a38aae44256f87cace",
+ "source": "local-git",
+ "public_filter": "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config",
+ "public_file": "src/session_start_plugins_test.zig",
+ "hidden": null,
+ "hidden_filter": null,
+ "hidden_file": null,
+ "check_timeout_s": 420,
+ "validation": {
+ "red": {
+ "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "missing"
+ },
+ "green": {
+ "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "pass"
+ },
+ "stable": {
+ "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config": "pass"
+ },
+ "ok": true
+ }
}
],
"mined": []
diff --git a/graff-evals/live/pr-1253/public_case.inc b/graff-evals/live/pr-1253/public_case.inc
index 2ccd0b12..e1598051 100644
--- a/graff-evals/live/pr-1253/public_case.inc
+++ b/graff-evals/live/pr-1253/public_case.inc
@@ -1,3 +1,22 @@
+//! Codex WS turns surface deltas as they arrive, through the same per-line
+//! hook (printDelta) as the SSE reader. Before, a WS turn buffered every frame
+//! and only rendered the answer after response.completed: ACP clients got the
+//! whole reply as one chunk at the end, and a turn that stalled mid-answer had
+//! shown nothing at all.
+
+const std = @import("std");
+const builtin = @import("builtin");
+const Io = std.Io;
+
+const http = @import("http.zig");
+const http_stall = @import("http_stall.zig");
+const ws = @import("ws.zig");
+const agent_ws = @import("agent_ws.zig");
+
+const mock = @import("agent_ws_mock.zig");
+const Mock = mock.Mock;
+const nowMs = mock.nowMs;
+const mockAgent = mock.mockAgent;
test "a WS text delta reaches the UI before the response completes" {
if (builtin.os.tag == .windows) return error.SkipZigTest;
diff --git a/graff-evals/live/pr-1256/public_case.inc b/graff-evals/live/pr-1256/public_case.inc
index ae4bccbf..9cb42ff2 100644
--- a/graff-evals/live/pr-1256/public_case.inc
+++ b/graff-evals/live/pr-1256/public_case.inc
@@ -1,3 +1,8 @@
+//! session_start.initRegistryConsent and the plugins switch, kept apart from
+//! session_start.zig (600-line ceiling).
+const std = @import("std");
+const Io = std.Io;
+const session_start = @import("session_start.zig");
test "initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config" {
const plugins = @import("plugins.zig");
diff --git a/graff-evals/tasks/mined-pr-1253.json b/graff-evals/tasks/mined-pr-1253.json
index 587e0b3f..0a2efd8c 100644
--- a/graff-evals/tasks/mined-pr-1253.json
+++ b/graff-evals/tasks/mined-pr-1253.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1253 — mined (ADR 0214)",
- "prompt": "ws: stream codex WS deltas through printDelta like SSE. Make the test named \"a WS text delta reaches the UI before the response completes\" in src/agent_ws_stream_test.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ws: stream codex WS deltas through printDelta like SSE. Make the test named \"a WS text delta reaches the UI before the response completes\" in src/agent_ws_stream_test.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1253 ."
],
diff --git a/graff-evals/tasks/mined-pr-1254.json b/graff-evals/tasks/mined-pr-1254.json
index a56f51ef..9a390d7c 100644
--- a/graff-evals/tasks/mined-pr-1254.json
+++ b/graff-evals/tasks/mined-pr-1254.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1254 — mined (ADR 0214)",
- "prompt": "Connect the MCP servers an ACP client passes. Make the test named \"parse: stdio with args and env, http with headers, sse and malformed left out\" in src/acp_mcp_servers.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Connect the MCP servers an ACP client passes. Make the test named \"parse: stdio with args and env, http with headers, sse and malformed left out\" in src/acp_mcp_servers.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1254 ."
],
diff --git a/graff-evals/tasks/mined-pr-1256.json b/graff-evals/tasks/mined-pr-1256.json
index 22413e22..dd2cb224 100644
--- a/graff-evals/tasks/mined-pr-1256.json
+++ b/graff-evals/tasks/mined-pr-1256.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1256 — mined (ADR 0214)",
- "prompt": "Apply GRAFF_NO_PLUGINS before connecting MCP servers. Make the test named \"initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config\" in src/session_start_plugins_test.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Apply GRAFF_NO_PLUGINS before connecting MCP servers. Make the test named \"initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config\" in src/session_start_plugins_test.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1256 ."
],
diff --git a/graff-evals/tasks/mined-pr-1257.json b/graff-evals/tasks/mined-pr-1257.json
index 2a7cb1b7..5e3fdcdc 100644
--- a/graff-evals/tasks/mined-pr-1257.json
+++ b/graff-evals/tasks/mined-pr-1257.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1257 — mined (ADR 0214)",
- "prompt": "stdin: read ACP and --json records past the 64 KiB buffer. Make the test named \"records larger than the reader buffer are returned whole\" in src/stdin_line.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "stdin: read ACP and --json records past the 64 KiB buffer. Make the test named \"records larger than the reader buffer are returned whole\" in src/stdin_line.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1257 ."
],
diff --git a/graff-evals/tasks/mined-pr-1302.json b/graff-evals/tasks/mined-pr-1302.json
index 089c9812..a1a1bf92 100644
--- a/graff-evals/tasks/mined-pr-1302.json
+++ b/graff-evals/tasks/mined-pr-1302.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1302 — mined (ADR 0214)",
- "prompt": "Bound publication review input for small diffs in large files. Make the test named \"claim review uses committed context when changed head source exceeds the budget\" in src/pr_review_input.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Bound publication review input for small diffs in large files. Make the test named \"claim review uses committed context when changed head source exceeds the budget\" in src/pr_review_input.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1302 ."
],
diff --git a/graff-evals/tasks/mined-pr-1306.json b/graff-evals/tasks/mined-pr-1306.json
index 69aea126..84b9f1df 100644
--- a/graff-evals/tasks/mined-pr-1306.json
+++ b/graff-evals/tasks/mined-pr-1306.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1306 — mined (ADR 0214)",
- "prompt": "Strip provider citation markers from ACP text. Make the test named \"a whole citation marker is removed and plain text passes through untouched\" in src/acp_citations.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Strip provider citation markers from ACP text. Make the test named \"a whole citation marker is removed and plain text passes through untouched\" in src/acp_citations.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1306 ."
],
diff --git a/graff-evals/tasks/mined-pr-1309.json b/graff-evals/tasks/mined-pr-1309.json
index 9a815856..8ef830a3 100644
--- a/graff-evals/tasks/mined-pr-1309.json
+++ b/graff-evals/tasks/mined-pr-1309.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1309 — mined (ADR 0214)",
- "prompt": "ACP: report context left with the standard usage_update. Make the test named \"usage_update carries used and size, and cost only when known\" in src/acp_context_meter.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ACP: report context left with the standard usage_update. Make the test named \"usage_update carries used and size, and cost only when known\" in src/acp_context_meter.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1309 ."
],
diff --git a/graff-evals/tasks/mined-pr-1327.json b/graff-evals/tasks/mined-pr-1327.json
index 66e63618..a07831df 100644
--- a/graff-evals/tasks/mined-pr-1327.json
+++ b/graff-evals/tasks/mined-pr-1327.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1327 — mined (ADR 0214)",
- "prompt": "ask_user: read the question from every argument shape models send. Make the test named \"the schema shape passes through unchanged\" in src/ask_user_args.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ask_user: read the question from every argument shape models send. Make the test named \"the schema shape passes through unchanged\" in src/ask_user_args.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1327 ."
],
diff --git a/graff-evals/tasks/mined-pr-1328.json b/graff-evals/tasks/mined-pr-1328.json
index 24a861c1..a603be54 100644
--- a/graff-evals/tasks/mined-pr-1328.json
+++ b/graff-evals/tasks/mined-pr-1328.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1328 — mined (ADR 0214)",
- "prompt": "Free the MCP shape cache at session teardown. Make the test named \"shutdown frees a filled cache and is a no-op when empty (#1196)\" in src/mcp_shapes.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Free the MCP shape cache at session teardown. Make the test named \"shutdown frees a filled cache and is a no-op when empty (#1196)\" in src/mcp_shapes.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1328 ."
],
diff --git a/graff-evals/tasks/mined-pr-1329.json b/graff-evals/tasks/mined-pr-1329.json
index f4310b10..c410c44a 100644
--- a/graff-evals/tasks/mined-pr-1329.json
+++ b/graff-evals/tasks/mined-pr-1329.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1329 — mined (ADR 0214)",
- "prompt": "ACP session/new stops waiting on unreachable MCP servers. Make the test named \"joinWithin (#1291): a server that fails fast ends the wait early\" in src/acp_mcp_servers.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ACP session/new stops waiting on unreachable MCP servers. Make the test named \"joinWithin (#1291): a server that fails fast ends the wait early\" in src/acp_mcp_servers.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1329 ."
],
diff --git a/graff-evals/tasks/mined-pr-1330.json b/graff-evals/tasks/mined-pr-1330.json
index 244785c9..24eb110b 100644
--- a/graff-evals/tasks/mined-pr-1330.json
+++ b/graff-evals/tasks/mined-pr-1330.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1330 — mined (ADR 0214)",
- "prompt": "Run tool calls a server left in content as inline markup. Make the test named \"recoverInlineCalls (#1247): markup after prose becomes calls, parameters in any order\" in src/tool_call_repair_tests.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Run tool calls a server left in content as inline markup. Make the test named \"recoverInlineCalls (#1247): markup after prose becomes calls, parameters in any order\" in src/tool_call_repair_tests.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1330 ."
],
diff --git a/graff-evals/tasks/mined-pr-1336.json b/graff-evals/tasks/mined-pr-1336.json
index 8d2228b6..1703eb59 100644
--- a/graff-evals/tasks/mined-pr-1336.json
+++ b/graff-evals/tasks/mined-pr-1336.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1336 — mined (ADR 0214)",
- "prompt": "Resolve a PR check that finished in the background. Make the test named \"a check promoted to the background resolves when its job exits 0 (#1326)\" in src/pr_local_checks.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Resolve a PR check that finished in the background. Make the test named \"a check promoted to the background resolves when its job exits 0 (#1326)\" in src/pr_local_checks.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1336 ."
],
diff --git a/graff-evals/tasks/mined-pr-1339.json b/graff-evals/tasks/mined-pr-1339.json
index a5fc5c72..6331319f 100644
--- a/graff-evals/tasks/mined-pr-1339.json
+++ b/graff-evals/tasks/mined-pr-1339.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1339 — mined (ADR 0214)",
- "prompt": "ask_user: a question that comes back unanswered is not a cancel. Make the test named \"decline, cancel, an empty accept and an error each end the question as themselves (#1322)\" in src/acp_elicit.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ask_user: a question that comes back unanswered is not a cancel. Make the test named \"decline, cancel, an empty accept and an error each end the question as themselves (#1322)\" in src/acp_elicit.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1339 ."
],
diff --git a/graff-evals/tasks/mined-pr-1348.json b/graff-evals/tasks/mined-pr-1348.json
index fd821c26..fdbb0f1a 100644
--- a/graff-evals/tasks/mined-pr-1348.json
+++ b/graff-evals/tasks/mined-pr-1348.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1348 — mined (ADR 0214)",
- "prompt": "PR readiness keys checks by what they run; rlm says when sleep_ms is capped. Make the test named \"one check in one directory is one entry whatever its spelling\" in src/pr_local_checks.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "PR readiness keys checks by what they run; rlm says when sleep_ms is capped. Make the test named \"one check in one directory is one entry whatever its spelling\" in src/pr_local_checks.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1348 ."
],
diff --git a/graff-evals/tasks/mined-pr-1364.json b/graff-evals/tasks/mined-pr-1364.json
index de6eaa77..2d1d5392 100644
--- a/graff-evals/tasks/mined-pr-1364.json
+++ b/graff-evals/tasks/mined-pr-1364.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1364 — mined (ADR 0214)",
- "prompt": "Informational subagents keep read tools (#1360). Make the test named \"a read-only child keeps reads and read-only shell, not writes (#1360)\" in src/agent_tool_gate.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Informational subagents keep read tools (#1360). Make the test named \"a read-only child keeps reads and read-only shell, not writes (#1360)\" in src/agent_tool_gate.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1364 ."
],
diff --git a/graff-evals/tasks/mined-pr-1365.json b/graff-evals/tasks/mined-pr-1365.json
index 06f37fbe..430ee743 100644
--- a/graff-evals/tasks/mined-pr-1365.json
+++ b/graff-evals/tasks/mined-pr-1365.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1365 — mined (ADR 0214)",
- "prompt": "Anthropic: refuse a cut-off tool input instead of running it as {} (refs #1218). Make the test named \"a cut-off Anthropic tool input is refused, not run as {} (#1218)\" in src/tool_call_args.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Anthropic: refuse a cut-off tool input instead of running it as {} (refs #1218). Make the test named \"a cut-off Anthropic tool input is refused, not run as {} (#1218)\" in src/tool_call_args.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1365 ."
],
diff --git a/graff-evals/tasks/mined-pr-1373.json b/graff-evals/tasks/mined-pr-1373.json
index d9ea24c1..fd6f99b4 100644
--- a/graff-evals/tasks/mined-pr-1373.json
+++ b/graff-evals/tasks/mined-pr-1373.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1373 — mined (ADR 0214)",
- "prompt": "ACP: frame Harness room deliveries as advisory room messages. Make the test named \"an agent's room line becomes an advisory room message; a person's stays a prompt\" in src/acp_room.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ACP: frame Harness room deliveries as advisory room messages. Make the test named \"an agent's room line becomes an advisory room message; a person's stays a prompt\" in src/acp_room.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1373 ."
],
diff --git a/graff-evals/tasks/mined-pr-1376.json b/graff-evals/tasks/mined-pr-1376.json
index 93973e2c..b4d3f3ef 100644
--- a/graff-evals/tasks/mined-pr-1376.json
+++ b/graff-evals/tasks/mined-pr-1376.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1376 — mined (ADR 0214)",
- "prompt": "ACP room deliveries: name another account's agent as such. Make the test named \"another account's agent is named as such, and framed text must say so to be kept\" in src/acp_room.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ACP room deliveries: name another account's agent as such. Make the test named \"another account's agent is named as such, and framed text must say so to be kept\" in src/acp_room.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1376 ."
],
diff --git a/graff-evals/tasks/mined-pr-1385.json b/graff-evals/tasks/mined-pr-1385.json
index 95fdc12e..94f1d7ed 100644
--- a/graff-evals/tasks/mined-pr-1385.json
+++ b/graff-evals/tasks/mined-pr-1385.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1385 — mined (ADR 0214)",
- "prompt": "ACP: saved views carry _meta[\"graff/view\"] on the completed tool update. Make the test named \"detect: graff's own view and MCP app snapshots\" in src/acp_view_meta.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ACP: saved views carry _meta[\"graff/view\"] on the completed tool update. Make the test named \"detect: graff's own view and MCP app snapshots\" in src/acp_view_meta.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1385 ."
],
diff --git a/graff-evals/tasks/mined-pr-1386.json b/graff-evals/tasks/mined-pr-1386.json
index eba2a348..13b03649 100644
--- a/graff-evals/tasks/mined-pr-1386.json
+++ b/graff-evals/tasks/mined-pr-1386.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1386 — mined (ADR 0214)",
- "prompt": "graff/models: each row names the model's maker. Make the test named \"maker: the gateway serves other labs' models\" in src/model_maker.zig pass. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "graff/models: each row names the model's maker. Make the test named \"maker: the gateway serves other labs' models\" in src/model_maker.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1386 ."
],