graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
{
"schema": "codegraff.eval.tier1.manifest.v1",
"comment": [
"Tier 1 of the internal eval set: the deterministic invariants the pre-push hook enforces.",
"No network, no model calls. Tier 2 (model-backed behavior) lives in evals/ and is opt-in.",
"test_count_baseline is a ratchet: the suite may grow, it may never shrink. A drop means a",
"module stopped being referenced from the test root and its tests silently vanished.",
"It is a FLOOR and nothing raises it automatically - bump it by hand at each release cut.",
"test_count_slack is how far the real suite may run ahead of that floor before the tests",
"check starts WARNING that the ratchet has stalled: it sat 350 behind for months (#439),",
"which is a floor low enough to catch nothing."
],
"test_roots": [
"src/main.zig",
"src/repl.zig"
],
"test_count_baseline": 3652,
"test_count_slack": 25,
"required_invariants": [
{
"id": "goal-epochs",
"test": "epochs: nextEpoch clears the live goal AND every parked epoch; currentEpoch 0 without a goal",
"why": "#318: a new goal inherited the previous goal's checklist and was born done."
},
{
"id": "todo-preserve",
"test": "a replace keeps omitted completed items and still drops omitted open ones",
"why": "2896b32: a replace deleted completed items the model omitted, erasing finished work."
},
{
"id": "todo-empty-write",
"test": "a write with zero usable items is rejected untouched, never completion evidence (#318)",
"why": "6300514: an empty todo_write wiped the checklist and read as completion."
},
{
"id": "loop-expired",
"test": "goal run: a passed wall-clock deadline stops as expired and flips nothing",
"why": "6300514: a timed run past its deadline must stop as expired, not as accepted."
},
{
"id": "loop-idle",
"test": "goal run: a restored all-[x] list is idle, not accepted (#318)",
"why": "#318: a leftover finished checklist ended a fresh run at iteration 1."
},
{
"id": "goal-loop-one-run",
"test": "/goal is the one autonomous run; /loop no longer starts one",
"why": "010b9ad, ADR 0216: /goal is the autonomous run; the retired /loop must not come back as a second entry point."
},
{
"id": "compaction-keeps-goal",
"test": "compactionSnapshot: no live goal, no checklist, and a capped render (#318)",
"why": "#318: compaction dropped the goal and its completed items mid-run."
},
{
"id": "completion-gate",
"test": "completionGate: open list refused once, no-checklist needs confirm, done list accepts, review exempt (#318)",
"why": "#318: the model could claim completion over an unfinished checklist."
},
{
"id": "child-deadline",
"test": "childTaskPrompt: the same absolute deadline reaches every child, minus a margin",
"why": "d33c4a4: a timed /goal run must hand its deadline down, or children outlive the run."
},
{
"id": "cache-affinity-scratch",
"test": "affinity: two scratch sandboxes share one project cache id",
"why": "289 remasure: hashing the leaf cwd minted a unique xAI partition per eval sandbox and paid the system+tools prefix at $2/M."
},
{
"id": "sse-error-not-truncated",
"test": "#748: error-only OpenAI SSE is an error envelope, not a missing stream",
"why": "#748: an explicit streamed error was retried as a truncated gateway body."
},
{
"id": "edit-selected-worktree",
"test": "#747: editing the same relative name after a tree switch lands on the selected root",
"why": "#747: an edit could report success while the selected worktree stayed unchanged."
},
{
"id": "http2-window",
"test": "http-zig credits DATA on the connection and the stream",
"why": "A second HTTP/2 stream stalls after 64KB if DATA is not credited."
},
{
"id": "http2-reuse",
"test": "keep the pooled session only after END_STREAM",
"why": "Reusing a half-read HTTP/2 session feeds the next turn the previous frame."
}
],
"docs_only_paths": [
"*.md",
"docs/*",
"plans/*",
"social/*",
"socials/*",
"assets/*",
"benchmarks/*.png",
"*.png",
"LICENSE"
]
}