Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
4.2 KB
Raw Blame
{
  "schema": "codegraff.eval.tier1.manifest.v1",
  "comment": [
    "Tier 1 of the internal eval set: the deterministic invariants the pre-push hook enforces.",
    "No network, no model calls. Tier 2 (model-backed behavior) lives in evals/ and is opt-in.",
    "test_count_baseline is a ratchet: the suite may grow, it may never shrink. A drop means a",
    "module stopped being referenced from the test root and its tests silently vanished.",
    "It is a FLOOR and nothing raises it automatically - bump it by hand at each release cut.",
    "test_count_slack is how far the real suite may run ahead of that floor before the tests",
    "check starts WARNING that the ratchet has stalled: it sat 350 behind for months (#439),",
    "which is a floor low enough to catch nothing."
  ],
  "test_roots": [
    "src/main.zig",
    "src/repl.zig"
  ],
  "test_count_baseline": 3652,
  "test_count_slack": 25,
  "required_invariants": [
    {
      "id": "goal-epochs",
      "test": "epochs: nextEpoch clears the live goal AND every parked epoch; currentEpoch 0 without a goal",
      "why": "#318: a new goal inherited the previous goal's checklist and was born done."
    },
    {
      "id": "todo-preserve",
      "test": "a replace keeps omitted completed items and still drops omitted open ones",
      "why": "2896b32: a replace deleted completed items the model omitted, erasing finished work."
    },
    {
      "id": "todo-empty-write",
      "test": "a write with zero usable items is rejected untouched, never completion evidence (#318)",
      "why": "6300514: an empty todo_write wiped the checklist and read as completion."
    },
    {
      "id": "loop-expired",
      "test": "goal run: a passed wall-clock deadline stops as expired and flips nothing",
      "why": "6300514: a timed run past its deadline must stop as expired, not as accepted."
    },
    {
      "id": "loop-idle",
      "test": "goal run: a restored all-[x] list is idle, not accepted (#318)",
      "why": "#318: a leftover finished checklist ended a fresh run at iteration 1."
    },
    {
      "id": "goal-loop-one-run",
      "test": "/goal is the one autonomous run; /loop no longer starts one",
      "why": "010b9ad, ADR 0216: /goal is the autonomous run; the retired /loop must not come back as a second entry point."
    },
    {
      "id": "compaction-keeps-goal",
      "test": "compactionSnapshot: no live goal, no checklist, and a capped render (#318)",
      "why": "#318: compaction dropped the goal and its completed items mid-run."
    },
    {
      "id": "completion-gate",
      "test": "completionGate: open list refused once, no-checklist needs confirm, done list accepts, review exempt (#318)",
      "why": "#318: the model could claim completion over an unfinished checklist."
    },
    {
      "id": "child-deadline",
      "test": "childTaskPrompt: the same absolute deadline reaches every child, minus a margin",
      "why": "d33c4a4: a timed /goal run must hand its deadline down, or children outlive the run."
    },
    {
      "id": "cache-affinity-scratch",
      "test": "affinity: two scratch sandboxes share one project cache id",
      "why": "289 remasure: hashing the leaf cwd minted a unique xAI partition per eval sandbox and paid the system+tools prefix at $2/M."
    },
    {
      "id": "sse-error-not-truncated",
      "test": "#748: error-only OpenAI SSE is an error envelope, not a missing stream",
      "why": "#748: an explicit streamed error was retried as a truncated gateway body."
    },
    {
      "id": "edit-selected-worktree",
      "test": "#747: editing the same relative name after a tree switch lands on the selected root",
      "why": "#747: an edit could report success while the selected worktree stayed unchanged."
    },
    {
      "id": "http2-window",
      "test": "http-zig credits DATA on the connection and the stream",
      "why": "A second HTTP/2 stream stalls after 64KB if DATA is not credited."
    },
    {
      "id": "http2-reuse",
      "test": "keep the pooled session only after END_STREAM",
      "why": "Reusing a half-read HTTP/2 session feeds the next turn the previous frame."
    }
  ],
  "docs_only_paths": [
    "*.md",
    "docs/*",
    "plans/*",
    "social/*",
    "socials/*",
    "assets/*",
    "benchmarks/*.png",
    "*.png",
    "LICENSE"
  ]
}