Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

graff-evals: subagents suite and its solo counterfactual

justrachebe51237936b
diff --git a/graff-evals/README.md b/graff-evals/README.md
index 19e0d2a4..e4cdeba3 100644
--- a/graff-evals/README.md
+++ b/graff-evals/README.md
@@ -24,6 +24,8 @@ harness under test spends model calls.
| `inhouse` | Distilled PR fixtures with SPEC.md. Cheap harness A/B — not the live badge. Opt-in. |
| `live` | Capped 12 gated PRs, no SPEC.md. Pass @ n=3; list$ on passing reps only. See [LIVE.md](../artifacts/graff-evals-live/LIVE.md). |
| `bg` | Commands that outlast the 15s `-p` foreground wait, so each run must collect background results. For A/Bs of how the harness words and delivers them. Opt-in. |
+| `subagents` | Work the prompt asks to hand to sub-agents in parallel: per-package bugfixes, a per-service log census, an API rename across three packages, an endpoint-coverage merge of two inventories, and a docs sidecar beside a fix. Fixtures are git repos; held-out checks. Opt-in. |
+| `subagents-solo` | The same five tasks with the delegation wording removed: the counterfactual that says whether the children paid for themselves. Opt-in. |
Terminal-Bench and other Harbor datasets run graff through the adapter in [harbor/](harbor/README.md).
@@ -56,6 +58,9 @@ CODEGRAFF_API_KEY=cg_sk_… ./run.py --suite swe --harness graff-dev,opencode-co
# Results: evals/graff-vs-codex-app-server/RESULTS.md
./run.py --suite mcp --harness graff-repl,codex-app-server -j 8 --interleave
./run.py --suite core --harness graff-repl,codex-app-server,pi-codex -j 8 --interleave
+# sub-agents and their counterfactual on one model; tools in the eval tree split each run into
+# parent and child time (the codex driver tags each child thread in EVAL_TRACE_DIR)
+./run.py --suite subagents,subagents-solo --harness graff-repl,codex-app-server --model gpt-6-astra -j 6 --interleave
# one gateway model in graff, Claude Code, OpenCode 2 and pi (set CODEGRAFF_API_KEY; OPENCODE2_BIN
# when `opencode` on PATH is 1.x; Claude Code needs an Anthropic-format proxy in front of the
# gateway, see the write-up). Results: evals/graff-vs-claude-code/RESULTS.md
diff --git a/graff-evals/codex_app_server.py b/graff-evals/codex_app_server.py
index 5da71afd..843a61bf 100755
--- a/graff-evals/codex_app_server.py
+++ b/graff-evals/codex_app_server.py
@@ -214,13 +214,13 @@ def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer:
state["after_item"] = time.monotonic()
elif method == "item/completed":
item = params.get("item") or {}
- trace(dict(item_sizes(item), thread="root" if root else "child"))
+ trace(dict(item_sizes(item), thread="root" if root else "child", tid=(tid or "")[-8:]))
if root and item.get("type") == "agentMessage":
state["answer"] = item.get("text") or state["answer"]
elif method == "thread/tokenUsage/updated":
totals[tid or thread_id] = (params.get("tokenUsage") or {}).get("total")
state["usage"] = summed(totals)
- trace({"ev": "usage", "thread": "root" if root else "child", "last": (params.get("tokenUsage") or {}).get("last")})
+ trace({"ev": "usage", "thread": "root" if root else "child", "tid": (tid or "")[-8:], "last": (params.get("tokenUsage") or {}).get("last")})
state["calls"] = state.get("calls", 0) + 1
if os.environ.get("EVAL_DEBUG"):
print(f"[usage-event] {json.dumps(params.get('tokenUsage'))}", file=sys.stderr)
diff --git a/graff-evals/fixtures/sa-coverage/api/__init__.py b/graff-evals/fixtures/sa-coverage/api/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/graff-evals/fixtures/sa-coverage/api/app.py b/graff-evals/fixtures/sa-coverage/api/app.py
new file mode 100644
index 00000000..ae1a3484
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/api/app.py
@@ -0,0 +1,13 @@
+"""Tiny router: endpoints register with @route(METHOD, PATH) or add_route(METHOD, PATH, handler)."""
+ROUTES = {}
+
+
+def add_route(method, path, handler):
+ ROUTES[(method, path)] = handler
+ return handler
+
+
+def route(method, path):
+ def wrap(handler):
+ return add_route(method, path, handler)
+ return wrap
diff --git a/graff-evals/fixtures/sa-coverage/api/routes/__init__.py b/graff-evals/fixtures/sa-coverage/api/routes/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/graff-evals/fixtures/sa-coverage/api/routes/admin.py b/graff-evals/fixtures/sa-coverage/api/routes/admin.py
new file mode 100644
index 00000000..316edc9f
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/api/routes/admin.py
@@ -0,0 +1,18 @@
+from api.app import add_route, route
+
+
+@route("GET", "/admin/stats")
+def stats(req):
+ return {}
+
+
+@route("POST", "/admin/reindex")
+def reindex(req):
+ return {}
+
+
+def flush(req):
+ return {}
+
+
+# add_route("POST", "/admin/flush", flush) # removed: flush runs from cron now
diff --git a/graff-evals/fixtures/sa-coverage/api/routes/billing.py b/graff-evals/fixtures/sa-coverage/api/routes/billing.py
new file mode 100644
index 00000000..940b76e5
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/api/routes/billing.py
@@ -0,0 +1,21 @@
+from api.app import route
+
+
+@route("GET", "/invoices")
+def list_invoices(req):
+ return []
+
+
+@route("GET", "/invoices/<id>")
+def get_invoice(req, id):
+ return {}
+
+
+@route("POST", "/invoices/<id>/pay")
+def pay_invoice(req, id):
+ return {}
+
+
+@route("PUT", "/billing/plan")
+def change_plan(req):
+ return {}
diff --git a/graff-evals/fixtures/sa-coverage/api/routes/health.py b/graff-evals/fixtures/sa-coverage/api/routes/health.py
new file mode 100644
index 00000000..ec9ae99b
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/api/routes/health.py
@@ -0,0 +1,13 @@
+from api.app import add_route
+
+
+def health(req):
+ return "ok"
+
+
+def ready(req):
+ return "ready"
+
+
+add_route("GET", "/health", health)
+add_route("GET", "/ready", ready)
diff --git a/graff-evals/fixtures/sa-coverage/api/routes/orders.py b/graff-evals/fixtures/sa-coverage/api/routes/orders.py
new file mode 100644
index 00000000..1265a132
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/api/routes/orders.py
@@ -0,0 +1,23 @@
+from api.app import add_route, route
+
+
+@route("GET", "/orders")
+def list_orders(req):
+ return []
+
+
+@route("GET", "/orders/<id>")
+def get_order(req, id):
+ return {}
+
+
+@route("POST", "/orders/<id>/cancel")
+def cancel_order(req, id):
+ return {}
+
+
+def refund(req, id):
+ return {}
+
+
+add_route("POST", "/orders/<id>/refund", refund)
diff --git a/graff-evals/fixtures/sa-coverage/api/routes/search.py b/graff-evals/fixtures/sa-coverage/api/routes/search.py
new file mode 100644
index 00000000..56b0ed0b
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/api/routes/search.py
@@ -0,0 +1,11 @@
+from api.app import route
+
+
+@route("GET", "/search")
+def search(req):
+ return []
+
+
+@route("GET", "/search/suggest")
+def suggest(req):
+ return []
diff --git a/graff-evals/fixtures/sa-coverage/api/routes/users.py b/graff-evals/fixtures/sa-coverage/api/routes/users.py
new file mode 100644
index 00000000..5bb8c6d3
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/api/routes/users.py
@@ -0,0 +1,21 @@
+from api.app import route
+
+
+@route("GET", "/users")
+def list_users(req):
+ return []
+
+
+@route("POST", "/users")
+def create_user(req):
+ return {}
+
+
+@route("GET", "/users/<id>")
+def get_user(req, id):
+ return {}
+
+
+# @route("DELETE", "/users/<id>") # disabled until soft-delete ships
+def delete_user(req, id):
+ return None
diff --git a/graff-evals/fixtures/sa-coverage/docs/api/billing.md b/graff-evals/fixtures/sa-coverage/docs/api/billing.md
new file mode 100644
index 00000000..ed61c879
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/docs/api/billing.md
@@ -0,0 +1,13 @@
+# Billing
+
+### GET /invoices
+Lists invoices.
+
+### POST /invoices/<id>/pay
+Pays an open invoice with the card on file.
+
+### PUT /billing/plan
+Switches plan; takes effect next cycle.
+
+### POST /billing/coupon
+Applies a coupon code to the account.
diff --git a/graff-evals/fixtures/sa-coverage/docs/api/ops.md b/graff-evals/fixtures/sa-coverage/docs/api/ops.md
new file mode 100644
index 00000000..0e7aa98b
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/docs/api/ops.md
@@ -0,0 +1,7 @@
+# Operations
+
+### GET /health
+Liveness probe; returns `ok`.
+
+### POST /admin/flush
+Flushes every cache.
diff --git a/graff-evals/fixtures/sa-coverage/docs/api/orders.md b/graff-evals/fixtures/sa-coverage/docs/api/orders.md
new file mode 100644
index 00000000..3a08643f
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/docs/api/orders.md
@@ -0,0 +1,13 @@
+# Orders
+
+### GET /orders
+Lists the caller's orders, newest first.
+
+### GET /orders/<id>
+One order with its line items.
+
+#### Example
+A cancelled order keeps its line items.
+
+### POST /orders/<id>/cancel
+Cancels an order that has not shipped.
diff --git a/graff-evals/fixtures/sa-coverage/docs/api/search.md b/graff-evals/fixtures/sa-coverage/docs/api/search.md
new file mode 100644
index 00000000..c4cdb6f1
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/docs/api/search.md
@@ -0,0 +1,7 @@
+# Search
+
+### GET /search
+Full-text search across users and orders.
+
+### GET /search/legacy
+The v1 search, kept for old clients.
diff --git a/graff-evals/fixtures/sa-coverage/docs/api/users.md b/graff-evals/fixtures/sa-coverage/docs/api/users.md
new file mode 100644
index 00000000..02d12910
--- /dev/null
+++ b/graff-evals/fixtures/sa-coverage/docs/api/users.md
@@ -0,0 +1,15 @@
+# Users
+
+All user endpoints return JSON. Use `GET /users` to page through accounts.
+
+### GET /users
+Lists users, 50 per page.
+
+### POST /users
+Creates a user from `{"name", "email"}`.
+
+### GET /users/<id>
+One user by id.
+
+### DELETE /users/<id>
+Deletes a user and their sessions.
diff --git a/graff-evals/fixtures/sa-fix-three/README.md b/graff-evals/fixtures/sa-fix-three/README.md
new file mode 100644
index 00000000..9581c3cc
--- /dev/null
+++ b/graff-evals/fixtures/sa-fix-three/README.md
@@ -0,0 +1,2 @@
+Three small, independent packages: geo (distances), textkit (strings), ledger (money in cents).
+Each has a test script: python3 <pkg>/test_<pkg>.py
diff --git a/graff-evals/fixtures/sa-fix-three/geo/__init__.py b/graff-evals/fixtures/sa-fix-three/geo/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/graff-evals/fixtures/sa-fix-three/geo/distance.py b/graff-evals/fixtures/sa-fix-three/geo/distance.py
new file mode 100644
index 00000000..72ea80f0
--- /dev/null
+++ b/graff-evals/fixtures/sa-fix-three/geo/distance.py
@@ -0,0 +1,31 @@
+"""Great-circle helpers."""
+import math
+
+EARTH_RADIUS_KM = 6371.0
+
+
+def haversine_km(a, b):
+ """Distance in km between two (lat, lon) points given in degrees."""
+ lat1, lon1 = a
+ lat2, lon2 = b
+ dlat = lat2 - lat1
+ dlon = lon2 - lon1
+ h = math.sin(dlat / 2) ** 2 + math.cos(lat1) * math.cos(lat2) * math.sin(dlon / 2) ** 2
+ return 2 * EARTH_RADIUS_KM * math.asin(math.sqrt(h))
+
+
+def bbox(points):
+ """(min_lat, min_lon, max_lat, max_lon) of a non-empty list of points."""
+ lats = [p[0] for p in points]
+ lons = [p[1] for p in points]
+ return (min(lats), max(lons), max(lats), min(lons))
+
+
+def nearest(origin, candidates):
+ """Name of the candidate closest to origin; candidates maps name -> (lat, lon)."""
+ best = None
+ for name, point in candidates.items():
+ d = haversine_km(origin, point)
+ if best is None or d > best[0]:
+ best = (d, name)
+ return best[1]
diff --git a/graff-evals/fixtures/sa-fix-three/geo/test_geo.py b/graff-evals/fixtures/sa-fix-three/geo/test_geo.py
new file mode 100644
index 00000000..c48af889
--- /dev/null
+++ b/graff-evals/fixtures/sa-fix-three/geo/test_geo.py
@@ -0,0 +1,16 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from geo.distance import bbox, haversine_km, nearest # noqa: E402
+
+PARIS = (48.8566, 2.3522)
+LONDON = (51.5074, -0.1278)
+BERLIN = (52.52, 13.405)
+
+d = haversine_km(PARIS, LONDON)
+assert 340 < d < 345, d
+assert haversine_km(PARIS, PARIS) < 1e-9
+assert bbox([PARIS, LONDON, BERLIN]) == (48.8566, -0.1278, 52.52, 13.405), bbox([PARIS, LONDON, BERLIN])
+assert nearest(PARIS, {"london": LONDON, "berlin": BERLIN}) == "london"
+print("geo OK")
diff --git a/graff-evals/fixtures/sa-fix-three/ledger/__init__.py b/graff-evals/fixtures/sa-fix-three/ledger/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/graff-evals/fixtures/sa-fix-three/ledger/money.py b/graff-evals/fixtures/sa-fix-three/ledger/money.py
new file mode 100644
index 00000000..410919fe
--- /dev/null
+++ b/graff-evals/fixtures/sa-fix-three/ledger/money.py
@@ -0,0 +1,20 @@
+"""Money parsing and arithmetic in integer cents."""
+
+
+def parse_amount(text):
+ """'$1,234.50' or '-12.5' -> integer cents."""
+ text = text.strip().replace("$", "")
+ return int(float(text) * 100)
+
+
+def format_cents(cents):
+ """Integer cents -> '$1,234.50'; negative amounts as '-$12.50'."""
+ sign = "-" if cents < 0 else ""
+ cents = abs(cents)
+ return f"{sign}${cents // 100}.{cents % 100:02d}"
+
+
+def split_evenly(cents, ways):
+ """Split cents into `ways` integer parts that sum to cents, larger parts first."""
+ share = cents // ways
+ return [share] * ways
diff --git a/graff-evals/fixtures/sa-fix-three/ledger/test_ledger.py b/graff-evals/fixtures/sa-fix-three/ledger/test_ledger.py
new file mode 100644
index 00000000..33acb73d
--- /dev/null
+++ b/graff-evals/fixtures/sa-fix-three/ledger/test_ledger.py
@@ -0,0 +1,13 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from ledger.money import format_cents, parse_amount, split_evenly # noqa: E402
+
+assert parse_amount("$1,234.50") == 123450, parse_amount("$1,234.50")
+assert parse_amount("0.29") == 29, parse_amount("0.29")
+assert parse_amount("-12.5") == -1250, parse_amount("-12.5")
+assert format_cents(123450) == "$1,234.50", format_cents(123450)
+assert format_cents(-1250) == "-$12.50", format_cents(-1250)
+assert split_evenly(100, 3) == [34, 33, 33], split_evenly(100, 3)
+print("ledger OK")
diff --git a/graff-evals/fixtures/sa-fix-three/textkit/__init__.py b/graff-evals/fixtures/sa-fix-three/textkit/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/graff-evals/fixtures/sa-fix-three/textkit/slug.py b/graff-evals/fixtures/sa-fix-three/textkit/slug.py
new file mode 100644
index 00000000..fdeed763
--- /dev/null
+++ b/graff-evals/fixtures/sa-fix-three/textkit/slug.py
@@ -0,0 +1,28 @@
+"""Text helpers."""
+import re
+
+
+def slugify(text):
+ """Lowercase ASCII slug: words joined by single dashes, no leading or trailing dash."""
+ text = text.lower()
+ text = re.sub(r"[^a-z0-9]", "-", text)
+ return text
+
+
+def truncate(text, width, ellipsis="..."):
+ """Cut text to at most width characters, ending with ellipsis when it was cut."""
+ if len(text) <= width:
+ return text
+ return text[:width] + ellipsis
+
+
+def title_case(text, minor=("a", "an", "and", "of", "the", "to")):
+ """Capitalize words except minor ones; the first word is always capitalized."""
+ words = text.split()
+ out = []
+ for w in words:
+ if w.lower() in minor:
+ out.append(w.lower())
+ else:
+ out.append(w.capitalize())
+ return " ".join(out)
diff --git a/graff-evals/fixtures/sa-fix-three/textkit/test_textkit.py b/graff-evals/fixtures/sa-fix-three/textkit/test_textkit.py
new file mode 100644
index 00000000..a7c85234
--- /dev/null
+++ b/graff-evals/fixtures/sa-fix-three/textkit/test_textkit.py
@@ -0,0 +1,12 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from textkit.slug import slugify, title_case, truncate # noqa: E402
+
+assert slugify("Hello, World!") == "hello-world", slugify("Hello, World!")
+assert slugify(" Many spaces -- here ") == "many-spaces-here", slugify(" Many spaces -- here ")
+assert truncate("abcdefghij", 6) == "abc...", truncate("abcdefghij", 6)
+assert truncate("short", 10) == "short"
+assert title_case("the lord of the rings") == "The Lord of the Rings", title_case("the lord of the rings")
+print("textkit OK")
diff --git a/graff-evals/fixtures/sa-rename-api/cli/__init__.py b/graff-evals/fixtures/sa-rename-api/cli/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/graff-evals/fixtures/sa-rename-api/cli/search.py b/graff-evals/fixtures/sa-rename-api/cli/search.py
new file mode 100644
index 00000000..f69bc29b
--- /dev/null
+++ b/graff-evals/fixtures/sa-rename-api/cli/search.py
@@ -0,0 +1,10 @@
+from core.users import all_ids, fetch_user
+
+
+def by_domain(domain):
+ hits = []
+ for uid in all_ids():
+ u = fetch_user(uid)
+ if u["email"].endswith("@" + domain):
+ hits.append(u["name"])
+ return hits
diff --git a/graff-evals/fixtures/sa-rename-api/cli/show.py b/graff-evals/fixtures/sa-rename-api/cli/show.py
new file mode 100644
index 00000000..3a519be2
--- /dev/null
+++ b/graff-evals/fixtures/sa-rename-api/cli/show.py
@@ -0,0 +1,14 @@
+import sys
+
+from core.users import fetch_user as lookup
+
+
+def main(argv):
+ u = lookup(int(argv[1]))
+ if not u:
+ return "no such user"
+ return "{name} ({email})".format(**u)
+
+
+if __name__ == "__main__":
+ print(main(sys.argv))
diff --git a/graff-evals/fixtures/sa-rename-api/core/__init__.py b/graff-evals/fixtures/sa-rename-api/core/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/graff-evals/fixtures/sa-rename-api/core/users.py b/graff-evals/fixtures/sa-rename-api/core/users.py
new file mode 100644
index 00000000..84836a17
--- /dev/null
+++ b/graff-evals/fixtures/sa-rename-api/core/users.py
@@ -0,0 +1,26 @@
+"""User store."""
+from dataclasses import dataclass
+
+_USERS = {
+ 1: {"name": "Ada", "email": "[email protected]", "active": True},
+ 2: {"name": "Grace", "email": "[email protected]", "active": True},
+ 3: {"name": "Linus", "email": "[email protected]", "active": False},
+}
+
+
+@dataclass(frozen=True)
+class User:
+ uid: int
+ name: str
+ email: str
+ active: bool
+
+
+def fetch_user(uid):
+ """Return the user record as a dict (with its uid), or None."""
+ row = _USERS.get(uid)
+ return dict(row, uid=uid) if row else None
+
+
+def all_ids():
+ return sorted(_USERS)
diff --git a/graff-evals/fixtures/sa-rename-api/jobs/__init__.py b/graff-evals/fixtures/sa-rename-api/jobs/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/graff-evals/fixtures/sa-rename-api/jobs/cleanup.py b/graff-evals/fixtures/sa-rename-api/jobs/cleanup.py
new file mode 100644
index 00000000..6c58b7e6
--- /dev/null
+++ b/graff-evals/fixtures/sa-rename-api/jobs/cleanup.py
@@ -0,0 +1,5 @@
+from core.users import all_ids, fetch_user
+
+
+def inactive_names():
+ return [fetch_user(uid)["name"] for uid in all_ids() if not fetch_user(uid)["active"]]
diff --git a/graff-evals/fixtures/sa-rename-api/jobs/digest.py b/graff-evals/fixtures/sa-rename-api/jobs/digest.py
new file mode 100644
index 00000000..e7d28af9
--- /dev/null
+++ b/graff-evals/fixtures/sa-rename-api/jobs/digest.py
@@ -0,0 +1,11 @@
+from core.users import all_ids, fetch_user
+
+
+def recipients():
+ """Emails of active users, in id order."""
+ out = []
+ for uid in all_ids():
+ u = fetch_user(uid)
+ if u and u["active"]:
+ out.append(u["email"])
+ return out
diff --git a/graff-evals/fixtures/sa-rename-api/jobs/export.py b/graff-evals/fixtures/sa-rename-api/jobs/export.py
new file mode 100644
index 00000000..db00014a
--- /dev/null
+++ b/graff-evals/fixtures/sa-rename-api/jobs/export.py
@@ -0,0 +1,8 @@
+import json
+
+from core.users import all_ids, fetch_user
+
+
+def export_json():
+ rows = [fetch_user(uid) for uid in all_ids()]
+ return json.dumps([{"id": r["uid"], "name": r["name"]} for r in rows if r["active"]])
diff --git a/graff-evals/fixtures/sa-rename-api/test_all.py b/graff-evals/fixtures/sa-rename-api/test_all.py
new file mode 100644
index 00000000..6ea19aa3
--- /dev/null
+++ b/graff-evals/fixtures/sa-rename-api/test_all.py
@@ -0,0 +1,26 @@
+import json
+
+from cli.search import by_domain
+from cli.show import main as show
+from jobs.cleanup import inactive_names
+from jobs.digest import recipients
+from jobs.export import export_json
+from web.admin import roster
+from web.feed import greeting
+from web.profile import profile_card
+
+assert profile_card(1) == "Ada <[email protected]>", profile_card(1)
+assert profile_card(3) == "unknown user", profile_card(3)
+assert profile_card(9) == "unknown user", profile_card(9)
+assert roster() == ["1: Ada", "2: Grace", "3: Linus (inactive)"], roster()
+assert greeting(2) == "Welcome back, Grace!", greeting(2)
+assert greeting(3) == "Welcome!", greeting(3)
+assert greeting(9) == "Welcome!", greeting(9)
+assert recipients() == ["[email protected]", "[email protected]"], recipients()
+assert inactive_names() == ["Linus"], inactive_names()
+assert json.loads(export_json()) == [{"id": 1, "name": "Ada"}, {"id": 2, "name": "Grace"}], export_json()
+assert show(["show", "3"]) == "Linus ([email protected])", show(["show", "3"])
+assert show(["show", "7"]) == "no such user", show(["show", "7"])
+assert by_domain("example.com") == ["Ada", "Grace"], by_domain("example.com")
+assert by_domain("example.org") == ["Linus"], by_domain("example.org")
+print("all OK")
diff --git a/graff-evals/fixtures/sa-rename-api/web/__init__.py b/graff-evals/fixtures/sa-rename-api/web/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/graff-evals/fixtures/sa-rename-api/web/admin.py b/graff-evals/fixtures/sa-rename-api/web/admin.py
new file mode 100644
index 00000000..2196a5c9
--- /dev/null
+++ b/graff-evals/fixtures/sa-rename-api/web/admin.py
@@ -0,0 +1,11 @@
+from core.users import all_ids, fetch_user
+
+
+def roster():
+ """Every user, inactive ones marked."""
+ lines = []
+ for uid in all_ids():
+ u = fetch_user(uid)
+ mark = "" if u["active"] else " (inactive)"
+ lines.append(f"{u['uid']}: {u['name']}{mark}")
+ return lines
diff --git a/graff-evals/fixtures/sa-rename-api/web/feed.py b/graff-evals/fixtures/sa-rename-api/web/feed.py
new file mode 100644
index 00000000..a9fda4f7
--- /dev/null
+++ b/graff-evals/fixtures/sa-rename-api/web/feed.py
@@ -0,0 +1,7 @@
+from core import users
+
+
+def greeting(uid):
+ user = users.fetch_user(uid)
+ name = user.get("name") if user else None
+ return f"Welcome back, {name}!" if name and user.get("active") else "Welcome!"
diff --git a/graff-evals/fixtures/sa-rename-api/web/profile.py b/graff-evals/fixtures/sa-rename-api/web/profile.py
new file mode 100644
index 00000000..aefe41cb
--- /dev/null
+++ b/graff-evals/fixtures/sa-rename-api/web/profile.py
@@ -0,0 +1,8 @@
+from core.users import fetch_user
+
+
+def profile_card(uid):
+ u = fetch_user(uid)
+ if u is None or not u["active"]:
+ return "unknown user"
+ return f"{u['name']} <{u['email']}>"
diff --git a/graff-evals/fixtures/sa-sidecar/confparse/__init__.py b/graff-evals/fixtures/sa-sidecar/confparse/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/graff-evals/fixtures/sa-sidecar/confparse/parser.py b/graff-evals/fixtures/sa-sidecar/confparse/parser.py
new file mode 100644
index 00000000..c543f9a8
--- /dev/null
+++ b/graff-evals/fixtures/sa-sidecar/confparse/parser.py
@@ -0,0 +1,64 @@
+"""Parser for the .conf format the deploy tool reads."""
+import os
+
+
+class ConfError(ValueError):
+ pass
+
+
+def parse(text, base_dir=".", _seen=None):
+ """Parse config text into a flat dict of "section.key" -> value."""
+ seen = set() if _seen is None else _seen
+ out = {}
+ aliases = {}
+ section = ""
+ for lineno, raw in enumerate(text.splitlines(), 1):
+ line = raw.split("#", 1)[0].strip()
+ if not line:
+ continue
+ if line.startswith("[") and line.endswith("]"):
+ section = line[1:-1].strip()
+ continue
+ word, _, rest = line.partition(" ")
+ rest = rest.strip()
+ if word == "include":
+ path = os.path.normpath(os.path.join(base_dir, rest))
+ if path in seen:
+ raise ConfError(f"line {lineno}: include cycle at {rest}")
+ seen.add(path)
+ with open(path) as f:
+ out.update(parse(f.read(), os.path.dirname(path), seen))
+ elif word == "set":
+ key, eq, value = rest.partition("=")
+ if not eq:
+ raise ConfError(f"line {lineno}: set needs key = value")
+ out[_qualify(section, key.strip())] = _coerce(value.strip())
+ elif word == "unset":
+ out.pop(rest, None)
+ elif word == "alias":
+ name, eq, target = rest.partition("=")
+ if not eq:
+ raise ConfError(f"line {lineno}: alias needs name = key")
+ aliases[name.strip()] = _qualify(section, target.strip())
+ else:
+ raise ConfError(f"line {lineno}: unknown directive {word!r}")
+ for name, target in aliases.items():
+ if target in out:
+ out[name] = out[target]
+ return out
+
+
+def _qualify(section, key):
+ return f"{section}.{key}" if section and "." not in key else key
+
+
+def _coerce(value):
+ if value in ("true", "false"):
+ return value == "true"
+ try:
+ return int(value)
+ except ValueError:
+ pass
+ if len(value) >= 2 and value[0] == value[-1] == '"':
+ return value[1:-1]
+ return value
diff --git a/graff-evals/fixtures/sa-sidecar/test_parser.py b/graff-evals/fixtures/sa-sidecar/test_parser.py
new file mode 100644
index 00000000..9cc40003
--- /dev/null
+++ b/graff-evals/fixtures/sa-sidecar/test_parser.py
@@ -0,0 +1,37 @@
+import os
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from confparse.parser import ConfError, parse # noqa: E402
+
+conf = """
+set name = "edge"
+[server]
+set port = 8080
+set debug = true
+set color = "#ff8800" # brand color
+unset debug
+alias listen = port
+"""
+got = parse(conf)
+assert got == {"name": "edge", "server.port": 8080, "server.color": "#ff8800", "listen": 8080}, got
+
+d = tempfile.mkdtemp()
+for name, body in {
+ "common.conf": "set shared = 1\n",
+ "b.conf": "include common.conf\nset b = 2\n",
+ "c.conf": "include common.conf\nset c = 3\n",
+ "loop1.conf": "include loop2.conf\n",
+ "loop2.conf": "include loop1.conf\n",
+}.items():
+ with open(os.path.join(d, name), "w") as f:
+ f.write(body)
+got = parse("include b.conf\ninclude c.conf\n", d)
+assert got == {"shared": 1, "b": 2, "c": 3}, got
+try:
+ parse("include loop1.conf\n", d)
+ raise SystemExit("expected ConfError for an include cycle")
+except ConfError:
+ pass
+print("parser OK")
diff --git a/graff-evals/hidden/sa-census-gen.py b/graff-evals/hidden/sa-census-gen.py
new file mode 100644
index 00000000..af35c0d5
--- /dev/null
+++ b/graff-evals/hidden/sa-census-gen.py
@@ -0,0 +1,60 @@
+"""Setup for sa-census: writes logs/<service>/ (plain + gzipped) into the cwd.
+
+Deterministic: fixed seed, fixed error-code counts per service, gzip mtime 0.
+WARN/INFO lines that merely mention an error code are noise; the level is the
+third whitespace-separated field.
+"""
+import gzip
+import os
+import random
+
+ERRORS = {
+ "auth": {"E401": 31, "E403": 18, "E500": 9, "E503": 7},
+ "billing": {"E402": 22, "E409": 22, "E500": 10},
+ "search": {"E504": 27, "E500": 26, "E429": 12},
+ "mailer": {"E550": 14, "E421": 19, "E451": 19},
+}
+NOISE = {"auth": "E503", "billing": "E500", "search": "E500", "mailer": "E550"}
+MESSAGES = {
+ "INFO": ["request ok id={id} latency_ms={ms}", "cache hit key=k{id}", "session refreshed id={id}"],
+ "DEBUG": ["pool size={ms} idle={id}", "tick seq={id}"],
+ "WARN": ["slow request id={id} latency_ms={ms}", "retrying id={id} attempt=2"],
+}
+
+
+def lines_for(service, rng):
+ out = []
+ for code, n in ERRORS[service].items():
+ out += [("ERROR", f"{code} request failed id={{id}}")] * n
+ out += [("WARN", f"recovered after ERROR {NOISE[service]} id={{id}}")] * 15
+ out += [("INFO", f"retry budget reset after ERROR {NOISE[service]} id={{id}}")] * 6
+ filler = 900 - len(out)
+ for _ in range(filler):
+ level = rng.choices(["INFO", "DEBUG", "WARN"], weights=[80, 8, 12])[0]
+ out.append((level, rng.choice(MESSAGES[level])))
+ rng.shuffle(out)
+ rendered = []
+ for i, (level, msg) in enumerate(out):
+ ts = f"2026-09-14T{8 + i // 3600:02d}:{(i // 60) % 60:02d}:{i % 60:02d}Z"
+ rendered.append(f"{ts} {service} {level} " + msg.format(id=1000 + i, ms=rng.randint(2, 900)))
+ return rendered
+
+
+def main():
+ rng = random.Random(1445)
+ for service in ERRORS:
+ d = os.path.join("logs", service)
+ os.makedirs(d, exist_ok=True)
+ lines = lines_for(service, rng)
+ cut1, cut2 = 280 + rng.randint(0, 60), 600 + rng.randint(0, 60)
+ parts = [lines[:cut1], lines[cut1:cut2], lines[cut2:]]
+ with open(os.path.join(d, f"{service}.log"), "w") as f:
+ f.write("\n".join(parts[2]) + "\n")
+ with open(os.path.join(d, f"{service}.1.log"), "w") as f:
+ f.write("\n".join(parts[1]) + "\n")
+ with gzip.GzipFile(os.path.join(d, f"{service}.2.log.gz"), "wb", mtime=0) as f:
+ f.write(("\n".join(parts[0]) + "\n").encode())
+
+
+if __name__ == "__main__":
+ main()
diff --git a/graff-evals/hidden/sa-census.py b/graff-evals/hidden/sa-census.py
new file mode 100644
index 00000000..2d78f081
--- /dev/null
+++ b/graff-evals/hidden/sa-census.py
@@ -0,0 +1,39 @@
+"""Held-out check for sa-census: recompute the census from logs/ and compare."""
+import collections
+import gzip
+import json
+import os
+
+
+def census():
+ services = {}
+ for service in sorted(os.listdir("logs")):
+ d = os.path.join("logs", service)
+ if not os.path.isdir(d) or service.startswith("."):
+ continue
+ lines = 0
+ codes = collections.Counter()
+ for name in sorted(os.listdir(d)):
+ path = os.path.join(d, name)
+ if name.endswith(".gz"):
+ text = gzip.open(path, "rt").read()
+ elif name.endswith(".log"):
+ text = open(path).read()
+ else:
+ continue
+ for line in text.splitlines():
+ if not line.strip():
+ continue
+ lines += 1
+ fields = line.split()
+ if len(fields) > 3 and fields[2] == "ERROR":
+ codes[fields[3]] += 1
+ top = min(codes, key=lambda c: (-codes[c], c))
+ services[service] = {"lines": lines, "errors": sum(codes.values()), "top_error": top}
+ return {"services": services, "total_errors": sum(s["errors"] for s in services.values())}
+
+
+want = census()
+got = json.load(open("summary.json"))
+assert got == want, f"summary.json differs:\n got {json.dumps(got, sort_keys=True)}\n want {json.dumps(want, sort_keys=True)}"
+print("sa-census hidden OK")
diff --git a/graff-evals/hidden/sa-coverage.py b/graff-evals/hidden/sa-coverage.py
new file mode 100644
index 00000000..1c9c3a21
--- /dev/null
+++ b/graff-evals/hidden/sa-coverage.py
@@ -0,0 +1,24 @@
+"""Held-out check for sa-coverage: exact endpoint coverage lists."""
+import json
+
+want = {
+ "undocumented": [
+ "GET /admin/stats",
+ "GET /invoices/<id>",
+ "GET /ready",
+ "GET /search/suggest",
+ "POST /admin/reindex",
+ "POST /orders/<id>/refund",
+ ],
+ "stale_docs": [
+ "DELETE /users/<id>",
+ "GET /search/legacy",
+ "POST /admin/flush",
+ "POST /billing/coupon",
+ ],
+}
+got = json.load(open("coverage.json"))
+assert isinstance(got, dict), got
+for key, value in want.items():
+ assert got.get(key) == value, f"{key}: got {got.get(key)!r}, want {value!r}"
+print("sa-coverage hidden OK")
diff --git a/graff-evals/hidden/sa-fix-three.py b/graff-evals/hidden/sa-fix-three.py
new file mode 100644
index 00000000..eaa87d8b
--- /dev/null
+++ b/graff-evals/hidden/sa-fix-three.py
@@ -0,0 +1,37 @@
+"""Held-out checks for sa-fix-three: each package's contract beyond its visible test."""
+import os
+import sys
+
+sys.path.insert(0, os.getcwd())
+from geo.distance import bbox, haversine_km, nearest # noqa: E402
+from ledger.money import format_cents, parse_amount, split_evenly # noqa: E402
+from textkit.slug import slugify, title_case, truncate # noqa: E402
+
+NYC = (40.7128, -74.006)
+LA = (34.0522, -118.2437)
+SYDNEY = (-33.8688, 151.2093)
+d = haversine_km(NYC, LA)
+assert 3930 < d < 3940, d
+assert abs(haversine_km(NYC, LA) - haversine_km(LA, NYC)) < 1e-6
+assert bbox([NYC, LA, SYDNEY]) == (-33.8688, -118.2437, 40.7128, 151.2093), bbox([NYC, LA, SYDNEY])
+assert bbox([NYC]) == (40.7128, -74.006, 40.7128, -74.006)
+assert nearest(NYC, {"sydney": SYDNEY, "la": LA, "nyc2": (40.73, -73.99)}) == "nyc2"
+
+assert slugify("--Already--Slugged--") == "already-slugged", slugify("--Already--Slugged--")
+assert slugify("C3PO & R2D2") == "c3po-r2d2", slugify("C3PO & R2D2")
+assert truncate("abcdefghij", 10) == "abcdefghij"
+assert truncate("abcdefghij", 9) == "abcdef...", truncate("abcdefghij", 9)
+assert len(truncate("x" * 50, 20)) == 20
+assert title_case("a tale of two cities") == "A Tale of Two Cities", title_case("a tale of two cities")
+assert title_case("war and peace") == "War and Peace"
+
+assert parse_amount("1,000,000") == 100000000, parse_amount("1,000,000")
+assert parse_amount("4.35") == 435, parse_amount("4.35")
+assert parse_amount(" $0.07 ") == 7, parse_amount(" $0.07 ")
+assert format_cents(5) == "$0.05", format_cents(5)
+assert format_cents(100000000) == "$1,000,000.00", format_cents(100000000)
+assert format_cents(-123456789) == "-$1,234,567.89", format_cents(-123456789)
+assert split_evenly(10, 4) == [3, 3, 2, 2], split_evenly(10, 4)
+assert split_evenly(7, 7) == [1] * 7
+assert sum(split_evenly(1001, 6)) == 1001
+print("sa-fix-three hidden OK")
diff --git a/graff-evals/hidden/sa-rename-api.py b/graff-evals/hidden/sa-rename-api.py
new file mode 100644
index 00000000..0fc9c8c3
--- /dev/null
+++ b/graff-evals/hidden/sa-rename-api.py
@@ -0,0 +1,29 @@
+"""Held-out check for sa-rename-api: the new API exists, the old one is gone everywhere."""
+import inspect
+import os
+import re
+import sys
+
+sys.path.insert(0, os.getcwd())
+import core.users as users # noqa: E402
+
+assert not hasattr(users, "fetch_user"), "fetch_user still defined in core/users.py"
+get_user, User = users.get_user, users.User
+assert get_user(1) == User(1, "Ada", "[email protected]", True), get_user(1)
+assert get_user(3) is None, get_user(3)
+assert get_user(9) is None and get_user(9, include_inactive=True) is None
+assert get_user(3, include_inactive=True) == User(3, "Linus", "[email protected]", False)
+param = inspect.signature(get_user).parameters.get("include_inactive")
+assert param is not None and param.kind is inspect.Parameter.KEYWORD_ONLY and param.default is False, param
+
+stale = []
+for pkg in ("core", "web", "jobs", "cli"):
+ for root, dirs, files in os.walk(pkg):
+ dirs[:] = [d for d in dirs if not d.startswith((".", "__"))]
+ for name in files:
+ if name.endswith(".py"):
+ path = os.path.join(root, name)
+ if re.search(r"\bfetch_user\b", open(path).read()):
+ stale.append(path)
+assert not stale, f"fetch_user still referenced in {stale}"
+print("sa-rename-api hidden OK")
diff --git a/graff-evals/hidden/sa-sidecar.py b/graff-evals/hidden/sa-sidecar.py
new file mode 100644
index 00000000..4de298f6
--- /dev/null
+++ b/graff-evals/hidden/sa-sidecar.py
@@ -0,0 +1,45 @@
+"""Held-out check for sa-sidecar: parser contract plus the sub-agent's docs/FORMAT.md."""
+import os
+import re
+import sys
+import tempfile
+
+sys.path.insert(0, os.getcwd())
+from confparse.parser import ConfError, parse # noqa: E402
+
+assert parse("set a = 1\nunset a\n") == {}, parse("set a = 1\nunset a\n")
+assert parse("[db]\nset host = x\nset port = 5\nunset db.port\n") == {"db.host": "x"}
+assert parse("[db]\nset host = x\nunset host\n[web]\nset host = y\n") == {"web.host": "y"}
+assert parse('set tag = "a#b" # trailing\n') == {"tag": "a#b"}, parse('set tag = "a#b" # trailing\n')
+assert parse("set n = 3 # three\n# set m = 4\n") == {"n": 3}
+d = tempfile.mkdtemp()
+for name, body in {
+ "base.conf": "set level = 1\n",
+ "x.conf": "include base.conf\nset x = 1\n",
+ "y.conf": "include x.conf\ninclude base.conf\n",
+ "self.conf": "include self.conf\n",
+}.items():
+ with open(os.path.join(d, name), "w") as f:
+ f.write(body)
+assert parse("include y.conf\ninclude x.conf\n", d) == {"level": 1, "x": 1}
+try:
+ parse("include self.conf\n", d)
+ raise SystemExit("expected ConfError for a self-include")
+except ConfError:
+ pass
+try:
+ parse("frobnicate now\n")
+ raise SystemExit("expected ConfError for an unknown directive")
+except ConfError:
+ pass
+
+doc_path = os.path.join("docs", "FORMAT.md")
+assert os.path.isfile(doc_path), "docs/FORMAT.md is missing"
+doc = open(doc_path).read()
+lines = [x for x in doc.splitlines() if x.strip()]
+assert len(lines) >= 12, f"docs/FORMAT.md is too thin ({len(lines)} non-empty lines)"
+for word in ("include", "set", "unset", "alias"):
+ assert re.search(rf"\b{word}\b", doc), f"docs/FORMAT.md never mentions the {word} directive"
+assert "[" in doc and re.search(r"section", doc, re.I), "docs/FORMAT.md does not describe [section] headers"
+assert "#" in doc, "docs/FORMAT.md does not describe comments"
+print("sa-sidecar hidden OK")
diff --git a/graff-evals/subagent_split.py b/graff-evals/subagent_split.py
new file mode 100644
index 00000000..c45f66ab
--- /dev/null
+++ b/graff-evals/subagent_split.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Split sub-agent runs into parent and child time.
+
+usage: subagent_split.py <output-root> [<codex-trace-dir>]
+
+<output-root> is a run.py --output-root (results.jsonl + sandboxes/). Codex
+runs need the driver's EVAL_TRACE_DIR; it defaults to <output-root>-codex-traces.
+
+Columns, in seconds from the start of the run:
+ first_spawn the parent started its first child
+ spawn_spread first to last spawn (one spawn per model call costs this)
+ child_crit first child start to last child finish
+ slowest the longest single child
+ par child time summed over child_crit (1.0 = no overlap)
+ parent_model the parent's own model time (graff traces only)
+ parent_wait the parent blocked on agent_output (graff traces only)
+ tail last child finish to the end of the run: integration and checks
+
+A graff child is read from its `subagent` trace line (ms and finish time);
+older traces fall back to the child's model calls.
+"""
+import collections
+import glob
+import json
+import os
+import re
+import sys
+
+COLS = ["wall", "ok", "root_calls", "children", "child_calls", "first_spawn", "spawn_spread",
+ "child_crit", "slowest", "par", "parent_model", "parent_wait", "tail"]
+
+
+def graff_run(sandbox):
+ traces = sorted(glob.glob(os.path.join(sandbox, ".graff", "traces", "*.jsonl")))
+ if not traces:
+ return None
+ ev = [json.loads(line) for line in open(traces[-1]) if line.strip()]
+ root_api = [(e["t"] - e.get("ms", 0), e["t"]) for e in ev if e.get("ev") == "api" and e.get("agent") == "repl"]
+ spawns = [e["t"] for e in ev if e.get("ev") == "tool" and e.get("name") == "subagent" and not e.get("from_sub")]
+ waits = [e.get("ms", 0) for e in ev if e.get("ev") == "tool" and e.get("name") == "agent_output" and not e.get("from_sub")]
+ calls = collections.Counter(e.get("agent") for e in ev if e.get("ev") == "api" and e.get("agent") not in (None, "repl"))
+ spans = {}
+ for e in ev:
+ if e.get("ev") == "subagent":
+ m = re.search(r"label=(.*?) ms=(\d+)", e.get("detail", ""))
+ if m:
+ spans[m.group(1)] = (e["t"] - int(m.group(2)), e["t"], calls.get(m.group(1), 0))
+ if not spans:
+ kids = collections.defaultdict(list)
+ for e in ev:
+ if e.get("ev") == "api" and e.get("agent") not in (None, "repl", "?"):
+ kids[e["agent"]].append((e["t"] - e.get("ms", 0), e["t"]))
+ spans = {k: (min(a for a, _ in v), max(b for _, b in v), len(v)) for k, v in kids.items()}
+ end = max([b for _, b in root_api] + [s[1] for s in spans.values()] + [0])
+ return summary(end, len(root_api), sum(b - a for a, b in root_api), sum(waits), spawns, spans)
+
+
+def codex_run(path):
+ ev = [json.loads(line) for line in open(path) if line.strip()]
+ root_calls = sum(1 for e in ev if e.get("ev") == "usage" and e.get("thread") == "root")
+ spawns = [e["t"] for e in ev if e.get("type") == "subAgentActivity" and e.get("thread") == "root"]
+ kids = collections.defaultdict(list)
+ for e in ev:
+ if e.get("thread") == "child":
+ kids[e.get("tid") or "child"].append(e)
+ spans = {}
+ for tid, items in kids.items():
+ first = min(e["t"] for e in items)
+ start = max([s for s in spawns if s <= first] or [first])
+ spans[tid] = (start, max(e["t"] for e in items), sum(1 for e in items if e.get("ev") == "usage"))
+ end = max([e["t"] for e in ev] + [0])
+ return summary(end, root_calls, None, None, spawns, spans)
+
+
+def summary(end, root_calls, parent_model, parent_wait, spawns, spans):
+ out = {"root_calls": root_calls, "children": len(spans)}
+ if parent_model is not None:
+ out["parent_model"] = parent_model / 1000
+ out["parent_wait"] = parent_wait / 1000
+ if spawns:
+ out["first_spawn"] = min(spawns) / 1000
+ out["spawn_spread"] = (max(spawns) - min(spawns)) / 1000
+ if spans:
+ lo = min(s[0] for s in spans.values())
+ hi = max(s[1] for s in spans.values())
+ crit = (hi - lo) / 1000
+ out.update(child_crit=crit, slowest=max(s[1] - s[0] for s in spans.values()) / 1000,
+ child_calls=sum(s[2] for s in spans.values()), tail=(end - hi) / 1000,
+ par=(sum(s[1] - s[0] for s in spans.values()) / 1000 / crit) if crit > 0 else 0.0)
+ return out
+
+
+def main():
+ root = sys.argv[1]
+ codex_dir = sys.argv[2] if len(sys.argv) > 2 else root.rstrip("/") + "-codex-traces"
+ rows = [json.loads(line) for line in open(os.path.join(root, "results.jsonl"))]
+ print(f"{'task':16} {'harness':17} " + " ".join(f"{c:>12}" for c in COLS))
+ for r in sorted(rows, key=lambda r: (r["task"], r["harness"], r.get("rep", 0))):
+ name = f"{r['harness']}-{r['task']}-r{r.get('rep', 1)}"
+ if r["harness"].startswith("codex"):
+ path = os.path.join(codex_dir, name + ".jsonl")
+ d = codex_run(path) if os.path.exists(path) else None
+ else:
+ d = graff_run(os.path.join(root, "sandboxes", name))
+ d = dict(d or {}, wall=r.get("wall_s"), ok=r.get("outcome_ok"))
+ cells = [f"{d[c]:>12.1f}" if isinstance(d.get(c), float) else f"{str(d.get(c, '-')):>12}" for c in COLS]
+ print(f"{r['task']:16} {r['harness']:17} " + " ".join(cells))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/graff-evals/tasks/62-sa-fix-three.json b/graff-evals/tasks/62-sa-fix-three.json
new file mode 100644
index 00000000..1b4da421
--- /dev/null
+++ b/graff-evals/tasks/62-sa-fix-three.json
@@ -0,0 +1,12 @@
+{
+ "id": "sa-fix-three",
+ "suite": "subagents",
+ "category": "subagents",
+ "files_dir": "fixtures/sa-fix-three",
+ "prompt": "geo/, textkit/ and ledger/ are independent packages, and each one's test script fails: `python3 geo/test_geo.py`, `python3 textkit/test_textkit.py`, `python3 ledger/test_ledger.py`. Hand each package to its own sub-agent and run the three in parallel; each fixes its package's source until its test passes, without editing tests. When they are done, run all three test scripts yourself and confirm they pass.",
+ "setup": [
+ "git init -q && git add -A && git -c user.name=eval -c [email protected] commit -qm init"
+ ],
+ "check": "python3 geo/test_geo.py && python3 textkit/test_textkit.py && python3 ledger/test_ledger.py && python3 \"$TASK_ROOT/hidden/sa-fix-three.py\"",
+ "timeout_s": 420
+}
diff --git a/graff-evals/tasks/63-sa-census.json b/graff-evals/tasks/63-sa-census.json
new file mode 100644
index 00000000..6fd098ab
--- /dev/null
+++ b/graff-evals/tasks/63-sa-census.json
@@ -0,0 +1,12 @@
+{
+ "id": "sa-census",
+ "suite": "subagents",
+ "category": "subagents",
+ "prompt": "logs/ has one directory per service (auth, billing, search, mailer), each holding plain and gzipped log files. Use one sub-agent per service, running in parallel, to count that service's log lines, its ERROR lines (the level is the third whitespace-separated field), and its most frequent error code among ERROR lines (the code is the fourth field; ties go to the smallest code). Then write summary.json as {\"services\": {\"<service>\": {\"lines\": N, \"errors\": N, \"top_error\": \"E###\"}, ...}, \"total_errors\": N}.",
+ "setup": [
+ "python3 \"$TASK_ROOT/hidden/sa-census-gen.py\"",
+ "git init -q && git add -A && git -c user.name=eval -c [email protected] commit -qm init"
+ ],
+ "check": "python3 \"$TASK_ROOT/hidden/sa-census.py\"",
+ "timeout_s": 420
+}
diff --git a/graff-evals/tasks/64-sa-rename-api.json b/graff-evals/tasks/64-sa-rename-api.json
new file mode 100644
index 00000000..d422926e
--- /dev/null
+++ b/graff-evals/tasks/64-sa-rename-api.json
@@ -0,0 +1,12 @@
+{
+ "id": "sa-rename-api",
+ "suite": "subagents",
+ "category": "subagents",
+ "files_dir": "fixtures/sa-rename-api",
+ "prompt": "Change the user API in core/users.py: replace fetch_user(uid), which returns a dict, with get_user(uid, *, include_inactive=False), which returns the User dataclass defined there, or None when the user is missing or (unless include_inactive=True) inactive. Remove fetch_user. Then update every caller so behavior is unchanged: give web/, jobs/ and cli/ to one sub-agent each, running in parallel while you change core. Finish with `python3 test_all.py` passing; do not edit the test.",
+ "setup": [
+ "git init -q && git add -A && git -c user.name=eval -c [email protected] commit -qm init"
+ ],
+ "check": "python3 test_all.py && python3 \"$TASK_ROOT/hidden/sa-rename-api.py\"",
+ "timeout_s": 420
+}
diff --git a/graff-evals/tasks/65-sa-coverage.json b/graff-evals/tasks/65-sa-coverage.json
new file mode 100644
index 00000000..5fe07f32
--- /dev/null
+++ b/graff-evals/tasks/65-sa-coverage.json
@@ -0,0 +1,12 @@
+{
+ "id": "sa-coverage",
+ "suite": "subagents",
+ "category": "subagents",
+ "files_dir": "fixtures/sa-coverage",
+ "prompt": "api/routes/ defines HTTP endpoints, with @route(METHOD, PATH) decorators or add_route(METHOD, PATH, handler) calls, and docs/api/ documents endpoints as `### METHOD PATH` headings. Use two sub-agents in parallel: one lists every endpoint the code defines, the other every endpoint the docs describe. Then write coverage.json as {\"undocumented\": [...], \"stale_docs\": [...]}: endpoints defined but not documented, and documented but not defined, each as a \"METHOD PATH\" string, each list sorted.",
+ "setup": [
+ "git init -q && git add -A && git -c user.name=eval -c [email protected] commit -qm init"
+ ],
+ "check": "python3 \"$TASK_ROOT/hidden/sa-coverage.py\"",
+ "timeout_s": 300
+}
diff --git a/graff-evals/tasks/66-sa-sidecar.json b/graff-evals/tasks/66-sa-sidecar.json
new file mode 100644
index 00000000..ae39185a
--- /dev/null
+++ b/graff-evals/tasks/66-sa-sidecar.json
@@ -0,0 +1,12 @@
+{
+ "id": "sa-sidecar",
+ "suite": "subagents",
+ "category": "subagents",
+ "files_dir": "fixtures/sa-sidecar",
+ "prompt": "`python3 test_parser.py` fails. Fix confparse/parser.py (not the test) until it passes. Meanwhile, have a sub-agent write docs/FORMAT.md: a reference for the config format parser.py accepts (sections, each directive's syntax with an example, comments, value types). Run that sub-agent in the background while you fix the parser; both must be done before you finish.",
+ "setup": [
+ "git init -q && git add -A && git -c user.name=eval -c [email protected] commit -qm init"
+ ],
+ "check": "python3 test_parser.py && python3 \"$TASK_ROOT/hidden/sa-sidecar.py\"",
+ "timeout_s": 420
+}
diff --git a/graff-evals/tasks/67-solo-fix-three.json b/graff-evals/tasks/67-solo-fix-three.json
new file mode 100644
index 00000000..8d8fdda7
--- /dev/null
+++ b/graff-evals/tasks/67-solo-fix-three.json
@@ -0,0 +1,13 @@
+{
+ "id": "solo-fix-three",
+ "suite": "subagents-solo",
+ "category": "subagents",
+ "files_dir": "fixtures/sa-fix-three",
+ "prompt": "geo/, textkit/ and ledger/ are independent packages, and each one's test script fails: `python3 geo/test_geo.py`, `python3 textkit/test_textkit.py`, `python3 ledger/test_ledger.py`. Fix each package's source until its test passes, without editing tests. Then run all three test scripts and confirm they pass.",
+ "setup": [
+ "git init -q && git add -A && git -c user.name=eval -c [email protected] commit -qm init"
+ ],
+ "check": "python3 geo/test_geo.py && python3 textkit/test_textkit.py && python3 ledger/test_ledger.py && python3 \"$TASK_ROOT/hidden/sa-fix-three.py\"",
+ "timeout_s": 420,
+ "counterfactual_of": "sa-fix-three"
+}
diff --git a/graff-evals/tasks/68-solo-census.json b/graff-evals/tasks/68-solo-census.json
new file mode 100644
index 00000000..cdc01dad
--- /dev/null
+++ b/graff-evals/tasks/68-solo-census.json
@@ -0,0 +1,13 @@
+{
+ "id": "solo-census",
+ "suite": "subagents-solo",
+ "category": "subagents",
+ "prompt": "logs/ has one directory per service (auth, billing, search, mailer), each holding plain and gzipped log files. For each service, count its log lines, its ERROR lines (the level is the third whitespace-separated field), and its most frequent error code among ERROR lines (the code is the fourth field; ties go to the smallest code). Then write summary.json as {\"services\": {\"<service>\": {\"lines\": N, \"errors\": N, \"top_error\": \"E###\"}, ...}, \"total_errors\": N}.",
+ "setup": [
+ "python3 \"$TASK_ROOT/hidden/sa-census-gen.py\"",
+ "git init -q && git add -A && git -c user.name=eval -c [email protected] commit -qm init"
+ ],
+ "check": "python3 \"$TASK_ROOT/hidden/sa-census.py\"",
+ "timeout_s": 420,
+ "counterfactual_of": "sa-census"
+}
diff --git a/graff-evals/tasks/69-solo-rename-api.json b/graff-evals/tasks/69-solo-rename-api.json
new file mode 100644
index 00000000..2ae092e8
--- /dev/null
+++ b/graff-evals/tasks/69-solo-rename-api.json
@@ -0,0 +1,13 @@
+{
+ "id": "solo-rename-api",
+ "suite": "subagents-solo",
+ "category": "subagents",
+ "files_dir": "fixtures/sa-rename-api",
+ "prompt": "Change the user API in core/users.py: replace fetch_user(uid), which returns a dict, with get_user(uid, *, include_inactive=False), which returns the User dataclass defined there, or None when the user is missing or (unless include_inactive=True) inactive. Remove fetch_user. Then update every caller in web/, jobs/ and cli/ so behavior is unchanged. Finish with `python3 test_all.py` passing; do not edit the test.",
+ "setup": [
+ "git init -q && git add -A && git -c user.name=eval -c [email protected] commit -qm init"
+ ],
+ "check": "python3 test_all.py && python3 \"$TASK_ROOT/hidden/sa-rename-api.py\"",
+ "timeout_s": 420,
+ "counterfactual_of": "sa-rename-api"
+}
diff --git a/graff-evals/tasks/70-solo-coverage.json b/graff-evals/tasks/70-solo-coverage.json
new file mode 100644
index 00000000..596cf437
--- /dev/null
+++ b/graff-evals/tasks/70-solo-coverage.json
@@ -0,0 +1,13 @@
+{
+ "id": "solo-coverage",
+ "suite": "subagents-solo",
+ "category": "subagents",
+ "files_dir": "fixtures/sa-coverage",
+ "prompt": "api/routes/ defines HTTP endpoints, with @route(METHOD, PATH) decorators or add_route(METHOD, PATH, handler) calls, and docs/api/ documents endpoints as `### METHOD PATH` headings. List every endpoint the code defines and every endpoint the docs describe. Then write coverage.json as {\"undocumented\": [...], \"stale_docs\": [...]}: endpoints defined but not documented, and documented but not defined, each as a \"METHOD PATH\" string, each list sorted.",
+ "setup": [
+ "git init -q && git add -A && git -c user.name=eval -c [email protected] commit -qm init"
+ ],
+ "check": "python3 \"$TASK_ROOT/hidden/sa-coverage.py\"",
+ "timeout_s": 300,
+ "counterfactual_of": "sa-coverage"
+}
diff --git a/graff-evals/tasks/71-solo-sidecar.json b/graff-evals/tasks/71-solo-sidecar.json
new file mode 100644
index 00000000..e36b1a4d
--- /dev/null
+++ b/graff-evals/tasks/71-solo-sidecar.json
@@ -0,0 +1,13 @@
+{
+ "id": "solo-sidecar",
+ "suite": "subagents-solo",
+ "category": "subagents",
+ "files_dir": "fixtures/sa-sidecar",
+ "prompt": "`python3 test_parser.py` fails. Fix confparse/parser.py (not the test) until it passes. Also write docs/FORMAT.md: a reference for the config format parser.py accepts (sections, each directive's syntax with an example, comments, value types).",
+ "setup": [
+ "git init -q && git add -A && git -c user.name=eval -c [email protected] commit -qm init"
+ ],
+ "check": "python3 test_parser.py && python3 \"$TASK_ROOT/hidden/sa-sidecar.py\"",
+ "timeout_s": 420,
+ "counterfactual_of": "sa-sidecar"
+}