graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
#!/usr/bin/env python3
"""Offline, private Graff/native-Cursor comparison reports; no provider calls."""
import argparse
from collections import Counter
import json
import math
from pathlib import Path
import statistics
from typing import Optional
ARMS = ("graff", "cursor")
FAILURE_FIELDS = (
"harness", "task", "rep", "artifact_ok", "outcome_ok", "exit", "timed_out",
"routing_ok", "native_completion_ok", "verifiers_unchanged", "check_note", "error",
)
def clean(row: dict) -> bool:
"""Match the receipt's complete clean-pass gate, not artifact success alone."""
return bool(
row.get("outcome_ok") and row.get("routing_ok")
and row.get("native_completion_ok") and row.get("exit") == 0
and not row.get("timed_out") and row.get("verifiers_unchanged")
)
def numeric(value) -> bool:
return (isinstance(value, (int, float)) and not isinstance(value, bool)
and math.isfinite(value))
def metric(values, median: bool = False) -> Optional[float]:
values = list(values)
if not values:
return None
return round((statistics.median if median else statistics.mean)(values), 3)
def aggregate(items: list, arm: str) -> dict:
wall = [r["wall_s"] for r in items if numeric(r.get("wall_s"))]
passed = [r["wall_s"] for r in items if clean(r) and numeric(r.get("wall_s"))]
# These are Cursor's own measurements, never a matched Graff metric.
native = [r["native_task_duration_s"] for r in items
if arm == "cursor" and numeric(r.get("native_task_duration_s"))]
overhead = [r["outside_native_duration_s"] for r in items
if arm == "cursor" and numeric(r.get("outside_native_duration_s"))]
return {
"runs": len(items), "clean_pass": sum(clean(r) for r in items),
"artifact_pass": sum(bool(r.get("artifact_ok")) for r in items),
"timeouts": sum(bool(r.get("timed_out")) for r in items),
"routing_verified": sum(bool(r.get("routing_ok")) for r in items),
"mean_wall_s": metric(wall), "median_wall_s": metric(wall, median=True),
"wall_reported_runs": len(wall),
"passing_mean_wall_s": metric(passed),
"passing_median_wall_s": metric(passed, median=True),
"passing_wall_reported_runs": len(passed),
"native_task_duration_mean_s": metric(native),
"native_task_duration_median_s": metric(native, median=True),
"outside_native_duration_mean_s": metric(overhead),
"outside_native_duration_median_s": metric(overhead, median=True),
"native_duration_reported_runs": len(native),
"outside_native_duration_reported_runs": len(overhead),
}
def paired_stats(items: list) -> dict:
pairs = {}
for row in items:
pairs.setdefault((row["task"], row["rep"]), {})[row["harness"]] = row
both_passed = [p for p in pairs.values() if all(clean(p[a]) for a in ARMS)]
timed = [p for p in both_passed if all(numeric(p[a].get("wall_s")) for a in ARMS)]
means = {a: metric(p[a]["wall_s"] for p in timed) for a in ARMS}
medians = {a: metric((p[a]["wall_s"] for p in timed), median=True) for a in ARMS}
reduction = None
if timed:
graff_mean = statistics.mean(p["graff"]["wall_s"] for p in timed)
cursor_mean = statistics.mean(p["cursor"]["wall_s"] for p in timed)
if cursor_mean > 0:
reduction = round((1 - graff_mean / cursor_mean) * 100, 1)
return {
"pairs": len(pairs), "both_clean_passed": len(both_passed),
"timed_both_clean_passed": len(timed),
"graff_faster_on_both_passing_pairs": sum(
p["graff"]["wall_s"] < p["cursor"]["wall_s"] for p in timed),
"mean_wall_s": means, "median_wall_s": medians,
"graff_wall_reduction_percent": reduction,
}
def validate(meta: dict, rows: list) -> None:
if not isinstance(meta, dict):
raise ValueError("metadata.json must contain an object")
for name, allowed in (
("preset", ("argon", "xai-sub")), ("provider", ("codegraff", "xai")),
("effort", ("low", "high")), ("mode", ("smoke", "subset", "full")),
):
if meta.get(name) not in allowed:
raise ValueError("metadata %s must be one of %s" % (name, ", ".join(allowed)))
for name in ("repetitions", "workers"):
if type(meta.get(name)) is not int or meta[name] < 1:
raise ValueError("metadata %s must be a positive integer" % name)
models = meta.get("models")
if not isinstance(models, dict) or any(
not isinstance(models.get(a), str) or not models[a] for a in ARMS
):
raise ValueError("metadata models must name both graff and cursor model ids")
tasks = meta.get("tasks")
if (not isinstance(tasks, list) or not tasks
or any(not isinstance(t, str) or not t for t in tasks)
or len(tasks) != len(set(tasks))):
raise ValueError("metadata tasks must be a nonempty list of unique task names")
expected = {(a, t, rep) for a in ARMS for t in tasks
for rep in range(1, meta["repetitions"] + 1)}
observed = Counter()
task_suites = {}
for index, row in enumerate(rows, 1):
if not isinstance(row, dict):
raise ValueError("result row %d must be an object" % index)
arm, task, rep = (row.get(k) for k in ("harness", "task", "rep"))
if not isinstance(arm, str) or not isinstance(task, str) or type(rep) is not int:
raise ValueError("result row %d needs harness/task strings and an integer rep" % index)
observed[(arm, task, rep)] += 1
suite = row.get("suite")
if not isinstance(suite, str) or not suite or suite == "all":
raise ValueError("result row %d needs a named suite (not 'all')" % index)
previous = task_suites.setdefault(task, suite)
if suite != previous:
raise ValueError("task %r has inconsistent suites: %r and %r" % (task, previous, suite))
for field in ("wall_s", "native_task_duration_s", "outside_native_duration_s"):
value = row.get(field)
if value is not None and (not numeric(value) or value < 0):
raise ValueError("result row %d has invalid %s" % (index, field))
if "mode" in row and row["mode"] != meta["mode"]:
raise ValueError("result row %d mode differs from metadata; do not mix smoke/full" % index)
missing = sorted(expected - set(observed))
extras = sorted(set(observed) - expected)
duplicates = sorted(k for k, count in observed.items() if count > 1)
if missing or extras or duplicates:
parts = []
for label, keys in (("missing", missing), ("unexpected", extras), ("duplicate", duplicates)):
if keys:
preview = ", ".join("%s/%s/rep=%s" % k for k in keys[:8])
parts.append("%s (%d): %s%s" % (
label, len(keys), preview, " ..." if len(keys) > 8 else ""))
raise ValueError("invalid task/repetition/arm matrix: " + "; ".join(parts))
def caveats(meta: dict) -> list:
return [
"Same named model, different provider backends: this is not a provider-controlled "
"causal ablation of the harness alone. See the recorded model ids and comparison kind.",
"Both arms request %s effort on their own scales; requested effort scales and "
"provider-side reasoning implementations differ." % meta["effort"],
"%d small deterministic fixtures, %d repetitions per arm, %d workers; mode: %s. "
"This is a small fixture sample, not evidence about large real-world repositories."
% (len(meta["tasks"]), meta["repetitions"], meta["workers"], meta["mode"]),
"Graff binary SHA-256 and Cursor version are recorded in metadata, not inferred "
"from current installations. No claim of a clean source tree is made.",
"Cursor uses native CLI agent -p, not desktop Harness or its SDK transport.",
"Synthetic sandboxes use isolated login snapshots, fixture-only MCP, and no "
"personal project context; credentials are not refreshed.",
"Smoke runs are excluded from full results. A smoke-mode report is only a smoke "
"report, not a full benchmark. Only the supplied root's result file is read.",
"All runs, including failures, are retained. All-run wall timings are separate "
"from passing-only matched task/repetition pairs; both-pass denominators are explicit.",
"first_out measures any output including startup events, not time to first model "
"token (TTFT). Wall time is task process lifetime.",
"Cursor native task duration and outside-native overhead are reported separately "
"with their own measurement counts. They are self-reported, not a comparable Graff "
"metric; outside-native wall time includes CLI startup/teardown.",
"Comparable dollar cost and model/API-call counts are unknown, never zero. "
"Native usage receipts, if present, do not establish comparable prices or call counts.",
"These are private local reports. The reporter never publishes them or makes "
"network or model calls.",
]
def cell(value) -> str:
return str(value).replace("|", "\\|").replace("\n", " ").replace("\r", " ")
def seconds(value) -> str:
return "unknown" if value is None else "%.2fs" % value
def markdown(report: dict) -> str:
meta, summary = report["metadata"], report["summary"]
lines = [
"# %s in Graff and native Cursor" % cell(meta["preset"]), "",
"Private local report — not published.", "",
"Created: %s. Mode: **%s**. Comparison: %s."
% (cell(meta.get("created_at", "unknown")), meta["mode"],
cell(meta.get("comparison_kind", "unknown"))), "",
"Graff: `%s` via `%s`. Cursor: `%s` via native `agent -p`. Requested effort: %s "
"on each arm's own scale. %d tasks × %d repetitions × two arms; %d workers."
% (cell(meta["models"]["graff"]), cell(meta["provider"]),
cell(meta["models"]["cursor"]), meta["effort"], len(meta["tasks"]),
meta["repetitions"], meta["workers"]), "",
"Graff SHA-256: `%s`. Cursor version: `%s`."
% (cell(meta.get("graff_sha256", "unknown")), cell(meta.get("cursor_version", "unknown"))),
"",
"| Suite | System | Clean pass | Mean wall (all runs) | Median wall (all runs) | Measured runs |",
"|---|---|---:|---:|---:|---:|",
]
for suite, arms in summary.items():
for arm, stats in arms.items():
lines.append("| %s | %s | %d/%d | %s | %s | %d/%d |" % (
cell(suite), arm, stats["clean_pass"], stats["runs"],
seconds(stats["mean_wall_s"]), seconds(stats["median_wall_s"]),
stats["wall_reported_runs"], stats["runs"]))
lines += ["", "## Matched passing-only pairs", "",
"Means and medians below use the same both-clean, timed pairs for both arms.", "",
"| Suite | Both clean / all pairs | Timed both-clean pairs | Graff mean / median | Cursor mean / median | Graff faster |",
"|---|---:|---:|---:|---:|---:|"]
for suite, paired in report["paired_by_suite"].items():
timed = paired["timed_both_clean_passed"]
lines.append("| %s | %d/%d | %d/%d | %s / %s | %s / %s | %d/%d |" % (
cell(suite), paired["both_clean_passed"], paired["pairs"], timed,
paired["both_clean_passed"], seconds(paired["mean_wall_s"]["graff"]),
seconds(paired["median_wall_s"]["graff"]),
seconds(paired["mean_wall_s"]["cursor"]),
seconds(paired["median_wall_s"]["cursor"]),
paired["graff_faster_on_both_passing_pairs"], timed))
lines += ["", "## Cursor native timing (not a matched Graff metric)", "",
"| Suite | Native mean / median | Native measured | Outside-native mean / median | Outside measured |",
"|---|---:|---:|---:|---:|"]
for suite, arms in summary.items():
stats = arms["cursor"]
lines.append("| %s | %s / %s | %d/%d | %s / %s | %d/%d |" % (
cell(suite), seconds(stats["native_task_duration_mean_s"]),
seconds(stats["native_task_duration_median_s"]),
stats["native_duration_reported_runs"], stats["runs"],
seconds(stats["outside_native_duration_mean_s"]),
seconds(stats["outside_native_duration_median_s"]),
stats["outside_native_duration_reported_runs"], stats["runs"]))
lines += ["", "## Measurement notes", ""]
for key in ("credentials", "latency_note", "runtime_cache"):
lines.append("- %s: %s" % (key, cell(meta.get(key, "unknown"))))
lines += ["", "## Qualification", ""]
lines += ["- " + c for c in report["caveats"]]
lines += ["", "## Non-clean runs", ""]
if not report["failures"]:
lines.append("None.")
else:
lines += ["| System | Task | Rep | Artifact | Outcome | Routing | Native completion | Exit | Timeout | Verifiers unchanged |",
"|---|---|---:|---|---|---|---|---:|---|---|"]
keys = ("harness", "task", "rep", "artifact_ok", "outcome_ok", "routing_ok",
"native_completion_ok", "exit", "timed_out", "verifiers_unchanged")
for row in report["failures"]:
lines.append("| " + " | ".join(cell(row.get(k)) for k in keys) + " |")
lines += ["", "Failure notes are retained in results.json; complete run receipts remain in the private comparison root."]
return "\n".join(lines) + "\n"
def without_root(value, roots: list):
"""Keep report contents relocatable, including root paths embedded in errors."""
if isinstance(value, str):
for root in roots:
value = value.replace(root, "<root>")
return value
if isinstance(value, list):
return [without_root(item, roots) for item in value]
if isinstance(value, dict):
return {without_root(k, roots): without_root(v, roots) for k, v in value.items()}
return value
def summarize(root: Path, output: Path) -> dict:
"""Validate one complete 1-based run matrix and write its private local report."""
root, output = Path(root), Path(output)
meta = json.loads((root / "metadata.json").read_text(encoding="utf-8"))
rows = []
for line_number, line in enumerate(
(root / "comparison" / "results.jsonl").read_text(encoding="utf-8").splitlines(), 1
):
if not line.strip():
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError as exc:
raise ValueError("invalid comparison/results.jsonl JSON at line %d" % line_number) from exc
validate(meta, rows)
suites = ["all", "core", "mcp"] + sorted({r["suite"] for r in rows} - {"core", "mcp"})
groups = {s: [r for r in rows if s == "all" or r["suite"] == s] for s in suites}
summary = {s: {a: aggregate([r for r in group if r["harness"] == a], a)
for a in ARMS} for s, group in groups.items()}
paired = {s: paired_stats(group) for s, group in groups.items()}
report = {
"metadata": meta, "summary": summary, "paired": paired["all"],
"paired_by_suite": paired,
"per_task": {t: {a: aggregate([r for r in rows if r["task"] == t
and r["harness"] == a], a) for a in ARMS}
for t in meta["tasks"]},
"failures": [{k: r.get(k) for k in FAILURE_FIELDS} for r in rows if not clean(r)],
"caveats": caveats(meta),
}
roots = sorted({str(root.absolute()), str(root.resolve())}, key=len, reverse=True)
# A filesystem root cannot be safely scrubbed by replacing every slash.
if any(Path(p).parent == Path(p) for p in roots):
raise ValueError("comparison root must not be the filesystem root")
report = without_root(report, roots)
output.mkdir(parents=True, exist_ok=True)
(output / "results.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8")
(output / "RESULTS.md").write_text(markdown(report), encoding="utf-8")
print(json.dumps({
"pass": {a: {"clean": summary["all"][a]["clean_pass"],
"runs": summary["all"][a]["runs"]} for a in ARMS},
"wall": {a: {k: summary["all"][a][k] for k in ("mean_wall_s", "median_wall_s")}
for a in ARMS},
"pairs": paired["all"], "failures": len(report["failures"]),
}, separators=(",", ":")))
return report
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
args = parser.parse_args()
try:
summarize(args.root, args.output)
except (OSError, ValueError) as exc:
parser.exit(1, "report failed: %s\n" % exc)
if __name__ == "__main__":
main()