graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
//! Bench-derived worker priors: `.harness/bench.json` (project, else
//! personal ~/.harness/bench.json) is a benchmark score/cost sheet — one
//! entry per (model, effort) config, e.g. a DeepSWE leaderboard export — and
//! this module turns it into per-provider tier ladders so worker assignment
//! starts from measured capability and cost instead of the compiled-in hand
//! table in subagent_tier_ladder.zig (which stays as the fallback for
//! providers the sheet cannot cover).
//!
//! Derivation, per provider, over the sheet models that RESOLVE on that
//! provider's live catalog (subagent_selection.modelForProvider — so one
//! sheet feeds every provider serving any of its models, and an unknown or
//! renamed model is skipped, never a load failure). Per-model bests are
//! folded first, then the losers are DISCARDED before any rung is seated
//! (#373): a candidate that another candidate matches-or-beats on score for
//! strictly less money is dominated, and a dominated model is never the
//! right answer at any budget. Rungs come only from the survivors — the
//! sheet's Pareto front for that provider:
//!
//! frontier = the front's top score (capability ceiling)
//! small = the front's best score-per-dollar below the frontier (the
//! efficiency knee the sheet's power-law tail points at)
//! mid = the best survivor STRICTLY between those two scores, else
//! null — a rung the data does not contain is not invented
//!
//! Seating from the front is what makes the ladder monotone by construction:
//! score(frontier) >= score(mid) >= score(small), effective cost ordered the
//! other way, and no rung beaten outright by a rung below it. Before #373
//! the three rungs were disjoint leftovers (mid = best score of whatever
//! frontier and small had not already claimed), which on the builtin sheet
//! seated codex mid = gpt-5.5 (0.54 @ $3.2) ABOVE small = gpt-5.6-luna
//! (0.67 @ $0.61) — so the #291 worker descent sent every worker to a model
//! that the rung below it beat on both axes at a fifth of the price.
//!
//! A provider needs at least two SURVIVING models to earn a derived ladder:
//! when one model dominates a provider's whole sheet there is no ladder to
//! derive (builtin openai: luna outright beats gpt-5.5, gpt-5.4 and terra),
//! and the compiled table stands rather than a fabricated descent. Scores
//! accept [0,1] or percent (0,100]; a model the sheet never prices costs
//! +inf — unpriced capability cannot be shown to be worth its price, so any
//! priced model matching its score dominates it, while two unpriced models
//! can never dominate each other. Loaded by fleet.loadAgentTypes, so the
//! session registry and auto-promote's hot-reload refresh it together.
//!
//! PRECEDENCE: project ./.harness/bench.json > personal ~/.harness/bench.json
//! > the BUILTIN sheet below — a shipped community leaderboard snapshot, so
//! every install starts with a measured cost/capability power law rather
//! than a blank. The intended refresher for the builtin is aggregated fleet
//! telemetry (users who opt into data sharing already submit signed
//! niche-tagged scores); serving that aggregate back is backend work — the
//! seam here is just "a sheet from anywhere, highest tier wins".
const std = @import("std");
const Io = std.Io;
const Allocator = std.mem.Allocator;
const tier_ladder = @import("subagent_tier_ladder.zig");
const selection = @import("subagent_selection.zig");
const provider_mod = @import("provider.zig");
const trace = @import("trace.zig");
pub const bench_path = ".harness/bench.json";
/// Bench-derived ladders, consulted by tier_ladder.forProvider ahead of the
/// compiled table. Session-arena strings; empty when no sheet is present.
pub var g_ladders: []const tier_ladder.TierLadder = &.{};
/// Which providers the user is actually logged into / keyed for, snapshotted
/// when startup resolves credentials (noteAvailability, spec-table order).
/// Derivation skips the rest: priors describe YOUR available fleet, not the
/// whole catalog. null (tests, pre-startup) = no opinion, derive everything.
/// A mid-session /login is picked up on next start, like persona files —
/// until then the compiled ladder covers the newly added provider.
pub var g_available: ?[provider_mod.provider_specs.len]bool = null;
/// By VALUE in, by value out — written to ride resolveKeys' return statement
/// (a pointer into its stack Keys would dangle) without costing the capped
/// file a line.
pub fn noteAvailability(keys: provider_mod.Keys) provider_mod.Keys {
var av: [provider_mod.provider_specs.len]bool = undefined;
for (keys.values, 0..) |v, i| av[i] = v != null;
g_available = av;
return keys;
}
pub const Entry = struct {
model: []const u8,
effort: ?[]const u8 = null, // recorded for provenance; ladders name models
score: f64,
cost: f64 = 0, // avg $ per task; <= 0 = unknown (no efficiency signal)
};
pub const Sheet = struct {
suite: []const u8 = "",
updated: []const u8 = "",
entries: []Entry = &.{},
};
/// The shipped default sheet: DeepSWE v1.1 (113 tasks, updated 2026-07-25),
/// avg $ per task. Chart-read values (±~2% score, ±~$0.5 cost) — plenty for
/// rung ORDERING, which is all derivation consumes. Models no provider
/// serves (gemini, muse-spark) still belong here: they resolve nowhere today
/// but start working the day a catalog serves them.
pub const builtin_entries = [_]Entry{
.{ .model = "claude-opus-5", .effort = "max", .score = 74, .cost = 12.0 },
.{ .model = "claude-fable-5", .effort = "high", .score = 70, .cost = 9.0 },
.{ .model = "claude-sonnet-5", .effort = "high", .score = 49, .cost = 10.5 },
.{ .model = "claude-opus-4-8", .effort = "high", .score = 41, .cost = 2.9 },
.{ .model = "claude-sonnet-4-6", .effort = "high", .score = 31, .cost = 6.5 },
.{ .model = "k3", .effort = "max", .score = 68, .cost = 4.6 },
.{ .model = "kimi-k2.7-code", .score = 30, .cost = 2.6 },
.{ .model = "gpt-5.6-sol", .effort = "max", .score = 73, .cost = 5.5 },
.{ .model = "gpt-5.6-sol", .effort = "medium", .score = 61, .cost = 1.86 },
.{ .model = "gpt-5.6-terra", .effort = "medium", .score = 35, .cost = 0.9 },
.{ .model = "gpt-5.6-luna", .effort = "max", .score = 67, .cost = 0.61 },
.{ .model = "gpt-5.6-luna", .effort = "medium", .score = 11, .cost = 0.15 },
.{ .model = "gpt-5.5", .effort = "medium", .score = 54, .cost = 3.2 },
.{ .model = "gpt-5.4", .effort = "xhigh", .score = 53, .cost = 5.2 },
.{ .model = "grok-4.5", .effort = "high", .score = 53, .cost = 3.0 },
.{ .model = "glm-5.2", .effort = "max", .score = 44, .cost = 3.9 },
.{ .model = "muse-spark-1.1", .effort = "xhigh", .score = 50, .cost = 3.0 },
.{ .model = "gemini-3.6-flash", .effort = "high", .score = 54, .cost = 3.5 },
.{ .model = "gemini-3.5-flash", .effort = "medium", .score = 38, .cost = 7.0 },
.{ .model = "gemini-3.1-pro", .effort = "high", .score = 12, .cost = 9.5 },
};
pub fn loadInto(io: Io, arena: Allocator, home: ?[]const u8) void {
var sheet = readSheet(io, arena, bench_path);
if (sheet == null) if (home) |h| {
const p = std.fmt.allocPrint(arena, "{s}/{s}", .{ h, bench_path }) catch return;
sheet = readSheet(io, arena, p);
};
g_entries = if (sheet) |s| s.entries else &builtin_entries;
// DGM feedback (#374 follow-on): lived, niche-scored runs from THIS
// workspace's archive re-weight the sheet before Pareto seating, so the
// front routes on what these models actually did here, not only on a
// leaderboard snapshot. Auto-promote's hot-reload re-runs this, so the
// front tightens as the session scores work.
const archive = trace.readTrajectoryArchive(io, arena, 8 << 20);
g_entries = blend(arena, g_entries, archive);
g_ladders = derive(arena, g_entries);
// #372: the SAME archive read also folds the (shape, role, tier, model)
// policy cells, so the session-wide ladders and the per-cell learned
// policy can never disagree about which runs they have seen.
@import("route_policy.zig").loadCells(arena, archive);
// …and the (task_class, budget_band, stratum) orchestration arms, hung on
// this same pass for the same reason and at no extra I/O: one archive
// read, three consumers, one consistent view of history.
@import("orchestration_policy.zig").loadArms(arena, archive);
}
fn readSheet(io: Io, arena: Allocator, path: []const u8) ?Sheet {
const data = Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(1 << 20)) catch return null;
return std.json.parseFromSliceLeaky(Sheet, arena, data, .{ .ignore_unknown_fields = true, .allocate = .alloc_always }) catch null;
}
/// [0,1] passes; (1,100] reads as percent; anything else is not a score.
// The fns below are pub only for bench_priors_tests.zig (600-cap split).
pub fn normalScore(s: f64) ?f64 {
if (!(s >= 0)) return null;
if (s <= 1) return s;
if (s <= 100) return s / 100.0;
return null;
}
/// The current sheet's entries (file else builtin) — the score authority
/// subscription-rung ranking consults (subagent_pin.subscriptionRung).
pub var g_entries: []const Entry = &builtin_entries;
/// The session Keys, captured where main() builds its loop context (the
/// pointer targets main's stack frame, alive for the whole loop). Lets the
/// spawn path build a Provider for a logged-in subscription — null in tests
/// and before startup, which reads as "no subscriptions reachable".
pub var g_keys: ?*const provider_mod.Keys = null;
pub fn noteKeys(keys: *provider_mod.Keys) *provider_mod.Keys {
g_keys = keys;
return keys;
}
/// Startup composition, riding resolveKeys' return: snapshot availability,
/// then load/derive the sheet — so the #291 worker default (resolved moments
/// later in main) sees the SAME bench ladders as every later /model
/// re-derivation (#371). Without this, startup derived workers from the
/// compiled table while a switch re-derived from the bench one, and the two
/// disagreed (terra vs the data-blessed gpt-5.5 on codex).
pub fn noteKeysAtStartup(keys: provider_mod.Keys, io: Io, arena: Allocator, home: ?[]const u8) provider_mod.Keys {
const kept = noteAvailability(keys);
loadInto(io, arena, home);
return kept;
}
/// Best sheet score for `model` as served by `provider_id`; null when the
/// sheet has no opinion.
pub fn scoreFor(provider_id: []const u8, model: []const u8) ?f64 {
var best: ?f64 = null;
for (g_entries) |e| {
const s = normalScore(e.score) orelse continue;
const r = selection.modelForProvider(provider_id, e.model) orelse continue;
if (!std.mem.eql(u8, r, model)) continue;
if (best == null or s > best.?) best = s;
}
return best;
}
fn strField(o: std.json.ObjectMap, key: []const u8) ?[]const u8 {
const v = o.get(key) orelse return null;
return if (v == .string) v.string else null;
}
/// One lived observation from the local trajectory archive: the pooled [0,1]
/// score seen for a (model, effort) config, joined score→config through
/// prompt_sha against the archive's recipe rows.
pub const Obs = struct { model: []const u8, effort: []const u8, sum: f64 = 0, n: u32 = 0 };
/// The DGM→Pareto feedback fold (#374 follow-on): pass 1 maps prompt_sha →
/// (model, effort) from kind:"recipe" rows, pass 2 joins kind:"score" rows
/// through that map. HONESTY NOTE: pooled per-config means still mix genome
/// quality into model quality — #290's confound — so this feeds a DAMPED
/// prior (blend's pseudo-counts), never a controlled comparison; matched-
/// genome stratification is the refinement path when archives grow.
pub fn foldObservations(arena: Allocator, archive: []const u8) []Obs {
const Recipe = struct { sha: []const u8, model: []const u8, effort: []const u8 };
var recipes: std.ArrayList(Recipe) = .empty;
var obs: std.ArrayList(Obs) = .empty;
var pass: usize = 0;
while (pass < 2) : (pass += 1) {
var it = std.mem.splitScalar(u8, archive, '\n');
while (it.next()) |ln| {
const t = std.mem.trim(u8, ln, " \t\r");
if (t.len == 0) continue;
const v = std.json.parseFromSliceLeaky(std.json.Value, arena, t, .{}) catch continue;
if (v != .object) continue;
const o = v.object;
const kind = strField(o, "kind") orelse continue;
const sha = strField(o, "prompt_sha") orelse continue;
if (pass == 0 and std.mem.eql(u8, kind, "recipe")) {
const model = strField(o, "model") orelse continue;
recipes.append(arena, .{ .sha = sha, .model = model, .effort = strField(o, "effort") orelse "" }) catch return obs.items;
} else if (pass == 1 and std.mem.eql(u8, kind, "score")) {
const sv = o.get("score") orelse continue;
const s: f64 = switch (sv) {
.float => |f| f,
.integer => |x| @floatFromInt(x),
else => continue,
};
if (!(s >= 0 and s <= 1)) continue; // archive scores are s01 by contract (#168)
const joined: ?Recipe = for (recipes.items) |rc| {
if (std.mem.eql(u8, rc.sha, sha)) break rc;
} else null;
const r = joined orelse continue;
const slot: *Obs = for (obs.items) |*x| {
if (std.mem.eql(u8, x.model, r.model) and std.mem.eql(u8, x.effort, r.effort)) break x;
} else blk: {
obs.append(arena, .{ .model = r.model, .effort = r.effort }) catch return obs.items;
break :blk &obs.items[obs.items.len - 1];
};
slot.sum += s;
slot.n += 1;
}
}
}
return obs.items;
}
/// The sheet speaks with the weight of this many lived runs: a couple of
/// real scores nudge the front, a season of them owns it. pub since #372 so
/// the per-cell learned policy damps its evidence with the SAME pseudo-count
/// discipline rather than inventing a second one.
pub const sheet_weight: f64 = 3;
/// Merge lived observations into the sheet before Pareto seating. A matching
/// (model[, effort]) entry gets a pseudo-count-blended score and KEEPS its
/// cost — units stay $/task; blending token prices in would poison the
/// domination comparisons. An unmatched config joins score-only with cost 0
/// (+inf effective cost): it may lead the front on capability but never
/// claims the efficiency rung until someone benches its price.
pub fn blend(arena: Allocator, sheet: []const Entry, archive: []const u8) []const Entry {
const obs = foldObservations(arena, archive);
if (obs.len == 0) return sheet;
var out: std.ArrayList(Entry) = .empty;
out.appendSlice(arena, sheet) catch return sheet;
for (obs) |ob| {
const mean = ob.sum / @as(f64, @floatFromInt(ob.n));
const matched: ?*Entry = for (out.items) |*e| {
if (!std.mem.eql(u8, e.model, ob.model)) continue;
if (e.effort) |ef| if (!std.mem.eql(u8, ef, ob.effort)) continue;
break e;
} else null;
if (matched) |e| {
const prior = normalScore(e.score) orelse continue;
const n: f64 = @floatFromInt(ob.n);
e.score = (prior * sheet_weight + mean * n) / (sheet_weight + n);
} else out.append(arena, .{ .model = ob.model, .effort = if (ob.effort.len == 0) null else ob.effort, .score = mean, .cost = 0 }) catch return out.items;
}
return out.items;
}
pub const Best = struct { name: []const u8, score: f64 = -1, spd: f64 = -1 };
/// One provider's per-model bests: the max score over that model's configs
/// and the max score-per-dollar, folded across every sheet entry that
/// resolves to it (several efforts of one model collapse into one candidate).
pub fn foldProvider(arena: Allocator, provider_id: []const u8, entries: []const Entry) []Best {
var models: std.ArrayList(Best) = .empty;
for (entries) |e| {
const score = normalScore(e.score) orelse continue;
const resolved = selection.modelForProvider(provider_id, e.model) orelse continue;
const slot = blk: {
for (models.items) |*m| if (std.mem.eql(u8, m.name, resolved)) break :blk m;
models.append(arena, .{ .name = resolved }) catch return models.items;
break :blk &models.items[models.items.len - 1];
};
if (score > slot.score) slot.score = score;
if (e.cost > 0 and score / e.cost > slot.spd) slot.spd = score / e.cost;
}
return models.items;
}
/// What a candidate's capability actually costs: the dollar figure implied
/// by its BEST measured efficiency (score / score-per-dollar). Deliberately
/// optimistic when a model's top score and top spd come from different
/// efforts (codex sol: 0.73 max @ $5.5 and 0.61 medium @ $1.86 fold to
/// ~$2.23) — an understated cost only makes a candidate HARDER to dominate,
/// and #373 is about never pruning a rung that might still earn its price.
/// Unpriced (spd < 0: no entry for this model carried a positive cost) is
/// +inf, so any priced model matching its score dominates it, and two
/// unpriced models never dominate each other.
pub fn effCost(m: Best) f64 {
return if (m.spd > 0) m.score / m.spd else std.math.inf(f64);
}
/// #373: does `a` beat `b` outright — at least `b`'s score for strictly less
/// money? Then `b` is not a rung, it is a mistake: every task `b` could run,
/// `a` runs at least as well for less.
///
/// Deliberately ONE cost axis. A second arm on list price was tried for #471
/// and reverted: `dominates` feeds a Pareto front, mixing two incomparable
/// cost axes makes it non-transitive, and a dominated model then evicts a
/// survivor the winner does not itself dominate (`gpt-5.4` knocked out
/// `gpt-5.6-terra` on list price while being pruned on $/task by `sol`,
/// which is unpriced and so could not evict terra itself). The list-price
/// rule #471 needs is a post-condition on the finished ladder instead —
/// tier_ladder.cheaperSeat.
pub fn dominates(a: Best, b: Best) bool {
return a.score >= b.score and effCost(a) < effCost(b);
}
/// The provider's Pareto front: fold's candidates minus everyone another
/// candidate beats outright. Exact ties (same score, same effective cost)
/// both survive — neither is strictly cheaper, so the rule can never eat a
/// whole tie group and leave the provider with nothing.
pub fn paretoFront(arena: Allocator, models: []const Best) []Best {
var front: std.ArrayList(Best) = .empty;
for (models, 0..) |m, i| {
const beaten = for (models, 0..) |o, j| {
if (i != j and dominates(o, m)) break true;
} else false;
if (!beaten) front.append(arena, m) catch return front.items;
}
return front.items;
}
pub fn derive(arena: Allocator, entries: []const Entry) []const tier_ladder.TierLadder {
var out: std.ArrayList(tier_ladder.TierLadder) = .empty;
for (provider_mod.provider_specs, 0..) |spec, i| {
if (g_available) |av| if (!av[i]) continue;
const models = paretoFront(arena, foldProvider(arena, spec.id, entries));
if (models.len < 2) continue; // one model beat the whole sheet: no ladder exists to derive
var frontier: *const Best = &models[0];
for (models) |*m| if (m.score > frontier.score) {
frontier = m;
};
var small: ?*const Best = null;
for (models) |*m| {
if (m == frontier or m.spd < 0) continue;
if (small == null or m.spd > small.?.spd) small = m;
}
const sm = small orelse continue; // capability alone cannot rank efficiency
// Only a survivor STRICTLY between the two seated rungs earns mid —
// which is what makes score(frontier) >= score(mid) >= score(small)
// true by construction (#373) instead of by leftover. The bounds
// also exclude frontier and sm themselves, so no pointer test is
// needed; a two-deep front simply has no mid, and the descent in
// subagent_selection steps frontier -> small on its own.
var mid: ?*const Best = null;
for (models) |*m| {
if (m.score >= frontier.score or m.score <= sm.score) continue;
if (mid == null or m.score > mid.?.score) mid = m;
}
out.append(arena, .{
.provider = spec.id,
.frontier = frontier.name,
.mid = if (mid) |m| m.name else null,
.small = sm.name,
}) catch return out.items;
}
return out.items;
}