The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
/** Routing by what choices actually cost and how they did (Scholes's goals, piece 3, 2026-10-31): a routing plug-in (route.ts
* RoutePlugin) that notes on a candidate its choice's measured results for the task's kind of work, over the project's last
* 7 days (backend superTabs/costs/read.ts, the same read as `harness super-tab costs`, which counts only
* the groups the routing agent may read and refuses a Buddy).
*
* - Cost: the full cost per shipped change, every ended attempt of the choice (failed and lost ones too) and every review of
* its work, rework rounds included. A choice's own attempts alone made one that passes 6 of 10 first reviews look cheaper
* than one that passes 9 of 10, its extra review rounds left out (Lead, 2026-10-31). The measured choices of the kind are
* put on the catalog's cost scale by it, so they compare with unmeasured ones on average.
* - Results: a first-review pass rate clearly under another measured choice's (their 95% ranges apart) is said, so another
* choice of its strength is preferred, never a weaker strength. With 5 to 10 reviews the ranges nearly always overlap, so
* cost decides.
* - Only with enough of it: at least MIN_SHIPPED shipped changes for cost and MIN_FIRST_REVIEWS first reviews for the pass
* rate (the nightly reflection's own thresholds, costs/reflection.ts); otherwise nothing is said.
* It never adds or refuses a candidate: what may run is the route's, and since task #2901 its finding is a note, never the
* order. The kind is the word the routing agent gives in its assessment (`kind`); none, or one nothing was measured for, says
* nothing. */
import { claudeAliasForModel } from '../engines/claude/runtimeProfile.js'
import { catalogRow, type Catalog, type Probe, type RoutePlugin } from './route.js'
import type { PluginMaker } from './routePlugins.js'
import type { ScopedRequest } from './start.js'
import { within } from './wire.js'
export const MEASURED = 'measured results'
export const DAYS = 7
export const MIN_FIRST_REVIEWS = 5
export const MIN_SHIPPED = 3
/** Which way a clearly worse pass rate leans, in catalog strength: less than one step, since only another choice of its
* strength is preferred. It moves nothing (task #2901). */
const WORSE = -0.5
const Z = 1.96
/** Well under the makers' 3 s (routePlugins.ts MAKER_TIMEOUT_MS), so a slow read is said as this piece's own failure. */
export const READ_MS = 2000
/** One choice's results for one kind of work, as the costs read gives them (backend costs/read.ts CostSummaryRow, FullCost). */
export interface MeasuredRow { engine: string; model: string; effort: string; kind: string; firstReviews: number; firstReviewPassed: number
shipped: number; costPerShipped: number | null; fullCostPerShipped: number | null
/** The share of the full cost's attempts whose cost is known; the rest were filled with their choice's mean. */
fullCostKnown: number | null }
/** `truncated`: the read reached its row limit, so it holds only the newest attempts. */
export interface Measured { rows: MeasuredRow[]; truncated: boolean }
const ENGINE: Record<string, string> = { claude: 'Claude', codex: 'Codex' }
const count = (v: unknown): v is number => Number.isInteger(v) && (v as number) >= 0
const money = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v) && v >= 0
const object = (v: unknown): Record<string, unknown> => v && typeof v === 'object' && !Array.isArray(v) ? v as Record<string, unknown> : {}
/** The rows of a costs answer it can go by, or null when the answer is no summary. A row it cannot read is dropped; a
* backend before full costs sends none, read as unknown. */
export function measuredFrom(answer: unknown): Measured | null {
const reply = object(answer), value = object(reply.value)
if (reply.ok !== true || !Array.isArray(value.rows)) return null
const rows = value.rows.map(object).flatMap((r): MeasuredRow[] => {
const full = r.fullCostPerShipped ?? null, own = r.costPerShipped ?? null, known = r.fullCostKnown ?? null
if (![r.engine, r.model, r.effort, r.kind].every(s => typeof s === 'string') || !KIND.test(r.kind as string) || !count(r.firstReviews) || !count(r.firstReviewPassed)
|| r.firstReviewPassed > r.firstReviews || !count(r.shipped) || !(full === null || money(full)) || !(own === null || money(own))
|| !(known === null || money(known) && known <= 1)) return []
return [{ engine: r.engine as string, model: r.model as string, effort: r.effort as string, kind: r.kind as string, firstReviews: r.firstReviews,
firstReviewPassed: r.firstReviewPassed, shipped: r.shipped, costPerShipped: own, fullCostPerShipped: full, fullCostKnown: known }]
})
return { rows, truncated: value.truncated === true }
}
/** Each reason a read failed, said once per process: the route runs about 30 times an hour, and a refusal that lasts (a Team
* account refused the costs route until 2026-11-03) would otherwise fill the log or, as before, say nothing at all. */
const saidWhy = new Set<string>()
function sayOnce(why: string): void {
if (saidWhy.has(why)) return
saidWhy.add(why)
console.warn(`[super-tabs] the route could not read cost results: ${why}; the route is shown without them`)
}
const reasonOf = (answer: Record<string, unknown>) =>
`${String(answer.error)}${Number.isInteger(answer.status) ? ` (${answer.status as number})` : ''}`
/** The project's measures, read through the route's own backend request: null when refused, failed, slower than [ms] or
* unreadable, so the route has no measured notes. A refusal is logged by its code, never its detail. The read is the costs
* command's, about 0.1 to 0.3 s beyond a plain request on the openharness Swarm (2026-10-31). */
export async function readMeasured(request: ScopedRequest, ms = READ_MS): Promise<Measured | null> {
const answer = await within(request({ action: 'costs', days: DAYS }).catch(() => { sayOnce('the request failed'); return null }), ms)
if (answer && typeof answer.error === 'string') sayOnce(reasonOf(answer))
return answer ? measuredFrom(answer) : null
}
/** The 95% Wilson range of `passed` of `n` (backend costs/reflection.ts wilson): honest about small samples. */
function wilson(passed: number, n: number): { low: number; high: number } {
const p = passed / n, z2 = Z * Z
const centre = p + z2 / (2 * n), spread = Z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n)), scale = 1 + z2 / n
return { low: Math.max(0, (centre - spread) / scale), high: Math.min(1, (centre + spread) / scale) }
}
/** The name a candidate's runs are kept under: a grid model's own, a Claude family for Claude's full ids. */
function ranAs(p: Pick<Probe, 'engine' | 'model' | 'gridModel'>): string {
if (p.gridModel) return p.gridModel
return p.engine === 'claude' ? claudeAliasForModel(p.model.replace(/\[1m\]$/i, '')) ?? p.model : p.model
}
const listed = (r: MeasuredRow, catalog: Catalog) => catalog.models.some(m => m.engine === r.engine && m.model === r.model && m.effort === r.effort)
/** Kinds whose attempts have no result of their own: a review is credited with the reviewed work's ship, and its cost is
* already in that build's full cost, so by cost alone Opus high reviews read as 76 shipped changes at $0.95 (review of #2423). */
const UNMEASURED_KINDS = new Set(['review'])
/** Another choice as a reason names it: only a model of the catalog, whose names every reader of the route log already sees.
* A row's model is what an agent was configured with, and a grid or local model's can be a file path or a person's share
* name: such a row is "another measured choice" (hook review, 2026-10-31: reasons hold numbers and kinds of work only). */
const nameOf = (r: MeasuredRow, catalog: Catalog) => listed(r, catalog) ? `${ENGINE[r.engine]} ${r.model} ${r.effort}` : 'another measured choice'
/** A kind of work as the route takes it (route.ts parseAssessment): one lowercase word. A row of any other kind is dropped. */
const KIND = /^[a-z][a-z0-9-]{0,39}$/
const costed = (r: MeasuredRow) => r.shipped >= MIN_SHIPPED && r.fullCostPerShipped !== null
const reviewed = (r: MeasuredRow) => r.firstReviews >= MIN_FIRST_REVIEWS
/** The plug-in over [measured]: pure, so the route notes every candidate with the same numbers. */
export function measuredPlugin(measured: Measured): RoutePlugin {
/** Per kind: its rows, and the measured costs' scale on the catalog's: dollars per cost point, so the measured choices compare
* with unmeasured ones on average. Null when fewer than two choices have a cost to compare. */
const byKind = new Map<string, { rows: MeasuredRow[]; scale: number | null; results: boolean }>()
const scaleOf = (rows: MeasuredRow[], catalog: Catalog) => {
// Only the catalog's own choices set the scale: a run of a model nobody offers (an effort the catalog has no row for, a
// model never reported) would shift every candidate's measured cost without being one (review of #2423).
const scaled = rows.filter(r => costed(r) && listed(r, catalog))
const points = scaled.reduce((sum, r) => sum + catalogRow(catalog, r.engine, r.model, r.effort)!.cost, 0)
const dollars = scaled.reduce((sum, r) => sum + r.fullCostPerShipped!, 0)
// All free (own hardware): nothing to tell them apart by.
return scaled.length >= 2 && dollars > 0 ? dollars / points : null
}
return {
name: MEASURED,
adjust(candidate, { assessment, catalog }) {
const kind = assessment.kind
if (!kind || UNMEASURED_KINDS.has(kind)) return null
let ofKind = byKind.get(kind)
if (!ofKind) {
const rows = measured.rows.filter(r => r.kind === kind)
// A kind with no first review at all has no results to go by: cost alone would judge it as nothing else is.
ofKind = { rows, scale: rows.some(r => r.firstReviews > 0) ? scaleOf(rows, catalog) : null, results: rows.some(r => r.firstReviews > 0) }
byKind.set(kind, ofKind)
}
if (!ofKind.results) return null
const name = ranAs(candidate)
const mine = ofKind.rows.find(r => r.engine === candidate.engine && r.model === name && r.effort === candidate.effort)
if (!mine) return null
// Clearly fewer passes than the best other measured choice: the top of its range under the bottom of that one's.
const range = reviewed(mine) ? wilson(mine.firstReviewPassed, mine.firstReviews) : null
const best = range ? ofKind.rows.filter(r => r !== mine && reviewed(r))
.map(r => ({ r, low: wilson(r.firstReviewPassed, r.firstReviews).low })).sort((x, y) => y.low - x.low)[0] : undefined
const worse = !!range && !!best && range.high < best.low
if (!costed(mine) && !worse) return null
const pass = !reviewed(mine) ? `first reviews too few to compare (${mine.firstReviews})`
: `${mine.firstReviewPassed} of ${mine.firstReviews} first reviews passed${worse
? `, clearly fewer than ${nameOf(best!.r, catalog)}'s ${best!.r.firstReviewPassed} of ${best!.r.firstReviews}, so prefer another choice of its strength` : ''}`
let cost: string, shift = 0
if (!costed(mine)) cost = `too few shipped changes with known costs to compare cost (${mine.shipped} shipped)`
else {
const estimated = mine.fullCostKnown !== null && mine.fullCostKnown < 1 ? `; ${Math.round((1 - mine.fullCostKnown) * 100)}% of the costs estimated` : ''
const full = `$${mine.fullCostPerShipped!.toFixed(2)} per shipped change over ${mine.shipped} (failed attempts and every review round included${estimated})`
if (ofKind.scale === null) cost = `${full}; no other choice for ${kind} work has a cost to compare it with`
else {
const points = mine.fullCostPerShipped! / ofKind.scale
shift = points - candidate.cost
cost = `${full}; cost ${points.toFixed(1)} in the catalog's terms, against its listed ${candidate.cost}`
}
}
return { reason: `${kind} work, last ${DAYS} days: ${pass}, ${cost}${measured.truncated ? '; only the newest attempts were read' : ''}`,
...(shift ? { cost: shift } : {}), ...(worse ? { strength: WORSE } : {}) }
},
}
}
/** The maker (routePlugins.ts): no kind of work, nothing to read; a read that fails says so through the route's warnings
* (the maker failed); a kind with no rows has nothing to say. */
export const MEASURED_MAKER: PluginMaker = {
name: MEASURED,
async make({ assessment, request }) {
const kind = assessment.kind
if (!kind) return null
const measured = await readMeasured(request)
if (!measured) throw new Error('The cost results could not be read.')
return measured.rows.some(r => r.kind === kind) ? measuredPlugin(measured) : null
},
}