Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat(route): a fit per candidate, and which router produced it

0xdiego25e6620bbc2ab9
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 67211671d..5e3c7f731 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -2876,7 +2876,7 @@ async function runForeground(session: AuthSession): Promise<void> {
// the screen, and predictable beat clever here (owner's call).
onRouteTask: async (text) => {
const host = cableHostRef
- if (!host) return { agentId: '', machineId: '', name: '', confidence: 0, reason: 'no agent list yet', candidates: [] }
+ if (!host) return { agentId: '', machineId: '', name: '', confidence: 0, reason: 'no agent list yet', candidates: [], weighed: 0, machines: 0, via: '' }
// Whatever the daemon knows right now. This also kicks a refresh of the remote machines, so a list
// that is short because a machine has not been asked yet fills in for the NEXT question rather than
// holding this one open.
@@ -2915,13 +2915,31 @@ async function runForeground(session: AuthSession): Promise<void> {
// the default; overshooting a deadline they DO have would turn a late answer into no answer.
const decision = await routeVoiceTask(text, candidates, undefined, ROUTE_CLASSIFY_APP_MS)
const named = (id: string) => candidates.find((agent) => agent.id === id)
- const others = candidates.filter((agent) => agent.id !== decision.agentId).slice(0, 2)
+ // The runners-up in the ROUTER's order when it gave one, and the list's own order when it did not.
+ // A picker that has to ask "which agent" is showing a ranking either way; this decides whose.
+ const ranking = (decision.scores ?? []).filter((score) => score.agentId !== decision.agentId)
+ const others = ranking.length
+ ? ranking.map((score) => named(score.agentId)).filter((agent): agent is RouterAgent => !!agent).slice(0, 2)
+ : candidates.filter((agent) => agent.id !== decision.agentId).slice(0, 2)
+ const fitOf = (id: string) => id === decision.agentId
+ ? decision.confidence
+ : ranking.find((score) => score.agentId === id)?.confidence ?? 0
return {
agentId: decision.agentId,
machineId: all.find((entry) => entry.id === decision.agentId)?.machineId ?? '',
name: named(decision.agentId)?.name ?? '',
confidence: decision.confidence,
reason: decision.reason,
+ // How many agents were actually WEIGHED, and across how many computers. The window says this
+ // while it waits, because the question a person has during those seconds is not "how long" —
+ // it is "did it even look at the agent I mean". The cap above can hide agents, and until now
+ // the only place that was said was this process's log.
+ weighed: ranked.length,
+ machines: new Set(ranked.map((agent) => agent.machine).filter(Boolean)).size,
+ // 'model' or 'heuristic', coarsened from the router's own label. The two arrive at the same low
+ // confidence BY DESIGN — an unsure model and a router that could not run must both stop and ask
+ // — and that is exactly why the window has to be able to tell them apart when it explains itself.
+ via: (decision.via ?? '').startsWith('heuristic') ? 'heuristic' : 'model',
candidates: [decision.agentId ? named(decision.agentId) : null, ...others]
.filter((agent): agent is RouterAgent => !!agent)
.map((agent) => {
@@ -2934,7 +2952,11 @@ async function runForeground(session: AuthSession): Promise<void> {
// the pane on the machine the agent actually lives on.
machineId: listed?.machineId ?? '',
machine: agent.machine ?? '',
+ engine: agent.engine ?? '',
recent: (agent.recentSummary ?? '').slice(0, 120),
+ // Drawn as a bar in the picker, never dispatched on. 0 = the router said nothing about
+ // this one, which the window renders as no bar rather than as a zero-length one.
+ confidence: fitOf(agent.id),
}
}),
}
diff --git a/cli/src/lib/voiceRouter.spec.ts b/cli/src/lib/voiceRouter.spec.ts
index d50e6b4cf..8efa76c39 100644
--- a/cli/src/lib/voiceRouter.spec.ts
+++ b/cli/src/lib/voiceRouter.spec.ts
@@ -36,7 +36,7 @@ const AGENTS: RouterAgent[] = [
describe('parseRouteOutput', () => {
it('parses a clean JSON object', () => {
const d = parseRouteOutput('{"agentId":"2","confidence":0.9,"reason":"login","needNewAgent":false}', AGENTS)
- expect(d).toEqual({ agentId: '2', confidence: 0.9, reason: 'login', needNewAgent: false })
+ expect(d).toMatchObject({ agentId: '2', confidence: 0.9, reason: 'login', needNewAgent: false })
})
it('extracts JSON from a markdown code fence', () => {
@@ -117,7 +117,7 @@ describe('routeVoiceTask', () => {
it('short-circuits to the only agent WITHOUT calling the LLM', async () => {
const d = await routeVoiceTask('anything', [{ id: '7', name: 'Solo' }])
- expect(d).toEqual({ agentId: '7', confidence: 1, reason: 'only agent in machine', needNewAgent: false })
+ expect(d).toMatchObject({ agentId: '7', confidence: 1, reason: 'only agent in machine', needNewAgent: false })
expect(runRouterOneShot).not.toHaveBeenCalled()
})
@@ -297,3 +297,68 @@ describe('routeVoiceTask engine selection', () => {
expect(d.confidence).toBeLessThanOrEqual(0.4) // and never enough to auto-dispatch
})
})
+
+describe('the runners-up carry a fit, and only for display', () => {
+ const agents = [
+ { id: 'a1', name: 'auth api' },
+ { id: 'a2', name: 'payment api' },
+ { id: 'a3', name: 'web' },
+ ]
+
+ it('reads alternates the model ranked', () => {
+ const decision = parseRouteOutput(
+ '{"agentId":"a1","confidence":0.44,"reason":"close","alternates":[{"agentId":"a2","confidence":0.3},{"agentId":"a3","confidence":0.12}]}',
+ agents,
+ )
+ expect(decision.agentId).toBe('a1')
+ expect(decision.scores).toEqual([
+ { agentId: 'a2', confidence: 0.3 },
+ { agentId: 'a3', confidence: 0.12 },
+ ])
+ })
+
+ it('drops an alternate the machine does not have, and the winner repeated', () => {
+ // A bar drawn for an agent that is not there is worse than no bar: the person would pick it.
+ const decision = parseRouteOutput(
+ '{"agentId":"a1","confidence":0.4,"alternates":[{"agentId":"ghost","confidence":0.9},{"agentId":"a1","confidence":0.4},{"agentId":"a2","confidence":"high"},{"agentId":"a3","confidence":0.2}]}',
+ agents,
+ )
+ expect(decision.scores).toEqual([{ agentId: 'a3', confidence: 0.2 }])
+ })
+
+ it('an answer with no alternates is still an answer', () => {
+ const decision = parseRouteOutput('{"agentId":"a2","confidence":0.9}', agents)
+ expect(decision.agentId).toBe('a2')
+ expect(decision.scores).toEqual([])
+ })
+
+ it('the heuristic ranks under its own winner and never over the cap', () => {
+ // The 0.4 cap is what stops a router that could not run from ever dispatching on its own, so the
+ // runners-up have to sit BELOW it — they are a picture of a ranking, not a threshold.
+ const decision = pickAgentHeuristic('fix the payment api retry', [
+ { id: 'a1', name: 'auth api' },
+ { id: 'a2', name: 'payment api' },
+ { id: 'a3', name: 'web' },
+ ])
+ expect(decision.agentId).toBe('a2')
+ expect(decision.confidence).toBe(0.4)
+ for (const score of decision.scores ?? []) {
+ expect(score.confidence).toBeLessThan(0.4)
+ expect(score.confidence).toBeGreaterThan(0)
+ }
+ })
+
+ it('with nothing matching, the winner is still the first agent', () => {
+ // Unchanged behaviour, pinned: the sort must not quietly reorder a no-match list.
+ const decision = pickAgentHeuristic('zzz qqq', agents)
+ expect(decision.agentId).toBe('a1')
+ expect(decision.confidence).toBe(0.2)
+ })
+
+ it('the label of how it was decided rides on the decision', () => {
+ // An unsure model and a router that could not run both land low ON PURPOSE. Without this they are
+ // the same thing on screen, and they need different sentences.
+ const only = pickAgentHeuristic('anything', [{ id: 'solo', name: 'solo' }])
+ expect(only.agentId).toBe('solo')
+ })
+})
diff --git a/cli/src/lib/voiceRouter.ts b/cli/src/lib/voiceRouter.ts
index 167877314..9c385e2db 100644
--- a/cli/src/lib/voiceRouter.ts
+++ b/cli/src/lib/voiceRouter.ts
@@ -32,6 +32,17 @@ export interface RouteDecision {
confidence: number
reason: string
needNewAgent: boolean
+ /** How this was decided, as logDecision labels it: an engine name, 'openrouter', or something
+ * starting with 'heuristic'. Carried on the decision — not just printed — because a caller that SHOWS
+ * the answer has to say which it is. A model that was unsure and a router that could not run land on
+ * the same low confidence by design, and without this they are indistinguishable on screen. */
+ via?: string
+ /** The runners-up and how well each fits, most confident first, never including the winner.
+ *
+ * Only ever DISPLAYED. Nothing dispatches on these: the winner is [agentId] and the number that
+ * gates auto-dispatch is [confidence]. They exist so a window that has to ask "which agent" can show
+ * whether it is a near-tie or a clear leader instead of three names in a row. */
+ scores?: Array<{ agentId: string; confidence: number }>
}
const ROUTE_SCRATCH = join(env.ADAPTER_DATA_DIR, 'voice-route-scratch')
@@ -129,20 +140,34 @@ function routeTokens(s: string): string[] {
// returns a concrete in-machine agent with a low, capped confidence — the voice still dispatches, never errors.
export function pickAgentHeuristic(transcript: string, agents: RouterAgent[]): RouteDecision {
const words = new Set(routeTokens(transcript))
- let best = agents[0]
- let bestScore = -1
- for (const a of agents) {
- const nameHits = routeTokens(a.name).filter((t) => words.has(t)).length
- const recentHits = routeTokens(a.recentSummary ?? '').filter((t) => words.has(t)).length
- const score = nameHits * 3 + recentHits
- if (score > bestScore) { bestScore = score; best = a }
- }
- const matched = bestScore > 0
+ const ranked = agents
+ .map((agent) => {
+ const nameHits = routeTokens(agent.name).filter((t) => words.has(t)).length
+ const recentHits = routeTokens(agent.recentSummary ?? '').filter((t) => words.has(t)).length
+ return { agent, score: nameHits * 3 + recentHits }
+ })
+ // STABLE on a tie, and that matters more than it looks: with no match at all every score is 0, and
+ // the winner has to stay "the first agent" — the same answer this function has always given — rather
+ // than whatever a sort happened to bubble up.
+ .sort((a, b) => b.score - a.score)
+ const best = ranked[0]
+ const matched = (best?.score ?? 0) > 0
+ const winner = matched ? 0.4 : 0.2
return {
- agentId: best?.id ?? agents[0]?.id ?? '',
- confidence: matched ? 0.4 : 0.2,
+ agentId: best?.agent.id ?? agents[0]?.id ?? '',
+ // UNCHANGED, and deliberately so: the dial and the backend gate auto-dispatch on this number, and
+ // the cap at 0.4 is what keeps a router that could not run from ever dispatching on its own.
+ confidence: winner,
reason: matched ? 'heuristic name/recent match' : 'closest agent (router unavailable)',
needNewAgent: false,
+ // Scaled UNDER the winner rather than invented: a name matcher can say "this one beat the others",
+ // and that is all these say. They are for a picker to draw a bar with, never for a threshold.
+ scores: ranked.slice(1, 4).map((entry) => ({
+ agentId: entry.agent.id,
+ confidence: best && best.score > 0
+ ? Math.max(0.05, Math.min(winner - 0.05, (entry.score / best.score) * winner))
+ : 0.1,
+ })),
}
}
@@ -155,6 +180,7 @@ function taskPreview(text: string): string {
/** Name the winner. Without this the log said an engine ran and never which agent it picked, so a wrong
* route looked identical to a right one. `via` separates a real classification from a fallback. */
function logDecision(via: string, agents: RouterAgent[], decision: RouteDecision): RouteDecision {
+ decision = { ...decision, via }
const chosen = agents.find((agent) => agent.id === decision.agentId)
console.log(
`[voice-route] → picked "${chosen?.name ?? '(unknown)'}" id=${decision.agentId || '(none)'}` +
@@ -181,8 +207,12 @@ export function buildRouterPrompt(transcript: string, agents: RouterAgent[]): st
`- 0.85+ when the name and/or recent activity clearly match\n` +
`- ~0.6 when it's a reasonable but not certain match\n` +
`- ~0.3 when nothing fits well but this is the closest agent.\n\n` +
+ `Also rank the next best fits in "alternates" (up to 2, most fitting first, never repeating your ` +
+ `pick). They are shown to the user when your confidence is low, so they can choose; they are never ` +
+ `dispatched to on their own. Omit the field when there is no other agent.\n\n` +
`Respond with ONLY a single JSON object, no prose, no markdown fence:\n` +
- `{"agentId":"<one id from the list>","confidence":<0..1>,"reason":"<max 12 words>"}`
+ `{"agentId":"<one id from the list>","confidence":<0..1>,"reason":"<max 12 words>",` +
+ `"alternates":[{"agentId":"<id>","confidence":<0..1>}]}`
)
}
@@ -210,10 +240,28 @@ export function parseRouteOutput(raw: string, agents: RouterAgent[]): RouteDecis
if (!Number.isFinite(confidence)) confidence = 0
confidence = Math.max(0, Math.min(1, confidence))
const reason = typeof obj.reason === 'string' ? obj.reason.slice(0, 120) : ''
+ // Alternates are optional and unvalidated on the way in: an id that is not on the list, a repeat of the
+ // winner, or a confidence that is not a number is DROPPED rather than corrected. A picker drawing a bar
+ // for an agent the machine does not have is worse than a picker drawing no bar.
+ const seen = new Set<string>()
+ const scores: Array<{ agentId: string; confidence: number }> = []
+ if (Array.isArray(obj.alternates)) {
+ for (const raw of obj.alternates) {
+ if (!raw || typeof raw !== 'object') continue
+ const entry = raw as Record<string, unknown>
+ const id = typeof entry.agentId === 'string' ? entry.agentId : ''
+ if (!id || !ids.has(id) || id === agentId || seen.has(id)) continue
+ const value = typeof entry.confidence === 'number' ? entry.confidence : Number(entry.confidence)
+ if (!Number.isFinite(value)) continue
+ seen.add(id)
+ scores.push({ agentId: id, confidence: Math.max(0, Math.min(1, value)) })
+ if (scores.length === 3) break
+ }
+ }
// An empty or unknown id ⇒ fall back to the closest (first) agent with capped confidence; a valid
// in-machine pick wins as-is. Either way we return a real agent and needNewAgent:false.
- if (!agentId || !ids.has(agentId)) return { agentId: fallbackId, confidence: Math.min(confidence, 0.3), reason: reason || 'closest agent', needNewAgent: false }
- return { agentId, confidence, reason, needNewAgent: false }
+ if (!agentId || !ids.has(agentId)) return { agentId: fallbackId, confidence: Math.min(confidence, 0.3), reason: reason || 'closest agent', needNewAgent: false, scores }
+ return { agentId, confidence, reason, needNewAgent: false, scores }
}
// Route a transcript to an agent. Skips the LLM for the trivial cases (0 / 1 agent).
diff --git a/cli/src/localWsServer.spec.ts b/cli/src/localWsServer.spec.ts
index e272c92aa..e7f0c2672 100644
--- a/cli/src/localWsServer.spec.ts
+++ b/cli/src/localWsServer.spec.ts
@@ -182,7 +182,8 @@ describe('local CLI WebSocket', () => {
asked.push(text)
return {
agentId: 'a1', machineId: 'm-local', name: 'auth-api', confidence: 0.86, reason: 'name matches',
- candidates: [{ agentId: 'a1', machineId: 'm-local', name: 'auth-api', machine: 'this computer', recent: 'token rotation' }],
+ weighed: 4, machines: 2, via: 'model',
+ candidates: [{ agentId: 'a1', machineId: 'm-local', name: 'auth-api', machine: 'this computer', engine: 'claude', recent: 'token rotation', confidence: 0.86 }],
}
},
})
diff --git a/cli/src/localWsServer.ts b/cli/src/localWsServer.ts
index a18ed0654..38eb737fc 100644
--- a/cli/src/localWsServer.ts
+++ b/cli/src/localWsServer.ts
@@ -70,6 +70,13 @@ export interface RouteCandidate {
machine: string
/** What it was last doing — the line under the name when the window has to ask. */
recent: string
+ /** Which CLI it runs on — 'claude', 'codex', … The window draws the same engine mark its rail does, so
+ * a row in the picker and the same agent in the rail are recognisably one thing. */
+ engine: string
+ /** How well the router thought this one fits, 0..1. DISPLAY ONLY — the pick is [RouteAnswer.agentId]
+ * and the number that gates it is [RouteAnswer.confidence]. 0 means the router said nothing about
+ * this candidate, which the window draws as no bar rather than as an empty one. */
+ confidence: number
}
export interface RouteAnswer {
@@ -80,6 +87,15 @@ export interface RouteAnswer {
reason: string
/** The best few, most confident first. Only read when the window decides to ask. */
candidates: RouteCandidate[]
+ /** How many agents were weighed, and across how many computers — what the window shows while it waits.
+ * The candidate list is capped, so this is the only place that says whether the right agent was even
+ * in the running. */
+ weighed: number
+ machines: number
+ /** 'model' when a classifier answered, 'heuristic' when name matching stood in for it. Both land on a
+ * low confidence by design; this is what lets the window say WHICH happened instead of showing the
+ * same sentence for a router that was unsure and one that never ran. */
+ via: string
}
export interface LocalWsServer {
@@ -276,12 +292,12 @@ export function attachLocalWsServer(server: http.Server, options: LocalWsServerO
const text = typeof payload?.text === 'string' ? payload.text.trim() : ''
// An answer ALWAYS goes back, even for a question we cannot serve: the window is holding a
// spinner open on this id, and silence is the one reply it cannot recover from.
- let answer: RouteAnswer = { agentId: '', machineId: '', name: '', confidence: 0, reason: 'empty task', candidates: [] }
+ let answer: RouteAnswer = { agentId: '', machineId: '', name: '', confidence: 0, reason: 'empty task', candidates: [], weighed: 0, machines: 0, via: '' }
if (text) {
try {
answer = await options.onRouteTask(text)
} catch (err) {
- answer = { agentId: '', machineId: '', name: '', confidence: 0, reason: (err as Error).message.slice(0, 120), candidates: [] }
+ answer = { agentId: '', machineId: '', name: '', confidence: 0, reason: (err as Error).message.slice(0, 120), candidates: [], weighed: 0, machines: 0, via: '' }
}
}
sink.sendFrame({ type: 'route_result', payload: { requestId, ...answer } })