Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Test steps take a mid-strength Tester; reviews keep the engineer's bar (#3774) (#3360)

harness-for-autonomous[bot]6aead4ecd74f
diff --git a/backend/src/superTabs/commands.ts b/backend/src/superTabs/commands.ts
index 2ebfb6b8b..3014eddce 100644
--- a/backend/src/superTabs/commands.ts
+++ b/backend/src/superTabs/commands.ts
@@ -26,7 +26,7 @@ import { jobKindOf } from './evidence/jobKind.js'
import { endSteps, stepEndedRows, isStep, stepName, stepsOf, taskOwner } from './steps.js'
import { closeMerged } from './mergedTasks.js'
import { reviewPlan, stepBrief, stepCheck, refreshStepBrief, type PlannedStep, type StepKind } from './reviewTable.js'
-import { builtBy, reviewerCandidates } from './reviewerPick.js'
+import { builtBy, reviewerCandidates, STRENGTH_RULE } from './reviewerPick.js'
import { recordCheckVerdict } from './checkVerdicts.js'
import { routeRecordSchema, routeTitle, startedAs, type KeptCandidate, type RouteDecision, type RouteFinal, type RouteOverride } from './routeRecord.js'
import { routingOverrideSaveSchema } from './routingPolicy.js'
@@ -1911,7 +1911,7 @@ function applyCommandOnce(original: SuperTabState, actor: Actor, input: Command,
*/
const openReviewSteps = (task: Conversation, read: NonNullable<CommandContext['pullRequestFiles']>) => {
const existing = stepsOf(state, task.id).filter(step => !step.archived)
- const taken = new Set<string>(), unassigned: string[] = []
+ const taken = new Set<string>(), unassigned: Array<{ name: string; kind: StepKind }> = []
for (const planned of reviewPlan(read.files, context.projectRules?.rules)) {
const free = (step: Conversation) => !taken.has(step.id)
let step = existing.find(s => free(s) && stepName(s) === planned.name) ?? existing.find(s => free(s) && stepKind(s) === planned.kind)
@@ -1933,13 +1933,14 @@ function applyCommandOnce(original: SuperTabState, actor: Actor, input: Command,
try { assignRefusal(target, candidate); return true } catch (error) { if (error instanceof SuperTabError) return false; throw error }
})
if (worker) assignTo(step, worker, stableId(operationId, 'review-step-assign', step.id))
- else unassigned.push(planned.name)
+ else unassigned.push({ name: planned.name, kind: planned.kind })
}
const leadId = conversation(state, groupProject(state, task).leadConversationId).leadId
if (unassigned.length && leadId) {
- message(task.id, `No free reviewer for ${unassigned.join(', ')} of #${task.number}: no agent of this project is free, never built it, `
- + 'runs a model at least as strong as its engineer and off the grid, and gave no verdict at this head. The step waits open: start or free one, '
- + 'and assign it.', [leadId], null, SYSTEM_ROW, undefined, undefined, undefined, stableId(operationId, 'review-unassigned'))
+ // Each step with the rule its pick applied (task #3774): a test's Tester needs a mid-strength model, a review the engineer's.
+ const steps = unassigned.map(step => `${step.name} (${STRENGTH_RULE[step.kind]})`).join('; ')
+ message(task.id, `No free reviewer for ${steps} of #${task.number}: no agent of this project is free in the step's role, never built it, `
+ + 'meets that model rule, and gave no verdict at this head. The step waits open: start or free one, and assign it.', [leadId], null, SYSTEM_ROW, undefined, undefined, undefined, stableId(operationId, 'review-unassigned'))
}
}
diff --git a/backend/src/superTabs/reviewSteps.test.ts b/backend/src/superTabs/reviewSteps.test.ts
index d4f9adcfc..3a3024c2b 100644
--- a/backend/src/superTabs/reviewSteps.test.ts
+++ b/backend/src/superTabs/reviewSteps.test.ts
@@ -276,3 +276,48 @@ describe('a handed task opens its review steps', () => {
expect(a).toBeTruthy()
})
})
+
+// Task #3774 (Ken's routing rule, restored 11-06: "A Tester runs where the change runs: the machine comes before the model, and a
+// mid-strength model is enough for most testing"). On 11-06 four Linux test steps waited for an Opus Tester while Sonnet Testers
+// sat idle on the same machine.
+describe('a test step\'s Tester runs a mid-strength model or stronger, whatever the engineer runs', () => {
+ const sonnet: AgentConfiguration = { ...opus, model: 'sonnet' }
+ function linux() {
+ const f = fixture()
+ const join = (name: string, roleName: string, configuration: AgentConfiguration) => f.run({ type: 'agent.add', conversationId: f.task, machineId: 'linux1',
+ harnessId: name, name, configuration, lead: false, roleName } as Command, owner, { contribution: { machineOwnerId: 'host-owner', contributorId: 'owner' } }).result.agentId as string
+ /** The Linux machine as its report lists it, once its agents joined. */
+ const listed = () => { f.state.machines = [{ machineId: 'linux1', name: 'linux1', contributorId: 'owner', membershipGeneration: 'g', generation: 'g1', cwd: '/swarm',
+ scope: 'swarm', engines: ['claude'], maxAgents: 4, paused: false,
+ observation: { generation: 'g1', receivedAt: 1_000_000 - 1000, sessionKind: 'computer', resources: { os: 'Linux' }, tools: {}, runningHarnessIds: [], stoppedHarnessIds: [],
+ readiness: { engines: [{ engine: 'claude', installed: true, signedIn: true }] },
+ quotas: [{ engine: 'claude', known: true, windows: [{ name: 'weekly', usedPercent: 10, resetsAt: null }] }] } }] as never }
+ return Object.assign(f, { join, listed })
+ }
+
+ it('gives an Opus-built task\'s Linux test step to an idle Sonnet Tester; its review still refuses a Sonnet Reviewer', () => {
+ const f = linux()
+ const tester = f.join('sonnet-tst', 'Tester', sonnet)
+ f.join('sonnet-rev', 'Reviewer', sonnet)
+ f.listed()
+ const effects = f.ready(['cli/src/superTabs/host.ts'])
+ expect(f.steps()).toEqual(expect.arrayContaining([{ name: 'Review', agent: null, status: null },
+ { name: 'Test on a Linux machine', agent: tester, status: 'working' }]))
+ // The notice names the rule it applied to the step left open.
+ const told = effects.messages.find(m => m.recipients.includes(f.lead) && /no free reviewer/i.test(m.text))!
+ expect(told.text).toContain('Review')
+ expect(told.text).toContain('at least as strong as its engineer')
+ })
+
+ it('never gives a test step to a grid model or a weaker one, and says the rule it applied', () => {
+ const f = linux()
+ const grid = f.join('grid-tst', 'Tester', sonnet)
+ f.state.agents.find(a => a.id === grid)!.gridModel = 'kimi k3'
+ f.join('haiku-tst', 'Tester', { ...opus, model: 'haiku', effort: 'auto' })
+ f.listed()
+ const effects = f.ready(['cli/src/superTabs/host.ts'])
+ expect(f.steps().find(s => s.name === 'Test on a Linux machine')).toMatchObject({ agent: null })
+ const told = effects.messages.find(m => m.recipients.includes(f.lead) && /no free reviewer/i.test(m.text))!
+ expect(told.text).toMatch(/Test on a Linux machine.*mid-strength model or stronger/s)
+ })
+})
diff --git a/backend/src/superTabs/reviewerPick.test.ts b/backend/src/superTabs/reviewerPick.test.ts
index 9c49ec77d..3cedee0ba 100644
--- a/backend/src/superTabs/reviewerPick.test.ts
+++ b/backend/src/superTabs/reviewerPick.test.ts
@@ -180,4 +180,17 @@ describe('reviewerCandidates', () => {
f.add('away', otherLead, opus, { roleName: 'Reviewer' })
expect(f.pick()).toEqual([])
})
+
+ // Task #3774: a test step's Tester needs a mid-strength model or stronger, whatever the engineer runs; a review keeps its bar.
+ it('takes a Sonnet Tester for an Opus engineer\'s test step, never for its review', () => {
+ const f = fixture()
+ f.add('s-tst', f.task, sonnet, { roleName: 'Tester' })
+ f.add('s-rev', f.task, sonnet, { roleName: 'Reviewer' })
+ const grid = f.add('g-tst', f.task, opus, { roleName: 'Tester' })
+ f.state.agents.find(a => a.id === grid)!.gridModel = 'kimi k3'
+ f.add('h-tst', f.task, { ...opus, model: 'haiku', effort: 'auto' }, { roleName: 'Tester' })
+ expect(f.pick('test')).toEqual(['s-tst'])
+ expect(f.pick('review')).toEqual([])
+ expect(f.pick('security')).toEqual([])
+ })
})
diff --git a/backend/src/superTabs/reviewerPick.ts b/backend/src/superTabs/reviewerPick.ts
index 53dd5e08c..c4bd3e065 100644
--- a/backend/src/superTabs/reviewerPick.ts
+++ b/backend/src/superTabs/reviewerPick.ts
@@ -1,7 +1,7 @@
/**
* Who may take a handed task's review, security review or test step (task #3401, anh Dee's Swarm redesign, 2026-11-05): a
- * free agent of the task's project that never worked on the task, runs no grid model, is at least as strong as its engineer,
- * and gave no verdict at this head. The project instructions' rules that the server now keeps: a Reviewer is never the
+ * free agent of the task's project that never worked on the task, runs no grid model, is at least as strong as its engineer
+ * (a test step's Tester: a mid-strength model or stronger, task #3774), and gave no verdict at this head. The project instructions' rules that the server now keeps: a Reviewer is never the
* Engineer of the same task (Roles, Reviewer), two security reviews are independent (Rules for every agent), a model on the
* grid is never offered for a review (route).
*/
@@ -40,6 +40,17 @@ export const strengthOf = (state: SuperTabState, agent: Agent): number | null =>
/** The strongest the catalog rates: what an engineer of unknown strength is taken to be, so its reviewer is never weaker. */
const TOP_STRENGTH = Math.max(...MODEL_CATALOG.models.map(row => row.strength))
+/** A test step's bar, whatever the engineer runs (task #3774, Ken's routing rule: "A Tester runs where the change runs: the machine
+ * comes before the model, and a mid-strength model is enough for most testing"): the catalog's middle strength, Sonnet's and
+ * above (Sol, Opus). On 11-06 four Linux test steps waited for an Opus Tester while Sonnet Testers sat idle on that machine. */
+export const TEST_STRENGTH = 2
+
+/** The rule [kind]'s pick applies to a model, as the PM's notice names it when a step stays open (commands.ts openReviewSteps). */
+export const STRENGTH_RULE: Record<StepKind, string> = {
+ review: 'runs a model at least as strong as its engineer and off the grid',
+ security: 'runs a model at least as strong as its engineer and off the grid',
+ test: 'runs a mid-strength model or stronger (Sonnet, Sol, Opus) off the grid, on the step\'s machine',
+}
/** Roles with work of their own that never take a step (project instructions, Roles): the PM, its Guide and Integrator, a Buddy.
* By role name, and by the agent's own name too: on the live Swarm the Guides carry the role Reviewer and the Integrator none
@@ -63,7 +74,8 @@ export function reviewerCandidates(state: SuperTabState, task: Conversation, kin
const index = workerIndex(state)
const built = builtBy(task)
const engineer = task.assignment ? state.agents.find(a => a.id === task.assignment!.agentId) : undefined
- const bar = (engineer && strengthOf(state, engineer)) ?? TOP_STRENGTH
+ // A review or security review: at least the engineer's strength. A test: mid-strength or more, whoever built it.
+ const bar = kind === 'test' ? TEST_STRENGTH : (engineer && strengthOf(state, engineer)) ?? TOP_STRENGTH
const steps = stepsOf(state, task.id)
const holding = new Set(steps.filter(step => activeWork(step)).map(step => step.assignment!.agentId))
const judged = new Set([task, ...steps].flatMap(group => group.checks.flatMap(check => check.verdicts ?? []))