Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

11.1 KB
Raw Blame
import { describe, expect, it } from 'vitest'
import type { Conversation } from '../model.js'
import type { RouteRun } from './attempts.js'
import type { CostRow } from './read.js'
import { proposalQuestion, proposalTitle, routingLine, routingReflection, wilson, type FailedStart, type ReflectionInput } from './reflection.js'

const MINUTE = 60_000, HOUR = 60 * MINUTE, DAY = 24 * HOUR
const NOW = 10 * DAY
const tokens = (model: string, input: number) => ({ [model]: { input, output: 0, cacheRead: 0, cacheWrite: 0 } })
/** An attempt [startedAt] ago that ran an hour and used 1M input tokens of [model]: $4 on Opus, $2 on Sonnet and gpt-6.1-sol. */
const run = (engine: string, model: string, effort: string, startedAt: number, over: Partial<RouteRun> = {}): RouteRun => ({ machineId: 'm', harnessId: 'h',
  engine, model, effort, billing: 'subscription', startedAt, start: {}, endedAt: startedAt + HOUR, end: tokens(model === 'opus' ? 'claude-opus-5-5'
    : model === 'sonnet' ? 'claude-sonnet-5-5' : model, 1e6), ...over })
const row = (assignmentId: string, conversationId: string, kind: string, r: RouteRun, stepUp?: CostRow['stepUp']): CostRow =>
  ({ conversationId, assignmentId, outcome: 'done', kind, fresh: false, run: r, ...stepUp ? { stepUp } : {} })
const group = (id: string, runs: Array<{ assignmentId: string; firstReview?: boolean }>, shipped?: { by: string; at: number }) => ({ id,
  title: 'Secret plan for Acme', ledger: { runs: runs.map(r => ({ engine: 'claude', model: 'opus', at: 0, ...r })),
    ...shipped ? { shippedAt: shipped.at, shippedBy: { assignmentId: shipped.by } } : {} } }) as unknown as Conversation

/**
 * A seeded week of one project:
 * - parser on Opus high: 5 attempts, all shipped, 4 of 5 first reviews passed, $4 each. Two of them in the last day.
 * - parser on Sonnet medium: 5 attempts, all shipped, 4 of 5 passed, $2 each: as good for half the price.
 * - ui on codex gpt-6.1-sol high: 6 attempts, 0 of 6 passed, one shipped later, $2 each.
 * - docs: three tasks whose grid attempt (deepseek, own hardware, 1, 1.5 and 0.5 h) failed and stepped up to Sonnet, which shipped.
 */
function seededWeek(): ReflectionInput {
  const rows: CostRow[] = [], groups: Conversation[] = []
  const at = (daysAgo: number) => NOW - daysAgo * DAY
  ;['p1', 'p2', 'p3', 'p4', 'p5'].forEach((id, i) => {
    rows.push(row(id, `g-${id}`, 'parser', run('claude', 'opus', 'high', at(i < 2 ? 0.5 : 3))))
    groups.push(group(`g-${id}`, [{ assignmentId: id, firstReview: i !== 4 }], { by: id, at: at(i < 2 ? 0.5 : 3) + HOUR }))
  })
  ;['s1', 's2', 's3', 's4', 's5'].forEach((id, i) => {
    rows.push(row(id, `g-${id}`, 'parser', run('claude', 'sonnet', 'medium', at(4))))
    groups.push(group(`g-${id}`, [{ assignmentId: id, firstReview: i !== 4 }], { by: id, at: at(4) + HOUR }))
  })
  ;['u1', 'u2', 'u3', 'u4', 'u5', 'u6'].forEach((id, i) => {
    rows.push(row(id, `g-${id}`, 'ui', run('codex', 'gpt-6.1-sol', 'high', at(5))))
    groups.push(group(`g-${id}`, [{ assignmentId: id, firstReview: false }], i === 5 ? { by: id, at: at(5) + HOUR } : undefined))
  })
  const grid = (minutes: number, daysAgo: number) => run('claude', 'deepseek/deepseek-v4.1-flash-1017', 'low', at(daysAgo),
    { grid: 'deepseek/deepseek-v4.1-flash-1017', billing: 'own-hardware', endedAt: at(daysAgo) + minutes * MINUTE })
  ;([[60, 6], [90, 2], [30, 1.5]] as const).forEach(([minutes, daysAgo], i) => {
    const g = `g-d${i}`
    rows.push(row(`d${i}`, g, 'docs', grid(minutes, daysAgo)),
      row(`d${i}b`, g, 'docs', run('claude', 'sonnet', 'medium', at(daysAgo) + 2 * HOUR), { attempt: 2, reason: 'Its first review said Ready no.' }))
    groups.push(group(g, [{ assignmentId: `d${i}`, firstReview: false }], { by: `d${i}b`, at: at(daysAgo) + 3 * HOUR }))
  })
  const failed: FailedStart = { engine: 'codex', model: 'gpt-6.1-sol', effort: 'high', at: at(0.2) }
  return { now: NOW, rows, groups, failedStarts: [failed, { ...failed, at: at(3) }], document: '# Project instructions\n\n## Routing\n\n- Opus builds.\n' }
}
/** [n] more first reviews of a kind on a model, all as [passed], each shipped at $[cost]. */
function more(week: ReflectionInput, kind: string, engine: string, model: string, effort: string, n: number, passed: boolean) {
  for (let i = 0; i < n; i++) {
    const id = `${kind}-${model}-${i}`
    week.rows.push(row(id, `g-${id}`, kind, run(engine, model, effort, NOW - 3 * DAY)))
    week.groups.push(group(`g-${id}`, [{ assignmentId: id, firstReview: passed }], { by: id, at: NOW - 3 * DAY + HOUR }))
  }
}
const UI = 'Stop routing ui work to codex gpt-6.1-sol high'
const PARSER = 'Route parser work to claude sonnet medium instead of claude opus high'
const DOCS = 'Start docs work on a subscription model instead of the grid'

describe('the nightly routing reflection', () => {
  it('returns proposals only, without a report or a report renderer', () => {
    const report = routingReflection(seededWeek())!
    expect(Object.keys(report)).toEqual(['proposals'])
  })

  it('proposes at most three changes, the largest saving first, each with its counts, its uncertainty and its API-equivalent saving', () => {
    const report = routingReflection(seededWeek())!
    expect(report.proposals.map(({ title, why, saving, dollars, hours, strong }) => ({ title, why, saving, dollars, hours, strong }))).toEqual([
      { title: UI, why: 'Its first review passed 0 of 6 (0%); even the top of its 95% range is 39%. Only 6 first reviews.',
        saving: 'about $12.00 a week at API prices of attempts that failed their first review', dollars: 12, hours: 0, strong: false },
      { title: PARSER, why: 'Its first review passed 4 of 5 (80%) against 4 of 5 (80%), at $2.00 instead of $4.00 per shipped change at API prices. '
        + 'Small samples (5 and 5 first reviews): the difference may be chance.', saving: 'about $10.00 a week at API prices', dollars: 10, hours: 0, strong: false },
      { title: DOCS, why: '3 of 3 docs tasks needed another model after their grid attempt failed. Only 3 tasks.',
        saving: 'about 3 hours to ship a week', dollars: 0, hours: 3, strong: false },
    ])
  })

  // Project instructions, phase 4 (decision 21): a Yes is one fixed line the coordinator writes into the Routing section.
  it('asks Yes as a change to the instructions, and recommends Yes only on strong evidence, else Not yet', () => {
    const [ui, cheaper, docs] = routingReflection(seededWeek())!.proposals
    expect(routingLine(ui!)).toBe('Ui work no longer runs on codex gpt-6.1-sol high.')
    expect(routingLine(cheaper!)).toBe('Parser work runs on claude sonnet medium instead of claude opus high.')
    expect(routingLine(docs!)).toBe('Docs work starts on a subscription model, not on the grid.')
    const asked = proposalQuestion(ui!)
    expect(asked.options[0]!.startsWith('Yes: the coordinator writes this into the Routing section of the instructions')).toBe(true)
    expect(asked).toEqual({ question: `${UI}?\nWhy: ${ui!.why} This could save ${ui!.saving}.`,
      options: [asked.options[0], 'No: keep routing as it is', 'Not yet: ask again when there is more evidence'],
      recommended: 2, unblocks: expect.stringContaining(routingLine(ui!)) })
    expect(asked.unblocks).not.toMatch(/reminded|30 days/)
    expect(proposalTitle(proposalQuestion(ui!).question)).toBe(UI)
    // Ten first reviews on each side: strong enough to recommend Yes, with no small-sample warning.
    const week = seededWeek()
    more(week, 'parser', 'claude', 'opus', 'high', 5, true)
    more(week, 'parser', 'claude', 'sonnet', 'medium', 5, true)
    const parser = routingReflection(week)!.proposals.find(p => p.title === PARSER)!
    expect(parser).toMatchObject({ strong: true, why: 'Its first review passed 9 of 10 (90%) against 9 of 10 (90%), at $2.00 instead of $4.00 per shipped change at API prices.' })
    expect(proposalQuestion(parser).recommended).toBe(0)
  })

  it('proposes to stop a model only when even the top of its 95% range is under half', () => {
    expect(wilson(0, 6).high).toBeCloseTo(0.39, 2)
    expect(wilson(1, 6).high).toBeGreaterThan(0.5)
    const week = seededWeek()
    // One of the six passed: it may still pass half the time, so nothing is proposed for it.
    week.groups = week.groups.map(g => g.id === 'g-u1' ? group('g-u1', [{ assignmentId: 'u1', firstReview: true }]) : g)
    expect(routingReflection(week)!.proposals.map(p => p.title)).toEqual([PARSER, DOCS])
  })

  it('keeps only the three largest savings', () => {
    const week = seededWeek()
    // A fourth: test work on codex xhigh whose first review passed 0 of 7, at $2 each: $14 a week, the largest.
    more(week, 'test', 'codex', 'gpt-6.1-sol', 'xhigh', 7, false)
    expect(routingReflection(week)!.proposals.map(p => p.title)).toEqual(['Stop routing test work to codex gpt-6.1-sol xhigh', UI, PARSER])
  })

  it('leaves out what the owner answered lately, without spending a slot on it', () => {
    const week = seededWeek()
    more(week, 'test', 'codex', 'gpt-6.1-sol', 'xhigh', 7, false)
    expect(routingReflection({ ...week, decided: ['Stop routing test work to codex gpt-6.1-sol xhigh'] })!.proposals.map(p => p.title)).toEqual([UI, PARSER, DOCS])
  })

  it('reads the instructions instead of an approved list', () => {
    const week = seededWeek()
    const [ui, parser] = routingReflection(week)!.proposals
    const document = `${week.document}- ${routingLine(ui!)}\n- ${routingLine(parser!)}\n`
    const report = routingReflection({ ...week, document })!
    expect(report.proposals.map(p => p.title)).toEqual([DOCS])
    // Without the line, the same change is proposed.
    expect(routingReflection(week)!.proposals.map(p => p.title)).toEqual([UI, PARSER, DOCS])
  })

  it('proposes nothing from too few first reviews, shipped changes or grid step-ups', () => {
    const week = seededWeek()
    week.rows = week.rows.filter(r => !['p5', 'u6', 'd2', 'd2b'].includes(r.assignmentId))
    week.groups = week.groups.map(g => g.id === 'g-u1' ? group('g-u1', [{ assignmentId: 'u1', firstReview: true }]) : g)
    expect(routingReflection(week)!.proposals).toEqual([])
  })

  it('proposes no cheaper model on fewer than five first reviews, however well they went', () => {
    const week = seededWeek()
    // Without its one failed review, Sonnet passed 4 of 4: better and cheaper, but on four reviews.
    week.rows = week.rows.filter(r => r.assignmentId !== 's5')
    expect(routingReflection(week)!.proposals.map(p => p.title)).not.toContain(PARSER)
    more(week, 'parser', 'claude', 'sonnet', 'medium', 1, true)
    expect(routingReflection(week)!.proposals.map(p => p.title)).toContain(PARSER)
  })

  it('is empty for a day with no attempt and no failed start: nothing is posted', () => {
    const week = seededWeek()
    week.rows = week.rows.filter(r => r.run.startedAt < NOW - DAY)
    week.failedStarts = week.failedStarts.filter(f => f.at < NOW - DAY)
    expect(routingReflection(week)).toBeNull()
  })

  it('says only aggregates: no task title, assignment or conversation id', () => {
    for (const proposal of routingReflection(seededWeek())!.proposals) {
      const question = proposalQuestion(proposal).question
      expect(question.split('\n')).toHaveLength(2)
      for (const secret of ['Secret plan', 'g-p1', 'p1', 'u3']) expect(question).not.toContain(secret)
    }
  })
})