Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

17.3 KB
Raw Blame
// cli/src/superTabs/windowPull.spec.ts
import { describe, expect, it } from 'vitest'
import { choose, decide, recordOf, type Assessment, type Catalog, type Commitment, type Fit, type Probe, type RoutePlugin } from './route.js'
import { makePlugins, ROUTE_PLUGIN_MAKERS, type PluginInput } from './routePlugins.js'
import { MEASURED, measuredPlugin } from './routeMeasured.js'
import { MAX_DISCOUNT, pendingPicks, PENDING_MS, WINDOW_PULL, windowPull, windowPullAdjust, windowPullMaker, weeklyUnused } from './windowPull.js'

/** Today's rows (route.spec.ts): Opus high and Codex default both strength 3, Codex cheaper; Sonnet weaker. */
const policy: Catalog = { grid: { strength: 1, models: [] }, models: [
  { engine: 'claude', model: 'opus', effort: 'high', strength: 3, cost: 3, offered: true },
  { engine: 'codex', model: 'default', effort: 'default', strength: 3, cost: 2, offered: true },
  { engine: 'claude', model: 'sonnet', effort: 'medium', strength: 2, cost: 2, offered: true },
] }
const NOW = 1_800_000_000_000, M = 60_000, H = 3_600_000, D = 86_400_000
const a = (patch: Partial<Assessment> = {}): Assessment => ({ difficulty: 'medium', risk: 'low', size: 'medium', why: 'API change.', needs: { tools: [], devices: [] }, ...patch })
/** A reading with a 5-hour and a weekly window, each as [percent left, resets in]. */
type Win = [left: number, resetsIn: number | null] | null
const windows = (five: Win, week: Win, account: string): Probe['quota'] => {
  const read = ([['5-hour', five], ['weekly', week]] as const).flatMap(([name, w]) => w ? [{ name, leftPercent: w[0], resetsAt: w[1] === null ? null : NOW + w[1] }] : [])
  return { known: true, leftPercent: Math.min(...read.map(w => w.leftPercent)), resetsAt: null, windows: read, account }
}
/** Fresh: a weekly window far from its reset. Soon: resets in 12 hours with 60% left, of which this week's pace (40% used over
 *  156 hours) would spend about 3%, so about 57% would go unused. */
const FRESH = windows([80, 3 * H], [90, 5 * D], 'acct-fresh'), SOON = windows([80, 3 * H], [60, 12 * H], 'acct-soon')
const codex = (machine: string, quota: Probe['quota'], patch: Partial<Probe> = {}): Probe => ({ kind: 'new', machineId: machine, machineName: machine,
  engine: 'codex', model: 'default', effort: 'default', quality: 3, cost: 2, quota, activity: 'new agent', known: true, refusals: [], notes: [], ...patch })
const opusMember = (machine: string, quota: Probe['quota']): Probe => ({ ...codex(machine, quota), kind: 'member', agentId: 'agent-1', name: 'Orion',
  engine: 'claude', model: 'opus', effort: 'high', cost: 3, activity: 'idle', frame: null })
const ok = (r: ReturnType<typeof choose>) => { if ('error' in r) throw new Error('no candidate'); return r }
const route = (probes: Probe[], plugins: RoutePlugin[], open: Commitment[] = [], task: Partial<Assessment> = {}) => ok(choose(probes, policy, a(task), NOW, open, [], [], plugins))
const pull = windowPull([])
const noteOf = (p: { notes: string[] }) => p.notes.find(n => n.startsWith(`${WINDOW_PULL}:`))
/** A candidate as the plug-in sees it, after the route's fit. */
const fitted = (p: Probe, fit: Partial<Fit> = {}): Probe & { fit: Fit } => ({ ...p, fit: { status: 'fits', coverage: 1, effectiveCost: p.cost, expiring: 0,
  spend: { '5-hour': 30, weekly: 4 }, account: p.quota.account ?? '', ...fit } })

describe('the window pull notes', () => {
  it('an account whose weekly window resets soon with quota left, and says why', () => {
    const probes = [codex('alpha', FRESH), codex('zulu', SOON)]
    expect(route(probes, []).shown.every(s => !noteOf(s))).toBe(true)
    const choice = route(probes, [pull])
    // Since task #2901 a plug-in's finding is a note, never the order.
    expect(choice.shown.map(s => s.machineId)).toEqual(['alpha', 'zulu'])
    const reason = 'Window pull: its account\'s weekly window resets in 12h, and at this week\'s pace about 57% of it would go unused; prefer this account to use it before the reset'
    expect(choice.shown[1]!.notes).toContain(reason)
    expect(choice.shown[1]!.label).toContain(reason)
    expect(noteOf(choice.shown[0]!)).toBeUndefined()
    const verdict = decide(choice)
    expect(verdict.via).toBe('instructions')
    expect(verdict.why).toContain('Plug-ins\' findings are in each candidate\'s notes.')
    expect(JSON.stringify(recordOf(choice, verdict, a()))).toContain(reason)
    // What the candidate is shown and recorded as stays as measured.
    expect(choice.shown[1]!.fit.effectiveCost).toBe(2)
  })
  it('an idle agent on such an account, beside a cheaper new agent: it could take the queued task', () => {
    const probes = [codex('alpha', FRESH), opusMember('zulu', SOON)]
    const choice = route(probes, [pull])
    // Since task #2901 a plug-in's finding is a note, never the order.
    expect(choice.shown.map(s => s.kind)).toEqual(['new', 'member'])
    expect(choice.shown[1]).toMatchObject({ kind: 'member', agentId: 'agent-1' })
    expect(noteOf(choice.shown[1]!)).toMatch(/this idle agent could take queued work to use it$/)
    expect(noteOf(choice.shown[0]!)).toBeUndefined()
  })
  it('a weaker model on such an account is noted, and the order stays the neutral one', () => {
    const sonnet = codex('zulu', SOON, { engine: 'claude', model: 'sonnet', effort: 'medium', quality: 2 })
    const choice = route([codex('alpha', FRESH), sonnet], [pull])
    // Since task #2901 a plug-in's finding is a note, never the order.
    expect(choice.shown.map(s => s.model)).toEqual(['default', 'sonnet'])
    expect(noteOf(choice.shown[1]!)).toMatch(/prefer this account to use it before the reset$/)
    expect(choice.shown[1]!.quality).toBe(2)
  })
  it('leans a choice at most half as costly, never free, and a clearly better record is said beside it (pieces 3 and 6 together)', () => {
    // At most half, however much would go unused.
    expect(windowPullAdjust(fitted(codex('zulu', SOON)), NOW, [])?.cost).toBe(-2 * MAX_DISCOUNT)
    expect(windowPullAdjust(fitted(codex('zulu', SOON), { effectiveCost: 1.2 }), NOW, [])?.cost).toBe(-0.6)
    // Piece 3's measured plug-in: Opus high passed 9 of 10 first reviews at $3 per shipped change in full, Codex 6 of 10 at $9.
    const row = (engine: string, model: string, effort: string, firstReviewPassed: number, fullCostPerShipped: number) => ({ engine, model, effort, kind: 'build',
      firstReviews: 10, firstReviewPassed, shipped: 10, costPerShipped: 1, fullCostPerShipped, fullCostKnown: 1 })
    const measured = measuredPlugin({ truncated: false, rows: [row('claude', 'opus', 'high', 9, 3), row('codex', 'default', 'default', 6, 9)] })
    const probes = [codex('alpha', FRESH, { engine: 'claude', model: 'opus', effort: 'high', cost: 3 }), codex('zulu', SOON)]
    expect(noteOf(route(probes, [pull], [], { kind: 'build' }).shown[1]!)).toBeDefined()
    for (const plugins of [[measured, pull], [pull, measured]]) {
      const choice = route(probes, plugins, [], { kind: 'build' })
      // Since task #2901 a plug-in's finding is a note, never the order.
      expect(choice.shown.map(s => s.machineId)).toEqual(['alpha', 'zulu'])
      expect(noteOf(choice.shown[1]!)).toBeDefined()
      expect(choice.shown[0]!.notes.some(n => n.startsWith(`${MEASURED}: build work`))).toBe(true)
    }
  })
  it('takes off only the part of the task the window would leave unused', () => {
    // 16% left a day before the reset, at a pace of 84% over six days: about 2% unused, half the task's 4%.
    const thin = codex('zulu', windows([80, 3 * H], [16, 24 * H], 'acct-soon'))
    expect(weeklyUnused(thin, NOW, [])?.unused).toBeCloseTo(2)
    expect(windowPullAdjust(fitted(thin), NOW, [])?.cost).toBe(-0.5)
  })
})

describe('the window pull leaves alone', () => {
  const none = (p: Probe, fit: Partial<Fit> = {}, open: Commitment[] = []) => expect(windowPullAdjust(fitted(p, fit), NOW, open)).toBeNull()
  it('a candidate whose quota would run out partway, as the route\'s fit says', () => {
    const short = codex('zulu', windows([5, 3 * H], [60, 12 * H], 'acct-soon'))
    const choice = route([codex('alpha', FRESH), short], [pull])
    expect(choice.shown.find(s => s.machineId === 'zulu')!.fit.status).toBe('short')
    expect(choice.shown.every(s => !noteOf(s))).toBe(true)
  })
  it('a candidate whose fit is not known to cover the task: a weekly window that could not be read (review of f97c022)', () => {
    // Opus with only its 5-hour window read: the fit is unknown, and half off would read it as cheaper than a Codex that fits.
    const unread = { ...opusMember('zulu', windows([90, 12 * H], null, 'acct-soon')), kind: 'new' as const, agentId: undefined, name: undefined }
    const choice = route([codex('alpha', FRESH), unread], [pull])
    expect(choice.shown.find(s => s.machineId === 'zulu')!.fit.status).toBe('unknown')
    expect(choice.shown.every(s => !noteOf(s))).toBe(true)
    none(unread, { status: 'unknown' })
  })
  it('but pulls an account that reports its weekly window alone, Codex or Claude, a complete reading that fits', () => {
    const weeklyOnly = codex('zulu', windows(null, [60, 12 * H], 'acct-soon'))
    const choice = route([codex('alpha', FRESH), weeklyOnly], [pull])
    expect(choice.shown[1]).toMatchObject({ machineId: 'zulu', fit: { status: 'fits' } })
    expect(noteOf(choice.shown[1]!)).toBeDefined()
    // A Team account's Claude, whose host relays no 5-hour window (Ken, 2026-11-03).
    const claude = { ...opusMember('zulu', windows(null, [60, 12 * H], 'acct-soon')), kind: 'new' as const, agentId: undefined, name: undefined }
    expect(route([codex('alpha', FRESH), claude], [pull]).shown.find(s => s.machineId === 'zulu')!.fit.status).toBe('fits')
  })
  it('a reading that says nothing of a weekly window about to reset', () => {
    none(codex('zulu', { known: false, leftPercent: null, resetsAt: null, account: 'acct-soon' }))
    none(codex('zulu', { known: true, leftPercent: null, resetsAt: null, metered: true }))
    none(codex('zulu', SOON, { gridModel: 'kimi-k3', gridName: 'grid' }))
    none(codex('zulu', { known: true, leftPercent: 60, resetsAt: null, account: 'acct-soon' }))
    none(codex('zulu', windows([80, 3 * H], null, 'acct-soon')))
    none(codex('zulu', windows([80, 3 * H], [60, null], 'acct-soon')))
    // A reset already past is a reading from before the window refilled.
    none(codex('zulu', windows([80, 3 * H], [60, -M], 'acct-soon')))
    none(codex('zulu', windows([80, 3 * H], [60, 0], 'acct-soon')))
    // More than two days away: this week's pace says too little yet.
    none(codex('zulu', windows([80, 3 * H], [90, 49 * H], 'acct-soon')))
  })
  it('an account whose pace this week would use what is left', () => {
    // 10% left a day before the reset, after 90% in six days.
    none(codex('zulu', windows([80, 3 * H], [10, 24 * H], 'acct-soon')))
    // Under 1% unused is no reason to move work.
    none(codex('zulu', windows([80, 3 * H], [15, 24 * H], 'acct-soon')))
  })
  it('a task that spends nothing of the week, or a candidate already free', () => {
    none(codex('zulu', SOON), { spend: { '5-hour': 0, weekly: 0 } })
    none(codex('zulu', SOON), { effectiveCost: 0 })
  })
  it('a window the picks still at work on that account already fill, but not one whose picks are over or elsewhere', () => {
    const thin = codex('zulu', windows([80, 3 * H], [16, 24 * H], 'acct-soon'))
    const held = (patch: Partial<Commitment>): Commitment => ({ account: 'acct-soon', spend: { weekly: 4 }, size: 'medium', createdAt: NOW - M, ...patch })
    none(thin, {}, [held({})])
    expect(windowPullAdjust(fitted(thin), NOW, [held({ createdAt: NOW - 2 * H })])).not.toBeNull()
    expect(windowPullAdjust(fitted(thin), NOW, [held({ account: 'acct-fresh' })])).not.toBeNull()
    expect(windowPullAdjust(fitted(thin), NOW, [held({ spend: { '5-hour': 30 } })])).not.toBeNull()
    // Through the route: its context's picks at work count.
    const choice = route([codex('alpha', FRESH), thin], [pull], [held({})])
    expect(choice.shown.every(s => !noteOf(s))).toBe(true)
  })
})

describe('the window pull\'s reason', () => {
  it('names the window, never the account', () => {
    const reading = codex('zulu', windows([80, 3 * H], [60, 40 * M], 'acct-7f3a9c'))
    const reason = windowPullAdjust(fitted(reading), NOW, [])!.reason
    expect(reason).toContain('resets in 40m')
    expect(reason).not.toMatch(/acct|7f3a9c/)
  })
  it('built from another group\'s pick, names nothing of it: only numbers and the kind of candidate', async () => {
    // Another group's pick, not started yet, on the same account, carrying all a history row can: its group, title, person, path.
    const theirs = { id: 'd-secret', conversationId: 'payroll-group', title: 'Rotate the payroll keys for Dee', account: '[email protected]',
      spend: { weekly: 20 }, size: 'large', outcome: null, final: null, createdAt: new Date(NOW - M).toISOString(),
      via: 'order', assignmentId: 'as-9', cwd: '/Users/dee/payroll', person: 'Dee' }
    const made = await makePlugins([windowPullMaker(() => NOW)], { conversationId: 'g2', assessment: a(), read: {}, catalog: policy,
      history: Promise.resolve({ ok: true, value: { decisions: [theirs] } }), request: async () => ({ ok: true }) })
    const soon = codex('zulu', windows([80, 3 * H], [60, 12 * H], '[email protected]'))
    const choice = route([codex('alpha', FRESH), soon], made.plugins)
    const reason = noteOf(choice.shown.find(s => s.machineId === 'zulu')!)!
    // Their pick is counted: about 57% less its 20% (a minute of a 3-hour task over) is about 37%.
    expect(reason).toBe('Window pull: its account\'s weekly window resets in 12h, and at this week\'s pace about 37% of it would go unused; prefer this account to use it before the reset')
    for (const leak of ['payroll', 'Payroll', 'Dee', 'dee', 'acct', 'example.com', 'd-secret', 'as-9', '/Users', 'Rotate']) expect(reason).not.toContain(leak)
  })
})

describe('picks not started yet, from the route history', () => {
  const pick = (patch: Record<string, unknown> = {}) => ({ id: 'd1', conversationId: 'other', account: 'acct-soon', spend: { '5-hour': 30, weekly: 4 }, size: 'medium',
    outcome: null, final: null, createdAt: new Date(NOW - M).toISOString(), ...patch })
  const history = (decisions: unknown) => ({ ok: true, value: { decisions } })
  it('holds quota for another group\'s pick nobody started yet, as a pick at work does', () => {
    expect(pendingPicks(history([pick()]), 'g1', NOW)).toEqual([{ account: 'acct-soon', size: 'medium', createdAt: NOW - M, spend: { weekly: 4 } }])
  })
  it('leaves out what the route already holds or what holds nothing', () => {
    expect(pendingPicks(history([
      pick({ final: { kind: 'new', engine: 'codex', model: 'default' } }), // started: the route's own picks at work hold it
      pick({ outcome: 'superseded' }), pick({ outcome: 'done' }),
      pick({ conversationId: 'g1' }), // this group's own pick: routing it again supersedes it
      pick({ account: undefined }), pick({ size: 'huge' }), pick({ spend: null }), pick({ spend: 'x' }),
      pick({ createdAt: new Date(NOW - PENDING_MS).toISOString() }), pick({ createdAt: 'never' }), pick({ createdAt: new Date(NOW + 2 * M).toISOString() }),
      null, 'x',
    ]), 'g1', NOW)).toEqual([])
  })
  it('counts a pick the server stamped a little ahead of this clock as just made, and one with no weekly spend as nothing', () => {
    expect(pendingPicks(history([pick({ createdAt: new Date(NOW + 20_000).toISOString() }), pick({ spend: { weekly: 'lots' } })]), 'g1', NOW))
      .toEqual([{ account: 'acct-soon', size: 'medium', createdAt: NOW, spend: { weekly: 4 } }, { account: 'acct-soon', size: 'medium', createdAt: NOW - M, spend: {} }])
  })
  it('is unknown when the history could not be read', () => {
    for (const unread of [null, { ok: false }, { ok: true }, { ok: true, value: 'x' }, { ok: true, value: { decisionsUnreadable: true } }]) {
      expect(pendingPicks(unread as never, 'g1', NOW)).toBeNull()
    }
  })
})

describe('the window pull\'s maker', () => {
  const input = (history: unknown): PluginInput => ({ conversationId: 'g2', assessment: a(), read: {}, catalog: policy, history: Promise.resolve(history as never),
    request: async () => ({ ok: true }) })
  it('is in the route\'s list of makers', () => {
    expect(ROUTE_PLUGIN_MAKERS.map(m => m.name)).toContain(WINDOW_PULL)
  })
  it('two routes a minute apart do not both pull onto one window', async () => {
    // About 2% would go unused, half a medium task: the first route notes the pull; the second sees that pick, not yet started.
    const thin = windows([80, 3 * H], [16, 24 * H], 'acct-soon')
    const probes = [codex('alpha', FRESH), codex('zulu', thin)]
    const first = await makePlugins([windowPullMaker(() => NOW)], input({ ok: true, value: { decisions: [] } }))
    expect(noteOf(route(probes, first.plugins).shown.find(s => s.machineId === 'zulu')!)).toBeDefined()
    const routed = { id: 'd1', conversationId: 'g1', account: 'acct-soon', spend: { weekly: 4 }, size: 'medium', outcome: null, final: null, createdAt: new Date(NOW - M).toISOString() }
    const second = await makePlugins([windowPullMaker(() => NOW)], input({ ok: true, value: { decisions: [routed] } }))
    const choice = route(probes, second.plugins)
    expect(choice.shown.every(s => !noteOf(s))).toBe(true)
  })
  it('is left out, and the route answer says so, when the history could not be read', async () => {
    expect(await makePlugins([windowPullMaker()], input(null))).toEqual({ plugins: [],
      warnings: ['The routing plug-in "Window pull" was skipped: its maker failed. The route is shown without it.'] })
  })
})