The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
import { randomUUID } from 'node:crypto'
import { execFile } from 'node:child_process'
import { promisify } from 'node:util'
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { MongoMemoryReplSet } from 'mongodb-memory-server-core'
import { PrismaClient } from '@prisma/client'
import type { AuthUser } from '../src/lib/ssoAuth.js'
import { SuperTabStore } from '../src/superTabs/store.js'
import type { Command } from '../src/superTabs/commands.js'
import type { MachineObservation } from '../src/superTabs/machines.js'
import { QUIET_CHECK_MS, runTimeCheckPass, timeCheckSwarms } from '../src/superTabs/timeChecks.js'
import { poolRows, rosterRows } from './hostRows.test-support.js'
import { plan, profiled, reads } from './plans.test-support.js'
// Task #3385 (anh Dee's Swarm redesign, 11-05): the stall, idle and lost-task follow-ups run in the backend worker, so they
// keep running while the PM is down. Before, only the PM's own poll ran them: with the PM hung, five people's asks waited 3 to
// 10 hours and the board showed nothing working. Nothing here polls as the PM. A fresh loopback replica set, created here.
let replica: MongoMemoryReplSet
let db: PrismaClient
const owner: AuthUser = { sub: '200000000000000000000001', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
const host: AuthUser = { sub: '200000000000000000000003', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
const config = { engine: 'codex', model: 'fixture-model', effort: 'high', contextWindow: 200000, engineVersion: 'fixture' }
const MINUTE = 60_000
beforeAll(async () => {
replica = await MongoMemoryReplSet.create({ binary: { version: '7.0.24', downloadDir: join(tmpdir(), 'harness-super-tab-mongodb-binaries') },
replSet: { count: 1, ip: '127.0.0.1', storageEngine: 'wiredTiger' } })
const uri = replica.getUri(`swarm_time_checks_test_${randomUUID().replaceAll('-', '_')}`)
await promisify(execFile)(join(process.cwd(), 'node_modules/.bin/prisma'), ['db', 'push', '--skip-generate'], {
env: { ...process.env, DATABASE_URL: uri }, timeout: 60000,
})
db = new PrismaClient({ datasources: { db: { url: uri } } })
for (const user of [owner, host]) await db.user.create({ data: { id: user.sub, externalId: `fixture-${user.sub}`, email: user.email, autonomousEnv: user.autonomousEnv } })
for (const machineId of ['lead-host', 'test-host']) {
await db.machine.create({ data: { machineId, userId: host.sub, apiKey: `fixture-${machineId}`, managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })
await db.machineShare.create({ data: { machineId, ownerId: host.sub, autonomousEnv: 'prod', emails: [owner.email] } })
}
})
afterAll(async () => {
await db?.$disconnect()
await replica?.stop()
})
/** A Swarm whose PM runs on one of host's machines and whose worker, on another, is assigned "Ship the parser". */
async function swarm() {
let now = Date.parse('2026-11-05T08:00:00.000Z')
const timed = new SuperTabStore(db, () => now)
const tab = (await timed.create(owner, randomUUID(), 'Time checks', 'public')).id
const execute = (command: Command, user = owner, asAgentId?: string) => timed.execute(user, tab, { operationId: randomUUID(), command }, asAgentId)
const invitation = await execute({ type: 'invite.create', email: host.email, role: 'collaborator', conversationId: 'lead' })
await execute({ type: 'invite.accept', inviteId: invitation.result.inviteId as string }, host)
for (const [machineId, name] of [['lead-host', 'Lead Mac'], ['test-host', 'Worker Mac']]) {
await execute({ type: 'machine.contribute', machineId: machineId!, name: name!, cwd: '/repo', engines: ['codex'], maxAgents: 4 }, host)
}
const generations = new Map<string, string>()
for (const machineId of ['lead-host', 'test-host']) generations.set(machineId, (await poolRows(timed, host, machineId)).find(p => p.tabId === tab)!.machine.generation)
const harness = { lead: randomUUID(), worker: randomUUID() }
// Added by the host, whose machines they run on: an agent is its own person's, on their own computer (task #4120).
const add = async (conversationId: string, machineId: string, harnessId: string, name: string, lead = false) => String((await execute({ type: 'agent.add',
conversationId, machineId, harnessId, name, configuration: config, lead }, host)).result.agentId)
const lead = await add('lead', 'lead-host', harness.lead, 'Lead', true)
const group = String((await execute({ type: 'conversation.create', title: 'Ship the parser', brief: 'Implement it',
checks: [{ description: 'Verified', independent: false }] })).result.conversationId)
const worker = await add(group, 'test-host', harness.worker, 'Worker')
const assignmentId = String((await execute({ type: 'work.assign', conversationId: group, agentId: worker })).result.assignmentId)
const observe = (machineId: string, running: string[], extra: Partial<MachineObservation> = {}) => timed.observeMachine(host, tab, machineId, {
generation: generations.get(machineId)!, resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9,
diskTotalBytes: 500e9, projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },
quotas: [], tools: {}, runningHarnessIds: running, stoppedHarnessIds: [], ...extra })
/** Only the worker's machine reports, its screen changing: the PM's host is down and never polls. */
const workerBeat = () => observe('test-host', [harness.worker], { harnessProgress: { [harness.worker]: now } })
const rows = async (kind: string) => (await db.superTabMessage.findMany({ where: { tabId: tab }, orderBy: { sequence: 'asc' } }))
.filter(m => (m.source as { kind?: string } | null)?.kind === kind)
return { timed, tab, execute, lead, group, worker, assignmentId, harness, observe, workerBeat, rows,
clock: { get now() { return now }, set now(value: number) { now = value } } }
}
describe('the time checks with the PM down', () => {
it('tell the PM of a task that went quiet, once, with no PM poll', async () => {
const f = await swarm()
const posted = f.clock.now
await f.execute({ type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId, status: 'working', result: 'Started' }, host, f.worker)
for (const minutes of [10, 20, 29]) {
f.clock.now = posted + minutes * MINUTE
await f.workerBeat()
await f.timed.runTimeChecks(f.tab)
}
expect(await f.rows('work-stalled')).toEqual([])
f.clock.now = posted + 30 * MINUTE
await f.workerBeat()
expect(await f.timed.runTimeChecks(f.tab)).toBeGreaterThan(0)
const stalled = await f.rows('work-stalled')
expect(stalled).toEqual([expect.objectContaining({ conversationId: 'lead', recipientIds: [f.lead] })])
expect(stalled[0]!.text).toContain('"Ship the parser": quiet, ')
for (const minutes of [31, 45]) {
f.clock.now = posted + minutes * MINUTE
await f.workerBeat()
await f.timed.runTimeChecks(f.tab)
}
expect(await f.rows('work-stalled')).toHaveLength(1)
})
it('tell the PM of an agent left idle for 15 minutes, once, with no PM poll', async () => {
const f = await swarm()
await f.execute({ type: 'work.cancel', conversationId: f.group, assignmentId: f.assignmentId, reason: 'Done elsewhere' })
// Its host took the cancel notice, as a running host does.
await db.superTabDelivery.updateMany({ where: { tabId: f.tab, agentId: f.worker, status: 'queued' }, data: { status: 'delivered' } })
f.clock.now += 14 * MINUTE
await f.workerBeat()
await f.timed.runTimeChecks(f.tab)
expect(await f.rows('idle')).toEqual([])
f.clock.now += MINUTE + 1_000
await f.workerBeat()
await f.timed.runTimeChecks(f.tab)
expect(await f.rows('idle')).toEqual([expect.objectContaining({ conversationId: 'lead', recipientIds: [f.lead], text: 'Worker has had no open task for 15 minutes.' })])
f.clock.now += 10 * MINUTE
await f.workerBeat()
await f.timed.runTimeChecks(f.tab)
expect(await f.rows('idle')).toHaveLength(1)
})
it('lose the task of a worker whose machine went silent, and tell the PM, with no report and no PM poll', async () => {
const f = await swarm()
await f.workerBeat()
f.clock.now += 2 * MINUTE + 5_000
// No machine reports from here on: the worker's machine died, and the PM's host is down too.
await f.timed.runTimeChecks(f.tab)
expect((await f.timed.read(owner, f.tab)).state.conversations.find(c => c.id === f.group)!.assignment!.status).toBe('lost')
expect((await f.rows('work-lost')).map(r => r.text)).toContainEqual(expect.stringContaining('Worker lost its task "Ship the parser"'))
})
// Task #3915: a task whose agent went down and nobody took tells its PM and its people within the checks' 5 minutes, once,
// in a line that tags them (their Needs you), and the tag resolves when someone takes it.
it('flag a task nobody took 10 minutes after its agent went down, once, tagging its requester, and clear it when taken', async () => {
const f = await swarm()
await f.workerBeat()
f.clock.now += 2 * MINUTE + 5_000
await f.timed.runTimeChecks(f.tab)
const stuck = async () => (await db.superTabMessage.findMany({ where: { tabId: f.tab, conversationId: f.group }, orderBy: { sequence: 'asc' } }))
.filter(m => /stuck/i.test(m.text))
expect(await stuck()).toEqual([])
f.clock.now += 10 * MINUTE
await f.timed.runTimeChecks(f.tab)
f.clock.now += 5 * MINUTE
await f.timed.runTimeChecks(f.tab)
const told = await stuck()
expect(told).toEqual([expect.objectContaining({ recipientIds: [f.lead], text: expect.stringContaining('Worker went down') })])
const tag = () => db.superTabMention.findMany({ where: { tabId: f.tab, messageId: told[0]!.id } })
expect(await tag()).toEqual([expect.objectContaining({ personId: owner.sub, conversationId: f.group, resolvedAt: null })])
const taker = String((await f.execute({ type: 'agent.add', conversationId: f.group, machineId: 'lead-host', harnessId: randomUUID(), name: 'Taker',
configuration: config, lead: false }, host)).result.agentId)
await f.execute({ type: 'work.assign', conversationId: f.group, agentId: taker })
f.clock.now += MINUTE
// Its tag resolves on the `[tabId, conversationId, personId]` index, examining only that person's tags there (backend AGENTS.md 3).
const entries = await profiled(db, () => f.timed.runTimeChecks(f.tab))
expect((await stuck()).map(m => m.text)).toEqual([told[0]!.text, 'No longer stuck: this task has an agent again.'])
expect(await tag()).toEqual([expect.objectContaining({ resolvedBy: 'fixed' })])
const resolve = reads(entries, 'SuperTabMention', ['messageId', 'personId'])
expect(resolve).toHaveLength(1)
expect(plan(resolve)).toEqual([expect.objectContaining({ plan: expect.stringMatching(/IXSCAN.*conversationId/), docsExamined: 1 })])
})
it('are no longer run by the PM\'s poll', async () => {
const f = await swarm()
await f.execute({ type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId, status: 'working', result: 'Started' }, host, f.worker)
f.clock.now += 31 * MINUTE
await f.workerBeat()
await f.timed.deliveries(host, f.tab, f.lead)
expect(await f.rows('work-stalled')).toEqual([])
expect((await f.timed.read(owner, f.tab)).state.conversations.find(c => c.id === f.group)!.liveness).toBeUndefined()
})
})
describe('the worker\'s pass', () => {
it('runs the checks of each Swarm active in the last week, and leaves a deleted one alone', async () => {
const f = await swarm(), quiet = await swarm()
await f.execute({ type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId, status: 'working', result: 'Started' }, host, f.worker)
f.clock.now += 31 * MINUTE
await f.workerBeat()
expect(await timeCheckSwarms(db, f.clock.now)).toEqual(expect.arrayContaining([f.tab, quiet.tab]))
await db.superTab.update({ where: { id: quiet.tab }, data: { deletedAt: new Date(f.clock.now) } })
const pass = await runTimeCheckPass(db, f.timed, f.clock.now)
expect(pass.failed).toBe(0)
expect(await f.rows('work-stalled')).toHaveLength(1)
expect(await quiet.rows('work-stalled')).toEqual([])
// A week later nothing there is read again.
expect(await timeCheckSwarms(db, f.clock.now + 8 * 24 * 60 * MINUTE)).not.toContain(f.tab)
})
})
describe('what the PM owes', () => {
const owedOf = async (f: Awaited<ReturnType<typeof swarm>>) => {
const head = await db.swarmHead.findFirstOrThrow({ where: { tabId: f.tab } })
return ((head.data as { projects?: Array<{ owed?: unknown }> }).projects ?? [])[0]?.owed
}
const pmRow = async (f: Awaited<ReturnType<typeof swarm>>, agentId = f.lead) =>
(await rosterRows(f.timed, host, agentId === f.lead ? 'lead-host' : 'test-host')).find(r => r.tabId === f.tab && 'agent' in r && r.agent.id === agentId) as Record<string, unknown>
it('is the oldest message a person sent it that it has not answered, until it answers, and only its own host row carries it', async () => {
const f = await swarm()
// An agent's message to the PM is never an ask it owes a person.
await f.execute({ type: 'message.send', conversationId: f.group, text: 'Halfway there.', replyTo: null, recipientIds: [f.lead] } as Command, host, f.worker)
await f.timed.runTimeChecks(f.tab)
expect(await owedOf(f)).toBeUndefined()
const sent = await f.execute({ type: 'message.send', conversationId: 'lead', text: 'Where is my parser?', replyTo: null, recipientIds: [] } as Command)
f.clock.now += MINUTE
const next = await f.execute({ type: 'message.send', conversationId: 'lead', text: 'And the docs?', replyTo: null, recipientIds: [] } as Command)
f.clock.now += MINUTE
await f.timed.runTimeChecks(f.tab)
const messageId = String(sent.result.messageId)
// When its delivery to the PM was written, as the server's clock read then.
const sentAt = async (id: unknown) => (await db.superTabDelivery.findFirstOrThrow({ where: { tabId: f.tab, messageId: String(id), agentId: f.lead } })).createdAt.getTime()
const asked = await sentAt(messageId)
expect(await owedOf(f)).toEqual({ messageId, since: asked })
expect((await pmRow(f)).owed).toEqual({ messageId, since: asked, restartAfterMs: 20 * MINUTE })
expect((await pmRow(f, f.worker)).owed).toBeNull()
// Readers never see it: it may name a message in a person's private chat with the PM.
expect((await f.timed.read(owner, f.tab)).state.projects![0]).not.toHaveProperty('owed')
await f.execute({ type: 'message.send', conversationId: 'lead', text: 'Parser ships today.', replyTo: null, recipientIds: [], answers: messageId } as Command, host, f.lead)
await f.timed.runTimeChecks(f.tab)
expect(await owedOf(f)).toEqual({ messageId: String(next.result.messageId), since: await sentAt(next.result.messageId) })
})
})
// Reviews of #3213 (Wren, Important 1; SecAlder, Minor 3): the instructions tell the PM to react instead of acknowledging, so
// a message it reacted to was owed for a day and every long PM turn read as hung.
describe('what the PM owes, once it reacted', () => {
it('is not a message the PM reacted to', async () => {
const f = await swarm()
const sent = await f.execute({ type: 'message.send', conversationId: 'lead', text: 'Thanks for the fix!', replyTo: null, recipientIds: [] } as Command)
const messageId = String(sent.result.messageId)
await f.timed.runTimeChecks(f.tab)
const head = async () => ((await db.swarmHead.findFirstOrThrow({ where: { tabId: f.tab } })).data as { projects?: Array<{ owed?: { messageId: string } }> }).projects![0]!.owed
expect((await head())?.messageId).toBe(messageId)
await f.execute({ type: 'message.react', conversationId: 'lead', messageId, emoji: 'check', on: true } as Command, host, f.lead)
expect((await db.superTabDelivery.findFirstOrThrow({ where: { tabId: f.tab, messageId, agentId: f.lead } })).status).toBe('answered')
await f.timed.runTimeChecks(f.tab)
expect(await head()).toBeUndefined()
})
})
// Review of #3213 (Wren, Important 2; AGENTS.md standard 4: a timer that wakes while nothing changed is a defect): a Swarm whose
// groups show no new activity is checked every few minutes, not every 30 s.
describe('the worker\'s pass on a quiet Swarm', () => {
it('checks a Swarm again at once when its activity moved, and otherwise only every QUIET_CHECK_MS', async () => {
const f = await swarm()
const calls: string[] = []
const counting = { runTimeChecks: async (tabId: string) => { if (tabId === f.tab) calls.push(tabId); return 0 } }
const seen = new Map()
await runTimeCheckPass(db, counting, f.clock.now, undefined, seen)
await runTimeCheckPass(db, counting, f.clock.now + 30_000, undefined, seen)
expect(calls).toHaveLength(1)
f.clock.now += 60_000
await f.execute({ type: 'message.send', conversationId: 'lead', text: 'Any news?', replyTo: null, recipientIds: [] } as Command)
await runTimeCheckPass(db, counting, f.clock.now, undefined, seen)
expect(calls).toHaveLength(2)
await runTimeCheckPass(db, counting, f.clock.now + QUIET_CHECK_MS - 1, undefined, seen)
expect(calls).toHaveLength(2)
await runTimeCheckPass(db, counting, f.clock.now + QUIET_CHECK_MS, undefined, seen)
expect(calls).toHaveLength(3)
})
})