The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Tell the Swarm Lead since when each worker has been down (#1601)
0xken15e2f1223a9dd7
diff --git a/backend/integration/superTabs.test.ts b/backend/integration/superTabs.test.tsindex ed39ac2b6..8b0f9f7e1 100644--- a/backend/integration/superTabs.test.ts+++ b/backend/integration/superTabs.test.ts@@ -600,6 +600,143 @@ describe('private swarm projects on the real control API', () => {})})+describe('since when a worker is down', () => {+ async function downFixture({ legacy = false } = {}) {+ let now = 1_000_000+ const timed = new SuperTabStore(db, () => now)+ const tab = (await timed.create(owner, randomUUID(), 'Down since', legacy ? undefined : 'public')).id+ const execute = (command: Command, user = owner, asAgentId?: string) =>+ timed.execute(user, tab, { operationId: randomUUID(), command }, asAgentId)+ const invitation = await execute({ type: 'invite.create', email: host.email, role: 'collaborator', conversationId: 'lead' })+ await execute({ type: 'invite.accept', inviteId: invitation.result.inviteId as string }, host)+ await execute({ type: 'machine.contribute', machineId: 'test-host', name: 'Host', cwd: '/repo', engines: ['codex'], maxAgents: 4 }, host)+ const leadHarness = randomUUID(), workerHarness = randomUUID()+ const lead = String((await execute({ type: 'agent.add', conversationId: 'lead', machineId: 'test-host', harnessId: leadHarness,+ name: 'Lead', configuration: config, lead: true })).result.agentId)+ const group = String((await execute({ type: 'conversation.create', title: 'Ship the parser', brief: 'Implement it',+ checks: [{ description: 'Verified', independent: false }] })).result.conversationId)+ const worker = String((await execute({ type: 'agent.add', conversationId: group, machineId: 'test-host', harnessId: workerHarness,+ name: 'Worker', configuration: config, lead: false })).result.agentId)+ const assignmentId = String((await execute({ type: 'work.assign', conversationId: group, agentId: worker })).result.assignmentId)+ const generation = (await timed.hostPools(host, 'test-host')).find(p => p.tabId === tab)!.machine.generation+ const observe = (running: string[], stopped: string[] = []) => timed.observeMachine(host, tab, 'test-host', { generation,+ resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9, diskTotalBytes: 500e9,+ projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },+ quotas: [], tools: {}, runningHarnessIds: running, stoppedHarnessIds: stopped })+ // What the Lead reads with `status`.+ const downSince = async (agentId: string) => (await timed.read(host, tab, lead)).state.agents.find(a => a.id === agentId)!.downSince+ return { timed, tab, execute, lead, group, worker, assignmentId, leadHarness, workerHarness, observe, downSince,+ clock: { get now() { return now }, set now(value: number) { now = value } } }+ }+ const iso = (ms: number) => new Date(ms).toISOString()++ it('is set by a gone session, a confirmed stop and an offline machine, and cleared when the worker is back', async () => {+ const f = await downFixture()+ await f.observe([f.leadHarness, f.workerHarness])+ expect(await f.downSince(f.worker)).toBeUndefined()++ // The host no longer lists the session as running.+ f.clock.now += 1_000+ const gone = f.clock.now+ await f.observe([f.leadHarness])+ f.clock.now += 1_000+ await f.observe([f.leadHarness])+ expect(await f.downSince(f.worker)).toBe(iso(gone))+ expect(await f.downSince(f.lead)).toBeUndefined()+ await f.observe([f.leadHarness, f.workerHarness])+ expect(await f.downSince(f.worker)).toBeUndefined()++ // The host confirms the session stopped, before its next reading.+ f.clock.now += 1_000+ const stopped = f.clock.now+ await f.timed.stopped(host, f.tab, f.assignmentId, 'fixture-native-stop')+ expect(await f.downSince(f.worker)).toBe(iso(stopped))+ f.clock.now += 1_000+ await f.observe([f.leadHarness], [f.workerHarness])+ expect(await f.downSince(f.worker)).toBe(iso(stopped))+ await f.observe([f.leadHarness, f.workerHarness])+ expect(await f.downSince(f.worker)).toBeUndefined()++ // The machine stops sending readings: down since its last one, kept when it returns without the session.+ f.clock.now += 1_000+ const lastReading = f.clock.now+ await f.observe([f.leadHarness, f.workerHarness])+ f.clock.now += 45_000+ expect(await f.downSince(f.worker)).toBeUndefined()+ f.clock.now += 1+ expect(await f.downSince(f.worker)).toBe(iso(lastReading))+ expect(await f.downSince(f.lead)).toBe(iso(lastReading))+ f.clock.now += 60_000+ await f.observe([f.leadHarness])+ expect(await f.downSince(f.worker)).toBe(iso(lastReading))+ expect(await f.downSince(f.lead)).toBeUndefined()+ })++ it('names a worker down two hours or more in the Lead\'s scheduled wake-up, once per time it went down', async () => {+ const f = await downFixture()+ const wakeUps = async () => (await db.superTabMessage.findMany({ where: { tabId: f.tab, conversationId: 'lead' }, orderBy: { sequence: 'asc' } }))+ .filter(m => (m.source as Record<string, unknown>)?.kind === 'update-request')+ const poll = async () => (await f.timed.deliveries(host, f.tab, f.lead)).filter(d => d.conversationId === 'lead')+ // The Lead acknowledges through the revision its wake-up names.+ const acknowledge = async () => f.execute({ type: 'project.report', projectId: 'lead',+ throughRevision: ((await wakeUps()).at(-1)!.source as { throughRevision: number }).throughRevision, text: null }, host, f.lead)+ await f.observe([f.leadHarness, f.workerHarness])+ f.clock.now += 300_000+ expect(await poll()).toHaveLength(1)+ await acknowledge()+ expect(await poll()).toEqual([])++ const down = f.clock.now+ await f.observe([f.leadHarness, f.workerHarness])+ await f.observe([f.leadHarness])+ f.clock.now = down + 2 * 3_600_000 - 60_000+ await f.execute({ ...message('A new report'), conversationId: f.group } as Command)+ f.clock.now += 59_999+ expect(await poll()).toEqual([])+ f.clock.now+++ expect(await poll()).toHaveLength(1)+ const named = (await wakeUps()).at(-1)!+ expect(named.text).toContain(`Worker (down since ${iso(down)}; task "Ship the parser")`)+ expect(named.text).not.toContain('Lead (down')+ expect(named.text).not.toContain('Consolidate')+ expect(await poll()).toHaveLength(1)+ await acknowledge()+ // The report that was not yet due still gets its own request.+ f.clock.now += 240_000+ expect(await poll()).toHaveLength(1)+ expect((await wakeUps()).at(-1)!.text).toContain('Consolidate new reports')+ await acknowledge()+ f.clock.now += 3 * 3_600_000+ expect(await poll()).toEqual([])++ await f.observe([f.leadHarness, f.workerHarness])+ const again = f.clock.now+ await f.observe([f.leadHarness])+ f.clock.now += 2 * 3_600_000+ expect(await poll()).toHaveLength(1)+ expect((await wakeUps()).at(-1)!.text).toContain(`Worker (down since ${iso(again)}`)+ expect(await wakeUps()).toHaveLength(4)+ })++ it('wakes the Lead of an older Swarm without stored projects', async () => {+ const f = await downFixture({ legacy: true })+ expect((await f.timed.read(owner, f.tab)).state.projects).toBeUndefined()+ const poll = async () => (await f.timed.deliveries(host, f.tab, f.lead)).filter(d => d.conversationId === 'lead')+ const down = f.clock.now+ await f.observe([f.leadHarness, f.workerHarness])+ await f.observe([f.leadHarness])+ f.clock.now = down + 2 * 3_600_000 - 1+ expect(await poll()).toEqual([])+ f.clock.now+++ const polls = [...await poll(), ...await poll()]+ expect(new Set(polls.map(d => d.id)).size).toBe(1)+ const wakeUps = (await db.superTabMessage.findMany({ where: { tabId: f.tab, conversationId: 'lead' } }))+ .filter(m => (m.source as Record<string, unknown>)?.kind === 'update-request')+ expect(wakeUps).toHaveLength(1)+ expect(wakeUps[0]!.text).toContain(`Worker (down since ${iso(down)}; task "Ship the parser")`)+ })+})+describe('Super Tabs on a real Mongo replica set', () => {it('contributes project capacity without sharing the computer and fences host execution', async () => {const tab = (await store.create(owner, randomUUID(), 'Machine pool')).iddiff --git a/backend/src/superTabs/machines.ts b/backend/src/superTabs/machines.tsindex 57fb48e2d..ae1e93be9 100644--- a/backend/src/superTabs/machines.ts+++ b/backend/src/superTabs/machines.ts@@ -1,5 +1,5 @@import { z } from 'zod'-import { fail, person, type AgentStart, type AgentStartRequest, type SuperTabState } from './model.js'+import { fail, person, type Agent, type AgentStart, type AgentStartRequest, type SuperTabState } from './model.js'import { groupProject } from './projects.js'import { conversation } from './model.js'@@ -39,6 +39,29 @@ export const machineObservationSchema = z.object({ generation: z.string().max(16}).strict()export type MachineObservation = z.infer<typeof machineObservationSchema>+/** A machine reports every 15 seconds (cli/src/superTabs/machines.ts); three missed beats is offline. */+export const MACHINE_FRESH_MS = 45_000++/** Since when an agent has been down: the stored time a reading first showed its session gone or the host confirmed its+ * stop, or its machine's last reading once that is stale. Nothing is written while a machine is silent, so that case is+ * read from the reading itself. Undefined while it is up. */+export function downSince(state: SuperTabState, target: Agent, now: number): string | undefined {+ const observation = state.machines?.find(m => m.machineId === target.machineId)?.observation+ const offline = observation && now - observation.receivedAt > MACHINE_FRESH_MS ? new Date(observation.receivedAt).toISOString() : undefined+ return [target.downSince, offline].filter(Boolean).sort()[0]+}++/** Before a new reading replaces the last one: a session it lists as running is back; any other agent on the machine+ * keeps the earliest time it was down, including the time its machine went silent. */+export function recordAgentPresence(state: SuperTabState, machineId: string, running: string[], now: number): void {+ for (const target of state.agents.filter(a => a.machineId === machineId)) {+ if (running.includes(target.harnessId)) {+ delete target.downSince+ delete target.downNamed+ } else target.downSince = downSince(state, target, now) ?? new Date(now).toISOString()+ }+}+export function machineContribution(state: SuperTabState, machineId: string) {const pool = state.machines?.find(m => m.machineId === machineId)if (!pool) return null@@ -95,7 +118,7 @@ export function admitMachineStart(state: SuperTabState, request: AgentStartRequeif (refused.length) {fail('OUTSIDE_CONTRIBUTION', `${pool.name} accepts only its contributed project folder, engines and default account settings, with full permissions for Swarm agents. Refused: ${refused.join('; ')}.`, 403)}- if (!pool.observation || now - pool.observation.receivedAt > 45_000) {+ if (!pool.observation || now - pool.observation.receivedAt > MACHINE_FRESH_MS) {fail('MACHINE_OFFLINE', 'Wait for a fresh reading from the contributed machine.')}if (!start && machineSlots(state, pool) >= pool.maxAgents) fail('MACHINE_FULL', 'This machine has reached its contributed agent limit.')diff --git a/backend/src/superTabs/model.ts b/backend/src/superTabs/model.tsindex 2d536680a..012d1cab6 100644--- a/backend/src/superTabs/model.ts+++ b/backend/src/superTabs/model.ts@@ -1,6 +1,7 @@import { createHash } from 'node:crypto'import type { RouteDecisionEffect } from './routeRecord.js'import { agentDisplayName, assignAgentDisplayNames } from './agentNames.js'+import { downSince } from './machines.js'import { canReadAgent, canReadProject, groupProject, placeRules, projectsOf, type ProjectVisibility, type SwarmProject } from './projects.js'export type Role = 'owner' | 'collaborator' | 'viewer'@@ -36,6 +37,10 @@ export interface Agent {configurationVersion?: string/** Fences a repeated Change agent report without rolling back later model observations. */replacement?: { previousHarnessId: string; harnessId: string }+ /** When a reading first showed its session gone or its host confirmed a stop; cleared when it is back (machines.ts). */+ downSince?: string+ /** The down time already named in the Lead's scheduled wake-up (projectUpdates.ts), so each time down is named once. */+ downNamed?: string}export type AgentRole = 'builder' | 'reviewer'export const agentRole = (agent: Pick<Agent, 'role'>): AgentRole => agent.role ?? 'builder'@@ -271,10 +276,15 @@ export function createSuperTab(id: string, name: string, owner: { id: string; em}/** Pending invitations are management data, not part of the shared conversation. */-export function visibleState(state: SuperTabState, actor: Actor): SuperTabState {+export function visibleState(state: SuperTabState, actor: Actor, now = Date.now()): SuperTabState {author(state, actor)const visible = structuredClone(state)assignAgentDisplayNames(visible.agents)+ for (const target of visible.agents) {+ const since = downSince(state, target, now)+ if (since) target.downSince = since+ delete target.downNamed+ }delete visible.sharingdelete visible.harnessReservationsdelete visible.releasedHarnessesdiff --git a/backend/src/superTabs/projectUpdates.ts b/backend/src/superTabs/projectUpdates.tsindex 5057a2040..ceea06fb3 100644--- a/backend/src/superTabs/projectUpdates.ts+++ b/backend/src/superTabs/projectUpdates.ts@@ -1,4 +1,5 @@-import { author, conversation, stableId, type Agent, type MessageDraft, type SuperTabState } from './model.js'+import { activeWork, author, conversation, stableId, type Agent, type MessageDraft, type SuperTabState } from './model.js'+import { downSince } from './machines.js'import { agentProjectId, groupProject, projectOf } from './projects.js'/** Workgroup reports mark a project dirty. No per-message model or timer is@@ -14,25 +15,50 @@ export function noteProjectReports(state: SuperTabState, messages: MessageDraft[}}+/** The Lead reassigns a Builder or Reviewer that has been down this long (the owner's rule, 2026-10-23). */+export const DOWN_REASSIGN_MS = 2 * 60 * 60_000++/** Builders and Reviewers of the project down for two hours, each named once per time it went down. A worker that is+ * already stopping was decided on. */+function downWorkers(state: SuperTabState, projectId: string, now: number) {+ return state.agents.flatMap(a => {+ const since = downSince(state, a, now)+ if (!since || a.downNamed === since || now - Date.parse(since) < DOWN_REASSIGN_MS || a.workDecision === 'stop' ||+ agentProjectId(state, a) !== projectId || state.conversations.some(c => c.leadId === a.id)) return []+ const task = state.conversations.find(c => activeWork(c) && c.assignment!.agentId === a.id)+ return [{ agentId: a.id, since, text: `${a.displayName ?? a.name} (down since ${since}${task ? `; task ${JSON.stringify(task.title)}` : ''})` }]+ })+}+export function projectUpdateRequest(state: SuperTabState, target: Agent, now: number): MessageDraft | null {- if (!state.projects || state.paused) return null+ // An older Swarm without stored projects is one synthesized project. It keeps no report progress (noteProjectReports),+ // so its Lead is woken only for down workers.+ if (state.paused) return nullconst project = projectOf(state, agentProjectId(state, target))const group = conversation(state, project.leadConversationId), progress = project.progress- if (group.leadId !== target.id || group.archived || !progress ||- progress.dirtyRevision <= progress.summarizedRevision ||- progress.dirtyRevision <= progress.requestedRevision ||- now < Math.max(progress.requestedAt ?? 0, progress.since) + project.updateMinutes * 60_000) return null- const id = stableId('project-update', state.id, project.id, String(progress.dirtyRevision))- const text = `Consolidate new reports for project ${JSON.stringify(project.name)}. ` +- `Read this project's workgroups with lastMessageSequence after ${progress.summarizedRevision}; inspect their new messages and current work. ` +- 'At your next natural checkpoint, tell the people what changed, what is blocked, and what comes next. Include useful workgroup references and result links. ' ++ if (group.leadId !== target.id || group.archived) return null+ const reports = !!progress && progress.dirtyRevision > progress.summarizedRevision &&+ progress.dirtyRevision > progress.requestedRevision &&+ now >= Math.max(progress.requestedAt ?? 0, progress.since) + project.updateMinutes * 60_000+ const down = downWorkers(state, project.id, now)+ if (!reports && !down.length) return null+ // A wake-up only for down workers is acknowledged through what was already summarized, so reports that are not yet due+ // keep their own request.+ const through = (reports ? progress!.dirtyRevision : progress?.summarizedRevision) ?? 0+ const id = stableId('project-update', state.id, project.id, String(through), ...down.map(d => `${d.agentId}@${d.since}`))+ const text = (reports ? `Consolidate new reports for project ${JSON.stringify(project.name)}. ` ++ `Read this project's workgroups with lastMessageSequence after ${progress!.summarizedRevision}; inspect their new messages and current work. ` ++ 'At your next natural checkpoint, tell the people what changed, what is blocked, and what comes next. Include useful workgroup references and result links. ' : '') ++ (down.length ? `Down for 2 hours or more in project ${JSON.stringify(project.name)}: ${down.map(d => d.text).join(', ')}. ` ++ 'Their machine is offline or their session ended or stopped. Decide whether to reassign their work to another agent. ' : '') +'Do not copy the terminal log or repeat unchanged status. This is a scheduled request, not a new task. ' +`Use harness super-tab --tab ${state.id} command UNIQUE_OPERATION_ID ` +- `JSON with {"type":"project.report","projectId":"${project.id}","throughRevision":${progress.dirtyRevision},"text":"your update"}. ` ++ `JSON with {"type":"project.report","projectId":"${project.id}","throughRevision":${through},"text":"your update"}. ` +'Use text:null to acknowledge if nothing merits an update. Reuse the same operation ID after a lost reply. ' +'Keep reporting human questions, serious blockers, and completion immediately; do not wait for this cadence.'return { id, conversationId: group.id,author: author(state, { kind: 'agent', agentId: target.id, machineOwnerId: target.machineOwnerId }),text, replyTo: null, recipients: [target.id],- source: { kind: 'update-request', projectId: project.id, throughRevision: progress.dirtyRevision } }+ source: { kind: 'update-request', projectId: project.id, throughRevision: through,+ ...(down.length ? { down: down.map(({ agentId, since }) => ({ agentId, since })) } : {}) } }}diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.tsindex 29c8a79f5..7c72844b3 100644--- a/backend/src/superTabs/store.ts+++ b/backend/src/superTabs/store.ts@@ -1,4 +1,4 @@-import { machineContribution, machineObservationSchema, machineSlots, type MachineObservation } from './machines.js'+import { machineContribution, machineObservationSchema, machineSlots, recordAgentPresence, type MachineObservation } from './machines.js'import { createHash } from 'node:crypto'import type { Prisma, PrismaClient, SuperTab, SuperTabDelivery } from '@prisma/client'import type { AuthUser } from '../lib/ssoAuth.js'@@ -282,7 +282,7 @@ export class SuperTabStore {const row = await this.row(tx, user, tabId)const state = stateOf(row)const actor = await this.actor(tx, user, state, asAgentId)- const visible = visibleState(state, actor)+ const visible = visibleState(state, actor, this.now())return { revision: row.revision, state: { ...visible, machines: visible.machines?.map(pool => ({...pool, availableSlots: Math.max(0, pool.maxAgents - machineSlots(state, state.machines!.find(m => m.machineId === pool.machineId)!)),})) } }@@ -769,6 +769,7 @@ export class SuperTabStore {if (!pool || pool.contributorId !== user.sub || pool.generation !== observation.generation) fail('CONTRIBUTION_REVOKED', 'Read the current machine contribution.', 403)const owned = await tx.machine.findFirst({ where: { machineId, userId: user.sub, autonomousEnv: user.autonomousEnv, ...machineAlive } })if (!owned || !machineBillingAllowsDataPlane(owned)) fail('HOST_UNAVAILABLE', 'This execution host is unavailable.', 403)+ recordAgentPresence(state, machineId, observation.runningHarnessIds, this.now())pool.observation = { ...observation, quotas: observation.quotas.filter(q => pool.engines.includes(q.engine)), receivedAt: this.now() }// Current-generation physical stop proof also finishes a close whose membership was already withdrawn.for (const proof of observation.stoppedReservations ?? []) {@@ -883,15 +884,21 @@ export class SuperTabStore {const update = projectUpdateRequest(state, target, this.now())if (update) {const project = groupProject(state, conversation(state, update.conversationId))- const progress = project.progress!+ // A worker down two hours asks for a wake-up even when no report has come in yet.+ const progress = project.progress ??= { dirtyRevision: 0, summarizedRevision: 0, requestedRevision: 0, since: this.now() }const pending = progress.requestMessageId? await tx.superTabDelivery.findFirst({ where: { tabId, agentId, messageId: progress.requestMessageId,status: { in: ['queued', 'claimed', 'uncertain', 'delivered'] } } }): nullif (!pending) {- progress.requestedRevision = progress.dirtyRevision- progress.requestedAt = this.now()+ if (update.source!.throughRevision === progress.dirtyRevision) {+ progress.requestedRevision = progress.dirtyRevision+ progress.requestedAt = this.now()+ }progress.requestMessageId = update.id+ for (const { agentId, since } of (update.source!.down ?? []) as Array<{ agentId: string; since: string }>) {+ state.agents.find(a => a.id === agentId)!.downNamed = since+ }const revision = row.revision + 1await this.writeMessages(tx, tabId, state, [update], revision)await tx.superTab.update({ where: { id: tabId, revision: row.revision }, data: { revision, state: json(state) } })@@ -1019,6 +1026,7 @@ export class SuperTabStore {// on purpose by someone who already knows.const worker = state.agents.find(a => a.id === assignment.agentId)if (worker) {+ worker.downSince ??= new Date(this.now()).toISOString()const lead = group.leadId && group.leadId !== worker.id && group.agentIds.includes(group.leadId) ? group.leadId : nullnotices.push({ id: stableId('stop-notice', tabId, assignmentId), conversationId: group.id,author: author(state, { kind: 'agent', agentId: worker.id, machineOwnerId: worker.machineOwnerId }),