Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Tell the Swarm Lead since when each worker has been down (#1601)

0xken15e2f1223a9dd7
diff --git a/backend/integration/superTabs.test.ts b/backend/integration/superTabs.test.ts
index ed39ac2b6..8b0f9f7e1 100644
--- a/backend/integration/superTabs.test.ts
+++ b/backend/integration/superTabs.test.ts
@@ -600,6 +600,143 @@ describe('private swarm projects on the real control API', () => {
})
})
+describe('since when a worker is down', () => {
+ async function downFixture({ legacy = false } = {}) {
+ let now = 1_000_000
+ const timed = new SuperTabStore(db, () => now)
+ const tab = (await timed.create(owner, randomUUID(), 'Down since', legacy ? undefined : 'public')).id
+ const execute = (command: Command, user = owner, asAgentId?: string) =>
+ timed.execute(user, tab, { operationId: randomUUID(), command }, asAgentId)
+ const invitation = await execute({ type: 'invite.create', email: host.email, role: 'collaborator', conversationId: 'lead' })
+ await execute({ type: 'invite.accept', inviteId: invitation.result.inviteId as string }, host)
+ await execute({ type: 'machine.contribute', machineId: 'test-host', name: 'Host', cwd: '/repo', engines: ['codex'], maxAgents: 4 }, host)
+ const leadHarness = randomUUID(), workerHarness = randomUUID()
+ const lead = String((await execute({ type: 'agent.add', conversationId: 'lead', machineId: 'test-host', harnessId: leadHarness,
+ name: 'Lead', configuration: config, lead: true })).result.agentId)
+ const group = String((await execute({ type: 'conversation.create', title: 'Ship the parser', brief: 'Implement it',
+ checks: [{ description: 'Verified', independent: false }] })).result.conversationId)
+ const worker = String((await execute({ type: 'agent.add', conversationId: group, machineId: 'test-host', harnessId: workerHarness,
+ name: 'Worker', configuration: config, lead: false })).result.agentId)
+ const assignmentId = String((await execute({ type: 'work.assign', conversationId: group, agentId: worker })).result.assignmentId)
+ const generation = (await timed.hostPools(host, 'test-host')).find(p => p.tabId === tab)!.machine.generation
+ const observe = (running: string[], stopped: string[] = []) => timed.observeMachine(host, tab, 'test-host', { generation,
+ resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9, diskTotalBytes: 500e9,
+ projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },
+ quotas: [], tools: {}, runningHarnessIds: running, stoppedHarnessIds: stopped })
+ // What the Lead reads with `status`.
+ const downSince = async (agentId: string) => (await timed.read(host, tab, lead)).state.agents.find(a => a.id === agentId)!.downSince
+ return { timed, tab, execute, lead, group, worker, assignmentId, leadHarness, workerHarness, observe, downSince,
+ clock: { get now() { return now }, set now(value: number) { now = value } } }
+ }
+ const iso = (ms: number) => new Date(ms).toISOString()
+
+ it('is set by a gone session, a confirmed stop and an offline machine, and cleared when the worker is back', async () => {
+ const f = await downFixture()
+ await f.observe([f.leadHarness, f.workerHarness])
+ expect(await f.downSince(f.worker)).toBeUndefined()
+
+ // The host no longer lists the session as running.
+ f.clock.now += 1_000
+ const gone = f.clock.now
+ await f.observe([f.leadHarness])
+ f.clock.now += 1_000
+ await f.observe([f.leadHarness])
+ expect(await f.downSince(f.worker)).toBe(iso(gone))
+ expect(await f.downSince(f.lead)).toBeUndefined()
+ await f.observe([f.leadHarness, f.workerHarness])
+ expect(await f.downSince(f.worker)).toBeUndefined()
+
+ // The host confirms the session stopped, before its next reading.
+ f.clock.now += 1_000
+ const stopped = f.clock.now
+ await f.timed.stopped(host, f.tab, f.assignmentId, 'fixture-native-stop')
+ expect(await f.downSince(f.worker)).toBe(iso(stopped))
+ f.clock.now += 1_000
+ await f.observe([f.leadHarness], [f.workerHarness])
+ expect(await f.downSince(f.worker)).toBe(iso(stopped))
+ await f.observe([f.leadHarness, f.workerHarness])
+ expect(await f.downSince(f.worker)).toBeUndefined()
+
+ // The machine stops sending readings: down since its last one, kept when it returns without the session.
+ f.clock.now += 1_000
+ const lastReading = f.clock.now
+ await f.observe([f.leadHarness, f.workerHarness])
+ f.clock.now += 45_000
+ expect(await f.downSince(f.worker)).toBeUndefined()
+ f.clock.now += 1
+ expect(await f.downSince(f.worker)).toBe(iso(lastReading))
+ expect(await f.downSince(f.lead)).toBe(iso(lastReading))
+ f.clock.now += 60_000
+ await f.observe([f.leadHarness])
+ expect(await f.downSince(f.worker)).toBe(iso(lastReading))
+ expect(await f.downSince(f.lead)).toBeUndefined()
+ })
+
+ it('names a worker down two hours or more in the Lead\'s scheduled wake-up, once per time it went down', async () => {
+ const f = await downFixture()
+ const wakeUps = async () => (await db.superTabMessage.findMany({ where: { tabId: f.tab, conversationId: 'lead' }, orderBy: { sequence: 'asc' } }))
+ .filter(m => (m.source as Record<string, unknown>)?.kind === 'update-request')
+ const poll = async () => (await f.timed.deliveries(host, f.tab, f.lead)).filter(d => d.conversationId === 'lead')
+ // The Lead acknowledges through the revision its wake-up names.
+ const acknowledge = async () => f.execute({ type: 'project.report', projectId: 'lead',
+ throughRevision: ((await wakeUps()).at(-1)!.source as { throughRevision: number }).throughRevision, text: null }, host, f.lead)
+ await f.observe([f.leadHarness, f.workerHarness])
+ f.clock.now += 300_000
+ expect(await poll()).toHaveLength(1)
+ await acknowledge()
+ expect(await poll()).toEqual([])
+
+ const down = f.clock.now
+ await f.observe([f.leadHarness, f.workerHarness])
+ await f.observe([f.leadHarness])
+ f.clock.now = down + 2 * 3_600_000 - 60_000
+ await f.execute({ ...message('A new report'), conversationId: f.group } as Command)
+ f.clock.now += 59_999
+ expect(await poll()).toEqual([])
+ f.clock.now++
+ expect(await poll()).toHaveLength(1)
+ const named = (await wakeUps()).at(-1)!
+ expect(named.text).toContain(`Worker (down since ${iso(down)}; task "Ship the parser")`)
+ expect(named.text).not.toContain('Lead (down')
+ expect(named.text).not.toContain('Consolidate')
+ expect(await poll()).toHaveLength(1)
+ await acknowledge()
+ // The report that was not yet due still gets its own request.
+ f.clock.now += 240_000
+ expect(await poll()).toHaveLength(1)
+ expect((await wakeUps()).at(-1)!.text).toContain('Consolidate new reports')
+ await acknowledge()
+ f.clock.now += 3 * 3_600_000
+ expect(await poll()).toEqual([])
+
+ await f.observe([f.leadHarness, f.workerHarness])
+ const again = f.clock.now
+ await f.observe([f.leadHarness])
+ f.clock.now += 2 * 3_600_000
+ expect(await poll()).toHaveLength(1)
+ expect((await wakeUps()).at(-1)!.text).toContain(`Worker (down since ${iso(again)}`)
+ expect(await wakeUps()).toHaveLength(4)
+ })
+
+ it('wakes the Lead of an older Swarm without stored projects', async () => {
+ const f = await downFixture({ legacy: true })
+ expect((await f.timed.read(owner, f.tab)).state.projects).toBeUndefined()
+ const poll = async () => (await f.timed.deliveries(host, f.tab, f.lead)).filter(d => d.conversationId === 'lead')
+ const down = f.clock.now
+ await f.observe([f.leadHarness, f.workerHarness])
+ await f.observe([f.leadHarness])
+ f.clock.now = down + 2 * 3_600_000 - 1
+ expect(await poll()).toEqual([])
+ f.clock.now++
+ const polls = [...await poll(), ...await poll()]
+ expect(new Set(polls.map(d => d.id)).size).toBe(1)
+ const wakeUps = (await db.superTabMessage.findMany({ where: { tabId: f.tab, conversationId: 'lead' } }))
+ .filter(m => (m.source as Record<string, unknown>)?.kind === 'update-request')
+ expect(wakeUps).toHaveLength(1)
+ expect(wakeUps[0]!.text).toContain(`Worker (down since ${iso(down)}; task "Ship the parser")`)
+ })
+})
+
describe('Super Tabs on a real Mongo replica set', () => {
it('contributes project capacity without sharing the computer and fences host execution', async () => {
const tab = (await store.create(owner, randomUUID(), 'Machine pool')).id
diff --git a/backend/src/superTabs/machines.ts b/backend/src/superTabs/machines.ts
index 57fb48e2d..ae1e93be9 100644
--- a/backend/src/superTabs/machines.ts
+++ b/backend/src/superTabs/machines.ts
@@ -1,5 +1,5 @@
import { z } from 'zod'
-import { fail, person, type AgentStart, type AgentStartRequest, type SuperTabState } from './model.js'
+import { fail, person, type Agent, type AgentStart, type AgentStartRequest, type SuperTabState } from './model.js'
import { groupProject } from './projects.js'
import { conversation } from './model.js'
@@ -39,6 +39,29 @@ export const machineObservationSchema = z.object({ generation: z.string().max(16
}).strict()
export type MachineObservation = z.infer<typeof machineObservationSchema>
+/** A machine reports every 15 seconds (cli/src/superTabs/machines.ts); three missed beats is offline. */
+export const MACHINE_FRESH_MS = 45_000
+
+/** Since when an agent has been down: the stored time a reading first showed its session gone or the host confirmed its
+ * stop, or its machine's last reading once that is stale. Nothing is written while a machine is silent, so that case is
+ * read from the reading itself. Undefined while it is up. */
+export function downSince(state: SuperTabState, target: Agent, now: number): string | undefined {
+ const observation = state.machines?.find(m => m.machineId === target.machineId)?.observation
+ const offline = observation && now - observation.receivedAt > MACHINE_FRESH_MS ? new Date(observation.receivedAt).toISOString() : undefined
+ return [target.downSince, offline].filter(Boolean).sort()[0]
+}
+
+/** Before a new reading replaces the last one: a session it lists as running is back; any other agent on the machine
+ * keeps the earliest time it was down, including the time its machine went silent. */
+export function recordAgentPresence(state: SuperTabState, machineId: string, running: string[], now: number): void {
+ for (const target of state.agents.filter(a => a.machineId === machineId)) {
+ if (running.includes(target.harnessId)) {
+ delete target.downSince
+ delete target.downNamed
+ } else target.downSince = downSince(state, target, now) ?? new Date(now).toISOString()
+ }
+}
+
export function machineContribution(state: SuperTabState, machineId: string) {
const pool = state.machines?.find(m => m.machineId === machineId)
if (!pool) return null
@@ -95,7 +118,7 @@ export function admitMachineStart(state: SuperTabState, request: AgentStartReque
if (refused.length) {
fail('OUTSIDE_CONTRIBUTION', `${pool.name} accepts only its contributed project folder, engines and default account settings, with full permissions for Swarm agents. Refused: ${refused.join('; ')}.`, 403)
}
- if (!pool.observation || now - pool.observation.receivedAt > 45_000) {
+ if (!pool.observation || now - pool.observation.receivedAt > MACHINE_FRESH_MS) {
fail('MACHINE_OFFLINE', 'Wait for a fresh reading from the contributed machine.')
}
if (!start && machineSlots(state, pool) >= pool.maxAgents) fail('MACHINE_FULL', 'This machine has reached its contributed agent limit.')
diff --git a/backend/src/superTabs/model.ts b/backend/src/superTabs/model.ts
index 2d536680a..012d1cab6 100644
--- a/backend/src/superTabs/model.ts
+++ b/backend/src/superTabs/model.ts
@@ -1,6 +1,7 @@
import { createHash } from 'node:crypto'
import type { RouteDecisionEffect } from './routeRecord.js'
import { agentDisplayName, assignAgentDisplayNames } from './agentNames.js'
+import { downSince } from './machines.js'
import { canReadAgent, canReadProject, groupProject, placeRules, projectsOf, type ProjectVisibility, type SwarmProject } from './projects.js'
export type Role = 'owner' | 'collaborator' | 'viewer'
@@ -36,6 +37,10 @@ export interface Agent {
configurationVersion?: string
/** Fences a repeated Change agent report without rolling back later model observations. */
replacement?: { previousHarnessId: string; harnessId: string }
+ /** When a reading first showed its session gone or its host confirmed a stop; cleared when it is back (machines.ts). */
+ downSince?: string
+ /** The down time already named in the Lead's scheduled wake-up (projectUpdates.ts), so each time down is named once. */
+ downNamed?: string
}
export type AgentRole = 'builder' | 'reviewer'
export const agentRole = (agent: Pick<Agent, 'role'>): AgentRole => agent.role ?? 'builder'
@@ -271,10 +276,15 @@ export function createSuperTab(id: string, name: string, owner: { id: string; em
}
/** Pending invitations are management data, not part of the shared conversation. */
-export function visibleState(state: SuperTabState, actor: Actor): SuperTabState {
+export function visibleState(state: SuperTabState, actor: Actor, now = Date.now()): SuperTabState {
author(state, actor)
const visible = structuredClone(state)
assignAgentDisplayNames(visible.agents)
+ for (const target of visible.agents) {
+ const since = downSince(state, target, now)
+ if (since) target.downSince = since
+ delete target.downNamed
+ }
delete visible.sharing
delete visible.harnessReservations
delete visible.releasedHarnesses
diff --git a/backend/src/superTabs/projectUpdates.ts b/backend/src/superTabs/projectUpdates.ts
index 5057a2040..ceea06fb3 100644
--- a/backend/src/superTabs/projectUpdates.ts
+++ b/backend/src/superTabs/projectUpdates.ts
@@ -1,4 +1,5 @@
-import { author, conversation, stableId, type Agent, type MessageDraft, type SuperTabState } from './model.js'
+import { activeWork, author, conversation, stableId, type Agent, type MessageDraft, type SuperTabState } from './model.js'
+import { downSince } from './machines.js'
import { agentProjectId, groupProject, projectOf } from './projects.js'
/** Workgroup reports mark a project dirty. No per-message model or timer is
@@ -14,25 +15,50 @@ export function noteProjectReports(state: SuperTabState, messages: MessageDraft[
}
}
+/** The Lead reassigns a Builder or Reviewer that has been down this long (the owner's rule, 2026-10-23). */
+export const DOWN_REASSIGN_MS = 2 * 60 * 60_000
+
+/** Builders and Reviewers of the project down for two hours, each named once per time it went down. A worker that is
+ * already stopping was decided on. */
+function downWorkers(state: SuperTabState, projectId: string, now: number) {
+ return state.agents.flatMap(a => {
+ const since = downSince(state, a, now)
+ if (!since || a.downNamed === since || now - Date.parse(since) < DOWN_REASSIGN_MS || a.workDecision === 'stop' ||
+ agentProjectId(state, a) !== projectId || state.conversations.some(c => c.leadId === a.id)) return []
+ const task = state.conversations.find(c => activeWork(c) && c.assignment!.agentId === a.id)
+ return [{ agentId: a.id, since, text: `${a.displayName ?? a.name} (down since ${since}${task ? `; task ${JSON.stringify(task.title)}` : ''})` }]
+ })
+}
+
export function projectUpdateRequest(state: SuperTabState, target: Agent, now: number): MessageDraft | null {
- if (!state.projects || state.paused) return null
+ // An older Swarm without stored projects is one synthesized project. It keeps no report progress (noteProjectReports),
+ // so its Lead is woken only for down workers.
+ if (state.paused) return null
const project = projectOf(state, agentProjectId(state, target))
const group = conversation(state, project.leadConversationId), progress = project.progress
- if (group.leadId !== target.id || group.archived || !progress ||
- progress.dirtyRevision <= progress.summarizedRevision ||
- progress.dirtyRevision <= progress.requestedRevision ||
- now < Math.max(progress.requestedAt ?? 0, progress.since) + project.updateMinutes * 60_000) return null
- const id = stableId('project-update', state.id, project.id, String(progress.dirtyRevision))
- const text = `Consolidate new reports for project ${JSON.stringify(project.name)}. ` +
- `Read this project's workgroups with lastMessageSequence after ${progress.summarizedRevision}; inspect their new messages and current work. ` +
- 'At your next natural checkpoint, tell the people what changed, what is blocked, and what comes next. Include useful workgroup references and result links. ' +
+ if (group.leadId !== target.id || group.archived) return null
+ const reports = !!progress && progress.dirtyRevision > progress.summarizedRevision &&
+ progress.dirtyRevision > progress.requestedRevision &&
+ now >= Math.max(progress.requestedAt ?? 0, progress.since) + project.updateMinutes * 60_000
+ const down = downWorkers(state, project.id, now)
+ if (!reports && !down.length) return null
+ // A wake-up only for down workers is acknowledged through what was already summarized, so reports that are not yet due
+ // keep their own request.
+ const through = (reports ? progress!.dirtyRevision : progress?.summarizedRevision) ?? 0
+ const id = stableId('project-update', state.id, project.id, String(through), ...down.map(d => `${d.agentId}@${d.since}`))
+ const text = (reports ? `Consolidate new reports for project ${JSON.stringify(project.name)}. ` +
+ `Read this project's workgroups with lastMessageSequence after ${progress!.summarizedRevision}; inspect their new messages and current work. ` +
+ 'At your next natural checkpoint, tell the people what changed, what is blocked, and what comes next. Include useful workgroup references and result links. ' : '') +
+ (down.length ? `Down for 2 hours or more in project ${JSON.stringify(project.name)}: ${down.map(d => d.text).join(', ')}. ` +
+ 'Their machine is offline or their session ended or stopped. Decide whether to reassign their work to another agent. ' : '') +
'Do not copy the terminal log or repeat unchanged status. This is a scheduled request, not a new task. ' +
`Use harness super-tab --tab ${state.id} command UNIQUE_OPERATION_ID ` +
- `JSON with {"type":"project.report","projectId":"${project.id}","throughRevision":${progress.dirtyRevision},"text":"your update"}. ` +
+ `JSON with {"type":"project.report","projectId":"${project.id}","throughRevision":${through},"text":"your update"}. ` +
'Use text:null to acknowledge if nothing merits an update. Reuse the same operation ID after a lost reply. ' +
'Keep reporting human questions, serious blockers, and completion immediately; do not wait for this cadence.'
return { id, conversationId: group.id,
author: author(state, { kind: 'agent', agentId: target.id, machineOwnerId: target.machineOwnerId }),
text, replyTo: null, recipients: [target.id],
- source: { kind: 'update-request', projectId: project.id, throughRevision: progress.dirtyRevision } }
+ source: { kind: 'update-request', projectId: project.id, throughRevision: through,
+ ...(down.length ? { down: down.map(({ agentId, since }) => ({ agentId, since })) } : {}) } }
}
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index 29c8a79f5..7c72844b3 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -1,4 +1,4 @@
-import { machineContribution, machineObservationSchema, machineSlots, type MachineObservation } from './machines.js'
+import { machineContribution, machineObservationSchema, machineSlots, recordAgentPresence, type MachineObservation } from './machines.js'
import { createHash } from 'node:crypto'
import type { Prisma, PrismaClient, SuperTab, SuperTabDelivery } from '@prisma/client'
import type { AuthUser } from '../lib/ssoAuth.js'
@@ -282,7 +282,7 @@ export class SuperTabStore {
const row = await this.row(tx, user, tabId)
const state = stateOf(row)
const actor = await this.actor(tx, user, state, asAgentId)
- const visible = visibleState(state, actor)
+ const visible = visibleState(state, actor, this.now())
return { revision: row.revision, state: { ...visible, machines: visible.machines?.map(pool => ({
...pool, availableSlots: Math.max(0, pool.maxAgents - machineSlots(state, state.machines!.find(m => m.machineId === pool.machineId)!)),
})) } }
@@ -769,6 +769,7 @@ export class SuperTabStore {
if (!pool || pool.contributorId !== user.sub || pool.generation !== observation.generation) fail('CONTRIBUTION_REVOKED', 'Read the current machine contribution.', 403)
const owned = await tx.machine.findFirst({ where: { machineId, userId: user.sub, autonomousEnv: user.autonomousEnv, ...machineAlive } })
if (!owned || !machineBillingAllowsDataPlane(owned)) fail('HOST_UNAVAILABLE', 'This execution host is unavailable.', 403)
+ recordAgentPresence(state, machineId, observation.runningHarnessIds, this.now())
pool.observation = { ...observation, quotas: observation.quotas.filter(q => pool.engines.includes(q.engine)), receivedAt: this.now() }
// Current-generation physical stop proof also finishes a close whose membership was already withdrawn.
for (const proof of observation.stoppedReservations ?? []) {
@@ -883,15 +884,21 @@ export class SuperTabStore {
const update = projectUpdateRequest(state, target, this.now())
if (update) {
const project = groupProject(state, conversation(state, update.conversationId))
- const progress = project.progress!
+ // A worker down two hours asks for a wake-up even when no report has come in yet.
+ const progress = project.progress ??= { dirtyRevision: 0, summarizedRevision: 0, requestedRevision: 0, since: this.now() }
const pending = progress.requestMessageId
? await tx.superTabDelivery.findFirst({ where: { tabId, agentId, messageId: progress.requestMessageId,
status: { in: ['queued', 'claimed', 'uncertain', 'delivered'] } } })
: null
if (!pending) {
- progress.requestedRevision = progress.dirtyRevision
- progress.requestedAt = this.now()
+ if (update.source!.throughRevision === progress.dirtyRevision) {
+ progress.requestedRevision = progress.dirtyRevision
+ progress.requestedAt = this.now()
+ }
progress.requestMessageId = update.id
+ for (const { agentId, since } of (update.source!.down ?? []) as Array<{ agentId: string; since: string }>) {
+ state.agents.find(a => a.id === agentId)!.downNamed = since
+ }
const revision = row.revision + 1
await this.writeMessages(tx, tabId, state, [update], revision)
await tx.superTab.update({ where: { id: tabId, revision: row.revision }, data: { revision, state: json(state) } })
@@ -1019,6 +1026,7 @@ export class SuperTabStore {
// on purpose by someone who already knows.
const worker = state.agents.find(a => a.id === assignment.agentId)
if (worker) {
+ worker.downSince ??= new Date(this.now()).toISOString()
const lead = group.leadId && group.leadId !== worker.id && group.agentIds.includes(group.leadId) ? group.leadId : null
notices.push({ id: stableId('stop-notice', tabId, assignmentId), conversationId: group.id,
author: author(state, { kind: 'agent', agentId: worker.id, machineOwnerId: worker.machineOwnerId }),