Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Wake Swarm Leads when recording gaps interrupt active work (#1608)

tony496s8eb6434c6de2
diff --git a/backend/integration/superTabs.test.ts b/backend/integration/superTabs.test.ts
index d5caff04b..ed39ac2b6 100644
--- a/backend/integration/superTabs.test.ts
+++ b/backend/integration/superTabs.test.ts
@@ -71,6 +71,194 @@ afterAll(async () => {
await replica?.stop()
})
+describe('native recording restart recovery', () => {
+ async function recoveryFixture(status: 'working' | 'review' = 'working') {
+ const { tab, lead } = await fixture()
+ const group = (await run(tab, { type: 'conversation.create', title: 'Recover interrupted work', brief: 'Finish the saved task',
+ checks: [{ description: 'Task has a checkpoint', independent: false }] })).result.conversationId as string
+ const worker = (await run(tab, { type: 'agent.add', conversationId: group, machineId: 'test-host', harnessId: randomUUID(),
+ name: 'Recovery Engineer', configuration: config, lead: false })).result.agentId as string
+ const assigned = await run(tab, { type: 'work.assign', conversationId: group, agentId: worker })
+ const assignmentId = assigned.result.assignmentId as string
+ if (status === 'review') await run(tab, { type: 'work.status', conversationId: group, assignmentId, status, result: 'Checkpoint ready' }, host, worker)
+ const { contextId } = await store.native.open(owner, tab, { operationId: randomUUID(), conversationId: group, agentId: worker })
+ const prompt = { turnId: 'saved_turn', kind: 'prompt' as const, previousEventId: null,
+ text: 'Continue from the saved commit', truncated: false, interrupted: false }
+ const checkpoint = await store.native.append(host, tab, contextId, prompt) as { eventId: string }
+ const checkpointRow = await db.superTabMessage.findUniqueOrThrow({ where: { id: checkpoint.eventId } })
+ const gap = { turnId: 'restart_observation', kind: 'gap' as const, previousEventId: checkpoint.eventId,
+ text: '', truncated: false, interrupted: false }
+ const interruption = (reason: 'recording_restarted' | 'history_lost' | 'agent_stopped' = 'recording_restarted') => ({ ...gap,
+ gap: { incidentId: 'restart_incident', reason, detectedAt: Date.now() + 1000 } })
+ const state = async () => (await store.read(owner, tab)).state.conversations.find(c => c.id === group)!.assignment!
+ const gapDeliveries = async () => {
+ const rows = await db.superTabMessage.findMany({ where: { tabId: tab, conversationId: group } })
+ const ids = rows.filter(m => (m.source as { phase?: string } | null)?.phase === 'gap').map(m => m.id)
+ return db.superTabDelivery.findMany({ where: { tabId: tab, messageId: { in: ids } } })
+ }
+ const append = async (body: Record<string, unknown>, context = contextId) => {
+ const response = await app.inject({ method: 'POST', url: `/api/super-tabs/${tab}/native/${context}`,
+ headers: { authorization: `Bearer ${host.sub}` }, payload: body })
+ expect(response.statusCode, response.body).toBe(200)
+ return response.json().data as { eventId: string }
+ }
+ return { tab, lead, group, worker, assignmentId, contextId, checkpoint, checkpointRow, gap, interruption, state, gapDeliveries, append }
+ }
+
+ it.each(['recording_restarted', 'history_lost', 'agent_stopped'] as const)('wakes only the Lead and blocks working work for %s with a checkpoint time and no visible ids', async reason => {
+ const f = await recoveryFixture(), event = f.interruption(reason)
+ const result = await store.native.append(host, f.tab, f.contextId, event) as { eventId: string }
+ expect(await f.state()).toMatchObject({ id: f.assignmentId, status: 'blocked', result: expect.stringContaining('Recovery Engineer') })
+ const message = await db.superTabMessage.findUniqueOrThrow({ where: { id: result.eventId } })
+ expect(message.text).toContain('Recovery Engineer')
+ expect(message.text).toContain('Recover interrupted work')
+ expect(message.text).toContain(f.checkpointRow.createdAt.toISOString())
+ expect(message.text).not.toContain(f.assignmentId)
+ expect(message.text).not.toContain(f.checkpoint.eventId)
+ expect(message.text).not.toContain('saved_turn')
+ expect((await f.state()).result).toBe(message.text)
+ expect(message.source).toMatchObject({ kind: 'native', phase: 'gap', recovery: {
+ assignmentId: f.assignmentId, checkpoint: { eventId: f.checkpoint.eventId, turnId: 'saved_turn', recordedAt: f.checkpointRow.createdAt.toISOString() } } })
+ expect(await f.gapDeliveries()).toMatchObject([{ agentId: f.lead, messageId: result.eventId }])
+ expect(await f.gapDeliveries()).toHaveLength(1)
+ expect(await store.native.append(host, f.tab, f.contextId, event)).toEqual(result)
+ expect(await f.gapDeliveries()).toHaveLength(1)
+ })
+
+ it('keeps a person’s unattributed terminal input from blocking working work or waking the Lead', async () => {
+ const f = await recoveryFixture(), before = await f.state()
+ const result = await f.append({ ...f.gap, gap: { incidentId: 'typed_input', reason: 'unattributed_input', detectedAt: Date.now() + 1000 } })
+ expect(await f.state()).toEqual(before)
+ expect(await f.gapDeliveries()).toEqual([])
+ const message = await db.superTabMessage.findUniqueOrThrow({ where: { id: result.eventId } })
+ expect(message.text).toBe('Some terminal activity could not be recorded. Check the native session for the full conversation.')
+ expect(message.source).not.toHaveProperty('recovery')
+ })
+
+ it.each(['legacy', 'context_expired', 'prompt_missing'] as const)('keeps a %s gap as a plain note without changing work', async reason => {
+ const f = await recoveryFixture(), before = await f.state()
+ const result = await f.append({ ...f.gap, ...(reason === 'legacy' ? {} : { gap: { incidentId: 'plain_gap', reason, detectedAt: Date.now() + 1000 } }) })
+ expect(await f.state()).toEqual(before)
+ expect(await f.gapDeliveries()).toEqual([])
+ const message = await db.superTabMessage.findUniqueOrThrow({ where: { id: result.eventId } })
+ expect(message.text).toBe('Some terminal activity could not be recorded. Check the native session for the full conversation.')
+ expect(message.source).not.toHaveProperty('recovery')
+ })
+
+ it('notifies once about an interruption during review while preserving the review state and result', async () => {
+ const f = await recoveryFixture('review'), before = await f.state(), event = f.interruption()
+ const first = await f.append(event)
+ await f.append({ ...event, turnId: 'another_gap', previousEventId: first.eventId })
+ expect(await f.state()).toEqual(before)
+ expect(await f.gapDeliveries()).toMatchObject([{ agentId: f.lead, messageId: first.eventId }])
+ expect(await f.gapDeliveries()).toHaveLength(1)
+ const message = await db.superTabMessage.findUniqueOrThrow({ where: { id: first.eventId } })
+ expect(message.text).toContain('Keep this task in review')
+ expect(message.text).not.toContain('report working')
+ expect(message.text).not.toContain('work was interrupted')
+ })
+
+ it('advertises recovery metadata without requiring it from older recorders', async () => {
+ const f = await recoveryFixture()
+ expect(await store.native.read(host, f.tab, f.contextId)).toMatchObject({ recovery: { version: 1, serverTime: expect.any(Number) } })
+ })
+
+ it('names the newer progress checkpoint when a later restart interrupts the task again', async () => {
+ const f = await recoveryFixture(), detectedAt = Date.now() + 1000
+ const first = await f.append({ ...f.gap, gap: { incidentId: 'first_restart', reason: 'recording_restarted', detectedAt } })
+ const later = new SuperTabStore(db, () => detectedAt + 1)
+ const progress = await later.execute(host, f.tab, { operationId: randomUUID(), command: {
+ type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId, status: 'working', result: 'Saved the next checkpoint' } }, f.worker)
+ const checkpoint = await db.superTabMessage.findUniqueOrThrow({ where: { id: progress.result.messageId as string } })
+ const second = await f.append({ ...f.gap, turnId: 'new_restart_observation', previousEventId: first.eventId,
+ gap: { incidentId: 'second_restart', reason: 'recording_restarted', detectedAt: detectedAt + 2 } })
+ expect(await f.state()).toMatchObject({ status: 'blocked' })
+ const message = await db.superTabMessage.findUniqueOrThrow({ where: { id: second.eventId } })
+ expect(message.text).toContain(checkpoint.createdAt.toISOString())
+ expect(message.text).not.toContain(checkpoint.id)
+ expect(message.source).toMatchObject({ recovery: { checkpoint: { eventId: checkpoint.id, recordedAt: checkpoint.createdAt.toISOString() } } })
+ expect(await f.gapDeliveries()).toHaveLength(2)
+ })
+
+ it('wakes once for the same restart across contexts and does not re-block after the Lead clears it', async () => {
+ const f = await recoveryFixture(), detectedAt = Date.now() + 1000
+ const metadata = { incidentId: 'same_restart', reason: 'recording_restarted', detectedAt }
+ const first = await f.append({ ...f.gap, gap: metadata })
+ expect(await f.state()).toMatchObject({ status: 'blocked' })
+ const later = new SuperTabStore(db, () => detectedAt + 1)
+ await later.execute(host, f.tab, { operationId: randomUUID(), command: {
+ type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId, status: 'working', result: 'Continuing from the checkpoint' } }, f.lead)
+ const { contextId } = await store.native.open(owner, f.tab, { operationId: randomUUID(), conversationId: f.group, agentId: f.worker })
+ await f.append({ ...f.gap, turnId: 'second_gap', previousEventId: null, gap: metadata }, contextId)
+ await f.append({ ...f.gap, turnId: 'third_gap', previousEventId: first.eventId, gap: metadata })
+ expect(await f.state()).toMatchObject({ status: 'working', result: 'Continuing from the checkpoint' })
+ expect(await f.gapDeliveries()).toHaveLength(1)
+ })
+
+ it.each(['worker', 'lead'] as const)('lets the %s resume a recovery block without a person, including a delayed legacy gap', async who => {
+ const f = await recoveryFixture()
+ const first = await store.native.append(host, f.tab, f.contextId, f.interruption()) as { eventId: string }
+ expect(await f.state()).toMatchObject({ status: 'blocked' })
+ await run(f.tab, { type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId,
+ status: 'working', result: 'Session checked; work continued' }, host, who === 'worker' ? f.worker : f.lead)
+ await store.native.append(host, f.tab, f.contextId, { ...f.gap, turnId: 'late_legacy_gap', previousEventId: first.eventId })
+ expect(await f.state()).toMatchObject({ status: 'working', result: 'Session checked; work continued' })
+ expect(await f.gapDeliveries()).toHaveLength(1)
+ })
+
+ it.each(['status', 'message'] as const)('does not block a delayed incident after the worker reports progress by %s', async progress => {
+ const f = await recoveryFixture(), detectedAt = Date.now() + 1000
+ const later = new SuperTabStore(db, () => detectedAt + 1)
+ const command: Command = progress === 'status'
+ ? { type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId, status: 'working', result: null }
+ : { type: 'message.send', conversationId: f.group, text: 'Recovered and committed the next step', replyTo: null, recipientIds: [] }
+ await later.execute(host, f.tab, { operationId: randomUUID(), command }, f.worker)
+ const first = await f.append({ ...f.gap, gap: { incidentId: 'delayed_restart', reason: 'recording_restarted', detectedAt } })
+ await f.append({ ...f.gap, turnId: 'later_retry', previousEventId: first.eventId,
+ gap: { incidentId: 'delayed_restart', reason: 'recording_restarted', detectedAt } })
+ expect(await f.state()).toMatchObject({ status: 'working', result: null })
+ expect(await f.gapDeliveries()).toHaveLength(0)
+ })
+
+ it.each(['done', 'blocked', 'needs_human'] as const)('does not replace %s work or notify the Lead', async status => {
+ const f = await recoveryFixture()
+ if (status === 'done') {
+ const checkId = (await store.read(owner, f.tab)).state.conversations.find(c => c.id === f.group)!.checks[0]!.id
+ await run(f.tab, { type: 'work.check', conversationId: f.group, assignmentId: f.assignmentId, checkId,
+ passed: true, evidence: 'The fixture recorded its checkpoint' }, host, f.worker)
+ }
+ await run(f.tab, { type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId,
+ status, result: 'Existing outcome' }, host, f.worker)
+ await store.native.append(host, f.tab, f.contextId, f.interruption())
+ expect(await f.state()).toMatchObject({ status, result: 'Existing outcome' })
+ expect(await f.gapDeliveries()).toHaveLength(0)
+ })
+
+ it('does not wake or block the Lead itself, even when it holds active work', async () => {
+ const { tab, lead } = await fixture()
+ await run(tab, { type: 'work.define', conversationId: 'lead', brief: 'Lead task', checks: [{ description: 'Checked', independent: false }] })
+ await run(tab, { type: 'work.assign', conversationId: 'lead', agentId: lead })
+ const { contextId } = await store.native.open(owner, tab, { operationId: randomUUID(), conversationId: 'lead', agentId: lead })
+ const receipt = await store.native.append(host, tab, contextId, { turnId: 'lead_restart', kind: 'gap', previousEventId: null,
+ text: '', truncated: false, interrupted: false,
+ gap: { incidentId: 'lead_restart', reason: 'recording_restarted', detectedAt: Date.now() + 1000 } }) as { eventId: string }
+ expect((await store.read(owner, tab)).state.conversations[0]!.assignment!.status).toBe('working')
+ expect(await db.superTabDelivery.findMany({ where: { messageId: receipt.eventId } })).toEqual([])
+ })
+
+ it('does not notify for a worker without an assignment', async () => {
+ const { tab } = await fixture()
+ const worker = (await run(tab, { type: 'agent.add', conversationId: 'lead', machineId: 'test-host', harnessId: randomUUID(),
+ name: 'Idle Engineer', configuration: config, lead: false })).result.agentId as string
+ const { contextId } = await store.native.open(owner, tab, { operationId: randomUUID(), conversationId: 'lead', agentId: worker })
+ const receipt = await store.native.append(host, tab, contextId, { turnId: 'idle_restart', kind: 'gap', previousEventId: null,
+ text: '', truncated: false, interrupted: false,
+ gap: { incidentId: 'idle_restart', reason: 'recording_restarted', detectedAt: Date.now() + 1000 } }) as { eventId: string }
+ expect((await store.read(owner, tab)).state.conversations[0]!.assignment).toBeNull()
+ expect(await db.superTabDelivery.findMany({ where: { messageId: receipt.eventId } })).toEqual([])
+ })
+})
+
describe('Change agent on a Swarm member', () => {
it('moves active execution and keeps queued, held, answered and uncertain receipts; Send again uses the current member', async () => {
const { tab, lead } = await fixture()
diff --git a/backend/src/superTabs/model.ts b/backend/src/superTabs/model.ts
index 8f2724bd0..2d536680a 100644
--- a/backend/src/superTabs/model.ts
+++ b/backend/src/superTabs/model.ts
@@ -89,6 +89,8 @@ export interface Assignment {
configuration: AgentConfiguration
execution: { machineId: string; harnessId: string; machineOwnerId: string }
result: string | null
+ /** Explicit progress fences delayed recording gaps; absent on older backends. */
+ recordingProgress?: { at: number; revision: number }
}
export interface WorkTask {
goal: string
diff --git a/backend/src/superTabs/native.ts b/backend/src/superTabs/native.ts
index 0058bf011..1b999e5ba 100644
--- a/backend/src/superTabs/native.ts
+++ b/backend/src/superTabs/native.ts
@@ -3,7 +3,7 @@ import type { Prisma, SuperTab, SuperTabNativeContext } from '@prisma/client'
import { z } from 'zod'
import type { AuthUser } from '../lib/ssoAuth.js'
import { activeWork, agent, author, conversation, fail, stableId, SuperTabError,
- type Actor, type Agent, type Author, type SuperTabState } from './model.js'
+ type Actor, type Agent, type Author, type Conversation, type SuperTabState } from './model.js'
import { canReadProject, groupProject, readConversation } from './projects.js'
import { noteProjectReports } from './projectUpdates.js'
@@ -12,7 +12,9 @@ export const openNativeSchema = z.object({ operationId: id, conversationId: id,
export const nativeEventSchema = z.object({
turnId: id, kind: z.enum(['prompt', 'answer', 'gap']), previousEventId: id.nullable(),
text: z.string().max(16_000), truncated: z.boolean(), interrupted: z.boolean(),
-}).strict()
+ gap: z.object({ incidentId: id, reason: z.enum(['recording_restarted', 'history_lost', 'unattributed_input', 'context_expired', 'agent_stopped', 'prompt_missing']),
+ detectedAt: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) }).strict().optional(),
+}).strict().refine(event => !event.gap || event.kind === 'gap', 'Recovery metadata belongs to a recording gap.')
type NativeEvent = z.infer<typeof nativeEventSchema>
type Transaction = Prisma.TransactionClient
type Grant = { machineOwnerId: string; contributorId: string; hostGrant: string | null }
@@ -28,6 +30,18 @@ interface Deps {
}
const json = (value: unknown): Prisma.InputJsonValue => JSON.parse(JSON.stringify(value)) as Prisma.InputJsonValue
const digest = (value: unknown) => createHash('sha256').update(JSON.stringify(value)).digest('hex')
+interface Checkpoint { eventId: string | null; turnId: string | null; recordedAt: string | null; sequence: number }
+interface Recovery {
+ assignmentId: string; incidentId: string; reason: string; detectedAt: number; checkpoint: Checkpoint; required: boolean
+}
+const gapReasons = {
+ recording_restarted: 'The work was interrupted by a restart of the terminal recording.',
+ history_lost: 'Part of the recorded terminal history was lost.',
+ unattributed_input: 'Terminal input could not be attributed to this conversation.',
+ context_expired: 'Terminal recording expired before the activity was captured.',
+ agent_stopped: 'The agent stopped during an unfinished recorded turn.',
+ prompt_missing: 'A terminal answer was observed without its recorded prompt.',
+}
/** A host reports what its native engine observed; it cannot create human-authored
* messages. The human attached to the source is the authenticated opener of that
@@ -99,7 +113,9 @@ export class SuperTabNativeStore {
async read(user: AuthUser, tabId: string, contextId: string) {
return this.deps.transaction(async tx => {
const { binding, record } = await this.hosted(tx, user, tabId, contextId)
- return { contextId, ...binding, lastEventId: record.lastEventId }
+ // Old recorders ignore this. New recorders keep the old strict request
+ // shape until this backend explicitly advertises recovery support.
+ return { contextId, ...binding, lastEventId: record.lastEventId, recovery: { version: 1, serverTime: Date.now() } }
})
}
@@ -110,6 +126,46 @@ export class SuperTabNativeStore {
catch (error) { if (error instanceof SuperTabError) return false; throw error }
}
+ private async recovery(tx: Transaction, tabId: string, group: Conversation, target: Agent, record: SuperTabNativeContext,
+ event: NativeEvent, eventId: string, revision: number): Promise<{ source: Recovery; text: string } | null> {
+ const assignment = group.assignment
+ // A recording gap can also mean ordinary human typing or an expired view.
+ // Older recorders do not distinguish those, so only known interruptions wake the Lead.
+ if (event.kind !== 'gap' || !event.gap || !['recording_restarted', 'history_lost', 'agent_stopped'].includes(event.gap.reason)
+ || !assignment || !activeWork(group) || assignment.agentId !== target.id
+ || !group.leadId || group.leadId === target.id) return null
+ const previous = await tx.superTabMessage.findFirst({ where: { id: record.lastEventId ?? assignment.id, tabId, conversationId: group.id } })
+ const source = previous?.source as { turnId?: string; recovery?: Recovery } | null
+ // Preserve the original checkpoint through repeated gaps. A gap is not new work progress.
+ let checkpoint: Checkpoint = source?.recovery?.checkpoint ?? {
+ eventId: previous?.id ?? null, turnId: source?.turnId ?? null,
+ recordedAt: previous?.createdAt.toISOString() ?? null, sequence: previous?.sequence ?? 0,
+ }
+ const incidentId = event.gap.incidentId
+ const receiptId = stableId('native-recovery', assignment.id, incidentId)
+ const prior = await tx.superTabOperation.findUnique({ where: { id: receiptId } })
+ const progress = assignment.recordingProgress
+ const progressed = !!progress && progress.at >= event.gap.detectedAt
+ if (progress && !progressed && progress.revision > checkpoint.sequence) {
+ const latest = await tx.superTabMessage.findFirst({ where: { tabId, conversationId: group.id, sequence: progress.revision } })
+ if (latest) checkpoint = { eventId: latest.id, turnId: null, recordedAt: latest.createdAt.toISOString(), sequence: latest.sequence }
+ }
+ const required = !prior && !progressed && ['working', 'review'].includes(assignment.status)
+ const recovery: Recovery = { assignmentId: assignment.id, incidentId, reason: event.gap.reason,
+ detectedAt: event.gap.detectedAt, checkpoint, required }
+ const reason = assignment.status === 'review' && event.gap.reason === 'recording_restarted'
+ ? 'Terminal recording restarted while the task was in review.' : gapReasons[event.gap.reason]
+ const next = assignment.status === 'review' ? 'Keep this task in review while you check the agent’s session.'
+ : 'Check the agent’s session, then ask it to continue from the checkpoint and report working.'
+ const text = `Recovery needed for ${target.displayName ?? target.name} on “${group.title}”. ${reason}\n\n`
+ + `Last recorded checkpoint: ${checkpoint.recordedAt ?? 'time unavailable'}.\n\n`
+ + `${next} Do not replay earlier input or commands; they may already have taken effect.`
+ if (required && assignment.status === 'working') { assignment.status = 'blocked'; assignment.result = text }
+ if (!prior) await tx.superTabOperation.create({ data: { id: receiptId, tabId, actorKey: `agent:${target.id}`,
+ requestHash: digest([assignment.id, incidentId]), revision, result: json({ eventId, recovery }) } })
+ return { source: recovery, text }
+ }
+
async append(user: AuthUser, tabId: string, contextId: string, input: NativeEvent) {
const event = nativeEventSchema.parse(input)
return this.deps.transaction(async tx => {
@@ -126,15 +182,17 @@ export class SuperTabNativeStore {
fail('NATIVE_PROMPT_MISSING', 'The terminal prompt was not recorded. Record the observation gap first.')
}
const revision = row.revision + 1
+ const recovery = await this.recovery(tx, tabId, group, target, record, event, eventId, revision)
group.lastMessageSequence = revision
const observedBy = author(state, { kind: 'agent', agentId: target.id, machineOwnerId: user.sub }, true)
- const text = event.kind === 'gap' ? 'Some terminal activity could not be recorded. Check the native session for the full conversation.' : event.text
+ const text = recovery?.source.required ? recovery.text : event.kind === 'gap'
+ ? 'Some terminal activity could not be recorded. Check the native session for the full conversation.' : event.text
// Inform the coordinating lead without replaying this input into the agent
// that already received it. Its resulting programmatic turn is not native input.
- const recipients = event.kind !== 'gap' && group.leadId && group.leadId !== target.id ? [group.leadId] : []
+ const recipients = (event.kind !== 'gap' || recovery?.source.required) && group.leadId && group.leadId !== target.id ? [group.leadId] : []
await tx.superTabMessage.create({ data: { id: eventId, tabId, conversationId: group.id, sequence: revision,
author: json(observedBy), source: json({ kind: 'native', phase: event.kind, turnId: event.turnId,
- operator: binding.operator, truncated: event.truncated, interrupted: event.interrupted }),
+ operator: binding.operator, truncated: event.truncated, interrupted: event.interrupted, ...(recovery ? { recovery: recovery.source } : {}) }),
text, replyTo: event.kind === 'answer' ? promptId : null, recipientIds: recipients } })
for (const recipient of recipients) {
const lead = agent(state, recipient)
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index 31a6e1777..29c8a79f5 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -456,6 +456,15 @@ export class SuperTabStore {
status: { in: ['queued', 'claimed', 'uncertain'] } }, data: { status: 'cancelled', holdReason: null, holdSince: null } })
}
const revision = row.revision + 1
+ if (command.type === 'work.assign' || command.type === 'work.status' || command.type === 'message.send') {
+ const group = conversation(effects.state, command.conversationId)
+ const assignment = group.assignment
+ if (assignment && (command.type !== 'message.send' || actor.kind === 'agent' && actor.agentId === assignment.agentId)) {
+ // Even an unchanged working status is fresh evidence that the agent
+ // continued. An older queued gap must not turn that progress into Stuck.
+ assignment.recordingProgress = { at: context.now, revision }
+ }
+ }
if (command.type === 'agent.add' || command.type === 'agent.started' || command.type === 'agent.configuration' || command.type === 'agent.rebind') {
for (const target of effects.state.agents) {
if (target.configurationVersion !== receiptId) continue
diff --git a/cli/src/lib/swarmSkills.spec.ts b/cli/src/lib/swarmSkills.spec.ts
index 8c0c33d63..d8352aa08 100644
--- a/cli/src/lib/swarmSkills.spec.ts
+++ b/cli/src/lib/swarmSkills.spec.ts
@@ -39,9 +39,9 @@ describe('built-in Swarm skills', () => {
expect(readdirSync(data)).toEqual([])
})
- // The owner's approved text (2026-10-22), byte for byte.
+ // The owner's approved text (2026-10-22), with Lead recovery guidance agreed on 2026-10-23.
it.each([
- ['lead', 'a8d54ee204209946486c8d99960f1317e4bd3e51434af894b174e712b67b7941'],
+ ['lead', '5fc83d8c43b66b44cdab27b48235dbc86b7ab11a57153edc1ae7e92831642a8e'],
['dev', '86caf697c83365f000317845b0aaa929947a9591d637ab2bd0f8536da8e86763'],
['review', '424dc9db5642ed7093128525f30877ffc301eb6957b33e8234916de0f0eaa208'],
] as const)('writes the owner\'s %s skill byte for byte', (role, sha256) => {
diff --git a/cli/src/lib/swarmSkills.ts b/cli/src/lib/swarmSkills.ts
index f82605620..6623f60cf 100644
--- a/cli/src/lib/swarmSkills.ts
+++ b/cli/src/lib/swarmSkills.ts
@@ -104,6 +104,7 @@ For large work, post the group plan and get the owner's ok before assigning; onc
- The brief is all a Builder gets. Never pass your chat history; put everything needed in the brief.
## 7. Run the work
+- When a restart interrupts active work, read the recovery notice and its last recorded checkpoint, then check the agent’s actual session and saved work. Ask it to continue from that checkpoint and report working; either you or the assigned agent can clear the recovery block. Never replay old input or commands: they may already have taken effect. A delayed recording gap after fresh progress is not a new interruption. Keep work already in review in review while you inspect the session. Ordinary typed input, an expired recording view, a missing recorded prompt, or an unclassified gap from an older recorder does not establish that work was interrupted.
- Needs-human: get the person's decision now.
- Blocked: change something. Give more context, re-route to a stronger model, cut the work smaller, or fix the plan. Never resend the same thing to the same model.
- Rulings, not stalls: when a Builder hits an ambiguity you can settle, settle it and record "Ruling: what — why — cost if wrong". Ask the person only what changes what gets built.
diff --git a/cli/src/superTabs/native.spec.ts b/cli/src/superTabs/native.spec.ts
index 503338464..103931987 100644
--- a/cli/src/superTabs/native.spec.ts
+++ b/cli/src/superTabs/native.spec.ts
@@ -11,7 +11,7 @@ import { NativeRecorder, nativeJournalFile } from './native.js'
const roots: string[] = []
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) })
const alpha = 'context_alpha_123456', beta = 'context_beta_1234567'
-function setup() {
+function setup(recovery = false) {
const root = mkdtempSync(join(tmpdir(), 'super-tab-native-')); roots.push(root)
let now = 100, revoked = false, loseReply = false
const observer = createNativeTurnObservations({ resolve: () => ({ sessionId: 'session', engine: 'codex', runtime: 'runtime' }), now: () => 100 })
@@ -20,7 +20,8 @@ function setup() {
const backend = vi.fn(async (method: string, path: string, body?: unknown) => {
if (revoked) return { status: 403, body: { success: false, error: { code: 'NATIVE_CONTEXT_REVOKED', message: 'Removed' } } }
const id = path.split('/').at(-1)!, context = contexts.get(id)!
- if (method === 'GET') return { status: 200, body: { data: { contextId: id, tabId: 'tab', machineId: 'machine-1', harnessId: 'harness', ...context } } }
+ if (method === 'GET') return { status: 200, body: { data: { contextId: id, tabId: 'tab', machineId: 'machine-1', harnessId: 'harness', ...context,
+ ...(recovery ? { recovery: { version: 1, serverTime: now + 900 } } : {}) } } }
const value = body as Record<string, unknown>
sent.push(value)
const eventId = createHash('sha256').update(JSON.stringify(['native-event', id, value.turnId, value.kind])).digest('hex').slice(0, 24)
@@ -47,6 +48,98 @@ function setup() {
}
describe('durable native conversation recording', () => {
+ it('identifies a recording restart when preparing a previously recorded context, without replaying input', async () => {
+ const t = setup(true)
+ t.contexts.get(alpha)!.lastEventId = 'previous_event'
+ const host = t.open()
+ await host.prepare('tab', alpha); await host.flush()
+ expect(t.sent).toEqual([expect.objectContaining({ kind: 'gap', previousEventId: 'previous_event',
+ gap: { incidentId: expect.any(String), reason: 'recording_restarted', detectedAt: 1000 } })])
+ expect(t.core.turns.deliver).not.toHaveBeenCalled()
+ })
+
+ it('uses the same restart incident across capture and preparation, and preserves it through a lost reply', async () => {
+ const t = setup(true), first = t.open()
+ await first.prepare('tab', alpha); t.prompt('Checkpoint'); await first.capture(); await first.flush()
+ t.contexts.get(beta)!.lastEventId = 'other_checkpoint'
+ t.observer.ingest('harness', [], true)
+ t.time(200)
+ await first.prepare('tab', beta)
+ await first.capture()
+ const gaps = t.saved().agents.harness.outbox
+ expect(gaps).toHaveLength(2)
+ expect(gaps[0].gap).toEqual({ incidentId: expect.any(String), reason: 'recording_restarted', detectedAt: 1100 })
+ expect(gaps[1].gap).toEqual(gaps[0].gap)
+ t.lose(); await expect(first.flush()).rejects.toThrow(); first.stop()
+ const restarted = t.open(); await restarted.capture(); await restarted.flush(); await restarted.capture(); await restarted.flush()
+ expect(t.sent.slice(1)).toHaveLength(3)
+ expect(t.sent[1]).toEqual(t.sent[2])
+ expect(t.records.size).toBe(3)
+ expect(t.core.turns.deliver).not.toHaveBeenCalled()
+ })
+
+ it('keeps the old wire format for a backend without recovery support, including saved outboxes', async () => {
+ const t = setup(), first = t.open()
+ t.contexts.get(alpha)!.lastEventId = 'previous_event'
+ await first.prepare('tab', alpha); first.stop()
+ const restarted = t.open(); await restarted.flush()
+ expect(t.sent).toEqual([{ turnId: expect.any(String), kind: 'gap', previousEventId: 'previous_event',
+ text: '', truncated: false, interrupted: false }])
+ })
+
+ it('reopens a retired context in the same epoch without reusing an observation with different content', async () => {
+ const t = setup(true), host = t.open()
+ t.contexts.get(alpha)!.lastEventId = 'previous_event'
+ await host.prepare('tab', alpha); await host.flush()
+ t.time(100_000); await host.capture()
+ expect(host.hasWork).toBe(false)
+ await host.prepare('tab', alpha); await host.flush()
+ expect(t.sent).toHaveLength(2)
+ expect(t.sent[0]).toMatchObject({ gap: { incidentId: expect.any(String) } })
+ expect(t.sent[1]!.gap).toMatchObject({ incidentId: (t.sent[0]!.gap as { incidentId: string }).incidentId })
+ expect(t.sent[1]!.turnId).not.toBe(t.sent[0]!.turnId)
+ expect(t.records.size).toBe(2)
+ })
+
+ it.each(['incident', 'reason', 'time', 'kind'] as const)('rejects corrupted recovery %s in a saved outbox', async field => {
+ const t = setup(true), host = t.open()
+ t.contexts.get(alpha)!.lastEventId = 'previous_event'
+ await host.prepare('tab', alpha); host.stop()
+ const saved = t.saved(), event = saved.agents.harness.outbox[0]
+ event.gap = { incidentId: 'restart', reason: 'recording_restarted', detectedAt: 1000 }
+ if (field === 'incident') event.gap.incidentId = '../invalid'
+ if (field === 'reason') event.gap.reason = 'not_a_reason'
+ if (field === 'time') event.gap.detectedAt = -1
+ if (field === 'kind') event.kind = 'prompt'
+ writeFileSync(t.file, JSON.stringify(saved))
+ expect(() => t.open()).toThrow('Invalid terminal recording gap')
+ expect(t.sent).toEqual([])
+ })
+
+ it('reports a confirmed stop with its detection time and retains the exact incident on retry', async () => {
+ const t = setup(true), host = t.open()
+ await host.prepare('tab', alpha); t.prompt('Unfinished'); await host.capture(); await host.flush()
+ vi.mocked(t.core.transcripts.nativeTurns!).mockResolvedValue(null)
+ t.core.agents.stopped = vi.fn(async () => ({ agentId: 'harness' }) as never)
+ t.time(300); await host.capture()
+ expect(t.saved().agents.harness.outbox[0]).toMatchObject({ kind: 'gap', interrupted: true,
+ gap: { incidentId: expect.any(String), reason: 'agent_stopped', detectedAt: 1200 } })
+ await host.flush(); await host.capture()
+ expect(t.sent).toHaveLength(2)
+ })
+
+ it('distinguishes lost history and unattributed input from a recording restart', async () => {
+ const t = setup(true), host = t.open()
+ await host.prepare('tab', alpha)
+ t.prompt('Unknown input', null); await host.capture(); await host.flush()
+ expect(t.sent[0]).toMatchObject({ gap: { reason: 'unattributed_input', detectedAt: 1000 } })
+ const cursor = t.saved().agents.harness.cursor
+ vi.mocked(t.core.transcripts.nativeTurns!).mockResolvedValueOnce({ cursor: { ...cursor, sequence: cursor.sequence + 1 }, observations: [], gap: true })
+ await host.capture(); await host.flush()
+ expect(t.sent[1]).toMatchObject({ gap: { reason: 'history_lost', detectedAt: 1000 } })
+ expect(t.sent[0]!.turnId).not.toEqual(t.sent[1]!.turnId)
+ })
+
it('says why an unidentified terminal cannot be recorded instead of asking for an update', async () => {
const t = setup()
const transcripts = t.core.transcripts as { nativeTurns?: unknown }
diff --git a/cli/src/superTabs/native.ts b/cli/src/superTabs/native.ts
index 35fc49693..44c997766 100644
--- a/cli/src/superTabs/native.ts
+++ b/cli/src/superTabs/native.ts
@@ -9,10 +9,14 @@ import { identifier, object } from './wire.js'
interface Context {
tabId: string; baseline: NativeTurnCursor; until: number; head: string | null; blocked?: string
+ recoveryClockOffset?: number; lastGapId?: string
}
+const GAP_REASONS = ['recording_restarted', 'history_lost', 'unattributed_input', 'context_expired', 'agent_stopped', 'prompt_missing'] as const
+interface Gap { incidentId: string; reason: typeof GAP_REASONS[number]; detectedAt: number }
interface Event {
contextId: string; turnId: string; kind: 'prompt' | 'answer' | 'gap'; previousEventId: string | null
text: string; truncated: boolean; interrupted: boolean
+ gap?: Gap
}
interface RecordedAgent {
cursor: NativeTurnCursor; contexts: Record<string, Context>; turns: Record<string, string>; outbox: Event[]
@@ -79,7 +83,9 @@ export class NativeRecorder {
for (const [id, entry] of Object.entries(contexts)) {
const c = object(entry)
if (!identifier(id) || !identifier(c.tabId) || !nativeTurnCursorIn(c.baseline) || !Number.isFinite(c.until)
- || !validIdOrNull(c.head) || (c.blocked !== undefined && typeof c.blocked !== 'string')) throw new Error('Invalid terminal context.')
+ || !validIdOrNull(c.head) || (c.blocked !== undefined && typeof c.blocked !== 'string')
+ || (c.recoveryClockOffset !== undefined && !Number.isFinite(c.recoveryClockOffset))
+ || (c.lastGapId !== undefined && !identifier(c.lastGapId))) throw new Error('Invalid terminal context.')
}
for (const [id, contextId] of Object.entries(turns)) if (!identifier(id) || !identifier(contextId) || !contexts[contextId]) throw new Error('Invalid terminal turn.')
for (const value of row.outbox) {
@@ -87,6 +93,11 @@ export class NativeRecorder {
if (!identifier(e.contextId) || !contexts[e.contextId] || !identifier(e.turnId) || !['prompt', 'answer', 'gap'].includes(String(e.kind))
|| !validIdOrNull(e.previousEventId) || typeof e.text !== 'string' || e.text.length > 16_000
|| typeof e.truncated !== 'boolean' || typeof e.interrupted !== 'boolean') throw new Error('Invalid terminal observation.')
+ if (e.gap !== undefined) {
+ const gap = object(e.gap)
+ if (e.kind !== 'gap' || !identifier(gap.incidentId) || !GAP_REASONS.includes(gap.reason as Gap['reason'])
+ || !Number.isSafeInteger(gap.detectedAt) || (gap.detectedAt as number) < 0) throw new Error('Invalid terminal recording gap.')
+ }
}
}
if (Object.keys(object(saved.agents)).length > 128) throw new Error('Too many recorded terminals.')
@@ -126,6 +137,9 @@ export class NativeRecorder {
this.available()
if (!identifier(tabId) || !identifier(contextId) || closed?.aborted) throw new Error('This terminal request is no longer available.')
const verified = object(await backend(this.core, 'GET', `/api/super-tabs/${tabId}/native/${contextId}`))
+ const recovery = object(verified.recovery)
+ const recoveryClockOffset = recovery.version === 1 && Number.isSafeInteger(recovery.serverTime)
+ ? (recovery.serverTime as number) - this.now() : undefined
this.available()
const harnessId = verified.harnessId
if (closed?.aborted || verified.contextId !== contextId || verified.tabId !== tabId || verified.machineId !== this.machineId
@@ -145,10 +159,14 @@ export class NativeRecorder {
if (!existing && Object.keys(this.journal.agents[harnessId]?.contexts ?? {}).length >= 256) throw new Error('Too many terminal contexts are active.')
this.commit(next => {
const row = next.agents[harnessId] ??= { cursor: page.cursor, contexts: {}, turns: {}, outbox: [] }
- if (existing) row.contexts[contextId]!.until = this.now() + CONTEXT_LEASE
+ if (existing) {
+ row.contexts[contextId]!.until = this.now() + CONTEXT_LEASE
+ row.contexts[contextId]!.recoveryClockOffset = recoveryClockOffset
+ }
else {
- row.contexts[contextId] = { tabId, baseline: page.cursor, until: this.now() + CONTEXT_LEASE, head: verified.lastEventId as string | null }
- if (verified.lastEventId) this.enqueue(row, contextId, { turnId: randomUUID(), kind: 'gap', text: '', truncated: false, interrupted: false })
+ row.contexts[contextId] = { tabId, baseline: page.cursor, until: this.now() + CONTEXT_LEASE,
+ head: verified.lastEventId as string | null, recoveryClockOffset }
+ if (verified.lastEventId) this.gap(row, contextId, harnessId, page.cursor.epoch, 'recording_restarted')
}
})
return { ok: true, inputContext: contextId }
@@ -162,6 +180,21 @@ export class NativeRecorder {
row.outbox.push({ ...event, contextId, previousEventId: last ? eventId(last) : context.head })
}
+ private gap(row: RecordedAgent, contextId: string, harnessId: string, boundary: string, reason: Gap['reason'], interrupted = false): void {
+ const context = row.contexts[contextId]!
+ // Preparation and capture can both see the same restarted core. The epoch
+ // identifies that incident across views and durable outbox retries.
+ const incidentId = createHash('sha256').update(JSON.stringify(['native-gap', this.machineId, harnessId, reason, boundary])).digest('hex').slice(0, 24)
+ if (context.lastGapId === incidentId) return
+ // An expired view can be prepared again in the same core epoch. It reports
+ // the same incident but appends at a new head, so it needs its own observation.
+ const last = row.outbox.findLast(event => event.contextId === contextId)
+ const turnId = createHash('sha256').update(JSON.stringify([incidentId, last ? eventId(last) : context.head])).digest('hex').slice(0, 24)
+ this.enqueue(row, contextId, { turnId, kind: 'gap', text: '', truncated: false, interrupted,
+ ...(context.recoveryClockOffset === undefined ? {} : { gap: { incidentId, reason, detectedAt: this.now() + context.recoveryClockOffset } }) })
+ context.lastGapId = incidentId
+ }
+
private expired(row: RecordedAgent): string[] {
const retained = new Set([...Object.values(row.turns), ...row.outbox.map(e => e.contextId)])
return Object.keys(row.contexts).filter(id => row.contexts[id]!.until <= this.now() && !retained.has(id))
@@ -188,8 +221,7 @@ export class NativeRecorder {
if (await this.core.agents.stopped?.(harnessId)) {
this.commit(next => {
const row = next.agents[harnessId]!
- for (const id of new Set(Object.values(row.turns))) this.enqueue(row, id,
- { turnId: randomUUID(), kind: 'gap', text: '', truncated: false, interrupted: true })
+ for (const id of new Set(Object.values(row.turns))) this.gap(row, id, harnessId, row.cursor.epoch, 'agent_stopped', true)
row.turns = {}
for (const c of Object.values(row.contexts)) c.until = 0
this.prune(next, harnessId)
@@ -206,14 +238,15 @@ export class NativeRecorder {
this.commit(next => {
const row = next.agents[harnessId]!
const gaps = new Set<string>()
- const gap = (id: string) => {
+ const gap = (id: string, reason: Gap['reason']) => {
if (gaps.has(id)) return
gaps.add(id)
- this.enqueue(row, id, { turnId: randomUUID(), kind: 'gap', text: '', truncated: false, interrupted: false })
+ this.gap(row, id, harnessId, reason === 'recording_restarted' ? page.cursor.epoch
+ : `${page.cursor.epoch}:${page.cursor.sequence}`, reason)
}
if (page.gap || page.observations.some(event => event.kind === 'prompt' && event.unattributedInput)) {
for (const [id, c] of Object.entries(row.contexts)) {
- if (!c.blocked) gap(id)
+ if (!c.blocked) gap(id, page.gap ? (page.cursor.epoch !== before.cursor.epoch ? 'recording_restarted' : 'history_lost') : 'unattributed_input')
}
if (page.gap) row.turns = {}
}
@@ -224,14 +257,14 @@ export class NativeRecorder {
if (!contextId || !context || context.blocked) continue
if (context.baseline.epoch === page.cursor.epoch && observation.sequence <= context.baseline.sequence) continue
if (observation.kind === 'prompt') {
- if (context.until <= this.now()) { gap(contextId); continue }
+ if (context.until <= this.now()) { gap(contextId, 'context_expired'); continue }
row.turns[observation.turnId] = contextId
this.enqueue(row, contextId, copyEvent(observation))
} else if (row.turns[observation.turnId] === contextId) {
this.enqueue(row, contextId, copyEvent(observation))
delete row.turns[observation.turnId]
} else if (context.until > this.now()) {
- this.enqueue(row, contextId, { turnId: observation.turnId, kind: 'gap', text: '', truncated: false, interrupted: true })
+ this.gap(row, contextId, harnessId, observation.turnId, 'prompt_missing', true)
}
}
row.cursor = page.cursor