Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Backend hotfix: idle delivery poll reads one row, host read skips the newest-change aggregate, host retry backs off (#1785)

0xken15db15e01c0474
diff --git a/backend/integration/superTabLiveness.test.ts b/backend/integration/superTabLiveness.test.ts
index 8d6e61d34..b89732245 100644
--- a/backend/integration/superTabLiveness.test.ts
+++ b/backend/integration/superTabLiveness.test.ts
@@ -44,9 +44,9 @@ afterAll(async () => {
})
/** A Swarm whose Lead runs on one of host's machines and whose worker runs on another, assigned "Ship the parser". */
-async function swarm({ legacy = false } = {}) {
+async function swarm({ legacy = false, client = db }: { legacy?: boolean; client?: PrismaClient } = {}) {
let now = 10_000_000
- const timed = new SuperTabStore(db, () => now)
+ const timed = new SuperTabStore(client, () => now)
const tab = (await timed.create(owner, randomUUID(), 'Liveness', legacy ? undefined : 'public')).id
if (legacy) await dropProjects(tab)
const execute = (command: Command, user = owner, asAgentId?: string) => timed.execute(user, tab, { operationId: randomUUID(), command }, asAgentId)
@@ -507,6 +507,62 @@ describe('a start that never runs', () => {
})
})
+/** Counts every Prisma operation a store makes through this client, as "Model.operation". */
+function counting() {
+ const ops: string[] = []
+ const client = db.$extends({ query: { $allModels: { async $allOperations({ model, operation, args, query }) {
+ ops.push(`${model}.${operation}`)
+ return query(args)
+ } } } }) as unknown as PrismaClient
+ return { client, ops }
+}
+
+// Every running agent's host polls its deliveries each second (a production backend went to 5 pods on this, 2026-10-27):
+// with nothing waiting the answer must cost one indexed read, and the Swarm is loaded only for the time checks, every 30 s.
+describe('the delivery poll of an agent with nothing waiting', () => {
+ it('reads one row, not the Swarm, until 30 seconds after its last full poll', async () => {
+ const counted = counting()
+ const f = await swarm({ client: counted.client })
+ expect(await f.timed.deliveries(host, f.tab, f.lead)).toEqual([])
+ counted.ops.length = 0
+ f.clock.now += 29_999
+ expect(await f.timed.deliveries(host, f.tab, f.lead)).toEqual([])
+ expect(counted.ops).toEqual(['SuperTabDelivery.findFirst'])
+ counted.ops.length = 0
+ f.clock.now += 1
+ expect(await f.timed.deliveries(host, f.tab, f.lead)).toEqual([])
+ expect(counted.ops).toContain('SwarmGroup.findMany')
+ })
+
+ it('still runs the Lead\'s time checks: the five-minute update request arrives once 30 seconds have passed', async () => {
+ const f = await swarm()
+ await f.leadInbox()
+ f.clock.now += 5 * MINUTE
+ const told = await f.leadInbox()
+ expect(told.map(m => (m.source as { kind?: string }).kind)).toEqual(['update-request'])
+ })
+
+ it('delivers a message on the very next poll, within a second of the last one', async () => {
+ const counted = counting()
+ const f = await swarm({ client: counted.client })
+ await f.timed.deliveries(host, f.tab, f.lead)
+ f.clock.now += 1_000
+ const sent = await f.execute({ type: 'message.send', conversationId: 'lead', text: 'Status?', replyTo: null, recipientIds: [] })
+ f.clock.now += 1_000
+ const rows = await f.timed.deliveries(host, f.tab, f.lead)
+ expect(rows.map(d => d.messageId)).toEqual([sent.result.messageId])
+ })
+
+ it('never answers a caller who is not the agent\'s host from another caller\'s quiet poll, before or after it fails', async () => {
+ const f = await swarm()
+ await f.timed.deliveries(host, f.tab, f.lead)
+ f.clock.now += 1_000
+ for (let attempt = 0; attempt < 2; attempt++) {
+ await expect(f.timed.deliveries(owner, f.tab, f.lead)).rejects.toMatchObject({ code: 'FORBIDDEN' })
+ }
+ })
+})
+
describe('the morning digest', () => {
it('is sent once when two of the Lead\'s polls run at the same moment', async () => {
const f = await swarm()
diff --git a/backend/integration/superTabRead.test.ts b/backend/integration/superTabRead.test.ts
index 7edc3054e..35966a54e 100644
--- a/backend/integration/superTabRead.test.ts
+++ b/backend/integration/superTabRead.test.ts
@@ -845,4 +845,32 @@ describe('what a read costs the database (group 3b)', () => {
// From version columns alone: building the rows from light conversations costs about 39 KB here, this about 10 KB.
expect(bytes).toBeLessThan(20_000)
})
+
+ // Every host reads each second (a production backend went to 5 pods on it, 2026-10-27): a Swarm with no change for it
+ // must not cost the newest-change aggregate, seven groupBy over all its parts.
+ it('the host form asks for the newest change only of the Swarms it answers for', async () => {
+ const machineId = `m-${randomUUID().slice(0, 8)}`
+ await db.machine.create({ data: { machineId, userId: owner.sub, apiKey: 'fixture-only', managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })
+ const s = await scene(f => { f.contribute(machineId, 'swarm') }, machineId)
+ const ops: string[] = []
+ const client = db.$extends({ query: { $allModels: { async $allOperations({ model, operation, args, query }) {
+ ops.push(`${model}.${operation}`)
+ return query(args)
+ } } } }) as unknown as PrismaClient
+ const hosts = new HostReads(new SuperTabStore(client), { now: () => clock })
+ clock = Date.now()
+ const first = await hosts.changes(owner, machineId, { full: true })
+ expect(ops.filter(op => op.endsWith('.groupBy'))).toHaveLength(7)
+ ops.length = 0
+ clock = Date.now()
+ const idle = await hosts.changes(owner, machineId, { cursor: first.cursor })
+ expect(idle.tabs).toEqual([])
+ expect(ops.filter(op => op.endsWith('.groupBy'))).toEqual([])
+ await new SuperTabStore(db).execute(owner, s.tabId, { operationId: randomUUID(),
+ command: { type: 'message.send', conversationId: s.web, text: 'Busy', replyTo: null, recipientIds: [] } })
+ clock = Date.now()
+ const woken = (await hosts.changes(owner, machineId, { cursor: idle.cursor })).tabs
+ const posted = (await db.swarmGroup.findFirst({ where: { tabId: s.tabId, key: s.web } }))!.changedAt.getTime()
+ expect(woken).toMatchObject([{ tabId: s.tabId, lastChangedAt: posted }])
+ })
})
diff --git a/backend/integration/superTabs.test.ts b/backend/integration/superTabs.test.ts
index 11c655eaa..996ba33af 100644
--- a/backend/integration/superTabs.test.ts
+++ b/backend/integration/superTabs.test.ts
@@ -694,7 +694,8 @@ describe('private swarm projects on the real control API', () => {
expect(await updates()).toEqual([])
now += 299_999
expect(await updates()).toEqual([])
- now++
+ // The quiet poll skips the time checks for 30 s after a full one (store.deliveries), so the request lands within that.
+ now += 30_000
const polls = await Promise.all(Array.from({ length: 3 }, () => updates()))
const ids = new Set(polls.flat().map(d => d.id))
expect(ids.size).toBe(1)
@@ -1203,7 +1204,8 @@ describe('Super Tabs on a real Mongo replica set', () => {
await store.delivery(host, tab, lead, first.id, { type: 'ack', claimId: 'claim', receipt: 'unclear', uncertain: true })
await run(tab, { type: 'project.delete', projectId: 'lead' })
expect(await store.deliveryReceipt(host, tab, lead, first.id)).toEqual({ id: first.id, status: 'cancelled' })
- await expect(store.deliveries(host, tab, lead)).rejects.toMatchObject({ code: 'AGENT_UNAVAILABLE' })
+ // A server that has not just polled for this host (the quiet poll answers an unchanged caller for 30 s) refuses at once.
+ await expect(new SuperTabStore(db).deliveries(host, tab, lead)).rejects.toMatchObject({ code: 'AGENT_UNAVAILABLE' })
await expect(store.deliveryReceipt(peer, tab, lead, first.id)).rejects.toMatchObject({ code: 'FORBIDDEN' })
// Reassigning the machine does not transfer the former owner's receipts.
await db.machine.update({ where: { machineId: 'test-host' }, data: { userId: peer.sub } })
diff --git a/backend/src/superTabs/read/host.ts b/backend/src/superTabs/read/host.ts
index 58901f17d..57b5ea716 100644
--- a/backend/src/superTabs/read/host.ts
+++ b/backend/src/superTabs/read/host.ts
@@ -57,9 +57,11 @@ export class HostReads {
orderBy: { updatedAt: 'desc' }, take: FULL_TABS, select: { id: true } })).map(r => r.id)
const teams = isTeamSession(user) || this.options.teamAccountsIdle ? [] : await tx.machine.findMany({ where: { userId: user.sub,
hostMachineId: machineId, sessionKind: 'team', autonomousEnv: user.autonomousEnv, ...machineAlive }, select: { machineId: true } })
- const newest = await parts.lastChangedAt(tx, ids)
const full = 'reload' in from
const woken = full ? null : await this.woken(tx, ids, machineId, from.since)
+ // Only for the Swarms answered for: this runs every second on every host, and seven groupBy over all of a Swarm's parts
+ // are no use for one that has nothing to say (a production backend at 5 pods, 2026-10-27).
+ const newest = await parts.lastChangedAt(tx, woken ? ids.filter(id => woken.has(id)) : ids)
const tabs = []
for (const tabId of ids) {
if (woken && !woken.has(tabId)) continue
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index 08ba1a3ab..ac5a6f55c 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -136,6 +136,10 @@ export async function transaction<T>(db: Pick<PrismaClient, '$transaction'>, bod
}
}
const TRANSACTION_OPTIONS = { maxWait: 5_000, timeout: 15_000 }
+/** The deliveries a host is still told about (the rest are answered, cancelled or confirmed). */
+const OPEN_DELIVERY = ['queued', 'claimed', 'uncertain'] as const
+/** Callers kept for the quiet delivery poll before the stale ones are dropped. */
+const POLLED_CAP = 5_000
const TRANSACTION_ATTEMPTS = 8
const TRANSACTION_BACKOFF_MS = 50
const TRANSACTION_BACKOFF_CAP_MS = 1_500
@@ -1550,7 +1554,26 @@ export class SuperTabStore {
catch (error) { if (error instanceof SuperTabError) return false; throw error }
}
+ /** When each caller's full delivery poll last succeeded for an agent, on this server: key is tab, agent and caller. */
+ private readonly polled = new Map<string, number>()
+
+ /**
+ * Every running agent's host polls this each second. With nothing waiting and the full poll (which loads the Swarm and
+ * runs the Lead's time checks) done within SWEEP_EVERY_MS, the answer is one indexed read: 15 agents were ~180 queries
+ * a second and a whole-Swarm rewrite check each, on 2026-10-27 the backend at 5 pods. Only a caller whose full poll
+ * succeeded is ever answered this way, so a stranger or a revoked host meets the authority checks every time.
+ */
async deliveries(user: AuthUser, tabId: string, agentId: string) {
+ const key = `${tabId}:${agentId}:${user.sub}`, last = this.polled.get(key)
+ if (last !== undefined && this.now() - last < SWEEP_EVERY_MS
+ && !await this.db.superTabDelivery.findFirst({ where: { tabId, agentId, status: { in: [...OPEN_DELIVERY] } }, select: { id: true } })) return []
+ const rows = await this.fullDeliveries(user, tabId, agentId)
+ this.polled.set(key, this.now())
+ if (this.polled.size > POLLED_CAP) for (const [k, at] of this.polled) if (this.now() - at >= SWEEP_EVERY_MS) this.polled.delete(k)
+ return rows
+ }
+
+ private async fullDeliveries(user: AuthUser, tabId: string, agentId: string) {
return transaction(this.db, async tx => {
let loaded = await this.load(tx, user, tabId)
// Work finished since the last digest and archived or asleep since still shipped, but ordinary loads leave it out
@@ -1599,7 +1622,7 @@ export class SuperTabStore {
const sequences = numberMessages(state, written, this.now())
await this.save(tx, loaded, state, clock.next(), [...judgedMachines(state, written), ...woken ? [{ kind: 'agent' as const, key: agentId }] : []])
await this.writeMessages(tx, tabId, state, written, sequences)
- const rows = await tx.superTabDelivery.findMany({ where: { tabId, agentId, status: { in: ['queued', 'claimed', 'uncertain'] } },
+ const rows = await tx.superTabDelivery.findMany({ where: { tabId, agentId, status: { in: [...OPEN_DELIVERY] } },
orderBy: { createdAt: 'asc' }, take: 100 })
const allowed: SuperTabDelivery[] = []
for (const delivery of rows) {
diff --git a/cli/src/superTabs/host.spec.ts b/cli/src/superTabs/host.spec.ts
index f4f3accf9..107d2aee9 100644
--- a/cli/src/superTabs/host.spec.ts
+++ b/cli/src/superTabs/host.spec.ts
@@ -460,7 +460,7 @@ describe('Super Tab native delivery handoff', () => {
await host.tick(); t.revoke(); await host.tick()
expect(t.core.turns.cancelDelivery).toHaveBeenCalledWith('team:service:superTabs:delivery')
expect(t.core.turns.leaseDeliveries).toHaveBeenLastCalledWith('superTabs', {})
- t.revoke(false); await host.tick()
+ t.revoke(false); t.wait(1_000); await host.tick()
expect(t.core.turns.deliver).toHaveBeenCalledTimes(1)
expect(t.receipts[0].uncertain).toBe(true)
t.remove(); await host.tick()
@@ -533,6 +533,148 @@ describe('Super Tab native delivery handoff', () => {
})
})
+describe('a delivery poll or claim that keeps failing', () => {
+ /** The calls the host makes for one failing path, counted as the backend refuses them with `status`. */
+ function failing(t: ReturnType<typeof setup>, refuse: (method: string, path: string) => boolean, status = 503) {
+ const original = vi.mocked(t.core.account.backend).getMockImplementation()!
+ const calls = { n: 0 }
+ vi.mocked(t.core.account.backend).mockImplementation(async (method, path, body) => {
+ if (!refuse(method, path)) return original(method, path, body)
+ calls.n++
+ return { status, body: { success: false, error: { code: 'SWARM_BUSY', message: 'This Swarm is busy. Try again in a moment.' } } }
+ })
+ return calls
+ }
+
+ it('asks again for the claim after 1 s, 2 s, 4 s and so on, never later than 30 s, and not in between', async () => {
+ const t = setup(), host = t.open()
+ const claims = failing(t, (method, path) => method === 'POST' && path.endsWith('/deliveries/delivery'))
+ await host.tick()
+ expect(claims.n).toBe(1)
+ await host.tick()
+ expect(claims.n).toBe(1)
+ t.wait(999); await host.tick()
+ expect(claims.n).toBe(1)
+ t.wait(1); await host.tick()
+ expect(claims.n).toBe(2)
+ t.wait(1_999); await host.tick()
+ expect(claims.n).toBe(2)
+ t.wait(1); await host.tick()
+ expect(claims.n).toBe(3)
+ // 4 s, 8 s, 16 s, then 30 s every time.
+ for (const [wait, attempts] of [[4_000, 4], [8_000, 5], [16_000, 6], [30_000, 7], [30_000, 8]] as const) {
+ t.wait(wait - 1); await host.tick()
+ expect(claims.n).toBe(attempts - 1)
+ t.wait(1); await host.tick()
+ expect(claims.n).toBe(attempts)
+ }
+ })
+
+ it('claims at once, on the next second, after the backend answers again', async () => {
+ const t = setup(), host = t.open()
+ let refuse = true
+ failing(t, (method, path) => refuse && method === 'POST' && path.endsWith('/deliveries/delivery'))
+ await host.tick()
+ t.wait(1_000); await host.tick()
+ refuse = false
+ t.wait(2_000); await host.tick()
+ expect(t.saved().entries.delivery.stage).toBe('submitting')
+ expect(t.core.turns.deliver).toHaveBeenCalledTimes(1)
+ })
+
+ it('keeps a claim\'s growing wait through a failed list, until a list that is read says the delivery is gone', async () => {
+ const t = setup(), host = t.open()
+ let refuseList = false, refuseClaim = true
+ const claims = { n: 0 }
+ const original = vi.mocked(t.core.account.backend).getMockImplementation()!
+ vi.mocked(t.core.account.backend).mockImplementation(async (method, path, body) => {
+ const list = method === 'GET' && path.endsWith('/deliveries'), claim = method === 'POST' && path.endsWith('/deliveries/delivery')
+ if ((list && refuseList) || (claim && refuseClaim)) {
+ if (claim) claims.n++
+ return { status: 503, body: { success: false, error: { code: 'SWARM_BUSY', message: 'This Swarm is busy. Try again in a moment.' } } }
+ }
+ return original(method, path, body)
+ })
+ await host.tick()
+ expect(claims.n).toBe(1)
+ // The list fails while the claim waits 1 s, then waits 1 s itself.
+ refuseList = true; t.wait(1_000); await host.tick()
+ refuseList = false; t.wait(1_000); await host.tick()
+ expect(claims.n).toBe(2)
+ // Two failures in a row: 2 s, not 1 s again.
+ t.wait(1_999); await host.tick()
+ expect(claims.n).toBe(2)
+ t.wait(1); await host.tick()
+ expect(claims.n).toBe(3)
+ })
+
+ it('keeps the wait of a delivery later in the list when an earlier one\'s claim throws', async () => {
+ const t = setup(), host = t.open()
+ const claims: Record<string, number> = { first: 0, second: 0 }
+ const original = vi.mocked(t.core.account.backend).getMockImplementation()!
+ vi.mocked(t.core.account.backend).mockImplementation(async (method, path, body) => {
+ if (method === 'GET' && path.endsWith('/deliveries')) {
+ return { status: 200, body: { success: true, data: ['first', 'second'].map(id => ({ id, conversationId: 'lead', claimId: null, status: 'queued' })) } }
+ }
+ const claim = /\/deliveries\/(first|second)$/.exec(path)
+ if (method === 'POST' && claim) {
+ claims[claim[1]!]!++
+ return { status: 503, body: { success: false, error: { code: 'SWARM_BUSY', message: 'This Swarm is busy. Try again in a moment.' } } }
+ }
+ return original(method, path, body)
+ })
+ // The first claim throws and ends the poll; while it waits, the second is tried and waits 1 s.
+ await host.tick(); await host.tick()
+ expect(claims).toEqual({ first: 1, second: 1 })
+ // Both are due; the first throws again before the second is reached, and the second is then tried and fails a second time.
+ t.wait(1_000); await host.tick(); await host.tick()
+ expect(claims).toEqual({ first: 2, second: 2 })
+ // The second waits 2 s now, not 1 s again.
+ t.wait(1_000); await host.tick()
+ expect(claims.second).toBe(2)
+ t.wait(1_000); await host.tick(); await host.tick()
+ expect(claims.second).toBe(3)
+ })
+
+ it('drops a claim\'s wait once a list that is read no longer has the delivery', async () => {
+ const t = setup(), host = t.open()
+ const claims = { n: 0 }
+ let gone = false
+ const original = vi.mocked(t.core.account.backend).getMockImplementation()!
+ vi.mocked(t.core.account.backend).mockImplementation(async (method, path, body) => {
+ if (method === 'GET' && path.endsWith('/deliveries') && gone) return { status: 200, body: { success: true, data: [] } }
+ if (method === 'POST' && path.endsWith('/deliveries/delivery')) {
+ claims.n++
+ return { status: 503, body: { success: false, error: { code: 'SWARM_BUSY', message: 'This Swarm is busy. Try again in a moment.' } } }
+ }
+ return original(method, path, body)
+ })
+ await host.tick()
+ t.wait(1_000); await host.tick()
+ expect(claims.n).toBe(2)
+ gone = true; t.wait(2_000); await host.tick()
+ gone = false; t.wait(0); await host.tick()
+ // Back again as a new delivery: asked at once, with a fresh 1 s wait.
+ expect(claims.n).toBe(3)
+ t.wait(1_000); await host.tick()
+ expect(claims.n).toBe(4)
+ })
+
+ it('asks again for a delivery list the backend refuses only after the same growing wait', async () => {
+ const t = setup(), host = t.open()
+ const lists = failing(t, (method, path) => method === 'GET' && path.endsWith('/deliveries'))
+ await host.tick()
+ await host.tick()
+ expect(lists.n).toBe(1)
+ t.wait(1_000); await host.tick()
+ expect(lists.n).toBe(2)
+ t.wait(1_999); await host.tick()
+ expect(lists.n).toBe(2)
+ t.wait(1); await host.tick()
+ expect(lists.n).toBe(3)
+ })
+})
+
it('confirms a stopped assignment only with fresh native evidence, including after contribution removal', async () => {
const t = setup()
const stopped = vi.fn(async (id: string) => id === 'gone')
diff --git a/cli/src/superTabs/host.ts b/cli/src/superTabs/host.ts
index 79c6c8c6b..7b9ac8b78 100644
--- a/cli/src/superTabs/host.ts
+++ b/cli/src/superTabs/host.ts
@@ -131,6 +131,10 @@ const STOPS_FRESH_MS = 5_000
const JOIN_FINAL = new Set(['MACHINES_FULL', 'OWNER_REQUIRED'])
/** A Lead's reopen that failed (an engine that cannot start, a busy pane) waits this long, doubling up to half an hour, so a
* crash loop never spins; one the core took waits a minute for its session to show before it is asked again. */
+/** A delivery poll or claim the backend refused is asked again after 1 s, doubling to 30 s: every host retried each second, which
+ * added to an overloaded backend (production at 5 pods, 2026-10-27). */
+const DELIVERY_RETRY_MS = 1_000
+const DELIVERY_RETRY_MAX_MS = 30_000
const LEAD_RETRY_MS = 5_000
const LEAD_RETRY_MAX_MS = 30 * 60_000
const LEAD_REOPEN_WAIT_MS = 60_000
@@ -205,6 +209,8 @@ export class SuperTabHost {
private replacementsRunning: Promise<void> | null = null
private nextReplacementsAt = 0
private timer: ReturnType<typeof setTimeout> | null = null
+ /** Per agent's delivery list and per delivery's claim, the failures in a row and when it may be asked again. */
+ private readonly deliveryRetries = new Map<string, { failures: number; retryAt: number; list?: string }>()
private native: NativeRecorder | null = null
private machines: ContributedMachines | null = null
private readonly machineDeps: Partial<MachineDeps>
@@ -522,6 +528,15 @@ export class SuperTabHost {
private async leases(): Promise<boolean> {
return this.core.turns.leaseDeliveries('superTabs', Object.fromEntries(this.authorized))
}
+ private backedOff(key: string): boolean {
+ const retry = this.deliveryRetries.get(key)
+ return !!retry && this.now() < retry.retryAt
+ }
+ private retryLater(key: string, list?: string): void {
+ const retry = this.deliveryRetries.get(key) ?? { failures: 0, retryAt: 0, ...(list ? { list } : {}) }
+ retry.retryAt = this.now() + Math.min(DELIVERY_RETRY_MAX_MS, DELIVERY_RETRY_MS * 2 ** retry.failures++)
+ this.deliveryRetries.set(key, retry)
+ }
private async poll(): Promise<void> {
const data = (await this.sources.feed.refresh()).roster
if (this.stopped) return
@@ -573,6 +588,10 @@ export class SuperTabHost {
const stopKeys = new Set(data.filter(item => object(item).stopRequested === true).map(item => { const row = object(item); return `${row.tabId}:${object(row.agent).id}` }))
for (const key of this.releaseAttempts.keys()) if (!stopKeys.has(key)) this.releaseAttempts.delete(key)
const seen = new Set<string>()
+ /** The agents whose delivery list was read in this poll: only such a list can say a delivery is gone. */
+ const listsRead = new Set<string>()
+ /** Every delivery of those lists, also those after one whose claim threw: only a list read without it says it is gone. */
+ const listed = new Set<string>()
const running: HostAgent[] = []
const leads = new Set<string>(), ended = new Set<string>()
for (const item of data) {
@@ -598,26 +617,37 @@ export class SuperTabHost {
if (!this.core.agents.byAgent(target.agent.harnessId) || !this.core.agents.terminalAvailable(target.agent.harnessId)) continue
running.push(target)
const path = `/api/super-tabs/${target.tabId}/agents/${target.agent.id}/deliveries`
+ const listKey = `list:${target.tabId}:${target.agent.id}`
+ if (this.backedOff(listKey)) continue
+ let listRead = false
try {
const deliveries = await backend(this.core, 'GET', path)
if (this.stopped) return
if (!Array.isArray(deliveries)) throw new Error('Invalid Super Tab delivery list.')
+ listRead = true
+ listsRead.add(listKey)
+ for (const item of deliveries) { const id = object(item).id; if (identifier(id)) listed.add(id) }
+ this.deliveryRetries.delete(listKey)
for (const item of deliveries) {
const delivery = object(item)
if (!identifier(delivery.id)) continue
seen.add(delivery.id)
+ const claimKey = `claim:${delivery.id}`
+ if (this.backedOff(claimKey)) continue
// Checked before anything is typed: a new delivery, or one held while its agent may since have stopped leading. A
// claimed one with no journal is never typed, only reported as uncertain; a delivered one only acknowledged.
const entry = this.journal.entries[delivery.id]
if ((entry ? entry.stage === 'prepared' : !delivery.claimId) && !this.mayRead(target, delivery)) continue
- try { await this.accept(target, delivery, path) } catch (error) {
+ try { await this.accept(target, delivery, path); this.deliveryRetries.delete(claimKey) } catch (error) {
// Answered (or resolved by a person) between the roster read and this call: nothing is owed, and the
// agent's other deliveries go on.
- if (!(error instanceof SuperTabHostError) || error.code !== 'DELIVERY_RESOLVED') throw error
+ if (!(error instanceof SuperTabHostError) || error.code !== 'DELIVERY_RESOLVED') { this.retryLater(claimKey, listKey); throw error }
+ this.deliveryRetries.delete(claimKey)
this.retire(delivery.id as string)
}
}
} catch (error) {
+ if (!listRead) this.retryLater(listKey)
// Authority checks and network failures both withhold input. A later fresh read may grant it again.
this.log(error instanceof Error ? error.message : String(error))
}
@@ -627,6 +657,13 @@ export class SuperTabHost {
for (const [harnessId, state] of this.keepAlives) if (!leads.has(harnessId) && !state.running) this.keepAlives.delete(harnessId)
for (const [key, state] of this.endings) if (!ended.has(key) && !state.running) this.endings.delete(key)
for (const deliveryId of this.unreadable) if (!seen.has(deliveryId)) this.unreadable.delete(deliveryId)
+ // A wait outlives a list that failed or was skipped: that says nothing of the delivery. It goes when a list that was read no
+ // longer has the delivery, or when its agent is no longer here.
+ for (const [key, retry] of this.deliveryRetries) {
+ const agentKey = retry.list ?? key
+ if (!running.some(r => `list:${r.tabId}:${r.agent.id}` === agentKey)) this.deliveryRetries.delete(key)
+ else if (key.startsWith('claim:') && listsRead.has(agentKey) && !listed.has(key.slice('claim:'.length))) this.deliveryRetries.delete(key)
+ }
for (const deliveryId of this.files.keys()) if (!this.journal.entries[deliveryId]) this.files.delete(deliveryId)
for (const entry of Object.values(this.journal.entries)) {
const id = typedId(entry)