The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Batch C: web reconnect backoff, deliveries over the daemon socket (#1920)
0xken1503b1a264ac07
diff --git a/backend/integration/superTabLiveness.test.ts b/backend/integration/superTabLiveness.test.tsindex 124208db0..ab94a629f 100644--- a/backend/integration/superTabLiveness.test.ts+++ b/backend/integration/superTabLiveness.test.ts@@ -7,7 +7,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'import { MongoMemoryReplSet } from 'mongodb-memory-server-core'import { PrismaClient } from '@prisma/client'import type { AuthUser } from '../src/lib/ssoAuth.js'-import { SuperTabStore } from '../src/superTabs/store.js'+import { SuperTabStore, type SuperTabStoreHooks } from '../src/superTabs/store.js'import type { Command } from '../src/superTabs/commands.js'import type { MachineObservation } from '../src/superTabs/machines.js'import * as parts from '../src/superTabs/parts/index.js'@@ -44,9 +44,9 @@ afterAll(async () => {})/** A Swarm whose Lead runs on one of host's machines and whose worker runs on another, assigned "Ship the parser". */-async function swarm({ legacy = false, client = db }: { legacy?: boolean; client?: PrismaClient } = {}) {+async function swarm({ legacy = false, client = db, hooks }: { legacy?: boolean; client?: PrismaClient; hooks?: SuperTabStoreHooks } = {}) {let now = 10_000_000- const timed = new SuperTabStore(client, () => now)+ const timed = new SuperTabStore(client, () => now, null, hooks)const tab = (await timed.create(owner, randomUUID(), 'Liveness', legacy ? undefined : 'public')).idif (legacy) await dropProjects(tab)const execute = (command: Command, user = owner, asAgentId?: string) => timed.execute(user, tab, { operationId: randomUUID(), command }, asAgentId)@@ -617,6 +617,34 @@ describe('the delivery poll of an agent with nothing waiting', () => {expect(rows.map(d => d.messageId)).toEqual([sent.result.messageId])})+ // Behind a load balancer each poll may land on another pod: a memory of the last full poll per pod made every pod run its+ // own full polls, so the one-row answer held only as often as a host met the same pod twice (stability roadmap 2.4).+ it('answers from one indexed read on another server once any server ran the full poll, when they share the memory', async () => {+ const counted = counting()+ const shared = new Map<string, number>()+ const polls = { get: async (key: string) => shared.get(key), set: async (key: string, at: number) => { shared.set(key, at) } }+ const f = await swarm({ client: counted.client, hooks: { polls } })+ const other = new SuperTabStore(counted.client, () => f.clock.now, null, { polls })+ expect(await f.timed.deliveries(host, f.tab, f.lead)).toEqual([])+ counted.ops.length = 0+ f.clock.now += 1_000+ expect(await other.deliveries(host, f.tab, f.lead)).toEqual([])+ expect(counted.ops).toEqual(['SuperTabDelivery.findFirst'])+ })++ it('runs the full checked poll each time while the shared memory cannot be read or written, and still answers', async () => {+ const counted = counting()+ const down = async () => { throw new Error('Redis unreachable') }+ const f = await swarm({ client: counted.client, hooks: { polls: { get: down, set: down } } })+ expect(await f.timed.deliveries(host, f.tab, f.lead)).toEqual([])+ counted.ops.length = 0+ f.clock.now += 1_000+ expect(await f.timed.deliveries(host, f.tab, f.lead)).toEqual([])+ expect(counted.ops).toContain('SwarmGroup.findMany')+ // A stranger still meets the authority checks, never a remembered answer.+ await expect(f.timed.deliveries(owner, f.tab, f.lead)).rejects.toMatchObject({ code: 'FORBIDDEN' })+ })+it('never answers a caller who is not the agent\'s host from another caller\'s quiet poll, before or after it fails', async () => {const f = await swarm()await f.timed.deliveries(host, f.tab, f.lead)@@ -627,6 +655,60 @@ describe('the delivery poll of an agent with nothing waiting', () => {})})+// Hosts read an idle agent's deliveries only every 30 s once the server pushes (stability roadmap 2.4): each new one is+// announced to the machine its agent runs on, once its write commits, so the host reads it at once.+describe('the push for a new delivery', () => {+ const announcing = () => {+ const waiting: Array<{ machineId: string; tabId: string; agentId: string }> = []+ return { waiting, hooks: { deliveriesWaiting: (w: { machineId: string; tabId: string; agentId: string }) => { waiting.push(w) } } }+ }++ it('names the machine, Swarm and agent of a message\'s recipient once it is written', async () => {+ const { waiting, hooks } = announcing()+ const f = await swarm({ hooks })+ waiting.length = 0+ await f.execute({ type: 'message.send', conversationId: f.group, text: 'Status?', replyTo: null, recipientIds: [f.worker] })+ expect(waiting).toEqual([{ machineId: 'test-host', tabId: f.tab, agentId: f.worker }])+ })++ // The host reads as soon as it hears: a push heard before the commit would find nothing and wait 30 s for the next read.+ it('is heard only once its write has committed', async () => {+ let open = 0+ const client = new Proxy(db, { get(target, key, receiver) {+ if (key !== '$transaction') return Reflect.get(target, key, receiver)+ return async (body: unknown, options: unknown) => {+ open+++ try { return await (target.$transaction as (b: unknown, o: unknown) => Promise<unknown>)(body, options) } finally { open-- }+ }+ } }) as PrismaClient+ const heard: number[] = []+ const f = await swarm({ client, hooks: { deliveriesWaiting: () => { heard.push(open) } } })+ heard.length = 0+ await f.execute({ type: 'message.send', conversationId: f.group, text: 'Status?', replyTo: null, recipientIds: [f.worker] })+ expect(heard).toEqual([0])+ })++ it('names the Lead\'s machine for a turn typed into a worker\'s terminal', async () => {+ const { waiting, hooks } = announcing()+ const f = await swarm({ hooks })+ const { contextId } = await f.timed.native.open(owner, f.tab, { operationId: randomUUID(), conversationId: f.group, agentId: f.worker })+ waiting.length = 0+ await f.timed.native.append(host, f.tab, contextId, { turnId: 'turn_1', kind: 'prompt', previousEventId: null, text: 'Typed by hand',+ truncated: false, interrupted: false })+ expect(waiting).toEqual([{ machineId: 'lead-host', tabId: f.tab, agentId: f.lead }])+ })++ it('announces the update request the Lead\'s own poll writes for it', async () => {+ const { waiting, hooks } = announcing()+ const f = await swarm({ hooks })+ await f.leadInbox()+ waiting.length = 0+ f.clock.now += 5 * MINUTE+ await f.timed.deliveries(host, f.tab, f.lead)+ expect(waiting).toContainEqual({ machineId: 'lead-host', tabId: f.tab, agentId: f.lead })+ })+})+describe('the morning digest', () => {// #1772 review: every open decision counts, however many there are, and private ones never crowd out shared ones.it('counts every decision waiting, past 200, and is not crowded out by newer private ones', async () => {diff --git a/backend/src/lib/adapterWs.ts b/backend/src/lib/adapterWs.tsindex 997fc3fd1..e79d125f5 100644--- a/backend/src/lib/adapterWs.ts+++ b/backend/src/lib/adapterWs.ts@@ -40,6 +40,7 @@ import { machineBillingAllowsDataPlane } from './billingState.js'import { normalizeComputerId } from './deviceAuth.js'import { authenticateAccessToken, SsoAuthError } from './ssoAuth.js'import { relayAccountPushes } from './adapterAccountPushes.js'+import { relaySwarmDeliveries } from './swarmDeliveryPushes.js'import { DAEMONS } from '../config/env.js'import { parseAutonomousEnvironment } from './autonomousEnvironment.js'import { machineService } from '../services/MachineService.js'@@ -234,6 +235,8 @@ async function attachAdapter(ws: WebSocket, machineId: string, userId: string, c// re-reads through its daemon rather than polling for it. See adapterAccountPushes.ts.// A team machine reads none of the account's desk, zoo, machines or connections: it hears only the device-key pushes.const accountPushesUnsub = await relayAccountPushes(userId, send, { zoo: DAEMONS.on, team })+ // Work waiting for this machine's Swarm agents, team machine or not: its host reads it at once (swarmDeliveryPushes.ts).+ const swarmPushesUnsub = await relaySwarmDeliveries(machineId, send)// Last desktop-app state THIS socket asserted. Socket-scoped on purpose: a new adapter connection// starts with no claim and must re-assert, so a fresh value can never renew a stale one.@@ -559,6 +562,7 @@ async function attachAdapter(ws: WebSocket, machineId: string, userId: string, creleaseLiveness()terminalDownUnsub()accountPushesUnsub()+ swarmPushesUnsub()// Only the CURRENT owner tears the role down — a superseded socket must not mark the fresh// connection offline. A superseded socket still releases its own liveness tracking above.if (owners.get(machineId) === ws) {diff --git a/backend/src/lib/bus.ts b/backend/src/lib/bus.tsindex 3fb71cf3a..3414faca3 100644--- a/backend/src/lib/bus.ts+++ b/backend/src/lib/bus.ts@@ -680,6 +680,20 @@ export function subscribeDeviceMachineListChanged(userId: string, cb: (msg: Devireturn addSub(deviceMachineListChannel(userId), cb as Cb)}+// Per-MACHINE Swarm deliveries: a delivery committed for one of this machine's agents (superTabs/store.ts deliveriesWaiting),+// on some REST worker; the worker holding that machine's adapter socket tells its host (lib/swarmDeliveryPushes.ts).+export interface SwarmDeliveriesMsg { tabId: string; agentId: string }++const swarmDeliveriesChannel = (machineId: string): string => `swarm-deliveries:${machineId}`++export function publishSwarmDeliveries(machineId: string, msg: SwarmDeliveriesMsg): Promise<number> {+ return safePublish(swarmDeliveriesChannel(machineId), JSON.stringify(msg))+}++export function subscribeSwarmDeliveries(machineId: string, cb: (msg: unknown) => void): Promise<() => void> {+ return addSub(swarmDeliveriesChannel(machineId), cb)+}+// Per-USER desk invalidation: the account's tabs changed (routes/desk.ts) on some REST worker; every// adapter socket of that user — one per computer, possibly on other workers — is told to re-fetch.export interface DeskChangedMsg { revision: number }diff --git a/backend/src/lib/swarmDeliveryPushes.test.ts b/backend/src/lib/swarmDeliveryPushes.test.tsnew file mode 100644index 000000000..bc762ff92--- /dev/null+++ b/backend/src/lib/swarmDeliveryPushes.test.ts@@ -0,0 +1,107 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'++/** The bus' per-machine channel, in memory: a publish reaches only the callbacks subscribed for that machine. */+const bus = vi.hoisted(() => {+ const channels = new Map<string, Set<(msg: unknown) => void>>()+ return {+ channels,+ async subscribe(machineId: string, cb: (msg: unknown) => void) {+ const set = channels.get(machineId) ?? new Set()+ channels.set(machineId, set)+ set.add(cb)+ return () => { set.delete(cb) }+ },+ publish(machineId: string, msg: unknown) { for (const cb of channels.get(machineId) ?? []) cb(msg) },+ }+})+vi.mock('./bus.js', () => ({+ subscribeSwarmDeliveries: (machineId: string, cb: (msg: unknown) => void) => bus.subscribe(machineId, cb),+ publishSwarmDeliveries: async (machineId: string, msg: unknown) => { bus.publish(machineId, JSON.parse(JSON.stringify(msg))); return 1 },+}))++import { announceSwarmDeliveries, relaySwarmDeliveries } from './swarmDeliveryPushes.js'++const pushes = (send: ReturnType<typeof vi.fn>) => send.mock.calls.map(([frame]) => frame)+const told = (payload: object) => ({ t: 'down', connId: '', frame: { type: 'super_tab_deliveries', payload } })++describe('Swarm delivery pushes on a daemon socket', () => {+ beforeEach(() => { bus.channels.clear(); vi.useFakeTimers() })+ afterEach(() => { vi.useRealTimers() })++ it('says it pushes as soon as the socket is attached, so the host reads every 30 s instead of each second', async () => {+ const send = vi.fn()+ await relaySwarmDeliveries('machine-a', send)+ expect(pushes(send)).toEqual([told({})])+ })++ it('names the Swarm and agent of a delivery waiting on its own machine', async () => {+ const send = vi.fn()+ await relaySwarmDeliveries('machine-a', send)+ send.mockClear()+ bus.publish('machine-a', { tabId: 'tab-1', agentId: 'agent-1' })+ expect(pushes(send)).toEqual([told({ tabId: 'tab-1', agentId: 'agent-1' })])+ })++ // The store's deliveriesWaiting on every server (routes/superTabs.ts, worker.ts), wherever the machine's socket is held.+ it('carries a delivery the store announces to the socket of the agent\'s machine, and to no other', async () => {+ const a = vi.fn(), b = vi.fn()+ await relaySwarmDeliveries('machine-a', a)+ await relaySwarmDeliveries('machine-b', b)+ a.mockClear()+ b.mockClear()+ announceSwarmDeliveries({ machineId: 'machine-a', tabId: 'tab-1', agentId: 'agent-1' })+ await vi.waitFor(() => expect(pushes(a)).toEqual([told({ tabId: 'tab-1', agentId: 'agent-1' })]))+ expect(pushes(b)).toEqual([])+ })++ // The machine is the one the socket signed in as (adapterWs.ts), never one the client names.+ it('never hears a delivery waiting on another machine', async () => {+ const send = vi.fn()+ await relaySwarmDeliveries('machine-a', send)+ send.mockClear()+ bus.publish('machine-b', { tabId: 'tab-1', agentId: 'agent-1' })+ expect(pushes(send)).toEqual([])+ })++ it('passes on nothing but a Swarm and an agent', async () => {+ const send = vi.fn()+ await relaySwarmDeliveries('machine-a', send)+ send.mockClear()+ bus.publish('machine-a', { tabId: 'tab-1', agentId: 'agent-1', text: 'secret' })+ bus.publish('machine-a', { tabId: 7, agentId: 'agent-1' })+ bus.publish('machine-a', null)+ expect(pushes(send)).toEqual([told({ tabId: 'tab-1', agentId: 'agent-1' })])+ })++ // The host goes back to reading each second 10 minutes after it last heard this (cli superTabs/host.ts PUSH_LEASE_MS).+ it('says again that it pushes every 4 minutes', async () => {+ const send = vi.fn()+ await relaySwarmDeliveries('machine-a', send)+ send.mockClear()+ await vi.advanceTimersByTimeAsync(4 * 60_000 - 1)+ expect(pushes(send)).toEqual([])+ await vi.advanceTimersByTimeAsync(1)+ expect(pushes(send)).toEqual([told({})])+ })++ // The host keeps reading each second while it has not heard that the server pushes: nothing is lost, only the saving.+ it('says nothing when it cannot listen, and never fails the socket for it', async () => {+ const send = vi.fn()+ const subscribe = vi.spyOn(bus, 'subscribe').mockRejectedValueOnce(new Error('Redis unreachable'))+ const stop = await relaySwarmDeliveries('machine-a', send)+ await vi.advanceTimersByTimeAsync(10 * 60_000)+ expect(pushes(send)).toEqual([])+ stop()+ subscribe.mockRestore()+ })++ it('stops listening and repeating once the socket closes', async () => {+ const send = vi.fn()+ const stop = await relaySwarmDeliveries('machine-a', send)+ send.mockClear()+ stop()+ bus.publish('machine-a', { tabId: 'tab-1', agentId: 'agent-1' })+ await vi.advanceTimersByTimeAsync(10 * 60_000)+ expect(pushes(send)).toEqual([])+ })+})diff --git a/backend/src/lib/swarmDeliveryPushes.ts b/backend/src/lib/swarmDeliveryPushes.tsnew file mode 100644index 000000000..0e028e9e4--- /dev/null+++ b/backend/src/lib/swarmDeliveryPushes.ts@@ -0,0 +1,48 @@+import { publishSwarmDeliveries, subscribeSwarmDeliveries } from './bus.js'+import { logger } from '../utils/logger.js'+import type { DeliveriesWaiting } from '../superTabs/store.js'++/** Repeated so a host keeps reading idle agents every 30 s: it goes back to each second 10 minutes after it last heard it+ * (cli superTabs/host.ts PUSH_LEASE_MS), which covers a missed repeat and a server rolled back to one that does not push. */+const PUSHES_REPEAT_MS = 4 * 60_000++/**+ * Stability roadmap 2.4: every Swarm host read each running agent's deliveries each second, almost always to find nothing.+ * This socket's machine now hears when a delivery waits for one of its agents (superTabs/store.ts deliveriesWaiting, after+ * the write commits), as a connection-less `super_tab_deliveries` down frame naming the Swarm and agent; the daemon hands it+ * to its Swarm host, which reads that list at once (cli gateway.ts, superTabs/host.ts).+ *+ * The frame with no payload says this server pushes: the host then reads an idle agent's list only every 30 s, the safety+ * net for a push lost on the way. A daemon from before this ignores the frame and keeps reading each second.+ *+ * `machineId` is the machine the socket signed in as (adapterWs.ts), never one the client names, so a socket hears only its+ * own machine's work. The frame carries nothing but the two ids; the host reads the deliveries through the checked poll.+ *+ * Resolves to the unsubscribe. Redis unreachable, it says nothing and never fails the socket.+ */+export async function relaySwarmDeliveries(machineId: string, send: (frame: unknown) => unknown): Promise<() => void> {+ const tell = (payload: { tabId?: string; agentId?: string }) => { send({ t: 'down', connId: '', frame: { type: 'super_tab_deliveries', payload } }) }+ let unsubscribe: () => void+ try {+ unsubscribe = await subscribeSwarmDeliveries(machineId, msg => {+ const { tabId, agentId } = (msg && typeof msg === 'object' ? msg : {}) as Record<string, unknown>+ if (typeof tabId === 'string' && typeof agentId === 'string') tell({ tabId, agentId })+ })+ } catch (error) {+ // Not told that the server pushes, the host keeps reading each second: nothing waits longer, only the saving is lost.+ logger.warn('[swarm] delivery pushes unavailable for this socket', { machineId, error: error instanceof Error ? error.message : String(error) })+ return () => {}+ }+ tell({})+ const repeat = setInterval(() => tell({}), PUSHES_REPEAT_MS)+ return () => {+ clearInterval(repeat)+ unsubscribe()+ }+}++/** The store's `deliveriesWaiting` on every server that writes Swarm messages (routes/superTabs.ts, worker.ts): the server+ * holding the machine's socket tells it. Dropped when Redis is away; the host's 30 s read still finds the delivery. */+export function announceSwarmDeliveries({ machineId, tabId, agentId }: DeliveriesWaiting): void {+ void publishSwarmDeliveries(machineId, { tabId, agentId })+}diff --git a/backend/src/lib/teamSession.test.ts b/backend/src/lib/teamSession.test.tsindex ccc37e0a3..ac62cb272 100644--- a/backend/src/lib/teamSession.test.ts+++ b/backend/src/lib/teamSession.test.ts@@ -31,7 +31,9 @@ const m = vi.hoisted(() => {}})vi.mock('./prisma.js', () => ({ prisma: m.prisma, machineAlive: { OR: [{ deletedAt: null }, { deletedAt: { isSet: false } }] } }))-vi.mock('./bus.js', () => ({ publishShareChanged: m.bus.changed, publishDeviceMachineListChanged: m.bus.listChanged, getAgentPresenceMany: m.bus.presence }))+// `pub`: the Swarm routes' store shares its delivery poll memory through it (routes/superTabs.ts); these tests never poll.+vi.mock('./bus.js', () => ({ publishShareChanged: m.bus.changed, publishDeviceMachineListChanged: m.bus.listChanged, getAgentPresenceMany: m.bus.presence,+ pub: {} }))vi.mock('./deviceKeyLog.js', () => ({ readDeviceKeyLog: m.deviceKeys.read, deviceKeysSeen: m.deviceKeys.seen, appendDeviceKey: m.deviceKeys.append,readDeviceAcks: m.deviceKeys.acks, appendDeviceAck: m.deviceKeys.ack, touchDeviceKey: m.deviceKeys.touch }))vi.mock('./machineAccess.js', () => ({ machinesSharedWith: vi.fn(async () => []) }))diff --git a/backend/src/routes/superTabs.ts b/backend/src/routes/superTabs.tsindex d9aabb4e5..21a8cc8b2 100644--- a/backend/src/routes/superTabs.ts+++ b/backend/src/routes/superTabs.ts@@ -9,6 +9,7 @@ import { sendError, sendSuccess } from '../utils/response.js'import { operationSchema } from '../superTabs/commands.js'import { fail, SuperTabError } from '../superTabs/model.js'import { SuperTabStore } from '../superTabs/store.js'+import { redisPollMemory } from '../superTabs/pollMemory.js'import { routingPolicySchema } from '../superTabs/routingPolicy.js'import { nativeEventSchema, openNativeSchema } from '../superTabs/native.js'import { machineObservationSchema } from '../superTabs/machines.js'@@ -122,6 +123,20 @@ function busyOnDuplicate(error: unknown): never {* before or right after any action, so it meets this at once. */const updateRequired = (_req: unknown, reply: FastifyReply) => sendError(reply, UPDATE_REQUIRED.message, UPDATE_REQUIRED.code, UPDATE_REQUIRED.status)+/** The store this server runs on. Loaded only here: importing the bus opens its Redis connections. */+async function productionStore(github: GitHubTokens | null): Promise<SuperTabStore> {+ const [{ pub }, { announceSwarmDeliveries }] = await Promise.all([import('../lib/bus.js'), import('../lib/swarmDeliveryPushes.js')])+ return new SuperTabStore(prisma, Date.now, attachmentsFromEnv(env),+ // A task that ends revokes the token it was given; a revoke that fails is logged inside and never blocks the write.+ { workEnded: (tabId, ended) => { void github?.release(tabId, ended).catch(error => logger.warn('[github] release failed', { tabId, error: String(error) })) },+ // A team account has no git login of its own: without the app it takes no teammates' work.+ teamAccountsIdle: !github,+ // Every server answers the quiet delivery poll from any server's full one, and tells the agent's machine of new work+ // through whichever server holds its daemon socket (stability roadmap 2.4). Both only ever drop, never fail, on Redis.+ polls: redisPollMemory(pub),+ deliveriesWaiting: announceSwarmDeliveries })+}+export async function superTabRoutes(app: FastifyInstance, options: { store?: SuperTabStore; github?: GitHubTokens | null; reads?: SwarmReads; hosts?: HostReads;approvals?: Approvals; signIn?: GitHubSignIn | null } = {}) {const github = options.github !== undefined ? options.github : await githubTokensFromEnv(env)@@ -141,11 +156,7 @@ export async function superTabRoutes(app: FastifyInstance, options: { store?: Suthrow new SuperTabError('GITHUB_REPOSITORY_NOT_APPROVED', `Nobody who can push to ${repository.fullName} has approved it for this project yet, `+ 'so teammates\' agents get no access to it. Someone who can push to it approves it in Harness: Project details, GitHub access.', 403)}- // A task that ends revokes the token it was given; a revoke that fails is logged inside and never blocks the write.- const store = options.store ?? new SuperTabStore(prisma, Date.now, attachmentsFromEnv(env),- { workEnded: (tabId, ended) => { void github?.release(tabId, ended).catch(error => logger.warn('[github] release failed', { tabId, error: String(error) })) },- // A team account has no git login of its own: without the app it takes no teammates' work.- teamAccountsIdle: !github })+ const store = options.store ?? await productionStore(github)const reads = options.reads ?? new SwarmReads(store, { githubApp: !!github })const hosts = options.hosts ?? new HostReads(store, { githubApp: !!github, teamAccountsIdle: !github })app.get('/api/super-tabs/grids', (req, reply) => respond(reply, () => store.hostGrids(req.user!)))diff --git a/backend/src/superTabs/native.ts b/backend/src/superTabs/native.tsindex 20c0afce1..9fe1da378 100644--- a/backend/src/superTabs/native.ts+++ b/backend/src/superTabs/native.ts@@ -9,6 +9,7 @@ import { canReadGroup, groupProject, readConversation } from './projects.js'import { noteProjectReports } from './projectUpdates.js'import { numberMessages, type Touch } from './parts/commit.js'import { redactCredentials } from './credentials.js'+import type { DeliveriesWaiting } from './store.js'const id = z.string().min(1).max(160).regex(/^[A-Za-z0-9_-]+$/)export const openNativeSchema = z.object({ operationId: id, conversationId: id, agentId: id }).strict()@@ -39,6 +40,8 @@ interface Deps {teamHost(tx: Transaction, user: AuthUser): Promise<string | null>/** Refuses a person no longer in the Swarm's Workspace (workspaceStore.ts requireWorkspaceAccess). */workspaceAccess(tx: Transaction, state: SuperTabState, userId: string): Promise<void>+ /** Tells the agent's machine, once `tx` commits, that a delivery waits for it (store.ts announce). */+ announce(tx: Transaction, waiting: DeliveriesWaiting): void}const json = (value: unknown): Prisma.InputJsonValue => JSON.parse(JSON.stringify(value)) as Prisma.InputJsonValueconst digest = (value: unknown) => createHash('sha256').update(JSON.stringify(value)).digest('hex')@@ -235,6 +238,7 @@ export class SuperTabNativeStore {...(team? { authorKind: 'agent', authorId: target.id, authorGeneration: target.membershipGeneration }: { authorKind: 'person', authorId: binding.operator.id, authorGeneration: binding.operator.membershipGeneration }) } })+ this.deps.announce(tx, { machineId: lead.machineId, tabId, agentId: recipient })}await tx.superTabNativeContext.update({ where: { id: contextId }, data: { lastEventId: eventId } })const result = { eventId, sequence }diff --git a/backend/src/superTabs/pollMemory.test.ts b/backend/src/superTabs/pollMemory.test.tsnew file mode 100644index 000000000..ae481cc24--- /dev/null+++ b/backend/src/superTabs/pollMemory.test.ts@@ -0,0 +1,52 @@+import { describe, expect, it } from 'vitest'+import { redisPollMemory } from './pollMemory.js'++/** The two Redis commands the memory uses, with Redis' own expiry (`PX` milliseconds), on a clock the test moves. */+function fakeRedis() {+ let now = 0+ const keys = new Map<string, { value: string; until: number }>()+ return {+ tick: (ms: number) => { now += ms },+ keys,+ async get(key: string) {+ const entry = keys.get(key)+ return entry && entry.until > now ? entry.value : null+ },+ async set(key: string, value: string, mode: 'PX', ms: number) {+ if (mode !== 'PX') throw new Error('expected an expiry')+ keys.set(key, { value, until: now + ms })+ return 'OK' as const+ },+ }+}++describe('the delivery poll memory every server shares', () => {+ it('answers with the time any server last remembered for a caller', async () => {+ const redis = fakeRedis()+ await redisPollMemory(redis).set('tab:agent:user', 1_234)+ expect(await redisPollMemory(redis).get('tab:agent:user')).toBe(1_234)+ expect(await redisPollMemory(redis).get('tab:agent:other')).toBeUndefined()+ })++ // A key per tab, agent and caller that never expired would grow Redis by one for every agent that ever polled.+ it('forgets each caller 30 seconds after it was remembered', async () => {+ const redis = fakeRedis()+ await redisPollMemory(redis).set('tab:agent:user', 1_234)+ redis.tick(29_999)+ expect(await redisPollMemory(redis).get('tab:agent:user')).toBe(1_234)+ redis.tick(1)+ expect(await redisPollMemory(redis).get('tab:agent:user')).toBeUndefined()+ })++ it('keeps its keys apart from the rest of Redis', async () => {+ const redis = fakeRedis()+ await redisPollMemory(redis).set('tab:agent:user', 1)+ expect([...redis.keys.keys()]).toEqual(['swarm:polled:tab:agent:user'])+ })++ it('reads a value it cannot use as never remembered', async () => {+ const redis = fakeRedis()+ await redis.set('swarm:polled:tab:agent:user', 'not a time', 'PX', 30_000)+ expect(await redisPollMemory(redis).get('tab:agent:user')).toBeUndefined()+ })+})diff --git a/backend/src/superTabs/pollMemory.ts b/backend/src/superTabs/pollMemory.tsnew file mode 100644index 000000000..28262e30d--- /dev/null+++ b/backend/src/superTabs/pollMemory.ts@@ -0,0 +1,23 @@+import { SWEEP_EVERY_MS } from './liveness.js'+import type { PollMemory } from './store.js'++/** The Redis commands the memory needs: the bus' fail-fast connection (lib/bus.ts `pub`) in production. */+export interface PollRedis {+ get(key: string): Promise<string | null>+ set(key: string, value: string, mode: 'PX', ms: number): Promise<unknown>+}++/**+ * Each caller's last full delivery poll, in Redis so every server answers the quiet poll from any server's full one (stability+ * roadmap 2.4). One key per tab, agent and caller, gone after SWEEP_EVERY_MS: an older one is never read anyway.+ */+export function redisPollMemory(redis: PollRedis): PollMemory {+ const key = (k: string) => `swarm:polled:${k}`+ return {+ async get(k) {+ const value = Number(await redis.get(key(k)) ?? NaN)+ return Number.isFinite(value) ? value : undefined+ },+ async set(k, at) { await redis.set(key(k), String(at), 'PX', SWEEP_EVERY_MS) },+ }+}diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.tsindex 89e202326..5dcf47788 100644--- a/backend/src/superTabs/store.ts+++ b/backend/src/superTabs/store.ts@@ -136,7 +136,14 @@ export async function loadRow(tx: Transaction, row: SwarmRow, named: Named = {})export async function transaction<T>(db: Pick<PrismaClient, '$transaction'>, body: (tx: Transaction) => Promise<T>,sleep: (ms: number) => Promise<void> = ms => new Promise(resolve => setTimeout(resolve, ms))): Promise<T> {for (let attempt = 1; ; attempt++) {- try { return await db.$transaction(body, TRANSACTION_OPTIONS) } catch (error) {+ let committed: Array<() => void> = []+ try {+ const result = await db.$transaction(tx => { committed = []; pendingCommits.set(tx, committed); return body(tx) }, TRANSACTION_OPTIONS)+ for (const callback of committed) {+ try { callback() } catch (error) { logger.warn('[super-tab] after-commit callback failed', { error: error instanceof Error ? error.message : String(error) }) }+ }+ return result+ } catch (error) {if (!retryable(error)) throw error// An expired or unstartable transaction (P2028) is load: one more try, then the client retries after Retry-After.if (attempt >= (errorCode(error) === 'P2028' ? EXPIRED_TRANSACTION_ATTEMPTS : TRANSACTION_ATTEMPTS)) {@@ -149,6 +156,24 @@ export async function transaction<T>(db: Pick<PrismaClient, '$transaction'>, bod}}const TRANSACTION_OPTIONS = { maxWait: 5_000, timeout: 15_000 }+/** Each attempt's after-commit work, by that attempt's client: a retried attempt's list is dropped with its client. */+const pendingCommits = new WeakMap<Transaction, Array<() => void>>()++/**+ * Runs `callback` once the transaction `tx` belongs to has committed, never for an attempt that failed or was retried. What+ * others hear of a write must follow its commit: told earlier, a reader could look before the write is visible, or hear of+ * one rolled back. The Swarm cache's stamp (Stability 2.2) and the delivery push (Stability 2.4) both go through here. It+ * runs synchronously after the commit; async work is fired and forgotten, and a throw is logged, never failing the write.+ */+export function afterCommit(tx: Transaction, callback: () => void): void {+ const pending = pendingCommits.get(tx)+ if (!pending) throw new Error('afterCommit needs a client from a Swarm transaction (store.ts transaction).')+ pending.push(callback)+}+/** Logged, never thrown: without the memory the delivery poll just runs in full. */+const pollMemoryFailed = (error: unknown): void => {+ logger.warn('[super-tab] delivery poll memory unavailable', { error: error instanceof Error ? error.message : String(error) })+}/** The deliveries a host is still told about (the rest are answered, cancelled or confirmed). */const OPEN_DELIVERY = ['queued', 'claimed', 'uncertain'] as const/** Callers kept for the quiet delivery poll before the stale ones are dropped. */@@ -187,6 +212,34 @@ export interface SuperTabStoreHooks {/** No Harness GitHub App on this server: a Mac's Harness Team account joins no Swarm and takes no teammates' work, since* it has no git login to fetch or push with (commands.ts `teamAccountsIdle`). */teamAccountsIdle?: boolean+ /** When each caller's full delivery poll last succeeded, shared by every server (Redis in production, pollMemory.ts). Left+ * out, this server remembers alone. */+ polls?: PollMemory+ /** A delivery now waits for this agent, committed: its machine's daemon socket tells its host to read it at once+ * (lib/swarmDeliveryPushes.ts). Called after the write commits; it must not throw. */+ deliveriesWaiting?: (waiting: DeliveriesWaiting) => void+}++/** Which agent, of which Swarm, on which machine, has a delivery to read. */+export interface DeliveriesWaiting { machineId: string; tabId: string; agentId: string }++/** Where `deliveries` remembers each caller's last full poll. Either call may throw: the poll then runs in full. */+export interface PollMemory {+ get(key: string): Promise<number | undefined>+ /** Remembered for SWEEP_EVERY_MS at least; an older entry is never read. */+ set(key: string, at: number): Promise<void>+}++/** One server's own memory: with several servers behind a load balancer, each runs its own full polls. */+export function localPollMemory(now: () => number): PollMemory {+ const polled = new Map<string, number>()+ return {+ get: async key => polled.get(key),+ set: async (key, at) => {+ polled.set(key, at)+ if (polled.size > POLLED_CAP) for (const [k, last] of polled) if (now() - last >= SWEEP_EVERY_MS) polled.delete(k)+ },+ }}/** Persistent coordination. Host adapters own execution and must confirm their local session grants. */@@ -201,6 +254,7 @@ export class SuperTabStore {get avatars(): AvatarStore | null { return this.attachments?.avatars ?? null }constructor(readonly db: PrismaClient, readonly now: () => number = Date.now,private readonly attachments: Attachments | null = null, private readonly hooks: SuperTabStoreHooks = {}) {+ this.polls = hooks.polls ?? localPollMemory(now)this.native = new SuperTabNativeStore({transaction: body => transaction(db, body),load: (tx, user, tabId, groupIds) => this.load(tx, user, tabId, { groupIds }),@@ -210,6 +264,7 @@ export class SuperTabStore {contribution: (tx, state, actor, command) => this.contribution(tx, state, actor, command),teamHost: (tx, user) => this.teamHost(tx, user),workspaceAccess: (tx, state, userId) => requireWorkspaceAccess(tx, state, userId),+ announce: (tx, waiting) => this.announce(tx, waiting),})this.workspaces = new SwarmWorkspaces(db, {transaction: body => transaction(db, body),@@ -1468,10 +1523,18 @@ export class SuperTabStore {assignmentId: activeWork(group) && group.assignment?.agentId === recipient ? group.assignment.id : null,membershipGeneration: target.membershipGeneration, authorKind: message.author.kind,authorId: message.author.id, authorGeneration: message.author.membershipGeneration } })+ this.announce(tx, { machineId: target.machineId, tabId, agentId: recipient })}}}+ /** Once `tx` commits, never for an attempt that failed or was retried: told earlier, a host could read before the delivery+ * is visible, and wait 30 s for its next read. */+ private announce(tx: Transaction, waiting: DeliveriesWaiting): void {+ const told = this.hooks.deliveriesWaiting+ if (told) afterCommit(tx, () => told(waiting))+ }+/** One page of a conversation, read by (sequence, id): rows written by one command before the store by parts share a* sequence, so a page edge inside such a run must not skip any (design section 4.6). `after` alone reads strictly after* a sequence, which is enough to poll for new messages. */@@ -1759,8 +1822,8 @@ export class SuperTabStore {catch (error) { if (error instanceof SuperTabError) return false; throw error }}- /** When each caller's full delivery poll last succeeded for an agent, on this server: key is tab, agent and caller. */- private readonly polled = new Map<string, number>()+ /** When each caller's full delivery poll last succeeded for an agent: key is tab, agent and caller. */+ private readonly polls: PollMemory/*** Every running agent's host polls this each second. With nothing waiting and the full poll (which loads the Swarm and@@ -1769,12 +1832,12 @@ export class SuperTabStore {* succeeded is ever answered this way, so a stranger or a revoked host meets the authority checks every time.*/async deliveries(user: AuthUser, tabId: string, agentId: string) {- const key = `${tabId}:${agentId}:${user.sub}`, last = this.polled.get(key)+ const key = `${tabId}:${agentId}:${user.sub}`+ const last = await this.polls.get(key).catch(error => { pollMemoryFailed(error); return undefined })if (last !== undefined && this.now() - last < SWEEP_EVERY_MS&& !await this.db.superTabDelivery.findFirst({ where: { tabId, agentId, status: { in: [...OPEN_DELIVERY] } }, select: { id: true } })) return []const rows = await this.fullDeliveries(user, tabId, agentId)- this.polled.set(key, this.now())- if (this.polled.size > POLLED_CAP) for (const [k, at] of this.polled) if (this.now() - at >= SWEEP_EVERY_MS) this.polled.delete(k)+ await this.polls.set(key, this.now()).catch(pollMemoryFailed)return rows}diff --git a/backend/src/superTabs/transaction.test.ts b/backend/src/superTabs/transaction.test.tsindex 6082fcaac..2b8850aff 100644--- a/backend/src/superTabs/transaction.test.ts+++ b/backend/src/superTabs/transaction.test.ts@@ -1,5 +1,5 @@import { describe, expect, it, vi } from 'vitest'-import { transaction } from './store.js'+import { afterCommit, transaction } from './store.js'import { SuperTabError } from './model.js'const conflict = () => Object.assign(new Error('Transaction failed due to a write conflict or a deadlock'), { code: 'P2034' })@@ -63,3 +63,63 @@ describe('Swarm transactions', () => {expect(other.$transaction).toHaveBeenCalledTimes(1)})})++/** A database whose commit fails with each of `failures` in turn, after the body has run, as a write conflict at commit does. */+function failingCommit(...failures: unknown[]) {+ const $transaction = vi.fn(async (body: (tx: never) => Promise<unknown>) => {+ const result = await body({} as never)+ const failure = failures.shift()+ if (failure) throw failure+ return result+ })+ return { $transaction } as never as { $transaction: typeof $transaction }+}++// Shared by the Swarm cache's stamp (Stability 2.2) and the delivery push (Stability 2.4): what others hear of a write must+// follow its commit, or a reader could be told of a change it cannot see yet, or of one that was rolled back.+describe('work after a Swarm transaction commits', () => {+ it('runs the committed attempt\'s callbacks once, in order, only after the body is done', async () => {+ const db = failingCommit()+ const ran: string[] = []+ await transaction(db as never, async tx => {+ afterCommit(tx, () => ran.push('first'))+ afterCommit(tx, () => ran.push('second'))+ ran.push('body')+ })+ expect(ran).toEqual(['body', 'first', 'second'])+ })++ it('drops the callbacks of an attempt whose commit failed, and runs only the one that committed', async () => {+ const db = failingCommit(conflict(), conflict())+ let attempt = 0+ const ran: number[] = []+ await transaction(db as never, async tx => { const mine = ++attempt; afterCommit(tx, () => ran.push(mine)) }, async () => {})+ expect(db.$transaction).toHaveBeenCalledTimes(3)+ expect(ran).toEqual([3])+ })++ it('runs nothing when the transaction fails for good', async () => {+ const ran = vi.fn()+ const rolledBack = failingCommit(new Error('boom'))+ await expect(transaction(rolledBack as never, async tx => { afterCommit(tx, ran) })).rejects.toThrow('boom')+ const refused = failingCommit()+ await expect(transaction(refused as never, async tx => { afterCommit(tx, ran); throw new SuperTabError('FORBIDDEN', 'No.', 403) }))+ .rejects.toMatchObject({ code: 'FORBIDDEN' })+ expect(ran).not.toHaveBeenCalled()+ })++ it('never fails a committed write for a callback that throws, and still runs the rest', async () => {+ const db = failingCommit()+ const later = vi.fn()+ expect(await transaction(db as never, async tx => {+ afterCommit(tx, () => { throw new Error('Redis is down') })+ afterCommit(tx, later)+ return 'saved'+ })).toBe('saved')+ expect(later).toHaveBeenCalledTimes(1)+ })++ it('refuses a callback for a client no Swarm transaction opened, so it is never silently lost', () => {+ expect(() => afterCommit({} as never, () => {})).toThrow(/Swarm transaction/)+ })+})diff --git a/backend/src/worker.ts b/backend/src/worker.tsindex 33763105b..cf633c5a4 100644--- a/backend/src/worker.ts+++ b/backend/src/worker.ts@@ -20,6 +20,7 @@ import { ensureMentionExpiry } from './superTabs/mentions.js'import { ensurePartExpiry, startSwarmSweeps } from './superTabs/sweeps.js'import { appPullRequests, markFinishedGroups, startSwarmArchive, unknownPullRequests } from './superTabs/archive.js'import { SuperTabStore } from './superTabs/store.js'+import { announceSwarmDeliveries } from './lib/swarmDeliveryPushes.js'import { GitHubApp, githubAppConfig } from './github/app.js'import { env } from './config/env.js'import { machineBillingWorkerService } from './services/MachineBillingWorkerService.js'@@ -71,7 +72,8 @@ async function start(): Promise<void> {machineBillingWorkerService.start()productUsagePeopleRollupService.start()- const swarms = new SuperTabStore(prisma)+ // Archive notes and Workspace removals write messages too: their recipients' hosts hear of them at once.+ const swarms = new SuperTabStore(prisma, Date.now, null, { deliveriesWaiting: announceSwarmDeliveries })stopSweeps = startSwarmSweeps(prisma, undefined, () => swarms.workspaces.finishRemovals())// Finished groups archive once GitHub says their pull requests are merged or closed; without the app, only groups that// link none do (superTabs/archive.ts).diff --git a/cli/src/backendSocket.spec.ts b/cli/src/backendSocket.spec.tsindex 27c84c39e..dfa9ab33b 100644--- a/cli/src/backendSocket.spec.ts+++ b/cli/src/backendSocket.spec.ts@@ -2786,6 +2786,30 @@ describe('desk_changed relay', () => {await socket.stop()})+ // Stability roadmap 2.4: the server says over this socket when work waits for an agent here (backend lib/adapterWs.ts).+ it('hands the backend\'s super_tab_deliveries to the Swarm host, never to the windows, and only the backend\'s', async () => {+ const socket = relaySocket('token')+ const heard: unknown[] = []+ const frames: Array<Record<string, unknown>> = []+ socket.onAccountNotice = (notice) => { heard.push(notice) }+ socket.registerLocalClient('local:swarm', { sendFrame: (frame) => { frames.push(frame); return true }, sendBinary: () => true })+ socket.connect()+ const ws = wsMock.instances[0]+ ws.open()+ ws.message({ t: 'down', connId: '', frame: { type: 'super_tab_deliveries', payload: { tabId: 'tab-1', agentId: 'agent-1' } } })+ // Naming no agent, or naming one wrongly, it only says the server pushes.+ ws.message({ t: 'down', connId: '', frame: { type: 'super_tab_deliveries', payload: { tabId: 'tab-1', agentId: 7 } } })+ await vi.waitFor(() => expect(heard).toEqual([{ type: 'super_tab_deliveries', tabId: 'tab-1', agentId: 'agent-1' }, { type: 'super_tab_deliveries' }]))+ // A client relayed with its own connId, or a local one, is not the backend.+ ws.message({ t: 'down', connId: 'web-1', frame: { type: 'super_tab_deliveries', payload: { tabId: 'tab-1', agentId: 'agent-2' } } })+ socket.handleLocalFrame('local:swarm', { type: 'super_tab_deliveries', payload: { tabId: 'tab-1', agentId: 'agent-3' } })+ await new Promise((r) => setTimeout(r, 20))+ expect(heard).toHaveLength(2)+ expect(frames.filter((f) => f.type === 'super_tab_deliveries')).toEqual([])+ await socket.unregisterLocalClient('local:swarm')+ await socket.stop()+ })+it('hands the backend\'s zoo_changed to the window as its own frame, and only the backend\'s', async () => {const socket = relaySocket('token')const frames: Array<Record<string, unknown>> = []diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.tsindex 5c9477c59..f68a9ecd6 100644--- a/cli/src/backendSocket.ts+++ b/cli/src/backendSocket.ts@@ -443,6 +443,8 @@ export class BackendSocket {},notice: (notice: BackendNotice) => {this.onAccountNotice?.(notice)+ // Only the Swarm host acts on it (superTabs/host.ts): the windows have nothing to re-read.+ if (notice.type === 'super_tab_deliveries') returnthis.sendLocal(notice.type === 'machines_changed' ? { type: notice.type, payload: { reason: notice.reason } }: notice.type === 'device_keys_changed' ? { type: notice.type, payload: {} }: { type: notice.type, payload: { revision: notice.revision } })diff --git a/cli/src/core/api.ts b/cli/src/core/api.tsindex 9d5366a8f..538b77937 100644--- a/cli/src/core/api.ts+++ b/cli/src/core/api.ts@@ -1510,6 +1510,9 @@ export type BackendNotice =| { type: 'machines_changed'; reason: string }| { type: 'device_keys_changed' }| { type: 'connectors_changed'; revision: number }+ /** Work waits for an agent of this machine's (backend lib/adapterWs.ts): its Swarm host reads that agent's deliveries now.+ * Naming none, the server says it pushes them, and every list is read once (superTabs/host.ts PUSH_LEASE_MS). */+ | { type: 'super_tab_deliveries'; tabId?: string; agentId?: string }/** What a person has open on this computer, per surface (lib/windowSurfaces.ts). */export type { LocalWindows, WindowSurface } from '../lib/windowSurfaces.js'diff --git a/cli/src/gateway/gateway.ts b/cli/src/gateway/gateway.tsindex 888ae25eb..ab1bcb084 100644--- a/cli/src/gateway/gateway.ts+++ b/cli/src/gateway/gateway.ts@@ -692,6 +692,15 @@ export class RelayGateway implements GatewayPort {return}+ // Work waits for an agent of this machine's (backend lib/adapterWs.ts): the Swarm host reads its deliveries now. Naming+ // none, the server says it pushes them. Backend-only like desk_changed: a client could only make the host read early.+ if (type === 'super_tab_deliveries') {+ const p = (typeof frame.payload === 'object' && frame.payload !== null ? frame.payload : {}) as Record<string, unknown>+ this.core.notice(typeof p.tabId === 'string' && typeof p.agentId === 'string'+ ? { type, tabId: p.tabId, agentId: p.agentId } : { type })+ return+ }+// The account's device key log grew: this daemon re-reads and verifies it (deviceLogSyncer.ts), and// the window re-reads its Devices list. Backend-only for the same reason as desk_changed.if (type === 'device_keys_changed') {diff --git a/cli/src/lib/relayFrames.ts b/cli/src/lib/relayFrames.tsindex 8df6b4a74..678f5826a 100644--- a/cli/src/lib/relayFrames.ts+++ b/cli/src/lib/relayFrames.ts@@ -38,7 +38,7 @@ export type DownTransport = 'relay' | 'local' | 'p2p'* lib/e2ee/deviceLogSyncer.ts): forged, the first only makes this daemon re-read and verify the log,* the second could fake an answer to its own append — neither is anything a client should be sending.*/-export const BACKEND_ONLY_DOWN_TYPES = new Set(['machine_meta', 'machine_revoked', 'desk_changed', 'zoo_changed', 'machines_changed', 'device_keys_changed', 'connectors_changed', 'devlog_append_result'])+export const BACKEND_ONLY_DOWN_TYPES = new Set(['machine_meta', 'machine_revoked', 'desk_changed', 'zoo_changed', 'machines_changed', 'device_keys_changed', 'connectors_changed', 'super_tab_deliveries', 'devlog_append_result'])/** A frame type as the sender spelled it, fit for one log line: the relay chooses it, so it is bounded* and escaped rather than trusted not to carry a newline that forges the next line. */diff --git a/cli/src/services/superTabsProcess.spec.ts b/cli/src/services/superTabsProcess.spec.tsindex 3198a42b7..e243cabc6 100644--- a/cli/src/services/superTabsProcess.spec.ts+++ b/cli/src/services/superTabsProcess.spec.ts@@ -82,3 +82,18 @@ it('uses the real service wrapper and process runner by default without connectiexpect(runServiceProcess).toHaveBeenCalledWith(expect.objectContaining({ name: 'superTabs' }))handle.stop()})++// Stability roadmap 2.4: the server's word that work waits reaches the host in this process as the core's notice.+it('hands the account\'s notices the core tells it to the host, and nothing else', () => {+ const t = setup()+ const heard: unknown[] = []+ const unsubscribe = t.api().account.onNotice(notice => { heard.push(notice) })+ t.wire().onEvent!({ kind: 'notice', notice: { type: 'super_tab_deliveries', tabId: 'tab', agentId: 'agent' } })+ t.wire().onEvent!({ kind: 'notice', notice: 'not a notice' })+ t.wire().onEvent!({ kind: 'delivery', event: { deliveryId: 'd', sessionId: 'one', state: 'queued' } })+ expect(heard).toEqual([{ type: 'super_tab_deliveries', tabId: 'tab', agentId: 'agent' }])+ unsubscribe()+ t.wire().onEvent!({ kind: 'notice', notice: { type: 'super_tab_deliveries' } })+ expect(heard).toHaveLength(1)+ t.process.stop()+})diff --git a/cli/src/services/superTabsProcess.ts b/cli/src/services/superTabsProcess.tsindex ed738285d..7dfa5301e 100644--- a/cli/src/services/superTabsProcess.ts+++ b/cli/src/services/superTabsProcess.ts@@ -1,4 +1,4 @@-import { emptyPorts, type DaemonAddress } from '../core/api.js'+import { emptyPorts, type BackendNotice, type DaemonAddress } from '../core/api.js'import { startSuperTabs } from './superTabs.js'import { runServiceProcess, type CoreConnection, type ServiceProcess } from './process.js'import { agentsIn, daemonIn, processCoreApi, type ShownAgent } from './processCoreApi.js'@@ -18,6 +18,9 @@ export function runSuperTabsService(options: SuperTabsServiceOptions): ServicePrconst ask = (query: string, payload: Record<string, unknown>) => core? core.query(query, payload) : Promise.reject(new Error('Core disconnected'))const turns = turnsLink(ask)+ /** The account's notices the core tells this process (`service_event` kind `notice`): the host hears from them that the+ * server says work waits (superTabs/host.ts). */+ const notices = new Set<(notice: BackendNotice) => void>()let reading: Promise<void> | null = nullconst ready = (): Promise<void> => {const connection = core@@ -27,7 +30,8 @@ export function runSuperTabsService(options: SuperTabsServiceOptions): ServicePrdaemon = daemonIn(address)}).finally(() => { reading = null })}- const base = processCoreApi(options.dataDir, 'superTabs', { live: () => live, ask, deliveries: turns, daemon: () => daemon })+ const base = processCoreApi(options.dataDir, 'superTabs', { live: () => live, ask, deliveries: turns, daemon: () => daemon,+ onNotice: listener => { notices.add(listener); return () => { notices.delete(listener) } } })// Every authority read also refreshes the native registry. A stopped/replaced// harness cannot receive work through a stale process-side snapshot.const api = { ...base, account: { ...base.account, backend: async (method: string, path: string, body?: unknown) => {@@ -39,7 +43,10 @@ export function runSuperTabsService(options: SuperTabsServiceOptions): ServicePrconst service = (options.run ?? runServiceProcess)({name: 'superTabs', socketPath: options.socketPath, machineId: options.machineId, token: options.token,requests: { super_tabs: async (payload, asker, closed) => { await ready(); return requests.super_tabs(payload, asker, closed) } },- onEvent: payload => { turns.heard(payload) },+ onEvent: payload => {+ if (payload.kind !== 'notice') { turns.heard(payload); return }+ if (payload.notice && typeof payload.notice === 'object') for (const listener of notices) listener(payload.notice as BackendNotice)+ },onConnected: connection => {core = connectionvoid ready().then(() => { if (core === connection) ports.superTabs?.connected() }).catch(() => {})diff --git a/cli/src/superTabs/host.spec.ts b/cli/src/superTabs/host.spec.tsindex eff5b7b76..dae826ea9 100644--- a/cli/src/superTabs/host.spec.ts+++ b/cli/src/superTabs/host.spec.ts@@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'import { dirname, join } from 'node:path'import { prepareSwarmSkill } from '../lib/swarmSkills.js'import { afterEach, describe, expect, it, vi } from 'vitest'-import type { CoreApi, TurnDelivery } from '../core/api.js'+import type { BackendNotice, CoreApi, TurnDelivery } from '../core/api.js'import { fakeCore } from '../testing/fakeCore.js'import { backend, swarmRoleFor, hostJournalFile, SuperTabHost, type SuperTabHostOptions } from './host.js'import { hostForm, swarmReply } from './fixtures/read/index.js'@@ -24,6 +24,7 @@ afterEach(() => { for (const host of hosts.splice(0)) host.stop(); for (const rofunction setup(messageFields: Record<string, unknown> = {}) {const dir = mkdtempSync(join(tmpdir(), 'super-tab-host-')); roots.push(dir)const listeners = new Set<(e: TurnDelivery) => void>()+ const notices = new Set<(notice: BackendNotice) => void>()const row: { tabId: string; agent: { id: string; harnessId: string; machineId: string }; lead?: boolean; leadsConversationId: string } ={ tabId: 'tab', agent: { id: 'agent', harnessId: 'harness', machineId: 'machine-1' }, leadsConversationId: 'leads' }/** The conversation the delivery comes from. */@@ -49,7 +50,7 @@ function setup(messageFields: Record<string, unknown> = {}) {})const core = fakeCore({ dataDir: dir,agents: { creationStatus: vi.fn(async creationId => ({ creationId, state: 'created' as const, agentId: 'harness' })), byAgent: vi.fn(() => ({ agentId: 'harness' }) as never), terminalAvailable: vi.fn(() => true) },- account: { backend: call },+ account: { backend: call, onNotice: vi.fn(fn => { notices.add(fn); return () => { notices.delete(fn) } }) },turns: { onDelivery: vi.fn(fn => { listeners.add(fn); return () => { listeners.delete(fn) } }) },})const log = vi.fn()@@ -58,7 +59,12 @@ function setup(messageFields: Record<string, unknown> = {}) {for (const fn of listeners) fn({ deliveryId: 'team:service:superTabs:delivery', sessionId: 'harness', state, reason })}const saved = () => JSON.parse(readFileSync(hostJournalFile(dir, 'machine-1'), 'utf8'))- return { core, call, open, event, receipts, holds, log, row, wait: (ms: number) => { clock += ms }, saved, dir, remove: () => { removed = true }, revoke: (value = true) => { revoked = value }, foreignClaim: () => { claimId = 'another-process' }, from: (id: string) => { from = id } }+ return { core, call, open, event, receipts, holds, log, row, wait: (ms: number) => { clock += ms }, saved, dir, remove: () => { removed = true },+ /** What the server's daemon socket tells this machine (backend lib/adapterWs.ts). */+ push: (notice: BackendNotice) => { for (const fn of notices) fn(notice) },+ /** Nothing waits for the agent until `send`, which queues a new delivery. */+ idle: () => { status = 'delivered' }, send: () => { status = 'queued'; claimId = null },+ revoke: (value = true) => { revoked = value }, foreignClaim: () => { claimId = 'another-process' }, from: (id: string) => { from = id } }}describe('current Swarm role lookup', () => {@@ -2219,3 +2225,112 @@ describe('a machine its owner removed from a Swarm', () => {expect(sent).toHaveLength(1)})})++// Stability roadmap 2.4 (2026-10-28): every running agent's host read its delivery list each second, which kept a 5-pod+// backend busy with almost nothing waiting. The server now says over the daemon socket when an agent has work; the read+// every 30 s stays as the safety net, and the read each second comes back whenever the server has not said it pushes.+describe('deliveries the server pushes', () => {+ const reads = (t: ReturnType<typeof setup>) => vi.mocked(t.call).mock.calls+ .filter(([method, path]) => method === 'GET' && path.endsWith('/deliveries')).length+ const announced: BackendNotice = { type: 'super_tab_deliveries' }+ const waiting: BackendNotice = { type: 'super_tab_deliveries', tabId: 'tab', agentId: 'agent' }+ async function seconds(t: ReturnType<typeof setup>, host: SuperTabHost, count: number) {+ for (let i = 0; i < count; i++) { t.wait(1_000); await host.tick() }+ }++ it('reads an idle agent\'s list each second while the server has not said it pushes', async () => {+ const t = setup(), host = t.open()+ t.idle()+ t.push({ type: 'desk_changed', revision: 1 })+ await host.tick()+ await seconds(t, host, 10)+ expect(reads(t)).toBe(11)+ })++ it('reads an idle agent\'s list every 30 s once the server pushes', async () => {+ const t = setup(), host = t.open()+ t.idle()+ t.push(announced)+ await host.tick()+ await seconds(t, host, 29)+ expect(reads(t)).toBe(1)+ await seconds(t, host, 1)+ expect(reads(t)).toBe(2)+ })++ it('reads the agent\'s list at once when the server says work waits, and types it', async () => {+ const t = setup(), host = t.open()+ t.idle()+ t.push(announced)+ await host.tick()+ await seconds(t, host, 5)+ t.send()+ t.push(waiting)+ await host.tick()+ expect(reads(t)).toBe(2)+ expect(t.core.turns.deliver).toHaveBeenCalledTimes(1)+ })++ it('does not wait for the next second when the push arrives during a read', async () => {+ const t = setup(), host = t.open()+ t.idle()+ t.push(announced)+ const first = host.tick()+ t.send()+ t.push(waiting)+ await first+ await vi.waitFor(() => expect(t.core.turns.deliver).toHaveBeenCalledTimes(1))+ })++ // The Lead's rule for this change: a lost push never loses work.+ it('still types work whose push was lost, within 30 s', async () => {+ const t = setup(), host = t.open()+ t.idle()+ t.push(announced)+ await host.tick()+ t.wait(500)+ t.send()+ await seconds(t, host, 29)+ expect(t.core.turns.deliver).not.toHaveBeenCalled()+ await seconds(t, host, 1)+ expect(t.core.turns.deliver).toHaveBeenCalledTimes(1)+ })++ // The core lets input be written only for 2.5 s after a fresh authority read (accept); a slower read would expire it.+ it('reads each second while the agent has input in flight, then goes quiet', async () => {+ const t = setup(), host = t.open()+ t.push(announced)+ await host.tick()+ expect(t.core.turns.deliver).toHaveBeenCalledTimes(1)+ await seconds(t, host, 3)+ expect(reads(t)).toBe(4)+ t.event('delivered')+ await seconds(t, host, 2)+ expect(t.receipts).toHaveLength(1)+ const settled = reads(t)+ await seconds(t, host, 10)+ expect(reads(t)).toBe(settled)+ })++ it('goes back to reading each second once the server has not said it pushes for 10 minutes', async () => {+ const t = setup(), host = t.open()+ t.idle()+ t.push(announced)+ await host.tick()+ t.wait(10 * 60_000)+ await host.tick()+ await seconds(t, host, 2)+ expect(reads(t)).toBe(4)+ })++ it('reads nothing more for a push naming an agent that does not run here', async () => {+ const t = setup(), host = t.open()+ t.idle()+ t.push(announced)+ await host.tick()+ t.push({ type: 'super_tab_deliveries', tabId: 'tab', agentId: 'other' })+ await host.tick()+ await seconds(t, host, 1)+ expect(reads(t)).toBe(1)+ })+})diff --git a/cli/src/superTabs/host.ts b/cli/src/superTabs/host.tsindex e22ee3033..0786b6f9b 100644--- a/cli/src/superTabs/host.ts+++ b/cli/src/superTabs/host.ts@@ -8,7 +8,7 @@ import { attachedFiles, attachmentsOf } from './attachments.js'import { randomUUID } from 'node:crypto'import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'import { dirname, join } from 'node:path'-import type { CoreApi, TurnDelivery } from '../core/api.js'+import type { BackendNotice, CoreApi, TurnDelivery } from '../core/api.js'import { shellQuote } from '../orchestrator/prompts.js'import { agentCommandPrefix, fullCommandPrefix } from './shortCommand.js'import { readAuthSession } from '../lib/authSession.js'@@ -137,6 +137,14 @@ const JOIN_FINAL = new Set(['MACHINES_FULL', 'OWNER_REQUIRED'])* added to an overloaded backend (production at 5 pods, 2026-10-27). */const DELIVERY_RETRY_MS = 1_000const DELIVERY_RETRY_MAX_MS = 30_000+/** Stability roadmap 2.4 (2026-10-28): every host read each running agent's delivery list each second, almost always to find+ * nothing. A server that says it pushes (a `super_tab_deliveries` notice over the daemon socket, backend lib/adapterWs.ts)+ * names the agent when work waits; an idle agent's list is then read only this often, as the safety net for a lost push.+ * It matches the backend's quiet poll (liveness.ts SWEEP_EVERY_MS), so each read still runs a Lead's time checks. */+const DELIVERY_FALLBACK_MS = 30_000+/** The server repeats that it pushes every 4 minutes on each socket: without it for this long (an older server, a rollback,+ * this process started after the last one), every list is read each second again. */+const PUSH_LEASE_MS = 10 * 60_000const LEAD_RETRY_MS = 5_000const LEAD_RETRY_MAX_MS = 30 * 60_000const LEAD_REOPEN_WAIT_MS = 60_000@@ -195,6 +203,15 @@ export class SuperTabHost {private readonly authorized = new Map<string, number>()private readonly withheld = new WithheldDeliveries()private readonly stopHearing: () => void+ private readonly stopNotices: () => void+ /** Until when the server is known to push deliveries (PUSH_LEASE_MS). */+ private pushUntil = 0+ /** The agents' lists the server said have work since the last poll took them, by list key; `pushAll` reads every one. */+ private readonly pushes = new Set<string>()+ private pushAll = false+ /** Per agent's list, when it was last read, what it held, and whether any of it is still in this host's hands: input in+ * flight needs a fresh read each second for its 2.5 s lease (accept), so only a list with nothing to do may wait. */+ private readonly lists = new Map<string, { readAt: number; ids: string[]; busy: boolean }>()private stopped = falseprivate running: Promise<void> | null = nullprivate stopsRunning: Promise<void> | null = null@@ -295,6 +312,7 @@ export class SuperTabHost {}this.persist()this.stopHearing = core.turns.onDelivery(event => this.heard(event))+ this.stopNotices = core.account.onNotice(notice => this.pushed(notice))if (existsSync(nativeJournalFile(core.dataDir, machineId))) {try { this.native = new NativeRecorder(core, machineId) } catch (error) { this.log(String(error)) }}@@ -316,6 +334,7 @@ export class SuperTabHost {if (this.timer) clearInterval(this.timer)this.timer = nullthis.stopHearing()+ this.stopNotices()this.native?.stop()this.machines?.stop()this.authorized.clear()@@ -356,7 +375,25 @@ export class SuperTabHost {}tick(): Promise<void> {if (this.stopped || this.now() < this.updateUntil) return Promise.resolve()- return this.running ??= this.poll().catch(error => this.failed(error)).finally(() => { this.running = null })+ return this.running ??= this.poll().catch(error => this.failed(error)).finally(() => {+ this.running = null+ // Work the server announced during that poll is read now, not at the next second.+ if (!this.stopped && (this.pushAll || this.pushes.size)) void this.tick()+ })+ }+ /** The server said work waits for one agent here, or (naming none) that it pushes: read at once. */+ private pushed(notice: BackendNotice): void {+ if (notice.type !== 'super_tab_deliveries' || this.stopped) return+ this.pushUntil = this.now() + PUSH_LEASE_MS+ if (identifier(notice.tabId) && identifier(notice.agentId)) this.pushes.add(`list:${notice.tabId}:${notice.agentId}`)+ else this.pushAll = true+ void this.tick()+ }+ /** A delivery this host has nothing more to do for until someone acts: withheld from an agent that may not read it, or+ * reported uncertain and waiting for a person. Any other keeps its agent's list read each second. */+ private settled(delivery: Json): boolean {+ const id = delivery.id as string, entry = this.journal.entries[id]+ return this.unreadable.has(id) || (!!entry && entry.stage === 'uncertain' && delivery.status === 'uncertain' && delivery.receipt === entry.receipt)}/** Logged, except that an update-required answer is said once and then waits a minute, rather than every second. */private failed(error: unknown): void {@@ -558,6 +595,10 @@ export class SuperTabHost {private async poll(): Promise<void> {const data = (await this.sources.feed.refresh()).rosterif (this.stopped) return+ // Taken once the roster is in: a push heard from here on is read by the next poll, which follows this one at once.+ const pushes = new Set(this.pushes), pushAll = this.pushAll || this.now() >= this.pushUntil+ this.pushes.clear()+ this.pushAll = false// Multiply was closed by the ordinary 30-minute idle cleanup while still a// shared member. Membership outlives its task, terminal view and host process.// Retention only pauses cleanup when it fails; it must never hold up delivery.@@ -637,6 +678,12 @@ export class SuperTabHost {const path = `/api/super-tabs/${target.tabId}/agents/${target.agent.id}/deliveries`const listKey = `list:${target.tabId}:${target.agent.id}`if (this.backedOff(listKey)) continue+ const last = this.lists.get(listKey)+ if (last && !pushAll && !last.busy && !pushes.has(listKey) && this.now() - last.readAt < DELIVERY_FALLBACK_MS) {+ // Unread this time, its deliveries are as the last read left them: none is taken as gone.+ for (const id of last.ids) seen.add(id)+ continue+ }let listRead = falsetry {const deliveries = await backend(this.core, 'GET', path)@@ -645,6 +692,8 @@ export class SuperTabHost {listRead = truelistsRead.add(listKey)for (const item of deliveries) { const id = object(item).id; if (identifier(id)) listed.add(id) }+ const read = { readAt: this.now(), ids: deliveries.map(item => object(item).id).filter(identifier), busy: true }+ this.lists.set(listKey, read)this.deliveryRetries.delete(listKey)for (const item of deliveries) {const delivery = object(item)@@ -664,6 +713,7 @@ export class SuperTabHost {this.retire(delivery.id as string)}}+ read.busy = deliveries.some(item => !this.settled(object(item)))} catch (error) {if (!listRead) this.retryLater(listKey)// Authority checks and network failures both withhold input. A later fresh read may grant it again.@@ -675,6 +725,7 @@ export class SuperTabHost {for (const [harnessId, state] of this.keepAlives) if (!leads.has(harnessId) && !state.running) this.keepAlives.delete(harnessId)for (const [key, state] of this.endings) if (!ended.has(key) && !state.running) this.endings.delete(key)for (const deliveryId of this.unreadable) if (!seen.has(deliveryId)) this.unreadable.delete(deliveryId)+ for (const key of this.lists.keys()) if (!running.some(r => `list:${r.tabId}:${r.agent.id}` === key)) this.lists.delete(key)// Diego's issue, 2026-10-27: kept for every session ever seen, a daemon running for weeks held each Custom role's text.for (const skills of [this.skillChecks, this.skillModes, this.roleTexts]) {for (const harnessId of skills.keys()) if (!this.rosterSkills.has(harnessId)) skills.delete(harnessId)diff --git a/desktop/lib/ws/ws_conn.dart b/desktop/lib/ws/ws_conn.dartindex 66d6649da..f5b6e42a0 100644--- a/desktop/lib/ws/ws_conn.dart+++ b/desktop/lib/ws/ws_conn.dart@@ -25,6 +25,29 @@ typedef AccessTokenProvider = Future<String> Function(enum WsTransportKind { cloudE2ee, localPlaintext }+/// Opens a backend socket ([WsConn.cloudSocket]).+typedef CloudSocketFactory = WebSocketChannel Function(+ Uri uri,+ Iterable<String>? protocols,+);++/// How long [WsConn] waits before its [attempt]th reconnect: a window that doubles from 1 s to a+/// 30 s ceiling, the wait somewhere in its second half. The jitter spreads the tabs a backend+/// restart dropped together; the half that is not random is what keeps a socket that fails at once+/// to a handful of dials a minute, which no draw can undercut.+Duration reconnectDelay(int attempt, Random random) {+ final exponent = min(max(attempt - 1, 0), 5);+ final window = min(30000, 1000 * (1 << exponent));+ return Duration(+ milliseconds: window ~/ 2 + (random.nextDouble() * (window / 2)).round(),+ );+}++/// How long a connection has to stay up before its next drop starts the backoff over. Resetting it+/// the moment a socket was up made a socket the backend dropped right after its handshake redial+/// every second, for good: about 54 dials a minute from one browser tab (prod, 2026-10-28).+const connectionStableAfter = Duration(minutes: 1);+/// A `request()` call got no reply within its timeout — distinct from other request-level errors/// (an explicit `{error: ...}` response) so callers can tell "the peer is unresponsive" apart from/// "the peer answered with a failure."@@ -133,6 +156,13 @@ class WsConn {/// local transport, the dev fixture — means every frame rides the socket.final TerminalTransportPluginFactory? transportPlugins;+ /// Opens the backend socket. Null dials a real one; tests pass a fake.+ @visibleForTesting+ final CloudSocketFactory? cloudSocket;++ /// The reconnect jitter's source. Null is a fresh [Random].+ final Random _random;+Future<Map<String, dynamic>> Function(String type,Map<String, dynamic> payload,@@ -154,8 +184,13 @@ class WsConn {bool _ready = false;int _attempt = 0;Timer? _reconnectTimer;+ Timer? _stableTimer;Timer? _observerHandshakeTimer;String? _tokenUsed;++ /// The token a 4401 refresh just handed over, until a connection is up on it. The backend+ /// refusing that one too means the sign-in itself is gone; refreshing again only loops.+ String? _refreshedToken;bool _forceRelayReconnect = false;final _pending = <String, _PendingRpc>{};final _readinessWaiters = <Completer<void>>{};@@ -235,7 +270,9 @@ class WsConn {this.fixedReconnectDelay,this.relayCodecs,this.transportPlugins,- });+ this.cloudSocket,+ Random? random,+ }) : _random = random ?? Random();/// The socket to dial, decided afresh on every connect: whether the file is/// there now, not whether it was the last time something looked. A dial@@ -374,9 +411,9 @@ class WsConn {final channel =overPipe ??(!isLocal- ? WebSocketChannel.connect(+ ? (cloudSocket ?? _connectCloud)(uri,- protocols: token == null || token.isEmpty ? null : [token],+ token == null || token.isEmpty ? null : [token],): socket != null? await _connectLocalSocket(uri, socket) ?? _connectLocalPort(uri)@@ -489,7 +526,9 @@ class WsConn {void _markReady() {_observerHandshakeTimer?.cancel();_ready = true;- _attempt = 0;+ _refreshedToken = null;+ _stableTimer?.cancel();+ _stableTimer = Timer(connectionStableAfter, () => _attempt = 0);onStatus(ConnectionStatus.connected);_flushQueue();_settleReadiness();@@ -946,6 +985,7 @@ class WsConn {void _onDone(WebSocketChannel channel) {if (!identical(_channel, channel)) return;_observerHandshakeTimer?.cancel();+ _stableTimer?.cancel();_channel = null;_sub = null;_codec = null;@@ -981,9 +1021,17 @@ class WsConn {return;}if (code == 4401) {+ final token = _tokenUsed;+ if (token != null && token == _refreshedToken) {+ _closing = true;+ onStatus(ConnectionStatus.disconnected);+ onAuthFailure('Your SSO session expired. Please sign in again.');+ return;+ }unawaited(_refreshAndReconnect());return;}+ _refreshedToken = null;if (code == 4403) {_closing = true;onStatus(ConnectionStatus.disconnected);@@ -1000,8 +1048,16 @@ class WsConn {Future<void> _refreshAndReconnect() async {onStatus(ConnectionStatus.reconnecting);try {- await accessTokenProvider(true, _tokenUsed);- if (!_closing) await connect();+ _refreshedToken = await accessTokenProvider(true, _tokenUsed);+ if (_closing) return;+ // The first refusal in a run dials straight back with the new token; a run that keeps+ // failing waits its turn like any other drop.+ if (_attempt == 0) {+ _attempt = 1;+ await connect();+ } else {+ _scheduleReconnect();+ }} catch (_) {_closing = true;onStatus(ConnectionStatus.disconnected);@@ -1014,13 +1070,13 @@ class WsConn {_attempt++;onStatus(ConnectionStatus.reconnecting);_reconnectTimer?.cancel();- final exponent = min(max(_attempt - 1, 0), 5);- final delay =- fixedReconnectDelay ??- Duration(milliseconds: min(30000, 1000 * (1 << exponent)));+ final delay = fixedReconnectDelay ?? reconnectDelay(_attempt, _random);_reconnectTimer = Timer(delay, connect);}+ static WebSocketChannel _connectCloud(Uri uri, Iterable<String>? protocols) =>+ WebSocketChannel.connect(uri, protocols: protocols);+void _rejectPending(String reason) {_settleReadiness(reason);for (final pending in _pending.values) {@@ -1041,6 +1097,7 @@ class WsConn {_disposePlugin();_reconnectTimer?.cancel();_reconnectTimer = null;+ _stableTimer?.cancel();_rejectPending('WS closed');final sub = _sub;_sub = null;diff --git a/desktop/test/ws_reconnect_backoff_test.dart b/desktop/test/ws_reconnect_backoff_test.dartnew file mode 100644index 000000000..482720494--- /dev/null+++ b/desktop/test/ws_reconnect_backoff_test.dart@@ -0,0 +1,276 @@+import 'dart:async';+import 'dart:convert';+import 'dart:math';++import 'package:fake_async/fake_async.dart';+import 'package:flutter_test/flutter_test.dart';+import 'package:harness/core/models.dart' show ConnectionStatus;+import 'package:harness/ws/ws_conn.dart';+import 'package:web_socket_channel/web_socket_channel.dart';++/// How long the fake backend takes to answer or drop a socket.+const _latency = Duration(milliseconds: 50);++/// A backend socket that answers after [_latency]: either the machine's+/// `connected` and then a drop ([readyFirst]), or a close straight away.+class _Socket implements WebSocketChannel {+ _Socket({+ required this.closeWith,+ required this.readyFirst,+ this.upFor = _latency,+ });++ final int closeWith;+ final bool readyFirst;++ /// How long a socket that came up stays up before the drop.+ final Duration upFor;+ final _down = StreamController<dynamic>();+ late final _sink = _Sink(this);+ int? _closeCode;++ @override+ Future<void> get ready => Future<void>.delayed(_latency);+ @override+ Stream<dynamic> get stream => _down.stream;+ @override+ WebSocketSink get sink => _sink;+ @override+ int? get closeCode => _closeCode;+ @override+ String? get closeReason => null;++ void _selected(String machineId) {+ Timer(_latency, () {+ if (readyFirst) {+ _down.add(+ jsonEncode({+ 'type': 'connected',+ 'payload': {'machineId': machineId},+ }),+ );+ }+ Timer(readyFirst ? upFor : _latency, () => _drop(closeWith));+ });+ }++ void _drop(int code) {+ if (_down.isClosed) return;+ _closeCode = code;+ unawaited(_down.close());+ }++ @override+ dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation);+}++class _Sink implements WebSocketSink {+ _Sink(this._socket);+ final _Socket _socket;++ @override+ void add(dynamic data) {+ final frame = jsonDecode(data as String) as Map<String, dynamic>;+ if (frame['type'] == 'machine_select') {+ _socket._selected((frame['payload'] as Map)['machineId'] as String);+ }+ }++ @override+ Future<void> close([int? closeCode, String? closeReason]) async =>+ _socket._drop(closeCode ?? 1000);++ @override+ dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation);+}++/// A [WsConn] to the backend whose every socket is a [_Socket].+class _Rig {+ _Rig({+ required int closeWith,+ bool readyFirst = false,+ Random? random,+ bool refreshFails = false,+ Duration Function()? now,+ }) {+ conn = WsConn(+ wsBaseUrl: 'wss://backend.test',+ autonomousEnv: 'prod',+ machineId: 'm1',+ accessTokenProvider: (force, _) async {+ if (force) {+ refreshes++;+ if (refreshFails) throw StateError('refresh failed');+ token = 'tok-$refreshes';+ }+ return token;+ },+ onAuthFailure: authFailures.add,+ onEvent: (_) {},+ onStatus: statuses.add,+ random: random,+ cloudSocket: (uri, protocols) {+ dials.add(protocols?.single);+ dialedAt.add(now?.call() ?? Duration.zero);+ final hold = holdNextFor;+ holdNextFor = null;+ return _Socket(+ closeWith: closeWith,+ readyFirst: readyFirst,+ upFor: hold ?? _latency,+ );+ },+ );+ }++ late final WsConn conn;+ final dials = <String?>[];+ final dialedAt = <Duration>[];++ /// Keeps the next socket up this long before it is dropped.+ Duration? holdNextFor;+ final authFailures = <String>[];+ final statuses = <ConnectionStatus>[];+ var token = 'tok-0';+ var refreshes = 0;+}++void main() {+ for (final (name, random) in [+ ('any', null),+ ('the shortest', _Fixed(0)),+ ('the longest', _Fixed(1)),+ ]) {+ test('a socket that comes up and is dropped again redials only a handful of times a minute '+ '($name jitter)', () {+ fakeAsync((async) {+ final rig = _Rig(closeWith: 1006, readyFirst: true, random: random);+ unawaited(rig.conn.connect());+ async.elapse(const Duration(minutes: 1));+ expect(rig.dials.length, lessThanOrEqualTo(8));++ // Steady state: after the first minute the pace is the 30 s ceiling, give or take jitter.+ final before = rig.dials.length;+ async.elapse(const Duration(minutes: 5));+ expect(rig.dials.length - before, inInclusiveRange(9, 20));+ unawaited(rig.conn.close());+ });+ });+ }++ test('a socket that closes before it is up redials only a handful of times a minute', () {+ fakeAsync((async) {+ final rig = _Rig(closeWith: 1013);+ unawaited(rig.conn.connect());+ async.elapse(const Duration(minutes: 1));+ expect(rig.dials.length, lessThanOrEqualTo(8));+ unawaited(rig.conn.close());+ });+ });++ test('a connection that stayed up for a minute retries its next drop promptly again', () {+ fakeAsync((async) {+ final rig = _Rig(+ closeWith: 1006,+ readyFirst: true,+ now: () => async.elapsed,+ );+ unawaited(rig.conn.connect());+ // Back off to the 30 s ceiling first, then let the next socket stay up for two minutes.+ async.elapse(const Duration(minutes: 3));+ rig.holdNextFor = const Duration(minutes: 2);+ final healthy = rig.dials.length;+ async.elapse(const Duration(minutes: 3));+ expect(rig.dials.length, greaterThan(healthy + 1));+ final gap = rig.dialedAt[healthy + 1] - rig.dialedAt[healthy];+ // Up two minutes, then at most the first step's second, not the 15-30 s ceiling.+ expect(gap, lessThan(const Duration(minutes: 2, seconds: 2)));+ unawaited(rig.conn.close());+ });+ });++ test('a short-lived connection keeps backing off', () {+ fakeAsync((async) {+ final rig = _Rig(+ closeWith: 1006,+ readyFirst: true,+ now: () => async.elapsed,+ );+ unawaited(rig.conn.connect());+ async.elapse(const Duration(minutes: 3));+ rig.holdNextFor = const Duration(seconds: 20);+ final brief = rig.dials.length;+ async.elapse(const Duration(minutes: 2));+ final gap = rig.dialedAt[brief + 1] - rig.dialedAt[brief];+ expect(gap, greaterThan(const Duration(seconds: 20 + 15)));+ unawaited(rig.conn.close());+ });+ });++ test('reconnect delays are jittered, so many tabs dropped at once do not return together', () {+ final delays = {+ for (var seed = 0; seed < 20; seed++)+ reconnectDelay(4, Random(seed)).inMilliseconds,+ };+ expect(delays.length, greaterThan(10));+ for (final delay in delays) {+ // The fourth attempt waits between half of 8 s and 8 s.+ expect(delay, inInclusiveRange(4000, 8000));+ }+ });++ test('reconnect delays double and stop at 30 s', () {+ Duration low(int attempt) => reconnectDelay(attempt, _Fixed(0));+ Duration high(int attempt) => reconnectDelay(attempt, _Fixed(1));+ expect(low(1), const Duration(milliseconds: 500));+ expect(high(1), const Duration(seconds: 1));+ expect(low(2), const Duration(seconds: 1));+ expect(high(3), const Duration(seconds: 4));+ expect(low(6), const Duration(seconds: 15));+ expect(high(6), const Duration(seconds: 30));+ expect(high(40), const Duration(seconds: 30));+ expect(low(40), const Duration(seconds: 15));+ });++ test(+ '4401 refreshes once; a refused fresh token stops and asks for sign-in',+ () {+ fakeAsync((async) {+ final rig = _Rig(closeWith: 4401);+ unawaited(rig.conn.connect());+ async.elapse(const Duration(minutes: 1));+ expect(rig.refreshes, 1);+ expect(rig.dials, ['tok-0', 'tok-1']);+ expect(rig.authFailures, hasLength(1));+ expect(rig.conn.isClosed, isTrue);+ expect(rig.statuses.last, ConnectionStatus.disconnected);+ });+ },+ );++ test(+ '4401 after a refreshed token served a working connection refreshes again',+ () {+ fakeAsync((async) {+ final rig = _Rig(closeWith: 4401, readyFirst: true);+ unawaited(rig.conn.connect());+ async.elapse(const Duration(minutes: 1));+ // Each socket was up before the backend said the token expired: never a sign-out, and+ // never a refresh loop either.+ expect(rig.authFailures, isEmpty);+ expect(rig.refreshes, greaterThanOrEqualTo(2));+ expect(rig.dials.length, lessThanOrEqualTo(8));+ unawaited(rig.conn.close());+ });+ },+ );+}++class _Fixed implements Random {+ _Fixed(this.value);+ final double value;+ @override+ double nextDouble() => value;+ @override+ dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation);+}