Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

backend: remove expensive loops: event-driven terminal wait, 60 s socket re-checks with instant access changes, cursor paging (#1905)

0xken152ebca76b4724
diff --git a/backend/integration/superTabObserver.test.ts b/backend/integration/superTabObserver.test.ts
new file mode 100644
index 000000000..7bc06a97a
--- /dev/null
+++ b/backend/integration/superTabObserver.test.ts
@@ -0,0 +1,112 @@
+import { randomUUID } from 'node:crypto'
+import { EventEmitter } from 'node:events'
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import type { PrismaClient } from '@prisma/client'
+import type { AuthUser } from '../src/lib/ssoAuth.js'
+import type { Command } from '../src/superTabs/commands.js'
+import { localDatabase, type LocalDatabase } from './bench/mongo.js'
+
+// A terminal observer socket against a real database and store. Redis is replaced by one process's channels, and
+// the observer relay's global client by this suite's database; every permission read is real.
+const shared = vi.hoisted(() => ({ db: null as unknown as PrismaClient, channels: null as unknown as EventEmitter }))
+vi.mock('../src/lib/prisma.js', async original => ({ ...await original<typeof import('../src/lib/prisma.js')>(), get prisma() { return shared.db } }))
+vi.mock('../src/lib/bus.js', async original => {
+ const listen = (channel: string, callback: () => void) => {
+ shared.channels.on(channel, callback)
+ return Promise.resolve(() => { shared.channels.off(channel, callback) })
+ }
+ return { ...await original<typeof import('../src/lib/bus.js')>(),
+ publishDown: async () => 1, subscribeUp: async () => () => {},
+ subscribeShareChanged: (id: string, callback: () => void) => listen(`share:${id}`, callback),
+ subscribeSwarmAccessChanged: (tabId: string, callback: () => void) => listen(`swarm:${tabId}`, callback),
+ publishSwarmAccessChanged: async (tabId: string) => { shared.channels.emit(`swarm:${tabId}`); return 1 } }
+})
+vi.mock('../src/lib/hub.js', () => ({ trackSocketLiveness: () => () => {} }))
+const { SuperTabStore } = await import('../src/superTabs/store.js')
+const { attachObserver } = await import('../src/lib/observerWs.js')
+const { publishSwarmAccessChanged } = await import('../src/lib/bus.js')
+
+let database: LocalDatabase
+let store: InstanceType<typeof SuperTabStore>
+const owner: AuthUser = { sub: '100000000000000000000001', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
+const peer: AuthUser = { sub: '100000000000000000000002', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
+const host: AuthUser = { sub: '100000000000000000000003', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
+const config = { engine: 'codex', model: 'fixture-model', effort: 'high', contextWindow: 200000, engineVersion: 'fixture' }
+const run = (tab: string, command: Command, user = owner) => store.execute(user, tab, { operationId: randomUUID(), command })
+
+class Socket extends EventEmitter {
+ readyState = 1; bufferedAmount = 0
+ send = vi.fn()
+ close = vi.fn((_code?: number, _reason?: string) => { this.readyState = 3; this.emit('close') })
+ terminate = vi.fn(() => this.close())
+}
+const until = async (condition: () => boolean, ms: number) => {
+ for (const end = Date.now() + ms; !condition(); await new Promise(resolve => setTimeout(resolve, 10))) {
+ if (Date.now() > end) return false
+ }
+ return true
+}
+
+beforeAll(async () => {
+ database = await localDatabase()
+ shared.db = database.db
+ // The routes' wiring (routes/superTabs.ts): a narrowing of who may watch is told after its commit.
+ store = new SuperTabStore(database.db, Date.now, null, { accessChanged: tabId => { void publishSwarmAccessChanged(tabId) } })
+ for (const user of [owner, peer, host]) await database.db.user.create({ data: { id: user.sub, externalId: `fixture-${user.sub}`,
+ email: user.email, autonomousEnv: user.autonomousEnv } })
+ await database.db.machine.create({ data: { machineId: 'test-host', userId: host.sub, apiKey: 'fixture-only', managerId: 'fixture',
+ autonomousEnv: 'prod', authMode: 'remote' } })
+ await database.db.machineShare.create({ data: { machineId: 'test-host', ownerId: host.sub, autonomousEnv: 'prod', emails: [owner.email, peer.email] } })
+})
+beforeEach(() => { shared.channels = new EventEmitter() })
+afterAll(async () => { await database?.stop() })
+
+describe('a terminal observer of a Swarm agent', () => {
+ /** Peer watches a Builder on host's machine through the Swarm's share. */
+ async function watching(withLead = true) {
+ const tab = (await store.create(owner, randomUUID(), 'Observed')).id
+ if (withLead) await run(tab, { type: 'agent.add', conversationId: 'lead', machineId: 'test-host', harnessId: randomUUID(), name: 'Lead', configuration: config, lead: true })
+ for (const person of [peer, host]) {
+ const invitation = await run(tab, { type: 'invite.create', email: person.email, role: 'collaborator', conversationId: 'lead' })
+ await run(tab, { type: 'invite.accept', inviteId: invitation.result.inviteId as string }, person)
+ }
+ const harnessId = randomUUID()
+ const builder = (await run(tab, { type: 'agent.add', conversationId: 'lead', machineId: 'test-host', harnessId, name: 'Builder', configuration: config, lead: false }))
+ .result.agentId as string
+ const share = await database.db.harnessShare.create({ data: { id: randomUUID(), superTabId: tab, ownerId: host.sub, machineId: 'test-host',
+ agentId: harnessId, recipientEmail: peer.email, autonomousEnv: 'prod', name: 'Builder', ownerPublicKey: 'fixture-key',
+ expiresAt: new Date(Date.now() + 86_400_000), revokedAt: null } })
+ const ws = new Socket()
+ await attachObserver(ws as never, peer, share)
+ expect(ws.send).toHaveBeenCalledWith(JSON.stringify({ type: 'observer_connected' }))
+ return { tab, ws, builder }
+ }
+
+ it('closes within a second of its viewer being removed from the Swarm', async () => {
+ const { tab, ws } = await watching()
+ await run(tab, { type: 'member.remove', userId: peer.sub })
+ expect(await until(() => ws.close.mock.calls.length > 0, 1_000)).toBe(true)
+ expect(ws.close).toHaveBeenCalledWith(4403, expect.any(String))
+ })
+
+ it('closes within a second of the agent’s project no longer being shared with its viewer', async () => {
+ const { tab, ws } = await watching()
+ await run(tab, { type: 'project.sharing', projectId: 'lead', visibility: 'private', memberIds: [host.sub] })
+ expect(await until(() => ws.close.mock.calls.length > 0, 1_000)).toBe(true)
+ expect(ws.close).toHaveBeenCalledWith(4403, expect.any(String))
+ })
+
+ it('closes within a second of the watched session turning confidential, here by becoming its project’s Lead', async () => {
+ const { tab, ws, builder } = await watching(false)
+ await run(tab, { type: 'conversation.agent', conversationId: 'lead', agentId: builder, present: true, lead: true })
+ expect(await until(() => ws.close.mock.calls.length > 0, 1_000)).toBe(true)
+ expect(ws.close).toHaveBeenCalledWith(4403, expect.any(String))
+ })
+
+ it('stays open through a change that leaves its viewer allowed', async () => {
+ const { tab, ws } = await watching()
+ await run(tab, { type: 'conversation.create', title: 'Another task', brief: '', checks: [] })
+ await run(tab, { type: 'project.sharing', projectId: 'lead', visibility: 'private', memberIds: [host.sub, peer.sub] })
+ expect(await until(() => ws.close.mock.calls.length > 0, 500)).toBe(false)
+ })
+})
diff --git a/backend/integration/superTabs.test.ts b/backend/integration/superTabs.test.ts
index baed01018..0cee5ff63 100644
--- a/backend/integration/superTabs.test.ts
+++ b/backend/integration/superTabs.test.ts
@@ -1,4 +1,5 @@
import { createHash, randomUUID } from 'node:crypto'
+import { EventEmitter } from 'node:events'
import { execFile } from 'node:child_process'
import { promisify } from 'node:util'
import { join } from 'node:path'
@@ -1124,6 +1125,31 @@ describe('Super Tabs on a real Mongo replica set', () => {
expect(rest.next).toBeNull()
})
+ it('reads each hidden row once on its way to a page, however many there are, even when they share a sequence', async () => {
+ const { tab, lead } = await fixture()
+ // 2,000 of the Lead's notices, as one command before the store by parts wrote them: on one sequence, between two posts.
+ const row = (id: string, sequence: number, hidden: boolean) => ({ id, tabId: tab, conversationId: 'lead', sequence,
+ author: { kind: 'person', id: owner.sub, name: owner.email }, text: id, replyTo: null, recipientIds: hidden ? [lead] : [],
+ ...(hidden ? { source: { kind: 'agent-only' } } : {}) })
+ await db.superTabMessage.createMany({ data: [row('a-first', 0, false),
+ ...Array.from({ length: 2_000 }, (_, i) => row(`b-notice-${String(i).padStart(4, '0')}`, 1, true)), row('c-last', 2, false)] })
+ // What MongoDB reads of the messages for one page: an offset is walked in the index (keys), rows are fetched (documents).
+ const examined = async (read: () => Promise<unknown>) => {
+ await db.$runCommandRaw({ profile: 2 })
+ try { await read() } finally { await db.$runCommandRaw({ profile: 0 }) }
+ const { cursor } = await db.$runCommandRaw({ find: 'system.profile', filter: {}, projection: { ns: 1, keysExamined: 1, docsExamined: 1 }, limit: 1000 }) as
+ { cursor: { firstBatch: Array<{ ns: string; keysExamined?: number; docsExamined?: number }> } }
+ await db.$runCommandRaw({ drop: 'system.profile' })
+ return cursor.firstBatch.filter(entry => entry.ns.endsWith('.SuperTabMessage'))
+ .reduce((sum, entry) => sum + (entry.keysExamined ?? 0) + (entry.docsExamined ?? 0), 0)
+ }
+ let opened!: Awaited<ReturnType<typeof store.messages>>
+ expect(await examined(async () => { opened = await store.messages(owner, tab, 'lead', {}) })).toBeLessThan(6_000)
+ expect(opened.messages.map(m => m.id)).toEqual(['a-first', 'c-last'])
+ expect(opened.next).toBeNull()
+ expect((await store.messages(owner, tab, 'lead', { after: 0 })).messages.map(m => m.id)).toEqual(['c-last'])
+ })
+
it('accepts an agent-only notice only from an agent to itself alone', async () => {
const { tab, lead } = await fixture()
const worker = (await run(tab, { type: 'agent.add', conversationId: 'lead', machineId: 'test-host', harnessId: randomUUID(), name: 'Worker', configuration: config, lead: false })).result.agentId as string
@@ -2461,6 +2487,90 @@ describe('the Lead reads and types in an agent terminal', () => {
.rejects.toThrow('no longer waiting')
})
+ /** A store whose terminal answers are told the way the routes tell them across instances (lib/bus.ts), here in process.
+ * `heard: false` loses every notice, as a dropped publish would. It counts the Swarm loads a wait makes. */
+ function watched(options: { heard?: boolean; now?: () => number } = {}) {
+ const notices = new EventEmitter()
+ const at = new SuperTabStore(db, options.now ?? Date.now, null, {
+ terminalAnswered: requestId => { if (options.heard !== false) notices.emit(requestId) },
+ watchTerminal: async (requestId, onChange) => { notices.on(requestId, onChange); return () => { notices.off(requestId, onChange) } },
+ })
+ const load = vi.spyOn(at as unknown as { load: (...args: unknown[]) => unknown }, 'load')
+ return { at, loads: () => load.mock.calls.length, listeners: (requestId: string) => notices.listenerCount(requestId) }
+ }
+
+ it('waits for an answer without loading the Swarm again and again', async () => {
+ const { tab, lead, worker, claim } = await terminalFixture()
+ const requestId = (await run(tab, { type: 'agent.screen', agentId: worker }, host, lead)).result.requestId as string
+ expect(await claim()).toHaveLength(1)
+ const { at, loads, listeners } = watched()
+ expect(await at.terminalResult(host, tab, requestId, lead, 3_000)).toEqual({ requestId, pending: true })
+ expect(loads()).toBeLessThanOrEqual(2)
+ expect(listeners(requestId)).toBe(0)
+ })
+
+ it('returns the answer as soon as the host’s answer commits', async () => {
+ const { tab, lead, worker, claim } = await terminalFixture()
+ const requestId = (await run(tab, { type: 'agent.screen', agentId: worker }, host, lead)).result.requestId as string
+ expect(await claim()).toHaveLength(1)
+ const { at, loads } = watched()
+ const waiting = at.terminalResult(host, tab, requestId, lead, 20_000).then(answer => ({ answer, at: Date.now() }))
+ await new Promise(resolve => setTimeout(resolve, 300))
+ await at.execute(host, tab, { operationId: randomUUID(), command: { type: 'agent.screenResult', requestId, ok: true, text: 'screen' } }, worker)
+ const answeredAt = Date.now()
+ const { answer, at: returnedAt } = await waiting
+ expect(answer).toEqual({ requestId, ok: true, text: 'screen' })
+ expect(returnedAt - answeredAt).toBeLessThan(100)
+ expect(loads()).toBeLessThanOrEqual(4)
+ })
+
+ it('wakes a wait when the request turns late, and when the host refuses it', async () => {
+ const { tab, lead, worker, claim } = await terminalFixture()
+ const unseen = (await run(tab, { type: 'agent.screen', agentId: worker }, host, lead)).result.requestId as string
+ // The pickup window ends 300 ms into this store's clock.
+ const late = watched({ now: () => Date.now() + 9_700 })
+ const started = Date.now()
+ expect(await late.at.terminalResult(host, tab, unseen, lead, 20_000)).toEqual({ requestId: unseen, ok: false, error: expect.stringContaining('did not pick up') })
+ expect(Date.now() - started).toBeLessThan(2_000)
+ const replaced = (await run(tab, { type: 'agent.input', agentId: worker, keys: ['Enter'] }, host, lead)).result.requestId as string
+ await db.superTabTerminalRequest.update({ where: { id: replaced }, data: { harnessId: 'replaced-terminal' } })
+ const { at } = watched()
+ const waiting = at.terminalResult(host, tab, replaced, lead, 20_000)
+ await new Promise(resolve => setTimeout(resolve, 300))
+ const refusedAt = Date.now()
+ expect(await claim(at)).toEqual([])
+ expect(await waiting).toEqual({ requestId: replaced, ok: false, error: expect.stringContaining('changed') })
+ expect(Date.now() - refusedAt).toBeLessThan(1_000)
+ })
+
+ it('neither waits for nor fails on a notice channel that cannot subscribe, as while Redis is down', async () => {
+ const { tab, lead, worker, claim } = await terminalFixture()
+ for (const watching of [() => new Promise<() => void>(() => {}), () => Promise.reject(new Error('Redis is down'))]) {
+ const at = new SuperTabStore(db, Date.now, null, { watchTerminal: watching })
+ const requestId = (await run(tab, { type: 'agent.screen', agentId: worker }, host, lead)).result.requestId as string
+ expect(await claim()).toHaveLength(1)
+ const started = Date.now()
+ expect(await at.terminalResult(host, tab, requestId, lead, 1_000)).toEqual({ requestId, pending: true })
+ expect(Date.now() - started).toBeLessThan(2_000)
+ await run(tab, { type: 'agent.screenResult', requestId, ok: true, text: 'screen' }, host, worker)
+ expect(await at.terminalResult(host, tab, requestId, lead, 20_000)).toEqual({ requestId, ok: true, text: 'screen' })
+ }
+ })
+
+ it('still returns an answer whose notice was lost, at the next re-check', async () => {
+ const { tab, lead, worker, claim } = await terminalFixture()
+ const requestId = (await run(tab, { type: 'agent.screen', agentId: worker }, host, lead)).result.requestId as string
+ expect(await claim()).toHaveLength(1)
+ const { at, loads } = watched({ heard: false })
+ const waiting = at.terminalResult(host, tab, requestId, lead, 20_000)
+ await new Promise(resolve => setTimeout(resolve, 300))
+ await run(tab, { type: 'agent.screenResult', requestId, ok: true, text: 'screen' }, host, worker)
+ const answeredAt = Date.now()
+ expect(await waiting).toEqual({ requestId, ok: true, text: 'screen' })
+ expect(Date.now() - answeredAt).toBeLessThan(6_000)
+ expect(loads()).toBeLessThanOrEqual(3)
+ })
+
it('expires an unread answer after two minutes, without waiting for another request', async () => {
const { tab, lead, worker } = await terminalFixture()
const requestId = await answered(tab, worker, host, lead)
diff --git a/backend/src/lib/agentMessagesWs.test.ts b/backend/src/lib/agentMessagesWs.test.ts
index 160e6a382..73a284e3f 100644
--- a/backend/src/lib/agentMessagesWs.test.ts
+++ b/backend/src/lib/agentMessagesWs.test.ts
@@ -202,12 +202,15 @@ describe('blind shared-tab relay WebSocket', () => {
expect(m.publish).toHaveBeenCalledTimes(1)
})
- it('closes 4403 when a share change or the 5 second re-check finds the socket no longer admitted', async () => {
+ it('closes 4403 when a share change or the 60 second re-check finds the socket no longer admitted', async () => {
await connect(); m.admits.mockResolvedValue(false); changed(); await flush()
expect(ws.close).toHaveBeenCalledWith(4403, expect.any(String))
m.admits.mockResolvedValue(true); ws = new Socket(); await connect()
- await vi.advanceTimersByTimeAsync(5000); expect(ws.close).not.toHaveBeenCalled()
- m.admits.mockResolvedValue(false); await vi.advanceTimersByTimeAsync(5000)
+ const checks = m.admits.mock.calls.length
+ await vi.advanceTimersByTimeAsync(60_000); expect(ws.close).not.toHaveBeenCalled()
+ expect(m.admits.mock.calls.length).toBe(checks + 1)
+ m.admits.mockResolvedValue(false); await vi.advanceTimersByTimeAsync(59_999); expect(ws.close).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(1)
expect(ws.close).toHaveBeenCalledWith(4403, expect.any(String))
})
diff --git a/backend/src/lib/agentMessagesWs.ts b/backend/src/lib/agentMessagesWs.ts
index 722713081..f520c62a5 100644
--- a/backend/src/lib/agentMessagesWs.ts
+++ b/backend/src/lib/agentMessagesWs.ts
@@ -74,7 +74,8 @@ async function attach(ws: WebSocket, userId: string, share: string, machine: str
try { if (!await admits(userId, machine, share, env)) ws.close(4403, 'Not in this shared tab') }
catch { ws.close(1013, 'Shared tab temporarily unavailable') }
}
- const timer = setInterval(() => { void check() }, 5000); timer.unref(); disposers.push(() => clearInterval(timer))
+ // Every change to the tab is told (routes/agentMessages.ts `tell`); the minute's re-check covers the rest.
+ const timer = setInterval(() => { void check() }, 60_000); timer.unref(); disposers.push(() => clearInterval(timer))
keep(await subscribeAgentMessageShareChanged(share, () => { void check() }))
keep(await subscribeTabRelay(share, machine, message => { if (!closed) send({ type: 'amsg_frame', ...(message as object) }) }))
if (closed) return
diff --git a/backend/src/lib/bus.ts b/backend/src/lib/bus.ts
index f2d91c2db..e20c08189 100644
--- a/backend/src/lib/bus.ts
+++ b/backend/src/lib/bus.ts
@@ -227,6 +227,23 @@ export function publishAgentMessageShareChanged(id: string): Promise<number> {
return safePublish(`amsg-share:${id}`, '{}')
}
+/** Who may watch an agent's terminal in a Swarm may have narrowed (a member removed, a project's sharing changed): its
+ * observer sockets, on any instance, check again at once. No content on this channel. */
+export function subscribeSwarmAccessChanged(tabId: string, cb: () => void): Promise<() => void> {
+ return addSub(`swarm-access:${tabId}`, cb)
+}
+export function publishSwarmAccessChanged(tabId: string): Promise<number> {
+ return safePublish(`swarm-access:${tabId}`, '{}')
+}
+
+/** A Swarm terminal request was answered or refused: its sender's wait, on any instance, reads it again. No content. */
+export function subscribeTerminalAnswered(requestId: string, cb: () => void): Promise<() => void> {
+ return addSub(`terminal-answer:${requestId}`, cb)
+}
+export function publishTerminalAnswered(requestId: string): Promise<number> {
+ return safePublish(`terminal-answer:${requestId}`, '{}')
+}
+
/** Frames between daemons of a shared tab, by tab and machine: sealed, never read here. */
export function subscribeTabRelay(share: string, machineId: string, cb: (msg: unknown) => void): Promise<() => void> {
return addSub(`amsg:${share}:${machineId}`, cb as Cb)
diff --git a/backend/src/lib/observerWs.test.ts b/backend/src/lib/observerWs.test.ts
index 6fad522a0..f52898b28 100644
--- a/backend/src/lib/observerWs.test.ts
+++ b/backend/src/lib/observerWs.test.ts
@@ -1,19 +1,21 @@
import { EventEmitter } from 'node:events'
import { beforeEach, afterEach, describe, expect, it, vi } from 'vitest'
const m = vi.hoisted(() => ({
- auth: vi.fn(), grant: vi.fn(), up: vi.fn(), changed: vi.fn(), down: vi.fn(), live: vi.fn(),
+ auth: vi.fn(), grant: vi.fn(), up: vi.fn(), changed: vi.fn(), swarm: vi.fn(), down: vi.fn(), live: vi.fn(),
upgrade: vi.fn(), link: vi.fn(), profile: vi.fn(),
}))
vi.mock('./ssoAuth.js', () => ({ authenticateAccessToken: m.auth }))
vi.mock('./prisma.js', () => ({ prisma: { user: { findUnique: m.profile } } }))
vi.mock('../routes/harnessLinks.js', () => ({ recipientLink: m.link }))
vi.mock('../routes/harnessShares.js', () => ({ recipientShare: m.grant }))
-vi.mock('./bus.js', () => ({ publishDown: m.down, subscribeUp: m.up, subscribeShareChanged: m.changed }))
+vi.mock('./bus.js', () => ({ publishDown: m.down, subscribeUp: m.up, subscribeShareChanged: m.changed, subscribeSwarmAccessChanged: m.swarm }))
vi.mock('./hub.js', () => ({ trackSocketLiveness: m.live }))
vi.mock('./wsServer.js', () => ({ WS_LIMITS: { web: 1 }, createWss: () => ({ handleUpgrade: m.upgrade }) }))
import { attachObserver, handleObserverUpgrade } from './observerWs.js'
const user = { sub: 'ken', email: ' [email protected] ', autonomousEnv: 'prod' as const, role: 'user' }
-const share = { id: '11111111-1111-4111-8111-111111111111', machineId: 'm', agentId: 'a', ownerId: 'owner' }
+// As recipientShare and recipientLink return them: an invitation ends and may belong to a Swarm; a link does neither.
+const share = { id: '11111111-1111-4111-8111-111111111111', machineId: 'm', agentId: 'a', ownerId: 'owner', superTabId: null, expiresAt: new Date(Date.UTC(2099, 0)) }
+const link = { id: share.id, machineId: 'm', agentId: 'a', ownerId: 'owner', visibility: 'public' }
class Socket extends EventEmitter {
readyState = 1; bufferedAmount = 0
send = vi.fn()
@@ -27,7 +29,7 @@ describe('isolated observer WebSocket boundary', () => {
beforeEach(() => {
vi.useFakeTimers(); vi.resetAllMocks(); ws = new Socket()
disposeUp = vi.fn(); disposeShare = vi.fn(); disposeLive = vi.fn()
- m.auth.mockResolvedValue(user); m.grant.mockResolvedValue(share); m.link.mockResolvedValue(share); m.profile.mockResolvedValue({ name: 'Ken' })
+ m.auth.mockResolvedValue(user); m.grant.mockResolvedValue(share); m.link.mockResolvedValue(link); m.profile.mockResolvedValue({ name: 'Ken' })
m.down.mockResolvedValue(1); m.live.mockReturnValue(disposeLive)
m.up.mockImplementation(async (_id, callback) => { upward = callback; return disposeUp })
m.changed.mockImplementation(async (_id, callback) => { changed = callback; return disposeShare })
@@ -97,9 +99,56 @@ describe('isolated observer WebSocket boundary', () => {
await attach(); m.grant.mockResolvedValue(null); changed(); await flush()
expect(ws.close).toHaveBeenCalledWith(4403, expect.any(String))
ws = new Socket(); m.grant.mockResolvedValue(share); await attach()
- m.grant.mockResolvedValue(null); await vi.advanceTimersByTimeAsync(5000)
+ const checks = m.grant.mock.calls.length
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(m.grant.mock.calls.length).toBe(checks + 1)
+ m.grant.mockResolvedValue(null); await vi.advanceTimersByTimeAsync(59_999)
+ expect(ws.close).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(1)
expect(ws.close).toHaveBeenCalledWith(4403, expect.any(String))
})
+ it('closes at the invitation’s expiry, read again from each check', async () => {
+ let expiresAt = new Date(Date.now() + 10_000)
+ m.grant.mockImplementation(async () => Date.now() < expiresAt.getTime() ? { ...share, expiresAt } : null)
+ await attach()
+ // Renewed before it ended: the check at the old expiry keeps the socket and waits for the new one.
+ expiresAt = new Date(Date.now() + 20_000)
+ await vi.advanceTimersByTimeAsync(10_000); expect(ws.close).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(10_000)
+ expect(ws.close).toHaveBeenCalledWith(4403, expect.any(String))
+ // A closed socket keeps no expiry timer.
+ ws = new Socket(); expiresAt = new Date(Date.now() + 10_000); await attach(); ws.close()
+ const checks = m.grant.mock.calls.length
+ await vi.advanceTimersByTimeAsync(10_000)
+ expect(m.grant.mock.calls.length).toBe(checks)
+ })
+ it('re-checks a Swarm’s share at once when who may watch in that Swarm changes', async () => {
+ let swarmChanged!: () => void
+ const disposeSwarm = vi.fn()
+ m.swarm.mockImplementation(async (_tabId, callback) => { swarmChanged = callback; return disposeSwarm })
+ await attach()
+ expect(m.swarm).not.toHaveBeenCalled()
+ ws = new Socket(); m.grant.mockResolvedValue({ ...share, superTabId: 'tab-1' })
+ await attachObserver(ws as never, user, { ...share, superTabId: 'tab-1' } as never)
+ expect(m.swarm).toHaveBeenCalledWith('tab-1', expect.any(Function))
+ // The share moved to another Swarm: its changes are the ones heard from now on.
+ m.grant.mockResolvedValue({ ...share, superTabId: 'tab-2' }); changed(); await flush()
+ expect(m.swarm).toHaveBeenLastCalledWith('tab-2', expect.any(Function))
+ expect(disposeSwarm).toHaveBeenCalledTimes(1)
+ m.grant.mockResolvedValue(null); swarmChanged(); await flush()
+ expect(ws.close).toHaveBeenCalledWith(4403, expect.any(String))
+ expect(disposeSwarm).toHaveBeenCalledTimes(2)
+ // A share taken out of its Swarm stops listening to it.
+ ws = new Socket(); m.grant.mockResolvedValue(share)
+ await attachObserver(ws as never, user, { ...share, superTabId: 'tab-1' } as never)
+ expect(disposeSwarm).toHaveBeenCalledTimes(3)
+ // A socket that closes while subscribing keeps no subscription.
+ ws = new Socket(); let subscribed!: () => void
+ m.swarm.mockImplementationOnce(() => new Promise(resolve => { subscribed = () => resolve(disposeSwarm) }))
+ const pending = attachObserver(ws as never, user, { ...share, superTabId: 'tab-1' } as never); await flush()
+ ws.close(); subscribed(); await pending
+ expect(disposeSwarm).toHaveBeenCalledTimes(4)
+ })
it('database and owner outages are recoverable; admission rechecks race-safe authority', async () => {
m.grant.mockResolvedValue(null); await attach()
expect(ws.send).not.toHaveBeenCalled()
diff --git a/backend/src/lib/observerWs.ts b/backend/src/lib/observerWs.ts
index 9497b0751..ee1c0845b 100644
--- a/backend/src/lib/observerWs.ts
+++ b/backend/src/lib/observerWs.ts
@@ -10,12 +10,13 @@ import { recipientShare } from '../routes/harnessShares.js'
import { recipientLink } from '../routes/harnessLinks.js'
import type { AutonomousEnvironment } from './autonomousEnvironment.js'
import { prisma } from './prisma.js'
-import { publishDown, subscribeUp, subscribeShareChanged } from './bus.js'
+import { publishDown, subscribeUp, subscribeShareChanged, subscribeSwarmAccessChanged } from './bus.js'
import { guardedSendJson } from './wsSend.js'
import { trackSocketLiveness } from './hub.js'
import { createOrderedInbox } from './orderedInbox.js'
const wss = createWss(WS_LIMITS.web, { echoFirstProtocol: true })
+const RECHECK_MS = 60_000
export function handleObserverUpgrade(req: IncomingMessage, socket: Duplex, head: Buffer): void {
void (async () => {
@@ -37,7 +38,7 @@ export function handleObserverUpgrade(req: IncomingMessage, socket: Duplex, head
/** Separate from the machine hub: no group key, machine inventory, P2P, or generic RPC route. */
export async function attachObserver(ws: WebSocket, user: AuthUser | null,
- share: { id: string; machineId: string; ownerId: string }, linkEnv?: AutonomousEnvironment): Promise<void> {
+ share: { id: string; machineId: string; ownerId: string; superTabId?: string | null }, linkEnv?: AutonomousEnvironment): Promise<void> {
const connId = `observer:${randomUUID()}`
const send = (frame: unknown) => guardedSendJson(ws, frame, 'must')
const disposers: Array<() => void> = [trackSocketLiveness(ws)]
@@ -50,17 +51,36 @@ export async function attachObserver(ws: WebSocket, user: AuthUser | null,
void down('observer_close', {}).catch(() => {})
}
ws.once('close', close)
+ // Each change to who may watch is told (the share or link, its Swarm); the minute's re-check covers the rest, and an
+ // invitation is read again the moment it ends.
+ let expiry: NodeJS.Timeout | undefined
+ let swarm: { tabId: string; dispose: () => void } | null = null
+ disposers.push(() => { clearTimeout(expiry); swarm?.dispose() })
+ const follow = async (tabId: string | null | undefined) => {
+ if ((swarm?.tabId ?? null) === (tabId ?? null)) return
+ swarm?.dispose(); swarm = null
+ if (!tabId) return
+ const dispose = await subscribeSwarmAccessChanged(tabId, () => { void check() })
+ if (closed) dispose(); else swarm = { tabId, dispose }
+ }
const check = async () => {
try {
const allowed = linkEnv ? await recipientLink(share.id, user, linkEnv) : await recipientShare(share.id, user!)
- if (!allowed) ws.close(4403, 'Sharing ended or invitation expired')
+ if (!allowed) { ws.close(4403, 'Sharing ended or invitation expired'); return }
+ if (closed || !('expiresAt' in allowed)) return
+ clearTimeout(expiry)
+ // Later ends are armed by a later minute's check (a timer past 24.8 days would fire at once).
+ const left = allowed.expiresAt.getTime() - Date.now()
+ if (left < RECHECK_MS) expiry = setTimeout(() => { void check() }, left)
+ await follow(allowed.superTabId)
} catch { ws.close(1013, 'Sharing temporarily unavailable') }
}
- const timer = setInterval(() => { void check() }, 5000)
+ const timer = setInterval(() => { void check() }, RECHECK_MS)
timer.unref()
disposers.push(() => clearInterval(timer))
const keep = (dispose: () => void) => { if (closed) dispose(); else disposers.push(dispose) }
keep(await subscribeShareChanged(share.id, () => { void check() }))
+ await follow(share.superTabId)
keep(await subscribeUp(share.machineId, message => {
if (closed) return
const frame = message.frame
diff --git a/backend/src/routes/harnessShares.test.ts b/backend/src/routes/harnessShares.test.ts
index 65daf1d4a..ee067ea7a 100644
--- a/backend/src/routes/harnessShares.test.ts
+++ b/backend/src/routes/harnessShares.test.ts
@@ -3,7 +3,7 @@ import Fastify, { type FastifyInstance } from 'fastify'
import { randomUUID } from 'node:crypto'
const mocks = vi.hoisted(() => ({
prisma: { harnessShare: { findFirst: vi.fn(), findMany: vi.fn(), findUnique: vi.fn(), upsert: vi.fn(), updateMany: vi.fn() },
- machine: { findFirst: vi.fn(), findMany: vi.fn() }, user: { findMany: vi.fn(), findUnique: vi.fn() } },
+ machine: { findFirst: vi.fn(), findMany: vi.fn() }, user: { findMany: vi.fn(), findUnique: vi.fn() }, harnessLink: { findMany: vi.fn() } },
mayOpenShare: vi.fn(), presence: vi.fn(), changed: vi.fn(), listChanged: vi.fn(), auth: vi.fn(),
}))
vi.mock('../superTabs/store.js', () => ({ SuperTabStore: class { mayOpenShare = mocks.mayOpenShare } }))
@@ -32,6 +32,7 @@ describe('account-bound harness invitations', () => {
mocks.prisma.machine.findMany.mockResolvedValue([machine])
mocks.prisma.user.findMany.mockResolvedValue([{ id: owner.sub, name: 'D', email: owner.email }])
mocks.prisma.harnessShare.findMany.mockResolvedValue([])
+ mocks.prisma.harnessLink.findMany.mockResolvedValue([])
mocks.prisma.harnessShare.updateMany.mockResolvedValue({ count: 1 })
mocks.presence.mockResolvedValue(new Map([['m1', true]]))
app = Fastify(); app.setErrorHandler(errorHandler); registerAuthMiddleware(app, mocks.auth)
@@ -189,6 +190,16 @@ describe('account-bound harness invitations', () => {
expect((await app.inject({ method: 'DELETE', url: `/api/harness-shares/${id}`, headers: auth })).statusCode).toBe(200)
expect(mocks.listChanged).toHaveBeenCalledWith('ken', { reason: 'harness_shares' })
})
+ it('makes the viewers of the agent’s private links check again when the invitation behind them changes', async () => {
+ const id = randomUUID()
+ mocks.prisma.harnessShare.findUnique.mockResolvedValue({ id, ...input(), ownerId: 'owner', autonomousEnv: 'prod', superTabId: null,
+ revokedAt: new Date(), expiresAt: new Date(input().expiresAt) })
+ mocks.prisma.harnessLink.findMany.mockResolvedValue([{ id: 'link-1' }, { id: 'link-2' }])
+ expect((await app.inject({ method: 'DELETE', url: `/api/harness-shares/${id}`, headers: auth })).statusCode).toBe(200)
+ expect(mocks.prisma.harnessLink.findMany).toHaveBeenCalledWith({ where: { ownerId: 'owner', machineId: 'm1', agentId: 'agent-1',
+ autonomousEnv: 'prod', revokedAt: null, visibility: { not: 'public' } }, select: { id: true } })
+ expect(mocks.changed.mock.calls).toEqual([[id], ['link-1'], ['link-2']])
+ })
it('invites an address with no account yet, and survives the lookup failing, without telling anyone', async () => {
const id = randomUUID(), remove = () => app.inject({ method: 'DELETE', url: `/api/harness-shares/${id}`, headers: auth })
mocks.prisma.harnessShare.findUnique.mockResolvedValue({ id, recipientEmail: '[email protected]' })
diff --git a/backend/src/routes/harnessShares.ts b/backend/src/routes/harnessShares.ts
index 8ef1374da..717a57026 100644
--- a/backend/src/routes/harnessShares.ts
+++ b/backend/src/routes/harnessShares.ts
@@ -40,9 +40,14 @@ async function tellRecipient(shareId: string): Promise<void> {
const share = await prisma.harnessShare.findUnique({ where: { id: shareId } })
if (!share) return
const recipient = await prisma.user.findUnique({ where: { email: share.recipientEmail }, select: { id: true } })
- if (!recipient) return
- await publishDeviceMachineListChanged(recipient.id, { reason: 'harness_shares' })
- logger.info('harness share: told the recipient it changed', { shareId, recipientId: recipient.id })
+ if (recipient) {
+ await publishDeviceMachineListChanged(recipient.id, { reason: 'harness_shares' })
+ logger.info('harness share: told the recipient it changed', { shareId, recipientId: recipient.id })
+ }
+ // A private link admits the recipient only through this invitation (harnessLinks.ts recipientLink): its viewers check again.
+ const links = await prisma.harnessLink.findMany({ where: { ownerId: share.ownerId, machineId: share.machineId, agentId: share.agentId,
+ autonomousEnv: share.autonomousEnv, revokedAt: null, visibility: { not: 'public' } }, select: { id: true } })
+ for (const link of links) await publishShareChanged(link.id)
} catch { /* best effort */ }
}
diff --git a/backend/src/routes/superTabs.ts b/backend/src/routes/superTabs.ts
index 21a8cc8b2..d6c035641 100644
--- a/backend/src/routes/superTabs.ts
+++ b/backend/src/routes/superTabs.ts
@@ -125,7 +125,7 @@ const updateRequired = (_req: unknown, reply: FastifyReply) => sendError(reply,
/** The store this server runs on. Loaded only here: importing the bus opens its Redis connections. */
async function productionStore(github: GitHubTokens | null): Promise<SuperTabStore> {
- const [{ pub }, { announceSwarmDeliveries }] = await Promise.all([import('../lib/bus.js'), import('../lib/swarmDeliveryPushes.js')])
+ const [bus, { announceSwarmDeliveries }] = await Promise.all([import('../lib/bus.js'), import('../lib/swarmDeliveryPushes.js')])
return new SuperTabStore(prisma, Date.now, attachmentsFromEnv(env),
// A task that ends revokes the token it was given; a revoke that fails is logged inside and never blocks the write.
{ workEnded: (tabId, ended) => { void github?.release(tabId, ended).catch(error => logger.warn('[github] release failed', { tabId, error: String(error) })) },
@@ -133,8 +133,13 @@ async function productionStore(github: GitHubTokens | null): Promise<SuperTabSto
teamAccountsIdle: !github,
// Every server answers the quiet delivery poll from any server's full one, and tells the agent's machine of new work
// through whichever server holds its daemon socket (stability roadmap 2.4). Both only ever drop, never fail, on Redis.
- polls: redisPollMemory(pub),
- deliveriesWaiting: announceSwarmDeliveries })
+ polls: redisPollMemory(bus.pub),
+ deliveriesWaiting: announceSwarmDeliveries,
+ // The host's answer often lands on another instance than the sender's wait.
+ terminalAnswered: requestId => { void bus.publishTerminalAnswered(requestId) },
+ watchTerminal: (requestId, onChange) => bus.subscribeTerminalAnswered(requestId, onChange),
+ // Its observer sockets may be on any instance.
+ accessChanged: tabId => { void bus.publishSwarmAccessChanged(tabId) } })
}
export async function superTabRoutes(app: FastifyInstance, options: { store?: SuperTabStore; github?: GitHubTokens | null; reads?: SwarmReads; hosts?: HostReads;
diff --git a/backend/src/superTabs/projects.test.ts b/backend/src/superTabs/projects.test.ts
index c141790d6..867f1d121 100644
--- a/backend/src/superTabs/projects.test.ts
+++ b/backend/src/superTabs/projects.test.ts
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'
import { describe, expect, it } from 'vitest'
import { applyCommand, commandSchema, type Command, type CommandContext } from './commands.js'
import { createSuperTab, visibleState, type Actor, type MessageDraft } from './model.js'
-import { canReadAgent, canWatchAgent, markConfidential, projectOf, projectsOf, readConversation, skillModeOf, skillSetOf } from './projects.js'
+import { canReadAgent, canWatchAgent, markConfidential, projectOf, projectsOf, readConversation, skillModeOf, skillSetOf, watchingNarrowed } from './projects.js'
const owner: Actor = { kind: 'person', userId: 'owner', email: '[email protected]' }
const peer: Actor = { kind: 'person', userId: 'peer', email: '[email protected]' }
@@ -548,3 +548,45 @@ describe('who may watch an agent\'s terminal (final review, Critical 1)', () =>
expect(f.state.agents.find(a => a.id === lead)!.confidential).toBe(true)
})
})
+
+describe('whether a commit narrows who may watch a terminal', () => {
+ // Each case starts where peer may watch a Builder in the main project, then runs one command.
+ function watched() {
+ const f = setup()
+ const builder = f.add('lead', 'builder', false)
+ expect(canWatchAgent(f.state, peer, f.state.agents.find(a => a.id === builder)!)).toBe(true)
+ const before = structuredClone(f.state)
+ return { f, builder, before, narrowed: () => watchingNarrowed(before, f.state) }
+ }
+
+ it('is narrowed when a session turns confidential, whatever the command: here made its project’s Lead', () => {
+ const { f, builder, narrowed } = watched()
+ f.run({ type: 'conversation.create', title: 'Private', brief: '', checks: [], audience: { personIds: ['owner'] } })
+ expect(narrowed()).toBe(false)
+ f.run({ type: 'conversation.agent', conversationId: 'lead', agentId: builder, present: true, lead: true })
+ expect(narrowed()).toBe(true)
+ })
+
+ it('is narrowed when a person leaves or a project stops being shared with them', () => {
+ const removed = watched()
+ removed.f.run({ type: 'member.remove', userId: 'peer' })
+ expect(removed.narrowed()).toBe(true)
+ const unshared = watched()
+ unshared.f.run({ type: 'project.sharing', projectId: 'lead', visibility: 'private', memberIds: [] })
+ expect(unshared.narrowed()).toBe(true)
+ })
+
+ it('is narrowed when a watched agent leaves', () => {
+ const { f, builder, narrowed } = watched()
+ f.run({ type: 'agent.remove', agentId: builder })
+ expect(narrowed()).toBe(true)
+ })
+
+ it('is not narrowed by a change that leaves every watcher watching', () => {
+ const { f, narrowed } = watched()
+ f.run({ type: 'conversation.create', title: 'Another task', brief: '', checks: [] })
+ f.run({ type: 'project.sharing', projectId: 'lead', visibility: 'private', memberIds: ['peer'] })
+ f.add('lead', 'second', false)
+ expect(narrowed()).toBe(false)
+ })
+})
diff --git a/backend/src/superTabs/projects.ts b/backend/src/superTabs/projects.ts
index 854b1949f..0b6ee53bd 100644
--- a/backend/src/superTabs/projects.ts
+++ b/backend/src/superTabs/projects.ts
@@ -190,6 +190,28 @@ export function canWatchAgent(state: SuperTabState, actor: Actor, target: Agent,
return canReadAgent(state, actor, target) && (!isConfidential(state, target, elsewhere) || ownSession(actor, target))
}
+/** Whether someone who could watch an agent's terminal before a commit cannot after it (canWatchAgent, by the session the
+ * observer relay watches: machine, harness and machine owner): a person left, the agent left, its project stopped being
+ * shared with them, or the session turned confidential, by whatever command. Its observer sockets then check again. */
+export function watchingNarrowed(before: SuperTabState, after: SuperTabState): boolean {
+ const session = (a: Agent) => `${a.machineId}\n${a.harnessId}\n${a.machineOwnerId}`
+ const now = new Map(after.agents.map(a => [session(a), a]))
+ // canWatchAgent per person, with the agent's confidentiality read once per state.
+ const watchers = (state: SuperTabState, target: Agent) => {
+ const confidential = isConfidential(state, target)
+ return state.people.map(p => ({ kind: 'person' as const, userId: p.id, email: p.email }))
+ .filter(actor => canReadAgent(state, actor, target) && (!confidential || ownSession(actor, target)))
+ }
+ return before.agents.some(target => {
+ const was = watchers(before, target)
+ if (!was.length) return false
+ const still = now.get(session(target))
+ if (!still) return true
+ const is = new Set(watchers(after, still).map(actor => actor.userId))
+ return was.some(actor => !is.has(actor.userId))
+ })
+}
+
/** The only actors a confidential session admits: the person whose machine runs it, or the session itself. */
export const ownSession = (actor: Actor, target: Agent): boolean =>
actor.kind === 'person' ? actor.userId === target.machineOwnerId : actor.agentId === target.id
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index 70e5e6653..98f7c2f15 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -16,7 +16,7 @@ import { attachmentMime, attachmentName, AttachmentTooLarge, AVATAR_PURPOSE, AVA
import { randomBytes } from 'node:crypto'
import type { Readable } from 'node:stream'
import { logger } from '../utils/logger.js'
-import { canReadAgent, canReadGroup, canWatchAgent, groupProject, ledProject, projectOf, projectsOf, agentProjectId, readConversation, roleVersion, type ProjectVisibility } from './projects.js'
+import { canReadAgent, canReadGroup, canWatchAgent, groupProject, ledProject, projectOf, projectsOf, agentProjectId, readConversation, roleVersion, watchingNarrowed, type ProjectVisibility } from './projects.js'
import { bindSession, makeProjectPrivate } from './sessionScope.js'
import { confidentialSessions, otherRegistrations, recordConfidential } from './confidential.js'
import { answersRequest, dirtyGroups, noteProjectReports, projectUpdateRequest } from './projectUpdates.js'
@@ -39,7 +39,7 @@ import type { HoldReason } from './holdReason.js'
import { agentDisplayName } from './agentNames.js'
import { endedHolds, HOLDING, type WorkHold } from '../github/holds.js'
import { accrueLent } from './leaderboard/lent.js'
-import { mayDrive, TERMINAL_ANSWER_MS, TERMINAL_EXPIRY_MS, TERMINAL_PICKUP_MS, type TerminalEffect } from './terminal.js'
+import { mayDrive, TERMINAL_ANSWER_MS, TERMINAL_EXPIRY_MS, TERMINAL_PICKUP_MS, TERMINAL_RECHECK_MS, type TerminalEffect } from './terminal.js'
import { closeAssignment, closeUnassignedStart, readDecision, recentDecisions, timedDecisionOf, writeRouteEffects } from './routeDecisions.js'
import { listMentions, markDone, mentionsWaiting, openTags, readMention, resolveByReply, writeMentions } from './mentions.js'
import { answerable, fullFor, listDecisions, readAskedDecision, recordAnswer, waitingIn, waitingInProject, writeDecisions } from './decisions.js'
@@ -218,6 +218,13 @@ export interface SuperTabStoreHooks {
/** No Harness GitHub App on this server: a Mac's Harness Team account joins no Swarm and takes no teammates' work, since
* it has no git login to fetch or push with (commands.ts `teamAccountsIdle`). */
teamAccountsIdle?: boolean
+ /** A terminal request was answered or refused, after the commit (lib/bus.ts tells every instance). It must not throw. */
+ terminalAnswered?: (requestId: string) => void
+ /** Calls `onChange` when that request is answered or refused on any instance; resolves to the call that stops listening. */
+ watchTerminal?: (requestId: string, onChange: () => void) => Promise<() => void>
+ /** Who may watch an agent's terminal in this Swarm may have narrowed, after the commit: its observer sockets check again
+ * at once (lib/observerWs.ts). It must not throw. */
+ accessChanged?: (tabId: string) => void
/** The read cache this store reads through (cache.ts): the process's by default, none when null. */
readCache?: SwarmCache | null
/** When each caller's full delivery poll last succeeded, shared by every server (Redis in production, pollMemory.ts). Left
@@ -1109,6 +1116,10 @@ export class SuperTabStore {
}
if (command.type === 'project.delete' || command.type.startsWith('project.role')) await this.writeRoleTexts(tx, tabId, effects.state, command)
if (effects.terminal) await this.writeTerminal(tx, tabId, effects.terminal, context.now)
+ if (this.hooks.accessChanged && watchingNarrowed(original, effects.state)) {
+ const told = this.hooks.accessChanged
+ afterCommit(tx, () => told(tabId))
+ }
if (command.type === 'work.cancel') {
await closeAssignment(tx, tabId, command.assignmentId, 'cancelled', context.now)
}
@@ -1414,34 +1425,57 @@ export class SuperTabStore {
const answered = await tx.superTabTerminalRequest.updateMany({ where: { id: effect.id, tabId, status: 'claimed', claimedAt: { gte: new Date(now - TERMINAL_ANSWER_MS) } },
data: { status: 'done', ok: effect.ok, result: effect.text, error: effect.error } })
if (!answered.count) fail('REQUEST_CLOSED', 'This terminal request is no longer waiting for an answer.')
+ this.terminalAnswered(tx, effect.id)
+ }
+
+ private terminalAnswered(tx: Transaction, requestId: string) {
+ const told = this.hooks.terminalAnswered
+ if (told) afterCommit(tx, () => told(requestId))
}
- /** The requester alone reads the answer, once: the screen text is deleted as it is returned. Waits up to `waitMs` for it. */
+ /** The requester alone reads the answer, once: the screen text is deleted as it is returned. Waits up to `waitMs` for it,
+ * reading again when the host's answer or refusal is told, when the request turns late, or every few seconds unwoken. */
async terminalResult(user: AuthUser, tabId: string, requestId: string, asAgentId?: string, waitMs = 0) {
const deadline = Date.now() + waitMs
- for (;;) {
- const answer = await transaction(this.db, async tx => {
- const { state } = await this.load(tx, user, tabId)
- const actor = await this.actor(tx, user, state, asAgentId)
- author(state, actor)
- const now = this.now()
- const row = await tx.superTabTerminalRequest.findFirst({ where: { id: requestId, tabId, createdAt: { gte: new Date(now - TERMINAL_EXPIRY_MS) } } })
- if (!row || row.requesterKey !== actorKey(actor)) fail('NOT_FOUND', 'This terminal request is unavailable.', 404)
- // A sender who has since lost write access, or is no longer the agent's project Lead, gets nothing.
- mayDrive(state, actor, agent(state, row.agentId), await this.confidential(tx, row.agentId))
- const late = row.status === 'pending' && now - row.createdAt.getTime() > TERMINAL_PICKUP_MS
- ? 'The agent’s machine did not pick up the request within 10 seconds. It may be offline or asleep.'
- : row.status === 'claimed' && now - row.claimedAt!.getTime() > TERMINAL_ANSWER_MS
- ? 'The agent’s machine picked up the request but did not answer within 30 seconds.' : null
- if (row.status !== 'done' && !late) return null
- await tx.superTabTerminalRequest.delete({ where: { id: row.id } })
- if (late || !row.ok) return { requestId, ok: false, error: late ?? row.error }
- return { requestId, ok: true, ...(row.result !== null ? { text: row.result } : {}) }
- })
- if (answer) return answer
- if (Date.now() >= deadline) return { requestId, pending: true }
- await new Promise(resolve => setTimeout(resolve, 250))
- }
+ let told = false, done = false, wake = () => {}, stop = () => {}
+ // Never awaited: while Redis is down a subscribe waits for it, and the wait reads on its own clock meanwhile.
+ if (waitMs > 0) this.hooks.watchTerminal?.(requestId, () => { told = true; wake() })
+ .then(dispose => { if (done) dispose(); else stop = dispose }, () => {})
+ try {
+ for (;;) {
+ told = false
+ const { answer, lateIn } = await transaction(this.db, async tx => {
+ const { state } = await this.load(tx, user, tabId)
+ const actor = await this.actor(tx, user, state, asAgentId)
+ author(state, actor)
+ const now = this.now()
+ const row = await tx.superTabTerminalRequest.findFirst({ where: { id: requestId, tabId, createdAt: { gte: new Date(now - TERMINAL_EXPIRY_MS) } } })
+ if (!row || row.requesterKey !== actorKey(actor)) fail('NOT_FOUND', 'This terminal request is unavailable.', 404)
+ // A sender who has since lost write access, or is no longer the agent's project Lead, gets nothing.
+ mayDrive(state, actor, agent(state, row.agentId), await this.confidential(tx, row.agentId))
+ // How long until a request still waiting turns late by itself.
+ const lateIn = row.status === 'pending' ? row.createdAt.getTime() + TERMINAL_PICKUP_MS - now
+ : row.status === 'claimed' ? row.claimedAt!.getTime() + TERMINAL_ANSWER_MS - now : 0
+ const late = row.status === 'pending' && lateIn < 0
+ ? 'The agent’s machine did not pick up the request within 10 seconds. It may be offline or asleep.'
+ : row.status === 'claimed' && lateIn < 0
+ ? 'The agent’s machine picked up the request but did not answer within 30 seconds.' : null
+ if (row.status !== 'done' && !late) return { answer: null, lateIn }
+ await tx.superTabTerminalRequest.delete({ where: { id: row.id } })
+ if (late || !row.ok) return { answer: { requestId, ok: false, error: late ?? row.error }, lateIn }
+ return { answer: { requestId, ok: true, ...(row.result !== null ? { text: row.result } : {}) }, lateIn }
+ })
+ if (answer) return answer
+ const left = deadline - Date.now()
+ if (left <= 0) return { requestId, pending: true }
+ if (told) continue
+ await new Promise<void>(resolve => {
+ const timer = setTimeout(resolve, Math.min(left, lateIn + 1, TERMINAL_RECHECK_MS))
+ wake = () => { clearTimeout(timer); resolve() }
+ })
+ wake = () => {}
+ }
+ } finally { done = true; stop() }
}
/** Each waiting request for this host's agents is handed over once. One that waited too long, or whose terminal, agent or
@@ -1465,7 +1499,8 @@ export class SuperTabStore {
? 'The agent’s machine did not pick up the request within 10 seconds. It may be offline or asleep.'
: await this.terminalRefusal(tx, states.get(row.tabId)!, row)
if (refusal) {
- await tx.superTabTerminalRequest.updateMany({ where: { id: row.id, status: 'pending' }, data: { status: 'done', ok: false, error: refusal } })
+ const refused = await tx.superTabTerminalRequest.updateMany({ where: { id: row.id, status: 'pending' }, data: { status: 'done', ok: false, error: refusal } })
+ if (refused.count) this.terminalAnswered(tx, row.id)
continue
}
const claimed = await tx.superTabTerminalRequest.updateMany({ where: { id: row.id, status: 'pending' }, data: { status: 'claimed', claimedAt: new Date(now) } })
@@ -1578,12 +1613,15 @@ export class SuperTabStore {
...(forward ? past(cursor.after!, cursor.afterId, 'gt') : cursor.before !== undefined ? past(cursor.before, cursor.beforeId, 'lt') : {}) }
// An agent-only notice is listed for its own recipient alone. Hidden rows never shorten a page: Desktop reads on from the
// last row it shows, so the page reads past them until it has 101 rows this reader may see, or reaches the end.
+ // Each batch reads on from the last row of the one before, never by an offset that walks every row before it again.
const rows: SuperTabMessage[] = []
- for (let skip = 0; rows.length <= 100; skip += 101) {
- const order = forward ? 'asc' as const : 'desc' as const
- const batch = await tx.superTabMessage.findMany({ where, orderBy: [{ sequence: order }, { id: order }], skip, take: 101 })
+ const order = forward ? 'asc' as const : 'desc' as const
+ for (let edge: SuperTabMessage | undefined; rows.length <= 100;) {
+ const batch = await tx.superTabMessage.findMany({ where: edge ? { AND: [where, past(edge.sequence, edge.id, forward ? 'gt' : 'lt')] } : where,
+ orderBy: [{ sequence: order }, { id: order }], take: 101 })
rows.push(...batch.filter(m => !agentOnly(m.source) || (actor.kind === 'agent' && m.recipientIds.includes(actor.agentId))))
if (batch.length < 101) break
+ edge = batch.at(-1)
}
const page = rows.slice(0, 100)
const next = rows.length > 100 ? { sequence: page.at(-1)!.sequence, id: page.at(-1)!.id } : null
diff --git a/backend/src/superTabs/terminal.ts b/backend/src/superTabs/terminal.ts
index 4c648360c..158286fa4 100644
--- a/backend/src/superTabs/terminal.ts
+++ b/backend/src/superTabs/terminal.ts
@@ -11,6 +11,8 @@ export const TERMINAL_KEYS = ['Enter', 'Esc', 'Tab', 'Up', 'Down', 'Left', 'Righ
/** The host has this long to pick a request up, then this long to answer it; after either, the request fails with a plain reason. */
export const TERMINAL_PICKUP_MS = 10_000
export const TERMINAL_ANSWER_MS = 30_000
+/** A sender's wait reads its request again this often even unwoken, in case the notice of its answer was lost. */
+export const TERMINAL_RECHECK_MS = 5_000
/** An unread answer, which can hold a screen's secrets, is gone after this long: refused on read, and removed by a TTL index. */
export const TERMINAL_EXPIRY_MS = 120_000
/** Run by the singleton worker at startup, like the product-usage expiry indexes. */
diff --git a/backend/vitest.super-tabs.config.ts b/backend/vitest.super-tabs.config.ts
index cb5657057..828110263 100644
--- a/backend/vitest.super-tabs.config.ts
+++ b/backend/vitest.super-tabs.config.ts
@@ -3,5 +3,5 @@ import { defineConfig } from 'vitest/config'
export default defineConfig({ test: { include: ['integration/superTabs.test.ts', 'integration/superTabSharing.test.ts', 'integration/superTabGithub.test.ts', 'integration/superTabTeam.test.ts', 'integration/superTabWorkspaces.test.ts',
'integration/superTabRead.test.ts', 'integration/superTabLeaderboard.test.ts', 'integration/superTabMigration.test.ts', 'integration/superTabLiveness.test.ts', 'integration/superTabTidy.test.ts',
'integration/superTabProfile.test.ts', 'integration/superTabLegacyDrop.test.ts', 'integration/shareRaces.test.ts',
- 'integration/superTabIndexes.test.ts', 'integration/superTabCache.test.ts'], hookTimeout: 120000, testTimeout: 30000, setupFiles: ['integration/cacheOn.test-support.ts'],
+ 'integration/superTabIndexes.test.ts', 'integration/superTabCache.test.ts', 'integration/superTabObserver.test.ts'], hookTimeout: 120000, testTimeout: 30000, setupFiles: ['integration/cacheOn.test-support.ts'],
fileParallelism: false, env: { NODE_ENV: 'test', HARNESS_CREDENTIAL_ENCRYPTION_KEY: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' } } })