Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: add viewer ledger for process cleanup and optimize remote inventory collection

0xken156aa085aa9289
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 390f40de1..06563c6c9 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -101,6 +101,7 @@ import { preTrustClaudeProject, preTrustCodexProject } from './lib/claudeTrust.j
import { materializeWorkspace } from './dsh/materialize.js'
import { dshLaunch } from './dsh/launch.js'
import { DshViewerManager } from './dsh/viewer.js'
+import { ViewerLedger } from './dsh/viewerLedger.js'
import { DshVerdictWatcher, type DshVerdict } from './dsh/verdict.js'
import { dshCommand, dshUsage } from './dsh/command.js'
import type { AgentDshContext } from './lib/agentFrame.js'
@@ -1608,6 +1609,11 @@ async function runForeground(session: AuthSession): Promise<void> {
const session = registry.byAgent(agentId)
if (session && registry.terminalAvailable(agentId)) syncSession(session)
}
+ // Viewers an earlier daemon started and never stopped (crash, force quit, SIGKILL) are still running
+ // and still polling; stop them BEFORE this daemon starts its own, or they accumulate a generation per
+ // restart. Only pids whose live start time matches what that daemon recorded are touched.
+ const viewerLedger = new ViewerLedger({ log: (line) => console.log(line) })
+ viewerLedger.reapOrphans()
const dshViewers = new DshViewerManager({
onUrl: (agentId, url) => {
dshFrameFor(agentId).viewerUrl = url
@@ -1615,6 +1621,7 @@ async function runForeground(session: AuthSession): Promise<void> {
syncCompanion(agentId)
},
log: (line) => console.log(line),
+ ledger: viewerLedger,
})
const dshVerdicts = new DshVerdictWatcher({
onChange: (agentId, verdict) => {
diff --git a/cli/src/dsh/shell.ts b/cli/src/dsh/shell.ts
index fcbe82d9e..949044fb2 100644
--- a/cli/src/dsh/shell.ts
+++ b/cli/src/dsh/shell.ts
@@ -138,6 +138,15 @@ export function runDshCommand(script: string, opts: DshCommandOptions): Promise<
export function killProcessGroup(child: ChildProcess, graceMs = 3_000): void {
const pid = child.pid
if (!pid) return
+ const timer = killPidGroup(pid, graceMs)
+ child.once('exit', () => clearTimeout(timer))
+}
+
+/** The same, for a group this process did not spawn (a viewer left over from an earlier daemon —
+ * see viewerLedger.ts): nothing to wait on, so the SIGKILL follow-up fires unconditionally. */
+export function killPidGroup(pid: number, graceMs = 3_000): NodeJS.Timeout {
+ // pid 1 would make `-pid` "every process in my session"; nothing this daemon reaps is init.
+ if (!Number.isInteger(pid) || pid <= 1) return setTimeout(() => {}, 0)
const signalGroup = (signal: NodeJS.Signals): void => {
try { process.kill(-pid, signal) } catch { /* already gone */ }
try { process.kill(pid, signal) } catch { /* already gone */ }
@@ -145,5 +154,5 @@ export function killProcessGroup(child: ChildProcess, graceMs = 3_000): void {
signalGroup('SIGTERM')
const timer = setTimeout(() => signalGroup('SIGKILL'), graceMs)
timer.unref?.()
- child.once('exit', () => clearTimeout(timer))
+ return timer
}
diff --git a/cli/src/dsh/viewer.ts b/cli/src/dsh/viewer.ts
index fdd53b380..e7f1710e2 100644
--- a/cli/src/dsh/viewer.ts
+++ b/cli/src/dsh/viewer.ts
@@ -20,6 +20,7 @@ import { installedDsh, type InstalledDsh } from './installed.js'
import { isViewerPackage } from './manifest.js'
import { resolveDshCommand } from './materialize.js'
import { isShellNoise, killProcessGroup, spawnDshCommand } from './shell.js'
+import type { ViewerLedger } from './viewerLedger.js'
import { viewerTarget } from '../lib/viewerWire.js'
/** The viewer that will actually run for a harness: its own, or the package it points at. */
@@ -75,6 +76,8 @@ export interface DshViewerDeps {
now?: () => number
/** How `viewer.use` finds its package; the installed index by default. */
lookup?: (id: string) => InstalledDsh | undefined
+ /** Where each spawned viewer's pid is recorded so the next daemon can reap it (viewerLedger.ts). */
+ ledger?: ViewerLedger
}
interface ViewerState {
@@ -266,6 +269,7 @@ export class DshViewerManager {
},
})
state.child = child
+ if (child.pid) this.deps.ledger?.add({ pid: child.pid, agentId: state.agentId, dshId: state.dsh.id, viewerDir: viewer.dir })
let logged = 0
const onData = (chunk: Buffer): void => {
for (const line of chunk.toString('utf8').split('\n')) {
@@ -277,6 +281,7 @@ export class DshViewerManager {
child.stderr?.on('data', onData)
child.on('error', (error) => this.log(`[dsh] ${state.dsh.id} viewer could not start · ${error.message}`))
child.on('exit', (code, signal) => {
+ if (child.pid) this.deps.ledger?.remove(child.pid)
if (state.child !== child) return
state.child = null
state.port = null
diff --git a/cli/src/dsh/viewerLedger.spec.ts b/cli/src/dsh/viewerLedger.spec.ts
new file mode 100644
index 000000000..18b8c479e
--- /dev/null
+++ b/cli/src/dsh/viewerLedger.spec.ts
@@ -0,0 +1,91 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { spawn, type ChildProcess } from 'node:child_process'
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { ViewerLedger, processStartedAt } from './viewerLedger.js'
+
+describe('ViewerLedger', () => {
+ let dir: string
+ beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'viewer-ledger-')) })
+ afterEach(() => { rmSync(dir, { recursive: true, force: true }) })
+
+ const entry = (pid: number, startedAt = 1_000_000) => ({ pid, startedAt, agentId: 'agent-1', dshId: 'acme/thing', viewerDir: '/i/acme/thing' })
+
+ it('records a spawned viewer and forgets it on exit, atomically on disk', () => {
+ const file = join(dir, 'viewers.json')
+ const ledger = new ViewerLedger({ file, now: () => 42 })
+ ledger.add({ pid: 101, agentId: 'a', dshId: 'x/y', viewerDir: '/v' })
+ ledger.add(entry(102))
+ expect(JSON.parse(readFileSync(file, 'utf8'))).toEqual([
+ { pid: 101, agentId: 'a', dshId: 'x/y', viewerDir: '/v', startedAt: 42 },
+ entry(102),
+ ])
+ ledger.remove(101)
+ expect(ledger.read()).toEqual([entry(102)])
+ // A pid seen again replaces its old row rather than duplicating it.
+ ledger.add(entry(102, 2_000_000))
+ expect(ledger.read()).toEqual([entry(102, 2_000_000)])
+ })
+
+ it('reads a missing or malformed file as empty', () => {
+ const ledger = new ViewerLedger({ file: join(dir, 'nope.json') })
+ expect(ledger.read()).toEqual([])
+ expect(ledger.reapOrphans()).toEqual([])
+ })
+
+ it('reaps only a pid that is alive AND started when the ledger says; then starts over', () => {
+ const file = join(dir, 'viewers.json')
+ const killed: number[] = []
+ const log: string[] = []
+ const ledger = new ViewerLedger({
+ file,
+ startedAt: (pid) => ({ 11: 1_000_500, 12: 5_000_000, 13: null }[pid] ?? null),
+ kill: (pid) => killed.push(pid),
+ log: (line) => log.push(line),
+ })
+ ledger.add(entry(11)) // alive, start agrees (within tolerance) → reaped
+ ledger.add(entry(12)) // alive, but a DIFFERENT process now wears this pid → left alone
+ ledger.add(entry(13)) // gone → nothing to do
+ ledger.add(entry(1)) // never: pid 1 is init, and its group is everything → refused at add
+ const reaped = ledger.reapOrphans()
+ expect(reaped.map((e) => e.pid)).toEqual([11])
+ expect(killed).toEqual([11])
+ expect(log.some((l) => l.includes('reaped orphaned acme/thing viewer pid 11'))).toBe(true)
+ expect(log.some((l) => l.includes('pid 12 is no longer'))).toBe(true)
+ expect(ledger.read()).toEqual([])
+ })
+
+ it('a ledger file naming pid 1 is not trusted either', () => {
+ const file = join(dir, 'viewers.json')
+ writeFileSync(file, JSON.stringify([entry(1), entry(0), entry(-5)]))
+ const killed: number[] = []
+ const ledger = new ViewerLedger({ file, startedAt: () => 1_000_000, kill: (pid) => killed.push(pid) })
+ expect(ledger.read()).toEqual([])
+ expect(ledger.reapOrphans()).toEqual([])
+ expect(killed).toEqual([])
+ })
+
+ it('processStartedAt reads a real process and reports a dead one as null', async () => {
+ const child: ChildProcess = spawn('sleep', ['30'], { stdio: 'ignore' })
+ try {
+ const at = processStartedAt(child.pid!)
+ expect(at).not.toBeNull()
+ expect(Math.abs(at! - Date.now())).toBeLessThan(10_000)
+ } finally {
+ child.kill('SIGKILL')
+ await new Promise((resolve) => child.once('exit', resolve))
+ }
+ expect(processStartedAt(child.pid!)).toBeNull()
+ })
+
+ it('the default reaper really stops a leftover process group', async () => {
+ const file = join(dir, 'viewers.json')
+ const child = spawn('sleep', ['30'], { stdio: 'ignore', detached: true })
+ const exited = new Promise<void>((resolve) => child.once('exit', () => resolve()))
+ const ledger = new ViewerLedger({ file, log: vi.fn() })
+ ledger.add({ pid: child.pid!, agentId: 'a', dshId: 'x/y', viewerDir: '/v' })
+ expect(ledger.reapOrphans().map((e) => e.pid)).toEqual([child.pid])
+ await exited
+ })
+})
diff --git a/cli/src/dsh/viewerLedger.ts b/cli/src/dsh/viewerLedger.ts
new file mode 100644
index 000000000..26800b957
--- /dev/null
+++ b/cli/src/dsh/viewerLedger.ts
@@ -0,0 +1,123 @@
+/**
+ * Which viewer processes THIS daemon has started, on disk, so the NEXT daemon can stop the ones it
+ * never got to.
+ *
+ * Viewers are spawned detached, in their own process group (shell.ts spawnDshCommand), and only the
+ * daemon's clean exits stop them (`dshViewers.stopAll()` on shutdown and on the self-update handoff).
+ * Every other way a daemon ends — a crash, a force quit, `harness stop` falling through to SIGKILL —
+ * leaves them running under launchd/systemd, still polling, still holding memory, and the daemon
+ * that comes up next starts a fresh set beside them: seven generations of one viewer were found on
+ * a single Mac. Nothing in the packages can fix that, so the daemon keeps a ledger.
+ *
+ * A pid alone is not proof — pids are reused — so each entry also records when the process started,
+ * and the reaper only touches a pid whose live start time agrees with the ledger's.
+ */
+import { execFileSync } from 'node:child_process'
+import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { env } from '../config/env.js'
+import { killPidGroup } from './shell.js'
+
+export interface ViewerLedgerEntry {
+ pid: number
+ /** ms epoch when the daemon spawned it; compared against the live process's start time. */
+ startedAt: number
+ agentId: string
+ dshId: string
+ viewerDir: string
+}
+
+/** How far the ledger's spawn time and the kernel's process start time may disagree and still be
+ * the same process. Spawn-to-exec is milliseconds; `ps` prints whole seconds. */
+const START_TOLERANCE_MS = 10_000
+
+export const VIEWER_LEDGER_FILE = join(env.ADAPTER_DATA_DIR, 'viewers.json')
+
+/** When `pid` started, per the kernel, or null when it is gone or `ps` cannot say. `lstart` is the
+ * one column both macOS and Linux ps print in a form Date.parse reads ("Mon Sep 22 10:00:01 2026"). */
+export function processStartedAt(pid: number): number | null {
+ try {
+ const out = execFileSync('ps', ['-o', 'lstart=', '-p', String(pid)], { encoding: 'utf8', timeout: 5_000 }).trim()
+ if (!out) return null
+ const at = Date.parse(out)
+ return Number.isFinite(at) ? at : null
+ } catch { return null }
+}
+
+export interface ViewerLedgerDeps {
+ file?: string
+ now?: () => number
+ /** Test seams for the reaper. */
+ startedAt?: (pid: number) => number | null
+ kill?: (pid: number) => void
+ log?: (line: string) => void
+}
+
+export class ViewerLedger {
+ private readonly file: string
+ constructor(private readonly deps: ViewerLedgerDeps = {}) {
+ this.file = deps.file ?? VIEWER_LEDGER_FILE
+ }
+
+ read(): ViewerLedgerEntry[] {
+ try {
+ const parsed: unknown = JSON.parse(readFileSync(this.file, 'utf8'))
+ if (!Array.isArray(parsed)) return []
+ return parsed.filter((e): e is ViewerLedgerEntry =>
+ !!e && typeof e === 'object'
+ // > 1, never merely > 0: signalling group -1 is "every process I own".
+ && Number.isInteger((e as ViewerLedgerEntry).pid) && (e as ViewerLedgerEntry).pid > 1
+ && Number.isFinite((e as ViewerLedgerEntry).startedAt)
+ && typeof (e as ViewerLedgerEntry).agentId === 'string'
+ && typeof (e as ViewerLedgerEntry).dshId === 'string'
+ && typeof (e as ViewerLedgerEntry).viewerDir === 'string')
+ } catch { return [] }
+ }
+
+ private write(entries: ViewerLedgerEntry[]): void {
+ try {
+ mkdirSync(dirname(this.file), { recursive: true })
+ const tmp = `${this.file}.${process.pid}.tmp`
+ writeFileSync(tmp, JSON.stringify(entries, null, 2) + '\n', { mode: 0o600 })
+ renameSync(tmp, this.file)
+ } catch { /* best effort: a ledger that cannot be written only means no reaping next time */ }
+ }
+
+ add(entry: Omit<ViewerLedgerEntry, 'startedAt'> & { startedAt?: number }): void {
+ if (!Number.isInteger(entry.pid) || entry.pid <= 1) return
+ const startedAt = entry.startedAt ?? (this.deps.now ?? Date.now)()
+ this.write([...this.read().filter((e) => e.pid !== entry.pid), { ...entry, startedAt }])
+ }
+
+ remove(pid: number): void {
+ const entries = this.read()
+ const kept = entries.filter((e) => e.pid !== pid)
+ if (kept.length !== entries.length) this.write(kept)
+ }
+
+ /**
+ * Stop every ledgered viewer that is still alive and provably the same process, then start the
+ * ledger over. Meant for daemon boot, BEFORE this daemon starts any viewer of its own. Returns the
+ * entries it signalled.
+ */
+ reapOrphans(): ViewerLedgerEntry[] {
+ const entries = this.read()
+ if (entries.length === 0) return []
+ const startedAt = this.deps.startedAt ?? processStartedAt
+ const kill = this.deps.kill ?? ((pid: number) => { killPidGroup(pid) })
+ const reaped: ViewerLedgerEntry[] = []
+ for (const entry of entries) {
+ const live = startedAt(entry.pid)
+ if (live === null) continue // gone, or unknowable — a pid we cannot vouch for is left alone
+ if (Math.abs(live - entry.startedAt) > START_TOLERANCE_MS) {
+ this.deps.log?.(`[dsh] pid ${entry.pid} is no longer the ${entry.dshId} viewer (started ${new Date(live).toISOString()}, ledger says ${new Date(entry.startedAt).toISOString()}) · left alone`)
+ continue
+ }
+ kill(entry.pid)
+ reaped.push(entry)
+ this.deps.log?.(`[dsh] reaped orphaned ${entry.dshId} viewer pid ${entry.pid} (started ${new Date(entry.startedAt).toISOString()}, agent ${entry.agentId.slice(0, 8)})`)
+ }
+ this.write([])
+ return reaped
+ }
+}
diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.ts
index 4859e0949..a2846916b 100644
--- a/cli/src/lib/remoteRelay.spec.ts
+++ b/cli/src/lib/remoteRelay.spec.ts
@@ -615,3 +615,108 @@ describe('RemoteRelayPool shares one upstream between every local client selecte
expect(live.sendBinary).toHaveBeenCalled()
})
})
+
+describe('RemoteRelayPool keeps background (isolated) clients cheap', () => {
+ const ctorArgs = [
+ { accessToken: async () => 'unused' } as never,
+ 'ws://unused',
+ { pub: new Uint8Array(), priv: new Uint8Array() } as never,
+ { pin: () => {}, get: () => null } as never,
+ ] as const
+ const sink = () => ({ sendFrame: vi.fn(() => true), sendBinary: vi.fn(() => true) })
+ const select = { type: 'machine_select', payload: { relayIsolation: true } }
+
+ it('a pool built with p2p off never starts a negotiation, even with a policy and on retry', () => {
+ const pool = new RemoteRelayPool(...ctorArgs, { p2p: false }) as unknown as {
+ startP2p: (machineId: string, entry: ReturnType<typeof fakeEntry>) => void
+ }
+ const entry = fakeEntry({ p2pPolicy: { enabled: true, protocolVersion: 1, stunUrls: [], openWaitMs: 1500 } })
+ pool.startP2p('m1', entry)
+ expect(entry.p2p).toBeNull()
+ })
+
+ it('a detached background session is shelved warm and handed to the next background client', async () => {
+ const pool = new RemoteRelayPool(...ctorArgs)
+ expect(pool.idleIsolatedCount('m1')).toBe(0)
+ // Every background pool's dial is stubbed to leave an entry the way a completed dial does; count
+ // how many dials happen across two sequential polls.
+ const made: Array<{ ws: { close: ReturnType<typeof vi.fn>; terminate: ReturnType<typeof vi.fn> }; lingerTimer: ReturnType<typeof setTimeout> | null }> = []
+ const dialSpy = vi.spyOn(RemoteRelayPool.prototype as unknown as { connect: (machineId: string) => Promise<unknown> }, 'connect')
+ .mockImplementation(async function (this: unknown, machineId: string) {
+ const entry = fakeEntry({ attached: new Set(), sink: null, ws: { send: vi.fn(), close: vi.fn(), terminate: vi.fn() }, viewers: { reset: vi.fn(), close: vi.fn() } })
+ made.push(entry as unknown as typeof made[number])
+ ;(this as { entries: Map<string, unknown> }).entries.set(machineId, entry)
+ return entry
+ })
+ try {
+ const first = sink()
+ const a = await pool.acquireIsolated('m1', 'env', select, first, vi.fn())
+ expect(first.sendFrame).toHaveBeenCalledWith({ type: 'connected', payload: { machineId: 'm1', e2ee: false, relayIsolation: true } })
+ a.detach()
+ // Detaching lingers the upstream and shelves the pool; nothing is terminated.
+ expect(made).toHaveLength(1)
+ expect(made[0].ws.terminate).not.toHaveBeenCalled()
+ expect(made[0].ws.close).not.toHaveBeenCalled()
+ expect(made[0].lingerTimer).not.toBeNull()
+ expect(pool.idleIsolatedCount('m1')).toBe(1)
+
+ const second = sink()
+ const b = await pool.acquireIsolated('m1', 'env', select, second, vi.fn())
+ expect(dialSpy).toHaveBeenCalledTimes(1) // the second poll took the warm session — no dial
+ expect(pool.idleIsolatedCount('m1')).toBe(0)
+ expect(made[0].lingerTimer).toBeNull()
+ expect(second.sendFrame).toHaveBeenCalledWith({ type: 'connected', payload: { machineId: 'm1', e2ee: false, relayIsolation: true } })
+
+ // Two background clients AT ONCE get distinct sessions: isolation between them is kept.
+ const third = sink()
+ const c = await pool.acquireIsolated('m1', 'env', select, third, vi.fn())
+ expect(dialSpy).toHaveBeenCalledTimes(2)
+ expect(made).toHaveLength(2)
+ b.detach(); c.detach()
+ expect(pool.idleIsolatedCount('m1')).toBe(2)
+ for (const entry of made) clearTimeout(entry.lingerTimer!)
+ // The window's own pool is untouched by any of this.
+ expect((pool as unknown as { entries: Map<string, unknown> }).entries.size).toBe(0)
+ // A background forceReconnect empties the shelf.
+ pool.invalidateIsolated('m1')
+ expect(pool.idleIsolatedCount('m1')).toBe(0)
+ expect(made.every((entry) => entry.ws.terminate.mock.calls.length === 1)).toBe(true)
+ } finally {
+ dialSpy.mockRestore()
+ }
+ })
+
+ it('a failed background dial is answered from the failure for the cooldown, not re-dialled per poll', async () => {
+ vi.useFakeTimers()
+ try {
+ const pool = new RemoteRelayPool(...ctorArgs, { dialCooldownMs: 30_000 }) as unknown as {
+ connect: (machineId: string, env: string, frame: unknown) => Promise<unknown>
+ dial: (...args: unknown[]) => Promise<unknown>
+ }
+ const dial = vi.spyOn(pool, 'dial').mockRejectedValue(new Error('NO_PEER_LINK'))
+ await expect(pool.connect('m1', 'env', select)).rejects.toThrow('NO_PEER_LINK')
+ expect(dial).toHaveBeenCalledTimes(1)
+ // Inside the cooldown: same rejection, no new dial.
+ vi.advanceTimersByTime(10_000)
+ await expect(pool.connect('m1', 'env', select)).rejects.toThrow('NO_PEER_LINK')
+ expect(dial).toHaveBeenCalledTimes(1)
+ // Past it: a fresh attempt.
+ vi.advanceTimersByTime(25_000)
+ await expect(pool.connect('m1', 'env', select)).rejects.toThrow('NO_PEER_LINK')
+ expect(dial).toHaveBeenCalledTimes(2)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('the window pool has no cooldown: a person clicking again always gets a fresh dial', async () => {
+ const pool = new RemoteRelayPool(...ctorArgs) as unknown as {
+ connect: (machineId: string, env: string, frame: unknown) => Promise<unknown>
+ dial: (...args: unknown[]) => Promise<unknown>
+ }
+ const dial = vi.spyOn(pool, 'dial').mockRejectedValue(new Error('relay connect timed out'))
+ await expect(pool.connect('m1', 'env', select)).rejects.toThrow()
+ await expect(pool.connect('m1', 'env', select)).rejects.toThrow()
+ expect(dial).toHaveBeenCalledTimes(2)
+ })
+})
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index b980c9038..cb0276540 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -37,6 +37,20 @@ import { isWrapped } from './e2ee/core.js'
const CONNECT_TIMEOUT_MS = 15_000
const LINGER_MS = 30_000
+// The background pools' linger. A pane that closes its bridge socket after every read polls on a
+// fixed tick (a minute, in the shipped monitors); anything at or under that tick would re-dial the
+// relay (and redo the E2EE handshake) on every poll, which is exactly the churn these pools exist to
+// stop. An idle upstream costs a heartbeat, so the linger is generous.
+const ISOLATED_LINGER_MS = 3 * 60_000
+// After a FAILED background dial, how long the same machine is answered from that failure instead of
+// dialled again. A poller that retries on every tick against an unlinked or unreachable machine would
+// otherwise turn one bad machine into a relay dial per tick, forever. Never applied to the window's
+// pool: a person clicking again deserves a fresh attempt.
+const ISOLATED_DIAL_COOLDOWN_MS = 30_000
+// Warm background sessions kept per machine. Each concurrent background client needs its own (they
+// must not see each other's frames), so the cap is "how many pollers may overlap on one machine and
+// still find a warm session"; beyond it the extra session is torn down on detach as it always was.
+const ISOLATED_IDLE_MAX = 4
// Same convention/value as localWsServer.ts's app<->daemon heartbeat. Without this, a machine-node
// cycling (e.g. `harness start` on the OTHER end after a crash/restart) can leave this daemon holding
// an upstream socket the backend silently dropped with no close frame — every RPC sent through it then
@@ -213,30 +227,89 @@ function framePayload(frame: Frame): Record<string, unknown> {
: {}
}
+export interface RemoteRelayPoolOptions {
+ /** Negotiate a WebRTC data channel for terminals once the E2EE session is up. Default on; the
+ * background pool turns it off — a session that will only ever carry `agents_list` has no terminal
+ * to move onto p2p, and the STUN/TURN round trips would be thrown away with it. */
+ p2p?: boolean
+ /** How long an entry nobody is attached to stays open for the next select. */
+ lingerMs?: number
+ /** After a failed dial, reject further dials for this machine for this long. 0 = never. */
+ dialCooldownMs?: number
+ /** Share one failure record between pools (the background pools all draw on the parent's). */
+ dialFailures?: Map<string, { at: number; error: unknown }>
+}
+
export class RemoteRelayPool {
private entries = new Map<string, Entry>()
private pending = new Map<string, Promise<Entry>>()
+ /** Last failed dial per machine, for the cooldown. Cleared by the next successful dial. */
+ private readonly lastDialFailure: Map<string, { at: number; error: unknown }>
+ private readonly p2pEnabled: boolean
+ private readonly lingerMs: number
+ private readonly dialCooldownMs: number
+ /** Warm background pools per machine, each holding one lingering session nobody is attached to —
+ * see acquireIsolated(). A pool is either here (idle) or in a client's hands, never both. */
+ private readonly idleIsolated = new Map<string, RemoteRelayPool[]>()
constructor(
private readonly auth: AuthSessionManager,
private readonly backendWsBase: string,
private readonly selfIdentity: Identity,
private readonly peers: MachinePeerStore,
- ) {}
+ opts: RemoteRelayPoolOptions = {},
+ ) {
+ this.p2pEnabled = opts.p2p !== false
+ this.lingerMs = opts.lingerMs ?? LINGER_MS
+ this.dialCooldownMs = opts.dialCooldownMs ?? 0
+ this.lastDialFailure = opts.dialFailures ?? new Map()
+ }
- /** Background CLI jobs must not replace the desktop window's one pooled sink. Each isolated
- * client gets its own encrypted session, released immediately instead of retained for reconnect. */
+ /** Background CLI jobs (a monitor pane polling `agents_list`, a script) must not replace the
+ * desktop window's one pooled sink, and must not see each other's frames either — so each one
+ * gets a pool of its OWN, separate from the window's, with its own encrypted session. What used to
+ * happen next was the expensive part: the pool was thrown away on detach, so every poll was a fresh
+ * relay socket, E2EE handshake AND WebRTC negotiation, terminated a second later (8.5k dials/72 min
+ * fleet-wide, 2026-09-22). Now a detached pool goes back on a per-machine shelf with its session
+ * lingering, and the next background client for that machine takes it — one dial per machine per
+ * linger, however often a pane polls. Concurrent clients still get distinct pools. These pools never
+ * negotiate p2p and share one dial-failure record, so a machine that refused is not re-dialled on
+ * every tick. */
async acquireIsolated(
machineId: string, autonomousEnv: string, selectFrame: Frame,
sink: LocalClientSink, onClosed: (code: number, reason: string) => void,
): Promise<RelaySession> {
- const isolated = new RemoteRelayPool(this.auth, this.backendWsBase, this.selfIdentity, this.peers)
- const session = await isolated.acquire(machineId, autonomousEnv, selectFrame, {
+ const shelf = this.idleIsolated.get(machineId) ?? []
+ const pool = shelf.pop() ?? new RemoteRelayPool(this.auth, this.backendWsBase, this.selfIdentity, this.peers,
+ { p2p: false, lingerMs: ISOLATED_LINGER_MS, dialCooldownMs: ISOLATED_DIAL_COOLDOWN_MS, dialFailures: this.lastDialFailure })
+ const session = await pool.acquire(machineId, autonomousEnv, selectFrame, {
...sink,
sendFrame: frame => sink.sendFrame(frame.type === 'connected'
? { ...frame, payload: { ...framePayload(frame), relayIsolation: true } } : frame),
}, onClosed)
- return { ...session, detach: () => isolated.invalidate(machineId) }
+ let detached = false
+ return {
+ ...session,
+ detach: () => {
+ if (detached) return
+ detached = true
+ session.detach() // the pool's own linger starts here
+ const idle = this.idleIsolated.get(machineId) ?? []
+ if (idle.length >= ISOLATED_IDLE_MAX) { pool.invalidate(machineId); return }
+ idle.push(pool)
+ this.idleIsolated.set(machineId, idle)
+ },
+ }
+ }
+
+ /** Test seam: how many warm background sessions are shelved for `machineId`. */
+ idleIsolatedCount(machineId: string): number { return this.idleIsolated.get(machineId)?.length ?? 0 }
+
+ /** `invalidate()` for the shelved background sessions — a background client's own `forceReconnect`
+ * says the machine behind them restarted, so none of them is worth handing to the next client. */
+ invalidateIsolated(machineId: string): void {
+ for (const pool of this.idleIsolated.get(machineId) ?? []) pool.invalidate(machineId)
+ this.idleIsolated.delete(machineId)
}
/** Force-drops a pooled entry so the next `acquire()` dials fresh instead of reusing it. For when
@@ -298,6 +371,10 @@ export class RemoteRelayPool {
}
private connect(machineId: string, autonomousEnv: string, selectFrame: Frame): Promise<Entry> {
+ const failed = this.lastDialFailure.get(machineId)
+ if (failed && this.dialCooldownMs > 0 && Date.now() - failed.at < this.dialCooldownMs) {
+ return Promise.reject(failed.error)
+ }
const attempt = (forceRefresh: boolean): Promise<Entry> => this.dial(machineId, autonomousEnv, selectFrame, forceRefresh)
// A stale access token is the single most likely reason the very first select fails (4401 on the
// upgrade) — one retry with a freshly-refreshed token is cheap next to surfacing that as a hard
@@ -307,6 +384,12 @@ export class RemoteRelayPool {
if (err instanceof RelayConnectError && err.closeCode === 4401) return attempt(true)
throw err
})
+ if (this.dialCooldownMs > 0) {
+ promise.then(
+ () => { this.lastDialFailure.delete(machineId) },
+ (error) => { this.lastDialFailure.set(machineId, { at: Date.now(), error }) },
+ )
+ }
this.pending.set(machineId, promise)
// `.finally()` re-throws on rejection, producing a SECOND promise distinct from the one returned
// below (which callers already await/catch) — left un-caught, every failed dial (e.g. NO_PEER_LINK
@@ -396,7 +479,7 @@ export class RemoteRelayPool {
// backendSocket, never this relay client), not in the badge, not in p2p_result — and its
// absence is indistinguishable from "TURN is configured but ICE preferred direct".
// Names only, never the credential itself.
- if (entry.p2pPolicy) {
+ if (entry.p2pPolicy && this.p2pEnabled) {
const p = entry.p2pPolicy
console.log(`[p2p] policy · machine=${sid(machineId)} stun=${p.stunUrls.length}`
+ ` turn=${p.turn ? `${p.turn.urls.length} urls` : 'NONE'} openWait=${p.openWaitMs}ms`)
@@ -421,7 +504,9 @@ export class RemoteRelayPool {
// the terminal just quietly stayed on the ws relay. The peer-version case is the important
// one: a machine whose CLI predates p2p answers no offer, so NO amount of STUN or TURN can
// help it. That is a very different problem from "ICE tried and failed".
- if (!entry.p2pPolicy) {
+ if (!this.p2pEnabled) {
+ console.log(`[p2p] off · machine=${sid(machineId)} background session, no terminals — ws relay only`)
+ } else if (!entry.p2pPolicy) {
console.log(`[p2p] off · machine=${sid(machineId)} backend sent no policy (rollout or kill switch)`)
} else if (crypto.terminalP2pVersion !== TERMINAL_P2P_PROTOCOL_VERSION) {
console.log(`[p2p] off · machine=${sid(machineId)} peer speaks p2p v${crypto.terminalP2pVersion},`
@@ -642,7 +727,7 @@ export class RemoteRelayPool {
try { entry.ws.close(1000, 'idle') } catch { /* ignore */ }
this.entries.delete(machineId)
}
- }, LINGER_MS)
+ }, this.lingerMs)
entry.lingerTimer.unref?.()
},
}
@@ -650,7 +735,9 @@ export class RemoteRelayPool {
private startP2p(machineId: string, entry: Entry): void {
const policy = entry.p2pPolicy
- if (!policy || entry.p2p) return
+ // The p2p-off guard lives HERE, not only at the dial-time call: scheduleP2pRetry() re-enters
+ // through this same method, and a background pool must stay relay-only through every retry too.
+ if (!policy || entry.p2p || !this.p2pEnabled) return
let wasDirect = false
const p2p = new TerminalP2pInitiator({
policy,
diff --git a/cli/src/localWsServer.ts b/cli/src/localWsServer.ts
index e02decc9d..d0a5721a4 100644
--- a/cli/src/localWsServer.ts
+++ b/cli/src/localWsServer.ts
@@ -327,7 +327,10 @@ export function attachLocalWsServer(server: http.Server, options: LocalWsServerO
// its pooled entry is suspect (most commonly the relayed machine's own Harness process
// restarted, dropping its E2EE session without the transport itself ever closing). Drop it
// so this select dials fresh instead of handing back the same dead session again.
- if (payload?.forceReconnect === true && payload?.relayIsolation !== true) options.relayPool.invalidate(requestedMachineId)
+ if (payload?.forceReconnect === true) {
+ if (payload?.relayIsolation === true) options.relayPool.invalidateIsolated(requestedMachineId)
+ else options.relayPool.invalidate(requestedMachineId)
+ }
try {
relay = payload?.relayIsolation === true
? await options.relayPool.acquireIsolated(requestedMachineId, options.autonomousEnv, frame, sink, close)
diff --git a/store/agents/autonomous-grid/test/server.test.mjs b/store/agents/autonomous-grid/test/server.test.mjs
index 11e3e2fb7..b24ed23af 100644
--- a/store/agents/autonomous-grid/test/server.test.mjs
+++ b/store/agents/autonomous-grid/test/server.test.mjs
@@ -22,5 +22,6 @@ test('viewer serves its real app and SSE, blocks mutations and never serves work
assert.equal((await fetch(`${base}/.harness/grid/../secret`)).status,404);
const blocked=await new Promise(resolve=>{request(base,{headers:{host:'attacker.test'}},res=>{res.resume();resolve(res.statusCode);}).end();});assert.equal(blocked,403);
const abort=new AbortController();const events=await fetch(`${base}/events`,{signal:abort.signal});const reader=events.body.getReader();const chunk=new TextDecoder().decode((await reader.read()).value);assert.match(chunk,/event: snapshot/);assert.match(chunk,/Mac Studio/);abort.abort();
- fail=true;await new Promise(resolve=>setTimeout(resolve,90));const stale=await (await fetch(`${base}/api/snapshot`)).json();assert.equal(stale.status,'unavailable');assert.ok(stale.nodes.every(n=>n.stale));assert.doesNotMatch(JSON.stringify(stale),/secret token/);
+ // No pane is attached any more, so the fleet is read again only when the observation is asked for and stale.
+ fail=true;let stale;for(let retry=0;retry<40;retry++){stale=await (await fetch(`${base}/api/snapshot`)).json();if(stale.status==='unavailable')break;await new Promise(resolve=>setTimeout(resolve,30));}assert.equal(stale.status,'unavailable');assert.ok(stale.nodes.every(n=>n.stale));assert.doesNotMatch(JSON.stringify(stale),/secret token/);
});
diff --git a/store/agents/autonomous-grid/viewer.mjs b/store/agents/autonomous-grid/viewer.mjs
index f71f5a5d1..06ca76d5e 100644
--- a/store/agents/autonomous-grid/viewer.mjs
+++ b/store/agents/autonomous-grid/viewer.mjs
@@ -8,7 +8,20 @@ import { createCollector } from './lib/telemetry.mjs';
export function createViewer({ workspace, port = 0, intervalMs = 8000, collect = createCollector(workspace, { intervalMs }), select = gridSelect }) {
const clients = new Set();
let snapshot = { spec: 1, status: 'connecting', grid: 'Your grid', nodes: [], machines: [], models: [], events: [], operations: [], history: {}, sources: {}, summary: {}, observedAt: null, pollIntervalMs: intervalMs };
- let stopped = false, pollTimer, opTimer, heartbeat;
+ let stopped = false, pollTimer, opTimer, heartbeat, inFlight = false, lastPollAt = 0;
+ // The next tick is booked only while a pane is connected: a viewer nobody is looking at must not keep
+ // running the grid CLI every few seconds. A request for the observation still wakes one read if the
+ // one on hand has gone stale.
+ const schedule = () => {
+ clearTimeout(pollTimer); pollTimer = null;
+ if (stopped || clients.size === 0) return;
+ pollTimer = setTimeout(poll, intervalMs);
+ };
+ const wake = () => {
+ if (stopped || inFlight || Date.now() - lastPollAt < intervalMs) return;
+ clearTimeout(pollTimer); pollTimer = null;
+ void poll();
+ };
const headers = { 'cache-control': 'no-store', 'x-content-type-options': 'nosniff', 'referrer-policy': 'no-referrer' };
const json = (res, code, value) => { res.writeHead(code, { ...headers, 'content-type': 'application/json; charset=utf-8' }); res.end(JSON.stringify(value)); };
const publish = () => {
@@ -38,13 +51,17 @@ export function createViewer({ workspace, port = 0, intervalMs = 8000, collect =
}
if (!['GET', 'HEAD'].includes(req.method)) { res.setHeader('allow', 'GET, HEAD, POST'); json(res, 405, { error: 'This viewer is read-only. Talk to the Grid agent to make changes.' }); return; }
if (url.pathname === '/health') { json(res, 200, { ok: true }); return; }
- if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); return; }
+ if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); wake(); return; }
if (url.pathname === '/events') {
if (req.method === 'HEAD') { res.writeHead(200, headers); res.end(); return; }
if (clients.size >= 32) { json(res, 503, { error: 'Too many viewer connections.' }); return; }
res.writeHead(200, { ...headers, 'content-type': 'text/event-stream', connection: 'keep-alive', 'x-accel-buffering': 'no' });
res.write(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`);
- clients.add(res); res.on('close', () => clients.delete(res)); return;
+ clients.add(res);
+ res.on('close', () => { clients.delete(res); if (clients.size === 0) { clearTimeout(pollTimer); pollTimer = null; } });
+ wake();
+ if (!pollTimer && !inFlight) schedule();
+ return;
}
const assets = { '/': ['index.html', 'text/html; charset=utf-8'], '/app.js': ['app.js', 'text/javascript; charset=utf-8'], '/app.css': ['app.css', 'text/css; charset=utf-8'] };
if (!assets[url.pathname]) { json(res, 404, { error: 'Not found' }); return; }
@@ -73,13 +90,15 @@ export function createViewer({ workspace, port = 0, intervalMs = 8000, collect =
}
async function poll() {
+ inFlight = true;
try { snapshot = await collect(); }
catch {
snapshot = { ...snapshot, status: 'unavailable', nodes: snapshot.nodes.map(n => ({ ...n, stale: true })), sources: { configuration: { ok: false, error: 'Cannot refresh the fleet. Ask the agent to check grid-fleet.json and run fleet refresh.' } } };
await atomicJson(join(workspace, '.harness', 'verdict.json'), { spec: 1, ready: false, summary: 'Grid telemetry could not refresh', findings: [{ severity: 'error', kind: 'telemetry', message: snapshot.sources.configuration.error }], updatedAt: now() }).catch(() => {});
}
+ inFlight = false; lastPollAt = Date.now();
if (stopped) return;
- publish(); pollTimer = setTimeout(poll, intervalMs);
+ publish(); schedule();
}
async function pollOperations() {
const latest = await operations(workspace).catch(() => []);
diff --git a/store/agents/harness-monitor/lib/bridge.mjs b/store/agents/harness-monitor/lib/bridge.mjs
index f44b52e89..a35767e6a 100644
--- a/store/agents/harness-monitor/lib/bridge.mjs
+++ b/store/agents/harness-monitor/lib/bridge.mjs
@@ -15,6 +15,7 @@
import { randomUUID } from 'node:crypto'
import { execFile } from 'node:child_process'
+import { homedir } from 'node:os'
import { promisify } from 'node:util'
const exec = promisify(execFile)
@@ -30,12 +31,15 @@ export function bridgeUrl(env = process.env) {
return url.href
}
-/** Every machine this daemon can reach, the current one flagged. `harness` not on PATH is an answer,
- * not a crash: the caller falls back to reading this machine's registry directly. */
-export async function machines(env = process.env) {
+/** Every machine this daemon can reach, the current one flagged, and — when the list could not be
+ * read — why. `harness` not on PATH is an answer, not a crash: the caller falls back to reading this
+ * machine's registry directly. The command runs from the home folder, never from this package's
+ * directory: a package update swaps that directory out from under a running viewer, and a CLI
+ * started in a directory that no longer exists fails in a way that reads exactly like a dead daemon. */
+export async function machinesReport(env = process.env) {
try {
- const { stdout } = await exec('harness', ['machines', '--json'], { timeout: 20_000, maxBuffer: 512 * 1024, env })
- return stdout.split('\n').filter(Boolean).map((line) => { try { return JSON.parse(line) } catch { return null } })
+ const { stdout } = await exec('harness', ['machines', '--json'], { timeout: 20_000, maxBuffer: 512 * 1024, env, cwd: homedir() })
+ const machines = stdout.split('\n').filter(Boolean).map((line) => { try { return JSON.parse(line) } catch { return null } })
.filter((row) => row && typeof row.machineId === 'string')
.map((row) => ({
machineId: row.machineId,
@@ -43,31 +47,62 @@ export async function machines(env = process.env) {
current: row.current === true,
online: row.status === 'running',
}))
- } catch { return [] }
+ return { machines, error: null }
+ } catch (error) {
+ const detail = error?.code === 'ENOENT'
+ ? 'the `harness` command is not on PATH'
+ : `the \`harness machines\` command failed${error?.stderr ? `: ${String(error.stderr).trim().split('\n')[0].slice(0, 200)}` : error?.message ? `: ${String(error.message).slice(0, 200)}` : ''}`
+ return { machines: [], error: detail }
+ }
+}
+
+/** The machine list alone, for callers that only need the rows. */
+export async function machines(env = process.env) {
+ return (await machinesReport(env)).machines
}
/**
- * One socket, one machine, however many calls the caller needs, then closed.
+ * One bridge socket per machine, opened on first use and kept for the next call.
+ *
+ * Behind a `machine_select` on this bridge, Harness dials its relay to that machine, runs an
+ * encryption handshake and — until recently — negotiated a WebRTC channel; opening a socket per poll
+ * threw all of that away every few seconds, for every machine, from every pane. So the select happens
+ * once, and a poll is one `agents_list` frame on a socket that already exists. A socket that closes,
+ * errors, or stops answering is dropped; the next call opens a fresh one.
*
- * `fn` receives `rpc(type, payload)`. Any failure — no daemon, an unlinked machine, a timeout —
- * arrives as an Error whose message is a sentence a person can act on, because these end up in a
- * pane header and a CLI's stderr rather than a log nobody reads.
+ * Any failure — no daemon, an unlinked machine, a timeout — arrives as an Error whose message is a
+ * sentence a person can act on, because these end up in a pane header and a CLI's stderr rather than
+ * a log nobody reads.
*/
-export async function withBridge(machineId, fn, { env = process.env, timeoutMs = 20_000, WebSocketImpl = globalThis.WebSocket } = {}) {
- if (!WebSocketImpl) throw new Error('Node 22 or newer is required to reach the Harness bridge.')
+const sessions = new Map() // WebSocketImpl -> Map<machineId, session>; keyed by impl so a test's scripted socket is its own world
+
+function sessionsFor(WebSocketImpl) {
+ let byMachine = sessions.get(WebSocketImpl)
+ if (!byMachine) { byMachine = new Map(); sessions.set(WebSocketImpl, byMachine) }
+ return byMachine
+}
+
+function openSession(machineId, { env, timeoutMs, WebSocketImpl, forceReconnect = false }) {
+ const byMachine = sessionsFor(WebSocketImpl)
const socket = new WebSocketImpl(bridgeUrl(env))
const pending = new Map()
let closed = null
+ const session = { socket, pending, ready: null, close: null }
const fail = (message) => {
closed ??= new Error(message)
- for (const [, entry] of pending) entry.reject(closed)
+ if (byMachine.get(machineId) === session) byMachine.delete(machineId)
+ for (const [, entry] of pending) { clearTimeout(entry.deadline); entry.reject(closed) }
pending.clear()
+ try { socket.close() } catch { /* already gone */ }
}
+ session.close = () => fail('The Harness bridge closed the connection.')
- const ready = new Promise((resolve, reject) => {
- const deadline = setTimeout(() => reject(new Error('The Harness daemon did not answer on the local bridge. Is Harness running?')), Math.min(timeoutMs, 15_000))
- socket.addEventListener('open', () => socket.send(JSON.stringify({ type: 'machine_select', payload: { machineId, localProtocolVersion: 1, relayIsolation: true } })))
- socket.addEventListener('error', () => { clearTimeout(deadline); reject(new Error('Could not open the local Harness bridge. Start Harness and try again.')) })
+ session.ready = new Promise((resolve, reject) => {
+ const deadline = setTimeout(() => { const m = 'The Harness daemon did not answer on the local bridge. Is Harness running?'; fail(m); reject(new Error(m)) }, Math.min(timeoutMs, 15_000))
+ // `forceReconnect` tells Harness the session it kept for this machine is dead (the machine's own
+ // Harness restarted under it) and must be dialled fresh rather than handed back once more.
+ socket.addEventListener('open', () => socket.send(JSON.stringify({ type: 'machine_select', payload: { machineId, localProtocolVersion: 1, relayIsolation: true, ...(forceReconnect ? { forceReconnect: true } : {}) } })))
+ socket.addEventListener('error', () => { clearTimeout(deadline); const m = 'Could not open the local Harness bridge. Start Harness and try again.'; fail(m); reject(new Error(m)) })
socket.addEventListener('close', (event) => {
clearTimeout(deadline)
const message = event?.code === 4404
@@ -92,28 +127,75 @@ export async function withBridge(machineId, fn, { env = process.env, timeoutMs =
else entry.resolve(payload)
})
})
+ session.ready.catch(() => {})
- const rpc = (type, payload = {}, { callTimeoutMs = timeoutMs } = {}) => new Promise((resolve, reject) => {
+ session.rpc = (type, payload = {}, { callTimeoutMs = timeoutMs } = {}) => new Promise((resolve, reject) => {
if (closed) { reject(closed); return }
const requestId = randomUUID()
- const deadline = setTimeout(() => { pending.delete(requestId); reject(new Error(`The daemon did not answer ${type} in time.`)) }, callTimeoutMs)
+ const deadline = setTimeout(() => {
+ pending.delete(requestId)
+ const error = new Error(`The daemon did not answer ${type} in time.`)
+ error.timedOut = true
+ reject(error)
+ }, callTimeoutMs)
pending.set(requestId, { type, resolve, reject, deadline })
- socket.send(JSON.stringify({ type, payload: { ...payload, requestId } }))
+ try { socket.send(JSON.stringify({ type, payload: { ...payload, requestId } })) }
+ catch { pending.delete(requestId); clearTimeout(deadline); reject(closed ?? new Error('The Harness bridge closed the connection.')) }
})
+ byMachine.set(machineId, session)
+ return session
+}
+
+/** The kept session for `machineId`, opening one if there is none. */
+async function bridgeSession(machineId, { env = process.env, timeoutMs = 20_000, WebSocketImpl = globalThis.WebSocket, forceReconnect = false } = {}) {
+ if (!WebSocketImpl) throw new Error('Node 22 or newer is required to reach the Harness bridge.')
+ const existing = sessionsFor(WebSocketImpl).get(machineId)
+ const session = existing ?? openSession(machineId, { env, timeoutMs, WebSocketImpl, forceReconnect })
+ await session.ready
+ return { session, reused: session === existing }
+}
+
+/**
+ * One socket, one machine, however many calls the caller needs, then closed — for a one-off job
+ * that must not share the kept session (a script, a test). `fn` receives `rpc(type, payload)`.
+ */
+export async function withBridge(machineId, fn, { env = process.env, timeoutMs = 20_000, WebSocketImpl = globalThis.WebSocket } = {}) {
+ if (!WebSocketImpl) throw new Error('Node 22 or newer is required to reach the Harness bridge.')
+ // Its own world, never the shared map: a fresh key that nothing else will look up.
+ const own = class extends WebSocketImpl {}
+ const session = openSession(machineId, { env, timeoutMs, WebSocketImpl: own })
try {
- await ready
- return await fn(rpc)
+ await session.ready
+ return await fn(session.rpc)
} finally {
- for (const [, entry] of pending) clearTimeout(entry.deadline)
- try { socket.close() } catch { /* already gone */ }
+ session.close()
+ sessions.delete(own)
}
}
-/** The fleet as the daemon sees it, for one machine. */
-export function listAgents(machineId, options) {
- return withBridge(machineId, async (rpc) => {
- const reply = await rpc('agents_list', {})
+/** The fleet as the daemon sees it, for one machine — on the kept session. A kept socket that has
+ * gone quiet (the daemon behind it restarted without closing it) is dropped and the read retried
+ * ONCE on a fresh one before the machine is reported silent. */
+export async function listAgents(machineId, options = {}) {
+ const ask = async ({ session }) => {
+ const reply = await session.rpc('agents_list', {})
return Array.isArray(reply.agents) ? reply.agents : []
- }, options)
+ }
+ const first = await bridgeSession(machineId, options)
+ try {
+ return await ask(first)
+ } catch (error) {
+ if (!error?.timedOut) throw error
+ first.session.close()
+ if (!first.reused) throw error
+ const second = await bridgeSession(machineId, { ...options, forceReconnect: true })
+ try { return await ask(second) } catch (again) { if (again?.timedOut) second.session.close(); throw again }
+ }
+}
+
+/** Drop every kept bridge session (the pane is closing). */
+export function closeBridges() {
+ for (const byMachine of sessions.values()) for (const session of [...byMachine.values()]) session.close()
+ sessions.clear()
}
diff --git a/store/agents/harness-monitor/lib/inventory.mjs b/store/agents/harness-monitor/lib/inventory.mjs
index bdece682d..3e521d065 100644
--- a/store/agents/harness-monitor/lib/inventory.mjs
+++ b/store/agents/harness-monitor/lib/inventory.mjs
@@ -16,7 +16,7 @@ import { existsSync } from 'node:fs'
import { homedir } from 'node:os'
import { basename, dirname } from 'node:path'
import { lastTurns } from './activity.mjs'
-import { listAgents, machines } from './bridge.mjs'
+import { listAgents, machinesReport } from './bridge.mjs'
import { engineProcess, panes, processTable } from './panes.mjs'
import { humanIdle } from './policy.mjs'
import { readRegistry, registryAsFrames } from './registry.mjs'
@@ -157,32 +157,56 @@ export function mergeRows(agents, { paneRows = new Map(), table = { byPid: new M
* Failures are per machine and reported, never fatal: a fleet view that refuses to draw because one
* linked laptop is asleep is worse than one that says which machine it could not reach.
*/
-export async function collect({ state = {}, now = Date.now(), includeRemote = true, timeoutMs = 8000, home = homedir(), cache = new Map() } = {}) {
- const [paneRows, table, machineList, registry] = await Promise.all([panes(), processTable(), machines(), readRegistry()])
+export async function collect({
+ state = {}, now = Date.now(), includeRemote = true, timeoutMs = 8000, home = homedir(), cache = new Map(),
+ // Remote machines are asked at most this often; between asks their last answer is re-merged against
+ // the current clock. A pane ticks every few seconds for the machine it is on — that is local reads —
+ // but every remote ask crosses the relay to another computer, so it gets its own, slower clock.
+ // `remote` is the holder for that answer, owned by the caller so it survives across ticks.
+ remoteIntervalMs = 60_000, remote = null,
+} = {}) {
+ const [paneRows, table, report, registry] = await Promise.all([panes(), processTable(), machinesReport(), readRegistry()])
+ const machineList = report.machines
const turns = await lastTurns(registry.rows.map((row) => ({ id: row.agentId, transcriptPath: row.transcriptPath })), { cache })
const current = machineList.find((m) => m.current) ?? null
- const targets = [current, ...(includeRemote ? machineList.filter((m) => !m.current && m.online) : [])].filter(Boolean)
+ const remoteMachines = includeRemote ? machineList.filter((m) => !m.current && m.online) : []
+ const remoteDue = !remote || remoteIntervalMs <= 0 || now - (remote.at ?? 0) >= remoteIntervalMs
+ const targets = [current, ...(remoteDue ? remoteMachines : [])].filter(Boolean)
const problems = []
let rows = []
// No daemon, no `harness` on PATH: fall back to the registry file this machine already has. Seeing
- // the fleet must not depend on the app running, and pausing is pure tmux either way.
+ // the fleet must not depend on the app running, and pausing is pure tmux either way. Say which of
+ // the two it was: a CLI that failed to run is a different repair from a daemon that is not there.
if (!targets.length) {
- if (!registry.rows.length) problems.push({ machine: 'this machine', error: 'No Harness daemon on the local bridge and no registry to read.' })
+ if (report.error) problems.push({ machine: 'this machine', error: `Read from the registry file — ${report.error}, so models, branches and remote machines are missing.` })
+ else if (!registry.rows.length) problems.push({ machine: 'this machine', error: 'No Harness daemon on the local bridge and no registry to read.' })
else problems.push({ machine: 'this machine', error: 'Read from the registry file — the Harness daemon is not answering, so models, branches and remote machines are missing.' })
const rowsOffline = mergeRows(registryAsFrames(registry.rows), { paneRows, table, state, machine: null, local: true, now, home, turns, registry: registry.byId })
rowsOffline.sort((a, b) => a.idleMs - b.idleMs || a.name.localeCompare(b.name))
return { rows: rowsOffline, machines: machineList, problems, observedAt: now, degraded: true }
}
+ // What each remote machine last answered, raw, so an off-tick merge still gets a fresh clock.
+ const answers = remoteDue ? new Map() : new Map(remote?.answers ?? [])
+ const remoteProblems = remoteDue ? [] : [...(remote?.problems ?? [])]
await Promise.all(targets.map(async (machine) => {
try {
const agents = await listAgents(machine.machineId, { timeoutMs })
- rows = rows.concat(mergeRows(agents, { paneRows, table, state, machine, local: machine.current === true, now, home, turns, registry: registry.byId }))
+ if (machine.current) rows = rows.concat(mergeRows(agents, { paneRows, table, state, machine, local: true, now, home, turns, registry: registry.byId }))
+ else answers.set(machine.machineId, agents)
} catch (error) {
- problems.push({ machine: machine.name, error: error instanceof Error ? error.message : String(error) })
+ const problem = { machine: machine.name, error: error instanceof Error ? error.message : String(error) }
+ if (machine.current) problems.push(problem)
+ else remoteProblems.push(problem)
}
}))
+ for (const machine of remoteMachines) {
+ const agents = answers.get(machine.machineId)
+ if (agents) rows = rows.concat(mergeRows(agents, { paneRows, table, state, machine, local: false, now, home, turns, registry: registry.byId }))
+ }
+ problems.push(...remoteProblems)
+ if (remote && remoteDue) { remote.at = now; remote.answers = answers; remote.problems = remoteProblems }
rows.sort((a, b) => a.idleMs - b.idleMs || a.name.localeCompare(b.name))
return { rows, machines: machineList, problems, observedAt: now }
diff --git a/store/agents/harness-monitor/test/bridge.test.mjs b/store/agents/harness-monitor/test/bridge.test.mjs
new file mode 100644
index 000000000..9d1a41310
--- /dev/null
+++ b/store/agents/harness-monitor/test/bridge.test.mjs
@@ -0,0 +1,85 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { closeBridges, listAgents, withBridge } from '../lib/bridge.mjs'
+
+/** A scripted bridge socket class: selects on open, answers agents_list, counts its instances. */
+function scripted({ answer = true, closeAfterAnswer = false, muteAfter = Infinity } = {}) {
+ const made = []
+ const sent = []
+ let answered = 0
+ const Impl = class {
+ constructor() {
+ made.push(this); this.listeners = {}
+ queueMicrotask(() => {
+ this.fire('open', {})
+ this.fire('message', { data: JSON.stringify({ type: 'connected', payload: { machineId: 'm', relayIsolation: true } }) })
+ })
+ }
+ addEventListener(type, handler) { (this.listeners[type] ??= []).push(handler) }
+ fire(type, event) { for (const handler of this.listeners[type] ?? []) handler(event) }
+ send(raw) {
+ const frame = JSON.parse(raw)
+ sent.push(frame)
+ if (frame.type !== 'agents_list' || !answer || answered >= muteAfter) return
+ answered += 1
+ this.fire('message', { data: JSON.stringify({ type: 'agents_list_result', payload: { requestId: frame.payload.requestId, agents: [{ id: `from-${made.indexOf(this)}` }] } }) })
+ if (closeAfterAnswer) this.fire('close', { code: 1000 })
+ }
+ close() {}
+ }
+ return { Impl, made, sent }
+}
+
+test('a second read on the same machine reuses the kept socket: one select, two asks', async () => {
+ const { Impl, made, sent } = scripted()
+ assert.deepEqual(await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 500 }), [{ id: 'from-0' }])
+ assert.deepEqual(await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 500 }), [{ id: 'from-0' }])
+ assert.equal(made.length, 1)
+ assert.deepEqual(sent.map((f) => f.type), ['machine_select', 'agents_list', 'agents_list'])
+ assert.equal(sent[1].payload.requestId === sent[2].payload.requestId, false)
+ closeBridges()
+})
+
+test('a kept socket that closed is replaced on the next read', async () => {
+ const { Impl, made, sent } = scripted({ closeAfterAnswer: true })
+ await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 500 })
+ await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 500 })
+ assert.equal(made.length, 2)
+ assert.deepEqual(sent.map((f) => f.type), ['machine_select', 'agents_list', 'machine_select', 'agents_list'])
+ closeBridges()
+})
+
+test('a kept socket that goes quiet is dropped and the read retried once on a fresh one', async () => {
+ const { Impl, made, sent } = scripted({ muteAfter: 1 })
+ assert.deepEqual(await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 80 }), [{ id: 'from-0' }])
+ // The first socket answers nothing more; the retry lands on a new socket, which is fresh and answers.
+ await assert.rejects(listAgents('m', { WebSocketImpl: Impl, timeoutMs: 80 }), /did not answer agents_list/)
+ assert.equal(made.length, 2, 'one retry, on a fresh socket, before giving up')
+ const selects = sent.filter((f) => f.type === 'machine_select').map((f) => f.payload.forceReconnect)
+ assert.deepEqual(selects, [undefined, true], 'the retry tells Harness its kept session is dead')
+ closeBridges()
+})
+
+test('an unlinked machine is named as such, and is not kept', async () => {
+ const Impl = class {
+ constructor() { this.listeners = {}; queueMicrotask(() => this.fire('close', { code: 4404 })) }
+ addEventListener(type, handler) { (this.listeners[type] ??= []).push(handler) }
+ fire(type, event) { for (const handler of this.listeners[type] ?? []) handler(event) }
+ send() {} close() {}
+ }
+ await assert.rejects(listAgents('m', { WebSocketImpl: Impl, timeoutMs: 200 }), /Link this machine/)
+ closeBridges()
+})
+
+test('withBridge is a one-off: its socket is closed after the call and never shared', async () => {
+ const { Impl, made, sent } = scripted()
+ const closedSockets = []
+ Impl.prototype.close = function () { closedSockets.push(this) }
+ const agents = await withBridge('m', async (rpc) => (await rpc('agents_list', {})).agents, { WebSocketImpl: Impl, timeoutMs: 500 })
+ assert.deepEqual(agents, [{ id: 'from-0' }])
+ assert.equal(closedSockets.length, 1)
+ await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 500 })
+ assert.equal(made.length, 2, 'the kept read did not reuse the one-off socket')
+ assert.deepEqual(sent.map((f) => f.type), ['machine_select', 'agents_list', 'machine_select', 'agents_list'])
+ closeBridges()
+})
diff --git a/store/agents/harness-monitor/test/server.test.mjs b/store/agents/harness-monitor/test/server.test.mjs
index 2d5fe036d..c6b57acd0 100644
--- a/store/agents/harness-monitor/test/server.test.mjs
+++ b/store/agents/harness-monitor/test/server.test.mjs
@@ -160,3 +160,35 @@ async function post(base, token, path, payload) {
})
return response.json()
}
+
+test('the fleet is read on a tick only while a pane is connected; remote machines on their own clock', async (t) => {
+ let reads = 0
+ const calls = []
+ const { viewer, base } = await serve(undefined, {
+ intervalMs: 40,
+ remoteIntervalMs: 150,
+ collect: async (options) => { reads += 1; calls.push(options.remoteIntervalMs); return { rows: [], machines: [], problems: [], observedAt: Date.now() } },
+ })
+ t.after(() => viewer.close())
+ const sleep = (ms) => new Promise((done) => setTimeout(done, ms))
+
+ await sleep(200)
+ assert.equal(reads, 1, 'the start-up read, and nothing more without a pane')
+
+ const response = await fetch(`${base}/events`)
+ const reader = response.body.getReader()
+ await reader.read()
+ await sleep(220)
+ assert.ok(reads >= 3, `a pane keeps the fleet read on a tick (${reads} reads)`)
+ assert.ok(calls.slice(1).every((v) => v === 150), 'ticks hand the collector the remote clock, not a forced remote read')
+
+ await reader.cancel()
+ await sleep(60)
+ const atLeave = reads
+ await sleep(200)
+ assert.ok(reads - atLeave <= 1, `reads stop once the last pane leaves (${reads - atLeave} more)`)
+
+ // Refresh is the person asking: it reads now and asks the remote machines too.
+ await fetch(`${base}/api/refresh`, { method: 'POST', headers: { 'content-type': 'application/json', 'x-hps-token': viewer.token }, body: '{}' })
+ assert.equal(calls.at(-1), 0, 'a refresh forces the remote read')
+})
diff --git a/store/agents/harness-monitor/viewer.mjs b/store/agents/harness-monitor/viewer.mjs
index d6201c778..6085a0549 100644
--- a/store/agents/harness-monitor/viewer.mjs
+++ b/store/agents/harness-monitor/viewer.mjs
@@ -21,6 +21,7 @@ import { homedir } from 'node:os'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
import { pause, resume } from './lib/actions.mjs'
+import { closeBridges } from './lib/bridge.mjs'
import { collect as collectFleet, summarize, tilde } from './lib/inventory.mjs'
import { capture, looksBlocked } from './lib/panes.mjs'
import { DEFAULT_POLICY, decide, normalizePolicy } from './lib/policy.mjs'
@@ -34,10 +35,13 @@ const VERBS = new Set(['pause', 'resume', 'pin', 'unpin'])
* is about to act on, which are the only rows where the answer changes anything. */
const BLOCKED_SCAN_LIMIT = 24
-export function createViewer({ workspace, port = 0, intervalMs = 4000, now = () => Date.now(), collect = collectFleet, scan = capture, verbs = { pause, resume } }) {
+export function createViewer({ workspace, port = 0, intervalMs = 4000, remoteIntervalMs = 60_000, now = () => Date.now(), collect = collectFleet, scan = capture, verbs = { pause, resume } }) {
const token = randomBytes(24).toString('base64url')
const clients = new Set()
const cache = new Map()
+ // The remote machines' last answers (lib/inventory.mjs collect): they are asked every
+ // `remoteIntervalMs`, the local machine every `intervalMs`, and only while a pane is connected.
+ const remote = { at: 0, answers: new Map(), problems: [] }
let snapshot = { spec: 1, status: 'starting', rows: [], summary: null, policy: DEFAULT_POLICY, plan: [], totals: null, problems: [], log: [], observedAt: null, intervalMs }
let stopped = false, timer, heartbeat, polling = null
// Resolves after the first observation lands. The server starts listening before it, so the pane draws
@@ -57,14 +61,14 @@ export function createViewer({ workspace, port = 0, intervalMs = 4000, now = ()
}
/** One observation: the fleet, what the policy would do to it, and the pane header's verdict. */
- async function observe() {
+ async function observe({ forceRemote = false } = {}) {
const state = await readState(workspace).catch(() => null)
if (!state) {
snapshot = { ...snapshot, status: 'unreadable', problems: [{ machine: 'monitor.json', error: 'The policy file could not be read. Ask the agent to check monitor.json.' }] }
return
}
const policy = normalizePolicy(state.policy, { home: homedir() })
- const { rows, problems, degraded } = await collect({ state, now: now(), includeRemote: true, cache })
+ const { rows, problems, degraded } = await collect({ state, now: now(), includeRemote: true, cache, remote, remoteIntervalMs: forceRemote ? 0 : remoteIntervalMs })
// Look for an open prompt only where it would change a decision, newest candidates first.
const candidates = rows
@@ -97,17 +101,33 @@ export function createViewer({ workspace, port = 0, intervalMs = 4000, now = ()
await writeVerdict(workspace, { summary, rows, plan: plan.entries, problems }).catch(() => {})
}
- async function poll({ immediate = false } = {}) {
+ let lastPollAt = 0
+ // The next tick is booked only while a pane is connected: a viewer nobody is looking at — a tab in
+ // the background, a pane the person closed but whose process outlived the daemon — must not keep
+ // every machine on the account answering it.
+ const schedule = () => {
+ clearTimeout(timer); timer = null
+ if (stopped || clients.size === 0) return
+ timer = setTimeout(() => { void poll() }, intervalMs)
+ }
+ // Someone wants the observation now: read again if the one we have is older than a tick.
+ const wake = () => {
+ if (stopped || polling || now() - lastPollAt < intervalMs) return
+ clearTimeout(timer); timer = null
+ void poll()
+ }
+
+ async function poll({ immediate = false, forceRemote = false } = {}) {
if (polling) return polling
polling = (async () => {
- try { await observe() } catch (error) {
+ try { await observe({ forceRemote }) } catch (error) {
snapshot = { ...snapshot, status: 'unavailable', problems: [{ machine: 'this machine', error: error instanceof Error ? error.message : String(error) }] }
}
if (!stopped) publish()
observed()
})()
- try { await polling } finally { polling = null }
- if (!stopped && !immediate) { clearTimeout(timer); timer = setTimeout(poll, intervalMs) }
+ try { await polling } finally { polling = null; lastPollAt = now() }
+ if (!stopped && !immediate) schedule()
}
/** The write surface. One verb, up to 64 rows, and a receipt per row — the same library call the CLI
@@ -170,19 +190,24 @@ export function createViewer({ workspace, port = 0, intervalMs = 4000, now = ()
let payload; try { payload = JSON.parse(body || '{}') } catch { json(res, 400, { error: 'Send JSON.' }); return }
if (url.pathname === '/api/act') { json(res, 200, await act(payload)); return }
if (url.pathname === '/api/policy') { json(res, 200, await savePolicy(payload.policy ?? {})); return }
- if (url.pathname === '/api/refresh') { await poll({ immediate: true }); json(res, 200, { ok: true }); return }
+ if (url.pathname === '/api/refresh') { await poll({ immediate: true, forceRemote: true }); json(res, 200, { ok: true }); return }
json(res, 404, { error: 'Not found' }); return
}
if (!['GET', 'HEAD'].includes(req.method)) { res.setHeader('allow', 'GET, HEAD, POST'); json(res, 405, { error: 'Not allowed.' }); return }
if (url.pathname === '/health') { json(res, 200, { ok: true }); return }
- if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); return }
+ if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); wake(); return }
if (url.pathname === '/events') {
if (req.method === 'HEAD') { res.writeHead(200, headers); res.end(); return }
if (clients.size >= 16) { json(res, 503, { error: 'Too many viewer connections.' }); return }
res.writeHead(200, { ...headers, 'content-type': 'text/event-stream', connection: 'keep-alive', 'x-accel-buffering': 'no' })
res.write(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`)
- clients.add(res); res.on('close', () => clients.delete(res)); return
+ clients.add(res)
+ res.on('close', () => { clients.delete(res); if (clients.size === 0) { clearTimeout(timer); timer = null } })
+ // A pane just opened: catch up if the observation is stale, and start ticking either way.
+ wake()
+ if (!timer && !polling) schedule()
+ return
}
const assets = {
@@ -225,6 +250,8 @@ export function createViewer({ workspace, port = 0, intervalMs = 4000, now = ()
}),
close: async () => {
stopped = true; clearTimeout(timer); clearInterval(heartbeat)
+ await polling?.catch(() => {})
+ closeBridges()
for (const client of clients) client.end()
server.closeAllConnections()
await new Promise((done) => server.close(done))
diff --git a/store/agents/machine-monitor/lib/daemon.mjs b/store/agents/machine-monitor/lib/daemon.mjs
index 1df37d87f..8f4f506c8 100644
--- a/store/agents/machine-monitor/lib/daemon.mjs
+++ b/store/agents/machine-monitor/lib/daemon.mjs
@@ -128,49 +128,133 @@ export async function remotePasswordSet(env = process.env) {
* `relayIsolation` asks for a session of this package's own, so polling a machine never disturbs the
* window the person is working in. Close code 4404 is the one answer that is not a failure: that
* machine is not linked from here yet, which is a thing to offer, not an error to report.
+ *
+ * ONE bridge socket per machine, kept open between polls. A poll used to open a socket, select the
+ * machine, ask, and close — and behind that select Harness dialled its relay, ran an encryption
+ * handshake and negotiated a WebRTC channel, all thrown away a second later, for every machine, every
+ * tick (thousands of relay dials an hour across a few panes, 2026-09-22). Now the select happens once;
+ * a poll is one `agents_list` frame on a socket that is already there. A socket that closes, or that
+ * stops answering, is dropped and the next poll opens a fresh one.
*/
-export async function listHarnesses(machineId, { timeoutMs = 20_000, env = process.env, WebSocketImpl = globalThis.WebSocket } = {}) {
- if (!WebSocketImpl) return { ok: false, error: 'Node 22 or newer is required to read a machine roster.' };
+const sessions = new Map(); // WebSocketImpl -> Map<machineId, session>; keyed by impl so a test's scripted socket is its own world
+
+function sessionsFor(WebSocketImpl) {
+ let byMachine = sessions.get(WebSocketImpl);
+ if (!byMachine) { byMachine = new Map(); sessions.set(WebSocketImpl, byMachine); }
+ return byMachine;
+}
+
+/** Open the bridge to `machineId`: resolves once Harness has selected it, rejects with the same
+ * answers `listHarnesses` used to give when it cannot. The session stays in the map until its socket
+ * closes or errors, whichever comes first; every waiter still on it hears that as a failure. */
+function openSession(machineId, { env, WebSocketImpl, timeoutMs, forceReconnect = false }) {
+ const byMachine = sessionsFor(WebSocketImpl);
let socket;
try { socket = new WebSocketImpl(bridgeUrl(env)); }
- catch { return { ok: false, error: DOWN }; }
- return await new Promise(resolve => {
- let done = false, deadline, heartbeat;
- const requestId = randomUUID();
- const finish = value => {
- if (done) return;
- done = true;
- clearTimeout(deadline); clearInterval(heartbeat);
- try { socket.close(); } catch { /* already gone */ }
- resolve(value);
- };
- deadline = setTimeout(() => finish({ ok: false, error: 'This machine did not answer in time.' }), timeoutMs);
+ catch { return Promise.reject({ ok: false, error: DOWN }); }
+ const session = { socket, waiters: new Map(), heartbeat: null };
+ const fail = (answer) => {
+ if (byMachine.get(machineId) === session) byMachine.delete(machineId);
+ clearInterval(session.heartbeat);
+ for (const waiter of session.waiters.values()) waiter(answer);
+ session.waiters.clear();
+ try { socket.close(); } catch { /* already gone */ }
+ };
+ session.close = () => fail({ ok: false, error: 'The connection to this machine closed before it answered.' });
+ const ready = new Promise((resolve, reject) => {
+ let settled = false;
+ const settle = (fn, value) => { if (settled) return; settled = true; clearTimeout(deadline); fn(value); };
+ const deadline = setTimeout(() => { settle(reject, { ok: false, error: 'This machine did not answer in time.' }); fail({ ok: false, error: 'This machine did not answer in time.' }); }, timeoutMs);
+ // `forceReconnect` tells Harness the session it kept for this machine is dead (the machine's own
+ // Harness restarted under it) and must be dialled fresh rather than handed back once more.
socket.addEventListener('open', () => socket.send(JSON.stringify({
type: 'machine_select',
- payload: { machineId, localProtocolVersion: 1, relayIsolation: true },
+ payload: { machineId, localProtocolVersion: 1, relayIsolation: true, ...(forceReconnect ? { forceReconnect: true } : {}) },
})));
- socket.addEventListener('error', () => finish({ ok: false, error: DOWN }));
+ socket.addEventListener('error', () => { settle(reject, { ok: false, error: DOWN }); fail({ ok: false, error: DOWN }); });
socket.addEventListener('close', event => {
- if (event.code === 4404) finish({ ok: false, needsLink: true, error: 'This machine is not linked from this computer yet.' });
- else finish({ ok: false, error: event.code === 4403 ? 'Harness would not connect to this machine.' : 'The connection to this machine closed before it answered.' });
+ const answer = event.code === 4404
+ ? { ok: false, needsLink: true, error: 'This machine is not linked from this computer yet.' }
+ : { ok: false, error: event.code === 4403 ? 'Harness would not connect to this machine.' : 'The connection to this machine closed before it answered.' };
+ settle(reject, answer); fail(answer);
});
socket.addEventListener('message', event => {
if (typeof event.data !== 'string' || event.data.length > 8 * 1024 * 1024) return;
let frame;
try { frame = JSON.parse(event.data); } catch { return; }
if (frame?.type === 'connected') {
- heartbeat = setInterval(() => { try { socket.send(JSON.stringify({ type: 'ping', payload: {} })); } catch { /* closing */ } }, 15_000);
- // The id rides in the payload — that is where the daemon reads it, and the reply carries it back.
- socket.send(JSON.stringify({ type: 'agents_list', payload: { requestId } }));
+ session.heartbeat = setInterval(() => { try { socket.send(JSON.stringify({ type: 'ping', payload: {} })); } catch { /* closing */ } }, 15_000);
+ session.heartbeat.unref?.();
+ settle(resolve);
return;
}
- if (frame?.type === 'agents_list_result' && frame.payload?.requestId === requestId) {
- finish({ ok: true, harnesses: Array.isArray(frame.payload.agents) ? frame.payload.agents : [] });
+ if (frame?.type === 'agents_list_result') {
+ const waiter = session.waiters.get(frame.payload?.requestId);
+ if (!waiter) return;
+ session.waiters.delete(frame.payload.requestId);
+ waiter({ ok: true, harnesses: Array.isArray(frame.payload.agents) ? frame.payload.agents : [] });
return;
}
if (['error', 'connection_error', 'local_protocol_error', 'machine_link_required'].includes(frame?.type)) {
- finish({ ok: false, needsLink: frame.type === 'machine_link_required', error: 'This machine is unavailable, or needs linking from this computer.' });
+ const answer = { ok: false, needsLink: frame.type === 'machine_link_required', error: 'This machine is unavailable, or needs linking from this computer.' };
+ settle(reject, answer); fail(answer);
}
});
});
+ session.ready = ready;
+ byMachine.set(machineId, session);
+ return ready.then(() => session);
+}
+
+/** One roster read on an open session; the whole call is bounded by `timeoutMs`. */
+function ask(session, timeoutMs) {
+ return new Promise(resolve => {
+ const requestId = randomUUID();
+ const deadline = setTimeout(() => {
+ if (!session.waiters.delete(requestId)) return;
+ resolve({ ok: false, timedOut: true, error: 'This machine did not answer in time.' });
+ }, timeoutMs);
+ session.waiters.set(requestId, answer => { clearTimeout(deadline); resolve(answer); });
+ // The id rides in the payload — that is where the daemon reads it, and the reply carries it back.
+ try { session.socket.send(JSON.stringify({ type: 'agents_list', payload: { requestId } })); }
+ catch { session.waiters.delete(requestId); clearTimeout(deadline); resolve({ ok: false, error: 'The connection to this machine closed before it answered.' }); }
+ });
+}
+
+export async function listHarnesses(machineId, { timeoutMs = 20_000, env = process.env, WebSocketImpl = globalThis.WebSocket } = {}) {
+ if (!WebSocketImpl) return { ok: false, error: 'Node 22 or newer is required to read a machine roster.' };
+ const byMachine = sessionsFor(WebSocketImpl);
+ const existing = byMachine.get(machineId);
+ let session;
+ try {
+ session = existing
+ ? await existing.ready.then(() => existing)
+ : await openSession(machineId, { env, WebSocketImpl, timeoutMs });
+ } catch (answer) {
+ return answer?.ok === false ? answer : { ok: false, error: DOWN };
+ }
+ const reused = session === existing;
+ let result = await ask(session, timeoutMs);
+ if (result.timedOut) {
+ // A socket that is open but no longer answers is the daemon's "transport alive, session dead"
+ // case: never keep it. On a REUSED one, try ONCE more on a fresh socket before reporting the
+ // machine silent; a fresh one that stayed silent has already had its chance.
+ session.close();
+ if (reused) {
+ try { session = await openSession(machineId, { env, WebSocketImpl, timeoutMs, forceReconnect: true }); }
+ catch (answer) { return answer?.ok === false ? answer : { ok: false, error: DOWN }; }
+ result = await ask(session, timeoutMs);
+ if (result.timedOut) session.close();
+ }
+ delete result.timedOut;
+ }
+ return result;
+}
+
+/** Drop every bridge session (the pane is closing). */
+export function closeSessions() {
+ for (const byMachine of sessions.values()) {
+ for (const session of [...byMachine.values()]) session.close();
+ }
+ sessions.clear();
}
diff --git a/store/agents/machine-monitor/test/daemon.test.mjs b/store/agents/machine-monitor/test/daemon.test.mjs
index 76ff9b618..150851827 100644
--- a/store/agents/machine-monitor/test/daemon.test.mjs
+++ b/store/agents/machine-monitor/test/daemon.test.mjs
@@ -1,6 +1,6 @@
import assert from 'node:assert/strict';
import test from 'node:test';
-import { bridgeUrl, daemonBase, listHarnesses } from '../lib/daemon.mjs';
+import { bridgeUrl, closeSessions, daemonBase, listHarnesses } from '../lib/daemon.mjs';
test('the daemon address must be loopback http, and the bridge follows it', () => {
assert.equal(daemonBase({}).href, 'http://127.0.0.1:18473/');
@@ -83,3 +83,83 @@ test('without a WebSocket there is an answer, not a crash', async () => {
assert.equal(result.ok, false);
assert.match(result.error, /Node 22/);
});
+
+/** A scripted socket class that also counts its instances and can be closed from the test. */
+function scriptedFactory({ answer = true, closeAfterAnswer = false } = {}) {
+ const made = [];
+ const sent = [];
+ const Impl = class {
+ constructor() {
+ made.push(this);
+ this.listeners = {};
+ queueMicrotask(() => {
+ this.fire('open', {});
+ this.fire('message', { data: JSON.stringify({ type: 'connected', payload: { machineId: 'a'.repeat(32), relayIsolation: true } }) });
+ });
+ }
+ addEventListener(type, handler) { (this.listeners[type] ??= []).push(handler); }
+ fire(type, event) { for (const handler of this.listeners[type] ?? []) handler(event); }
+ send(raw) {
+ const frame = JSON.parse(raw);
+ sent.push(frame);
+ if (frame.type === 'agents_list' && answer) {
+ this.fire('message', { data: JSON.stringify({ type: 'agents_list_result', payload: { requestId: frame.payload.requestId, agents: [{ id: 'h' + made.length }] } }) });
+ if (closeAfterAnswer) this.fire('close', { code: 1000 });
+ }
+ }
+ close() {}
+ };
+ return { Impl, made, sent };
+}
+
+test('a second read on the same machine reuses the bridge socket: one select, two asks', async () => {
+ const { Impl, made, sent } = scriptedFactory();
+ const first = await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 500 });
+ const second = await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 500 });
+ assert.equal(first.ok, true); assert.equal(second.ok, true);
+ assert.equal(made.length, 1, 'no second socket');
+ assert.deepEqual(sent.map(f => f.type), ['machine_select', 'agents_list', 'agents_list']);
+ assert.notEqual(sent[1].payload.requestId, sent[2].payload.requestId);
+ closeSessions();
+});
+
+test('a socket that closed is replaced on the next read', async () => {
+ const { Impl, made, sent } = scriptedFactory({ closeAfterAnswer: true });
+ assert.equal((await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 500 })).ok, true);
+ assert.equal((await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 500 })).ok, true);
+ assert.equal(made.length, 2, 'the close was noticed and a fresh socket opened');
+ assert.deepEqual(sent.map(f => f.type), ['machine_select', 'agents_list', 'machine_select', 'agents_list']);
+ closeSessions();
+});
+
+test('a reused socket that stops answering is dropped and the read retried once on a fresh one', async () => {
+ let answering = true;
+ const made = [];
+ const selects = [];
+ const Impl = class {
+ constructor() {
+ made.push(this); this.listeners = {};
+ queueMicrotask(() => { this.fire('open', {}); this.fire('message', { data: JSON.stringify({ type: 'connected', payload: {} }) }); });
+ }
+ addEventListener(type, handler) { (this.listeners[type] ??= []).push(handler); }
+ fire(type, event) { for (const handler of this.listeners[type] ?? []) handler(event); }
+ send(raw) {
+ const frame = JSON.parse(raw);
+ if (frame.type === 'machine_select') selects.push(frame.payload);
+ if (frame.type !== 'agents_list') return;
+ // The first socket answers once, then goes mute; every later socket answers.
+ if (made.indexOf(this) === 0 && !answering) return;
+ this.fire('message', { data: JSON.stringify({ type: 'agents_list_result', payload: { requestId: frame.payload.requestId, agents: [{ id: 'from-' + made.indexOf(this) }] } }) });
+ }
+ close() {}
+ };
+ assert.deepEqual((await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 100 })).harnesses, [{ id: 'from-0' }]);
+ answering = false;
+ const result = await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 100 });
+ assert.equal(result.ok, true);
+ assert.deepEqual(result.harnesses, [{ id: 'from-1' }], 'answered by the replacement socket');
+ assert.equal(made.length, 2);
+ assert.equal(selects[0].forceReconnect, undefined);
+ assert.equal(selects[1].forceReconnect, true, 'the retry tells Harness its kept session is dead');
+ closeSessions();
+});
diff --git a/store/agents/machine-monitor/test/server.test.mjs b/store/agents/machine-monitor/test/server.test.mjs
index 872ffd3f9..a3fd4fa1b 100644
--- a/store/agents/machine-monitor/test/server.test.mjs
+++ b/store/agents/machine-monitor/test/server.test.mjs
@@ -137,3 +137,34 @@ test('the event stream opens with the current observation', async () => {
assert.match(text, /"status":"live"/);
await reader.cancel();
});
+
+test('the fleet is read on a tick only while a pane is connected; an unattended viewer reads once', async () => {
+ let reads = 0;
+ const workspace = await mkdtemp(join(tmpdir(), 'machines-server-'));
+ const viewer = createViewer({ workspace, port: 0, intervalMs: 40, collect: async () => { reads += 1; return SNAPSHOT; } });
+ const port = await viewer.start();
+ test.after(async () => { await viewer.close(); await rm(workspace, { recursive: true, force: true }); });
+ const sleep = ms => new Promise(done => setTimeout(done, ms));
+
+ await sleep(200);
+ assert.equal(reads, 1, 'the start-up read, and nothing more without a pane');
+
+ const response = await fetch(`http://127.0.0.1:${port}/events`);
+ const reader = response.body.getReader();
+ await reader.read();
+ await sleep(220);
+ const withPane = reads;
+ assert.ok(withPane >= 3, `a pane keeps the fleet read on a tick (${withPane} reads)`);
+
+ await reader.cancel();
+ await sleep(60);
+ const atLeave = reads;
+ await sleep(200);
+ assert.ok(reads - atLeave <= 1, `reads stop once the last pane leaves (${reads - atLeave} more)`);
+
+ // A one-off request for the observation wakes exactly one read when it has gone stale.
+ const before = reads;
+ await fetch(`http://127.0.0.1:${port}/api/snapshot`);
+ await sleep(60);
+ assert.equal(reads, before + 1);
+});
diff --git a/store/agents/machine-monitor/viewer.mjs b/store/agents/machine-monitor/viewer.mjs
index 55eab0d99..ad9429886 100644
--- a/store/agents/machine-monitor/viewer.mjs
+++ b/store/agents/machine-monitor/viewer.mjs
@@ -16,23 +16,46 @@ import { readFile } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import { atomicJson, operations, PACKAGE, recordOperation, stateDir, writeVerdict } from './lib/fleet.mjs';
+import { closeSessions } from './lib/daemon.mjs';
import { linkMachine } from './lib/ops.mjs';
import { createCollector } from './lib/snapshot.mjs';
+/**
+ * How often the fleet is re-read while a pane is looking at it. Every remote machine on the account
+ * is asked over Harness's bridge each time, so this is the one number that decides what a pane costs
+ * the rest of the fleet; a minute is fast enough for a map, and nothing is read at all while no pane
+ * is open (a request for the observation still wakes one read if it has gone stale).
+ */
+const POLL_INTERVAL_MS = 60_000;
+
const EMPTY = {
- spec: 1, status: 'connecting', message: null, observedAt: null, pollIntervalMs: 15_000,
+ spec: 1, status: 'connecting', message: null, observedAt: null, pollIntervalMs: POLL_INTERVAL_MS,
account: null, localMachineId: null, thisComputer: null,
machines: [], projects: [], operations: [], totals: null,
summary: { machines: 0, online: 0, needsLink: 0, harnesses: 0, open: 0, projects: 0, engines: {} },
sources: {},
};
-export function createViewer({ workspace, port = 0, intervalMs = 15_000, collect = createCollector(workspace, { intervalMs }) }) {
+export function createViewer({ workspace, port = 0, intervalMs = POLL_INTERVAL_MS, collect = createCollector(workspace, { intervalMs }) }) {
const clients = new Set();
let snapshot = { ...EMPTY, pollIntervalMs: intervalMs };
let stopped = false, pollTimer, opTimer, heartbeat, linking = false;
// The poll in flight, so close() can wait for it instead of leaving a write racing the exit.
- let polling = null;
+ let polling = null, inFlight = false, lastPollAt = 0;
+ // The next tick is booked only while a pane is connected: an unattended viewer (a tab in the
+ // background, a pane the person closed but whose process outlived the daemon) must not keep every
+ // machine on the account answering it.
+ const schedule = () => {
+ clearTimeout(pollTimer); pollTimer = null;
+ if (stopped || clients.size === 0) return;
+ pollTimer = setTimeout(() => { polling = poll(); }, intervalMs);
+ };
+ // Someone wants the observation now: read again if the one we have is older than a tick.
+ const wake = () => {
+ if (stopped || inFlight || Date.now() - lastPollAt < intervalMs) return;
+ clearTimeout(pollTimer); pollTimer = null;
+ polling = poll();
+ };
const headers = { 'cache-control': 'no-store', 'x-content-type-options': 'nosniff', 'referrer-policy': 'no-referrer' };
const json = (res, code, value) => { res.writeHead(code, { ...headers, 'content-type': 'application/json; charset=utf-8' }); res.end(JSON.stringify(value)); };
const publish = () => {
@@ -74,8 +97,8 @@ export function createViewer({ workspace, port = 0, intervalMs = 15_000, collect
});
if (!result.ok) { json(res, 400, { error: result.error }); return; }
json(res, 200, { ok: true });
- clearTimeout(pollTimer);
- polling = poll();
+ clearTimeout(pollTimer); pollTimer = null;
+ if (!inFlight) polling = poll();
} finally { linking = false; }
return;
}
@@ -85,13 +108,17 @@ export function createViewer({ workspace, port = 0, intervalMs = 15_000, collect
return;
}
if (url.pathname === '/health') { json(res, 200, { ok: true }); return; }
- if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); return; }
+ if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); wake(); return; }
if (url.pathname === '/events') {
if (req.method === 'HEAD') { res.writeHead(200, headers); res.end(); return; }
if (clients.size >= 32) { json(res, 503, { error: 'Too many viewer connections.' }); return; }
res.writeHead(200, { ...headers, 'content-type': 'text/event-stream', connection: 'keep-alive', 'x-accel-buffering': 'no' });
res.write(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`);
- clients.add(res); res.on('close', () => clients.delete(res));
+ clients.add(res);
+ res.on('close', () => { clients.delete(res); if (clients.size === 0) { clearTimeout(pollTimer); pollTimer = null; } });
+ // A pane just opened: catch up if the observation is stale, and start ticking either way.
+ wake();
+ if (!pollTimer && !inFlight) schedule();
return;
}
const assets = {
@@ -113,6 +140,7 @@ export function createViewer({ workspace, port = 0, intervalMs = 15_000, collect
});
async function poll() {
+ inFlight = true;
try {
const observed = await collect();
// A poll that lands after close() has nothing to publish: the workspace may already be gone,
@@ -132,9 +160,11 @@ export function createViewer({ workspace, port = 0, intervalMs = 15_000, collect
};
await writeVerdict(workspace, snapshot).catch(() => {});
}
+ inFlight = false;
+ lastPollAt = Date.now();
if (stopped) return;
publish();
- pollTimer = setTimeout(() => { polling = poll(); }, intervalMs);
+ schedule();
}
async function pollOperations() {
@@ -161,6 +191,7 @@ export function createViewer({ workspace, port = 0, intervalMs = 15_000, collect
stopped = true;
clearTimeout(pollTimer); clearTimeout(opTimer); clearInterval(heartbeat);
await polling?.catch(() => {});
+ closeSessions();
for (const client of clients) client.end();
server.closeAllConnections();
await new Promise(done => server.close(done));