The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat: add viewer ledger for process cleanup and optimize remote inventory collection
0xken156aa085aa9289
diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex 390f40de1..06563c6c9 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -101,6 +101,7 @@ import { preTrustClaudeProject, preTrustCodexProject } from './lib/claudeTrust.jimport { materializeWorkspace } from './dsh/materialize.js'import { dshLaunch } from './dsh/launch.js'import { DshViewerManager } from './dsh/viewer.js'+import { ViewerLedger } from './dsh/viewerLedger.js'import { DshVerdictWatcher, type DshVerdict } from './dsh/verdict.js'import { dshCommand, dshUsage } from './dsh/command.js'import type { AgentDshContext } from './lib/agentFrame.js'@@ -1608,6 +1609,11 @@ async function runForeground(session: AuthSession): Promise<void> {const session = registry.byAgent(agentId)if (session && registry.terminalAvailable(agentId)) syncSession(session)}+ // Viewers an earlier daemon started and never stopped (crash, force quit, SIGKILL) are still running+ // and still polling; stop them BEFORE this daemon starts its own, or they accumulate a generation per+ // restart. Only pids whose live start time matches what that daemon recorded are touched.+ const viewerLedger = new ViewerLedger({ log: (line) => console.log(line) })+ viewerLedger.reapOrphans()const dshViewers = new DshViewerManager({onUrl: (agentId, url) => {dshFrameFor(agentId).viewerUrl = url@@ -1615,6 +1621,7 @@ async function runForeground(session: AuthSession): Promise<void> {syncCompanion(agentId)},log: (line) => console.log(line),+ ledger: viewerLedger,})const dshVerdicts = new DshVerdictWatcher({onChange: (agentId, verdict) => {diff --git a/cli/src/dsh/shell.ts b/cli/src/dsh/shell.tsindex fcbe82d9e..949044fb2 100644--- a/cli/src/dsh/shell.ts+++ b/cli/src/dsh/shell.ts@@ -138,6 +138,15 @@ export function runDshCommand(script: string, opts: DshCommandOptions): Promise<export function killProcessGroup(child: ChildProcess, graceMs = 3_000): void {const pid = child.pidif (!pid) return+ const timer = killPidGroup(pid, graceMs)+ child.once('exit', () => clearTimeout(timer))+}++/** The same, for a group this process did not spawn (a viewer left over from an earlier daemon —+ * see viewerLedger.ts): nothing to wait on, so the SIGKILL follow-up fires unconditionally. */+export function killPidGroup(pid: number, graceMs = 3_000): NodeJS.Timeout {+ // pid 1 would make `-pid` "every process in my session"; nothing this daemon reaps is init.+ if (!Number.isInteger(pid) || pid <= 1) return setTimeout(() => {}, 0)const signalGroup = (signal: NodeJS.Signals): void => {try { process.kill(-pid, signal) } catch { /* already gone */ }try { process.kill(pid, signal) } catch { /* already gone */ }@@ -145,5 +154,5 @@ export function killProcessGroup(child: ChildProcess, graceMs = 3_000): void {signalGroup('SIGTERM')const timer = setTimeout(() => signalGroup('SIGKILL'), graceMs)timer.unref?.()- child.once('exit', () => clearTimeout(timer))+ return timer}diff --git a/cli/src/dsh/viewer.ts b/cli/src/dsh/viewer.tsindex fdd53b380..e7f1710e2 100644--- a/cli/src/dsh/viewer.ts+++ b/cli/src/dsh/viewer.ts@@ -20,6 +20,7 @@ import { installedDsh, type InstalledDsh } from './installed.js'import { isViewerPackage } from './manifest.js'import { resolveDshCommand } from './materialize.js'import { isShellNoise, killProcessGroup, spawnDshCommand } from './shell.js'+import type { ViewerLedger } from './viewerLedger.js'import { viewerTarget } from '../lib/viewerWire.js'/** The viewer that will actually run for a harness: its own, or the package it points at. */@@ -75,6 +76,8 @@ export interface DshViewerDeps {now?: () => number/** How `viewer.use` finds its package; the installed index by default. */lookup?: (id: string) => InstalledDsh | undefined+ /** Where each spawned viewer's pid is recorded so the next daemon can reap it (viewerLedger.ts). */+ ledger?: ViewerLedger}interface ViewerState {@@ -266,6 +269,7 @@ export class DshViewerManager {},})state.child = child+ if (child.pid) this.deps.ledger?.add({ pid: child.pid, agentId: state.agentId, dshId: state.dsh.id, viewerDir: viewer.dir })let logged = 0const onData = (chunk: Buffer): void => {for (const line of chunk.toString('utf8').split('\n')) {@@ -277,6 +281,7 @@ export class DshViewerManager {child.stderr?.on('data', onData)child.on('error', (error) => this.log(`[dsh] ${state.dsh.id} viewer could not start · ${error.message}`))child.on('exit', (code, signal) => {+ if (child.pid) this.deps.ledger?.remove(child.pid)if (state.child !== child) returnstate.child = nullstate.port = nulldiff --git a/cli/src/dsh/viewerLedger.spec.ts b/cli/src/dsh/viewerLedger.spec.tsnew file mode 100644index 000000000..18b8c479e--- /dev/null+++ b/cli/src/dsh/viewerLedger.spec.ts@@ -0,0 +1,91 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { spawn, type ChildProcess } from 'node:child_process'+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { ViewerLedger, processStartedAt } from './viewerLedger.js'++describe('ViewerLedger', () => {+ let dir: string+ beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'viewer-ledger-')) })+ afterEach(() => { rmSync(dir, { recursive: true, force: true }) })++ const entry = (pid: number, startedAt = 1_000_000) => ({ pid, startedAt, agentId: 'agent-1', dshId: 'acme/thing', viewerDir: '/i/acme/thing' })++ it('records a spawned viewer and forgets it on exit, atomically on disk', () => {+ const file = join(dir, 'viewers.json')+ const ledger = new ViewerLedger({ file, now: () => 42 })+ ledger.add({ pid: 101, agentId: 'a', dshId: 'x/y', viewerDir: '/v' })+ ledger.add(entry(102))+ expect(JSON.parse(readFileSync(file, 'utf8'))).toEqual([+ { pid: 101, agentId: 'a', dshId: 'x/y', viewerDir: '/v', startedAt: 42 },+ entry(102),+ ])+ ledger.remove(101)+ expect(ledger.read()).toEqual([entry(102)])+ // A pid seen again replaces its old row rather than duplicating it.+ ledger.add(entry(102, 2_000_000))+ expect(ledger.read()).toEqual([entry(102, 2_000_000)])+ })++ it('reads a missing or malformed file as empty', () => {+ const ledger = new ViewerLedger({ file: join(dir, 'nope.json') })+ expect(ledger.read()).toEqual([])+ expect(ledger.reapOrphans()).toEqual([])+ })++ it('reaps only a pid that is alive AND started when the ledger says; then starts over', () => {+ const file = join(dir, 'viewers.json')+ const killed: number[] = []+ const log: string[] = []+ const ledger = new ViewerLedger({+ file,+ startedAt: (pid) => ({ 11: 1_000_500, 12: 5_000_000, 13: null }[pid] ?? null),+ kill: (pid) => killed.push(pid),+ log: (line) => log.push(line),+ })+ ledger.add(entry(11)) // alive, start agrees (within tolerance) → reaped+ ledger.add(entry(12)) // alive, but a DIFFERENT process now wears this pid → left alone+ ledger.add(entry(13)) // gone → nothing to do+ ledger.add(entry(1)) // never: pid 1 is init, and its group is everything → refused at add+ const reaped = ledger.reapOrphans()+ expect(reaped.map((e) => e.pid)).toEqual([11])+ expect(killed).toEqual([11])+ expect(log.some((l) => l.includes('reaped orphaned acme/thing viewer pid 11'))).toBe(true)+ expect(log.some((l) => l.includes('pid 12 is no longer'))).toBe(true)+ expect(ledger.read()).toEqual([])+ })++ it('a ledger file naming pid 1 is not trusted either', () => {+ const file = join(dir, 'viewers.json')+ writeFileSync(file, JSON.stringify([entry(1), entry(0), entry(-5)]))+ const killed: number[] = []+ const ledger = new ViewerLedger({ file, startedAt: () => 1_000_000, kill: (pid) => killed.push(pid) })+ expect(ledger.read()).toEqual([])+ expect(ledger.reapOrphans()).toEqual([])+ expect(killed).toEqual([])+ })++ it('processStartedAt reads a real process and reports a dead one as null', async () => {+ const child: ChildProcess = spawn('sleep', ['30'], { stdio: 'ignore' })+ try {+ const at = processStartedAt(child.pid!)+ expect(at).not.toBeNull()+ expect(Math.abs(at! - Date.now())).toBeLessThan(10_000)+ } finally {+ child.kill('SIGKILL')+ await new Promise((resolve) => child.once('exit', resolve))+ }+ expect(processStartedAt(child.pid!)).toBeNull()+ })++ it('the default reaper really stops a leftover process group', async () => {+ const file = join(dir, 'viewers.json')+ const child = spawn('sleep', ['30'], { stdio: 'ignore', detached: true })+ const exited = new Promise<void>((resolve) => child.once('exit', () => resolve()))+ const ledger = new ViewerLedger({ file, log: vi.fn() })+ ledger.add({ pid: child.pid!, agentId: 'a', dshId: 'x/y', viewerDir: '/v' })+ expect(ledger.reapOrphans().map((e) => e.pid)).toEqual([child.pid])+ await exited+ })+})diff --git a/cli/src/dsh/viewerLedger.ts b/cli/src/dsh/viewerLedger.tsnew file mode 100644index 000000000..26800b957--- /dev/null+++ b/cli/src/dsh/viewerLedger.ts@@ -0,0 +1,123 @@+/**+ * Which viewer processes THIS daemon has started, on disk, so the NEXT daemon can stop the ones it+ * never got to.+ *+ * Viewers are spawned detached, in their own process group (shell.ts spawnDshCommand), and only the+ * daemon's clean exits stop them (`dshViewers.stopAll()` on shutdown and on the self-update handoff).+ * Every other way a daemon ends — a crash, a force quit, `harness stop` falling through to SIGKILL —+ * leaves them running under launchd/systemd, still polling, still holding memory, and the daemon+ * that comes up next starts a fresh set beside them: seven generations of one viewer were found on+ * a single Mac. Nothing in the packages can fix that, so the daemon keeps a ledger.+ *+ * A pid alone is not proof — pids are reused — so each entry also records when the process started,+ * and the reaper only touches a pid whose live start time agrees with the ledger's.+ */+import { execFileSync } from 'node:child_process'+import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'+import { dirname, join } from 'node:path'+import { env } from '../config/env.js'+import { killPidGroup } from './shell.js'++export interface ViewerLedgerEntry {+ pid: number+ /** ms epoch when the daemon spawned it; compared against the live process's start time. */+ startedAt: number+ agentId: string+ dshId: string+ viewerDir: string+}++/** How far the ledger's spawn time and the kernel's process start time may disagree and still be+ * the same process. Spawn-to-exec is milliseconds; `ps` prints whole seconds. */+const START_TOLERANCE_MS = 10_000++export const VIEWER_LEDGER_FILE = join(env.ADAPTER_DATA_DIR, 'viewers.json')++/** When `pid` started, per the kernel, or null when it is gone or `ps` cannot say. `lstart` is the+ * one column both macOS and Linux ps print in a form Date.parse reads ("Mon Sep 22 10:00:01 2026"). */+export function processStartedAt(pid: number): number | null {+ try {+ const out = execFileSync('ps', ['-o', 'lstart=', '-p', String(pid)], { encoding: 'utf8', timeout: 5_000 }).trim()+ if (!out) return null+ const at = Date.parse(out)+ return Number.isFinite(at) ? at : null+ } catch { return null }+}++export interface ViewerLedgerDeps {+ file?: string+ now?: () => number+ /** Test seams for the reaper. */+ startedAt?: (pid: number) => number | null+ kill?: (pid: number) => void+ log?: (line: string) => void+}++export class ViewerLedger {+ private readonly file: string+ constructor(private readonly deps: ViewerLedgerDeps = {}) {+ this.file = deps.file ?? VIEWER_LEDGER_FILE+ }++ read(): ViewerLedgerEntry[] {+ try {+ const parsed: unknown = JSON.parse(readFileSync(this.file, 'utf8'))+ if (!Array.isArray(parsed)) return []+ return parsed.filter((e): e is ViewerLedgerEntry =>+ !!e && typeof e === 'object'+ // > 1, never merely > 0: signalling group -1 is "every process I own".+ && Number.isInteger((e as ViewerLedgerEntry).pid) && (e as ViewerLedgerEntry).pid > 1+ && Number.isFinite((e as ViewerLedgerEntry).startedAt)+ && typeof (e as ViewerLedgerEntry).agentId === 'string'+ && typeof (e as ViewerLedgerEntry).dshId === 'string'+ && typeof (e as ViewerLedgerEntry).viewerDir === 'string')+ } catch { return [] }+ }++ private write(entries: ViewerLedgerEntry[]): void {+ try {+ mkdirSync(dirname(this.file), { recursive: true })+ const tmp = `${this.file}.${process.pid}.tmp`+ writeFileSync(tmp, JSON.stringify(entries, null, 2) + '\n', { mode: 0o600 })+ renameSync(tmp, this.file)+ } catch { /* best effort: a ledger that cannot be written only means no reaping next time */ }+ }++ add(entry: Omit<ViewerLedgerEntry, 'startedAt'> & { startedAt?: number }): void {+ if (!Number.isInteger(entry.pid) || entry.pid <= 1) return+ const startedAt = entry.startedAt ?? (this.deps.now ?? Date.now)()+ this.write([...this.read().filter((e) => e.pid !== entry.pid), { ...entry, startedAt }])+ }++ remove(pid: number): void {+ const entries = this.read()+ const kept = entries.filter((e) => e.pid !== pid)+ if (kept.length !== entries.length) this.write(kept)+ }++ /**+ * Stop every ledgered viewer that is still alive and provably the same process, then start the+ * ledger over. Meant for daemon boot, BEFORE this daemon starts any viewer of its own. Returns the+ * entries it signalled.+ */+ reapOrphans(): ViewerLedgerEntry[] {+ const entries = this.read()+ if (entries.length === 0) return []+ const startedAt = this.deps.startedAt ?? processStartedAt+ const kill = this.deps.kill ?? ((pid: number) => { killPidGroup(pid) })+ const reaped: ViewerLedgerEntry[] = []+ for (const entry of entries) {+ const live = startedAt(entry.pid)+ if (live === null) continue // gone, or unknowable — a pid we cannot vouch for is left alone+ if (Math.abs(live - entry.startedAt) > START_TOLERANCE_MS) {+ this.deps.log?.(`[dsh] pid ${entry.pid} is no longer the ${entry.dshId} viewer (started ${new Date(live).toISOString()}, ledger says ${new Date(entry.startedAt).toISOString()}) · left alone`)+ continue+ }+ kill(entry.pid)+ reaped.push(entry)+ this.deps.log?.(`[dsh] reaped orphaned ${entry.dshId} viewer pid ${entry.pid} (started ${new Date(entry.startedAt).toISOString()}, agent ${entry.agentId.slice(0, 8)})`)+ }+ this.write([])+ return reaped+ }+}diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.tsindex 4859e0949..a2846916b 100644--- a/cli/src/lib/remoteRelay.spec.ts+++ b/cli/src/lib/remoteRelay.spec.ts@@ -615,3 +615,108 @@ describe('RemoteRelayPool shares one upstream between every local client selecteexpect(live.sendBinary).toHaveBeenCalled()})})++describe('RemoteRelayPool keeps background (isolated) clients cheap', () => {+ const ctorArgs = [+ { accessToken: async () => 'unused' } as never,+ 'ws://unused',+ { pub: new Uint8Array(), priv: new Uint8Array() } as never,+ { pin: () => {}, get: () => null } as never,+ ] as const+ const sink = () => ({ sendFrame: vi.fn(() => true), sendBinary: vi.fn(() => true) })+ const select = { type: 'machine_select', payload: { relayIsolation: true } }++ it('a pool built with p2p off never starts a negotiation, even with a policy and on retry', () => {+ const pool = new RemoteRelayPool(...ctorArgs, { p2p: false }) as unknown as {+ startP2p: (machineId: string, entry: ReturnType<typeof fakeEntry>) => void+ }+ const entry = fakeEntry({ p2pPolicy: { enabled: true, protocolVersion: 1, stunUrls: [], openWaitMs: 1500 } })+ pool.startP2p('m1', entry)+ expect(entry.p2p).toBeNull()+ })++ it('a detached background session is shelved warm and handed to the next background client', async () => {+ const pool = new RemoteRelayPool(...ctorArgs)+ expect(pool.idleIsolatedCount('m1')).toBe(0)+ // Every background pool's dial is stubbed to leave an entry the way a completed dial does; count+ // how many dials happen across two sequential polls.+ const made: Array<{ ws: { close: ReturnType<typeof vi.fn>; terminate: ReturnType<typeof vi.fn> }; lingerTimer: ReturnType<typeof setTimeout> | null }> = []+ const dialSpy = vi.spyOn(RemoteRelayPool.prototype as unknown as { connect: (machineId: string) => Promise<unknown> }, 'connect')+ .mockImplementation(async function (this: unknown, machineId: string) {+ const entry = fakeEntry({ attached: new Set(), sink: null, ws: { send: vi.fn(), close: vi.fn(), terminate: vi.fn() }, viewers: { reset: vi.fn(), close: vi.fn() } })+ made.push(entry as unknown as typeof made[number])+ ;(this as { entries: Map<string, unknown> }).entries.set(machineId, entry)+ return entry+ })+ try {+ const first = sink()+ const a = await pool.acquireIsolated('m1', 'env', select, first, vi.fn())+ expect(first.sendFrame).toHaveBeenCalledWith({ type: 'connected', payload: { machineId: 'm1', e2ee: false, relayIsolation: true } })+ a.detach()+ // Detaching lingers the upstream and shelves the pool; nothing is terminated.+ expect(made).toHaveLength(1)+ expect(made[0].ws.terminate).not.toHaveBeenCalled()+ expect(made[0].ws.close).not.toHaveBeenCalled()+ expect(made[0].lingerTimer).not.toBeNull()+ expect(pool.idleIsolatedCount('m1')).toBe(1)++ const second = sink()+ const b = await pool.acquireIsolated('m1', 'env', select, second, vi.fn())+ expect(dialSpy).toHaveBeenCalledTimes(1) // the second poll took the warm session — no dial+ expect(pool.idleIsolatedCount('m1')).toBe(0)+ expect(made[0].lingerTimer).toBeNull()+ expect(second.sendFrame).toHaveBeenCalledWith({ type: 'connected', payload: { machineId: 'm1', e2ee: false, relayIsolation: true } })++ // Two background clients AT ONCE get distinct sessions: isolation between them is kept.+ const third = sink()+ const c = await pool.acquireIsolated('m1', 'env', select, third, vi.fn())+ expect(dialSpy).toHaveBeenCalledTimes(2)+ expect(made).toHaveLength(2)+ b.detach(); c.detach()+ expect(pool.idleIsolatedCount('m1')).toBe(2)+ for (const entry of made) clearTimeout(entry.lingerTimer!)+ // The window's own pool is untouched by any of this.+ expect((pool as unknown as { entries: Map<string, unknown> }).entries.size).toBe(0)+ // A background forceReconnect empties the shelf.+ pool.invalidateIsolated('m1')+ expect(pool.idleIsolatedCount('m1')).toBe(0)+ expect(made.every((entry) => entry.ws.terminate.mock.calls.length === 1)).toBe(true)+ } finally {+ dialSpy.mockRestore()+ }+ })++ it('a failed background dial is answered from the failure for the cooldown, not re-dialled per poll', async () => {+ vi.useFakeTimers()+ try {+ const pool = new RemoteRelayPool(...ctorArgs, { dialCooldownMs: 30_000 }) as unknown as {+ connect: (machineId: string, env: string, frame: unknown) => Promise<unknown>+ dial: (...args: unknown[]) => Promise<unknown>+ }+ const dial = vi.spyOn(pool, 'dial').mockRejectedValue(new Error('NO_PEER_LINK'))+ await expect(pool.connect('m1', 'env', select)).rejects.toThrow('NO_PEER_LINK')+ expect(dial).toHaveBeenCalledTimes(1)+ // Inside the cooldown: same rejection, no new dial.+ vi.advanceTimersByTime(10_000)+ await expect(pool.connect('m1', 'env', select)).rejects.toThrow('NO_PEER_LINK')+ expect(dial).toHaveBeenCalledTimes(1)+ // Past it: a fresh attempt.+ vi.advanceTimersByTime(25_000)+ await expect(pool.connect('m1', 'env', select)).rejects.toThrow('NO_PEER_LINK')+ expect(dial).toHaveBeenCalledTimes(2)+ } finally {+ vi.useRealTimers()+ }+ })++ it('the window pool has no cooldown: a person clicking again always gets a fresh dial', async () => {+ const pool = new RemoteRelayPool(...ctorArgs) as unknown as {+ connect: (machineId: string, env: string, frame: unknown) => Promise<unknown>+ dial: (...args: unknown[]) => Promise<unknown>+ }+ const dial = vi.spyOn(pool, 'dial').mockRejectedValue(new Error('relay connect timed out'))+ await expect(pool.connect('m1', 'env', select)).rejects.toThrow()+ await expect(pool.connect('m1', 'env', select)).rejects.toThrow()+ expect(dial).toHaveBeenCalledTimes(2)+ })+})diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.tsindex b980c9038..cb0276540 100644--- a/cli/src/lib/remoteRelay.ts+++ b/cli/src/lib/remoteRelay.ts@@ -37,6 +37,20 @@ import { isWrapped } from './e2ee/core.js'const CONNECT_TIMEOUT_MS = 15_000const LINGER_MS = 30_000+// The background pools' linger. A pane that closes its bridge socket after every read polls on a+// fixed tick (a minute, in the shipped monitors); anything at or under that tick would re-dial the+// relay (and redo the E2EE handshake) on every poll, which is exactly the churn these pools exist to+// stop. An idle upstream costs a heartbeat, so the linger is generous.+const ISOLATED_LINGER_MS = 3 * 60_000+// After a FAILED background dial, how long the same machine is answered from that failure instead of+// dialled again. A poller that retries on every tick against an unlinked or unreachable machine would+// otherwise turn one bad machine into a relay dial per tick, forever. Never applied to the window's+// pool: a person clicking again deserves a fresh attempt.+const ISOLATED_DIAL_COOLDOWN_MS = 30_000+// Warm background sessions kept per machine. Each concurrent background client needs its own (they+// must not see each other's frames), so the cap is "how many pollers may overlap on one machine and+// still find a warm session"; beyond it the extra session is torn down on detach as it always was.+const ISOLATED_IDLE_MAX = 4// Same convention/value as localWsServer.ts's app<->daemon heartbeat. Without this, a machine-node// cycling (e.g. `harness start` on the OTHER end after a crash/restart) can leave this daemon holding// an upstream socket the backend silently dropped with no close frame — every RPC sent through it then@@ -213,30 +227,89 @@ function framePayload(frame: Frame): Record<string, unknown> {: {}}+export interface RemoteRelayPoolOptions {+ /** Negotiate a WebRTC data channel for terminals once the E2EE session is up. Default on; the+ * background pool turns it off — a session that will only ever carry `agents_list` has no terminal+ * to move onto p2p, and the STUN/TURN round trips would be thrown away with it. */+ p2p?: boolean+ /** How long an entry nobody is attached to stays open for the next select. */+ lingerMs?: number+ /** After a failed dial, reject further dials for this machine for this long. 0 = never. */+ dialCooldownMs?: number+ /** Share one failure record between pools (the background pools all draw on the parent's). */+ dialFailures?: Map<string, { at: number; error: unknown }>+}+export class RemoteRelayPool {private entries = new Map<string, Entry>()private pending = new Map<string, Promise<Entry>>()+ /** Last failed dial per machine, for the cooldown. Cleared by the next successful dial. */+ private readonly lastDialFailure: Map<string, { at: number; error: unknown }>+ private readonly p2pEnabled: boolean+ private readonly lingerMs: number+ private readonly dialCooldownMs: number+ /** Warm background pools per machine, each holding one lingering session nobody is attached to —+ * see acquireIsolated(). A pool is either here (idle) or in a client's hands, never both. */+ private readonly idleIsolated = new Map<string, RemoteRelayPool[]>()constructor(private readonly auth: AuthSessionManager,private readonly backendWsBase: string,private readonly selfIdentity: Identity,private readonly peers: MachinePeerStore,- ) {}+ opts: RemoteRelayPoolOptions = {},+ ) {+ this.p2pEnabled = opts.p2p !== false+ this.lingerMs = opts.lingerMs ?? LINGER_MS+ this.dialCooldownMs = opts.dialCooldownMs ?? 0+ this.lastDialFailure = opts.dialFailures ?? new Map()+ }- /** Background CLI jobs must not replace the desktop window's one pooled sink. Each isolated- * client gets its own encrypted session, released immediately instead of retained for reconnect. */+ /** Background CLI jobs (a monitor pane polling `agents_list`, a script) must not replace the+ * desktop window's one pooled sink, and must not see each other's frames either — so each one+ * gets a pool of its OWN, separate from the window's, with its own encrypted session. What used to+ * happen next was the expensive part: the pool was thrown away on detach, so every poll was a fresh+ * relay socket, E2EE handshake AND WebRTC negotiation, terminated a second later (8.5k dials/72 min+ * fleet-wide, 2026-09-22). Now a detached pool goes back on a per-machine shelf with its session+ * lingering, and the next background client for that machine takes it — one dial per machine per+ * linger, however often a pane polls. Concurrent clients still get distinct pools. These pools never+ * negotiate p2p and share one dial-failure record, so a machine that refused is not re-dialled on+ * every tick. */async acquireIsolated(machineId: string, autonomousEnv: string, selectFrame: Frame,sink: LocalClientSink, onClosed: (code: number, reason: string) => void,): Promise<RelaySession> {- const isolated = new RemoteRelayPool(this.auth, this.backendWsBase, this.selfIdentity, this.peers)- const session = await isolated.acquire(machineId, autonomousEnv, selectFrame, {+ const shelf = this.idleIsolated.get(machineId) ?? []+ const pool = shelf.pop() ?? new RemoteRelayPool(this.auth, this.backendWsBase, this.selfIdentity, this.peers,+ { p2p: false, lingerMs: ISOLATED_LINGER_MS, dialCooldownMs: ISOLATED_DIAL_COOLDOWN_MS, dialFailures: this.lastDialFailure })+ const session = await pool.acquire(machineId, autonomousEnv, selectFrame, {...sink,sendFrame: frame => sink.sendFrame(frame.type === 'connected'? { ...frame, payload: { ...framePayload(frame), relayIsolation: true } } : frame),}, onClosed)- return { ...session, detach: () => isolated.invalidate(machineId) }+ let detached = false+ return {+ ...session,+ detach: () => {+ if (detached) return+ detached = true+ session.detach() // the pool's own linger starts here+ const idle = this.idleIsolated.get(machineId) ?? []+ if (idle.length >= ISOLATED_IDLE_MAX) { pool.invalidate(machineId); return }+ idle.push(pool)+ this.idleIsolated.set(machineId, idle)+ },+ }+ }++ /** Test seam: how many warm background sessions are shelved for `machineId`. */+ idleIsolatedCount(machineId: string): number { return this.idleIsolated.get(machineId)?.length ?? 0 }++ /** `invalidate()` for the shelved background sessions — a background client's own `forceReconnect`+ * says the machine behind them restarted, so none of them is worth handing to the next client. */+ invalidateIsolated(machineId: string): void {+ for (const pool of this.idleIsolated.get(machineId) ?? []) pool.invalidate(machineId)+ this.idleIsolated.delete(machineId)}/** Force-drops a pooled entry so the next `acquire()` dials fresh instead of reusing it. For when@@ -298,6 +371,10 @@ export class RemoteRelayPool {}private connect(machineId: string, autonomousEnv: string, selectFrame: Frame): Promise<Entry> {+ const failed = this.lastDialFailure.get(machineId)+ if (failed && this.dialCooldownMs > 0 && Date.now() - failed.at < this.dialCooldownMs) {+ return Promise.reject(failed.error)+ }const attempt = (forceRefresh: boolean): Promise<Entry> => this.dial(machineId, autonomousEnv, selectFrame, forceRefresh)// A stale access token is the single most likely reason the very first select fails (4401 on the// upgrade) — one retry with a freshly-refreshed token is cheap next to surfacing that as a hard@@ -307,6 +384,12 @@ export class RemoteRelayPool {if (err instanceof RelayConnectError && err.closeCode === 4401) return attempt(true)throw err})+ if (this.dialCooldownMs > 0) {+ promise.then(+ () => { this.lastDialFailure.delete(machineId) },+ (error) => { this.lastDialFailure.set(machineId, { at: Date.now(), error }) },+ )+ }this.pending.set(machineId, promise)// `.finally()` re-throws on rejection, producing a SECOND promise distinct from the one returned// below (which callers already await/catch) — left un-caught, every failed dial (e.g. NO_PEER_LINK@@ -396,7 +479,7 @@ export class RemoteRelayPool {// backendSocket, never this relay client), not in the badge, not in p2p_result — and its// absence is indistinguishable from "TURN is configured but ICE preferred direct".// Names only, never the credential itself.- if (entry.p2pPolicy) {+ if (entry.p2pPolicy && this.p2pEnabled) {const p = entry.p2pPolicyconsole.log(`[p2p] policy · machine=${sid(machineId)} stun=${p.stunUrls.length}`+ ` turn=${p.turn ? `${p.turn.urls.length} urls` : 'NONE'} openWait=${p.openWaitMs}ms`)@@ -421,7 +504,9 @@ export class RemoteRelayPool {// the terminal just quietly stayed on the ws relay. The peer-version case is the important// one: a machine whose CLI predates p2p answers no offer, so NO amount of STUN or TURN can// help it. That is a very different problem from "ICE tried and failed".- if (!entry.p2pPolicy) {+ if (!this.p2pEnabled) {+ console.log(`[p2p] off · machine=${sid(machineId)} background session, no terminals — ws relay only`)+ } else if (!entry.p2pPolicy) {console.log(`[p2p] off · machine=${sid(machineId)} backend sent no policy (rollout or kill switch)`)} else if (crypto.terminalP2pVersion !== TERMINAL_P2P_PROTOCOL_VERSION) {console.log(`[p2p] off · machine=${sid(machineId)} peer speaks p2p v${crypto.terminalP2pVersion},`@@ -642,7 +727,7 @@ export class RemoteRelayPool {try { entry.ws.close(1000, 'idle') } catch { /* ignore */ }this.entries.delete(machineId)}- }, LINGER_MS)+ }, this.lingerMs)entry.lingerTimer.unref?.()},}@@ -650,7 +735,9 @@ export class RemoteRelayPool {private startP2p(machineId: string, entry: Entry): void {const policy = entry.p2pPolicy- if (!policy || entry.p2p) return+ // The p2p-off guard lives HERE, not only at the dial-time call: scheduleP2pRetry() re-enters+ // through this same method, and a background pool must stay relay-only through every retry too.+ if (!policy || entry.p2p || !this.p2pEnabled) returnlet wasDirect = falseconst p2p = new TerminalP2pInitiator({policy,diff --git a/cli/src/localWsServer.ts b/cli/src/localWsServer.tsindex e02decc9d..d0a5721a4 100644--- a/cli/src/localWsServer.ts+++ b/cli/src/localWsServer.ts@@ -327,7 +327,10 @@ export function attachLocalWsServer(server: http.Server, options: LocalWsServerO// its pooled entry is suspect (most commonly the relayed machine's own Harness process// restarted, dropping its E2EE session without the transport itself ever closing). Drop it// so this select dials fresh instead of handing back the same dead session again.- if (payload?.forceReconnect === true && payload?.relayIsolation !== true) options.relayPool.invalidate(requestedMachineId)+ if (payload?.forceReconnect === true) {+ if (payload?.relayIsolation === true) options.relayPool.invalidateIsolated(requestedMachineId)+ else options.relayPool.invalidate(requestedMachineId)+ }try {relay = payload?.relayIsolation === true? await options.relayPool.acquireIsolated(requestedMachineId, options.autonomousEnv, frame, sink, close)diff --git a/store/agents/autonomous-grid/test/server.test.mjs b/store/agents/autonomous-grid/test/server.test.mjsindex 11e3e2fb7..b24ed23af 100644--- a/store/agents/autonomous-grid/test/server.test.mjs+++ b/store/agents/autonomous-grid/test/server.test.mjs@@ -22,5 +22,6 @@ test('viewer serves its real app and SSE, blocks mutations and never serves workassert.equal((await fetch(`${base}/.harness/grid/../secret`)).status,404);const blocked=await new Promise(resolve=>{request(base,{headers:{host:'attacker.test'}},res=>{res.resume();resolve(res.statusCode);}).end();});assert.equal(blocked,403);const abort=new AbortController();const events=await fetch(`${base}/events`,{signal:abort.signal});const reader=events.body.getReader();const chunk=new TextDecoder().decode((await reader.read()).value);assert.match(chunk,/event: snapshot/);assert.match(chunk,/Mac Studio/);abort.abort();- fail=true;await new Promise(resolve=>setTimeout(resolve,90));const stale=await (await fetch(`${base}/api/snapshot`)).json();assert.equal(stale.status,'unavailable');assert.ok(stale.nodes.every(n=>n.stale));assert.doesNotMatch(JSON.stringify(stale),/secret token/);+ // No pane is attached any more, so the fleet is read again only when the observation is asked for and stale.+ fail=true;let stale;for(let retry=0;retry<40;retry++){stale=await (await fetch(`${base}/api/snapshot`)).json();if(stale.status==='unavailable')break;await new Promise(resolve=>setTimeout(resolve,30));}assert.equal(stale.status,'unavailable');assert.ok(stale.nodes.every(n=>n.stale));assert.doesNotMatch(JSON.stringify(stale),/secret token/);});diff --git a/store/agents/autonomous-grid/viewer.mjs b/store/agents/autonomous-grid/viewer.mjsindex f71f5a5d1..06ca76d5e 100644--- a/store/agents/autonomous-grid/viewer.mjs+++ b/store/agents/autonomous-grid/viewer.mjs@@ -8,7 +8,20 @@ import { createCollector } from './lib/telemetry.mjs';export function createViewer({ workspace, port = 0, intervalMs = 8000, collect = createCollector(workspace, { intervalMs }), select = gridSelect }) {const clients = new Set();let snapshot = { spec: 1, status: 'connecting', grid: 'Your grid', nodes: [], machines: [], models: [], events: [], operations: [], history: {}, sources: {}, summary: {}, observedAt: null, pollIntervalMs: intervalMs };- let stopped = false, pollTimer, opTimer, heartbeat;+ let stopped = false, pollTimer, opTimer, heartbeat, inFlight = false, lastPollAt = 0;+ // The next tick is booked only while a pane is connected: a viewer nobody is looking at must not keep+ // running the grid CLI every few seconds. A request for the observation still wakes one read if the+ // one on hand has gone stale.+ const schedule = () => {+ clearTimeout(pollTimer); pollTimer = null;+ if (stopped || clients.size === 0) return;+ pollTimer = setTimeout(poll, intervalMs);+ };+ const wake = () => {+ if (stopped || inFlight || Date.now() - lastPollAt < intervalMs) return;+ clearTimeout(pollTimer); pollTimer = null;+ void poll();+ };const headers = { 'cache-control': 'no-store', 'x-content-type-options': 'nosniff', 'referrer-policy': 'no-referrer' };const json = (res, code, value) => { res.writeHead(code, { ...headers, 'content-type': 'application/json; charset=utf-8' }); res.end(JSON.stringify(value)); };const publish = () => {@@ -38,13 +51,17 @@ export function createViewer({ workspace, port = 0, intervalMs = 8000, collect =}if (!['GET', 'HEAD'].includes(req.method)) { res.setHeader('allow', 'GET, HEAD, POST'); json(res, 405, { error: 'This viewer is read-only. Talk to the Grid agent to make changes.' }); return; }if (url.pathname === '/health') { json(res, 200, { ok: true }); return; }- if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); return; }+ if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); wake(); return; }if (url.pathname === '/events') {if (req.method === 'HEAD') { res.writeHead(200, headers); res.end(); return; }if (clients.size >= 32) { json(res, 503, { error: 'Too many viewer connections.' }); return; }res.writeHead(200, { ...headers, 'content-type': 'text/event-stream', connection: 'keep-alive', 'x-accel-buffering': 'no' });res.write(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`);- clients.add(res); res.on('close', () => clients.delete(res)); return;+ clients.add(res);+ res.on('close', () => { clients.delete(res); if (clients.size === 0) { clearTimeout(pollTimer); pollTimer = null; } });+ wake();+ if (!pollTimer && !inFlight) schedule();+ return;}const assets = { '/': ['index.html', 'text/html; charset=utf-8'], '/app.js': ['app.js', 'text/javascript; charset=utf-8'], '/app.css': ['app.css', 'text/css; charset=utf-8'] };if (!assets[url.pathname]) { json(res, 404, { error: 'Not found' }); return; }@@ -73,13 +90,15 @@ export function createViewer({ workspace, port = 0, intervalMs = 8000, collect =}async function poll() {+ inFlight = true;try { snapshot = await collect(); }catch {snapshot = { ...snapshot, status: 'unavailable', nodes: snapshot.nodes.map(n => ({ ...n, stale: true })), sources: { configuration: { ok: false, error: 'Cannot refresh the fleet. Ask the agent to check grid-fleet.json and run fleet refresh.' } } };await atomicJson(join(workspace, '.harness', 'verdict.json'), { spec: 1, ready: false, summary: 'Grid telemetry could not refresh', findings: [{ severity: 'error', kind: 'telemetry', message: snapshot.sources.configuration.error }], updatedAt: now() }).catch(() => {});}+ inFlight = false; lastPollAt = Date.now();if (stopped) return;- publish(); pollTimer = setTimeout(poll, intervalMs);+ publish(); schedule();}async function pollOperations() {const latest = await operations(workspace).catch(() => []);diff --git a/store/agents/harness-monitor/lib/bridge.mjs b/store/agents/harness-monitor/lib/bridge.mjsindex f44b52e89..a35767e6a 100644--- a/store/agents/harness-monitor/lib/bridge.mjs+++ b/store/agents/harness-monitor/lib/bridge.mjs@@ -15,6 +15,7 @@import { randomUUID } from 'node:crypto'import { execFile } from 'node:child_process'+import { homedir } from 'node:os'import { promisify } from 'node:util'const exec = promisify(execFile)@@ -30,12 +31,15 @@ export function bridgeUrl(env = process.env) {return url.href}-/** Every machine this daemon can reach, the current one flagged. `harness` not on PATH is an answer,- * not a crash: the caller falls back to reading this machine's registry directly. */-export async function machines(env = process.env) {+/** Every machine this daemon can reach, the current one flagged, and — when the list could not be+ * read — why. `harness` not on PATH is an answer, not a crash: the caller falls back to reading this+ * machine's registry directly. The command runs from the home folder, never from this package's+ * directory: a package update swaps that directory out from under a running viewer, and a CLI+ * started in a directory that no longer exists fails in a way that reads exactly like a dead daemon. */+export async function machinesReport(env = process.env) {try {- const { stdout } = await exec('harness', ['machines', '--json'], { timeout: 20_000, maxBuffer: 512 * 1024, env })- return stdout.split('\n').filter(Boolean).map((line) => { try { return JSON.parse(line) } catch { return null } })+ const { stdout } = await exec('harness', ['machines', '--json'], { timeout: 20_000, maxBuffer: 512 * 1024, env, cwd: homedir() })+ const machines = stdout.split('\n').filter(Boolean).map((line) => { try { return JSON.parse(line) } catch { return null } }).filter((row) => row && typeof row.machineId === 'string').map((row) => ({machineId: row.machineId,@@ -43,31 +47,62 @@ export async function machines(env = process.env) {current: row.current === true,online: row.status === 'running',}))- } catch { return [] }+ return { machines, error: null }+ } catch (error) {+ const detail = error?.code === 'ENOENT'+ ? 'the `harness` command is not on PATH'+ : `the \`harness machines\` command failed${error?.stderr ? `: ${String(error.stderr).trim().split('\n')[0].slice(0, 200)}` : error?.message ? `: ${String(error.message).slice(0, 200)}` : ''}`+ return { machines: [], error: detail }+ }+}++/** The machine list alone, for callers that only need the rows. */+export async function machines(env = process.env) {+ return (await machinesReport(env)).machines}/**- * One socket, one machine, however many calls the caller needs, then closed.+ * One bridge socket per machine, opened on first use and kept for the next call.+ *+ * Behind a `machine_select` on this bridge, Harness dials its relay to that machine, runs an+ * encryption handshake and — until recently — negotiated a WebRTC channel; opening a socket per poll+ * threw all of that away every few seconds, for every machine, from every pane. So the select happens+ * once, and a poll is one `agents_list` frame on a socket that already exists. A socket that closes,+ * errors, or stops answering is dropped; the next call opens a fresh one.*- * `fn` receives `rpc(type, payload)`. Any failure — no daemon, an unlinked machine, a timeout —- * arrives as an Error whose message is a sentence a person can act on, because these end up in a- * pane header and a CLI's stderr rather than a log nobody reads.+ * Any failure — no daemon, an unlinked machine, a timeout — arrives as an Error whose message is a+ * sentence a person can act on, because these end up in a pane header and a CLI's stderr rather than+ * a log nobody reads.*/-export async function withBridge(machineId, fn, { env = process.env, timeoutMs = 20_000, WebSocketImpl = globalThis.WebSocket } = {}) {- if (!WebSocketImpl) throw new Error('Node 22 or newer is required to reach the Harness bridge.')+const sessions = new Map() // WebSocketImpl -> Map<machineId, session>; keyed by impl so a test's scripted socket is its own world++function sessionsFor(WebSocketImpl) {+ let byMachine = sessions.get(WebSocketImpl)+ if (!byMachine) { byMachine = new Map(); sessions.set(WebSocketImpl, byMachine) }+ return byMachine+}++function openSession(machineId, { env, timeoutMs, WebSocketImpl, forceReconnect = false }) {+ const byMachine = sessionsFor(WebSocketImpl)const socket = new WebSocketImpl(bridgeUrl(env))const pending = new Map()let closed = null+ const session = { socket, pending, ready: null, close: null }const fail = (message) => {closed ??= new Error(message)- for (const [, entry] of pending) entry.reject(closed)+ if (byMachine.get(machineId) === session) byMachine.delete(machineId)+ for (const [, entry] of pending) { clearTimeout(entry.deadline); entry.reject(closed) }pending.clear()+ try { socket.close() } catch { /* already gone */ }}+ session.close = () => fail('The Harness bridge closed the connection.')- const ready = new Promise((resolve, reject) => {- const deadline = setTimeout(() => reject(new Error('The Harness daemon did not answer on the local bridge. Is Harness running?')), Math.min(timeoutMs, 15_000))- socket.addEventListener('open', () => socket.send(JSON.stringify({ type: 'machine_select', payload: { machineId, localProtocolVersion: 1, relayIsolation: true } })))- socket.addEventListener('error', () => { clearTimeout(deadline); reject(new Error('Could not open the local Harness bridge. Start Harness and try again.')) })+ session.ready = new Promise((resolve, reject) => {+ const deadline = setTimeout(() => { const m = 'The Harness daemon did not answer on the local bridge. Is Harness running?'; fail(m); reject(new Error(m)) }, Math.min(timeoutMs, 15_000))+ // `forceReconnect` tells Harness the session it kept for this machine is dead (the machine's own+ // Harness restarted under it) and must be dialled fresh rather than handed back once more.+ socket.addEventListener('open', () => socket.send(JSON.stringify({ type: 'machine_select', payload: { machineId, localProtocolVersion: 1, relayIsolation: true, ...(forceReconnect ? { forceReconnect: true } : {}) } })))+ socket.addEventListener('error', () => { clearTimeout(deadline); const m = 'Could not open the local Harness bridge. Start Harness and try again.'; fail(m); reject(new Error(m)) })socket.addEventListener('close', (event) => {clearTimeout(deadline)const message = event?.code === 4404@@ -92,28 +127,75 @@ export async function withBridge(machineId, fn, { env = process.env, timeoutMs =else entry.resolve(payload)})})+ session.ready.catch(() => {})- const rpc = (type, payload = {}, { callTimeoutMs = timeoutMs } = {}) => new Promise((resolve, reject) => {+ session.rpc = (type, payload = {}, { callTimeoutMs = timeoutMs } = {}) => new Promise((resolve, reject) => {if (closed) { reject(closed); return }const requestId = randomUUID()- const deadline = setTimeout(() => { pending.delete(requestId); reject(new Error(`The daemon did not answer ${type} in time.`)) }, callTimeoutMs)+ const deadline = setTimeout(() => {+ pending.delete(requestId)+ const error = new Error(`The daemon did not answer ${type} in time.`)+ error.timedOut = true+ reject(error)+ }, callTimeoutMs)pending.set(requestId, { type, resolve, reject, deadline })- socket.send(JSON.stringify({ type, payload: { ...payload, requestId } }))+ try { socket.send(JSON.stringify({ type, payload: { ...payload, requestId } })) }+ catch { pending.delete(requestId); clearTimeout(deadline); reject(closed ?? new Error('The Harness bridge closed the connection.')) }})+ byMachine.set(machineId, session)+ return session+}++/** The kept session for `machineId`, opening one if there is none. */+async function bridgeSession(machineId, { env = process.env, timeoutMs = 20_000, WebSocketImpl = globalThis.WebSocket, forceReconnect = false } = {}) {+ if (!WebSocketImpl) throw new Error('Node 22 or newer is required to reach the Harness bridge.')+ const existing = sessionsFor(WebSocketImpl).get(machineId)+ const session = existing ?? openSession(machineId, { env, timeoutMs, WebSocketImpl, forceReconnect })+ await session.ready+ return { session, reused: session === existing }+}++/**+ * One socket, one machine, however many calls the caller needs, then closed — for a one-off job+ * that must not share the kept session (a script, a test). `fn` receives `rpc(type, payload)`.+ */+export async function withBridge(machineId, fn, { env = process.env, timeoutMs = 20_000, WebSocketImpl = globalThis.WebSocket } = {}) {+ if (!WebSocketImpl) throw new Error('Node 22 or newer is required to reach the Harness bridge.')+ // Its own world, never the shared map: a fresh key that nothing else will look up.+ const own = class extends WebSocketImpl {}+ const session = openSession(machineId, { env, timeoutMs, WebSocketImpl: own })try {- await ready- return await fn(rpc)+ await session.ready+ return await fn(session.rpc)} finally {- for (const [, entry] of pending) clearTimeout(entry.deadline)- try { socket.close() } catch { /* already gone */ }+ session.close()+ sessions.delete(own)}}-/** The fleet as the daemon sees it, for one machine. */-export function listAgents(machineId, options) {- return withBridge(machineId, async (rpc) => {- const reply = await rpc('agents_list', {})+/** The fleet as the daemon sees it, for one machine — on the kept session. A kept socket that has+ * gone quiet (the daemon behind it restarted without closing it) is dropped and the read retried+ * ONCE on a fresh one before the machine is reported silent. */+export async function listAgents(machineId, options = {}) {+ const ask = async ({ session }) => {+ const reply = await session.rpc('agents_list', {})return Array.isArray(reply.agents) ? reply.agents : []- }, options)+ }+ const first = await bridgeSession(machineId, options)+ try {+ return await ask(first)+ } catch (error) {+ if (!error?.timedOut) throw error+ first.session.close()+ if (!first.reused) throw error+ const second = await bridgeSession(machineId, { ...options, forceReconnect: true })+ try { return await ask(second) } catch (again) { if (again?.timedOut) second.session.close(); throw again }+ }+}++/** Drop every kept bridge session (the pane is closing). */+export function closeBridges() {+ for (const byMachine of sessions.values()) for (const session of [...byMachine.values()]) session.close()+ sessions.clear()}diff --git a/store/agents/harness-monitor/lib/inventory.mjs b/store/agents/harness-monitor/lib/inventory.mjsindex bdece682d..3e521d065 100644--- a/store/agents/harness-monitor/lib/inventory.mjs+++ b/store/agents/harness-monitor/lib/inventory.mjs@@ -16,7 +16,7 @@ import { existsSync } from 'node:fs'import { homedir } from 'node:os'import { basename, dirname } from 'node:path'import { lastTurns } from './activity.mjs'-import { listAgents, machines } from './bridge.mjs'+import { listAgents, machinesReport } from './bridge.mjs'import { engineProcess, panes, processTable } from './panes.mjs'import { humanIdle } from './policy.mjs'import { readRegistry, registryAsFrames } from './registry.mjs'@@ -157,32 +157,56 @@ export function mergeRows(agents, { paneRows = new Map(), table = { byPid: new M* Failures are per machine and reported, never fatal: a fleet view that refuses to draw because one* linked laptop is asleep is worse than one that says which machine it could not reach.*/-export async function collect({ state = {}, now = Date.now(), includeRemote = true, timeoutMs = 8000, home = homedir(), cache = new Map() } = {}) {- const [paneRows, table, machineList, registry] = await Promise.all([panes(), processTable(), machines(), readRegistry()])+export async function collect({+ state = {}, now = Date.now(), includeRemote = true, timeoutMs = 8000, home = homedir(), cache = new Map(),+ // Remote machines are asked at most this often; between asks their last answer is re-merged against+ // the current clock. A pane ticks every few seconds for the machine it is on — that is local reads —+ // but every remote ask crosses the relay to another computer, so it gets its own, slower clock.+ // `remote` is the holder for that answer, owned by the caller so it survives across ticks.+ remoteIntervalMs = 60_000, remote = null,+} = {}) {+ const [paneRows, table, report, registry] = await Promise.all([panes(), processTable(), machinesReport(), readRegistry()])+ const machineList = report.machinesconst turns = await lastTurns(registry.rows.map((row) => ({ id: row.agentId, transcriptPath: row.transcriptPath })), { cache })const current = machineList.find((m) => m.current) ?? null- const targets = [current, ...(includeRemote ? machineList.filter((m) => !m.current && m.online) : [])].filter(Boolean)+ const remoteMachines = includeRemote ? machineList.filter((m) => !m.current && m.online) : []+ const remoteDue = !remote || remoteIntervalMs <= 0 || now - (remote.at ?? 0) >= remoteIntervalMs+ const targets = [current, ...(remoteDue ? remoteMachines : [])].filter(Boolean)const problems = []let rows = []// No daemon, no `harness` on PATH: fall back to the registry file this machine already has. Seeing- // the fleet must not depend on the app running, and pausing is pure tmux either way.+ // the fleet must not depend on the app running, and pausing is pure tmux either way. Say which of+ // the two it was: a CLI that failed to run is a different repair from a daemon that is not there.if (!targets.length) {- if (!registry.rows.length) problems.push({ machine: 'this machine', error: 'No Harness daemon on the local bridge and no registry to read.' })+ if (report.error) problems.push({ machine: 'this machine', error: `Read from the registry file — ${report.error}, so models, branches and remote machines are missing.` })+ else if (!registry.rows.length) problems.push({ machine: 'this machine', error: 'No Harness daemon on the local bridge and no registry to read.' })else problems.push({ machine: 'this machine', error: 'Read from the registry file — the Harness daemon is not answering, so models, branches and remote machines are missing.' })const rowsOffline = mergeRows(registryAsFrames(registry.rows), { paneRows, table, state, machine: null, local: true, now, home, turns, registry: registry.byId })rowsOffline.sort((a, b) => a.idleMs - b.idleMs || a.name.localeCompare(b.name))return { rows: rowsOffline, machines: machineList, problems, observedAt: now, degraded: true }}+ // What each remote machine last answered, raw, so an off-tick merge still gets a fresh clock.+ const answers = remoteDue ? new Map() : new Map(remote?.answers ?? [])+ const remoteProblems = remoteDue ? [] : [...(remote?.problems ?? [])]await Promise.all(targets.map(async (machine) => {try {const agents = await listAgents(machine.machineId, { timeoutMs })- rows = rows.concat(mergeRows(agents, { paneRows, table, state, machine, local: machine.current === true, now, home, turns, registry: registry.byId }))+ if (machine.current) rows = rows.concat(mergeRows(agents, { paneRows, table, state, machine, local: true, now, home, turns, registry: registry.byId }))+ else answers.set(machine.machineId, agents)} catch (error) {- problems.push({ machine: machine.name, error: error instanceof Error ? error.message : String(error) })+ const problem = { machine: machine.name, error: error instanceof Error ? error.message : String(error) }+ if (machine.current) problems.push(problem)+ else remoteProblems.push(problem)}}))+ for (const machine of remoteMachines) {+ const agents = answers.get(machine.machineId)+ if (agents) rows = rows.concat(mergeRows(agents, { paneRows, table, state, machine, local: false, now, home, turns, registry: registry.byId }))+ }+ problems.push(...remoteProblems)+ if (remote && remoteDue) { remote.at = now; remote.answers = answers; remote.problems = remoteProblems }rows.sort((a, b) => a.idleMs - b.idleMs || a.name.localeCompare(b.name))return { rows, machines: machineList, problems, observedAt: now }diff --git a/store/agents/harness-monitor/test/bridge.test.mjs b/store/agents/harness-monitor/test/bridge.test.mjsnew file mode 100644index 000000000..9d1a41310--- /dev/null+++ b/store/agents/harness-monitor/test/bridge.test.mjs@@ -0,0 +1,85 @@+import assert from 'node:assert/strict'+import { test } from 'node:test'+import { closeBridges, listAgents, withBridge } from '../lib/bridge.mjs'++/** A scripted bridge socket class: selects on open, answers agents_list, counts its instances. */+function scripted({ answer = true, closeAfterAnswer = false, muteAfter = Infinity } = {}) {+ const made = []+ const sent = []+ let answered = 0+ const Impl = class {+ constructor() {+ made.push(this); this.listeners = {}+ queueMicrotask(() => {+ this.fire('open', {})+ this.fire('message', { data: JSON.stringify({ type: 'connected', payload: { machineId: 'm', relayIsolation: true } }) })+ })+ }+ addEventListener(type, handler) { (this.listeners[type] ??= []).push(handler) }+ fire(type, event) { for (const handler of this.listeners[type] ?? []) handler(event) }+ send(raw) {+ const frame = JSON.parse(raw)+ sent.push(frame)+ if (frame.type !== 'agents_list' || !answer || answered >= muteAfter) return+ answered += 1+ this.fire('message', { data: JSON.stringify({ type: 'agents_list_result', payload: { requestId: frame.payload.requestId, agents: [{ id: `from-${made.indexOf(this)}` }] } }) })+ if (closeAfterAnswer) this.fire('close', { code: 1000 })+ }+ close() {}+ }+ return { Impl, made, sent }+}++test('a second read on the same machine reuses the kept socket: one select, two asks', async () => {+ const { Impl, made, sent } = scripted()+ assert.deepEqual(await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 500 }), [{ id: 'from-0' }])+ assert.deepEqual(await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 500 }), [{ id: 'from-0' }])+ assert.equal(made.length, 1)+ assert.deepEqual(sent.map((f) => f.type), ['machine_select', 'agents_list', 'agents_list'])+ assert.equal(sent[1].payload.requestId === sent[2].payload.requestId, false)+ closeBridges()+})++test('a kept socket that closed is replaced on the next read', async () => {+ const { Impl, made, sent } = scripted({ closeAfterAnswer: true })+ await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 500 })+ await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 500 })+ assert.equal(made.length, 2)+ assert.deepEqual(sent.map((f) => f.type), ['machine_select', 'agents_list', 'machine_select', 'agents_list'])+ closeBridges()+})++test('a kept socket that goes quiet is dropped and the read retried once on a fresh one', async () => {+ const { Impl, made, sent } = scripted({ muteAfter: 1 })+ assert.deepEqual(await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 80 }), [{ id: 'from-0' }])+ // The first socket answers nothing more; the retry lands on a new socket, which is fresh and answers.+ await assert.rejects(listAgents('m', { WebSocketImpl: Impl, timeoutMs: 80 }), /did not answer agents_list/)+ assert.equal(made.length, 2, 'one retry, on a fresh socket, before giving up')+ const selects = sent.filter((f) => f.type === 'machine_select').map((f) => f.payload.forceReconnect)+ assert.deepEqual(selects, [undefined, true], 'the retry tells Harness its kept session is dead')+ closeBridges()+})++test('an unlinked machine is named as such, and is not kept', async () => {+ const Impl = class {+ constructor() { this.listeners = {}; queueMicrotask(() => this.fire('close', { code: 4404 })) }+ addEventListener(type, handler) { (this.listeners[type] ??= []).push(handler) }+ fire(type, event) { for (const handler of this.listeners[type] ?? []) handler(event) }+ send() {} close() {}+ }+ await assert.rejects(listAgents('m', { WebSocketImpl: Impl, timeoutMs: 200 }), /Link this machine/)+ closeBridges()+})++test('withBridge is a one-off: its socket is closed after the call and never shared', async () => {+ const { Impl, made, sent } = scripted()+ const closedSockets = []+ Impl.prototype.close = function () { closedSockets.push(this) }+ const agents = await withBridge('m', async (rpc) => (await rpc('agents_list', {})).agents, { WebSocketImpl: Impl, timeoutMs: 500 })+ assert.deepEqual(agents, [{ id: 'from-0' }])+ assert.equal(closedSockets.length, 1)+ await listAgents('m', { WebSocketImpl: Impl, timeoutMs: 500 })+ assert.equal(made.length, 2, 'the kept read did not reuse the one-off socket')+ assert.deepEqual(sent.map((f) => f.type), ['machine_select', 'agents_list', 'machine_select', 'agents_list'])+ closeBridges()+})diff --git a/store/agents/harness-monitor/test/server.test.mjs b/store/agents/harness-monitor/test/server.test.mjsindex 2d5fe036d..c6b57acd0 100644--- a/store/agents/harness-monitor/test/server.test.mjs+++ b/store/agents/harness-monitor/test/server.test.mjs@@ -160,3 +160,35 @@ async function post(base, token, path, payload) {})return response.json()}++test('the fleet is read on a tick only while a pane is connected; remote machines on their own clock', async (t) => {+ let reads = 0+ const calls = []+ const { viewer, base } = await serve(undefined, {+ intervalMs: 40,+ remoteIntervalMs: 150,+ collect: async (options) => { reads += 1; calls.push(options.remoteIntervalMs); return { rows: [], machines: [], problems: [], observedAt: Date.now() } },+ })+ t.after(() => viewer.close())+ const sleep = (ms) => new Promise((done) => setTimeout(done, ms))++ await sleep(200)+ assert.equal(reads, 1, 'the start-up read, and nothing more without a pane')++ const response = await fetch(`${base}/events`)+ const reader = response.body.getReader()+ await reader.read()+ await sleep(220)+ assert.ok(reads >= 3, `a pane keeps the fleet read on a tick (${reads} reads)`)+ assert.ok(calls.slice(1).every((v) => v === 150), 'ticks hand the collector the remote clock, not a forced remote read')++ await reader.cancel()+ await sleep(60)+ const atLeave = reads+ await sleep(200)+ assert.ok(reads - atLeave <= 1, `reads stop once the last pane leaves (${reads - atLeave} more)`)++ // Refresh is the person asking: it reads now and asks the remote machines too.+ await fetch(`${base}/api/refresh`, { method: 'POST', headers: { 'content-type': 'application/json', 'x-hps-token': viewer.token }, body: '{}' })+ assert.equal(calls.at(-1), 0, 'a refresh forces the remote read')+})diff --git a/store/agents/harness-monitor/viewer.mjs b/store/agents/harness-monitor/viewer.mjsindex d6201c778..6085a0549 100644--- a/store/agents/harness-monitor/viewer.mjs+++ b/store/agents/harness-monitor/viewer.mjs@@ -21,6 +21,7 @@ import { homedir } from 'node:os'import { dirname, join, resolve } from 'node:path'import { fileURLToPath, pathToFileURL } from 'node:url'import { pause, resume } from './lib/actions.mjs'+import { closeBridges } from './lib/bridge.mjs'import { collect as collectFleet, summarize, tilde } from './lib/inventory.mjs'import { capture, looksBlocked } from './lib/panes.mjs'import { DEFAULT_POLICY, decide, normalizePolicy } from './lib/policy.mjs'@@ -34,10 +35,13 @@ const VERBS = new Set(['pause', 'resume', 'pin', 'unpin'])* is about to act on, which are the only rows where the answer changes anything. */const BLOCKED_SCAN_LIMIT = 24-export function createViewer({ workspace, port = 0, intervalMs = 4000, now = () => Date.now(), collect = collectFleet, scan = capture, verbs = { pause, resume } }) {+export function createViewer({ workspace, port = 0, intervalMs = 4000, remoteIntervalMs = 60_000, now = () => Date.now(), collect = collectFleet, scan = capture, verbs = { pause, resume } }) {const token = randomBytes(24).toString('base64url')const clients = new Set()const cache = new Map()+ // The remote machines' last answers (lib/inventory.mjs collect): they are asked every+ // `remoteIntervalMs`, the local machine every `intervalMs`, and only while a pane is connected.+ const remote = { at: 0, answers: new Map(), problems: [] }let snapshot = { spec: 1, status: 'starting', rows: [], summary: null, policy: DEFAULT_POLICY, plan: [], totals: null, problems: [], log: [], observedAt: null, intervalMs }let stopped = false, timer, heartbeat, polling = null// Resolves after the first observation lands. The server starts listening before it, so the pane draws@@ -57,14 +61,14 @@ export function createViewer({ workspace, port = 0, intervalMs = 4000, now = ()}/** One observation: the fleet, what the policy would do to it, and the pane header's verdict. */- async function observe() {+ async function observe({ forceRemote = false } = {}) {const state = await readState(workspace).catch(() => null)if (!state) {snapshot = { ...snapshot, status: 'unreadable', problems: [{ machine: 'monitor.json', error: 'The policy file could not be read. Ask the agent to check monitor.json.' }] }return}const policy = normalizePolicy(state.policy, { home: homedir() })- const { rows, problems, degraded } = await collect({ state, now: now(), includeRemote: true, cache })+ const { rows, problems, degraded } = await collect({ state, now: now(), includeRemote: true, cache, remote, remoteIntervalMs: forceRemote ? 0 : remoteIntervalMs })// Look for an open prompt only where it would change a decision, newest candidates first.const candidates = rows@@ -97,17 +101,33 @@ export function createViewer({ workspace, port = 0, intervalMs = 4000, now = ()await writeVerdict(workspace, { summary, rows, plan: plan.entries, problems }).catch(() => {})}- async function poll({ immediate = false } = {}) {+ let lastPollAt = 0+ // The next tick is booked only while a pane is connected: a viewer nobody is looking at — a tab in+ // the background, a pane the person closed but whose process outlived the daemon — must not keep+ // every machine on the account answering it.+ const schedule = () => {+ clearTimeout(timer); timer = null+ if (stopped || clients.size === 0) return+ timer = setTimeout(() => { void poll() }, intervalMs)+ }+ // Someone wants the observation now: read again if the one we have is older than a tick.+ const wake = () => {+ if (stopped || polling || now() - lastPollAt < intervalMs) return+ clearTimeout(timer); timer = null+ void poll()+ }++ async function poll({ immediate = false, forceRemote = false } = {}) {if (polling) return pollingpolling = (async () => {- try { await observe() } catch (error) {+ try { await observe({ forceRemote }) } catch (error) {snapshot = { ...snapshot, status: 'unavailable', problems: [{ machine: 'this machine', error: error instanceof Error ? error.message : String(error) }] }}if (!stopped) publish()observed()})()- try { await polling } finally { polling = null }- if (!stopped && !immediate) { clearTimeout(timer); timer = setTimeout(poll, intervalMs) }+ try { await polling } finally { polling = null; lastPollAt = now() }+ if (!stopped && !immediate) schedule()}/** The write surface. One verb, up to 64 rows, and a receipt per row — the same library call the CLI@@ -170,19 +190,24 @@ export function createViewer({ workspace, port = 0, intervalMs = 4000, now = ()let payload; try { payload = JSON.parse(body || '{}') } catch { json(res, 400, { error: 'Send JSON.' }); return }if (url.pathname === '/api/act') { json(res, 200, await act(payload)); return }if (url.pathname === '/api/policy') { json(res, 200, await savePolicy(payload.policy ?? {})); return }- if (url.pathname === '/api/refresh') { await poll({ immediate: true }); json(res, 200, { ok: true }); return }+ if (url.pathname === '/api/refresh') { await poll({ immediate: true, forceRemote: true }); json(res, 200, { ok: true }); return }json(res, 404, { error: 'Not found' }); return}if (!['GET', 'HEAD'].includes(req.method)) { res.setHeader('allow', 'GET, HEAD, POST'); json(res, 405, { error: 'Not allowed.' }); return }if (url.pathname === '/health') { json(res, 200, { ok: true }); return }- if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); return }+ if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); wake(); return }if (url.pathname === '/events') {if (req.method === 'HEAD') { res.writeHead(200, headers); res.end(); return }if (clients.size >= 16) { json(res, 503, { error: 'Too many viewer connections.' }); return }res.writeHead(200, { ...headers, 'content-type': 'text/event-stream', connection: 'keep-alive', 'x-accel-buffering': 'no' })res.write(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`)- clients.add(res); res.on('close', () => clients.delete(res)); return+ clients.add(res)+ res.on('close', () => { clients.delete(res); if (clients.size === 0) { clearTimeout(timer); timer = null } })+ // A pane just opened: catch up if the observation is stale, and start ticking either way.+ wake()+ if (!timer && !polling) schedule()+ return}const assets = {@@ -225,6 +250,8 @@ export function createViewer({ workspace, port = 0, intervalMs = 4000, now = ()}),close: async () => {stopped = true; clearTimeout(timer); clearInterval(heartbeat)+ await polling?.catch(() => {})+ closeBridges()for (const client of clients) client.end()server.closeAllConnections()await new Promise((done) => server.close(done))diff --git a/store/agents/machine-monitor/lib/daemon.mjs b/store/agents/machine-monitor/lib/daemon.mjsindex 1df37d87f..8f4f506c8 100644--- a/store/agents/machine-monitor/lib/daemon.mjs+++ b/store/agents/machine-monitor/lib/daemon.mjs@@ -128,49 +128,133 @@ export async function remotePasswordSet(env = process.env) {* `relayIsolation` asks for a session of this package's own, so polling a machine never disturbs the* window the person is working in. Close code 4404 is the one answer that is not a failure: that* machine is not linked from here yet, which is a thing to offer, not an error to report.+ *+ * ONE bridge socket per machine, kept open between polls. A poll used to open a socket, select the+ * machine, ask, and close — and behind that select Harness dialled its relay, ran an encryption+ * handshake and negotiated a WebRTC channel, all thrown away a second later, for every machine, every+ * tick (thousands of relay dials an hour across a few panes, 2026-09-22). Now the select happens once;+ * a poll is one `agents_list` frame on a socket that is already there. A socket that closes, or that+ * stops answering, is dropped and the next poll opens a fresh one.*/-export async function listHarnesses(machineId, { timeoutMs = 20_000, env = process.env, WebSocketImpl = globalThis.WebSocket } = {}) {- if (!WebSocketImpl) return { ok: false, error: 'Node 22 or newer is required to read a machine roster.' };+const sessions = new Map(); // WebSocketImpl -> Map<machineId, session>; keyed by impl so a test's scripted socket is its own world++function sessionsFor(WebSocketImpl) {+ let byMachine = sessions.get(WebSocketImpl);+ if (!byMachine) { byMachine = new Map(); sessions.set(WebSocketImpl, byMachine); }+ return byMachine;+}++/** Open the bridge to `machineId`: resolves once Harness has selected it, rejects with the same+ * answers `listHarnesses` used to give when it cannot. The session stays in the map until its socket+ * closes or errors, whichever comes first; every waiter still on it hears that as a failure. */+function openSession(machineId, { env, WebSocketImpl, timeoutMs, forceReconnect = false }) {+ const byMachine = sessionsFor(WebSocketImpl);let socket;try { socket = new WebSocketImpl(bridgeUrl(env)); }- catch { return { ok: false, error: DOWN }; }- return await new Promise(resolve => {- let done = false, deadline, heartbeat;- const requestId = randomUUID();- const finish = value => {- if (done) return;- done = true;- clearTimeout(deadline); clearInterval(heartbeat);- try { socket.close(); } catch { /* already gone */ }- resolve(value);- };- deadline = setTimeout(() => finish({ ok: false, error: 'This machine did not answer in time.' }), timeoutMs);+ catch { return Promise.reject({ ok: false, error: DOWN }); }+ const session = { socket, waiters: new Map(), heartbeat: null };+ const fail = (answer) => {+ if (byMachine.get(machineId) === session) byMachine.delete(machineId);+ clearInterval(session.heartbeat);+ for (const waiter of session.waiters.values()) waiter(answer);+ session.waiters.clear();+ try { socket.close(); } catch { /* already gone */ }+ };+ session.close = () => fail({ ok: false, error: 'The connection to this machine closed before it answered.' });+ const ready = new Promise((resolve, reject) => {+ let settled = false;+ const settle = (fn, value) => { if (settled) return; settled = true; clearTimeout(deadline); fn(value); };+ const deadline = setTimeout(() => { settle(reject, { ok: false, error: 'This machine did not answer in time.' }); fail({ ok: false, error: 'This machine did not answer in time.' }); }, timeoutMs);+ // `forceReconnect` tells Harness the session it kept for this machine is dead (the machine's own+ // Harness restarted under it) and must be dialled fresh rather than handed back once more.socket.addEventListener('open', () => socket.send(JSON.stringify({type: 'machine_select',- payload: { machineId, localProtocolVersion: 1, relayIsolation: true },+ payload: { machineId, localProtocolVersion: 1, relayIsolation: true, ...(forceReconnect ? { forceReconnect: true } : {}) },})));- socket.addEventListener('error', () => finish({ ok: false, error: DOWN }));+ socket.addEventListener('error', () => { settle(reject, { ok: false, error: DOWN }); fail({ ok: false, error: DOWN }); });socket.addEventListener('close', event => {- if (event.code === 4404) finish({ ok: false, needsLink: true, error: 'This machine is not linked from this computer yet.' });- else finish({ ok: false, error: event.code === 4403 ? 'Harness would not connect to this machine.' : 'The connection to this machine closed before it answered.' });+ const answer = event.code === 4404+ ? { ok: false, needsLink: true, error: 'This machine is not linked from this computer yet.' }+ : { ok: false, error: event.code === 4403 ? 'Harness would not connect to this machine.' : 'The connection to this machine closed before it answered.' };+ settle(reject, answer); fail(answer);});socket.addEventListener('message', event => {if (typeof event.data !== 'string' || event.data.length > 8 * 1024 * 1024) return;let frame;try { frame = JSON.parse(event.data); } catch { return; }if (frame?.type === 'connected') {- heartbeat = setInterval(() => { try { socket.send(JSON.stringify({ type: 'ping', payload: {} })); } catch { /* closing */ } }, 15_000);- // The id rides in the payload — that is where the daemon reads it, and the reply carries it back.- socket.send(JSON.stringify({ type: 'agents_list', payload: { requestId } }));+ session.heartbeat = setInterval(() => { try { socket.send(JSON.stringify({ type: 'ping', payload: {} })); } catch { /* closing */ } }, 15_000);+ session.heartbeat.unref?.();+ settle(resolve);return;}- if (frame?.type === 'agents_list_result' && frame.payload?.requestId === requestId) {- finish({ ok: true, harnesses: Array.isArray(frame.payload.agents) ? frame.payload.agents : [] });+ if (frame?.type === 'agents_list_result') {+ const waiter = session.waiters.get(frame.payload?.requestId);+ if (!waiter) return;+ session.waiters.delete(frame.payload.requestId);+ waiter({ ok: true, harnesses: Array.isArray(frame.payload.agents) ? frame.payload.agents : [] });return;}if (['error', 'connection_error', 'local_protocol_error', 'machine_link_required'].includes(frame?.type)) {- finish({ ok: false, needsLink: frame.type === 'machine_link_required', error: 'This machine is unavailable, or needs linking from this computer.' });+ const answer = { ok: false, needsLink: frame.type === 'machine_link_required', error: 'This machine is unavailable, or needs linking from this computer.' };+ settle(reject, answer); fail(answer);}});});+ session.ready = ready;+ byMachine.set(machineId, session);+ return ready.then(() => session);+}++/** One roster read on an open session; the whole call is bounded by `timeoutMs`. */+function ask(session, timeoutMs) {+ return new Promise(resolve => {+ const requestId = randomUUID();+ const deadline = setTimeout(() => {+ if (!session.waiters.delete(requestId)) return;+ resolve({ ok: false, timedOut: true, error: 'This machine did not answer in time.' });+ }, timeoutMs);+ session.waiters.set(requestId, answer => { clearTimeout(deadline); resolve(answer); });+ // The id rides in the payload — that is where the daemon reads it, and the reply carries it back.+ try { session.socket.send(JSON.stringify({ type: 'agents_list', payload: { requestId } })); }+ catch { session.waiters.delete(requestId); clearTimeout(deadline); resolve({ ok: false, error: 'The connection to this machine closed before it answered.' }); }+ });+}++export async function listHarnesses(machineId, { timeoutMs = 20_000, env = process.env, WebSocketImpl = globalThis.WebSocket } = {}) {+ if (!WebSocketImpl) return { ok: false, error: 'Node 22 or newer is required to read a machine roster.' };+ const byMachine = sessionsFor(WebSocketImpl);+ const existing = byMachine.get(machineId);+ let session;+ try {+ session = existing+ ? await existing.ready.then(() => existing)+ : await openSession(machineId, { env, WebSocketImpl, timeoutMs });+ } catch (answer) {+ return answer?.ok === false ? answer : { ok: false, error: DOWN };+ }+ const reused = session === existing;+ let result = await ask(session, timeoutMs);+ if (result.timedOut) {+ // A socket that is open but no longer answers is the daemon's "transport alive, session dead"+ // case: never keep it. On a REUSED one, try ONCE more on a fresh socket before reporting the+ // machine silent; a fresh one that stayed silent has already had its chance.+ session.close();+ if (reused) {+ try { session = await openSession(machineId, { env, WebSocketImpl, timeoutMs, forceReconnect: true }); }+ catch (answer) { return answer?.ok === false ? answer : { ok: false, error: DOWN }; }+ result = await ask(session, timeoutMs);+ if (result.timedOut) session.close();+ }+ delete result.timedOut;+ }+ return result;+}++/** Drop every bridge session (the pane is closing). */+export function closeSessions() {+ for (const byMachine of sessions.values()) {+ for (const session of [...byMachine.values()]) session.close();+ }+ sessions.clear();}diff --git a/store/agents/machine-monitor/test/daemon.test.mjs b/store/agents/machine-monitor/test/daemon.test.mjsindex 76ff9b618..150851827 100644--- a/store/agents/machine-monitor/test/daemon.test.mjs+++ b/store/agents/machine-monitor/test/daemon.test.mjs@@ -1,6 +1,6 @@import assert from 'node:assert/strict';import test from 'node:test';-import { bridgeUrl, daemonBase, listHarnesses } from '../lib/daemon.mjs';+import { bridgeUrl, closeSessions, daemonBase, listHarnesses } from '../lib/daemon.mjs';test('the daemon address must be loopback http, and the bridge follows it', () => {assert.equal(daemonBase({}).href, 'http://127.0.0.1:18473/');@@ -83,3 +83,83 @@ test('without a WebSocket there is an answer, not a crash', async () => {assert.equal(result.ok, false);assert.match(result.error, /Node 22/);});++/** A scripted socket class that also counts its instances and can be closed from the test. */+function scriptedFactory({ answer = true, closeAfterAnswer = false } = {}) {+ const made = [];+ const sent = [];+ const Impl = class {+ constructor() {+ made.push(this);+ this.listeners = {};+ queueMicrotask(() => {+ this.fire('open', {});+ this.fire('message', { data: JSON.stringify({ type: 'connected', payload: { machineId: 'a'.repeat(32), relayIsolation: true } }) });+ });+ }+ addEventListener(type, handler) { (this.listeners[type] ??= []).push(handler); }+ fire(type, event) { for (const handler of this.listeners[type] ?? []) handler(event); }+ send(raw) {+ const frame = JSON.parse(raw);+ sent.push(frame);+ if (frame.type === 'agents_list' && answer) {+ this.fire('message', { data: JSON.stringify({ type: 'agents_list_result', payload: { requestId: frame.payload.requestId, agents: [{ id: 'h' + made.length }] } }) });+ if (closeAfterAnswer) this.fire('close', { code: 1000 });+ }+ }+ close() {}+ };+ return { Impl, made, sent };+}++test('a second read on the same machine reuses the bridge socket: one select, two asks', async () => {+ const { Impl, made, sent } = scriptedFactory();+ const first = await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 500 });+ const second = await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 500 });+ assert.equal(first.ok, true); assert.equal(second.ok, true);+ assert.equal(made.length, 1, 'no second socket');+ assert.deepEqual(sent.map(f => f.type), ['machine_select', 'agents_list', 'agents_list']);+ assert.notEqual(sent[1].payload.requestId, sent[2].payload.requestId);+ closeSessions();+});++test('a socket that closed is replaced on the next read', async () => {+ const { Impl, made, sent } = scriptedFactory({ closeAfterAnswer: true });+ assert.equal((await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 500 })).ok, true);+ assert.equal((await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 500 })).ok, true);+ assert.equal(made.length, 2, 'the close was noticed and a fresh socket opened');+ assert.deepEqual(sent.map(f => f.type), ['machine_select', 'agents_list', 'machine_select', 'agents_list']);+ closeSessions();+});++test('a reused socket that stops answering is dropped and the read retried once on a fresh one', async () => {+ let answering = true;+ const made = [];+ const selects = [];+ const Impl = class {+ constructor() {+ made.push(this); this.listeners = {};+ queueMicrotask(() => { this.fire('open', {}); this.fire('message', { data: JSON.stringify({ type: 'connected', payload: {} }) }); });+ }+ addEventListener(type, handler) { (this.listeners[type] ??= []).push(handler); }+ fire(type, event) { for (const handler of this.listeners[type] ?? []) handler(event); }+ send(raw) {+ const frame = JSON.parse(raw);+ if (frame.type === 'machine_select') selects.push(frame.payload);+ if (frame.type !== 'agents_list') return;+ // The first socket answers once, then goes mute; every later socket answers.+ if (made.indexOf(this) === 0 && !answering) return;+ this.fire('message', { data: JSON.stringify({ type: 'agents_list_result', payload: { requestId: frame.payload.requestId, agents: [{ id: 'from-' + made.indexOf(this) }] } }) });+ }+ close() {}+ };+ assert.deepEqual((await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 100 })).harnesses, [{ id: 'from-0' }]);+ answering = false;+ const result = await listHarnesses('a'.repeat(32), { WebSocketImpl: Impl, timeoutMs: 100 });+ assert.equal(result.ok, true);+ assert.deepEqual(result.harnesses, [{ id: 'from-1' }], 'answered by the replacement socket');+ assert.equal(made.length, 2);+ assert.equal(selects[0].forceReconnect, undefined);+ assert.equal(selects[1].forceReconnect, true, 'the retry tells Harness its kept session is dead');+ closeSessions();+});diff --git a/store/agents/machine-monitor/test/server.test.mjs b/store/agents/machine-monitor/test/server.test.mjsindex 872ffd3f9..a3fd4fa1b 100644--- a/store/agents/machine-monitor/test/server.test.mjs+++ b/store/agents/machine-monitor/test/server.test.mjs@@ -137,3 +137,34 @@ test('the event stream opens with the current observation', async () => {assert.match(text, /"status":"live"/);await reader.cancel();});++test('the fleet is read on a tick only while a pane is connected; an unattended viewer reads once', async () => {+ let reads = 0;+ const workspace = await mkdtemp(join(tmpdir(), 'machines-server-'));+ const viewer = createViewer({ workspace, port: 0, intervalMs: 40, collect: async () => { reads += 1; return SNAPSHOT; } });+ const port = await viewer.start();+ test.after(async () => { await viewer.close(); await rm(workspace, { recursive: true, force: true }); });+ const sleep = ms => new Promise(done => setTimeout(done, ms));++ await sleep(200);+ assert.equal(reads, 1, 'the start-up read, and nothing more without a pane');++ const response = await fetch(`http://127.0.0.1:${port}/events`);+ const reader = response.body.getReader();+ await reader.read();+ await sleep(220);+ const withPane = reads;+ assert.ok(withPane >= 3, `a pane keeps the fleet read on a tick (${withPane} reads)`);++ await reader.cancel();+ await sleep(60);+ const atLeave = reads;+ await sleep(200);+ assert.ok(reads - atLeave <= 1, `reads stop once the last pane leaves (${reads - atLeave} more)`);++ // A one-off request for the observation wakes exactly one read when it has gone stale.+ const before = reads;+ await fetch(`http://127.0.0.1:${port}/api/snapshot`);+ await sleep(60);+ assert.equal(reads, before + 1);+});diff --git a/store/agents/machine-monitor/viewer.mjs b/store/agents/machine-monitor/viewer.mjsindex 55eab0d99..ad9429886 100644--- a/store/agents/machine-monitor/viewer.mjs+++ b/store/agents/machine-monitor/viewer.mjs@@ -16,23 +16,46 @@ import { readFile } from 'node:fs/promises';import { join, resolve } from 'node:path';import { pathToFileURL } from 'node:url';import { atomicJson, operations, PACKAGE, recordOperation, stateDir, writeVerdict } from './lib/fleet.mjs';+import { closeSessions } from './lib/daemon.mjs';import { linkMachine } from './lib/ops.mjs';import { createCollector } from './lib/snapshot.mjs';+/**+ * How often the fleet is re-read while a pane is looking at it. Every remote machine on the account+ * is asked over Harness's bridge each time, so this is the one number that decides what a pane costs+ * the rest of the fleet; a minute is fast enough for a map, and nothing is read at all while no pane+ * is open (a request for the observation still wakes one read if it has gone stale).+ */+const POLL_INTERVAL_MS = 60_000;+const EMPTY = {- spec: 1, status: 'connecting', message: null, observedAt: null, pollIntervalMs: 15_000,+ spec: 1, status: 'connecting', message: null, observedAt: null, pollIntervalMs: POLL_INTERVAL_MS,account: null, localMachineId: null, thisComputer: null,machines: [], projects: [], operations: [], totals: null,summary: { machines: 0, online: 0, needsLink: 0, harnesses: 0, open: 0, projects: 0, engines: {} },sources: {},};-export function createViewer({ workspace, port = 0, intervalMs = 15_000, collect = createCollector(workspace, { intervalMs }) }) {+export function createViewer({ workspace, port = 0, intervalMs = POLL_INTERVAL_MS, collect = createCollector(workspace, { intervalMs }) }) {const clients = new Set();let snapshot = { ...EMPTY, pollIntervalMs: intervalMs };let stopped = false, pollTimer, opTimer, heartbeat, linking = false;// The poll in flight, so close() can wait for it instead of leaving a write racing the exit.- let polling = null;+ let polling = null, inFlight = false, lastPollAt = 0;+ // The next tick is booked only while a pane is connected: an unattended viewer (a tab in the+ // background, a pane the person closed but whose process outlived the daemon) must not keep every+ // machine on the account answering it.+ const schedule = () => {+ clearTimeout(pollTimer); pollTimer = null;+ if (stopped || clients.size === 0) return;+ pollTimer = setTimeout(() => { polling = poll(); }, intervalMs);+ };+ // Someone wants the observation now: read again if the one we have is older than a tick.+ const wake = () => {+ if (stopped || inFlight || Date.now() - lastPollAt < intervalMs) return;+ clearTimeout(pollTimer); pollTimer = null;+ polling = poll();+ };const headers = { 'cache-control': 'no-store', 'x-content-type-options': 'nosniff', 'referrer-policy': 'no-referrer' };const json = (res, code, value) => { res.writeHead(code, { ...headers, 'content-type': 'application/json; charset=utf-8' }); res.end(JSON.stringify(value)); };const publish = () => {@@ -74,8 +97,8 @@ export function createViewer({ workspace, port = 0, intervalMs = 15_000, collect});if (!result.ok) { json(res, 400, { error: result.error }); return; }json(res, 200, { ok: true });- clearTimeout(pollTimer);- polling = poll();+ clearTimeout(pollTimer); pollTimer = null;+ if (!inFlight) polling = poll();} finally { linking = false; }return;}@@ -85,13 +108,17 @@ export function createViewer({ workspace, port = 0, intervalMs = 15_000, collectreturn;}if (url.pathname === '/health') { json(res, 200, { ok: true }); return; }- if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); return; }+ if (url.pathname === '/api/snapshot') { json(res, 200, snapshot); wake(); return; }if (url.pathname === '/events') {if (req.method === 'HEAD') { res.writeHead(200, headers); res.end(); return; }if (clients.size >= 32) { json(res, 503, { error: 'Too many viewer connections.' }); return; }res.writeHead(200, { ...headers, 'content-type': 'text/event-stream', connection: 'keep-alive', 'x-accel-buffering': 'no' });res.write(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`);- clients.add(res); res.on('close', () => clients.delete(res));+ clients.add(res);+ res.on('close', () => { clients.delete(res); if (clients.size === 0) { clearTimeout(pollTimer); pollTimer = null; } });+ // A pane just opened: catch up if the observation is stale, and start ticking either way.+ wake();+ if (!pollTimer && !inFlight) schedule();return;}const assets = {@@ -113,6 +140,7 @@ export function createViewer({ workspace, port = 0, intervalMs = 15_000, collect});async function poll() {+ inFlight = true;try {const observed = await collect();// A poll that lands after close() has nothing to publish: the workspace may already be gone,@@ -132,9 +160,11 @@ export function createViewer({ workspace, port = 0, intervalMs = 15_000, collect};await writeVerdict(workspace, snapshot).catch(() => {});}+ inFlight = false;+ lastPollAt = Date.now();if (stopped) return;publish();- pollTimer = setTimeout(() => { polling = poll(); }, intervalMs);+ schedule();}async function pollOperations() {@@ -161,6 +191,7 @@ export function createViewer({ workspace, port = 0, intervalMs = 15_000, collectstopped = true;clearTimeout(pollTimer); clearTimeout(opTimer); clearInterval(heartbeat);await polling?.catch(() => {});+ closeSessions();for (const client of clients) client.end();server.closeAllConnections();await new Promise(done => server.close(done));