Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(store): Harness Monitor asks an unreachable machine later and later (#1725)

0xken150b31e5ac2ad0
diff --git a/store/agents/harness-monitor/lib/inventory.mjs b/store/agents/harness-monitor/lib/inventory.mjs
index e061cf758..f12cbcc87 100644
--- a/store/agents/harness-monitor/lib/inventory.mjs
+++ b/store/agents/harness-monitor/lib/inventory.mjs
@@ -86,6 +86,9 @@ export function mergeRows(agents, { state = {}, machine = null, local = true, no
})
}
+const RETRY_MS = 15_000
+const RETRY_MAX_MS = 5 * 60_000
+
/** Cache is scoped to this viewer, never an account-global singleton. Offline machines retain rows. */
export async function collect({
state = {}, now = Date.now(), includeRemote = true, timeoutMs = 8000, home = homedir(),
@@ -98,20 +101,31 @@ export async function collect({
const answers = remote.answers ??= new Map()
const failed = remote.failed ??= new Set()
const previouslyOnline = remote.online ??= new Set()
+ const retryAt = remote.retryAt ??= new Map()
const due = remoteIntervalMs <= 0 || !remote.at || now - remote.at >= remoteIntervalMs
- await Promise.all(machines.filter(m => m.online && (m.current || due || !answers.has(m.machineId) || !previouslyOnline.has(m.machineId) || failed.has(m.machineId))).map(async machine => {
+ // A machine that would not answer is asked again later and later, not on every 4 s local tick: three unreachable
+ // machines behind one open Monitor were each dialed every few seconds, a relay session apiece (2026-10-26). A
+ // refresh the person asks for (remoteIntervalMs 0) still asks at once.
+ const wanted = m => failed.has(m.machineId) ? remoteIntervalMs <= 0 || now >= (retryAt.get(m.machineId)?.at ?? 0)
+ : due || !answers.has(m.machineId) || !previouslyOnline.has(m.machineId)
+ await Promise.all(machines.filter(m => m.online && (m.current || wanted(m))).map(async machine => {
try {
const answer = agentsFor ? { agents: await agentsFor(machine.machineId, { timeoutMs }), shared: [] }
: await inventoryFor(machine.machineId, { timeoutMs })
- answers.set(machine.machineId, { ...answer, receivedAt: now }); failed.delete(machine.machineId)
+ answers.set(machine.machineId, { ...answer, receivedAt: now }); failed.delete(machine.machineId); retryAt.delete(machine.machineId)
+ }
+ catch (error) {
+ failed.add(machine.machineId); problems.push({ machine: machine.name, error: error.message })
+ const misses = (retryAt.get(machine.machineId)?.misses ?? 0) + 1
+ retryAt.set(machine.machineId, { misses, at: now + Math.min(RETRY_MS * 2 ** (misses - 1), RETRY_MAX_MS) })
}
- catch (error) { failed.add(machine.machineId); problems.push({ machine: machine.name, error: error.message }) }
}))
if (due) remote.at = now
remote.online = new Set(machines.filter(m => !report.error && m.online).map(m => m.machineId))
if (!report.error) {
const known = new Set(machines.map(m => m.machineId))
for (const id of answers.keys()) if (!known.has(id)) { answers.delete(id); failed.delete(id) }
+ for (const id of retryAt.keys()) if (!known.has(id)) retryAt.delete(id)
remote.machines = machines
}
const visibleMachines = report.error ? (remote.machines ?? []) : machines
diff --git a/store/agents/harness-monitor/test/inventory.test.mjs b/store/agents/harness-monitor/test/inventory.test.mjs
index 5ad2a1f24..3e42af0d4 100644
--- a/store/agents/harness-monitor/test/inventory.test.mjs
+++ b/store/agents/harness-monitor/test/inventory.test.mjs
@@ -60,7 +60,20 @@ test('reconnect and failed reads refresh before the regular remote interval', as
await read(); await read(); assert.equal(reads, 1)
online = false; assert.equal((await read()).rows[0].activity, 'offline')
online = true; fail = true; assert.equal((await read()).rows[0].canOpen, false); assert.equal(reads, 2)
- fail = false; assert.equal((await read()).rows[0].canOpen, true); assert.equal(reads, 3)
+ // Asked again 15 s after a failed read, not on the next tick.
+ fail = false; await read(); assert.equal(reads, 2)
+ now += 15_000; assert.equal((await read()).rows[0].canOpen, true); assert.equal(reads, 3)
+})
+test('a machine that keeps failing is asked later and later, at most every five minutes, and at once on a refresh', async () => {
+ const remote = { at: 0, answers: new Map() }; let now = 0, reads = 0
+ const options = { remote, remoteIntervalMs: 15_000, reportMachines: async () => ({ machines: [{ ...machine, online: true }] }),
+ agentsFor: async () => { reads++; throw Error('Disconnected') } }
+ const asks = []
+ for (; now <= 20 * 60_000; now += 4_000) { const before = reads; await collect({ ...options, now }); if (reads > before) asks.push(now) }
+ const gaps = asks.slice(1).map((at, i) => at - asks[i])
+ assert.deepEqual(gaps.slice(0, 5).map(gap => Math.ceil(gap / 4_000) * 4_000), [16_000, 32_000, 60_000, 120_000, 240_000])
+ assert.ok(gaps.every(gap => gap <= 5 * 60_000 + 4_000))
+ const before = reads; await collect({ ...options, remoteIntervalMs: 0, now }); assert.equal(reads, before + 1)
})
test('ambiguous IDs and pane numbers cannot choose another machine', () => {
const a = merge(), b = merge({}, { machine: { machineId: 'm2' } })