Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: add detailed logging for P2P demotion and retry lifecycle events to improve observability

0xken152ec84e617af2
diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.ts
index 67f00c593..c0505ab5b 100644
--- a/cli/src/lib/remoteRelay.spec.ts
+++ b/cli/src/lib/remoteRelay.spec.ts
@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { RemoteRelayPool } from './remoteRelay.js'
/** Minimal fake of the private `Entry` shape `noteTerminalResponse`/`demoteP2p` operate on — exercised
@@ -160,6 +160,14 @@ describe('RemoteRelayPool p2p retry policy', () => {
demoteP2p: (machineId: string, entry: ReturnType<typeof fakeEntry>, reason: string) => void
}
+ // Regression: demoteP2p/scheduleP2pRetry used to produce zero log output — the exact gap that made
+ // "did a demoted connection actually try to come back?" unanswerable from harness.log for a real
+ // production case (machine-remote-2 staying stuck on relay with no trace of why). Every outcome
+ // below must now be a distinct, findable line rather than three different ways to stay silent.
+ let logSpy: ReturnType<typeof vi.spyOn>
+ beforeEach(() => { logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) })
+ afterEach(() => { logSpy.mockRestore() })
+
it('schedules up to 10 retries within an hour, then refuses an 11th', () => {
vi.useFakeTimers()
const HOURLY_CAP = 10 // mirrors the unexported P2P_RETRY_HOURLY_CAP constant in remoteRelay.ts
@@ -171,12 +179,14 @@ describe('RemoteRelayPool p2p retry policy', () => {
pool.scheduleP2pRetry('machine-1', entry)
expect(entry.p2pRetryTimer, `retry #${i} should have been scheduled`).not.toBeNull()
expect(entry.p2pRetryTimestamps).toHaveLength(i)
+ expect(logSpy).toHaveBeenLastCalledWith(expect.stringContaining('[p2p] retry scheduled'))
vi.advanceTimersByTime(60_000)
entry.p2pRetryTimer = null // the real timer callback nulls this before re-dialing; simulate that
}
pool.scheduleP2pRetry('machine-1', entry) // one retry past the cap — refused, quota spent for this hour
expect(entry.p2pRetryTimer).toBeNull()
expect(entry.p2pRetryTimestamps).toHaveLength(HOURLY_CAP)
+ expect(logSpy).toHaveBeenLastCalledWith(expect.stringContaining('[p2p] retry budget exhausted'))
} finally {
vi.useRealTimers()
}
@@ -191,6 +201,7 @@ describe('RemoteRelayPool p2p retry policy', () => {
pool.scheduleP2pRetry('machine-1', entry) // a second failure before the first retry even fires
expect(entry.p2pRetryTimer).toBe(firstTimer)
expect(entry.p2pRetryTimestamps).toHaveLength(1)
+ expect(logSpy).toHaveBeenLastCalledWith(expect.stringContaining('[p2p] retry already pending'))
} finally {
vi.useRealTimers()
}
@@ -217,11 +228,28 @@ describe('RemoteRelayPool p2p retry policy', () => {
expect(entry.p2pRetryTimestamps).toHaveLength(10) // refused — nothing new was recorded
})
+ it('logs when a retry fires but finds its entry already torn down/replaced', () => {
+ vi.useFakeTimers()
+ try {
+ const entry = fakeEntry()
+ pool.scheduleP2pRetry('machine-1', entry)
+ // Nothing ever registered this entry into pool.entries (this test drives scheduleP2pRetry
+ // directly, same shortcut every other test here takes) — so `entries.get('machine-1') !== entry`
+ // holds by construction once the timer fires, exactly like a real entry replaced mid-flight.
+ vi.advanceTimersByTime(60_000)
+ expect(logSpy).toHaveBeenLastCalledWith(expect.stringContaining('[p2p] retry orphaned'))
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
it('demoteP2p (case B: was direct, then demoted) schedules a retry through the same policy', () => {
const entry = fakeEntry({ p2pStreams: new Set(['stream-a']) })
pool.demoteP2p('machine-1', entry, 'send_failed')
expect(entry.p2pRetryTimer).not.toBeNull()
expect(entry.p2pRetryTimestamps).toHaveLength(1)
+ // sid() truncates to 8 chars — 'machine-1'.slice(0, 8) is 'machine-'
+ expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[p2p] demoted · machine=machine- reason=send_failed streams=1'))
})
})
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index e2737009c..464e7ac8d 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -676,6 +676,14 @@ export class RemoteRelayPool {
}
private demoteP2p(machineId: string, entry: Entry, reason: string): void {
+ // The ONE place every demotion converges, regardless of cause (a bad send, ICE failing outright,
+ // or ICE failing after this connection had already been direct for a while) — log here rather than
+ // at each call site so none of them can silently fall through with zero trace. This was the exact
+ // gap that made a real production case ("machine-remote-2 stuck on relay, remote-1 stayed p2p")
+ // untraceable: the connection HAD been direct, then failed, and nothing printed a single line
+ // about it — 'failed && !wasDirect' in startP2p()'s onState is the only sibling that already logs,
+ // and it deliberately does not cover this case (see that method's own comment).
+ console.log(`[p2p] demoted · machine=${sid(machineId)} reason=${reason} streams=${entry.p2pStreams.size}`)
// The primary just died (or is dying) — an upgrade trial in flight for it no longer means anything,
// and scheduleP2pRetry() below is about to take over recovery via the normal path anyway.
if (entry.upgradeTimer) { clearTimeout(entry.upgradeTimer); entry.upgradeTimer = null }
@@ -710,14 +718,30 @@ export class RemoteRelayPool {
* should still get another shot rather than being stuck on relay for the rest of the daemon's uptime.
*/
private scheduleP2pRetry(machineId: string, entry: Entry): void {
- if (entry.p2pRetryTimer) return
+ // Every outcome below gets a line — this used to be silent end to end, which is exactly what made
+ // "did it even try to retry?" unanswerable from harness.log for the machine-remote-2 case this
+ // was added for: a retry can be skipped (already pending, or budget spent this hour) or scheduled
+ // and then orphaned by the time it fires (the entry it was scheduled for got replaced), and each of
+ // those needs to be a different, findable line rather than three ways to produce the same silence.
+ if (entry.p2pRetryTimer) {
+ console.log(`[p2p] retry already pending · machine=${sid(machineId)}`)
+ return
+ }
const windowStart = Date.now() - P2P_RETRY_WINDOW_MS
entry.p2pRetryTimestamps = entry.p2pRetryTimestamps.filter((t) => t > windowStart)
- if (entry.p2pRetryTimestamps.length >= P2P_RETRY_HOURLY_CAP) return
+ if (entry.p2pRetryTimestamps.length >= P2P_RETRY_HOURLY_CAP) {
+ console.log(`[p2p] retry budget exhausted · machine=${sid(machineId)} (${entry.p2pRetryTimestamps.length}/${P2P_RETRY_HOURLY_CAP} in the last hour)`)
+ return
+ }
entry.p2pRetryTimestamps.push(Date.now())
+ console.log(`[p2p] retry scheduled · machine=${sid(machineId)} in ${P2P_RETRY_DELAY_MS / 1000}s`
+ + ` (${entry.p2pRetryTimestamps.length}/${P2P_RETRY_HOURLY_CAP} this hour)`)
entry.p2pRetryTimer = setTimeout(() => {
entry.p2pRetryTimer = null
- if (this.entries.get(machineId) !== entry) return // entry was torn down/replaced meanwhile
+ if (this.entries.get(machineId) !== entry) {
+ console.log(`[p2p] retry orphaned · machine=${sid(machineId)} — entry was torn down/replaced meanwhile`)
+ return
+ }
entry.p2p = null
this.startP2p(machineId, entry)
}, P2P_RETRY_DELAY_MS)