The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat: add detailed logging for P2P demotion and retry lifecycle events to improve observability
0xken152ec84e617af2
diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.tsindex 67f00c593..c0505ab5b 100644--- a/cli/src/lib/remoteRelay.spec.ts+++ b/cli/src/lib/remoteRelay.spec.ts@@ -1,4 +1,4 @@-import { afterEach, describe, expect, it, vi } from 'vitest'+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import { RemoteRelayPool } from './remoteRelay.js'/** Minimal fake of the private `Entry` shape `noteTerminalResponse`/`demoteP2p` operate on — exercised@@ -160,6 +160,14 @@ describe('RemoteRelayPool p2p retry policy', () => {demoteP2p: (machineId: string, entry: ReturnType<typeof fakeEntry>, reason: string) => void}+ // Regression: demoteP2p/scheduleP2pRetry used to produce zero log output — the exact gap that made+ // "did a demoted connection actually try to come back?" unanswerable from harness.log for a real+ // production case (machine-remote-2 staying stuck on relay with no trace of why). Every outcome+ // below must now be a distinct, findable line rather than three different ways to stay silent.+ let logSpy: ReturnType<typeof vi.spyOn>+ beforeEach(() => { logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) })+ afterEach(() => { logSpy.mockRestore() })+it('schedules up to 10 retries within an hour, then refuses an 11th', () => {vi.useFakeTimers()const HOURLY_CAP = 10 // mirrors the unexported P2P_RETRY_HOURLY_CAP constant in remoteRelay.ts@@ -171,12 +179,14 @@ describe('RemoteRelayPool p2p retry policy', () => {pool.scheduleP2pRetry('machine-1', entry)expect(entry.p2pRetryTimer, `retry #${i} should have been scheduled`).not.toBeNull()expect(entry.p2pRetryTimestamps).toHaveLength(i)+ expect(logSpy).toHaveBeenLastCalledWith(expect.stringContaining('[p2p] retry scheduled'))vi.advanceTimersByTime(60_000)entry.p2pRetryTimer = null // the real timer callback nulls this before re-dialing; simulate that}pool.scheduleP2pRetry('machine-1', entry) // one retry past the cap — refused, quota spent for this hourexpect(entry.p2pRetryTimer).toBeNull()expect(entry.p2pRetryTimestamps).toHaveLength(HOURLY_CAP)+ expect(logSpy).toHaveBeenLastCalledWith(expect.stringContaining('[p2p] retry budget exhausted'))} finally {vi.useRealTimers()}@@ -191,6 +201,7 @@ describe('RemoteRelayPool p2p retry policy', () => {pool.scheduleP2pRetry('machine-1', entry) // a second failure before the first retry even firesexpect(entry.p2pRetryTimer).toBe(firstTimer)expect(entry.p2pRetryTimestamps).toHaveLength(1)+ expect(logSpy).toHaveBeenLastCalledWith(expect.stringContaining('[p2p] retry already pending'))} finally {vi.useRealTimers()}@@ -217,11 +228,28 @@ describe('RemoteRelayPool p2p retry policy', () => {expect(entry.p2pRetryTimestamps).toHaveLength(10) // refused — nothing new was recorded})+ it('logs when a retry fires but finds its entry already torn down/replaced', () => {+ vi.useFakeTimers()+ try {+ const entry = fakeEntry()+ pool.scheduleP2pRetry('machine-1', entry)+ // Nothing ever registered this entry into pool.entries (this test drives scheduleP2pRetry+ // directly, same shortcut every other test here takes) — so `entries.get('machine-1') !== entry`+ // holds by construction once the timer fires, exactly like a real entry replaced mid-flight.+ vi.advanceTimersByTime(60_000)+ expect(logSpy).toHaveBeenLastCalledWith(expect.stringContaining('[p2p] retry orphaned'))+ } finally {+ vi.useRealTimers()+ }+ })+it('demoteP2p (case B: was direct, then demoted) schedules a retry through the same policy', () => {const entry = fakeEntry({ p2pStreams: new Set(['stream-a']) })pool.demoteP2p('machine-1', entry, 'send_failed')expect(entry.p2pRetryTimer).not.toBeNull()expect(entry.p2pRetryTimestamps).toHaveLength(1)+ // sid() truncates to 8 chars — 'machine-1'.slice(0, 8) is 'machine-'+ expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[p2p] demoted · machine=machine- reason=send_failed streams=1'))})})diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.tsindex e2737009c..464e7ac8d 100644--- a/cli/src/lib/remoteRelay.ts+++ b/cli/src/lib/remoteRelay.ts@@ -676,6 +676,14 @@ export class RemoteRelayPool {}private demoteP2p(machineId: string, entry: Entry, reason: string): void {+ // The ONE place every demotion converges, regardless of cause (a bad send, ICE failing outright,+ // or ICE failing after this connection had already been direct for a while) — log here rather than+ // at each call site so none of them can silently fall through with zero trace. This was the exact+ // gap that made a real production case ("machine-remote-2 stuck on relay, remote-1 stayed p2p")+ // untraceable: the connection HAD been direct, then failed, and nothing printed a single line+ // about it — 'failed && !wasDirect' in startP2p()'s onState is the only sibling that already logs,+ // and it deliberately does not cover this case (see that method's own comment).+ console.log(`[p2p] demoted · machine=${sid(machineId)} reason=${reason} streams=${entry.p2pStreams.size}`)// The primary just died (or is dying) — an upgrade trial in flight for it no longer means anything,// and scheduleP2pRetry() below is about to take over recovery via the normal path anyway.if (entry.upgradeTimer) { clearTimeout(entry.upgradeTimer); entry.upgradeTimer = null }@@ -710,14 +718,30 @@ export class RemoteRelayPool {* should still get another shot rather than being stuck on relay for the rest of the daemon's uptime.*/private scheduleP2pRetry(machineId: string, entry: Entry): void {- if (entry.p2pRetryTimer) return+ // Every outcome below gets a line — this used to be silent end to end, which is exactly what made+ // "did it even try to retry?" unanswerable from harness.log for the machine-remote-2 case this+ // was added for: a retry can be skipped (already pending, or budget spent this hour) or scheduled+ // and then orphaned by the time it fires (the entry it was scheduled for got replaced), and each of+ // those needs to be a different, findable line rather than three ways to produce the same silence.+ if (entry.p2pRetryTimer) {+ console.log(`[p2p] retry already pending · machine=${sid(machineId)}`)+ return+ }const windowStart = Date.now() - P2P_RETRY_WINDOW_MSentry.p2pRetryTimestamps = entry.p2pRetryTimestamps.filter((t) => t > windowStart)- if (entry.p2pRetryTimestamps.length >= P2P_RETRY_HOURLY_CAP) return+ if (entry.p2pRetryTimestamps.length >= P2P_RETRY_HOURLY_CAP) {+ console.log(`[p2p] retry budget exhausted · machine=${sid(machineId)} (${entry.p2pRetryTimestamps.length}/${P2P_RETRY_HOURLY_CAP} in the last hour)`)+ return+ }entry.p2pRetryTimestamps.push(Date.now())+ console.log(`[p2p] retry scheduled · machine=${sid(machineId)} in ${P2P_RETRY_DELAY_MS / 1000}s`+ + ` (${entry.p2pRetryTimestamps.length}/${P2P_RETRY_HOURLY_CAP} this hour)`)entry.p2pRetryTimer = setTimeout(() => {entry.p2pRetryTimer = null- if (this.entries.get(machineId) !== entry) return // entry was torn down/replaced meanwhile+ if (this.entries.get(machineId) !== entry) {+ console.log(`[p2p] retry orphaned · machine=${sid(machineId)} — entry was torn down/replaced meanwhile`)+ return+ }entry.p2p = nullthis.startP2p(machineId, entry)}, P2P_RETRY_DELAY_MS)