Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: support stale machine list caching and handle unknown machine states in panes during outages

0xken152f5c6f54b885
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 685400e37..ecd4d42fc 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -34,7 +34,7 @@ import { buildLogBundle, bundleFileName, redactSecretsInText } from './lib/logBu
import { CableSession } from './cable/cableSession.js'
import { DaemonCableHost, cableEventFor, cableQuestionFor, cableQuestionCloseFor } from './cable/cableHost.js'
-import { MachineListCache } from './device/machineList.js'
+import { MachineListCache, machineListCachePath, withStaleMarker } from './device/machineList.js'
import { DeviceLink } from './device/deviceLink.js'
import { DeviceFleet } from './device/deviceFleet.js'
import { registry, projectDisplayName, type RegisteredSession } from './lib/registry.js'
@@ -956,6 +956,9 @@ async function logout(): Promise<void> {
await stopDaemonProcess()
clearAuthSession()
rmSync(MACHINE_NAME_FILE, { force: true })
+ // Same reason as the name above: the cached machine list describes the account that just left, and the
+ // local `/api/machines` fallback would otherwise hand it to whoever signs in next on this computer.
+ rmSync(machineListCachePath(), { force: true })
console.log('Signed out. Run `harness login`, then `harness start`, to reconnect this computer.')
// Existence is the whole of the test — nothing is read out of the store, and nothing is written
// into it. On stderr, so a script reading this command's output is unaffected by it.
@@ -2573,6 +2576,43 @@ async function runForeground(session: AuthSession): Promise<void> {
return { status: res.status, body: json }
}
+ // Built HERE rather than beside the cable stack that also uses it (further down), because the hook
+ // server starts long before that point and agent restore can sit between the two. A cache bound late
+ // is a cache that is still null exactly when a cold boot during an outage needs it most.
+ const machineListCache = new MachineListCache(
+ () => proxyBackend('GET', '/api/machines'),
+ computerId,
+ (line) => console.log(`[cable] ${line}`),
+ undefined,
+ // A machine row is per (user, computer): the one local fact that distinguishes two ACCOUNTS here.
+ // Read fresh each time — a re-login swaps it under a daemon that never restarted.
+ () => readAuthSession()?.machineId ?? null,
+ )
+
+ /**
+ * `GET /api/machines` for local clients, answered from the last known-good list when the backend leg
+ * is down.
+ *
+ * The daemon already keeps that list: it re-reads it every 60s for the dial's wheel and persists it to
+ * `machines.json`, with the explicit policy that an outage keeps the rows and stops claiming they are
+ * live. The desktop app was the one consumer that got none of that — a bare pass-through handed it the
+ * 502 and it had nothing to draw, so a ten-second network blip emptied the machine list and left every
+ * pane spinning. Stale rows are not wrong rows; the marker below says which they are.
+ */
+ async function machinesListWithFallback(): Promise<{ status: number; body: Record<string, unknown> }> {
+ const res = await proxyBackend('GET', '/api/machines')
+ if (res.status === 200) {
+ // Feed the cache the answer we already have rather than making it fetch the same thing again.
+ machineListCache.adopt(res.body)
+ return res
+ }
+ // A real end of session is the caller's answer, not an outage: never serve a list from behind it.
+ if (res.status === 401 || res.status === 403) return res
+ const cached = machineListCache.lastResponse()
+ if (!cached) return res
+ return { status: 200, body: withStaleMarker(cached.body, cached.fetchedAt) }
+ }
+
// Update-handoff state, declared here — ahead of the /api/status handler that reads `restarting` —
// rather than beside the updater that writes it, so the closure never reaches a `let` in its TDZ.
let restarting = false
@@ -2951,7 +2991,7 @@ async function runForeground(session: AuthSession): Promise<void> {
try { return readFileSync(LOG_FILE, 'utf-8').split('\n').slice(-120).join('\n') } catch { return '' }
},
onStop: () => { setTimeout(() => process.kill(process.pid, 'SIGTERM'), 50) }, // let the 200 flush first
- onMachinesList: () => proxyBackend('GET', '/api/machines'),
+ onMachinesList: () => machinesListWithFallback(),
onMachineRename: (machineId, name) => proxyBackend('PATCH', `/api/machines/${encodeURIComponent(machineId)}`, { name }),
onMachineDelete: (machineId) => proxyBackend('DELETE', `/api/machines/${encodeURIComponent(machineId)}`),
onAuthMe: () => proxyBackend('GET', '/api/auth/me'),
@@ -4247,11 +4287,9 @@ async function runForeground(session: AuthSession): Promise<void> {
// Three independent things, on purpose. The LIST is a REST read that works while the backend socket is
// down; `local` is derived from the computer id and needs no network at all; and the LANE is a device
// socket that only exists while the dial is actually looking at another machine.
- const machineList = new MachineListCache(
- () => proxyBackend('GET', '/api/machines'),
- computerId,
- (line) => console.log(`[cable] ${line}`),
- )
+ // The same cache the local `/api/machines` handler answers from (built up near `proxyBackend`), so the
+ // dial's wheel and the desktop's list cannot disagree — and neither can go stale while the other is fresh.
+ const machineList = machineListCache
void machineList.refresh()
const machineListTimer = setInterval(() => void machineList.refresh(), 60_000)
machineListTimer.unref?.()
diff --git a/cli/src/device/machineList.spec.ts b/cli/src/device/machineList.spec.ts
index ee9002f32..a3d16f898 100644
--- a/cli/src/device/machineList.spec.ts
+++ b/cli/src/device/machineList.spec.ts
@@ -3,15 +3,18 @@
// `local` is derived, not declared, and getting it wrong is invisible: every machine — including this
// computer's own — silently reads as remote, so the dial's own row goes missing and the daemon opens a
// cloud socket to reach agents that are in this very process.
-import { mkdtempSync } from 'node:fs'
+import { mkdtempSync, readFileSync, statSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it, vi } from 'vitest'
-import { MachineListCache, sameComputer } from './machineList.js'
+import { MachineListCache, sameComputer, withStaleMarker } from './machineList.js'
const DIR = (): string => mkdtempSync(join(tmpdir(), 'machines-'))
+/** The machine row a session belongs to — per (user, computer), so it is what tells two accounts apart. */
+const OWNER = 'machine-of-user-a'
+
/** The shape `GET /api/machines` actually answers with (MachineService.toOwner). */
function row(over: Record<string, unknown> = {}): Record<string, unknown> {
return { machineId: 'm1', computerId: 'aabbccdd', name: 'office-imac', status: 'running', authMode: 'remote', ...over }
@@ -116,3 +119,137 @@ describe('MachineListCache', () => {
expect(cache.list().source).toBe('local')
})
})
+
+// The desktop app reads this same endpoint through the daemon, so an outage must leave it with the last
+// known list rather than nothing — see `machinesListWithFallback` in cli.ts.
+describe('MachineListCache last-known-good body', () => {
+ it('keeps the successful response verbatim, so a fallback cannot change the wire shape', async () => {
+ const dir = DIR()
+ // A row carries fields the dial's ListedMachine projection drops (computerId, hostname) but the
+ // desktop reads. Serving the projection back would silently break it.
+ const body = { success: true, data: { machines: [row({ hostname: 'imac-office' })] } } as Record<string, unknown>
+ const cache = new MachineListCache(async () => ({ status: 200, body }), () => 'aabbccdd', () => {}, dir, () => OWNER)
+ await cache.refresh()
+
+ const kept = cache.lastResponse()
+ expect(kept?.body).toEqual(body)
+ expect(kept!.fetchedAt).toBeGreaterThan(0)
+ // And it survives a restart: the body is on disk beside the rows.
+ const onDisk = JSON.parse(readFileSync(join(dir, 'machines.json'), 'utf8')) as Record<string, unknown>
+ expect(onDisk.body).toEqual(body)
+ })
+
+ it('has nothing to serve until a read succeeds', async () => {
+ const cache = new MachineListCache(async () => ({ status: 502, body: {} }), () => 'z', () => {}, DIR())
+ await cache.refresh()
+ expect(cache.lastResponse()).toBeNull()
+ })
+
+ it('reads a cache file written before this field existed, without throwing', () => {
+ const dir = DIR()
+ // The shape an older daemon wrote: rows only.
+ writeFileSync(join(dir, 'machines.json'), JSON.stringify({ machines: [{ machineId: 'm1', name: 'old', state: 'ready', authMode: 'remote', local: false }] }))
+ const cache = new MachineListCache(ok([]), () => 'z', () => {}, dir)
+ expect(cache.find('m1')?.name).toBe('old') // the wheel still draws
+ expect(cache.lastResponse()).toBeNull() // but there is no body to answer with yet
+ })
+
+ it('restores the body from disk so a daemon that starts offline can still answer', async () => {
+ const dir = DIR()
+ const body = { success: true, data: { machines: [row()] } } as Record<string, unknown>
+ await new MachineListCache(async () => ({ status: 200, body }), () => 'aabbccdd', () => {}, dir, () => OWNER).refresh()
+
+ const restarted = new MachineListCache(async () => ({ status: 502, body: {} }), () => 'aabbccdd', () => {}, dir, () => OWNER)
+ expect(restarted.lastResponse()?.body).toEqual(body)
+ })
+
+ it('forgets the body when the session ends, so no list outlives its owner', async () => {
+ const dir = DIR()
+ const body = { success: true, data: { machines: [row()] } } as Record<string, unknown>
+ let status = 200
+ const cache = new MachineListCache(async () => ({ status, body }), () => 'aabbccdd', () => {}, dir, () => OWNER)
+ await cache.refresh()
+ expect(cache.lastResponse()).not.toBeNull()
+
+ status = 401
+ await cache.refresh()
+ expect(cache.lastResponse()).toBeNull()
+ expect(cache.list().machines).toHaveLength(0)
+ // Cleared on disk too — a restart must not resurrect it.
+ expect(JSON.parse(readFileSync(join(dir, 'machines.json'), 'utf8')).body).toBeNull()
+ })
+
+ it('adopt() takes a body the caller already has, and refuses one with no machines', () => {
+ const cache = new MachineListCache(ok([]), () => 'aabbccdd', () => {}, DIR())
+ expect(cache.adopt({ machines: [row()] })).toBe(true)
+ expect(cache.find('m1')?.local).toBe(true)
+ expect(cache.adopt({ nope: true })).toBe(false)
+ })
+})
+
+describe('withStaleMarker', () => {
+ it('marks inside `data`, which is where a local client unwraps to', () => {
+ const at = Date.parse('2026-09-15T12:00:00.000Z')
+ const out = withStaleMarker({ success: true, data: { machines: [row()] } }, at)
+ const data = out.data as Record<string, unknown>
+ expect(data.stale).toBe(true)
+ expect(data.staleSince).toBe('2026-09-15T12:00:00.000Z')
+ expect(out.stale).toBeUndefined() // not at the top level, where nobody looks
+ expect((data.machines as unknown[])).toHaveLength(1) // the payload itself is untouched
+ })
+
+ it('falls back to the top level for a body that has no `data`', () => {
+ const out = withStaleMarker({ machines: [row()] }, Date.parse('2026-09-15T12:00:00.000Z'))
+ expect(out.stale).toBe(true)
+ expect(out.staleSince).toBe('2026-09-15T12:00:00.000Z')
+ })
+})
+
+// `harness logout` deletes this file, but a session can also be replaced in place — signing in as someone
+// else without logging out first. Answering that with the previous account's machines would disclose them.
+describe('MachineListCache account guard', () => {
+ const body = { success: true, data: { machines: [row()] } } as Record<string, unknown>
+
+ it('will not answer a different account from the list it cached for the previous one', async () => {
+ const dir = DIR()
+ let owner = 'machine-of-user-a';
+ const cache = new MachineListCache(async () => ({ status: 200, body }), () => 'aabbccdd', () => {}, dir, () => owner)
+ await cache.refresh()
+ expect(cache.lastResponse()).not.toBeNull()
+
+ owner = 'machine-of-user-b' // same computer, different account
+ expect(cache.lastResponse()).toBeNull()
+ })
+
+ it('will not answer when this session has no machine of its own to compare', async () => {
+ const dir = DIR()
+ const cache = new MachineListCache(async () => ({ status: 200, body }), () => 'aabbccdd', () => {}, dir, () => null)
+ await cache.refresh()
+ // Nothing to prove ownership with is not the same as proving it: stay quiet.
+ expect(cache.lastResponse()).toBeNull()
+ })
+
+ it('will not answer from a cache written before the stamp existed', () => {
+ const dir = DIR()
+ writeFileSync(join(dir, 'machines.json'), JSON.stringify({ machines: [], body, fetchedAt: 1 }))
+ const cache = new MachineListCache(ok([]), () => 'aabbccdd', () => {}, dir, () => OWNER)
+ expect(cache.lastResponse()).toBeNull()
+ })
+})
+
+describe('MachineListCache disk writes', () => {
+ it('does not rewrite the file when the list has not changed', async () => {
+ const dir = DIR()
+ const path = join(dir, 'machines.json')
+ const cache = new MachineListCache(ok([row()]), () => 'aabbccdd', () => {}, dir, () => OWNER)
+ await cache.refresh()
+ const first = statSync(path).mtimeMs
+
+ // The desktop asking again is the common case; an unchanged answer must cost no synchronous write.
+ for (let i = 0; i < 5; i++) cache.adopt({ machines: [row()] })
+ expect(statSync(path).mtimeMs).toBe(first)
+
+ cache.adopt({ machines: [row({ machineId: 'm2' })] }) // a real change still lands
+ expect(statSync(path).mtimeMs).not.toBe(first)
+ })
+})
diff --git a/cli/src/device/machineList.ts b/cli/src/device/machineList.ts
index 1756eb210..df7f5b4f2 100644
--- a/cli/src/device/machineList.ts
+++ b/cli/src/device/machineList.ts
@@ -41,6 +41,14 @@ export function sameComputer(a: string | null | undefined, b: string | null | un
export type MachineSource = 'backend' | 'local' | 'signed-out'
+/** How long an UNCHANGED list may go without a rewrite — see `saveCache`. Matches the refresh poll. */
+const REWRITE_UNCHANGED_AFTER_MS = 60_000
+
+/** Where the cache lives. Exported so sign-out can delete exactly the file this class writes. */
+export function machineListCachePath(dataDir = env.ADAPTER_DATA_DIR): string {
+ return join(dataDir, 'machines.json')
+}
+
function stateOf(status: string): FleetMachine['state'] {
// `toOwner.status` is resolved live for computer-backed machines (`applyRemoteStatus`), so it is a real
// presence signal for exactly the machines where presence is a question.
@@ -49,18 +57,47 @@ function stateOf(status: string): FleetMachine['state'] {
return 'unknown'
}
+/**
+ * Tag a cached machine-list body as stale, where the client actually reads it.
+ *
+ * The backend answers `{success:true,data:{…}}` and this daemon forwards that verbatim, so the flag has
+ * to go INSIDE `data` — a local client unwraps to `data` and would never see a top-level field. Older
+ * shapes that return the machines at the top level get it there instead.
+ */
+export function withStaleMarker(body: Record<string, unknown>, fetchedAt: number): Record<string, unknown> {
+ const marker = { stale: true, staleSince: new Date(fetchedAt).toISOString() }
+ const data = body.data
+ if (data && typeof data === 'object' && !Array.isArray(data)) {
+ return { ...body, data: { ...(data as Record<string, unknown>), ...marker } }
+ }
+ return { ...body, ...marker }
+}
+
export class MachineListCache {
private machines: ListedMachine[] = []
private source: MachineSource = 'local'
private readonly path: string
+ // The last SUCCESSFUL `GET /api/machines` body, kept verbatim. The dial only needs `ListedMachine`, but
+ // the desktop app reads the raw backend row (computerId, hostname, …) straight off this endpoint, so a
+ // fallback that served the lossy projection would silently change the wire shape. Keeping the original
+ // is what lets `onMachinesList` answer from cache during an outage without the app noticing a difference.
+ private lastBody: Record<string, unknown> | null = null
+ private fetchedAt = 0
+ private bodyOwner: string | null = null
+ /** The CONTENT `saveCache` last wrote (timestamp excluded), so an unchanged list costs no disk write. */
+ private lastWrittenContent = ''
+ private lastWrittenAt = 0
constructor(
private readonly fetchMachines: () => Promise<{ status: number; body: Record<string, unknown> }>,
private readonly localComputerId: () => string,
private readonly log: (line: string) => void,
dataDir = env.ADAPTER_DATA_DIR,
+ /** The machine row this session belongs to. A machine row is per (user, computer), so it is the one
+ * local fact that tells two ACCOUNTS on this computer apart — see `lastResponse`. */
+ private readonly owner: () => string | null = () => null,
) {
- this.path = join(dataDir, 'machines.json')
+ this.path = machineListCachePath(dataDir)
this.loadCache()
}
@@ -73,6 +110,21 @@ export class MachineListCache {
return this.machines.find((m) => m.machineId === machineId)
}
+ /**
+ * The last known-good response body and when it was read, or null if there is none to serve.
+ *
+ * Withheld unless it provably belongs to the session asking for it. `harness logout` deletes this file,
+ * but a session can also be replaced in place (sign in as someone else without logging out), and
+ * answering THAT with the previous account's machines would be a straight disclosure. Both ids must be
+ * present and equal: a cache with no stamp — written by a daemon older than this check — is not served.
+ */
+ lastResponse(): { body: Record<string, unknown>; fetchedAt: number } | null {
+ if (!this.lastBody) return null
+ const mine = this.owner()
+ if (!mine || !this.bodyOwner || this.bodyOwner !== mine) return null
+ return { body: this.lastBody, fetchedAt: this.fetchedAt }
+ }
+
/**
* Re-read the list.
*
@@ -90,14 +142,30 @@ export class MachineListCache {
}
if (res.status === 401 || res.status === 403) { this.signedOut(); return }
if (res.status >= 400) { this.degrade(`HTTP ${res.status}`); return }
+ if (!this.adopt(res.body)) this.degrade('no machines in the response')
+ }
- const raw = (res.body?.machines ?? (res.body?.data as Record<string, unknown> | undefined)?.machines) as unknown
- if (!Array.isArray(raw)) { this.degrade('no machines in the response'); return }
+ /**
+ * Take a body that a `GET /api/machines` just returned as the new known-good list.
+ *
+ * Split out of `refresh()` so the proxy handler can hand over the response it already has instead of
+ * spending a second round trip to tell this cache the same thing.
+ *
+ * Returns false when the body carries no machine array — the caller decides whether that is a degrade
+ * (a real read that came back wrong) or simply not its business.
+ */
+ adopt(body: Record<string, unknown>): boolean {
+ const raw = (body?.machines ?? (body?.data as Record<string, unknown> | undefined)?.machines) as unknown
+ if (!Array.isArray(raw)) return false
const mine = this.localComputerId()
this.machines = raw.map((r) => this.toListed(r as OwnerMachine, mine)).filter((m) => m.machineId)
this.source = 'backend'
+ this.lastBody = body
+ this.bodyOwner = this.owner()
+ this.fetchedAt = Date.now()
this.saveCache()
+ return true
}
private toListed(r: OwnerMachine, mine: string): ListedMachine {
@@ -155,19 +223,54 @@ export class MachineListCache {
if (this.source !== 'signed-out') this.log('machines: not signed in')
this.source = 'signed-out'
this.machines = []
+ // Drop the body as well: a signed-out session must never be answered from a previous user's list.
+ this.lastBody = null
+ this.bodyOwner = null
+ this.fetchedAt = 0
+ this.saveCache()
}
/** So a daemon that starts offline still draws the list the user saw last time. */
private loadCache(): void {
try {
- const parsed = JSON.parse(readFileSync(this.path, 'utf8')) as { machines?: ListedMachine[] }
+ const parsed = JSON.parse(readFileSync(this.path, 'utf8')) as {
+ machines?: ListedMachine[]
+ body?: Record<string, unknown>
+ fetchedAt?: number
+ bodyOwner?: string
+ }
if (Array.isArray(parsed.machines)) {
this.machines = parsed.machines.map((m) => ({ ...m, state: 'unknown' as const }))
}
+ // `body`/`fetchedAt` arrived after this file already existed in the wild, so a cache written by an
+ // older daemon has rows but no body. Rows still draw the wheel; the proxy fallback simply has
+ // nothing to serve until the next successful read fills it in.
+ if (parsed.body && typeof parsed.body === 'object') {
+ this.lastBody = parsed.body
+ this.fetchedAt = typeof parsed.fetchedAt === 'number' ? parsed.fetchedAt : 0
+ this.bodyOwner = typeof parsed.bodyOwner === 'string' ? parsed.bodyOwner : null
+ }
} catch { /* no cache yet, or unreadable — an empty wheel plus the local row is correct */ }
}
private saveCache(): void {
- try { writeFileSync(this.path, JSON.stringify({ machines: this.machines }), { mode: 0o600 }) } catch { /* best effort */ }
+ // `adopt` now runs on every local `/api/machines` too, not just the 60s poll, and this is a
+ // synchronous write on the event loop of a daemon that is streaming terminals. An unchanged list is
+ // the common case, so skip those. `fetchedAt` is deliberately NOT part of the comparison — it moves
+ // on every read and would defeat the check entirely — but it must not drift arbitrarily either,
+ // since it is what `staleSince` reports after a restart, so an unchanged list is still rewritten
+ // once a minute (the poll's own cadence).
+ const content = JSON.stringify({
+ machines: this.machines,
+ body: this.lastBody,
+ bodyOwner: this.bodyOwner,
+ })
+ const now = Date.now()
+ if (content === this.lastWrittenContent && now - this.lastWrittenAt < REWRITE_UNCHANGED_AFTER_MS) return
+ try {
+ writeFileSync(this.path, JSON.stringify({ ...JSON.parse(content), fetchedAt: this.fetchedAt }), { mode: 0o600 })
+ this.lastWrittenContent = content
+ this.lastWrittenAt = now
+ } catch { /* best effort */ }
}
}
diff --git a/desktop/lib/api/api_client.dart b/desktop/lib/api/api_client.dart
index b2274f617..46a895236 100644
--- a/desktop/lib/api/api_client.dart
+++ b/desktop/lib/api/api_client.dart
@@ -50,9 +50,16 @@ class ApiClient {
}
// -- machines (control plane, proxied by the local CLI) --
+ /// Whether the last [machines] answer came from the daemon's cache rather than the backend.
+ ///
+ /// The daemon answers 200 with the last known-good list when the backend leg is unreachable, so a
+ /// caller that only checked the status code would mistake an outage for a healthy, current read.
+ bool lastMachinesStale = false;
+
Future<List<Machine>> machines() async {
final res = await _dio.get('/api/machines');
final data = _unwrap(res) as Map<String, dynamic>;
+ lastMachinesStale = data['stale'] == true;
final list = data['machines'] as List<dynamic>? ?? [];
return list
.map((e) => Machine.fromJson(e as Map<String, dynamic>))
diff --git a/desktop/lib/state/app_state.dart b/desktop/lib/state/app_state.dart
index 84c85056e..da8953bea 100644
--- a/desktop/lib/state/app_state.dart
+++ b/desktop/lib/state/app_state.dart
@@ -1099,6 +1099,19 @@ class AppNotifier extends ChangeNotifier {
}
String? get lastError => _lastError;
+
+ /// The machine list's own failure, separate from [lastError] — that slot is shared with agent-launch
+ /// and other one-off errors, and is cleared by [dismissError]. A pane asking "is this machine missing
+ /// because we could not read the list?" needs the narrower question.
+ String? get machineListError => _machineLoadError;
+
+ /// True when the machine list on screen came from the daemon's cache because the backend could not be
+ /// reached. The rows are the last known-good ones, not current.
+ bool machinesAreStale = false;
+
+ /// Whether a machine-list recovery is armed. Mirrors [environmentRecheckPending] — the honest way for a
+ /// test to ask "is this still trying?" without reaching into a private timer.
+ bool get machineRecoveryPending => _machineRecoveryTimer != null;
bool get lastErrorRetryable => _lastErrorRetryable;
String? get bootStatusMessage => _bootStatusMessage;
@@ -2663,7 +2676,15 @@ class AppNotifier extends ChangeNotifier {
try {
await _refreshMachines(revision);
if (_authWorkCurrent(revision)) {
- _stopMachineRecovery();
+ // A cached answer is readable but not current, so the job is not done: leave the recovery timer
+ // running and it converges on its own once the backend is back. Without this the daemon's 200
+ // would read as success, recovery would stop, and the app would sit on stale rows until
+ // somebody pressed reload.
+ if (machinesAreStale) {
+ _scheduleMachineRecovery(revision);
+ } else {
+ _stopMachineRecovery();
+ }
if (_lastError != null && _lastError == _machineLoadError) {
_lastError = null;
notifyListeners();
@@ -2732,6 +2753,34 @@ class AppNotifier extends ChangeNotifier {
_machineLoadError = message;
}
+ /// What the loopback probe means for one machine's transport.
+ ///
+ /// Written once because two callers need the same answer: the refresh loop, and the failure path that
+ /// keeps this computer usable when the backend list could not be read.
+ void _applyLocalTransport(
+ MachineState state,
+ LocalCliEndpoint? localEndpoint,
+ String? localComputerId,
+ ) {
+ if (state.localOnly && localEndpoint?.computerId == localComputerId) {
+ state.localEndpoint = localEndpoint;
+ state.transportMode =
+ state.connectionStatus == ConnectionStatus.connected
+ ? MachineTransportMode.localPlaintext
+ : MachineTransportMode.localOffline;
+ } else if (state.localOnly) {
+ // The token still identifies this as local, but the CLI is offline or
+ // failed its identity/capability check. Never fall back to cloud E2EE.
+ state.localEndpoint = null;
+ state.transportMode = MachineTransportMode.localOffline;
+ state.nodeOnline = false;
+ _startOfflineRetry(state);
+ } else {
+ state.localEndpoint = null;
+ state.transportMode = MachineTransportMode.cloudE2ee;
+ }
+ }
+
Future<void> _refreshMachines(int revision) async {
final discovery = _discovery;
// The CLI computer id is the local identity source of truth. The loopback
@@ -2739,9 +2788,45 @@ class AppNotifier extends ChangeNotifier {
final localComputerId = await discovery.computerId();
if (!_authWorkCurrent(revision)) return;
final localFuture = discovery.discover(expectedComputerId: localComputerId);
- final list = await _fetchMachines();
- final localEndpoint = await localFuture;
+ // The two legs stay independent. The loopback probe reads a local file and asks 127.0.0.1, so it
+ // cannot fail for a network reason — but awaiting it BEHIND the backend call meant a cloud outage
+ // threw first and threw away an answer that was already correct, while awaiting it FIRST would let a
+ // slow probe hold up the list. Latch it as it lands instead, and use it on both paths.
+ LocalCliEndpoint? probed;
+ final localSettled = localFuture.then((value) => probed = value).catchError((
+ Object error,
+ ) {
+ // A probe that fails is the CLI being unreachable, which the transport decision below already
+ // handles — but swallowing it silently leaves nothing to diagnose from.
+ debugPrint('local CLI probe failed: $error');
+ return null;
+ });
+ final List<Machine> list;
+ try {
+ list = await _fetchMachines();
+ machinesAreStale = api.lastMachinesStale;
+ } catch (_) {
+ // A backend outage must not cost this computer its own transport. Without this the probe result
+ // stayed unapplied, so `usesLocalTransport` went false and `_connectMachine` skipped the local
+ // machine — while relayed machines, which never consult it, kept streaming. That asymmetry was the
+ // bug: the local terminal stopped rendering and the relayed ones did not.
+ await localSettled;
+ if (_authWorkCurrent(revision)) {
+ final endpoint = probed;
+ for (final state in machineStates.values) {
+ _applyLocalTransport(state, endpoint, localComputerId);
+ _connectMachine(state);
+ }
+ if (endpoint != null) _updateLocalProjectSnapshot(endpoint);
+ notifyListeners();
+ }
+ rethrow; // the recovery timer owns the retry; this only protects what already works
+ }
+ await localSettled;
+ // Both legs are in: this is the one gate that decides whether a result that arrived after a
+ // sign-out or a dispose may still be published.
if (!_authWorkCurrent(revision)) return;
+ final localEndpoint = probed;
machines = list
.where((machine) => machine.authMode == MachineAuthMode.remote)
.toList();
@@ -2764,24 +2849,7 @@ class AppNotifier extends ChangeNotifier {
state.localOnly =
localComputerId != null &&
_normalizeComputerId(machine.computerId) == localComputerId;
- if (state.localOnly && localEndpoint?.computerId == localComputerId) {
- state.localEndpoint = localEndpoint;
- if (state.connectionStatus == ConnectionStatus.connected) {
- state.transportMode = MachineTransportMode.localPlaintext;
- } else {
- state.transportMode = MachineTransportMode.localOffline;
- }
- } else if (state.localOnly) {
- // The token still identifies this as local, but the CLI is offline or
- // failed its identity/capability check. Never fall back to cloud E2EE.
- state.localEndpoint = null;
- state.transportMode = MachineTransportMode.localOffline;
- state.nodeOnline = false;
- _startOfflineRetry(state);
- } else {
- state.localEndpoint = null;
- state.transportMode = MachineTransportMode.cloudE2ee;
- }
+ _applyLocalTransport(state, localEndpoint, localComputerId);
final reportedOnline = _nodeOnlineFromStatus(machine.status);
if (!state.isLocalMachine &&
reportedOnline != null &&
diff --git a/desktop/lib/widgets/machine_rail.dart b/desktop/lib/widgets/machine_rail.dart
index 1fdecfbfe..2dda90352 100644
--- a/desktop/lib/widgets/machine_rail.dart
+++ b/desktop/lib/widgets/machine_rail.dart
@@ -288,6 +288,31 @@ class _MachineRailState extends State<MachineRail> {
],
),
),
+ // Says out loud that these rows are a cached copy. Without it a backend outage looks
+ // identical to a healthy list, and the only clue is machines whose state never changes.
+ if (widget.notifier.machinesAreStale && machines.isNotEmpty)
+ Padding(
+ padding: const EdgeInsets.fromLTRB(16, 8, 16, 0),
+ child: Row(
+ children: [
+ Icon(
+ LucideIcons.cloudOff300,
+ size: 13,
+ color: grid.AppPalette.textSecondary,
+ ),
+ const SizedBox(width: 6),
+ Expanded(
+ child: Text(
+ 'Offline copy — backend unreachable',
+ style: TextStyle(
+ color: grid.AppPalette.textSecondary,
+ fontSize: 11,
+ ),
+ ),
+ ),
+ ],
+ ),
+ ),
Expanded(
child: machines.isEmpty
// Two kinds of empty, and they must not look the same: the
diff --git a/desktop/lib/widgets/pane_grid.dart b/desktop/lib/widgets/pane_grid.dart
index b569b5a4c..3643508c4 100644
--- a/desktop/lib/widgets/pane_grid.dart
+++ b/desktop/lib/widgets/pane_grid.dart
@@ -1381,12 +1381,24 @@ class _PaneContent extends StatelessWidget {
// A never-attached view has no output to preserve: keep its setup guidance.
if (machine == null) {
+ // "Waiting" is only honest while there is still something to wait FOR. When the machine list
+ // itself could not be read, this machine is not slow — it is unknown, and a spinner that never
+ // ends is the wrong answer. Keyed off `machineListError`, not `lastError`: that slot is shared
+ // with agent-launch failures and is cleared by `dismissError`.
+ final listFailed = notifier.machineListError != null;
+ final retrying = notifier.machinesRefreshing;
return _PaneStatus(
title: wantedAgentId ?? pane.machineId,
- icon: Icons.hourglass_empty,
- message: 'Waiting for this machine to answer…',
+ icon: listFailed ? Icons.cloud_off : Icons.hourglass_empty,
+ message: listFailed
+ ? 'Could not reach the Harness backend, so this machine is unknown right now.'
+ : 'Waiting for this machine to answer…',
onClose: single && !swarmMode ? null : close,
- busy: true,
+ busy: !listFailed || retrying,
+ // The automatic recovery is already retrying in the background; this is for someone who does not
+ // want to wait for the next tick. `retryMachines` coalesces, so pressing it during a run joins it.
+ actionLabel: listFailed && !retrying ? 'RETRY' : null,
+ onAction: listFailed ? notifier.retryMachines : null,
);
}
if (needsLink) {
@@ -1863,6 +1875,8 @@ class _PaneStatus extends StatelessWidget {
required this.message,
this.onClose,
this.busy = false,
+ this.actionLabel,
+ this.onAction,
});
final String title;
@@ -1871,6 +1885,11 @@ class _PaneStatus extends StatelessWidget {
final VoidCallback? onClose;
final bool busy;
+ /// An optional way out of the state being described. A pane that is merely waiting has none; one
+ /// reporting a failure the user can retry does, and it reads the same as the error strip's RETRY.
+ final String? actionLabel;
+ final VoidCallback? onAction;
+
@override
Widget build(BuildContext context) {
grid.AppTheme.watch(context);
@@ -1903,6 +1922,10 @@ class _PaneStatus extends StatelessWidget {
fontSize: 11.5,
),
),
+ if (actionLabel != null && onAction != null) ...[
+ const SizedBox(height: 4),
+ TextButton(onPressed: onAction, child: Text(actionLabel!)),
+ ],
],
),
),
diff --git a/desktop/test/machine_list_outage_test.dart b/desktop/test/machine_list_outage_test.dart
new file mode 100644
index 000000000..3d146047c
--- /dev/null
+++ b/desktop/test/machine_list_outage_test.dart
@@ -0,0 +1,213 @@
+// A backend outage must not take this computer's own machine down with it.
+//
+// The machine list is a cloud read; the local CLI endpoint is a loopback probe. They used to be awaited
+// in that order, so a cloud timeout threw before the probe's answer was applied — `usesLocalTransport`
+// went false and `_connectMachine` skipped the LOCAL machine, while relayed machines, which never
+// consult the probe, kept streaming. "Local terminal dead, relayed terminal fine" was the symptom.
+import 'dart:async';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness/api/api_client.dart';
+import 'package:harness/auth/auth_session.dart';
+import 'package:harness/auth/cli_login.dart';
+import 'package:harness/core/config.dart';
+import 'package:harness/core/models.dart';
+import 'package:harness/state/app_state.dart';
+import 'package:harness/ws/local_cli_discovery.dart';
+import 'package:harness/ws/ws_conn.dart';
+
+const _computerId = 'aabbccddeeff0011';
+
+class _Api extends ApiClient {
+ _Api() : super(config: AppConfig.dev, session: AuthSession());
+ final lists = <Completer<List<Machine>>>[];
+
+ /// What the real client reads off the daemon's `stale` marker.
+ bool nextIsStale = false;
+
+ @override
+ Future<Map<String, dynamic>?> me() async => null;
+
+ @override
+ Future<List<Machine>> machines() {
+ final result = Completer<List<Machine>>();
+ lists.add(result);
+ return result.future.then((value) {
+ lastMachinesStale = nextIsStale;
+ return value;
+ });
+ }
+}
+
+class _Cli extends CliLogin {
+ @override
+ Future<CliAuthStatus> checkStatus() async =>
+ const CliAuthStatus(loggedIn: true);
+ @override
+ Future<void> login({void Function(String url)? onAuthorizeUrl}) async {}
+ @override
+ Future<void> logout() async {}
+}
+
+/// A daemon that is up and answering on loopback, whatever the cloud is doing.
+class _Discovery extends LocalCliDiscovery {
+ _Discovery() : super(config: AppConfig.dev);
+ @override
+ Future<String?> computerId() async => _computerId;
+ @override
+ Future<LocalCliEndpoint?> discover({String? expectedComputerId}) async =>
+ LocalCliEndpoint(
+ computerId: _computerId,
+ wsUri: Uri.parse('ws://127.0.0.1:18473/ws'),
+ protocolVersion: localWsProtocolVersion,
+ terminalProtocolVersion: 1,
+ );
+}
+
+class _Connection extends WsConn {
+ _Connection()
+ : super(
+ wsBaseUrl: 'ws://fixture.invalid',
+ autonomousEnv: 'test',
+ machineId: 'local-machine',
+ accessTokenProvider: (_, _) async => '',
+ onAuthFailure: (_) {},
+ onEvent: (_) {},
+ onStatus: (_) {},
+ );
+ @override
+ Future<void> waitUntilReady({required Duration timeout}) async {}
+ @override
+ Future<Map<String, dynamic>> request(
+ String type, {
+ Map<String, dynamic> payload = const {},
+ Duration timeout = const Duration(seconds: 20),
+ }) async => const {'agents': []};
+}
+
+class _App extends AppNotifier {
+ _App(_Api api, _Connection connection)
+ : super(
+ config: AppConfig.dev,
+ authSession: AuthSession(),
+ cliLogin: _Cli(),
+ localCliDiscovery: _Discovery(),
+ connectionForTest: (_) => connection,
+ ) {
+ this.api = api;
+ }
+ @override
+ Future<void> ensureCliDaemonReady() async {}
+}
+
+const _localMachine = Machine(
+ machineId: 'local-machine',
+ name: 'This computer',
+ computerId: _computerId,
+ authMode: MachineAuthMode.remote,
+ status: 'online',
+);
+
+Future<void> _tick() => Future.delayed(Duration.zero);
+
+void main() {
+ late _Api api;
+ late _App app;
+ // Some tests deliberately end with the recovery timer armed; the binding checks for pending timers
+ // when the BODY ends, so those dispose themselves and tell tearDown not to do it twice.
+ var disposedEarly = false;
+
+ setUp(() {
+ disposedEarly = false;
+ api = _Api();
+ app = _App(api, _Connection());
+ app.status = AppStatus.authenticated;
+ // The state a running session is in: this computer's machine, already known to be local.
+ app.machines = [_localMachine];
+ app.machineStates['local-machine'] = MachineState(_localMachine)
+ ..localOnly = true;
+ });
+
+ tearDown(() {
+ if (!disposedEarly) app.dispose();
+ });
+
+ test(
+ 'a failed machine list still applies the loopback endpoint to the local machine',
+ () async {
+ final refresh = app.refreshMachines();
+ await _tick();
+ api.lists.single.completeError(
+ ApiException('Could not reach the Harness backend', status: 502),
+ );
+ await expectLater(refresh, throwsA(isA<ApiException>()));
+ await _tick();
+
+ final local = app.machineStates['local-machine']!;
+ // The probe answered, so this machine can still be reached over loopback — which is exactly what
+ // `_connectMachine` gates the local machine on.
+ expect(local.localEndpoint, isNotNull);
+ expect(local.usesLocalTransport, isTrue);
+ expect(local.transportMode, isNot(MachineTransportMode.cloudE2ee));
+ },
+ );
+
+ test('the machine list failure is reported on its own signal', () async {
+ // Through `retryMachines`, the entry point a user or the recovery timer actually uses — that is
+ // where the failure is turned into a reportable error.
+ final retry = app.retryMachines();
+ await _tick();
+ api.lists.single.completeError(ApiException('boom', status: 502));
+ await retry;
+
+ // The pane affordance keys off this, not off the shared `lastError` slot, which agent-launch
+ // failures also write and `dismissError()` clears.
+ expect(app.machineListError, isNotNull);
+ expect(app.lastErrorRetryable, isTrue);
+ });
+
+ test('a successful list still resolves the local endpoint as before', () async {
+ final refresh = app.refreshMachines();
+ await _tick();
+ api.lists.single.complete([_localMachine]);
+ await refresh;
+ await _tick();
+
+ final local = app.machineStates['local-machine']!;
+ expect(local.localOnly, isTrue);
+ expect(local.localEndpoint, isNotNull);
+ expect(app.machineListError, isNull);
+ });
+
+ test('a cached answer is not the end of the job: it keeps recovering', () async {
+ // The daemon answers 200 from its own cache when the backend is unreachable. Treating that as a
+ // clean success would stop the retry and leave the app on stale rows until somebody pressed reload.
+ api.nextIsStale = true;
+ final retry = app.retryMachines();
+ await _tick();
+ api.lists.single.complete([_localMachine]);
+ await retry;
+ await _tick();
+
+ expect(app.machinesAreStale, isTrue);
+ expect(app.machines, isNotEmpty); // usable rows, so no blocking error
+ expect(app.machineListError, isNull);
+ // Still trying: a timer is armed, which is what converges once the backend returns.
+ expect(app.machineRecoveryPending, isTrue);
+
+ app.dispose();
+ disposedEarly = true;
+ });
+
+ test('a fresh answer ends the job', () async {
+ api.nextIsStale = false;
+ final retry = app.retryMachines();
+ await _tick();
+ api.lists.single.complete([_localMachine]);
+ await retry;
+ await _tick();
+
+ expect(app.machinesAreStale, isFalse);
+ expect(app.machineRecoveryPending, isFalse);
+ });
+}
diff --git a/desktop/test/pane_machine_unknown_test.dart b/desktop/test/pane_machine_unknown_test.dart
new file mode 100644
index 000000000..6446a6153
--- /dev/null
+++ b/desktop/test/pane_machine_unknown_test.dart
@@ -0,0 +1,168 @@
+// What a pane says when its machine is not in the list.
+//
+// "Waiting for this machine to answer…" is only honest while there is something to wait FOR. When the
+// machine LIST itself could not be read, the machine is not slow — it is unknown, and a spinner that
+// never ends tells the user nothing and offers them nothing. This is the difference between those two.
+import 'dart:async';
+
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness/api/api_client.dart';
+import 'package:harness/auth/auth_session.dart';
+import 'package:harness/auth/cli_login.dart';
+import 'package:harness/core/config.dart';
+import 'package:harness/core/models.dart';
+import 'package:harness/state/app_state.dart';
+import 'package:harness/state/terminal_pane.dart';
+import 'package:harness/widgets/pane_grid.dart';
+import 'package:harness/ws/local_cli_discovery.dart';
+
+class _Api extends ApiClient {
+ _Api() : super(config: AppConfig.dev, session: AuthSession());
+ final lists = <Completer<List<Machine>>>[];
+
+ @override
+ Future<Map<String, dynamic>?> me() async => null;
+
+ @override
+ Future<List<Machine>> machines() {
+ final result = Completer<List<Machine>>();
+ lists.add(result);
+ return result.future;
+ }
+}
+
+class _Cli extends CliLogin {
+ @override
+ Future<CliAuthStatus> checkStatus() async =>
+ const CliAuthStatus(loggedIn: true);
+ @override
+ Future<void> login({void Function(String url)? onAuthorizeUrl}) async {}
+ @override
+ Future<void> logout() async {}
+}
+
+class _Discovery extends LocalCliDiscovery {
+ _Discovery() : super(config: AppConfig.dev);
+ @override
+ Future<String?> computerId() async => null;
+ @override
+ Future<LocalCliEndpoint?> discover({String? expectedComputerId}) async =>
+ null;
+}
+
+class _App extends AppNotifier {
+ _App(_Api api)
+ : super(
+ config: AppConfig.dev,
+ authSession: AuthSession(),
+ cliLogin: _Cli(),
+ localCliDiscovery: _Discovery(),
+ configStore: null,
+ ) {
+ this.api = api;
+ }
+ @override
+ Future<void> ensureCliDaemonReady() async {}
+}
+
+const _waiting = 'Waiting for this machine to answer…';
+
+/// Run the refresh chain until it has actually asked for the list.
+///
+/// It must be `pump`, not `await`ing a signal from the stub: inside `testWidgets` the clock is fake, and
+/// awaiting a bare future parks the test on a chain that only `pump` can advance — the whole test then
+/// hangs until its timeout. Pumping frames is what drives it.
+Future<void> _untilRequested(WidgetTester tester, _Api api, int count) async {
+ for (var i = 0; i < 20 && api.lists.length < count; i++) {
+ await tester.pump(Duration.zero);
+ }
+ expect(
+ api.lists.length,
+ greaterThanOrEqualTo(count),
+ reason: 'the machine list was never requested',
+ );
+}
+
+Future<void> _pump(WidgetTester tester, AppNotifier app) async {
+ await tester.pumpWidget(
+ MaterialApp(home: PaneGrid(notifier: app, swarmMode: false)),
+ );
+ await tester.pump();
+}
+
+void main() {
+ late _Api api;
+ late _App app;
+ late bool disposed;
+
+ setUp(() {
+ api = _Api();
+ app = _App(api);
+ disposed = false;
+ app.status = AppStatus.authenticated;
+ // A pane restored from the saved layout: it knows its machineId, and nothing else does yet.
+ app.panes.add(TerminalPane(id: 0, machineId: 'm', agentId: 'a0'));
+ });
+
+ tearDown(() {
+ if (!disposed) app.dispose();
+ });
+
+ // A failed load leaves the automatic recovery timer armed (by design — it is what gets the list back).
+ // The widget-test binding checks for pending timers when the BODY ends, before tearDown runs, so a test
+ // that deliberately ends in the failed state has to close the notifier itself.
+ void end() {
+ app.dispose();
+ disposed = true;
+ }
+
+ testWidgets('keeps waiting while the list has simply not arrived', (
+ tester,
+ ) async {
+ await _pump(tester, app);
+ expect(find.text(_waiting), findsOneWidget);
+ expect(find.text('RETRY'), findsNothing);
+ });
+
+ testWidgets('says the machine is unknown, and offers a retry, once the list fails', (
+ tester,
+ ) async {
+ final retry = app.retryMachines();
+ await _untilRequested(tester, api, 1);
+ api.lists.single.completeError(
+ ApiException('Could not reach the Harness backend', status: 502),
+ );
+ await retry;
+ await _pump(tester, app);
+
+ expect(find.text(_waiting), findsNothing);
+ expect(find.textContaining('this machine is unknown'), findsOneWidget);
+ expect(find.byIcon(Icons.cloud_off), findsOneWidget);
+
+ // And the button reaches the same reload the error strip's RETRY does.
+ expect(api.lists, hasLength(1));
+ await tester.tap(find.text('RETRY'));
+ await _untilRequested(tester, api, 2);
+ end();
+ });
+
+ testWidgets('goes back to waiting once the list loads again', (tester) async {
+ final failed = app.retryMachines();
+ await _untilRequested(tester, api, 1);
+ api.lists.single.completeError(ApiException('boom', status: 502));
+ await failed;
+ await _pump(tester, app);
+ expect(find.byIcon(Icons.cloud_off), findsOneWidget);
+
+ final ok = app.retryMachines();
+ await _untilRequested(tester, api, 2);
+ api.lists.last.complete(const []);
+ await ok;
+ await _pump(tester, app);
+
+ // The list is readable again; this machine is merely absent from it.
+ expect(find.text(_waiting), findsOneWidget);
+ expect(find.text('RETRY'), findsNothing);
+ });
+}