Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

refactor: extract the grid-attach runner, and stop churn from burning its budget

Kelvin7e5ee0ea8725
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 224d12bd5..dccc81eef 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -59,7 +59,7 @@ import { ensureHarnessGrid, type EnsureStatus } from './lib/gridEnsure.js'
import { passThroughToGridLogout } from './lib/gridLogout.js'
import { clearGridMcpUrlCache } from './lib/gridMcpUrl.js'
import { warnIfGridSignInRemains } from './lib/gridCredentials.js'
-import { reconcileGridAttach, gridNamesLocal } from './lib/gridAttach.js'
+import { reconcileGridAttach, gridNamesLocal, createGridAttachRunner } from './lib/gridAttach.js'
import { signedInGridEmail, resetGridDeriveMemo } from './lib/gridDerive.js'
import { forgetGridModels } from './lib/gridModels.js'
import { gridAvailable } from './lib/gridExec.js'
@@ -285,10 +285,15 @@ const GRID_MINT_TIMEOUT_MS = 10_000
* not it has settled — the ceiling that keeps a stuck attempt from degrading every grid RPC for the
* daemon's whole life (`gridReadyProbe`). */
const GRID_ATTACH_CEILING_MS = 30_000
-/** How many grid-attach attempts one daemon makes before giving up until its next start. Retries are
- * driven by backend reconnects, and a link that flaps could otherwise sign in to grid on every one;
- * a machine still unattached after this many reachable moments has a problem retrying won't fix. */
+/** How many grid-attach attempts one daemon makes before giving up until its next start. A machine
+ * still unattached after this many reachable moments has a problem that retrying will not fix, and
+ * each attempt past that is a grid sign-in — and a token rotation — bought for nothing. */
const GRID_ATTACH_MAX_ATTEMPTS = 5
+/** The soonest one attempt may follow another. Retries are driven by backend reconnects, and waking
+ * a laptop, changing network or toggling a VPN produces several within seconds; without this floor
+ * one moment of ordinary churn spent the whole allowance above and the feature went quiet for the
+ * daemon's life. Requests inside the window are deferred to its end, not dropped. */
+const GRID_ATTACH_MIN_INTERVAL_MS = 60_000
/** Between session-binding attempts for a process whose engine store is not resolvable yet. */
const REPAIR_RETRY_MS = 60_000
@@ -1619,21 +1624,15 @@ async function runForeground(session: AuthSession): Promise<void> {
// single attempt that lost to a control plane which was not reachable yet — the ordinary shape of
// a daemon coming up with the network — would leave the account with no grid until the next real
// restart. A connect is the evidence the control plane is reachable, so it is when to try again.
- let gridAttachInFlight: Promise<unknown> | null = null
- let gridAttachDone = false
- let gridAttachAttempts = 0
- // A hard ceiling on how long the RPCs will gate on an attempt, INDEPENDENT of whether it settled.
- // Every step inside an attempt is bounded on its own now — `mintName` by its abort signal, the
- // hand-off and the ensure by their own watchdogs — so this is the backstop rather than the only
- // guard: whatever an attempt does, the RPCs stop paying for it after the ceiling. Past it the
- // probe returns null; an attempt that lands later still publishes the name through `onName`.
- let gridAttachDeadline = 0
-
- const runGridAttach = (): void => {
- if (gridAttachDone || gridAttachInFlight || gridAttachAttempts >= GRID_ATTACH_MAX_ATTEMPTS) return
- gridAttachAttempts += 1
- gridAttachDeadline = Date.now() + GRID_ATTACH_CEILING_MS
- gridAttachInFlight = reconcileGridAttach({
+ // How long the RPCs gate on an attempt, and how many attempts there are, live in the runner —
+ // `lib/gridAttach.ts`, beside the reconcile itself, so the coordination has a unit test rather
+ // than only a comment. Everything below is the daemon-shaped half: what one attempt actually does.
+ const gridAttach = createGridAttachRunner({
+ maxAttempts: GRID_ATTACH_MAX_ATTEMPTS,
+ minIntervalMs: GRID_ATTACH_MIN_INTERVAL_MS,
+ ceilingMs: GRID_ATTACH_CEILING_MS,
+ log: (line) => console.log(`[grid-attach] ${line}`),
+ attempt: () => reconcileGridAttach({
managedGridReady,
gridAvailable: () => gridAvailable(),
// The backend mints and remembers the name; this CLI holds neither the account's email nor its
@@ -1657,21 +1656,14 @@ async function runForeground(session: AuthSession): Promise<void> {
clearGridMcpUrlCache()
},
log: (line) => console.log(`[grid-attach] ${line}`),
- }).then((result) => {
- // Only an outcome that actually attached stops the retries. `no-cli`, `no-name` and
- // `handoff-failed` are all "the control plane or this machine was not ready", which is exactly
- // what a later connect may have fixed.
- if (result.status === 'converged' || result.status === 'signed-in') gridAttachDone = true
- }).catch((err) => {
- console.error('[grid-attach] reconcile failed:', err instanceof Error ? (err.stack ?? err.message) : err)
- }).finally(() => { gridAttachInFlight = null })
- }
+ }),
+ })
// While an attempt is running AND within its ceiling, the RPCs that need the grid name wait
// briefly on it; otherwise they read the name directly.
- backend.gridReadyProbe = () => (gridAttachInFlight && Date.now() < gridAttachDeadline ? gridAttachInFlight : null)
- onBackendConnected = runGridAttach
- runGridAttach()
+ backend.gridReadyProbe = () => gridAttach.probe()
+ onBackendConnected = () => gridAttach.run()
+ gridAttach.run()
/**
* Is ANY device surface watching this machine?
diff --git a/cli/src/lib/gridAttach.spec.ts b/cli/src/lib/gridAttach.spec.ts
index 576e12f4a..0dc1cd96c 100644
--- a/cli/src/lib/gridAttach.spec.ts
+++ b/cli/src/lib/gridAttach.spec.ts
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from 'vitest'
-import { reconcileGridAttach, type GridAttachDeps } from './gridAttach.js'
+import { reconcileGridAttach, createGridAttachRunner, type GridAttachDeps, type GridAttachResult } from './gridAttach.js'
import type { GridHandoffResult } from './gridHandoff.js'
import type { EnsureResult } from './gridEnsure.js'
@@ -161,3 +161,145 @@ describe('reconcileGridAttach — the daemon-start convergence', () => {
await expect(reconcileGridAttach(d)).resolves.toMatchObject({ status: 'no-cli' })
})
})
+
+/**
+ * The coordination half: WHEN a reconcile runs. The daemon calls `run()` at start and again on every
+ * backend reconnect, so every property here is about the interaction of those calls.
+ */
+describe('createGridAttachRunner — when a reconcile runs', () => {
+ const result = (status: GridAttachResult['status']): GridAttachResult => ({ status, name: NAME, detail: '' })
+
+ /** A clock the test moves by hand, so nothing here waits on real time. */
+ function clock(start = 1_000_000) {
+ let t = start
+ return { now: () => t, advance: (ms: number) => { t += ms } }
+ }
+
+ function runner(over: Partial<Parameters<typeof createGridAttachRunner>[0]> = {}, c = clock()) {
+ const attempt = vi.fn(async () => result('converged'))
+ const logs: string[] = []
+ const r = createGridAttachRunner({
+ attempt,
+ maxAttempts: 3,
+ minIntervalMs: 1_000,
+ ceilingMs: 5_000,
+ now: c.now,
+ log: (line) => logs.push(line),
+ ...over,
+ })
+ return { r, attempt: (over.attempt ?? attempt) as ReturnType<typeof vi.fn>, logs, clock: c }
+ }
+
+ it('runs once and stops once attached — a later reconnect costs nothing', async () => {
+ const { r, attempt, clock: c } = runner()
+
+ r.run()
+ await r.probe()
+ expect(attempt).toHaveBeenCalledOnce()
+
+ c.advance(10_000) // well past the interval, so only "already attached" can stop a second run
+ r.run()
+ expect(attempt).toHaveBeenCalledOnce()
+ })
+
+ it('does not overlap: a reconnect during an in-flight attempt is a no-op', async () => {
+ let release = (): void => {}
+ const gate = new Promise<void>((res) => { release = res })
+ const attempt = vi.fn(async () => { await gate; return result('converged') })
+ const { r, clock: c } = runner({ attempt })
+
+ r.run()
+ c.advance(10_000)
+ r.run()
+ r.run()
+ expect(attempt).toHaveBeenCalledOnce()
+
+ release()
+ await r.probe()
+ expect(attempt).toHaveBeenCalledOnce()
+ })
+
+ it('retries after an attempt that did not attach, once the interval has passed', async () => {
+ const attempt = vi.fn(async () => result('no-name'))
+ const { r, clock: c } = runner({ attempt })
+
+ r.run()
+ await r.probe()
+ expect(attempt).toHaveBeenCalledOnce()
+
+ c.advance(1_000)
+ r.run()
+ await r.probe()
+ expect(attempt).toHaveBeenCalledTimes(2)
+ })
+
+ it('collapses a burst of reconnects into ONE deferred attempt instead of burning the budget', async () => {
+ vi.useFakeTimers()
+ try {
+ const attempt = vi.fn(async () => result('no-name'))
+ // A real clock is not advanced by fake timers, so the runner reads the same injected one.
+ const c = clock()
+ const { r } = runner({ attempt }, c)
+
+ r.run()
+ await r.probe()
+ expect(attempt).toHaveBeenCalledTimes(1)
+
+ // Waking a laptop: several reconnects inside the interval. None may start an attempt, and
+ // together they must cost exactly one — the whole point of deferring rather than dropping.
+ for (let i = 0; i < 5; i++) r.run()
+ expect(attempt).toHaveBeenCalledTimes(1)
+ expect(r.attempts()).toBe(1)
+
+ c.advance(1_000)
+ await vi.advanceTimersByTimeAsync(1_000)
+ expect(attempt).toHaveBeenCalledTimes(2)
+ // Four attempts' worth of churn cost one, so the cap of 3 is not spent.
+ expect(r.attempts()).toBe(2)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('gives up out loud at the cap, and says it only once', async () => {
+ const attempt = vi.fn(async () => result('no-name'))
+ const { r, logs, clock: c } = runner({ attempt })
+
+ for (let i = 0; i < 3; i++) {
+ c.advance(1_000)
+ r.run()
+ await r.probe()
+ }
+ expect(attempt).toHaveBeenCalledTimes(3)
+
+ c.advance(1_000)
+ r.run()
+ c.advance(1_000)
+ r.run()
+ expect(attempt).toHaveBeenCalledTimes(3)
+ expect(logs.filter((l) => l.includes('giving up'))).toHaveLength(1)
+ })
+
+ it('a rejected attempt is logged and retried, never thrown', async () => {
+ const attempt = vi.fn(async () => { throw new Error('boom') })
+ const { r, logs, clock: c } = runner({ attempt })
+
+ expect(() => r.run()).not.toThrow()
+ await r.probe()
+ expect(logs.some((l) => l.includes('boom'))).toBe(true)
+
+ c.advance(1_000)
+ r.run()
+ expect(attempt).toHaveBeenCalledTimes(2)
+ })
+
+ it('probe stops offering the attempt once its ceiling passes, so an RPC never waits on a stuck one', async () => {
+ const attempt = vi.fn(async () => new Promise<GridAttachResult>(() => {})) // never settles
+ const { r, clock: c } = runner({ attempt })
+
+ r.run()
+ expect(r.probe()).not.toBeNull()
+ c.advance(5_000)
+ expect(r.probe()).toBeNull()
+ })
+})
diff --git a/cli/src/lib/gridAttach.ts b/cli/src/lib/gridAttach.ts
index 2f6831f13..326ec0677 100644
--- a/cli/src/lib/gridAttach.ts
+++ b/cli/src/lib/gridAttach.ts
@@ -174,6 +174,93 @@ export async function reconcileGridAttach(deps: GridAttachDeps): Promise<GridAtt
* and makes no network call (autonomous-grid `cli/remote_grid.py`), so the reconcile's gate stays
* cheap. An unparsable answer is no names, which the caller treats as "ensure it".
*/
+export interface GridAttachRunnerOptions {
+ /** Runs one reconcile. Injected so the coordination below is testable without a daemon. */
+ attempt: () => Promise<GridAttachResult>
+ /** How many attempts this runner will ever make before it stops trying. */
+ maxAttempts: number
+ /** The soonest a new attempt may START after the previous one did. */
+ minIntervalMs: number
+ /** How long an in-flight attempt is worth making an RPC wait for, from when it started. */
+ ceilingMs: number
+ /** Injected clock, for tests. */
+ now?: () => number
+ log: (line: string) => void
+}
+
+export interface GridAttachRunner {
+ /** Start an attempt if one is warranted. Safe to call as often as a reconnect fires. */
+ run: () => void
+ /** The in-flight attempt while it is still worth waiting for, else null. */
+ probe: () => Promise<unknown> | null
+ /** How many attempts have been started. Diagnostics and tests. */
+ attempts: () => number
+}
+
+/**
+ * Decides WHEN to reconcile, as opposed to {@link reconcileGridAttach}, which decides what to do.
+ *
+ * Its own unit because the daemon calls `run()` from two places — once at start, then on every
+ * backend reconnect — and every interesting property is about the interaction of those calls: two
+ * attempts must not overlap, a burst of reconnects must not turn into a burst of grid sign-ins, and
+ * the whole thing must eventually stop rather than rotate this account's tokens forever.
+ *
+ * **A burst collapses into one attempt, it does not burn the budget.** A laptop waking, changing
+ * network or toggling a VPN produces several reconnects in seconds. Counting each as an attempt
+ * spent the entire allowance on one moment of ordinary churn and then went quiet for the daemon's
+ * life — which can be days. Requests inside `minIntervalMs` are therefore DEFERRED to the end of
+ * that window rather than dropped, so the churn yields exactly one attempt and the opportunity is
+ * not lost either. The timer is unref'd: it must never be a reason a process stays alive.
+ *
+ * **It gives up out loud.** Reaching the cap says so once. Silence there was indistinguishable from
+ * a feature that was working.
+ */
+export function createGridAttachRunner(opts: GridAttachRunnerOptions): GridAttachRunner {
+ const now = opts.now ?? Date.now
+ let inFlight: Promise<unknown> | null = null
+ let deferred: ReturnType<typeof setTimeout> | null = null
+ let done = false
+ let attempts = 0
+ let lastStartedAt = Number.NEGATIVE_INFINITY
+ let deadline = 0
+ let saidGaveUp = false
+
+ const run = (): void => {
+ if (done || inFlight || deferred) return
+ if (attempts >= opts.maxAttempts) {
+ if (!saidGaveUp) {
+ saidGaveUp = true
+ opts.log(`no grid after ${attempts} attempts — giving up until this machine restarts, or you sign in to Harness again`)
+ }
+ return
+ }
+ const wait = opts.minIntervalMs - (now() - lastStartedAt)
+ if (wait > 0) {
+ deferred = setTimeout(() => { deferred = null; run() }, wait)
+ deferred.unref?.()
+ return
+ }
+ attempts += 1
+ lastStartedAt = now()
+ deadline = lastStartedAt + opts.ceilingMs
+ inFlight = opts.attempt()
+ .then((result) => {
+ // Only an outcome that actually attached stops the retries. `no-cli`, `no-name` and
+ // `handoff-failed` are all "this machine or the control plane was not ready", which is
+ // exactly what a later connect may have fixed.
+ if (result.status === 'converged' || result.status === 'signed-in') done = true
+ })
+ .catch((err) => { opts.log(`attempt failed: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`) })
+ .finally(() => { inFlight = null })
+ }
+
+ return {
+ run,
+ probe: () => (inFlight && now() < deadline ? inFlight : null),
+ attempts: () => attempts,
+ }
+}
+
export async function gridNamesLocal(): Promise<string[]> {
const { value, result } = await gridJson<Array<{ grid?: unknown }>>(['--remote', 'ls'])
// ⚠️ A FAILED read is NOT an empty list, and conflating the two is how "this machine has no grids"
diff --git a/cli/src/lib/gridHandoff.spec.ts b/cli/src/lib/gridHandoff.spec.ts
index 3f7e83091..c865ea384 100644
--- a/cli/src/lib/gridHandoff.spec.ts
+++ b/cli/src/lib/gridHandoff.spec.ts
@@ -112,11 +112,16 @@ describe('handOffToGrid — which grid it runs', () => {
* login` that never returned, and a daemon reconcile that never settled.
*/
describe('handOffToGrid — a child that never answers', () => {
- /** A `grid` that reads the token and then hangs, the shape of a black-holed control plane. */
+ /** A `grid` that reads the token and then hangs, the shape of a black-holed control plane.
+ *
+ * It IGNORES SIGTERM on purpose: that is the case the watchdog's `SIGKILL` exists for, and a fake
+ * that died politely would let a child which cannot be asked to stop pass as one that can. (The
+ * signal itself is not observable through this module's result — the kill and the settle happen
+ * together — so what this pins is that a TERM-immune child still ends the call.) */
function hangingGrid(dir: string): string {
mkdirSync(dir, { recursive: true })
const bin = join(dir, 'grid')
- writeFileSync(bin, ['#!/bin/sh', '/bin/cat > /dev/null', 'sleep 60', ''].join('\n'), { mode: 0o755 })
+ writeFileSync(bin, ['#!/bin/sh', "trap '' TERM", '/bin/cat > /dev/null', 'sleep 60', ''].join('\n'), { mode: 0o755 })
return bin
}