The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
refactor: extract the grid-attach runner, and stop churn from burning its budget
Kelvin7e5ee0ea8725
diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex 224d12bd5..dccc81eef 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -59,7 +59,7 @@ import { ensureHarnessGrid, type EnsureStatus } from './lib/gridEnsure.js'import { passThroughToGridLogout } from './lib/gridLogout.js'import { clearGridMcpUrlCache } from './lib/gridMcpUrl.js'import { warnIfGridSignInRemains } from './lib/gridCredentials.js'-import { reconcileGridAttach, gridNamesLocal } from './lib/gridAttach.js'+import { reconcileGridAttach, gridNamesLocal, createGridAttachRunner } from './lib/gridAttach.js'import { signedInGridEmail, resetGridDeriveMemo } from './lib/gridDerive.js'import { forgetGridModels } from './lib/gridModels.js'import { gridAvailable } from './lib/gridExec.js'@@ -285,10 +285,15 @@ const GRID_MINT_TIMEOUT_MS = 10_000* not it has settled — the ceiling that keeps a stuck attempt from degrading every grid RPC for the* daemon's whole life (`gridReadyProbe`). */const GRID_ATTACH_CEILING_MS = 30_000-/** How many grid-attach attempts one daemon makes before giving up until its next start. Retries are- * driven by backend reconnects, and a link that flaps could otherwise sign in to grid on every one;- * a machine still unattached after this many reachable moments has a problem retrying won't fix. */+/** How many grid-attach attempts one daemon makes before giving up until its next start. A machine+ * still unattached after this many reachable moments has a problem that retrying will not fix, and+ * each attempt past that is a grid sign-in — and a token rotation — bought for nothing. */const GRID_ATTACH_MAX_ATTEMPTS = 5+/** The soonest one attempt may follow another. Retries are driven by backend reconnects, and waking+ * a laptop, changing network or toggling a VPN produces several within seconds; without this floor+ * one moment of ordinary churn spent the whole allowance above and the feature went quiet for the+ * daemon's life. Requests inside the window are deferred to its end, not dropped. */+const GRID_ATTACH_MIN_INTERVAL_MS = 60_000/** Between session-binding attempts for a process whose engine store is not resolvable yet. */const REPAIR_RETRY_MS = 60_000@@ -1619,21 +1624,15 @@ async function runForeground(session: AuthSession): Promise<void> {// single attempt that lost to a control plane which was not reachable yet — the ordinary shape of// a daemon coming up with the network — would leave the account with no grid until the next real// restart. A connect is the evidence the control plane is reachable, so it is when to try again.- let gridAttachInFlight: Promise<unknown> | null = null- let gridAttachDone = false- let gridAttachAttempts = 0- // A hard ceiling on how long the RPCs will gate on an attempt, INDEPENDENT of whether it settled.- // Every step inside an attempt is bounded on its own now — `mintName` by its abort signal, the- // hand-off and the ensure by their own watchdogs — so this is the backstop rather than the only- // guard: whatever an attempt does, the RPCs stop paying for it after the ceiling. Past it the- // probe returns null; an attempt that lands later still publishes the name through `onName`.- let gridAttachDeadline = 0-- const runGridAttach = (): void => {- if (gridAttachDone || gridAttachInFlight || gridAttachAttempts >= GRID_ATTACH_MAX_ATTEMPTS) return- gridAttachAttempts += 1- gridAttachDeadline = Date.now() + GRID_ATTACH_CEILING_MS- gridAttachInFlight = reconcileGridAttach({+ // How long the RPCs gate on an attempt, and how many attempts there are, live in the runner —+ // `lib/gridAttach.ts`, beside the reconcile itself, so the coordination has a unit test rather+ // than only a comment. Everything below is the daemon-shaped half: what one attempt actually does.+ const gridAttach = createGridAttachRunner({+ maxAttempts: GRID_ATTACH_MAX_ATTEMPTS,+ minIntervalMs: GRID_ATTACH_MIN_INTERVAL_MS,+ ceilingMs: GRID_ATTACH_CEILING_MS,+ log: (line) => console.log(`[grid-attach] ${line}`),+ attempt: () => reconcileGridAttach({managedGridReady,gridAvailable: () => gridAvailable(),// The backend mints and remembers the name; this CLI holds neither the account's email nor its@@ -1657,21 +1656,14 @@ async function runForeground(session: AuthSession): Promise<void> {clearGridMcpUrlCache()},log: (line) => console.log(`[grid-attach] ${line}`),- }).then((result) => {- // Only an outcome that actually attached stops the retries. `no-cli`, `no-name` and- // `handoff-failed` are all "the control plane or this machine was not ready", which is exactly- // what a later connect may have fixed.- if (result.status === 'converged' || result.status === 'signed-in') gridAttachDone = true- }).catch((err) => {- console.error('[grid-attach] reconcile failed:', err instanceof Error ? (err.stack ?? err.message) : err)- }).finally(() => { gridAttachInFlight = null })- }+ }),+ })// While an attempt is running AND within its ceiling, the RPCs that need the grid name wait// briefly on it; otherwise they read the name directly.- backend.gridReadyProbe = () => (gridAttachInFlight && Date.now() < gridAttachDeadline ? gridAttachInFlight : null)- onBackendConnected = runGridAttach- runGridAttach()+ backend.gridReadyProbe = () => gridAttach.probe()+ onBackendConnected = () => gridAttach.run()+ gridAttach.run()/*** Is ANY device surface watching this machine?diff --git a/cli/src/lib/gridAttach.spec.ts b/cli/src/lib/gridAttach.spec.tsindex 576e12f4a..0dc1cd96c 100644--- a/cli/src/lib/gridAttach.spec.ts+++ b/cli/src/lib/gridAttach.spec.ts@@ -1,5 +1,5 @@import { describe, expect, it, vi } from 'vitest'-import { reconcileGridAttach, type GridAttachDeps } from './gridAttach.js'+import { reconcileGridAttach, createGridAttachRunner, type GridAttachDeps, type GridAttachResult } from './gridAttach.js'import type { GridHandoffResult } from './gridHandoff.js'import type { EnsureResult } from './gridEnsure.js'@@ -161,3 +161,145 @@ describe('reconcileGridAttach — the daemon-start convergence', () => {await expect(reconcileGridAttach(d)).resolves.toMatchObject({ status: 'no-cli' })})})++/**+ * The coordination half: WHEN a reconcile runs. The daemon calls `run()` at start and again on every+ * backend reconnect, so every property here is about the interaction of those calls.+ */+describe('createGridAttachRunner — when a reconcile runs', () => {+ const result = (status: GridAttachResult['status']): GridAttachResult => ({ status, name: NAME, detail: '' })++ /** A clock the test moves by hand, so nothing here waits on real time. */+ function clock(start = 1_000_000) {+ let t = start+ return { now: () => t, advance: (ms: number) => { t += ms } }+ }++ function runner(over: Partial<Parameters<typeof createGridAttachRunner>[0]> = {}, c = clock()) {+ const attempt = vi.fn(async () => result('converged'))+ const logs: string[] = []+ const r = createGridAttachRunner({+ attempt,+ maxAttempts: 3,+ minIntervalMs: 1_000,+ ceilingMs: 5_000,+ now: c.now,+ log: (line) => logs.push(line),+ ...over,+ })+ return { r, attempt: (over.attempt ?? attempt) as ReturnType<typeof vi.fn>, logs, clock: c }+ }++ it('runs once and stops once attached — a later reconnect costs nothing', async () => {+ const { r, attempt, clock: c } = runner()++ r.run()+ await r.probe()+ expect(attempt).toHaveBeenCalledOnce()++ c.advance(10_000) // well past the interval, so only "already attached" can stop a second run+ r.run()+ expect(attempt).toHaveBeenCalledOnce()+ })++ it('does not overlap: a reconnect during an in-flight attempt is a no-op', async () => {+ let release = (): void => {}+ const gate = new Promise<void>((res) => { release = res })+ const attempt = vi.fn(async () => { await gate; return result('converged') })+ const { r, clock: c } = runner({ attempt })++ r.run()+ c.advance(10_000)+ r.run()+ r.run()+ expect(attempt).toHaveBeenCalledOnce()++ release()+ await r.probe()+ expect(attempt).toHaveBeenCalledOnce()+ })++ it('retries after an attempt that did not attach, once the interval has passed', async () => {+ const attempt = vi.fn(async () => result('no-name'))+ const { r, clock: c } = runner({ attempt })++ r.run()+ await r.probe()+ expect(attempt).toHaveBeenCalledOnce()++ c.advance(1_000)+ r.run()+ await r.probe()+ expect(attempt).toHaveBeenCalledTimes(2)+ })++ it('collapses a burst of reconnects into ONE deferred attempt instead of burning the budget', async () => {+ vi.useFakeTimers()+ try {+ const attempt = vi.fn(async () => result('no-name'))+ // A real clock is not advanced by fake timers, so the runner reads the same injected one.+ const c = clock()+ const { r } = runner({ attempt }, c)++ r.run()+ await r.probe()+ expect(attempt).toHaveBeenCalledTimes(1)++ // Waking a laptop: several reconnects inside the interval. None may start an attempt, and+ // together they must cost exactly one — the whole point of deferring rather than dropping.+ for (let i = 0; i < 5; i++) r.run()+ expect(attempt).toHaveBeenCalledTimes(1)+ expect(r.attempts()).toBe(1)++ c.advance(1_000)+ await vi.advanceTimersByTimeAsync(1_000)+ expect(attempt).toHaveBeenCalledTimes(2)+ // Four attempts' worth of churn cost one, so the cap of 3 is not spent.+ expect(r.attempts()).toBe(2)+ } finally {+ vi.useRealTimers()+ }+ })++ it('gives up out loud at the cap, and says it only once', async () => {+ const attempt = vi.fn(async () => result('no-name'))+ const { r, logs, clock: c } = runner({ attempt })++ for (let i = 0; i < 3; i++) {+ c.advance(1_000)+ r.run()+ await r.probe()+ }+ expect(attempt).toHaveBeenCalledTimes(3)++ c.advance(1_000)+ r.run()+ c.advance(1_000)+ r.run()+ expect(attempt).toHaveBeenCalledTimes(3)+ expect(logs.filter((l) => l.includes('giving up'))).toHaveLength(1)+ })++ it('a rejected attempt is logged and retried, never thrown', async () => {+ const attempt = vi.fn(async () => { throw new Error('boom') })+ const { r, logs, clock: c } = runner({ attempt })++ expect(() => r.run()).not.toThrow()+ await r.probe()+ expect(logs.some((l) => l.includes('boom'))).toBe(true)++ c.advance(1_000)+ r.run()+ expect(attempt).toHaveBeenCalledTimes(2)+ })++ it('probe stops offering the attempt once its ceiling passes, so an RPC never waits on a stuck one', async () => {+ const attempt = vi.fn(async () => new Promise<GridAttachResult>(() => {})) // never settles+ const { r, clock: c } = runner({ attempt })++ r.run()+ expect(r.probe()).not.toBeNull()+ c.advance(5_000)+ expect(r.probe()).toBeNull()+ })+})diff --git a/cli/src/lib/gridAttach.ts b/cli/src/lib/gridAttach.tsindex 2f6831f13..326ec0677 100644--- a/cli/src/lib/gridAttach.ts+++ b/cli/src/lib/gridAttach.ts@@ -174,6 +174,93 @@ export async function reconcileGridAttach(deps: GridAttachDeps): Promise<GridAtt* and makes no network call (autonomous-grid `cli/remote_grid.py`), so the reconcile's gate stays* cheap. An unparsable answer is no names, which the caller treats as "ensure it".*/+export interface GridAttachRunnerOptions {+ /** Runs one reconcile. Injected so the coordination below is testable without a daemon. */+ attempt: () => Promise<GridAttachResult>+ /** How many attempts this runner will ever make before it stops trying. */+ maxAttempts: number+ /** The soonest a new attempt may START after the previous one did. */+ minIntervalMs: number+ /** How long an in-flight attempt is worth making an RPC wait for, from when it started. */+ ceilingMs: number+ /** Injected clock, for tests. */+ now?: () => number+ log: (line: string) => void+}++export interface GridAttachRunner {+ /** Start an attempt if one is warranted. Safe to call as often as a reconnect fires. */+ run: () => void+ /** The in-flight attempt while it is still worth waiting for, else null. */+ probe: () => Promise<unknown> | null+ /** How many attempts have been started. Diagnostics and tests. */+ attempts: () => number+}++/**+ * Decides WHEN to reconcile, as opposed to {@link reconcileGridAttach}, which decides what to do.+ *+ * Its own unit because the daemon calls `run()` from two places — once at start, then on every+ * backend reconnect — and every interesting property is about the interaction of those calls: two+ * attempts must not overlap, a burst of reconnects must not turn into a burst of grid sign-ins, and+ * the whole thing must eventually stop rather than rotate this account's tokens forever.+ *+ * **A burst collapses into one attempt, it does not burn the budget.** A laptop waking, changing+ * network or toggling a VPN produces several reconnects in seconds. Counting each as an attempt+ * spent the entire allowance on one moment of ordinary churn and then went quiet for the daemon's+ * life — which can be days. Requests inside `minIntervalMs` are therefore DEFERRED to the end of+ * that window rather than dropped, so the churn yields exactly one attempt and the opportunity is+ * not lost either. The timer is unref'd: it must never be a reason a process stays alive.+ *+ * **It gives up out loud.** Reaching the cap says so once. Silence there was indistinguishable from+ * a feature that was working.+ */+export function createGridAttachRunner(opts: GridAttachRunnerOptions): GridAttachRunner {+ const now = opts.now ?? Date.now+ let inFlight: Promise<unknown> | null = null+ let deferred: ReturnType<typeof setTimeout> | null = null+ let done = false+ let attempts = 0+ let lastStartedAt = Number.NEGATIVE_INFINITY+ let deadline = 0+ let saidGaveUp = false++ const run = (): void => {+ if (done || inFlight || deferred) return+ if (attempts >= opts.maxAttempts) {+ if (!saidGaveUp) {+ saidGaveUp = true+ opts.log(`no grid after ${attempts} attempts — giving up until this machine restarts, or you sign in to Harness again`)+ }+ return+ }+ const wait = opts.minIntervalMs - (now() - lastStartedAt)+ if (wait > 0) {+ deferred = setTimeout(() => { deferred = null; run() }, wait)+ deferred.unref?.()+ return+ }+ attempts += 1+ lastStartedAt = now()+ deadline = lastStartedAt + opts.ceilingMs+ inFlight = opts.attempt()+ .then((result) => {+ // Only an outcome that actually attached stops the retries. `no-cli`, `no-name` and+ // `handoff-failed` are all "this machine or the control plane was not ready", which is+ // exactly what a later connect may have fixed.+ if (result.status === 'converged' || result.status === 'signed-in') done = true+ })+ .catch((err) => { opts.log(`attempt failed: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`) })+ .finally(() => { inFlight = null })+ }++ return {+ run,+ probe: () => (inFlight && now() < deadline ? inFlight : null),+ attempts: () => attempts,+ }+}+export async function gridNamesLocal(): Promise<string[]> {const { value, result } = await gridJson<Array<{ grid?: unknown }>>(['--remote', 'ls'])// ⚠️ A FAILED read is NOT an empty list, and conflating the two is how "this machine has no grids"diff --git a/cli/src/lib/gridHandoff.spec.ts b/cli/src/lib/gridHandoff.spec.tsindex 3f7e83091..c865ea384 100644--- a/cli/src/lib/gridHandoff.spec.ts+++ b/cli/src/lib/gridHandoff.spec.ts@@ -112,11 +112,16 @@ describe('handOffToGrid — which grid it runs', () => {* login` that never returned, and a daemon reconcile that never settled.*/describe('handOffToGrid — a child that never answers', () => {- /** A `grid` that reads the token and then hangs, the shape of a black-holed control plane. */+ /** A `grid` that reads the token and then hangs, the shape of a black-holed control plane.+ *+ * It IGNORES SIGTERM on purpose: that is the case the watchdog's `SIGKILL` exists for, and a fake+ * that died politely would let a child which cannot be asked to stop pass as one that can. (The+ * signal itself is not observable through this module's result — the kill and the settle happen+ * together — so what this pins is that a TERM-immune child still ends the call.) */function hangingGrid(dir: string): string {mkdirSync(dir, { recursive: true })const bin = join(dir, 'grid')- writeFileSync(bin, ['#!/bin/sh', '/bin/cat > /dev/null', 'sleep 60', ''].join('\n'), { mode: 0o755 })+ writeFileSync(bin, ['#!/bin/sh', "trap '' TERM", '/bin/cat > /dev/null', 'sleep 60', ''].join('\n'), { mode: 0o755 })return bin}