Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: the harness reads grids without waking them

Kelvin7715ca701ee4
diff --git a/cli/src/backendSocket.gridReads.spec.ts b/cli/src/backendSocket.gridReads.spec.ts
new file mode 100644
index 000000000..ef559098e
--- /dev/null
+++ b/cli/src/backendSocket.gridReads.spec.ts
@@ -0,0 +1,155 @@
+/**
+ * The daemon's grid RPCs as a desktop drives them — `grid_models_list`, the Model Manager's
+ * `grid_fleet_models_list` and `agent_retarget` — against a fake relay that records every request it
+ * receives, and a fake `grid` first on PATH (grid-reads-without-waking issue 02).
+ *
+ * The relay is the spy: nothing here recomputes what the daemon sends, it reads what arrived.
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { createServer, type IncomingHttpHeaders, type Server } from 'node:http'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { BackendSocket } from './backendSocket.js'
+import { env } from './config/env.js'
+import { fakeGridAnswers, installFakeGrid, type FakeGrid, type FakeGridPlan } from './lib/__fixtures__/fakeGrid.js'
+import { clearGridMcpUrlCache } from './lib/gridMcpUrl.js'
+import { resetGridModels, type GridModelsService } from './lib/gridModels.js'
+
+const OWN = 'mine', OWN_ID = 'net-own'
+const OVERVIEW = '/relay/v1/grid/overview'
+
+let server: Server, base: string, seen: Array<{ path: string; headers: IncomingHttpHeaders }>
+let overview: { status: number; body: unknown }
+let root: string, gridHome: string, previousData: string, grid: FakeGrid, service: GridModelsService, clock: number
+let socket: BackendSocket, frames: Array<{ type: string; payload: Record<string, unknown> }>
+
+function plan(status: string | null = 'running'): FakeGridPlan {
+ return {
+ ls: { stdout: JSON.stringify([{ grid: OWN, id: OWN_ID, type: 'permissioned-public' }]) },
+ [`info ${OWN}`]: { stdout: JSON.stringify({ grid: OWN, status, grid_url: `${base}/g/${OWN_ID}` }) },
+ }
+}
+
+async function ask(type: string, payload: Record<string, unknown> = {}): Promise<Record<string, unknown>> {
+ const requestId = `${type}-${frames.length}`
+ socket.handleLocalFrame('local:grid-reads', { type, payload: { ...payload, requestId } })
+ await vi.waitFor(() => expect(frames.some((f) => f.payload.requestId === requestId)).toBe(true), { timeout: 10_000 })
+ return frames.find((f) => f.payload.requestId === requestId)!.payload
+}
+
+function expectNoCredential(): void {
+ for (const request of seen) {
+ expect(request.headers.authorization).toBeUndefined()
+ expect(request.headers['x-api-key']).toBeUndefined()
+ }
+}
+
+beforeEach(async () => {
+ seen = []
+ overview = { status: 200, body: { nodes: [{ name: 'mac', engine: 'llama.cpp', models: ['small-q4'], online: true }], models: [{ id: 'Small-Q4' }] } }
+ server = createServer((req, res) => {
+ seen.push({ path: req.url ?? '', headers: req.headers })
+ const found = req.url === `/g/${OWN_ID}${OVERVIEW}` ? overview : { status: 404, body: {} }
+ res.writeHead(found.status, { 'content-type': 'application/json' })
+ res.end(JSON.stringify(found.body))
+ })
+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
+ base = `http://127.0.0.1:${(server.address() as { port: number }).port}`
+ root = mkdtempSync(join(tmpdir(), 'grid-reads-rpc-'))
+ gridHome = join(root, 'grid-home')
+ mkdirSync(gridHome, { recursive: true })
+ vi.stubEnv('GRID_HOME', gridHome)
+ previousData = env.ADAPTER_DATA_DIR
+ env.ADAPTER_DATA_DIR = join(root, 'data')
+ clock = Date.parse('2026-09-24T10:00:00Z')
+ // Before the socket, which subscribes to the service it finds for its `grid_models_changed` push.
+ service = resetGridModels({ now: () => clock, dataDir: () => join(root, 'data'), gridHome: () => gridHome, email: () => null })
+ grid = installFakeGrid(plan())
+ socket = new BackendSocket('token')
+ socket.setHarnessGridName(OWN)
+ frames = []
+ socket.registerLocalClient('local:grid-reads', { sendFrame: (frame) => { frames.push(frame as typeof frames[number]); return true }, sendBinary: () => true })
+})
+
+afterEach(async () => {
+ await service.settled()
+ await socket.unregisterLocalClient('local:grid-reads')
+ await socket.stop()
+ grid.dispose()
+ clearGridMcpUrlCache()
+ env.ADAPTER_DATA_DIR = previousData
+ vi.unstubAllEnvs()
+ server.closeAllConnections()
+ await new Promise<void>((resolve) => server.close(() => resolve()))
+ rmSync(root, { recursive: true, force: true })
+})
+
+describe('grid_models_list', () => {
+ it('answers in the old shape with the three new fields, and reads the grid with no credential', async () => {
+ const reply = await ask('grid_models_list')
+
+ expect(reply).toMatchObject({ gridName: OWN, models: [{ id: 'Small-Q4', node: 'mac' }] })
+ expect(reply.grids).toEqual([{
+ name: OWN, type: 'permissioned-public', own: true, models: [{ id: 'Small-Q4', node: 'mac' }],
+ state: 'awake', seenAt: new Date(clock).toISOString(), lastKnownAge: 0,
+ }])
+ expect(seen.map((r) => r.path)).toEqual([`/g/${OWN_ID}${OVERVIEW}`])
+ expectNoCredential()
+ })
+
+ it('pushes grid_models_changed when a read it did not wait for changes the list', async () => {
+ await ask('grid_models_list')
+ clock += 20_000
+ overview = { status: 200, body: { nodes: [{ name: 'mac', engine: 'llama.cpp', models: ['small-q4', 'big-q8'] }], models: [] } }
+
+ const reply = await ask('grid_models_list')
+ // Answered at once from the picture; the read it started lands behind it and is pushed.
+ expect((reply.models as unknown[]).length).toBe(1)
+ await vi.waitFor(() => expect(frames.some((f) => f.type === 'grid_models_changed')).toBe(true), { timeout: 5_000 })
+
+ const pushed = frames.find((f) => f.type === 'grid_models_changed')!.payload
+ expect(pushed).not.toHaveProperty('requestId')
+ expect(pushed).toMatchObject({ gridName: OWN, models: [{ id: 'Small-Q4', node: 'mac' }, { id: 'big-q8', node: 'mac' }], supportsModelLaunch: true })
+ // Two overview reads, and one of provider discovery: `big-q8` had no spelling from any other source.
+ expect(seen.map((r) => r.path)).toEqual([`/g/${OWN_ID}${OVERVIEW}`, `/g/${OWN_ID}${OVERVIEW}`, `/g/${OWN_ID}/nodes/discover`])
+ expectNoCredential()
+ })
+})
+
+describe('grid_fleet_models_list (the Model Manager) on a sleeping own grid', () => {
+ it('is not an inventory error, keeps Start and Pause enabled, and reports a parked engine running', async () => {
+ grid.replan(plan('asleep'))
+ overview = { status: 503, body: { detail: 'resting', code: 'grid_asleep' } }
+ const records = join(gridHome, 'run', 'engines', OWN_ID)
+ mkdirSync(records, { recursive: true })
+ writeFileSync(join(records, 'remote.json'), JSON.stringify({
+ node_id: 'node-1', meta_name: 'mac', pid: process.pid, engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'] }], advertise_as: [],
+ }))
+
+ const reply = await ask('grid_fleet_models_list', { refresh: true })
+
+ expect(reply.error).not.toBe('Running models could not be checked. Try again.')
+ expect(reply.models).toEqual([expect.objectContaining({ id: 'local:Small-Q4.gguf', state: 'running', gridAsleep: true, canStop: true })])
+ expect(seen.map((r) => r.path)).toEqual([`/g/${OWN_ID}${OVERVIEW}`])
+ expectNoCredential()
+ // The owner status is asked once and remembered, not asked on every tick.
+ await ask('grid_fleet_models_list', { refresh: true })
+ await ask('grid_fleet_models_list', { refresh: true })
+ expect(grid.calls().filter((argv) => argv.includes('info'))).toHaveLength(1)
+ expect(grid.calls().some((argv) => argv.includes('engines'))).toBe(false)
+ })
+})
+
+describe('agent_retarget', () => {
+ it('moves an agent onto a grid model without reading the grid at all', async () => {
+ const answers = fakeGridAnswers()
+ grid.replan(answers.plan)
+ socket.onRetargetAgent = vi.fn(async () => ({ ok: true as const }))
+
+ const reply = await ask('agent_retarget', { agentId: 'agent-1', gridModel: 'GLM-4.7-Flash', gridName: answers.gridName })
+
+ expect(reply).toMatchObject({ retargeted: true })
+ expect(seen).toEqual([])
+ })
+})
diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.ts
index 6f7fa2c1d..ca6b3288a 100644
--- a/cli/src/backendSocket.ts
+++ b/cli/src/backendSocket.ts
@@ -36,7 +36,7 @@ import { GridFleetRpc, GRID_FLEET_PROTOCOL, GRID_FLEET_MAX_TIMEOUT_MS, parseGrid
import { LocalModels } from './lib/localModels.js'
import { ApiConnections, apiConnectionsRequest } from './lib/apiConnections.js'
import { gridCapableEngines, parseGridLaunchOverride, type GridLaunchOverride } from './lib/gridLaunch.js'
-import { listAllGridModels, resolveGridTarget } from './lib/gridModels.js'
+import { forgetGridModels, gridInventory, listAllGridModels, onGridModelsChanged, resolveGridTarget, type GridSection } from './lib/gridModels.js'
import { parseNewAgentModel, resolveNewAgentModel } from './lib/newAgentModel.js'
import { deriveHarnessGridName } from './lib/gridDerive.js'
import { AGENT_NAME_RE, FirstPromptUnsupportedError, MAX_FIRST_PROMPT_CHARS, NamedAgentUnsupportedError, permissionModeApproves, permissionModeFlags, supportsFirstPrompt, supportsNamedAgent } from './lib/engineLaunch.js'
@@ -365,9 +365,42 @@ async function enrichSubagentStats(events: SessionEvent[], transcriptPath: strin
}
}
+/** What `grid_models_list` answers and `grid_models_changed` pushes — one shape, so a window parses both
+ * with one reader. `gridName` and `models` keep naming the own grid alone, for an app that predates
+ * `grids`; each section's `state`, `seenAt` and `lastKnownAge` are additive. */
+function gridModelsPayload(gridName: string | null, grids: GridSection[]): Record<string, unknown> {
+ return {
+ gridName,
+ models: grids.find((g) => g.own)?.models ?? [],
+ grids,
+ // Which engines a Local model can be offered to at all. Static per CLI version — it is
+ // the set of launch contracts in `gridLaunch.ts` — and answered here, beside the list,
+ // so the picker can say "Cursor runs only on its own login" instead of offering a row
+ // whose retarget the daemon would refuse. An older app ignores the field; an older
+ // daemon omits it, which the app reads as "offer everything", as before.
+ localModelEngines: gridCapableEngines(),
+ supportsModelLaunch: true,
+ // Whether this MACHINE has a `grid` to run at all — `managed`, `path` or `missing` —
+ // as distinct from `gridName`, which is about the account. The Local model dialog was
+ // gating on the account alone and starting an agent whose second step is `grid`; this
+ // is what lets it, and the picker, say so first. An older app ignores the field.
+ gridCli: gridCliPresence(),
+ }
+}
+
export class BackendSocket {
private readonly gridFleet = new GridFleetRpc()
- private readonly localModels = new LocalModels({ stateDir: join(env.ADAPTER_DATA_DIR, 'local-models') })
+ // The Model Manager reads the grid it runs on through the same credential-less reader as every picker
+ // (never `grid engines`, which carries the grid credential and so wakes a sleeping grid on every tick),
+ // and a start or stop it finishes makes every list read again — pushed to the window when it changes.
+ private readonly localModels = new LocalModels({
+ stateDir: join(env.ADAPTER_DATA_DIR, 'local-models'),
+ inventory: gridInventory,
+ onChanged: () => { forgetGridModels(); void this.pushGridModels() },
+ })
+ /** A read the window did not ask for changed what a picker would show: tell the window
+ * (`grid_models_changed`), so its one picture stays current without polling. */
+ private readonly stopGridModelsPush = onGridModelsChanged(() => { void this.pushGridModels() })
private readonly apiConnections = new ApiConnections(env.ADAPTER_DATA_DIR)
private ws: WebSocket | null = null
private connecting = false
@@ -763,6 +796,17 @@ export class BackendSocket {
* workspace that must be told which grid is "yours" rather than work it out or ask. */
privateGridName(): Promise<string | null> { return this.resolveGridName() }
+ /** `grid_models_changed` to the windows on this computer: the same payload `grid_models_list` answers,
+ * built from the pictures as they stand — no read is started to build it, so a push never causes one. */
+ private async pushGridModels(): Promise<void> {
+ if (this.closed || this.localClients.size === 0) return
+ try {
+ const gridName = await this.resolveGridName()
+ const grids = await listAllGridModels(gridName, { refresh: false })
+ this.sendLocal({ type: 'grid_models_changed', payload: gridModelsPayload(gridName, grids) })
+ } catch { /* the next ask answers the same thing */ }
+ }
+
/**
* The account's private grid: the backend's word when it gave one, else what this machine can
* work out for itself (`lib/gridDerive.ts`). A backend that predates `machine_meta.gridName`
@@ -930,6 +974,7 @@ export class BackendSocket {
async stop(): Promise<void> {
this.closed = true
+ this.stopGridModelsPush()
this.orchestratorService?.stop()
this.viewerForwarder.closeAll()
if (this.heartbeat) this.heartbeat.stop()
@@ -1869,7 +1914,8 @@ export class BackendSocket {
// and `models` keep naming the own grid alone, for an app that predates `grids`.
//
// DETACHED from this connection's ordered RPC chain, like `engines_probe`: it waits on a
- // grid reconcile (up to 6s), then `grid` spawns that each go to the network (up to 30s).
+ // grid reconcile (up to 6s), then a `grid ls` spawn and — for a grid this daemon has never
+ // read — up to 4s of its first read (`gridModels.ts`); every other grid answers from its picture.
// The desktop asks for it in the same breath as `terminal_capabilities` and `agents_list`
// on every connect, and awaited here it held both behind it — with no network, past the
// app's 10s request timeout, on which the app forces a reconnect and asks all three again.
@@ -1878,9 +1924,8 @@ export class BackendSocket {
// until the wifi came back. Request ids make the reply safe to land out of order.
//
// One computation at a time: detached, a second ask that lands while the first is still
- // out (the app re-asks on every connect, and its own 12s timeout is shorter than the grid
- // spawns' 30s) would start more `grid` processes for the same answer. Later askers share
- // the one in flight; the cache in listGridModels covers the settled case.
+ // out (the app re-asks on every connect) would spawn another `grid ls` for the same answer.
+ // Later askers share the one in flight; each grid's reads are single-flight in the service.
void (async () => {
const gridName = await this.resolveGridName()
const inFlight = this.gridModelsInFlight
@@ -1892,24 +1937,7 @@ export class BackendSocket {
if (this.gridModelsInFlight?.gridName === gridName) this.gridModelsInFlight = null
}),
}).grids
- const grids = await listing
- reply(type, requestId, {
- gridName,
- models: grids.find((g) => g.own)?.models ?? [],
- grids,
- // Which engines a Local model can be offered to at all. Static per CLI version — it is
- // the set of launch contracts in `gridLaunch.ts` — and answered here, beside the list,
- // so the picker can say "Cursor runs only on its own login" instead of offering a row
- // whose retarget the daemon would refuse. An older app ignores the field; an older
- // daemon omits it, which the app reads as "offer everything", as before.
- localModelEngines: gridCapableEngines(),
- supportsModelLaunch: true,
- // Whether this MACHINE has a `grid` to run at all — `managed`, `path` or `missing` —
- // as distinct from `gridName`, which is about the account. The Local model dialog was
- // gating on the account alone and starting an agent whose second step is `grid`; this
- // is what lets it, and the picker, say so first. An older app ignores the field.
- gridCli: gridCliPresence(),
- })
+ reply(type, requestId, gridModelsPayload(gridName, await listing))
})().catch(() => reply(type, requestId, { error: 'GRID_MODELS_FAILED' }))
return
}
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index e051fb827..9f10efc42 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -185,7 +185,7 @@ import {
type Poller, type UpdateEntry,
} from './lib/selfUpdate.js'
import { managedNodePath } from './lib/nodeRuntime.js'
-import { ensureLauncher, ensureManagedGrid, ensureManagedRuntime } from './lib/runtimeInstall.js'
+import { ensureLauncher, ensureManagedGrid, ensureManagedRuntime, startGridPinRecheck } from './lib/runtimeInstall.js'
import { stat } from 'fs/promises'
import { CodexNormalizer, codexTaskError, lastCodexTurnText } from './engines/codex/normalizer.js'
import { codexSubagentResolverFor } from './engines/codex/subagent.js'
@@ -1323,6 +1323,9 @@ async function runForeground(session: AuthSession | null): Promise<void> {
// pinned binary before it hands a token over.
const managedGridReady = ensureManagedGrid((m) => console.log(`[grid-runtime] ${m}`))
void managedGridReady
+ // …and keeps following it while this daemon runs: a pin moved after the start reaches it within ten
+ // minutes rather than at the next restart (`startGridPinRecheck`).
+ startGridPinRecheck()
registry.load()
// Persisted locators are hints until this process has observed their terminal root and PID/start marker.
diff --git a/cli/src/lib/__fixtures__/fakeGrid.ts b/cli/src/lib/__fixtures__/fakeGrid.ts
index ba51f5d18..72ab25cde 100644
--- a/cli/src/lib/__fixtures__/fakeGrid.ts
+++ b/cli/src/lib/__fixtures__/fakeGrid.ts
@@ -11,7 +11,7 @@
* multi-turn answers are the parts that drift when copied, and a fake that means something slightly
* different in each file is worse than none.
*/
-import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { chmodSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -21,14 +21,18 @@ const log = process.env.FAKE_GRID_LOG
const plan = JSON.parse(fs.readFileSync(process.env.FAKE_GRID_PLAN, 'utf8'))
const args = process.argv.slice(2).filter((a) => a !== '--remote')
const verb = args[0] || ''
-const calls = fs.existsSync(log) ? JSON.parse(fs.readFileSync(log, 'utf8')) : []
-calls.push(process.argv.slice(2))
-fs.writeFileSync(log, JSON.stringify(calls))
-const step = plan[verb]
+// One line per call, APPENDED: the daemon runs several \`grid\`s at once (one per grid it reads), and a
+// read-modify-write of one JSON document lost calls and crashed a reader that caught it half written.
+fs.appendFileSync(log, JSON.stringify(process.argv.slice(2)) + '\\n')
+const calls = fs.readFileSync(log, 'utf8').split('\\n').filter(Boolean).map((line) => JSON.parse(line))
+// A plan may answer one grid differently from another: \`info mine\` is asked before \`info\`.
+const keyed = plan[verb + ' ' + (args[1] || '')]
+const step = keyed || plan[verb]
if (!step) { process.exit(0) }
// A verb may answer differently on successive calls (\`ls\` before and after a create), so its
// answer can be a LIST of turns; the last turn repeats once the list runs out.
-const turn = Array.isArray(step) ? (step[calls.filter((c) => c.includes(verb)).length - 1] ?? step[step.length - 1]) : step
+const asked = calls.filter((c) => c.includes(verb) && (!keyed || c.includes(args[1]))).length
+const turn = Array.isArray(step) ? (step[asked - 1] ?? step[step.length - 1]) : step
if (turn.stdout) process.stdout.write(turn.stdout)
if (turn.stderr) process.stderr.write(turn.stderr)
process.exit(turn.exit ?? 0)
@@ -37,7 +41,8 @@ process.exit(turn.exit ?? 0)
/** What one invocation of a verb prints and how it exits. */
export interface FakeGridTurn { stdout?: string; stderr?: string; exit?: number }
-/** Keyed by the FIRST argument after `--remote` is stripped — `mcp` for `grid mcp config …`. */
+/** Keyed by the FIRST argument after `--remote` is stripped — `mcp` for `grid mcp config …` — or by that
+ * and the next one, `info mine`, which wins over the verb alone. */
export type FakeGridPlan = Record<string, FakeGridTurn | FakeGridTurn[]>
export interface FakeGrid {
@@ -45,6 +50,8 @@ export interface FakeGrid {
calls: () => string[][]
/** The verbs called, in order, with `--remote` stripped — `['mcp', 'info', 'ls']`. */
verbs: () => string[]
+ /** Answer every later call from `plan` instead — a grid that changes while a test watches. */
+ replan: (plan: FakeGridPlan) => void
/** Delete the fake and unset the variables that point at it. */
dispose: () => void
}
@@ -62,13 +69,13 @@ export function installFakeGrid(plan: FakeGridPlan): FakeGrid {
chmodSync(bin, 0o755)
const planFile = join(root, 'plan.json')
writeFileSync(planFile, JSON.stringify(plan))
- const log = join(root, 'calls.json')
+ const log = join(root, 'calls.jsonl')
process.env.HARNESS_GRID_BIN = bin
process.env.FAKE_GRID_PLAN = planFile
process.env.FAKE_GRID_LOG = log
const calls = (): string[][] => {
try {
- return JSON.parse(readFileSync(log, 'utf8')) as string[][]
+ return readFileSync(log, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line) as string[])
} catch {
return []
}
@@ -76,6 +83,11 @@ export function installFakeGrid(plan: FakeGridPlan): FakeGrid {
return {
calls,
verbs: () => calls().map((argv) => argv.find((arg) => arg !== '--remote') ?? ''),
+ // Replaced, never rewritten in place: a fake still running must read the old plan or the new one.
+ replan: (next) => {
+ writeFileSync(`${planFile}.next`, JSON.stringify(next))
+ renameSync(`${planFile}.next`, planFile)
+ },
dispose: () => {
rmSync(root, { recursive: true, force: true })
delete process.env.HARNESS_GRID_BIN
diff --git a/cli/src/lib/gridModels.ts b/cli/src/lib/gridModels.ts
index a18151326..c280d289f 100644
--- a/cli/src/lib/gridModels.ts
+++ b/cli/src/lib/gridModels.ts
@@ -1,88 +1,45 @@
/**
- * What the account's private harness grid can answer right now.
+ * What the grids this computer is signed into can answer — read WITHOUT waking any of them.
*
- * One `grid models --json <grid>` away, so this module is a shape and a filter rather than a
- * protocol. It exists because two things have to be decided in one place: which rows are offered to
- * a person, and what an agent is told to ask for.
+ * Every picker, the Models panel and the macOS Models menu ask here, and until issue 02 of
+ * grid-reads-without-waking each ask was a `grid models` spawn plus a SIGNED-IN read of the relay's
+ * model list. On the platform a signed-in read of a sleeping grid wakes it, so an open app kept every
+ * grid of the account awake all day — and the read that woke one usually came back empty. Now:
+ *
+ * - each grid is read through `gridReader.ts`, with no credential: an awake grid answers as before, a
+ * sleeping one says so at once (with the platform's record of what it served) and is not started;
+ * - what was read is kept as a PICTURE per grid (`gridPicture.ts`, persisted), so an answer is given at
+ * once and a sleeping grid shows its last known models instead of vanishing;
+ * - the account's OWN grid, whose status the owner can read, is read at most once per asleep episode.
+ *
+ * The credentialed model-list read is gone from every automatic path. What still wakes a grid is a
+ * person's act, and it does not go through here.
*/
+import { createHash, randomUUID } from 'node:crypto'
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import { env as config } from '../config/env.js'
+import { gridCredentialsPath } from './gridCredentials.js'
+import { signedInGridEmail } from './gridDerive.js'
import { gridExec, gridJson } from './gridExec.js'
import { resolveGridMcpUrl } from './gridMcpUrl.js'
-
-/** A row of `grid models --json`. `node` names the machine serving it — on a private grid, one of
- * the user's own. */
-interface GridModelRow {
- model?: unknown
- engine?: unknown
- node?: unknown
-}
+import {
+ emptyPicture, mergeAwake, idKey, parsePicture, provenStopped, sectionView, servedKey, unspelled, withAsleep,
+ withSpellings, withUnknown, type GridPicture, type LocalRecord, type PictureState, type SectionView, type ServedHere,
+} from './gridPicture.js'
+import {
+ OWNER_ASLEEP_STATUS, readBase, readDiscoveryIds, readGridInfo, readOverview, readViaCli, type GridInfo, type GridRead,
+ type ReadNode,
+} from './gridReader.js'
+import { readRunRecords } from './localModels.js'
export interface GridModel {
- /** The id an engine is pointed at, verbatim from the grid. */
+ /** The id an engine is pointed at, in the exact case the grid serves it. */
id: string
/** Which machine answers it, or empty when the grid does not say. Display only. */
node: string
}
-/**
- * The grid's router entry, which is deliberately NOT offered.
- *
- * `auto` is served by the `grid-router` engine and picks a model per request. It is being
- * reimplemented once the grid path is end-to-end encrypted, and offering it first would teach a
- * selection that is about to change meaning. Matched on the ENGINE rather than the name `auto`, so a
- * renamed router cannot leak back into the picker.
- */
-const ROUTER_ENGINE = 'grid-router'
-
-/**
- * Live models on `gridName`, router excluded. An empty list is a real answer — a grid nobody is
- * serving yet has none — and is not distinguished here from a grid that could not be reached; the
- * caller shows "no models" either way, and the reason is in the daemon log.
- */
-export async function listGridModels(gridName: string | null): Promise<GridModel[]> {
- if (!gridName?.trim()) return []
- const fresh = cached(gridName)
- if (fresh) return fresh
-
- // ⚠️ Concurrently, and the ordering used to cost a full second of the click. `grid models` is
- // ~830ms and the relay catalogue another ~430ms, both of them a Python CLI spawn plus a network
- // round trip; run one after the other they are the whole of what a person waits through before
- // the menu appears. Neither needs the other's answer.
- const [listed, served] = await Promise.all([
- gridJson<GridModelRow[]>(['--remote', 'models', gridName]),
- relayModelIds(gridName),
- ])
- const value = listed.value
- if (!Array.isArray(value)) return []
-
- // ⚠️ `grid models` LOWERCASES the model id; the relay does not. Measured on a real engine:
- // `grid models --json` says `qwen3.6-35b-a3b-ud-q5_k_xl` while the relay serves
- // `Qwen3.6-35B-A3B-UD-Q5_K_XL`, and posting the lowercased one answers
- // `{"detail":"No providers available for this model"}`. The id an engine is pointed at has to be
- // the one the RELAY knows, so the relay's own catalogue is the authority here and `grid models`
- // is kept only for the node label. Case is unrecoverable from the lowercased string, so this
- // cannot be fixed by mapping.
- const nodeFor = new Map<string, string>()
- for (const row of value) {
- const id = typeof row.model === 'string' ? row.model.trim() : ''
- if (id && row.engine !== ROUTER_ENGINE) nodeFor.set(id.toLowerCase(), typeof row.node === 'string' ? row.node : '')
- }
-
- // The relay could not be reached (offline, no token yet) → fall back to what `grid models` said.
- // A list with the wrong case is still better than an empty picker: the engine reports the refusal
- // plainly, whereas an empty menu looks like the grid has nothing on it.
- const ids = served.length ? served : [...nodeFor.keys()]
- const seen = new Set<string>()
- const models: GridModel[] = []
- for (const id of ids) {
- const key = id.toLowerCase()
- if (!nodeFor.has(key) || seen.has(key)) continue
- seen.add(key)
- models.push({ id, node: nodeFor.get(key) ?? '' })
- }
- remember(gridName, models)
- return models
-}
-
/** One grid this computer is signed into, with what it serves — the picker's section. */
export interface GridSection {
/** The grid's name as `grid ls` prints it. */
@@ -91,84 +48,407 @@ export interface GridSection {
type: string
/** True for the account's private grid — the picker labels that one "Local". */
own: boolean
+ /** Its models: live while it is awake, the last known ones while it is not — never blanked. */
models: GridModel[]
+ /** Additive (an older app ignores these three): what the last read said, when the grid was last seen
+ * awake, and how old the list above is. */
+ state?: PictureState
+ seenAt?: string | null
+ lastKnownAge?: number | null
}
-/**
- * Every grid this computer is signed into, each with its live models, own grid first.
- *
- * The picker used to ask only the private grid ("models my machines serve"). A person in a
- * shared grid (a team's, a company's) has models there they can switch an agent to just the
- * same, and had to leave the app to find out. So all of them, in sections, the private one first
- * and marked so the app can call it Local. Listed concurrently: each grid costs a `grid models`
- * spawn and a relay round trip, and three of them one after another was the whole of what a
- * person waited through. A grid that fails to answer is an empty section, not a missing one.
- */
-export async function listAllGridModels(ownGridName: string | null): Promise<GridSection[]> {
- const { value: rows } = await gridJson<Array<{ grid?: unknown; type?: unknown }>>(['--remote', 'ls'])
- if (!Array.isArray(rows)) {
- return ownGridName ? [{ name: ownGridName, type: 'permissioned-public', own: true, models: await listGridModels(ownGridName) }] : []
- }
- const grids = rows
- .filter((row): row is { grid: string; type?: unknown } => typeof row.grid === 'string' && row.grid.trim().length > 0)
- .map((row) => ({ name: row.grid.trim(), type: typeof row.type === 'string' ? row.type : '' }))
- const sections = await Promise.all(grids.map(async (grid) => ({
- name: grid.name,
- type: grid.type,
- own: grid.name === ownGridName,
- models: await listGridModels(grid.name),
- })))
- sections.sort((a, b) => Number(b.own) - Number(a.own))
- return sections
+/** What the Model Manager is told about the grid it runs on (`localModels.ts`'s `GridInventory`). */
+export interface GridInventoryAnswer {
+ state: PictureState
+ nodes: Record<string, unknown>[]
+}
+
+/** How long one read's answer stands before the next is made: the list is live while a grid is awake,
+ * and a sleeping grid changes only when something wakes it. A failure is retried sooner than sleep. */
+const AWAKE_MEMO_MS = 15_000
+const ASLEEP_MEMO_MS = 120_000
+const FAILED_MEMO_MS = 30_000
+
+/** `grid info` is a control-plane call: a member's grid address barely moves; the OWNER's status is the
+ * thing that says a grid fell asleep or woke, so it is asked more often, and less often while asleep. */
+const ADDRESS_MEMO_MS = 10 * 60_000
+const OWN_STATUS_AWAKE_MEMO_MS = 15_000
+const OWN_STATUS_ASLEEP_MEMO_MS = 60_000
+
+/** Several changes landing together (three grids refreshed at once) are one push, not three. */
+const CHANGE_COALESCE_MS = 250
+
+/** How long an ask waits for the FIRST read of a grid this daemon has never seen before answering with
+ * what it has. Kept well inside the app's 12s `grid_models_list` timeout, which the grid-name wait
+ * (`GRID_ATTACH_WAIT_MS`, 6s) shares: a read slower than this lands behind the answer and is pushed. */
+const FIRST_READ_WAIT_MS = 4_000
+
+interface GridRow { name: string; type: string; id: string }
+
+/** The key a grid is tracked under when `grid ls` gave it no network id — its name, marked as such, so
+ * it can never collide with an id and never names a run-record directory. */
+const NAME_ONLY_PREFIX = 'name:'
+const nameOnlyRow = (name: string, type = ''): GridRow => ({ name, type, id: `${NAME_ONLY_PREFIX}${name}` })
+
+/** Everything this daemon holds about one grid, keyed by its network id. */
+interface Tracked {
+ id: string
+ /** The name `grid` knows it by — what goes into `grid info <name>` and the fallback's argv. */
+ name: string
+ /** The account's own grid. A property of the GRID, learnt from whoever last knew (the model list,
+ * the Model Manager), so an asker that does not know — a launch check — never changes how it is read. */
+ own: boolean
+ picture: GridPicture
+ loaded: boolean
+ /** Whether a picture existed before this process read anything (on disk, or read since). */
+ known: boolean
+ /** The picture as last written, so a read that changed nothing writes nothing. */
+ saved: string
+ readAt: number | null
+ readTtl: number
+ info: { at: number; value: GridInfo | null } | null
+ /** The own grid's one read of its current asleep episode has been made. */
+ episodeRead: boolean
+ /** What this daemon has watched a live run record serve here since the last awake read. */
+ seen: Set<string>
+ /** Ids discovery has already been asked to spell, so an id no provider spells is asked about once. */
+ spelled: Set<string>
+ /** The last awake overview's own node objects, for the Model Manager. Not persisted. */
+ rawNodes: Record<string, unknown>[]
+ pending: Promise<void> | null
+}
+
+export interface GridModelsDeps {
+ now: () => number
+ /** Where pictures are kept (`<data dir>/grid-pictures`). */
+ dataDir: () => string
+ /** `~/.grid`, whose `run/engines/<grid id>/` holds this computer's run records. */
+ gridHome: () => string
+ /** The signed-in account — a node published under it is the account's own. */
+ email: () => string | null
+ /** [FIRST_READ_WAIT_MS], injectable so a test need not wait it out. */
+ firstReadWaitMs: number
+}
+
+const defaultDeps: GridModelsDeps = {
+ now: () => Date.now(),
+ dataDir: () => config.ADAPTER_DATA_DIR,
+ gridHome: () => dirname(gridCredentialsPath()),
+ email: () => signedInGridEmail(),
+ firstReadWaitMs: FIRST_READ_WAIT_MS,
}
/**
- * The last answer for a grid, for a few seconds.
- *
- * The picker asks on every open, and the answer costs two subprocess spawns and two network round
- * trips — measured at ~1.4s, which is a person watching a header do nothing after a click. Reopening
- * a menu twice in a row should not pay that twice.
+ * The pictures, the reads that keep them current, and the answers built from them.
*
- * Short on purpose. The list is genuinely live — an engine can join or leave between two opens — so
- * this is only the window in which a click is free, not a cache in the sense of holding state. Longer
- * would start offering models nobody is serving, which is the failure the no-caching rule existed to
- * avoid.
+ * Nothing here runs on a timer: a grid is read only when something asks about it and its last answer
+ * has aged past its memo. So an app nobody looks at costs nothing, and an app somebody looks at costs a
+ * credential-less read every 15s at most per grid — and never a wake.
*/
-const TTL_MS = 15_000
-const memo = new Map<string, { at: number; models: GridModel[] }>()
+export class GridModelsService {
+ private readonly deps: GridModelsDeps
+ private readonly tracked = new Map<string, Tracked>()
+ private readonly listeners = new Set<() => void>()
+ private rows: GridRow[] | null = null
+ private changeTimer: NodeJS.Timeout | null = null
+
+ constructor(deps: Partial<GridModelsDeps> = {}) {
+ this.deps = { ...defaultDeps, ...deps }
+ }
+
+ /** Called (coalesced) whenever a read changed what a picker would be told. */
+ onChange(listener: () => void): () => void {
+ this.listeners.add(listener)
+ return () => { this.listeners.delete(listener) }
+ }
+
+ /** Resolves once every read already started has landed — for a test, and for a caller that must see
+ * the answer a read it triggered produced. */
+ async settled(): Promise<void> {
+ await Promise.all([...this.tracked.values()].map((tracked) => tracked.pending?.catch(() => {})))
+ }
+
+ /** Every read is due again — for a caller that has just changed what a grid serves, or which grid is
+ * whose. The pictures stay: they are what is shown until the next read lands. */
+ forget(): void {
+ for (const tracked of this.tracked.values()) {
+ tracked.readAt = null
+ tracked.info = null
+ }
+ this.rows = null
+ }
+
+ /**
+ * Every grid this computer is signed into, own grid first, each with its models.
+ *
+ * Answered from the pictures at once. A grid whose answer has aged is read again in the background —
+ * one read per grid at a time — and a change is announced through [onChange]. Only a grid this daemon
+ * has never seen is waited for, and only for [FIRST_READ_WAIT_MS]: an answer made of nothing is worse
+ * than a short wait, and a timed-out ask is worse than both.
+ */
+ async sections(ownGridName: string | null, opts: { refresh?: boolean } = {}): Promise<GridSection[]> {
+ const rows = await this.gridRows(ownGridName)
+ const sections = await Promise.all(rows.map(async (row) => {
+ const tracked = await this.track(row, row.name === ownGridName)
+ if (opts.refresh !== false && this.due(tracked)) {
+ const refreshing = this.refresh(tracked)
+ if (!tracked.known) await this.within(refreshing, this.deps.firstReadWaitMs)
+ else void refreshing.catch(() => {})
+ }
+ return { name: row.name, type: row.type, own: tracked.own, ...await this.view(tracked) }
+ }))
+ sections.sort((a, b) => Number(b.own) - Number(a.own))
+ return sections
+ }
+
+ /** One grid's models, read now if its answer has aged — for a launch that must not start on a model
+ * nobody serves. Still never a waking read: a sleeping grid answers with what it last served. */
+ async models(gridName: string): Promise<GridModel[]> {
+ const tracked = await this.track(await this.rowFor(gridName))
+ if (this.due(tracked)) await this.refresh(tracked)
+ return (await this.view(tracked)).models
+ }
+
+ /** What the Model Manager needs to tell a running model from a stopped one, on the account's own grid. */
+ async inventory(gridName: string, force: boolean): Promise<GridInventoryAnswer> {
+ const tracked = await this.track(await this.rowFor(gridName), true)
+ if (force) tracked.readAt = null
+ if (this.due(tracked)) await this.refresh(tracked)
+ return { state: tracked.picture.state, nodes: tracked.picture.state === 'awake' ? tracked.rawNodes : [] }
+ }
-function cached(gridName: string): GridModel[] | null {
- const hit = memo.get(gridName)
- return hit && Date.now() - hit.at < TTL_MS ? hit.models : null
+ private async rowFor(gridName: string): Promise<GridRow> {
+ return (await this.gridRows(null)).find((row) => row.name === gridName) ?? nameOnlyRow(gridName)
+ }
+
+ /** `grid ls --json` — a LOCAL registry read, no network. The last good answer stands in for a failed
+ * one; with none, the own grid alone, as before this module knew about shared grids. */
+ private async gridRows(ownGridName: string | null): Promise<GridRow[]> {
+ const { value } = await gridJson<Array<{ grid?: unknown; type?: unknown; id?: unknown }>>(['--remote', 'ls'])
+ if (Array.isArray(value)) {
+ this.rows = value
+ // A name goes into a `grid` argv (`info <grid>`, the fallback's `models <grid>`): one that reads
+ // as a flag, or carries a control character, is not one this daemon will pass along.
+ .filter((row) => typeof row?.grid === 'string' && /^[^-\x00-\x1f\x7f][^\x00-\x1f\x7f]*$/.test(row.grid.trim()))
+ .map((row) => {
+ const name = (row.grid as string).trim()
+ const type = typeof row.type === 'string' ? row.type : ''
+ return typeof row.id === 'string' && row.id.trim() ? { name, type, id: row.id.trim() } : nameOnlyRow(name, type)
+ })
+ }
+ if (this.rows) return this.rows
+ return ownGridName ? [nameOnlyRow(ownGridName, 'permissioned-public')] : []
+ }
+
+ /** The grid behind `row`, loaded from disk the first time. `own`, when the caller knows it, is recorded. */
+ private async track(row: GridRow, own?: boolean): Promise<Tracked> {
+ let tracked = this.tracked.get(row.id)
+ if (!tracked) {
+ tracked = {
+ id: row.id, name: row.name, own: false, picture: emptyPicture(), loaded: false, known: false, saved: '',
+ readAt: null, readTtl: 0, info: null, episodeRead: false, seen: new Set(), spelled: new Set(), rawNodes: [],
+ pending: null,
+ }
+ this.tracked.set(row.id, tracked)
+ }
+ tracked.name = row.name
+ if (own !== undefined) tracked.own = own
+ if (!tracked.loaded) {
+ tracked.loaded = true
+ await this.load(tracked)
+ }
+ return tracked
+ }
+
+ /** `work`, or nothing more than `ms` of it — what it produces meanwhile lands on its own. */
+ private async within(work: Promise<void>, ms: number): Promise<void> {
+ let timer: NodeJS.Timeout | undefined
+ await Promise.race([work.catch(() => {}), new Promise<void>((resolve) => { timer = setTimeout(resolve, ms) })])
+ clearTimeout(timer)
+ }
+
+ private due(tracked: Tracked): boolean {
+ const now = this.deps.now()
+ if (tracked.readAt === null || now - tracked.readAt >= tracked.readTtl) return true
+ // The owner's status is what says the own grid fell asleep or woke; it is checked on its own clock.
+ return tracked.own && (tracked.info === null || now - tracked.info.at >= this.statusTtl(tracked))
+ }
+
+ private statusTtl(tracked: Tracked): number {
+ if (!tracked.own) return ADDRESS_MEMO_MS
+ return tracked.info?.value?.status === OWNER_ASLEEP_STATUS ? OWN_STATUS_ASLEEP_MEMO_MS : OWN_STATUS_AWAKE_MEMO_MS
+ }
+
+ /** One refresh per grid at a time; every asker shares it. */
+ private refresh(tracked: Tracked): Promise<void> {
+ tracked.pending ??= this.read(tracked).finally(() => { tracked.pending = null })
+ return tracked.pending
+ }
+
+ private async read(tracked: Tracked): Promise<void> {
+ const before = JSON.stringify(await this.signature(tracked))
+ const info = await this.info(tracked)
+ const asleepByStatus = tracked.own && info?.status === OWNER_ASLEEP_STATUS
+ // The own grid, while its status says asleep, is read exactly once per asleep episode — to fetch the
+ // platform's record of what it served — and then left alone until the status changes.
+ if (asleepByStatus && tracked.episodeRead) {
+ await this.apply(tracked, { kind: 'asleep', lastKnown: null }, null)
+ } else {
+ tracked.episodeRead = asleepByStatus
+ const base = readBase(info?.gridUrl ?? null)
+ let read: GridRead = base ? await readOverview(base) : { kind: 'unreachable' }
+ // Only when nothing answered at all — never to second-guess an answer the grid gave.
+ if (read.kind === 'unreachable') read = await readViaCli(tracked.name)
+ await this.apply(tracked, read, base)
+ }
+ await this.save(tracked)
+ tracked.known = true
+ if (JSON.stringify(await this.signature(tracked)) !== before) this.changed()
+ }
+
+ private async info(tracked: Tracked): Promise<GridInfo | null> {
+ const now = this.deps.now()
+ if (tracked.info && now - tracked.info.at < this.statusTtl(tracked)) return tracked.info.value
+ // A failed ask keeps the last answer (the address is still the address), and is not re-asked at once.
+ const value = await readGridInfo(tracked.name) ?? tracked.info?.value ?? null
+ tracked.info = { at: now, value }
+ if (value?.status !== OWNER_ASLEEP_STATUS) tracked.episodeRead = false
+ return value
+ }
+
+ private async apply(tracked: Tracked, read: GridRead, base: string | null): Promise<void> {
+ const now = this.deps.now()
+ tracked.readAt = now
+ if (read.kind === 'asleep') {
+ tracked.picture = withAsleep(tracked.picture, read.lastKnown, now)
+ tracked.readTtl = ASLEEP_MEMO_MS
+ return
+ }
+ if (read.kind !== 'awake') {
+ tracked.picture = withUnknown(tracked.picture)
+ tracked.readTtl = FAILED_MEMO_MS
+ return
+ }
+ const here = await this.servedHere(tracked)
+ const email = this.deps.email()?.trim().toLowerCase() ?? ''
+ const isMine = (node: ReadNode): boolean => !!email && node.providerEmail?.trim().toLowerCase() === email
+ const previous = tracked.picture
+ let picture = mergeAwake(previous, read.nodes, now, isMine, (name, key) => provenStopped(previous, here, name, key))
+ picture = { ...picture, caseMap: withSpellings(picture.caseMap, [...read.curatedIds, ...here.records.flatMap((r) => r.ids)]) }
+ const unknownIds = unspelled(picture, read.nodes).filter((key) => !tracked.spelled.has(key))
+ if (unknownIds.length && base) {
+ unknownIds.forEach((key) => tracked.spelled.add(key))
+ picture = { ...picture, caseMap: withSpellings(picture.caseMap, await readDiscoveryIds(base)) }
+ }
+ tracked.picture = picture
+ tracked.rawNodes = read.rawNodes
+ tracked.readTtl = AWAKE_MEMO_MS
+ // What this computer serves is re-learnt from what is live now: the grid has just said what it serves.
+ tracked.seen = new Set(liveKeys(here.records))
+ }
+
+ /**
+ * This computer's own records for the grid, and what they have been seen serving. Deliberately updates
+ * `seen` on EVERY look, answers included: "seen served since the last awake read" is only true if each
+ * look that found a live record counted.
+ */
+ private async servedHere(tracked: Tracked): Promise<ServedHere> {
+ const records: LocalRecord[] = tracked.id.startsWith(NAME_ONLY_PREFIX) ? [] : await readRunRecords(this.deps.gridHome(), tracked.id)
+ for (const key of liveKeys(records)) tracked.seen.add(key)
+ return { records, seen: tracked.seen, own: tracked.own }
+ }
+
+ private async view(tracked: Tracked): Promise<Omit<SectionView, 'models'> & { models: GridModel[] }> {
+ return sectionView(tracked.picture, await this.servedHere(tracked), this.deps.now())
+ }
+
+ /** What a push is about: the rows and the state — not the ages, which move every second. */
+ private async signature(tracked: Tracked): Promise<unknown> {
+ const view = await this.view(tracked)
+ return { models: view.models, state: view.state }
+ }
+
+ private changed(): void {
+ if (this.changeTimer) return
+ this.changeTimer = setTimeout(() => {
+ this.changeTimer = null
+ for (const listener of this.listeners) {
+ try { listener() } catch { /* a listener's failure is its own */ }
+ }
+ }, CHANGE_COALESCE_MS)
+ this.changeTimer.unref?.()
+ }
+
+ private file(tracked: Tracked): string {
+ const name = createHash('sha256').update(tracked.id).digest('hex').slice(0, 24)
+ return join(this.deps.dataDir(), 'grid-pictures', `${name}.json`)
+ }
+
+ private async load(tracked: Tracked): Promise<void> {
+ try {
+ const raw = JSON.parse(await readFile(this.file(tracked), 'utf8')) as { networkId?: unknown; picture?: unknown }
+ const picture = raw.networkId === tracked.id ? parsePicture(raw.picture) : null
+ if (!picture) return
+ tracked.picture = picture
+ tracked.known = true
+ tracked.saved = JSON.stringify(picture)
+ } catch { /* never read on this computer, or not a file this module wrote */ }
+ }
+
+ /** Owner-only, replaced atomically — a crash mid-write leaves the previous picture, never half of one. */
+ private async save(tracked: Tracked): Promise<void> {
+ const text = JSON.stringify(tracked.picture)
+ if (text === tracked.saved) return
+ try {
+ const file = this.file(tracked)
+ await mkdir(dirname(file), { recursive: true, mode: 0o700 })
+ const temp = `${file}.${randomUUID()}.tmp`
+ await writeFile(temp, JSON.stringify({ networkId: tracked.id, picture: tracked.picture }), { mode: 0o600 })
+ await rename(temp, file)
+ tracked.saved = text
+ } catch { /* the picture is still in memory; the next read tries again */ }
+ }
}
-function remember(gridName: string, models: GridModel[]): void {
- memo.set(gridName, { at: Date.now(), models })
+function liveKeys(records: readonly LocalRecord[]): string[] {
+ return records
+ .filter((record) => record.pid !== null && record.alive)
+ .flatMap((record) => record.ids.map((id) => servedKey(record.name, idKey(id))))
}
-/** Drop the memo — for a caller that has just changed what the grid serves. */
+/** The daemon's one service. Module-level like the memo it replaces, so every caller shares one picture. */
+let service = new GridModelsService()
+
+/** Live models on `gridName`, router excluded, as the picture has them after a read if one was due. */
+export async function listGridModels(gridName: string | null): Promise<GridModel[]> {
+ if (!gridName?.trim()) return []
+ return service.models(gridName.trim())
+}
+
+/** Every grid this computer is signed into, each with its models, own grid first. */
+export function listAllGridModels(ownGridName: string | null, opts: { refresh?: boolean } = {}): Promise<GridSection[]> {
+ return service.sections(ownGridName, opts)
+}
+
+/** The Model Manager's inventory of the own grid (`LocalModels`' injected `inventory`). */
+export function gridInventory(gridName: string, force: boolean): Promise<GridInventoryAnswer> {
+ return service.inventory(gridName, force)
+}
+
+/** Make every read due again — for a caller that has just changed what a grid serves. */
export function forgetGridModels(): void {
- memo.clear()
+ service.forget()
}
-/** Model ids exactly as the grid's relay serves them — the names an engine must send. Empty when the
- * relay cannot be asked, which the caller treats as "fall back", never as "no models". */
-async function relayModelIds(gridName: string): Promise<string[]> {
- const info = await gridExec(['--remote', 'info', gridName, '--env'])
- if (info.code !== 'OK') return []
- const env = readEnvExports(info.stdout)
- if (!env.baseUrl || !env.apiKey) return []
- try {
- const response = await fetch(`${env.baseUrl.replace(/\/$/, '')}/models`, {
- headers: { authorization: `Bearer ${env.apiKey}` },
- signal: AbortSignal.timeout(10_000),
- })
- if (!response.ok) return []
- const body = await response.json() as { data?: Array<{ id?: unknown }> }
- return (body.data ?? []).map((m) => (typeof m.id === 'string' ? m.id : '')).filter(Boolean)
- } catch {
- return []
- }
+/** Be told (coalesced) when a background read changed what a picker would be told. */
+export function onGridModelsChanged(listener: () => void): () => void {
+ return service.onChange(listener)
+}
+
+/** For tests: a fresh service, optionally with its clock and places injected. */
+export function resetGridModels(deps: Partial<GridModelsDeps> = {}): GridModelsService {
+ service = new GridModelsService(deps)
+ return service
}
diff --git a/cli/src/lib/gridPicture.ts b/cli/src/lib/gridPicture.ts
new file mode 100644
index 000000000..517c86558
--- /dev/null
+++ b/cli/src/lib/gridPicture.ts
@@ -0,0 +1,340 @@
+/**
+ * What this daemon remembers about each grid: what it serves, in exact letter case, and when it was
+ * last seen awake — so every picker is answered at once, and a sleeping grid shows its last known models
+ * instead of vanishing.
+ *
+ * Pure: no I/O and no clock of its own (`now` is passed in), so the rules below are the whole of it.
+ *
+ * **The rule every function here keeps: a row is REMOVED only on positive evidence** — the grid's own
+ * answer or record, or this computer's own run records. Everything that is not evidence leaves the list
+ * stale, never shorter, and never wakes anything.
+ */
+import type { LastKnown, ReadNode } from './gridReader.js'
+
+/**
+ * How long a model must be missing from awake answers before it leaves the list — one node TTL (120s on
+ * the master) plus one 30s heartbeat from a pre-0.3.48 provider. A just-woken master has not heard from
+ * its providers, so without this the read that follows a wake blanks every list. ⚠️ Cross-repo timing
+ * (the lockstep register); the public repo's pin finds it by this exact name.
+ */
+export const GRID_MODEL_RETENTION_MS = 150_000
+
+/**
+ * …and across at least this many awake answers. Implied today — the window is measured from the FIRST
+ * answer that missed a model, so it can only have elapsed by a later one (a mutation that drops this
+ * floor survives every test for exactly that reason) — and kept as the spec's own statement of the rule,
+ * so that measuring the window from anything earlier cannot let one read after a long silence count as
+ * the whole of it.
+ */
+const MIN_ABSENT_READS = 2
+
+/**
+ * A sleep record older than this is ignored: the master forgets node rows after 30 days asleep, so an
+ * older record lists models nobody serves any more. ⚠️ Cross-repo timing, found by this exact name.
+ */
+export const GRID_LAST_KNOWN_MAX_AGE_MS = 29 * 24 * 60 * 60 * 1000
+
+/** Engines that are a router rather than a model (`grid-router` serves `auto`) are never offered. */
+const ROUTER_ENGINE = 'grid-router'
+
+/**
+ * `awake` — the last read answered. `asleep` — the platform says it is resting. `unknown` — the last
+ * read failed any other way, or nothing has been read yet (the list shown, if any, is the last known).
+ * `waking` — a person asked for it to start (issue 03's explicit wake; never set by a read).
+ */
+export type PictureState = 'awake' | 'asleep' | 'waking' | 'unknown'
+
+export interface PictureModel {
+ /** The id without case — the join key across sources. Its spelling lives in `caseMap`. */
+ key: string
+ /** When a read last listed it (for a record, when the record was taken). */
+ seenAt: number
+ /** The first awake read it was missing from, while it has been missing since; null when listed. */
+ absentSince: number | null
+ /** How many awake reads in a row it has been missing from. */
+ absentReads: number
+}
+
+export interface PictureNode {
+ /** The node's published name — the machine's, or the anonymous label the grid gives it. */
+ name: string
+ engine: string
+ /** The account's own node, as the last live read said (its published provider). Never guessed. */
+ isMine: boolean
+ models: PictureModel[]
+}
+
+export interface GridPicture {
+ spec: 1
+ state: PictureState
+ /** The last AWAKE read, epoch ms. */
+ seenAt: number | null
+ /** When the node list below was true — the last awake read, or the record it came from. */
+ listAt: number | null
+ nodes: PictureNode[]
+ /** lower-case id → exact id. */
+ caseMap: Record<string, string>
+}
+
+export function emptyPicture(): GridPicture {
+ return { spec: 1, state: 'unknown', seenAt: null, listAt: null, nodes: [], caseMap: {} }
+}
+
+/** The join key for a model id across every source: trimmed, without case. (Not `localModels.ts`'s own
+ * private key, which also strips `.gguf` for a different comparison.) */
+export const idKey = (id: string): string => id.trim().toLowerCase()
+const hasUpperCase = (id: string): boolean => id !== id.toLowerCase()
+
+/**
+ * `caseMap` with `ids` folded in. The first spelling of a name wins, except that one carrying an
+ * upper-case letter always replaces an all-lower-case one: the lower-case form is what the overview
+ * already showed, so it is never the better answer — and it never overwrites a better one.
+ */
+export function withSpellings(caseMap: Record<string, string>, ids: readonly string[]): Record<string, string> {
+ let next = caseMap
+ for (const raw of ids) {
+ const exact = raw.trim()
+ const key = idKey(exact)
+ if (!key) continue
+ const known = next[key]
+ if (known === undefined || (!hasUpperCase(known) && hasUpperCase(exact))) {
+ if (next === caseMap) next = { ...caseMap }
+ next[key] = exact
+ }
+ }
+ return next
+}
+
+/** Whether some model an awake answer names has no spelling yet — the only reason discovery is read. */
+export function unspelled(picture: GridPicture, nodes: readonly ReadNode[]): string[] {
+ const missing = new Set<string>()
+ for (const node of nodes) {
+ for (const model of node.models) {
+ const key = idKey(model)
+ if (key && picture.caseMap[key] === undefined && !hasUpperCase(model)) missing.add(key)
+ }
+ }
+ return [...missing]
+}
+
+const listed = (key: string, at: number): PictureModel => ({ key, seenAt: at, absentSince: null, absentReads: 0 })
+
+function nodeFrom(read: { name: string; engine: string; models: string[] }, at: number, isMine: boolean): PictureNode {
+ const keys = [...new Set(read.models.map(idKey).filter(Boolean))]
+ return { name: read.name, engine: read.engine, isMine, models: keys.map((key) => listed(key, at)) }
+}
+
+/**
+ * An awake answer merged in, per model. A model the answer does not list stays until it has been missing
+ * for [GRID_MODEL_RETENTION_MS] across at least two awake answers — so a cold wake that answers empty
+ * blanks nothing — unless `provenStopped` says this computer itself stopped serving it, which is positive
+ * evidence and takes effect at once.
+ *
+ * Previous entries are paired with live ones by name and engine, first fit, so two same-named machines
+ * stay two entries.
+ */
+export function mergeAwake(
+ previous: GridPicture,
+ nodes: readonly ReadNode[],
+ now: number,
+ isMine: (node: ReadNode) => boolean,
+ provenStopped: (name: string, key: string) => boolean,
+): GridPicture {
+ const live = nodes.map((node) => nodeFrom(node, now, isMine(node)))
+ const paired = new Set<PictureNode>()
+ const kept: PictureNode[] = []
+ const carry = (entry: PictureNode, model: PictureModel): PictureModel | null => {
+ if (provenStopped(entry.name, model.key)) return null
+ const absentSince = model.absentSince ?? now
+ const absentReads = model.absentReads + 1
+ const gone = now - absentSince >= GRID_MODEL_RETENTION_MS && absentReads >= MIN_ABSENT_READS
+ return gone ? null : { ...model, absentSince, absentReads }
+ }
+ for (const entry of previous.nodes) {
+ const match = live.find((node) => !paired.has(node) && node.name === entry.name && node.engine === entry.engine)
+ const carried = entry.models.flatMap((model) => {
+ if (match?.models.some((current) => current.key === model.key)) return []
+ const next = carry(entry, model)
+ return next ? [next] : []
+ })
+ if (match) {
+ paired.add(match)
+ match.models.push(...carried)
+ } else if (carried.length) {
+ kept.push({ ...entry, models: carried })
+ }
+ }
+ return { ...previous, state: 'awake', seenAt: now, listAt: now, nodes: [...live, ...kept] }
+}
+
+/**
+ * The asleep answer taken in. Its record, when there is one, is a read made `ageSeconds` ago: it replaces
+ * the node entries only when it is not older than the last awake read, and a record past
+ * [GRID_LAST_KNOWN_MAX_AGE_MS] is ignored. `isMine` carries over only when exactly one previous entry had
+ * that name — a record never says whose a machine is.
+ */
+export function withAsleep(previous: GridPicture, lastKnown: LastKnown | null, now: number): GridPicture {
+ const asleep: GridPicture = { ...previous, state: 'asleep' }
+ if (!lastKnown) return asleep
+ const ageMs = lastKnown.ageSeconds * 1000
+ if (ageMs > GRID_LAST_KNOWN_MAX_AGE_MS) return asleep
+ const takenAt = now - ageMs
+ const spelled = { ...asleep, caseMap: withSpellings(previous.caseMap, lastKnown.ids) }
+ if (previous.seenAt !== null && takenAt < previous.seenAt) return spelled
+ const mine = (name: string): boolean => {
+ const same = previous.nodes.filter((node) => node.name === name)
+ return same.length === 1 && same[0]!.isMine
+ }
+ return {
+ ...spelled,
+ listAt: takenAt,
+ nodes: lastKnown.nodes.map((node) => nodeFrom(node, takenAt, mine(node.name))),
+ }
+}
+
+/** A read that failed: the state says so, and every row stays exactly as it was. */
+export function withUnknown(previous: GridPicture): GridPicture {
+ return { ...previous, state: 'unknown' }
+}
+
+/** One of this computer's own run records for a grid, as `servedHere` reads it. */
+export interface LocalRecord {
+ /** The node name it registers under (`meta_name`). */
+ name: string
+ /** What it advertises, in the case it advertised it. */
+ ids: string[]
+ /** An integer > 0 while a process holds it; null while a join is mid-spawn (it writes `pid: 0`). */
+ pid: number | null
+ /** Whether that pid is a live process. False for a null pid. */
+ alive: boolean
+}
+
+/**
+ * What this computer's own records prove about a grid's rows — never decided by hostname, the machine
+ * list, or the heartbeat sidecar (the sidecar goes stale while a healthy provider is parked).
+ *
+ * `seen` is what this daemon has watched a LIVE record serve since the grid's last awake read, keyed
+ * `name\0key`; the caller rebuilds it at every awake read and adds to it at every look.
+ */
+export interface ServedHere {
+ records: readonly LocalRecord[]
+ seen: ReadonlySet<string>
+ /** The account's own grid, where every node is the account's. */
+ own: boolean
+}
+
+export const servedKey = (name: string, key: string): string => `${name}\u0000${key}`
+
+/**
+ * Whether this computer has positive evidence that `entry` no longer serves `key`. All must hold: a live
+ * record here was seen serving it since the last awake read; no record serves it now, or every one that
+ * does names a pid that no longer exists (a pid-0 record is a join mid-spawn — not evidence); exactly one
+ * entry in the picture carries that name (so a same-named machine elsewhere is never touched); and the
+ * grid is the account's own, or the entry is marked as the account's.
+ */
+export function provenStopped(picture: GridPicture, here: ServedHere, name: string, key: string): boolean {
+ if (!here.seen.has(servedKey(name, key))) return false
+ const serving = here.records.filter((record) => record.name === name && record.ids.some((id) => idKey(id) === key))
+ if (serving.some((record) => record.pid === null || record.alive)) return false
+ const same = picture.nodes.filter((node) => node.name === name)
+ if (same.length !== 1) return false
+ return here.own || same[0]!.isMine
+}
+
+/** One row of the picker's section: the id an engine is pointed at, and which machine answers it. */
+export interface GridModelRow {
+ id: string
+ node: string
+}
+
+export interface SectionView {
+ models: GridModelRow[]
+ state: PictureState
+ /** ISO-8601 of the last awake read. */
+ seenAt: string | null
+ /** How old the list shown is, in whole seconds. */
+ lastKnownAge: number | null
+}
+
+/**
+ * What the picker is told about one grid. While the grid is not awake, this computer's own live records
+ * are shown under its name (a model this computer serves is never "set up your first model"), and a row
+ * this computer has proof it stopped is taken out at once.
+ */
+export function sectionView(picture: GridPicture, here: ServedHere, now: number): SectionView {
+ const awake = picture.state === 'awake'
+ const caseMap = withSpellings(picture.caseMap, here.records.flatMap((record) => record.ids))
+ const rows: GridModelRow[] = []
+ const offered = new Set<string>()
+ const offer = (key: string, node: string): void => {
+ if (!key || offered.has(key)) return
+ offered.add(key)
+ rows.push({ id: caseMap[key] ?? key, node })
+ }
+ for (const node of picture.nodes) {
+ if (node.engine === ROUTER_ENGINE) continue
+ for (const model of node.models) {
+ if (!awake && provenStopped(picture, here, node.name, model.key)) continue
+ offer(model.key, node.name)
+ }
+ }
+ if (!awake) {
+ for (const record of here.records) {
+ if (record.pid === null || !record.alive) continue
+ for (const id of record.ids) offer(idKey(id), record.name)
+ }
+ }
+ return {
+ models: rows,
+ state: picture.state,
+ seenAt: picture.seenAt === null ? null : new Date(picture.seenAt).toISOString(),
+ lastKnownAge: picture.listAt === null ? null : Math.max(0, Math.round((now - picture.listAt) / 1000)),
+ }
+}
+
+/** Bounds on a picture read back from disk — the same as on an answer read off the wire (`gridReader.ts`),
+ * except the spellings, which accumulate across answers, records and discovery rather than arriving in
+ * one. A file past them is cut, never trusted to be small. */
+const MAX_SAVED_NODES = 256
+const MAX_SAVED_MODELS_PER_NODE = 256
+const MAX_SAVED_TEXT = 256
+const MAX_SAVED_SPELLINGS = 4096
+
+/** A picture read back from disk, or null when it is not one this module wrote. Never trusts the file. */
+export function parsePicture(value: unknown): GridPicture | null {
+ const record = value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : null
+ if (!record || record.spec !== 1) return null
+ const states: PictureState[] = ['awake', 'asleep', 'waking', 'unknown']
+ const time = (v: unknown): number | null => typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : null
+ const str = (v: unknown): string => typeof v === 'string' ? v.slice(0, MAX_SAVED_TEXT) : ''
+ if (!states.includes(record.state as PictureState) || !Array.isArray(record.nodes)) return null
+ const nodes: PictureNode[] = []
+ for (const raw of record.nodes.slice(0, MAX_SAVED_NODES)) {
+ if (!raw || typeof raw !== 'object' || !Array.isArray((raw as PictureNode).models)) continue
+ const node = raw as Record<string, unknown>
+ const models = (node.models as unknown[]).slice(0, MAX_SAVED_MODELS_PER_NODE).flatMap((m): PictureModel[] => {
+ const model = m as Record<string, unknown> | null
+ const seenAt = time(model?.seenAt)
+ const key = str(model?.key)
+ if (!model || !key || seenAt === null) return []
+ const absentReads = typeof model.absentReads === 'number' && Number.isInteger(model.absentReads) && model.absentReads >= 0 ? model.absentReads : 0
+ return [{ key, seenAt, absentSince: time(model.absentSince), absentReads }]
+ })
+ nodes.push({ name: str(node.name), engine: str(node.engine), isMine: node.isMine === true, models })
+ }
+ const caseMap: Record<string, string> = {}
+ if (record.caseMap && typeof record.caseMap === 'object' && !Array.isArray(record.caseMap)) {
+ for (const [key, exact] of Object.entries(record.caseMap as Record<string, unknown>).slice(0, MAX_SAVED_SPELLINGS)) {
+ if (typeof exact === 'string' && exact && idKey(exact) === key) caseMap[key] = exact.slice(0, MAX_SAVED_TEXT)
+ }
+ }
+ return {
+ spec: 1,
+ // Waking is a person's request in flight, and that request died with the process that made it.
+ state: record.state === 'waking' ? 'unknown' : record.state as PictureState,
+ seenAt: time(record.seenAt),
+ listAt: time(record.listAt),
+ nodes,
+ caseMap,
+ }
+}
diff --git a/cli/src/lib/gridReader.ts b/cli/src/lib/gridReader.ts
new file mode 100644
index 000000000..33772c68a
--- /dev/null
+++ b/cli/src/lib/gridReader.ts
@@ -0,0 +1,303 @@
+/**
+ * Reading a grid the way a person glances at it — without knocking.
+ *
+ * A grid sleeps when nobody uses it, and on the platform a SIGNED-IN read of a sleeping grid wakes it
+ * (grid-apis `wake_routes`: presence of a credential, never its validity). This daemon used to read
+ * every grid an account is in about once a minute, with the account's credential, so none of them ever
+ * slept — and the read that woke one usually came back empty, because a just-woken master has not heard
+ * from its providers yet. Everything here reads with NO credential:
+ *
+ * - The master's overview and its provider discovery are public (grid-src serves both unauthenticated),
+ * so an AWAKE grid answers exactly as it would a signed-in read;
+ * - a SLEEPING grid answers at once `503 {code: "grid_asleep"}` and is not started, and that answer may
+ * carry `last_known` — what the grid was serving when the platform put it to sleep.
+ *
+ * ⚠️ **Never add a credential to anything in this file.** Not an Authorization header, not x-api-key,
+ * not a token in the query. The whole resource saving rests on it, and its failure is SILENT: every list
+ * still looks perfect while every open harness keeps every grid awake again. A person's act that SHOULD
+ * wake a grid (a message, Start, an explicit wake) goes out another way, on purpose.
+ *
+ * The values below are cross-repo (the lockstep register; `tests/test_grid_reads_lockstep.py` in the
+ * public `grid` repository finds the two literals by these exact quoted spellings).
+ */
+import { VERSION } from '../version.js'
+import { gridExec, gridJson } from './gridExec.js'
+
+/** The master's two public reads, under a grid's address. ⚠️ Cross-repo literals (grid-src's routes,
+ * grid-apis' route rules and sleep record, the public CLI's `remote_overview`). */
+export const GRID_OVERVIEW_PATH = '/relay/v1/grid/overview'
+export const GRID_DISCOVER_PATH = '/nodes/discover'
+
+/** The code beside `detail` on a sleeping grid's 503 (grid-apis `grid_proxy.GRID_ASLEEP_CODE`). Compared
+ * for EQUALITY only: any other code, or none, is "not answering" — never asleep, never awake. */
+export const GRID_ASLEEP_CODE = 'grid_asleep'
+
+/** The public CLI's flag that reads without a credential (`grid` ≥ 0.3.49). A `grid` too old for it exits
+ * 2 at argparse, before anything is sent — so the fallback below can never turn into a waking read. */
+export const GRID_NO_WAKE_FLAG = '--no-wake'
+
+/** The owner status of a grid the platform's reaper slept (grid-apis `grid_sleep_state.ASLEEP`), as
+ * `grid info --json` passes it through. Only the grid's owner sees a status at all. */
+export const OWNER_ASLEEP_STATUS = 'asleep'
+
+/** A read that has not answered in this long is not going to; the list shown meanwhile is the last one. */
+const READ_TIMEOUT_MS = 10_000
+
+/** An overview is a few KB; anything past this is not one, and is not held in memory to find out. */
+const MAX_BODY_BYTES = 2 * 1024 * 1024
+
+/** Bounds on what a hostile or broken answer can put into the picture (and so onto disk). */
+const MAX_NODES = 256
+const MAX_MODELS_PER_NODE = 256
+const MAX_IDS = 1024
+const MAX_TEXT = 256
+
+/** Why a request is being made — the User-Agent's trailing word, read by people counting the platform's
+ * "wake fired" journal lines (the zero-baseline alarm keys on `(read)`). Attribution only. */
+export type ReadPurpose = 'read' | 'wake' | 'prewarm' | 'prewarm-key'
+
+export function harnessUserAgent(purpose: ReadPurpose): string {
+ return `autonomous-harness/${VERSION} (${purpose})`
+}
+
+/** One engine node as a read reports it. `models` are ids as the grid lists them — lower-cased by the
+ * overview, exact from the CLI fallback; the picture keys them without case either way. */
+export interface ReadNode {
+ name: string
+ engine: string
+ models: string[]
+ /** The provider's account, when the grid publishes it (it withholds it on an `os-community` grid).
+ * Read only to decide whether a node is the account's own, and never persisted. */
+ providerEmail: string | null
+}
+
+/** What the platform recorded a grid serving when it slept it (grid-apis `sleep_record.answer_field`). */
+export interface LastKnown {
+ /** How long ago the record was taken, computed by the SERVER so no clock here matters. */
+ ageSeconds: number
+ nodes: Array<{ name: string; engine: string; models: string[] }>
+ /** Exact-case candidate ids — a spelling source for the case map, never a list of what is served. */
+ ids: string[]
+}
+
+export type GridRead =
+ /** It answered: these are the live nodes. `rawNodes` is the overview's own node objects, for the Model
+ * Manager, which reads telemetry off them; through the CLI fallback, the little `grid models` says. */
+ | { kind: 'awake'; nodes: ReadNode[]; curatedIds: string[]; rawNodes: Array<Record<string, unknown>> }
+ /** The platform says it is resting. `lastKnown` is its record, when it has a valid one. */
+ | { kind: 'asleep'; lastKnown: LastKnown | null }
+ /** Anything else: stopped by its owner, master down, deleted, a codeless 503, garbage, a timeout. */
+ | { kind: 'unanswered' }
+ /** No HTTP answer at all — the one case the CLI fallback is for. */
+ | { kind: 'unreachable' }
+
+type Json = Record<string, unknown>
+const isObject = (value: unknown): value is Json => !!value && typeof value === 'object' && !Array.isArray(value)
+const text = (value: unknown): string =>
+ typeof value === 'string' ? value.replace(/[\u0000-\u001f\u007f]/g, ' ').trim().slice(0, MAX_TEXT) : ''
+const strings = (value: unknown, limit: number): string[] =>
+ Array.isArray(value) ? value.map(text).filter(Boolean).slice(0, limit) : []
+
+/**
+ * The address a read goes to, or null when it is not one this daemon will send anything to: `https`, or
+ * plain `http` to this computer only (a developer's local grid), with no credentials in the URL itself.
+ * Null is not a verdict on the grid — the caller falls back to the CLI, which knows its own transport.
+ */
+export function readBase(gridUrl: string | null): string | null {
+ if (!gridUrl) return null
+ let url: URL
+ try { url = new URL(gridUrl) } catch { return null }
+ const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)
+ if (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) return null
+ if (url.username || url.password) return null
+ url.search = ''
+ url.hash = ''
+ return url.href.replace(/\/+$/, '')
+}
+
+/** A GET with no credential, bounded in time, size and redirects. Throws only on a network failure. */
+async function get(base: string, path: string, purpose: ReadPurpose): Promise<{ status: number; body: unknown }> {
+ const response = await fetch(`${base}${path}`, {
+ method: 'GET',
+ headers: { 'user-agent': harnessUserAgent(purpose), accept: 'application/json' },
+ // A redirect is answered as what it is — not followed to wherever it points.
+ redirect: 'manual',
+ signal: AbortSignal.timeout(READ_TIMEOUT_MS),
+ })
+ return { status: response.status, body: await boundedJson(response) }
+}
+
+/** The body as JSON, or undefined when it is not JSON or is larger than any real answer. */
+async function boundedJson(response: Response): Promise<unknown> {
+ const declared = Number(response.headers.get('content-length'))
+ if (Number.isFinite(declared) && declared > MAX_BODY_BYTES) {
+ await response.body?.cancel().catch(() => {})
+ return undefined
+ }
+ const reader = response.body?.getReader()
+ if (!reader) return undefined
+ const chunks: Uint8Array[] = []
+ let size = 0
+ for (;;) {
+ const { done, value } = await reader.read()
+ if (done) break
+ size += value.byteLength
+ if (size > MAX_BODY_BYTES) {
+ await reader.cancel().catch(() => {})
+ return undefined
+ }
+ chunks.push(value)
+ }
+ try { return JSON.parse(Buffer.concat(chunks).toString('utf8')) } catch { return undefined }
+}
+
+function readNodes(value: unknown): ReadNode[] {
+ if (!Array.isArray(value)) return []
+ return value.filter(isObject).slice(0, MAX_NODES).map((node) => ({
+ name: text(node.name),
+ engine: text(node.engine),
+ models: strings(node.models, MAX_MODELS_PER_NODE),
+ providerEmail: typeof node.provider_email === 'string' && node.provider_email ? node.provider_email : null,
+ }))
+}
+
+/** `last_known`, or null when it is absent or not in the shape grid-apis writes. Parsing it sends nothing. */
+export function parseLastKnown(value: unknown): LastKnown | null {
+ if (!isObject(value)) return null
+ const age = value.age_seconds
+ if (typeof age !== 'number' || !Number.isFinite(age) || age < 0) return null
+ if (!Array.isArray(value.nodes)) return null
+ const nodes = value.nodes.filter(isObject).slice(0, MAX_NODES).map((node) => ({
+ name: text(node.name),
+ engine: text(node.engine),
+ models: strings(node.models, MAX_MODELS_PER_NODE),
+ }))
+ return { ageSeconds: age, nodes, ids: strings(value.ids, MAX_IDS) }
+}
+
+/**
+ * Read a grid's public overview, with no credential, and say which of the four it is.
+ *
+ * Asleep is a 503 whose JSON `code` EQUALS `grid_asleep` and nothing else: `grid_stopped` (its owner
+ * stopped it), `grid_master_down`, a 410 for a deleted grid and a codeless 503 are all "not answering",
+ * and so is a 200 that is not an overview.
+ */
+export async function readOverview(base: string): Promise<GridRead> {
+ let answer: { status: number; body: unknown }
+ try {
+ answer = await get(base, GRID_OVERVIEW_PATH, 'read')
+ } catch {
+ return { kind: 'unreachable' }
+ }
+ const { status, body } = answer
+ if (status === 503 && isObject(body) && body.code === GRID_ASLEEP_CODE) {
+ return { kind: 'asleep', lastKnown: parseLastKnown(body.last_known) }
+ }
+ // A 200 that is not an overview (a maintenance page, a proxy's own JSON) is not "awake with nothing":
+ // read as that, it would count as an empty answer and start taking models off every list.
+ if (status !== 200 || !isObject(body) || !Array.isArray(body.nodes)) return { kind: 'unanswered' }
+ const curatedIds = Array.isArray(body.models)
+ ? body.models.filter(isObject).map((entry) => text(entry.id)).filter(Boolean).slice(0, MAX_IDS)
+ : []
+ const rawNodes = body.nodes.filter(isObject).slice(0, MAX_NODES)
+ return { kind: 'awake', nodes: readNodes(rawNodes), curatedIds, rawNodes }
+}
+
+/** The master's display rule (grid-src `model_ids.display_model_name`): a trailing `.gguf` removed,
+ * compared without regard to case, and the case of what is left KEPT. */
+export function displayModelName(raw: string): string {
+ return raw.toLowerCase().endsWith('.gguf') ? raw.slice(0, -'.gguf'.length) : raw
+}
+
+/**
+ * Every served model's exact-case name from the grid's public provider discovery, under the display rule.
+ * Asked only when an awake answer names a model no other source can spell. Any failure is an empty list,
+ * which leaves the lower-case id the overview gave — a spelling problem, never a reason to fail a read.
+ */
+export async function readDiscoveryIds(base: string): Promise<string[]> {
+ try {
+ const { status, body } = await get(base, GRID_DISCOVER_PATH, 'read')
+ if (status !== 200 || !isObject(body) || !Array.isArray(body.providers)) return []
+ const ids: string[] = []
+ for (const provider of body.providers.filter(isObject)) {
+ const capabilities = isObject(provider.capabilities) ? provider.capabilities : {}
+ const entries = isObject(capabilities.models) ? capabilities.models : {}
+ if (!Array.isArray(provider.models)) continue
+ for (const route of provider.models) {
+ const entry = typeof route === 'string' ? entries[route] : undefined
+ const raw = isObject(entry) ? text(entry.raw_model_id) : ''
+ if (raw) ids.push(displayModelName(raw))
+ }
+ }
+ return ids.slice(0, MAX_IDS)
+ } catch {
+ return []
+ }
+}
+
+/** `grid info <grid> --json`, reduced to what a read needs. Null when `grid` could not answer. */
+export interface GridInfo {
+ /** The owner status (`running`, `asleep`, `stopped`, …); null for a member, who cannot read it. */
+ status: string | null
+ /** The grid's address; reads go to `<gridUrl>/relay/v1/grid/overview`. */
+ gridUrl: string | null
+}
+
+export async function readGridInfo(gridName: string): Promise<GridInfo | null> {
+ const { value } = await gridJson<Json>(['--remote', 'info', gridName])
+ if (!isObject(value)) return null
+ return {
+ status: typeof value.status === 'string' && value.status ? value.status : null,
+ gridUrl: typeof value.grid_url === 'string' && value.grid_url.trim() ? value.grid_url.trim() : null,
+ }
+}
+
+/** The `code` of `grid`'s `--json` refusal envelope (one JSON line on stderr), or null. */
+function envelopeCode(stderr: string): string | null {
+ for (const line of stderr.split('\n')) {
+ const trimmed = line.trim()
+ if (!trimmed.startsWith('{')) continue
+ try {
+ const parsed = JSON.parse(trimmed) as unknown
+ const error = isObject(parsed) && isObject(parsed.error) ? parsed.error : null
+ if (error && typeof error.code === 'string') return error.code
+ } catch { /* not the envelope */ }
+ }
+ return null
+}
+
+/**
+ * The fallback for a machine that reaches the internet only through something the daemon's own `fetch`
+ * does not use (an HTTP proxy the `grid` CLI honours): `grid models <grid> --json --no-wake`, ONCE.
+ *
+ * ⚠️ Never retried without the flag. A `grid` too old for it exits 2 before sending anything, and that is
+ * "no read" — a stale list, which is the direction every failure here must take.
+ */
+export async function readViaCli(gridName: string): Promise<GridRead> {
+ const result = await gridExec(['--remote', 'models', gridName, GRID_NO_WAKE_FLAG, '--json'])
+ if (result.code !== 'OK') {
+ return envelopeCode(result.stderr) === GRID_ASLEEP_CODE ? { kind: 'asleep', lastKnown: null } : { kind: 'unanswered' }
+ }
+ let rows: unknown
+ try { rows = JSON.parse(result.stdout) } catch { return { kind: 'unanswered' } }
+ if (!Array.isArray(rows)) return { kind: 'unanswered' }
+ // `grid models` answers one row per (model, node); the picture wants nodes.
+ const byNode = new Map<string, ReadNode>()
+ const exact: string[] = []
+ for (const row of rows.filter(isObject).slice(0, MAX_NODES * MAX_MODELS_PER_NODE)) {
+ const model = text(row.model)
+ if (!model) continue
+ const name = text(row.node)
+ const engine = text(row.engine)
+ const key = `${name}\u0000${engine}`
+ const node = byNode.get(key) ?? { name, engine, models: [], providerEmail: null }
+ node.models.push(model)
+ byNode.set(key, node)
+ exact.push(model)
+ }
+ const nodes = [...byNode.values()].slice(0, MAX_NODES)
+ // The Model Manager reads node objects; these carry what `grid models` knows, and a node it lists is
+ // one the grid is routing to right now.
+ const rawNodes = nodes.map((node) => ({ name: node.name, engine: node.engine, models: node.models, online: true }))
+ return { kind: 'awake', nodes, curatedIds: exact.slice(0, MAX_IDS), rawNodes }
+}
diff --git a/cli/src/lib/gridReads.spec.ts b/cli/src/lib/gridReads.spec.ts
new file mode 100644
index 000000000..20931d9e9
--- /dev/null
+++ b/cli/src/lib/gridReads.spec.ts
@@ -0,0 +1,480 @@
+/**
+ * The model list against a fake relay and a fake `grid` first on PATH — the seam a desktop, an old
+ * desktop and the Model Manager all read through (grid-reads-without-waking issue 02).
+ *
+ * Every assertion is on what leaves the seam: the sections a picker is given, and the requests that
+ * reached the relay (method, path, headers), recorded by the relay itself. The clock is the service's
+ * own injected `now`, so the 15s / 120s / 150s rules are exercised without waiting for them.
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { spawnSync } from 'node:child_process'
+import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs'
+import { createServer, type IncomingHttpHeaders, type Server } from 'node:http'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { installFakeGrid, type FakeGrid, type FakeGridPlan, type FakeGridTurn } from './__fixtures__/fakeGrid.js'
+import { forgetGridModels, gridInventory, listAllGridModels, listGridModels, resetGridModels, type GridModelsService, type GridSection } from './gridModels.js'
+
+const OWN = 'mine', OWN_ID = 'net-own', TEAM = 'team', TEAM_ID = 'net-team'
+const EMAIL = '[email protected]'
+const OVERVIEW = '/relay/v1/grid/overview', DISCOVER = '/nodes/discover'
+
+interface Seen { method: string; path: string; headers: IncomingHttpHeaders }
+interface Answer { status: number; body?: unknown; delayMs?: number }
+
+let server: Server, base: string, seen: Seen[], answers: Map<string, Answer>
+let root: string, gridHome: string, grid: FakeGrid, service: GridModelsService, clock: number
+
+const answer = (gridId: string, path: string, value: Answer): void => { answers.set(`/g/${gridId}${path}`, value) }
+const node = (name: string, models: string[], extra: Record<string, unknown> = {}) =>
+ ({ name, engine: 'llama.cpp', models, online: true, provider_email: EMAIL, ...extra })
+const awake = (nodes: unknown[], curated: string[] = []): Answer => ({ status: 200, body: { nodes, models: curated.map((id) => ({ id })) } })
+const asleep = (lastKnown?: unknown): Answer =>
+ ({ status: 503, body: { detail: 'resting', code: 'grid_asleep', ...(lastKnown === undefined ? {} : { last_known: lastKnown }) } })
+const record = (ageSeconds: number, nodes: Array<{ name: string; models: string[] }>, ids: string[] = []) =>
+ ({ age_seconds: ageSeconds, nodes: nodes.map((n) => ({ engine: 'llama.cpp', ...n })), ids })
+
+/** A grid's `grid info --json`, as the public CLI prints it (a member sees no status). */
+const info = (name: string, status: string | null, url: string): FakeGridTurn =>
+ ({ stdout: JSON.stringify({ grid: name, type: '', status, grid_url: url }) })
+
+function plan(options: { grids?: 'own' | 'both'; ownStatus?: string | null; ownUrl?: string; teamUrl?: string; models?: FakeGridTurn } = {}): FakeGridPlan {
+ const rows = [{ grid: OWN, id: OWN_ID, type: 'permissioned-public' }]
+ if (options.grids !== 'own') rows.push({ grid: TEAM, id: TEAM_ID, type: 'permissioned-providers' })
+ return {
+ ls: { stdout: JSON.stringify(rows) },
+ [`info ${OWN}`]: info(OWN, options.ownStatus === undefined ? 'running' : options.ownStatus, options.ownUrl ?? `${base}/g/${OWN_ID}`),
+ [`info ${TEAM}`]: info(TEAM, null, options.teamUrl ?? `${base}/g/${TEAM_ID}`),
+ ...(options.models ? { models: options.models } : {}),
+ }
+}
+
+const reads = (gridId: string, path = OVERVIEW): number => seen.filter((r) => r.path === `/g/${gridId}${path}`).length
+const section = (sections: GridSection[], name: string): GridSection => sections.find((s) => s.name === name)!
+const ids = (s: GridSection): string[] => s.models.map((m) => m.id)
+
+/** Ask as a picker does, then let every read it started land, then ask again for what they produced. */
+async function look(own: string | null = OWN): Promise<GridSection[]> {
+ await listAllGridModels(own)
+ await service.settled()
+ return listAllGridModels(own, { refresh: false })
+}
+
+/** Move the clock past every memo a read could be sitting in. */
+const later = (seconds = 125): void => { clock += seconds * 1000 }
+
+/** A run record `grid join` writes for this computer on grid `gridId`. */
+function runRecord(gridId: string, file: string, fields: Record<string, unknown>): void {
+ const dir = join(gridHome, 'run', 'engines', gridId)
+ mkdirSync(dir, { recursive: true })
+ writeFileSync(join(dir, file), JSON.stringify({ engines: [], advertise_as: [], ...fields }))
+}
+const removeRecord = (gridId: string, file: string): void => rmSync(join(gridHome, 'run', 'engines', gridId, file), { force: true })
+
+/** A pid that existed a moment ago and no longer does. */
+function deadPid(): number {
+ const child = spawnSync(process.execPath, ['-e', 'process.stdout.write(String(process.pid))'], { encoding: 'utf8' })
+ return Number(child.stdout)
+}
+
+beforeEach(async () => {
+ seen = []
+ answers = new Map()
+ server = createServer((req, res) => {
+ seen.push({ method: req.method ?? '', path: req.url ?? '', headers: req.headers })
+ const found = answers.get(req.url ?? '') ?? { status: 404, body: { detail: 'Not Found' } }
+ setTimeout(() => {
+ res.writeHead(found.status, { 'content-type': 'application/json' })
+ res.end(JSON.stringify(found.body ?? {}))
+ }, found.delayMs ?? 0)
+ })
+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
+ base = `http://127.0.0.1:${(server.address() as { port: number }).port}`
+ root = mkdtempSync(join(tmpdir(), 'grid-reads-'))
+ gridHome = join(root, 'grid-home')
+ mkdirSync(gridHome, { recursive: true })
+ clock = Date.parse('2026-09-24T10:00:00Z')
+ service = resetGridModels({ now: () => clock, dataDir: () => join(root, 'data'), gridHome: () => gridHome, email: () => EMAIL })
+ grid = installFakeGrid(plan())
+})
+
+afterEach(async () => {
+ await service.settled()
+ grid.dispose()
+ server.closeAllConnections()
+ await new Promise<void>((resolve) => server.close(() => resolve()))
+ rmSync(root, { recursive: true, force: true })
+})
+
+describe('no automatic read carries a credential', () => {
+ it('the model list, the Model Manager and a launch check read with no Authorization and no x-api-key', async () => {
+ answer(OWN_ID, OVERVIEW, awake([node('mac', ['small-q4'])]))
+ answer(TEAM_ID, OVERVIEW, awake([node('rig', ['big-model'], { provider_email: '[email protected]' })]))
+ answer(TEAM_ID, DISCOVER, { status: 200, body: { providers: [] } })
+
+ await look()
+ await gridInventory(OWN, true)
+ forgetGridModels()
+ await listGridModels(TEAM)
+
+ expect(seen.length).toBeGreaterThanOrEqual(4)
+ for (const request of seen) {
+ expect(request.method).toBe('GET')
+ expect(request.headers.authorization).toBeUndefined()
+ expect(request.headers['x-api-key']).toBeUndefined()
+ expect(request.headers['user-agent']).toMatch(/^autonomous-harness\/\S+ \(read\)$/)
+ }
+ // The credentialed model-list read is gone, and so is the only place its credential came from.
+ expect(seen.some((r) => r.path.endsWith('/relay/v1/models'))).toBe(false)
+ expect(grid.calls().some((argv) => argv.includes('--env'))).toBe(false)
+ })
+
+ it('parsing a sleep record sends nothing of its own', async () => {
+ grid.replan(plan({ grids: 'own' }))
+ answer(OWN_ID, OVERVIEW, asleep(record(3600, [{ name: 'mac', models: ['small-q4'] }], ['Small-Q4'])))
+
+ const sections = await look()
+
+ expect(seen.map((r) => r.path)).toEqual([`/g/${OWN_ID}${OVERVIEW}`])
+ expect(section(sections, OWN)).toMatchObject({ state: 'asleep', models: [{ id: 'Small-Q4', node: 'mac' }], lastKnownAge: 3600 })
+ })
+})
+
+describe('what an answer means', () => {
+ it('a 200 reads awake, and a 503 whose code is grid_asleep reads asleep', async () => {
+ answer(TEAM_ID, OVERVIEW, awake([node('rig', ['big-model'])], ['Big-Model']))
+ expect(section(await look(), TEAM)).toMatchObject({ state: 'awake', models: [{ id: 'Big-Model', node: 'rig' }], lastKnownAge: 0 })
+ later()
+ answer(TEAM_ID, OVERVIEW, asleep())
+ expect(section(await look(), TEAM)).toMatchObject({ state: 'asleep', models: [{ id: 'Big-Model', node: 'rig' }] })
+ })
+
+ it.each<[string, Answer]>([
+ ['a codeless 503', { status: 503, body: { detail: 'upstream unavailable' } }],
+ ['grid_stopped', { status: 503, body: { detail: 'stopped by its owner', code: 'grid_stopped' } }],
+ ['grid_master_down', { status: 503, body: { detail: 'master down', code: 'grid_master_down' } }],
+ ['a 410', { status: 410, body: { detail: 'deleted', code: 'grid_deleted' } }],
+ ['an asleep code on a 200', { status: 200, body: { code: 'grid_asleep' } }],
+ ['a 200 that is not JSON', { status: 200, body: undefined }],
+ ])('%s reads "not answering" and keeps the last known list', async (_name, failure) => {
+ answer(TEAM_ID, OVERVIEW, awake([node('rig', ['big-model'])], ['Big-Model']))
+ await look()
+ later()
+ answer(TEAM_ID, OVERVIEW, failure.body === undefined ? { status: 200, body: 'not an overview' } : failure)
+
+ expect(section(await look(), TEAM)).toMatchObject({ state: 'unknown', models: [{ id: 'Big-Model', node: 'rig' }] })
+ })
+
+ it('a refused connection reads "not answering", keeps the list, and falls back to `grid` once, with --no-wake', async () => {
+ answer(TEAM_ID, OVERVIEW, awake([node('rig', ['big-model'])], ['Big-Model']))
+ await look()
+ const closed = createServer()
+ await new Promise<void>((resolve) => closed.listen(0, '127.0.0.1', resolve))
+ const refused = `http://127.0.0.1:${(closed.address() as { port: number }).port}`
+ await new Promise<void>((resolve) => closed.close(() => resolve()))
+ grid.replan(plan({ teamUrl: refused, models: { exit: 1, stderr: 'Could not reach grid team\n' } }))
+ forgetGridModels()
+
+ expect(section(await look(), TEAM)).toMatchObject({ state: 'unknown', models: [{ id: 'Big-Model', node: 'rig' }] })
+ expect(grid.calls().filter((argv) => argv.includes('models'))).toEqual([['--remote', 'models', TEAM, '--no-wake', '--json']])
+ })
+
+ it('the fallback reads asleep when `grid` refuses with the grid_asleep code, and awake from its rows', async () => {
+ grid.replan(plan({ grids: 'own', ownUrl: 'ftp://nowhere.example', models: { exit: 1, stderr: `${JSON.stringify({ error: { code: 'grid_asleep', message: 'asleep', status: 503 } })}\n` } }))
+ expect(section(await look(), OWN).state).toBe('asleep')
+
+ grid.replan(plan({ grids: 'own', ownUrl: 'ftp://nowhere.example', models: { stdout: JSON.stringify([
+ { model: 'auto', engine: 'grid-router', node: '' }, { model: 'Small-Q4', engine: 'llama.cpp', node: 'mac' },
+ ]) } }))
+ forgetGridModels()
+ later()
+ expect(section(await look(), OWN)).toMatchObject({ state: 'awake', models: [{ id: 'Small-Q4', node: 'mac' }] })
+ expect(seen).toEqual([])
+ })
+
+ it('a `grid` too old for --no-wake yields no read, and is never asked again without the flag', async () => {
+ grid.replan(plan({ grids: 'own', ownUrl: 'ftp://nowhere.example', models: { exit: 2, stderr: 'grid: error: unrecognized arguments: --no-wake\n' } }))
+
+ expect(section(await look(), OWN)).toMatchObject({ state: 'unknown', models: [] })
+ later(); forgetGridModels()
+ await look()
+
+ const asked = grid.calls().filter((argv) => argv.includes('models'))
+ expect(asked).toHaveLength(2)
+ expect(asked.every((argv) => argv.includes('--no-wake'))).toBe(true)
+ expect(seen).toEqual([])
+ })
+})
+
+describe('the own grid while its owner status says asleep', () => {
+ it('is read exactly once per asleep episode, and not again until the status changes', async () => {
+ grid.replan(plan({ grids: 'own', ownStatus: 'asleep' }))
+ answer(OWN_ID, OVERVIEW, asleep(record(600, [{ name: 'mac', models: ['small-q4'] }])))
+
+ expect(section(await look(), OWN).state).toBe('asleep')
+ for (let i = 0; i < 6; i++) { later(); await look() }
+ expect(reads(OWN_ID)).toBe(1)
+
+ grid.replan(plan({ grids: 'own', ownStatus: 'running' }))
+ answer(OWN_ID, OVERVIEW, awake([node('mac', ['small-q4'])]))
+ later()
+ expect(section(await look(), OWN).state).toBe('awake')
+ expect(reads(OWN_ID)).toBe(2)
+
+ grid.replan(plan({ grids: 'own', ownStatus: 'asleep' }))
+ answer(OWN_ID, OVERVIEW, asleep())
+ later(); await look()
+ for (let i = 0; i < 4; i++) { later(); await look() }
+ expect(reads(OWN_ID)).toBe(3)
+ })
+
+ it('a launch check during the episode makes no read of its own, and does not restart the episode', async () => {
+ grid.replan(plan({ grids: 'own', ownStatus: 'asleep' }))
+ answer(OWN_ID, OVERVIEW, asleep(record(600, [{ name: 'mac', models: ['small-q4'] }], ['Small-Q4'])))
+ await look()
+
+ later()
+ expect(await listGridModels(OWN)).toEqual([{ id: 'Small-Q4', node: 'mac' }])
+ later(); await look()
+ expect(reads(OWN_ID)).toBe(1)
+ })
+
+ it('a member grid, whose status nobody but its owner can read, is read on its own memo', async () => {
+ answer(TEAM_ID, OVERVIEW, asleep())
+ await look()
+ later(60); await look()
+ expect(reads(TEAM_ID)).toBe(1)
+ later(61); await look()
+ expect(reads(TEAM_ID)).toBe(2)
+ })
+})
+
+describe('exact case', () => {
+ it('comes from the curated ids, this computer\'s run records and discovery — .gguf stripped, case kept', async () => {
+ grid.replan(plan({ grids: 'own' }))
+ runRecord(OWN_ID, 'remote.json', { meta_name: 'mac', pid: process.pid, advertise_as: ['Small-Q4'] })
+ answer(OWN_ID, OVERVIEW, awake([node('mac', ['small-q4', 'qwen3.6-35b-a3b', 'glm-4.7-flash'])], ['GLM-4.7-Flash']))
+ answer(OWN_ID, DISCOVER, { status: 200, body: { providers: [
+ { models: ['provider:a'], capabilities: { models: { 'provider:a': { raw_model_id: 'Qwen3.6-35B-A3B.GGUF' } } } },
+ ] } })
+
+ expect(ids(section(await look(), OWN))).toEqual(['Small-Q4', 'Qwen3.6-35B-A3B', 'GLM-4.7-Flash'])
+ later(20); await look()
+ expect(reads(OWN_ID, DISCOVER)).toBe(1)
+ })
+
+ it('is never replaced by a lower-case form, and discovery is not asked when every id is spelled', async () => {
+ answer(TEAM_ID, OVERVIEW, awake([node('rig', ['glm-4.7-flash'])], ['GLM-4.7-Flash']))
+ await look()
+ later()
+ answer(TEAM_ID, OVERVIEW, asleep(record(10, [{ name: 'rig', models: ['glm-4.7-flash'] }], ['glm-4.7-flash'])))
+
+ expect(ids(section(await look(), TEAM))).toEqual(['GLM-4.7-Flash'])
+ expect(reads(TEAM_ID, DISCOVER)).toBe(0)
+ })
+})
+
+describe('the picture', () => {
+ it('a cold wake that answers empty blanks nothing; a model leaves after 150s across two awake reads', async () => {
+ answer(TEAM_ID, OVERVIEW, awake([node('rig', ['big-model', 'small-model'])]))
+ await look()
+ clock += 3 * 3600_000
+ answer(TEAM_ID, OVERVIEW, awake([node('rig', ['small-model'])]))
+
+ expect(ids(section(await look(), TEAM))).toEqual(['small-model', 'big-model'])
+ later(20); expect(ids(section(await look(), TEAM))).toEqual(['small-model', 'big-model'])
+ later(20); expect(ids(section(await look(), TEAM))).toEqual(['small-model', 'big-model'])
+ later(111); expect(ids(section(await look(), TEAM))).toEqual(['small-model'])
+ })
+
+ it('the 150s run from the first answer that missed a model, not from when it was last seen', async () => {
+ answer(TEAM_ID, OVERVIEW, awake([node('rig', ['big-model'])]))
+ await look()
+ later(600)
+ answer(TEAM_ID, OVERVIEW, awake([]))
+ expect(ids(section(await look(), TEAM))).toEqual(['big-model'])
+ later(16)
+ expect(ids(section(await look(), TEAM))).toEqual(['big-model'])
+ later(134)
+ expect(ids(section(await look(), TEAM))).toEqual([])
+ })
+
+ it('a sleep record older than the last awake read does not replace it; a newer one does', async () => {
+ answer(TEAM_ID, OVERVIEW, awake([node('rig', ['big-model'])]))
+ await look()
+ later(300)
+ answer(TEAM_ID, OVERVIEW, asleep(record(400, [{ name: 'other', models: ['ghost'] }])))
+ expect(ids(section(await look(), TEAM))).toEqual(['big-model'])
+
+ later()
+ answer(TEAM_ID, OVERVIEW, asleep(record(30, [{ name: 'other', models: ['fresh-model'] }])))
+ expect(section(await look(), TEAM)).toMatchObject({ state: 'asleep', models: [{ id: 'fresh-model', node: 'other' }], lastKnownAge: 30 })
+ })
+
+ it('a record older than 29 days is ignored', async () => {
+ answer(TEAM_ID, OVERVIEW, asleep(record(30 * 86400, [{ name: 'rig', models: ['ancient'] }])))
+ expect(section(await look(), TEAM)).toMatchObject({ state: 'asleep', models: [], lastKnownAge: null })
+ later()
+ answer(TEAM_ID, OVERVIEW, asleep(record(28 * 86400, [{ name: 'rig', models: ['recent'] }])))
+ expect(ids(section(await look(), TEAM))).toEqual(['recent'])
+ })
+
+ it('waits for the first read of a grid it has never seen only so long, and the read lands behind', async () => {
+ service = resetGridModels({ now: () => clock, dataDir: () => join(root, 'data'), gridHome: () => gridHome, email: () => EMAIL, firstReadWaitMs: 50 })
+ const changed: number[] = []
+ service.onChange(() => changed.push(1))
+ grid.replan(plan({ grids: 'own' }))
+ answer(OWN_ID, OVERVIEW, { ...awake([node('mac', ['small-q4'])]), delayMs: 1_500 })
+
+ // Answered before the read could land: nothing is known yet, and it says so — no clock is asserted,
+ // so a loaded machine cannot turn this into a flake.
+ const first = section(await listAllGridModels(OWN), OWN)
+ expect(first).toMatchObject({ state: 'unknown', models: [], lastKnownAge: null })
+
+ await service.settled()
+ expect(section(await listAllGridModels(OWN, { refresh: false }), OWN)).toMatchObject({ state: 'awake', models: [{ id: 'small-q4', node: 'mac' }] })
+ await vi.waitFor(() => expect(changed).toEqual([1]))
+ })
+
+ it('is answered at once from what was read before, even by a daemon that just started', async () => {
+ answer(TEAM_ID, OVERVIEW, awake([node('rig', ['big-model'])]))
+ await look()
+ service = resetGridModels({ now: () => clock, dataDir: () => join(root, 'data'), gridHome: () => gridHome, email: () => EMAIL })
+ answer(TEAM_ID, OVERVIEW, asleep())
+ later()
+
+ // The first answer is the persisted picture; the read it started lands behind it.
+ expect(ids(section(await listAllGridModels(OWN), TEAM))).toEqual(['big-model'])
+ await service.settled()
+ expect(section(await listAllGridModels(OWN, { refresh: false }), TEAM).state).toBe('asleep')
+ })
+})
+
+describe('this computer\'s own models while a grid is not awake (servedHere)', () => {
+ /** The own grid read awake with `nodes`, then asleep with no record. */
+ async function ownAwakeThenAsleep(nodes: unknown[]): Promise<void> {
+ grid.replan(plan({ grids: 'own' }))
+ answer(OWN_ID, OVERVIEW, awake(nodes))
+ await look()
+ later()
+ answer(OWN_ID, OVERVIEW, asleep())
+ await look()
+ }
+
+ it('shows a model this computer serves under its name, though the grid never listed it', async () => {
+ grid.replan(plan({ grids: 'own' }))
+ runRecord(OWN_ID, 'remote.json', { meta_name: 'mac', pid: process.pid, advertise_as: ['Small-Q4'] })
+ answer(OWN_ID, OVERVIEW, asleep())
+
+ expect(section(await look(), OWN)).toMatchObject({ state: 'asleep', models: [{ id: 'Small-Q4', node: 'mac' }] })
+ })
+
+ it('takes a model paused on this computer out at once', async () => {
+ runRecord(OWN_ID, 'remote.json', { meta_name: 'mac', pid: process.pid, models: ['Small-Q4.gguf'] })
+ await ownAwakeThenAsleep([node('mac', ['small-q4']), node('studio', ['big-model'])])
+ removeRecord(OWN_ID, 'remote.json')
+
+ expect(ids(section(await listAllGridModels(OWN, { refresh: false }), OWN))).toEqual(['big-model'])
+ })
+
+ it('takes it out at once when its process died, and while the grid is awake too', async () => {
+ grid.replan(plan({ grids: 'own' }))
+ runRecord(OWN_ID, 'remote.json', { meta_name: 'mac', pid: process.pid, advertise_as: ['Small-Q4'] })
+ answer(OWN_ID, OVERVIEW, awake([node('mac', ['small-q4'])]))
+ await look()
+ runRecord(OWN_ID, 'remote.json', { meta_name: 'mac', pid: deadPid(), advertise_as: ['Small-Q4'] })
+ answer(OWN_ID, OVERVIEW, awake([]))
+ later(20)
+
+ expect(ids(section(await look(), OWN))).toEqual([])
+ })
+
+ it('a model served by two nodes survives the removal of one', async () => {
+ runRecord(OWN_ID, 'remote.json', { meta_name: 'mac', pid: process.pid, advertise_as: ['Small-Q4'] })
+ await ownAwakeThenAsleep([node('mac', ['small-q4']), node('studio', ['small-q4'])])
+ removeRecord(OWN_ID, 'remote.json')
+
+ expect(section(await listAllGridModels(OWN, { refresh: false }), OWN).models).toEqual([{ id: 'Small-Q4', node: 'studio' }])
+ })
+
+ it.each<[string, () => Promise<void>]>([
+ ['a teammate\'s same-named node on a shared grid', async () => {
+ runRecord(TEAM_ID, 'remote.json', { meta_name: 'mac', pid: process.pid, advertise_as: ['Small-Q4'] })
+ answer(TEAM_ID, OVERVIEW, awake([node('mac', ['small-q4'], { provider_email: '[email protected]' })]))
+ await look()
+ later()
+ answer(TEAM_ID, OVERVIEW, asleep())
+ await look()
+ removeRecord(TEAM_ID, 'remote.json')
+ }],
+ ['an empty records directory', async () => {
+ mkdirSync(join(gridHome, 'run', 'engines', TEAM_ID), { recursive: true })
+ answer(TEAM_ID, OVERVIEW, awake([node('mac', ['small-q4'])]))
+ await look()
+ later()
+ answer(TEAM_ID, OVERVIEW, asleep())
+ await look()
+ }],
+ ['a pid-0 record (a join mid-spawn)', async () => {
+ runRecord(TEAM_ID, 'remote.json', { meta_name: 'mac', pid: process.pid, advertise_as: ['Small-Q4'] })
+ answer(TEAM_ID, OVERVIEW, awake([node('mac', ['small-q4'])]))
+ await look()
+ later()
+ answer(TEAM_ID, OVERVIEW, asleep())
+ await look()
+ runRecord(TEAM_ID, 'remote.json', { meta_name: 'mac', pid: 0, advertise_as: ['Small-Q4'] })
+ }],
+ ['a stale heartbeat sidecar beside a parked provider', async () => {
+ runRecord(TEAM_ID, 'remote.json', { meta_name: 'mac', pid: process.pid, advertise_as: ['Small-Q4'] })
+ const sidecar = join(gridHome, 'run', 'engines', TEAM_ID, 'remote.heartbeat')
+ writeFileSync(sidecar, '')
+ utimesSync(sidecar, new Date(0), new Date(0))
+ answer(TEAM_ID, OVERVIEW, awake([node('mac', ['small-q4'])]))
+ await look()
+ later()
+ answer(TEAM_ID, OVERVIEW, asleep())
+ await look()
+ }],
+ ['two same-named entries', async () => {
+ runRecord(TEAM_ID, 'remote.json', { meta_name: 'mac', pid: process.pid, advertise_as: ['Small-Q4'] })
+ answer(TEAM_ID, OVERVIEW, awake([node('mac', ['small-q4']), node('mac', ['small-q4'], { engine: 'vllm' })]))
+ await look()
+ later()
+ answer(TEAM_ID, OVERVIEW, asleep())
+ await look()
+ removeRecord(TEAM_ID, 'remote.json')
+ }],
+ ])('removes nothing for %s', async (_name, arrange) => {
+ await arrange()
+ expect(ids(section(await listAllGridModels(OWN, { refresh: false }), TEAM)).map((id) => id.toLowerCase())).toEqual(['small-q4'])
+ })
+})
+
+describe('the grid list', () => {
+ it('never passes a name that reads as a flag, or carries a control character, to `grid`', async () => {
+ grid.replan({ ...plan({ grids: 'own' }), ls: { stdout: JSON.stringify([
+ { grid: OWN, id: OWN_ID, type: 'permissioned-public' }, { grid: '--json', id: 'x' }, { grid: 'a\nb', id: 'y' },
+ ]) } })
+ answer(OWN_ID, OVERVIEW, awake([]))
+
+ expect((await look()).map((s) => s.name)).toEqual([OWN])
+ expect(grid.calls().filter((argv) => argv[1] === 'info').map((argv) => argv[2])).toEqual([OWN])
+ })
+})
+
+describe('what an older desktop reads', () => {
+ it('every section stays a valid {name, own, models:[{id,node}]}, and a sleeping one keeps its models', async () => {
+ answer(OWN_ID, OVERVIEW, asleep(record(60, [{ name: 'mac', models: ['small-q4'] }])))
+ answer(TEAM_ID, OVERVIEW, { status: 503, body: { detail: 'down' } })
+
+ const sections = await look()
+
+ expect(sections.map((s) => [s.name, s.own])).toEqual([[OWN, true], [TEAM, false]])
+ for (const s of sections) {
+ expect(typeof s.name).toBe('string')
+ expect(typeof s.own).toBe('boolean')
+ for (const row of s.models) expect(Object.keys(row).sort()).toEqual(['id', 'node'])
+ }
+ expect(section(sections, OWN).models).toEqual([{ id: 'small-q4', node: 'mac' }])
+ })
+})
diff --git a/cli/src/lib/localModels.integration.spec.ts b/cli/src/lib/localModels.integration.spec.ts
index f6b7bb60e..0840f1c68 100644
--- a/cli/src/lib/localModels.integration.spec.ts
+++ b/cli/src/lib/localModels.integration.spec.ts
@@ -1,6 +1,7 @@
import { it, expect } from 'vitest'
import { createServer } from 'node:http'
import { mkdtemp, mkdir, writeFile, readFile, rm, stat } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { LocalModels } from './localModels.js'
@@ -57,7 +58,11 @@ else if (args.includes('leave')) { rmSync(record); rmSync(registered); }
else if (args.includes('info')) console.log("export OPENAI_BASE_URL='${base}/v1'\\nexport OPENAI_API_KEY='inference-fixture'");
else process.exitCode = 2;
`, { mode: 0o700 })
- const options = { stateDir, processEnv: { GRID_HOME: gridHome, HARNESS_GRID_BIN: executable, PATH: root } }
+ // The grid's own answer, as the credential-less reader would give it: the engine is listed while its
+ // run record exists (the fake's `join` writes it, `leave` removes it).
+ const record = join(gridHome, 'run', 'engines', 'fixture-grid', 'remote.json')
+ const inventory = async () => ({ state: 'awake' as const, nodes: existsSync(record) ? [{ node_id: 'fixture-node', online: true, models: ['tiny'] }] : [] })
+ const options = { stateDir, processEnv: { GRID_HOME: gridHome, HARNESS_GRID_BIN: executable, PATH: root }, inventory }
service = new LocalModels(options)
expect((await service.list('home')).models[0]).toMatchObject({ state: 'available', canStart: true })
expect(replies).toBe(0)
diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.ts
index 89ee6500e..f4fc39091 100644
--- a/cli/src/lib/localModels.spec.ts
+++ b/cli/src/lib/localModels.spec.ts
@@ -4,7 +4,8 @@ import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { createHash } from 'node:crypto'
import { rmSync, writeFileSync } from 'node:fs'
-import { compatibleModels, LocalModels } from './localModels.js'
+import { spawnSync } from 'node:child_process'
+import { compatibleModels, LocalModels, readRunRecords, type GridInventory } from './localModels.js'
import { GridFleetRpc, type GridFleetResult } from './gridFleetRpc.js'
import { encryptDownFrame, encryptRpcResult } from './e2ee/applicationFrames.js'
@@ -18,6 +19,11 @@ let calls: string[][], serving: boolean, catalogCards: ReturnType<typeof card>[]
let request: Mock<(url: string | URL, init?: RequestInit) => Promise<Response>>
let run: Mock<(args: string[], output?: (s: string) => void) => Promise<GridFleetResult>>
let downloadFails: boolean, catalogFails: boolean
+// What the grid says it serves. Production reads it without a credential (`gridModels.gridInventory`);
+// here it is the scenario the fake `grid` below answers for `engines`, so each test keeps its meaning,
+// and `gridState` is what the grid's own answer said about it.
+let gridState: GridInventory['state']
+let inventory: Mock<(grid: string, force: boolean) => Promise<GridInventory>>
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'local-models-'))
home = join(root, 'grid'); stateDir = join(root, 'receipts'); records = join(home, 'run', 'engines', 'grid-home')
@@ -55,7 +61,13 @@ beforeEach(async () => {
}
return response({ choices: [{ message: { content: 'ok' } }] })
})
- service = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch })
+ gridState = 'awake'
+ inventory = vi.fn(async (grid: string) => {
+ const answer = await run(['--remote', 'engines', grid, '--json'])
+ if (!answer.ok) throw new Error('unreadable')
+ return { state: gridState, nodes: gridState === 'awake' ? JSON.parse(answer.stdout) : [] }
+ })
+ service = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory })
})
afterEach(async () => { await service.settled(); vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); await rm(root, { recursive: true, force: true }) })
@@ -175,7 +187,7 @@ describe('local model discovery and lifecycle', () => {
it('keeps existing local engines visible when the catalog cannot be reached', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
catalogFails = true
- const fresh = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch })
+ const fresh = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch , inventory })
const snapshot = await fresh.list('home')
expect(snapshot.models[0]).toMatchObject({ state: 'running', canStop: true })
expect(snapshot.error).toContain('unavailable')
@@ -210,7 +222,7 @@ describe('local model discovery and lifecycle', () => {
const first = await service.list('home')
expect(first.models[0]).toMatchObject({ id: 'local:Small-Q4.gguf', state: 'running', canStop: true })
await service.act('home', first.models[0].id, 'stop'); await service.settled()
- const fresh = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch })
+ const fresh = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch , inventory })
expect((await fresh.list('home')).models[0]).toMatchObject({ state: 'downloaded', canStart: true })
await fresh.act('home', first.models[0].id, 'start'); await fresh.settled()
expect(calls.some(args => args[0] === 'pull')).toBe(false)
@@ -231,7 +243,7 @@ describe('local model discovery and lifecycle', () => {
if (scenario === 'legacy') delete saved[0].aliases
if (scenario === 'malformed') saved[0].aliases = [42, '', '--invalid', 'invalid\nname']
await writeFile(path, JSON.stringify(saved))
- const fresh = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch })
+ const fresh = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch , inventory })
await fresh.act('home', id, 'start'); await fresh.settled()
const expectedAlias = scenario === 'current' ? 'team/my-model' : scenario === 'legacy' ? 'my-model' : null
const joined = calls.find(args => args.includes('join'))!
@@ -348,7 +360,7 @@ describe('local model discovery and lifecycle', () => {
it.each([undefined, 'https://alternate.example.test'])('uses the configured or default catalog when the credential store omits it', async base => {
await writeFile(join(home, 'credentials.toml'), 'session_token = "test-only-token"\n')
- service = new LocalModels({ stateDir, processEnv: { GRID_HOME: home, GRID_CONTROL_PLANE_URL: base }, run, request: request as typeof fetch })
+ service = new LocalModels({ stateDir, processEnv: { GRID_HOME: home, GRID_CONTROL_PLANE_URL: base }, run, request: request as typeof fetch , inventory })
await service.list('home')
expect(String(request.mock.calls[0][0])).toBe(`${base ?? 'https://api-grid.autonomous.ai'}/v1/grid/catalog`)
})
@@ -536,7 +548,7 @@ describe('local model discovery and lifecycle', () => {
it.each([false, true])('handles an explicit engine override (available=%s)', async available => {
const binary = join(root, 'custom-llama')
if (available) await writeFile(binary, '#!/bin/sh\nexit 0', { mode: 0o700 })
- service = new LocalModels({ stateDir, processEnv: { GRID_HOME: home, LLAMA_SERVER: binary }, run, request: request as typeof fetch })
+ service = new LocalModels({ stateDir, processEnv: { GRID_HOME: home, LLAMA_SERVER: binary }, run, request: request as typeof fetch , inventory })
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const operation = (await service.list('home')).models[0].operation
expect(operation?.phase).toBe(available ? 'done' : 'failed')
@@ -589,7 +601,7 @@ describe('local model discovery and lifecycle', () => {
vi.stubEnv('GRID_HOME', home)
vi.stubGlobal('fetch', request)
const rpc = vi.spyOn(GridFleetRpc.prototype, 'run').mockImplementation(async (_owner, _id, options, output) => run(options.args, output))
- service = new LocalModels({ stateDir })
+ service = new LocalModels({ stateDir, inventory })
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation?.phase).toBe('done')
expect(rpc).toHaveBeenCalledWith('local-models', expect.any(String), expect.objectContaining({ timeoutMs: 30_000, thinking: false }), undefined, 4 * 1024 * 1024)
@@ -688,3 +700,95 @@ describe('local model discovery and lifecycle', () => {
expect((await service.list('home')).models[0]).toMatchObject({ name: 'Small-Q4', state: 'running', operation: { phase: 'failed' } })
})
})
+
+/** A pid that existed a moment ago and no longer does. */
+function deadPid(): number {
+ return Number(spawnSync(process.execPath, ['-e', 'process.stdout.write(String(process.pid))'], { encoding: 'utf8' }).stdout)
+}
+
+describe('the Model Manager while its grid sleeps (grid-reads-without-waking issue 02)', () => {
+ const parkedRecord = (pid: unknown) => JSON.stringify({ node_id: 'local-node', meta_name: 'This computer', pid,
+ engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'] }], advertise_as: [] })
+
+ it('is not an inventory error: Start and Pause stay enabled, and a parked engine reads running', async () => {
+ gridState = 'asleep'
+ const next = card('org/Next-GGUF'); next.versions[0].pull_spec = 'org/Next-GGUF:Next.gguf'
+ catalogCards = [card(), next]
+ await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
+ await writeFile(join(records, 'remote.json'), parkedRecord(process.pid))
+
+ const snapshot = await service.list('home')
+
+ expect(snapshot.error).not.toBe('Running models could not be checked. Try again.')
+ expect(snapshot.models[0]).toMatchObject({ id: 'org/Small-GGUF', state: 'running', gridAsleep: true, canStop: true })
+ expect(snapshot.models[1]).toMatchObject({ id: 'org/Next-GGUF', canStart: true })
+ expect(inventory).toHaveBeenCalledWith('home', false)
+ })
+
+ it('reads a parked engine running on a row the catalog does not know, too', async () => {
+ gridState = 'asleep'; catalogCards = []
+ await writeFile(join(records, 'remote.json'), parkedRecord(process.pid))
+ expect((await service.list('home')).models).toEqual([expect.objectContaining({ id: 'local:Small-Q4.gguf', state: 'running', gridAsleep: true, canStop: true })])
+ })
+
+ it.each([['a process that is gone', () => deadPid()], ['a join mid-spawn (pid 0)', () => 0], ['no pid at all', () => 'x']])(
+ 'does not call %s parked', async (_name, pid) => {
+ gridState = 'asleep'; catalogCards = []
+ await writeFile(join(records, 'remote.json'), parkedRecord(pid()))
+ const [row] = (await service.list('home')).models
+ expect(row).toMatchObject({ id: 'local:Small-Q4.gguf', state: 'available', canStop: true })
+ expect(row).not.toHaveProperty('gridAsleep')
+ })
+
+ it('an engine the awake grid lists is running with no asleep flag', async () => {
+ serving = true
+ await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
+ await writeFile(join(records, 'remote.json'), parkedRecord(process.pid))
+ await writeFile(join(records, 'remote.heartbeat'), '')
+ const [row] = (await service.list('home')).models
+ expect(row).toMatchObject({ state: 'running' })
+ expect(row).not.toHaveProperty('gridAsleep')
+ })
+
+ it('a grid that answered nothing readable is still an inventory error', async () => {
+ gridState = 'unknown'
+ expect((await service.list('home')).error).toBe('Running models could not be checked. Try again.')
+ })
+
+ it.each([['failed', { ...ok(), ok: false, code: 1 }], ['answered with something that is not JSON', { ...ok(), stdout: 'not json' }]])(
+ 'so is a grid list that %s', async (_how, answer) => {
+ const original = run.getMockImplementation()!
+ run.mockImplementation(async (args, output) => args.includes('ls') ? answer : original(args, output))
+ expect((await service.list('home')).error).toBe('Running models could not be checked. Try again.')
+ })
+
+ it('tells its owner when a start or stop has finished, so the lists are read again', async () => {
+ const onChanged = vi.fn()
+ service = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory, onChanged })
+ await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
+ expect(onChanged).toHaveBeenCalledOnce()
+ })
+})
+
+describe('readRunRecords — what servedHere knows about this computer', () => {
+ it('reads the name, the advertised ids (else the models under the display rule) and whether the pid lives', async () => {
+ await writeFile(join(records, 'a.json'), JSON.stringify({ meta_name: 'mac', pid: process.pid, advertise_as: ['Team/Model', 7, ''], models: ['ignored.gguf'] }))
+ await writeFile(join(records, 'b.json'), JSON.stringify({ meta_name: 'mac', pid: deadPid(), models: ['Qwen3-8B-Q4_K_M.GGUF', 'plain'] }))
+ await writeFile(join(records, 'c.json'), JSON.stringify({ pid: 0, models: 'not a list' }))
+ await writeFile(join(records, 'd.json'), '{ not json')
+ await writeFile(join(records, 'e.heartbeat'), '')
+
+ const found = (await readRunRecords(home, 'grid-home')).sort((x, y) => x.ids.join().localeCompare(y.ids.join()))
+
+ expect(found).toEqual([
+ { name: '', ids: [], pid: null, alive: false },
+ { name: 'mac', ids: ['Qwen3-8B-Q4_K_M', 'plain'], pid: expect.any(Number), alive: false },
+ { name: 'mac', ids: ['Team/Model'], pid: process.pid, alive: true },
+ ])
+ })
+
+ it.each(['', '../grid-home', 'missing'])('answers nothing for grid id %j', async (gridId) => {
+ await writeFile(join(records, 'a.json'), JSON.stringify({ meta_name: 'mac', pid: process.pid, advertise_as: ['X'] }))
+ expect(await readRunRecords(home, gridId)).toEqual([])
+ })
+})
diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.ts
index afc3cc8f1..ad56d5ebb 100644
--- a/cli/src/lib/localModels.ts
+++ b/cli/src/lib/localModels.ts
@@ -5,8 +5,10 @@ import { readFile, readdir, stat, mkdir, rename, writeFile, statfs } from 'node:
import { basename, dirname, join } from 'node:path'
import { GridFleetRpc, type GridFleetResult } from './gridFleetRpc.js'
import { gridCredentialsPath } from './gridCredentials.js'
-import { forgetGridModels } from './gridModels.js'
import { binaryOnPath } from './binaryOnPath.js'
+import { processExists } from './processLiveness.js'
+import type { LocalRecord, PictureState } from './gridPicture.js'
+import { displayModelName } from './gridReader.js'
const GiB = 1024 ** 3
const obj = (v: unknown): Record<string, any> => v && typeof v === 'object' && !Array.isArray(v) ? v : {}
@@ -24,6 +26,8 @@ export interface LocalModel {
sizeBytes?: number; quant?: string; recommended?: boolean; canStart: boolean; canStop: boolean
tokensPerSecond?: number; requests?: number; windowSeconds?: number
operation?: ModelOperation
+ /** Running here, parked while its grid sleeps (additive: an older desktop reads plain `running`). */
+ gridAsleep?: boolean
}
export interface ModelOperation {
id: string; modelId: string; action: 'start' | 'stop'; phase: 'running' | 'done' | 'failed'
@@ -38,12 +42,22 @@ interface Candidate {
id: string; name: string; pull: string; file: string; files: string[]; size: number; quant: string; context: number
aliases?: string[]
}
-interface Owned { file: string; selector: string; aliases: string[]; nodeId: string; name: string; live: boolean; context: number; siblings: number }
+/** `live`: its heartbeat sidecar is fresh (the grid is hearing from it). `pidAlive`: the process its run
+ * record names exists — the only liveness that holds while the grid sleeps and the sidecar goes stale. */
+interface Owned { file: string; selector: string; aliases: string[]; nodeId: string; name: string; live: boolean; pidAlive: boolean; context: number; siblings: number }
interface Receipt { spec: 1; grid: string; operation?: ModelOperation }
+/** What the grid says is running on it, read WITHOUT waking it (`gridModels.gridInventory`): the
+ * overview's node objects while it is awake, nothing while it sleeps. */
+export interface GridInventory { state: PictureState; nodes: Record<string, unknown>[] }
interface Options {
stateDir: string; processEnv?: NodeJS.ProcessEnv
run?: (args: string[], output?: (chunk: string) => void, timeout?: number) => Promise<GridFleetResult>
request?: typeof fetch
+ /** Injected, never a `grid engines` call: that read carries the grid credential, and a signed-in read
+ * of a sleeping grid WAKES it — once a tick, for as long as the panel is open. */
+ inventory: (grid: string, force: boolean) => Promise<GridInventory>
+ /** Told when a start or stop has finished, so the model lists are read again. */
+ onChanged?: () => void
}
/** One concrete, machine-fitted version per model. Non-chat and unprobed offline
@@ -199,7 +213,8 @@ export class LocalModels {
const advertised = Array.isArray(record.advertise_as) && specs.length === 1
? record.advertise_as.filter((v: unknown) => typeof v === 'string' && v.length > 0) : []
const aliases: string[] = advertised.length ? advertised : [file]
- result.push({ file: basename(file), selector: file, aliases, nodeId: str(record.node_id), name: str(record.meta_name), live, context: Math.min(16384, num(record.ctx_size) || 8192), siblings: specs.length + (record.media ? 1 : 0) })
+ const pid = recordPid(record)
+ result.push({ file: basename(file), selector: file, aliases, nodeId: str(record.node_id), name: str(record.meta_name), live, pidAlive: pid !== null && processExists(pid), context: Math.min(16384, num(record.ctx_size) || 8192), siblings: specs.length + (record.media ? 1 : 0) })
this.blockers.set(grid, `Stop ${cleanName(aliases[0])} first to start another local model.`)
}
}
@@ -284,9 +299,15 @@ export class LocalModels {
}
private async readList(grid: string, force: boolean): Promise<LocalModelsSnapshot> {
await Promise.all([this.loadCatalog(force), this.readReceipt(grid)])
- let owned: Owned[] = [], nodes: Record<string, any>[] = [], inventoryError: string | undefined
+ let owned: Owned[] = [], nodes: Record<string, any>[] = [], inventoryError: string | undefined, asleep = false
try {
- [owned, nodes] = await Promise.all([this.owned(grid), this.json(['--remote', 'engines', grid, '--json']).then(rows)])
+ let inventory: GridInventory
+ [owned, inventory] = await Promise.all([this.owned(grid), this.options.inventory(grid, force)])
+ // Asleep is not an inventory error: the grid is resting, what runs here is still known here, and
+ // Start and Pause stay enabled. Only a read that failed some other way is.
+ if (inventory.state === 'unknown') throw new Error('unanswered')
+ asleep = inventory.state !== 'awake'
+ nodes = rows(inventory.nodes)
} catch { inventoryError = 'Running models could not be checked. Try again.' }
const operation = this.active?.grid === grid ? this.active.operation : this.receipt?.grid === grid ? this.receipt.operation : undefined
const choices = [...this.candidates, ...(await this.known(grid, owned)).filter(k => !this.candidates.some(c => c.file === k.file))]
@@ -305,10 +326,14 @@ export class LocalModels {
modelKey(str(typeof m === 'string' ? m : obj(m).model)) === modelKey(alias)))) : []
return matches.length === 1 ? matches[0] : undefined
}
+ // A parked engine: its grid sleeps, so the grid lists nothing, but the process that serves it is alive
+ // here and rejoins the moment the grid wakes. Liveness is the record's pid — never the heartbeat
+ // sidecar, which goes stale while a healthy provider is parked.
+ const parked = (instance?: Owned): boolean => asleep && !!instance?.pidAlive
const models = choices.map((candidate, index): LocalModel => {
const instance = owned.find(o => owners.get(o.file) === candidate)
const node = servingNode(instance)
- const running = !!node
+ const running = !!node || parked(instance)
const available = downloaded[index]
const answered = obj(node?.answered)
const perModel = rows(answered.by_model).find(a => instance?.aliases.some(alias => modelKey(alias) === modelKey(str(a.model))))
@@ -320,12 +345,15 @@ export class LocalModels {
tokensPerSecond: single && running ? num(node?.throughput_tok_s) : undefined,
requests: running ? num(perModel?.requests) : undefined,
windowSeconds: running ? num(answered.window_seconds) : undefined,
- operation: operation?.modelId === candidate.id ? operation : undefined }
+ operation: operation?.modelId === candidate.id ? operation : undefined,
+ ...(!node && parked(instance) ? { gridAsleep: true } : {}) }
})
for (const instance of owned.filter(o => !choices.some(c => c.file === o.file))) {
+ const serving = !!servingNode(instance)
models.push({ id: `local:${instance.file}`, name: cleanName(instance.aliases[0]),
- state: servingNode(instance) ? 'running' : 'available', canStart: false, canStop: !inventoryError,
- operation: operation?.modelId === `local:${instance.file}` ? operation : undefined })
+ state: serving || parked(instance) ? 'running' : 'available', canStart: false, canStop: !inventoryError,
+ operation: operation?.modelId === `local:${instance.file}` ? operation : undefined,
+ ...(!serving && parked(instance) ? { gridAsleep: true } : {}) })
}
const value: LocalModelsSnapshot = { models, memoryBytes: num(obj(this.device.memory).total_gb) === undefined ? undefined : this.device.memory.total_gb * GiB,
hardware: str(obj(this.device.machine).model || obj(this.device.cpu).brand), error: inventoryError || this.catalogError,
@@ -349,7 +377,7 @@ export class LocalModels {
operation.phase = 'failed'
operation.error = error instanceof ModelError ? error.message : 'The model could not finish. Start again to retry.'
await this.save(grid, operation).catch(() => {})
- }).finally(() => { this.active = undefined; this.cached = undefined; forgetGridModels() })
+ }).finally(() => { this.active = undefined; this.cached = undefined; this.options.onChanged?.() })
return { operation }
}
@@ -453,3 +481,39 @@ export class LocalModels {
}
}
class ModelError extends Error {}
+
+/** More names than any one engine advertises: a record past it is cut, never trusted to be small. */
+const MAX_RECORD_IDS = 64
+
+/** A run record's pid while a process holds it; null while a join is mid-spawn (`grid join` writes 0). */
+function recordPid(record: Record<string, any>): number | null {
+ return Number.isSafeInteger(record.pid) && record.pid > 0 ? record.pid : null
+}
+
+/**
+ * This computer's own run records for one grid, as `servedHere` reads them (`gridPicture.ts`): the node
+ * name each registers under, what it advertises, and whether a process still holds it.
+ *
+ * ⚠️ The keys are the ones `grid join` writes (autonomous-grid `remote_provider._build_record`), and that
+ * repository's `tests/test_grid_reads_lockstep.py` pins every `record.<key>` read in THIS file against
+ * them — which is why this lives here rather than in a module of its own. A key `grid join` stopped
+ * writing is a model this computer silently stops recognising as its own: stale, never a wrong hide.
+ */
+export async function readRunRecords(home: string, gridId: string): Promise<LocalRecord[]> {
+ if (!gridId || basename(gridId) !== gridId) return []
+ const folder = join(home, 'run', 'engines', gridId)
+ const names = await readdir(folder).catch(() => [])
+ const result: LocalRecord[] = []
+ for (const name of names.filter(n => n.endsWith('.json'))) {
+ let record: Record<string, any>
+ try { record = obj(JSON.parse(await readFile(join(folder, name), 'utf8'))) } catch { continue }
+ const strings = (v: unknown): string[] => Array.isArray(v) ? v.filter((s): s is string => typeof s === 'string' && s.length > 0) : []
+ // What the grid lists it as: the alias it advertises when it has one, else its model under the
+ // master's display rule (a trailing .gguf removed, case kept).
+ const advertised = strings(record.advertise_as)
+ const ids = (advertised.length ? advertised : strings(record.models).map(displayModelName)).slice(0, MAX_RECORD_IDS)
+ const pid = recordPid(record)
+ result.push({ name: str(record.meta_name), ids, pid, alive: pid !== null && processExists(pid) })
+ }
+ return result
+}
diff --git a/cli/src/lib/runtimeInstall.spec.ts b/cli/src/lib/runtimeInstall.spec.ts
index 60d5c51f8..373b7c78b 100644
--- a/cli/src/lib/runtimeInstall.spec.ts
+++ b/cli/src/lib/runtimeInstall.spec.ts
@@ -337,6 +337,83 @@ describe('ensureManagedGrid', () => {
})
})
+describe('startGridPinRecheck', () => {
+ const key = currentPlatformKey()
+
+ beforeEach(() => {
+ root = mkdtempSync(join(tmpdir(), 'harness-grid-recheck-'))
+ runtimeDir = join(root, '.harness', 'runtime')
+ binDir = join(root, '.local', 'bin')
+ cliDir = join(root, '.harness', 'cli')
+ for (const dir of [runtimeDir, binDir, cliDir]) mkdirSync(dir, { recursive: true })
+ vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] })
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.unstubAllGlobals()
+ for (const entry of readdirSync(runtimeDir)) {
+ try { chmodSync(join(runtimeDir, entry, 'bin'), 0o755) } catch { /* not a runtime dir */ }
+ }
+ rmSync(root, { recursive: true, force: true })
+ for (const key of ['ADAPTER_RUNTIME_DIR', 'HARNESS_BIN_DIR', 'ADAPTER_CLI_DIR', 'ADAPTER_RUNTIME_METADATA_URL', 'ADAPTER_GRID_RUNTIME_METADATA_URL', 'HARNESS_GRID_BIN']) {
+ delete process.env[key]
+ }
+ })
+
+ /** A snapshot of everything under the runtime dir: names, sizes and times. */
+ const tree = (): string[] => readdirSync(runtimeDir, { recursive: true, withFileTypes: true })
+ .map((entry) => { const path = join(entry.parentPath, entry.name); const st = statSync(path); return `${path}:${st.size}:${st.mtimeMs}` })
+ .sort()
+
+ it('re-reads the pin every ten minutes, writes nothing while it matches, and a moved pin reaches the next grid call', async () => {
+ const installed = installedGrid('0.3.47', key, true)
+ const old = buildArchive('0.3.47', key, 'grid')
+ stubFetch(manifestFor('grid', key, '0.3.47', old.bytes, old.root), old.bytes)
+ const { ensureManagedGrid, startGridPinRecheck, GRID_PIN_RECHECK_MS } = await load()
+ const { gridBinaryPath } = await import('./gridExec.js')
+ let last: Promise<unknown> = Promise.resolve()
+ const ensure = vi.fn(() => (last = ensureManagedGrid()))
+ // The check, and the bookkeeping that frees the next one.
+ const checked = async (): Promise<void> => { await last; await new Promise((resolve) => setImmediate(resolve)) }
+ const stop = startGridPinRecheck({ ensure })
+ const before = tree()
+
+ expect(GRID_PIN_RECHECK_MS).toBe(10 * 60_000)
+ vi.advanceTimersByTime(GRID_PIN_RECHECK_MS - 1)
+ expect(ensure).not.toHaveBeenCalled()
+ vi.advanceTimersByTime(1)
+ await checked()
+ expect(ensure).toHaveBeenCalledOnce()
+ expect(tree()).toEqual(before)
+ expect(gridBinaryPath({})).toBe(installed)
+
+ const next = buildArchive('0.3.49', key, 'grid')
+ stubFetch(manifestFor('grid', key, '0.3.49', next.bytes, next.root), next.bytes)
+ vi.advanceTimersByTime(GRID_PIN_RECHECK_MS)
+ await checked()
+ expect(gridBinaryPath({})).toBe(join(runtimeDir, `grid-0.3.49-${key}`, 'bin', 'grid'))
+ stop()
+ vi.advanceTimersByTime(GRID_PIN_RECHECK_MS * 3)
+ expect(ensure).toHaveBeenCalledTimes(2)
+ })
+
+ it('runs one check at a time: a slow one is not stacked on', async () => {
+ const { startGridPinRecheck, GRID_PIN_RECHECK_MS } = await load()
+ let release: () => void = () => {}
+ const ensure = vi.fn(() => new Promise<void>((resolve) => { release = resolve }))
+ const stop = startGridPinRecheck({ ensure })
+
+ vi.advanceTimersByTime(GRID_PIN_RECHECK_MS * 3)
+ expect(ensure).toHaveBeenCalledOnce()
+ release()
+ await new Promise((resolve) => setImmediate(resolve))
+ vi.advanceTimersByTime(GRID_PIN_RECHECK_MS)
+ expect(ensure).toHaveBeenCalledTimes(2)
+ stop()
+ })
+})
+
describe('ensureLauncher', () => {
const NODE = '/opt/harness/runtime/node-v22/bin/node'
diff --git a/cli/src/lib/runtimeInstall.ts b/cli/src/lib/runtimeInstall.ts
index a31725c98..1053e7923 100644
--- a/cli/src/lib/runtimeInstall.ts
+++ b/cli/src/lib/runtimeInstall.ts
@@ -246,6 +246,33 @@ export async function ensureManagedGrid(log: (message: string) => void = () => {
})
}
+/** How often a running daemon re-reads the managed `grid` pin. */
+export const GRID_PIN_RECHECK_MS = 10 * 60_000
+
+/**
+ * Follow the pin while the daemon RUNS, not only when it starts.
+ *
+ * A daemon updates itself within about a minute of a release, but it can then run for days, and a pin
+ * moved in between would wait for the next restart. That matters here because the pin and the daemon's
+ * code are released in an ORDER (grid-reads-without-waking: CLI tag → harness tag → pin, never 0.3.48
+ * alone): the pin has to reach the daemons already running the new code, the same day.
+ *
+ * Idempotent by construction — [ensureManagedGrid] fetches only the manifest and writes nothing while the
+ * installed grid IS the pin — and one check at a time. Every `grid` call resolves the binary afresh
+ * (`gridBinaryPath` reads `current-grid`), so a moved pin takes effect on the next call. Returns a stop.
+ */
+export function startGridPinRecheck(options: { ensure?: () => Promise<unknown>; intervalMs?: number } = {}): () => void {
+ const ensure = options.ensure ?? (() => ensureManagedGrid((m) => console.log(`[grid-runtime] ${m}`)))
+ let checking = false
+ const timer = setInterval(() => {
+ if (checking) return
+ checking = true
+ void ensure().catch(() => {}).finally(() => { checking = false })
+ }, options.intervalMs ?? GRID_PIN_RECHECK_MS)
+ timer.unref?.()
+ return () => clearInterval(timer)
+}
+
/** The trailing `exec …` line, which is the only line any launcher we ship varies. */
const EXEC_LINE = /^exec .*$/m
diff --git a/desktop/lib/core/models.dart b/desktop/lib/core/models.dart
index 66953865f..ab7d405ed 100644
--- a/desktop/lib/core/models.dart
+++ b/desktop/lib/core/models.dart
@@ -1048,6 +1048,28 @@ enum GridCli {
};
}
+/// What the daemon last learned about one grid, as it reports beside the section (`state`).
+///
+/// The daemon reads grids without waking them, so a section can describe a grid that is resting:
+/// [asleep] keeps its last known models rather than blanking them, [unknown] is a read that failed
+/// some other way (or none yet), and [waking] is reserved for a person asking it to start. An older
+/// daemon sends no field, read as null: nothing is claimed either way, and the section draws as it
+/// always did.
+enum GridSectionState {
+ awake,
+ asleep,
+ waking,
+ unknown;
+
+ static GridSectionState? parse(Object? raw) => switch (raw) {
+ 'awake' => GridSectionState.awake,
+ 'asleep' => GridSectionState.asleep,
+ 'waking' => GridSectionState.waking,
+ 'unknown' => GridSectionState.unknown,
+ _ => null,
+ };
+}
+
/// The picker's whole answer: which grid was asked, and what it offers.
///
/// One grid the machine is signed into, with what it serves. [own] marks the account's private
@@ -1057,10 +1079,23 @@ class GridSection {
final bool own;
final List<GridModel> models;
+ /// How the grid answered the daemon's last look — see [GridSectionState]. Null from an older
+ /// daemon, and nothing here draws it yet.
+ final GridSectionState? state;
+
+ /// When the grid was last seen awake, or null.
+ final DateTime? seenAt;
+
+ /// How old [models] is, in seconds, or null when nothing is known.
+ final int? lastKnownAge;
+
const GridSection({
required this.name,
required this.own,
required this.models,
+ this.state,
+ this.seenAt,
+ this.lastKnownAge,
});
}
@@ -1105,6 +1140,58 @@ class GridModels {
this.reachable = true,
});
+ /// The daemon's `grid_models_list` reply — and the `grid_models_changed` push, which carries the
+ /// same document — read defensively: the fields of an older or newer daemon are simply absent or
+ /// ignored, never a failure. `reachable` is false only when the daemon answered with an `error`.
+ factory GridModels.fromReply(Map<String, dynamic> reply) {
+ List<GridModel> parseModels(Object? raw, {String? grid}) =>
+ (raw is List ? raw : const <dynamic>[])
+ .whereType<Map<String, dynamic>>()
+ .map(
+ (m) => GridModel(
+ id: m['id'] is String ? m['id'] as String : '',
+ node: m['node'] is String ? m['node'] as String : '',
+ grid: grid,
+ ),
+ )
+ .where((m) => m.id.isNotEmpty)
+ .toList();
+ final rawGrids = reply['grids'];
+ final grids = (rawGrids is List ? rawGrids : const <dynamic>[])
+ .whereType<Map<String, dynamic>>()
+ .where((g) => g['name'] is String && (g['name'] as String).isNotEmpty)
+ .map((g) {
+ final name = g['name'] as String;
+ final seen = g['seenAt'];
+ final known = g['lastKnownAge'];
+ return GridSection(
+ name: name,
+ own: g['own'] == true,
+ models: parseModels(g['models'], grid: name),
+ state: GridSectionState.parse(g['state']),
+ seenAt: seen is String ? DateTime.tryParse(seen) : null,
+ lastKnownAge: known is num && known.isFinite && known >= 0
+ ? known.round()
+ : null,
+ );
+ })
+ .toList();
+ final capable = reply['localModelEngines'];
+ return GridModels(
+ gridName: reply['gridName'] is String
+ ? reply['gridName'] as String
+ : null,
+ models: parseModels(reply['models']),
+ grids: grids,
+ localModelEngines: capable is List
+ ? capable.whereType<String>().map((e) => e.toLowerCase()).toSet()
+ : null,
+ gridCli: GridCli.parse(reply['gridCli']),
+ supportsModelLaunch: reply['supportsModelLaunch'] == true,
+ reachable: reply['error'] == null,
+ );
+ }
+
/// The sections to draw: [grids] when the daemon sent them, else the own grid alone.
List<GridSection> get sections => grids.isNotEmpty
? grids
diff --git a/desktop/lib/models/model_manager_controller.dart b/desktop/lib/models/model_manager_controller.dart
index e246ed17e..1fb55720a 100644
--- a/desktop/lib/models/model_manager_controller.dart
+++ b/desktop/lib/models/model_manager_controller.dart
@@ -94,10 +94,15 @@ class ModelManagerController extends ChangeNotifier {
if (_started || _disposed) return;
_started = true;
app.addListener(_observe);
+ app.foreground.addListener(_foregroundChanged);
+ app.gridPictures.addListener(_pictureChanged);
unawaited(_loadPreferences());
_observe();
if (poll) {
_timer = Timer.periodic(const Duration(seconds: 4), (_) {
+ // A minimised or background app reads nothing, busy or not: the daemon owns the operation
+ // either way, and the one refresh on return ([_foregroundChanged]) catches up.
+ if (!app.inForeground) return;
if (busy ||
_panelVisible ||
_lastLocalRead == null ||
@@ -155,11 +160,33 @@ class ModelManagerController extends ChangeNotifier {
!_autoPrepared) {
_autoPrepared = true;
unawaited(prepare());
- unawaited(refresh());
+ // In the background the first read waits for the app to come back ([_foregroundChanged]).
+ if (app.inForeground) unawaited(refresh());
}
_changed();
}
+ /// Back in front of the person: exactly one refresh, whatever the background skipped.
+ void _foregroundChanged() {
+ if (_disposed || !app.inForeground) return;
+ final owner = machine;
+ if (owner == null || owner.connectionStatus != ConnectionStatus.connected) {
+ return;
+ }
+ unawaited(refresh());
+ }
+
+ /// Another surface's read, or the daemon's `grid_models_changed` push, changed the app's picture
+ /// of this machine's grids: take it, with no read of our own.
+ void _pictureChanged() {
+ final owner = machine;
+ if (_disposed || owner == null) return;
+ final picture = app.gridPictures[owner.machine.machineId];
+ if (picture == null || identical(picture, models)) return;
+ models = picture;
+ _changed();
+ }
+
bool _current(MachineState owner) => !_disposed && identical(machine, owner);
void _changed() {
if (!_disposed) notifyListeners();
@@ -330,9 +357,14 @@ class ModelManagerController extends ChangeNotifier {
const Duration(seconds: 20);
await Future.wait([
if (readShared)
- app.gridModels(owner.machine.machineId).then((answer) {
+ app.refreshGridModels(owner.machine.machineId).then((answer) {
if (_current(owner)) {
- models = answer;
+ // The panel says so when ITS read could not reach the machine ("Shared models are
+ // unavailable." with Try again) — that failure is kept out of the shared picture, so no
+ // other surface blanks. Answered, it shows the picture: a push that landed meanwhile is newer.
+ models = answer.reachable
+ ? app.gridPictures[owner.machine.machineId] ?? answer
+ : answer;
_lastModelsRead = DateTime.now();
}
}),
@@ -427,7 +459,11 @@ class ModelManagerController extends ChangeNotifier {
_disposed = true;
_apis?.dispose();
_timer?.cancel();
- if (_started) app.removeListener(_observe);
+ if (_started) {
+ app.removeListener(_observe);
+ app.foreground.removeListener(_foregroundChanged);
+ app.gridPictures.removeListener(_pictureChanged);
+ }
super.dispose();
}
}
diff --git a/desktop/lib/screens/swarm_screen.dart b/desktop/lib/screens/swarm_screen.dart
index 71469c4cb..5b1f57307 100644
--- a/desktop/lib/screens/swarm_screen.dart
+++ b/desktop/lib/screens/swarm_screen.dart
@@ -267,7 +267,13 @@ class _SwarmScreenState extends State<SwarmScreen>
app.agentUnread.addListener(_unreadChanged);
// Coming back to the window puts the tab in front of the person again, and
// nothing in the app necessarily changes when that happens — so it is told.
- _lifecycle = AppLifecycleListener(onResume: app.seeWatchedAgents);
+ _lifecycle = AppLifecycleListener(
+ onResume: app.seeWatchedAgents,
+ // Whether the app is in front of anyone decides whether the model surfaces refresh at all
+ // (`AppNotifier.foreground`): a minimised or background window asks no daemon anything.
+ onStateChange: app.appLifecycleChanged,
+ );
+ app.appLifecycleChanged(WidgetsBinding.instance.lifecycleState);
app.addListener(_syncToolbarNotices);
_toolbarNotices.addListener(_toolbarNoticesChanged);
_onboarding.addListener(_onboardingChanged);
@@ -299,6 +305,8 @@ class _SwarmScreenState extends State<SwarmScreen>
}
if (_native) {
_modelsMenu!.addListener(_syncModels);
+ app.gridPictures.addListener(_gridPictureChanged);
+ app.foreground.addListener(_foregroundChanged);
// Same trigger as the subscription rows: the daemon memoises its answer, so opening the menu
// repeatedly costs nothing after the first.
unawaited(_refreshLocalModels());
@@ -395,6 +403,8 @@ class _SwarmScreenState extends State<SwarmScreen>
);
unawaited(_channel.invokeMethod<void>('machinesState', {'machines': []}));
app.removeListener(_syncNative);
+ app.gridPictures.removeListener(_gridPictureChanged);
+ app.foreground.removeListener(_foregroundChanged);
_channel.setMethodCallHandler(null);
unawaited(
_channel.invokeMethod<void>('update', {'tabs': [], 'enabled': false}),
@@ -978,6 +988,10 @@ class _SwarmScreenState extends State<SwarmScreen>
{
'name': s.name,
'own': s.own,
+ // How the grid answered the daemon's last look (`GridSectionState`), when a daemon new
+ // enough to say sends it. Nothing native draws it yet; it is here so a change of state
+ // alone reaches the menu.
+ if (s.state != null) 'state': s.state!.name,
'models': [
for (final m in s.models) {'id': m.id, 'node': m.node},
],
@@ -1010,26 +1024,63 @@ class _SwarmScreenState extends State<SwarmScreen>
DateTime? _localModelsAt;
- Future<void> _refreshLocalModels() async {
- final machines = app.machines
- .where((machine) => !machine.isShared)
- .toList();
- if (machines.isEmpty) return;
+ Future<void> _refreshLocalModels({bool force = false}) async {
+ // A minimised or background window asks nothing: every read here is for a menu nobody can see
+ // until the app comes back, and it refreshes once when it does ([_foregroundChanged]). A forced
+ // read is a person's act — the menu opening — and runs whatever the lifecycle last said: a
+ // window can read as inactive while its own menu is tracking.
+ if (!app.inForeground && !force) return;
+ final preferred = _modelsMachine();
+ if (preferred == null) return;
// The daemon memoises its answer, so a repeat is nearly free — but this is called on every app
// change, and an RPC per keystroke-sized notification is not free. One read per window is
- // plenty for a list that changes when someone starts or stops serving a model.
+ // plenty for a list that changes when someone starts or stops serving a model; opening the menu
+ // and coming back to the app ([force]) are the exceptions, and read at once.
final now = DateTime.now();
final last = _localModelsAt;
- if (last != null && now.difference(last) < const Duration(seconds: 10)) {
+ if (!force &&
+ last != null &&
+ now.difference(last) < const Duration(seconds: 10)) {
return;
}
_localModelsAt = now;
- final preferred = machines.firstWhere(
- (m) => app.stateOf(m.machineId)?.isLocalMachine == true,
- orElse: () => machines.first,
- );
- final answer = await app.gridModels(preferred.machineId);
+ final picture = await app.readGridPicture(preferred);
if (!mounted) return;
+ _adoptLocalSections(picture);
+ }
+
+ /// The machine whose daemon answers for the Models menu — the grid is per ACCOUNT, so any of them
+ /// answers the same, and the local one is asked first because its daemon is a loopback away.
+ String? _modelsMachine() {
+ final machines = app.machines
+ .where((machine) => !machine.isShared)
+ .toList();
+ if (machines.isEmpty) return null;
+ return machines
+ .firstWhere(
+ (m) => app.stateOf(m.machineId)?.isLocalMachine == true,
+ orElse: () => machines.first,
+ )
+ .machineId;
+ }
+
+ /// A new picture — from a read here, another surface's read, or the daemon's push — redraws the
+ /// menu when it changes what the menu shows. No request of its own.
+ void _gridPictureChanged() {
+ if (!mounted) return;
+ final machine = _modelsMachine();
+ final picture = machine == null ? null : app.gridPictures[machine];
+ if (picture != null) _adoptLocalSections(picture);
+ }
+
+ /// Back in front of the person: the one refresh the background skipped.
+ void _foregroundChanged() {
+ if (mounted && app.inForeground) {
+ unawaited(_refreshLocalModels(force: true));
+ }
+ }
+
+ void _adoptLocalSections(GridModels answer) {
final sections = _sectionsForModelMenu(answer);
if (_sameSections(sections, _localSections)) return;
_localSections = sections;
@@ -1054,13 +1105,23 @@ class _SwarmScreenState extends State<SwarmScreen>
];
}
+ /// Whether the menu would draw [a] exactly as [b]. The machine serving a row and the state of a
+ /// section are compared too: a model that moved to another computer, or a grid that went to sleep,
+ /// is a different menu even when every id is the same.
static bool _sameSections(List<GridSection> a, List<GridSection> b) {
if (a.length != b.length) return false;
for (var i = 0; i < a.length; i++) {
- if (a[i].name != b[i].name || a[i].own != b[i].own) return false;
+ if (a[i].name != b[i].name ||
+ a[i].own != b[i].own ||
+ a[i].state != b[i].state) {
+ return false;
+ }
if (a[i].models.length != b[i].models.length) return false;
for (var j = 0; j < a[i].models.length; j++) {
- if (a[i].models[j].id != b[i].models[j].id) return false;
+ if (a[i].models[j].id != b[i].models[j].id ||
+ a[i].models[j].node != b[i].models[j].node) {
+ return false;
+ }
}
}
return true;
@@ -1102,6 +1163,9 @@ class _SwarmScreenState extends State<SwarmScreen>
return;
}
if (call.method == 'modelsOpened') {
+ // The menu opens from its cache; the models are re-read beside the subscriptions, and a
+ // change lands in the open menu through `modelsState`.
+ unawaited(_refreshLocalModels(force: true));
await _modelsMenu?.refresh();
return;
}
diff --git a/desktop/lib/state/app_state.dart b/desktop/lib/state/app_state.dart
index e23cff2f1..0151c1853 100644
--- a/desktop/lib/state/app_state.dart
+++ b/desktop/lib/state/app_state.dart
@@ -51,6 +51,7 @@ import '../widgets/engine_identity.dart'
show allEngines, engineIdentity, isTerminalEngine;
import '../store/store_screen.dart' show openStoreAgent;
import 'dial_status.dart';
+import 'grid_pictures.dart';
import 'harness_placement.dart';
import 'desk_sync.dart';
import 'pane_layout_store.dart';
@@ -559,6 +560,25 @@ class AppNotifier extends ChangeNotifier {
final WsConn Function(String machineId)? connectionForTest;
final Map<String, Timer> _turnActivityWatchdogs = {};
+ /// What each machine's daemon last said the account's grids serve — the one list the pane
+ /// picker, the macOS Models menu and the Model Manager all read. See [GridPictures].
+ final GridPictures gridPictures = GridPictures();
+
+ /// Whether the app is in front of the person (`AppLifecycleState.resumed`). The three surfaces
+ /// that refresh [gridPictures] on their own — the Model Manager's timer, the macOS menu's refresh
+ /// and the pane picker's prefetch — run only while this is true, and each refreshes once when it
+ /// turns true again. A minimised or background app asks nothing. True until the platform says
+ /// otherwise, so a state nobody reported (a test, the first frame) behaves as it always did.
+ final ValueNotifier<bool> foreground = ValueNotifier(true);
+
+ bool get inForeground => foreground.value;
+
+ /// Fed by the workspace's [AppLifecycleListener]. A null state is unknown, read as foreground.
+ void appLifecycleChanged(AppLifecycleState? state) {
+ if (_disposed) return;
+ foreground.value = state == null || state == AppLifecycleState.resumed;
+ }
+
late final sessionPreviews = SessionPreviewStore(
canFetch: _canFetchPreview,
fetchRecent: (key) => _conn(key.machineId).request(
@@ -2955,6 +2975,7 @@ class AppNotifier extends ChangeNotifier {
machinesAreStale = false;
machineStates.clear();
sessionPreviews.clear();
+ gridPictures.clear();
expandedMachines.clear();
selectedMachineId = null;
_ensurePool();
@@ -3294,6 +3315,7 @@ class AppNotifier extends ChangeNotifier {
machinesAreStale = false;
machineStates.clear();
sessionPreviews.clear();
+ gridPictures.clear();
expandedMachines.clear();
selectedMachineId = null;
_closedHistory.clear();
@@ -5025,11 +5047,12 @@ class AppNotifier extends ChangeNotifier {
}
}
- /// Live models on the account's private harness grid, for the pane header's picker.
+ /// One `grid_models_list` read of [machineId]'s daemon, as it answered — the raw read behind
+ /// [refreshGridModels], which keeps the result as the app's one picture ([gridPictures]).
///
- /// Asked of the machine the pane belongs to rather than kept in app state: the answer is whatever
- /// that machine's `grid` reports at this moment (an engine can join or leave between two opens),
- /// and a cached list would offer a model nobody is serving any more.
+ /// The daemon answers from ITS picture of each grid, read without waking any of them, so a sleeping
+ /// grid keeps its last known models. Surfaces read [readGridPicture]; a caller that must know this
+ /// machine answered right now (the creation check) reads [refreshGridModels].
///
/// Never throws — a machine whose daemon is too old to know the RPC, one with no grid, and one
/// that timed out are all "nothing to offer", which is what the picker shows.
@@ -5037,52 +5060,7 @@ class AppNotifier extends ChangeNotifier {
try {
final response = await _conn(machineId)
.request('grid_models_list', timeout: const Duration(seconds: 12));
- final models = (response['models'] as List<dynamic>? ?? [])
- .whereType<Map<String, dynamic>>()
- .map(
- (m) => GridModel(
- id: (m['id'] as String?) ?? '',
- node: (m['node'] as String?) ?? '',
- ),
- )
- .where((m) => m.id.isNotEmpty)
- .toList();
- final capable = response['localModelEngines'];
- List<GridModel> parseModels(Object? raw) => (raw as List<dynamic>? ?? [])
- .whereType<Map<String, dynamic>>()
- .map(
- (m) => GridModel(
- id: (m['id'] as String?) ?? '',
- node: (m['node'] as String?) ?? '',
- ),
- )
- .where((m) => m.id.isNotEmpty)
- .toList();
- final grids = (response['grids'] as List<dynamic>? ?? [])
- .whereType<Map<String, dynamic>>()
- .where((g) => (g['name'] as String?)?.isNotEmpty == true)
- .map(
- (g) => GridSection(
- name: g['name'] as String,
- own: g['own'] == true,
- models: [
- for (final m in parseModels(g['models']))
- GridModel(id: m.id, node: m.node, grid: g['name'] as String),
- ],
- ),
- )
- .toList();
- return GridModels(
- gridName: response['gridName'] as String?,
- models: models,
- grids: grids,
- localModelEngines: capable is List
- ? capable.whereType<String>().map((e) => e.toLowerCase()).toSet()
- : null,
- gridCli: GridCli.parse(response['gridCli']),
- supportsModelLaunch: response['supportsModelLaunch'] == true,
- reachable: response['error'] == null,
- );
+ return GridModels.fromReply(response);
} catch (_) {
// NOT `gridName: null` with an empty list — that is the shape of "this account has no grid",
// and a caller cannot tell it from "the machine did not answer". A signed-in user whose daemon
@@ -5091,6 +5069,44 @@ class AppNotifier extends ChangeNotifier {
}
}
+ final Map<String, Future<GridModels>> _gridReads = {};
+
+ /// Ask [machineId]'s daemon for the model list and take the answer as its [gridPictures] entry.
+ ///
+ /// One read feeds every surface. Reads for one machine that overlap are one request — the Model
+ /// Manager, the menu and every pane's picker all refresh on the same return to the foreground, and
+ /// that is one question to the daemon, not one per surface. An answer that was already on its way
+ /// when a `grid_models_changed` push landed is returned to its caller but not adopted: the push is
+ /// newer. An answer that is not one — the machine could not be asked — is returned but not adopted
+ /// either: one surface's timed-out read must not blank the list every other surface is showing.
+ Future<GridModels> refreshGridModels(String machineId) {
+ final inFlight = _gridReads[machineId];
+ if (inFlight != null) return inFlight;
+ final epoch = gridPictures.epochOf(machineId);
+ late final Future<GridModels> read;
+ read = gridModels(machineId)
+ .then((answer) {
+ if (!_disposed && answer.reachable) {
+ gridPictures.adopt(machineId, answer, ifEpoch: epoch);
+ }
+ return answer;
+ })
+ .whenComplete(() {
+ if (identical(_gridReads[machineId], read)) {
+ _gridReads.remove(machineId);
+ }
+ });
+ return _gridReads[machineId] = read;
+ }
+
+ /// What a surface shows after a read: the app's picture — newer than the answer when a push landed
+ /// meanwhile, and the last good list when this read could not reach the machine — or the answer
+ /// itself when there has never been a picture.
+ Future<GridModels> readGridPicture(String machineId) async {
+ final answer = await refreshGridModels(machineId);
+ return gridPictures[machineId] ?? answer;
+ }
+
/// Model Manager keeps the original package ID for installed workspaces.
static const gridHarness = 'autonomous/autonomous-grid';
ModelManagerController? _modelManager;
@@ -6979,7 +6995,7 @@ class AppNotifier extends ChangeNotifier {
}
final connection = _conn(machineId);
if (!creation.awaitingConfirmation && choices['gridModel'] != null) {
- final models = await gridModels(machineId);
+ final models = await refreshGridModels(machineId);
if (!models.reachable) {
return creation._complete(
'Could not verify models on $machineName. Refresh models or use your subscription.',
@@ -10673,6 +10689,12 @@ class AppNotifier extends ChangeNotifier {
}
}
break;
+ case 'grid_models_changed':
+ // The daemon's picture of the account's grids changed — read without waking any of them,
+ // so this arrives for a grid going to sleep as well as for a model coming up. The payload
+ // is the whole `grid_models_list` document: adopted as it is, with no request.
+ gridPictures.adopt(machineId, GridModels.fromReply(payload));
+ break;
case 'machines_changed':
// The account's machine list changed somewhere: a machine created,
// renamed or deleted, or a shared harness invited or taken back. The
@@ -10996,6 +11018,8 @@ class AppNotifier extends ChangeNotifier {
terminalThemeStore.removeListener(_announceTerminalThemeEverywhere);
_localGitProjects.dispose();
sessionPreviews.dispose();
+ gridPictures.dispose();
+ foreground.dispose();
// Its sweep timer would otherwise outlive the window it was drawing into.
agentAlerts.dispose();
agentUnread.dispose();
diff --git a/desktop/lib/state/grid_pictures.dart b/desktop/lib/state/grid_pictures.dart
new file mode 100644
index 000000000..625e42845
--- /dev/null
+++ b/desktop/lib/state/grid_pictures.dart
@@ -0,0 +1,48 @@
+import 'package:flutter/foundation.dart';
+
+import '../core/models.dart';
+
+/// The app's one picture of what each machine's daemon says the account's grids serve.
+///
+/// The pane picker, the macOS Models menu and the Model Manager used to keep an answer each, asked
+/// for on their own schedules — three reads for one list, and three chances to disagree. They now
+/// read this, and it is fed from two places: every `grid_models_list` answer ([AppNotifier.refreshGridModels])
+/// and the daemon's `grid_models_changed` push, which carries the same document and needs no
+/// request at all.
+///
+/// Its own notifier rather than [AppNotifier]'s, so a surface listening for a new list is not
+/// rebuilt on every terminal frame, and a list changing does not rebuild the workspace.
+class GridPictures extends ChangeNotifier {
+ final Map<String, GridModels> _byMachine = {};
+ final Map<String, int> _epochs = {};
+
+ /// Moves on every [clear], so a read that started under the previous account can never land.
+ int _generation = 0;
+
+ /// The latest answer from [machineId]'s daemon, or null before it has said anything.
+ GridModels? operator [](String machineId) => _byMachine[machineId];
+
+ /// How many answers [machineId]'s picture has taken so far. A read captures it when it starts, so
+ /// that an answer which was already on its way when a push landed cannot overwrite the push.
+ int epochOf(String machineId) =>
+ (_generation << 32) | (_epochs[machineId] ?? 0);
+
+ /// Take [answer] as [machineId]'s picture. With [ifEpoch], only when nothing else has been
+ /// adopted for that machine since the caller read [epochOf]. Returns whether it was adopted.
+ bool adopt(String machineId, GridModels answer, {int? ifEpoch}) {
+ if (ifEpoch != null && epochOf(machineId) != ifEpoch) return false;
+ _byMachine[machineId] = answer;
+ _epochs[machineId] = (_epochs[machineId] ?? 0) + 1;
+ notifyListeners();
+ return true;
+ }
+
+ /// Forget every machine — a sign-out, where the next account's grids are not these.
+ void clear() {
+ _generation++;
+ _epochs.clear();
+ if (_byMachine.isEmpty) return;
+ _byMachine.clear();
+ notifyListeners();
+ }
+}
diff --git a/desktop/lib/state/new_harness.dart b/desktop/lib/state/new_harness.dart
index 87aabff50..a7c0139d1 100644
--- a/desktop/lib/state/new_harness.dart
+++ b/desktop/lib/state/new_harness.dart
@@ -920,7 +920,7 @@ class NewHarnessController extends ChangeNotifier {
// Native credential reads stay out of fixture tests, as in the session picker.
if (!kUnderTest) unawaited(_modelUsage!.refresh());
_refresh();
- final answer = await app.gridModels(machine);
+ final answer = await app.refreshGridModels(machine);
if (_disposed || request != _modelRequest || machine != _machineId) return;
_modelCatalog = answer;
_loadingModels = false;
diff --git a/desktop/lib/widgets/grid_model_picker.dart b/desktop/lib/widgets/grid_model_picker.dart
index a83706adf..2f3b6f008 100644
--- a/desktop/lib/widgets/grid_model_picker.dart
+++ b/desktop/lib/widgets/grid_model_picker.dart
@@ -97,14 +97,44 @@ class _GridModelPickerState extends State<GridModelPicker> {
ModelsMenuController? _usage;
GridModels? _last;
+ /// Set when the control was built while the app was in the background: the warm-up it skipped is
+ /// owed, and paid once when the app comes back to the foreground.
+ bool _prefetchOwed = false;
+
@override
void initState() {
super.initState();
+ // The app's one picture of the grids, when some other surface has already read it — or the
+ // daemon has pushed it — is a warm answer for this control's first click.
+ _last = widget.notifier.gridPictures[widget.machineId];
+ widget.notifier.gridPictures.addListener(_pictureChanged);
+ widget.notifier.foreground.addListener(_foregroundChanged);
// Warm the answer as soon as the control exists, so a click lands on a memo rather than on two
// subprocess spawns and two network round trips — measured at ~1.4s, which is a person watching
// a header do nothing. Fire-and-forget: nothing here waits on it, and a failure just means the
- // first open pays what it used to.
- unawaited(_prefetch());
+ // first open pays what it used to. Not while the app is in the background: a pane restored
+ // behind a minimised window asks nothing until somebody can see it.
+ if (widget.notifier.inForeground) {
+ unawaited(_prefetch());
+ } else {
+ _prefetchOwed = true;
+ }
+ }
+
+ void _foregroundChanged() {
+ if (!mounted || !_prefetchOwed || !widget.notifier.inForeground) return;
+ _prefetchOwed = false;
+ unawaited(_prefetch().then((_) => _refreshOpenMenu()));
+ }
+
+ /// The app's picture of this machine's grids changed — another surface read it, or the daemon
+ /// pushed it. Taken as this control's memo, and drawn into the menu if one is open.
+ void _pictureChanged() {
+ if (!mounted) return;
+ final fresh = widget.notifier.gridPictures[widget.machineId];
+ if (fresh == null || identical(fresh, _last)) return;
+ _last = fresh;
+ _refreshOpenMenu();
}
Future<void> _prefetch() async {
@@ -116,8 +146,9 @@ class _GridModelPickerState extends State<GridModelPicker> {
// usage rail keeps out of tests (kUnderTest) — here every test that drew a pane header left that
// work's timers pending after the tree was gone. Opening the menu still refreshes.
if (!kUnderTest) unawaited(_usage!.refresh().catchError((_) {}));
- final answer = await widget.notifier.gridModels(widget.machineId);
- if (mounted) _last = answer;
+ _prefetchOwed = false;
+ final picture = await widget.notifier.readGridPicture(widget.machineId);
+ if (mounted) _last = picture;
}
/// Closes the menu this control has open, if any. Set while one is showing.
@@ -200,6 +231,9 @@ class _GridModelPickerState extends State<GridModelPicker> {
WidgetsBinding.instance.addPostFrameCallback((_) {
if (mounted && _entry == entry) entry.markNeedsBuild();
});
+ // A post-frame callback waits for a frame, and nothing else may be drawing: a list pushed by the
+ // daemon lands in an idle window, where the redraw would sit until some unrelated repaint.
+ WidgetsBinding.instance.scheduleFrame();
}
/// The answer the OPEN menu is drawing, and the overlay entry drawing it. Both set only while a
@@ -212,6 +246,8 @@ class _GridModelPickerState extends State<GridModelPicker> {
@override
void dispose() {
+ widget.notifier.gridPictures.removeListener(_pictureChanged);
+ widget.notifier.foreground.removeListener(_foregroundChanged);
_expiry?.cancel();
// A pane can go away under an open menu — closed, moved, or its swarm switched — and an overlay
// entry outlives the State that inserted it.
@@ -266,7 +302,7 @@ class _GridModelPickerState extends State<GridModelPicker> {
// This open uses whatever is cached; the refresh lands for the next one. `refresh()` is itself
// capped at once a minute, so opening the menu repeatedly costs nothing.
unawaited(_usage!.refresh().catchError((_) {}));
- answer = await widget.notifier.gridModels(widget.machineId);
+ answer = await widget.notifier.readGridPicture(widget.machineId);
} finally {
if (mounted) setState(() => _loading = false);
}
@@ -426,6 +462,7 @@ class _GridModelPickerState extends State<GridModelPicker> {
final right = b.grids[i];
if (left.name != right.name ||
left.own != right.own ||
+ left.state != right.state ||
left.models.length != right.models.length) {
return false;
}
diff --git a/desktop/test/grid_picture_test.dart b/desktop/test/grid_picture_test.dart
new file mode 100644
index 000000000..f16ddbdc1
--- /dev/null
+++ b/desktop/test/grid_picture_test.dart
@@ -0,0 +1,508 @@
+// The app-level picture of what the account's grids serve, and the three surfaces that refresh it
+// (the Model Manager, the macOS Models menu, the pane picker) keeping quiet while the app is in the
+// background. grid-reads-without-waking issue 02, desktop half.
+import 'dart:async';
+
+import 'package:flutter/material.dart';
+import 'package:flutter/services.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness/core/models.dart';
+import 'package:harness/models/model_manager_controller.dart';
+import 'package:harness/screens/swarm_screen.dart';
+import 'package:harness/state/app_state.dart';
+import 'package:harness/state/swarm_catalog.dart';
+import 'package:harness/usage/models_menu_controller.dart';
+import 'package:harness/usage/usage_controller.dart';
+import 'package:harness/widgets/grid_model_picker.dart';
+import 'package:harness/ws/ws_conn.dart';
+
+import 'support/model_manager.dart';
+import 'swarm_state_test.dart' show MemoryStore, createApp;
+
+/// A daemon that answers `grid_models_list` from [reply] and counts every time it is asked.
+class _Daemon extends WsConn {
+ _Daemon(this.reply)
+ : super(
+ wsBaseUrl: 'ws://fixture.invalid',
+ autonomousEnv: 'test',
+ machineId: 'm',
+ accessTokenProvider: (_, _) async => '',
+ onAuthFailure: (_) {},
+ onEvent: (_) {},
+ onStatus: (_) {},
+ );
+
+ Map<String, dynamic> reply;
+ int asked = 0;
+
+ /// When set, the next `grid_models_list` waits on it — so a test can land a push while a read is
+ /// still out.
+ Completer<void>? hold;
+
+ @override
+ Future<Map<String, dynamic>> request(
+ String type, {
+ Map<String, dynamic> payload = const {},
+ Duration timeout = const Duration(seconds: 20),
+ }) async {
+ if (type != 'grid_models_list') return {};
+ asked++;
+ final held = hold;
+ if (held != null) {
+ hold = null;
+ final answer = reply;
+ await held.future;
+ return answer;
+ }
+ return reply;
+ }
+}
+
+Map<String, dynamic> _reply({
+ String state = 'awake',
+ String node = 'macbook',
+ List<String> ids = const ['Qwen3.5-4B'],
+ bool withNewFields = true,
+}) => {
+ 'gridName': 'home',
+ 'models': [
+ for (final id in ids) {'id': id, 'node': node},
+ ],
+ 'grids': [
+ {
+ 'name': 'home',
+ 'type': 'permissioned-public',
+ 'own': true,
+ 'models': [
+ for (final id in ids) {'id': id, 'node': node},
+ ],
+ if (withNewFields) ...{
+ 'state': state,
+ 'seenAt': '2026-09-24T08:00:00.000Z',
+ 'lastKnownAge': 42,
+ },
+ },
+ ],
+ 'supportsModelLaunch': true,
+};
+
+void main() {
+ TestWidgetsFlutterBinding.ensureInitialized();
+
+ group('GridModels.fromReply', () {
+ test('reads the three optional fields a newer daemon adds', () {
+ final answer = GridModels.fromReply(_reply(state: 'asleep'));
+ final section = answer.sections.single;
+ expect(section.state, GridSectionState.asleep);
+ expect(section.seenAt, DateTime.utc(2026, 9, 24, 8));
+ expect(section.lastKnownAge, 42);
+ // The sleeping section keeps its last known models — never blanked.
+ expect(section.models.single.id, 'Qwen3.5-4B');
+ });
+
+ test('an older daemon that sends none of them reads exactly as before', () {
+ final answer = GridModels.fromReply(_reply(withNewFields: false));
+ final section = answer.sections.single;
+ expect(section.state, isNull);
+ expect(section.seenAt, isNull);
+ expect(section.lastKnownAge, isNull);
+ expect(section.name, 'home');
+ expect(section.own, isTrue);
+ expect(section.models.single.node, 'macbook');
+ expect(section.models.single.grid, 'home');
+ expect(answer.models.single.id, 'Qwen3.5-4B');
+ expect(answer.reachable, isTrue);
+ expect(answer.supportsModelLaunch, isTrue);
+ });
+
+ test('a state it does not know, a negative age and a bad time are no claim at all', () {
+ final raw = _reply();
+ final grid = (raw['grids'] as List).single as Map<String, dynamic>;
+ grid['state'] = 'hibernating';
+ grid['lastKnownAge'] = -5;
+ grid['seenAt'] = 'yesterday';
+ final section = GridModels.fromReply(raw).sections.single;
+ expect(section.state, isNull);
+ expect(section.lastKnownAge, isNull);
+ expect(section.seenAt, isNull);
+ });
+ });
+
+ group('the app-level picture', () {
+ late _Daemon daemon;
+ late AppNotifier app;
+ setUp(() {
+ daemon = _Daemon(_reply());
+ app = createApp(connectionForTest: (_) => daemon);
+ });
+ tearDown(() => app.dispose());
+
+ test('a grid_models_changed push replaces it with no request', () async {
+ expect(app.gridPictures['m'], isNull);
+ await app.handleEventForTest('m', {
+ 'type': 'grid_models_changed',
+ 'payload': _reply(state: 'asleep', node: 'studio'),
+ });
+ final picture = app.gridPictures['m']!;
+ expect(picture.sections.single.state, GridSectionState.asleep);
+ expect(picture.sections.single.models.single.node, 'studio');
+ expect(daemon.asked, 0);
+ });
+
+ test(
+ 'every read feeds it, and reads that overlap are one request',
+ () async {
+ final first = app.refreshGridModels('m');
+ final second = app.refreshGridModels('m');
+ expect(identical(await first, await second), isTrue);
+ expect(daemon.asked, 1);
+ expect(
+ app.gridPictures['m']!.sections.single.state,
+ GridSectionState.awake,
+ );
+ },
+ );
+
+ test(
+ 'a read that was out when a push landed does not overwrite the push',
+ () async {
+ final hold = daemon.hold = Completer<void>();
+ final read = app.refreshGridModels('m');
+ await app.handleEventForTest('m', {
+ 'type': 'grid_models_changed',
+ 'payload': _reply(state: 'asleep'),
+ });
+ hold.complete();
+ await read;
+ expect(
+ app.gridPictures['m']!.sections.single.state,
+ GridSectionState.asleep,
+ );
+ },
+ );
+
+ test(
+ 'a read that could not reach the machine keeps the last good picture for every surface',
+ () async {
+ await app.refreshGridModels('m');
+ daemon.reply = {'error': 'GRID_MODELS_FAILED'};
+
+ // The caller that must know (the creation check) still hears that the machine did not answer…
+ expect((await app.refreshGridModels('m')).reachable, isFalse);
+ // …while the picture every other surface shows stays the last list it had.
+ final picture = await app.readGridPicture('m');
+ expect(picture.reachable, isTrue);
+ expect(picture.sections.single.models.single.id, 'Qwen3.5-4B');
+ expect(app.gridPictures['m']!.reachable, isTrue);
+ },
+ );
+
+ test('the foreground flag follows the lifecycle, unknown counting as foreground', () {
+ expect(app.inForeground, isTrue);
+ app.appLifecycleChanged(AppLifecycleState.inactive);
+ expect(app.inForeground, isFalse);
+ app.appLifecycleChanged(AppLifecycleState.hidden);
+ expect(app.inForeground, isFalse);
+ app.appLifecycleChanged(AppLifecycleState.resumed);
+ expect(app.inForeground, isTrue);
+ app.appLifecycleChanged(null);
+ expect(app.inForeground, isTrue);
+ });
+ });
+
+ group('the Model Manager timer', () {
+ late ModelManagerTestApp app;
+ setUp(() => app = ModelManagerTestApp(ModelManagerConnection()));
+ tearDown(() => app.dispose());
+
+ // Each test owns its controller and disposes it before the test ends: its 4 s timer is a fake
+ // timer inside `testWidgets`, and one left running fails the test.
+ ModelManagerController manager() =>
+ ModelManagerController(app, storage: MemoryStore())
+ // The visible panel is the tick that reads every 4 s whatever the clock says — the idle
+ // 60 s read compares `DateTime.now()`, which fake time does not move.
+ ..setPanelVisible(true);
+
+ testWidgets(
+ 'a background app runs none of it, and one refresh runs on return',
+ (tester) async {
+ final controller = manager();
+ app.appLifecycleChanged(AppLifecycleState.hidden);
+ controller.start();
+ await tester.pump();
+ for (var i = 0; i < 30; i++) {
+ await tester.pump(const Duration(seconds: 4));
+ }
+ expect(app.localReads, 0);
+ expect(app.gridReads, 0);
+
+ app.appLifecycleChanged(AppLifecycleState.resumed);
+ await tester.pump();
+ expect(app.localReads, 1);
+ expect(app.gridReads, 1);
+ controller.dispose();
+ await tester.pump(const Duration(seconds: 10));
+ },
+ );
+
+ testWidgets('in the foreground it reads as it always did', (tester) async {
+ final controller = manager();
+ controller.start();
+ await tester.pump();
+ final first = app.localReads;
+ expect(first, 1);
+ await tester.pump(const Duration(seconds: 4));
+ expect(app.localReads, 2);
+ controller.dispose();
+ await tester.pump(const Duration(seconds: 10));
+ });
+
+ testWidgets('a push reaches the manager with no read of its own', (
+ tester,
+ ) async {
+ final controller = manager();
+ controller.start();
+ await tester.pump();
+ final reads = app.gridReads;
+ await app.handleEventForTest('m', {
+ 'type': 'grid_models_changed',
+ 'payload': _reply(ids: ['Pushed-Model']),
+ });
+ expect(controller.sections.single.models.single.id, 'Pushed-Model');
+ expect(app.gridReads, reads);
+ controller.dispose();
+ await tester.pump(const Duration(seconds: 10));
+ });
+ });
+
+ group('the pane picker prefetch', () {
+ late _Daemon daemon;
+ late AppNotifier app;
+ setUp(() {
+ daemon = _Daemon(_reply());
+ app = createApp(connectionForTest: (_) => daemon);
+ });
+ tearDown(() => app.dispose());
+
+ Future<void> mount(WidgetTester tester) => tester.pumpWidget(
+ MaterialApp(
+ home: Scaffold(
+ body: Center(
+ child: GridModelPicker(
+ notifier: app,
+ machineId: 'm',
+ engineLabel: 'claude',
+ ),
+ ),
+ ),
+ ),
+ );
+
+ testWidgets('does not run in the background, and runs once on return', (
+ tester,
+ ) async {
+ app.appLifecycleChanged(AppLifecycleState.inactive);
+ await mount(tester);
+ await tester.pump();
+ expect(daemon.asked, 0);
+ app.appLifecycleChanged(AppLifecycleState.resumed);
+ await tester.pump();
+ expect(daemon.asked, 1);
+ app.appLifecycleChanged(AppLifecycleState.inactive);
+ app.appLifecycleChanged(AppLifecycleState.resumed);
+ await tester.pump();
+ // A picker that is already warm owes nothing on a second return.
+ expect(daemon.asked, 1);
+ });
+
+ testWidgets('in the foreground it warms as it always did', (tester) async {
+ await mount(tester);
+ await tester.pump();
+ expect(daemon.asked, 1);
+ });
+
+ testWidgets('a push lands in the open menu with no request', (
+ tester,
+ ) async {
+ tester.view.physicalSize = const Size(1200, 900);
+ tester.view.devicePixelRatio = 1;
+ addTearDown(tester.view.reset);
+ await mount(tester);
+ await tester.tap(find.byType(GridModelPicker));
+ await tester.pumpAndSettle();
+ expect(find.text('Qwen3.5-4B'), findsOneWidget);
+ final asked = daemon.asked;
+ await app.handleEventForTest('m', {
+ 'type': 'grid_models_changed',
+ 'payload': _reply(ids: ['Pushed-Model']),
+ });
+ await tester.pumpAndSettle();
+ expect(find.text('Pushed-Model'), findsOneWidget);
+ expect(daemon.asked, asked);
+ });
+ });
+
+ group('the macOS Models menu', () {
+ const channel = MethodChannel('harness/swarm_tabs');
+ late _Daemon daemon;
+ late AppNotifier app;
+ late List<MethodCall> messages;
+ late ModelsMenuController menu;
+ late UsageController usage;
+ late SwarmProjectStore projects;
+
+ setUp(() {
+ daemon = _Daemon(_reply());
+ app = createApp(connectionForTest: (_) => daemon);
+ usage = UsageController(sources: const [], autoStart: false);
+ menu = ModelsMenuController(usage: usage);
+ projects = SwarmProjectStore();
+ });
+ tearDown(() {
+ menu.dispose();
+ usage.dispose();
+ app.dispose();
+ projects.dispose();
+ });
+
+ Future<void> mount(WidgetTester tester) async {
+ final messenger = tester.binding.defaultBinaryMessenger;
+ messages = <MethodCall>[];
+ messenger.setMockMethodCallHandler(channel, (call) async {
+ messages.add(call);
+ return true;
+ });
+ addTearDown(() => messenger.setMockMethodCallHandler(channel, null));
+ await tester.pumpWidget(
+ MaterialApp(
+ home: SwarmScreen(
+ notifier: app,
+ nativeTabs: true,
+ projectStore: projects,
+ modelsMenu: menu,
+ ),
+ ),
+ );
+ await tester.pump();
+ }
+
+ Future<void> native(WidgetTester tester, String method) async {
+ final reply = Completer<void>();
+ tester.binding.defaultBinaryMessenger.handlePlatformMessage(
+ channel.name,
+ const StandardMethodCodec().encodeMethodCall(MethodCall(method)),
+ (_) => reply.complete(),
+ );
+ await reply.future;
+ await tester.pump();
+ }
+
+ List<Map<Object?, Object?>> lastSections() =>
+ ((messages.lastWhere((c) => c.method == 'modelsState').arguments
+ as Map)['sections']
+ as List)
+ .cast<Map<Object?, Object?>>();
+
+ testWidgets(
+ 'does not refresh in the background, and refreshes once on return',
+ (tester) async {
+ tester.binding.handleAppLifecycleStateChanged(
+ AppLifecycleState.inactive,
+ );
+ addTearDown(
+ () => tester.binding.handleAppLifecycleStateChanged(
+ AppLifecycleState.resumed,
+ ),
+ );
+ await mount(tester);
+ // Every app change asks the menu to refresh; in the background none of them may.
+ for (var i = 0; i < 5; i++) {
+ app.notifyListeners();
+ await tester.pump(const Duration(seconds: 15));
+ }
+ expect(daemon.asked, 0);
+ tester.binding.handleAppLifecycleStateChanged(
+ AppLifecycleState.resumed,
+ );
+ await tester.pump();
+ expect(daemon.asked, 1);
+ await tester.pumpWidget(const SizedBox());
+ },
+ );
+
+ testWidgets('refreshes when it opens, inside the ten-second throttle', (
+ tester,
+ ) async {
+ await mount(tester);
+ final before = daemon.asked;
+ expect(before, 1);
+ await native(tester, 'modelsOpened');
+ expect(daemon.asked, before + 1);
+ await tester.pumpWidget(const SizedBox());
+ });
+
+ testWidgets('opening it refreshes even when the window reads as inactive', (
+ tester,
+ ) async {
+ await mount(tester);
+ final before = daemon.asked;
+ tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.inactive);
+ addTearDown(
+ () => tester.binding.handleAppLifecycleStateChanged(
+ AppLifecycleState.resumed,
+ ),
+ );
+ await tester.pump();
+ await native(tester, 'modelsOpened');
+ expect(daemon.asked, before + 1);
+ await tester.pumpWidget(const SizedBox());
+ });
+
+ testWidgets('updates when only a node changed', (tester) async {
+ await mount(tester);
+ expect(lastSections().single['models'], [
+ {'id': 'Qwen3.5-4B', 'node': 'macbook'},
+ ]);
+ daemon.reply = _reply(node: 'studio');
+ await native(tester, 'modelsOpened');
+ expect(lastSections().single['models'], [
+ {'id': 'Qwen3.5-4B', 'node': 'studio'},
+ ]);
+ await tester.pumpWidget(const SizedBox());
+ });
+
+ testWidgets('updates when only the state changed', (tester) async {
+ await mount(tester);
+ expect(lastSections().single['state'], 'awake');
+ daemon.reply = _reply(state: 'asleep');
+ await native(tester, 'modelsOpened');
+ expect(lastSections().single['state'], 'asleep');
+ await tester.pumpWidget(const SizedBox());
+ });
+
+ testWidgets('a push updates it with no request', (tester) async {
+ await mount(tester);
+ final asked = daemon.asked;
+ await app.handleEventForTest('m', {
+ 'type': 'grid_models_changed',
+ 'payload': _reply(ids: ['Pushed-Model']),
+ });
+ await tester.pump();
+ expect((lastSections().single['models'] as List).single, {
+ 'id': 'Pushed-Model',
+ 'node': 'macbook',
+ });
+ expect(daemon.asked, asked);
+ await tester.pumpWidget(const SizedBox());
+ });
+
+ testWidgets('an older daemon sends no state, and the menu says none', (
+ tester,
+ ) async {
+ daemon.reply = _reply(withNewFields: false);
+ await mount(tester);
+ expect(lastSections().single.containsKey('state'), isFalse);
+ await tester.pumpWidget(const SizedBox());
+ });
+ });
+}
diff --git a/desktop/test/support/model_manager.dart b/desktop/test/support/model_manager.dart
index 09af2bd53..f675eb19f 100644
--- a/desktop/test/support/model_manager.dart
+++ b/desktop/test/support/model_manager.dart
@@ -92,7 +92,7 @@ class ModelManagerTestApp extends AppNotifier {
bool installed = true;
String? installError, probeError, resumeError;
bool resumeThrows = false;
- int localReads = 0, resumes = 0;
+ int localReads = 0, resumes = 0, gridReads = 0;
Completer<Map<String, dynamic>>? localReply;
Completer<void>? probeReply;
String? sendError;
@@ -179,7 +179,11 @@ class ModelManagerTestApp extends AppNotifier {
}
@override
- Future<GridModels> gridModels(String machineId) async => inventory;
+ Future<GridModels> gridModels(String machineId) async {
+ gridReads++;
+ return inventory;
+ }
+
@override
Future<RestartAgentResult> resumeAgent(
String machineId,
diff --git a/store/agents/autonomous-grid/lib/connect.mjs b/store/agents/autonomous-grid/lib/connect.mjs
index c14a54cde..7e4a7c3a9 100644
--- a/store/agents/autonomous-grid/lib/connect.mjs
+++ b/store/agents/autonomous-grid/lib/connect.mjs
@@ -1,7 +1,7 @@
import { readFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { isAbsolute, join } from 'node:path';
-import { atomicJson, gridJson, gridSelect, now, readConfig, readJson, stateDir, text, validateConfig } from './fleet.mjs';
+import { ASLEEP_CODE, atomicJson, gridJson, gridSelect, NO_WAKE, now, readConfig, readJson, stateDir, text, validateConfig } from './fleet.mjs';
import { discoverMachines } from './harness.mjs';
/** The email `grid login` recorded in the credential store, or null. `GRID_HOME` is honoured the way grid honours it. */
@@ -56,6 +56,12 @@ export async function initializeWorkspace(workspace, { runJson = gridJson, selec
let config = await readConfig(workspace), source = 'workspace', message = '', candidates = [];
const controller = () => config.machines.find(m => m.id === config.controller);
const read = (mode, args) => runJson(controller(), mode, args, { timeoutMs: 5000 }).catch(() => ({ ok: false }));
+ // "Does this grid answer?" — asked on a workspace's first start, which nobody clicked, so it must not
+ // wake a sleeping grid to find out (NO_WAKE on a remote grid). Asleep is an answer: the grid is there.
+ // A `grid` too old for the flag cannot ask without waking; the grids probed here are the account's own
+ // or its active selection, so that is taken as there too rather than choosing nothing.
+ const probe = (mode, grid) => read(mode, ['engines', grid, ...(mode === 'remote' ? [NO_WAKE] : [])]);
+ const answers = result => (result.ok && Array.isArray(result.value)) || result.refusal === ASLEEP_CODE || result.outdated === true;
if (!config.grid) {
const saved = await readJson(profilePath, null);
if (saved) {
@@ -75,8 +81,7 @@ export async function initializeWorkspace(workspace, { runJson = gridJson, selec
// and the viewer agree from the first minute — never a team's or a community grid by guess.
const active = text((await read(preferred, ['use'])).value?.active);
if (active && !active.startsWith('-')) {
- const engines = await read(preferred, ['engines', active]);
- if (engines.ok && Array.isArray(engines.value)) { config.grid = active; source = 'Grid selection'; }
+ if (answers(await probe(preferred, active))) { config.grid = active; source = 'Grid selection'; }
}
const remote = listings.find(l => l.mode === 'remote')?.result;
// Harness names the account's grid (HARNESS_PRIVATE_GRID, from the backend that minted it);
@@ -85,8 +90,7 @@ export async function initializeWorkspace(workspace, { runJson = gridJson, selec
config.personalGrid = env.HARNESS_PRIVATE_GRID?.trim() || pickPrivateGrid(await email(), remote?.value);
const own = config.grid ? null : config.personalGrid;
if (own) {
- const engines = await read('remote', ['engines', own]);
- if (engines.ok && Array.isArray(engines.value)) {
+ if (answers(await probe('remote', own))) {
config.mode = 'remote'; config.grid = own; source = 'your private grid';
// The write form of `use` answers in prose, not JSON (lib/fleet.mjs gridSelect).
await select(controller(), 'remote', own, { timeoutMs: 5000 }).catch(() => ({ ok: false }));
@@ -100,8 +104,8 @@ export async function initializeWorkspace(workspace, { runJson = gridJson, selec
// Only a unique known grid is an unambiguous fallback. Multiple grids stay a choice,
// even when one happens to have more traffic. Probe only the candidate we may select.
if (candidates.length === 1) {
- const candidate = candidates[0], engines = await read(candidate.mode,['engines',candidate.grid]);
- if (engines.ok && Array.isArray(engines.value)) {
+ const candidate = candidates[0];
+ if (answers(await probe(candidate.mode, candidate.grid))) {
config.mode = candidate.mode; config.grid = candidate.grid; source = 'only reachable grid';
}
}
diff --git a/store/agents/autonomous-grid/lib/fleet.mjs b/store/agents/autonomous-grid/lib/fleet.mjs
index a3ce29152..a01c5836e 100644
--- a/store/agents/autonomous-grid/lib/fleet.mjs
+++ b/store/agents/autonomous-grid/lib/fleet.mjs
@@ -13,6 +13,47 @@ const idPattern = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,63}$/;
export const text = (value, max = 240) => typeof value === 'string' ? value.replace(/[\u0000-\u001f\u007f]/g, ' ').slice(0, max) : '';
export const number = value => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
+/**
+ * The flag that reads a grid WITHOUT waking it (`grid` ≥ 0.3.49): the overview goes out with no
+ * credential, so a sleeping grid answers "asleep" at once instead of being started for a glance.
+ * Every automatic engines/models/stats read of a REMOTE grid carries it (lib/telemetry.mjs), always,
+ * with no version check — the binary `toolchain/grid.sh` resolves can change between two polls when
+ * the managed pin moves. A `grid` too old for it refuses the whole command with argparse's exit 2,
+ * which is loud and wakes nothing; the caller shows its last reading and never asks again without it.
+ *
+ * ⚠️ A cross-repo literal: the public CLI's `cli/remote_overview.NO_WAKE_FLAG`, pinned by its
+ * `tests/test_grid_reads_lockstep.py`, which finds it here by this exact quoted spelling.
+ */
+export const NO_WAKE = '--no-wake';
+/**
+ * The code beside `detail` when the platform says a grid is resting (grid-apis' proxy), carried to us
+ * in `grid`'s `--json` error envelope. Compared for EQUALITY and nothing else: any other refusal —
+ * codeless, stopped, master down, deleted — is a failed read and keeps today's handling.
+ * ⚠️ Cross-repo like NO_WAKE, and pinned the same way.
+ */
+export const ASLEEP_CODE = 'grid_asleep';
+/** The owner status of a grid the platform put to sleep (`grid info --json`'s `status`). Only the
+ * grid's owner is shown a status; a member sees null and learns it from ASLEEP_CODE instead. */
+export const ASLEEP_STATE = 'asleep';
+/** argparse's exit status for an argument it does not know — how an old `grid` refuses NO_WAKE. */
+const USAGE_EXIT = 2;
+
+/** The `code` of `grid`'s JSON error envelope (`{"error": {"code": …, "message": …}}`, one line on
+ * either stream), or null. A program branches on this; the sentence beside it is for people. */
+export function refusalCode(stdout, stderr) {
+ for (const chunk of [stderr, stdout]) {
+ for (const line of String(chunk || '').split('\n')) {
+ const trimmed = line.trim();
+ if (!trimmed.startsWith('{')) continue;
+ try {
+ const code = JSON.parse(trimmed)?.error?.code;
+ if (typeof code === 'string' && code) return code;
+ } catch { /* not an envelope */ }
+ }
+ }
+ return null;
+}
+
export async function readJson(file, fallback, limit = 2 * 1024 * 1024) {
try {
if ((await stat(file)).size > limit) throw new Error('File is too large');
@@ -150,25 +191,36 @@ export function cleanDetail(message) {
.replace(/(^[a-z][a-z0-9+.-]*:\/\/)[^/@\s]+@/i, '$1…@'), 280);
}
+/**
+ * Run a `--json` read and classify it. A failure carries two things a caller may branch on beside the
+ * sentence: `refusal`, the envelope's code (ASLEEP_CODE is the one this package reads), and
+ * `outdated`, true when the binary refused an argument it does not know — argparse's exit 2 with its
+ * usage line, which is how a `grid` older than NO_WAKE answers it. The usage line is required as well
+ * as the status because a Harness machine's own RPC also answers 2 for a request it would not start.
+ */
export async function gridJson(machine, mode, args, options) {
const result = await execute(machine, [`--${mode}`, ...args, '--json'], options);
if (!result.ok) {
+ const refusal = refusalCode(result.stdout, result.stderr);
+ const outdated = result.code === USAGE_EXIT && /\busage:|unrecognized arguments/i.test(`${result.stderr}\n${result.stdout}`);
+ const fail = error => ({ ok: false, error, refusal, outdated });
const blocked = /(?:could not reach|network|socket)[\s\S]{0,600}(?:operation not permitted|EPERM|denied|blocked)/i.test(`${result.stdout}\n${result.stderr}`);
- if (blocked) return { ok: false, error: 'Network access was blocked by the agent sandbox. Use fleet status for viewer observations, or request scoped network approval before retrying this Grid command.' };
- if (result.code === 127) return { ok: false, error: result.error };
+ if (blocked) return fail('Network access was blocked by the agent sandbox. Use fleet status for viewer observations, or request scoped network approval before retrying this Grid command.');
+ if (result.code === 127) return fail(result.error);
// Exit 1 carries the reason on stderr (e.g. a dead relay URL): repeat it so the
// viewer and the agent see "could not reach grid …", not just an exit code.
const detail = cliDetail(result.stdout, result.stderr);
- return { ok: false, error: detail ? `grid ${args[0]} failed (${result.code}): ${detail}` : `grid ${args[0]} failed (${result.code}). Run it in the agent terminal for details.` };
+ return fail(detail ? `grid ${args[0]} failed (${result.code}): ${detail}` : `grid ${args[0]} failed (${result.code}). Run it in the agent terminal for details.`);
}
try {
const value = JSON.parse(result.stdout);
if (value?.error) {
const detail = typeof value.error === 'string' ? cleanDetail(value.error) : cleanDetail(value.error.message || 'Grid reported an error.');
- return { ok: false, error: `grid ${args[0]} reported: ${detail}` };
+ const refusal = typeof value.error?.code === 'string' && value.error.code ? value.error.code : null;
+ return { ok: false, error: `grid ${args[0]} reported: ${detail}`, refusal, outdated: false };
}
return { ok: true, value };
- } catch { return { ok: false, error: `grid ${args[0]} did not return valid JSON.` }; }
+ } catch { return { ok: false, error: `grid ${args[0]} did not return valid JSON.`, refusal: null, outdated: false }; }
}
/**
diff --git a/store/agents/autonomous-grid/lib/telemetry.mjs b/store/agents/autonomous-grid/lib/telemetry.mjs
index fde6d4127..4068ae7da 100644
--- a/store/agents/autonomous-grid/lib/telemetry.mjs
+++ b/store/agents/autonomous-grid/lib/telemetry.mjs
@@ -1,6 +1,6 @@
import { createHash } from 'node:crypto';
import { join } from 'node:path';
-import { atomicJson, gridJson, now, number, operations, readConfig, readJson, stateDir, text } from './fleet.mjs';
+import { ASLEEP_CODE, ASLEEP_STATE, atomicJson, gridJson, NO_WAKE, now, number, operations, readConfig, readJson, stateDir, text } from './fleet.mjs';
const array = value => Array.isArray(value) ? value : [];
const objects = value => array(value).filter(v => v && typeof v === 'object' && !Array.isArray(v));
@@ -68,15 +68,41 @@ export function normalizeDevice(machine, raw, observedAt) {
};
}
-export function assemble(config, reads, previous = null, observedAt = now()) {
+/** How often a SLEEPING grid is looked at again. Nothing about it changes until something wakes it, and
+ * each look is a status read plus, for a member, one credential-less refusal — cheap, but not free. */
+export const ASLEEP_POLL_MS = 30_000;
+
+/** `HH:MM` on this computer's clock — the time a person reads next to "showing the reading from". */
+const clock = iso => { const d = new Date(iso); return Number.isFinite(d.getTime()) ? `${String(d.getHours()).padStart(2, '0')}:${String(d.getMinutes()).padStart(2, '0')}` : ''; };
+
+/**
+ * One observation from `reads`. `hold` is set when this poll deliberately showed nothing new:
+ * `'asleep'` (the platform says the grid is resting) or `'updating'` (this computer's `grid` is too
+ * old for NO_WAKE, so it could not be read without waking it). Either way the previous nodes stay on
+ * screen, marked stale, and NO event is derived — a resting grid has not lost its engines, and saying
+ * "no longer serving" about every one of them was the false alarm this replaces.
+ */
+export function assemble(config, reads, previous = null, observedAt = now(), hold = null) {
const scope = JSON.stringify([config.mode, config.grid, config.controller]);
if (previous?.scope !== scope) previous = null;
const sources = Object.fromEntries(Object.entries(reads).map(([name, result]) => [name, { ok: result.ok, error: result.ok ? null : result.error, observedAt }]));
+ // Asleep is an answer, not a failed read: a member learns it from engines' refusal, and reporting
+ // that as a broken source would put an error finding in the agent's verdict for a grid at rest.
+ if (hold === 'asleep' && sources.engines) sources.engines = { ok: true, error: null, observedAt };
const info = reads.info?.ok && reads.info.value && typeof reads.info.value === 'object' ? reads.info.value : {};
const stats = reads.stats?.ok && reads.stats.value && typeof reads.stats.value === 'object' ? reads.stats.value : {};
const modelNames = new Map(objects(reads.models?.value).map(m => [text(m.model).toLowerCase(), text(m.model)]).filter(([a,b]) => a && b));
let nodes;
- const fresh = reads.engines?.ok && Array.isArray(reads.engines.value);
+ const fresh = !hold && reads.engines?.ok && Array.isArray(reads.engines.value);
+ // The time of the reading on screen: now when this poll read one, otherwise whatever the previous
+ // observation was showing. An older snapshot on disk predates the field; its own time counts only if
+ // it was itself a reading.
+ const readingFrom = fresh ? observedAt : previous ? (previous.readingFrom ?? (['live', 'partial'].includes(previous.status) ? previous.observedAt : null)) : null;
+ const controllerName = text(config.machines?.find(m => m.id === config.controller)?.name) || 'This computer';
+ const notice = hold === 'asleep' ? 'Asleep · starts when you send a message'
+ : hold === 'updating' ? `${controllerName} is being updated — ${readingFrom ? `showing the reading from ${clock(readingFrom)}` : 'there is no earlier reading to show'}`
+ : undefined;
+ if (hold === 'updating') sources.engines = { ok: false, error: notice, observedAt };
if (fresh) {
const cards = objects(stats.engines);
nodes = objects(reads.engines.value).slice(0, 256).map((raw,i) => {
@@ -107,18 +133,48 @@ export function assemble(config, reads, previous = null, observedAt = now()) {
if (fresh) for (const n of nodes) {
const old = previous?.nodes?.find(p => p.id === n.id);
if ((!old || old.online !== true) && n.online === true) addEvent('online', `${n.name} joined the grid`, n.id);
- if (old?.online === true && n.online === false) addEvent('offline', `${n.name} is no longer serving`, n.id);
+ // Only a node last SEEN serving can be said to have stopped. One held stale through a sleep (or a
+ // failed read) is a last-known entry: a cold wake that has not heard from it yet is not news.
+ if (old?.online === true && !old.stale && n.online === false) addEvent('offline', `${n.name} is no longer serving`, n.id);
if (old && n.online === true && JSON.stringify(old.models) !== JSON.stringify(n.models)) addEvent('models', `${n.name} updated its models`, n.id);
}
return {
- spec: 1, scope, observedAt, mode: config.mode, grid: text(info.grid || stats.grid || config.grid) || 'Your grid', endpoint: safeUrl(info.grid_url),
- status: !fresh ? 'unavailable' : Object.values(sources).some(s => !s.ok) ? 'partial' : 'live',
+ spec: 1, scope, observedAt, readingFrom, ...(notice ? { notice } : {}), mode: config.mode, grid: text(info.grid || stats.grid || config.grid) || 'Your grid', endpoint: safeUrl(info.grid_url),
+ status: hold || (!fresh ? 'unavailable' : Object.values(sources).some(s => !s.ok) ? 'partial' : 'live'),
sources, nodes, models, history, events: events.slice(0, 40),
summary: { enginesOnline: online.length, enginesKnown: nodes.length, modelsServing: models.length, answered: normalizeAnswered(stats.answered), uptimePct: number(stats.uptime_pct), concurrency: number(stats.parallel), activeRequests: online.length && online.every(n => n.activeRequests !== null) ? online.reduce((sum,n) => sum + n.activeRequests, 0) : null },
preferences: config.preferences,
};
}
+/**
+ * A remote grid's reads, in the order that lets a sleeping grid be SEEN without being started.
+ *
+ * `info` first: it asks the control plane, never the grid, and for the grid's owner its status says
+ * `asleep` outright — then nothing else is read. A member is shown no status, so `engines` goes next,
+ * alone, with NO_WAKE: a sleeping grid refuses it with ASLEEP_CODE, and models and stats would only be
+ * refused the same way. A `grid` too old for NO_WAKE refuses engines outright (exit 2); the others would
+ * be refused too, and running them WITHOUT the flag is exactly the wake this exists to stop — so the
+ * poll ends there and the last reading stays up. Only an engines answer earns models and stats.
+ */
+async function readRemote(read) {
+ const reads = { info: await read('info') };
+ if (reads.info.ok && reads.info.value?.status === ASLEEP_STATE) return { reads, hold: 'asleep' };
+ reads.engines = await read('engines', [NO_WAKE]);
+ if (reads.engines.refusal === ASLEEP_CODE) return { reads, hold: 'asleep' };
+ if (reads.engines.outdated) return { reads, hold: 'updating' };
+ const [models, stats] = await Promise.all([read('models', [NO_WAKE]), read('stats', [NO_WAKE])]);
+ return { reads: { ...reads, models, stats }, hold: null };
+}
+
+/** A LAN grid has no proxy and nothing to wake, so its reads are asked together as they always were —
+ * and without NO_WAKE, which a user's own older `grid` would refuse for no benefit at all. */
+async function readLocal(read) {
+ const names = ['info', 'engines', 'models'];
+ const results = await Promise.all(names.map(name => read(name)));
+ return { reads: Object.fromEntries(names.map((name, i) => [name, results[i]])), hold: null };
+}
+
export function createCollector(workspace, { runJson = gridJson, intervalMs = 8000 } = {}) {
let inFlight, deviceCache = new Map(), previous;
async function followSelection(config) {
@@ -150,18 +206,18 @@ export function createCollector(workspace, { runJson = gridJson, intervalMs = 80
}
const controller = config.machines.find(m => m.id === config.controller);
const selector = config.grid ? [config.grid] : [];
- const names = ['info', 'engines', 'models', ...(config.mode === 'remote' ? ['stats'] : [])];
+ const read = (command, extra = []) => runJson(controller, config.mode, [command, ...selector, ...extra]).catch(() => ({ ok: false, error: `grid ${command} could not be read.` }));
// `ls` beside the per-grid reads: the viewer's grid dropdown is every grid this account is in,
// read fresh each poll so a grid joined a minute ago is offered without a restart.
- const results = await Promise.allSettled([...names.map(command => runJson(controller, config.mode, [command, ...selector])), runJson(controller, config.mode, ['ls'])]);
- const reads = Object.fromEntries(results.slice(0, names.length).map((result,i) => [names[i], result.status === 'fulfilled' ? result.value : { ok: false, error: `grid ${names[i]} could not be read.` }]));
- const listed = results[names.length];
- const grids = listed.status === 'fulfilled' && Array.isArray(listed.value?.value)
- ? listed.value.value.map(row => ({ name: text(row.grid || row.name || row.id), type: text(row.type) })).filter(g => g.name && !g.name.startsWith('-'))
+ const listing = runJson(controller, config.mode, ['ls']).catch(() => null);
+ const { reads, hold } = config.mode === 'remote' ? await readRemote(read) : await readLocal(read);
+ const listed = await listing;
+ const grids = Array.isArray(listed?.value)
+ ? listed.value.map(row => ({ name: text(row.grid || row.name || row.id), type: text(row.type) })).filter(g => g.name && !g.name.startsWith('-'))
: previous?.grids || [];
previous ||= await readJson(join(stateDir(workspace), 'snapshot.json'), null).catch(() => null);
const observedAt = now();
- const snapshot = assemble(config, reads, previous, observedAt);
+ const snapshot = assemble(config, reads, previous, observedAt, hold);
snapshot.grids = grids;
snapshot.machines = await Promise.all(config.machines.map(async machine => {
const cacheKey = JSON.stringify(machine);
@@ -176,13 +232,13 @@ export function createCollector(workspace, { runJson = gridJson, intervalMs = 80
const activeKeys = new Set(config.machines.map(m => JSON.stringify(m)));
for (const k of deviceCache.keys()) if (!activeKeys.has(k)) deviceCache.delete(k);
snapshot.operations = await operations(workspace);
- snapshot.pollIntervalMs = intervalMs;
+ snapshot.pollIntervalMs = hold === 'asleep' ? ASLEEP_POLL_MS : intervalMs;
await atomicJson(join(stateDir(workspace), 'snapshot.json'), snapshot);
const failures = Object.entries(snapshot.sources).filter(([,s]) => !s.ok);
await atomicJson(join(workspace, '.harness', 'verdict.json'), {
spec: 1, ready: snapshot.status !== 'unavailable',
- summary: snapshot.status === 'unavailable' ? 'Grid unavailable · ask the agent to connect your fleet' : `${snapshot.summary.enginesOnline} engines online · ${snapshot.summary.modelsServing} models serving${snapshot.status === 'partial' ? ' · partial telemetry' : ''}`,
- findings: failures.map(([kind,s]) => ({ severity: kind === 'engines' ? 'error' : 'warning', kind, message: s.error })),
+ summary: snapshot.status === 'unavailable' ? 'Grid unavailable · ask the agent to connect your fleet' : snapshot.notice || `${snapshot.summary.enginesOnline} engines online · ${snapshot.summary.modelsServing} models serving${snapshot.status === 'partial' ? ' · partial telemetry' : ''}`,
+ findings: failures.map(([kind,s]) => ({ severity: kind === 'engines' && !hold ? 'error' : 'warning', kind, message: s.error })),
phases: [{ id: 'connect', name: 'Connect', state: snapshot.status === 'unavailable' ? 'active' : 'done' }, { id: 'observe', name: 'Observe', state: snapshot.status === 'unavailable' ? 'pending' : 'active' }], updatedAt: observedAt,
});
previous = snapshot;
diff --git a/store/agents/autonomous-grid/test/connect.test.mjs b/store/agents/autonomous-grid/test/connect.test.mjs
index 865619938..1da5d9df4 100644
--- a/store/agents/autonomous-grid/test/connect.test.mjs
+++ b/store/agents/autonomous-grid/test/connect.test.mjs
@@ -34,7 +34,7 @@ test('fresh initialization follows the CLI\'s active selection when there is one
const selected=[];const select=async(_host,mode,grid)=>{selected.push([mode,grid]);return {ok:true,active:grid};};
const {config,connection}=await initializeWorkspace(dir,{profilePath,discover,runJson,select,email:async()=>'[email protected]'});
assert.equal(config.grid,'autonomous.ai');assert.equal(connection.source,'Grid selection');
- assert.deepEqual(calls.filter(c=>c[1]==='engines'),[['remote','engines','autonomous.ai']]);
+ assert.deepEqual(calls.filter(c=>c[1]==='engines'),[['remote','engines','autonomous.ai','--no-wake']]);
assert.deepEqual(selected,[],'an existing selection is never rewritten');
});
test('with no selection yet, the signed-in user\'s own private grid is chosen AND selected, so the CLI agrees',async t=>{
diff --git a/store/agents/autonomous-grid/test/nowake.test.mjs b/store/agents/autonomous-grid/test/nowake.test.mjs
new file mode 100644
index 000000000..058e4ad4b
--- /dev/null
+++ b/store/agents/autonomous-grid/test/nowake.test.mjs
@@ -0,0 +1,203 @@
+// grid-reads-without-waking issue 02, the viewer's half: every automatic read goes out with
+// `--no-wake`, a sleeping grid is shown asleep instead of down, a `grid` too old for the flag leaves
+// the last reading on screen, and a hidden page stops listening.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { runInNewContext } from 'node:vm';
+import { atomicJson, gridJson, PACKAGE, stateDir } from '../lib/fleet.mjs';
+import { assemble, createCollector } from '../lib/telemetry.mjs';
+import { initializeWorkspace } from '../lib/connect.mjs';
+import { createViewer } from '../viewer.mjs';
+import { config, device, reads, remoteNodes } from './fixtures.mjs';
+
+const temporary = async t => { const dir=await mkdtemp(join(tmpdir(),'grid-nowake-'));t.after(()=>rm(dir,{recursive:true,force:true}));return dir; };
+const READS = ['engines','models','stats'];
+const asleepEnvelope = JSON.stringify({error:{code:'grid_asleep',message:'Grid test-grid is asleep: this grid is resting',status:503}});
+
+/** A fake `grid` of one version, answering each verb from `verbs` and logging every argv it saw. An
+ * "old" one refuses `--no-wake` exactly as argparse does: a usage line on stderr and exit 2. */
+async function fakeGrid(dir, name, { rejectsNoWake = false, verbs = {} }, log) {
+ const file = join(dir, name), plan = join(dir, `${name}.json`);
+ await writeFile(plan, JSON.stringify({ rejectsNoWake, verbs }));
+ await writeFile(file, `#!/usr/bin/env node
+const fs=require('fs');const plan=JSON.parse(fs.readFileSync(${JSON.stringify(plan)},'utf8'));
+const argv=process.argv.slice(2);const log=${JSON.stringify(log)};
+fs.appendFileSync(log,JSON.stringify({binary:${JSON.stringify(name)},argv})+'\\n');
+if(plan.rejectsNoWake&&argv.includes('--no-wake')){process.stderr.write('usage: grid [-h] [--remote | --local] {models,engines,stats,info} ...\\ngrid: error: unrecognized arguments: --no-wake\\n');process.exit(2);}
+const verb=argv.find(a=>!a.startsWith('-'))||'';const turn=plan.verbs[verb];
+if(!turn){process.stdout.write('[]');process.exit(0);}
+if(turn.stdout!==undefined)process.stdout.write(typeof turn.stdout==='string'?turn.stdout:JSON.stringify(turn.stdout));
+if(turn.stderr)process.stderr.write(turn.stderr);process.exit(turn.exit??0);
+`, { mode: 0o755 });
+ return file;
+}
+const awakeVerbs = {
+ use:{stdout:{mode:'remote',active:'test-grid'}}, ls:{stdout:[{grid:'test-grid',type:'permissioned-public'}]},
+ info:{stdout:{grid:'test-grid',type:'permissioned-public',status:'running',grid_url:'https://relay.example'}},
+ engines:{stdout:remoteNodes}, models:{stdout:reads.models.value}, stats:{stdout:reads.stats.value}, 'device-info':{stdout:device},
+};
+// One line per call, APPENDED: the viewer runs several \`grid\` reads at once, and a read-modify-write of
+// one JSON document crashed a fake that caught it half written — which read as the grid being down.
+const logged = async log => (await readFile(log,'utf8').catch(()=>'')).split('\n').filter(Boolean).map(line=>JSON.parse(line));
+const verbOf = argv => argv.find(a=>!a.startsWith('-'));
+
+test('gridJson carries the refusal code a program branches on, and tells an old grid from a failure',async t=>{
+ const dir=await temporary(t),log=join(dir,'calls.json');
+ const asleep=await fakeGrid(dir,'asleep',{verbs:{engines:{stderr:`${asleepEnvelope}\nGrid test-grid is asleep: this grid is resting\n`,exit:1}}},log);
+ const result=await gridJson({transport:'local',gridBinary:asleep},'remote',['engines','test-grid','--no-wake']);
+ assert.equal(result.ok,false);assert.equal(result.refusal,'grid_asleep');assert.equal(result.outdated,false);
+ // A codeless refusal and a master that is down keep today's shape: no code a caller could act on.
+ const down=await fakeGrid(dir,'down',{verbs:{engines:{stderr:'{"error":{"code":"grid_master_down","message":"down","status":503}}\n',exit:1}}},log);
+ assert.equal((await gridJson({transport:'local',gridBinary:down},'remote',['engines','x'])).refusal,'grid_master_down');
+ const old=await fakeGrid(dir,'old',{rejectsNoWake:true},log);
+ const refused=await gridJson({transport:'local',gridBinary:old},'remote',['engines','test-grid','--no-wake']);
+ assert.equal(refused.ok,false);assert.equal(refused.outdated,true);assert.equal(refused.refusal,null);
+});
+
+test('a grid too old for --no-wake leaves the last reading, stale, and is never asked again without the flag',async t=>{
+ const dir=await temporary(t),log=join(dir,'calls.json');
+ const old=await fakeGrid(dir,'grid-0.3.47',{rejectsNoWake:true,verbs:awakeVerbs},log);
+ const current=await fakeGrid(dir,'grid-0.3.49',{verbs:awakeVerbs},log);
+ const workspace=join(dir,'ws');
+ await atomicJson(join(workspace,'grid-fleet.json'),config);
+ // An earlier reading on disk, as a viewer that ran yesterday leaves it.
+ const earlier=assemble(config,reads,null,'2026-09-24T08:05:00Z');
+ await atomicJson(join(stateDir(workspace),'snapshot.json'),earlier);
+ const saved=process.env.HARNESS_GRID_BIN;t.after(()=>{if(saved===undefined)delete process.env.HARNESS_GRID_BIN;else process.env.HARNESS_GRID_BIN=saved;});
+ const collect=createCollector(workspace);
+
+ process.env.HARNESS_GRID_BIN=old;
+ const first=await collect();
+ assert.equal(first.status,'updating');
+ assert.equal(first.nodes.length,earlier.nodes.length);assert.ok(first.nodes.every(n=>n.stale));
+ assert.match(first.notice,/^Test workstation is being updated — showing the reading from \d{2}:\d{2}$/);
+ assert.equal(first.readingFrom,'2026-09-24T08:05:00Z');
+ assert.ok(!first.events.some(e=>e.kind==='offline'));
+ const firstCalls=(await logged(log)).filter(c=>READS.includes(verbOf(c.argv)));
+ assert.deepEqual(firstCalls.map(c=>verbOf(c.argv)),['engines'],'one refused read, then nothing — no retry and no models/stats');
+
+ process.env.HARNESS_GRID_BIN=current;
+ const second=await collect();
+ assert.equal(second.status,'live');assert.equal(second.summary.enginesOnline,7);assert.equal(second.notice,undefined);
+ const calls=(await logged(log)).filter(c=>READS.includes(verbOf(c.argv)));
+ assert.deepEqual(calls.filter(c=>c.binary==='grid-0.3.49').map(c=>verbOf(c.argv)).sort(),['engines','models','stats']);
+ for(const call of calls)assert.ok(call.argv.includes('--no-wake'),`${call.binary} ran ${call.argv.join(' ')} without --no-wake`);
+});
+
+test('the own grid asleep is read from its status alone: nothing else is asked, and nothing it served is called gone',async t=>{
+ const dir=await temporary(t);await atomicJson(join(dir,'grid-fleet.json'),config);
+ const calls=[];let status='running';
+ const collect=createCollector(dir,{runJson:async(_m,_mode,args)=>{calls.push(args);
+ if(args[0]==='info')return {ok:true,value:{grid:'test-grid',status,grid_url:'https://relay.example'}};
+ if(args[0]==='device-info')return {ok:true,value:device};if(args[0]==='use')return {ok:true,value:{active:'test-grid'}};
+ return reads[args[0]]||{ok:true,value:[]};}});
+ const awake=await collect();assert.equal(awake.status,'live');
+ status='asleep';calls.length=0;
+ const asleep=await collect();
+ assert.deepEqual(calls.filter(a=>READS.includes(a[0])),[],'no engines/models/stats while the status says asleep');
+ assert.equal(asleep.status,'asleep');assert.equal(asleep.pollIntervalMs,30_000);
+ assert.equal(asleep.nodes.length,awake.nodes.length);assert.ok(asleep.nodes.every(n=>n.stale));
+ assert.ok(!asleep.events.some(e=>e.kind==='offline'),'a resting grid did not lose its engines');
+ assert.equal(asleep.readingFrom,awake.observedAt);
+ const verdict=JSON.parse(await readFile(join(dir,'.harness/verdict.json'),'utf8'));
+ assert.equal(verdict.ready,true);assert.doesNotMatch(verdict.summary,/grid/i);
+});
+
+test('the first reading after a sleep does not call a node that has not rejoined yet gone',async t=>{
+ const dir=await temporary(t);await atomicJson(join(dir,'grid-fleet.json'),config);
+ let status='running',engines=reads.engines;
+ const collect=createCollector(dir,{runJson:async(_m,_mode,args)=>{
+ if(args[0]==='info')return {ok:true,value:{grid:'test-grid',status,grid_url:'https://relay.example'}};
+ if(args[0]==='device-info')return {ok:true,value:device};if(args[0]==='use')return {ok:true,value:{active:'test-grid'}};
+ if(args[0]==='engines')return engines;
+ return reads[args[0]]||{ok:true,value:[]};}});
+ const awake=await collect();assert.ok(awake.nodes.length>1);
+ status='asleep';await collect();
+ // Woken, and only one engine has heard of it so far.
+ status='running';engines={ok:true,value:remoteNodes.slice(0,1)};
+ const woken=await collect();
+ assert.equal(woken.status,'live');
+ assert.ok(!woken.events.some(e=>e.kind==='offline'),'a cold wake is not every other engine leaving');
+});
+
+test('a member grid is asleep when engines answers grid_asleep, and models and stats are skipped',async t=>{
+ const dir=await temporary(t),log=join(dir,'calls.json');
+ const verbs={...awakeVerbs,info:{stdout:{grid:'test-grid',type:'domain-restricted',status:null,grid_url:'https://relay.example'}},
+ engines:{stderr:`${asleepEnvelope}\nGrid test-grid is asleep: this grid is resting\n`,exit:1}};
+ const binary=await fakeGrid(dir,'grid-0.3.49',{verbs},log);
+ const workspace=join(dir,'ws');await atomicJson(join(workspace,'grid-fleet.json'),{...config,machines:[{...config.machines[0],gridBinary:binary}]});
+ const earlier=assemble(config,reads,null,'2026-09-24T08:05:00Z');
+ await atomicJson(join(stateDir(workspace),'snapshot.json'),earlier);
+ const snapshot=await createCollector(workspace)();
+ assert.equal(snapshot.status,'asleep');assert.equal(snapshot.pollIntervalMs,30_000);
+ assert.ok(snapshot.nodes.length>0&&snapshot.nodes.every(n=>n.stale));
+ assert.ok(!snapshot.events.some(e=>e.kind==='offline'));
+ assert.equal(snapshot.sources.engines?.ok,true,'asleep is an answer, not a failed read');
+ const read=(await logged(log)).map(c=>c.argv).filter(a=>READS.includes(verbOf(a)));
+ assert.deepEqual(read,[['--remote','engines','test-grid','--no-wake','--json']]);
+});
+
+test('the viewer books its next read 30 s out while the grid is asleep, and at its own interval otherwise',async t=>{
+ const workspace=await temporary(t);
+ let asleep=false;const booked=[];
+ const collect=async()=>({...assemble(config,reads),status:asleep?'asleep':'live',pollIntervalMs:asleep?30_000:50,machines:[],operations:[]});
+ const timer=(fn,ms)=>{booked.push(ms);return setTimeout(fn,Math.min(ms,20));};
+ const viewer=createViewer({workspace,intervalMs:50,collect,timer});
+ const port=await viewer.start();t.after(()=>viewer.close());
+ const abort=new AbortController();t.after(()=>abort.abort());
+ const events=await fetch(`http://127.0.0.1:${port}/events`,{signal:abort.signal});const reader=events.body.getReader();await reader.read();
+ await new Promise(r=>setTimeout(r,60));
+ assert.ok(booked.length>0&&booked.every(ms=>ms===50),`live: ${booked}`);
+ asleep=true;booked.length=0;
+ await new Promise(r=>setTimeout(r,80));
+ assert.ok(booked.includes(30_000),`asleep: ${booked}`);
+});
+
+/** stream.js in a sandbox with a fake document and EventSource — the shipped file, not a copy. */
+async function loadStream() {
+ const code=await readFile(join(PACKAGE,'viewer','stream.js'),'utf8');
+ const listeners={},opened=[];
+ const doc={hidden:false,addEventListener:(type,fn)=>{(listeners[type]||=[]).push(fn);}};
+ class FakeEventSource{constructor(url){this.url=url;this.closed=false;this.handlers={};opened.push(this);}addEventListener(type,fn){this.handlers[type]=fn;}close(){this.closed=true;}}
+ const sandbox={};runInNewContext(code,sandbox);
+ return {api:sandbox.harnessViewerStream,doc,opened,FakeEventSource,flip:hidden=>{doc.hidden=hidden;for(const fn of listeners.visibilitychange||[])fn();}};
+}
+
+test('a hidden page closes its event stream and reopens it when shown again',async()=>{
+ const {api,doc,opened,FakeEventSource,flip}=await loadStream();
+ const received=[];
+ const live=api.liveStream({doc,open:url=>new FakeEventSource(url),onSnapshot:e=>received.push(e),onError:()=>{}});
+ assert.equal(opened.length,1);assert.equal(opened[0].url,'events');assert.equal(live.connected(),true);
+ flip(true);assert.equal(opened[0].closed,true);assert.equal(live.connected(),false);
+ flip(true);assert.equal(opened.length,1,'hidden twice is still one close, no reopen');
+ flip(false);assert.equal(opened.length,2);assert.equal(opened[1].closed,false);assert.equal(live.connected(),true);
+ opened[1].handlers.snapshot({data:'{}'});assert.equal(received.length,1);
+ // A page that loads hidden (a background tab) does not open a stream until it is looked at.
+ const late=await loadStream();late.doc.hidden=true;
+ late.api.liveStream({doc:late.doc,open:url=>new late.FakeEventSource(url),onSnapshot:()=>{},onError:()=>{}});
+ assert.equal(late.opened.length,0);late.flip(false);assert.equal(late.opened.length,1);
+});
+
+test('the viewer serves stream.js to the page that loads it',async t=>{
+ const workspace=await temporary(t);
+ const viewer=createViewer({workspace,intervalMs:1000,collect:async()=>({...assemble(config,reads),machines:[],operations:[]})});
+ const port=await viewer.start();t.after(()=>viewer.close());
+ const page=await (await fetch(`http://127.0.0.1:${port}/`)).text();
+ assert.ok(page.indexOf('src="stream.js"')>=0&&page.indexOf('src="stream.js"')<page.indexOf('src="app.js"'));
+ assert.equal((await fetch(`http://127.0.0.1:${port}/stream.js`)).status,200);
+});
+
+test('a fresh workspace probes with --no-wake and takes a sleeping own grid as an answer',async t=>{
+ const dir=await temporary(t);await atomicJson(join(dir,'grid-fleet.json'),{...config,grid:null,mode:'remote'});
+ const calls=[];
+ const runJson=async(_host,mode,args)=>{calls.push([mode,...args]);
+ if(args[0]==='engines')return {ok:false,error:'grid engines failed (1): asleep',refusal:'grid_asleep',outdated:false};
+ return {ok:true,value:args[0]==='mode'?{mode:'remote'}:args[0]==='use'?{mode:'remote',active:null}:args[0]==='ls'&&mode==='remote'?[{grid:'tuan-dev-991371e4',type:'permissioned-public'}]:[]};};
+ const {config:result}=await initializeWorkspace(dir,{profilePath:join(dir,'defaults.json'),discover:async()=>[],runJson,select:async(_h,_m,grid)=>({ok:true,active:grid}),email:async()=>'[email protected]',env:{}});
+ assert.equal(result.grid,'tuan-dev-991371e4');
+ const probes=calls.filter(c=>c[1]==='engines');
+ assert.ok(probes.length>0);for(const probe of probes)assert.ok(probe.includes('--no-wake'),probe.join(' '));
+});
diff --git a/store/agents/autonomous-grid/viewer.mjs b/store/agents/autonomous-grid/viewer.mjs
index 94d939081..44e639efa 100644
--- a/store/agents/autonomous-grid/viewer.mjs
+++ b/store/agents/autonomous-grid/viewer.mjs
@@ -5,20 +5,22 @@ import { pathToFileURL } from 'node:url';
import { atomicJson, gridSelect, now, operations, PACKAGE, readConfig } from './lib/fleet.mjs';
import { createCollector } from './lib/telemetry.mjs';
-export function createViewer({ workspace, port = 0, intervalMs = 8000, collect = createCollector(workspace, { intervalMs }), select = gridSelect }) {
+export function createViewer({ workspace, port = 0, intervalMs = 8000, collect = createCollector(workspace, { intervalMs }), select = gridSelect, timer = setTimeout }) {
const clients = new Set();
let snapshot = { spec: 1, status: 'connecting', grid: 'Your grid', nodes: [], machines: [], models: [], events: [], operations: [], history: {}, sources: {}, summary: {}, observedAt: null, pollIntervalMs: intervalMs };
let stopped = false, pollTimer, opTimer, heartbeat, inFlight = false, lastPollAt = 0;
// The next tick is booked only while a pane is connected: a viewer nobody is looking at must not keep
// running the grid CLI every few seconds. A request for the observation still wakes one read if the
- // one on hand has gone stale.
+ // one on hand has gone stale. The observation names its own pace — a sleeping grid is looked at every
+ // 30 s rather than every few (lib/telemetry.mjs) — and a stale-read request honours the same pace.
+ const pace = () => snapshot.pollIntervalMs || intervalMs;
const schedule = () => {
clearTimeout(pollTimer); pollTimer = null;
if (stopped || clients.size === 0) return;
- pollTimer = setTimeout(poll, intervalMs);
+ pollTimer = timer(poll, pace());
};
const wake = () => {
- if (stopped || inFlight || Date.now() - lastPollAt < intervalMs) return;
+ if (stopped || inFlight || Date.now() - lastPollAt < pace()) return;
clearTimeout(pollTimer); pollTimer = null;
void poll();
};
@@ -63,7 +65,7 @@ export function createViewer({ workspace, port = 0, intervalMs = 8000, collect =
if (!pollTimer && !inFlight) schedule();
return;
}
- const assets = { '/': ['index.html', 'text/html; charset=utf-8'], '/app.js': ['app.js', 'text/javascript; charset=utf-8'], '/app.css': ['app.css', 'text/css; charset=utf-8'] };
+ const assets = { '/': ['index.html', 'text/html; charset=utf-8'], '/stream.js': ['stream.js', 'text/javascript; charset=utf-8'], '/app.js': ['app.js', 'text/javascript; charset=utf-8'], '/app.css': ['app.css', 'text/css; charset=utf-8'] };
if (!assets[url.pathname]) { json(res, 404, { error: 'Not found' }); return; }
const [file, contentType] = assets[url.pathname];
const content = await readFile(join(PACKAGE, 'viewer', file));
diff --git a/store/agents/autonomous-grid/viewer/app.js b/store/agents/autonomous-grid/viewer/app.js
index cc71b739d..c21f7d5a8 100644
--- a/store/agents/autonomous-grid/viewer/app.js
+++ b/store/agents/autonomous-grid/viewer/app.js
@@ -22,13 +22,15 @@
if (!snapshot) return;
const late = lastReceived && Date.now() - lastReceived > Math.max(30_000, (snapshot.pollIntervalMs || 8000)*3);
const state = transportLost || late ? 'disconnected' : snapshot.status;
- const labels={live:'Grid connected',partial:'Partial telemetry',unavailable:'Grid unavailable',unconfigured:'Choose a grid',connecting:'Connecting',disconnected:'Connection lost'};
+ const labels={live:'Grid connected',partial:'Partial telemetry',unavailable:'Grid unavailable',unconfigured:'Choose a grid',connecting:'Connecting',disconnected:'Connection lost',asleep:'Asleep',updating:'Showing an earlier reading'};
// The connection state rides on the grid dropdown's title: the header that carried a dot and a
// label is gone, and the dropdown is where the grid is named now.
$('grid-select').title=labels[state] || 'Waiting for Grid';
// The footer that used to say "ask Grid to reconnect" is gone (it cost a row of height for a
// sentence nobody needed while things worked); the activity heading says it when it matters.
- $('observed').textContent=['unavailable','disconnected'].includes(state)?`${labels[state]} · ask Grid to reconnect`:snapshot.observedAt?`Observed ${age(snapshot.observedAt)}`:'Waiting for Grid';
+ // A resting grid, or a reading held because this computer's grid tool is being updated, says so in
+ // the viewer's own sentence (lib/telemetry.mjs) instead of an age that would read as a live one.
+ $('observed').textContent=['unavailable','disconnected'].includes(state)?`${labels[state]} · ask Grid to reconnect`:snapshot.notice?snapshot.notice:snapshot.observedAt?`Observed ${age(snapshot.observedAt)}`:'Waiting for Grid';
}
function consume(data) {
if (!data || data.spec!==1 || !Array.isArray(data.nodes)) return;
@@ -42,7 +44,10 @@
function render() {
if (!snapshot) return;
renderGridSelect();
- const available=!['connecting','unavailable','unconfigured'].includes(snapshot.status);
+ // Held (asleep, or an earlier reading kept on screen): every node is a last-known one, so the
+ // totals would count nothing live. A dash says "not measured now" where a zero would say "none".
+ const held=['asleep','updating'].includes(snapshot.status);
+ const available=!held&&!['connecting','unavailable','unconfigured'].includes(snapshot.status);
$('total-engines').textContent=available?fmt(snapshot.summary.enginesOnline,0):'—';
$('total-known').textContent=snapshot.nodes.length?`${snapshot.nodes.length} known to this grid`:'waiting for engines';
$('total-models').textContent=available?fmt(snapshot.summary.modelsServing,0):'—';
@@ -54,8 +59,8 @@
$('map-subtitle').textContent=snapshot.nodes.length>12?`12 of ${snapshot.nodes.length} engines · see every engine in Rack`:'Every machine has a place.';
$('map-empty').hidden=graphNodes.length>0;
$('hub').hidden=graphNodes.length===0;
- $('empty-message').textContent=snapshot.status==='unconfigured'?'Ask the Model Manager to connect one of your existing grids, or help you create your first fleet.':snapshot.status==='unavailable'?'Grid isn’t reachable yet. Ask the agent to check your connection, choose a grid, or start one.':'Ask the Model Manager to discover this machine and deploy your first model.';
- $('hub-status').textContent=available?`${snapshot.summary.enginesOnline || 0} engines online`:'awaiting connection';
+ $('empty-message').textContent=snapshot.status==='asleep'?'Resting to save resources. It starts by itself when you send a message.':snapshot.status==='unconfigured'?'Ask the Model Manager to connect one of your existing grids, or help you create your first fleet.':snapshot.status==='unavailable'?'Grid isn’t reachable yet. Ask the agent to check your connection, choose a grid, or start one.':'Ask the Model Manager to discover this machine and deploy your first model.';
+ $('hub-status').textContent=snapshot.status==='asleep'?'asleep':snapshot.status==='updating'?'earlier reading':available?`${snapshot.summary.enginesOnline || 0} engines online`:'awaiting connection';
const ids=new Set(graphNodes.map(n=>n.id));
for (const [id,b] of nodeButtons) if (!ids.has(id)) { b.remove();nodeButtons.delete(id); }
for (const n of graphNodes) {
@@ -278,10 +283,15 @@
else if(copy){try{await navigator.clipboard.writeText(copy.dataset.prompt);toast('Request copied. Paste it into your Model Manager.');}catch{toast('Select the request text and paste it into your Model Manager.');}}
});
new ResizeObserver(()=>resize()).observe(stage);
- const stream=new EventSource('events');
- stream.addEventListener('snapshot',event=>{try{consume(JSON.parse(event.data));}catch{transportLost=true;status();}});
- stream.onerror=()=>{transportLost=true;status();};
+ // Open only while the page is visible (stream.js): a hidden pane that kept listening kept the viewer
+ // reading for nobody.
+ window.harnessViewerStream.liveStream({
+ doc:document,open:url=>new EventSource(url),
+ onSnapshot:event=>{try{consume(JSON.parse(event.data));}catch{transportLost=true;status();}},
+ onError:()=>{transportLost=true;status();},
+ });
const fetchSnapshot=()=>fetch('api/snapshot',{signal:AbortSignal.timeout(12_000)}).then(r=>{if(!r.ok)throw new Error();return r.json();}).then(consume).catch(()=>{transportLost=true;status();});
- fetchSnapshot();setInterval(()=>{status();if(transportLost)fetchSnapshot();},8000);
+ // The fallback read asks for a fresh observation too, so it waits for the page to be seen as well.
+ fetchSnapshot();setInterval(()=>{status();if(transportLost&&!document.hidden)fetchSnapshot();},8000);
requestAnimationFrame(frame);
})();
diff --git a/store/agents/autonomous-grid/viewer/index.html b/store/agents/autonomous-grid/viewer/index.html
index 564ea4485..e0074013c 100644
--- a/store/agents/autonomous-grid/viewer/index.html
+++ b/store/agents/autonomous-grid/viewer/index.html
@@ -6,6 +6,7 @@
<meta name="color-scheme" content="dark">
<title>Model Manager</title>
<link rel="stylesheet" href="app.css">
+ <script src="stream.js" defer></script>
<script src="app.js" defer></script>
</head>
<body>
diff --git a/store/agents/autonomous-grid/viewer/stream.js b/store/agents/autonomous-grid/viewer/stream.js
new file mode 100644
index 000000000..2ee9279ac
--- /dev/null
+++ b/store/agents/autonomous-grid/viewer/stream.js
@@ -0,0 +1,27 @@
+// The page's one event stream, open only while the page can be seen.
+//
+// The viewer books its next read only while a client is listening (viewer.mjs), so a stream left open
+// by a pane nobody is looking at — a background tab, a hidden desktop pane — kept it reading every few
+// seconds for no one. Closed on hide, the last listener is gone and the reads stop; reopened on show,
+// the server answers with its snapshot at once and reads again if that one has gone stale.
+//
+// A file of its own, loaded before app.js, so a test can run exactly this code against a stand-in
+// document without a browser (test/nowake.test.mjs).
+(root => {
+ 'use strict';
+ function liveStream({ doc, open, onSnapshot, onError }) {
+ let source = null;
+ const connect = () => {
+ if (source || doc.hidden) return;
+ source = open('events');
+ source.addEventListener('snapshot', onSnapshot);
+ source.onerror = onError;
+ };
+ // `close()` fires no error event, so a deliberate close never reads as a lost connection.
+ const disconnect = () => { if (!source) return; source.close(); source = null; };
+ doc.addEventListener('visibilitychange', () => { if (doc.hidden) disconnect(); else connect(); });
+ connect();
+ return { connected: () => source !== null, close: disconnect };
+ }
+ root.harnessViewerStream = { liveStream };
+})(globalThis);