Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Models: a grid that is down runs nothing, and Start brings it back up

bitcoinbuilderd6e7d82b4afc
diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.ts
index 89ee6500e..4d25a1968 100644
--- a/cli/src/lib/localModels.spec.ts
+++ b/cli/src/lib/localModels.spec.ts
@@ -18,16 +18,24 @@ let calls: string[][], serving: boolean, catalogCards: ReturnType<typeof card>[]
let request: Mock<(url: string | URL, init?: RequestInit) => Promise<Response>>
let run: Mock<(args: string[], output?: (s: string) => void) => Promise<GridFleetResult>>
let downloadFails: boolean, catalogFails: boolean
+/** What `grid info` says of the grid. Down (`stopped`/`asleep`) refuses `engines` and `join`, as
+ * the real grid does, until `grid start` brings it back. */
+let gridState: 'running' | 'stopped' | 'asleep'
+const refused = (message: string) => ({ ok: false, code: 1, stdout: '', stderr: message, error: 'Grid command failed.' })
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'local-models-'))
home = join(root, 'grid'); stateDir = join(root, 'receipts'); records = join(home, 'run', 'engines', 'grid-home')
await mkdir(records, { recursive: true }); await mkdir(join(home, 'models'))
await writeFile(join(home, 'credentials.toml'), 'session_token = "test-only-token"\napi_url = "https://catalog.example.test"\n')
- calls = []; serving = false; catalogCards = [card()]; downloadFails = false; catalogFails = false
+ calls = []; serving = false; catalogCards = [card()]; downloadFails = false; catalogFails = false; gridState = 'running'
run = vi.fn(async (args: string[], output?: (s: string) => void) => {
calls.push(args)
if (args[0] === 'device-info') return ok({ device_class: 'apple-silicon', backend: 'metal', usable_bytes: 54 * 1024 ** 3, memory: { total_gb: 64 } })
if (args.includes('ls')) return ok([{ grid: 'home', id: 'grid-home' }])
+ if (args.includes('info') && args.includes('--json')) return ok({ grid: 'home', status: gridState })
+ if (args[1] === 'start') { gridState = 'running'; return ok() }
+ const down = gridState !== 'running'
+ if (down && (args.includes('engines') || args.includes('join'))) return refused("Grid home isn't up; run `grid start home` first.")
if (args.includes('engines')) return ok(serving ? [{ node_id: 'local-node', online: true, models: ['Small-Q4.gguf'], throughput_tok_s: 17.6,
vram_used_mb: 99999, answered: { window_seconds: 3600, requests: 4, by_model: [{ model: 'Small-Q4.gguf', requests: 4 }] } }] : [])
if (args[0] === 'pull') {
@@ -83,7 +91,7 @@ describe('local model discovery and lifecycle', () => {
request.mockResolvedValue(response({ models: [card()], runnable_total: 100, pagination: { page: 1, total_pages: 7 } }))
// Fresh Response objects (bodies are single-consumption).
request.mockImplementation(async () => response({ models: [card()], runnable_total: 100, pagination: { page: 1, total_pages: 7 } }))
- expect((await service.list('home')).error).toContain('incomplete')
+ expect((await service.list('home')).notice).toContain('incomplete')
})
it('downloads, loads and verifies without another user step', async () => {
@@ -178,7 +186,7 @@ describe('local model discovery and lifecycle', () => {
const fresh = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch })
const snapshot = await fresh.list('home')
expect(snapshot.models[0]).toMatchObject({ state: 'running', canStop: true })
- expect(snapshot.error).toContain('unavailable')
+ expect(snapshot.notice).toContain('unavailable')
expect(JSON.stringify(snapshot)).not.toContain('private raw')
})
@@ -318,7 +326,7 @@ describe('local model discovery and lifecycle', () => {
expect(calls).toHaveLength(before)
await service.list('home', true)
expect(calls.filter(args => args[0] === 'device-info')).toHaveLength(2)
- expect(await service.list(null)).toMatchObject({ models: [], busy: false, error: expect.stringContaining('Sign in') })
+ expect(await service.list(null)).toMatchObject({ models: [], busy: false, notice: expect.stringContaining('Sign in') })
expect((await service.act(null, 'x', 'start')).error).toBeTruthy()
expect((await service.act('home', {}, 'start')).error).toBeTruthy()
})
@@ -329,19 +337,19 @@ describe('local model discovery and lifecycle', () => {
['malformed token', 'session_token = "invalid\\q"'],
])('reports sign-in for %s without sending credentials', async (_name, credentials) => {
await writeFile(join(home, 'credentials.toml'), credentials)
- expect((await service.list('home')).error).toContain('Sign in')
+ expect((await service.list('home')).notice).toContain('Sign in')
expect(request).not.toHaveBeenCalled()
})
it.each(['http://catalog.example.test', 'file:///tmp/catalog'])('refuses insecure catalog address %s', async base => {
await writeFile(join(home, 'credentials.toml'), `session_token = 'private'\napi_url = '${base}'`)
- expect((await service.list('home')).error).toContain('address')
+ expect((await service.list('home')).notice).toContain('address')
expect(request).not.toHaveBeenCalled()
})
it.each(['http://localhost:1234', 'http://127.0.0.1:1234', 'http://[::1]:1234'])('permits an explicitly configured local catalog at %s', async base => {
await writeFile(join(home, 'credentials.toml'), `session_token = 'test-only-token'\napi_url = '${base}'`)
- expect((await service.list('home')).error).toBeUndefined()
+ expect((await service.list('home')).notice).toBeUndefined()
expect(String(request.mock.calls[0][0])).toBe(`${base}/v1/grid/catalog`)
expect(request.mock.calls[0][1]).toMatchObject({ redirect: 'error' })
})
@@ -362,7 +370,7 @@ describe('local model discovery and lifecycle', () => {
})
const snapshot = await service.list('home')
expect(snapshot.models).toEqual([])
- expect(snapshot.error).toBeTruthy()
+ expect(snapshot.notice).toBeTruthy()
expect(JSON.stringify(snapshot)).not.toContain('private detail')
expect(request.mock.calls.length).toBeLessThanOrEqual(100)
})
@@ -383,10 +391,61 @@ describe('local model discovery and lifecycle', () => {
run.mockImplementation(async (args, output) => args.includes('engines')
? { ...ok(), ok: scenario !== 'failed command', stdout: 'not json' } : original(args, output))
const snapshot = await service.list('home', true)
- expect(snapshot.error).toContain('Running models could not be checked')
+ expect(snapshot.notice).toContain('Running models could not be checked')
expect(snapshot.models.every(m => !m.canStart && !m.canStop)).toBe(true)
})
+ // ── a grid that is down ──────────────────────────────────────────────────────────────────────
+ //
+ // ⚠️ REGRESSION. Grid refuses `engines` on a grid that is not up, and that refusal was read as
+ // "running models unknown": every Start in the list went dark, so the one state in which a person
+ // most needs to start a model was the one in which none could be. And a Start that did get through
+ // failed at `join`, which a down grid refuses too.
+
+ it.each(['stopped', 'asleep'] as const)('a %s grid runs nothing: every model can start, and nothing is wrong', async state => {
+ gridState = state
+ const snapshot = await service.list('home')
+ expect(snapshot.models).toMatchObject([{ name: 'Small', state: 'available', canStart: true, canStop: false }])
+ expect(snapshot.notice).toBeUndefined()
+ // Never `error`: on this protocol that field fails the request and the app keeps no list.
+ expect(snapshot).not.toHaveProperty('error')
+ })
+
+ it('a refusal from a grid that IS running is still a gap, not an empty list', async () => {
+ run.mockImplementation(async args => args.includes('engines') ? refused('private detail')
+ : args.includes('info') ? ok({ status: 'running' }) : ok([{ grid: 'home', id: 'grid-home' }]))
+ await service.list('home', true)
+ const snapshot = await service.list('home', true)
+ expect(snapshot.notice).toContain('Running models could not be checked')
+ expect(JSON.stringify(snapshot)).not.toContain('private detail')
+ })
+
+ it.each(['stopped', 'asleep'] as const)('Start brings a %s grid up before joining it', async state => {
+ gridState = state
+ await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
+ const started = calls.findIndex(args => args[0] === '--remote' && args[1] === 'start')
+ const joined = calls.findIndex(args => args.includes('join'))
+ expect(calls[started]).toEqual(['--remote', 'start', 'home'])
+ expect(started).toBeGreaterThanOrEqual(0)
+ expect(started).toBeLessThan(joined)
+ expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'running', operation: { phase: 'done' } })
+ })
+
+ it('Start leaves a running grid alone', async () => {
+ await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
+ expect(calls.some(args => args[0] === '--remote' && args[1] === 'start')).toBe(false)
+ expect((await service.list('home', true)).models[0].state).toBe('running')
+ })
+
+ it('a grid that will not come up fails the Start in words, before any join', async () => {
+ gridState = 'stopped'
+ const original = run.getMockImplementation()!
+ run.mockImplementation(async (args, output) => args[1] === 'start' ? refused('private detail') : original(args, output))
+ await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
+ expect(calls.some(args => args.includes('join'))).toBe(false)
+ expect((await service.list('home')).models[0].operation).toMatchObject({ phase: 'failed', error: 'Your grid could not start. Try again.' })
+ })
+
it.each(['missing grid', 'invalid grid id', 'missing record directory', 'corrupt record'])('ignores %s without exposing a Stop action', async scenario => {
const original = run.getMockImplementation()!
if (scenario === 'missing grid' || scenario === 'invalid grid id') {
diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.ts
index afc3cc8f1..34a105bb6 100644
--- a/cli/src/lib/localModels.ts
+++ b/cli/src/lib/localModels.ts
@@ -18,6 +18,9 @@ const cleanName = (v: string): string => basename(v).replace(/\.gguf$/i, '').rep
// Grid advertises a GGUF filename as a lowercase model name without its extension.
const modelKey = (v: string): string => v.toLowerCase().replace(/\.gguf$/, '')
const validArg = (v: string): boolean => !!v && !v.startsWith('-') && !/[\x00-\x1f]/.test(v)
+/** `grid info` statuses of a grid that is not up. Grid refuses `engines` and `join` on one, and
+ * names `grid start` as the way back. Anything else unknown stays unknown rather than "down". */
+const DOWN = new Set(['stopped', 'asleep'])
export interface LocalModel {
id: string; name: string; state: 'available' | 'downloaded' | 'running'
@@ -31,7 +34,11 @@ export interface ModelOperation {
progress?: number; error?: string; updatedAt: string
}
export interface LocalModelsSnapshot {
- models: LocalModel[]; memoryBytes?: number; hardware?: string; error?: string
+ models: LocalModel[]; memoryBytes?: number; hardware?: string
+ /** A sentence to show BESIDE the list — never `error`. On this protocol an `error` field fails
+ * the whole request: the app's RPC layer drops the reply and keeps nothing of it, so a list sent
+ * with a warning in `error` arrived as no list at all and the Local tab read 0. */
+ notice?: string
observedAt: string; busy: boolean
}
interface Candidate {
@@ -272,7 +279,7 @@ export class LocalModels {
}
async list(grid: string | null, force = false): Promise<LocalModelsSnapshot> {
- if (!grid) return { models: [], error: 'Sign in to find models for this computer.', observedAt: new Date().toISOString(), busy: false }
+ if (!grid) return { models: [], notice: 'Sign in to find models for this computer.', observedAt: new Date().toISOString(), busy: false }
if (!force && this.cached?.grid === grid && Date.now() - this.cached.at < 2500) return this.cached.value
if (this.listPending) {
if (this.listGrid === grid) return this.listPending
@@ -286,7 +293,7 @@ export class LocalModels {
await Promise.all([this.loadCatalog(force), this.readReceipt(grid)])
let owned: Owned[] = [], nodes: Record<string, any>[] = [], inventoryError: string | undefined
try {
- [owned, nodes] = await Promise.all([this.owned(grid), this.json(['--remote', 'engines', grid, '--json']).then(rows)])
+ [owned, nodes] = await Promise.all([this.owned(grid), this.servingNodes(grid)])
} catch { inventoryError = 'Running models could not be checked. Try again.' }
const operation = this.active?.grid === grid ? this.active.operation : this.receipt?.grid === grid ? this.receipt.operation : undefined
const choices = [...this.candidates, ...(await this.known(grid, owned)).filter(k => !this.candidates.some(c => c.file === k.file))]
@@ -328,12 +335,34 @@ export class LocalModels {
operation: operation?.modelId === `local:${instance.file}` ? operation : undefined })
}
const value: LocalModelsSnapshot = { models, memoryBytes: num(obj(this.device.memory).total_gb) === undefined ? undefined : this.device.memory.total_gb * GiB,
- hardware: str(obj(this.device.machine).model || obj(this.device.cpu).brand), error: inventoryError || this.catalogError,
+ hardware: str(obj(this.device.machine).model || obj(this.device.cpu).brand), notice: inventoryError || this.catalogError,
observedAt: new Date().toISOString(), busy: !!this.active }
this.cached = { grid, at: Date.now(), value }
return value
}
+ /** What `grid info` says of [grid]: `running`, `stopped`, `asleep`. */
+ private async gridStatus(grid: string): Promise<string> {
+ return str(obj(await this.json(['--remote', 'info', grid, '--json'])).status)
+ }
+
+ /** The nodes serving [grid] — none at all when the grid is down.
+ *
+ * ⚠️ A stopped or asleep grid REFUSES `engines` ("isn't up; run `grid start` first") rather than
+ * answering an empty list, and reading that refusal as "unknown" disabled every Start in the
+ * list: the one state in which a person most needs to start a model was the one in which none
+ * could be. Stopping the last model from this list leaves the grid up, so it was only reached by
+ * a grid taken down by hand or gone to sleep — but then there was no way back from here. A grid
+ * that is down runs nothing, which is an answer, not a gap. Any other refusal still is one. */
+ private async servingNodes(grid: string): Promise<Record<string, any>[]> {
+ const result = await this.run(['--remote', 'engines', grid, '--json'])
+ if (result.ok) {
+ try { return rows(JSON.parse(result.stdout)) } catch { throw new Error('Running models could not be checked. Try again.') }
+ }
+ if (DOWN.has(await this.gridStatus(grid))) return []
+ throw new Error('Running models could not be checked. Try again.')
+ }
+
/** A repeated click or lost RPC acknowledgement joins the same operation.
* A different model waits: starts/stops must not race on the machine's budget. */
async act(grid: string | null, modelId: unknown, action: 'start' | 'stop'): Promise<{ operation?: ModelOperation; error?: string }> {
@@ -412,6 +441,11 @@ export class LocalModels {
if (!await this.downloaded(candidate)) throw new ModelError('The download is incomplete. Start again to resume.')
}
await change('starting')
+ // A grid that is down refuses a join outright ("The model could not start"), and `sync`
+ // above restores its registration, not its process. Bring it up the way its own refusal
+ // says to. A grid already running is left alone: `start` is only for one that is not.
+ const status = await this.gridStatus(grid).catch(() => { throw new ModelError('Your grid could not be checked. Try again.') })
+ if (DOWN.has(status)) await must(['--remote', 'start', grid], 'Your grid could not start. Try again.')
const override = this.processEnv.LLAMA_SERVER
const installed = override ? binaryOnPath(override, this.processEnv)
: binaryOnPath(join(this.home, 'bin', 'llama-server'), this.processEnv) || binaryOnPath('llama-server', this.processEnv)
diff --git a/desktop/lib/models/model_manager_controller.dart b/desktop/lib/models/model_manager_controller.dart
index e246ed17e..e2e33d8fd 100644
--- a/desktop/lib/models/model_manager_controller.dart
+++ b/desktop/lib/models/model_manager_controller.dart
@@ -344,7 +344,11 @@ class ModelManagerController extends ChangeNotifier {
refresh: force,
);
if (!_current(owner) || revision != _actionRevision) return;
- error = answer['error'] as String?;
+ // `notice` is a sentence BESIDE the list. A reply carrying `error` never reaches here:
+ // the RPC layer fails it whole and keeps nothing, which is how a daemon that sent its
+ // warning in `error` once emptied this list. `error` is still read for a daemon too old
+ // to send models at all.
+ error = (answer['notice'] ?? answer['error']) as String?;
if (answer['models'] is! List) {
inventoryAvailable = false;
return;
diff --git a/desktop/test/model_manager_controller_test.dart b/desktop/test/model_manager_controller_test.dart
index eb9787a1a..d3e6059cf 100644
--- a/desktop/test/model_manager_controller_test.dart
+++ b/desktop/test/model_manager_controller_test.dart
@@ -356,6 +356,20 @@ void main() {
expect(controller.localModels, isEmpty);
});
+ test('a notice beside the list keeps the list, and every row it offers', () async {
+ // ⚠️ REGRESSION. The daemon's sentence for a grid it could not fully read rode in `error`,
+ // and an `error` fails the request whole — the rows it came with were thrown away and the
+ // Local tab read 0. The sentence now arrives as `notice`, and the list with it.
+ app.localInventory = {
+ ...modelInventory(),
+ 'notice': 'Running models could not be checked. Try again.',
+ };
+ await controller.refresh();
+ expect(controller.localModels, hasLength(5));
+ expect(controller.inventoryAvailable, isTrue);
+ expect(controller.error, 'Running models could not be checked. Try again.');
+ });
+
test('an old daemon or malformed inventory disables actions without clearing the last view', () async {
await controller.refresh();
app.localInventory = {'error': 'Update Harness to manage local models.'};