The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Models: a grid that is down runs nothing, and Start brings it back up
bitcoinbuilderd6e7d82b4afc
diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.tsindex 89ee6500e..4d25a1968 100644--- a/cli/src/lib/localModels.spec.ts+++ b/cli/src/lib/localModels.spec.ts@@ -18,16 +18,24 @@ let calls: string[][], serving: boolean, catalogCards: ReturnType<typeof card>[]let request: Mock<(url: string | URL, init?: RequestInit) => Promise<Response>>let run: Mock<(args: string[], output?: (s: string) => void) => Promise<GridFleetResult>>let downloadFails: boolean, catalogFails: boolean+/** What `grid info` says of the grid. Down (`stopped`/`asleep`) refuses `engines` and `join`, as+ * the real grid does, until `grid start` brings it back. */+let gridState: 'running' | 'stopped' | 'asleep'+const refused = (message: string) => ({ ok: false, code: 1, stdout: '', stderr: message, error: 'Grid command failed.' })beforeEach(async () => {root = await mkdtemp(join(tmpdir(), 'local-models-'))home = join(root, 'grid'); stateDir = join(root, 'receipts'); records = join(home, 'run', 'engines', 'grid-home')await mkdir(records, { recursive: true }); await mkdir(join(home, 'models'))await writeFile(join(home, 'credentials.toml'), 'session_token = "test-only-token"\napi_url = "https://catalog.example.test"\n')- calls = []; serving = false; catalogCards = [card()]; downloadFails = false; catalogFails = false+ calls = []; serving = false; catalogCards = [card()]; downloadFails = false; catalogFails = false; gridState = 'running'run = vi.fn(async (args: string[], output?: (s: string) => void) => {calls.push(args)if (args[0] === 'device-info') return ok({ device_class: 'apple-silicon', backend: 'metal', usable_bytes: 54 * 1024 ** 3, memory: { total_gb: 64 } })if (args.includes('ls')) return ok([{ grid: 'home', id: 'grid-home' }])+ if (args.includes('info') && args.includes('--json')) return ok({ grid: 'home', status: gridState })+ if (args[1] === 'start') { gridState = 'running'; return ok() }+ const down = gridState !== 'running'+ if (down && (args.includes('engines') || args.includes('join'))) return refused("Grid home isn't up; run `grid start home` first.")if (args.includes('engines')) return ok(serving ? [{ node_id: 'local-node', online: true, models: ['Small-Q4.gguf'], throughput_tok_s: 17.6,vram_used_mb: 99999, answered: { window_seconds: 3600, requests: 4, by_model: [{ model: 'Small-Q4.gguf', requests: 4 }] } }] : [])if (args[0] === 'pull') {@@ -83,7 +91,7 @@ describe('local model discovery and lifecycle', () => {request.mockResolvedValue(response({ models: [card()], runnable_total: 100, pagination: { page: 1, total_pages: 7 } }))// Fresh Response objects (bodies are single-consumption).request.mockImplementation(async () => response({ models: [card()], runnable_total: 100, pagination: { page: 1, total_pages: 7 } }))- expect((await service.list('home')).error).toContain('incomplete')+ expect((await service.list('home')).notice).toContain('incomplete')})it('downloads, loads and verifies without another user step', async () => {@@ -178,7 +186,7 @@ describe('local model discovery and lifecycle', () => {const fresh = new LocalModels({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch })const snapshot = await fresh.list('home')expect(snapshot.models[0]).toMatchObject({ state: 'running', canStop: true })- expect(snapshot.error).toContain('unavailable')+ expect(snapshot.notice).toContain('unavailable')expect(JSON.stringify(snapshot)).not.toContain('private raw')})@@ -318,7 +326,7 @@ describe('local model discovery and lifecycle', () => {expect(calls).toHaveLength(before)await service.list('home', true)expect(calls.filter(args => args[0] === 'device-info')).toHaveLength(2)- expect(await service.list(null)).toMatchObject({ models: [], busy: false, error: expect.stringContaining('Sign in') })+ expect(await service.list(null)).toMatchObject({ models: [], busy: false, notice: expect.stringContaining('Sign in') })expect((await service.act(null, 'x', 'start')).error).toBeTruthy()expect((await service.act('home', {}, 'start')).error).toBeTruthy()})@@ -329,19 +337,19 @@ describe('local model discovery and lifecycle', () => {['malformed token', 'session_token = "invalid\\q"'],])('reports sign-in for %s without sending credentials', async (_name, credentials) => {await writeFile(join(home, 'credentials.toml'), credentials)- expect((await service.list('home')).error).toContain('Sign in')+ expect((await service.list('home')).notice).toContain('Sign in')expect(request).not.toHaveBeenCalled()})it.each(['http://catalog.example.test', 'file:///tmp/catalog'])('refuses insecure catalog address %s', async base => {await writeFile(join(home, 'credentials.toml'), `session_token = 'private'\napi_url = '${base}'`)- expect((await service.list('home')).error).toContain('address')+ expect((await service.list('home')).notice).toContain('address')expect(request).not.toHaveBeenCalled()})it.each(['http://localhost:1234', 'http://127.0.0.1:1234', 'http://[::1]:1234'])('permits an explicitly configured local catalog at %s', async base => {await writeFile(join(home, 'credentials.toml'), `session_token = 'test-only-token'\napi_url = '${base}'`)- expect((await service.list('home')).error).toBeUndefined()+ expect((await service.list('home')).notice).toBeUndefined()expect(String(request.mock.calls[0][0])).toBe(`${base}/v1/grid/catalog`)expect(request.mock.calls[0][1]).toMatchObject({ redirect: 'error' })})@@ -362,7 +370,7 @@ describe('local model discovery and lifecycle', () => {})const snapshot = await service.list('home')expect(snapshot.models).toEqual([])- expect(snapshot.error).toBeTruthy()+ expect(snapshot.notice).toBeTruthy()expect(JSON.stringify(snapshot)).not.toContain('private detail')expect(request.mock.calls.length).toBeLessThanOrEqual(100)})@@ -383,10 +391,61 @@ describe('local model discovery and lifecycle', () => {run.mockImplementation(async (args, output) => args.includes('engines')? { ...ok(), ok: scenario !== 'failed command', stdout: 'not json' } : original(args, output))const snapshot = await service.list('home', true)- expect(snapshot.error).toContain('Running models could not be checked')+ expect(snapshot.notice).toContain('Running models could not be checked')expect(snapshot.models.every(m => !m.canStart && !m.canStop)).toBe(true)})+ // ── a grid that is down ──────────────────────────────────────────────────────────────────────+ //+ // ⚠️ REGRESSION. Grid refuses `engines` on a grid that is not up, and that refusal was read as+ // "running models unknown": every Start in the list went dark, so the one state in which a person+ // most needs to start a model was the one in which none could be. And a Start that did get through+ // failed at `join`, which a down grid refuses too.++ it.each(['stopped', 'asleep'] as const)('a %s grid runs nothing: every model can start, and nothing is wrong', async state => {+ gridState = state+ const snapshot = await service.list('home')+ expect(snapshot.models).toMatchObject([{ name: 'Small', state: 'available', canStart: true, canStop: false }])+ expect(snapshot.notice).toBeUndefined()+ // Never `error`: on this protocol that field fails the request and the app keeps no list.+ expect(snapshot).not.toHaveProperty('error')+ })++ it('a refusal from a grid that IS running is still a gap, not an empty list', async () => {+ run.mockImplementation(async args => args.includes('engines') ? refused('private detail')+ : args.includes('info') ? ok({ status: 'running' }) : ok([{ grid: 'home', id: 'grid-home' }]))+ await service.list('home', true)+ const snapshot = await service.list('home', true)+ expect(snapshot.notice).toContain('Running models could not be checked')+ expect(JSON.stringify(snapshot)).not.toContain('private detail')+ })++ it.each(['stopped', 'asleep'] as const)('Start brings a %s grid up before joining it', async state => {+ gridState = state+ await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()+ const started = calls.findIndex(args => args[0] === '--remote' && args[1] === 'start')+ const joined = calls.findIndex(args => args.includes('join'))+ expect(calls[started]).toEqual(['--remote', 'start', 'home'])+ expect(started).toBeGreaterThanOrEqual(0)+ expect(started).toBeLessThan(joined)+ expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'running', operation: { phase: 'done' } })+ })++ it('Start leaves a running grid alone', async () => {+ await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()+ expect(calls.some(args => args[0] === '--remote' && args[1] === 'start')).toBe(false)+ expect((await service.list('home', true)).models[0].state).toBe('running')+ })++ it('a grid that will not come up fails the Start in words, before any join', async () => {+ gridState = 'stopped'+ const original = run.getMockImplementation()!+ run.mockImplementation(async (args, output) => args[1] === 'start' ? refused('private detail') : original(args, output))+ await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()+ expect(calls.some(args => args.includes('join'))).toBe(false)+ expect((await service.list('home')).models[0].operation).toMatchObject({ phase: 'failed', error: 'Your grid could not start. Try again.' })+ })+it.each(['missing grid', 'invalid grid id', 'missing record directory', 'corrupt record'])('ignores %s without exposing a Stop action', async scenario => {const original = run.getMockImplementation()!if (scenario === 'missing grid' || scenario === 'invalid grid id') {diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.tsindex afc3cc8f1..34a105bb6 100644--- a/cli/src/lib/localModels.ts+++ b/cli/src/lib/localModels.ts@@ -18,6 +18,9 @@ const cleanName = (v: string): string => basename(v).replace(/\.gguf$/i, '').rep// Grid advertises a GGUF filename as a lowercase model name without its extension.const modelKey = (v: string): string => v.toLowerCase().replace(/\.gguf$/, '')const validArg = (v: string): boolean => !!v && !v.startsWith('-') && !/[\x00-\x1f]/.test(v)+/** `grid info` statuses of a grid that is not up. Grid refuses `engines` and `join` on one, and+ * names `grid start` as the way back. Anything else unknown stays unknown rather than "down". */+const DOWN = new Set(['stopped', 'asleep'])export interface LocalModel {id: string; name: string; state: 'available' | 'downloaded' | 'running'@@ -31,7 +34,11 @@ export interface ModelOperation {progress?: number; error?: string; updatedAt: string}export interface LocalModelsSnapshot {- models: LocalModel[]; memoryBytes?: number; hardware?: string; error?: string+ models: LocalModel[]; memoryBytes?: number; hardware?: string+ /** A sentence to show BESIDE the list — never `error`. On this protocol an `error` field fails+ * the whole request: the app's RPC layer drops the reply and keeps nothing of it, so a list sent+ * with a warning in `error` arrived as no list at all and the Local tab read 0. */+ notice?: stringobservedAt: string; busy: boolean}interface Candidate {@@ -272,7 +279,7 @@ export class LocalModels {}async list(grid: string | null, force = false): Promise<LocalModelsSnapshot> {- if (!grid) return { models: [], error: 'Sign in to find models for this computer.', observedAt: new Date().toISOString(), busy: false }+ if (!grid) return { models: [], notice: 'Sign in to find models for this computer.', observedAt: new Date().toISOString(), busy: false }if (!force && this.cached?.grid === grid && Date.now() - this.cached.at < 2500) return this.cached.valueif (this.listPending) {if (this.listGrid === grid) return this.listPending@@ -286,7 +293,7 @@ export class LocalModels {await Promise.all([this.loadCatalog(force), this.readReceipt(grid)])let owned: Owned[] = [], nodes: Record<string, any>[] = [], inventoryError: string | undefinedtry {- [owned, nodes] = await Promise.all([this.owned(grid), this.json(['--remote', 'engines', grid, '--json']).then(rows)])+ [owned, nodes] = await Promise.all([this.owned(grid), this.servingNodes(grid)])} catch { inventoryError = 'Running models could not be checked. Try again.' }const operation = this.active?.grid === grid ? this.active.operation : this.receipt?.grid === grid ? this.receipt.operation : undefinedconst choices = [...this.candidates, ...(await this.known(grid, owned)).filter(k => !this.candidates.some(c => c.file === k.file))]@@ -328,12 +335,34 @@ export class LocalModels {operation: operation?.modelId === `local:${instance.file}` ? operation : undefined })}const value: LocalModelsSnapshot = { models, memoryBytes: num(obj(this.device.memory).total_gb) === undefined ? undefined : this.device.memory.total_gb * GiB,- hardware: str(obj(this.device.machine).model || obj(this.device.cpu).brand), error: inventoryError || this.catalogError,+ hardware: str(obj(this.device.machine).model || obj(this.device.cpu).brand), notice: inventoryError || this.catalogError,observedAt: new Date().toISOString(), busy: !!this.active }this.cached = { grid, at: Date.now(), value }return value}+ /** What `grid info` says of [grid]: `running`, `stopped`, `asleep`. */+ private async gridStatus(grid: string): Promise<string> {+ return str(obj(await this.json(['--remote', 'info', grid, '--json'])).status)+ }++ /** The nodes serving [grid] — none at all when the grid is down.+ *+ * ⚠️ A stopped or asleep grid REFUSES `engines` ("isn't up; run `grid start` first") rather than+ * answering an empty list, and reading that refusal as "unknown" disabled every Start in the+ * list: the one state in which a person most needs to start a model was the one in which none+ * could be. Stopping the last model from this list leaves the grid up, so it was only reached by+ * a grid taken down by hand or gone to sleep — but then there was no way back from here. A grid+ * that is down runs nothing, which is an answer, not a gap. Any other refusal still is one. */+ private async servingNodes(grid: string): Promise<Record<string, any>[]> {+ const result = await this.run(['--remote', 'engines', grid, '--json'])+ if (result.ok) {+ try { return rows(JSON.parse(result.stdout)) } catch { throw new Error('Running models could not be checked. Try again.') }+ }+ if (DOWN.has(await this.gridStatus(grid))) return []+ throw new Error('Running models could not be checked. Try again.')+ }+/** A repeated click or lost RPC acknowledgement joins the same operation.* A different model waits: starts/stops must not race on the machine's budget. */async act(grid: string | null, modelId: unknown, action: 'start' | 'stop'): Promise<{ operation?: ModelOperation; error?: string }> {@@ -412,6 +441,11 @@ export class LocalModels {if (!await this.downloaded(candidate)) throw new ModelError('The download is incomplete. Start again to resume.')}await change('starting')+ // A grid that is down refuses a join outright ("The model could not start"), and `sync`+ // above restores its registration, not its process. Bring it up the way its own refusal+ // says to. A grid already running is left alone: `start` is only for one that is not.+ const status = await this.gridStatus(grid).catch(() => { throw new ModelError('Your grid could not be checked. Try again.') })+ if (DOWN.has(status)) await must(['--remote', 'start', grid], 'Your grid could not start. Try again.')const override = this.processEnv.LLAMA_SERVERconst installed = override ? binaryOnPath(override, this.processEnv): binaryOnPath(join(this.home, 'bin', 'llama-server'), this.processEnv) || binaryOnPath('llama-server', this.processEnv)diff --git a/desktop/lib/models/model_manager_controller.dart b/desktop/lib/models/model_manager_controller.dartindex e246ed17e..e2e33d8fd 100644--- a/desktop/lib/models/model_manager_controller.dart+++ b/desktop/lib/models/model_manager_controller.dart@@ -344,7 +344,11 @@ class ModelManagerController extends ChangeNotifier {refresh: force,);if (!_current(owner) || revision != _actionRevision) return;- error = answer['error'] as String?;+ // `notice` is a sentence BESIDE the list. A reply carrying `error` never reaches here:+ // the RPC layer fails it whole and keeps nothing, which is how a daemon that sent its+ // warning in `error` once emptied this list. `error` is still read for a daemon too old+ // to send models at all.+ error = (answer['notice'] ?? answer['error']) as String?;if (answer['models'] is! List) {inventoryAvailable = false;return;diff --git a/desktop/test/model_manager_controller_test.dart b/desktop/test/model_manager_controller_test.dartindex eb9787a1a..d3e6059cf 100644--- a/desktop/test/model_manager_controller_test.dart+++ b/desktop/test/model_manager_controller_test.dart@@ -356,6 +356,20 @@ void main() {expect(controller.localModels, isEmpty);});+ test('a notice beside the list keeps the list, and every row it offers', () async {+ // ⚠️ REGRESSION. The daemon's sentence for a grid it could not fully read rode in `error`,+ // and an `error` fails the request whole — the rows it came with were thrown away and the+ // Local tab read 0. The sentence now arrives as `notice`, and the list with it.+ app.localInventory = {+ ...modelInventory(),+ 'notice': 'Running models could not be checked. Try again.',+ };+ await controller.refresh();+ expect(controller.localModels, hasLength(5));+ expect(controller.inventoryAvailable, isTrue);+ expect(controller.error, 'Running models could not be checked. Try again.');+ });+test('an old daemon or malformed inventory disables actions without clearing the last view', () async {await controller.refresh();app.localInventory = {'error': 'Update Harness to manage local models.'};