The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest'
import { mkdtemp, mkdir, writeFile, readFile, readdir, rm, stat, statfs, readlink } from 'node:fs/promises'
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { createHash } from 'node:crypto'
import { existsSync, rmSync, writeFileSync } from 'node:fs'
import { spawnSync } from 'node:child_process'
import type { GpuProcess } from './gpuProcesses.js'
import { baseModel, compatibleModels, contextLadder, fitOf, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'
import { GridFleetRpc, type GridFleetResult } from './gridFleetRpc.js'
import { AppStartError, type AppEngineOps, type AppEngineRecord, type AppModel } from './appModels.js'
import { encryptDownFrame, encryptRpcResult } from './e2ee/applicationFrames.js'
// The coverage audit needs real filesystem work with explicit disk/read faults at the I/O boundary.
vi.mock('node:fs/promises', async (original) => {
const fs = await original<typeof import('node:fs/promises')>()
return { ...fs, statfs: vi.fn(fs.statfs), readlink: vi.fn(fs.readlink) }
})
const card = (id = 'org/Small-GGUF') => ({ repo_id: id, runnable: true, task: 'text-generation', format: 'GGUF',
fit: { version: 'Q4', ctx: 131072, size: 64 },
versions: [{ version: 'Q4', size_bytes: 64, pull_spec: `${id}:Small-Q4.gguf`, urls: ['https://example.test/Small-Q4.gguf'] }] })
const ok = (value: unknown = {}) => ({ ok: true, code: 0, stdout: JSON.stringify(value), stderr: '', error: null })
const response = (body: unknown) => new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } })
// Found by QA on a quiet machine: secondary instances also keep background scans that write receipts.
const modelFixtures: LocalModels[] = []
function modelFixture(options: ConstructorParameters<typeof LocalModels>[0]): LocalModels {
const models = new LocalModels(options)
modelFixtures.push(models)
return models
}
let root: string, home: string, stateDir: string, records: string
let calls: string[][], serving: boolean, catalogCards: ReturnType<typeof card>[], service: LocalModels
let request: Mock<(url: string | URL, init?: RequestInit) => Promise<Response>>
let run: Mock<(args: string[], output?: (s: string) => void) => Promise<GridFleetResult>>
let downloadFails: boolean, catalogFails: boolean
/** What `grid info` says of the grid. Down (`stopped`/`asleep`) refuses `engines` and `join`, as
* the real grid does, until `grid start` brings it back. */
let gridState: 'running' | 'stopped' | 'asleep'
/** What `grid ctx FILE` reads from a file's header (null: the command fails), and the window the
* serving engine reports once started (undefined: Grid does not say). */
let trainedWindow: number | null, servedWindow: number | undefined
/** The relay's own test request — as opposed to the probe sent straight to the engine on this
* machine (`127.0.0.1`), which a start now makes first. */
const RELAY_CHAT = 'inference.example.test/v1/chat/completions'
const refused = (message: string) => ({ ok: false, code: 1, stdout: '', stderr: message, error: 'Grid command failed.' })
/** The grid's own answer when a test pins it (null: derived from [gridState], as production derives it). */
let answered: GridInventory['state'] | null
/** What `gridModels.gridInventory` would hand the Model Manager — read without a credential. */
let inventory: Mock<(grid: string, force: boolean) => Promise<GridInventory>>
beforeEach(async () => {
vi.mocked(statfs).mockReset(); vi.mocked(readlink).mockReset()
root = await mkdtemp(join(tmpdir(), 'local-models-'))
home = join(root, 'grid'); stateDir = join(root, 'receipts'); records = join(home, 'run', 'engines', 'grid-home')
await mkdir(records, { recursive: true }); await mkdir(join(home, 'models'))
await writeFile(join(home, 'credentials.toml'), 'session_token = "test-only-token"\napi_url = "https://catalog.example.test"\n')
calls = []; serving = false; catalogCards = [card()]; downloadFails = false; catalogFails = false; gridState = 'running'
trainedWindow = null; servedWindow = undefined
run = vi.fn(async (args: string[], output?: (s: string) => void) => {
calls.push(args)
if (args[0] === 'device-info') return ok({ device_class: 'apple-silicon', backend: 'metal', usable_bytes: 54 * 1024 ** 3, memory: { total_gb: 64 } })
if (args.includes('ls')) return ok([{ grid: 'home', id: 'grid-home' }])
if (args.includes('info') && args.includes('--json')) return ok({ grid: 'home', status: gridState })
if (args[1] === 'start') { gridState = 'running'; return ok() }
const down = gridState !== 'running'
if (down && (args.includes('engines') || args.includes('join'))) return refused("Grid home isn't up; run `grid start home` first.")
if (args[0] === 'ctx') return trainedWindow === null ? refused('no header') : ok({ file: args[1], context_length: trainedWindow })
if (args.includes('engines')) return ok(serving ? [{ node_id: 'local-node', online: true, models: ['Small-Q4.gguf'], throughput_tok_s: 17.6,
...(servedWindow === undefined ? {} : { model_capabilities: { 'small-q4': { context_length: servedWindow, vision: false } } }),
vram_used_mb: 99999, answered: { window_seconds: 3600, requests: 4, by_model: [{ model: 'Small-Q4.gguf', requests: 4 }] } }] : [])
if (args[0] === 'pull') {
output?.('42%')
if (downloadFails) return { ...ok(), ok: false, code: 1 }
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
}
if (args[0] === 'engine' && args[1] === 'install') {
await mkdir(join(home, 'bin'), { recursive: true })
await writeFile(join(home, 'bin', 'llama-server'), '#!/bin/sh\nexit 0\n', { mode: 0o755 })
}
if (args.includes('join')) {
serving = true
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'] }], advertise_as: [] }))
await writeFile(join(records, 'remote.heartbeat'), '')
}
if (args.includes('leave')) { serving = false; await rm(join(records, 'remote.json'), { force: true }) }
if (args.includes('info')) return { ...ok(), stdout: "export OPENAI_BASE_URL='https://inference.example.test/v1'\nexport OPENAI_API_KEY='test-inference-token'\n" }
return ok()
})
request = vi.fn(async (url: string | URL, _init?: RequestInit) => {
if (String(url).includes('/catalog')) {
if (catalogFails) throw new Error('private raw error must never escape')
return response({ models: catalogCards, runnable_total: catalogCards.length, pagination: { page: 1, total_pages: 1 } })
}
return response({ choices: [{ message: { content: 'ok' } }] })
})
// The inventory as production reads it (`gridModels.gridInventory`), over this fake's grid: the owner
// status is `gridState`; a sleeping grid answers asleep, a stopped one answers nothing readable (the
// proxy's grid_stopped), and a running one lists what the fake's `engines` lists.
answered = null
inventory = vi.fn(async (grid: string): Promise<GridInventory> => {
const status = gridState
if (answered) return { state: answered, nodes: [], status }
if (status === 'asleep') return { state: 'asleep', nodes: [], status }
if (status !== 'running') return { state: 'unknown', nodes: [], status }
const answer = await run(['--remote', 'engines', grid, '--json'])
if (!answer.ok) return { state: 'unknown', nodes: [], status }
try { return { state: 'awake', nodes: JSON.parse(answer.stdout), status } } catch { return { state: 'unknown', nodes: [], status } }
})
service = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory })
})
afterEach(async () => {
try { await Promise.all(modelFixtures.splice(0).map(models => models.settled())) } finally {
vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.unstubAllGlobals()
await rm(root, { recursive: true, force: true })
}
})
describe('local model discovery and lifecycle', () => {
it('accepts only catalog fits that carry a finite coding context', () => {
// QA's coverage audit: both candidates in a context comparison have passed this catalog validation.
for (const ctx of [undefined, null, NaN, Infinity, -1, 0, 32768]) {
expect(compatibleModels({ models: [{ ...card(), fit: { version: 'Q4', ctx } }] })).toEqual([])
}
expect(compatibleModels({ models: [card()] })).toMatchObject([{ context: 131072 }])
})
it('inspects silently and offers only confirmed compatible chat models', async () => {
expect(compatibleModels([card()])).toEqual([])
expect(compatibleModels({ models: [card(), { ...card('no'), runnable: false }, { ...card('image'), task: 'image-generation' }] })).toHaveLength(1)
const snapshot = await service.list('home')
expect(snapshot.models).toMatchObject([{ name: 'Small', state: 'available', sizeBytes: 64, canStart: true, canStop: false }])
expect(snapshot.memoryBytes).toBe(64 * 1024 ** 3)
expect(snapshot).toMatchObject({ fitBytes: 32 * 1024 ** 3, fitMemory: 'unified' })
expect(calls.some(args => ['pull', 'join', 'leave', 'install'].some(a => args.includes(a)))).toBe(false)
expect(JSON.stringify(snapshot)).not.toContain('test-only-token')
})
it('fetches only the first ranked page of popular models, like grid catalog', async () => {
request.mockResolvedValue(response({ models: [card('org/Model1-GGUF'), card('org/Model2-GGUF')], pagination: { page: 1, total_pages: 7 } }))
expect((await service.list('home')).models).toHaveLength(2)
expect(request).toHaveBeenCalledTimes(1)
expect(JSON.parse(String(request.mock.calls[0][1]?.body))).toMatchObject({ browse: true, page: 1, page_size: 50 })
})
it('returns the first page as-is without looping pages or a pagination notice', async () => {
request.mockResolvedValue(response({ models: [card()], runnable_total: 100, pagination: { page: 1, total_pages: 7 } }))
const snapshot = await service.list('home')
expect(snapshot.models).toHaveLength(1)
expect(snapshot.notice).toBeFalsy()
})
it('downloads, loads and verifies without another user step', async () => {
// Not running: nothing serves it, so no name a harness could be on.
expect((await service.list('home')).models[0]).not.toHaveProperty('servedAs')
const ack = await service.act('home', 'org/Small-GGUF', 'start')
expect(ack.operation?.phase).toBe('running')
await service.settled()
const snapshot = await service.list('home')
// Shown as the catalog's `Small`, served under its file's name — the one the grid lists, and a harness on it says.
expect(snapshot.models[0]).toMatchObject({ name: 'Small', state: 'running', canStop: true, servedAs: ['Small-Q4'],
tokensPerSecond: 17.6, requests: 4, windowSeconds: 3600, operation: { phase: 'done', stage: 'verifying' } })
expect(snapshot.models[0]).not.toHaveProperty('memoryBytes')
expect(calls.find(args => args.includes('join'))).toEqual(['--remote', 'join', 'home', '--serve', 'Small-Q4.gguf', '--max-concurrency', '1', '--parallel', '1', '--ctx-size', '131072', '--endpoint-port', expect.stringMatching(/^\d+$/), '--reasoning-budget', '0'])
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(true)
expect(await readFile(join(stateDir, (await readdir(stateDir))[0]), 'utf8')).not.toContain('token')
})
// The node takes one request at a time (NODE_CONCURRENCY): a harness's model is one conversation in one slot,
// so a second request waits in the grid's queue. It is the same number whatever else the node runs — here an
// engine Grid started on it while the weights downloaded (from a terminal, say) — because a join that changed
// it would restart the node, and the restart reload every engine Grid runs there.
it.each([
['no slot count of its own', { engine: {}, record: {} }],
['a slot count of its own', { engine: { launch: { parallel: 4 } }, record: {} }],
['a slot count on a record written before engines had launch settings', { engine: {}, record: { parallel: 4 } }],
])('joins at one request at a time beside an engine Grid runs with %s', async (_kind, other) => {
const base = run.getMockImplementation()!
run.mockImplementation(async (args, output) => {
if (args[0] === 'pull') {
await writeFile(join(records, 'other.json'), JSON.stringify({ node_id: 'local-node', ...other.record,
engines: [{ endpoint_url: null, models: ['Other-Q4.gguf'], ...other.engine }], advertise_as: [] }))
}
return base(args, output)
})
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const joined = calls.find(args => args.includes('join'))!
expect(joined[joined.indexOf('--max-concurrency') + 1]).toBe('1')
// A harness's model keeps one slot and its whole window.
expect(joined[joined.indexOf('--parallel') + 1]).toBe('1')
})
// grid-reads-without-waking issue 03: the reply test is an inference THROUGH the grid, so on a sleeping
// grid it starts it — and the platform then keeps it up for hours. A stray Start on a model that is
// already serving has nothing to check.
it('Start on a model already serving at the last read skips the reply test', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home', true)).models[0].state).toBe('running')
request.mockClear(); calls.length = 0
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
// Nothing that reads through the grid with its credential: no reply test, no `info --env` for its
// key, no signed-in `engines` for the served window — and nothing started.
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(false)
expect(calls.filter(args => args.includes('--env') || args.includes('engines') || args.includes('join'))).toEqual([])
expect((await service.list('home')).models[0]).toMatchObject({ state: 'running', operation: { phase: 'done' } })
})
it('Start on a model this computer holds but that was not serving at the last read runs the reply test, as before', async () => {
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'] }], advertise_as: [] }))
expect((await service.list('home', true)).models[0].state).not.toBe('running')
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(true)
expect(calls.some(args => args.includes('join'))).toBe(false)
})
it('uses an already downloaded complete file, and Stop retains it', async () => {
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
expect((await service.list('home')).models[0].state).toBe('downloaded')
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(calls.some(args => args[0] === 'pull')).toBe(false)
await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
expect(calls.find(args => args.includes('leave'))).toEqual(['--remote', 'leave', 'home', '--engine', 'Small-Q4.gguf'])
expect((await stat(join(home, 'models', 'Small-Q4.gguf'))).size).toBe(64)
expect((await service.list('home')).models[0]).toMatchObject({ state: 'downloaded', canStop: false, canStart: true })
})
it('reuses the installed engine on subsequent starts', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(calls.filter(args => args[0] === 'engine' && args[1] === 'install')).toHaveLength(1)
expect(calls.filter(args => args[0] === 'pull')).toHaveLength(1)
})
it('does not count partial files as downloaded', async () => {
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(12))
await writeFile(join(home, 'models', 'Small-Q4.gguf.part'), Buffer.alloc(64))
expect((await service.list('home')).models[0].state).toBe('available')
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(calls.some(args => args[0] === 'pull')).toBe(true)
})
it('joins repeated clicks, serializes other models, and keeps status independent of its caller', async () => {
let release!: () => void
const barrier = new Promise<void>(resolve => { release = resolve })
const original = request.getMockImplementation()!
request.mockImplementation(async (url, init) => { if (String(url).includes(RELAY_CHAT)) await barrier; return original(url, init) })
const first = await service.act('home', 'org/Small-GGUF', 'start')
const second = await service.act('home', 'org/Small-GGUF', 'start')
expect(second.operation?.id).toBe(first.operation?.id)
expect((await service.act('home', 'other', 'start')).error).toContain('Wait')
await vi.waitFor(() => expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(true))
expect((await service.list('home')).models[0].operation).toMatchObject({ stage: 'verifying', phase: 'running' })
release(); await service.settled()
expect(calls.filter(args => args.includes('join'))).toHaveLength(1)
})
it('reports a failed download and retries it without starting an incomplete model', async () => {
downloadFails = true
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation).toMatchObject({ phase: 'failed', error: 'The download stopped. Start again to resume.' })
expect(calls.some(args => args.includes('join'))).toBe(false)
downloadFails = false
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation?.phase).toBe('done')
})
it('keeps a failed Grid registration refresh retryable without starting or downloading', async () => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('sync') ? { ...ok(), ok: false } : original(args, output))
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation).toMatchObject({ phase: 'failed', stage: 'checking' })
expect(calls.some(args => args.includes('join') || args.includes('pull'))).toBe(false)
run.mockImplementation(original)
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].state).toBe('running')
})
it('lets a model start beside the APIs a Swarm shares, which run nothing here, and not beside another engine', async () => {
const share = (port: number) => ({ endpoint_url: `http://127.0.0.1:${port}/${'a'.repeat(36)}/v1`, models: ['deepseek/x'] })
await writeFile(join(records, 'shared.json'), JSON.stringify({ node_id: 'local-node', engines: [share(41001), share(41002)] }))
await writeFile(join(records, 'shared.heartbeat'), '')
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home', true)).models.find(m => m.id === 'org/Small-GGUF')!.operation).toMatchObject({ phase: 'done' })
await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
// One that is not a share's endpoint is an engine like any other.
await writeFile(join(records, 'shared.json'), JSON.stringify({ node_id: 'local-node', engines: [share(41001), { endpoint_url: 'http://127.0.0.1:41003/v1', models: ['y'] }] }))
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home', true)).models.find(m => m.id === 'org/Small-GGUF')!.operation)
.toMatchObject({ phase: 'failed', error: 'Open Model Manager to add a model alongside those already running.' })
// An app's engine this daemon joined there is named, so the person knows what to stop.
await mkdir(stateDir, { recursive: true })
await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([{ spec: 1, modelId: 'app:ollama:llama3.2:3b', grid: 'home',
name: 'llama3.2:3b', engine: 'ollama', served: 'llama3.2:3b', alias: 'llama3.2:3b', port: 41003, binary: '/usr/local/bin/ollama' }]))
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home', true)).models.find(m => m.id === 'org/Small-GGUF')!.operation)
.toMatchObject({ phase: 'failed', error: 'Stop llama3.2:3b first to start another local model.' })
})
it('never offers Stop for an external endpoint or another machine', async () => {
serving = true
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', engines: [{ endpoint_url: 'http://localhost:1234', models: ['Small-Q4.gguf'] }] }))
await writeFile(join(records, 'remote.heartbeat'), '')
const snapshot = await service.list('home')
expect(snapshot.models[0].canStop).toBe(false)
await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
expect(calls.some(args => args.includes('leave'))).toBe(false)
})
it('keeps existing local engines visible when the catalog cannot be reached', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
catalogFails = true
const fresh = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch , inventory })
const snapshot = await fresh.list('home')
expect(snapshot.models[0]).toMatchObject({ state: 'running', canStop: true })
expect(snapshot.notice).toContain('unavailable')
expect(JSON.stringify(snapshot)).not.toContain('private raw')
})
it('does not silently evict an existing model to fit the next one', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const next = card('org/Next-GGUF'); next.versions[0].pull_spec = 'org/Next-GGUF:Next.gguf'; next.versions[0].urls = ['https://example.test/Next.gguf']
catalogCards = [card(), next]
await service.list('home', true)
request.mockImplementation(async (_url, init) => response({ models: JSON.parse(String(init?.body)).device.usable_bytes < 54 * 1024 ** 3 ? [] : catalogCards, pagination: { total_pages: 1 } }))
await service.act('home', 'org/Next-GGUF', 'start'); await service.settled()
const snapshot = await service.list('home')
expect(snapshot.models.find(m => m.id === 'org/Next-GGUF')?.operation?.error).toContain('Stop Small-Q4 first')
expect(calls.some(args => args.includes('leave'))).toBe(false)
})
it('rejects arbitrary model ids without passing them to Grid', async () => {
await service.act('home', '--serve /arbitrary', 'start'); await service.settled()
expect(calls.some(args => args.includes('--serve /arbitrary'))).toBe(false)
expect(calls.some(args => args[0] === 'pull')).toBe(false)
})
it('imports an existing local setup, keeps it after Stop, and can start it again', async () => {
catalogCards = []
serving = true
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', meta_name: 'This computer', ctx_size: 8192,
engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'] }], advertise_as: ['Small-Q4.gguf'] }))
await writeFile(join(records, 'remote.heartbeat'), '')
const first = await service.list('home')
expect(first.models[0]).toMatchObject({ id: 'local:Small-Q4.gguf', state: 'running', canStop: true, servedAs: ['Small-Q4'] })
await service.act('home', first.models[0].id, 'stop'); await service.settled()
const fresh = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch , inventory })
expect((await fresh.list('home')).models[0]).toMatchObject({ state: 'downloaded', canStart: true })
await fresh.act('home', first.models[0].id, 'start'); await fresh.settled()
expect(calls.some(args => args[0] === 'pull')).toBe(false)
expect((await fresh.list('home')).models[0].operation?.phase).toBe('done')
})
it("knows Grid's own engine by its own names when another engine joins beside it", async () => {
catalogCards = []
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
// An engine at its own address joined beside it: each engine carries its own names, the record's flat list all of them.
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', advertise_as: ['small', 'gemma-4-e2b'],
engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'], advertise_as: ['small'] },
{ endpoint_url: 'http://127.0.0.1:50872/v1', models: ['gemma-4-e2b'], advertise_as: ['gemma-4-e2b'] }] }))
await writeFile(join(records, 'remote.heartbeat'), '')
inventory.mockImplementation(async () => ({ state: 'awake', status: 'running',
nodes: [{ node_id: 'local-node', online: true, models: ['small', 'gemma-4-e2b'] }] }))
expect((await service.list('home')).models.find(m => m.id === 'local:Small-Q4.gguf'))
.toMatchObject({ state: 'running', canStop: true })
})
it("never pins a record's flat list of names on one of two engines: each is known by its own file", async () => {
catalogCards = []
// Written before engines carried their own names: the flat list only ever named a sole engine, and here it
// cannot say which of the two it named.
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', advertise_as: ['team/my-model'],
engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'] }, { endpoint_url: null, models: ['Big-Q4.gguf'] }] }))
await writeFile(join(records, 'remote.heartbeat'), '')
inventory.mockImplementation(async () => ({ state: 'awake', status: 'running',
nodes: [{ node_id: 'local-node', online: true, models: ['small-q4', 'big-q4'] }] }))
expect((await service.list('home')).models.filter(m => m.id.startsWith('local:')).map(m => [m.id, m.name, m.state]))
.toEqual([['local:Small-Q4.gguf', 'Small-Q4', 'running'], ['local:Big-Q4.gguf', 'Big-Q4', 'running']])
})
it.each(['current', 'legacy', 'malformed'])('preserves imported routing names across restart with %s receipts', async scenario => {
catalogCards = []; serving = true
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node',
engines: [{ models: ['Small-Q4.gguf'] }], advertise_as: ['team/my-model'] }))
await writeFile(join(records, 'remote.heartbeat'), '')
const id = (await service.list('home')).models[0].id
await service.act('home', id, 'stop'); await service.settled()
const path = join(stateDir, (await readdir(stateDir)).find(name => name.endsWith('.known.json'))!)
const saved = JSON.parse(await readFile(path, 'utf8'))
expect(saved[0].aliases).toEqual(['team/my-model'])
// What receipts from before the 64K floor carried: a pinned 16K that must not be replayed.
saved[0].context = 16384
if (scenario === 'legacy') delete saved[0].aliases
if (scenario === 'malformed') saved[0].aliases = [42, '', '--invalid', 'invalid\nname']
await writeFile(path, JSON.stringify(saved))
const fresh = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch , inventory })
await fresh.act('home', id, 'start'); await fresh.settled()
const expectedAlias = scenario === 'current' ? 'team/my-model' : scenario === 'legacy' ? 'my-model' : null
const joined = calls.find(args => args.includes('join'))!
expect(joined.slice(joined.indexOf('--reasoning-budget') + 2)).toEqual(expectedAlias ? ['--advertise-as', expectedAlias] : [])
// Not sized by any catalog and its header unread here, so the start begins at 128K — and a
// receipt's saved context, which older ones pinned at 16K, is never replayed.
expect(joined[joined.indexOf('--ctx-size') + 1]).toBe('131072')
const inference = request.mock.calls.find(([url]) => String(url).includes(RELAY_CHAT))!
expect(JSON.parse(String(inference[1]?.body)).model).toBe(expectedAlias ?? 'Small-Q4.gguf')
expect((await fresh.list('home')).models[0].operation?.phase).toBe('done')
})
it('lists a model this computer served on one grid on every other grid too', async () => {
catalogCards = []; serving = true
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', engines: [{ models: ['Small-Q4.gguf'] }] }))
await writeFile(join(records, 'remote.heartbeat'), '')
const id = (await service.list('home')).models[0].id
await service.act('home', id, 'stop'); await service.settled()
// Another Swarm's grid, read by a fresh process: the file is known there as well, and a broken file beside it
// reads as nothing.
await writeFile(join(stateDir, 'broken.known.json'), '{')
// Known on two grids, it is listed once.
const known = (await readdir(stateDir)).find(name => name.endsWith('.known.json') && name !== 'broken.known.json')!
await writeFile(join(stateDir, 'copy.known.json'), await readFile(join(stateDir, known)))
const fresh = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory })
expect((await fresh.list('swarm-0123456789ab')).models.map(m => m.id)).toEqual([id])
// With no state folder yet, nothing is known.
const blank = modelFixture({ stateDir: join(stateDir, 'none'), processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory })
expect((await blank.list('swarm-0123456789ab')).models.map(m => m.id)).toEqual([])
})
it('reuses an already downloaded fitting quant of the same model', async () => {
catalogCards[0].versions.push({ version: 'Q3', size_bytes: 48, pull_spec: 'org/Small-GGUF:Small-Q3.gguf', urls: ['https://example.test/Small-Q3.gguf'] })
await writeFile(join(home, 'models', 'Small-Q3.gguf'), Buffer.alloc(48))
const snapshot = await service.list('home')
expect(snapshot.models[0]).toMatchObject({ state: 'downloaded', sizeBytes: 48, quant: 'Q3' })
})
it('matches the current CLI engine shape without attributing another model’s requests', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const record = JSON.parse(await readFile(join(records, 'remote.json'), 'utf8'))
record.meta_name = 'My laptop'
await writeFile(join(records, 'remote.json'), JSON.stringify(record))
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('engines') ? ok([{
name: 'My laptop', online: true, models: ['Small-Q4.gguf'], throughput_tok_s: 12,
answered: { requests: 99, window_seconds: 86400, by_model: [{ model: 'prior-model', requests: 99 }] },
}]) : original(args, output))
const view = await service.list('home')
expect(view.models[0]).toMatchObject({ state: 'running', tokensPerSecond: 12 })
expect(view.models[0].requests).toBeUndefined()
})
it.each(['small-q4', { model: 'small-q4' }])('matches Grid’s canonical model names after start: %j', async alias => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const record = JSON.parse(await readFile(join(records, 'remote.json'), 'utf8'))
record.meta_name = 'My laptop'
await writeFile(join(records, 'remote.json'), JSON.stringify(record))
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('engines') ? ok([{
name: 'My laptop', online: true, models: [alias], throughput_tok_s: 12,
answered: { requests: 99, window_seconds: 86400, by_model: [
{ model: 'small-q8', requests: 95 }, { model: 'small-q4', requests: 4 },
] },
}]) : original(args, output))
expect((await service.list('home')).models[0]).toMatchObject({
state: 'running', canStop: true, tokensPerSecond: 12, requests: 4, windowSeconds: 86400,
})
})
it('assigns an engine only to the downloaded variant when catalog filenames collide', async () => {
const other = card('org/Small-MTP-GGUF')
other.versions[0].size_bytes = 80
catalogCards = [other, card()]
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const snapshot = await service.list('home')
expect(snapshot.models.find(m => m.id === 'org/Small-GGUF')).toMatchObject({
state: 'running', canStop: true, requests: 4,
})
expect(snapshot.models.find(m => m.id === other.repo_id)).toMatchObject({
state: 'available', canStop: false, requests: undefined, tokensPerSecond: undefined,
})
})
it('can still stop a running catalog model after its weights are removed', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
await rm(join(home, 'models', 'Small-Q4.gguf'))
expect((await service.list('home')).models[0]).toMatchObject({ state: 'running', canStop: true })
await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
expect((await service.list('home')).models[0]).toMatchObject({ state: 'available', canStop: false })
})
it('encrypts inventory and lifecycle requests and responses', () => {
for (const type of ['grid_fleet_models_list', 'grid_fleet_model_download', 'grid_fleet_model_start', 'grid_fleet_model_stop']) {
expect(encryptDownFrame(type)).toBe(true); expect(encryptRpcResult(`${type}_result`)).toBe(true)
}
})
it('coalesces discovery, caches it briefly, and isolates different accounts', async () => {
const [first, duplicate, other] = await Promise.all([service.list('home'), service.list('home'), service.list('other')])
expect(duplicate).toEqual(first)
expect(other.busy).toBe(false)
expect(calls.filter(args => args[0] === 'device-info')).toHaveLength(1)
const before = calls.length
expect(await service.list('other')).toEqual(other)
expect(calls).toHaveLength(before)
await service.list('home', true)
expect(calls.filter(args => args[0] === 'device-info')).toHaveLength(2)
expect(await service.list(null)).toMatchObject({ models: [], busy: false, notice: expect.stringContaining('Sign in') })
expect((await service.act(null, 'x', 'start')).error).toBeTruthy()
expect((await service.act('home', {}, 'start')).error).toBeTruthy()
})
it.each([
['missing credentials', ''],
['no session token', 'api_url = "https://catalog.example.test"'],
['malformed token', 'session_token = "invalid\\q"'],
])('reports sign-in for %s without sending credentials', async (_name, credentials) => {
await writeFile(join(home, 'credentials.toml'), credentials)
expect((await service.list('home')).notice).toContain('Sign in')
expect(request).not.toHaveBeenCalled()
})
it.each(['http://catalog.example.test', 'file:///tmp/catalog'])('refuses insecure catalog address %s', async base => {
await writeFile(join(home, 'credentials.toml'), `session_token = 'private'\napi_url = '${base}'`)
expect((await service.list('home')).notice).toContain('address')
expect(request).not.toHaveBeenCalled()
})
it.each(['http://localhost:1234', 'http://127.0.0.1:1234', 'http://[::1]:1234'])('permits an explicitly configured local catalog at %s', async base => {
await writeFile(join(home, 'credentials.toml'), `session_token = 'test-only-token'\napi_url = '${base}'`)
expect((await service.list('home')).notice).toBeUndefined()
expect(String(request.mock.calls[0][0])).toBe(`${base}/v1/grid/catalog`)
expect(request.mock.calls[0][1]).toMatchObject({ redirect: 'error' })
})
it.each([undefined, 'https://alternate.example.test'])('uses the configured or default catalog when the credential store omits it', async base => {
await writeFile(join(home, 'credentials.toml'), 'session_token = "test-only-token"\n')
service = modelFixture({ stateDir, processEnv: { GRID_HOME: home, GRID_CONTROL_PLANE_URL: base }, run, request: request as typeof fetch , inventory })
await service.list('home')
expect(String(request.mock.calls[0][0])).toBe(`${base ?? 'https://api-grid.autonomous.ai'}/v1/grid/catalog`)
})
it.each(['http failure', 'missing rows'])('handles a catalog %s without inventing compatibility', async scenario => {
request.mockImplementation(async () =>
scenario === 'http failure' ? new Response('private detail', { status: 401 })
: response({ error: 'private detail' }))
const snapshot = await service.list('home')
expect(snapshot.models).toEqual([])
expect(snapshot.notice).toBeTruthy()
expect(JSON.stringify(snapshot)).not.toContain('private detail')
})
it('supports an unpaginated catalog, offers a fast model before a big slow one, and skips unfitted versions', async () => {
const large = card('org/Large-GGUF'), small = { ...card('org/Fast-GGUF'), fit: { ...card().fit, est_tok_s: 30 } }
large.versions[0].size_bytes = 20 * 1024 ** 3
small.versions[0].size_bytes = 2 * 1024 ** 3
small.versions.push({ ...small.versions[0], version: 'Q8', size_bytes: 4 * 1024 ** 3 })
small.versions.push({ ...small.versions[0], version: 'unknown', size_bytes: undefined as any })
request.mockImplementation(async () => response({ models: [large, small] }))
expect((await service.list('home')).models.map(m => [m.name, m.recommended])).toEqual([['Fast', true], ['Large', false]])
})
it('offers what a coding agent can work with first: a faithful quant, fast enough, then bigger, then the catalog order', async () => {
// [id, estimated tok/s, billions of parameters, quant] in the catalog's own order.
const rows: [string, number | undefined, number | undefined, string?][] = [
['org/Dense-70B-GGUF', 8, 70], // big, and slow on this machine
['org/Qwen-35B-A3B-GGUF', 21, 35],
['org/Qwen-35B-A3B-MTP-GGUF', 21, 35], // a variant of the one above
['org/Tiny-4B-GGUF', 27, 4],
['org/Retrained-35B-A3B-GGUF', 22, 35], // a fine-tune: the same MoE shape
['org/Renamed-35B-GGUF', 21, 35], // a fine-tune: the same size, speed and weights
['org/Coder-Next-GGUF', 25, undefined, 'MXFP4_MOE'], // no count: read off 48 GB of MXFP4 weights
['org/Unknown-9B-GGUF', undefined, 9], // no estimate: not known to be fast
['org/gpt-oss-safeguard-20b-GGUF', 24, 20], // a safety classifier, not a coding model
['org/Giant-122B-A10B-GGUF', 30, 122, 'Q2_K_XL'], // biggest and fast, at 2 bits a weight
]
catalogCards = rows.map(([id, estimate, params, quant]) => {
const row: Record<string, any> = card(id)
row.versions[0].pull_spec = `${id}:${id.split('/')[1]}.gguf`
if (estimate !== undefined) row.fit = { ...row.fit, est_tok_s: estimate }
if (params !== undefined) row.params_b = params
if (quant) {
row.fit = { ...row.fit, version: quant }
row.versions[0] = { ...row.versions[0], version: quant, ...(quant === 'MXFP4_MOE' ? { size_bytes: 48e9 } : {}) }
}
return row as ReturnType<typeof card>
})
const snapshot = await service.list('home')
expect(snapshot.models.map(model => model.name)).toEqual([
'Coder-Next', 'Qwen-35B-A3B', 'Tiny-4B',
'Dense-70B', 'Unknown-9B',
'Giant-122B-A10B',
'Qwen-35B-A3B-MTP', 'Retrained-35B-A3B', 'Renamed-35B',
])
expect(snapshot.models[0].recommended).toBe(true)
})
it.each(['BF16', 'FP16', 'F16'])('sizes %s weights as sixteen bits when the catalog omits parameter counts', quant => {
const candidate = (name: string, quant: string, size: number) => ({ id: name, name, quant, size,
pull: `${name}:${name}.gguf`, file: `${name}.gguf`, files: [`${name}.gguf`], estTokS: 30 })
expect(rankForCoding([candidate('Full precision', quant, 32e9), candidate('Quantized', 'Q4', 12e9)]).map(model => model.name))
.toEqual(['Quantized', 'Full precision'])
})
it('uses IQ overhead and the unknown-quant fallback when ranking models of the same inferred size', () => {
const candidate = (name: string, quant: string, size: number) => ({ id: name, name, quant, size,
pull: `${name}:${name}.gguf`, file: `${name}.gguf`, files: [`${name}.gguf`], estTokS: 30 })
expect(rankForCoding([candidate('IQ model', 'IQ4_XS', 8.6e9), candidate('Unknown quant', '', 9.6e9)]).map(model => model.name))
.toEqual(['IQ model', 'Unknown quant'])
})
it('ranks without regard to where a model was in the catalog once speed or size tells them apart', () => {
const at = (name: string, estTokS: number, paramsB: number, size = 1) =>
({ id: `org/${name}`, name, pull: `org/${name}:${name}.gguf`, file: `${name}.gguf`, files: [`${name}.gguf`], size, quant: 'Q4', estTokS, paramsB })
expect(rankForCoding([at('Slow-Big', 5, 70), at('Fast-Small', 25, 4), at('Fast-Big', 25, 30)]).map(c => c.name))
.toEqual(['Fast-Big', 'Fast-Small', 'Slow-Big'])
// At exactly the floor a model counts as fast enough.
expect(rankForCoding([at('Slow', 19.9, 30), at('Enough', 20, 4)]).map(c => c.name)).toEqual(['Enough', 'Slow'])
// Two models of one size are two models when their weights or speed differ.
expect(rankForCoding([at('Moe-35B', 78, 35, 20e9), at('Dense-35B', 30, 35, 20e9), at('Other-35B', 78, 35, 24e9)]).map(c => c.name))
.toEqual(['Moe-35B', 'Dense-35B', 'Other-35B'])
})
it('fits the catalog to half the machine, and never to more than grid says is free', () => {
const device = (usable: number, total?: number, backend = 'metal') =>
({ backend, usable_bytes: usable * 1024 ** 3, ...(total ? { memory: { total_gb: total } } : {}) })
expect(modelBudget(device(54, 64))).toBe(32 * 1024 ** 3)
expect(modelBudget(device(20, 64))).toBe(20 * 1024 ** 3)
expect(modelBudget(device(54))).toBe(54 * 1024 ** 3)
// No GPU: the model lives in system RAM, shared like unified memory.
expect(modelBudget(device(28, 32, 'cpu'))).toBe(16 * 1024 ** 3)
expect(modelBudget({})).toBeUndefined()
})
it("gives the catalog all of an NVIDIA card's free VRAM, whatever the system RAM", () => {
// A 24 GB card in a 32 GB PC: VRAM is not system RAM, and nothing else is waiting for it.
expect(modelBudget({ backend: 'cuda', usable_bytes: 23 * 1024 ** 3, memory: { total_gb: 32 } })).toBe(23 * 1024 ** 3)
// What the apps are told with it: an NVIDIA card's free VRAM and all its VRAM, not the PC's RAM.
expect(fitOf({ backend: 'cuda', usable_bytes: 22 * 1024 ** 3, memory: { total_gb: 64 }, gpus: [{ backend: 'cuda', memory_total_mb: 24576 }] }))
.toEqual({ fitBytes: 22 * 1024 ** 3, fitMemory: 'vram', gpuMemoryBytes: 24 * 1024 ** 3 })
expect(fitOf({ backend: 'metal', usable_bytes: 54 * 1024 ** 3, memory: { total_gb: 64 }, gpus: [{ backend: 'metal', memory_total_mb: 65536 }] }))
.toEqual({ fitBytes: 32 * 1024 ** 3, fitMemory: 'unified' })
expect(fitOf({ backend: 'cpu', usable_bytes: 20 * 1024 ** 3, memory: { total_gb: 32 } })).toEqual({ fitBytes: 16 * 1024 ** 3, fitMemory: 'ram' })
expect(fitOf({})).toEqual({})
// A card that names no memory adds none, and none at all is no GPU figure.
expect(fitOf({ backend: 'cuda', usable_bytes: 7 * 1024 ** 3, gpus: [{ backend: 'cuda' }] })).toEqual({ fitBytes: 7 * 1024 ** 3, fitMemory: 'vram' })
// Two 24 GB cards in a 64 GB PC.
expect(modelBudget({ backend: 'cuda', usable_bytes: 46 * 1024 ** 3, memory: { total_gb: 64 } })).toBe(46 * 1024 ** 3)
})
it('names one base model across its variants, and a fine-tune under its own name apart', () => {
for (const name of ['Qwen3.6-35B-A3B', 'Qwen3.6-35B-A3B-MTP', 'unsloth/Qwen3.6-35B-A3B-MTP-GGUF', 'Qwen3.6-35B-A3B-UD-Q5_K_XL.gguf']) {
expect(baseModel(name)).toBe('qwen3.6-35b-a3b')
}
expect(baseModel('gemma-4-E4B-it-qat')).toBe(baseModel('gemma-4-E4B-it'))
expect(baseModel('GLM-4.7-Flash-REAP-23B-A3B')).toBe(baseModel('GLM-4.7-Flash'))
expect(baseModel('Qwen3-VL-30B-A3B-Thinking')).toBe(baseModel('Qwen3-VL-30B-A3B-Instruct'))
expect(baseModel('Qwen-AgentWorld-35B-A3B')).not.toBe(baseModel('Qwen3.6-35B-A3B'))
expect(baseModel('Qwen3.5-9B')).not.toBe(baseModel('Qwen3.5-4B'))
})
it.each(['failed command', 'invalid json'])('disables actions when inventory returns %s', async scenario => {
await service.list('home')
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('engines')
? { ...ok(), ok: scenario !== 'failed command', stdout: 'not json' } : original(args, output))
const snapshot = await service.list('home', true)
expect(snapshot.notice).toContain('Running models could not be checked')
expect(snapshot.models.every(m => !m.canStart && !m.canStop)).toBe(true)
})
// ── a grid that is down ──────────────────────────────────────────────────────────────────────
//
// ⚠️ REGRESSION. Grid refuses `engines` on a grid that is not up, and that refusal was read as
// "running models unknown": every Start in the list went dark, so the one state in which a person
// most needs to start a model was the one in which none could be. And a Start that did get through
// failed at `join`, which a down grid refuses too.
it.each(['stopped', 'asleep'] as const)('a %s grid runs nothing: every model can start, and nothing is wrong', async state => {
gridState = state
const snapshot = await service.list('home')
expect(snapshot.models).toMatchObject([{ name: 'Small', state: 'available', canStart: true, canStop: false }])
expect(snapshot.notice).toBeUndefined()
// Never `error`: on this protocol that field fails the request and the app keeps no list.
expect(snapshot).not.toHaveProperty('error')
})
it('a refusal from a grid that IS running is still a gap, not an empty list', async () => {
run.mockImplementation(async args => args.includes('engines') ? refused('private detail')
: args.includes('info') ? ok({ status: 'running' }) : ok([{ grid: 'home', id: 'grid-home' }]))
await service.list('home', true)
const snapshot = await service.list('home', true)
expect(snapshot.notice).toContain('Running models could not be checked')
expect(JSON.stringify(snapshot)).not.toContain('private detail')
})
// 2026-10-26: a new account's own grid is made only when something needs it, while its Swarm's grid was ready. Read
// as "could not be checked", every Start went dark, Run for Swarm too.
it('an own grid this account does not have yet runs nothing: every model can start, and nothing is wrong', async () => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('ls')
? ok([{ grid: 'swarm-0123456789ab', id: 'grid-swarm' }]) : original(args, output))
inventory.mockImplementation(async () => ({ state: 'unknown', nodes: [], status: null }))
const snapshot = await service.list('home', true)
expect(snapshot.notice).toBeUndefined()
expect(snapshot.models[0]).toMatchObject({ canStart: true })
// A grid list that did not answer a list says nothing about the grid: still unchecked.
run.mockImplementation(async (args, output) => args.includes('ls') ? ok({ error: 'signed out' }) : original(args, output))
expect((await service.list('home', true)).notice).toContain('Running models could not be checked')
})
it.each(['stopped', 'asleep'] as const)('Start brings a %s grid up before joining it', async state => {
gridState = state
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const started = calls.findIndex(args => args[0] === '--remote' && args[1] === 'start')
const joined = calls.findIndex(args => args.includes('join'))
expect(calls[started]).toEqual(['--remote', 'start', 'home'])
expect(started).toBeGreaterThanOrEqual(0)
expect(started).toBeLessThan(joined)
expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'running', operation: { phase: 'done' } })
})
it('Start leaves a running grid alone', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(calls.some(args => args[0] === '--remote' && args[1] === 'start')).toBe(false)
expect((await service.list('home', true)).models[0].state).toBe('running')
})
it('an unreadable grid status blocks nothing: the start goes on to the join', async () => {
// A Grid too old to answer `info --json` joined fine before this read existed.
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('info') && args.includes('--json') ? refused('usage') : original(args, output))
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(calls.some(args => args[1] === 'start')).toBe(false)
expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'running', operation: { phase: 'done' } })
})
it('a grid that will not come up fails the Start in words, before any join', async () => {
gridState = 'stopped'
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args[1] === 'start' ? refused('private detail') : original(args, output))
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(calls.some(args => args.includes('join'))).toBe(false)
expect((await service.list('home')).models[0].operation).toMatchObject({ phase: 'failed', error: 'Your grid could not start. Try again.' })
})
it.each(['missing grid', 'invalid grid id', 'missing record directory', 'corrupt record'])('ignores %s without exposing a Stop action', async scenario => {
const original = run.getMockImplementation()!
if (scenario === 'missing grid' || scenario === 'invalid grid id') {
run.mockImplementation(async (args, output) => args.includes('ls') ? ok(scenario === 'missing grid' ? [] : [{ grid: 'home', id: '../escape' }]) : original(args, output))
} else if (scenario === 'missing record directory') await rm(records, { recursive: true })
else await writeFile(join(records, 'broken.json'), 'not-json')
expect((await service.list('home')).models[0].canStop).toBe(false)
expect((await service.list('-invalid', true)).models[0].canStop).toBe(false)
})
it.each([
{}, { engines: [{ models: ['--bad.gguf'] }] }, { engines: [{ models: ['file.bin'] }] },
{ engines: [{ api_kind: 'openai', models: ['Small-Q4.gguf'] }] },
{ engines: [{ models: ['one.gguf', 'two.gguf'] }] },
])('does not treat an unowned or malformed run record as a controllable model: %j', async record => {
await writeFile(join(records, 'remote.json'), JSON.stringify(record))
expect((await service.list('home')).models[0].canStop).toBe(false)
})
it.each(['missing heartbeat', 'offline node', 'ambiguous node', 'missing models', 'missing file'])('does not claim running from %s alone', async scenario => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
if (scenario === 'missing heartbeat') await rm(join(records, 'remote.heartbeat'))
if (scenario === 'missing file') { catalogCards = []; await rm(join(home, 'models', 'Small-Q4.gguf')); serving = false }
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => {
const result = await original(args, output)
if (!args.includes('engines') || scenario === 'missing heartbeat' || scenario === 'missing file') return result
const node = JSON.parse(result.stdout)[0]
return ok(scenario === 'ambiguous node' ? [node, node] : [{ ...node, ...(scenario === 'offline node' ? { online: false } : { models: null }) }])
})
const snapshot = await service.list('home', true)
expect(snapshot.models[0]).toMatchObject({ canStop: true })
expect(snapshot.models[0].state).not.toBe('running')
expect(snapshot.models[0].tokensPerSecond).toBeUndefined()
})
it('keeps a missing-file engine stoppable and requires live inventory to call it running', async () => {
catalogCards = []
serving = true
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', engines: [{ models: ['Small-Q4.gguf'] }] }))
await writeFile(join(records, 'remote.heartbeat'), '')
expect((await service.list('home')).models[0]).toMatchObject({ state: 'running', canStart: false, canStop: true })
await service.act('home', 'local:Small-Q4.gguf', 'stop'); await service.settled()
expect(calls.some(args => args.includes('leave'))).toBe(true)
expect((await service.list('home')).models).toEqual([])
})
it('matches object-shaped aliases and never substitutes engine-total request counts', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('engines') ? ok([{
id: 'local-node', online: true, models: [{ model: 'SMALL-Q4.GGUF' }],
answered: { requests: 200, window_seconds: 3600 },
}]) : original(args, output))
expect((await service.list('home')).models[0]).toMatchObject({ state: 'running', requests: undefined })
})
it('marks an unfinished receipt interrupted after restart without replaying a command', async () => {
await mkdir(stateDir)
const hash = createHash('sha256').update('home').digest('hex').slice(0, 24)
await writeFile(join(stateDir, `${hash}.json`), JSON.stringify({ spec: 1, grid: 'home', operation: {
id: 'interrupted', modelId: 'org/Small-GGUF', action: 'start', stage: 'downloading', phase: 'running', updatedAt: '2026-01-01T00:00:00Z',
} }))
expect((await service.list('home')).models[0].operation).toMatchObject({ phase: 'failed', error: expect.stringContaining('interrupted') })
expect(calls.some(args => args[0] === 'pull' || args.includes('join'))).toBe(false)
})
it.each([{ spec: 9, grid: 'home' }, { spec: 1, grid: 'somebody-else' }])('discards an invalid or different-account receipt: %j', async receipt => {
await mkdir(stateDir)
const hash = createHash('sha256').update('home').digest('hex').slice(0, 24)
await writeFile(join(stateDir, `${hash}.json`), JSON.stringify({ ...receipt, operation: { id: 'private', modelId: 'org/Small-GGUF' } }))
expect((await service.list('home')).models[0].operation).toBeUndefined()
})
it('refuses to start if a durable receipt cannot be created', async () => {
await writeFile(stateDir, 'not a directory')
expect((await service.act('home', 'org/Small-GGUF', 'start')).error).toContain('saved')
expect(calls).toEqual([])
await rm(stateDir)
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation?.phase).toBe('done')
})
it.each(['model no longer fits', 'smaller fit only', 'no machine budget'])('checks changed hardware before downloading: %s', async scenario => {
await service.list('home')
if (scenario === 'no machine budget') {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args[0] === 'device-info' ? ok({}) : original(args, output))
}
request.mockImplementation(async () => response({ models: scenario === 'smaller fit only'
? [{ ...card(), versions: [{ ...card().versions[0], size_bytes: 32 }] }] : [] }))
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation?.error).toContain('make room')
expect(calls.some(args => args[0] === 'pull' || args.includes('join'))).toBe(false)
})
it('does not download when free disk space is insufficient', async () => {
catalogCards[0].versions[0].size_bytes = Number.MAX_SAFE_INTEGER
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation?.error).toContain('disk space')
expect(calls.some(args => args[0] === 'pull')).toBe(false)
})
it('does not start an incomplete download even after the downloader exits successfully', async () => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args[0] === 'pull' ? ok() : original(args, output))
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation?.error).toContain('incomplete')
expect(calls.some(args => args.includes('join'))).toBe(false)
})
it.each(['engine', 'join', 'info', 'leave'])('reports a failed %s without inventing success', async step => {
if (step === 'leave') { await service.act('home', 'org/Small-GGUF', 'start'); await service.settled() }
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes(step) ? { ...ok(), ok: false, stderr: 'private-token' } : original(args, output))
await service.act('home', 'org/Small-GGUF', step === 'leave' ? 'stop' : 'start'); await service.settled()
const snapshot = await service.list('home')
expect(snapshot.models[0].operation?.phase).toBe('failed')
expect(JSON.stringify(snapshot)).not.toContain('private-token')
})
it('keeps Stop available while a successful leave has not removed its run record', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('leave') ? ok() : original(args, output))
await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
expect((await service.list('home')).models[0]).toMatchObject({ canStop: true, operation: { phase: 'failed', error: expect.stringContaining('still stopping') } })
})
it('protects a shared runtime from simple Stop and Start', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const record = JSON.parse(await readFile(join(records, 'remote.json'), 'utf8'))
record.media = { port: 1234 }
await writeFile(join(records, 'remote.json'), JSON.stringify(record))
await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
expect((await service.list('home')).models[0].operation?.error).toContain('share its engine')
expect(calls.some(args => args.includes('leave'))).toBe(false)
})
it('stops a model beside the APIs a Swarm shares, which run nothing here, and not beside another engine', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const record = JSON.parse(await readFile(join(records, 'remote.json'), 'utf8'))
const share = { endpoint_url: `http://127.0.0.1:41001/${'a'.repeat(36)}/v1`, models: ['deepseek/x'] }
await writeFile(join(records, 'remote.json'), JSON.stringify({ ...record, engines: [...record.engines, share] }))
await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
expect((await service.list('home')).models[0].operation?.error ?? '').not.toContain('share its engine')
expect(calls.some(args => args.includes('leave'))).toBe(true)
// An engine that is not a share's endpoint still is one.
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const again = JSON.parse(await readFile(join(records, 'remote.json'), 'utf8'))
await writeFile(join(records, 'remote.json'), JSON.stringify({ ...again, engines: [...again.engines, { endpoint_url: 'http://127.0.0.1:41003/v1', models: ['y'] }] }))
calls.length = 0
await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
expect((await service.list('home')).models[0].operation?.error).toContain('share its engine')
expect(calls.some(args => args.includes('leave'))).toBe(false)
})
it('reverifies an already owned instance without downloading or joining again', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(calls.filter(args => args.includes('join'))).toHaveLength(1)
expect(request.mock.calls.filter(([url]) => String(url).includes(RELAY_CHAT))).toHaveLength(2)
})
it.each([false, true])('handles an explicit engine override (available=%s)', async available => {
const binary = join(root, 'custom-llama')
if (available) await writeFile(binary, '#!/bin/sh\nexit 0', { mode: 0o700 })
service = modelFixture({ stateDir, processEnv: { GRID_HOME: home, LLAMA_SERVER: binary }, run, request: request as typeof fetch , inventory })
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const operation = (await service.list('home')).models[0].operation
expect(operation?.phase).toBe(available ? 'done' : 'failed')
expect(calls.some(args => args[0] === 'engine')).toBe(false)
})
it('keeps a missing endpoint recoverable without sending a test to an unknown address', async () => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('info') ? ok() : original(args, output))
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation?.phase).toBe('failed')
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(false)
})
it('gives a model that thinks first room to finish, and starts it on its real reply', async () => {
// As Muse-Glimmer-30B: 8 tokens are all thinking, its content empty; with room it answers ok.
const original = request.getMockImplementation()!
const asked: number[] = []
request.mockImplementation(async (url, init) => {
if (!String(url).includes(RELAY_CHAT)) return original(url, init)
const tokens = JSON.parse(String(init?.body)).max_tokens as number
asked.push(tokens)
return response({ choices: [{ message: tokens < 512 ? { content: '', reasoning_content: 'Reply with the single word' } : { content: 'ok', reasoning_content: 'Probably just ok.' } }] })
})
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation).toMatchObject({ action: 'start', phase: 'done' })
expect(asked).toEqual([8, 512])
})
it.each(['reasoning only', 'http error', 'network error'])('requires a real test reply and bounds retries for %s', async scenario => {
const original = request.getMockImplementation()!
let attempted!: () => void
const first = new Promise<void>(resolve => { attempted = resolve })
request.mockImplementation(async (url, init) => {
// The catalog, and the engine on this machine, answer normally: this is about the RELAY check.
if (String(url).includes('/catalog') || String(url).includes('127.0.0.1')) return original(url, init)
attempted()
if (scenario === 'network error') throw new Error('private-token')
return scenario === 'http error' ? new Response('private-token', { status: 503 })
: response({ choices: [{ message: { reasoning_content: 'ok' } }] })
})
vi.useFakeTimers({ toFake: ['setTimeout', 'Date'] })
await service.act('home', 'org/Small-GGUF', 'start')
await first
await vi.advanceTimersByTimeAsync(182_000)
await service.settled()
expect((await service.list('home')).models[0].operation).toMatchObject({ phase: 'failed', error: expect.stringContaining('did not answer') })
expect(JSON.stringify(await service.list('home'))).not.toContain('private-token')
vi.useRealTimers()
})
it('rejects unsafe, incomplete and duplicate fit cards while supporting split downloads', () => {
const invalid = [null, {}, { ...card(), runnable: false }, { ...card(), format: 'safetensors' },
{ ...card(), repo_id: '' }, { ...card(), fit: {} },
...['--flag:model.gguf', 'repo-only', 'org/repo:model.bin'].map(pull_spec => ({ ...card(), versions: [{ ...card().versions[0], pull_spec }] })),
{ ...card(), fit: { version: 'Q4', ctx: 0 } },
{ ...card(), versions: [{ ...card().versions[0], size_bytes: -1 }] }]
expect(compatibleModels({ models: invalid })).toEqual([])
expect(compatibleModels({ models: [card(), card()] })).toHaveLength(1)
expect(compatibleModels({ models: [{ ...card(), versions: [{ ...card().versions[0], urls: ['bad-url'] }] }] })[0].files).toEqual(['Small-Q4.gguf'])
expect(compatibleModels({ models: [{ ...card(), versions: [{ ...card().versions[0], urls: null }] }] })[0].files).toEqual(['Small-Q4.gguf'])
expect(compatibleModels({ models: [{ ...card(), task: 'image-text-to-text', versions: [{ ...card().versions[0], urls: ['https://example.test/part-1.gguf', 'https://example.test/part-2.gguf'] }] }] })[0].files).toEqual(['part-1.gguf', 'part-2.gguf'])
})
it('routes production-default dependencies through the bounded Grid subprocess runner', async () => {
vi.stubEnv('GRID_HOME', home)
vi.stubGlobal('fetch', request)
const rpc = vi.spyOn(GridFleetRpc.prototype, 'run').mockImplementation(async (_owner, _id, options, output) => run(options.args, output))
service = modelFixture({ stateDir, inventory })
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation?.phase).toBe('done')
expect(rpc).toHaveBeenCalledWith('local-models', expect.any(String), expect.objectContaining({ timeoutMs: 30_000, thinking: false }), undefined, 4 * 1024 * 1024)
expect(rpc.mock.calls.find(call => call[2].args[0] === 'pull')?.[2].timeoutMs).toBe(30 * 60_000)
})
it.each(['memory changed', 'file removed'])('rechecks an imported deployment before restarting when %s', async scenario => {
catalogCards = []
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await run(['--remote', 'join', 'home'])
await service.list('home')
await service.act('home', 'local:Small-Q4.gguf', 'stop'); await service.settled()
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => {
if (args[0] === 'device-info') {
if (scenario === 'file removed') await rm(join(home, 'models', 'Small-Q4.gguf'))
else return ok({ usable_bytes: 0 })
}
return original(args, output)
})
const ack = await service.act('home', 'local:Small-Q4.gguf', 'start'); await service.settled()
expect(ack.operation).toMatchObject({ phase: 'failed', error: expect.stringContaining(scenario === 'memory changed' ? 'make room' : 'no longer available') })
expect(calls.filter(args => args.includes('join'))).toHaveLength(1)
})
it('tells, per model, the window and speed the catalog expects on this machine, and the free disk', async () => {
catalogCards = [{ ...card(), params_b: 9, fit: { ...card().fit, est_tok_s: 21.3 } } as ReturnType<typeof card>, card('org/Bare-GGUF')]
catalogCards[1].versions[0].pull_spec = 'org/Bare-GGUF:Bare-Q4.gguf'
const snapshot = await service.list('home')
expect(snapshot.models[0]).toMatchObject({ name: 'Small', contextWindow: 131072, estTokS: 21.3, paramsB: 9 })
// A catalog row that does not say is shown without, never as zero.
expect(snapshot.models[1]).not.toHaveProperty('estTokS')
expect(snapshot.models[1]).not.toHaveProperty('paramsB')
expect(snapshot.freeDiskBytes).toBeGreaterThan(0)
})
it("tells the catalog the machine's measured bandwidth and compute, which its speed estimates rest on", async () => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args[0] === 'device-info'
? ok({ device_class: 'apple-silicon', backend: 'metal', usable_bytes: 54 * 1024 ** 3, memory: { total_gb: 64 },
mem_bandwidth_gbps: 400, compute_gflops: 15600 })
: original(args, output))
await service.list('home')
const sent = request.mock.calls.find(([url]) => String(url).includes('/catalog'))!
// Half of its 64 GB, not the 54 GB grid reports free: the rest stays for everything else it runs.
expect(JSON.parse(String(sent[1]?.body)).device).toEqual({ device_class: 'apple-silicon', usable_bytes: 32 * 1024 ** 3,
backend: 'metal', mem_bandwidth_gbps: 400, compute_gflops: 15600 })
})
it("counts the VRAM this picker's own engine holds as the next model's, on an NVIDIA card", async () => {
// A 24 GB card running our 27B: 2.7 GB free beside it. Starting another model stops ours first, so
// the catalog is fitted to what ours holds as well, never to what is left beside it.
const MiB = 1024 ** 2, original = run.getMockImplementation()!
const card = { device_class: 'nvidia', backend: 'cuda', usable_bytes: 2746 * MiB, memory: { total_gb: 16 }, gpus: [{ backend: 'cuda', memory_total_mb: 24564 }] }
const grid = { pid: 30956, name: join(home, 'bin', 'llama-server'), bytes: 21000 * MiB }
const browser = { pid: 2014, name: '/usr/lib/chromium/chromium', bytes: 300 * MiB }
// An app's engine this picker started, known by its record: its binary, or its pid.
await mkdir(stateDir, { recursive: true })
await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([{ spec: 1, modelId: 'app:llama.cpp:Other.gguf', grid: 'home',
name: 'Other', engine: 'llama.cpp', served: 'other', alias: 'other', port: 9000, binary: '/opt/llama/llama-server', pid: 4242 }]))
const fitted = async (device: Record<string, unknown>, processes?: GpuProcess[]): Promise<number | undefined> => {
run.mockImplementation(async (args, output) => args[0] === 'device-info' ? ok(device) : original(args, output))
request.mockClear()
const models = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory,
...(processes ? { gpuProcesses: async () => processes } : {}) })
const snapshot = await models.list('home')
const sent = JSON.parse(String(request.mock.calls.find(([url]) => String(url).includes('/catalog'))![1]?.body)).device.usable_bytes
expect(snapshot.fitBytes).toBe(sent)
return sent
}
expect(await fitted(card, [grid, browser])).toBe((2746 + 21000) * MiB)
expect(await fitted(card, [{ pid: 777, name: '/opt/llama/llama-server', bytes: 4000 * MiB }, { pid: 4242, name: 'ollama', bytes: 1000 * MiB }]))
.toBe((2746 + 5000) * MiB)
// Never more than the card has, and with no card's size to hold it to, all of it.
expect(await fitted(card, [{ ...grid, bytes: 30000 * MiB }])).toBe(24564 * MiB)
expect(await fitted({ ...card, gpus: [] }, [grid])).toBe((2746 + 21000) * MiB)
// Nothing of ours on it, no way to tell, no figure to add to, or no NVIDIA card: as grid says.
expect(await fitted(card, [browser])).toBe(2746 * MiB)
expect(await fitted(card)).toBe(2746 * MiB)
expect(await fitted({ ...card, usable_bytes: undefined }, [grid])).toBeUndefined()
expect(await fitted({ device_class: 'apple-silicon', backend: 'metal', usable_bytes: 54 * 1024 ** 3, memory: { total_gb: 64 } }, [grid]))
.toBe(32 * 1024 ** 3)
})
it('leaves out a measurement the machine did not report, so the catalog falls back to its own', async () => {
await service.list('home')
const sent = request.mock.calls.find(([url]) => String(url).includes('/catalog'))!
const device = JSON.parse(String(sent[1]?.body)).device
expect(device).not.toHaveProperty('mem_bandwidth_gbps')
expect(device).not.toHaveProperty('compute_gflops')
})
it('names one model the same whatever its quantization', () => {
for (const name of ['Qwen3.6-35B-A3B', 'unsloth/Qwen3.6-35B-A3B-GGUF', 'Qwen3.6-35B-A3B-UD-Q5_K_XL.gguf', 'Qwen3.6-35B-A3B-Q4_K_M',
'qwen3.6-35b-a3b-bf16', 'Qwen3.6-35B-A3B-IQ4_XS', 'Qwen3.6-35B-A3B-MXFP4_MOE']) {
expect(modelFamily(name)).toBe('qwen3.6-35b-a3b')
}
// Another model of the family stays another model.
expect(modelFamily('Qwen3.6-35B-A3B-MTP')).not.toBe(modelFamily('Qwen3.6-35B-A3B'))
})
it('does not offer to download another quantization of a model this machine already has', async () => {
catalogCards = [card(), card('org/Other-GGUF')]
catalogCards[1].versions[0].pull_spec = 'org/Other-GGUF:Other-Q4.gguf'
// Small at Q8, imported from an existing setup and serving: the catalog's Q4 of it is a second copy.
serving = true
await writeFile(join(home, 'models', 'Small-Q8_0.gguf'), Buffer.alloc(96))
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node',
engines: [{ endpoint_url: null, models: ['Small-Q8_0.gguf'] }], advertise_as: [] }))
await writeFile(join(records, 'remote.heartbeat'), '')
run.mockImplementation(async (args: string[]) => {
calls.push(args)
if (args[0] === 'device-info') return ok({ device_class: 'apple-silicon', backend: 'metal', usable_bytes: 54 * 1024 ** 3, memory: { total_gb: 64 } })
if (args.includes('ls')) return ok([{ grid: 'home', id: 'grid-home' }])
if (args.includes('info') && args.includes('--json')) return ok({ grid: 'home', status: 'running' })
if (args.includes('engines')) return ok([{ node_id: 'local-node', online: true, models: ['Small-Q8_0.gguf'] }])
return ok()
})
const snapshot = await service.list('home')
expect(snapshot.models.map(model => [model.id, model.state])).toEqual([
['org/Other-GGUF', 'available'],
['local:Small-Q8_0.gguf', 'running'],
])
})
it('keeps imported downloads visible, merges catalog matches, and removes deleted files', async () => {
catalogCards = []
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await run(['--remote', 'join', 'home'])
await service.list('home')
await service.act('home', 'local:Small-Q4.gguf', 'stop'); await service.settled()
expect((await service.list('home')).models[0].state).toBe('downloaded')
catalogCards = [card()]
expect((await service.list('home', true)).models).toHaveLength(1)
catalogCards = []
await rm(join(home, 'models', 'Small-Q4.gguf'))
expect((await service.list('home', true)).models).toEqual([])
})
it('does not mistake a directory for downloaded weights or fabricate unavailable device memory', async () => {
await mkdir(join(home, 'models', 'Small-Q4.gguf'))
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args[0] === 'device-info' ? ok({}) : original(args, output))
const view = await service.list('home')
expect(view.models[0].state).toBe('available')
expect(view.memoryBytes).toBeUndefined()
})
it('cannot start a stale catalog choice while compatibility is unavailable', async () => {
await service.list('home')
catalogFails = true
await service.list('home', true)
const ack = await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(ack.operation).toMatchObject({ phase: 'failed', error: expect.stringContaining('could not be checked') })
expect(calls.some(args => args[0] === 'pull')).toBe(false)
})
it('ignores non-progress output and boundedly saves valid download progress', async () => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => {
if (args[0] === 'pull') { output?.('connecting…'); output?.('150%'); output?.('0%'); output?.('2%') }
return original(args, output)
})
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation?.phase).toBe('done')
})
it('releases the operation after receipt storage fails during a download', async () => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => {
if (args[0] === 'pull') {
rmSync(stateDir, { recursive: true, force: true }); writeFileSync(stateDir, 'storage unavailable')
output?.('42%')
await new Promise(resolve => setImmediate(resolve))
throw new Error('private filesystem detail')
}
return original(args, output)
})
const ack = await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(ack.operation).toMatchObject({ phase: 'failed', error: 'The model could not finish. Start again to retry.' })
expect((await service.list('home')).busy).toBe(false)
await rm(stateDir)
run.mockImplementation(original)
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect((await service.list('home')).models[0].operation?.phase).toBe('done')
})
it('recovers malformed aliases and preserves failure status for a model with missing weights', async () => {
catalogCards = []
serving = true
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', advertise_as: [42, ''], engines: [{ models: ['Small-Q4.gguf'] }] }))
await writeFile(join(records, 'remote.heartbeat'), '')
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('leave') ? { ...ok(), ok: false } : original(args, output))
await service.act('home', 'local:Small-Q4.gguf', 'stop'); await service.settled()
expect((await service.list('home')).models[0]).toMatchObject({ name: 'Small-Q4', state: 'running', operation: { phase: 'failed' } })
})
})
describe('a run record its engine did not outlive', () => {
// As on an NVIDIA machine after a restart: grid's record of the engine survives in run/engines, its
// process does not, and the model read as held by it — "It cannot start now. Try again in a moment."
it('holds nothing: the model reads downloaded and starts, the record cleared before the join', async () => {
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', meta_name: 'harness', pid: deadPid(),
engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'] }], advertise_as: [] }))
expect((await service.list('home', true)).models[0]).toMatchObject({ id: 'org/Small-GGUF', state: 'downloaded', canStart: true, canStop: false })
const ack = await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(ack.error).toBeUndefined()
const leave = calls.findIndex(args => args.join(' ') === '--remote leave home --engine Small-Q4.gguf')
const joined = calls.findIndex(args => args.includes('join'))
expect(leave).toBeGreaterThanOrEqual(0)
expect(joined).toBeGreaterThan(leave)
expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'running', operation: { phase: 'done' } })
})
it('a start goes on when the record cannot be left, and nothing stands in another model\'s way', async () => {
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'stale.json'), JSON.stringify({ node_id: 'old-node', pid: deadPid(),
engines: [{ endpoint_url: null, models: ['Other-Q4.gguf'] }], advertise_as: [] }))
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => { if (!args.includes('leave')) return original(args, output); calls.push(args); return { ...ok(), ok: false } })
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(calls).toContainEqual(['--remote', 'leave', 'home', '--engine', 'Other-Q4.gguf'])
expect(calls.some(args => args.includes('join'))).toBe(true)
expect((await service.list('home', true)).models[0].operation).toMatchObject({ phase: 'done' })
})
})
describe('a download that stops, or is cancelled', () => {
/** A pull that sends nothing more, as grid's did when its connection stalled (no read timeout): it ends
* only when it is aborted. */
const hang = () => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args: string[], output?: (s: string) => void, _timeout?: number, signal?: AbortSignal) => {
if (args[0] !== 'pull') return original(args, output)
calls.push(args)
await writeFile(join(home, 'models', 'Small-Q4.gguf.part'), Buffer.alloc(24))
output?.('36%')
return new Promise<GridFleetResult>(resolve => signal?.addEventListener('abort', () => resolve({ ok: false, code: 124, stdout: '', stderr: '', error: 'interrupted' })))
})
}
it('is not downloaded while a part of it is unfinished — its projector, say — and can be got again', async () => {
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(home, 'models', 'Small-Q4.mmproj.gguf.part'), Buffer.alloc(8))
expect((await service.list('home', true)).models[0]).toMatchObject({ id: 'org/Small-GGUF', state: 'available', canStart: true })
await rm(join(home, 'models', 'Small-Q4.mmproj.gguf.part'))
expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'downloaded' })
})
it('that has stalled is ended, to be started again where it stopped', async () => {
hang()
service = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory, pullStallMs: 40, pullWatchMs: 5 })
await service.act('home', 'org/Small-GGUF', 'download'); await service.settled()
expect((await service.list('home', true)).models[0].operation).toMatchObject({ action: 'download', phase: 'failed', error: 'The download stalled. Start again to resume.' })
// What it got is kept: a start again resumes from it.
expect(existsSync(join(home, 'models', 'Small-Q4.gguf.part'))).toBe(true)
})
it('is cancelled by a stop, and what it downloaded goes: the row is one to get again, not a failure', async () => {
hang()
await writeFile(join(home, 'models', 'Small-Q4.mmproj.gguf.part'), Buffer.alloc(8))
await writeFile(join(home, 'models', 'Other-Q4.gguf'), Buffer.alloc(8))
const started = await service.act('home', 'org/Small-GGUF', 'download')
await vi.waitFor(() => expect(calls.some(args => args[0] === 'pull')).toBe(true))
const stop = await service.act('home', 'org/Small-GGUF', 'stop')
expect(stop.operation?.id).toBe(started.operation?.id)
await service.settled()
expect((await readdir(join(home, 'models'))).sort()).toEqual(['Other-Q4.gguf'])
const [model] = (await service.list('home', true)).models
expect(model).toMatchObject({ state: 'available', operation: { action: 'stop', phase: 'done' } })
expect(model.operation).not.toHaveProperty('error')
// Another model's operation is not cancelled by a stop of this one.
hang()
await service.act('home', 'org/Small-GGUF', 'download')
await vi.waitFor(() => expect(calls.filter(args => args[0] === 'pull')).toHaveLength(2))
expect(await service.act('home', 'org/Other-GGUF', 'stop')).toEqual({ error: 'Wait for the current model operation to finish.' })
await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
})
})
/** A pid that existed a moment ago and no longer does. */
function deadPid(): number {
return Number(spawnSync(process.execPath, ['-e', 'process.stdout.write(String(process.pid))'], { encoding: 'utf8' }).stdout)
}
describe('the Model Manager while its grid sleeps (grid-reads-without-waking issue 02)', () => {
const parkedRecord = (pid: unknown) => JSON.stringify({ node_id: 'local-node', meta_name: 'This computer', pid,
engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'] }], advertise_as: [] })
it('is not an inventory error: Start and Pause stay enabled, and a parked engine reads running', async () => {
gridState = 'asleep'
const next = card('org/Next-GGUF'); next.versions[0].pull_spec = 'org/Next-GGUF:Next.gguf'
catalogCards = [card(), next]
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'remote.json'), parkedRecord(process.pid))
const snapshot = await service.list('home')
expect(snapshot.notice).not.toBe('Running models could not be checked. Try again.')
expect(snapshot.models[0]).toMatchObject({ id: 'org/Small-GGUF', state: 'running', gridAsleep: true, canStop: true })
expect(snapshot.models[1]).toMatchObject({ id: 'org/Next-GGUF', canStart: true })
expect(inventory).toHaveBeenCalledWith('home', false)
})
it('reads a parked engine running on a row the catalog does not know, too', async () => {
gridState = 'asleep'; catalogCards = []
await writeFile(join(records, 'remote.json'), parkedRecord(process.pid))
expect((await service.list('home')).models).toEqual([expect.objectContaining({ id: 'local:Small-Q4.gguf', state: 'running', gridAsleep: true, canStop: true })])
})
it.each([['a join mid-spawn (pid 0)', () => 0], ['no pid at all', () => 'x']])(
'does not call %s parked', async (_name, pid) => {
gridState = 'asleep'; catalogCards = []
await writeFile(join(records, 'remote.json'), parkedRecord(pid()))
const [row] = (await service.list('home')).models
expect(row).toMatchObject({ id: 'local:Small-Q4.gguf', state: 'available', canStop: true })
expect(row).not.toHaveProperty('gridAsleep')
})
it('does not call a process that is gone parked, nor offer it at all: nothing of it is left to stop', async () => {
gridState = 'asleep'; catalogCards = []
await writeFile(join(records, 'remote.json'), parkedRecord(deadPid()))
expect((await service.list('home')).models).toEqual([])
})
it('an engine the awake grid lists is running with no asleep flag', async () => {
serving = true
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'remote.json'), parkedRecord(process.pid))
await writeFile(join(records, 'remote.heartbeat'), '')
const [row] = (await service.list('home')).models
expect(row).toMatchObject({ state: 'running' })
expect(row).not.toHaveProperty('gridAsleep')
})
it('a grid that answered nothing readable is still an inventory error', async () => {
answered = 'unknown'
expect((await service.list('home')).notice).toBe('Running models could not be checked. Try again.')
})
it('and so is one whose owner status could not be read either — nothing says it is down', async () => {
inventory.mockResolvedValueOnce({ state: 'unknown', nodes: [], status: null })
expect((await service.list('home')).notice).toBe('Running models could not be checked. Try again.')
})
it.each([['failed', { ...ok(), ok: false, code: 1 }], ['answered with something that is not JSON', { ...ok(), stdout: 'not json' }]])(
'so is a grid list that %s', async (_how, answer) => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('ls') ? answer : original(args, output))
expect((await service.list('home')).notice).toBe('Running models could not be checked. Try again.')
})
it('tells its owner when a start or stop has finished, so the lists are read again', async () => {
const onChanged = vi.fn()
service = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory, onChanged })
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(onChanged).toHaveBeenCalledOnce()
})
})
describe('readRunRecords — what servedHere knows about this computer', () => {
it('reads the name, the advertised ids (else the models under the display rule) and whether the pid lives', async () => {
await writeFile(join(records, 'a.json'), JSON.stringify({ meta_name: 'mac', pid: process.pid, advertise_as: ['Team/Model', 7, ''], models: ['ignored.gguf'] }))
await writeFile(join(records, 'b.json'), JSON.stringify({ meta_name: 'mac', pid: deadPid(), models: ['Qwen3-8B-Q4_K_M.GGUF', 'plain'] }))
await writeFile(join(records, 'c.json'), JSON.stringify({ pid: 0, models: 'not a list' }))
await writeFile(join(records, 'd.json'), '{ not json')
await writeFile(join(records, 'e.heartbeat'), '')
const found = (await readRunRecords(home, 'grid-home')).sort((x, y) => x.ids.join().localeCompare(y.ids.join()))
expect(found).toEqual([
{ name: '', ids: [], pid: null, alive: false, advertised: false },
{ name: 'mac', ids: ['Qwen3-8B-Q4_K_M', 'plain'], pid: expect.any(Number), alive: false, advertised: false },
// Its own aliases, as given: the one spelling that is the name served now.
{ name: 'mac', ids: ['Team/Model'], pid: process.pid, alive: true, advertised: true },
])
})
it.each(['', '../grid-home', 'missing'])('answers nothing for grid id %j', async (gridId) => {
await writeFile(join(records, 'a.json'), JSON.stringify({ meta_name: 'mac', pid: process.pid, advertise_as: ['X'] }))
expect(await readRunRecords(home, gridId)).toEqual([])
})
})
describe('a context a coding agent can work in', () => {
// ⚠️ REGRESSION. Every model was offered and started at 16K or less — a model that loads, passes
// its one-word check, and then cannot hold a coding agent's first prompt. Codex, Claude Code and
// OpenCode all need 64K at the least; the most this machine can give is what a start now asks for.
it.each([
[32768, false], [65535, false], [65536, true], [262144, true],
])("a catalog fit of %i on this machine is offered: %s", async (ctx, offered) => {
catalogCards[0].fit.ctx = ctx
expect((await service.list('home')).models.length).toBe(offered ? 1 : 0)
})
it('pins the whole fit, capped at the most the model was trained for', async () => {
Object.assign(catalogCards[0].fit, { ctx: 262144, max_ctx: 196608 })
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const joined = calls.find(args => args.includes('join'))!
expect(joined[joined.indexOf('--ctx-size') + 1]).toBe('196608')
})
it('shrinks to what fits at start when memory has since tightened, but never under 64K', async () => {
catalogCards[0].fit.ctx = 262144
await service.list('home')
catalogCards[0].fit.ctx = 98304
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const joined = calls.find(args => args.includes('join'))!
expect(joined[joined.indexOf('--ctx-size') + 1]).toBe('98304')
})
it('refuses to start at all when the fit at start is under 64K', async () => {
await service.list('home')
catalogCards[0].fit.ctx = 32768
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(calls.some(args => args.includes('join'))).toBe(false)
expect((await service.list('home')).models[0].operation).toMatchObject({ phase: 'failed', error: expect.stringContaining('make room') })
})
it('hides a model on disk whose file can never hold 64K, and keeps one it cannot read', async () => {
catalogCards = []; serving = true
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', engines: [{ models: ['Small-Q4.gguf'] }] }))
await writeFile(join(records, 'remote.heartbeat'), '')
const id = (await service.list('home')).models[0].id
await service.act('home', id, 'stop'); await service.settled()
// Unreadable header: unknown is not "too small".
expect((await service.list('home', true)).models.map(m => m.id)).toEqual([id])
trainedWindow = 32768
const fresh = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory })
expect((await fresh.list('home')).models).toEqual([])
})
it.each([
[131072, 'done'], [undefined, 'done'], [32768, 'failed'],
] as const)('after a start that served %s tokens, the start is %s', async (window, phase) => {
servedWindow = window
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const operation = (await service.list('home', true)).models[0].operation
expect(operation?.phase).toBe(phase)
if (phase === 'failed') {
// Served too small: taken back down rather than left answering prompts it cannot hold.
expect(calls.find(args => args.includes('leave'))).toEqual(['--remote', 'leave', 'home', '--engine', 'Small-Q4.gguf'])
expect(operation?.error).toBe('Small could only get a 32K context here. Coding agents need at least 64K. Close some apps, or choose a smaller model.')
} else {
expect(calls.some(args => args.includes('leave'))).toBe(false)
}
})
})
describe('stepping down to the context that actually runs', () => {
// ⚠️ REGRESSION, measured on a 64 GB M1 Max. Left to the engine, a 35B model took its whole
// trained 256K, loaded, and then failed its very first request — "Insufficient Memory
// (kIOGPUCommandBufferCallbackErrorOutOfMemory)", then "Compute error." for every request after.
// The relay check waited three minutes and said only "did not answer". The same file ran at 128K.
const joinedAt = () => calls.filter(args => args.includes('join')).map(args => Number(args[args.indexOf('--ctx-size') + 1]))
/** The engine on this machine runs out of memory at any context above [limit]. */
const gpuHolds = (limit: number) => {
const original = request.getMockImplementation()!
request.mockImplementation(async (url, init) => {
if (String(url).includes('127.0.0.1') && String(url).endsWith('/chat/completions') && joinedAt().at(-1)! > limit) {
return new Response(JSON.stringify({ error: { code: 500, message: 'Compute error.', type: 'server_error' } }), { status: 500 })
}
return original(url, init)
})
}
it('halves to the 64K floor and no further', () => {
expect(contextLadder(262144)).toEqual([262144, 131072, 65536])
expect(contextLadder(196608)).toEqual([196608, 98304, 65536])
expect(contextLadder(100000)).toEqual([100000, 65536])
expect(contextLadder(65536)).toEqual([65536])
expect(contextLadder(50000)).toEqual([])
})
it('takes a size the GPU cannot run back down, and settles on the next that does', async () => {
catalogCards[0].fit.ctx = 262144
gpuHolds(131072)
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(joinedAt()).toEqual([262144, 131072])
// Down between the two, and Grid's registration restored before the second join.
const between = calls.slice(calls.findIndex(a => a.includes('join')) + 1, calls.findLastIndex(a => a.includes('join')))
expect(between).toContainEqual(['--remote', 'leave', 'home', '--engine', 'Small-Q4.gguf'])
expect(between).toContainEqual(['--remote', 'sync'])
expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'running', operation: { phase: 'done' } })
})
it('says it is memory, in words, when not even 64K runs — without waiting on the relay', async () => {
catalogCards[0].fit.ctx = 262144
gpuHolds(0)
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(joinedAt()).toEqual([262144, 131072, 65536])
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(false)
expect((await service.list('home', true)).models[0].operation).toMatchObject({ phase: 'failed',
error: 'This computer does not have the memory to run Small with a 64K context. Close some apps, or choose a smaller model.' })
})
it('steps down from a size whose join fails, since a failed allocation at load looks like that', async () => {
catalogCards[0].fit.ctx = 262144
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('join') && args.includes('262144') ? refused('engine exited') : original(args, output))
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const joins = run.mock.calls.map(([args]) => args).filter(args => args.includes('join'))
expect(joins.map(a => a[a.indexOf('--ctx-size') + 1])).toEqual(['262144', '131072'])
expect((await service.list('home', true)).models[0].operation?.phase).toBe('done')
})
it.each([
['nothing is listening on this machine', () => { throw new Error('ECONNREFUSED') }],
['the engine fails for another reason', () => new Response('{"error":{"message":"template error"}}', { status: 500 })],
// Not a load in progress (503), so not waited on for ten minutes.
['its health check answers something unexpected', () => new Response('teapot', { status: 418 })],
])('leaves the judgement to the relay check when %s', async (_case, answer) => {
const original = request.getMockImplementation()!
request.mockImplementation(async (url, init) => String(url).includes('127.0.0.1') ? answer() : original(url, init))
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(joinedAt()).toEqual([131072])
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(true)
expect((await service.list('home', true)).models[0].operation?.phase).toBe('done')
})
it('starts a file on disk at the most its header says it holds', async () => {
catalogCards = []; serving = true
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', engines: [{ models: ['Small-Q4.gguf'] }] }))
await writeFile(join(records, 'remote.heartbeat'), '')
const id = (await service.list('home')).models[0].id
await service.act('home', id, 'stop'); await service.settled()
trainedWindow = 262144
const fresh = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory })
await fresh.act('home', id, 'start'); await fresh.settled()
expect(joinedAt()).toEqual([262144])
})
it.each(['request disconnected', 'error body disconnected'])('leaves the final judgement to the relay when a local probe has %s', async scenario => {
const original = request.getMockImplementation()!
request.mockImplementation(async (url, init) => {
if (String(url).includes('127.0.0.1') && String(url).endsWith('/chat/completions')) {
if (scenario === 'request disconnected') throw new Error('socket closed')
return new Response(new ReadableStream({ start(controller) { controller.error(new Error('body socket closed')) } }), { status: 500 })
}
return original(url, init)
})
const result = await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(result.operation?.phase).toBe('done')
expect(joinedAt()).toEqual([131072])
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(true)
})
it('bounds an engine that keeps reporting loading and still lets the relay establish readiness', async () => {
const original = request.getMockImplementation()!
let attempted!: () => void
const firstAttempt = new Promise<void>(resolve => { attempted = resolve })
request.mockImplementation(async (url, init) => {
if (String(url).includes('127.0.0.1') && String(url).endsWith('/health')) {
attempted(); return new Response('loading', { status: 503 })
}
return original(url, init)
})
vi.useFakeTimers({ toFake: ['Date', 'setTimeout'] })
const result = await service.act('home', 'org/Small-GGUF', 'start')
await firstAttempt
await vi.advanceTimersByTimeAsync(601_000)
await service.settled()
expect(result.operation?.phase).toBe('done')
expect(joinedAt()).toEqual([131072])
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(true)
})
it.each(['grid sleeps after leave', 'old grid cannot report status'])('restores the grid between context attempts when the %s', async scenario => {
catalogCards[0].fit.ctx = 262144
gpuHolds(131072)
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => {
if (args.includes('leave') && scenario === 'grid sleeps after leave') gridState = 'asleep'
if (args.includes('info') && args.includes('--json') && scenario === 'old grid cannot report status') return refused('status unavailable')
return original(args, output)
})
const result = await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
expect(result.operation?.phase).toBe('done')
expect(joinedAt()).toEqual([262144, 131072])
expect(calls.filter(args => args.join(' ') === '--remote start home')).toHaveLength(scenario === 'grid sleeps after leave' ? 1 : 0)
})
it.each(['unavailable', 'legacy id', 'one unnamed provider', 'several other providers'])('uses only identifiable context evidence when inventory is %s', async scenario => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => {
if (!args.includes('engines')) return original(args, output)
if (scenario === 'unavailable') return refused('inventory unavailable')
const capabilities = { 'Small-Q4': { context_length: 16384 } }
if (scenario === 'legacy id') return ok([{ id: 'local-node', model_capabilities: capabilities }])
if (scenario === 'one unnamed provider') return ok([{ model_capabilities: capabilities }])
return ok([{ node_id: 'other-one', model_capabilities: capabilities }, { node_id: 'other-two', model_capabilities: capabilities }])
})
const result = await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const identified = scenario === 'legacy id' || scenario === 'one unnamed provider'
expect(result.operation?.phase).toBe(identified ? 'failed' : 'done')
expect(calls.some(args => args.includes('leave'))).toBe(identified)
if (identified) expect(result.operation?.error).toContain('could only get a 16K context')
})
})
describe("a model is labelled with the machine's name as Harness shows it", () => {
// ⚠️ REGRESSION. Started from the Models modal, a model joined with no `--name`, so Grid labelled
// it with the host name: `mac.lan` under "On your machines" while Machines called it `M2`.
const joined = () => calls.find(args => args.includes('join'))!
it('joins under the Harness name', async () => {
const named = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, machineName: () => ' M2 ', inventory })
await named.act('home', 'org/Small-GGUF', 'start'); await named.settled()
expect(joined().slice(joined().indexOf('--name'), joined().indexOf('--name') + 2)).toEqual(['--name', 'M2'])
})
it.each([
['is not known yet', () => null],
['would read as a flag', () => '--all'],
['carries a control character', () => 'M2\nevil'],
])('leaves --name off when the name %s, rather than pass it', async (_case, machineName) => {
const named = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, machineName, inventory })
await named.act('home', 'org/Small-GGUF', 'start'); await named.settled()
expect(joined()).not.toContain('--name')
expect((await named.list('home', true)).models[0].operation?.phase).toBe('done')
})
})
describe('download without starting', () => {
it('stores the weights without waking a grid, installing an engine, or sending a prompt', async () => {
gridState = 'asleep'
const result = await service.act('home', 'org/Small-GGUF', 'download')
await service.settled()
expect(result.operation).toMatchObject({ action: 'download', phase: 'done' })
expect(calls.filter(args => args[0] === 'pull')).toHaveLength(1)
expect(calls.some(args => args.includes('sync') || args.includes('join') || args.includes('start') || args.includes('install'))).toBe(false)
expect(request.mock.calls.some(([url]) => String(url).includes('chat/completions'))).toBe(false)
expect(gridState).toBe('asleep')
expect((await service.list('home')).models[0]).toMatchObject({ state: 'downloaded', canStart: true, canStop: false })
expect((await service.list('home')).supportsDownload).toBe(true)
})
it('a download request for a running model never runs another reply test', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
calls.length = 0; request.mockClear()
const result = await service.act('home', 'org/Small-GGUF', 'download'); await service.settled()
expect(result.operation?.phase).toBe('done')
expect(serving).toBe(true)
expect(calls.some(args => args.includes('join') || args.includes('pull') || args.includes('sync'))).toBe(false)
expect(request.mock.calls.some(([url]) => String(url).includes('chat/completions'))).toBe(false)
})
it('keeps failed downloads retryable and starts only after a separate Start', async () => {
downloadFails = true
const first = await service.act('home', 'org/Small-GGUF', 'download')
await service.settled()
expect(first.operation?.phase).toBe('failed')
expect(serving).toBe(false)
downloadFails = false
await service.act('home', 'org/Small-GGUF', 'download')
await service.settled()
expect(serving).toBe(false)
const pulls = calls.filter(args => args[0] === 'pull').length
await service.act('home', 'org/Small-GGUF', 'start')
await service.settled()
expect(serving).toBe(true)
expect(calls.filter(args => args[0] === 'pull')).toHaveLength(pulls)
})
})
describe('models other apps downloaded, started in their own app', () => {
const GiB = 1024 ** 3
const ollama: AppModel = { id: 'app:ollama:llama3.2:3b', name: 'llama3.2:3b', app: 'ollama', engine: 'ollama', ref: 'llama3.2:3b',
binary: '/usr/local/bin/ollama', sizeBytes: 2 * GiB, quant: 'Q4_K_M', contextLength: 131072, kvBytesPerToken: 112 * 1024 }
const studio: AppModel = { id: 'app:lm-studio:google/gemma-4-e2b', name: 'gemma-4-e2b', app: 'lm-studio', engine: 'lm-studio',
ref: 'google/gemma-4-e2b', binary: '/home/me/.lmstudio/bin/lms', sizeBytes: 4 * GiB, contextLength: 131072 }
let apps: AppModel[], up: Set<string>, joinedAliases: Set<string>, window: number | undefined
let ops: { [K in keyof AppEngineOps]: Mock<AppEngineOps[K]> }
/** The grid fake, with a `join --at` that lists the alias rather than writing a `--serve` run record. */
const appService = (over: Partial<ConstructorParameters<typeof LocalModels>[0]> = {}) => {
const base = run.getMockImplementation()!
run.mockImplementation(async (args, output) => {
if (args.includes('join') && args.includes('--at')) {
calls.push(args)
joinedAliases.add(args[args.indexOf('--advertise-as') + 1]!)
return ok()
}
if (args.includes('leave') && args.includes('--engine') && joinedAliases.has(args[args.indexOf('--engine') + 1]!)) {
calls.push(args)
joinedAliases.delete(args[args.indexOf('--engine') + 1]!)
return ok()
}
return base(args, output)
})
inventory.mockImplementation(async () => ({ state: 'awake', status: 'running',
nodes: [...joinedAliases].map(alias => ({ node_id: 'local-node', online: true, models: [alias] })) }))
return modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory,
appModels: async () => apps, appEngines: ops as unknown as AppEngineOps, ...over })
}
beforeEach(() => {
apps = [ollama, studio]; up = new Set(); joinedAliases = new Set(); window = 131072
ops = {
start: vi.fn(async (model: AppModel) => {
up.add(model.id)
return { engine: model.engine as AppEngineRecord['engine'], served: model.ref, alias: model.name, port: 41000, binary: model.binary!, pid: 4242 }
}),
alive: vi.fn(async (record: AppEngineRecord) => up.has(record.modelId)),
loadedContext: vi.fn(async () => window),
stop: vi.fn(async (record: AppEngineRecord) => { up.delete(record.modelId) }),
}
})
it('lists them under the app they came from, ready to start, beside Grid\'s own', async () => {
// A catalog model not here yet runs in nothing; once Grid has downloaded it, it runs in Grid, named
// like the apps beside it.
const models = appService()
expect((await models.list('home')).models.find(m => m.id === 'org/Small-GGUF')).toMatchObject({ state: 'available' })
expect((await models.list('home')).models.find(m => m.id === 'org/Small-GGUF')).not.toHaveProperty('app')
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
const snapshot = await models.list('home', true)
expect(snapshot.models.find(m => m.id === 'org/Small-GGUF')).toMatchObject({ state: 'downloaded', app: 'Grid' })
expect(snapshot.models.filter(m => m.app && m.app !== 'Grid')).toEqual([
expect.objectContaining({ id: ollama.id, name: 'llama3.2:3b', app: 'Ollama', state: 'downloaded', sizeBytes: 2 * GiB, canStart: true, canStop: false }),
expect.objectContaining({ id: studio.id, name: 'gemma-4-e2b', app: 'LM Studio', state: 'downloaded', canStart: true, canStop: false }),
])
expect(ops.start).not.toHaveBeenCalled()
})
it('starts one in its app, joins it at its own address, checks it through the grid, and stops what it started', async () => {
const models = appService()
await models.act('home', ollama.id, 'start'); await models.settled()
expect(ops.start).toHaveBeenCalledWith(ollama, 131072, join(stateDir, 'logs'))
expect(calls.find(args => args.includes('--at'))).toEqual(['--remote', 'join', 'home', '--at', 'http://127.0.0.1:41000/v1',
'-m', 'llama3.2:3b', '--advertise-as', 'llama3.2:3b', '--max-concurrency', '1'])
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(true)
let row = (await models.list('home', true)).models.find(m => m.id === ollama.id)!
expect(row).toMatchObject({ state: 'running', canStart: false, canStop: true, operation: { phase: 'done' } })
// One engine of this picker's at a time.
expect(await models.act('home', studio.id, 'start').then(() => models.settled()).then(() => models.list('home', true)))
.toMatchObject({ models: expect.arrayContaining([expect.objectContaining({ id: studio.id, operation: expect.objectContaining({ phase: 'failed', error: 'Stop llama3.2:3b first to start another local model.' }) })]) })
expect(ops.start).toHaveBeenCalledTimes(1)
await models.act('home', ollama.id, 'stop'); await models.settled()
expect(calls.some(args => args.join(' ') === '--remote leave home --engine llama3.2:3b')).toBe(true)
expect(ops.stop).toHaveBeenCalledWith(expect.objectContaining({ modelId: ollama.id, pid: 4242 }))
row = (await models.list('home', true)).models.find(m => m.id === ollama.id)!
expect(row).toMatchObject({ state: 'downloaded', canStart: true, canStop: false })
expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8'))).toEqual([])
})
it('says a model is served on a Swarm\'s grid while its engine runs there, and keeps it from starting twice', async () => {
const swarm = 'swarm-0123456789ab'
const models = appService()
await models.act(swarm, ollama.id, 'start'); await models.settled()
let row = (await models.list('home', true)).models.find(m => m.id === ollama.id)!
expect(row).toMatchObject({ state: 'downloaded', canStart: false, servedOn: [{ grid: swarm, servedAs: ['llama3.2:3b'] }] })
// Its engine gone: nothing is served there, and it starts again.
up.clear()
row = (await models.list('home', true)).models.find(m => m.id === ollama.id)!
expect(row).toMatchObject({ canStart: true })
expect(row).not.toHaveProperty('servedOn')
})
it('names the live engines it joined to a grid, and joins the first again as it was started', async () => {
const swarm = 'swarm-0123456789ab'
const record = (grid: string, modelId = ollama.id) => ({ spec: 1, modelId, grid, name: 'llama3.2:3b', engine: 'ollama', served: 'llama3.2:3b',
alias: 'llama3.2:3b', port: 41000, binary: '/usr/local/bin/ollama', pid: 4242 })
await mkdir(stateDir, { recursive: true })
await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([record('home'), record(swarm, 'app:gone'), record(swarm)]))
up.add(ollama.id)
const models = appService({ machineName: () => 'Studio Mac' })
expect(await models.servedApps(swarm)).toEqual(['llama3.2:3b'])
expect(await models.rejoinApp(swarm)).toBe(true)
expect(calls.at(-1)).toEqual(['--remote', 'join', swarm, '--at', 'http://127.0.0.1:41000/v1', '-m', 'llama3.2:3b',
'--advertise-as', 'llama3.2:3b', '--max-concurrency', '1', '--name', 'Studio Mac'])
// No name, or none an argument can carry: joined without one. Nothing alive there: nothing joined.
expect(await appService({ machineName: () => '-x' }).rejoinApp(swarm)).toBe(true)
expect(calls.at(-1)).not.toContain('--name')
// An engine that cannot be checked is not served.
ops.alive.mockRejectedValueOnce(new Error('ps')).mockRejectedValueOnce(new Error('ps'))
expect(await models.servedApps(swarm)).toEqual([])
ops.alive.mockRejectedValueOnce(new Error('ps')).mockRejectedValueOnce(new Error('ps'))
expect(await models.rejoinApp(swarm)).toBe(false)
up.clear()
expect(await models.servedApps(swarm)).toEqual([])
expect(await models.rejoinApp(swarm)).toBe(false)
expect(await appService({ appEngines: undefined }).servedApps(swarm)).toEqual([])
})
it('lists the own grid beside a Swarm\'s when one model is served on both, and reads none without its app\'s engines', async () => {
const record = (grid: string) => ({ spec: 1, modelId: ollama.id, grid, name: 'llama3.2:3b', engine: 'ollama', served: 'llama3.2:3b',
alias: 'llama3.2:3b', port: 41000, binary: '/usr/local/bin/ollama', pid: 4242 })
await mkdir(stateDir, { recursive: true })
await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([record('home'), record('swarm-0123456789ab')]))
up.add(ollama.id); joinedAliases.add('llama3.2:3b')
const row = (await appService().list('home', true)).models.find(m => m.id === ollama.id)!
expect(row.servedOn).toEqual([{ grid: 'home', servedAs: ['llama3.2:3b'] }, { grid: 'swarm-0123456789ab', servedAs: ['llama3.2:3b'] }])
expect((await appService({ appEngines: undefined }).list('home', true)).models.find(m => m.id === ollama.id)).not.toHaveProperty('servedOn')
})
it('keeps an engine it started stoppable when a later scan reads the model differently', async () => {
const models = appService()
await models.act('home', ollama.id, 'start'); await models.settled()
// The next scan missed the app (a slow probe) and would serve the file with Grid's engine instead.
apps = [{ ...ollama, engine: 'grid', ref: join(root, 'blob') }]
const row = (await models.list('home', true)).models.filter(m => m.id === ollama.id)
expect(row).toEqual([expect.objectContaining({ state: 'running', canStop: true, canStart: false })])
await models.act('home', ollama.id, 'stop'); await models.settled()
expect(ops.stop).toHaveBeenCalledWith(expect.objectContaining({ modelId: ollama.id }))
expect(joinedAliases.size).toBe(0)
expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8'))).toEqual([])
})
it('refreshes the picker without rescanning apps until their 30-second cache expires', async () => {
// Found by QA's coverage audit: refreshing the picker should not repeatedly probe installed engines.
vi.useFakeTimers({ toFake: ['Date'] })
const start = Date.now(), scan = vi.fn(async () => apps)
service = appService({ appModels: scan })
await service.list('home')
expect(scan).toHaveBeenCalledTimes(1)
vi.setSystemTime(start + 3000)
await service.list('home')
expect(scan).toHaveBeenCalledTimes(1)
vi.setSystemTime(start + 30_000)
await service.list('home')
expect(scan).toHaveBeenCalledTimes(2)
})
it('answers from the last scan while a slow one runs, and the list after it has the new one', async () => {
let scans = 0, release!: () => void
const models = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory,
appEngines: ops as unknown as AppEngineOps,
appModels: async () => {
if (++scans === 1) return [ollama]
// QA's coverage CI exposed an unending third scan: only the second is deliberately held.
if (scans === 2) await new Promise<void>(resolve => { release = resolve })
return [ollama, studio]
} })
expect((await models.list('home')).models.filter(m => m.app && m.app !== 'Grid').map(m => m.id)).toEqual([ollama.id])
// A forced read starts a scan that hangs (a busy llama-server answering --version late): the list
// answers at once from the last scan rather than waiting on it.
expect((await models.list('home', true)).models.filter(m => m.app && m.app !== 'Grid').map(m => m.id)).toEqual([ollama.id])
expect(scans).toBe(2)
release()
// QA's coverage CI let the receipt finish before the next forced read; make that ordering explicit.
await models.settled()
await vi.waitFor(async () => expect((await models.list('home', true)).models.some(m => m.id === studio.id)).toBe(true))
})
it('answers from the scan it saved when it starts again, and a scan that never answers is given up', async () => {
await appService().list('home')
let scans = 0
const appIds = (snapshot: Awaited<ReturnType<LocalModels['list']>>) => snapshot.models.filter(m => m.app && m.app !== 'Grid').map(m => m.id)
// Started again, with every scan hanging: the saved one answers at once, nothing waits on a scan.
const restarted = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory,
appEngines: ops as unknown as AppEngineOps, appScanMs: 50,
appModels: () => { scans++; return new Promise<AppModel[]>(() => {}) } })
expect(appIds(await restarted.list('home'))).toEqual([ollama.id, studio.id])
// That scan is given up at its deadline, the saved answer kept, and the next read starts another.
await new Promise(resolve => setTimeout(resolve, 80))
expect(appIds(await restarted.list('home', true))).toEqual([ollama.id, studio.id])
expect(scans).toBe(2)
})
it('starts another right after a stop, without a list read between to clear the one that ran', async () => {
const models = appService()
await models.act('home', ollama.id, 'start'); await models.settled()
await models.list('home', true)
// A switch: stop, then start at once — the list read above still says llama3.2:3b runs.
await models.act('home', ollama.id, 'stop'); await models.settled()
await models.act('home', studio.id, 'start'); await models.settled()
expect(ops.start).toHaveBeenLastCalledWith(studio, 131072, join(stateDir, 'logs'))
expect((await models.list('home', true)).models.find(m => m.id === studio.id)).toMatchObject({ state: 'running', operation: { phase: 'done' } })
})
it('takes it down again, and says so, when the app loaded it with less than 64K', async () => {
window = 16384
const models = appService()
await models.act('home', ollama.id, 'start'); await models.settled()
const row = (await models.list('home', true)).models.find(m => m.id === ollama.id)!
expect(row.operation).toMatchObject({ phase: 'failed', error: expect.stringContaining('could only get a 16K context in Ollama') })
expect(row).toMatchObject({ state: 'downloaded', canStart: true })
expect(ops.stop).toHaveBeenCalledTimes(1)
expect(joinedAliases.size).toBe(0)
})
it('settles a background app scan and its saved receipt before returning', async () => {
// Found by QA on a quiet machine: teardown removed receipts while an app scan was still saving them.
let scans = 0
let finishScan!: (models: AppModel[]) => void
service = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory,
appModels: () => ++scans === 1 ? Promise.resolve([ollama]) : new Promise(resolve => { finishScan = resolve }),
appEngines: ops as unknown as AppEngineOps })
await service.list('home')
await service.list('home', true)
const settled = vi.fn()
const drained = service.settled().then(settled)
try {
await new Promise(resolve => setImmediate(resolve))
expect(settled).not.toHaveBeenCalled()
} finally {
finishScan([{ ...ollama, name: 'after the scan' }])
await vi.waitFor(async () => {
expect(JSON.parse(await readFile(join(stateDir, 'app-models.json'), 'utf8'))[0].name).toBe('after the scan')
})
await drained
}
})
it('starts nothing that cannot get 64K beside its weights', async () => {
apps = [{ ...ollama, sizeBytes: 60 * GiB }]
const models = appService()
await models.act('home', ollama.id, 'start'); await models.settled()
expect((await models.list('home', true)).models.find(m => m.id === ollama.id)?.operation)
.toMatchObject({ phase: 'failed', error: expect.stringContaining('with a 64K context') })
expect(ops.start).not.toHaveBeenCalled()
})
it("serves one with Grid's llama.cpp, through a link to the app's file, when the app is not installed", async () => {
const file = join(root, 'ollama-blob')
await writeFile(file, Buffer.alloc(64))
apps = [{ id: 'app:ollama:qwen3:8b', name: 'qwen3:8b', app: 'ollama', engine: 'grid', ref: file, sizeBytes: 64 }]
const models = appService()
const listed = (await models.list('home')).models.find(m => m.id === 'app:ollama:qwen3:8b')!
// Its app is not here, so it runs in Grid, and says so.
expect(listed).toMatchObject({ app: 'Grid', state: 'downloaded', canStart: true })
await models.act('home', 'app:ollama:qwen3:8b', 'start'); await models.settled()
const { readlink } = await import('node:fs/promises')
expect(await readlink(join(home, 'models', 'qwen3-8b.gguf'))).toBe(file)
expect(calls.find(args => args.includes('--serve'))?.slice(0, 5)).toEqual(['--remote', 'join', 'home', '--serve', 'qwen3-8b.gguf'])
expect(ops.start).not.toHaveBeenCalled()
})
// Quiet-machine QA found the app-engine recovery paths had no behavior coverage.
it.each([
[new AppStartError('Ollama could not reserve memory.'), 'Ollama could not reserve memory.'],
[new Error('private engine diagnostic'), 'The model could not start. Try again.'],
])('keeps an app start failure retryable and reports only its public message', async (error, message) => {
service = appService()
ops.start.mockRejectedValueOnce(error)
const failed = await service.act('home', ollama.id, 'start'); await service.settled()
expect(failed.operation).toMatchObject({ phase: 'failed', error: message })
expect(joinedAliases.size).toBe(0)
expect(ops.stop).not.toHaveBeenCalled()
const retried = await service.act('home', ollama.id, 'start'); await service.settled()
expect(retried.operation?.phase).toBe('done')
expect((await service.list('home')).models.find(model => model.id === ollama.id))
.toMatchObject({ state: 'running', canStop: true })
})
it('forgets the engines the Jev decision models left, which blocked every chat model\'s start', async () => {
// As on a computer that ran them before #1141: laya-english's engine long gone, kev's still up.
const jev = (modelId: string, name: string, port: number): AppEngineRecord => ({ spec: 1, modelId, grid: 'home', name,
engine: 'llama.cpp', served: name, alias: name, port, binary: '/opt/llama-server', pid: 4300 + port % 100 })
const laya = jev('jev:ggml-org/Laya-GGUF', 'laya-english', 52954), kev = jev('jev:ggml-org/Kev-0.8B-GGUF', 'kev-0.8b', 60437)
up.add(kev.modelId); joinedAliases.add(kev.alias)
await mkdir(stateDir, { recursive: true })
await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([laya, kev]))
service = appService()
const snapshot = await service.list('home', true)
expect(snapshot.models.map(m => m.name)).not.toEqual(expect.arrayContaining(['laya-english']))
expect(snapshot.models.some(m => m.id.startsWith('jev:'))).toBe(false)
expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8'))).toEqual([])
await vi.waitFor(() => expect(ops.stop).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ modelId: kev.modelId })))
await vi.waitFor(() => expect(joinedAliases.has(kev.alias)).toBe(false))
// A chat model starts: nothing left to "stop first".
const started = await service.act('home', ollama.id, 'start'); await service.settled()
expect(started.operation).not.toMatchObject({ phase: 'failed' })
expect(ops.start).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ id: ollama.id }), expect.anything(), expect.anything())
})
it.each(['join refused', 'credentials unavailable', 'credentials threw'])('takes an app engine down after %s, retaining another grid\'s record', async scenario => {
service = appService()
const elsewhere: AppEngineRecord = { spec: 1, modelId: studio.id, grid: 'elsewhere', name: studio.name,
engine: 'lm-studio', served: studio.ref, alias: studio.name, port: 41002, binary: studio.binary!, pid: 4243 }
await mkdir(stateDir, { recursive: true })
await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([elsewhere]))
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => {
if (scenario === 'join refused' && args.includes('--at')) return refused('private join diagnostic')
if (scenario !== 'join refused' && args.includes('--env')) {
if (scenario === 'credentials threw') throw new Error('private credential diagnostic')
return refused('private credential diagnostic')
}
return original(args, output)
})
const failed = await service.act('home', ollama.id, 'start'); await service.settled()
expect(failed.operation).toMatchObject({ phase: 'failed', error: scenario === 'join refused'
? 'The model could not join your grid. Try again.' : scenario === 'credentials threw'
? 'The model did not answer. Try again.' : 'The model is starting, but could not be checked. Try again shortly.' })
expect(ops.stop).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ modelId: ollama.id, grid: 'home' }))
expect(up.size).toBe(0)
expect(joinedAliases.size).toBe(0)
expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8'))).toEqual([elsewhere])
})
it.each(['asleep', 'stopped', 'status unavailable'] as const)('starts an app engine when the grid is %s', async state => {
service = appService({ machineName: () => ' QA machine ' })
const original = run.getMockImplementation()!
gridState = state === 'status unavailable' ? 'running' : state
run.mockImplementation(async (args, output) => state === 'status unavailable' && args.includes('info') && args.includes('--json')
? refused('old grid cannot read status') : original(args, output))
const started = await service.act('home', ollama.id, 'start'); await service.settled()
expect(started.operation?.phase).toBe('done')
expect(calls.filter(args => args.join(' ') === '--remote start home')).toHaveLength(state === 'status unavailable' ? 0 : 1)
expect(calls.find(args => args.includes('--at'))).toEqual(expect.arrayContaining(['--name', 'QA machine']))
})
it('does not allocate an app engine when the sleeping grid refuses to start', async () => {
service = appService()
gridState = 'asleep'
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args[1] === 'start' ? refused('private start diagnostic') : original(args, output))
const result = await service.act('home', ollama.id, 'start'); await service.settled()
expect(result.operation).toMatchObject({ phase: 'failed', error: 'Your grid could not start. Try again.' })
expect(ops.start).not.toHaveBeenCalled()
})
it('leaves a running Grid engine alone when an app model is requested', async () => {
service = appService()
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
const result = await service.act('home', ollama.id, 'start'); await service.settled()
expect(result.operation).toMatchObject({ phase: 'failed', error: 'Stop Small-Q4 first to start another local model.' })
expect(ops.start).not.toHaveBeenCalled()
expect(serving).toBe(true)
})
it.each(['model disappeared', 'engine support unavailable'])('refuses an app start when its %s', async scenario => {
service = appService(scenario === 'engine support unavailable' ? { appEngines: undefined } : {})
if (scenario === 'model disappeared') apps = []
const result = await service.act('home', ollama.id, 'start'); await service.settled()
expect(result.operation).toMatchObject({ phase: 'failed', error: 'This model could not be checked. Refresh and try again.' })
expect(ops.start).not.toHaveBeenCalled()
})
it('keeps a model stoppable after its app scan loses it, including while the grid sleeps', async () => {
service = appService()
await service.act('home', ollama.id, 'start'); await service.settled()
apps = []
await service.list('home', true); await service.settled()
inventory.mockResolvedValue({ state: 'asleep', status: 'asleep', nodes: [] })
expect((await service.list('home', true)).models.find(model => model.id === ollama.id))
.toMatchObject({ state: 'running', app: 'Ollama', gridAsleep: true, canStart: false, canStop: true })
const stopped = await service.act('home', ollama.id, 'stop'); await service.settled()
expect(stopped.operation?.phase).toBe('done')
expect(ops.stop).toHaveBeenCalledOnce()
expect((await service.list('home', true)).models.some(model => model.id === ollama.id)).toBe(false)
})
it('does not allocate an app model twice, or download weights its app already owns', async () => {
service = appService()
window = undefined
for (const action of ['download', 'stop', 'start', 'start', 'download'] as const) {
const result = await service.act('home', ollama.id, action); await service.settled()
expect(result.operation?.phase).toBe('done')
}
expect(ops.start).toHaveBeenCalledOnce()
expect(ops.stop).not.toHaveBeenCalled()
expect(calls.some(args => args[0] === 'pull')).toBe(false)
})
it.each(['same link', 'different link', 'ordinary file', 'missing weights'])('protects Grid\'s model filename when there is a %s', async scenario => {
const { symlink, readlink } = await import('node:fs/promises')
const file = join(root, 'app-weights'), link = join(home, 'models', 'app-model.gguf')
const otherFile = join(root, 'other-weights')
if (scenario !== 'missing weights') await writeFile(file, Buffer.alloc(64))
await writeFile(otherFile, 'a different model')
apps = [{ id: 'app:ollama:app-model', name: 'app-model', app: 'ollama', engine: 'grid', ref: file, sizeBytes: 64 }]
if (scenario === 'same link') await symlink(file, link)
if (scenario === 'different link') await symlink(otherFile, link)
if (scenario === 'ordinary file') await writeFile(link, 'a different model')
service = appService()
const result = await service.act('home', apps[0].id, 'start'); await service.settled()
if (scenario === 'same link') {
expect(result.operation?.phase).toBe('done')
expect(await readlink(link)).toBe(file)
expect(calls.find(args => args.includes('--serve'))).toContain('app-model.gguf')
} else {
expect(result.operation).toMatchObject({ phase: 'failed', error: scenario === 'missing weights'
? 'The downloaded file is no longer available. Open Model Manager to restore it.'
: 'A different file has this name in Grid\'s models folder. Open Model Manager to start this model.' })
expect(calls.some(args => args.includes('--serve'))).toBe(false)
if (scenario === 'different link') expect(await readlink(link)).toBe(otherFile)
if (scenario === 'ordinary file') expect(await readFile(link, 'utf8')).toBe('a different model')
}
})
it('does not replace a model link whose target cannot be read', async () => {
const { symlink } = await import('node:fs/promises')
const file = join(root, 'app-weights'), link = join(home, 'models', 'app-model.gguf')
await writeFile(file, Buffer.alloc(64)); await symlink(file, link)
apps = [{ id: 'app:ollama:app-model', name: 'app-model', app: 'ollama', engine: 'grid', ref: file, sizeBytes: 64 }]
service = appService()
vi.mocked(readlink).mockRejectedValueOnce(new Error('EACCES'))
const result = await service.act('home', apps[0].id, 'start'); await service.settled()
expect(result.operation).toMatchObject({ phase: 'failed', error: expect.stringContaining('A different file has this name') })
expect(await readlink(link)).toBe(file)
expect(calls.some(args => args.includes('--serve'))).toBe(false)
})
it('recovers a first app scan that fails and keeps the catalog usable meanwhile', async () => {
const scan = vi.fn<() => Promise<AppModel[]>>().mockRejectedValueOnce(new Error('app unavailable')).mockResolvedValue([ollama])
service = appService({ appModels: scan })
const first = await service.list('home')
expect(first.notice).toBeUndefined()
expect(first.models.some(model => model.id === 'org/Small-GGUF')).toBe(true)
expect(first.models.some(model => model.id === ollama.id)).toBe(false)
expect((await service.list('home', true)).models.find(model => model.id === ollama.id))
.toMatchObject({ state: 'downloaded', canStart: true })
expect(scan).toHaveBeenCalledTimes(2)
})
it('lists app models even when their scan receipt cannot be saved, then persists the next successful scan', async () => {
service = appService()
await writeFile(stateDir, 'storage unavailable')
expect((await service.list('home')).models.find(model => model.id === ollama.id)).toMatchObject({ state: 'downloaded' })
expect(await readFile(stateDir, 'utf8')).toBe('storage unavailable')
await rm(stateDir); await mkdir(stateDir)
await service.list('home', true)
await vi.waitFor(async () => expect(JSON.parse(await readFile(join(stateDir, 'app-models.json'), 'utf8'))).toEqual(apps))
})
it('keeps models visible without inventing free disk space when both filesystem probes fail', async () => {
service = appService()
vi.mocked(statfs).mockRejectedValueOnce(new Error('models volume unavailable')).mockRejectedValueOnce(new Error('home volume unavailable'))
const snapshot = await service.list('home')
expect(snapshot).not.toHaveProperty('freeDiskBytes')
expect(snapshot.models.find(model => model.id === ollama.id)).toMatchObject({ state: 'downloaded', canStart: true })
})
it.each([true, false])('reads object-shaped model names without assuming every provider has a models array (listed=%s)', async listed => {
service = appService()
await service.act('home', ollama.id, 'start'); await service.settled()
inventory.mockResolvedValue({ state: 'awake', status: 'running', nodes: [
{ online: false, models: [ollama.name] }, { online: true },
{ online: true, models: [{ model: listed ? ollama.name : 'someone-elses-model' }] },
] })
expect((await service.list('home', true)).models.find(model => model.id === ollama.id))
.toMatchObject({ state: listed ? 'running' : 'downloaded', canStop: true })
})
it('unregisters a stale app record even when its engine adapter is unavailable', async () => {
await mkdir(stateDir)
await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([{ spec: 1, modelId: ollama.id, grid: 'home', name: ollama.name,
engine: 'ollama', served: ollama.ref, alias: ollama.name, port: 41000, binary: ollama.binary, pid: 4242 }]))
service = appService({ appEngines: undefined })
expect((await service.list('home')).models.find(model => model.id === ollama.id))
.toMatchObject({ state: 'downloaded', canStart: false, canStop: true, servedAs: [ollama.name] })
const result = await service.act('home', ollama.id, 'stop'); await service.settled()
expect(result.operation?.phase).toBe('done')
expect(calls).toContainEqual(['--remote', 'leave', 'home', '--engine', ollama.name])
expect(ops.stop).not.toHaveBeenCalled()
expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8'))).toEqual([])
})
// A record left by a kind of model the picker no longer offers, under an id that is not an app model's: it keeps
// its row and holds the next start, so its Stop must take the engine down rather than look for one of Grid's.
it('stops an engine it started for a model it no longer offers, whatever the id', async () => {
const left: AppEngineRecord = { spec: 1, modelId: 'retired:org/old-model', grid: 'home', name: 'old-model',
engine: 'llama.cpp', served: 'old-model', alias: 'old-model', port: 41003, binary: '/opt/llama-server', pid: 4244 }
await mkdir(stateDir); await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([left]))
up.add(left.modelId); joinedAliases.add(left.alias)
service = appService()
expect((await service.list('home')).models.find(model => model.id === left.modelId))
.toMatchObject({ name: 'old-model', app: 'llama.cpp', state: 'running', canStart: false, canStop: true })
await service.act('home', ollama.id, 'start'); await service.settled()
expect((await service.list('home', true)).models.find(model => model.id === ollama.id)?.operation)
.toMatchObject({ phase: 'failed', error: 'Stop old-model first to start another local model.' })
const result = await service.act('home', left.modelId, 'stop'); await service.settled()
expect(result.operation?.phase).toBe('done')
expect(ops.stop).toHaveBeenCalledWith(expect.objectContaining({ modelId: left.modelId }))
expect(joinedAliases.size).toBe(0)
expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8'))).toEqual([])
})
it('preserves the same app model\'s record on another grid through start and stop here', async () => {
const elsewhere: AppEngineRecord = { spec: 1, modelId: ollama.id, grid: 'elsewhere', name: ollama.name,
engine: 'ollama', served: ollama.ref, alias: ollama.name, port: 41002, binary: ollama.binary!, pid: 4243 }
await mkdir(stateDir); await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([elsewhere]))
service = appService()
const started = await service.act('home', ollama.id, 'start'); await service.settled()
expect(started.operation?.phase).toBe('done')
expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8')))
.toEqual([elsewhere, expect.objectContaining({ modelId: ollama.id, grid: 'home' })])
const stopped = await service.act('home', ollama.id, 'stop'); await service.settled()
expect(stopped.operation?.phase).toBe('done')
expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8'))).toEqual([elsewhere])
})
it.each([undefined, null, '', '--unsafe', 'bad\nname'])('omits an unusable machine name from an app join (%j)', async name => {
service = appService({ machineName: () => name })
const result = await service.act('home', ollama.id, 'start'); await service.settled()
expect(result.operation?.phase).toBe('done')
expect(calls.find(args => args.includes('--at'))).not.toContain('--name')
})
})
describe('the grids this computer serves a model on', () => {
/** Omagrid beside the own grid: `grid ls` lists both, and this computer's engine for Small-Q4 has a run
* record there too (as `grid join` on Omagrid writes it). */
const onOmagrid = async (record: Record<string, unknown> = {}) => {
const folder = join(home, 'run', 'engines', 'grid-oma')
await mkdir(folder, { recursive: true })
await writeFile(join(folder, 'remote.json'), JSON.stringify({ node_id: 'oma-node', engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'] }], advertise_as: ['Small-Q4'], ...record }))
await writeFile(join(folder, 'remote.heartbeat'), '')
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('ls')
? ok([{ grid: 'home', id: 'grid-home' }, { grid: 'Omagrid', id: 'grid-oma' }]) : original(args, output))
}
/** Omagrid's overview: a stranger's node serving the same model, and this computer's. */
const omagrid = (): GridInventory => ({ state: 'awake', status: null, nodes: [
{ node_id: 'stranger', online: true, models: ['small-q4'], answered: { window_seconds: 86400, requests: 900, by_model: [{ model: 'small-q4', requests: 900, tokens_out: 9_000_000 }] } },
{ node_id: 'oma-node', online: true, models: ['small-q4'], answered: { window_seconds: 86400, requests: 268, by_model: [{ model: 'small-q4', requests: 268, tokens_out: 1_100_000, tokens_in: 2_400_000, tokens_cached: 1_800_000 }] } },
] })
const withServingNodes = (servingNodes: (grid: string) => Promise<GridInventory>, extra: Partial<ConstructorParameters<typeof LocalModels>[0]> = {}) =>
modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory, servingNodes, ...extra })
it('names the own grid and every other grid it serves on, with only its own engine’s figures there', async () => {
const servingNodes = vi.fn(async () => omagrid())
service = withServingNodes(servingNodes)
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
await onOmagrid()
expect((await service.list('home', true)).models[0].servedOn).toEqual([
// Each with the name that grid lists it under — what a harness moves onto it there by.
{ grid: 'home', servedAs: ['Small-Q4'], requests: 4, windowSeconds: 3600 },
{ grid: 'Omagrid', servedAs: ['Small-Q4'], requests: 268, tokensOut: 1_100_000, tokensIn: 2_400_000, tokensCached: 1_800_000, windowSeconds: 86400 },
])
// Read as another grid's nodes, never through the own grid's inventory, which takes a grid for the account's own.
expect(servingNodes.mock.calls).toEqual([['Omagrid']])
expect(inventory.mock.calls.every(([grid]) => grid === 'home')).toBe(true)
})
it('names no grid for a model it serves nowhere, and reads no other grid without a run record here', async () => {
const servingNodes = vi.fn(async () => omagrid())
service = withServingNodes(servingNodes)
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('ls')
? ok([{ grid: 'home', id: 'grid-home' }, { grid: 'Omagrid', id: 'grid-oma' }]) : original(args, output))
expect((await service.list('home')).models[0]).not.toHaveProperty('servedOn')
expect(servingNodes).not.toHaveBeenCalled()
})
it('names a grid that sleeps while its engine here is alive, with no figures; not one whose engine is gone', async () => {
service = withServingNodes(async () => ({ state: 'asleep', status: 'asleep', nodes: [] }))
await onOmagrid({ pid: process.pid })
expect((await service.list('home', true)).models[0].servedOn).toEqual([{ grid: 'Omagrid', servedAs: ['Small-Q4'] }])
await onOmagrid({ pid: 0 })
expect((await service.list('home', true)).models[0]).not.toHaveProperty('servedOn')
})
it('names a grid whose overview does not list this computer’s engine yet, or could not be read, while its process is alive', async () => {
// No process for its record (gone, or mid-spawn): not serving there.
service = withServingNodes(async () => ({ state: 'awake', status: null, nodes: [omagrid().nodes[0]] }))
await onOmagrid()
expect((await service.list('home', true)).models[0]).not.toHaveProperty('servedOn')
service = withServingNodes(async () => { throw new Error('unreachable') })
expect((await service.list('home', true)).models[0]).not.toHaveProperty('servedOn')
// Its process alive: serving there, figures unknown — so the own grid never starts a second engine of it.
await onOmagrid({ pid: process.pid })
expect((await service.list('home', true)).models[0].servedOn).toEqual([{ grid: 'Omagrid', servedAs: ['Small-Q4'] }])
service = withServingNodes(async () => ({ state: 'awake', status: null, nodes: [omagrid().nodes[0]] }))
expect((await service.list('home', true)).models[0].servedOn).toEqual([{ grid: 'Omagrid', servedAs: ['Small-Q4'] }])
})
it('starts a model this computer knows from another grid’s list on the community grid by the same id', async () => {
// Small-Q4 served on the own grid, then its run record gone: known there as `local:Small-Q4.gguf` only.
catalogCards = []
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'] }], advertise_as: [] }))
expect((await service.list('home', true)).models.map(m => m.id)).toContain('local:Small-Q4.gguf')
await rm(join(records, 'remote.json'))
await service.act('Omagrid', 'local:Small-Q4.gguf', 'start'); await service.settled()
expect(calls.find(args => args.includes('join'))?.slice(0, 5)).toEqual(['--remote', 'join', 'Omagrid', '--serve', 'Small-Q4.gguf'])
})
it('reads only the own grid for a caller that gives no other grid’s reader (as before)', async () => {
await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
await onOmagrid()
expect((await service.list('home', true)).models[0].servedOn).toEqual([{ grid: 'home', servedAs: ['Small-Q4'], requests: 4, windowSeconds: 3600 }])
})
it('says which OS grid this computer claims, and nothing for none', async () => {
expect((await withServingNodes(async () => omagrid(), { os: () => 'omarchy' }).list('home')).os).toBe('omarchy')
expect(await withServingNodes(async () => omagrid(), { os: () => undefined }).list('home')).not.toHaveProperty('os')
})
it('names the community grid of this computer’s OS that the account is in, for a start there', async () => {
const original = run.getMockImplementation()!
run.mockImplementation(async (args, output) => args.includes('ls') ? ok([
{ grid: 'home', id: 'grid-home', type: 'permissioned-public' }, { grid: '--flag', id: 'x', type: 'os-community' },
{ grid: 'Omagrid', id: 'grid-oma', type: 'os-community' },
]) : original(args, output))
expect((await service.list('home')).osGrid).toBe('Omagrid')
expect(await service.communityGrid()).toBe('Omagrid')
run.mockImplementation(original)
expect(await service.communityGrid()).toBeNull()
expect(await service.list('home', true)).not.toHaveProperty('osGrid')
})
it('a list of the own grid says how a start on the community grid goes, and how it ended', async () => {
const ack = await service.act('Omagrid', 'org/Small-GGUF', 'start')
expect((await service.list('home', true)).models[0].operation).toMatchObject({ id: ack.operation!.id, phase: 'running' })
await service.settled()
expect((await service.list('home', true)).models[0].operation).toMatchObject({ id: ack.operation!.id, phase: 'done' })
// An operation on the own grid after it: its own receipt again.
const own = await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
expect((await service.list('home', true)).models[0].operation).toMatchObject({ id: own.operation!.id, action: 'stop' })
})
})