Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat(cli): Ollama's decision models in the picker, and Start on one its grid dropped

bitcoinbuilderba2f4be495d8
diff --git a/cli/src/lib/appModels.spec.ts b/cli/src/lib/appModels.spec.ts
index 4e5128bf5..f18691a93 100644
--- a/cli/src/lib/appModels.spec.ts
+++ b/cli/src/lib/appModels.spec.ts
@@ -1,7 +1,13 @@
-import { describe, expect, it } from 'vitest'
-import { APP_CONTEXT, appContext, cacheName, llamaServerArgs, scanAppModels } from './appModels.js'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { createHash } from 'node:crypto'
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { APP_CONTEXT, appContext, appEngineOps, cacheName, llamaServerArgs, ollamaDecisionModels, scanAppModels, type AppEngineRecord, type AppModel } from './appModels.js'
const GiB = 1024 ** 3
+/** Never the developer's own Ollama store: a scan reads it from `OLLAMA_MODELS`. */
+const env = { OLLAMA_MODELS: '/nonexistent/ollama-models' }
const grid = { kind: 'llama.cpp', path: '/home/me/.grid/bin/llama-server', version: 'version: 10369', note: "Grid's own engine" }
const gguf = (over: Record<string, unknown> = {}) => ({ format: 'gguf', bytes: 2 * GiB, quant: 'Q4_K_M',
gguf: { contextLength: 131072, toolCalls: true, kvBytesPerToken: 114688 }, ...over })
@@ -37,7 +43,7 @@ describe('models other apps downloaded', () => {
gguf({ name: 'Loose', source: 'folder', path: '/d/Loose.gguf', startWith: { engine: 'llama.cpp', label: "Grid's llama.cpp" } }),
],
})
- const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env: {}, run })
+ const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env, run })
expect(seen[0]).toEqual(['/node', '/pkg/toolchain/fleet.mjs', 'models', '--json'])
expect(models).toEqual([
{ id: 'app:ollama:llama3.2:3b', name: 'llama3.2:3b', app: 'ollama', engine: 'ollama', ref: 'llama3.2:3b', binary: '/usr/local/bin/ollama',
@@ -65,7 +71,7 @@ describe('models other apps downloaded', () => {
gguf({ name: 'new-arch:7b', source: 'ollama', path: '/m/blobs/sha256-f', gguf: { contextLength: 131072, unsupportedTensorTypes: [143] }, startWith: { engine: 'llama.cpp' } }),
],
})
- const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env: {}, run })
+ const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env, run })
expect(models.map(m => [m.name, m.app, m.engine, m.ref])).toEqual([
['qwen3:8b', 'ollama', 'grid', '/m/blobs/sha256-b'],
// LM Studio's app is there but has no `lms` yet: its GGUF is still a file Grid's engine reads.
@@ -86,7 +92,7 @@ describe('models other apps downloaded', () => {
{ type: 'llm', modelKey: 'old/short', format: 'gguf', sizeBytes: 1e9, maxContextLength: 8192 },
{ type: 'llm', modelKey: 'org/no-tools', format: 'gguf', sizeBytes: 1e9, maxContextLength: 131072, trainedForToolUse: false },
])
- const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env: {}, run })
+ const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env, run })
expect(models).toEqual([
{ id: 'app:lm-studio:google/gemma-4-e2b', name: 'gemma-4-e2b', app: 'lm-studio', engine: 'lm-studio', ref: 'google/gemma-4-e2b', binary: lms,
sizeBytes: 4414806160, contextLength: 131072 },
@@ -96,11 +102,11 @@ describe('models other apps downloaded', () => {
})
it('is no models, never an error, without the Model Manager or with a scan that fails', async () => {
- expect(await scanAppModels({ node: '/node', packageDir: null, env: {} })).toEqual([])
+ expect(await scanAppModels({ node: '/node', packageDir: null, env })).toEqual([])
const failing = async () => ({ ok: false, stdout: '', stderr: 'boom' })
- expect(await scanAppModels({ node: '/node', packageDir: '/pkg', env: {}, run: failing })).toEqual([])
+ expect(await scanAppModels({ node: '/node', packageDir: '/pkg', env, run: failing })).toEqual([])
const garbled = async () => ({ ok: true, stdout: 'not json', stderr: '' })
- expect(await scanAppModels({ node: '/node', packageDir: '/pkg', env: {}, run: garbled })).toEqual([])
+ expect(await scanAppModels({ node: '/node', packageDir: '/pkg', env, run: garbled })).toEqual([])
})
it('gives 128K when it fits beside the weights, 64K when only that does, and nothing below', () => {
@@ -132,3 +138,98 @@ describe('llama-server for a model', () => {
expect(args.slice(args.indexOf('--ctx-size'), args.indexOf('-ngl'))).toEqual(['--ctx-size', '8192', '--parallel', '4', '--kv-unified'])
})
})
+
+describe("Ollama's decision models", () => {
+ let store: string
+ /** A model in Ollama's store as `ollama pull` leaves it: a manifest naming a config blob and the weights. */
+ const pull = async (path: string, config: Record<string, unknown> | string, layers = [3 * GiB, 1 * GiB]) => {
+ const text = typeof config === 'string' ? config : JSON.stringify(config)
+ const digest = createHash('sha256').update(text).digest('hex')
+ await mkdir(join(store, 'blobs'), { recursive: true })
+ await writeFile(join(store, 'blobs', `sha256-${digest}`), text)
+ const manifest = join(store, 'manifests', path)
+ await mkdir(join(manifest, '..'), { recursive: true })
+ await writeFile(manifest, JSON.stringify({ schemaVersion: 2, config: { digest: `sha256:${digest}` },
+ layers: layers.map(size => ({ mediaType: 'application/vnd.ollama.image.tensor', size })) }))
+ }
+ const decision = { model_format: 'safetensors', file_type: 'mxfp8', capabilities: ['decision'], capabilities_explicit: true }
+ beforeEach(async () => { store = await mkdtemp(join(tmpdir(), 'ollama-store-')) })
+ afterEach(() => rm(store, { recursive: true, force: true }))
+
+ it('reads them from its manifests, by the name `ollama run` takes, and says which need a newer Ollama', async () => {
+ await pull('registry.ollama.ai/library/tev1/latest', { ...decision, requires: '0.40.0' })
+ await pull('registry.ollama.ai/someone/nimble/q4', { ...decision, file_type: 'Q4_K_M' }, [6 * GiB])
+ await pull('hf.co/org/clef-flash/latest', decision)
+ // Chat models, and anything Ollama itself could not read, are not decision models.
+ await pull('registry.ollama.ai/library/llama3.2/3b', { model_format: 'gguf', capabilities: ['completion', 'tools'] })
+ await pull('registry.ollama.ai/library/broken/latest', 'not json')
+ await mkdir(join(store, 'manifests', 'registry.ollama.ai', 'library', 'odd'), { recursive: true })
+ await writeFile(join(store, 'manifests', 'registry.ollama.ai', 'library', 'odd', 'latest'), JSON.stringify({ config: { digest: '../../etc' } }))
+
+ const models = await ollamaDecisionModels(store, '/usr/local/bin/ollama', '0.35.2')
+ expect(models.sort((a, b) => a.id.localeCompare(b.id))).toEqual([
+ { id: 'jev:ollama:hf.co/org/clef-flash:latest', name: 'hf.co/org/clef-flash', app: 'ollama', engine: 'ollama',
+ ref: 'hf.co/org/clef-flash:latest', binary: '/usr/local/bin/ollama', sizeBytes: 4 * GiB, quant: 'MXFP8', kind: 'decision' },
+ { id: 'jev:ollama:someone/nimble:q4', name: 'someone/nimble:q4', app: 'ollama', engine: 'ollama',
+ ref: 'someone/nimble:q4', binary: '/usr/local/bin/ollama', sizeBytes: 6 * GiB, quant: 'Q4_K_M', kind: 'decision' },
+ { id: 'jev:ollama:tev1:latest', name: 'tev1', app: 'ollama', engine: 'ollama', ref: 'tev1:latest',
+ binary: '/usr/local/bin/ollama', sizeBytes: 4 * GiB, quant: 'MXFP8', kind: 'decision', needs: '0.40.0' },
+ ])
+ // Before 0.35 no Ollama answers `/v1/systemone`, whatever the model asks for.
+ expect((await ollamaDecisionModels(store, '/usr/local/bin/ollama', '0.32.5')).map(m => [m.name, m.needs]))
+ .toEqual(expect.arrayContaining([['tev1', '0.40.0'], ['someone/nimble:q4', '0.35.0']]))
+ expect(await ollamaDecisionModels(join(store, 'missing'), '/usr/local/bin/ollama', '0.40.0')).toEqual([])
+ })
+
+ it('are listed by the scan where Ollama is installed, once, in place of a chat model of the same name', async () => {
+ await pull('registry.ollama.ai/library/tev1/latest', { ...decision, requires: '0.40.0' })
+ const found = (engines: unknown[]) => fake({ machine: { canRun: ['ollama'], engines }, models: [
+ gguf({ name: 'tev1:latest', source: 'ollama', path: '/m/blobs/sha256-t', startWith: { engine: 'ollama', label: 'ollama' } }),
+ gguf({ name: 'llama3.2:3b', source: 'ollama', path: '/m/blobs/sha256-a', startWith: { engine: 'ollama', label: 'ollama' } })] }).run
+ const installed = await scanAppModels({ node: '/node', packageDir: '/pkg', env: { OLLAMA_MODELS: store },
+ run: found([{ kind: 'ollama', path: '/usr/local/bin/ollama', version: '0.40.0' }]) })
+ expect(installed.map(m => [m.id, m.kind])).toEqual([['app:ollama:llama3.2:3b', undefined], ['jev:ollama:tev1:latest', 'decision']])
+ // Without Ollama nothing can serve one: not a decision model here, and never a chat model either.
+ const absent = await scanAppModels({ node: '/node', packageDir: '/pkg', env: { OLLAMA_MODELS: store }, run: found([grid]) })
+ expect(absent.map(m => m.id)).toEqual([])
+ })
+})
+
+describe("a decision model's Ollama", () => {
+ let dir: string, records: AppEngineRecord[]
+ const model: AppModel = { id: 'jev:ollama:tev1:latest', name: 'tev1', app: 'ollama', engine: 'ollama', ref: 'tev1:latest',
+ sizeBytes: 4 * GiB, kind: 'decision' }
+ beforeEach(async () => {
+ dir = await mkdtemp(join(tmpdir(), 'ollama-engine-')); records = []
+ // A stand-in `ollama serve`: it only has to stay up until it is stopped.
+ model.binary = join(dir, 'ollama')
+ await writeFile(model.binary, '#!/bin/sh\nexec sleep 60\n', { mode: 0o755 })
+ })
+ afterEach(async () => {
+ for (const record of records) { try { process.kill(-record.pid!, 'SIGKILL') } catch { /* gone */ } }
+ await rm(dir, { recursive: true, force: true })
+ })
+ const ollama = (decides: boolean) => {
+ const asked: string[] = []
+ const request = (async (url: string | URL, init?: RequestInit) => {
+ asked.push(String(url))
+ if (String(url).endsWith('/api/version')) return Response.json({ version: '0.40.0' })
+ return decides ? Response.json({ answers: { refund: { type: 'noul', noul: 0.98 } } })
+ : Response.json({ error: `"${JSON.parse(String(init?.body)).model}" not found` }, { status: 404 })
+ }) as typeof fetch
+ return { ops: appEngineOps({}, request), asked }
+ }
+
+ it('is asked one question before Grid is, so it joins loaded', async () => {
+ const { ops, asked } = ollama(true)
+ const started = await ops.start(model, 8192, dir, 4)
+ records.push({ spec: 1, modelId: model.id, grid: 'home', name: model.name, ...started })
+ expect(started).toMatchObject({ engine: 'ollama', served: 'tev1:latest', alias: 'tev1' })
+ expect(asked.at(-1)).toBe(`http://127.0.0.1:${started.port}/v1/systemone`)
+ })
+
+ it('is stopped, and says so, when it cannot answer one', async () => {
+ const { ops } = ollama(false)
+ await expect(ops.start(model, 8192, dir, 4)).rejects.toThrow('Ollama could not load tev1. Close some apps, then try again.')
+ })
+})
diff --git a/cli/src/lib/appModels.ts b/cli/src/lib/appModels.ts
index a553301aa..f841743fd 100644
--- a/cli/src/lib/appModels.ts
+++ b/cli/src/lib/appModels.ts
@@ -4,8 +4,9 @@
* architecture. Grid's llama.cpp starts one only when its app is not installed here. */
import { execFile, spawn } from 'node:child_process'
import { closeSync, openSync } from 'node:fs'
-import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'
+import { mkdir, readdir, readFile, rename, writeFile } from 'node:fs/promises'
import { createServer, type AddressInfo } from 'node:net'
+import { homedir } from 'node:os'
import { basename, join } from 'node:path'
import { processExists } from './processLiveness.js'
@@ -27,6 +28,11 @@ export interface AppModel {
/** The app's own binary (`ollama`, `lms`, `llama-server`); absent for `grid`. */
binary?: string
sizeBytes: number; quant?: string; contextLength?: number; kvBytesPerToken?: number
+ /** `decision`: a decision (System One) model its app answers at `/v1/systemone` — listed with the Jev models,
+ * never a harness's model. */
+ kind?: 'decision'
+ /** The app version it needs, when the one installed here is older: Start says to update rather than fail. */
+ needs?: string
}
/** One engine this daemon started for a picker model, kept on disk so Stop and the list outlive a restart. */
@@ -60,7 +66,8 @@ export function cacheName(name: string): string {
* The picker's models from other apps: Ollama's and llama.cpp's from the Model Manager's own scan
* (`fleet models --json`, whose `startWith` is the rule), LM Studio's from `lms ls`, the only place that
* names the key `lms load` takes. Only what a coding agent can run: 64K or more, tool calls not ruled out,
- * and a file the engine reads. A scan that fails is no models, never an error over the whole picker.
+ * and a file the engine reads — and Ollama's decision models ([ollamaDecisionModels]), marked as such. A scan
+ * that fails is no models, never an error over the whole picker.
*/
export async function scanAppModels({ node, packageDir, env, run = exec }: {
node: string; packageDir: string | null; env: NodeJS.ProcessEnv; run?: Exec
@@ -99,6 +106,14 @@ export async function scanAppModels({ node, packageDir, env, run = exec }: {
...(context ? { contextLength: context } : {}), ...(num(gguf.kvBytesPerToken) ? { kvBytesPerToken: num(gguf.kvBytesPerToken) } : {}) })
}
+ // Ollama's decision models are read from its own store: they are never a coding agent's, and the GGUF one
+ // that passes for a chat model above is listed once, as what it is.
+ const ollama = installed('ollama')
+ const decisions = await ollamaDecisionModels(ollamaStore(env), str(ollama?.path), str(ollama?.version))
+ const deciding = new Set(decisions.map(d => d.ref))
+ for (let i = result.length - 1; i >= 0; i--) if (result[i]!.app === 'ollama' && deciding.has(result[i]!.ref)) result.splice(i, 1)
+ if (ollama) result.push(...decisions)
+
if (lmsUsable) {
const listed = await run(lms, ['ls', '--json'], { env, timeout: 30_000 })
let models: Record<string, any>[] = []
@@ -119,6 +134,67 @@ export async function scanAppModels({ node, packageDir, env, run = exec }: {
return result
}
+/** The first Ollama that answers System One decisions at `/v1/systemone`. */
+export const MIN_OLLAMA_DECISION = '0.35.0'
+
+/** Where Ollama keeps its models: `OLLAMA_MODELS`, else its default — the store an `ollama serve` started with
+ * this daemon's environment reads, so a model listed from it is one that serve can load. */
+export const ollamaStore = (env: NodeJS.ProcessEnv): string => env.OLLAMA_MODELS || join(env.HOME || homedir(), '.ollama', 'models')
+
+const versionParts = (v: string): number[] | null => {
+ const match = /^v?(\d+)\.(\d+)\.(\d+)/.exec(v.trim())
+ return match ? match.slice(1).map(Number) : null
+}
+/** Whether version [a] is older than [b]; false when either cannot be read. */
+const olderThan = (a: string, b: string): boolean => {
+ const [x, y] = [versionParts(a), versionParts(b)]
+ if (!x || !y) return false
+ const at = x.findIndex((part, i) => part !== y[i])
+ return at >= 0 && x[at]! < y[at]!
+}
+
+/**
+ * Ollama's decision models (Tev, Nimble, Clef in its library), read from its manifests: the Model Manager's
+ * scan reads only GGUF weights, and Ollama 0.40 keeps these as MLX safetensors, so tev1 was in no list [run].
+ * A model's config says it decides (`"capabilities": ["decision"]`) and the Ollama it needs (`requires`). One
+ * the [installed] Ollama is too old for is still listed, with what it [AppModel.needs], so Start says to update.
+ * Layout: `manifests/<registry>/<namespace>/<model>/<tag>`, named `<model>:<tag>` in Ollama's own library.
+ */
+export async function ollamaDecisionModels(store: string, binary: string, installed: string): Promise<AppModel[]> {
+ const dirs = async (path: string) => (await readdir(path, { withFileTypes: true }).catch(() => [])).filter(d => d.isDirectory()).map(d => d.name)
+ const blob = (digest: unknown) => /^sha256:[0-9a-f]{64}$/.test(str(digest)) ? join(store, 'blobs', str(digest).replace(':', '-')) : null
+ const result: AppModel[] = []
+ const manifests = join(store, 'manifests')
+ for (const registry of await dirs(manifests)) {
+ for (const namespace of await dirs(join(manifests, registry))) {
+ for (const model of await dirs(join(manifests, registry, namespace))) {
+ const folder = join(manifests, registry, namespace, model)
+ for (const tag of (await readdir(folder, { withFileTypes: true }).catch(() => [])).filter(d => d.isFile()).map(d => d.name)) {
+ let manifest: Record<string, any>, config: Record<string, any>
+ try {
+ manifest = obj(JSON.parse(await readFile(join(folder, tag), 'utf8')))
+ const path = blob(obj(manifest.config).digest)
+ if (!path) continue
+ config = obj(JSON.parse(await readFile(path, 'utf8')))
+ } catch { continue }
+ if (!Array.isArray(config.capabilities) || !config.capabilities.includes('decision')) continue
+ const named = registry === 'registry.ollama.ai' ? namespace === 'library' ? model : `${namespace}/${model}` : `${registry}/${namespace}/${model}`
+ const ref = `${named}:${tag}`
+ // The grid lists it under its plain name, as `ollama run` takes it.
+ const name = tag === 'latest' ? named : ref
+ if (!safe(name) || !safe(ref)) continue
+ const required = [str(config.requires), MIN_OLLAMA_DECISION].filter(versionParts).reduce((most, v) => olderThan(most, v) ? v : most)
+ const sizeBytes = rows(manifest.layers).reduce((sum, layer) => sum + (num(layer.size) ?? 0), 0)
+ result.push({ id: `jev:ollama:${ref}`, name, app: 'ollama', engine: 'ollama', ref, ...(binary ? { binary } : {}), sizeBytes,
+ ...(str(config.file_type) ? { quant: str(config.file_type).toUpperCase() } : {}), kind: 'decision',
+ ...(olderThan(installed, required) ? { needs: required } : {}) })
+ }
+ }
+ }
+ }
+ return result
+}
+
/** The window to start [model] with: [APP_CONTEXT] when it holds that, else 64K, within [budget] bytes of
* memory when the model's cache per token is known. Null when not even 64K fits. */
export function appContext(model: Pick<AppModel, 'contextLength' | 'kvBytesPerToken' | 'sizeBytes'>, budget: number | undefined): number | null {
@@ -170,6 +246,18 @@ export function appEngineOps(env: NodeJS.ProcessEnv, request: typeof fetch = fet
return { status: response.status, body: obj(await response.json().catch(() => ({}))) }
} catch { return { status: 0, body: {} } }
}
+ /** Whether [model] at [base] answers one System One question within [ms]. */
+ const decides = async (base: string, model: string, ms: number): Promise<boolean> => {
+ try {
+ const response = await request(`${base}/systemone`, {
+ method: 'POST', headers: { 'content-type': 'application/json' },
+ body: JSON.stringify({ model, state: 'I was charged twice. Please refund the duplicate.',
+ questions: { refund: { type: 'noul', instructions: 'Is a refund requested?' } } }),
+ signal: AbortSignal.timeout(ms), redirect: 'error',
+ })
+ return typeof obj(obj(obj(response.ok ? await response.json() : {}).answers).refund).noul === 'number'
+ } catch { return false }
+ }
/** Up when [url] answers 200; a 503 is a model still loading. Gives up early if the process exits. */
const ready = async (url: string, ms: number, pid?: number): Promise<boolean> => {
for (const deadline = Date.now() + ms; Date.now() < deadline; await sleep(500)) {
@@ -210,6 +298,13 @@ export function appEngineOps(env: NodeJS.ProcessEnv, request: typeof fetch = fet
const pid = await background(binary, ['serve'], { OLLAMA_HOST: `127.0.0.1:${port}`, OLLAMA_CONTEXT_LENGTH: String(ctx),
OLLAMA_NUM_PARALLEL: String(slots), OLLAMA_KEEP_ALIVE: '-1' }, logDir, `ollama-${port}.log`)
if (!await ready(`http://127.0.0.1:${port}/api/version`, 60_000, pid)) throw new AppStartError('Ollama could not start. Try again.')
+ // Ollama loads a model on its first request, and Grid's join asks a decision model one question with 15 s
+ // to answer (remote/probe.py `_PROBE_TIMEOUT`): a model still loading then joins without its decisions.
+ // So it is asked here first, with a load's time, and an Ollama that cannot run it says so now.
+ if (model.kind === 'decision' && !await decides(`http://127.0.0.1:${port}/v1`, model.ref, 5 * 60_000)) {
+ try { process.kill(-pid, 'SIGTERM') } catch { /* gone */ }
+ throw new AppStartError(`Ollama could not load ${model.name}. Close some apps, then try again.`)
+ }
return { engine: 'ollama', served: model.ref, alias: model.name, port, binary, pid }
}
if (model.engine === 'llama.cpp') {
diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.ts
index d7d212d92..4689e5f74 100644
--- a/cli/src/lib/localModels.spec.ts
+++ b/cli/src/lib/localModels.spec.ts
@@ -2305,4 +2305,75 @@ describe('Jev models: Get brings Grid\'s llama.cpp up to a build that serves the
expect(stopped.operation?.phase).toBe('done')
expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8'))).toEqual([elsewhere])
})
+
+ it('offers Start again for one its grid no longer lists, and Start puts it back', async () => {
+ await engine('0.5.0-dev (build 11378, commit edd6e2bbd)')
+ const models = jevService()
+ await models.act('home', LAYA, 'start'); await models.settled()
+ // The Grid app's `grid leave` takes it off the grid; its engine runs on.
+ joined.delete('laya-english')
+ expect(await laya(models)).toMatchObject({ state: 'downloaded', canStart: true, canStop: true })
+ await models.act('home', LAYA, 'start'); await models.settled()
+ expect(ops.stop).toHaveBeenCalledOnce()
+ expect(ops.start).toHaveBeenCalledTimes(2)
+ expect(await laya(models)).toMatchObject({ state: 'running', canStart: false, canStop: true, operation: { phase: 'done' } })
+ // One the grid serves is left as it is, and so is one whose grid cannot be read.
+ await models.act('home', LAYA, 'start'); await models.settled()
+ inventory.mockRejectedValue(new Error('unreadable'))
+ await models.act('home', LAYA, 'start'); await models.settled()
+ expect(ops.start).toHaveBeenCalledTimes(2)
+ expect(ops.stop).toHaveBeenCalledOnce()
+ })
+
+ describe("another app's decision model", () => {
+ const TEV = 'jev:ollama:tev1:latest'
+ const tev1: AppModel = { id: TEV, name: 'tev1', app: 'ollama', engine: 'ollama', ref: 'tev1:latest', binary: '/usr/local/bin/ollama',
+ sizeBytes: 4_480_000_000, quant: 'MXFP8', kind: 'decision' }
+ const tev = async (models: LocalModels) => (await models.list('home', true)).models.filter(m => m.id === TEV)
+
+ it('is listed once, as a decision model in its own app, and started there beside a chat model', async () => {
+ const models = jevService({}, { appModels: async () => [gemma, tev1] })
+ expect(await tev(models)).toEqual([expect.objectContaining({ name: 'tev1', kind: 'decision', app: 'Ollama', state: 'downloaded',
+ sizeBytes: 4_480_000_000, quant: 'MXFP8', canStart: true, canStop: false })])
+ await models.act('home', gemma.id, 'start'); await models.settled()
+ await models.act('home', TEV, 'start'); await models.settled()
+ // Here already, in the engine that downloaded it: nothing to pull, and Grid's llama.cpp is not asked.
+ expect(calls.some(args => args[0] === 'pull' || args[0] === 'engine')).toBe(false)
+ expect(ops.start).toHaveBeenLastCalledWith(tev1, 8192, join(stateDir, 'logs'), 4)
+ expect(calls.find(args => args.includes('--at') && args.includes('tev1'))).toEqual(['--remote', 'join', 'home', '--at',
+ 'http://127.0.0.1:41001/v1', '-m', 'tev1', '--advertise-as', 'tev1', '--max-concurrency', '5'])
+ expect(decisions).toEqual(['tev1'])
+ expect(await tev(models)).toEqual([expect.objectContaining({ kind: 'decision', app: 'Ollama', state: 'running', canStop: true,
+ operation: expect.objectContaining({ phase: 'done' }) })])
+ expect((await models.list('home', true)).models.find(m => m.id === gemma.id)).toMatchObject({ state: 'running' })
+
+ await models.act('home', TEV, 'stop'); await models.settled()
+ expect(calls.some(args => args.join(' ') === '--remote leave home --engine tev1')).toBe(true)
+ expect(ops.stop).toHaveBeenCalledWith(expect.objectContaining({ modelId: TEV }))
+ expect(await tev(models)).toEqual([expect.objectContaining({ state: 'downloaded', canStart: true, canStop: false })])
+ })
+
+ it.each([
+ ['an Ollama too old for it', { needs: '0.40.0' }, undefined, 'tev1 needs Ollama 0.40.0 or newer. Update Ollama, then start again.'],
+ ['no memory for it', { sizeBytes: 12 * GiB }, card10, 'This computer does not have the memory to run tev1. Close some apps, or choose a smaller model.'],
+ ])('says so before starting anything when there is %s', async (_case, over, device, error) => {
+ const models = jevService({}, { appModels: async () => [{ ...tev1, ...over }] }, device)
+ // Listed whatever it needs: it is here, and its Start is what says what to do.
+ expect(await tev(models)).toEqual([expect.objectContaining({ state: 'downloaded', canStart: true })])
+ await models.act('home', TEV, 'start'); await models.settled()
+ expect((await tev(models))[0]!.operation).toMatchObject({ phase: 'failed', error })
+ expect(ops.start).not.toHaveBeenCalled()
+ })
+
+ it('keeps a row to stop one whose model has since gone', async () => {
+ let apps = [tev1]
+ const models = jevService({}, { appModels: async () => apps })
+ await models.act('home', TEV, 'start'); await models.settled()
+ apps = []
+ expect(await tev(models)).toEqual([expect.objectContaining({ kind: 'decision', state: 'running', canStart: false, canStop: true })])
+ await models.act('home', TEV, 'stop'); await models.settled()
+ expect(ops.stop).toHaveBeenCalledWith(expect.objectContaining({ modelId: TEV }))
+ expect(await tev(models)).toEqual([])
+ })
+ })
})
diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.ts
index 40544b59b..d71c7b9e5 100644
--- a/cli/src/lib/localModels.ts
+++ b/cli/src/lib/localModels.ts
@@ -704,7 +704,7 @@ export class LocalModels {
private async savedApps(): Promise<{ at: number; value: AppModel[] } | undefined> {
try {
const value = rows(JSON.parse(await readFile(this.appsFile, 'utf8'))).filter(a =>
- str(a.id).startsWith('app:') && str(a.name) && ['ollama', 'lm-studio', 'llama.cpp'].includes(a.app) &&
+ (str(a.id).startsWith('app:') || str(a.id).startsWith('jev:ollama:')) && str(a.name) && ['ollama', 'lm-studio', 'llama.cpp'].includes(a.app) &&
['ollama', 'lm-studio', 'llama.cpp', 'grid'].includes(a.engine) && str(a.ref)) as AppModel[]
return { at: 0, value }
} catch { return undefined }
@@ -830,7 +830,7 @@ export class LocalModels {
canStart: !inventoryError && !record, canStop: !inventoryError && !!record,
operation: operation?.modelId === id ? operation : undefined, ...(running && !listed ? { gridAsleep: true } : {}) }
}
- for (const app of apps.filter(a => a.engine !== 'grid' || recorded(a.id))) {
+ for (const app of apps.filter(a => a.kind !== 'decision' && (a.engine !== 'grid' || recorded(a.id)))) {
models.push(await appRow(app.id, app.name, app.app, records.find(r => r.modelId === app.id),
{ sizeBytes: app.sizeBytes, ...(app.quant ? { quant: app.quant } : {}) }))
}
@@ -843,6 +843,10 @@ export class LocalModels {
// Only those that can start here are offered: one this computer has no memory for is listed only once it
// is here — running, downloaded, or part way.
const jevBudget = modelBudget(this.device)
+ // An engine of ours the grid does not list is started again: the Grid app's `grid leave` took Kev off its grid
+ // while its llama-server ran on, and the row, neither serving nor startable, said only "Downloaded" [run].
+ const restartable = (row: LocalModel, record?: AppEngineRecord): Partial<LocalModel> =>
+ record && row.state !== 'running' && !inventoryError ? { canStart: true } : {}
for (const jev of this.options.appEngines ? JEV_MODELS : []) {
const record = jevRecords.find(r => r.modelId === jev.id)
const quant = await this.jevQuant(jev, jevBudget)
@@ -850,7 +854,17 @@ export class LocalModels {
const shown = quant ?? jev.quants[0]!
const have = await this.fileComplete(shown.file, shown.size)
const row = await appRow(jev.id, jev.name, 'llama.cpp', record, { sizeBytes: shown.size, quant: shown.quant, kind: 'decision' })
- models.push({ ...row, app: GRID_LABEL, state: row.state === 'running' ? 'running' : have ? 'downloaded' : 'available' })
+ models.push({ ...row, app: GRID_LABEL, state: row.state === 'running' ? 'running' : have ? 'downloaded' : 'available', ...restartable(row, record) })
+ }
+ // Decision models another app downloaded (Ollama's tev1): listed whatever their size, since they are here —
+ // Start is what says when there is no memory for one. A record whose model has since gone keeps a row to stop it by.
+ for (const app of this.options.appEngines ? apps.filter(a => a.kind === 'decision') : []) {
+ const record = jevRecords.find(r => r.modelId === app.id)
+ const row = await appRow(app.id, app.name, app.app, record, { sizeBytes: app.sizeBytes, ...(app.quant ? { quant: app.quant } : {}), kind: 'decision' })
+ models.push({ ...row, ...restartable(row, record) })
+ }
+ for (const record of jevRecords.filter(r => !JEV_MODELS.some(j => j.id === r.modelId) && !apps.some(a => a.id === r.modelId))) {
+ models.push({ ...await appRow(record.modelId, record.name, record.engine, record, { kind: 'decision' }), canStart: false })
}
const freeDiskBytes = await statfs(join(this.home, 'models')).catch(() => statfs(this.home))
.then(disk => disk.bavail * disk.bsize, () => undefined)
@@ -1116,8 +1130,9 @@ export class LocalModels {
}
/**
- * A Jev model's Get, Start and Stop. Get is the whole of it: the weights, an engine that can serve them,
- * and the model on the grid — so a person never meets "too old" for an engine they did not choose.
+ * A decision model's Get, Start and Stop. For a Jev model Get is the whole of it: the weights, an engine that
+ * can serve them, and the model on the grid — so a person never meets "too old" for an engine they did not
+ * choose. Another app's decision model (Ollama's tev1) is here already, so Start is all there is to it.
*
* ⚠️ Grid's own llama.cpp is updated in place (`grid engine install llama.cpp`) when it is older than
* [MIN_JEV_BUILD], exactly as a missing one is installed for a chat model: nothing else upgrades the engine
@@ -1127,67 +1142,115 @@ export class LocalModels {
private async performJev(grid: string, operation: ModelOperation,
change: (stage: ModelOperation['stage']) => Promise<void>,
must: (args: string[], message: string, output?: (chunk: string) => void) => Promise<void>): Promise<void> {
- const jev = JEV_MODELS.find(m => m.id === operation.modelId)
- const ops = this.options.appEngines
- if (!jev || !ops) throw new ModelError('This model could not be checked. Refresh and try again.')
- const records = await readAppRecords(this.appRecordsFile)
- const record = records.find(r => r.modelId === jev.id && r.grid === grid)
- const takeDown = async (started: AppEngineRecord) => {
- await this.run(['--remote', 'leave', grid, '--engine', started.alias], undefined, 5 * 60_000)
- await ops.stop(started)
- await writeAppRecords(this.appRecordsFile, (await readAppRecords(this.appRecordsFile)).filter(r => !(r.modelId === jev.id && r.grid === grid)))
+ if (!this.options.appEngines) throw new ModelError('This model could not be checked. Refresh and try again.')
+ let record = (await readAppRecords(this.appRecordsFile)).find(r => r.modelId === operation.modelId && r.grid === grid)
+ // Start on one the grid does not serve takes its engine down first, and starts it again: its process may
+ // have gone with it, and a fresh start is the one path that checks every step.
+ if (record && operation.action === 'start' && !await this.decisionServing(grid, record)) {
+ await this.stopDecision(grid, record)
+ record = undefined
}
if (operation.action === 'stop') {
- if (record) await takeDown(record)
+ if (record) await this.stopDecision(grid, record)
} else if (!record) {
- const quant = await this.jevQuant(jev, modelBudget(await this.readDevice()))
- if (!quant) throw new ModelError(`This computer does not have the memory to run ${jev.name}. Close some apps, or choose a smaller model.`)
- if (!await this.fileComplete(quant.file, quant.size)) {
- await mkdir(join(this.home, 'models'), { recursive: true })
- const disk = await statfs(join(this.home, 'models'))
- if (disk.bavail * disk.bsize < quant.size + GiB) throw new ModelError('Free up disk space, then start again.')
- await change('downloading')
- let last = 0, progressWrites = Promise.resolve()
- await must(['pull', `${jev.repo}:${quant.file}`], 'The download stopped. Start again to resume.', chunk => {
- const matches = [...chunk.matchAll(/(\d+(?:\.\d+)?)\s*%/g)]
- const percent = matches.length ? Number(matches.at(-1)![1]) : NaN
- if (Number.isFinite(percent) && percent >= 0 && percent <= 100 && Date.now() - last > 500) {
- last = Date.now(); operation.progress = percent / 100
- progressWrites = progressWrites.then(() => this.save(grid, operation)).catch(() => {})
- }
- })
- await progressWrites
- if (!await this.fileComplete(quant.file, quant.size)) throw new ModelError('The download is incomplete. Start again to resume.')
- }
- if (operation.action === 'start') {
- const binary = await this.jevEngine(jev, change, must)
- await change('starting')
- if (DOWN.has(await this.gridStatus(grid).catch(() => ''))) await must(['--remote', 'start', grid], 'Your grid could not start. Try again.')
- const model: AppModel = { id: jev.id, name: jev.name, app: 'llama.cpp', engine: 'llama.cpp', ref: join(this.home, 'models', quant.file), binary, sizeBytes: quant.size }
- let started: AppEngineRecord
- try {
- started = { spec: 1, modelId: jev.id, grid, name: jev.name, ...await ops.start(model, JEV_CONTEXT, join(this.options.stateDir, 'logs'), JEV_SLOTS) }
- } catch (error) {
- throw new ModelError(error instanceof AppStartError ? error.message : 'The model could not start. Try again.')
- }
- await writeAppRecords(this.appRecordsFile, [...(await readAppRecords(this.appRecordsFile)).filter(r => !(r.modelId === jev.id && r.grid === grid)), started])
- const named = this.options.machineName?.()?.trim()
- const joined = await this.run(['--remote', 'join', grid, '--at', `http://127.0.0.1:${started.port}/v1`, '-m', started.served,
- '--advertise-as', started.alias, ...concurrencyArgs(await this.gridConcurrency(grid, false)),
- ...(named && validArg(named) ? ['--name', named] : [])], undefined, 10 * 60_000)
- if (!joined.ok) { await takeDown(started); throw new ModelError('The model could not join your grid. Try again.') }
- await change('verifying')
- try { await this.verifyDecision(grid, started.alias) } catch (error) {
- await takeDown(started)
- throw error instanceof ModelError ? error : new ModelError('The model did not answer. Try again.')
- }
- }
+ const jev = JEV_MODELS.find(m => m.id === operation.modelId)
+ const model = jev ? await this.getJev(grid, jev, operation, change, must) : await this.appDecision(operation.modelId)
+ if (model && operation.action === 'start') await this.startDecision(grid, model, change, must)
}
operation.phase = 'done'
delete operation.progress
await this.save(grid, operation)
}
+ /** [jev]'s weights, downloaded when they are not here; for a start, the model as Grid's llama.cpp runs it. */
+ private async getJev(grid: string, jev: JevModel, operation: ModelOperation,
+ change: (stage: ModelOperation['stage']) => Promise<void>,
+ must: (args: string[], message: string, output?: (chunk: string) => void) => Promise<void>): Promise<AppModel | undefined> {
+ const quant = await this.jevQuant(jev, modelBudget(await this.readDevice()))
+ if (!quant) throw new ModelError(`This computer does not have the memory to run ${jev.name}. Close some apps, or choose a smaller model.`)
+ if (!await this.fileComplete(quant.file, quant.size)) {
+ await mkdir(join(this.home, 'models'), { recursive: true })
+ const disk = await statfs(join(this.home, 'models'))
+ if (disk.bavail * disk.bsize < quant.size + GiB) throw new ModelError('Free up disk space, then start again.')
+ await change('downloading')
+ let last = 0, progressWrites = Promise.resolve()
+ await must(['pull', `${jev.repo}:${quant.file}`], 'The download stopped. Start again to resume.', chunk => {
+ const matches = [...chunk.matchAll(/(\d+(?:\.\d+)?)\s*%/g)]
+ const percent = matches.length ? Number(matches.at(-1)![1]) : NaN
+ if (Number.isFinite(percent) && percent >= 0 && percent <= 100 && Date.now() - last > 500) {
+ last = Date.now(); operation.progress = percent / 100
+ progressWrites = progressWrites.then(() => this.save(grid, operation)).catch(() => {})
+ }
+ })
+ await progressWrites
+ if (!await this.fileComplete(quant.file, quant.size)) throw new ModelError('The download is incomplete. Start again to resume.')
+ }
+ if (operation.action !== 'start') return undefined
+ return { id: jev.id, name: jev.name, app: 'llama.cpp', engine: 'llama.cpp', ref: join(this.home, 'models', quant.file),
+ binary: await this.jevEngine(jev, change, must), sizeBytes: quant.size }
+ }
+
+ /** Another app's decision model, from the last scan — or a new one, when the model clicked is not in it — once
+ * its app is new enough and this computer has the memory for it. */
+ private async appDecision(modelId: string): Promise<AppModel> {
+ let apps = await this.apps()
+ if (!apps.some(a => a.id === modelId)) apps = await this.scanApps()
+ const app = apps.find(a => a.id === modelId && a.kind === 'decision')
+ if (!app) throw new ModelError('This model could not be checked. Refresh and try again.')
+ const label = APP_LABEL[app.app]
+ if (app.needs) throw new ModelError(`${app.name} needs ${label} ${app.needs} or newer. Update ${label}, then start again.`)
+ const budget = modelBudget(await this.readDevice())
+ if (budget !== undefined && jevMemory(app.sizeBytes) > budget) {
+ throw new ModelError(`This computer does not have the memory to run ${app.name}. Close some apps, or choose a smaller model.`)
+ }
+ return app
+ }
+
+ /** Runs decision [model] on its own loopback port with [JEV_SLOTS] slots and joins it to [grid] `--at`, beside
+ * whatever runs there; up once a decision through the grid is answered, and taken down when it is not. */
+ private async startDecision(grid: string, model: AppModel,
+ change: (stage: ModelOperation['stage']) => Promise<void>,
+ must: (args: string[], message: string) => Promise<void>): Promise<void> {
+ await change('starting')
+ if (DOWN.has(await this.gridStatus(grid).catch(() => ''))) await must(['--remote', 'start', grid], 'Your grid could not start. Try again.')
+ let started: AppEngineRecord
+ try {
+ started = { spec: 1, modelId: model.id, grid, name: model.name,
+ ...await this.options.appEngines!.start(model, JEV_CONTEXT, join(this.options.stateDir, 'logs'), JEV_SLOTS) }
+ } catch (error) {
+ throw new ModelError(error instanceof AppStartError ? error.message : 'The model could not start. Try again.')
+ }
+ await writeAppRecords(this.appRecordsFile, [...(await readAppRecords(this.appRecordsFile)).filter(r => !(r.modelId === model.id && r.grid === grid)), started])
+ const named = this.options.machineName?.()?.trim()
+ const joined = await this.run(['--remote', 'join', grid, '--at', `http://127.0.0.1:${started.port}/v1`, '-m', started.served,
+ '--advertise-as', started.alias, ...concurrencyArgs(await this.gridConcurrency(grid, false)),
+ ...(named && validArg(named) ? ['--name', named] : [])], undefined, 10 * 60_000)
+ if (!joined.ok) { await this.stopDecision(grid, started); throw new ModelError('The model could not join your grid. Try again.') }
+ await change('verifying')
+ try { await this.verifyDecision(grid, started.alias) } catch (error) {
+ await this.stopDecision(grid, started)
+ throw error instanceof ModelError ? error : new ModelError('The model did not answer. Try again.')
+ }
+ }
+
+ /** Whether [record]'s engine is up and [grid] lists it — or the grid sleeps, which lists nothing, and it is parked. */
+ private async decisionServing(grid: string, record: AppEngineRecord): Promise<boolean> {
+ if (!await this.options.appEngines!.alive(record)) return false
+ const inventory = await this.options.inventory(grid, true).catch(() => null)
+ // Unread is not "not served": an engine that may be serving is never taken down on a guess.
+ if (!inventory || (inventory.state === 'unknown' && !DOWN.has(inventory.status ?? ''))) return true
+ if (inventory.state === 'asleep' || inventory.status === 'asleep') return true
+ return rows(inventory.nodes).some(n => n.online === true && (Array.isArray(n.models) ? n.models : [])
+ .some((m: unknown) => modelKey(str(typeof m === 'string' ? m : obj(m).model)) === modelKey(record.alias)))
+ }
+
+ /** A decision engine off [grid], stopped, and its record gone. */
+ private async stopDecision(grid: string, started: AppEngineRecord): Promise<void> {
+ await this.run(['--remote', 'leave', grid, '--engine', started.alias], undefined, 5 * 60_000)
+ await this.options.appEngines?.stop(started)
+ await writeAppRecords(this.appRecordsFile, (await readAppRecords(this.appRecordsFile)).filter(r => !(r.modelId === started.modelId && r.grid === grid)))
+ }
+
/** Grid's llama-server, brought up to a build that runs [jev] first when it is older (or missing). */
private async jevEngine(jev: JevModel, change: (stage: ModelOperation['stage']) => Promise<void>,
must: (args: string[], message: string) => Promise<void>): Promise<string> {