The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat(cli): Ollama's decision models in the picker, and Start on one its grid dropped
bitcoinbuilderba2f4be495d8
diff --git a/cli/src/lib/appModels.spec.ts b/cli/src/lib/appModels.spec.tsindex 4e5128bf5..f18691a93 100644--- a/cli/src/lib/appModels.spec.ts+++ b/cli/src/lib/appModels.spec.ts@@ -1,7 +1,13 @@-import { describe, expect, it } from 'vitest'-import { APP_CONTEXT, appContext, cacheName, llamaServerArgs, scanAppModels } from './appModels.js'+import { afterEach, beforeEach, describe, expect, it } from 'vitest'+import { createHash } from 'node:crypto'+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { APP_CONTEXT, appContext, appEngineOps, cacheName, llamaServerArgs, ollamaDecisionModels, scanAppModels, type AppEngineRecord, type AppModel } from './appModels.js'const GiB = 1024 ** 3+/** Never the developer's own Ollama store: a scan reads it from `OLLAMA_MODELS`. */+const env = { OLLAMA_MODELS: '/nonexistent/ollama-models' }const grid = { kind: 'llama.cpp', path: '/home/me/.grid/bin/llama-server', version: 'version: 10369', note: "Grid's own engine" }const gguf = (over: Record<string, unknown> = {}) => ({ format: 'gguf', bytes: 2 * GiB, quant: 'Q4_K_M',gguf: { contextLength: 131072, toolCalls: true, kvBytesPerToken: 114688 }, ...over })@@ -37,7 +43,7 @@ describe('models other apps downloaded', () => {gguf({ name: 'Loose', source: 'folder', path: '/d/Loose.gguf', startWith: { engine: 'llama.cpp', label: "Grid's llama.cpp" } }),],})- const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env: {}, run })+ const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env, run })expect(seen[0]).toEqual(['/node', '/pkg/toolchain/fleet.mjs', 'models', '--json'])expect(models).toEqual([{ id: 'app:ollama:llama3.2:3b', name: 'llama3.2:3b', app: 'ollama', engine: 'ollama', ref: 'llama3.2:3b', binary: '/usr/local/bin/ollama',@@ -65,7 +71,7 @@ describe('models other apps downloaded', () => {gguf({ name: 'new-arch:7b', source: 'ollama', path: '/m/blobs/sha256-f', gguf: { contextLength: 131072, unsupportedTensorTypes: [143] }, startWith: { engine: 'llama.cpp' } }),],})- const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env: {}, run })+ const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env, run })expect(models.map(m => [m.name, m.app, m.engine, m.ref])).toEqual([['qwen3:8b', 'ollama', 'grid', '/m/blobs/sha256-b'],// LM Studio's app is there but has no `lms` yet: its GGUF is still a file Grid's engine reads.@@ -86,7 +92,7 @@ describe('models other apps downloaded', () => {{ type: 'llm', modelKey: 'old/short', format: 'gguf', sizeBytes: 1e9, maxContextLength: 8192 },{ type: 'llm', modelKey: 'org/no-tools', format: 'gguf', sizeBytes: 1e9, maxContextLength: 131072, trainedForToolUse: false },])- const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env: {}, run })+ const models = await scanAppModels({ node: '/node', packageDir: '/pkg', env, run })expect(models).toEqual([{ id: 'app:lm-studio:google/gemma-4-e2b', name: 'gemma-4-e2b', app: 'lm-studio', engine: 'lm-studio', ref: 'google/gemma-4-e2b', binary: lms,sizeBytes: 4414806160, contextLength: 131072 },@@ -96,11 +102,11 @@ describe('models other apps downloaded', () => {})it('is no models, never an error, without the Model Manager or with a scan that fails', async () => {- expect(await scanAppModels({ node: '/node', packageDir: null, env: {} })).toEqual([])+ expect(await scanAppModels({ node: '/node', packageDir: null, env })).toEqual([])const failing = async () => ({ ok: false, stdout: '', stderr: 'boom' })- expect(await scanAppModels({ node: '/node', packageDir: '/pkg', env: {}, run: failing })).toEqual([])+ expect(await scanAppModels({ node: '/node', packageDir: '/pkg', env, run: failing })).toEqual([])const garbled = async () => ({ ok: true, stdout: 'not json', stderr: '' })- expect(await scanAppModels({ node: '/node', packageDir: '/pkg', env: {}, run: garbled })).toEqual([])+ expect(await scanAppModels({ node: '/node', packageDir: '/pkg', env, run: garbled })).toEqual([])})it('gives 128K when it fits beside the weights, 64K when only that does, and nothing below', () => {@@ -132,3 +138,98 @@ describe('llama-server for a model', () => {expect(args.slice(args.indexOf('--ctx-size'), args.indexOf('-ngl'))).toEqual(['--ctx-size', '8192', '--parallel', '4', '--kv-unified'])})})++describe("Ollama's decision models", () => {+ let store: string+ /** A model in Ollama's store as `ollama pull` leaves it: a manifest naming a config blob and the weights. */+ const pull = async (path: string, config: Record<string, unknown> | string, layers = [3 * GiB, 1 * GiB]) => {+ const text = typeof config === 'string' ? config : JSON.stringify(config)+ const digest = createHash('sha256').update(text).digest('hex')+ await mkdir(join(store, 'blobs'), { recursive: true })+ await writeFile(join(store, 'blobs', `sha256-${digest}`), text)+ const manifest = join(store, 'manifests', path)+ await mkdir(join(manifest, '..'), { recursive: true })+ await writeFile(manifest, JSON.stringify({ schemaVersion: 2, config: { digest: `sha256:${digest}` },+ layers: layers.map(size => ({ mediaType: 'application/vnd.ollama.image.tensor', size })) }))+ }+ const decision = { model_format: 'safetensors', file_type: 'mxfp8', capabilities: ['decision'], capabilities_explicit: true }+ beforeEach(async () => { store = await mkdtemp(join(tmpdir(), 'ollama-store-')) })+ afterEach(() => rm(store, { recursive: true, force: true }))++ it('reads them from its manifests, by the name `ollama run` takes, and says which need a newer Ollama', async () => {+ await pull('registry.ollama.ai/library/tev1/latest', { ...decision, requires: '0.40.0' })+ await pull('registry.ollama.ai/someone/nimble/q4', { ...decision, file_type: 'Q4_K_M' }, [6 * GiB])+ await pull('hf.co/org/clef-flash/latest', decision)+ // Chat models, and anything Ollama itself could not read, are not decision models.+ await pull('registry.ollama.ai/library/llama3.2/3b', { model_format: 'gguf', capabilities: ['completion', 'tools'] })+ await pull('registry.ollama.ai/library/broken/latest', 'not json')+ await mkdir(join(store, 'manifests', 'registry.ollama.ai', 'library', 'odd'), { recursive: true })+ await writeFile(join(store, 'manifests', 'registry.ollama.ai', 'library', 'odd', 'latest'), JSON.stringify({ config: { digest: '../../etc' } }))++ const models = await ollamaDecisionModels(store, '/usr/local/bin/ollama', '0.35.2')+ expect(models.sort((a, b) => a.id.localeCompare(b.id))).toEqual([+ { id: 'jev:ollama:hf.co/org/clef-flash:latest', name: 'hf.co/org/clef-flash', app: 'ollama', engine: 'ollama',+ ref: 'hf.co/org/clef-flash:latest', binary: '/usr/local/bin/ollama', sizeBytes: 4 * GiB, quant: 'MXFP8', kind: 'decision' },+ { id: 'jev:ollama:someone/nimble:q4', name: 'someone/nimble:q4', app: 'ollama', engine: 'ollama',+ ref: 'someone/nimble:q4', binary: '/usr/local/bin/ollama', sizeBytes: 6 * GiB, quant: 'Q4_K_M', kind: 'decision' },+ { id: 'jev:ollama:tev1:latest', name: 'tev1', app: 'ollama', engine: 'ollama', ref: 'tev1:latest',+ binary: '/usr/local/bin/ollama', sizeBytes: 4 * GiB, quant: 'MXFP8', kind: 'decision', needs: '0.40.0' },+ ])+ // Before 0.35 no Ollama answers `/v1/systemone`, whatever the model asks for.+ expect((await ollamaDecisionModels(store, '/usr/local/bin/ollama', '0.32.5')).map(m => [m.name, m.needs]))+ .toEqual(expect.arrayContaining([['tev1', '0.40.0'], ['someone/nimble:q4', '0.35.0']]))+ expect(await ollamaDecisionModels(join(store, 'missing'), '/usr/local/bin/ollama', '0.40.0')).toEqual([])+ })++ it('are listed by the scan where Ollama is installed, once, in place of a chat model of the same name', async () => {+ await pull('registry.ollama.ai/library/tev1/latest', { ...decision, requires: '0.40.0' })+ const found = (engines: unknown[]) => fake({ machine: { canRun: ['ollama'], engines }, models: [+ gguf({ name: 'tev1:latest', source: 'ollama', path: '/m/blobs/sha256-t', startWith: { engine: 'ollama', label: 'ollama' } }),+ gguf({ name: 'llama3.2:3b', source: 'ollama', path: '/m/blobs/sha256-a', startWith: { engine: 'ollama', label: 'ollama' } })] }).run+ const installed = await scanAppModels({ node: '/node', packageDir: '/pkg', env: { OLLAMA_MODELS: store },+ run: found([{ kind: 'ollama', path: '/usr/local/bin/ollama', version: '0.40.0' }]) })+ expect(installed.map(m => [m.id, m.kind])).toEqual([['app:ollama:llama3.2:3b', undefined], ['jev:ollama:tev1:latest', 'decision']])+ // Without Ollama nothing can serve one: not a decision model here, and never a chat model either.+ const absent = await scanAppModels({ node: '/node', packageDir: '/pkg', env: { OLLAMA_MODELS: store }, run: found([grid]) })+ expect(absent.map(m => m.id)).toEqual([])+ })+})++describe("a decision model's Ollama", () => {+ let dir: string, records: AppEngineRecord[]+ const model: AppModel = { id: 'jev:ollama:tev1:latest', name: 'tev1', app: 'ollama', engine: 'ollama', ref: 'tev1:latest',+ sizeBytes: 4 * GiB, kind: 'decision' }+ beforeEach(async () => {+ dir = await mkdtemp(join(tmpdir(), 'ollama-engine-')); records = []+ // A stand-in `ollama serve`: it only has to stay up until it is stopped.+ model.binary = join(dir, 'ollama')+ await writeFile(model.binary, '#!/bin/sh\nexec sleep 60\n', { mode: 0o755 })+ })+ afterEach(async () => {+ for (const record of records) { try { process.kill(-record.pid!, 'SIGKILL') } catch { /* gone */ } }+ await rm(dir, { recursive: true, force: true })+ })+ const ollama = (decides: boolean) => {+ const asked: string[] = []+ const request = (async (url: string | URL, init?: RequestInit) => {+ asked.push(String(url))+ if (String(url).endsWith('/api/version')) return Response.json({ version: '0.40.0' })+ return decides ? Response.json({ answers: { refund: { type: 'noul', noul: 0.98 } } })+ : Response.json({ error: `"${JSON.parse(String(init?.body)).model}" not found` }, { status: 404 })+ }) as typeof fetch+ return { ops: appEngineOps({}, request), asked }+ }++ it('is asked one question before Grid is, so it joins loaded', async () => {+ const { ops, asked } = ollama(true)+ const started = await ops.start(model, 8192, dir, 4)+ records.push({ spec: 1, modelId: model.id, grid: 'home', name: model.name, ...started })+ expect(started).toMatchObject({ engine: 'ollama', served: 'tev1:latest', alias: 'tev1' })+ expect(asked.at(-1)).toBe(`http://127.0.0.1:${started.port}/v1/systemone`)+ })++ it('is stopped, and says so, when it cannot answer one', async () => {+ const { ops } = ollama(false)+ await expect(ops.start(model, 8192, dir, 4)).rejects.toThrow('Ollama could not load tev1. Close some apps, then try again.')+ })+})diff --git a/cli/src/lib/appModels.ts b/cli/src/lib/appModels.tsindex a553301aa..f841743fd 100644--- a/cli/src/lib/appModels.ts+++ b/cli/src/lib/appModels.ts@@ -4,8 +4,9 @@* architecture. Grid's llama.cpp starts one only when its app is not installed here. */import { execFile, spawn } from 'node:child_process'import { closeSync, openSync } from 'node:fs'-import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'+import { mkdir, readdir, readFile, rename, writeFile } from 'node:fs/promises'import { createServer, type AddressInfo } from 'node:net'+import { homedir } from 'node:os'import { basename, join } from 'node:path'import { processExists } from './processLiveness.js'@@ -27,6 +28,11 @@ export interface AppModel {/** The app's own binary (`ollama`, `lms`, `llama-server`); absent for `grid`. */binary?: stringsizeBytes: number; quant?: string; contextLength?: number; kvBytesPerToken?: number+ /** `decision`: a decision (System One) model its app answers at `/v1/systemone` — listed with the Jev models,+ * never a harness's model. */+ kind?: 'decision'+ /** The app version it needs, when the one installed here is older: Start says to update rather than fail. */+ needs?: string}/** One engine this daemon started for a picker model, kept on disk so Stop and the list outlive a restart. */@@ -60,7 +66,8 @@ export function cacheName(name: string): string {* The picker's models from other apps: Ollama's and llama.cpp's from the Model Manager's own scan* (`fleet models --json`, whose `startWith` is the rule), LM Studio's from `lms ls`, the only place that* names the key `lms load` takes. Only what a coding agent can run: 64K or more, tool calls not ruled out,- * and a file the engine reads. A scan that fails is no models, never an error over the whole picker.+ * and a file the engine reads — and Ollama's decision models ([ollamaDecisionModels]), marked as such. A scan+ * that fails is no models, never an error over the whole picker.*/export async function scanAppModels({ node, packageDir, env, run = exec }: {node: string; packageDir: string | null; env: NodeJS.ProcessEnv; run?: Exec@@ -99,6 +106,14 @@ export async function scanAppModels({ node, packageDir, env, run = exec }: {...(context ? { contextLength: context } : {}), ...(num(gguf.kvBytesPerToken) ? { kvBytesPerToken: num(gguf.kvBytesPerToken) } : {}) })}+ // Ollama's decision models are read from its own store: they are never a coding agent's, and the GGUF one+ // that passes for a chat model above is listed once, as what it is.+ const ollama = installed('ollama')+ const decisions = await ollamaDecisionModels(ollamaStore(env), str(ollama?.path), str(ollama?.version))+ const deciding = new Set(decisions.map(d => d.ref))+ for (let i = result.length - 1; i >= 0; i--) if (result[i]!.app === 'ollama' && deciding.has(result[i]!.ref)) result.splice(i, 1)+ if (ollama) result.push(...decisions)+if (lmsUsable) {const listed = await run(lms, ['ls', '--json'], { env, timeout: 30_000 })let models: Record<string, any>[] = []@@ -119,6 +134,67 @@ export async function scanAppModels({ node, packageDir, env, run = exec }: {return result}+/** The first Ollama that answers System One decisions at `/v1/systemone`. */+export const MIN_OLLAMA_DECISION = '0.35.0'++/** Where Ollama keeps its models: `OLLAMA_MODELS`, else its default — the store an `ollama serve` started with+ * this daemon's environment reads, so a model listed from it is one that serve can load. */+export const ollamaStore = (env: NodeJS.ProcessEnv): string => env.OLLAMA_MODELS || join(env.HOME || homedir(), '.ollama', 'models')++const versionParts = (v: string): number[] | null => {+ const match = /^v?(\d+)\.(\d+)\.(\d+)/.exec(v.trim())+ return match ? match.slice(1).map(Number) : null+}+/** Whether version [a] is older than [b]; false when either cannot be read. */+const olderThan = (a: string, b: string): boolean => {+ const [x, y] = [versionParts(a), versionParts(b)]+ if (!x || !y) return false+ const at = x.findIndex((part, i) => part !== y[i])+ return at >= 0 && x[at]! < y[at]!+}++/**+ * Ollama's decision models (Tev, Nimble, Clef in its library), read from its manifests: the Model Manager's+ * scan reads only GGUF weights, and Ollama 0.40 keeps these as MLX safetensors, so tev1 was in no list [run].+ * A model's config says it decides (`"capabilities": ["decision"]`) and the Ollama it needs (`requires`). One+ * the [installed] Ollama is too old for is still listed, with what it [AppModel.needs], so Start says to update.+ * Layout: `manifests/<registry>/<namespace>/<model>/<tag>`, named `<model>:<tag>` in Ollama's own library.+ */+export async function ollamaDecisionModels(store: string, binary: string, installed: string): Promise<AppModel[]> {+ const dirs = async (path: string) => (await readdir(path, { withFileTypes: true }).catch(() => [])).filter(d => d.isDirectory()).map(d => d.name)+ const blob = (digest: unknown) => /^sha256:[0-9a-f]{64}$/.test(str(digest)) ? join(store, 'blobs', str(digest).replace(':', '-')) : null+ const result: AppModel[] = []+ const manifests = join(store, 'manifests')+ for (const registry of await dirs(manifests)) {+ for (const namespace of await dirs(join(manifests, registry))) {+ for (const model of await dirs(join(manifests, registry, namespace))) {+ const folder = join(manifests, registry, namespace, model)+ for (const tag of (await readdir(folder, { withFileTypes: true }).catch(() => [])).filter(d => d.isFile()).map(d => d.name)) {+ let manifest: Record<string, any>, config: Record<string, any>+ try {+ manifest = obj(JSON.parse(await readFile(join(folder, tag), 'utf8')))+ const path = blob(obj(manifest.config).digest)+ if (!path) continue+ config = obj(JSON.parse(await readFile(path, 'utf8')))+ } catch { continue }+ if (!Array.isArray(config.capabilities) || !config.capabilities.includes('decision')) continue+ const named = registry === 'registry.ollama.ai' ? namespace === 'library' ? model : `${namespace}/${model}` : `${registry}/${namespace}/${model}`+ const ref = `${named}:${tag}`+ // The grid lists it under its plain name, as `ollama run` takes it.+ const name = tag === 'latest' ? named : ref+ if (!safe(name) || !safe(ref)) continue+ const required = [str(config.requires), MIN_OLLAMA_DECISION].filter(versionParts).reduce((most, v) => olderThan(most, v) ? v : most)+ const sizeBytes = rows(manifest.layers).reduce((sum, layer) => sum + (num(layer.size) ?? 0), 0)+ result.push({ id: `jev:ollama:${ref}`, name, app: 'ollama', engine: 'ollama', ref, ...(binary ? { binary } : {}), sizeBytes,+ ...(str(config.file_type) ? { quant: str(config.file_type).toUpperCase() } : {}), kind: 'decision',+ ...(olderThan(installed, required) ? { needs: required } : {}) })+ }+ }+ }+ }+ return result+}+/** The window to start [model] with: [APP_CONTEXT] when it holds that, else 64K, within [budget] bytes of* memory when the model's cache per token is known. Null when not even 64K fits. */export function appContext(model: Pick<AppModel, 'contextLength' | 'kvBytesPerToken' | 'sizeBytes'>, budget: number | undefined): number | null {@@ -170,6 +246,18 @@ export function appEngineOps(env: NodeJS.ProcessEnv, request: typeof fetch = fetreturn { status: response.status, body: obj(await response.json().catch(() => ({}))) }} catch { return { status: 0, body: {} } }}+ /** Whether [model] at [base] answers one System One question within [ms]. */+ const decides = async (base: string, model: string, ms: number): Promise<boolean> => {+ try {+ const response = await request(`${base}/systemone`, {+ method: 'POST', headers: { 'content-type': 'application/json' },+ body: JSON.stringify({ model, state: 'I was charged twice. Please refund the duplicate.',+ questions: { refund: { type: 'noul', instructions: 'Is a refund requested?' } } }),+ signal: AbortSignal.timeout(ms), redirect: 'error',+ })+ return typeof obj(obj(obj(response.ok ? await response.json() : {}).answers).refund).noul === 'number'+ } catch { return false }+ }/** Up when [url] answers 200; a 503 is a model still loading. Gives up early if the process exits. */const ready = async (url: string, ms: number, pid?: number): Promise<boolean> => {for (const deadline = Date.now() + ms; Date.now() < deadline; await sleep(500)) {@@ -210,6 +298,13 @@ export function appEngineOps(env: NodeJS.ProcessEnv, request: typeof fetch = fetconst pid = await background(binary, ['serve'], { OLLAMA_HOST: `127.0.0.1:${port}`, OLLAMA_CONTEXT_LENGTH: String(ctx),OLLAMA_NUM_PARALLEL: String(slots), OLLAMA_KEEP_ALIVE: '-1' }, logDir, `ollama-${port}.log`)if (!await ready(`http://127.0.0.1:${port}/api/version`, 60_000, pid)) throw new AppStartError('Ollama could not start. Try again.')+ // Ollama loads a model on its first request, and Grid's join asks a decision model one question with 15 s+ // to answer (remote/probe.py `_PROBE_TIMEOUT`): a model still loading then joins without its decisions.+ // So it is asked here first, with a load's time, and an Ollama that cannot run it says so now.+ if (model.kind === 'decision' && !await decides(`http://127.0.0.1:${port}/v1`, model.ref, 5 * 60_000)) {+ try { process.kill(-pid, 'SIGTERM') } catch { /* gone */ }+ throw new AppStartError(`Ollama could not load ${model.name}. Close some apps, then try again.`)+ }return { engine: 'ollama', served: model.ref, alias: model.name, port, binary, pid }}if (model.engine === 'llama.cpp') {diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.tsindex d7d212d92..4689e5f74 100644--- a/cli/src/lib/localModels.spec.ts+++ b/cli/src/lib/localModels.spec.ts@@ -2305,4 +2305,75 @@ describe('Jev models: Get brings Grid\'s llama.cpp up to a build that serves theexpect(stopped.operation?.phase).toBe('done')expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8'))).toEqual([elsewhere])})++ it('offers Start again for one its grid no longer lists, and Start puts it back', async () => {+ await engine('0.5.0-dev (build 11378, commit edd6e2bbd)')+ const models = jevService()+ await models.act('home', LAYA, 'start'); await models.settled()+ // The Grid app's `grid leave` takes it off the grid; its engine runs on.+ joined.delete('laya-english')+ expect(await laya(models)).toMatchObject({ state: 'downloaded', canStart: true, canStop: true })+ await models.act('home', LAYA, 'start'); await models.settled()+ expect(ops.stop).toHaveBeenCalledOnce()+ expect(ops.start).toHaveBeenCalledTimes(2)+ expect(await laya(models)).toMatchObject({ state: 'running', canStart: false, canStop: true, operation: { phase: 'done' } })+ // One the grid serves is left as it is, and so is one whose grid cannot be read.+ await models.act('home', LAYA, 'start'); await models.settled()+ inventory.mockRejectedValue(new Error('unreadable'))+ await models.act('home', LAYA, 'start'); await models.settled()+ expect(ops.start).toHaveBeenCalledTimes(2)+ expect(ops.stop).toHaveBeenCalledOnce()+ })++ describe("another app's decision model", () => {+ const TEV = 'jev:ollama:tev1:latest'+ const tev1: AppModel = { id: TEV, name: 'tev1', app: 'ollama', engine: 'ollama', ref: 'tev1:latest', binary: '/usr/local/bin/ollama',+ sizeBytes: 4_480_000_000, quant: 'MXFP8', kind: 'decision' }+ const tev = async (models: LocalModels) => (await models.list('home', true)).models.filter(m => m.id === TEV)++ it('is listed once, as a decision model in its own app, and started there beside a chat model', async () => {+ const models = jevService({}, { appModels: async () => [gemma, tev1] })+ expect(await tev(models)).toEqual([expect.objectContaining({ name: 'tev1', kind: 'decision', app: 'Ollama', state: 'downloaded',+ sizeBytes: 4_480_000_000, quant: 'MXFP8', canStart: true, canStop: false })])+ await models.act('home', gemma.id, 'start'); await models.settled()+ await models.act('home', TEV, 'start'); await models.settled()+ // Here already, in the engine that downloaded it: nothing to pull, and Grid's llama.cpp is not asked.+ expect(calls.some(args => args[0] === 'pull' || args[0] === 'engine')).toBe(false)+ expect(ops.start).toHaveBeenLastCalledWith(tev1, 8192, join(stateDir, 'logs'), 4)+ expect(calls.find(args => args.includes('--at') && args.includes('tev1'))).toEqual(['--remote', 'join', 'home', '--at',+ 'http://127.0.0.1:41001/v1', '-m', 'tev1', '--advertise-as', 'tev1', '--max-concurrency', '5'])+ expect(decisions).toEqual(['tev1'])+ expect(await tev(models)).toEqual([expect.objectContaining({ kind: 'decision', app: 'Ollama', state: 'running', canStop: true,+ operation: expect.objectContaining({ phase: 'done' }) })])+ expect((await models.list('home', true)).models.find(m => m.id === gemma.id)).toMatchObject({ state: 'running' })++ await models.act('home', TEV, 'stop'); await models.settled()+ expect(calls.some(args => args.join(' ') === '--remote leave home --engine tev1')).toBe(true)+ expect(ops.stop).toHaveBeenCalledWith(expect.objectContaining({ modelId: TEV }))+ expect(await tev(models)).toEqual([expect.objectContaining({ state: 'downloaded', canStart: true, canStop: false })])+ })++ it.each([+ ['an Ollama too old for it', { needs: '0.40.0' }, undefined, 'tev1 needs Ollama 0.40.0 or newer. Update Ollama, then start again.'],+ ['no memory for it', { sizeBytes: 12 * GiB }, card10, 'This computer does not have the memory to run tev1. Close some apps, or choose a smaller model.'],+ ])('says so before starting anything when there is %s', async (_case, over, device, error) => {+ const models = jevService({}, { appModels: async () => [{ ...tev1, ...over }] }, device)+ // Listed whatever it needs: it is here, and its Start is what says what to do.+ expect(await tev(models)).toEqual([expect.objectContaining({ state: 'downloaded', canStart: true })])+ await models.act('home', TEV, 'start'); await models.settled()+ expect((await tev(models))[0]!.operation).toMatchObject({ phase: 'failed', error })+ expect(ops.start).not.toHaveBeenCalled()+ })++ it('keeps a row to stop one whose model has since gone', async () => {+ let apps = [tev1]+ const models = jevService({}, { appModels: async () => apps })+ await models.act('home', TEV, 'start'); await models.settled()+ apps = []+ expect(await tev(models)).toEqual([expect.objectContaining({ kind: 'decision', state: 'running', canStart: false, canStop: true })])+ await models.act('home', TEV, 'stop'); await models.settled()+ expect(ops.stop).toHaveBeenCalledWith(expect.objectContaining({ modelId: TEV }))+ expect(await tev(models)).toEqual([])+ })+ })})diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.tsindex 40544b59b..d71c7b9e5 100644--- a/cli/src/lib/localModels.ts+++ b/cli/src/lib/localModels.ts@@ -704,7 +704,7 @@ export class LocalModels {private async savedApps(): Promise<{ at: number; value: AppModel[] } | undefined> {try {const value = rows(JSON.parse(await readFile(this.appsFile, 'utf8'))).filter(a =>- str(a.id).startsWith('app:') && str(a.name) && ['ollama', 'lm-studio', 'llama.cpp'].includes(a.app) &&+ (str(a.id).startsWith('app:') || str(a.id).startsWith('jev:ollama:')) && str(a.name) && ['ollama', 'lm-studio', 'llama.cpp'].includes(a.app) &&['ollama', 'lm-studio', 'llama.cpp', 'grid'].includes(a.engine) && str(a.ref)) as AppModel[]return { at: 0, value }} catch { return undefined }@@ -830,7 +830,7 @@ export class LocalModels {canStart: !inventoryError && !record, canStop: !inventoryError && !!record,operation: operation?.modelId === id ? operation : undefined, ...(running && !listed ? { gridAsleep: true } : {}) }}- for (const app of apps.filter(a => a.engine !== 'grid' || recorded(a.id))) {+ for (const app of apps.filter(a => a.kind !== 'decision' && (a.engine !== 'grid' || recorded(a.id)))) {models.push(await appRow(app.id, app.name, app.app, records.find(r => r.modelId === app.id),{ sizeBytes: app.sizeBytes, ...(app.quant ? { quant: app.quant } : {}) }))}@@ -843,6 +843,10 @@ export class LocalModels {// Only those that can start here are offered: one this computer has no memory for is listed only once it// is here — running, downloaded, or part way.const jevBudget = modelBudget(this.device)+ // An engine of ours the grid does not list is started again: the Grid app's `grid leave` took Kev off its grid+ // while its llama-server ran on, and the row, neither serving nor startable, said only "Downloaded" [run].+ const restartable = (row: LocalModel, record?: AppEngineRecord): Partial<LocalModel> =>+ record && row.state !== 'running' && !inventoryError ? { canStart: true } : {}for (const jev of this.options.appEngines ? JEV_MODELS : []) {const record = jevRecords.find(r => r.modelId === jev.id)const quant = await this.jevQuant(jev, jevBudget)@@ -850,7 +854,17 @@ export class LocalModels {const shown = quant ?? jev.quants[0]!const have = await this.fileComplete(shown.file, shown.size)const row = await appRow(jev.id, jev.name, 'llama.cpp', record, { sizeBytes: shown.size, quant: shown.quant, kind: 'decision' })- models.push({ ...row, app: GRID_LABEL, state: row.state === 'running' ? 'running' : have ? 'downloaded' : 'available' })+ models.push({ ...row, app: GRID_LABEL, state: row.state === 'running' ? 'running' : have ? 'downloaded' : 'available', ...restartable(row, record) })+ }+ // Decision models another app downloaded (Ollama's tev1): listed whatever their size, since they are here —+ // Start is what says when there is no memory for one. A record whose model has since gone keeps a row to stop it by.+ for (const app of this.options.appEngines ? apps.filter(a => a.kind === 'decision') : []) {+ const record = jevRecords.find(r => r.modelId === app.id)+ const row = await appRow(app.id, app.name, app.app, record, { sizeBytes: app.sizeBytes, ...(app.quant ? { quant: app.quant } : {}), kind: 'decision' })+ models.push({ ...row, ...restartable(row, record) })+ }+ for (const record of jevRecords.filter(r => !JEV_MODELS.some(j => j.id === r.modelId) && !apps.some(a => a.id === r.modelId))) {+ models.push({ ...await appRow(record.modelId, record.name, record.engine, record, { kind: 'decision' }), canStart: false })}const freeDiskBytes = await statfs(join(this.home, 'models')).catch(() => statfs(this.home)).then(disk => disk.bavail * disk.bsize, () => undefined)@@ -1116,8 +1130,9 @@ export class LocalModels {}/**- * A Jev model's Get, Start and Stop. Get is the whole of it: the weights, an engine that can serve them,- * and the model on the grid — so a person never meets "too old" for an engine they did not choose.+ * A decision model's Get, Start and Stop. For a Jev model Get is the whole of it: the weights, an engine that+ * can serve them, and the model on the grid — so a person never meets "too old" for an engine they did not+ * choose. Another app's decision model (Ollama's tev1) is here already, so Start is all there is to it.** ⚠️ Grid's own llama.cpp is updated in place (`grid engine install llama.cpp`) when it is older than* [MIN_JEV_BUILD], exactly as a missing one is installed for a chat model: nothing else upgrades the engine@@ -1127,67 +1142,115 @@ export class LocalModels {private async performJev(grid: string, operation: ModelOperation,change: (stage: ModelOperation['stage']) => Promise<void>,must: (args: string[], message: string, output?: (chunk: string) => void) => Promise<void>): Promise<void> {- const jev = JEV_MODELS.find(m => m.id === operation.modelId)- const ops = this.options.appEngines- if (!jev || !ops) throw new ModelError('This model could not be checked. Refresh and try again.')- const records = await readAppRecords(this.appRecordsFile)- const record = records.find(r => r.modelId === jev.id && r.grid === grid)- const takeDown = async (started: AppEngineRecord) => {- await this.run(['--remote', 'leave', grid, '--engine', started.alias], undefined, 5 * 60_000)- await ops.stop(started)- await writeAppRecords(this.appRecordsFile, (await readAppRecords(this.appRecordsFile)).filter(r => !(r.modelId === jev.id && r.grid === grid)))+ if (!this.options.appEngines) throw new ModelError('This model could not be checked. Refresh and try again.')+ let record = (await readAppRecords(this.appRecordsFile)).find(r => r.modelId === operation.modelId && r.grid === grid)+ // Start on one the grid does not serve takes its engine down first, and starts it again: its process may+ // have gone with it, and a fresh start is the one path that checks every step.+ if (record && operation.action === 'start' && !await this.decisionServing(grid, record)) {+ await this.stopDecision(grid, record)+ record = undefined}if (operation.action === 'stop') {- if (record) await takeDown(record)+ if (record) await this.stopDecision(grid, record)} else if (!record) {- const quant = await this.jevQuant(jev, modelBudget(await this.readDevice()))- if (!quant) throw new ModelError(`This computer does not have the memory to run ${jev.name}. Close some apps, or choose a smaller model.`)- if (!await this.fileComplete(quant.file, quant.size)) {- await mkdir(join(this.home, 'models'), { recursive: true })- const disk = await statfs(join(this.home, 'models'))- if (disk.bavail * disk.bsize < quant.size + GiB) throw new ModelError('Free up disk space, then start again.')- await change('downloading')- let last = 0, progressWrites = Promise.resolve()- await must(['pull', `${jev.repo}:${quant.file}`], 'The download stopped. Start again to resume.', chunk => {- const matches = [...chunk.matchAll(/(\d+(?:\.\d+)?)\s*%/g)]- const percent = matches.length ? Number(matches.at(-1)![1]) : NaN- if (Number.isFinite(percent) && percent >= 0 && percent <= 100 && Date.now() - last > 500) {- last = Date.now(); operation.progress = percent / 100- progressWrites = progressWrites.then(() => this.save(grid, operation)).catch(() => {})- }- })- await progressWrites- if (!await this.fileComplete(quant.file, quant.size)) throw new ModelError('The download is incomplete. Start again to resume.')- }- if (operation.action === 'start') {- const binary = await this.jevEngine(jev, change, must)- await change('starting')- if (DOWN.has(await this.gridStatus(grid).catch(() => ''))) await must(['--remote', 'start', grid], 'Your grid could not start. Try again.')- const model: AppModel = { id: jev.id, name: jev.name, app: 'llama.cpp', engine: 'llama.cpp', ref: join(this.home, 'models', quant.file), binary, sizeBytes: quant.size }- let started: AppEngineRecord- try {- started = { spec: 1, modelId: jev.id, grid, name: jev.name, ...await ops.start(model, JEV_CONTEXT, join(this.options.stateDir, 'logs'), JEV_SLOTS) }- } catch (error) {- throw new ModelError(error instanceof AppStartError ? error.message : 'The model could not start. Try again.')- }- await writeAppRecords(this.appRecordsFile, [...(await readAppRecords(this.appRecordsFile)).filter(r => !(r.modelId === jev.id && r.grid === grid)), started])- const named = this.options.machineName?.()?.trim()- const joined = await this.run(['--remote', 'join', grid, '--at', `http://127.0.0.1:${started.port}/v1`, '-m', started.served,- '--advertise-as', started.alias, ...concurrencyArgs(await this.gridConcurrency(grid, false)),- ...(named && validArg(named) ? ['--name', named] : [])], undefined, 10 * 60_000)- if (!joined.ok) { await takeDown(started); throw new ModelError('The model could not join your grid. Try again.') }- await change('verifying')- try { await this.verifyDecision(grid, started.alias) } catch (error) {- await takeDown(started)- throw error instanceof ModelError ? error : new ModelError('The model did not answer. Try again.')- }- }+ const jev = JEV_MODELS.find(m => m.id === operation.modelId)+ const model = jev ? await this.getJev(grid, jev, operation, change, must) : await this.appDecision(operation.modelId)+ if (model && operation.action === 'start') await this.startDecision(grid, model, change, must)}operation.phase = 'done'delete operation.progressawait this.save(grid, operation)}+ /** [jev]'s weights, downloaded when they are not here; for a start, the model as Grid's llama.cpp runs it. */+ private async getJev(grid: string, jev: JevModel, operation: ModelOperation,+ change: (stage: ModelOperation['stage']) => Promise<void>,+ must: (args: string[], message: string, output?: (chunk: string) => void) => Promise<void>): Promise<AppModel | undefined> {+ const quant = await this.jevQuant(jev, modelBudget(await this.readDevice()))+ if (!quant) throw new ModelError(`This computer does not have the memory to run ${jev.name}. Close some apps, or choose a smaller model.`)+ if (!await this.fileComplete(quant.file, quant.size)) {+ await mkdir(join(this.home, 'models'), { recursive: true })+ const disk = await statfs(join(this.home, 'models'))+ if (disk.bavail * disk.bsize < quant.size + GiB) throw new ModelError('Free up disk space, then start again.')+ await change('downloading')+ let last = 0, progressWrites = Promise.resolve()+ await must(['pull', `${jev.repo}:${quant.file}`], 'The download stopped. Start again to resume.', chunk => {+ const matches = [...chunk.matchAll(/(\d+(?:\.\d+)?)\s*%/g)]+ const percent = matches.length ? Number(matches.at(-1)![1]) : NaN+ if (Number.isFinite(percent) && percent >= 0 && percent <= 100 && Date.now() - last > 500) {+ last = Date.now(); operation.progress = percent / 100+ progressWrites = progressWrites.then(() => this.save(grid, operation)).catch(() => {})+ }+ })+ await progressWrites+ if (!await this.fileComplete(quant.file, quant.size)) throw new ModelError('The download is incomplete. Start again to resume.')+ }+ if (operation.action !== 'start') return undefined+ return { id: jev.id, name: jev.name, app: 'llama.cpp', engine: 'llama.cpp', ref: join(this.home, 'models', quant.file),+ binary: await this.jevEngine(jev, change, must), sizeBytes: quant.size }+ }++ /** Another app's decision model, from the last scan — or a new one, when the model clicked is not in it — once+ * its app is new enough and this computer has the memory for it. */+ private async appDecision(modelId: string): Promise<AppModel> {+ let apps = await this.apps()+ if (!apps.some(a => a.id === modelId)) apps = await this.scanApps()+ const app = apps.find(a => a.id === modelId && a.kind === 'decision')+ if (!app) throw new ModelError('This model could not be checked. Refresh and try again.')+ const label = APP_LABEL[app.app]+ if (app.needs) throw new ModelError(`${app.name} needs ${label} ${app.needs} or newer. Update ${label}, then start again.`)+ const budget = modelBudget(await this.readDevice())+ if (budget !== undefined && jevMemory(app.sizeBytes) > budget) {+ throw new ModelError(`This computer does not have the memory to run ${app.name}. Close some apps, or choose a smaller model.`)+ }+ return app+ }++ /** Runs decision [model] on its own loopback port with [JEV_SLOTS] slots and joins it to [grid] `--at`, beside+ * whatever runs there; up once a decision through the grid is answered, and taken down when it is not. */+ private async startDecision(grid: string, model: AppModel,+ change: (stage: ModelOperation['stage']) => Promise<void>,+ must: (args: string[], message: string) => Promise<void>): Promise<void> {+ await change('starting')+ if (DOWN.has(await this.gridStatus(grid).catch(() => ''))) await must(['--remote', 'start', grid], 'Your grid could not start. Try again.')+ let started: AppEngineRecord+ try {+ started = { spec: 1, modelId: model.id, grid, name: model.name,+ ...await this.options.appEngines!.start(model, JEV_CONTEXT, join(this.options.stateDir, 'logs'), JEV_SLOTS) }+ } catch (error) {+ throw new ModelError(error instanceof AppStartError ? error.message : 'The model could not start. Try again.')+ }+ await writeAppRecords(this.appRecordsFile, [...(await readAppRecords(this.appRecordsFile)).filter(r => !(r.modelId === model.id && r.grid === grid)), started])+ const named = this.options.machineName?.()?.trim()+ const joined = await this.run(['--remote', 'join', grid, '--at', `http://127.0.0.1:${started.port}/v1`, '-m', started.served,+ '--advertise-as', started.alias, ...concurrencyArgs(await this.gridConcurrency(grid, false)),+ ...(named && validArg(named) ? ['--name', named] : [])], undefined, 10 * 60_000)+ if (!joined.ok) { await this.stopDecision(grid, started); throw new ModelError('The model could not join your grid. Try again.') }+ await change('verifying')+ try { await this.verifyDecision(grid, started.alias) } catch (error) {+ await this.stopDecision(grid, started)+ throw error instanceof ModelError ? error : new ModelError('The model did not answer. Try again.')+ }+ }++ /** Whether [record]'s engine is up and [grid] lists it — or the grid sleeps, which lists nothing, and it is parked. */+ private async decisionServing(grid: string, record: AppEngineRecord): Promise<boolean> {+ if (!await this.options.appEngines!.alive(record)) return false+ const inventory = await this.options.inventory(grid, true).catch(() => null)+ // Unread is not "not served": an engine that may be serving is never taken down on a guess.+ if (!inventory || (inventory.state === 'unknown' && !DOWN.has(inventory.status ?? ''))) return true+ if (inventory.state === 'asleep' || inventory.status === 'asleep') return true+ return rows(inventory.nodes).some(n => n.online === true && (Array.isArray(n.models) ? n.models : [])+ .some((m: unknown) => modelKey(str(typeof m === 'string' ? m : obj(m).model)) === modelKey(record.alias)))+ }++ /** A decision engine off [grid], stopped, and its record gone. */+ private async stopDecision(grid: string, started: AppEngineRecord): Promise<void> {+ await this.run(['--remote', 'leave', grid, '--engine', started.alias], undefined, 5 * 60_000)+ await this.options.appEngines?.stop(started)+ await writeAppRecords(this.appRecordsFile, (await readAppRecords(this.appRecordsFile)).filter(r => !(r.modelId === started.modelId && r.grid === grid)))+ }+/** Grid's llama-server, brought up to a build that runs [jev] first when it is older (or missing). */private async jevEngine(jev: JevModel, change: (stage: ModelOperation['stage']) => Promise<void>,must: (args: string[], message: string) => Promise<void>): Promise<string> {