Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Jev catalog sized to the machine, Chat/Decision models in the picker, and Jev engines that never restart a chat model (#888)

scholes-autonomousfcd107497bff
diff --git a/cli/src/lib/appModels.spec.ts b/cli/src/lib/appModels.spec.ts
index 8d3035547..4e5128bf5 100644
--- a/cli/src/lib/appModels.spec.ts
+++ b/cli/src/lib/appModels.spec.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest'
-import { APP_CONTEXT, appContext, cacheName, scanAppModels } from './appModels.js'
+import { APP_CONTEXT, appContext, cacheName, llamaServerArgs, scanAppModels } from './appModels.js'
const GiB = 1024 ** 3
const grid = { kind: 'llama.cpp', path: '/home/me/.grid/bin/llama-server', version: 'version: 10369', note: "Grid's own engine" }
@@ -117,3 +117,18 @@ describe('models other apps downloaded', () => {
expect(cacheName('plain-model-Q4_K_M')).toBe('plain-model-Q4_K_M')
})
})
+
+describe('llama-server for a model', () => {
+ const model = { ref: '/m/Laya-Q8_0.gguf', name: 'laya-english' }
+
+ it("gives a harness's model one slot and its whole window", () => {
+ const args = llamaServerArgs(model, 131072, 41001)
+ expect(args).toEqual(['-m', '/m/Laya-Q8_0.gguf', '--alias', 'laya-english', '--ctx-size', '131072', '--parallel', '1',
+ '-ngl', '999', '--host', '127.0.0.1', '--port', '41001'])
+ })
+
+ it('shares one window between several slots, so any one request can still use all of it', () => {
+ const args = llamaServerArgs(model, 8192, 41001, 4)
+ expect(args.slice(args.indexOf('--ctx-size'), args.indexOf('-ngl'))).toEqual(['--ctx-size', '8192', '--parallel', '4', '--kv-unified'])
+ })
+})
diff --git a/cli/src/lib/appModels.ts b/cli/src/lib/appModels.ts
index c33609652..a553301aa 100644
--- a/cli/src/lib/appModels.ts
+++ b/cli/src/lib/appModels.ts
@@ -143,9 +143,18 @@ function freePort(): Promise<number> {
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
+/** llama-server's arguments for [model] on loopback [port]. Several slots share one window (`--kv-unified`),
+ * as llama-server's own default does, so any one request can still use all of it. */
+export function llamaServerArgs(model: Pick<AppModel, 'ref' | 'name'>, ctx: number, port: number, slots = 1): string[] {
+ return ['-m', model.ref, '--alias', model.name, '--ctx-size', String(ctx), '--parallel', String(slots),
+ ...(slots > 1 ? ['--kv-unified'] : []), '-ngl', '999', '--host', '127.0.0.1', '--port', String(port)]
+}
+
/** Starting, checking and stopping the engines, injected so the lifecycle can be tested without them. */
export interface AppEngineOps {
- start(model: AppModel, ctx: number, logDir: string): Promise<Omit<AppEngineRecord, 'spec' | 'modelId' | 'grid' | 'name'>>
+ /** `slots`: how many requests the engine works on at once. A harness's model has one, so its whole window
+ * is one conversation's; a Jev model's questions are short and many, and each is its own request. */
+ start(model: AppModel, ctx: number, logDir: string, slots?: number): Promise<Omit<AppEngineRecord, 'spec' | 'modelId' | 'grid' | 'name'>>
alive(record: AppEngineRecord): Promise<boolean>
/** The window the engine says it loaded, after a first request; undefined when it does not say. */
loadedContext(record: AppEngineRecord): Promise<number | undefined>
@@ -192,21 +201,20 @@ export function appEngineOps(env: NodeJS.ProcessEnv, request: typeof fetch = fet
}
return {
- async start(model, ctx, logDir) {
+ async start(model, ctx, logDir, slots = 1) {
const binary = model.binary!
if (model.engine === 'ollama') {
// Its own server beside the person's Ollama app, never on the app's 11434: the window is a server
// setting, and theirs ran every model at 16K [run]. Same model store, so nothing is copied.
const port = await freePort()
const pid = await background(binary, ['serve'], { OLLAMA_HOST: `127.0.0.1:${port}`, OLLAMA_CONTEXT_LENGTH: String(ctx),
- OLLAMA_NUM_PARALLEL: '1', OLLAMA_KEEP_ALIVE: '-1' }, logDir, `ollama-${port}.log`)
+ OLLAMA_NUM_PARALLEL: String(slots), OLLAMA_KEEP_ALIVE: '-1' }, logDir, `ollama-${port}.log`)
if (!await ready(`http://127.0.0.1:${port}/api/version`, 60_000, pid)) throw new AppStartError('Ollama could not start. Try again.')
return { engine: 'ollama', served: model.ref, alias: model.name, port, binary, pid }
}
if (model.engine === 'llama.cpp') {
const port = await freePort()
- const pid = await background(binary, ['-m', model.ref, '--alias', model.name, '--ctx-size', String(ctx), '--parallel', '1',
- '-ngl', '999', '--host', '127.0.0.1', '--port', String(port)], {}, logDir, `llama-${port}.log`)
+ const pid = await background(binary, llamaServerArgs(model, ctx, port, slots), {}, logDir, `llama-${port}.log`)
if (!await ready(`http://127.0.0.1:${port}/health`, 10 * 60_000, pid)) {
try { process.kill(-pid, 'SIGTERM') } catch { /* gone */ }
throw new AppStartError('llama.cpp could not load the model. Try again.')
@@ -224,7 +232,7 @@ export function appEngineOps(env: NodeJS.ProcessEnv, request: typeof fetch = fet
}
const port = typeof status.port === 'number' ? status.port : 0
if (!port) throw new AppStartError('LM Studio could not start its server. Open LM Studio and try again.')
- const loaded = await run(binary, ['load', model.ref, '--context-length', String(ctx), '--gpu', 'max', '--parallel', '1',
+ const loaded = await run(binary, ['load', model.ref, '--context-length', String(ctx), '--gpu', 'max', '--parallel', String(slots),
'--identifier', model.name, '-y'], { env, timeout: 10 * 60_000 })
if (!loaded.ok) {
if (startedServer) await run(binary, ['server', 'stop'], { env, timeout: 30_000 })
diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.ts
index 31a0ff0d5..de00d8ed0 100644
--- a/cli/src/lib/localModels.spec.ts
+++ b/cli/src/lib/localModels.spec.ts
@@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'
import { createHash } from 'node:crypto'
import { rmSync, writeFileSync } from 'node:fs'
import { spawnSync } from 'node:child_process'
-import { baseModel, compatibleModels, contextLadder, llamaBuild, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'
+import { baseModel, compatibleModels, contextLadder, JEV_MODELS, jevMemory, llamaBuild, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'
import { GridFleetRpc, type GridFleetResult } from './gridFleetRpc.js'
import { AppStartError, type AppEngineOps, type AppEngineRecord, type AppModel } from './appModels.js'
import { encryptDownFrame, encryptRpcResult } from './e2ee/applicationFrames.js'
@@ -163,7 +163,7 @@ describe('local model discovery and lifecycle', () => {
expect(snapshot.models[0]).toMatchObject({ state: 'running', canStop: true,
tokensPerSecond: 17.6, requests: 4, windowSeconds: 3600, operation: { phase: 'done', stage: 'verifying' } })
expect(snapshot.models[0]).not.toHaveProperty('memoryBytes')
- expect(calls.find(args => args.includes('join'))).toEqual(['--remote', 'join', 'home', '--serve', 'Small-Q4.gguf', '--max-concurrency', '1', '--ctx-size', '131072', '--endpoint-port', expect.stringMatching(/^\d+$/), '--reasoning-budget', '0'])
+ expect(calls.find(args => args.includes('join'))).toEqual(['--remote', 'join', 'home', '--serve', 'Small-Q4.gguf', '--max-concurrency', '5', '--parallel', '1', '--ctx-size', '131072', '--endpoint-port', expect.stringMatching(/^\d+$/), '--reasoning-budget', '0'])
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(true)
expect(await readFile(join(stateDir, (await readdir(stateDir))[0]), 'utf8')).not.toContain('token')
})
@@ -314,6 +314,20 @@ describe('local model discovery and lifecycle', () => {
expect((await fresh.list('home')).models[0].operation?.phase).toBe('done')
})
+ it("knows Grid's own engine by its own names when another engine joins beside it", async () => {
+ catalogCards = []
+ await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
+ // A Jev model joined beside it: each engine carries its own names, the record's flat list all of them.
+ await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', advertise_as: ['small', 'kev-0.8b'],
+ engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'], advertise_as: ['small'] },
+ { endpoint_url: 'http://127.0.0.1:50872/v1', models: ['kev-0.8b'], advertise_as: ['kev-0.8b'] }] }))
+ await writeFile(join(records, 'remote.heartbeat'), '')
+ inventory.mockImplementation(async () => ({ state: 'awake', status: 'running',
+ nodes: [{ node_id: 'local-node', online: true, models: ['small', 'kev-0.8b'] }] }))
+ expect((await service.list('home')).models.find(m => m.id === 'local:Small-Q4.gguf'))
+ .toMatchObject({ state: 'running', canStop: true })
+ })
+
it.each(['current', 'legacy', 'malformed'])('preserves imported routing names across restart with %s receipts', async scenario => {
catalogCards = []; serving = true
await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
@@ -1451,7 +1465,7 @@ describe('models other apps downloaded, started in their own app', () => {
await models.act('home', ollama.id, 'start'); await models.settled()
expect(ops.start).toHaveBeenCalledWith(ollama, 131072, join(stateDir, 'logs'))
expect(calls.find(args => args.includes('--at'))).toEqual(['--remote', 'join', 'home', '--at', 'http://127.0.0.1:41000/v1',
- '-m', 'llama3.2:3b', '--advertise-as', 'llama3.2:3b', '--max-concurrency', '1'])
+ '-m', 'llama3.2:3b', '--advertise-as', 'llama3.2:3b', '--max-concurrency', '5'])
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(true)
let row = (await models.list('home', true)).models.find(m => m.id === ollama.id)!
expect(row).toMatchObject({ state: 'running', canStart: false, canStop: true, operation: { phase: 'done' } })
@@ -1836,13 +1850,17 @@ describe('Jev models: Get brings Grid\'s llama.cpp up to a build that serves the
await mkdir(join(home, 'bin'), { recursive: true })
await writeFile(join(home, 'bin', 'llama-server'), `#!/bin/sh\necho "version: ${build}"\n`, { mode: 0o755 })
}
- const jevService = (env: NodeJS.ProcessEnv = {}, over: Partial<ConstructorParameters<typeof LocalModels>[0]> = {}) => {
+ const jevService = (env: NodeJS.ProcessEnv = {}, over: Partial<ConstructorParameters<typeof LocalModels>[0]> = {},
+ device?: Record<string, unknown> | null) => {
const base = run.getMockImplementation()!
run.mockImplementation(async (args, output) => {
- if (args[0] === 'pull' && args[1] === 'ggml-org/Laya-GGUF:Laya-Q8_0.gguf') {
+ if (device !== undefined && args[0] === 'device-info') { calls.push(args); return device ? ok(device) : refused('no device') }
+ const pulled = args[0] === 'pull' ? JEV_MODELS.flatMap(m => m.quants.map(q => ({ ...q, ref: `${m.repo}:${q.file}` })))
+ .find(q => q.ref === args[1]) : undefined
+ if (pulled) {
calls.push(args); output?.('100%')
- const file = join(home, 'models', 'Laya-Q8_0.gguf')
- await writeFile(file, ''); await (await import('node:fs/promises')).truncate(file, LAYA_SIZE)
+ const file = join(home, 'models', pulled.file)
+ await writeFile(file, ''); await (await import('node:fs/promises')).truncate(file, pulled.size)
return ok()
}
if (args[0] === 'engine' && args[1] === 'install') { calls.push(args); await engine(installBuild); return ok() }
@@ -1901,9 +1919,9 @@ describe('Jev models: Get brings Grid\'s llama.cpp up to a build that serves the
expect(installed).toBeGreaterThan(pulled)
expect(stages).toContain('updating')
expect(ops.start).toHaveBeenLastCalledWith(expect.objectContaining({ id: LAYA, engine: 'llama.cpp', binary: join(home, 'bin', 'llama-server'),
- ref: join(home, 'models', 'Laya-Q8_0.gguf') }), 8192, join(stateDir, 'logs'))
+ ref: join(home, 'models', 'Laya-Q8_0.gguf') }), 8192, join(stateDir, 'logs'), 4)
expect(calls.find(args => args.includes('--at') && args.includes('laya-english'))).toEqual(['--remote', 'join', 'home', '--at',
- 'http://127.0.0.1:41001/v1', '-m', 'laya-english', '--advertise-as', 'laya-english', '--max-concurrency', '1'])
+ 'http://127.0.0.1:41001/v1', '-m', 'laya-english', '--advertise-as', 'laya-english', '--max-concurrency', '5'])
expect(decisions).toEqual(['laya-english'])
expect(await laya(models)).toMatchObject({ kind: 'decision', state: 'running', canStop: true, operation: { phase: 'done' } })
// And a running Jev model blocks no chat model either.
@@ -1916,6 +1934,68 @@ describe('Jev models: Get brings Grid\'s llama.cpp up to a build that serves the
expect(await laya(models)).toMatchObject({ state: 'downloaded', canStart: true, canStop: false })
})
+ it("takes its grid's one request limit, and never changes it where Grid runs an engine itself", async () => {
+ await engine('0.5.0-dev (build 11378, commit edd6e2bbd)')
+ const limit = () => {
+ const join = calls.filter(args => args.includes('--at') && args.includes('laya-english')).at(-1)!
+ return join.includes('--max-concurrency') ? join[join.indexOf('--max-concurrency') + 1] : 'left as it is'
+ }
+ const models = jevService()
+ // Nothing of Grid's runs here: the limit every join of this computer's sets.
+ await models.act('home', LAYA, 'start'); await models.settled()
+ expect(limit()).toBe('5')
+ await models.act('home', LAYA, 'stop'); await models.settled()
+ // A chat model Grid runs itself — with a slot count of its own or without: a join that changed the limit would
+ // restart the node, and the restart reload that model, so the Jev model's join leaves it.
+ for (const launch of [{ parallel: 1 }, undefined]) {
+ await writeFile(join(records, 'remote.json'), JSON.stringify({ node_id: 'local-node', max_concurrency: 1,
+ engines: [{ endpoint_url: null, models: ['Small-Q4.gguf'], ...(launch ? { launch } : {}) }] }))
+ await models.act('home', LAYA, 'start'); await models.settled()
+ expect(limit()).toBe('left as it is')
+ expect(await laya(models)).toMatchObject({ state: 'running' })
+ await models.act('home', LAYA, 'stop'); await models.settled()
+ }
+ })
+
+ // An NVIDIA card with 10 GiB free: its VRAM is the model's own, all of it the budget.
+ const card10 = { device_class: 'nvidia', backend: 'cuda', usable_bytes: 10 * GiB, memory: { total_gb: 32 } }
+ const offered = async (models: LocalModels) => Object.fromEntries((await models.list('home', true)).models
+ .filter(m => m.kind === 'decision').map(m => [m.name, m.quant]))
+
+ it('offers only the Jev models this computer can start, each at the best quant that fits', async () => {
+ const models = jevService({}, {}, card10)
+ expect(await offered(models)).toEqual({ 'laya-english': 'Q8_0', 'kev-0.8b': 'Q8_0', 'kev-4b': 'Q8_0', lev: 'Q8_0',
+ 'kev-9b': 'Q4_K_M', 'nimble-9b': 'Q4_K_M', 'clef-flash': 'Q4_K_M' })
+ // Clef needs 23 GB even at Q4_K_M; Kev 9B's Q8_0 would need 12.
+ expect(jevMemory(JEV_MODELS.find(m => m.name === 'clef')!.quants.at(-1)!.size)).toBeGreaterThan(10 * GiB)
+ // Get downloads the quant it was offered at.
+ await models.act('home', 'jev:ggml-org/Kev-9B-GGUF', 'download'); await models.settled()
+ expect(calls.filter(args => args[0] === 'pull')).toEqual([['pull', 'ggml-org/Kev-9B-GGUF:Kev-9B-Q4_K_M.gguf']])
+ })
+
+ it('offers none it cannot size, but keeps listing the ones already here, at the quant that is here', async () => {
+ await writeFile(join(home, 'models', 'Kev-9B-Q8_0.gguf'), '')
+ await (await import('node:fs/promises')).truncate(join(home, 'models', 'Kev-9B-Q8_0.gguf'), 9_529_735_648)
+ await writeFile(join(home, 'models', 'Clef-Q4_K_M.gguf.part'), 'half')
+ // grid cannot say how much memory there is: nothing new is offered.
+ expect(await offered(jevService({}, {}, null))).toEqual({ 'kev-9b': 'Q8_0', clef: 'Q4_K_M' })
+ // On a card too small for either, they stay too — one downloaded, one to resume, never swapped for a smaller file.
+ const models = jevService({}, {}, card10)
+ expect(await offered(models)).toMatchObject({ 'kev-9b': 'Q8_0', clef: 'Q4_K_M' })
+ await models.act('home', 'jev:ggml-org/Clef-GGUF', 'download'); await models.settled()
+ expect(calls.filter(args => args[0] === 'pull')).toEqual([['pull', 'ggml-org/Clef-GGUF:Clef-Q4_K_M.gguf']])
+ })
+
+ it('updates an engine new enough for Jev models but not for Clef, whose architecture came later', async () => {
+ await engine('0.5.0-dev (build 11365, commit 1a2b3c4d5)')
+ const models = jevService({}, {}, { ...card10, usable_bytes: 48 * GiB })
+ await models.act('home', LAYA, 'start'); await models.settled()
+ expect(calls.some(args => args[0] === 'engine')).toBe(false)
+ await models.act('home', 'jev:ggml-org/Clef-GGUF', 'start'); await models.settled()
+ expect(calls.filter(args => args[0] === 'engine')).toEqual([['engine', 'install', 'llama.cpp']])
+ expect((await models.list('home', true)).models.find(m => m.name === 'clef')).toMatchObject({ state: 'running', quant: 'Q8_0' })
+ })
+
it('leaves an engine new enough alone', async () => {
await engine('0.5.0-dev (build 11378, commit edd6e2bbd)')
const models = jevService()
@@ -1929,7 +2009,7 @@ describe('Jev models: Get brings Grid\'s llama.cpp up to a build that serves the
installBuild = '10369 (6e62ba538)'
const models = jevService()
await models.act('home', LAYA, 'start'); await models.settled()
- expect((await laya(models)).operation).toMatchObject({ phase: 'failed', error: expect.stringContaining('still too old for Jev models (build 10369') })
+ expect((await laya(models)).operation).toMatchObject({ phase: 'failed', error: expect.stringContaining('still too old for laya-english (build 10369') })
expect(ops.start).not.toHaveBeenCalled()
})
@@ -1938,7 +2018,7 @@ describe('Jev models: Get brings Grid\'s llama.cpp up to a build that serves the
await writeFile(custom, '#!/bin/sh\necho "version: 0.5.0 (build 11146, commit 7fe450e19)"\n', { mode: 0o755 })
const models = jevService({ LLAMA_SERVER: custom })
await models.act('home', LAYA, 'start'); await models.settled()
- expect((await laya(models)).operation).toMatchObject({ phase: 'failed', error: 'LLAMA_SERVER is llama.cpp build 11146; Jev models need build 11361 or newer.' })
+ expect((await laya(models)).operation).toMatchObject({ phase: 'failed', error: 'LLAMA_SERVER is llama.cpp build 11146; laya-english needs build 11361 or newer.' })
expect(calls.some(args => args[0] === 'engine')).toBe(false)
})
@@ -1999,7 +2079,7 @@ describe('Jev models: Get brings Grid\'s llama.cpp up to a build that serves the
const result = await service.act('home', LAYA, 'start'); await service.settled()
expect(result.operation).toMatchObject({ phase: 'failed', error: scenario === 'install failed'
? 'The model engine could not be updated. Try again.'
- : "Grid's model engine is still too old for Jev models (build unknown; they need 11361 or newer). Update Grid, then try again." })
+ : "Grid's model engine is still too old for laya-english (build unknown; it needs 11361 or newer). Update Grid, then try again." })
expect(ops.start).not.toHaveBeenCalled()
})
@@ -2009,7 +2089,7 @@ describe('Jev models: Get brings Grid\'s llama.cpp up to a build that serves the
service = jevService({ LLAMA_SERVER: binary })
const result = await service.act('home', LAYA, 'start'); await service.settled()
expect(result.operation?.phase).toBe('done')
- expect(ops.start).toHaveBeenCalledWith(expect.objectContaining({ binary }), 8192, join(stateDir, 'logs'))
+ expect(ops.start).toHaveBeenCalledWith(expect.objectContaining({ binary }), 8192, join(stateDir, 'logs'), 4)
expect(calls.some(args => args[0] === 'engine')).toBe(false)
expect(decisions).toEqual(['laya-english'])
})
diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.ts
index 559720a92..c3b887c5d 100644
--- a/cli/src/lib/localModels.ts
+++ b/cli/src/lib/localModels.ts
@@ -42,15 +42,51 @@ const UNREAD_CONTEXT = 128 * 1024
* with autonomous-grid's catalog (`shared/models/catalog.py`, its `kind="decision"` rows); `size` is the
* file's own, which is how a finished download is told from a partial one.
*/
-export const JEV_MODELS: ReadonlyArray<{ id: string; name: string; pull: string; file: string; size: number; quant: string }> = [
- { id: 'jev:ggml-org/Laya-GGUF', name: 'laya-english', pull: 'ggml-org/Laya-GGUF:Laya-Q8_0.gguf', file: 'Laya-Q8_0.gguf', size: 449_397_600, quant: 'Q8_0' },
+/** One file a Jev model is published as. */
+export interface JevQuant { quant: string; file: string; size: number }
+/** A Jev (System One) model: ggml-org's official GGUF for llama.cpp's `/v1/systemone`, which reads the model's
+ * decision type (Laya, Kev, Lev, Nimble, Clef) from the file. [quants] best first — Q8_0, and Q4_K_M for a
+ * computer that has no room for it. */
+export interface JevModel {
+ id: string; name: string; repo: string; quants: ReadonlyArray<JevQuant>
+ /** The first llama.cpp build that runs it, when that is later than [MIN_JEV_BUILD]'s. */
+ minBuild?: number
+}
+export const JEV_MODELS: ReadonlyArray<JevModel> = [
+ { id: 'jev:ggml-org/Laya-GGUF', name: 'laya-english', repo: 'ggml-org/Laya-GGUF',
+ quants: [{ quant: 'Q8_0', file: 'Laya-Q8_0.gguf', size: 449_397_600 }] },
+ { id: 'jev:ggml-org/Kev-0.8B-GGUF', name: 'kev-0.8b', repo: 'ggml-org/Kev-0.8B-GGUF',
+ quants: [{ quant: 'Q8_0', file: 'Kev-0.8B-Q8_0.gguf', size: 812_406_304 }] },
+ { id: 'jev:ggml-org/Kev-4B-GGUF', name: 'kev-4b', repo: 'ggml-org/Kev-4B-GGUF', quants: [
+ { quant: 'Q8_0', file: 'Kev-4B-Q8_0.gguf', size: 4_483_801_504 }, { quant: 'Q4_K_M', file: 'Kev-4B-Q4_K_M.gguf', size: 3_033_489_824 }] },
+ { id: 'jev:ggml-org/lev-GGUF', name: 'lev', repo: 'ggml-org/lev-GGUF', quants: [
+ { quant: 'Q8_0', file: 'lev-Q8_0.gguf', size: 4_482_405_280 }, { quant: 'Q4_K_M', file: 'lev-Q4_K_M.gguf', size: 3_011_777_440 }] },
+ { id: 'jev:ggml-org/Kev-9B-GGUF', name: 'kev-9b', repo: 'ggml-org/Kev-9B-GGUF', quants: [
+ { quant: 'Q8_0', file: 'Kev-9B-Q8_0.gguf', size: 9_529_735_648 }, { quant: 'Q4_K_M', file: 'Kev-9B-Q4_K_M.gguf', size: 6_358_923_744 }] },
+ { id: 'jev:ggml-org/Bespoke-Nimble-9B-v3-GGUF', name: 'nimble-9b', repo: 'ggml-org/Bespoke-Nimble-9B-v3-GGUF', quants: [
+ { quant: 'Q8_0', file: 'Bespoke-Nimble-9B-v3-Q8_0.gguf', size: 9_527_503_392 },
+ { quant: 'Q4_K_M', file: 'Bespoke-Nimble-9B-v3-Q4_K_M.gguf', size: 6_324_185_632 }] },
+ // Clef's own architecture came after `/v1/systemone` itself (ggml-org/llama.cpp#29831, b11371).
+ { id: 'jev:ggml-org/Clef-Flash-GGUF', name: 'clef-flash', repo: 'ggml-org/Clef-Flash-GGUF', minBuild: 11371, quants: [
+ { quant: 'Q8_0', file: 'Clef-Flash-Q8_0.gguf', size: 9_657_260_192 }, { quant: 'Q4_K_M', file: 'Clef-Flash-Q4_K_M.gguf', size: 6_486_448_288 }] },
+ { id: 'jev:ggml-org/Clef-GGUF', name: 'clef', repo: 'ggml-org/Clef-GGUF', minBuild: 11371, quants: [
+ { quant: 'Q8_0', file: 'Clef-Q8_0.gguf', size: 28_732_215_360 }, { quant: 'Q4_K_M', file: 'Clef-Q4_K_M.gguf', size: 19_232_219_200 }] },
]
+/** The memory a Jev model of [size] bytes takes to start: its weights, the window its slots share and
+ * llama.cpp's working buffers — 15% over the file, and a GiB. */
+export const jevMemory = (size: number): number => Math.ceil(size * 1.15) + GiB
const isJev = (id: string): boolean => id.startsWith('jev:')
+const concurrencyArgs = (limit: number | undefined): string[] => limit === undefined ? [] : ['--max-concurrency', String(limit)]
/** The first llama.cpp build that serves `/v1/systemone` (ggml-org/llama.cpp#29818). An older one
* cannot even load a decision GGUF, so Get updates Grid's engine below it — and Grid itself refuses. */
export const MIN_JEV_BUILD = 11361
/** A decision model's window: its state and options, never a conversation. */
const JEV_CONTEXT = 8192
+/** Questions a Jev engine answers at once. Each question of a request is its own task in llama.cpp, so a
+ * request's questions are answered side by side: 8 took 0.11 s on 4 slots against 0.17 s on 1 (Laya, M-series). */
+export const JEV_SLOTS = 4
+/** Requests a node of this computer's takes at once, through Grid: its chat model's one, and a Jev model's slots. */
+export const NODE_CONCURRENCY = 1 + JEV_SLOTS
/** The build a llama-server reports (`version: 0.5.0 (build 11146, …)`, or `version: 10369 (…)` before
* llama.cpp's semver releases); undefined when it is missing or does not say. 30 s: a binary macOS has
@@ -141,7 +177,9 @@ export interface Candidate {
}
/** `live`: its heartbeat sidecar is fresh (the grid is hearing from it). `pidAlive`: the process its run
* record names exists — the only liveness that holds while the grid sleeps and the sidecar goes stale. */
-interface Owned { file: string; selector: string; aliases: string[]; nodeId: string; name: string; live: boolean; pidAlive: boolean; siblings: number }
+/** `slotted`: it launches with a slot count of its own (`--parallel`). Grid gives a built-in engine without one a
+ * slot per request its node takes, each with the engine's whole window. */
+interface Owned { file: string; selector: string; aliases: string[]; nodeId: string; name: string; live: boolean; pidAlive: boolean; siblings: number; slotted: boolean }
interface Receipt { spec: 1; grid: string; operation?: ModelOperation }
/** What the grid says is running on it, read WITHOUT waking it (`gridModels.gridInventory`): the
* overview's node objects while it is awake, nothing while it sleeps — and its owner status (`running`,
@@ -423,10 +461,29 @@ export class LocalModels {
})().finally(() => { this.catalogPending = undefined })
}
+ /** This computer's profile now — the memory free on an NVIDIA card moves — or the last one read when grid
+ * does not answer. */
+ private async readDevice(): Promise<Record<string, any>> {
+ try { this.device = obj(await this.json(['device-info', '--json'])) } catch { /* the last read stands */ }
+ return this.device
+ }
+
private async fileComplete(file: string, size: number): Promise<boolean> {
return stat(join(this.home, 'models', file)).then(s => s.isFile() && s.size === size, () => false)
}
+ /** The file of [jev] this computer has or would get: one already here — whole, or a download to resume —
+ * else the best that fits [budget] ([modelBudget]: an NVIDIA card's free VRAM, half of an Apple Silicon
+ * Mac's unified memory). Undefined when none fits, or the budget is unknown: only a model that can start
+ * is offered. */
+ private async jevQuant(jev: JevModel, budget: number | undefined): Promise<JevQuant | undefined> {
+ for (const quant of jev.quants) if (await this.fileComplete(quant.file, quant.size)) return quant
+ for (const quant of jev.quants) {
+ if (await stat(join(this.home, 'models', `${quant.file}.part`)).then(s => s.isFile(), () => false)) return quant
+ }
+ return budget === undefined ? undefined : jev.quants.find(quant => jevMemory(quant.size) <= budget)
+ }
+
private async downloaded(candidate: Candidate): Promise<boolean> {
if (candidate.appPath) return stat(candidate.appPath).then(s => s.isFile() && s.size === candidate.size, () => false)
const sizes = await Promise.all(candidate.files.map(file => stat(join(this.home, 'models', file)).then(s => s.isFile() ? s.size : 0).catch(() => 0)))
@@ -455,17 +512,36 @@ export class LocalModels {
if (spec.endpoint_url || spec.api_kind || !Array.isArray(spec.models) || spec.models.length !== 1) continue
const file = str(spec.models[0])
if (!validArg(file) || !file.toLowerCase().endsWith('.gguf')) continue
- const advertised = Array.isArray(record.advertise_as) && specs.length === 1
- ? record.advertise_as.filter((v: unknown) => typeof v === 'string' && v.length > 0) : []
+ // Grid's `spec_aliases`: the engine's own names (ADR 0045) — or the record's flat list, which only ever
+ // named a sole engine. With a Jev model joined beside it, the flat list was ignored and the engine was
+ // looked for by its file name, which the grid does not list: a running model read as only downloaded.
+ const named = 'advertise_as' in spec ? spec.advertise_as : specs.length === 1 ? record.advertise_as : undefined
+ const advertised = Array.isArray(named) ? named.filter((v: unknown) => typeof v === 'string' && v.length > 0) : []
const aliases: string[] = advertised.length ? advertised : [file]
const pid = recordPid(record)
- result.push({ file: basename(file), selector: file, aliases, nodeId: str(record.node_id), name: str(record.meta_name), live, pidAlive: pid !== null && processExists(pid), siblings: specs.length + (record.media ? 1 : 0) })
+ // Grid's `builtin_launch`: the spec's own launch settings, or a record written before specs had them.
+ const launch = spec.launch && typeof spec.launch === 'object' ? spec.launch : record
+ result.push({ file: basename(file), selector: file, aliases, nodeId: str(record.node_id), name: str(record.meta_name), live,
+ pidAlive: pid !== null && processExists(pid), siblings: specs.length + (record.media ? 1 : 0), slotted: (num(launch.parallel) ?? 0) > 0 })
this.blockers.set(grid, `Stop ${cleanName(aliases[0])} first to start another local model.`)
}
}
return result
}
+ /** The `--max-concurrency` to join [grid]'s node with, or undefined to leave its limit as it is. Grid's limit is
+ * the node's, shared by every engine on it: [NODE_CONCURRENCY], at one a decision waited for a chat model's whole
+ * reply. It is one number, the same for every join, because a join that changes it restarts the node, and a
+ * restart relaunches every engine Grid runs itself — starting a 0.8 GB Kev beside a running Qwen reloaded all
+ * 25 GB of it. So a join that launches none of Grid's ([launches] false: an app's engine, a Jev model) leaves the
+ * limit alone wherever one of Grid's runs; a launch restarts the node anyway, and sets it. It is one while an engine
+ * of Grid's without its own slot count runs there: that one takes a slot per request, each its whole window. */
+ private async gridConcurrency(grid: string, launches: boolean): Promise<number | undefined> {
+ const engines = await this.owned(grid)
+ if (!launches && engines.length) return undefined
+ return engines.some(engine => !engine.slotted) ? 1 : NODE_CONCURRENCY
+ }
+
/** Keep models imported from an existing Grid setup after Stop removes its
* run record. Only completed local files that this computer already served
* become restartable; arbitrary downloaded GGUFs are not assumed compatible. */
@@ -765,10 +841,16 @@ export class LocalModels {
// Jev models: Get downloads one, brings Grid's llama.cpp up to a build that serves it, and runs it on
// the grid beside whatever else is there; it is listed `running` once the grid lists it. Offered only
// where this daemon can start an engine of its own, which is how one runs.
+ // Only those that can start here are offered: one this computer has no memory for is listed only once it
+ // is here — running, downloaded, or part way.
+ const jevBudget = modelBudget(this.device)
for (const jev of this.options.appEngines ? JEV_MODELS : []) {
const record = jevRecords.find(r => r.modelId === jev.id)
- const have = await this.fileComplete(jev.file, jev.size)
- const row = await appRow(jev.id, jev.name, 'llama.cpp', record, { sizeBytes: jev.size, quant: jev.quant, kind: 'decision' })
+ const quant = await this.jevQuant(jev, jevBudget)
+ if (!quant && !record && operation?.modelId !== jev.id) continue
+ const shown = quant ?? jev.quants[0]!
+ const have = await this.fileComplete(shown.file, shown.size)
+ const row = await appRow(jev.id, jev.name, 'llama.cpp', record, { sizeBytes: shown.size, quant: shown.quant, kind: 'decision' })
models.push({ ...row, app: GRID_LABEL, state: row.state === 'running' ? 'running' : have ? 'downloaded' : 'available' })
}
const freeDiskBytes = await statfs(join(this.home, 'models')).catch(() => statfs(this.home))
@@ -917,7 +999,9 @@ export class LocalModels {
const port = await freePort()
const joined = await this.run(['--remote', 'join', grid, '--serve', candidate.file,
...(machineName ? ['--name', machineName] : []),
- '--max-concurrency', '1', '--ctx-size', String(ctx), '--endpoint-port', String(port),
+ // One slot, its whole window: a harness's model is one conversation, whatever the node's concurrency.
+ ...concurrencyArgs(await this.gridConcurrency(grid, true)), '--parallel', '1',
+ '--ctx-size', String(ctx), '--endpoint-port', String(port),
'--reasoning-budget', '0',
...(candidate.aliases ?? []).flatMap(alias => ['--advertise-as', alias])], undefined, 30 * 60_000)
// A join that fails at a size is also stepped down from: an allocation that fails at load
@@ -1016,7 +1100,8 @@ export class LocalModels {
}
const named = this.options.machineName?.()?.trim()
const joined = await this.run(['--remote', 'join', grid, '--at', `http://127.0.0.1:${started.port}/v1`, '-m', started.served,
- '--advertise-as', started.alias, '--max-concurrency', '1', ...(named && validArg(named) ? ['--name', named] : [])], undefined, 10 * 60_000)
+ '--advertise-as', started.alias, ...concurrencyArgs(await this.gridConcurrency(grid, false)),
+ ...(named && validArg(named) ? ['--name', named] : [])], undefined, 10 * 60_000)
if (!joined.ok) await failed('The model could not join your grid. Try again.')
operation.stage = 'verifying'; await this.save(grid, operation)
try { await this.verify(grid, started.alias) } catch (error) {
@@ -1056,13 +1141,15 @@ export class LocalModels {
if (operation.action === 'stop') {
if (record) await takeDown(record)
} else if (!record) {
- if (!await this.fileComplete(jev.file, jev.size)) {
+ const quant = await this.jevQuant(jev, modelBudget(await this.readDevice()))
+ if (!quant) throw new ModelError(`This computer does not have the memory to run ${jev.name}. Close some apps, or choose a smaller model.`)
+ if (!await this.fileComplete(quant.file, quant.size)) {
await mkdir(join(this.home, 'models'), { recursive: true })
const disk = await statfs(join(this.home, 'models'))
- if (disk.bavail * disk.bsize < jev.size + GiB) throw new ModelError('Free up disk space, then start again.')
+ if (disk.bavail * disk.bsize < quant.size + GiB) throw new ModelError('Free up disk space, then start again.')
await change('downloading')
let last = 0, progressWrites = Promise.resolve()
- await must(['pull', jev.pull], 'The download stopped. Start again to resume.', chunk => {
+ await must(['pull', `${jev.repo}:${quant.file}`], 'The download stopped. Start again to resume.', chunk => {
const matches = [...chunk.matchAll(/(\d+(?:\.\d+)?)\s*%/g)]
const percent = matches.length ? Number(matches.at(-1)![1]) : NaN
if (Number.isFinite(percent) && percent >= 0 && percent <= 100 && Date.now() - last > 500) {
@@ -1071,23 +1158,24 @@ export class LocalModels {
}
})
await progressWrites
- if (!await this.fileComplete(jev.file, jev.size)) throw new ModelError('The download is incomplete. Start again to resume.')
+ if (!await this.fileComplete(quant.file, quant.size)) throw new ModelError('The download is incomplete. Start again to resume.')
}
if (operation.action === 'start') {
- const binary = await this.jevEngine(change, must)
+ const binary = await this.jevEngine(jev, change, must)
await change('starting')
if (DOWN.has(await this.gridStatus(grid).catch(() => ''))) await must(['--remote', 'start', grid], 'Your grid could not start. Try again.')
- const model: AppModel = { id: jev.id, name: jev.name, app: 'llama.cpp', engine: 'llama.cpp', ref: join(this.home, 'models', jev.file), binary, sizeBytes: jev.size }
+ const model: AppModel = { id: jev.id, name: jev.name, app: 'llama.cpp', engine: 'llama.cpp', ref: join(this.home, 'models', quant.file), binary, sizeBytes: quant.size }
let started: AppEngineRecord
try {
- started = { spec: 1, modelId: jev.id, grid, name: jev.name, ...await ops.start(model, JEV_CONTEXT, join(this.options.stateDir, 'logs')) }
+ started = { spec: 1, modelId: jev.id, grid, name: jev.name, ...await ops.start(model, JEV_CONTEXT, join(this.options.stateDir, 'logs'), JEV_SLOTS) }
} catch (error) {
throw new ModelError(error instanceof AppStartError ? error.message : 'The model could not start. Try again.')
}
await writeAppRecords(this.appRecordsFile, [...(await readAppRecords(this.appRecordsFile)).filter(r => !(r.modelId === jev.id && r.grid === grid)), started])
const named = this.options.machineName?.()?.trim()
const joined = await this.run(['--remote', 'join', grid, '--at', `http://127.0.0.1:${started.port}/v1`, '-m', started.served,
- '--advertise-as', started.alias, '--max-concurrency', '1', ...(named && validArg(named) ? ['--name', named] : [])], undefined, 10 * 60_000)
+ '--advertise-as', started.alias, ...concurrencyArgs(await this.gridConcurrency(grid, false)),
+ ...(named && validArg(named) ? ['--name', named] : [])], undefined, 10 * 60_000)
if (!joined.ok) { await takeDown(started); throw new ModelError('The model could not join your grid. Try again.') }
await change('verifying')
try { await this.verifyDecision(grid, started.alias) } catch (error) {
@@ -1101,23 +1189,24 @@ export class LocalModels {
await this.save(grid, operation)
}
- /** Grid's llama-server, brought up to a build that serves Jev models first when it is older (or missing). */
- private async jevEngine(change: (stage: ModelOperation['stage']) => Promise<void>,
+ /** Grid's llama-server, brought up to a build that runs [jev] first when it is older (or missing). */
+ private async jevEngine(jev: JevModel, change: (stage: ModelOperation['stage']) => Promise<void>,
must: (args: string[], message: string) => Promise<void>): Promise<string> {
+ const needed = Math.max(MIN_JEV_BUILD, jev.minBuild ?? 0)
const override = this.processEnv.LLAMA_SERVER
if (override) {
const build = await llamaBuild(override, this.processEnv)
- if (build !== undefined && build < MIN_JEV_BUILD) throw new ModelError(`LLAMA_SERVER is llama.cpp build ${build}; Jev models need build ${MIN_JEV_BUILD} or newer.`)
+ if (build !== undefined && build < needed) throw new ModelError(`LLAMA_SERVER is llama.cpp build ${build}; ${jev.name} needs build ${needed} or newer.`)
return override
}
const managed = join(this.home, 'bin', 'llama-server')
const build = binaryOnPath(managed, this.processEnv) ? await llamaBuild(managed, this.processEnv) : undefined
- if (build !== undefined && build >= MIN_JEV_BUILD) return managed
+ if (build !== undefined && build >= needed) return managed
await change('updating')
await must(['engine', 'install', 'llama.cpp'], 'The model engine could not be updated. Try again.')
const updated = await llamaBuild(managed, this.processEnv)
- if (updated === undefined || updated < MIN_JEV_BUILD) {
- throw new ModelError(`Grid's model engine is still too old for Jev models (build ${updated ?? 'unknown'}; they need ${MIN_JEV_BUILD} or newer). Update Grid, then try again.`)
+ if (updated === undefined || updated < needed) {
+ throw new ModelError(`Grid's model engine is still too old for ${jev.name} (build ${updated ?? 'unknown'}; it needs ${needed} or newer). Update Grid, then try again.`)
}
return managed
}
diff --git a/desktop/lib/models/model_search_catalog.dart b/desktop/lib/models/model_search_catalog.dart
index 1e839a7eb..795e7d080 100644
--- a/desktop/lib/models/model_search_catalog.dart
+++ b/desktop/lib/models/model_search_catalog.dart
@@ -14,22 +14,41 @@ import 'model_manager_controller.dart';
/// Between a shared model's name and the machine serving it, in its row's title.
const modelMachineSeparator = ' · ';
+/// The two kinds of model the picker lists, each under its own heading: chat models, which a harness
+/// runs on, and decision models (Jev, System One), which answer typed questions and run no harness.
+enum ModelSearchGroup {
+ chat('Chat models'),
+ decision('Decision models');
+
+ const ModelSearchGroup(this.label);
+ final String label;
+}
+
enum ModelSearchSection {
subscriptions('Subscriptions'),
apis('APIs'),
local('Your models'),
/// Models a machine of yours can download: the grid catalog's picks for it. Its heading names the
- /// machine and its memory ([ModelSearchCatalog.catalogHeading]); this label is its search word.
+ /// machine and its memory ([ModelSearchCatalog.catalogHeadingFor]); this label is its search word.
catalog('Get models'),
shared('Shared with you'),
- /// Jev (System One) decision models, from any grid: they answer typed questions and cannot run
- /// a harness, so they are listed last, apart from every model a harness can use.
- jev('Jev models');
+ /// Decision models, listed last, apart from every model a harness can use: a machine of yours
+ /// runs or has them, ones it can download (only those it has the memory to start), and the ones
+ /// a grid shared with you serves.
+ jevLocal('Your models', ModelSearchGroup.decision),
+ jevCatalog('Get models', ModelSearchGroup.decision),
+ jevShared('Shared with you', ModelSearchGroup.decision);
- const ModelSearchSection(this.label);
+ const ModelSearchSection(this.label, [this.group = ModelSearchGroup.chat]);
final String label;
+ final ModelSearchGroup group;
+
+ /// Names the section's heading in the list: its label, which a decision section shares with its
+ /// chat namesake, so that one also names its group.
+ String get key =>
+ group == ModelSearchGroup.chat ? label : '${group.label}: $label';
}
/// Public model metadata shared by the picker and its preview. Filtering this
@@ -74,7 +93,11 @@ class ModelSearchEntry {
: api != null
? ModelSearchSection.apis
: isJev
- ? ModelSearchSection.jev
+ ? isDownload
+ ? ModelSearchSection.jevCatalog
+ : own
+ ? ModelSearchSection.jevLocal
+ : ModelSearchSection.jevShared
: isDownload
? ModelSearchSection.catalog
: own
@@ -122,6 +145,7 @@ class ModelSearchEntry {
api?.host,
api?.name,
apiModel?.name,
+ if (isJev) ...[ModelSearchGroup.decision.label, 'jev'],
...searchAliases,
],
);
@@ -511,15 +535,18 @@ class ModelSearchCatalog extends ChangeNotifier {
: 'Not downloaded';
}
- /// The downloads' heading: the machine they are for and its memory, `Get for this Mac · 64 GB`.
- /// Downloads for more than one machine are headed plainly; each row's preview names its machine.
- String get catalogHeading {
+ /// The chat downloads' heading ([catalogHeadingFor]).
+ String get catalogHeading => catalogHeadingFor(ModelSearchSection.catalog);
+
+ /// A downloads section's heading — chat models' or decision models' — the machine they are for and its
+ /// memory, `Get for this Mac · 64 GB`. Downloads for more than one machine are headed plainly; each
+ /// row's preview names its machine.
+ String catalogHeadingFor(ModelSearchSection section) {
final owners = {
for (final entry in entries.values)
- if (entry.section == ModelSearchSection.catalog)
- entry.controller ?? manager,
+ if (entry.section == section) entry.controller ?? manager,
};
- if (owners.length != 1) return ModelSearchSection.catalog.label;
+ if (owners.length != 1) return section.label;
final owner = owners.single;
final machine = owner.machine;
final memory = owner.memoryBytes;
diff --git a/desktop/lib/shared/theme/app_icons.dart b/desktop/lib/shared/theme/app_icons.dart
index 765ed14b4..599b1e8c1 100644
--- a/desktop/lib/shared/theme/app_icons.dart
+++ b/desktop/lib/shared/theme/app_icons.dart
@@ -150,6 +150,7 @@ abstract final class AppIcons {
static const slidersHorizontal = LucideIcons.slidersHorizontal400;
static const smartphone = LucideIcons.smartphone400;
static const snowflake = LucideIcons.snowflake400;
+ static const signpost = LucideIcons.signpost400;
static const sparkles = LucideIcons.sparkles400;
static const square = LucideIcons.square400;
static const squareCheck = LucideIcons.squareCheck400;
diff --git a/desktop/lib/state/swarm_search.dart b/desktop/lib/state/swarm_search.dart
index 7cc5a0264..bbb89bc42 100644
--- a/desktop/lib/state/swarm_search.dart
+++ b/desktop/lib/state/swarm_search.dart
@@ -244,10 +244,13 @@ class SwarmSearchController extends ChangeNotifier {
].any((word) => word.contains(words) || words.contains(word));
}
- /// A section's heading in the list. The downloads name the machine they are for.
+ /// A section's heading in the list. The downloads, chat models' and decision models', name the
+ /// machine they are for.
String modelSectionLabel(ModelSearchSection section) =>
- section == ModelSearchSection.catalog && models != null
- ? models!.catalogHeading
+ models != null &&
+ (section == ModelSearchSection.catalog ||
+ section == ModelSearchSection.jevCatalog)
+ ? models!.catalogHeadingFor(section)
: section.label;
/// A Jev (System One) model's row: nothing to Use, its pane says how to call it.
@@ -296,7 +299,11 @@ class SwarmSearchController extends ChangeNotifier {
// A subscription says how much of it is left — the one figure worth a glance; Enter on it is
// the preview's to say. Other rows with no weights here say what Enter does.
if (entry.subscription != null) return entry.status;
- if (local == null) return modelRowAction(row) ?? entry.status;
+ if (local == null) {
+ return _jevServing(entry)
+ ? 'Serving'
+ : modelRowAction(row) ?? entry.status;
+ }
final owner = entry.controller ?? catalog.manager;
final operation = owner.operationFor(local);
if (operation?.active == true ||
@@ -309,6 +316,9 @@ class SwarmSearchController extends ChangeNotifier {
if (usingModelId == row.modelId) {
return local.downloaded ? 'Starting' : 'Downloading';
}
+ // A decision model on a grid says it is serving. Enter copies how to call it, which the pane and
+ // its hint say; "Copy" at the end of the row read as the model's state.
+ if (_jevServing(entry)) return 'Serving';
return modelRowAction(row) ??
(local.running
? 'Running'
@@ -317,6 +327,12 @@ class SwarmSearchController extends ChangeNotifier {
: null);
}
+ /// A decision model a grid serves now, to be called — its row says Serving, live.
+ bool _jevServing(ModelSearchEntry entry) =>
+ entry.isJev &&
+ entry.gridModel != null &&
+ entry.gridModel!.unavailable == null;
+
static const inUseWord = '● In use';
/// A model row's title on the desktop list: the two action rows named for what they do, without
@@ -370,7 +386,7 @@ class SwarmSearchController extends ChangeNotifier {
if (entry == null || catalog == null) return false;
if (modelRowInUse(row)) return true;
final local = entry.local;
- if (local == null) return false;
+ if (local == null) return _jevServing(entry);
final owner = entry.controller ?? catalog.manager;
return owner.operationFor(local)?.active == true ||
owner.pendingId == local.id ||
@@ -405,6 +421,14 @@ class SwarmSearchController extends ChangeNotifier {
return '${size.padLeft(6)} ${speed.padLeft(9)}';
}
+ /// A chat model's download, folded under "More models" past the first few. Never a Jev model's: its
+ /// own section is the only place it is offered, and it ranks after every chat download, so folding
+ /// it with them hid it.
+ bool _foldedDownload(SwarmDestination row) {
+ final entry = models?.entries[row.modelId];
+ return entry != null && entry.needsDownload && !entry.isJev;
+ }
+
bool canGetModel(SwarmDestination? row) {
if (!isModelMode) return false;
final entry = models?.entries[row?.modelId];
@@ -688,13 +712,16 @@ class SwarmSearchController extends ChangeNotifier {
_modelChoices?.canRunLocally(modelSelectionEngine) == true &&
models!.entries[row.modelId]!.own;
- /// Another model of yours running on [row]'s machine — the one to stop before [row]'s can run.
+ /// Another chat model of yours running on [row]'s machine — the one to stop before [row]'s can run.
+ /// Never a Jev model: it runs beside chat models, and Use on a chat model stopped one.
LocalModel? otherRunningModel(SwarmDestination? row) {
final entry = models?.entries[row?.modelId];
final local = entry?.local, owner = entry?.controller;
- if (local == null || owner == null) return null;
+ if (local == null || owner == null || local.decision) return null;
return owner.localModels
- .where((model) => model.canStop && model.id != local.id)
+ .where(
+ (model) => model.canStop && model.id != local.id && !model.decision,
+ )
.firstOrNull;
}
@@ -1765,25 +1792,19 @@ class SwarmSearchController extends ChangeNotifier {
)
.toList();
if (isModelMode) {
- _downloadCount = candidates
- .where((row) => models?.entries[row.modelId]?.needsDownload == true)
- .length;
+ _downloadCount = candidates.where(_foldedDownload).length;
}
if (isModelMode && matchQuery.trim().isEmpty && !modelDownloadsVisible) {
// Always surface the top few catalog models (in the daemon's order) so the
// picker opens with them already in view; the rest stay hidden until
// "More models" is pressed.
final topCatalog = candidates
- .where((row) => models?.entries[row.modelId]?.needsDownload == true)
+ .where(_foldedDownload)
.take(shownDownloads)
.map((row) => row.id)
.toSet();
candidates = candidates
- .where(
- (row) =>
- models?.entries[row.modelId]?.needsDownload != true ||
- topCatalog.contains(row.id),
- )
+ .where((row) => !_foldedDownload(row) || topCatalog.contains(row.id))
.toList();
}
if (isModelMode && matchQuery.trim().isEmpty) {
@@ -2187,9 +2208,7 @@ class SwarmSearchController extends ChangeNotifier {
modelDownloadsVisible = !modelDownloadsVisible;
_filter();
if (modelDownloadsVisible) {
- final index = rows.indexWhere(
- (row) => models?.entries[row.modelId]?.needsDownload == true,
- );
+ final index = rows.indexWhere(_foldedDownload);
if (index >= 0) {
cursor = index;
_selectedId = selected!.id;
diff --git a/desktop/lib/widgets/swarm_switcher.dart b/desktop/lib/widgets/swarm_switcher.dart
index bb76e6d86..0ae87c479 100644
--- a/desktop/lib/widgets/swarm_switcher.dart
+++ b/desktop/lib/widgets/swarm_switcher.dart
@@ -547,6 +547,7 @@ class _SwarmSearchResultsState extends State<SwarmSearchResults> {
final _resultsKey = GlobalKey();
double _rowHeight = 56;
double _modelHeadingHeight = 28;
+ double _modelGroupHeadingHeight = 36;
double _twoLineRowHeight = 56;
bool _desktopRows = false;
bool _revealScheduled = false;
@@ -558,11 +559,13 @@ class _SwarmSearchResultsState extends State<SwarmSearchResults> {
(Size, double)? _geometry;
SwarmSearchController get search => widget.search;
- /// The model list as drawn: each section's heading, then its rows. [key] is the section's own
- /// label, which stays put while a heading such as `Get for this Mac · 64 GB` follows its machine.
+ /// The model list as drawn: each section's heading, then its rows — under a heading for its group,
+ /// Chat models or Decision models, whenever a decision model is listed. [key] is the section's own
+ /// ([ModelSearchSection.key]), which stays put while a heading such as `Get for this Mac · 64 GB`
+ /// follows its machine; a group's is [_groupKey] and its label.
List<({int? index, String? heading, String? key})> get _modelRows {
if (!search.isModelMode) return const [];
- final items = <({int? index, String? heading, String? key})>[];
+ final shown = <ModelSearchSection, List<int>>{};
for (final section in ModelSearchSection.values) {
final indices = [
for (final (index, row) in search.rows.indexed)
@@ -571,20 +574,37 @@ class _SwarmSearchResultsState extends State<SwarmSearchResults> {
// An empty section still says it is there, so there is somewhere to add to — except the
// downloads, which a machine with the whole catalog, or none, has nothing under, what is
// shared with you while Grid is not set up (the Set up row under Your models stands for
- // it), and Jev models, which nobody adds from here.
+ // it), and decision models, which nobody adds from here.
if (indices.isEmpty &&
(search.matchQuery.trim().isNotEmpty ||
section == ModelSearchSection.catalog ||
- section == ModelSearchSection.jev ||
+ section.group == ModelSearchGroup.decision ||
(section == ModelSearchSection.shared &&
search.gridSetupOffered))) {
continue;
}
+ shown[section] = indices;
+ }
+ // One kind alone needs no heading for it: the list reads as it did before decision models.
+ final grouped = shown.keys.any(
+ (section) => section.group == ModelSearchGroup.decision,
+ );
+ final items = <({int? index, String? heading, String? key})>[];
+ ModelSearchGroup? group;
+ for (final MapEntry(key: section, value: indices) in shown.entries) {
if (items.isNotEmpty) items.add((index: null, heading: null, key: null));
+ if (grouped && section.group != group) {
+ group = section.group;
+ items.add((
+ index: null,
+ heading: group.label,
+ key: '$_groupKey${group.label}',
+ ));
+ }
items.add((
index: null,
heading: search.modelSectionLabel(section),
- key: section.label,
+ key: section.key,
));
items.addAll(
indices.map((index) => (index: index, heading: null, key: null)),
@@ -593,10 +613,21 @@ class _SwarmSearchResultsState extends State<SwarmSearchResults> {
return items;
}
+ static const _groupKey = 'group:';
+
+ bool _isGroupHeading(({int? index, String? heading, String? key}) item) =>
+ item.key?.startsWith(_groupKey) == true;
+
double _modelItemHeight(({int? index, String? heading, String? key}) item) {
if (!_desktopRows) return _rowHeight;
final index = item.index;
- if (index == null) return item.heading == null ? 8 : _modelHeadingHeight;
+ if (index == null) {
+ return item.heading == null
+ ? 8
+ : _isGroupHeading(item)
+ ? _modelGroupHeadingHeight
+ : _modelHeadingHeight;
+ }
// A shared model's row is two lines: its name, and the machine serving it under it.
return search.modelRowNameEnd(search.rows[index]) != null
? _twoLineRowHeight
@@ -876,6 +907,7 @@ class _SwarmSearchResultsState extends State<SwarmSearchResults> {
}
_desktopRows = desktop;
_modelHeadingHeight = scale.scale(12) * 1.45 + 12;
+ _modelGroupHeadingHeight = scale.scale(16) * 1.45 + 26;
final height = widget.fitRows
? _fittedHeight(
constraints,
@@ -1311,6 +1343,16 @@ class _SwarmSearchResultsState extends State<SwarmSearchResults> {
if (item.heading == null) {
return const SizedBox();
}
+ if (desktop && _isGroupHeading(item)) {
+ return _ModelGroupHeading(
+ key: ValueKey('model-section:${item.key}'),
+ label: item.heading!,
+ decision:
+ item.key ==
+ '$_groupKey${ModelSearchGroup.decision.label}',
+ first: index == 0,
+ );
+ }
return Semantics(
header: true,
child: Padding(
@@ -2442,3 +2484,58 @@ class SwarmSearchCount extends StatelessWidget {
);
}
}
+
+/// A kind of model at the head of its sections — Chat models, Decision models — set apart from the
+/// section headings under it so the list reads as two parts: its mark, a larger and heavier label, and
+/// a rule above every kind after the first.
+class _ModelGroupHeading extends StatelessWidget {
+ const _ModelGroupHeading({
+ super.key,
+ required this.label,
+ required this.decision,
+ required this.first,
+ });
+
+ final String label;
+ final bool decision;
+ final bool first;
+
+ @override
+ Widget build(BuildContext context) {
+ grid.AppTheme.watch(context);
+ return Semantics(
+ header: true,
+ child: Container(
+ margin: const EdgeInsets.symmetric(horizontal: 14),
+ padding: EdgeInsets.only(top: first ? 6 : 14),
+ decoration: first
+ ? null
+ : BoxDecoration(
+ border: Border(top: BorderSide(color: DesktopChrome.rim)),
+ ),
+ alignment: Alignment.centerLeft,
+ child: Row(
+ children: [
+ Icon(
+ decision ? AppIcons.signpost : AppIcons.messagesSquare,
+ size: 18,
+ color: DesktopChrome.foreground,
+ ),
+ const SizedBox(width: 8),
+ Flexible(
+ child: Text(
+ label,
+ maxLines: 1,
+ overflow: TextOverflow.ellipsis,
+ style: DesktopChrome.text(
+ size: 16,
+ color: DesktopChrome.foreground,
+ ).copyWith(fontWeight: FontWeight.w600),
+ ),
+ ),
+ ],
+ ),
+ ),
+ );
+ }
+}
diff --git a/desktop/test/local_model_picker_test.dart b/desktop/test/local_model_picker_test.dart
index 19dcd2ef7..f8273a393 100644
--- a/desktop/test/local_model_picker_test.dart
+++ b/desktop/test/local_model_picker_test.dart
@@ -710,6 +710,217 @@ void main() {
},
);
+ testWidgets(
+ 'Jev models to get are in their own section, never folded under the chat downloads',
+ (tester) async {
+ final app = await _localFixture();
+ try {
+ app.localInventory = {
+ ..._inventory(),
+ 'models': [
+ ...(_inventory()['models'] as List),
+ // More chat downloads than the list shows unfolded, ranked before the Jev models as the
+ // daemon lists them.
+ for (var i = 0; i < 8; i++)
+ {
+ 'id': 'unsloth/Chat-$i-GGUF',
+ 'name': 'Chat-$i',
+ 'state': 'available',
+ 'quant': 'Q4_K_M',
+ 'sizeBytes': 4 * _gib,
+ 'canStart': true,
+ },
+ for (final (id, name) in [
+ ('jev:ggml-org/Kev-0.8B-GGUF', 'kev-0.8b'),
+ ('jev:ggml-org/lev-GGUF', 'lev'),
+ ('jev:ggml-org/Clef-GGUF', 'clef'),
+ ])
+ {
+ 'id': id,
+ 'name': name,
+ 'kind': 'decision',
+ 'state': 'available',
+ 'quant': 'Q8_0',
+ 'sizeBytes': 2 * _gib,
+ 'canStart': true,
+ },
+ ],
+ };
+ await app.modelManager.refresh(force: true);
+ final picker = await _open(tester, app);
+ for (final name in ['kev-0.8b', 'lev', 'clef']) {
+ final row = _row(picker, name);
+ expect(picker.modelSection(row), ModelSearchSection.jevCatalog);
+ expect(picker.modelRowAction(row), 'Get');
+ }
+ // Under Decision models, Get for this Mac is headed like the chat models' downloads.
+ expect(
+ picker.modelSectionLabel(ModelSearchSection.jevCatalog),
+ 'Get for ${thisComputerName()} · 64 GB',
+ );
+ // Each kind has its heading: Chat models over the chat sections, Decision models over its own —
+ // here only Get for this Mac, since none is on this computer yet.
+ expect(
+ find.byKey(const ValueKey('model-section:group:Chat models')),
+ findsOneWidget,
+ );
+ final decision = find.byKey(
+ const ValueKey('model-section:group:Decision models'),
+ );
+ await tester.scrollUntilVisible(
+ decision,
+ 200,
+ scrollable: find
+ .descendant(
+ of: find.byType(SwarmSearchResults),
+ matching: find.byType(Scrollable),
+ )
+ .first,
+ );
+ expect(decision, findsOneWidget);
+ expect(
+ find.byKey(
+ const ValueKey('model-section:Decision models: Get models'),
+ ),
+ findsOneWidget,
+ );
+ expect(
+ find.byKey(
+ const ValueKey('model-section:Decision models: Your models'),
+ ),
+ findsNothing,
+ );
+ // The fold counts chat downloads alone: ten (the fixture's two and eight more), five shown.
+ expect(
+ picker.modelRowTitle(
+ picker.rows.singleWhere(picker.isModelDownloadsRow),
+ ),
+ 'More models (5)',
+ );
+ } finally {
+ await tester.pumpWidget(const SizedBox());
+ app.dispose();
+ }
+ },
+ );
+
+ testWidgets(
+ 'a running Jev model is never stopped to start a chat model, nor stops one',
+ (tester) async {
+ final app = await _localFixture(running: false);
+ try {
+ Map<String, dynamic> jev(String id, String name, String state) => {
+ 'id': id,
+ 'name': name,
+ 'kind': 'decision',
+ 'state': state,
+ 'quant': 'Q8_0',
+ 'sizeBytes': _gib,
+ 'canStart': state != 'running',
+ 'canStop': state == 'running',
+ };
+ app.localInventory = {
+ ..._inventory(running: false),
+ 'models': [
+ ...(_inventory(running: false)['models'] as List),
+ jev('jev:ggml-org/Laya-GGUF', 'laya-english', 'running'),
+ jev('jev:ggml-org/Kev-4B-GGUF', 'kev-4b', 'downloaded'),
+ ],
+ };
+ await app.modelManager.refresh(force: true);
+ final picker = await _open(tester, app);
+ // No chat model runs: Use on one has nothing to stop, the Jev model running included.
+ expect(picker.otherRunningModel(_row(picker, 'gemma-4-12B')), isNull);
+ // Starting another Jev model stops nothing either.
+ expect(picker.otherRunningModel(_row(picker, 'kev-4b')), isNull);
+
+ // A chat model running is still the one to stop.
+ final withChat = _inventory();
+ app.localInventory = {
+ ...withChat,
+ 'models': [
+ ...(withChat['models'] as List),
+ jev('jev:ggml-org/Laya-GGUF', 'laya-english', 'running'),
+ ],
+ };
+ await app.modelManager.refresh(force: true);
+ await tester.pumpAndSettle();
+ expect(
+ picker.otherRunningModel(_row(picker, 'gemma-4-12B'))?.name,
+ 'Qwen3.6-35B-A3B',
+ );
+ } finally {
+ await tester.pumpWidget(const SizedBox());
+ app.dispose();
+ }
+ },
+ );
+
+ testWidgets(
+ 'the arrows walk the Jev models in the order they are drawn, a running one first',
+ (tester) async {
+ final app = await _localFixture();
+ try {
+ Map<String, dynamic> jev(String id, String name, String state) => {
+ 'id': id,
+ 'name': name,
+ 'kind': 'decision',
+ 'state': state,
+ 'quant': 'Q8_0',
+ 'sizeBytes': _gib,
+ 'canStart': state != 'running',
+ 'canStop': state == 'running',
+ };
+ app.localInventory = {
+ ..._inventory(),
+ 'models': [
+ ...(_inventory()['models'] as List),
+ jev('jev:ggml-org/Laya-GGUF', 'laya-english', 'downloaded'),
+ jev('jev:ggml-org/Kev-0.8B-GGUF', 'kev-0.8b', 'running'),
+ jev('jev:ggml-org/Kev-4B-GGUF', 'kev-4b', 'available'),
+ jev('jev:ggml-org/Clef-GGUF', 'clef', 'available'),
+ ],
+ };
+ app.inventory = const GridModels(
+ gridName: 'home',
+ models: [
+ GridModel(id: 'Qwen3.6-35B-A3B', node: 'This Mac'),
+ GridModel(id: 'kev-0.8b', node: 'This Mac', decision: true),
+ ],
+ );
+ await app.modelManager.refresh(force: true);
+ final picker = await _open(tester, app);
+ final drawn = picker.rows
+ .where(
+ (row) =>
+ picker.modelSection(row).group == ModelSearchGroup.decision,
+ )
+ .map((row) => row.title)
+ .toList();
+ expect(drawn, ['kev-0.8b', 'laya-english', 'kev-4b', 'clef']);
+ // The one running, which the grid lists, says it is serving — live; Enter copies how to call it.
+ final serving = _row(picker, 'kev-0.8b');
+ expect(picker.modelRowStatus(serving), 'Serving');
+ expect(picker.modelRowLive(serving), isTrue);
+ expect(picker.modelRowAction(serving), 'Copy');
+ expect(picker.modelRowStatus(_row(picker, 'laya-english')), 'Start');
+ picker.move(
+ picker.rows.indexOf(_row(picker, 'laya-english')) - picker.cursor,
+ );
+ await tester.pumpAndSettle();
+ await key(tester, LogicalKeyboardKey.arrowUp);
+ await tester.pumpAndSettle();
+ expect(picker.selected!.title, 'kev-0.8b');
+ await key(tester, LogicalKeyboardKey.arrowDown);
+ await tester.pumpAndSettle();
+ expect(picker.selected!.title, 'laya-english');
+ } finally {
+ await tester.pumpWidget(const SizedBox());
+ app.dispose();
+ }
+ },
+ );
+
testWidgets(
'Get on a Jev model starts it, never moves the harness, and then it is one Jev row to stop or call',
(tester) async {
@@ -734,7 +945,7 @@ void main() {
await app.modelManager.refresh(force: true);
final picker = await _open(tester, app);
var row = _row(picker, 'laya-english');
- expect(picker.modelSection(row), ModelSearchSection.jev);
+ expect(picker.modelSection(row), ModelSearchSection.jevCatalog);
expect(picker.canSelectModel(row), isFalse);
expect(picker.canGetModelForUse(row), isFalse);
expect(picker.modelRowAction(row), 'Get');
@@ -791,7 +1002,7 @@ void main() {
.toList();
expect(layas, hasLength(1));
row = layas.single;
- expect(picker.modelSection(row), ModelSearchSection.jev);
+ expect(picker.modelSection(row), ModelSearchSection.jevLocal);
expect(picker.modelRowAction(row), 'Copy');
picker.move(picker.rows.indexOf(row) - picker.cursor);
await tester.pumpAndSettle();
@@ -804,6 +1015,42 @@ void main() {
findsOneWidget,
);
expect(_inPreview(jevRequest('home', 'laya-english')), findsOneWidget);
+
+ // Stopped, it is the same row, still selected: Enter must not pass to whatever row the list
+ // then has under the cursor.
+ final layaId = row.id;
+ app.localInventory = {
+ ...running,
+ 'models': [
+ for (final model in running['models'] as List)
+ if ((model as Map<String, dynamic>)['id'] == laya)
+ {
+ ...model,
+ 'state': 'downloaded',
+ 'canStart': true,
+ 'canStop': false,
+ }
+ else
+ model,
+ {
+ 'id': 'jev:ggml-org/Clef-GGUF',
+ 'name': 'clef',
+ 'kind': 'decision',
+ 'state': 'available',
+ 'sizeBytes': 28732215360,
+ 'quant': 'Q8_0',
+ 'canStart': true,
+ },
+ ],
+ };
+ app.inventory = const GridModels(
+ gridName: 'home',
+ models: [GridModel(id: 'Qwen3.6-35B-A3B', node: 'This Mac')],
+ );
+ await app.modelManager.refresh(force: true);
+ await tester.pumpAndSettle();
+ expect(picker.selected!.title, 'laya-english');
+ expect(picker.selected!.id, layaId);
expect(tester.takeException(), isNull);
} finally {
await tester.pumpWidget(const SizedBox());
diff --git a/desktop/test/resource_picker_test.dart b/desktop/test/resource_picker_test.dart
index abf90d9b8..d21600222 100644
--- a/desktop/test/resource_picker_test.dart
+++ b/desktop/test/resource_picker_test.dart
@@ -14,7 +14,7 @@ import 'package:harness/auth/cli_link.dart';
import 'package:harness/core/models.dart';
import 'package:harness/models/api_connections_controller.dart';
import 'package:harness/models/model_search_catalog.dart'
- show ModelSearchSection, jevRequest;
+ show ModelSearchGroup, ModelSearchSection, jevRequest;
import 'package:harness/widgets/api_picker_form.dart';
import 'package:harness/widgets/desktop_chrome.dart';
import 'package:harness/shared/theme/app_theme.dart' as grid;
@@ -1068,19 +1068,20 @@ void main() {
await configured.mount(tester, app, map);
await key(tester, LogicalKeyboardKey.keyI, cmd: true);
final rows = search(tester).rows.where((row) => row.isModel);
- String section(String title) => search(tester)
- .modelSection(rows.firstWhere((row) => row.title.startsWith(title)))
- .label;
- // From the own grid and a shared one alike, a Jev model is under its own heading, last.
- expect(section('laya-english'), 'Jev models');
- expect(section('nimble'), 'Jev models');
- expect(section('qwen3.8-27b'), isNot('Jev models'));
+ ModelSearchSection section(String title) => search(
+ tester,
+ ).modelSection(rows.firstWhere((row) => row.title.startsWith(title)));
+ // A Jev model is a decision model, listed apart from the chat models: Your models when the
+ // own grid serves it, Shared with you when a shared grid does.
+ expect(section('laya-english'), ModelSearchSection.jevLocal);
+ expect(section('nimble'), ModelSearchSection.jevShared);
+ expect(section('qwen3.8-27b').group, ModelSearchGroup.chat);
expect(
rows.where((row) => row.title.startsWith('laya-english')),
hasLength(1),
reason: rows.map((row) => row.title).join('\n'),
);
- expect(ModelSearchSection.values.last, ModelSearchSection.jev);
+ expect(ModelSearchSection.values.last.group, ModelSearchGroup.decision);
// Clicking a Jev row shows how to call it, and neither copies nor moves the harness.
expect(search(tester).isJevRow(search(tester).selected), isFalse);
@@ -1089,12 +1090,14 @@ void main() {
layaRow,
100,
scrollable: find
- .ancestor(
- of: find.text('Your models', findRichText: true),
+ .descendant(
+ of: find.byType(SwarmSearchResults),
matching: find.byType(Scrollable),
)
.first,
);
+ await tester.ensureVisible(layaRow);
+ await tester.pumpAndSettle();
await tester.tap(layaRow);
await tester.pumpAndSettle();
expect(search(tester).selected!.title, 'laya-english');
@@ -1109,6 +1112,9 @@ void main() {
expect(search(tester).isJevRow(laya), isTrue);
expect(search(tester).canSelectModel(laya), isFalse);
expect(search(tester).modelRowAction(laya), 'Copy');
+ // Served by another machine of yours: its row says Serving, live, as one served here does.
+ expect(search(tester).modelRowStatus(laya), 'Serving');
+ expect(search(tester).modelRowLive(laya), isTrue);
expect(search(tester).actionLabel(laya), 'Copy request');
expect(find.text('Call it from a terminal:'), findsOneWidget);
expect(find.text(jevRequest('home', 'laya-english')), findsOneWidget);