Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(models): fit the catalog to the VRAM our own model holds; names without their quant; forget Jev's engine records

bitcoinbuilderbee01667f8a0
diff --git a/cli/src/lib/gpuProcesses.spec.ts b/cli/src/lib/gpuProcesses.spec.ts
new file mode 100644
index 000000000..d0137c487
--- /dev/null
+++ b/cli/src/lib/gpuProcesses.spec.ts
@@ -0,0 +1,36 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const execFile = vi.hoisted(() => vi.fn())
+vi.mock('node:child_process', () => ({ execFile }))
+
+import { nvidiaComputeApps, parseComputeApps } from './gpuProcesses.js'
+
+const MiB = 1024 ** 2
+
+describe('the processes holding NVIDIA VRAM', () => {
+ beforeEach(() => execFile.mockReset())
+
+ it('reads nvidia-smi\'s compute apps, leaving out a line that does not read', () => {
+ expect(parseComputeApps([
+ '30956, /home/me/.grid/bin/llama-server, 21769',
+ '2014, /opt/My App, Inc/app, 300',
+ '99, /usr/bin/ollama, [N/A]',
+ '0, /bin/zero-pid, 10',
+ '12, , 10',
+ 'garbage',
+ '',
+ ].join('\n'))).toEqual([
+ { pid: 30956, name: '/home/me/.grid/bin/llama-server', bytes: 21769 * MiB },
+ { pid: 2014, name: '/opt/My App,Inc/app', bytes: 300 * MiB },
+ ])
+ })
+
+ it('asks nvidia-smi for them, and has none where it is missing or fails', async () => {
+ execFile.mockImplementationOnce((_file, _args, _options, done) => done(null, '30956, /home/me/.grid/bin/llama-server, 21769\n'))
+ expect(await nvidiaComputeApps()).toEqual([{ pid: 30956, name: '/home/me/.grid/bin/llama-server', bytes: 21769 * MiB }])
+ expect(execFile).toHaveBeenCalledWith('nvidia-smi', ['--query-compute-apps=pid,process_name,used_memory', '--format=csv,noheader,nounits'],
+ { timeout: 5_000 }, expect.any(Function))
+ execFile.mockImplementationOnce((_file, _args, _options, done) => done(Object.assign(new Error('spawn nvidia-smi ENOENT'), { code: 'ENOENT' }), ''))
+ expect(await nvidiaComputeApps()).toEqual([])
+ })
+})
diff --git a/cli/src/lib/gpuProcesses.ts b/cli/src/lib/gpuProcesses.ts
new file mode 100644
index 000000000..d1d0fde81
--- /dev/null
+++ b/cli/src/lib/gpuProcesses.ts
@@ -0,0 +1,23 @@
+import { execFile } from 'node:child_process'
+
+/** A process holding memory on an NVIDIA card: its pid, its executable and the bytes it holds. */
+export interface GpuProcess { pid: number; name: string; bytes: number }
+
+/** `nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader,nounits`, read:
+ * `30956, /home/me/.grid/bin/llama-server, 21769` (MiB). A line that does not read — `[N/A]` where the
+ * driver will not say — is left out. */
+export function parseComputeApps(csv: string): GpuProcess[] {
+ return csv.split('\n').flatMap(line => {
+ const fields = line.split(',').map(field => field.trim())
+ if (fields.length < 3) return []
+ const pid = Number(fields[0]), mib = Number(fields[fields.length - 1])
+ const name = fields.slice(1, -1).join(',')
+ return Number.isInteger(pid) && pid > 0 && Number.isFinite(mib) && mib > 0 && name ? [{ pid, name, bytes: mib * 1024 ** 2 }] : []
+ })
+}
+
+/** The processes computing on this machine's NVIDIA cards: none where there is no `nvidia-smi`, or it fails. */
+export function nvidiaComputeApps(): Promise<GpuProcess[]> {
+ return new Promise(resolve => execFile('nvidia-smi', ['--query-compute-apps=pid,process_name,used_memory', '--format=csv,noheader,nounits'],
+ { timeout: 5_000 }, (error, stdout) => resolve(error ? [] : parseComputeApps(String(stdout)))))
+}
diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.ts
index 87aef2562..6546de008 100644
--- a/cli/src/lib/localModels.spec.ts
+++ b/cli/src/lib/localModels.spec.ts
@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os'
import { createHash } from 'node:crypto'
import { rmSync, writeFileSync } from 'node:fs'
import { spawnSync } from 'node:child_process'
+import type { GpuProcess } from './gpuProcesses.js'
import { baseModel, compatibleModels, contextLadder, fitOf, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'
import { GridFleetRpc, type GridFleetResult } from './gridFleetRpc.js'
import { AppStartError, type AppEngineOps, type AppEngineRecord, type AppModel } from './appModels.js'
@@ -601,6 +602,8 @@ describe('local model discovery and lifecycle', () => {
.toEqual({ fitBytes: 32 * 1024 ** 3, fitMemory: 'unified' })
expect(fitOf({ backend: 'cpu', usable_bytes: 20 * 1024 ** 3, memory: { total_gb: 32 } })).toEqual({ fitBytes: 16 * 1024 ** 3, fitMemory: 'ram' })
expect(fitOf({})).toEqual({})
+ // A card that names no memory adds none, and none at all is no GPU figure.
+ expect(fitOf({ backend: 'cuda', usable_bytes: 7 * 1024 ** 3, gpus: [{ backend: 'cuda' }] })).toEqual({ fitBytes: 7 * 1024 ** 3, fitMemory: 'vram' })
// Two 24 GB cards in a 64 GB PC.
expect(modelBudget({ backend: 'cuda', usable_bytes: 46 * 1024 ** 3, memory: { total_gb: 64 } })).toBe(46 * 1024 ** 3)
})
@@ -939,6 +942,41 @@ describe('local model discovery and lifecycle', () => {
backend: 'metal', mem_bandwidth_gbps: 400, compute_gflops: 15600 })
})
+ it("counts the VRAM this picker's own engine holds as the next model's, on an NVIDIA card", async () => {
+ // A 24 GB card running our 27B: 2.7 GB free beside it. Starting another model stops ours first, so
+ // the catalog is fitted to what ours holds as well, never to what is left beside it.
+ const MiB = 1024 ** 2, original = run.getMockImplementation()!
+ const card = { device_class: 'nvidia', backend: 'cuda', usable_bytes: 2746 * MiB, memory: { total_gb: 16 }, gpus: [{ backend: 'cuda', memory_total_mb: 24564 }] }
+ const grid = { pid: 30956, name: join(home, 'bin', 'llama-server'), bytes: 21000 * MiB }
+ const browser = { pid: 2014, name: '/usr/lib/chromium/chromium', bytes: 300 * MiB }
+ // An app's engine this picker started, known by its record: its binary, or its pid.
+ await mkdir(stateDir, { recursive: true })
+ await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([{ spec: 1, modelId: 'app:llama.cpp:Other.gguf', grid: 'home',
+ name: 'Other', engine: 'llama.cpp', served: 'other', alias: 'other', port: 9000, binary: '/opt/llama/llama-server', pid: 4242 }]))
+ const fitted = async (device: Record<string, unknown>, processes?: GpuProcess[]): Promise<number | undefined> => {
+ run.mockImplementation(async (args, output) => args[0] === 'device-info' ? ok(device) : original(args, output))
+ request.mockClear()
+ const models = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory,
+ ...(processes ? { gpuProcesses: async () => processes } : {}) })
+ const snapshot = await models.list('home')
+ const sent = JSON.parse(String(request.mock.calls.find(([url]) => String(url).includes('/catalog'))![1]?.body)).device.usable_bytes
+ expect(snapshot.fitBytes).toBe(sent)
+ return sent
+ }
+ expect(await fitted(card, [grid, browser])).toBe((2746 + 21000) * MiB)
+ expect(await fitted(card, [{ pid: 777, name: '/opt/llama/llama-server', bytes: 4000 * MiB }, { pid: 4242, name: 'ollama', bytes: 1000 * MiB }]))
+ .toBe((2746 + 5000) * MiB)
+ // Never more than the card has, and with no card's size to hold it to, all of it.
+ expect(await fitted(card, [{ ...grid, bytes: 30000 * MiB }])).toBe(24564 * MiB)
+ expect(await fitted({ ...card, gpus: [] }, [grid])).toBe((2746 + 21000) * MiB)
+ // Nothing of ours on it, no way to tell, no figure to add to, or no NVIDIA card: as grid says.
+ expect(await fitted(card, [browser])).toBe(2746 * MiB)
+ expect(await fitted(card)).toBe(2746 * MiB)
+ expect(await fitted({ ...card, usable_bytes: undefined }, [grid])).toBeUndefined()
+ expect(await fitted({ device_class: 'apple-silicon', backend: 'metal', usable_bytes: 54 * 1024 ** 3, memory: { total_gb: 64 } }, [grid]))
+ .toBe(32 * 1024 ** 3)
+ })
+
it('leaves out a measurement the machine did not report, so the catalog falls back to its own', async () => {
await service.list('home')
const sent = request.mock.calls.find(([url]) => String(url).includes('/catalog'))!
@@ -1669,6 +1707,27 @@ describe('models other apps downloaded, started in their own app', () => {
.toMatchObject({ state: 'running', canStop: true })
})
+ it('forgets the engines the Jev decision models left, which blocked every chat model\'s start', async () => {
+ // As on a computer that ran them before #1141: laya-english's engine long gone, kev's still up.
+ const jev = (modelId: string, name: string, port: number): AppEngineRecord => ({ spec: 1, modelId, grid: 'home', name,
+ engine: 'llama.cpp', served: name, alias: name, port, binary: '/opt/llama-server', pid: 4300 + port % 100 })
+ const laya = jev('jev:ggml-org/Laya-GGUF', 'laya-english', 52954), kev = jev('jev:ggml-org/Kev-0.8B-GGUF', 'kev-0.8b', 60437)
+ up.add(kev.modelId); joinedAliases.add(kev.alias)
+ await mkdir(stateDir, { recursive: true })
+ await writeFile(join(stateDir, 'app-engines.json'), JSON.stringify([laya, kev]))
+ service = appService()
+ const snapshot = await service.list('home', true)
+ expect(snapshot.models.map(m => m.name)).not.toEqual(expect.arrayContaining(['laya-english']))
+ expect(snapshot.models.some(m => m.id.startsWith('jev:'))).toBe(false)
+ expect(JSON.parse(await readFile(join(stateDir, 'app-engines.json'), 'utf8'))).toEqual([])
+ await vi.waitFor(() => expect(ops.stop).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ modelId: kev.modelId })))
+ await vi.waitFor(() => expect(joinedAliases.has(kev.alias)).toBe(false))
+ // A chat model starts: nothing left to "stop first".
+ const started = await service.act('home', ollama.id, 'start'); await service.settled()
+ expect(started.operation).not.toMatchObject({ phase: 'failed' })
+ expect(ops.start).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ id: ollama.id }), expect.anything(), expect.anything())
+ })
+
it.each(['join refused', 'credentials unavailable', 'credentials threw'])('takes an app engine down after %s, retaining another grid\'s record', async scenario => {
service = appService()
const elsewhere: AppEngineRecord = { spec: 1, modelId: studio.id, grid: 'elsewhere', name: studio.name,
diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.ts
index 52d506961..35dc48cde 100644
--- a/cli/src/lib/localModels.ts
+++ b/cli/src/lib/localModels.ts
@@ -12,6 +12,7 @@ import type { LocalRecord, PictureState } from './gridPicture.js'
import { displayModelName } from './gridReader.js'
import { readEnvExports } from './gridWake.js'
import { MIN_CODING_CONTEXT } from './codingContext.js'
+import type { GpuProcess } from './gpuProcesses.js'
import { APP_LABEL, AppStartError, GRID_LABEL, appContext, readAppRecords, writeAppRecords, type AppEngine, type AppEngineOps, type AppEngineRecord, type AppModel } from './appModels.js'
// Its own module, so the API launches the core keeps need not load this one (codingContext.ts).
@@ -146,6 +147,9 @@ interface Options {
appEngines?: AppEngineOps
/** How long a scan is waited for ([APP_SCAN_MS]); a test shortens it. */
appScanMs?: number
+ /** The processes holding NVIDIA VRAM here ([nvidiaComputeApps]), so the catalog counts what this
+ * picker's own engine holds as the next model's ([LocalModels.withOwnVram]). Absent: not counted. */
+ gpuProcesses?: () => Promise<GpuProcess[]>
}
/** One concrete, machine-fitted version per model. Non-chat and unprobed offline
@@ -230,6 +234,12 @@ export function fitOf(device: Record<string, any>): Pick<LocalModelsSnapshot, 'f
return { ...(fit === undefined ? {} : { fitBytes: fit }), ...(fitMemory ? { fitMemory } : {}), ...(gpuMemoryBytes > 0 ? { gpuMemoryBytes } : {}) }
}
+/** An engine record of a Jev decision model's (`jev:` ids — ggml-org's Kev and Laya, Ollama's tev1),
+ * from before #1141 removed them: [LocalModels.records] forgets it. */
+export function isJevRecord(record: Pick<AppEngineRecord, 'modelId'>): boolean {
+ return record.modelId.startsWith('jev:')
+}
+
/** Catalog models that are not for running an agent on: safety classifiers such as gpt-oss-safeguard
* or Llama Guard, which answer "safe"/"unsafe" rather than code. */
const NOT_FOR_AGENTS = /(?:^|[-_/])(?:safeguard|guard|shieldgemma)(?:[-_.]|$)/i
@@ -389,7 +399,7 @@ export class LocalModels {
if (!force && this.catalogAt && Date.now() - this.catalogAt < 5 * 60_000) return
return this.catalogPending ??= (async () => {
try {
- this.device = obj(await this.json(['device-info', '--json']))
+ this.device = await this.withOwnVram(obj(await this.json(['device-info', '--json'])))
const catalog = obj(await this.catalog(this.device))
this.candidates = rankForCoding(compatibleModels(catalog).filter(candidate => !NOT_FOR_AGENTS.test(candidate.id)))
// Prefer an already downloaded fitting quant rather than downloading
@@ -413,6 +423,26 @@ export class LocalModels {
})().finally(() => { this.catalogPending = undefined })
}
+ /** An NVIDIA card's `usable_bytes` is the VRAM free on it now: less what the model this picker runs
+ * holds. One local model runs at a time and a start stops the running one first, so that VRAM is the
+ * next model's too, and it is counted back in. Left out, the catalog was fitted to what was left
+ * beside our own model (2.9 GB of a 24 GB card running a 27B): it offered nothing to get, and the
+ * running model fell out of it, so its row came back under its file name (`Qwen3.8-27B-UD-Q4_K_M`)
+ * and the Use that had started it lost the row and never moved the harness. Grid's engine is known by
+ * its binary; an app's engine this picker started by its record. Everyone else's VRAM (a desktop, a
+ * browser, a llama-server of the person's own) stays taken. */
+ private async withOwnVram(device: Record<string, any>): Promise<Record<string, any>> {
+ const usable = num(device.usable_bytes)
+ if (device.backend !== 'cuda' || usable === undefined || !this.options.gpuProcesses) return device
+ const records = await this.records()
+ const engines = new Set([join(this.home, 'bin', 'llama-server'), ...records.map(record => record.binary)])
+ const pids = new Set(records.map(record => record.pid))
+ const ours = (await this.options.gpuProcesses()).filter(p => engines.has(p.name) || pids.has(p.pid)).reduce((sum, p) => sum + p.bytes, 0)
+ if (!ours) return device
+ const cards = fitOf(device).gpuMemoryBytes
+ return { ...device, usable_bytes: cards ? Math.min(usable + ours, cards) : usable + ours }
+ }
+
private async downloaded(candidate: Candidate): Promise<boolean> {
if (candidate.appPath) return stat(candidate.appPath).then(s => s.isFile() && s.size === candidate.size, () => false)
const sizes = await Promise.all(candidate.files.map(file => stat(join(this.home, 'models', file)).then(s => s.isFile() ? s.size : 0).catch(() => 0)))
@@ -644,6 +674,30 @@ export class LocalModels {
private get appRecordsFile(): string { return join(this.options.stateDir, 'app-engines.json') }
+ /** The engines this daemon started — less any left by the Jev decision models Harness no longer has
+ * (#1141). Their records name models no list knows, and as engine records each one blocked every
+ * chat model's start ("Stop laya-english first to start another local model."), its engine long
+ * gone or not, and listed itself as a model to use that cannot chat. Each is forgotten the first
+ * time it is read, its engine stopped and taken off its grid first should it still run. */
+ private async records(): Promise<AppEngineRecord[]> {
+ const all = await readAppRecords(this.appRecordsFile)
+ const left = all.filter(isJevRecord)
+ if (!left.length) return all
+ const kept = all.filter(record => !isJevRecord(record))
+ await writeAppRecords(this.appRecordsFile, kept)
+ for (const record of left) void this.retire(record)
+ return kept
+ }
+
+ /** A Jev engine still running: stopped, and off its grid (best effort — its record is already gone). */
+ private async retire(record: AppEngineRecord): Promise<void> {
+ try {
+ if (!await this.options.appEngines?.alive(record)) return
+ await this.options.appEngines?.stop(record)
+ await this.run(['--remote', 'leave', record.grid, '--engine', record.alias], undefined, 60_000)
+ } catch { /* forgotten either way */ }
+ }
+
async list(grid: string | null, force = false): Promise<LocalModelsSnapshot> {
if (!grid) return { models: [], notice: 'Sign in to find models for this computer.', observedAt: new Date().toISOString(), busy: false }
if (!force && this.cached?.grid === grid && Date.now() - this.cached.at < 2500) return this.cached.value
@@ -670,7 +724,7 @@ export class LocalModels {
} catch { inventoryError = 'Running models could not be checked. Try again.' }
const operation = this.active?.grid === grid ? this.active.operation : this.receipt?.grid === grid ? this.receipt.operation : undefined
const apps = await this.apps(force)
- const records = (await readAppRecords(this.appRecordsFile)).filter(record => record.grid === grid)
+ const records = (await this.records()).filter(record => record.grid === grid)
// An engine this daemon started is read from its record, whatever a later scan makes of the model: a
// scan that missed the person's llama-server (busy serving, it answered `--version` late) once listed
// a running engine as Grid's, and Stop then had nothing to stop.
@@ -811,7 +865,7 @@ export class LocalModels {
const app = apps.find(a => a.id === operation.modelId)
// An engine this daemon started is stopped the way it was started, whatever its id: a record left by a kind of
// model the picker no longer offers keeps a row (and holds the next start), so its Stop must take it down.
- const started = (await readAppRecords(this.appRecordsFile)).some(r => r.modelId === operation.modelId && r.grid === grid)
+ const started = (await this.records()).some(r => r.modelId === operation.modelId && r.grid === grid)
if (started || (operation.modelId.startsWith('app:') && app?.engine !== 'grid')) {
return this.performApp(grid, operation, app?.engine === 'grid' ? undefined : app)
}
@@ -960,7 +1014,7 @@ export class LocalModels {
* address, answered through the grid, and held to the 64K floor by what the engine says it loaded. */
private async performApp(grid: string, operation: ModelOperation, app: AppModel | undefined): Promise<void> {
const ops = this.options.appEngines
- const records = await readAppRecords(this.appRecordsFile)
+ const records = await this.records()
const record = records.find(r => r.modelId === operation.modelId && r.grid === grid)
const without = (gone: AppEngineRecord) => writeAppRecords(this.appRecordsFile, records.filter(r => r !== gone))
const takeDown = async (started: AppEngineRecord) => {
@@ -993,7 +1047,7 @@ export class LocalModels {
await writeAppRecords(this.appRecordsFile, [...records.filter(r => !(r.modelId === app.id && r.grid === grid)), started])
const failed = async (message: string): Promise<never> => {
await takeDown(started)
- await writeAppRecords(this.appRecordsFile, (await readAppRecords(this.appRecordsFile)).filter(r => !(r.modelId === app.id && r.grid === grid)))
+ await writeAppRecords(this.appRecordsFile, (await this.records()).filter(r => !(r.modelId === app.id && r.grid === grid)))
throw new ModelError(message)
}
const named = this.options.machineName?.()?.trim()
diff --git a/cli/src/services/models.ts b/cli/src/services/models.ts
index e50afff72..349424e7c 100644
--- a/cli/src/services/models.ts
+++ b/cli/src/services/models.ts
@@ -36,6 +36,7 @@ import {
import { gridModelsPayload } from '../lib/gridModelsPayload.js'
import { resolveGridTarget } from '../lib/gridTarget.js'
import { LocalModels } from '../lib/localModels.js'
+import { nvidiaComputeApps } from '../lib/gpuProcesses.js'
import type { NewAgentModel } from '../lib/newAgentModel.js'
import type { RuntimeModelOption } from '../lib/runtimeProfile.js'
import { parseRuntimeProfile } from '../lib/runtimeProfileWire.js'
@@ -247,6 +248,7 @@ function modelsRequests(core: CoreApi, grid: GridSetUp, privateGridName: () => P
// bundled harness), so the picker and that harness agree on what is here and what starts it.
appModels: () => scanAppModels({ node: baseNode(process.execPath), packageDir: installedDsh(MODEL_MANAGER_ID)?.realDir ?? null, env: process.env }),
appEngines: appEngineOps(process.env),
+ gpuProcesses: nvidiaComputeApps,
})
/** The grid listing currently out, shared by every `grid_models_list` for the same own grid that lands
* meanwhile. */
diff --git a/desktop/lib/models/model_manager_controller.dart b/desktop/lib/models/model_manager_controller.dart
index 8a703edf6..559013f76 100644
--- a/desktop/lib/models/model_manager_controller.dart
+++ b/desktop/lib/models/model_manager_controller.dart
@@ -69,7 +69,8 @@ class ModelManagerController extends ChangeNotifier {
/// The machine's RAM. On an NVIDIA machine a model runs in the card, not in this: see [fitsInVram].
double? memoryBytes;
- /// What the catalog fits models to (`fitBytes`) — on an NVIDIA machine the VRAM free on its card —
+ /// What the catalog fits models to (`fitBytes`) — on an NVIDIA machine the VRAM its card has for
+ /// models: what is free on it and what our own running model holds, which starting another frees —
/// and that card's VRAM in all (`gpuMemoryBytes`); null when the daemon did not say.
double? fitBytes, gpuMemoryBytes;
@@ -78,13 +79,13 @@ class ModelManagerController extends ChangeNotifier {
bool fitsInVram = false;
/// The figure a heading names this machine's memory by, and what follows it: an NVIDIA card's VRAM
- /// (`VRAM`, or `VRAM free` where only what is free on it is known), else the machine's memory
+ /// (`VRAM`, or `VRAM for models` where only that is known), else the machine's memory
/// (`unit` empty).
({double bytes, String unit})? get headingMemory => fitsInVram
? gpuMemoryBytes != null
? (bytes: gpuMemoryBytes!, unit: 'VRAM')
: fitBytes != null
- ? (bytes: fitBytes!, unit: 'VRAM free')
+ ? (bytes: fitBytes!, unit: 'VRAM for models')
: null
: memoryBytes != null
? (bytes: memoryBytes!, unit: '')
diff --git a/desktop/lib/widgets/swarm_resource_preview.dart b/desktop/lib/widgets/swarm_resource_preview.dart
index 7cb5da0cc..7ac439c18 100644
--- a/desktop/lib/widgets/swarm_resource_preview.dart
+++ b/desktop/lib/widgets/swarm_resource_preview.dart
@@ -1397,16 +1397,16 @@ class _SwarmResourcePreviewState extends State<SwarmResourcePreview> {
'${gigabytesLabel(owner.freeDiskBytes!)} free',
].join(' · '),
),
- // An NVIDIA machine's against the VRAM free on its card, which the catalog fits to.
+ // An NVIDIA machine's against the VRAM its card has for models, which the catalog fits to.
if (owner.fitsInVram) ...[
if ((local.sizeBytes, owner.fitBytes) case (
final size?,
- final free?,
+ final room?,
))
labelValue(
'VRAM',
- '${size <= free ? 'fits' : 'needs ${gigabytesLabel(size)}'} · '
- 'its GPU has ${gigabytesLabel(free)} free',
+ '${size <= room ? 'fits' : 'needs ${gigabytesLabel(size)}'} · '
+ 'its GPU has ${gigabytesLabel(room)} for models',
),
] else if ((local.sizeBytes, owner.memoryBytes) case (
final size?,
diff --git a/desktop/test/model_search_catalog_test.dart b/desktop/test/model_search_catalog_test.dart
index 04ca244a6..294dd6098 100644
--- a/desktop/test/model_search_catalog_test.dart
+++ b/desktop/test/model_search_catalog_test.dart
@@ -164,7 +164,7 @@ void main() {
await app.modelManager.refresh();
expect(
grouped.catalogHeading,
- 'Get for ${thisComputerName()} · 22 GB VRAM free',
+ 'Get for ${thisComputerName()} · 22 GB VRAM for models',
);
} finally {
grouped.dispose();
diff --git a/tui/src/models.rs b/tui/src/models.rs
index 41e3fc618..1f30303c7 100644
--- a/tui/src/models.rs
+++ b/tui/src/models.rs
@@ -89,6 +89,9 @@ impl LocalModel {
})
}
pub fn running(&self) -> bool { self.state == "running" }
+ /// Its name as shown: without the quantization a file name ends in, which the Quant line says.
+ /// [name] stays whole — it is what the grid serves it as, and what a harness is matched to it by.
+ pub fn shown(&self) -> String { without_quant(&self.name).to_string() }
pub fn downloaded(&self) -> bool { self.state == "downloaded" || self.running() }
/// Running, parked while this computer's models rest: it answers again on the next message.
pub fn resting(&self) -> bool { self.running() && self.asleep }
@@ -96,14 +99,32 @@ impl LocalModel {
/// guessed from its size.
pub fn quantization(&self) -> Option<String> {
if let Some(q) = &self.quant { return Some(q.clone()) }
- self.id.split(|c: char| !(c.is_ascii_alphanumeric() || c == '_')).find(|w| {
- let u = w.to_ascii_uppercase();
- let digit_after = |p: &str| u.strip_prefix(p).is_some_and(|r| r.starts_with(|c: char| c.is_ascii_digit()));
- digit_after("IQ") || digit_after("Q") || digit_after("MXFP") || ["BF16", "FP16", "F16"].contains(&u.as_str())
- }).map(str::to_ascii_uppercase)
+ quant_of(&self.id)
}
}
+/// The quantization a model's file name says (`…-Q4_K_M.gguf`): `Q4_K_M`.
+pub fn quant_of(name: &str) -> Option<String> {
+ name.split(|c: char| !(c.is_ascii_alphanumeric() || c == '_')).find(|w| {
+ let u = w.to_ascii_uppercase();
+ let digit_after = |p: &str| u.strip_prefix(p).is_some_and(|r| r.starts_with(|c: char| c.is_ascii_digit()));
+ digit_after("IQ") || digit_after("Q") || digit_after("MXFP") || ["BF16", "FP16", "F16"].contains(&u.as_str())
+ }).map(str::to_ascii_uppercase)
+}
+
+/// [name] without the quantization its file name ends in: `Qwen3.8-27B-UD-Q4_K_M` is `Qwen3.8-27B`,
+/// `qwen3:4b-instruct-2507-q4_k_m` is `qwen3:4b-instruct-2507`. A name that is all quant stays.
+pub fn without_quant(name: &str) -> &str {
+ let quant = |rest: &str| {
+ let u = rest.to_ascii_uppercase();
+ let q = u.strip_prefix("UD-").or_else(|| u.strip_prefix("UD_")).unwrap_or(&u);
+ let digit_after = |p: &str| q.strip_prefix(p).is_some_and(|r| r.starts_with(|c: char| c.is_ascii_digit()) && r.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'));
+ digit_after("IQ") || digit_after("Q") || digit_after("MXFP") || ["BF16", "FP16", "F16"].contains(&q)
+ };
+ name.char_indices().filter(|&(i, c)| i > 0 && matches!(c, '-' | '_' | '.')).map(|(i, _)| i)
+ .find(|&i| quant(&name[i + 1..])).map_or(name, |i| &name[..i])
+}
+
/// What `grid_fleet_models_list` answers: the models, the machine's memory and free disk, and
/// whether an operation is under way. `grid_setup_needed`: grid is not set up here, so local and
/// shared models are one Set up row (the desktop's); `grid_setup_error`: why the last Set up failed.
@@ -626,7 +647,7 @@ pub fn facts(app: &App, m: &LocalModel, t: Option<&Target>) -> Facts {
let active = op.as_ref().is_some_and(Operation::active);
let busy = v.busy(machine);
let online = up(app, machine) && v.local.contains_key(machine);
- let other = other_running(app, m).map(|o| o.name);
+ let other = other_running(app, m).map(|o| o.shown());
let runs = t.is_some_and(|t| can_run(app, &t.engine));
let served = served_in(v.grids.get(machine), m);
let in_use = t.is_some_and(|t| m.running() && !t.grid_model.is_empty()
@@ -673,7 +694,7 @@ pub fn catalog_heading(snap: Option<&Snapshot>) -> String {
match snap {
Some(s) if s.fit_vram => match (s.gpu_memory, s.fit) {
(Some(all), _) => h.push_str(&format!(" · {} VRAM", gb(all))),
- (None, Some(free)) => h.push_str(&format!(" · {} VRAM free", gb(free))),
+ (None, Some(room)) => h.push_str(&format!(" · {} VRAM for models", gb(room))),
_ => {}
},
Some(s) => if let Some(m) = s.memory { h.push_str(&format!(" · {}", gb(m))) },
@@ -744,7 +765,7 @@ fn local_row(app: &App, m: &LocalModel, t: Option<&Target>, group: &str) -> Row
// quantization and app are in the preview, and found by a search.
let quant = m.quantization().unwrap_or_default();
let app_name = m.app.clone().unwrap_or_default();
- Row::new(format!("mv:local:{}", m.id), m.name.clone()).group(group.to_string())
+ Row::new(format!("mv:local:{}", m.id), m.shown()).group(group.to_string())
.extra(format!("{} {quant} {app_name} {word} local{}", m.id, if m.recommended { " recommended" } else { "" }))
.lead(vec![span(dot, fg(color))])
.right(columns(&m.size.map(gb).unwrap_or_default(), &speed(m), &word))
@@ -765,8 +786,9 @@ fn grid_word(app: &App, x: &GridModel, t: Option<&Target>) -> String {
fn grid_row(app: &App, s: &Section, x: &GridModel, t: Option<&Target>, group: &str) -> Row {
let word = grid_word(app, x, t);
let (dot, color) = if word.starts_with('✓') { ("✓ ", theme::accent()) } else if x.offline.is_some() { ("· ", theme::MUTED) } else { ("◆ ", theme::TEAL) };
- Row::new(format!("mv:grid:{}\t{}\t{}", s.name, x.node, x.id), x.id.clone()).group(group.to_string())
- .extra(format!("{} {} {} {}", s.name, x.node, if s.own { "local" } else { "shared" }, word))
+ // Shown without its quant, as Your models are; switched onto (and found by) the name it is served as.
+ Row::new(format!("mv:grid:{}\t{}\t{}", s.name, x.node, x.id), without_quant(&x.id).to_string()).group(group.to_string())
+ .extra(format!("{} {} {} {} {}", x.id, s.name, x.node, if s.own { "local" } else { "shared" }, word))
.lead(vec![span(dot, fg(color))])
.detail(vec![span(x.node.clone(), fg(theme::MUTED))])
.right(word)
@@ -1076,7 +1098,7 @@ pub fn plan_enter(app: &App, id: &str) -> Plan {
if let Some(u) = v.using.as_ref().filter(|u| u.model == m.id) { return Plan::Say(using_label(app, u)) }
if f.active || f.pending.is_some() || f.busy { return say("Wait for the current model operation to finish.") }
if let (true, Some(t), Some((grid, served))) = (f.use_running, t, f.served.clone()) {
- return Plan::Retarget { machine: t.machine.clone(), payload: json!({ "agentId": t.agent, "gridModel": served, "gridName": grid }), name: m.name.clone() };
+ return Plan::Retarget { machine: t.machine.clone(), payload: json!({ "agentId": t.agent, "gridModel": served, "gridName": grid }), name: m.shown() };
}
if m.running() {
return match t {
@@ -1089,11 +1111,11 @@ pub fn plan_enter(app: &App, id: &str) -> Plan {
// first when another harness answers with that one.
let other = other_running(app, &m);
let users = other.as_ref().map(|o| harnesses_on(app, o, t)).unwrap_or_default();
- let stop_said = other.as_ref().and_then(|o| stop_words(&o.name, &users));
+ let stop_said = other.as_ref().and_then(|o| stop_words(&o.shown(), &users));
if m.downloaded() {
if f.use_start {
let go = Plan::Use { model: m.id.clone(), download: false };
- return match (&other, &stop_said) { (Some(o), Some(said)) => Plan::Arm(format!("Stop {}? {said} Enter · Esc", o.name), Box::new(go)), _ => go };
+ return match (&other, &stop_said) { (Some(o), Some(said)) => Plan::Arm(format!("Stop {}? {said} Enter · Esc", o.shown()), Box::new(go)), _ => go };
}
if let Some(o) = &f.other { return Plan::Say(format!("Stop {o} first: one local model runs at a time — ^S on it")) }
return match t {
@@ -1109,7 +1131,7 @@ pub fn plan_enter(app: &App, id: &str) -> Plan {
let size = m.size.map(gb).map(|s| format!(" ({s})")).unwrap_or_default();
let then = match (f.get_use, &f.other) { (true, Some(o)) => format!(", stop {o}, start it, move this harness onto it"), (true, None) => ", start it, move this harness onto it".into(), _ => String::new() };
let said = stop_said.filter(|_| f.get_use).map(|s| format!(" {s}")).unwrap_or_default();
- Plan::Arm(format!("Get {}{size}{then}?{said} Enter · Esc", m.name), Box::new(go))
+ Plan::Arm(format!("Get {}{size}{then}?{said} Enter · Esc", m.shown()), Box::new(go))
}
/// What ^S does on row [id]: stop a local model (asked first).
@@ -1118,7 +1140,7 @@ pub fn plan_stop(app: &App, id: &str) -> Plan {
let f = facts(app, &m, None);
if f.busy { return Plan::Say("Wait for the current model operation to finish.".into()) }
if !m.can_stop { return Plan::Say(if m.running() { "Other models share its engine — stop it from the desktop's Model Manager".into() } else { "It is not running".into() }) }
- Plan::Arm(format!("^S again stops {} — the download is kept", m.name), Box::new(Plan::Act { model: m.id.clone(), action: "stop" }))
+ Plan::Arm(format!("^S again stops {} — the download is kept", m.shown()), Box::new(Plan::Act { model: m.id.clone(), action: "stop" }))
}
/// Enter (or ^S, [stop]) on row [id] of the view: planned, then done.
@@ -1846,10 +1868,10 @@ fn start_use(app: &mut App, model: &str, download: bool) {
let supports = app.models_view.local.get(&app.fleet.local_id).is_some_and(|s| s.supports_download);
// One local model runs at a time: the one running here stops first — after a download, so it
// answers until this one is ready to start.
- let stops = other_running(app, &m).map(|o| (o.id, o.name));
+ let stops = other_running(app, &m).map(|o| { let shown = o.shown(); (o.id, shown) });
let step = match (&stops, download && supports) { (_, true) => Step::Get, (Some(_), false) => Step::Stop, (None, false) => if download { Step::Get } else { Step::Start } };
let identity = app.fleet.agent(&t.machine, &t.agent).map(|a| crate::session_close::Identity::capture(app, a));
- app.models_view.using = Some(Use { model: m.id.clone(), name: m.name.clone(), machine: t.machine, agent: t.agent, step, stops: stops.clone(), since: Instant::now(), identity });
+ app.models_view.using = Some(Use { model: m.id.clone(), name: m.shown(), machine: t.machine, agent: t.agent, step, stops: stops.clone(), since: Instant::now(), identity });
app.models_view.ended = None;
match (step, stops) {
(Step::Stop, Some((other, _))) => act(app, &other, "stop"),
@@ -2049,7 +2071,7 @@ pub fn preview(app: &App, id: &str) -> Vec<Line<'static>> {
let status = if let Some(o) = f.op.as_ref().filter(|o| o.active()) { o.word() } else if let Some(p) = f.pending { p.to_string() } else if f.failed { "Failed · try again".into() }
else if m.running() { "Running".into() } else if m.downloaded() { "Downloaded".into() } else { "Not downloaded".into() };
let status = if f.switching { format!("{SWITCHING} this harness is moving onto it") } else if f.in_use { format!("{status} · this harness is on it") } else { status };
- let mut out = vec![bold(m.name.clone()), Line::raw(status)];
+ let mut out = vec![bold(m.shown()), Line::raw(status)];
// (A Use's end says why it failed itself.)
if !v.ended.as_ref().is_some_and(|e| e.row == id) { out.extend(failed) }
out.extend(use_lines(app, &m, id));
@@ -2059,9 +2081,10 @@ pub fn preview(app: &App, id: &str) -> Vec<Line<'static>> {
if let Some(size) = m.size {
let free = snap.and_then(|s| s.free_disk).filter(|_| !m.downloaded()).map(|d| format!(" · {} free", gb(d))).unwrap_or_default();
out.push(kv(if m.downloaded() { "Size" } else { "Download" }, format!("{}{free}", gb(size))));
- // (An NVIDIA machine's against the VRAM free on its card, which the catalog fits to.)
+ // (An NVIDIA machine's against the VRAM its card has for models — what is free on it and
+ // what our own running model holds, which a start of another frees — the catalog's fit.)
match snap {
- Some(s) if s.fit_vram => if let Some(free) = s.fit { out.push(kv("VRAM", format!("{} · its GPU has {} free", if size <= free { "fits".to_string() } else { format!("needs {}", gb(size)) }, gb(free)))) },
+ Some(s) if s.fit_vram => if let Some(room) = s.fit { out.push(kv("VRAM", format!("{} · its GPU has {} for models", if size <= room { "fits".to_string() } else { format!("needs {}", gb(size)) }, gb(room)))) },
_ => if let Some(mem) = snap.and_then(|s| s.memory) { out.push(kv("Memory", format!("{} · {here} has {}", if size <= mem { "fits".to_string() } else { format!("needs {}", gb(size)) }, gb(mem)))) },
}
}
@@ -2105,10 +2128,11 @@ pub fn preview(app: &App, id: &str) -> Vec<Line<'static>> {
"✓ In use" => format!("{status} · this harness is on it"),
_ => status,
};
- let mut out = vec![bold(model.to_string()), Line::raw(status)];
+ let mut out = vec![bold(without_quant(model).to_string()), Line::raw(status)];
out.extend(failed(model));
out.push(Line::raw(""));
out.push(kv("Machine", node.to_string()));
+ if let Some(q) = quant_of(model) { out.push(kv("Quant", q)) }
out.push(kv("Source", if own { "On your machines".to_string() } else { format!("Shared · {grid}") }));
let said = match plan_enter(app, id) { Plan::Retarget { .. } => "Enter moves this harness onto it.".to_string(), Plan::Arm(..) => "Enter twice moves this harness onto it anyway.".into(), Plan::Say(w) if w == NO_HARNESS => no_target_why(app), Plan::Say(w) => w, _ => String::new() };
if !said.is_empty() { out.push(Line::raw("")); out.push(Line::from(span(said, fg(theme::accent())))) }
@@ -3531,7 +3555,8 @@ mod tests {
}
/// An NVIDIA machine's models run in its card: the heading says its VRAM and "fits" is judged
- /// against the VRAM free on it — not the machine's RAM, which a Mac's unified memory still shows.
+ /// against the VRAM it has for models — what is free on it and what our own running model holds —
+ /// not the machine's RAM, which a Mac's unified memory still shows.
#[test]
fn an_nvidia_machine_is_told_its_vram_not_its_ram() {
let mut app = app();
@@ -3546,12 +3571,65 @@ mod tests {
on_local(&mut app, "local", Ok(reply.clone()));
assert_eq!(catalog_heading(app.models_view.local.get("local")), format!("Get for {} · 16 GB VRAM", this_computer()));
let panel = text(&preview(&app, "mv:local:qwen-35b"));
- assert!(panel.contains("VRAM needs 20 GB · its GPU has 14 GB free"), "{panel}");
+ assert!(panel.contains("VRAM needs 20 GB · its GPU has 14 GB for models"), "{panel}");
assert!(!panel.contains("has 64 GB"), "{panel}");
- // A daemon that names no card's total: what is free on it.
+ // A daemon that names no card's total: what it has for models.
reply.as_object_mut().unwrap().remove("gpuMemoryBytes");
on_local(&mut app, "local", Ok(reply));
- assert_eq!(catalog_heading(app.models_view.local.get("local")), format!("Get for {} · 14 GB VRAM free", this_computer()));
+ assert_eq!(catalog_heading(app.models_view.local.get("local")), format!("Get for {} · 14 GB VRAM for models", this_computer()));
+ }
+
+ #[test]
+ fn a_name_is_shown_without_the_quant_its_file_ends_in() {
+ for (name, shown) in [("Qwen3.8-27B-UD-Q4_K_M", "Qwen3.8-27B"), ("Qwen3-4B-Instruct-2507-Q4_K_M", "Qwen3-4B-Instruct-2507"),
+ ("qwen3:4b-instruct-2507-q4_k_m", "qwen3:4b-instruct-2507"), ("Kev-0.8B-Q8_0", "Kev-0.8B"), ("gpt-oss-20b-MXFP4", "gpt-oss-20b"),
+ ("Mistral-7B-v0.1.IQ4_XS", "Mistral-7B-v0.1"), ("gemma-4-E4B-it-BF16", "gemma-4-E4B-it"), ("Qwen3.6-35B-A3B-UD_Q5_K_XL", "Qwen3.6-35B-A3B"),
+ // Nothing to take off — and a word that only starts like one (`Q3-Coder`, `Qwen`) is the name's.
+ ("Qwen3.6-35B-A3B", "Qwen3.6-35B-A3B"), ("llama3.2:3b", "llama3.2:3b"), ("gemma-4-e2b", "gemma-4-e2b"), ("Qwen-Q3-Coder", "Qwen-Q3-Coder"),
+ ("Q4_K_M", "Q4_K_M"), ("-Q4", "-Q4"), ("Qwen/Qwen3.8-27B", "Qwen/Qwen3.8-27B")] {
+ assert_eq!(without_quant(name), shown, "{name}");
+ }
+ }
+
+ #[test]
+ fn a_shared_model_is_shown_without_its_quant_and_switched_onto_by_its_served_name() {
+ let mut app = loaded();
+ let mut g = grids_reply(&[]);
+ g["grids"][1]["models"][0]["id"] = json!("Qwen3.8-27B-UD-Q4_K_M");
+ on_grids(&mut app, "local", Ok(g));
+ let id = "mv:grid:team\tgpu-box\tQwen3.8-27B-UD-Q4_K_M";
+ assert_eq!(row(&rows_of(&app), id).label, "Qwen3.8-27B");
+ assert!(row(&rows_of(&app), id).extra.contains("Qwen3.8-27B-UD-Q4_K_M"));
+ let panel = text(&preview(&app, id));
+ assert!(panel.starts_with("Qwen3.8-27B\n"), "{panel}");
+ assert!(panel.contains("Quant Q4_K_M"), "{panel}");
+ match plan_enter(&app, id) {
+ Plan::Retarget { payload, .. } => assert_eq!(payload["gridModel"], "Qwen3.8-27B-UD-Q4_K_M"),
+ other => panic!("{other:?}"),
+ }
+ }
+
+ /// A model a Grid setup left running, which the catalog does not list: grid serves it under its
+ /// file's name, quant and all. It is shown without the quant (the Quant line says it), and a
+ /// harness still moves onto it by the name grid serves.
+ #[test]
+ fn a_running_model_is_shown_without_its_quant_and_still_switched_onto_by_its_served_name() {
+ let mut app = app();
+ let file = json!({ "id": "local:Qwen3.8-27B-UD-Q4_K_M.gguf", "name": "Qwen3.8-27B-UD-Q4_K_M", "state": "running", "quant": "", "canStart": false, "canStop": true, "app": "Grid" });
+ on_local(&mut app, "local", Ok(local_reply(file, false)));
+ on_grids(&mut app, "local", Ok(grids_reply(&["Qwen3.8-27B-UD-Q4_K_M"])));
+ let id = "mv:local:local:Qwen3.8-27B-UD-Q4_K_M.gguf";
+ assert_eq!(row(&rows_of(&app), id).label, "Qwen3.8-27B");
+ let panel = text(&preview(&app, id));
+ assert!(panel.starts_with("Qwen3.8-27B\n"), "{panel}");
+ assert!(panel.contains("Quant Q4_K_M"), "{panel}");
+ match plan_enter(&app, id) {
+ Plan::Retarget { payload, name, .. } => {
+ assert_eq!(payload["gridModel"], "Qwen3.8-27B-UD-Q4_K_M");
+ assert_eq!(name, "Qwen3.8-27B");
+ }
+ other => panic!("{other:?}"),
+ }
}
}