Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(models): an NVIDIA machine is told its VRAM, not its RAM

bitcoinbuilder72355f09732b
diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.ts
index 2cf562710..87aef2562 100644
--- a/cli/src/lib/localModels.spec.ts
+++ b/cli/src/lib/localModels.spec.ts
@@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'
import { createHash } from 'node:crypto'
import { rmSync, writeFileSync } from 'node:fs'
import { spawnSync } from 'node:child_process'
-import { baseModel, compatibleModels, contextLadder, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'
+import { baseModel, compatibleModels, contextLadder, fitOf, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'
import { GridFleetRpc, type GridFleetResult } from './gridFleetRpc.js'
import { AppStartError, type AppEngineOps, type AppEngineRecord, type AppModel } from './appModels.js'
import { encryptDownFrame, encryptRpcResult } from './e2ee/applicationFrames.js'
@@ -129,6 +129,7 @@ describe('local model discovery and lifecycle', () => {
const snapshot = await service.list('home')
expect(snapshot.models).toMatchObject([{ name: 'Small', state: 'available', sizeBytes: 64, canStart: true, canStop: false }])
expect(snapshot.memoryBytes).toBe(64 * 1024 ** 3)
+ expect(snapshot).toMatchObject({ fitBytes: 32 * 1024 ** 3, fitMemory: 'unified' })
expect(calls.some(args => ['pull', 'join', 'leave', 'install'].some(a => args.includes(a)))).toBe(false)
expect(JSON.stringify(snapshot)).not.toContain('test-only-token')
})
@@ -593,6 +594,13 @@ describe('local model discovery and lifecycle', () => {
it("gives the catalog all of an NVIDIA card's free VRAM, whatever the system RAM", () => {
// A 24 GB card in a 32 GB PC: VRAM is not system RAM, and nothing else is waiting for it.
expect(modelBudget({ backend: 'cuda', usable_bytes: 23 * 1024 ** 3, memory: { total_gb: 32 } })).toBe(23 * 1024 ** 3)
+ // What the apps are told with it: an NVIDIA card's free VRAM and all its VRAM, not the PC's RAM.
+ expect(fitOf({ backend: 'cuda', usable_bytes: 22 * 1024 ** 3, memory: { total_gb: 64 }, gpus: [{ backend: 'cuda', memory_total_mb: 24576 }] }))
+ .toEqual({ fitBytes: 22 * 1024 ** 3, fitMemory: 'vram', gpuMemoryBytes: 24 * 1024 ** 3 })
+ expect(fitOf({ backend: 'metal', usable_bytes: 54 * 1024 ** 3, memory: { total_gb: 64 }, gpus: [{ backend: 'metal', memory_total_mb: 65536 }] }))
+ .toEqual({ fitBytes: 32 * 1024 ** 3, fitMemory: 'unified' })
+ expect(fitOf({ backend: 'cpu', usable_bytes: 20 * 1024 ** 3, memory: { total_gb: 32 } })).toEqual({ fitBytes: 16 * 1024 ** 3, fitMemory: 'ram' })
+ expect(fitOf({})).toEqual({})
// Two 24 GB cards in a 64 GB PC.
expect(modelBudget({ backend: 'cuda', usable_bytes: 46 * 1024 ** 3, memory: { total_gb: 64 } })).toBe(46 * 1024 ** 3)
})
diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.ts
index 443b43798..52d506961 100644
--- a/cli/src/lib/localModels.ts
+++ b/cli/src/lib/localModels.ts
@@ -89,7 +89,14 @@ export interface ModelOperation {
progress?: number; error?: string; updatedAt: string
}
export interface LocalModelsSnapshot {
+ /** [memoryBytes]: the machine's RAM — on an NVIDIA machine not what a model runs in. */
models: LocalModel[]; memoryBytes?: number; hardware?: string
+ /** What the catalog fits models to ([modelBudget]) and what memory that is: an NVIDIA card's free
+ * VRAM (`vram`), an Apple Silicon Mac's unified memory, or system RAM where there is no usable
+ * GPU. The apps say VRAM, and judge "fits" against it, where a model runs on the card. */
+ fitBytes?: number; fitMemory?: 'vram' | 'unified' | 'ram'
+ /** An NVIDIA machine's cards' VRAM, all of it (grid's `gpus[].memory_total_mb`). */
+ gpuMemoryBytes?: number
/** A sentence to show BESIDE the list — never `error`. On this protocol an `error` field fails
* the whole request: the app's RPC layer drops the reply and keeps nothing of it, so a list sent
* with a warning in `error` arrived as no list at all and the Local tab read 0. */
@@ -213,6 +220,16 @@ export function modelBudget(device: Record<string, any>): number | undefined {
return Math.min(usable, Math.floor(total * GiB * MODEL_MEMORY_SHARE))
}
+/** What the catalog fits to on [device] and what memory that is ([LocalModelsSnapshot.fitBytes]). */
+export function fitOf(device: Record<string, any>): Pick<LocalModelsSnapshot, 'fitBytes' | 'fitMemory' | 'gpuMemoryBytes'> {
+ const fit = modelBudget(device)
+ const backend = str(device.backend)
+ const fitMemory = backend === 'cuda' ? 'vram' as const : backend === 'metal' ? 'unified' as const : backend ? 'ram' as const : undefined
+ const cards = backend === 'cuda' && Array.isArray(device.gpus) ? device.gpus : []
+ const gpuMemoryBytes = cards.reduce((sum: number, gpu: unknown) => sum + (num(obj(gpu).memory_total_mb) ?? 0), 0) * 1024 ** 2
+ return { ...(fit === undefined ? {} : { fitBytes: fit }), ...(fitMemory ? { fitMemory } : {}), ...(gpuMemoryBytes > 0 ? { gpuMemoryBytes } : {}) }
+}
+
/** Catalog models that are not for running an agent on: safety classifiers such as gpt-oss-safeguard
* or Llama Guard, which answer "safe"/"unsafe" rather than code. */
const NOT_FOR_AGENTS = /(?:^|[-_/])(?:safeguard|guard|shieldgemma)(?:[-_.]|$)/i
@@ -735,6 +752,7 @@ export class LocalModels {
const freeDiskBytes = await statfs(join(this.home, 'models')).catch(() => statfs(this.home))
.then(disk => disk.bavail * disk.bsize, () => undefined)
const value: LocalModelsSnapshot = { models: withoutCopiesOfOwned(models), memoryBytes: num(obj(this.device.memory).total_gb) === undefined ? undefined : this.device.memory.total_gb * GiB,
+ ...fitOf(this.device),
hardware: str(obj(this.device.machine).model || obj(this.device.cpu).brand), notice: inventoryError || this.catalogError,
observedAt: new Date().toISOString(), busy: !!this.active, supportsDownload: true,
...(freeDiskBytes === undefined ? {} : { freeDiskBytes }) }
diff --git a/desktop/lib/models/model_manager_controller.dart b/desktop/lib/models/model_manager_controller.dart
index 6bbabf905..8a703edf6 100644
--- a/desktop/lib/models/model_manager_controller.dart
+++ b/desktop/lib/models/model_manager_controller.dart
@@ -65,8 +65,31 @@ class ModelManagerController extends ChangeNotifier {
String? _dismissedReadyId;
GridModels? models;
List<LocalModel> localModels = const [];
+
+ /// The machine's RAM. On an NVIDIA machine a model runs in the card, not in this: see [fitsInVram].
double? memoryBytes;
+ /// What the catalog fits models to (`fitBytes`) — on an NVIDIA machine the VRAM free on its card —
+ /// and that card's VRAM in all (`gpuMemoryBytes`); null when the daemon did not say.
+ double? fitBytes, gpuMemoryBytes;
+
+ /// Models run in an NVIDIA card's VRAM here (`fitMemory: vram`): the figures to show are the card's,
+ /// and "fits" is judged against [fitBytes], not the machine's RAM.
+ bool fitsInVram = false;
+
+ /// The figure a heading names this machine's memory by, and what follows it: an NVIDIA card's VRAM
+ /// (`VRAM`, or `VRAM free` where only what is free on it is known), else the machine's memory
+ /// (`unit` empty).
+ ({double bytes, String unit})? get headingMemory => fitsInVram
+ ? gpuMemoryBytes != null
+ ? (bytes: gpuMemoryBytes!, unit: 'VRAM')
+ : fitBytes != null
+ ? (bytes: fitBytes!, unit: 'VRAM free')
+ : null
+ : memoryBytes != null
+ ? (bytes: memoryBytes!, unit: '')
+ : null;
+
/// Free space where downloads land, or null when the daemon did not say.
double? freeDiskBytes;
String? hardware, error, _managerError;
@@ -225,6 +248,9 @@ class ModelManagerController extends ChangeNotifier {
models = null;
localModels = const [];
memoryBytes = null;
+ fitBytes = null;
+ gpuMemoryBytes = null;
+ fitsInVram = false;
freeDiskBytes = null;
hardware = null;
error = null;
@@ -579,10 +605,14 @@ class ModelManagerController extends ChangeNotifier {
.map(LocalModel.fromJson)
.where((m) => m.id.isNotEmpty)
.toList();
- final memory = answer['memoryBytes'];
- memoryBytes = memory is num && memory.isFinite && memory > 0
- ? memory.toDouble()
+ double? positive(Object? value) =>
+ value is num && value.isFinite && value > 0
+ ? value.toDouble()
: null;
+ memoryBytes = positive(answer['memoryBytes']);
+ fitBytes = positive(answer['fitBytes']);
+ gpuMemoryBytes = positive(answer['gpuMemoryBytes']);
+ fitsInVram = answer['fitMemory'] == 'vram';
final disk = answer['freeDiskBytes'];
freeDiskBytes = disk is num && disk.isFinite && disk >= 0
? disk.toDouble()
diff --git a/desktop/lib/models/model_search_catalog.dart b/desktop/lib/models/model_search_catalog.dart
index b13dbcd1a..1d8873f2f 100644
--- a/desktop/lib/models/model_search_catalog.dart
+++ b/desktop/lib/models/model_search_catalog.dart
@@ -492,10 +492,15 @@ class ModelSearchCatalog extends ChangeNotifier {
if (owners.length != 1) return ModelSearchSection.catalog.label;
final owner = owners.single;
final machine = owner.machine;
- final memory = owner.memoryBytes;
+ // (An NVIDIA machine's models run in its card: its VRAM, not the machine's RAM.)
+ final memory = owner.headingMemory;
return [
'Get for ${machine == null || machine.isLocalMachine ? thisComputerName() : machine.machine.displayName}',
- if (memory != null) gigabytesLabel(memory),
+ if (memory != null)
+ [
+ gigabytesLabel(memory.bytes),
+ if (memory.unit.isNotEmpty) memory.unit,
+ ].join(' '),
].join(' · ');
}
diff --git a/desktop/lib/models/models_panel.dart b/desktop/lib/models/models_panel.dart
index 8bea64b3a..80af4c202 100644
--- a/desktop/lib/models/models_panel.dart
+++ b/desktop/lib/models/models_panel.dart
@@ -323,8 +323,8 @@ class _ModelsPanelState extends State<ModelsPanel> with SectionWakes {
child: Text(
[
'This computer',
- if (controller.memoryBytes case final memory?)
- '${_size(memory)} memory',
+ if (controller.headingMemory case final memory?)
+ '${_size(memory.bytes)} ${memory.unit.isEmpty ? 'memory' : memory.unit}',
].join(' · '),
style: AppType.monoMeta(
color: AppPalette.textSecondary,
diff --git a/desktop/lib/widgets/swarm_resource_preview.dart b/desktop/lib/widgets/swarm_resource_preview.dart
index b920dbf00..7cb5da0cc 100644
--- a/desktop/lib/widgets/swarm_resource_preview.dart
+++ b/desktop/lib/widgets/swarm_resource_preview.dart
@@ -1397,7 +1397,18 @@ class _SwarmResourcePreviewState extends State<SwarmResourcePreview> {
'${gigabytesLabel(owner.freeDiskBytes!)} free',
].join(' · '),
),
- if ((local.sizeBytes, owner.memoryBytes) case (
+ // An NVIDIA machine's against the VRAM free on its card, which the catalog fits to.
+ if (owner.fitsInVram) ...[
+ if ((local.sizeBytes, owner.fitBytes) case (
+ final size?,
+ final free?,
+ ))
+ labelValue(
+ 'VRAM',
+ '${size <= free ? 'fits' : 'needs ${gigabytesLabel(size)}'} · '
+ 'its GPU has ${gigabytesLabel(free)} free',
+ ),
+ ] else if ((local.sizeBytes, owner.memoryBytes) case (
final size?,
final memory?,
))
diff --git a/desktop/test/model_search_catalog_test.dart b/desktop/test/model_search_catalog_test.dart
index d35d7be0a..04ca244a6 100644
--- a/desktop/test/model_search_catalog_test.dart
+++ b/desktop/test/model_search_catalog_test.dart
@@ -130,6 +130,47 @@ void main() {
},
);
+ test('an NVIDIA machine is told its VRAM, not its RAM', () async {
+ final grouped = ModelSearchCatalog(
+ app.modelManager,
+ _WithSubscriptions(),
+ pollHosts: false,
+ );
+ try {
+ Map<String, Object?> answer({bool vram = true, bool total = true}) => {
+ 'memoryBytes': 64 * 1024 * 1024 * 1024,
+ if (vram) ...{
+ 'fitMemory': 'vram',
+ 'fitBytes': 22 * 1024 * 1024 * 1024,
+ if (total) 'gpuMemoryBytes': 24 * 1024 * 1024 * 1024,
+ },
+ 'models': [
+ {'id': 'catalog-A', 'name': 'AAA catalog'},
+ ],
+ };
+ app.localInventory = answer(vram: false);
+ await app.modelManager.refresh();
+ expect(app.modelManager.fitsInVram, isFalse);
+ expect(grouped.catalogHeading, 'Get for ${thisComputerName()} · 64 GB');
+ app.localInventory = answer();
+ await app.modelManager.refresh();
+ expect(app.modelManager.fitsInVram, isTrue);
+ expect(app.modelManager.fitBytes, 22 * 1024 * 1024 * 1024);
+ expect(
+ grouped.catalogHeading,
+ 'Get for ${thisComputerName()} · 24 GB VRAM',
+ );
+ app.localInventory = answer(total: false);
+ await app.modelManager.refresh();
+ expect(
+ grouped.catalogHeading,
+ 'Get for ${thisComputerName()} · 22 GB VRAM free',
+ );
+ } finally {
+ grouped.dispose();
+ }
+ });
+
test('sections separate your models from downloads, and fold the catalog past five', () async {
final usage = _WithSubscriptions();
final grouped = ModelSearchCatalog(
diff --git a/tui/src/models.rs b/tui/src/models.rs
index 6e58ff5f0..e9766923a 100644
--- a/tui/src/models.rs
+++ b/tui/src/models.rs
@@ -109,6 +109,9 @@ impl LocalModel {
/// shared models are one Set up row (the desktop's); `grid_setup_error`: why the last Set up failed.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Snapshot { pub models: Vec<LocalModel>, pub memory: Option<f64>, pub free_disk: Option<f64>, pub hardware: Option<String>, pub notice: Option<String>, pub busy: bool, pub supports_download: bool,
+ /// What the catalog fits models to, and whether that is an NVIDIA card's VRAM (`fitMemory: vram`)
+ /// with its cards' VRAM in all: [memory] is the machine's RAM, not what such a model runs in.
+ pub fit: Option<f64>, pub fit_vram: bool, pub gpu_memory: Option<f64>,
pub grid_setup_needed: bool, pub grid_setup_error: Option<String> }
/// None when there is no list — a daemon too old for it, or a reply that failed.
@@ -119,6 +122,7 @@ pub fn parse_local(reply: &Value) -> Option<Snapshot> {
// A sentence beside the list, never a failure (an `error` fails the whole request).
notice: text(&reply["notice"]), busy: reply["busy"] == true, supports_download: reply["supportsDownload"] == true,
grid_setup_needed: reply["gridSetupNeeded"] == true, grid_setup_error: text(&reply["gridSetupError"]),
+ fit: num(&reply["fitBytes"]).filter(|m| *m > 0.0), fit_vram: reply["fitMemory"] == "vram", gpu_memory: num(&reply["gpuMemoryBytes"]).filter(|m| *m > 0.0),
})
}
@@ -646,7 +650,16 @@ fn capital(s: &str) -> String { let mut c = s.chars(); c.next().map(|f| f.to_upp
/// `Get for this Mac · 64 GB`: the downloads, and the memory they were chosen for.
pub fn catalog_heading(snap: Option<&Snapshot>) -> String {
let mut h = format!("Get for {}", this_computer());
- if let Some(m) = snap.and_then(|s| s.memory) { h.push_str(&format!(" · {}", gb(m))) }
+ // (An NVIDIA machine's models run in its card: its VRAM, not the machine's RAM.)
+ match snap {
+ Some(s) if s.fit_vram => match (s.gpu_memory, s.fit) {
+ (Some(all), _) => h.push_str(&format!(" · {} VRAM", gb(all))),
+ (None, Some(free)) => h.push_str(&format!(" · {} VRAM free", gb(free))),
+ _ => {}
+ },
+ Some(s) => if let Some(m) = s.memory { h.push_str(&format!(" · {}", gb(m))) },
+ None => {}
+ }
h
}
@@ -2000,7 +2013,11 @@ pub fn preview(app: &App, id: &str) -> Vec<Line<'static>> {
if let Some(size) = m.size {
let free = snap.and_then(|s| s.free_disk).filter(|_| !m.downloaded()).map(|d| format!(" · {} free", gb(d))).unwrap_or_default();
out.push(kv(if m.downloaded() { "Size" } else { "Download" }, format!("{}{free}", gb(size))));
- if let Some(mem) = snap.and_then(|s| s.memory) { out.push(kv("Memory", format!("{} · {here} has {}", if size <= mem { "fits".to_string() } else { format!("needs {}", gb(size)) }, gb(mem)))) }
+ // (An NVIDIA machine's against the VRAM free on its card, which the catalog fits to.)
+ match snap {
+ Some(s) if s.fit_vram => if let Some(free) = s.fit { out.push(kv("VRAM", format!("{} · its GPU has {} free", if size <= free { "fits".to_string() } else { format!("needs {}", gb(size)) }, gb(free)))) },
+ _ => if let Some(mem) = snap.and_then(|s| s.memory) { out.push(kv("Memory", format!("{} · {here} has {}", if size <= mem { "fits".to_string() } else { format!("needs {}", gb(size)) }, gb(mem)))) },
+ }
}
match (m.running().then_some(m.tok_s).flatten(), m.est_tok_s) {
(Some(s), _) => out.push(kv("Speed", format!("{s:.1} tok/s"))),
@@ -3402,4 +3419,29 @@ mod tests {
assert_eq!(url_problem("https://api.example.com/v1"), None);
assert_eq!(url_problem("http://localhost:8080/v1"), None);
}
+
+ /// An NVIDIA machine's models run in its card: the heading says its VRAM and "fits" is judged
+ /// against the VRAM free on it — not the machine's RAM, which a Mac's unified memory still shows.
+ #[test]
+ fn an_nvidia_machine_is_told_its_vram_not_its_ram() {
+ let mut app = app();
+ let mut reply = local_reply(qwen("available", Value::Null), false);
+ on_local(&mut app, "local", Ok(reply.clone()));
+ assert_eq!(catalog_heading(app.models_view.local.get("local")), format!("Get for {} · 64 GB", this_computer()));
+ assert!(text(&preview(&app, "mv:local:qwen-35b")).contains("Memory fits"), "{}", text(&preview(&app, "mv:local:qwen-35b")));
+
+ reply["fitBytes"] = json!(14.0 * GIB);
+ reply["fitMemory"] = json!("vram");
+ reply["gpuMemoryBytes"] = json!(16.0 * GIB);
+ on_local(&mut app, "local", Ok(reply.clone()));
+ assert_eq!(catalog_heading(app.models_view.local.get("local")), format!("Get for {} · 16 GB VRAM", this_computer()));
+ let panel = text(&preview(&app, "mv:local:qwen-35b"));
+ assert!(panel.contains("VRAM needs 20 GB · its GPU has 14 GB free"), "{panel}");
+ assert!(!panel.contains("has 64 GB"), "{panel}");
+
+ // A daemon that names no card's total: what is free on it.
+ reply.as_object_mut().unwrap().remove("gpuMemoryBytes");
+ on_local(&mut app, "local", Ok(reply));
+ assert_eq!(catalog_heading(app.models_view.local.get("local")), format!("Get for {} · 14 GB VRAM free", this_computer()));
+ }
}