The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(models): an NVIDIA machine is told its VRAM, not its RAM
bitcoinbuilder72355f09732b
diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.tsindex 2cf562710..87aef2562 100644--- a/cli/src/lib/localModels.spec.ts+++ b/cli/src/lib/localModels.spec.ts@@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'import { createHash } from 'node:crypto'import { rmSync, writeFileSync } from 'node:fs'import { spawnSync } from 'node:child_process'-import { baseModel, compatibleModels, contextLadder, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'+import { baseModel, compatibleModels, contextLadder, fitOf, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'import { GridFleetRpc, type GridFleetResult } from './gridFleetRpc.js'import { AppStartError, type AppEngineOps, type AppEngineRecord, type AppModel } from './appModels.js'import { encryptDownFrame, encryptRpcResult } from './e2ee/applicationFrames.js'@@ -129,6 +129,7 @@ describe('local model discovery and lifecycle', () => {const snapshot = await service.list('home')expect(snapshot.models).toMatchObject([{ name: 'Small', state: 'available', sizeBytes: 64, canStart: true, canStop: false }])expect(snapshot.memoryBytes).toBe(64 * 1024 ** 3)+ expect(snapshot).toMatchObject({ fitBytes: 32 * 1024 ** 3, fitMemory: 'unified' })expect(calls.some(args => ['pull', 'join', 'leave', 'install'].some(a => args.includes(a)))).toBe(false)expect(JSON.stringify(snapshot)).not.toContain('test-only-token')})@@ -593,6 +594,13 @@ describe('local model discovery and lifecycle', () => {it("gives the catalog all of an NVIDIA card's free VRAM, whatever the system RAM", () => {// A 24 GB card in a 32 GB PC: VRAM is not system RAM, and nothing else is waiting for it.expect(modelBudget({ backend: 'cuda', usable_bytes: 23 * 1024 ** 3, memory: { total_gb: 32 } })).toBe(23 * 1024 ** 3)+ // What the apps are told with it: an NVIDIA card's free VRAM and all its VRAM, not the PC's RAM.+ expect(fitOf({ backend: 'cuda', usable_bytes: 22 * 1024 ** 3, memory: { total_gb: 64 }, gpus: [{ backend: 'cuda', memory_total_mb: 24576 }] }))+ .toEqual({ fitBytes: 22 * 1024 ** 3, fitMemory: 'vram', gpuMemoryBytes: 24 * 1024 ** 3 })+ expect(fitOf({ backend: 'metal', usable_bytes: 54 * 1024 ** 3, memory: { total_gb: 64 }, gpus: [{ backend: 'metal', memory_total_mb: 65536 }] }))+ .toEqual({ fitBytes: 32 * 1024 ** 3, fitMemory: 'unified' })+ expect(fitOf({ backend: 'cpu', usable_bytes: 20 * 1024 ** 3, memory: { total_gb: 32 } })).toEqual({ fitBytes: 16 * 1024 ** 3, fitMemory: 'ram' })+ expect(fitOf({})).toEqual({})// Two 24 GB cards in a 64 GB PC.expect(modelBudget({ backend: 'cuda', usable_bytes: 46 * 1024 ** 3, memory: { total_gb: 64 } })).toBe(46 * 1024 ** 3)})diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.tsindex 443b43798..52d506961 100644--- a/cli/src/lib/localModels.ts+++ b/cli/src/lib/localModels.ts@@ -89,7 +89,14 @@ export interface ModelOperation {progress?: number; error?: string; updatedAt: string}export interface LocalModelsSnapshot {+ /** [memoryBytes]: the machine's RAM — on an NVIDIA machine not what a model runs in. */models: LocalModel[]; memoryBytes?: number; hardware?: string+ /** What the catalog fits models to ([modelBudget]) and what memory that is: an NVIDIA card's free+ * VRAM (`vram`), an Apple Silicon Mac's unified memory, or system RAM where there is no usable+ * GPU. The apps say VRAM, and judge "fits" against it, where a model runs on the card. */+ fitBytes?: number; fitMemory?: 'vram' | 'unified' | 'ram'+ /** An NVIDIA machine's cards' VRAM, all of it (grid's `gpus[].memory_total_mb`). */+ gpuMemoryBytes?: number/** A sentence to show BESIDE the list — never `error`. On this protocol an `error` field fails* the whole request: the app's RPC layer drops the reply and keeps nothing of it, so a list sent* with a warning in `error` arrived as no list at all and the Local tab read 0. */@@ -213,6 +220,16 @@ export function modelBudget(device: Record<string, any>): number | undefined {return Math.min(usable, Math.floor(total * GiB * MODEL_MEMORY_SHARE))}+/** What the catalog fits to on [device] and what memory that is ([LocalModelsSnapshot.fitBytes]). */+export function fitOf(device: Record<string, any>): Pick<LocalModelsSnapshot, 'fitBytes' | 'fitMemory' | 'gpuMemoryBytes'> {+ const fit = modelBudget(device)+ const backend = str(device.backend)+ const fitMemory = backend === 'cuda' ? 'vram' as const : backend === 'metal' ? 'unified' as const : backend ? 'ram' as const : undefined+ const cards = backend === 'cuda' && Array.isArray(device.gpus) ? device.gpus : []+ const gpuMemoryBytes = cards.reduce((sum: number, gpu: unknown) => sum + (num(obj(gpu).memory_total_mb) ?? 0), 0) * 1024 ** 2+ return { ...(fit === undefined ? {} : { fitBytes: fit }), ...(fitMemory ? { fitMemory } : {}), ...(gpuMemoryBytes > 0 ? { gpuMemoryBytes } : {}) }+}+/** Catalog models that are not for running an agent on: safety classifiers such as gpt-oss-safeguard* or Llama Guard, which answer "safe"/"unsafe" rather than code. */const NOT_FOR_AGENTS = /(?:^|[-_/])(?:safeguard|guard|shieldgemma)(?:[-_.]|$)/i@@ -735,6 +752,7 @@ export class LocalModels {const freeDiskBytes = await statfs(join(this.home, 'models')).catch(() => statfs(this.home)).then(disk => disk.bavail * disk.bsize, () => undefined)const value: LocalModelsSnapshot = { models: withoutCopiesOfOwned(models), memoryBytes: num(obj(this.device.memory).total_gb) === undefined ? undefined : this.device.memory.total_gb * GiB,+ ...fitOf(this.device),hardware: str(obj(this.device.machine).model || obj(this.device.cpu).brand), notice: inventoryError || this.catalogError,observedAt: new Date().toISOString(), busy: !!this.active, supportsDownload: true,...(freeDiskBytes === undefined ? {} : { freeDiskBytes }) }diff --git a/desktop/lib/models/model_manager_controller.dart b/desktop/lib/models/model_manager_controller.dartindex 6bbabf905..8a703edf6 100644--- a/desktop/lib/models/model_manager_controller.dart+++ b/desktop/lib/models/model_manager_controller.dart@@ -65,8 +65,31 @@ class ModelManagerController extends ChangeNotifier {String? _dismissedReadyId;GridModels? models;List<LocalModel> localModels = const [];++ /// The machine's RAM. On an NVIDIA machine a model runs in the card, not in this: see [fitsInVram].double? memoryBytes;+ /// What the catalog fits models to (`fitBytes`) — on an NVIDIA machine the VRAM free on its card —+ /// and that card's VRAM in all (`gpuMemoryBytes`); null when the daemon did not say.+ double? fitBytes, gpuMemoryBytes;++ /// Models run in an NVIDIA card's VRAM here (`fitMemory: vram`): the figures to show are the card's,+ /// and "fits" is judged against [fitBytes], not the machine's RAM.+ bool fitsInVram = false;++ /// The figure a heading names this machine's memory by, and what follows it: an NVIDIA card's VRAM+ /// (`VRAM`, or `VRAM free` where only what is free on it is known), else the machine's memory+ /// (`unit` empty).+ ({double bytes, String unit})? get headingMemory => fitsInVram+ ? gpuMemoryBytes != null+ ? (bytes: gpuMemoryBytes!, unit: 'VRAM')+ : fitBytes != null+ ? (bytes: fitBytes!, unit: 'VRAM free')+ : null+ : memoryBytes != null+ ? (bytes: memoryBytes!, unit: '')+ : null;+/// Free space where downloads land, or null when the daemon did not say.double? freeDiskBytes;String? hardware, error, _managerError;@@ -225,6 +248,9 @@ class ModelManagerController extends ChangeNotifier {models = null;localModels = const [];memoryBytes = null;+ fitBytes = null;+ gpuMemoryBytes = null;+ fitsInVram = false;freeDiskBytes = null;hardware = null;error = null;@@ -579,10 +605,14 @@ class ModelManagerController extends ChangeNotifier {.map(LocalModel.fromJson).where((m) => m.id.isNotEmpty).toList();- final memory = answer['memoryBytes'];- memoryBytes = memory is num && memory.isFinite && memory > 0- ? memory.toDouble()+ double? positive(Object? value) =>+ value is num && value.isFinite && value > 0+ ? value.toDouble(): null;+ memoryBytes = positive(answer['memoryBytes']);+ fitBytes = positive(answer['fitBytes']);+ gpuMemoryBytes = positive(answer['gpuMemoryBytes']);+ fitsInVram = answer['fitMemory'] == 'vram';final disk = answer['freeDiskBytes'];freeDiskBytes = disk is num && disk.isFinite && disk >= 0? disk.toDouble()diff --git a/desktop/lib/models/model_search_catalog.dart b/desktop/lib/models/model_search_catalog.dartindex b13dbcd1a..1d8873f2f 100644--- a/desktop/lib/models/model_search_catalog.dart+++ b/desktop/lib/models/model_search_catalog.dart@@ -492,10 +492,15 @@ class ModelSearchCatalog extends ChangeNotifier {if (owners.length != 1) return ModelSearchSection.catalog.label;final owner = owners.single;final machine = owner.machine;- final memory = owner.memoryBytes;+ // (An NVIDIA machine's models run in its card: its VRAM, not the machine's RAM.)+ final memory = owner.headingMemory;return ['Get for ${machine == null || machine.isLocalMachine ? thisComputerName() : machine.machine.displayName}',- if (memory != null) gigabytesLabel(memory),+ if (memory != null)+ [+ gigabytesLabel(memory.bytes),+ if (memory.unit.isNotEmpty) memory.unit,+ ].join(' '),].join(' · ');}diff --git a/desktop/lib/models/models_panel.dart b/desktop/lib/models/models_panel.dartindex 8bea64b3a..80af4c202 100644--- a/desktop/lib/models/models_panel.dart+++ b/desktop/lib/models/models_panel.dart@@ -323,8 +323,8 @@ class _ModelsPanelState extends State<ModelsPanel> with SectionWakes {child: Text(['This computer',- if (controller.memoryBytes case final memory?)- '${_size(memory)} memory',+ if (controller.headingMemory case final memory?)+ '${_size(memory.bytes)} ${memory.unit.isEmpty ? 'memory' : memory.unit}',].join(' · '),style: AppType.monoMeta(color: AppPalette.textSecondary,diff --git a/desktop/lib/widgets/swarm_resource_preview.dart b/desktop/lib/widgets/swarm_resource_preview.dartindex b920dbf00..7cb5da0cc 100644--- a/desktop/lib/widgets/swarm_resource_preview.dart+++ b/desktop/lib/widgets/swarm_resource_preview.dart@@ -1397,7 +1397,18 @@ class _SwarmResourcePreviewState extends State<SwarmResourcePreview> {'${gigabytesLabel(owner.freeDiskBytes!)} free',].join(' · '),),- if ((local.sizeBytes, owner.memoryBytes) case (+ // An NVIDIA machine's against the VRAM free on its card, which the catalog fits to.+ if (owner.fitsInVram) ...[+ if ((local.sizeBytes, owner.fitBytes) case (+ final size?,+ final free?,+ ))+ labelValue(+ 'VRAM',+ '${size <= free ? 'fits' : 'needs ${gigabytesLabel(size)}'} · '+ 'its GPU has ${gigabytesLabel(free)} free',+ ),+ ] else if ((local.sizeBytes, owner.memoryBytes) case (final size?,final memory?,))diff --git a/desktop/test/model_search_catalog_test.dart b/desktop/test/model_search_catalog_test.dartindex d35d7be0a..04ca244a6 100644--- a/desktop/test/model_search_catalog_test.dart+++ b/desktop/test/model_search_catalog_test.dart@@ -130,6 +130,47 @@ void main() {},);+ test('an NVIDIA machine is told its VRAM, not its RAM', () async {+ final grouped = ModelSearchCatalog(+ app.modelManager,+ _WithSubscriptions(),+ pollHosts: false,+ );+ try {+ Map<String, Object?> answer({bool vram = true, bool total = true}) => {+ 'memoryBytes': 64 * 1024 * 1024 * 1024,+ if (vram) ...{+ 'fitMemory': 'vram',+ 'fitBytes': 22 * 1024 * 1024 * 1024,+ if (total) 'gpuMemoryBytes': 24 * 1024 * 1024 * 1024,+ },+ 'models': [+ {'id': 'catalog-A', 'name': 'AAA catalog'},+ ],+ };+ app.localInventory = answer(vram: false);+ await app.modelManager.refresh();+ expect(app.modelManager.fitsInVram, isFalse);+ expect(grouped.catalogHeading, 'Get for ${thisComputerName()} · 64 GB');+ app.localInventory = answer();+ await app.modelManager.refresh();+ expect(app.modelManager.fitsInVram, isTrue);+ expect(app.modelManager.fitBytes, 22 * 1024 * 1024 * 1024);+ expect(+ grouped.catalogHeading,+ 'Get for ${thisComputerName()} · 24 GB VRAM',+ );+ app.localInventory = answer(total: false);+ await app.modelManager.refresh();+ expect(+ grouped.catalogHeading,+ 'Get for ${thisComputerName()} · 22 GB VRAM free',+ );+ } finally {+ grouped.dispose();+ }+ });+test('sections separate your models from downloads, and fold the catalog past five', () async {final usage = _WithSubscriptions();final grouped = ModelSearchCatalog(diff --git a/tui/src/models.rs b/tui/src/models.rsindex 6e58ff5f0..e9766923a 100644--- a/tui/src/models.rs+++ b/tui/src/models.rs@@ -109,6 +109,9 @@ impl LocalModel {/// shared models are one Set up row (the desktop's); `grid_setup_error`: why the last Set up failed.#[derive(Clone, Debug, Default, PartialEq)]pub struct Snapshot { pub models: Vec<LocalModel>, pub memory: Option<f64>, pub free_disk: Option<f64>, pub hardware: Option<String>, pub notice: Option<String>, pub busy: bool, pub supports_download: bool,+ /// What the catalog fits models to, and whether that is an NVIDIA card's VRAM (`fitMemory: vram`)+ /// with its cards' VRAM in all: [memory] is the machine's RAM, not what such a model runs in.+ pub fit: Option<f64>, pub fit_vram: bool, pub gpu_memory: Option<f64>,pub grid_setup_needed: bool, pub grid_setup_error: Option<String> }/// None when there is no list — a daemon too old for it, or a reply that failed.@@ -119,6 +122,7 @@ pub fn parse_local(reply: &Value) -> Option<Snapshot> {// A sentence beside the list, never a failure (an `error` fails the whole request).notice: text(&reply["notice"]), busy: reply["busy"] == true, supports_download: reply["supportsDownload"] == true,grid_setup_needed: reply["gridSetupNeeded"] == true, grid_setup_error: text(&reply["gridSetupError"]),+ fit: num(&reply["fitBytes"]).filter(|m| *m > 0.0), fit_vram: reply["fitMemory"] == "vram", gpu_memory: num(&reply["gpuMemoryBytes"]).filter(|m| *m > 0.0),})}@@ -646,7 +650,16 @@ fn capital(s: &str) -> String { let mut c = s.chars(); c.next().map(|f| f.to_upp/// `Get for this Mac · 64 GB`: the downloads, and the memory they were chosen for.pub fn catalog_heading(snap: Option<&Snapshot>) -> String {let mut h = format!("Get for {}", this_computer());- if let Some(m) = snap.and_then(|s| s.memory) { h.push_str(&format!(" · {}", gb(m))) }+ // (An NVIDIA machine's models run in its card: its VRAM, not the machine's RAM.)+ match snap {+ Some(s) if s.fit_vram => match (s.gpu_memory, s.fit) {+ (Some(all), _) => h.push_str(&format!(" · {} VRAM", gb(all))),+ (None, Some(free)) => h.push_str(&format!(" · {} VRAM free", gb(free))),+ _ => {}+ },+ Some(s) => if let Some(m) = s.memory { h.push_str(&format!(" · {}", gb(m))) },+ None => {}+ }h}@@ -2000,7 +2013,11 @@ pub fn preview(app: &App, id: &str) -> Vec<Line<'static>> {if let Some(size) = m.size {let free = snap.and_then(|s| s.free_disk).filter(|_| !m.downloaded()).map(|d| format!(" · {} free", gb(d))).unwrap_or_default();out.push(kv(if m.downloaded() { "Size" } else { "Download" }, format!("{}{free}", gb(size))));- if let Some(mem) = snap.and_then(|s| s.memory) { out.push(kv("Memory", format!("{} · {here} has {}", if size <= mem { "fits".to_string() } else { format!("needs {}", gb(size)) }, gb(mem)))) }+ // (An NVIDIA machine's against the VRAM free on its card, which the catalog fits to.)+ match snap {+ Some(s) if s.fit_vram => if let Some(free) = s.fit { out.push(kv("VRAM", format!("{} · its GPU has {} free", if size <= free { "fits".to_string() } else { format!("needs {}", gb(size)) }, gb(free)))) },+ _ => if let Some(mem) = snap.and_then(|s| s.memory) { out.push(kv("Memory", format!("{} · {here} has {}", if size <= mem { "fits".to_string() } else { format!("needs {}", gb(size)) }, gb(mem)))) },+ }}match (m.running().then_some(m.tok_s).flatten(), m.est_tok_s) {(Some(s), _) => out.push(kv("Speed", format!("{s:.1} tok/s"))),@@ -3402,4 +3419,29 @@ mod tests {assert_eq!(url_problem("https://api.example.com/v1"), None);assert_eq!(url_problem("http://localhost:8080/v1"), None);}++ /// An NVIDIA machine's models run in its card: the heading says its VRAM and "fits" is judged+ /// against the VRAM free on it — not the machine's RAM, which a Mac's unified memory still shows.+ #[test]+ fn an_nvidia_machine_is_told_its_vram_not_its_ram() {+ let mut app = app();+ let mut reply = local_reply(qwen("available", Value::Null), false);+ on_local(&mut app, "local", Ok(reply.clone()));+ assert_eq!(catalog_heading(app.models_view.local.get("local")), format!("Get for {} · 64 GB", this_computer()));+ assert!(text(&preview(&app, "mv:local:qwen-35b")).contains("Memory fits"), "{}", text(&preview(&app, "mv:local:qwen-35b")));++ reply["fitBytes"] = json!(14.0 * GIB);+ reply["fitMemory"] = json!("vram");+ reply["gpuMemoryBytes"] = json!(16.0 * GIB);+ on_local(&mut app, "local", Ok(reply.clone()));+ assert_eq!(catalog_heading(app.models_view.local.get("local")), format!("Get for {} · 16 GB VRAM", this_computer()));+ let panel = text(&preview(&app, "mv:local:qwen-35b"));+ assert!(panel.contains("VRAM needs 20 GB · its GPU has 14 GB free"), "{panel}");+ assert!(!panel.contains("has 64 GB"), "{panel}");++ // A daemon that names no card's total: what is free on it.+ reply.as_object_mut().unwrap().remove("gpuMemoryBytes");+ on_local(&mut app, "local", Ok(reply));+ assert_eq!(catalog_heading(app.models_view.local.get("local")), format!("Get for {} · 14 GB VRAM free", this_computer()));+ }}