The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat(local models): offer what fits beside everything else, one pick per model
bitcoinbuilderfeb12c55b7b5
diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.tsindex 59e96c648..c2a8ea109 100644--- a/cli/src/lib/localModels.spec.ts+++ b/cli/src/lib/localModels.spec.ts@@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'import { createHash } from 'node:crypto'import { rmSync, writeFileSync } from 'node:fs'import { spawnSync } from 'node:child_process'-import { baseModel, compatibleModels, contextLadder, LocalModels, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'+import { baseModel, compatibleModels, contextLadder, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'import { GridFleetRpc, type GridFleetResult } from './gridFleetRpc.js'import { encryptDownFrame, encryptRpcResult } from './e2ee/applicationFrames.js'@@ -439,46 +439,69 @@ describe('local model discovery and lifecycle', () => {expect((await service.list('home')).models.map(m => [m.name, m.recommended])).toEqual([['Fast', true], ['Large', false]])})- it('offers what a coding agent can work with first: fast enough, then bigger, then the catalog order', async () => {- // [id, estimated tok/s, billions of parameters] in the catalog's own order.- const rows: [string, number | undefined, number | undefined][] = [- ['org/Dense-70B-GGUF', 8, 70], // big, and slow on this machine+ it('offers what a coding agent can work with first: a faithful quant, fast enough, then bigger, then the catalog order', async () => {+ // [id, estimated tok/s, billions of parameters, quant] in the catalog's own order.+ const rows: [string, number | undefined, number | undefined, string?][] = [+ ['org/Dense-70B-GGUF', 8, 70], // big, and slow on this machine['org/Qwen-35B-A3B-GGUF', 21, 35],- ['org/Qwen-35B-A3B-MTP-GGUF', 21, 35], // a variant of the one above- ['org/Tiny-4B-GGUF', 17, 4],- ['org/Other-35B-A3B-GGUF', 22, 35], // as big as Qwen, later in the catalog- ['org/Coder-Next-GGUF', 25, undefined], // no count: read off 48 GB of MXFP4 weights- ['org/Unknown-9B-GGUF', undefined, 9], // no estimate: not known to be fast- ['org/gpt-oss-safeguard-20b-GGUF', 24, 20], // a safety classifier, not a coding model+ ['org/Qwen-35B-A3B-MTP-GGUF', 21, 35], // a variant of the one above+ ['org/Tiny-4B-GGUF', 27, 4],+ ['org/Retrained-35B-A3B-GGUF', 22, 35], // a fine-tune: the same MoE shape+ ['org/Renamed-35B-GGUF', 21, 35], // a fine-tune: the same size, speed and weights+ ['org/Coder-Next-GGUF', 25, undefined, 'MXFP4_MOE'], // no count: read off 48 GB of MXFP4 weights+ ['org/Unknown-9B-GGUF', undefined, 9], // no estimate: not known to be fast+ ['org/gpt-oss-safeguard-20b-GGUF', 24, 20], // a safety classifier, not a coding model+ ['org/Giant-122B-A10B-GGUF', 30, 122, 'Q2_K_XL'], // biggest and fast, at 2 bits a weight]- catalogCards = rows.map(([id, estimate, params]) => {+ catalogCards = rows.map(([id, estimate, params, quant]) => {const row: Record<string, any> = card(id)- const file = `${id.split('/')[1]}.gguf`- row.versions[0].pull_spec = `${id}:${file}`+ row.versions[0].pull_spec = `${id}:${id.split('/')[1]}.gguf`if (estimate !== undefined) row.fit = { ...row.fit, est_tok_s: estimate }if (params !== undefined) row.params_b = params- if (id.includes('Coder-Next')) {- row.fit = { ...row.fit, version: 'MXFP4_MOE' }- row.versions[0] = { ...row.versions[0], version: 'MXFP4_MOE', size_bytes: 48e9 }+ if (quant) {+ row.fit = { ...row.fit, version: quant }+ row.versions[0] = { ...row.versions[0], version: quant, ...(quant === 'MXFP4_MOE' ? { size_bytes: 48e9 } : {}) }}return row as ReturnType<typeof card>})const snapshot = await service.list('home')expect(snapshot.models.map(model => model.name)).toEqual([- 'Coder-Next', 'Qwen-35B-A3B', 'Other-35B-A3B', 'Tiny-4B',+ 'Coder-Next', 'Qwen-35B-A3B', 'Tiny-4B','Dense-70B', 'Unknown-9B',- 'Qwen-35B-A3B-MTP',+ 'Giant-122B-A10B',+ 'Qwen-35B-A3B-MTP', 'Retrained-35B-A3B', 'Renamed-35B',])expect(snapshot.models[0].recommended).toBe(true)})it('ranks without regard to where a model was in the catalog once speed or size tells them apart', () => {- const at = (name: string, estTokS: number, paramsB: number) =>- ({ id: `org/${name}`, name, pull: `org/${name}:${name}.gguf`, file: `${name}.gguf`, files: [`${name}.gguf`], size: 1, quant: 'Q4', estTokS, paramsB })- expect(rankForCoding([at('Slow-Big', 5, 70), at('Fast-Small', 20, 4), at('Fast-Big', 20, 30)]).map(c => c.name))+ const at = (name: string, estTokS: number, paramsB: number, size = 1) =>+ ({ id: `org/${name}`, name, pull: `org/${name}:${name}.gguf`, file: `${name}.gguf`, files: [`${name}.gguf`], size, quant: 'Q4', estTokS, paramsB })+ expect(rankForCoding([at('Slow-Big', 5, 70), at('Fast-Small', 25, 4), at('Fast-Big', 25, 30)]).map(c => c.name)).toEqual(['Fast-Big', 'Fast-Small', 'Slow-Big'])// At exactly the floor a model counts as fast enough.- expect(rankForCoding([at('Slow', 14.9, 30), at('Enough', 15, 4)]).map(c => c.name)).toEqual(['Enough', 'Slow'])+ expect(rankForCoding([at('Slow', 19.9, 30), at('Enough', 20, 4)]).map(c => c.name)).toEqual(['Enough', 'Slow'])+ // Two models of one size are two models when their weights or speed differ.+ expect(rankForCoding([at('Moe-35B', 78, 35, 20e9), at('Dense-35B', 30, 35, 20e9), at('Other-35B', 78, 35, 24e9)]).map(c => c.name))+ .toEqual(['Moe-35B', 'Dense-35B', 'Other-35B'])+ })++ it('fits the catalog to half the machine, and never to more than grid says is free', () => {+ const device = (usable: number, total?: number, backend = 'metal') =>+ ({ backend, usable_bytes: usable * 1024 ** 3, ...(total ? { memory: { total_gb: total } } : {}) })+ expect(modelBudget(device(54, 64))).toBe(32 * 1024 ** 3)+ expect(modelBudget(device(20, 64))).toBe(20 * 1024 ** 3)+ expect(modelBudget(device(54))).toBe(54 * 1024 ** 3)+ // No GPU: the model lives in system RAM, shared like unified memory.+ expect(modelBudget(device(28, 32, 'cpu'))).toBe(16 * 1024 ** 3)+ expect(modelBudget({})).toBeUndefined()+ })++ it("gives the catalog all of an NVIDIA card's free VRAM, whatever the system RAM", () => {+ // A 24 GB card in a 32 GB PC: VRAM is not system RAM, and nothing else is waiting for it.+ expect(modelBudget({ backend: 'cuda', usable_bytes: 23 * 1024 ** 3, memory: { total_gb: 32 } })).toBe(23 * 1024 ** 3)+ // Two 24 GB cards in a 64 GB PC.+ expect(modelBudget({ backend: 'cuda', usable_bytes: 46 * 1024 ** 3, memory: { total_gb: 64 } })).toBe(46 * 1024 ** 3)})it('names one base model across its variants, and a fine-tune under its own name apart', () => {@@ -810,7 +833,8 @@ describe('local model discovery and lifecycle', () => {: original(args, output))await service.list('home')const sent = request.mock.calls.find(([url]) => String(url).includes('/catalog'))!- expect(JSON.parse(String(sent[1]?.body)).device).toEqual({ device_class: 'apple-silicon', usable_bytes: 54 * 1024 ** 3,+ // Half of its 64 GB, not the 54 GB grid reports free: the rest stays for everything else it runs.+ expect(JSON.parse(String(sent[1]?.body)).device).toEqual({ device_class: 'apple-silicon', usable_bytes: 32 * 1024 ** 3,backend: 'metal', mem_bandwidth_gbps: 400, compute_gflops: 15600 })})diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.tsindex 3ac54b03e..76ef2e6fd 100644--- a/cli/src/lib/localModels.ts+++ b/cli/src/lib/localModels.ts@@ -169,8 +169,30 @@ export function withoutCopiesOfOwned(models: LocalModel[]): LocalModel[] {}/** The speed a coding agent can work at: an agent waits on every token it writes, and below this the- * catalog's estimate for this machine reads as a model that crawls. */-export const USABLE_TOK_S = 15+ * catalog's estimate for this machine reads as a model that crawls. On a bandwidth-bound Mac this is+ * what separates an MoE (35-80 tok/s on an M1 Max) from a dense model of its size (9-17). */+export const USABLE_TOK_S = 20++/** The share of a machine's memory one model may fill — weights, a coding agent's context and the+ * engine's overhead together, as the catalog fits them. The rest stays for everything else the person+ * runs: on a 64 GB Mac, a 45 GB model with its context left the editor, the browser and the agents+ * themselves fighting over what remained. */+export const MODEL_MEMORY_SHARE = 0.5++/** The budget the catalog fits models to: what the machine can spare ([MODEL_MEMORY_SHARE] of its+ * memory), never more than grid says it has free. Undefined when grid reports neither.+ *+ * Only where a model shares its memory with everything else — an Apple Silicon Mac's unified memory,+ * or system RAM on a machine with no usable GPU. An NVIDIA card's VRAM is the model's own: grid+ * reports what is free on it now (total less what is already in use, the desktop's share included),+ * and all of that goes to the catalog, whose fit keeps its own 10% margin. Capping it at half the+ * system RAM would have held a 24 GB card in a 32 GB PC to 16 GB. */+export function modelBudget(device: Record<string, any>): number | undefined {+ const usable = num(device.usable_bytes), total = num(obj(device.memory).total_gb)+ if (usable === undefined) return undefined+ if (device.backend === 'cuda' || total === undefined) return usable+ return Math.min(usable, Math.floor(total * GiB * MODEL_MEMORY_SHARE))+}/** Catalog models that are not for running an agent on: safety classifiers such as gpt-oss-safeguard* or Llama Guard, which answer "safe"/"unsafe" rather than code. */@@ -202,26 +224,51 @@ export function baseModel(name: string): string {}}+/** A quant under this many bits per weight (Q2, IQ2, IQ1) trades too much of the model away: a+ * 2-bit 122B is not the better model than a 4-bit 35B beside it. */+const MIN_FAITHFUL_BITS = 3++/** The shape a model's name gives an MoE, `35B-A3B` — shared by its fine-tunes under other names+ * (`Qwen-AgentWorld-35B-A3B` is Qwen3.6-35B-A3B retrained). Null for a name that gives none. */+function moeShape(name: string): string | null {+ const shape = /(?:^|[-_])(\d+(?:\.\d+)?)B-A(\d+(?:\.\d+)?)B(?:[-_.]|$)/i.exec(name)+ return shape ? `${shape[1]}b-a${shape[2]}b` : null+}++/** Two catalog rows with the same parameter count, the same file size and the same speed estimate are+ * one architecture: the estimate is the machine's bandwidth over the bytes a token reads, which only+ * the same geometry at the same quant repeats. That is how a fine-tune under a name of its own+ * (Ornith-1.0-35B beside Qwen3.6-35B-A3B: 20.6 GB, 78.2 tok/s both) reads as the model it is. */+function sameArchitecture(a: Candidate, b: Candidate): boolean {+ return Math.round(paramsOf(a)) === Math.round(paramsOf(b)) && Math.abs(a.size - b.size) <= 0.03 * b.size &&+ !!a.estTokS && !!b.estTokS && Math.abs(a.estTokS - b.estTokS) <= 0.02 * b.estTokS+}+/** The order the Get list offers a machine's downloads in. Every candidate already fits the machine- * with a coding agent's context ([compatibleModels]); among them:- * 1. fast enough to work with first — the catalog's estimate for this machine at [USABLE_TOK_S] or+ * with a coding agent's context, within its [MODEL_MEMORY_SHARE] ([compatibleModels]); among them:+ * 1. a faithful quant first — under [MIN_FAITHFUL_BITS] a model goes after every model at one;+ * 2. fast enough to work with first — the catalog's estimate for this machine at [USABLE_TOK_S] or* more. That is what preferring MoE models on a bandwidth-bound Mac comes to, read per machine, and* it lets dense models in on a machine fast enough for them;- * 2. then bigger first, the better model;- * 3. then the catalog's own order, which is popularity among what fits.- * And one version of each base model ([baseModel]) before any second one, so MTP, QAT and pruned- * copies of one model never fill the top of the list: they follow every other model, in this order. */+ * 3. then bigger first, the better model;+ * 4. then the catalog's own order, which is popularity among what fits.+ * And one version of each model before any second one, so its MTP, QAT and pruned variants+ * ([baseModel]), its fine-tunes sharing its MoE shape ([moeShape]) or its architecture+ * ([sameArchitecture]) never fill the top of the list: they follow every other model, in this order. */export function rankForCoding<T extends Candidate>(candidates: T[]): T[] {+ const faithful = (candidate: Candidate): number => bitsPerWeight(candidate.quant) >= MIN_FAITHFUL_BITS ? 1 : 0const fast = (candidate: Candidate): number => (candidate.estTokS ?? 0) >= USABLE_TOK_S ? 1 : 0const sorted = candidates.map((candidate, index) => ({ candidate, index })).sort((a, b) =>+ faithful(b.candidate) - faithful(a.candidate) ||fast(b.candidate) - fast(a.candidate) ||Math.round(paramsOf(b.candidate)) - Math.round(paramsOf(a.candidate)) ||a.index - b.index).map(({ candidate }) => candidate)const seen = new Set<string>(), first: T[] = [], again: T[] = []for (const candidate of sorted) {- const base = baseModel(candidate.name)- ;(seen.has(base) ? again : first).push(candidate)- seen.add(base)+ const names = [baseModel(candidate.name), moeShape(candidate.name)].filter((name): name is string => !!name)+ const repeat = names.some(name => seen.has(name)) || first.some(leader => sameArchitecture(candidate, leader))+ ;(repeat ? again : first).push(candidate)+ for (const name of names) seen.add(name)}return [...first, ...again]}@@ -286,10 +333,11 @@ export class LocalModels {// any Mac — an M1 Max's 400 GB/s read as 2.7x slower than it is, and the models whose prompt it// then judged too slow to process were not offered at all.const bandwidth = num(device.mem_bandwidth_gbps), compute = num(device.compute_gflops)+ const budget = modelBudget(device)const response = await this.request(url, {method: 'POST', headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },body: JSON.stringify({ browse: true, page: 1, page_size: 50, device: {- device_class: device.device_class, usable_bytes: device.usable_bytes, backend: device.backend,+ device_class: device.device_class, usable_bytes: budget ?? device.usable_bytes, backend: device.backend,...(bandwidth ? { mem_bandwidth_gbps: bandwidth } : {}), ...(compute ? { compute_gflops: compute } : {}),} }), signal: AbortSignal.timeout(20_000), redirect: 'error',})diff --git a/desktop/design/terminal-dialogs.md b/desktop/design/terminal-dialogs.mdindex ff63f134b..d6fec2c99 100644--- a/desktop/design/terminal-dialogs.md+++ b/desktop/design/terminal-dialogs.md@@ -141,11 +141,17 @@ the downloads, and **Shared with you**. Headers are plain muted text and nevertake selection. Put **[ Add ]** within APIs. Your models puts downloaded modelsfirst, followed by models served on the user's machines. The downloads areheaded with the machine they are for and its memory (`Get for this Mac · 64 GB`),-and a machine with nothing to download has no such heading. They are ordered-for a coding agent on that machine: the ones the catalog estimates at 15 tok/s-or more first, then bigger models first, then the catalog's own (popularity)-order, with one version of each base model before any MTP, QAT or pruned-variant; safety classifiers are never offered. The list shows the first five;+and a machine with nothing to download has no such heading. Where a model+shares memory with everything else (a Mac's unified memory, or system RAM with+no GPU), only models that fit half of it with a coding agent's 64K context are+offered, and the other half stays for everything else it runs; an NVIDIA+card's free VRAM is the model's own, all of it. They are ordered for a+coding agent on that machine: quants of 3 bits or more first, then the ones the+catalog estimates at 20 tok/s or more, then bigger models first, then the+catalog's own (popularity) order — with one version of each model before any+second: its MTP, QAT or pruned variants, and fine-tunes sharing its MoE shape+(`35B-A3B`) or its size and speed. Safety classifiers are never offered. The+list shows the first five;the rest stay behind **[ More models (N) ]**, where Enterexpands them in place and selects the first catalog row, and **[ Show fewer ]**collapses them. The catalog never offers another quantization of a model thediff --git a/desktop/design/terminal-review-checklist.md b/desktop/design/terminal-review-checklist.mdindex e8036c0e3..ca785ff26 100644--- a/desktop/design/terminal-review-checklist.md+++ b/desktop/design/terminal-review-checklist.md@@ -194,8 +194,9 @@ Escape or Cmd-Shift-J closes the experimental prompt.the running sessions. Cmd-I opens Models in Cmd-P with `:` already entered.Models have Subscriptions, APIs, Your models, the downloads (headed`Get for this Mac · 64 GB`) and Shared with you sections, with downloaded- models first in Your models. The top five downloads are listed — fast- enough (~15 tok/s or more) first, then bigger, one version per model; the rest+ models first in Your models. Downloads fit half of shared memory (all of+ an NVIDIA card's free VRAM) with a 64K context; the top five are listed — 3-bit quants or better, fast enough+ (~20 tok/s or more) first, then bigger, one version per model; the restappear when expanding [ More models (N) ] or searching explicitly;[ Show fewer ] collapses them. No download is another quantization of amodel already there. Rows line up size and speed columns; usable rows say