The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(models): a stalled download ends, a download can be cancelled, and a thinking model starts
bitcoinbuilder7c571771fd62
diff --git a/cli/src/lib/localModels.integration.spec.ts b/cli/src/lib/localModels.integration.spec.tsindex 6fb4a1508..8195f25d3 100644--- a/cli/src/lib/localModels.integration.spec.ts+++ b/cli/src/lib/localModels.integration.spec.ts@@ -97,3 +97,48 @@ else process.exitCode = 2;await rm(root, { recursive: true, force: true })}}, 20_000)++it('cancels a download that hangs: the real grid process ends, and what it wrote goes', async () => {+ const root = await mkdtemp(join(tmpdir(), 'models-cancel-'))+ const gridHome = join(root, 'grid-home'), stateDir = join(root, 'state')+ const modelId = 'fixture/Tiny-GGUF'+ const server = createServer(async (req, res) => {+ for await (const _ of req) { /* drain */ }+ res.setHeader('content-type', 'application/json')+ res.end(JSON.stringify({ models: [{ repo_id: modelId, runnable: true, task: 'text-generation', format: 'GGUF',+ fit: { version: 'Q4', ctx: 131072 }, versions: [{ version: 'Q4', size_bytes: 64, pull_spec: `${modelId}:tiny.gguf` }] }] }))+ })+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))+ const base = `http://127.0.0.1:${(server.address() as { port: number }).port}`+ let service: LocalModels | undefined+ try {+ await mkdir(join(gridHome, 'models'), { recursive: true })+ await writeFile(join(gridHome, 'credentials.toml'), `session_token = 'catalog-fixture'\napi_url = '${base}'\n`)+ const executable = join(root, 'grid.mjs')+ // `grid pull` as it hung when its connection stopped sending: a `.part` written, then nothing, forever.+ await writeFile(executable, `#!${process.execPath}+import { writeFileSync } from 'node:fs';+import { join } from 'node:path';+const home = process.env.GRID_HOME, args = process.argv.slice(2);+if (args[0] === 'device-info') console.log(JSON.stringify({ usable_bytes: 16 * 1024 ** 3, device_class: 'cpu', backend: 'cpu', memory: { total_gb: 16 } }));+else if (args.includes('ls')) console.log(JSON.stringify([{ grid: 'home', id: 'fixture-grid' }]));+else if (args[0] === 'pull') { writeFileSync(join(home, 'models', 'tiny.gguf.part'), Buffer.alloc(24)); writeFileSync(join(home, 'pulling'), String(process.pid)); setInterval(() => {}, 1000); }+else process.exitCode = 2;+`, { mode: 0o700 })+ const inventory = async () => ({ state: 'awake' as const, status: 'running', nodes: [] })+ service = new LocalModels({ stateDir, processEnv: { GRID_HOME: gridHome, HARNESS_GRID_BIN: executable, PATH: root }, inventory })+ await service.act('home', modelId, 'download')+ const pulling = join(gridHome, 'pulling')+ for (let i = 0; i < 100 && !existsSync(pulling); i++) await new Promise(resolve => setTimeout(resolve, 50))+ const pid = Number(await readFile(pulling, 'utf8'))+ await service.act('home', modelId, 'stop'); await service.settled()+ expect(() => process.kill(pid, 0)).toThrow()+ expect(existsSync(join(gridHome, 'models', 'tiny.gguf.part'))).toBe(false)+ expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'available', operation: { action: 'stop', phase: 'done' } })+ } finally {+ await service?.settled()+ server.closeAllConnections()+ await new Promise<void>(resolve => server.close(() => resolve()))+ await rm(root, { recursive: true, force: true })+ }+}, 20_000)diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.tsindex 8d2ee68a3..4a83b07a9 100644--- a/cli/src/lib/localModels.spec.ts+++ b/cli/src/lib/localModels.spec.ts@@ -3,7 +3,7 @@ import { mkdtemp, mkdir, writeFile, readFile, readdir, rm, stat, statfs, readlinimport { join } from 'node:path'import { tmpdir } from 'node:os'import { createHash } from 'node:crypto'-import { rmSync, writeFileSync } from 'node:fs'+import { existsSync, rmSync, writeFileSync } from 'node:fs'import { spawnSync } from 'node:child_process'import type { GpuProcess } from './gpuProcesses.js'import { baseModel, compatibleModels, contextLadder, fitOf, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'@@ -856,6 +856,21 @@ describe('local model discovery and lifecycle', () => {expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(false)})+ it('gives a model that thinks first room to finish, and starts it on its real reply', async () => {+ // As Muse-Glimmer-30B: 8 tokens are all thinking, its content empty; with room it answers ok.+ const original = request.getMockImplementation()!+ const asked: number[] = []+ request.mockImplementation(async (url, init) => {+ if (!String(url).includes(RELAY_CHAT)) return original(url, init)+ const tokens = JSON.parse(String(init?.body)).max_tokens as number+ asked.push(tokens)+ return response({ choices: [{ message: tokens < 512 ? { content: '', reasoning_content: 'Reply with the single word' } : { content: 'ok', reasoning_content: 'Probably just ok.' } }] })+ })+ await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()+ expect((await service.list('home')).models[0].operation).toMatchObject({ action: 'start', phase: 'done' })+ expect(asked).toEqual([8, 512])+ })+it.each(['reasoning only', 'http error', 'network error'])('requires a real test reply and bounds retries for %s', async scenario => {const original = request.getMockImplementation()!let attempted!: () => void@@ -1125,6 +1140,59 @@ describe('a run record its engine did not outlive', () => {})})+describe('a download that stops, or is cancelled', () => {+ /** A pull that sends nothing more, as grid's did when its connection stalled (no read timeout): it ends+ * only when it is aborted. */+ const hang = () => {+ const original = run.getMockImplementation()!+ run.mockImplementation(async (args: string[], output?: (s: string) => void, _timeout?: number, signal?: AbortSignal) => {+ if (args[0] !== 'pull') return original(args, output)+ calls.push(args)+ await writeFile(join(home, 'models', 'Small-Q4.gguf.part'), Buffer.alloc(24))+ output?.('36%')+ return new Promise<GridFleetResult>(resolve => signal?.addEventListener('abort', () => resolve({ ok: false, code: 124, stdout: '', stderr: '', error: 'interrupted' })))+ })+ }++ it('is not downloaded while a part of it is unfinished — its projector, say — and can be got again', async () => {+ await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))+ await writeFile(join(home, 'models', 'Small-Q4.mmproj.gguf.part'), Buffer.alloc(8))+ expect((await service.list('home', true)).models[0]).toMatchObject({ id: 'org/Small-GGUF', state: 'available', canStart: true })+ await rm(join(home, 'models', 'Small-Q4.mmproj.gguf.part'))+ expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'downloaded' })+ })++ it('that has stalled is ended, to be started again where it stopped', async () => {+ hang()+ service = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory, pullStallMs: 40, pullWatchMs: 5 })+ await service.act('home', 'org/Small-GGUF', 'download'); await service.settled()+ expect((await service.list('home', true)).models[0].operation).toMatchObject({ action: 'download', phase: 'failed', error: 'The download stalled. Start again to resume.' })+ // What it got is kept: a start again resumes from it.+ expect(existsSync(join(home, 'models', 'Small-Q4.gguf.part'))).toBe(true)+ })++ it('is cancelled by a stop, and what it downloaded goes: the row is one to get again, not a failure', async () => {+ hang()+ await writeFile(join(home, 'models', 'Small-Q4.mmproj.gguf.part'), Buffer.alloc(8))+ await writeFile(join(home, 'models', 'Other-Q4.gguf'), Buffer.alloc(8))+ const started = await service.act('home', 'org/Small-GGUF', 'download')+ await vi.waitFor(() => expect(calls.some(args => args[0] === 'pull')).toBe(true))+ const stop = await service.act('home', 'org/Small-GGUF', 'stop')+ expect(stop.operation?.id).toBe(started.operation?.id)+ await service.settled()+ expect((await readdir(join(home, 'models'))).sort()).toEqual(['Other-Q4.gguf'])+ const [model] = (await service.list('home', true)).models+ expect(model).toMatchObject({ state: 'available', operation: { action: 'stop', phase: 'done' } })+ expect(model.operation).not.toHaveProperty('error')+ // Another model's operation is not cancelled by a stop of this one.+ hang()+ await service.act('home', 'org/Small-GGUF', 'download')+ await vi.waitFor(() => expect(calls.filter(args => args[0] === 'pull')).toHaveLength(2))+ expect(await service.act('home', 'org/Other-GGUF', 'stop')).toEqual({ error: 'Wait for the current model operation to finish.' })+ await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()+ })+})+/** A pid that existed a moment ago and no longer does. */function deadPid(): number {return Number(spawnSync(process.execPath, ['-e', 'process.stdout.write(String(process.pid))'], { encoding: 'utf8' }).stdout)diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.tsindex 1dad1f9e1..c8acc43cb 100644--- a/cli/src/lib/localModels.ts+++ b/cli/src/lib/localModels.ts@@ -13,6 +13,7 @@ import { displayModelName } from './gridReader.js'import { readEnvExports } from './gridWake.js'import { MIN_CODING_CONTEXT } from './codingContext.js'import type { GpuProcess } from './gpuProcesses.js'+import { downloadedBytes, removeDownload, unfinishedDownload } from './modelFiles.js'import { APP_LABEL, AppStartError, GRID_LABEL, appContext, readAppRecords, writeAppRecords, type AppEngine, type AppEngineOps, type AppEngineRecord, type AppModel } from './appModels.js'import { osToken, type OsToken } from './osToken.js'@@ -154,7 +155,11 @@ interface Options {* one it takes the host name — so a model started here read `mac.lan` under "On your machines"* while Machines called the same computer `M2`. */machineName?: () => string | null | undefined- run?: (args: string[], output?: (chunk: string) => void, timeout?: number) => Promise<GridFleetResult>+ run?: (args: string[], output?: (chunk: string) => void, timeout?: number, signal?: AbortSignal) => Promise<GridFleetResult>+ /** How long a download's files may stop growing before it is called stalled, and how often they are+ * looked at ([PULL_STALL_MS], [PULL_WATCH_MS]); a test shortens both. */+ pullStallMs?: number+ pullWatchMs?: numberrequest?: typeof fetch/** Injected, never a `grid engines` call: that read carries the grid credential, and a signed-in read* of a sleeping grid WAKES it — once a tick, for as long as the panel is open. */@@ -176,6 +181,11 @@ interface Options {gpuProcesses?: () => Promise<GpuProcess[]>}+/** A download whose files have not grown in this long has stalled ([LocalModels.perform]): long enough for a+ * slow mirror's pauses, short of the 30 minutes a command may run. Looked at every [PULL_WATCH_MS]. */+const PULL_STALL_MS = 3 * 60_000+const PULL_WATCH_MS = 10_000+/** One concrete, machine-fitted version per model. Non-chat and unprobed offline* catalog rows are never called compatible. Split GGUF files stay one model.*@@ -347,6 +357,8 @@ export class LocalModels {private readonly processEnv: NodeJS.ProcessEnvprivate readonly home: stringprivate readonly run: NonNullable<Options['run']>+ private readonly pullStallMs: number+ private readonly pullWatchMs: numberprivate readonly request: typeof fetchprivate candidates: Candidate[] = []private device: Record<string, any> = {}@@ -358,7 +370,7 @@ export class LocalModels {private cached?: { at: number; grid: string; value: LocalModelsSnapshot }/** The models the last read of each grid found running — kept past `cached`, which any save clears. */private readonly runningAtLastRead = new Map<string, Set<string>>()- private active?: { grid: string; operation: ModelOperation; done: Promise<void> }+ private active?: { grid: string; operation: ModelOperation; done: Promise<void>; pull: AbortController; cancelled: boolean }private receipt?: Receiptprivate receiptScope?: stringprivate knownByGrid = new Map<string, Candidate[]>()@@ -375,9 +387,14 @@ export class LocalModels {this.processEnv = options.processEnv ?? process.envthis.home = dirname(gridCredentialsPath(this.processEnv))const rpc = new GridFleetRpc(this.processEnv)- this.run = options.run ?? ((args, output, timeout = 30_000) =>- rpc.run('local-models', randomUUID(), { args, timeoutMs: timeout, thinking: false }, output, 4 * 1024 * 1024))+ this.run = options.run ?? ((args, output, timeout = 30_000, signal) => {+ const id = randomUUID()+ signal?.addEventListener('abort', () => rpc.cancel('local-models', id), { once: true })+ return rpc.run('local-models', id, { args, timeoutMs: timeout, thinking: false }, output, 4 * 1024 * 1024)+ })this.request = options.request ?? fetch+ this.pullStallMs = options.pullStallMs ?? PULL_STALL_MS+ this.pullWatchMs = options.pullWatchMs ?? PULL_WATCH_MS}private async json(args: string[]): Promise<any> {@@ -474,6 +491,9 @@ export class LocalModels {private async downloaded(candidate: Candidate): Promise<boolean> {if (candidate.appPath) return stat(candidate.appPath).then(s => s.isFile() && s.size === candidate.size, () => false)+ // A `.part` beside it is a download that did not finish — its projector, say, while its weights did.+ // Read as downloaded, it could not be got again, and the rest of it was never fetched.+ if (await unfinishedDownload(join(this.home, 'models'), candidate.file)) return falseconst sizes = await Promise.all(candidate.files.map(file => stat(join(this.home, 'models', file)).then(s => s.isFile() ? s.size : 0).catch(() => 0)))return sizes.every(size => size > 0) && sizes.reduce((sum, size) => sum + size, 0) === candidate.size}@@ -937,11 +957,20 @@ export class LocalModels {* A different model waits: starts/stops must not race on the machine's budget. */async act(grid: string | null, modelId: unknown, action: 'download' | 'start' | 'stop'): Promise<{ operation?: ModelOperation; error?: string }> {if (!grid || typeof modelId !== 'string') return { error: 'The model is unavailable. Refresh and try again.' }+ // Stop on a model that is downloading cancels the download, and what it downloaded goes ([perform]).+ // Without it, a download that hung (its connection stalled after the disk filled) held the view on+ // "Downloading 36%" with nothing to press but Wait.+ const running = this.active+ if (action === 'stop' && running?.grid === grid && running.operation.modelId === modelId && running.operation.stage === 'downloading') {+ running.cancelled = true+ running.pull.abort()+ return { operation: running.operation }+ }if (this.active) return this.active.grid === grid && this.active.operation.modelId === modelId && this.active.operation.action === action? { operation: this.active.operation } : { error: 'Wait for the current model operation to finish.' }const operation: ModelOperation = { id: randomUUID(), modelId, action, stage: action === 'stop' ? 'stopping' : 'checking', phase: 'running', updatedAt: new Date().toISOString() }// Reserve before any await; independent RPCs may arrive on separate connections.- const active = { grid, operation, done: Promise.resolve() }+ const active = { grid, operation, done: Promise.resolve(), pull: new AbortController(), cancelled: false }this.active = activetry { await this.save(grid, operation) } catch { this.active = undefined; return { error: 'Setup could not be saved. Try again.' } }active.done = this.perform(grid, operation).catch(async error => {@@ -963,8 +992,8 @@ export class LocalModels {const change = async (stage: ModelOperation['stage']) => {operation.stage = stage; delete operation.progress; await this.save(grid, operation)}- const must = async (args: string[], message: string, output?: (chunk: string) => void) => {- if (!(await this.run(args, output, 30 * 60_000)).ok) throw new ModelError(message)+ const must = async (args: string[], message: string, output?: (chunk: string) => void, signal?: AbortSignal) => {+ if (!(await this.run(args, output, 30 * 60_000, signal)).ok) throw new ModelError(message)}if (operation.action !== 'stop') await this.loadCatalog()if (operation.action === 'start') {@@ -1022,14 +1051,36 @@ export class LocalModels {if (disk.bavail * disk.bsize < candidate.size + GiB) throw new ModelError('Free up disk space, then start again.')await change('downloading')let last = 0, progressWrites = Promise.resolve()- await must(['pull', candidate.pull], 'The download stopped. Start again to resume.', chunk => {- const matches = [...chunk.matchAll(/(\d+(?:\.\d+)?)\s*%/g)]- const percent = matches.length ? Number(matches.at(-1)![1]) : NaN- if (Number.isFinite(percent) && percent >= 0 && percent <= 100 && Date.now() - last > 500) {- last = Date.now(); operation.progress = percent / 100- progressWrites = progressWrites.then(() => this.save(grid, operation)).catch(() => {})+ // The pull is watched by what it writes: grid reads its download with no read timeout, so a+ // connection that stops sending holds it forever — after a full disk on an NVIDIA machine, 30+ // minutes at 36% with nothing to press. Files that stop growing for [PULL_STALL_MS] end it.+ const active = this.active!, pull = active.pull+ let size = -1, grewAt = Date.now(), stalled = false+ const watch = setInterval(() => void downloadedBytes(join(this.home, 'models'), candidate.file).then(bytes => {+ if (bytes !== size) { size = bytes; grewAt = Date.now() } else if (Date.now() - grewAt >= this.pullStallMs) { stalled = true; pull.abort() }+ }), this.pullWatchMs)+ try {+ await must(['pull', candidate.pull], 'The download stopped. Start again to resume.', chunk => {+ const matches = [...chunk.matchAll(/(\d+(?:\.\d+)?)\s*%/g)]+ const percent = matches.length ? Number(matches.at(-1)![1]) : NaN+ if (Number.isFinite(percent) && percent >= 0 && percent <= 100 && Date.now() - last > 500) {+ last = Date.now(); operation.progress = percent / 100+ progressWrites = progressWrites.then(() => this.save(grid, operation)).catch(() => {})+ }+ }, pull.signal)+ } catch (error) {+ if (active.cancelled) {+ await progressWrites+ // (The model was not on this computer before: a download starts only for one [downloaded] says+ // is not, so what goes is only what this download wrote.)+ await removeDownload(join(this.home, 'models'), candidate.file)+ // A cancel is a stop that finished, not a failure: the row goes back to Get.+ operation.action = 'stop'; operation.stage = 'stopping'; operation.phase = 'done'; delete operation.progress+ await this.save(grid, operation)+ return}- })+ throw stalled ? new ModelError('The download stalled. Start again to resume.') : error+ } finally { clearInterval(watch) }await progressWritesif (!await this.downloaded(candidate)) throw new ModelError('The download is incomplete. Start again to resume.')}@@ -1190,15 +1241,22 @@ export class LocalModels {const { baseUrl, apiKey } = readEnvExports(info.stdout)if (!info.ok || !baseUrl || !apiKey) throw new ModelError('The model is starting, but could not be checked. Try again shortly.')const deadline = Date.now() + 180_000+ let maxTokens = 8do {try {const response = await this.request(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {method: 'POST', headers: { authorization: `Bearer ${apiKey}`, 'content-type': 'application/json' },- body: JSON.stringify({ model, messages: [{ role: 'user', content: 'Reply with the single word: ok' }], max_tokens: 8 }),- signal: AbortSignal.timeout(30_000), redirect: 'error',+ body: JSON.stringify({ model, messages: [{ role: 'user', content: 'Reply with the single word: ok' }], max_tokens: maxTokens }),+ signal: AbortSignal.timeout(maxTokens === THINKING_TEST_TOKENS ? 90_000 : 30_000), redirect: 'error',})const body = response.ok ? obj(await response.json()) : {}- if (/^\s*ok[.!]?\s*$/i.test(str(obj(rows(body.choices)[0]?.message).content))) return+ const message = obj(rows(body.choices)[0]?.message)+ if (/^\s*ok[.!]?\s*$/i.test(str(message.content))) return+ // A model that thinks first spends a reply this short thinking: Muse-Glimmer-30B (which thinks+ // whatever --reasoning-budget says) put all 8 tokens in its reasoning, its content stayed empty,+ // and every start of it failed after 3 minutes of "Testing" while it answered each try. Asked+ // again with room to finish thinking, it says ok. A real reply is still what passes.+ if (!str(message.content).trim() && str(message.reasoning_content || message.reasoning).trim()) maxTokens = THINKING_TEST_TOKENS} catch { /* registration and first warmup take time; status remains verifying */ }await new Promise(resolve => setTimeout(resolve, 1500))} while (Date.now() < deadline)@@ -1207,6 +1265,10 @@ export class LocalModels {}class ModelError extends Error {}+/** The reply a model that thinks first is given to finish thinking and answer the start's test in+ * ([LocalModels.verify]): Muse-Glimmer-30B needed under 200. */+const THINKING_TEST_TOKENS = 512+/** More names than any one engine advertises: a record past it is cut, never trusted to be small. */const MAX_RECORD_IDS = 64diff --git a/cli/src/lib/modelFiles.spec.ts b/cli/src/lib/modelFiles.spec.tsnew file mode 100644index 000000000..f757ad6f4--- /dev/null+++ b/cli/src/lib/modelFiles.spec.ts@@ -0,0 +1,44 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'+import { join } from 'node:path'+import { tmpdir } from 'node:os'++vi.mock('node:fs/promises', async original => {+ const fs = await original<typeof import('node:fs/promises')>()+ return { ...fs, stat: vi.fn(fs.stat) }+})+const { stat } = await import('node:fs/promises')+const { downloadFiles, downloadedBytes, removeDownload, unfinishedDownload } = await import('./modelFiles.js')++describe("a download's files in grid's models folder", () => {+ let dir: string+ beforeEach(async () => {+ dir = await mkdtemp(join(tmpdir(), 'model-files-'))+ for (const [name, size] of [['Glimmer-Q4.gguf', 16], ['Glimmer-Q4.mmproj.gguf.part', 4], ['Glimmer-Q4.notes.txt', 2], ['Other-Q4.gguf', 8]] as const) {+ await writeFile(join(dir, name), Buffer.alloc(size))+ }+ })+ afterEach(async () => { vi.mocked(stat).mockReset(); await rm(dir, { recursive: true, force: true }) })++ it('are its weights, its projector and either one\'s .part — nothing else of the folder', async () => {+ expect((await downloadFiles(dir, 'Glimmer-Q4.gguf')).sort()).toEqual(['Glimmer-Q4.gguf', 'Glimmer-Q4.mmproj.gguf.part'])+ expect(await downloadFiles(join(dir, 'missing'), 'Glimmer-Q4.gguf')).toEqual([])+ })++ it('are unfinished while a .part is there', async () => {+ expect(await unfinishedDownload(dir, 'Glimmer-Q4.gguf')).toBe(true)+ expect(await unfinishedDownload(dir, 'Other-Q4.gguf')).toBe(false)+ })++ it('are measured complete and unfinished alike, a file gone meanwhile counting nothing', async () => {+ expect(await downloadedBytes(dir, 'Glimmer-Q4.gguf')).toBe(20)+ const real = await vi.importActual<typeof import('node:fs/promises')>('node:fs/promises')+ vi.mocked(stat).mockImplementation(async path => String(path).endsWith('.part') ? Promise.reject(new Error('ENOENT')) : real.stat(path))+ expect(await downloadedBytes(dir, 'Glimmer-Q4.gguf')).toBe(16)+ })++ it('go when the download is cancelled, and nothing else does', async () => {+ await removeDownload(dir, 'Glimmer-Q4.gguf')+ expect((await readdir(dir)).sort()).toEqual(['Glimmer-Q4.notes.txt', 'Other-Q4.gguf'])+ })+})diff --git a/cli/src/lib/modelFiles.ts b/cli/src/lib/modelFiles.tsnew file mode 100644index 000000000..caf3aa5a1--- /dev/null+++ b/cli/src/lib/modelFiles.ts@@ -0,0 +1,27 @@+import { readdir, rm, stat } from 'node:fs/promises'+import { basename, join } from 'node:path'++/** The files a download of [file] writes in grid's models folder [dir]: the weights, the vision projector+ * `grid pull` fetches beside them (`<stem>.mmproj.gguf`), and either one's unfinished `.part`. */+export async function downloadFiles(dir: string, file: string): Promise<string[]> {+ const stem = basename(file).replace(/\.gguf$/i, '')+ const names = await readdir(dir).catch(() => [] as string[])+ return names.filter(name => name.startsWith(stem) && (/\.gguf$/i.test(name) || name.endsWith('.part')))+}++/** Whether a download of [file] did not finish: a `.part` of it, or of its projector, is still there. */+export async function unfinishedDownload(dir: string, file: string): Promise<boolean> {+ return (await downloadFiles(dir, file)).some(name => name.endsWith('.part'))+}++/** How much of a download of [file] is on disk now, complete and unfinished files alike — what a stall+ * watch measures a download by. A file gone between the listing and its size counts nothing. */+export async function downloadedBytes(dir: string, file: string): Promise<number> {+ const sizes = await Promise.all((await downloadFiles(dir, file)).map(name => stat(join(dir, name)).then(s => s.size, () => 0)))+ return sizes.reduce((sum, size) => sum + size, 0)+}++/** A cancelled download's files, gone. */+export async function removeDownload(dir: string, file: string): Promise<void> {+ for (const name of await downloadFiles(dir, file)) await rm(join(dir, name), { force: true })+}diff --git a/desktop/lib/models/model_manager_controller.dart b/desktop/lib/models/model_manager_controller.dartindex 559013f76..12f414568 100644--- a/desktop/lib/models/model_manager_controller.dart+++ b/desktop/lib/models/model_manager_controller.dart@@ -672,6 +672,53 @@ class ModelManagerController extends ChangeNotifier {Future<void> toggle(LocalModel model) =>control(model, model.canStop ? 'stop' : 'start');+ /// Whether [model]'s download is under way here: what [cancelDownload] stops. Every other action+ /// waits while it runs, so a download that hung (its connection stalled after a full disk) once left+ /// nothing to press.+ bool downloading(LocalModel model) {+ final operation = operationFor(model);+ return operation?.active == true && operation?.stage == 'downloading';+ }++ /// [model]'s download cancelled: the daemon's stop of a model that is downloading ends the download+ /// and deletes what it downloaded, and the model can be got again.+ Future<void> cancelDownload(LocalModel model) async {+ final owner = machine;+ if (owner == null ||+ owner.connectionStatus != ConnectionStatus.connected ||+ owner.needsLink ||+ owner.machine.isShared ||+ pendingId != null ||+ !downloading(model)) {+ return;+ }+ pendingId = model.id;+ _actionRevision++;+ error = null;+ _changed();+ try {+ final answer = await app.controlLocalModel(+ owner.machine.machineId,+ model.id,+ start: false,+ );+ if (!_current(owner)) return;+ error = answer['error'] as String?;+ } catch (_) {+ if (_current(owner)) {+ error =+ 'Checking the model operation. Your click will not be repeated.';+ }+ } finally {+ if (_current(owner)) {+ pendingId = null;+ _changed();+ await _refreshing;+ if (_current(owner)) await refresh();+ }+ }+ }+Future<void> control(LocalModel model, String action) async {final owner = machine;if (owner == null ||diff --git a/desktop/lib/models/models_panel.dart b/desktop/lib/models/models_panel.dartindex 80af4c202..82418ec00 100644--- a/desktop/lib/models/models_panel.dart+++ b/desktop/lib/models/models_panel.dart@@ -528,7 +528,11 @@ class _ModelsPanelState extends State<ModelsPanel> with SectionWakes {: model.downloaded? AppIcons.play: AppIcons.download;- final tooltip = active+ // A download under way can be cancelled — the one action open while it runs.+ final cancel = !pending && controller.downloading(model);+ final tooltip = cancel+ ? 'Cancel the download. What it downloaded is deleted.'+ : active? status: model.canStop? 'Pause ${model.name} and free memory. The download is kept.'@@ -603,7 +607,21 @@ class _ModelsPanelState extends State<ModelsPanel> with SectionWakes {child: SizedBox(width: 40,height: 40,- child: active+ child: cancel+ ? IconButton(+ key: ValueKey('model-cancel-${model.id}'),+ onPressed: () =>+ unawaited(controller.cancelDownload(model)),+ icon: Icon(+ AppIcons.close,+ size: 16,+ semanticLabel: 'Cancel downloading ${model.name}',+ ),+ style: IconButton.styleFrom(+ foregroundColor: AppPalette.textPrimary,+ ),+ )+ : active? Semantics(liveRegion: true,label: '$status ${model.name}',diff --git a/desktop/lib/widgets/swarm_resource_preview.dart b/desktop/lib/widgets/swarm_resource_preview.dartindex 7ac439c18..93d7e1cbe 100644--- a/desktop/lib/widgets/swarm_resource_preview.dart+++ b/desktop/lib/widgets/swarm_resource_preview.dart@@ -891,6 +891,26 @@ class _SwarmResourcePreviewState extends State<SwarmResourcePreview> {final local = model?.local;final api = model?.api;final owner = model?.controller ?? catalog.manager;+ if (local != null && owner.downloading(local)) {+ // A download under way: cancelling it is the one action open while it runs.+ return [+ _ResourceAction(+ 'Cancel download',+ !busy &&+ owner.pendingId == null &&+ owner.machine?.connectionStatus ==+ ConnectionStatus.connected+ ? () => unawaited(+ _run(() async {+ await owner.cancelDownload(local);+ return owner.error;+ }),+ )+ : null,+ command: 'picker.model_stop',+ ),+ ];+ }if (local != null) {final enabled =!busy &&diff --git a/desktop/test/models_panel_test.dart b/desktop/test/models_panel_test.dartindex 4924c4fdc..e8c9fac9b 100644--- a/desktop/test/models_panel_test.dart+++ b/desktop/test/models_panel_test.dart@@ -244,6 +244,60 @@ void main() {variant: TargetPlatformVariant.only(TargetPlatform.macOS),);+ testWidgets('a download under way can be cancelled, the one action open while it runs', (+ tester,+ ) async {+ tester.view.devicePixelRatio = 1;+ tester.view.physicalSize = const Size(700, 800);+ addTearDown(tester.view.reset);+ final app = ModelManagerTestApp(ModelManagerConnection());+ final controller = ModelManagerController(app, poll: false);+ final subscriptions = _SubscriptionRows();+ addTearDown(() {+ controller.dispose();+ subscriptions.dispose();+ app.dispose();+ });+ // Nothing downloading: Cancel asks the daemon nothing.+ await controller.refresh();+ await controller.cancelDownload(controller.localModels.first);+ expect(app.actions, isEmpty);+ app.localInventory = modelInventory(scenario: 'downloading');+ await controller.refresh();+ expect(controller.downloading(controller.localModels.first), isTrue);+ await tester.pumpWidget(+ MaterialApp(+ home: Scaffold(+ body: Center(+ child: SizedBox(+ width: 640,+ child: ModelsPanel(+ controller: controller,+ subscriptions: subscriptions,+ onClose: () {},+ onManage: () {},+ ),+ ),+ ),+ ),+ ),+ );+ await tester.tap(find.text('Local 5'));+ await tester.pump();+ // In place of the spinner: the download can be cancelled, though every other action waits.+ final cancel = find.byKey(const ValueKey('model-cancel-qwen'));+ expect(cancel, findsOneWidget);+ expect(find.bySemanticsLabel('Cancel downloading Qwen3.8-27B'), findsOneWidget);+ expect(+ tester.widget<IconButton>(find.byKey(const ValueKey('model-action-gemma'))).onPressed,+ isNull,+ );+ await tester.tap(cancel);+ await tester.pump();+ expect(app.actions, [(machine: 'm', model: 'qwen', start: false)]);+ await tester.pumpWidget(const SizedBox());+ });+testWidgets('download, play and pause reflect file and engine state', (tester,) async {@@ -398,7 +452,9 @@ void main() {expect(find.text('Downloading · 42%').hitTestable(), findsOneWidget);expect(app.actions, hasLength(1));expect(controller.busy, isTrue);- expect(find.byType(CircularProgressIndicator), findsOneWidget);+ // While it downloads, the download can be cancelled: a Cancel in place of the spinner.+ expect(find.byKey(const ValueKey('model-cancel-qwen')), findsOneWidget);+ expect(find.byType(CircularProgressIndicator), findsNothing);expect(tester.takeException(), isNull);await tester.pumpWidget(const SizedBox());});diff --git a/desktop/test/resource_picker_test.dart b/desktop/test/resource_picker_test.dartindex a2d16b606..0044e9c4e 100644--- a/desktop/test/resource_picker_test.dart+++ b/desktop/test/resource_picker_test.dart@@ -508,13 +508,22 @@ void main() {await capture(tester, 'model-manage-focused');await key(tester, LogicalKeyboardKey.space);expect(app.downloads, [(machine: 'm', model: 'qwen')]);- expect(button('picker.model_download').onPressed, isNull);+ // While it downloads, the one action is to cancel it.+ expect(+ find.byKey(const ValueKey('resource-action:picker.model_download')),+ findsNothing,+ );+ expect(find.text('Cancel download'), findsOneWidget);await key(tester, LogicalKeyboardKey.enter);expect(app.downloads, hasLength(1));expect(app.actions, isEmpty);expect(tester.widget<TextField>(field).focusNode!.hasFocus, isTrue);expect(controller.query, ':Qwen3.8-27B');expect(tester.takeException(), isNull);+ // (Enter again did not cancel it: only Cancel download does.)+ await tester.tap(find.text('Cancel download'));+ await tester.pump();+ expect(app.actions, [(machine: 'm', model: 'qwen', start: false)]);} finally {await tester.pumpWidget(const SizedBox());app.dispose();diff --git a/tui/src/models.rs b/tui/src/models.rsindex 501fc276f..eed77eb79 100644--- a/tui/src/models.rs+++ b/tui/src/models.rs@@ -55,6 +55,8 @@ impl Operation {Some(Operation { id: s("id")?, model: s("modelId")?, action, stage, phase, progress: v.get("progress").and_then(num).map(|p| p.clamp(0.0, 1.0)), error: s("error") })}pub fn active(&self) -> bool { self.phase == "running" }+ /// A download under way: what ^S cancels ([plan_stop]).+ pub fn downloading(&self) -> bool { self.active() && self.stage == "downloading" }pub fn failed(&self) -> bool { self.phase == "failed" }pub fn label(&self) -> &'static str {match self.stage.as_str() { "downloading" => "Downloading", "updating" => "Updating engine", "starting" => "Starting", "verifying" => "Testing", "stopping" => "Stopping", _ => "Checking" }@@ -1216,6 +1218,11 @@ pub fn plan_stop(app: &App, id: &str) -> Plan {}let Some(m) = id.strip_prefix("mv:local:").and_then(|model| local_model(app, model)) else { return Plan::Nothing };let f = facts(app, &m, None);+ // A download under way is cancelled, and what it downloaded deleted — the daemon's stop of it. A+ // download that hung had nothing to press but Wait.+ if f.op.as_ref().is_some_and(Operation::downloading) {+ return Plan::Arm(format!("^S again cancels downloading {} — what it downloaded is deleted", m.shown()), Box::new(Plan::Act { model: m.id.clone(), action: "stop" }));+ }if f.busy { return Plan::Say("Wait for the current model operation to finish.".into()) }// Run on the community grid alone: stopped there.if !m.can_stop && on_community(app, &m) {@@ -1815,7 +1822,11 @@ pub fn on_api_models(app: &mut App, id: &str, reply: Result<Value, RpcError>) {}/// A download, start or stop of this computer's [model].-fn act(app: &mut App, model: &str, action: &'static str) { act_on(app, model, action, false) }+fn act(app: &mut App, model: &str, action: &'static str) {+ // A stop of the model a Use is getting cancels that Use too: there is nothing left for it to start.+ if action == "stop" && app.models_view.using.as_ref().is_some_and(|u| u.model == model) { app.models_view.using = None }+ act_on(app, model, action, false)+}/// [act], on the community grid of this computer's OS when [community] (a start or stop; the daemon's/// `community`).@@ -2439,7 +2450,8 @@ pub fn preview(app: &App, id: &str) -> Vec<Line<'static>> {else if f.use_start { said.push(match &f.other { None => "Enter starts it and moves this harness onto it.".into(), Some(o) => format!("Enter stops {o}, starts this one, and moves this harness onto it.") }) }else if t.is_none() && m.downloaded() && !m.running() && m.can_start { said.push("Enter starts it.".into()) }}- if m.can_stop { said.push("^S stops it and frees its memory. The download is kept.".into()) }+ if f.op.as_ref().is_some_and(Operation::downloading) { said.push("^S cancels the download and deletes what it downloaded.".into()) }+ else if m.can_stop { said.push("^S stops it and frees its memory. The download is kept.".into()) }else if let Some(label) = &community { said.push(format!("^S stops it on {label}. The download is kept.")) }if !said.is_empty() { out.push(Line::raw("")); out.extend(said.into_iter().map(|s| Line::from(span(s, fg(theme::accent()))))) }if let Some(e) = f.op.as_ref().and_then(|o| o.error.clone()) { out.push(warn(e)) }@@ -4355,6 +4367,30 @@ mod tests {assert_eq!(app.models_view.using.as_ref().map(|u| (u.step, u.stops.clone())), Some((Step::Start, None)));}+ #[test]+ fn stop_on_a_downloading_model_cancels_the_download_and_the_use_that_began_it() {+ let mut app = loaded();+ let mut picker = Picker::new("Models", "");+ let id = "mv:local:qwen-35b";+ choose(&mut app, &mut picker, id, false);+ choose(&mut app, &mut picker, id, false);+ on_act(&mut app, "local", Ok(json!({ "operation": op("download", "downloading", "running", Some(0.1)) })));+ on_local(&mut app, "local", Ok(local_reply(qwen("available", op("download", "downloading", "running", Some(0.36))), true)));+ assert!(app.models_view.using.is_some());+ let p = text(&preview(&app, id));+ assert!(p.contains("^S cancels the download and deletes what it downloaded."), "{p}");+ // Not "Wait for the current model operation to finish", as it was while a download hung.+ assert!(matches!(plan_stop(&app, id), Plan::Arm(ref w, _) if w == "^S again cancels downloading Qwen3.6-35B-A3B — what it downloaded is deleted"));+ choose(&mut app, &mut picker, id, true);+ assert!(app.models_view.sent.iter().all(|s| s.1 != "grid_fleet_model_stop"), "the first ^S only asks");+ choose(&mut app, &mut picker, id, true);+ assert_eq!(last_sent(&app), ("local".into(), "grid_fleet_model_stop".into(), json!({ "modelId": "qwen-35b" })));+ assert!(app.models_view.using.is_none());+ // Another operation under way is still waited for.+ on_local(&mut app, "local", Ok(local_reply(qwen("downloaded", op("start", "starting", "running", None)), true)));+ assert!(matches!(plan_stop(&app, id), Plan::Say(ref w) if w == "Wait for the current model operation to finish."));+ }+/// A model a Grid setup left running, which the catalog does not list: grid serves it under its/// file's name, quant and all. It is shown without the quant (the Quant line says it — on Omarchy,/// only beside another download of the model), and a harness still moves onto it by the name grid serves.