Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(models): a stalled download ends, a download can be cancelled, and a thinking model starts

bitcoinbuilder7c571771fd62
diff --git a/cli/src/lib/localModels.integration.spec.ts b/cli/src/lib/localModels.integration.spec.ts
index 6fb4a1508..8195f25d3 100644
--- a/cli/src/lib/localModels.integration.spec.ts
+++ b/cli/src/lib/localModels.integration.spec.ts
@@ -97,3 +97,48 @@ else process.exitCode = 2;
await rm(root, { recursive: true, force: true })
}
}, 20_000)
+
+it('cancels a download that hangs: the real grid process ends, and what it wrote goes', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'models-cancel-'))
+ const gridHome = join(root, 'grid-home'), stateDir = join(root, 'state')
+ const modelId = 'fixture/Tiny-GGUF'
+ const server = createServer(async (req, res) => {
+ for await (const _ of req) { /* drain */ }
+ res.setHeader('content-type', 'application/json')
+ res.end(JSON.stringify({ models: [{ repo_id: modelId, runnable: true, task: 'text-generation', format: 'GGUF',
+ fit: { version: 'Q4', ctx: 131072 }, versions: [{ version: 'Q4', size_bytes: 64, pull_spec: `${modelId}:tiny.gguf` }] }] }))
+ })
+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))
+ const base = `http://127.0.0.1:${(server.address() as { port: number }).port}`
+ let service: LocalModels | undefined
+ try {
+ await mkdir(join(gridHome, 'models'), { recursive: true })
+ await writeFile(join(gridHome, 'credentials.toml'), `session_token = 'catalog-fixture'\napi_url = '${base}'\n`)
+ const executable = join(root, 'grid.mjs')
+ // `grid pull` as it hung when its connection stopped sending: a `.part` written, then nothing, forever.
+ await writeFile(executable, `#!${process.execPath}
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+const home = process.env.GRID_HOME, args = process.argv.slice(2);
+if (args[0] === 'device-info') console.log(JSON.stringify({ usable_bytes: 16 * 1024 ** 3, device_class: 'cpu', backend: 'cpu', memory: { total_gb: 16 } }));
+else if (args.includes('ls')) console.log(JSON.stringify([{ grid: 'home', id: 'fixture-grid' }]));
+else if (args[0] === 'pull') { writeFileSync(join(home, 'models', 'tiny.gguf.part'), Buffer.alloc(24)); writeFileSync(join(home, 'pulling'), String(process.pid)); setInterval(() => {}, 1000); }
+else process.exitCode = 2;
+`, { mode: 0o700 })
+ const inventory = async () => ({ state: 'awake' as const, status: 'running', nodes: [] })
+ service = new LocalModels({ stateDir, processEnv: { GRID_HOME: gridHome, HARNESS_GRID_BIN: executable, PATH: root }, inventory })
+ await service.act('home', modelId, 'download')
+ const pulling = join(gridHome, 'pulling')
+ for (let i = 0; i < 100 && !existsSync(pulling); i++) await new Promise(resolve => setTimeout(resolve, 50))
+ const pid = Number(await readFile(pulling, 'utf8'))
+ await service.act('home', modelId, 'stop'); await service.settled()
+ expect(() => process.kill(pid, 0)).toThrow()
+ expect(existsSync(join(gridHome, 'models', 'tiny.gguf.part'))).toBe(false)
+ expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'available', operation: { action: 'stop', phase: 'done' } })
+ } finally {
+ await service?.settled()
+ server.closeAllConnections()
+ await new Promise<void>(resolve => server.close(() => resolve()))
+ await rm(root, { recursive: true, force: true })
+ }
+}, 20_000)
diff --git a/cli/src/lib/localModels.spec.ts b/cli/src/lib/localModels.spec.ts
index 8d2ee68a3..4a83b07a9 100644
--- a/cli/src/lib/localModels.spec.ts
+++ b/cli/src/lib/localModels.spec.ts
@@ -3,7 +3,7 @@ import { mkdtemp, mkdir, writeFile, readFile, readdir, rm, stat, statfs, readlin
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { createHash } from 'node:crypto'
-import { rmSync, writeFileSync } from 'node:fs'
+import { existsSync, rmSync, writeFileSync } from 'node:fs'
import { spawnSync } from 'node:child_process'
import type { GpuProcess } from './gpuProcesses.js'
import { baseModel, compatibleModels, contextLadder, fitOf, LocalModels, modelBudget, modelFamily, rankForCoding, readRunRecords, type GridInventory } from './localModels.js'
@@ -856,6 +856,21 @@ describe('local model discovery and lifecycle', () => {
expect(request.mock.calls.some(([url]) => String(url).includes(RELAY_CHAT))).toBe(false)
})
+ it('gives a model that thinks first room to finish, and starts it on its real reply', async () => {
+ // As Muse-Glimmer-30B: 8 tokens are all thinking, its content empty; with room it answers ok.
+ const original = request.getMockImplementation()!
+ const asked: number[] = []
+ request.mockImplementation(async (url, init) => {
+ if (!String(url).includes(RELAY_CHAT)) return original(url, init)
+ const tokens = JSON.parse(String(init?.body)).max_tokens as number
+ asked.push(tokens)
+ return response({ choices: [{ message: tokens < 512 ? { content: '', reasoning_content: 'Reply with the single word' } : { content: 'ok', reasoning_content: 'Probably just ok.' } }] })
+ })
+ await service.act('home', 'org/Small-GGUF', 'start'); await service.settled()
+ expect((await service.list('home')).models[0].operation).toMatchObject({ action: 'start', phase: 'done' })
+ expect(asked).toEqual([8, 512])
+ })
+
it.each(['reasoning only', 'http error', 'network error'])('requires a real test reply and bounds retries for %s', async scenario => {
const original = request.getMockImplementation()!
let attempted!: () => void
@@ -1125,6 +1140,59 @@ describe('a run record its engine did not outlive', () => {
})
})
+describe('a download that stops, or is cancelled', () => {
+ /** A pull that sends nothing more, as grid's did when its connection stalled (no read timeout): it ends
+ * only when it is aborted. */
+ const hang = () => {
+ const original = run.getMockImplementation()!
+ run.mockImplementation(async (args: string[], output?: (s: string) => void, _timeout?: number, signal?: AbortSignal) => {
+ if (args[0] !== 'pull') return original(args, output)
+ calls.push(args)
+ await writeFile(join(home, 'models', 'Small-Q4.gguf.part'), Buffer.alloc(24))
+ output?.('36%')
+ return new Promise<GridFleetResult>(resolve => signal?.addEventListener('abort', () => resolve({ ok: false, code: 124, stdout: '', stderr: '', error: 'interrupted' })))
+ })
+ }
+
+ it('is not downloaded while a part of it is unfinished — its projector, say — and can be got again', async () => {
+ await writeFile(join(home, 'models', 'Small-Q4.gguf'), Buffer.alloc(64))
+ await writeFile(join(home, 'models', 'Small-Q4.mmproj.gguf.part'), Buffer.alloc(8))
+ expect((await service.list('home', true)).models[0]).toMatchObject({ id: 'org/Small-GGUF', state: 'available', canStart: true })
+ await rm(join(home, 'models', 'Small-Q4.mmproj.gguf.part'))
+ expect((await service.list('home', true)).models[0]).toMatchObject({ state: 'downloaded' })
+ })
+
+ it('that has stalled is ended, to be started again where it stopped', async () => {
+ hang()
+ service = modelFixture({ stateDir, processEnv: { GRID_HOME: home }, run, request: request as typeof fetch, inventory, pullStallMs: 40, pullWatchMs: 5 })
+ await service.act('home', 'org/Small-GGUF', 'download'); await service.settled()
+ expect((await service.list('home', true)).models[0].operation).toMatchObject({ action: 'download', phase: 'failed', error: 'The download stalled. Start again to resume.' })
+ // What it got is kept: a start again resumes from it.
+ expect(existsSync(join(home, 'models', 'Small-Q4.gguf.part'))).toBe(true)
+ })
+
+ it('is cancelled by a stop, and what it downloaded goes: the row is one to get again, not a failure', async () => {
+ hang()
+ await writeFile(join(home, 'models', 'Small-Q4.mmproj.gguf.part'), Buffer.alloc(8))
+ await writeFile(join(home, 'models', 'Other-Q4.gguf'), Buffer.alloc(8))
+ const started = await service.act('home', 'org/Small-GGUF', 'download')
+ await vi.waitFor(() => expect(calls.some(args => args[0] === 'pull')).toBe(true))
+ const stop = await service.act('home', 'org/Small-GGUF', 'stop')
+ expect(stop.operation?.id).toBe(started.operation?.id)
+ await service.settled()
+ expect((await readdir(join(home, 'models'))).sort()).toEqual(['Other-Q4.gguf'])
+ const [model] = (await service.list('home', true)).models
+ expect(model).toMatchObject({ state: 'available', operation: { action: 'stop', phase: 'done' } })
+ expect(model.operation).not.toHaveProperty('error')
+ // Another model's operation is not cancelled by a stop of this one.
+ hang()
+ await service.act('home', 'org/Small-GGUF', 'download')
+ await vi.waitFor(() => expect(calls.filter(args => args[0] === 'pull')).toHaveLength(2))
+ expect(await service.act('home', 'org/Other-GGUF', 'stop')).toEqual({ error: 'Wait for the current model operation to finish.' })
+ await service.act('home', 'org/Small-GGUF', 'stop'); await service.settled()
+ })
+})
+
/** A pid that existed a moment ago and no longer does. */
function deadPid(): number {
return Number(spawnSync(process.execPath, ['-e', 'process.stdout.write(String(process.pid))'], { encoding: 'utf8' }).stdout)
diff --git a/cli/src/lib/localModels.ts b/cli/src/lib/localModels.ts
index 1dad1f9e1..c8acc43cb 100644
--- a/cli/src/lib/localModels.ts
+++ b/cli/src/lib/localModels.ts
@@ -13,6 +13,7 @@ import { displayModelName } from './gridReader.js'
import { readEnvExports } from './gridWake.js'
import { MIN_CODING_CONTEXT } from './codingContext.js'
import type { GpuProcess } from './gpuProcesses.js'
+import { downloadedBytes, removeDownload, unfinishedDownload } from './modelFiles.js'
import { APP_LABEL, AppStartError, GRID_LABEL, appContext, readAppRecords, writeAppRecords, type AppEngine, type AppEngineOps, type AppEngineRecord, type AppModel } from './appModels.js'
import { osToken, type OsToken } from './osToken.js'
@@ -154,7 +155,11 @@ interface Options {
* one it takes the host name — so a model started here read `mac.lan` under "On your machines"
* while Machines called the same computer `M2`. */
machineName?: () => string | null | undefined
- run?: (args: string[], output?: (chunk: string) => void, timeout?: number) => Promise<GridFleetResult>
+ run?: (args: string[], output?: (chunk: string) => void, timeout?: number, signal?: AbortSignal) => Promise<GridFleetResult>
+ /** How long a download's files may stop growing before it is called stalled, and how often they are
+ * looked at ([PULL_STALL_MS], [PULL_WATCH_MS]); a test shortens both. */
+ pullStallMs?: number
+ pullWatchMs?: number
request?: typeof fetch
/** Injected, never a `grid engines` call: that read carries the grid credential, and a signed-in read
* of a sleeping grid WAKES it — once a tick, for as long as the panel is open. */
@@ -176,6 +181,11 @@ interface Options {
gpuProcesses?: () => Promise<GpuProcess[]>
}
+/** A download whose files have not grown in this long has stalled ([LocalModels.perform]): long enough for a
+ * slow mirror's pauses, short of the 30 minutes a command may run. Looked at every [PULL_WATCH_MS]. */
+const PULL_STALL_MS = 3 * 60_000
+const PULL_WATCH_MS = 10_000
+
/** One concrete, machine-fitted version per model. Non-chat and unprobed offline
* catalog rows are never called compatible. Split GGUF files stay one model.
*
@@ -347,6 +357,8 @@ export class LocalModels {
private readonly processEnv: NodeJS.ProcessEnv
private readonly home: string
private readonly run: NonNullable<Options['run']>
+ private readonly pullStallMs: number
+ private readonly pullWatchMs: number
private readonly request: typeof fetch
private candidates: Candidate[] = []
private device: Record<string, any> = {}
@@ -358,7 +370,7 @@ export class LocalModels {
private cached?: { at: number; grid: string; value: LocalModelsSnapshot }
/** The models the last read of each grid found running — kept past `cached`, which any save clears. */
private readonly runningAtLastRead = new Map<string, Set<string>>()
- private active?: { grid: string; operation: ModelOperation; done: Promise<void> }
+ private active?: { grid: string; operation: ModelOperation; done: Promise<void>; pull: AbortController; cancelled: boolean }
private receipt?: Receipt
private receiptScope?: string
private knownByGrid = new Map<string, Candidate[]>()
@@ -375,9 +387,14 @@ export class LocalModels {
this.processEnv = options.processEnv ?? process.env
this.home = dirname(gridCredentialsPath(this.processEnv))
const rpc = new GridFleetRpc(this.processEnv)
- this.run = options.run ?? ((args, output, timeout = 30_000) =>
- rpc.run('local-models', randomUUID(), { args, timeoutMs: timeout, thinking: false }, output, 4 * 1024 * 1024))
+ this.run = options.run ?? ((args, output, timeout = 30_000, signal) => {
+ const id = randomUUID()
+ signal?.addEventListener('abort', () => rpc.cancel('local-models', id), { once: true })
+ return rpc.run('local-models', id, { args, timeoutMs: timeout, thinking: false }, output, 4 * 1024 * 1024)
+ })
this.request = options.request ?? fetch
+ this.pullStallMs = options.pullStallMs ?? PULL_STALL_MS
+ this.pullWatchMs = options.pullWatchMs ?? PULL_WATCH_MS
}
private async json(args: string[]): Promise<any> {
@@ -474,6 +491,9 @@ export class LocalModels {
private async downloaded(candidate: Candidate): Promise<boolean> {
if (candidate.appPath) return stat(candidate.appPath).then(s => s.isFile() && s.size === candidate.size, () => false)
+ // A `.part` beside it is a download that did not finish — its projector, say, while its weights did.
+ // Read as downloaded, it could not be got again, and the rest of it was never fetched.
+ if (await unfinishedDownload(join(this.home, 'models'), candidate.file)) return false
const sizes = await Promise.all(candidate.files.map(file => stat(join(this.home, 'models', file)).then(s => s.isFile() ? s.size : 0).catch(() => 0)))
return sizes.every(size => size > 0) && sizes.reduce((sum, size) => sum + size, 0) === candidate.size
}
@@ -937,11 +957,20 @@ export class LocalModels {
* A different model waits: starts/stops must not race on the machine's budget. */
async act(grid: string | null, modelId: unknown, action: 'download' | 'start' | 'stop'): Promise<{ operation?: ModelOperation; error?: string }> {
if (!grid || typeof modelId !== 'string') return { error: 'The model is unavailable. Refresh and try again.' }
+ // Stop on a model that is downloading cancels the download, and what it downloaded goes ([perform]).
+ // Without it, a download that hung (its connection stalled after the disk filled) held the view on
+ // "Downloading 36%" with nothing to press but Wait.
+ const running = this.active
+ if (action === 'stop' && running?.grid === grid && running.operation.modelId === modelId && running.operation.stage === 'downloading') {
+ running.cancelled = true
+ running.pull.abort()
+ return { operation: running.operation }
+ }
if (this.active) return this.active.grid === grid && this.active.operation.modelId === modelId && this.active.operation.action === action
? { operation: this.active.operation } : { error: 'Wait for the current model operation to finish.' }
const operation: ModelOperation = { id: randomUUID(), modelId, action, stage: action === 'stop' ? 'stopping' : 'checking', phase: 'running', updatedAt: new Date().toISOString() }
// Reserve before any await; independent RPCs may arrive on separate connections.
- const active = { grid, operation, done: Promise.resolve() }
+ const active = { grid, operation, done: Promise.resolve(), pull: new AbortController(), cancelled: false }
this.active = active
try { await this.save(grid, operation) } catch { this.active = undefined; return { error: 'Setup could not be saved. Try again.' } }
active.done = this.perform(grid, operation).catch(async error => {
@@ -963,8 +992,8 @@ export class LocalModels {
const change = async (stage: ModelOperation['stage']) => {
operation.stage = stage; delete operation.progress; await this.save(grid, operation)
}
- const must = async (args: string[], message: string, output?: (chunk: string) => void) => {
- if (!(await this.run(args, output, 30 * 60_000)).ok) throw new ModelError(message)
+ const must = async (args: string[], message: string, output?: (chunk: string) => void, signal?: AbortSignal) => {
+ if (!(await this.run(args, output, 30 * 60_000, signal)).ok) throw new ModelError(message)
}
if (operation.action !== 'stop') await this.loadCatalog()
if (operation.action === 'start') {
@@ -1022,14 +1051,36 @@ export class LocalModels {
if (disk.bavail * disk.bsize < candidate.size + GiB) throw new ModelError('Free up disk space, then start again.')
await change('downloading')
let last = 0, progressWrites = Promise.resolve()
- await must(['pull', candidate.pull], 'The download stopped. Start again to resume.', chunk => {
- const matches = [...chunk.matchAll(/(\d+(?:\.\d+)?)\s*%/g)]
- const percent = matches.length ? Number(matches.at(-1)![1]) : NaN
- if (Number.isFinite(percent) && percent >= 0 && percent <= 100 && Date.now() - last > 500) {
- last = Date.now(); operation.progress = percent / 100
- progressWrites = progressWrites.then(() => this.save(grid, operation)).catch(() => {})
+ // The pull is watched by what it writes: grid reads its download with no read timeout, so a
+ // connection that stops sending holds it forever — after a full disk on an NVIDIA machine, 30
+ // minutes at 36% with nothing to press. Files that stop growing for [PULL_STALL_MS] end it.
+ const active = this.active!, pull = active.pull
+ let size = -1, grewAt = Date.now(), stalled = false
+ const watch = setInterval(() => void downloadedBytes(join(this.home, 'models'), candidate.file).then(bytes => {
+ if (bytes !== size) { size = bytes; grewAt = Date.now() } else if (Date.now() - grewAt >= this.pullStallMs) { stalled = true; pull.abort() }
+ }), this.pullWatchMs)
+ try {
+ await must(['pull', candidate.pull], 'The download stopped. Start again to resume.', chunk => {
+ const matches = [...chunk.matchAll(/(\d+(?:\.\d+)?)\s*%/g)]
+ const percent = matches.length ? Number(matches.at(-1)![1]) : NaN
+ if (Number.isFinite(percent) && percent >= 0 && percent <= 100 && Date.now() - last > 500) {
+ last = Date.now(); operation.progress = percent / 100
+ progressWrites = progressWrites.then(() => this.save(grid, operation)).catch(() => {})
+ }
+ }, pull.signal)
+ } catch (error) {
+ if (active.cancelled) {
+ await progressWrites
+ // (The model was not on this computer before: a download starts only for one [downloaded] says
+ // is not, so what goes is only what this download wrote.)
+ await removeDownload(join(this.home, 'models'), candidate.file)
+ // A cancel is a stop that finished, not a failure: the row goes back to Get.
+ operation.action = 'stop'; operation.stage = 'stopping'; operation.phase = 'done'; delete operation.progress
+ await this.save(grid, operation)
+ return
}
- })
+ throw stalled ? new ModelError('The download stalled. Start again to resume.') : error
+ } finally { clearInterval(watch) }
await progressWrites
if (!await this.downloaded(candidate)) throw new ModelError('The download is incomplete. Start again to resume.')
}
@@ -1190,15 +1241,22 @@ export class LocalModels {
const { baseUrl, apiKey } = readEnvExports(info.stdout)
if (!info.ok || !baseUrl || !apiKey) throw new ModelError('The model is starting, but could not be checked. Try again shortly.')
const deadline = Date.now() + 180_000
+ let maxTokens = 8
do {
try {
const response = await this.request(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
method: 'POST', headers: { authorization: `Bearer ${apiKey}`, 'content-type': 'application/json' },
- body: JSON.stringify({ model, messages: [{ role: 'user', content: 'Reply with the single word: ok' }], max_tokens: 8 }),
- signal: AbortSignal.timeout(30_000), redirect: 'error',
+ body: JSON.stringify({ model, messages: [{ role: 'user', content: 'Reply with the single word: ok' }], max_tokens: maxTokens }),
+ signal: AbortSignal.timeout(maxTokens === THINKING_TEST_TOKENS ? 90_000 : 30_000), redirect: 'error',
})
const body = response.ok ? obj(await response.json()) : {}
- if (/^\s*ok[.!]?\s*$/i.test(str(obj(rows(body.choices)[0]?.message).content))) return
+ const message = obj(rows(body.choices)[0]?.message)
+ if (/^\s*ok[.!]?\s*$/i.test(str(message.content))) return
+ // A model that thinks first spends a reply this short thinking: Muse-Glimmer-30B (which thinks
+ // whatever --reasoning-budget says) put all 8 tokens in its reasoning, its content stayed empty,
+ // and every start of it failed after 3 minutes of "Testing" while it answered each try. Asked
+ // again with room to finish thinking, it says ok. A real reply is still what passes.
+ if (!str(message.content).trim() && str(message.reasoning_content || message.reasoning).trim()) maxTokens = THINKING_TEST_TOKENS
} catch { /* registration and first warmup take time; status remains verifying */ }
await new Promise(resolve => setTimeout(resolve, 1500))
} while (Date.now() < deadline)
@@ -1207,6 +1265,10 @@ export class LocalModels {
}
class ModelError extends Error {}
+/** The reply a model that thinks first is given to finish thinking and answer the start's test in
+ * ([LocalModels.verify]): Muse-Glimmer-30B needed under 200. */
+const THINKING_TEST_TOKENS = 512
+
/** More names than any one engine advertises: a record past it is cut, never trusted to be small. */
const MAX_RECORD_IDS = 64
diff --git a/cli/src/lib/modelFiles.spec.ts b/cli/src/lib/modelFiles.spec.ts
new file mode 100644
index 000000000..f757ad6f4
--- /dev/null
+++ b/cli/src/lib/modelFiles.spec.ts
@@ -0,0 +1,44 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+
+vi.mock('node:fs/promises', async original => {
+ const fs = await original<typeof import('node:fs/promises')>()
+ return { ...fs, stat: vi.fn(fs.stat) }
+})
+const { stat } = await import('node:fs/promises')
+const { downloadFiles, downloadedBytes, removeDownload, unfinishedDownload } = await import('./modelFiles.js')
+
+describe("a download's files in grid's models folder", () => {
+ let dir: string
+ beforeEach(async () => {
+ dir = await mkdtemp(join(tmpdir(), 'model-files-'))
+ for (const [name, size] of [['Glimmer-Q4.gguf', 16], ['Glimmer-Q4.mmproj.gguf.part', 4], ['Glimmer-Q4.notes.txt', 2], ['Other-Q4.gguf', 8]] as const) {
+ await writeFile(join(dir, name), Buffer.alloc(size))
+ }
+ })
+ afterEach(async () => { vi.mocked(stat).mockReset(); await rm(dir, { recursive: true, force: true }) })
+
+ it('are its weights, its projector and either one\'s .part — nothing else of the folder', async () => {
+ expect((await downloadFiles(dir, 'Glimmer-Q4.gguf')).sort()).toEqual(['Glimmer-Q4.gguf', 'Glimmer-Q4.mmproj.gguf.part'])
+ expect(await downloadFiles(join(dir, 'missing'), 'Glimmer-Q4.gguf')).toEqual([])
+ })
+
+ it('are unfinished while a .part is there', async () => {
+ expect(await unfinishedDownload(dir, 'Glimmer-Q4.gguf')).toBe(true)
+ expect(await unfinishedDownload(dir, 'Other-Q4.gguf')).toBe(false)
+ })
+
+ it('are measured complete and unfinished alike, a file gone meanwhile counting nothing', async () => {
+ expect(await downloadedBytes(dir, 'Glimmer-Q4.gguf')).toBe(20)
+ const real = await vi.importActual<typeof import('node:fs/promises')>('node:fs/promises')
+ vi.mocked(stat).mockImplementation(async path => String(path).endsWith('.part') ? Promise.reject(new Error('ENOENT')) : real.stat(path))
+ expect(await downloadedBytes(dir, 'Glimmer-Q4.gguf')).toBe(16)
+ })
+
+ it('go when the download is cancelled, and nothing else does', async () => {
+ await removeDownload(dir, 'Glimmer-Q4.gguf')
+ expect((await readdir(dir)).sort()).toEqual(['Glimmer-Q4.notes.txt', 'Other-Q4.gguf'])
+ })
+})
diff --git a/cli/src/lib/modelFiles.ts b/cli/src/lib/modelFiles.ts
new file mode 100644
index 000000000..caf3aa5a1
--- /dev/null
+++ b/cli/src/lib/modelFiles.ts
@@ -0,0 +1,27 @@
+import { readdir, rm, stat } from 'node:fs/promises'
+import { basename, join } from 'node:path'
+
+/** The files a download of [file] writes in grid's models folder [dir]: the weights, the vision projector
+ * `grid pull` fetches beside them (`<stem>.mmproj.gguf`), and either one's unfinished `.part`. */
+export async function downloadFiles(dir: string, file: string): Promise<string[]> {
+ const stem = basename(file).replace(/\.gguf$/i, '')
+ const names = await readdir(dir).catch(() => [] as string[])
+ return names.filter(name => name.startsWith(stem) && (/\.gguf$/i.test(name) || name.endsWith('.part')))
+}
+
+/** Whether a download of [file] did not finish: a `.part` of it, or of its projector, is still there. */
+export async function unfinishedDownload(dir: string, file: string): Promise<boolean> {
+ return (await downloadFiles(dir, file)).some(name => name.endsWith('.part'))
+}
+
+/** How much of a download of [file] is on disk now, complete and unfinished files alike — what a stall
+ * watch measures a download by. A file gone between the listing and its size counts nothing. */
+export async function downloadedBytes(dir: string, file: string): Promise<number> {
+ const sizes = await Promise.all((await downloadFiles(dir, file)).map(name => stat(join(dir, name)).then(s => s.size, () => 0)))
+ return sizes.reduce((sum, size) => sum + size, 0)
+}
+
+/** A cancelled download's files, gone. */
+export async function removeDownload(dir: string, file: string): Promise<void> {
+ for (const name of await downloadFiles(dir, file)) await rm(join(dir, name), { force: true })
+}
diff --git a/desktop/lib/models/model_manager_controller.dart b/desktop/lib/models/model_manager_controller.dart
index 559013f76..12f414568 100644
--- a/desktop/lib/models/model_manager_controller.dart
+++ b/desktop/lib/models/model_manager_controller.dart
@@ -672,6 +672,53 @@ class ModelManagerController extends ChangeNotifier {
Future<void> toggle(LocalModel model) =>
control(model, model.canStop ? 'stop' : 'start');
+ /// Whether [model]'s download is under way here: what [cancelDownload] stops. Every other action
+ /// waits while it runs, so a download that hung (its connection stalled after a full disk) once left
+ /// nothing to press.
+ bool downloading(LocalModel model) {
+ final operation = operationFor(model);
+ return operation?.active == true && operation?.stage == 'downloading';
+ }
+
+ /// [model]'s download cancelled: the daemon's stop of a model that is downloading ends the download
+ /// and deletes what it downloaded, and the model can be got again.
+ Future<void> cancelDownload(LocalModel model) async {
+ final owner = machine;
+ if (owner == null ||
+ owner.connectionStatus != ConnectionStatus.connected ||
+ owner.needsLink ||
+ owner.machine.isShared ||
+ pendingId != null ||
+ !downloading(model)) {
+ return;
+ }
+ pendingId = model.id;
+ _actionRevision++;
+ error = null;
+ _changed();
+ try {
+ final answer = await app.controlLocalModel(
+ owner.machine.machineId,
+ model.id,
+ start: false,
+ );
+ if (!_current(owner)) return;
+ error = answer['error'] as String?;
+ } catch (_) {
+ if (_current(owner)) {
+ error =
+ 'Checking the model operation. Your click will not be repeated.';
+ }
+ } finally {
+ if (_current(owner)) {
+ pendingId = null;
+ _changed();
+ await _refreshing;
+ if (_current(owner)) await refresh();
+ }
+ }
+ }
+
Future<void> control(LocalModel model, String action) async {
final owner = machine;
if (owner == null ||
diff --git a/desktop/lib/models/models_panel.dart b/desktop/lib/models/models_panel.dart
index 80af4c202..82418ec00 100644
--- a/desktop/lib/models/models_panel.dart
+++ b/desktop/lib/models/models_panel.dart
@@ -528,7 +528,11 @@ class _ModelsPanelState extends State<ModelsPanel> with SectionWakes {
: model.downloaded
? AppIcons.play
: AppIcons.download;
- final tooltip = active
+ // A download under way can be cancelled — the one action open while it runs.
+ final cancel = !pending && controller.downloading(model);
+ final tooltip = cancel
+ ? 'Cancel the download. What it downloaded is deleted.'
+ : active
? status
: model.canStop
? 'Pause ${model.name} and free memory. The download is kept.'
@@ -603,7 +607,21 @@ class _ModelsPanelState extends State<ModelsPanel> with SectionWakes {
child: SizedBox(
width: 40,
height: 40,
- child: active
+ child: cancel
+ ? IconButton(
+ key: ValueKey('model-cancel-${model.id}'),
+ onPressed: () =>
+ unawaited(controller.cancelDownload(model)),
+ icon: Icon(
+ AppIcons.close,
+ size: 16,
+ semanticLabel: 'Cancel downloading ${model.name}',
+ ),
+ style: IconButton.styleFrom(
+ foregroundColor: AppPalette.textPrimary,
+ ),
+ )
+ : active
? Semantics(
liveRegion: true,
label: '$status ${model.name}',
diff --git a/desktop/lib/widgets/swarm_resource_preview.dart b/desktop/lib/widgets/swarm_resource_preview.dart
index 7ac439c18..93d7e1cbe 100644
--- a/desktop/lib/widgets/swarm_resource_preview.dart
+++ b/desktop/lib/widgets/swarm_resource_preview.dart
@@ -891,6 +891,26 @@ class _SwarmResourcePreviewState extends State<SwarmResourcePreview> {
final local = model?.local;
final api = model?.api;
final owner = model?.controller ?? catalog.manager;
+ if (local != null && owner.downloading(local)) {
+ // A download under way: cancelling it is the one action open while it runs.
+ return [
+ _ResourceAction(
+ 'Cancel download',
+ !busy &&
+ owner.pendingId == null &&
+ owner.machine?.connectionStatus ==
+ ConnectionStatus.connected
+ ? () => unawaited(
+ _run(() async {
+ await owner.cancelDownload(local);
+ return owner.error;
+ }),
+ )
+ : null,
+ command: 'picker.model_stop',
+ ),
+ ];
+ }
if (local != null) {
final enabled =
!busy &&
diff --git a/desktop/test/models_panel_test.dart b/desktop/test/models_panel_test.dart
index 4924c4fdc..e8c9fac9b 100644
--- a/desktop/test/models_panel_test.dart
+++ b/desktop/test/models_panel_test.dart
@@ -244,6 +244,60 @@ void main() {
variant: TargetPlatformVariant.only(TargetPlatform.macOS),
);
+ testWidgets('a download under way can be cancelled, the one action open while it runs', (
+ tester,
+ ) async {
+ tester.view.devicePixelRatio = 1;
+ tester.view.physicalSize = const Size(700, 800);
+ addTearDown(tester.view.reset);
+ final app = ModelManagerTestApp(ModelManagerConnection());
+ final controller = ModelManagerController(app, poll: false);
+ final subscriptions = _SubscriptionRows();
+ addTearDown(() {
+ controller.dispose();
+ subscriptions.dispose();
+ app.dispose();
+ });
+ // Nothing downloading: Cancel asks the daemon nothing.
+ await controller.refresh();
+ await controller.cancelDownload(controller.localModels.first);
+ expect(app.actions, isEmpty);
+ app.localInventory = modelInventory(scenario: 'downloading');
+ await controller.refresh();
+ expect(controller.downloading(controller.localModels.first), isTrue);
+ await tester.pumpWidget(
+ MaterialApp(
+ home: Scaffold(
+ body: Center(
+ child: SizedBox(
+ width: 640,
+ child: ModelsPanel(
+ controller: controller,
+ subscriptions: subscriptions,
+ onClose: () {},
+ onManage: () {},
+ ),
+ ),
+ ),
+ ),
+ ),
+ );
+ await tester.tap(find.text('Local 5'));
+ await tester.pump();
+ // In place of the spinner: the download can be cancelled, though every other action waits.
+ final cancel = find.byKey(const ValueKey('model-cancel-qwen'));
+ expect(cancel, findsOneWidget);
+ expect(find.bySemanticsLabel('Cancel downloading Qwen3.8-27B'), findsOneWidget);
+ expect(
+ tester.widget<IconButton>(find.byKey(const ValueKey('model-action-gemma'))).onPressed,
+ isNull,
+ );
+ await tester.tap(cancel);
+ await tester.pump();
+ expect(app.actions, [(machine: 'm', model: 'qwen', start: false)]);
+ await tester.pumpWidget(const SizedBox());
+ });
+
testWidgets('download, play and pause reflect file and engine state', (
tester,
) async {
@@ -398,7 +452,9 @@ void main() {
expect(find.text('Downloading · 42%').hitTestable(), findsOneWidget);
expect(app.actions, hasLength(1));
expect(controller.busy, isTrue);
- expect(find.byType(CircularProgressIndicator), findsOneWidget);
+ // While it downloads, the download can be cancelled: a Cancel in place of the spinner.
+ expect(find.byKey(const ValueKey('model-cancel-qwen')), findsOneWidget);
+ expect(find.byType(CircularProgressIndicator), findsNothing);
expect(tester.takeException(), isNull);
await tester.pumpWidget(const SizedBox());
});
diff --git a/desktop/test/resource_picker_test.dart b/desktop/test/resource_picker_test.dart
index a2d16b606..0044e9c4e 100644
--- a/desktop/test/resource_picker_test.dart
+++ b/desktop/test/resource_picker_test.dart
@@ -508,13 +508,22 @@ void main() {
await capture(tester, 'model-manage-focused');
await key(tester, LogicalKeyboardKey.space);
expect(app.downloads, [(machine: 'm', model: 'qwen')]);
- expect(button('picker.model_download').onPressed, isNull);
+ // While it downloads, the one action is to cancel it.
+ expect(
+ find.byKey(const ValueKey('resource-action:picker.model_download')),
+ findsNothing,
+ );
+ expect(find.text('Cancel download'), findsOneWidget);
await key(tester, LogicalKeyboardKey.enter);
expect(app.downloads, hasLength(1));
expect(app.actions, isEmpty);
expect(tester.widget<TextField>(field).focusNode!.hasFocus, isTrue);
expect(controller.query, ':Qwen3.8-27B');
expect(tester.takeException(), isNull);
+ // (Enter again did not cancel it: only Cancel download does.)
+ await tester.tap(find.text('Cancel download'));
+ await tester.pump();
+ expect(app.actions, [(machine: 'm', model: 'qwen', start: false)]);
} finally {
await tester.pumpWidget(const SizedBox());
app.dispose();
diff --git a/tui/src/models.rs b/tui/src/models.rs
index 501fc276f..eed77eb79 100644
--- a/tui/src/models.rs
+++ b/tui/src/models.rs
@@ -55,6 +55,8 @@ impl Operation {
Some(Operation { id: s("id")?, model: s("modelId")?, action, stage, phase, progress: v.get("progress").and_then(num).map(|p| p.clamp(0.0, 1.0)), error: s("error") })
}
pub fn active(&self) -> bool { self.phase == "running" }
+ /// A download under way: what ^S cancels ([plan_stop]).
+ pub fn downloading(&self) -> bool { self.active() && self.stage == "downloading" }
pub fn failed(&self) -> bool { self.phase == "failed" }
pub fn label(&self) -> &'static str {
match self.stage.as_str() { "downloading" => "Downloading", "updating" => "Updating engine", "starting" => "Starting", "verifying" => "Testing", "stopping" => "Stopping", _ => "Checking" }
@@ -1216,6 +1218,11 @@ pub fn plan_stop(app: &App, id: &str) -> Plan {
}
let Some(m) = id.strip_prefix("mv:local:").and_then(|model| local_model(app, model)) else { return Plan::Nothing };
let f = facts(app, &m, None);
+ // A download under way is cancelled, and what it downloaded deleted — the daemon's stop of it. A
+ // download that hung had nothing to press but Wait.
+ if f.op.as_ref().is_some_and(Operation::downloading) {
+ return Plan::Arm(format!("^S again cancels downloading {} — what it downloaded is deleted", m.shown()), Box::new(Plan::Act { model: m.id.clone(), action: "stop" }));
+ }
if f.busy { return Plan::Say("Wait for the current model operation to finish.".into()) }
// Run on the community grid alone: stopped there.
if !m.can_stop && on_community(app, &m) {
@@ -1815,7 +1822,11 @@ pub fn on_api_models(app: &mut App, id: &str, reply: Result<Value, RpcError>) {
}
/// A download, start or stop of this computer's [model].
-fn act(app: &mut App, model: &str, action: &'static str) { act_on(app, model, action, false) }
+fn act(app: &mut App, model: &str, action: &'static str) {
+ // A stop of the model a Use is getting cancels that Use too: there is nothing left for it to start.
+ if action == "stop" && app.models_view.using.as_ref().is_some_and(|u| u.model == model) { app.models_view.using = None }
+ act_on(app, model, action, false)
+}
/// [act], on the community grid of this computer's OS when [community] (a start or stop; the daemon's
/// `community`).
@@ -2439,7 +2450,8 @@ pub fn preview(app: &App, id: &str) -> Vec<Line<'static>> {
else if f.use_start { said.push(match &f.other { None => "Enter starts it and moves this harness onto it.".into(), Some(o) => format!("Enter stops {o}, starts this one, and moves this harness onto it.") }) }
else if t.is_none() && m.downloaded() && !m.running() && m.can_start { said.push("Enter starts it.".into()) }
}
- if m.can_stop { said.push("^S stops it and frees its memory. The download is kept.".into()) }
+ if f.op.as_ref().is_some_and(Operation::downloading) { said.push("^S cancels the download and deletes what it downloaded.".into()) }
+ else if m.can_stop { said.push("^S stops it and frees its memory. The download is kept.".into()) }
else if let Some(label) = &community { said.push(format!("^S stops it on {label}. The download is kept.")) }
if !said.is_empty() { out.push(Line::raw("")); out.extend(said.into_iter().map(|s| Line::from(span(s, fg(theme::accent()))))) }
if let Some(e) = f.op.as_ref().and_then(|o| o.error.clone()) { out.push(warn(e)) }
@@ -4355,6 +4367,30 @@ mod tests {
assert_eq!(app.models_view.using.as_ref().map(|u| (u.step, u.stops.clone())), Some((Step::Start, None)));
}
+ #[test]
+ fn stop_on_a_downloading_model_cancels_the_download_and_the_use_that_began_it() {
+ let mut app = loaded();
+ let mut picker = Picker::new("Models", "");
+ let id = "mv:local:qwen-35b";
+ choose(&mut app, &mut picker, id, false);
+ choose(&mut app, &mut picker, id, false);
+ on_act(&mut app, "local", Ok(json!({ "operation": op("download", "downloading", "running", Some(0.1)) })));
+ on_local(&mut app, "local", Ok(local_reply(qwen("available", op("download", "downloading", "running", Some(0.36))), true)));
+ assert!(app.models_view.using.is_some());
+ let p = text(&preview(&app, id));
+ assert!(p.contains("^S cancels the download and deletes what it downloaded."), "{p}");
+ // Not "Wait for the current model operation to finish", as it was while a download hung.
+ assert!(matches!(plan_stop(&app, id), Plan::Arm(ref w, _) if w == "^S again cancels downloading Qwen3.6-35B-A3B — what it downloaded is deleted"));
+ choose(&mut app, &mut picker, id, true);
+ assert!(app.models_view.sent.iter().all(|s| s.1 != "grid_fleet_model_stop"), "the first ^S only asks");
+ choose(&mut app, &mut picker, id, true);
+ assert_eq!(last_sent(&app), ("local".into(), "grid_fleet_model_stop".into(), json!({ "modelId": "qwen-35b" })));
+ assert!(app.models_view.using.is_none());
+ // Another operation under way is still waited for.
+ on_local(&mut app, "local", Ok(local_reply(qwen("downloaded", op("start", "starting", "running", None)), true)));
+ assert!(matches!(plan_stop(&app, id), Plan::Say(ref w) if w == "Wait for the current model operation to finish."));
+ }
+
/// A model a Grid setup left running, which the catalog does not list: grid serves it under its
/// file's name, quant and all. It is shown without the quant (the Quant line says it — on Omarchy,
/// only beside another download of the model), and a harness still moves onto it by the name grid serves.