Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: settle canceled model downloads and explain import interruptions
[._.]/ Adam Eivyc46ea3475ef1
diff --git a/server/services/hfDownload.js b/server/services/hfDownload.jsindex c68d90f924..2d22c4ec1c 100644--- a/server/services/hfDownload.js+++ b/server/services/hfDownload.js@@ -32,6 +32,7 @@ import { PATHS } from '../lib/fileUtils.js';import { getHfTokenInfo } from './hfToken.js';import { safeChildProcessEnv, safeChildProcessOptions } from '../lib/processEnv.js';import { createLineReader } from '../lib/streamLines.js';+import { killWithEscalation } from '../lib/killWithEscalation.js';import { getSettings } from './settings.js';const HELPER_SCRIPT = join(PATHS.root, 'scripts', 'hf_download_repo.py');@@ -191,8 +192,8 @@ export function buildHfDownloadArgs({ repo, revision = null, only = null, ignore}// Returns `{ promise, kill }`. The promise resolves with `{ ok, sizeBytes,-// errorKind, errorMessage }`. `kill()` SIGTERMs the python child so the-// SSE handler can stop the download when the EventSource client closes.+// errorKind, errorMessage }`. `kill()` cancels the python child and escalates+// if needed so a disconnected SSE client cannot strand the download slot.//// `only` is an array of exact repo-relative filenames. When set, the helper// runs in SINGLE-FILE mode: it never enumerates the repo and fetches only those@@ -208,10 +209,23 @@ export function buildHfDownloadArgs({ repo, revision = null, only = null, ignoreexport function downloadHfRepo({ repo, revision = null, only = null, ignore = null, pythonPath: preferredPython = null, onEvent }) {let proc = null;let killed = false;+ let processClosed = false;+ let cancelEscalation = null;let errorKind = null;let errorMessage = null;let sizeBytes = 0;+ const cancelChild = () => {+ if (!proc || processClosed || cancelEscalation || proc.exitCode !== null || proc.signalCode !== null) return;+ cancelEscalation = killWithEscalation(proc, {+ label: 'HuggingFace download', stillRunning: () => !processClosed,+ });+ };+ const finishProcess = () => {+ processClosed = true;+ clearTimeout(cancelEscalation);+ };+const promise = (async () => {const pythonPath = await resolveHfDownloadPython(preferredPython);// Cancel-before-spawn check. resolveHfDownloadPython runs an@@ -247,10 +261,6 @@ export function downloadHfRepo({ repo, revision = null, only = null, ignore = nureturn new Promise((resolve) => {proc = spawn(pythonPath, args, safeChildProcessOptions({ env, stdio: ['ignore', 'pipe', 'pipe'] }));- // Window: kill() could have fired between the second `if (killed)`- // check and the spawn returning the proc handle. Re-check now that we- // own a proc — if it raced, kill it immediately.- if (killed) proc.kill('SIGTERM');const handleLine = (raw) => {const parsed = parseHfDownloadLine(raw);@@ -281,11 +291,13 @@ export function downloadHfRepo({ repo, revision = null, only = null, ignore = nuproc.stderr.on('data', stderrReader.push);proc.stdout.on('data', stdoutReader.push);proc.on('error', (err) => {+ finishProcess();const msg = `Failed to spawn python: ${err.message}`;onEvent({ type: 'error', message: msg, kind: 'spawn_failed' });resolve({ ok: false, errorKind: 'spawn_failed', errorMessage: msg });});proc.on('close', (code, signal) => {+ finishProcess();// Flush any trailing partial line the python helper emitted without a// newline before exit (rare with line-buffered stderr but possible// when the process is SIGKILL'd mid-write).@@ -303,14 +315,17 @@ export function downloadHfRepo({ repo, revision = null, only = null, ignore = nuonEvent({ type: 'error', message: msg, kind: errorKind, repo });return resolve({ ok: false, errorKind, errorMessage: msg });});+ // Terminal listeners must already own the child when cancellation starts.+ if (killed) cancelChild();});})();return {promise,kill: () => {+ if (killed) return;killed = true;- if (proc && !proc.killed) proc.kill('SIGTERM');+ cancelChild();},};}diff --git a/server/services/hfDownload.lifecycle.test.js b/server/services/hfDownload.lifecycle.test.jsnew file mode 100644index 0000000000..9909ffc2ae--- /dev/null+++ b/server/services/hfDownload.lifecycle.test.js@@ -0,0 +1,55 @@+import { EventEmitter } from 'node:events';+import { PassThrough } from 'node:stream';+import { afterEach, describe, expect, it, vi } from 'vitest';+import { spawn } from '../lib/childProcess.js';+import { downloadHfRepo } from './hfDownload.js';++vi.mock('../lib/childProcess.js', async (importOriginal) => ({+ ...await importOriginal(), spawn: vi.fn(),+}));+vi.mock('./hfToken.js', () => ({ getHfTokenInfo: async () => ({ token: null }) }));++afterEach(() => {+ vi.useRealTimers();+ vi.clearAllMocks();+});++describe('Hugging Face download cancellation', () => {+ it.each([false, true])('settles cancellation when SIGTERM is ignored: %s', async (ignoresTerm) => {+ vi.useFakeTimers();+ const child = Object.assign(new EventEmitter(), {+ stdout: new PassThrough(), stderr: new PassThrough(),+ exitCode: null, signalCode: null, killed: false,+ });+ const close = (signal) => {+ child.signalCode = signal;+ child.emit('close', null, signal);+ };+ child.kill = vi.fn((signal) => {+ child.killed = true;+ if (signal === 'SIGKILL' || !ignoresTerm) queueMicrotask(() => close(signal));+ return true;+ });+ const spawned = Promise.withResolvers();+ spawn.mockImplementation(() => { spawned.resolve(); return child; });+ const onEvent = vi.fn();+ const handle = downloadHfRepo({ repo: 'example/model', pythonPath: 'fixture-python', onEvent });+ try {+ await spawned.promise;+ handle.kill();+ handle.kill();+ await vi.advanceTimersByTimeAsync(8_000);+ expect(child.kill.mock.calls.map(([signal]) => signal)).toEqual(+ ignoresTerm ? ['SIGTERM', 'SIGKILL'] : ['SIGTERM'],+ );+ await expect(handle.promise).resolves.toMatchObject({ ok: false, errorKind: 'cancelled' });+ expect(onEvent).toHaveBeenLastCalledWith(expect.objectContaining({ kind: 'cancelled' }));+ expect(vi.getTimerCount()).toBe(0);+ } finally {+ if (child.signalCode === null) close('SIGKILL');+ await handle.promise;+ child.stdout.destroy();+ child.stderr.destroy();+ }+ });+});diff --git a/server/services/ollamaManager.js b/server/services/ollamaManager.jsindex a1f08d209f..7e5ffbf19d 100644--- a/server/services/ollamaManager.js+++ b/server/services/ollamaManager.js@@ -1521,16 +1521,21 @@ function hfImportError(status, statusText) {async function downloadHfImportFile({ repo, remotePath, destination, headers, progress }) {const url = buildHfResolveUrl(repo, 'main', remotePath)- const response = await fetchWithTimeout(url, { headers }, 0)- if (!response.ok || !response.body) throw new Error(hfImportError(response.status, response.statusText))- await ensureDir(dirname(destination))- const tracker = new Transform({- transform(chunk, _encoding, callback) {- progress(chunk.length)- callback(null, chunk)- }- })- await pipeline(Readable.fromWeb(response.body), tracker, createWriteStream(destination))+ try {+ const response = await fetchWithTimeout(url, { headers }, 0)+ if (!response.ok || !response.body) throw new Error(hfImportError(response.status, response.statusText))+ await ensureDir(dirname(destination))+ const tracker = new Transform({+ transform(chunk, _encoding, callback) {+ progress(chunk.length)+ callback(null, chunk)+ }+ })+ await pipeline(Readable.fromWeb(response.body), tracker, createWriteStream(destination))+ } catch (err) {+ if (!(err instanceof TypeError) || !/^(fetch failed|terminated)$/.test(err.message)) throw err+ throw new Error('Hugging Face checkpoint download was interrupted. Check the connection and retry the installation.', { cause: err })+ }}/**diff --git a/server/services/ollamaManager.test.js b/server/services/ollamaManager.test.jsindex afe3260759..60cb329319 100644--- a/server/services/ollamaManager.test.js+++ b/server/services/ollamaManager.test.js@@ -733,6 +733,37 @@ describe('ollamaManager curated Hugging Face Safetensors import', () => {expect(onProgress).toHaveBeenCalledWith(expect.objectContaining({ importing: true }))})+ it.each(['request', 'body'])('explains how to recover from an interrupted checkpoint %s', async (phase) => {+ hfTokenMock.token = 'hf_example_token'+ const transportError = new TypeError(phase === 'request' ? 'fetch failed' : 'terminated', {+ cause: Object.assign(new Error('other side closed'), { code: 'UND_ERR_SOCKET' })+ })+ vi.stubGlobal('fetch', vi.fn(async (url) => {+ const href = String(url)+ if (href.endsWith('/api/version')) return versionResponse()+ if (href.includes('/api/models/example/Qwen-MLX')) {+ return Response.json({ siblings: [+ { rfilename: '4-bit/config.json', size: 2 },+ { rfilename: '4-bit/model.safetensors', size: 7 }+ ] })+ }+ if (href.endsWith('/config.json')) return new Response('{}')+ if (href.endsWith('/model.safetensors')) {+ if (phase === 'request') throw transportError+ return new Response(new ReadableStream({ start(controller) { controller.error(transportError) } }))+ }+ throw new Error(`unexpected fetch: ${href}`)+ }))+ execMock.impl = (_cmd, args, _opts, cb) => args[0] === '--version'+ ? cb(null, { stdout: 'ollama 0.31.0', stderr: '' })+ : cb(new Error('An interrupted download must not reach ollama create'))+ const { importModelFromHfSafetensors } = await loadManager()++ await expect(importModelFromHfSafetensors(spec)).resolves.toMatchObject({+ success: false, error: expect.stringMatching(/Hugging Face checkpoint download.*interrupted.*connection.*retry/i)+ })+ })+it('fails before downloading when the gated repository has no token', async () => {stubFetch([])const { importModelFromHfSafetensors } = await loadManager()