Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: settle canceled model downloads and explain import interruptions

[._.]/ Adam Eivyc46ea3475ef1
diff --git a/server/services/hfDownload.js b/server/services/hfDownload.js
index c68d90f924..2d22c4ec1c 100644
--- a/server/services/hfDownload.js
+++ b/server/services/hfDownload.js
@@ -32,6 +32,7 @@ import { PATHS } from '../lib/fileUtils.js';
import { getHfTokenInfo } from './hfToken.js';
import { safeChildProcessEnv, safeChildProcessOptions } from '../lib/processEnv.js';
import { createLineReader } from '../lib/streamLines.js';
+import { killWithEscalation } from '../lib/killWithEscalation.js';
import { getSettings } from './settings.js';
const HELPER_SCRIPT = join(PATHS.root, 'scripts', 'hf_download_repo.py');
@@ -191,8 +192,8 @@ export function buildHfDownloadArgs({ repo, revision = null, only = null, ignore
}
// Returns `{ promise, kill }`. The promise resolves with `{ ok, sizeBytes,
-// errorKind, errorMessage }`. `kill()` SIGTERMs the python child so the
-// SSE handler can stop the download when the EventSource client closes.
+// errorKind, errorMessage }`. `kill()` cancels the python child and escalates
+// if needed so a disconnected SSE client cannot strand the download slot.
//
// `only` is an array of exact repo-relative filenames. When set, the helper
// runs in SINGLE-FILE mode: it never enumerates the repo and fetches only those
@@ -208,10 +209,23 @@ export function buildHfDownloadArgs({ repo, revision = null, only = null, ignore
export function downloadHfRepo({ repo, revision = null, only = null, ignore = null, pythonPath: preferredPython = null, onEvent }) {
let proc = null;
let killed = false;
+ let processClosed = false;
+ let cancelEscalation = null;
let errorKind = null;
let errorMessage = null;
let sizeBytes = 0;
+ const cancelChild = () => {
+ if (!proc || processClosed || cancelEscalation || proc.exitCode !== null || proc.signalCode !== null) return;
+ cancelEscalation = killWithEscalation(proc, {
+ label: 'HuggingFace download', stillRunning: () => !processClosed,
+ });
+ };
+ const finishProcess = () => {
+ processClosed = true;
+ clearTimeout(cancelEscalation);
+ };
+
const promise = (async () => {
const pythonPath = await resolveHfDownloadPython(preferredPython);
// Cancel-before-spawn check. resolveHfDownloadPython runs an
@@ -247,10 +261,6 @@ export function downloadHfRepo({ repo, revision = null, only = null, ignore = nu
return new Promise((resolve) => {
proc = spawn(pythonPath, args, safeChildProcessOptions({ env, stdio: ['ignore', 'pipe', 'pipe'] }));
- // Window: kill() could have fired between the second `if (killed)`
- // check and the spawn returning the proc handle. Re-check now that we
- // own a proc — if it raced, kill it immediately.
- if (killed) proc.kill('SIGTERM');
const handleLine = (raw) => {
const parsed = parseHfDownloadLine(raw);
@@ -281,11 +291,13 @@ export function downloadHfRepo({ repo, revision = null, only = null, ignore = nu
proc.stderr.on('data', stderrReader.push);
proc.stdout.on('data', stdoutReader.push);
proc.on('error', (err) => {
+ finishProcess();
const msg = `Failed to spawn python: ${err.message}`;
onEvent({ type: 'error', message: msg, kind: 'spawn_failed' });
resolve({ ok: false, errorKind: 'spawn_failed', errorMessage: msg });
});
proc.on('close', (code, signal) => {
+ finishProcess();
// Flush any trailing partial line the python helper emitted without a
// newline before exit (rare with line-buffered stderr but possible
// when the process is SIGKILL'd mid-write).
@@ -303,14 +315,17 @@ export function downloadHfRepo({ repo, revision = null, only = null, ignore = nu
onEvent({ type: 'error', message: msg, kind: errorKind, repo });
return resolve({ ok: false, errorKind, errorMessage: msg });
});
+ // Terminal listeners must already own the child when cancellation starts.
+ if (killed) cancelChild();
});
})();
return {
promise,
kill: () => {
+ if (killed) return;
killed = true;
- if (proc && !proc.killed) proc.kill('SIGTERM');
+ cancelChild();
},
};
}
diff --git a/server/services/hfDownload.lifecycle.test.js b/server/services/hfDownload.lifecycle.test.js
new file mode 100644
index 0000000000..9909ffc2ae
--- /dev/null
+++ b/server/services/hfDownload.lifecycle.test.js
@@ -0,0 +1,55 @@
+import { EventEmitter } from 'node:events';
+import { PassThrough } from 'node:stream';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { spawn } from '../lib/childProcess.js';
+import { downloadHfRepo } from './hfDownload.js';
+
+vi.mock('../lib/childProcess.js', async (importOriginal) => ({
+ ...await importOriginal(), spawn: vi.fn(),
+}));
+vi.mock('./hfToken.js', () => ({ getHfTokenInfo: async () => ({ token: null }) }));
+
+afterEach(() => {
+ vi.useRealTimers();
+ vi.clearAllMocks();
+});
+
+describe('Hugging Face download cancellation', () => {
+ it.each([false, true])('settles cancellation when SIGTERM is ignored: %s', async (ignoresTerm) => {
+ vi.useFakeTimers();
+ const child = Object.assign(new EventEmitter(), {
+ stdout: new PassThrough(), stderr: new PassThrough(),
+ exitCode: null, signalCode: null, killed: false,
+ });
+ const close = (signal) => {
+ child.signalCode = signal;
+ child.emit('close', null, signal);
+ };
+ child.kill = vi.fn((signal) => {
+ child.killed = true;
+ if (signal === 'SIGKILL' || !ignoresTerm) queueMicrotask(() => close(signal));
+ return true;
+ });
+ const spawned = Promise.withResolvers();
+ spawn.mockImplementation(() => { spawned.resolve(); return child; });
+ const onEvent = vi.fn();
+ const handle = downloadHfRepo({ repo: 'example/model', pythonPath: 'fixture-python', onEvent });
+ try {
+ await spawned.promise;
+ handle.kill();
+ handle.kill();
+ await vi.advanceTimersByTimeAsync(8_000);
+ expect(child.kill.mock.calls.map(([signal]) => signal)).toEqual(
+ ignoresTerm ? ['SIGTERM', 'SIGKILL'] : ['SIGTERM'],
+ );
+ await expect(handle.promise).resolves.toMatchObject({ ok: false, errorKind: 'cancelled' });
+ expect(onEvent).toHaveBeenLastCalledWith(expect.objectContaining({ kind: 'cancelled' }));
+ expect(vi.getTimerCount()).toBe(0);
+ } finally {
+ if (child.signalCode === null) close('SIGKILL');
+ await handle.promise;
+ child.stdout.destroy();
+ child.stderr.destroy();
+ }
+ });
+});
diff --git a/server/services/ollamaManager.js b/server/services/ollamaManager.js
index a1f08d209f..7e5ffbf19d 100644
--- a/server/services/ollamaManager.js
+++ b/server/services/ollamaManager.js
@@ -1521,16 +1521,21 @@ function hfImportError(status, statusText) {
async function downloadHfImportFile({ repo, remotePath, destination, headers, progress }) {
const url = buildHfResolveUrl(repo, 'main', remotePath)
- const response = await fetchWithTimeout(url, { headers }, 0)
- if (!response.ok || !response.body) throw new Error(hfImportError(response.status, response.statusText))
- await ensureDir(dirname(destination))
- const tracker = new Transform({
- transform(chunk, _encoding, callback) {
- progress(chunk.length)
- callback(null, chunk)
- }
- })
- await pipeline(Readable.fromWeb(response.body), tracker, createWriteStream(destination))
+ try {
+ const response = await fetchWithTimeout(url, { headers }, 0)
+ if (!response.ok || !response.body) throw new Error(hfImportError(response.status, response.statusText))
+ await ensureDir(dirname(destination))
+ const tracker = new Transform({
+ transform(chunk, _encoding, callback) {
+ progress(chunk.length)
+ callback(null, chunk)
+ }
+ })
+ await pipeline(Readable.fromWeb(response.body), tracker, createWriteStream(destination))
+ } catch (err) {
+ if (!(err instanceof TypeError) || !/^(fetch failed|terminated)$/.test(err.message)) throw err
+ throw new Error('Hugging Face checkpoint download was interrupted. Check the connection and retry the installation.', { cause: err })
+ }
}
/**
diff --git a/server/services/ollamaManager.test.js b/server/services/ollamaManager.test.js
index afe3260759..60cb329319 100644
--- a/server/services/ollamaManager.test.js
+++ b/server/services/ollamaManager.test.js
@@ -733,6 +733,37 @@ describe('ollamaManager curated Hugging Face Safetensors import', () => {
expect(onProgress).toHaveBeenCalledWith(expect.objectContaining({ importing: true }))
})
+ it.each(['request', 'body'])('explains how to recover from an interrupted checkpoint %s', async (phase) => {
+ hfTokenMock.token = 'hf_example_token'
+ const transportError = new TypeError(phase === 'request' ? 'fetch failed' : 'terminated', {
+ cause: Object.assign(new Error('other side closed'), { code: 'UND_ERR_SOCKET' })
+ })
+ vi.stubGlobal('fetch', vi.fn(async (url) => {
+ const href = String(url)
+ if (href.endsWith('/api/version')) return versionResponse()
+ if (href.includes('/api/models/example/Qwen-MLX')) {
+ return Response.json({ siblings: [
+ { rfilename: '4-bit/config.json', size: 2 },
+ { rfilename: '4-bit/model.safetensors', size: 7 }
+ ] })
+ }
+ if (href.endsWith('/config.json')) return new Response('{}')
+ if (href.endsWith('/model.safetensors')) {
+ if (phase === 'request') throw transportError
+ return new Response(new ReadableStream({ start(controller) { controller.error(transportError) } }))
+ }
+ throw new Error(`unexpected fetch: ${href}`)
+ }))
+ execMock.impl = (_cmd, args, _opts, cb) => args[0] === '--version'
+ ? cb(null, { stdout: 'ollama 0.31.0', stderr: '' })
+ : cb(new Error('An interrupted download must not reach ollama create'))
+ const { importModelFromHfSafetensors } = await loadManager()
+
+ await expect(importModelFromHfSafetensors(spec)).resolves.toMatchObject({
+ success: false, error: expect.stringMatching(/Hugging Face checkpoint download.*interrupted.*connection.*retry/i)
+ })
+ })
+
it('fails before downloading when the gated repository has no token', async () => {
stubFetch([])
const { importModelFromHfSafetensors } = await loadManager()