Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
// SSE driver for HuggingFace repo pre-download endpoints. Image gen and video
// gen both expose `POST /…/models/:id/download` (and video also a separate
// `/text-encoder/download`); without this helper each route open-codes the
// same `writeHead → send → cache-check → in-flight check → spawn → cleanup`
// flow. A single in-flight Map keyed by repo also dedupes across routes:
// FLUX-family repos referenced by both image and video gen would otherwise
// spawn two concurrent children.
//
// Lives in services rather than lib because it drives `downloadHfRepo` — a
// python child process that reads settings for its venv and HF token. The pure
// SSE plumbing it builds on (`openSseStream`, `onClientDisconnect`) stays in
// `lib/sseDownload.js` (issue #4901).
import { findCachedRepoFile, inspectModelCache, repoToDirName } from '../lib/hfCache.js';
import { hfInventoryRow } from '../lib/modelInventory.js';
import { openSseStream, onClientDisconnect } from '../lib/sseDownload.js';
import { downloadHfRepo } from './hfDownload.js';
import { recordModelInstall } from './modelManifest.js';
const inFlight = new Map(); // repo -> { promise, kill }
export async function startHfDownloadStream({ req, res, repo, revision = null, repos, fallbacks, cachedFile = null, alreadyDownloadedMessage, force = false, pythonPath = null }) {
// Three input shapes, two semantics:
// - `repo` (single string) / `repos` (ordered array) — ALL must succeed. Used
// when a model has auxiliary repos that must be present alongside the main
// weights (e.g. HiDream's separate Llama-3.1 text encoder). Sequential;
// short-circuits on any single-repo error.
// - `fallbacks` (ordered array of `{ repo, only? }`) — FIRST SUCCESS wins, and
// a failure advances to the next entry. Used when the same file lives in
// several repos with different access (Ingredients: gated first-party repo,
// then an un-gated mirror), so a user with no HF token still succeeds.
//
// `only` (array of exact repo-relative filenames, per entry) forwards to the
// helper's single-file mode — MANDATORY for aggregate repos where a snapshot is
// catastrophic (the ~708 GB DeepBeepMeep/LTX-2 mirror).
//
// `ignore` (array of fnmatch globs, per entry) is the inverse: still snapshot
// the repo, but drop paths the runtime never loads — extra checkpoint formats
// or a bundled sibling model. Ignored when `only` is set for the same entry.
//
// `cachedFile` is the "is it already here?" predicate for a single-file pull:
// `inspectModelCache` reports the whole repo cached as soon as ANY weight is
// resident, which for an aggregate mirror is true after the first unrelated
// file — so the generic already-cached short-circuit would skip a weight the
// user doesn't have. Callers pass an async `() => boolean` that checks the one
// file instead.
const normalizeOnly = (v) => (Array.isArray(v) ? v.filter((f) => typeof f === 'string' && f.length > 0) : []);
const normalizeRevision = (v) => (typeof v === 'string' && v.length > 0 ? v : null);
const normalizeTarget = (target) => {
if (typeof target === 'string' && target.length > 0) return { repo: target, only: [], ignore: [], revision: null };
if (!target || typeof target.repo !== 'string' || target.repo.length === 0) return null;
return {
repo: target.repo,
only: normalizeOnly(target.only),
ignore: normalizeOnly(target.ignore),
revision: normalizeRevision(target.revision),
};
};
const firstSuccessWins = Array.isArray(fallbacks);
const targetInputs = firstSuccessWins
? fallbacks
: (Array.isArray(repos) ? repos : [revision ? { repo, revision } : repo]);
const targets = targetInputs
.map(normalizeTarget)
.filter(Boolean);
const { send, safeEnd } = openSseStream(res);
if (targets.length === 0) {
send({ type: 'error', message: 'No repo specified for download.' });
return safeEnd();
}
// Disconnect bookkeeping wired BEFORE the cache-inspection await. The
// inspection can take double-digit ms on a cold cache; without this the
// client closing mid-await would land after spawn with no kill path.
let currentHandle = null;
let aborted = false;
onClientDisconnect(req, res, () => {
aborted = true;
// Only a live handle means a download was mid-flight, so this keeps the
// cancel line out of the log when the client simply navigated away between
// repos.
if (currentHandle) {
console.log(`🛑 HuggingFace download cancelled (client disconnect)`);
currentHandle.kill();
}
safeEnd();
});
let downloadedAny = false;
let totalSize = 0;
// Only meaningful in first-success-wins mode: which repo actually delivered,
// and the per-attempt failures to report if none did.
let succeededRepo = null;
const attemptErrors = [];
// The typed error kinds the failed attempts reported. When every attempt
// failed the SAME way — always true for a single-candidate spec — that kind is
// still the honest verdict, so it survives into the final frame instead of
// being flattened to a generic `all_sources_failed` the UI can't act on. The
// one that matters is `gated_repo`: the client keys its "accept the license
// and save an HF token" affordance off it.
const attemptKinds = [];
for (let i = 0; i < targets.length; i += 1) {
const { repo: r, only: onlyFiles, ignore: ignorePatterns, revision } = targets[i];
const isLastTarget = i === targets.length - 1;
if (aborted) return;
// For a single-file pull the repo-wide cache verdict is both the wrong
// question AND too expensive to ask: `inspectModelCache` recursively stats
// every weight in the snapshot, and an aggregate mirror reports `cached` off
// any unrelated resident file — so it would skip the file the user actually
// asked for after walking hundreds of GB to get there. Skip the walk entirely
// and defer to the caller's per-file predicate.
const singleFile = onlyFiles.length > 0;
const existing = singleFile
? { cached: false, sizeBytes: 0 }
: await inspectModelCache(r, { revision });
if (aborted) return;
const alreadyHave = singleFile
? (typeof cachedFile === 'function'
? await cachedFile({ repo: r, only: onlyFiles, revision })
: (await Promise.all(onlyFiles.map((file) => findCachedRepoFile(r, file, { revision })))).every(Boolean))
: existing.cached;
if (aborted) return;
// `force` is set by a repair-initiated re-download: repairModelCache()
// deleted the corrupt file(s), but for a multi-shard repo the remaining
// shards still make inspectModelCache() report `cached` — so skipping here
// would leave the just-deleted shard missing and never re-fetched. Forcing
// re-runs the HF fetch, which is a cheap no-op for files already present
// (etag match) and pulls only what's gone.
if (alreadyHave && !force) {
// A single-file pull has no repo-wide total to report (we deliberately
// never computed one) — and claiming the aggregate mirror's resident bytes
// would be a wildly wrong number on the badge anyway. Report 0 and let the
// badge fall back to the registry's size estimate.
if (singleFile) {
console.log(`📦 HuggingFace weight already cached: ${onlyFiles.join(', ')} (${r})`);
send({ type: 'log', message: `${onlyFiles.join(', ')} already cached.`, repo: r, sizeBytes: 0 });
} else {
totalSize += existing.sizeBytes || 0;
console.log(`📦 HuggingFace repo already cached: ${r} (${existing.sizeBytes} bytes)`);
send({ type: 'log', message: `${r} already cached (${existing.sizeBytes} bytes).`, repo: r, sizeBytes: existing.sizeBytes });
}
// In first-success-wins mode the file being present is the whole goal —
// don't fall through and try the mirror for something we already have.
if (firstSuccessWins) { succeededRepo = r; break; }
continue;
}
// Dedupe key includes the filenames: an aggregate repo hosts many unrelated
// weights, so two single-file pulls of DIFFERENT files from the same repo are
// not duplicates and must not block each other.
const revisionKey = revision ? `@${revision}` : '';
const flightKey = singleFile ? `${r}${revisionKey}::${onlyFiles.join(',')}` : `${r}${revisionKey}`;
if (inFlight.has(flightKey)) {
console.log(`⏭️ HuggingFace download already running for ${flightKey} — refusing duplicate`);
send({ type: 'error', message: `Another download for ${r} is already running.`, kind: 'already_running', repo: r });
return safeEnd();
}
// Server-side visibility for a pull that used to surface only on the SSE
// stream (the browser) — a headless/PM2 log had no record the multi-GB
// fetch ever ran. Log the start, each file as it streams, and the outcome.
console.log(`⬇️ Downloading HuggingFace repo: ${r}${singleFile ? ` (${onlyFiles.length} file(s) only)` : ''}${force ? ' (forced re-fetch)' : ''}`);
const handle = downloadHfRepo({
repo: r,
revision,
only: singleFile ? onlyFiles : null,
ignore: ignorePatterns,
pythonPath,
onEvent: (ev) => {
// File-start frames only. Byte ticks arrive several times a second on
// a large single-file pull and would drown the PM2 log.
if (ev.type === 'progress' && ev.file && ev.downloaded == null && ev.stage !== 'verify') {
console.log(`⬇️ ${r}: ${ev.file} (${ev.step}/${ev.total})`);
}
// A failure that we're about to retry against the next candidate isn't a
// user-facing error — downgrade it to a log so the UI doesn't flash a red
// "gated repo" banner for something the mirror then delivers fine.
if (firstSuccessWins && ev.type === 'error') {
send({
type: 'log',
message: `${r}: ${ev.message}${isLastTarget ? '' : ' — trying the next source.'}`,
repo: r,
});
return;
}
send({ ...ev, repo: r });
},
});
currentHandle = handle;
inFlight.set(flightKey, handle);
let result;
try {
result = await handle.promise;
if (result?.ok) {
console.log(`✅ HuggingFace download complete: ${r} (${result.sizeBytes || 0} bytes)`);
// Every HF weight pull in PortOS lands here — image gen, video gen and
// music all drive this one stream — so this is the single place the model
// manifest learns a Hub repo is now on disk. A single-file pull reports
// only the bytes IT moved, which is a floor rather than the directory's
// size, hence the estimate flag; the next reconcile replaces it with the
// measured total.
await recordModelInstall({
...hfInventoryRow({
dirName: repoToDirName(r),
name: r,
detail: r,
sizeBytes: result.sizeBytes,
sizeIsEstimate: singleFile,
}),
source: 'download',
});
} else if (result?.errorKind !== 'cancelled') {
console.error(`❌ HuggingFace download failed: ${r} — ${result?.errorMessage || 'unknown error'}`);
}
} finally {
inFlight.delete(flightKey);
currentHandle = null;
}
if (!firstSuccessWins) {
// ALL must succeed here — unlike the fallback-chain branch below, there
// is no next candidate to fall through to. Silently treating a failed
// repo as done (the prior bug) let the loop reach `complete` even
// though one of the required repos never downloaded.
if (!result?.ok) {
if (result?.errorKind === 'cancelled') return safeEnd();
send({
type: 'error',
kind: result?.errorKind || 'download_failed',
message: `${r}: ${result?.errorMessage || 'unknown error'}`,
repo: r,
});
return safeEnd();
}
downloadedAny = true;
totalSize += result.sizeBytes || 0;
continue;
}
if (result?.ok) {
downloadedAny = true;
succeededRepo = r;
totalSize += result.sizeBytes || 0;
break;
}
// A user cancel must not silently roll onto the next candidate.
if (result?.errorKind === 'cancelled') return safeEnd();
attemptErrors.push(`${r}: ${result?.errorMessage || 'unknown error'}`);
attemptKinds.push(result?.errorKind || 'download_failed');
}
if (!aborted) {
if (firstSuccessWins) {
if (!succeededRepo) {
const sharedKind = attemptKinds.length > 0 && attemptKinds.every((k) => k === attemptKinds[0])
? attemptKinds[0]
: 'all_sources_failed';
send({
type: 'error',
kind: sharedKind,
message: `Every source failed — ${attemptErrors.join('; ')}`,
});
return safeEnd();
}
send({
type: 'complete',
message: downloadedAny
? `${succeededRepo} downloaded.`
: (alreadyDownloadedMessage || `${succeededRepo} already downloaded.`),
repos: [succeededRepo],
sizeBytes: totalSize,
});
return safeEnd();
}
const names = targets.map((t) => t.repo);
let message;
if (names.length === 1) {
// Preserve legacy single-repo `complete` message semantics: if it was
// already cached on entry, surface the caller's `alreadyDownloadedMessage`
// (or the default already-downloaded line).
message = !downloadedAny
? (alreadyDownloadedMessage || `${names[0]} already downloaded.`)
: `${names[0]} downloaded.`;
} else {
message = downloadedAny
? `Downloaded ${names.length} repos: ${names.join(', ')}`
: `All ${names.length} repos already cached: ${names.join(', ')}`;
}
send({ type: 'complete', message, repos: names, sizeBytes: totalSize });
}
safeEnd();
}