Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

13.9 KB
Raw Blame
// Pre-download a HuggingFace repo snapshot into the local HF cache, with
// SSE-friendly progress events. Used by the inline "Download" badge on
// the image + video gen forms so users don't discover a multi-GB pull
// the first time they hit Render.
//
// Picks a Python venv that has `huggingface_hub` installed. The FLUX.2 venv
// is the preferred choice (always installed when image gen is set up); the
// mflux/legacy pythonPath is the fallback for installs that use only mflux.
//
// Wire protocol parses the stdout/stderr lines from scripts/hf_download_repo.py:
//   STAGE:list                                  -> { type: 'stage', stage: 'list' }
//   STAGE:download:<n>/<total>:<file>           -> progress at the START of file n
//   STAGE:bytes:<n>/<total>:<got>/<size>:<file> -> byte progress for file n
//   STAGE:verify:<n>/<total>:<file>             -> transfer done; committing/hashing
//   STAGE:complete:<bytes>                      -> { type: 'complete', sizeBytes }
//   USER_ERROR:<kind>:<detail>                  -> typed-error capture; <detail>
//                                                  is the repo id for list/auth
//                                                  failures and the filename
//                                                  for per-file download errors
//   ❌ <prose>                                   -> errorMessage
// Unknown lines fall through as raw `{ type: 'log', message }`.
//
// parseHfDownloadLine is the single decoder — downloadHfRepo and the tests
// share it so a wire-shape change cannot drift between them.

import { spawn } from '../lib/childProcess.js';
import { join } from 'node:path';
import {
  resolveFlux2Python, isFlux2VenvHealthy, HF_HUB_PYTHON_RESOLVERS,
} from '../lib/pythonSetup.js';
import { PATHS } from '../lib/fileUtils.js';
import { getHfTokenInfo } from './hfToken.js';
import { safeChildProcessEnv, safeChildProcessOptions } from '../lib/processEnv.js';
import { createLineReader } from '../lib/streamLines.js';
import { killWithEscalation } from '../lib/killWithEscalation.js';
import { getSettings } from './settings.js';

const HELPER_SCRIPT = join(PATHS.root, 'scripts', 'hf_download_repo.py');

/**
 * Decode one helper stdout/stderr line into a typed action.
 *
 *   { type: 'event', event }         — emit via onEvent
 *   { type: 'complete', sizeBytes }  — record resident bytes; do not emit yet
 *   { type: 'user_error', errorKind }
 *   { type: 'error_message', message }
 *   { type: 'ignore' }
 */
export function parseHfDownloadLine(raw) {
  const line = String(raw).trim();
  if (!line) return { type: 'ignore' };

  if (line.startsWith('STAGE:')) {
    const body = line.slice('STAGE:'.length);
    const colon = body.indexOf(':');
    const stage = colon === -1 ? body : body.slice(0, colon);
    const detail = colon === -1 ? '' : body.slice(colon + 1);

    if (stage === 'download') {
      // `3/47:model-00003-of-00047.safetensors` — this fires when a file
      // STARTS, so the bar sits at the beginning of that file. Using
      // step/total here made a 1-file 50 GB pull report 100% for the entire
      // transfer.
      const m = detail.match(/^(\d+)\/(\d+):(.+)$/);
      if (m) {
        const step = parseInt(m[1], 10);
        const total = parseInt(m[2], 10);
        return {
          type: 'event',
          event: {
            type: 'progress',
            stage: 'download',
            progress: total > 0 ? (step - 1) / total : 0,
            step,
            total,
            file: m[3],
          },
        };
      }
    }

    if (stage === 'bytes') {
      // `1/1:26843545600/51506295440:qwen3vl_….safetensors`
      const m = detail.match(/^(\d+)\/(\d+):(\d+)\/(\d+):(.+)$/);
      if (m) {
        const step = parseInt(m[1], 10);
        const files = parseInt(m[2], 10);
        const downloaded = parseInt(m[3], 10);
        const totalBytes = parseInt(m[4], 10);
        const fileFrac = totalBytes > 0 ? Math.min(1, downloaded / totalBytes) : 0;
        return {
          type: 'event',
          event: {
            type: 'progress',
            stage: 'download',
            progress: files > 0 ? ((step - 1) + fileFrac) / files : fileFrac,
            step,
            total: files,
            downloaded,
            totalBytes,
            file: m[5],
          },
        };
      }
    }

    if (stage === 'verify') {
      const m = detail.match(/^(\d+)\/(\d+):(.+)$/);
      if (m) {
        const step = parseInt(m[1], 10);
        const total = parseInt(m[2], 10);
        return {
          type: 'event',
          event: {
            type: 'progress',
            stage: 'verify',
            progress: total > 0 ? step / total : 1,
            step,
            total,
            file: m[3],
          },
        };
      }
    }

    if (stage === 'complete') {
      const bytes = parseInt(detail, 10);
      return { type: 'complete', sizeBytes: Number.isFinite(bytes) ? bytes : 0 };
    }

    return { type: 'event', event: { type: 'stage', stage, detail } };
  }

  if (line.startsWith('USER_ERROR:')) {
    const body = line.slice('USER_ERROR:'.length);
    const colon = body.indexOf(':');
    return { type: 'user_error', errorKind: colon === -1 ? body : body.slice(0, colon) };
  }

  if (line.startsWith('❌')) {
    return { type: 'error_message', message: line.replace(/^❌\s*/, '') };
  }

  // Mirrored by STAGE:download — the helper prints both so older regexes still match.
  if (line.startsWith('DOWNLOAD:')) return { type: 'ignore' };

  return { type: 'event', event: { type: 'log', message: line } };
}

// Resolve a Python interpreter with huggingface_hub installed. Order: FLUX.2
// venv (the modern path; always has hf_hub via diffusers), then the
// settings.imageGen.local.pythonPath (mflux installs), then any provisioned
// MUSIC venv (acestep/audioldm2/musicgen all install huggingface_hub) so the
// Track editor's "Install model" button works in a MUSIC-ONLY setup with no
// image-gen configured. Returns null if none is available — the caller
// surfaces a user-facing error rather than silently failing the download.
//
// Health-gate the FLUX.2 venv: an interrupted install leaves the binary in
// place but no packages, and resolveFlux2Python() alone would still return
// that broken interpreter. Every download would then fail on the broken
// venv before reaching the working fallback.
export async function resolveHfDownloadPython(preferredPython = null) {
  if (preferredPython) return preferredPython;
  const flux2 = resolveFlux2Python();
  if (flux2 && await isFlux2VenvHealthy()) return flux2;
  const settings = await getSettings();
  if (settings?.imageGen?.local?.pythonPath) return settings.imageGen.local.pythonPath;
  // Any provisioned music venv is a valid hf_hub host — used when only music
  // gen is set up. pythonSetup owns the list so it can't drift out of step with
  // the resolvers themselves (see HF_HUB_PYTHON_RESOLVERS).
  for (const resolve of HF_HUB_PYTHON_RESOLVERS) {
    const found = resolve();
    if (found) return found;
  }
  return null;
}

// Argv for scripts/hf_download_repo.py. Pure and exported so the flag contract
// is unit-tested without spawning python — the same split as buildSidecarArgs in
// services/pipeline/musicGen.js. Returns `onlyFiles` too because the caller
// branches its progress copy on single-file mode.
export function buildHfDownloadArgs({ repo, revision = null, only = null, ignore = null, tokenEnv = 'HF_TOKEN' }) {
  const globList = (v) => (Array.isArray(v) ? v.filter((f) => typeof f === 'string' && f.length > 0) : []);
  const args = [HELPER_SCRIPT, '--repo', repo, '--token-env', tokenEnv];
  if (revision) args.push('--revision', revision);
  const onlyFiles = globList(only);
  for (const f of onlyFiles) args.push('--only', f);
  // The helper hard-errors on --only with --ignore (single-file mode never
  // enumerates the repo, so there is nothing to filter). Single-file mode wins.
  if (!onlyFiles.length) for (const pat of globList(ignore)) args.push('--ignore', pat);
  return { args, onlyFiles };
}

// Returns `{ promise, kill }`. The promise resolves with `{ ok, sizeBytes,
// errorKind, errorMessage }`. `kill()` cancels the python child and escalates
// if needed so a disconnected SSE client cannot strand the download slot.
//
// `only` is an array of exact repo-relative filenames. When set, the helper
// runs in SINGLE-FILE mode: it never enumerates the repo and fetches only those
// files. This is MANDATORY for aggregate repos — `DeepBeepMeep/LTX-2` mirrors
// every LTX weight in one ~708 GB repo, so a snapshot would fill the user's
// disk to pull one 1.3 GB IC-LoRA (see server/lib/icLoraWeights.js).
//
// `ignore` is an array of fnmatch globs dropped from the enumerated file list —
// the inverse tool, for repos that ship extra checkpoint formats the runtime
// never loads (MiniMax Music 3 carries a 20 GB captioning model and 10 GB of
// original-format .pth beside the 29 GB the diffusers pipeline actually reads).
// It is ignored in single-file mode, where the helper never enumerates at all.
export function downloadHfRepo({ repo, revision = null, only = null, ignore = null, pythonPath: preferredPython = null, onEvent }) {
  let proc = null;
  let killed = false;
  let processClosed = false;
  let cancelEscalation = null;
  let errorKind = null;
  let errorMessage = null;
  let sizeBytes = 0;

  const cancelChild = () => {
    if (!proc || processClosed || cancelEscalation || proc.exitCode !== null || proc.signalCode !== null) return;
    cancelEscalation = killWithEscalation(proc, {
      label: 'HuggingFace download', stillRunning: () => !processClosed,
    });
  };
  const finishProcess = () => {
    processClosed = true;
    clearTimeout(cancelEscalation);
  };

  const promise = (async () => {
    const pythonPath = await resolveHfDownloadPython(preferredPython);
    // Cancel-before-spawn check. resolveHfDownloadPython runs an
    // isFlux2VenvHealthy() probe (several hundred ms cold) and getHfTokenInfo
    // does file I/O; a kill() landing inside either await otherwise still
    // lets the spawn fire below, leaving a multi-GB HF download running with
    // no SSE client to consume progress and holding the inFlight slot until
    // the whole snapshot finishes.
    if (killed) return { ok: false, errorKind: 'cancelled', errorMessage: 'Cancelled' };
    if (!pythonPath) {
      const msg = 'No Python runtime with huggingface_hub is ready. Install or repair the selected runtime from the model setup panel.';
      onEvent({ type: 'error', message: msg, kind: 'venv_missing' });
      return { ok: false, errorKind: 'venv_missing', errorMessage: msg };
    }
    const { token } = await getHfTokenInfo();
    if (killed) return { ok: false, errorKind: 'cancelled', errorMessage: 'Cancelled' };
    const env = safeChildProcessEnv();
    // The Python helper looks up the token by env-var name so we don't have
    // to pass secrets on argv. Strip any stale value when the user has
    // explicitly cleared their stored token.
    if (token) env.HF_TOKEN = token;
    else delete env.HF_TOKEN;

    const { args, onlyFiles } = buildHfDownloadArgs({ repo, revision, only, ignore });

    onEvent({
      type: 'stage',
      stage: 'starting',
      message: onlyFiles.length
        ? `Downloading ${onlyFiles.join(', ')} from ${repo}…`
        : `Downloading ${repo}…`,
    });

    return new Promise((resolve) => {
      proc = spawn(pythonPath, args, safeChildProcessOptions({ env, stdio: ['ignore', 'pipe', 'pipe'] }));

      const handleLine = (raw) => {
        const parsed = parseHfDownloadLine(raw);
        if (parsed.type === 'ignore') return;
        if (parsed.type === 'complete') {
          sizeBytes = parsed.sizeBytes;
          // Don't emit a `complete` event here — wait for the close
          // handler so a successful exit code is required.
          return;
        }
        if (parsed.type === 'user_error') {
          errorKind = parsed.errorKind;
          return;
        }
        if (parsed.type === 'error_message') {
          errorMessage = parsed.message;
          return;
        }
        onEvent(parsed.event);
      };

      // Line-buffer across chunks so a STAGE:/USER_ERROR: marker split across
      // pipe boundaries isn't truncated and routed to the generic log path
      // (which loses the typed-error / progress wire shape). Separate reader
      // per stream — a shared carry buffer would splice stdout onto stderr.
      const stdoutReader = createLineReader(handleLine);
      const stderrReader = createLineReader(handleLine);
      proc.stderr.on('data', stderrReader.push);
      proc.stdout.on('data', stdoutReader.push);
      proc.on('error', (err) => {
        finishProcess();
        const msg = `Failed to spawn python: ${err.message}`;
        onEvent({ type: 'error', message: msg, kind: 'spawn_failed' });
        resolve({ ok: false, errorKind: 'spawn_failed', errorMessage: msg });
      });
      proc.on('close', (code, signal) => {
        finishProcess();
        // Flush any trailing partial line the python helper emitted without a
        // newline before exit (rare with line-buffered stderr but possible
        // when the process is SIGKILL'd mid-write).
        stdoutReader.flush();
        stderrReader.flush();
        if (killed) {
          onEvent({ type: 'error', message: 'Cancelled', kind: 'cancelled' });
          return resolve({ ok: false, errorKind: 'cancelled', errorMessage: 'Cancelled' });
        }
        if (code === 0) {
          onEvent({ type: 'complete', sizeBytes, repo });
          return resolve({ ok: true, sizeBytes });
        }
        const msg = errorMessage || (signal ? `Killed by ${signal}` : `Exit code ${code}`);
        onEvent({ type: 'error', message: msg, kind: errorKind, repo });
        return resolve({ ok: false, errorKind, errorMessage: msg });
      });
      // Terminal listeners must already own the child when cancellation starts.
      if (killed) cancelChild();
    });
  })();

  return {
    promise,
    kill: () => {
      if (killed) return;
      killed = true;
      cancelChild();
    },
  };
}