Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

5.7 KB
Raw Blame
/**
 * User-installed audio model registry — augments the shipped music-gen engine
 * model lists (server/services/pipeline/musicGen.js `ENGINES[id].models`) with
 * extra HuggingFace checkpoints the user downloads via the Music studio.
 *
 * The SHIPPED models live in code (musicGen.js); this registry holds only the
 * user's ADDITIONS, persisted to `data/audio-models.json` keyed by engine id:
 *
 *   { "musicgen": [ { "id": "facebook/musicgen-large", "repo": "facebook/musicgen-large", "name": "MusicGen Large" } ],
 *     "audioldm2": [], "acestep": [] }
 *
 * `listEngineModels(engineId)` merges shipped + user models (shipped first,
 * deduped by id) so the route/UI present one combined list. A user model's `id`
 * IS its HF repo id, which the sidecar passes straight to `--model`, so a
 * downloaded checkpoint is selectable for generation immediately (no restart —
 * the JSON is read on each list call, unlike the code-level defaults).
 *
 * Install flow: the route downloads the HF repo via `downloadHfRepo` (shared
 * with the image/video model installer) into the local HF cache, then calls
 * `addAudioModel` here to register it. Removing a model only de-registers it
 * (the cached weights are managed by the HF cache, not deleted here).
 */

import { join } from 'path';
import { readJSONFile, atomicWrite, ensureDir } from '../lib/fileUtils.js';
import { PATHS } from '../lib/fileUtils.js';
import { getEngine, ENGINES } from './pipeline/musicGen.js';
import { isNonBlankStr } from '../lib/textUtils.js';

const REGISTRY_FILE = join(PATHS.data, 'audio-models.json');

// HF repo ids look like `org/name` (optionally with extra path segments). Keep
// the guard permissive but reject path-traversal / whitespace / control chars.
const REPO_RE = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._\/-]+$/;

export function isValidRepoId(repo) {
  return isNonBlankStr(repo) && REPO_RE.test(repo.trim()) && !repo.includes('..');
}

// One sanitized user-model entry, or null to drop. `id` defaults to the repo id
// (what the sidecar's --model wants); `name` defaults to the repo's basename.
function sanitizeUserModel(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const repo = isNonBlankStr(raw.repo) ? raw.repo.trim() : '';
  if (!isValidRepoId(repo)) return null;
  const id = isNonBlankStr(raw.id) ? raw.id.trim() : repo;
  const name = isNonBlankStr(raw.name) ? raw.name.trim().slice(0, 200) : repo.split('/').pop();
  return { id, repo, name };
}

async function loadRegistry() {
  // Strict: add/remove both persist the loaded registry. A swallowed failure
  // would replace every other engine's user-installed model rows.
  const raw = await readJSONFile(REGISTRY_FILE, {}, { strict: true });
  const out = {};
  if (raw && typeof raw === 'object') {
    for (const engineId of Object.keys(ENGINES)) {
      const list = Array.isArray(raw[engineId]) ? raw[engineId] : [];
      out[engineId] = list.map(sanitizeUserModel).filter(Boolean);
    }
  } else {
    for (const engineId of Object.keys(ENGINES)) out[engineId] = [];
  }
  return out;
}

async function saveRegistry(registry) {
  await ensureDir(PATHS.data);
  await atomicWrite(REGISTRY_FILE, registry);
}

/** User-added models for one engine (sanitized). */
export async function listUserModels(engineId) {
  const reg = await loadRegistry();
  return reg[engineId] || [];
}

/**
 * The combined model list for an engine: shipped defaults (from musicGen.js)
 * first, then user-added, deduped by id. Returned shape is `{ id, name, repo?, revision? }`
 * matching the shipped models so the route/UI treat them uniformly. `userAdded`
 * marks the entries this registry contributed (so the UI can offer "remove").
 */
export async function listEngineModels(engineId) {
  const engine = getEngine(engineId);
  const shipped = engine.models.map((m) => ({
    id: m.id,
    name: m.name,
    repo: m.repo,
    ...(m.revision ? { revision: m.revision } : {}),
    userAdded: false,
  }));
  const seen = new Set(shipped.map((m) => m.id));
  const user = (await listUserModels(engine.id))
    .filter((m) => !seen.has(m.id))
    .map((m) => ({ id: m.id, name: m.name, repo: m.repo, userAdded: true }));
  return [...shipped, ...user];
}

/**
 * Register a downloaded HF checkpoint for an engine. Idempotent: re-adding the
 * same repo updates its display name rather than duplicating. Rejects an unknown
 * engine or an invalid repo id. Returns the sanitized entry.
 */
export async function addAudioModel({ engine: engineId, repo, name }) {
  if (!ENGINES[engineId]) {
    throw Object.assign(new Error(`Unknown audio engine: ${engineId}`), { status: 400, code: 'AUDIO_MODEL_UNKNOWN_ENGINE' });
  }
  const entry = sanitizeUserModel({ repo, name, id: repo });
  if (!entry) {
    throw Object.assign(new Error('Invalid HuggingFace repo id'), { status: 400, code: 'AUDIO_MODEL_INVALID_REPO' });
  }
  const reg = await loadRegistry();
  const list = reg[engineId] || [];
  const idx = list.findIndex((m) => m.id === entry.id);
  if (idx >= 0) list[idx] = entry;
  else list.push(entry);
  reg[engineId] = list;
  await saveRegistry(reg);
  console.log(`🎚️ Registered audio model for ${engineId}: ${entry.repo}`);
  return entry;
}

/**
 * De-register a user-added model (does NOT touch the cached weights — the HF
 * cache owns those). Returns true if an entry was removed. A shipped default id
 * is a no-op (not in the user registry) — shipped models can't be removed here.
 */
export async function removeAudioModel({ engine: engineId, id }) {
  const reg = await loadRegistry();
  const list = reg[engineId] || [];
  const next = list.filter((m) => m.id !== id);
  if (next.length === list.length) return false;
  reg[engineId] = next;
  await saveRegistry(reg);
  console.log(`🎚️ De-registered audio model for ${engineId}: ${id}`);
  return true;
}