Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

13.5 KB
Raw Blame
/**
 * Audio → MIDI transcription via MuScriptor — shared by the Rounds workbench
 * (a reference's attached audio) and the Music Video parsing system (the
 * project's source track).
 *
 * MuScriptor (https://github.com/muscriptor/muscriptor) is a local
 * multi-instrument music-transcription model. It runs in an opt-in venv
 * (`INSTALL_MUSCRIPTOR=1 bash scripts/setup-image-video.sh`) through the
 * sidecar `scripts/transcribe_muscriptor.py`, which mirrors the generate_*
 * audio sidecars' STAGE:/RESULT: wire protocol. Model weights auto-download
 * from HuggingFace on first use; the sidecar emits a distinct `download-model`
 * stage (vs `load-model` for a cached load) so the client can show an honest
 * "Downloading model weights…" phase + toast instead of an opaque multi-minute
 * spinner — the STAGE lines keep the whole run visible.
 *
 * Job shape mirrors roundReferenceAudioImport.js (the lightweight SSE-job
 * pattern): kickoff returns `{ jobId }` immediately, the transcription runs
 * detached and streams `{ type: 'progress' | 'complete' | 'error' |
 * 'canceled' }` frames, and the produced `.mid` lands in the caller's
 * `destDir` — uploads for rounds (served at /api/uploads/<filename>), the
 * music dir for music video (so the file federates with the project's other
 * audio) — a filename pointer, consistent with every other audio field.
 */

import { randomUUID } from 'crypto';
import { existsSync, statSync } from 'fs';
import { unlink } from 'fs/promises';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { tmpdir } from 'os';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { PATHS, shortId, importFileToDir } from '../lib/fileUtils.js';
import { broadcastSse, attachSseClient as attachSse, closeJobAfterDelay } from '../lib/sseUtils.js';
import { MUSCRIPTOR_MODELS } from '../lib/muscriptorModels.js';
import { killWithEscalation } from '../lib/killWithEscalation.js';
import { hfChildEnv } from './hfToken.js';
import { isGatedRepoError, extractGatedRepo } from '../lib/hfErrors.js';
import { resolveMuscriptorPython, isMuscriptorRuntimeReady, MUSCRIPTOR_VENV_DEFAULT } from '../lib/pythonSetup.js';
import { runSidecarProcess, parseSidecarResult } from '../lib/sidecarProcess.js';
import { ServerError } from '../lib/errorHandler.js';

const __dirname = dirname(fileURLToPath(import.meta.url));
// The sidecar lives at the repo root — resolve module-relative so the path is
// correct regardless of the server process's cwd (mirrors pipeline/musicGen.js).
const MUSCRIPTOR_SCRIPT = join(__dirname, '../../scripts/transcribe_muscriptor.py');

// MuScriptor model sizes (weights auto-download on first use). Medium is the
// library default — a quality/speed balance; small is the CPU-friendly tier.
export { MUSCRIPTOR_MODELS };
export const DEFAULT_MUSCRIPTOR_MODEL = 'medium';

/** Clamp a requested model size onto the known set (route validates shape; this guards the value). */
export const resolveMuscriptorModel = (model) =>
  (MUSCRIPTOR_MODELS.includes(model) ? model : DEFAULT_MUSCRIPTOR_MODEL);

// The sidecar's `@install_hf_error_handler` (scripts/_runner_common.py) emits a
// structured `USER_ERROR:gated_repo:<repo>` line on a gated download — prefer it
// over prose-matching a stderr tail that sidecarProcess has already truncated to
// the last few lines. The repo group is optional (older/marker-less output).
const GATED_MARKER_RE = /USER_ERROR:gated_repo(?::(\S+))?/;

/**
 * Map a failed sidecar reason onto the SSE error frame's typed fields. Pure —
 * unit-tested without spawning Python.
 *
 * MuScriptor's weights live in a *gated* HuggingFace repo
 * (`MuScriptor/muscriptor-*`), so the first download with no accepted license
 * (or no token) 403s. Surface that as a typed `gated_repo` frame carrying the
 * repo so the client can deep-link the license page + token entry (reusing the
 * same `gated_repo` code the image runner emits) instead of dead-ending on a
 * raw traceback toast. Detection prefers the sidecar's structured marker and
 * falls back to the gated-error prose so a marker-less sidecar still classifies.
 * Any other failure passes through as a plain error string.
 */
export function classifyMidiFailure(reason, model) {
  const text = String(reason || '');
  const marker = text.match(GATED_MARKER_RE);
  if (marker || isGatedRepoError(text)) {
    const repo = (marker && marker[1]) || extractGatedRepo(text) || `MuScriptor/muscriptor-${resolveMuscriptorModel(model)}`;
    return {
      code: 'gated_repo',
      repo,
      error:
        `Access to ${repo} is gated on HuggingFace. Accept the license at `
        + `https://huggingface.co/${repo} and add your HuggingFace token in Image Gen settings, then retry.`,
    };
  }
  return { error: text || 'MIDI transcription failed' };
}

const MUSCRIPTOR_INSTALL_HINT =
  `MuScriptor runtime not found. Run \`INSTALL_MUSCRIPTOR=1 bash scripts/setup-image-video.sh\` `
  + `to bootstrap it (expected venv at ${MUSCRIPTOR_VENV_DEFAULT}).`;

/**
 * Build the sidecar `{ bin, args }`. Pure — unit-tested without spawning
 * Python (the buildSidecarArgs pattern from pipeline/musicGen.js).
 */
export function buildMuscriptorArgs({ pythonPath, scriptPath = MUSCRIPTOR_SCRIPT, audioPath, outputPath, model }) {
  return {
    bin: pythonPath,
    args: [
      scriptPath,
      '--audio', audioPath,
      '--output', outputPath,
      '--model', resolveMuscriptorModel(model),
    ],
  };
}

// jobId -> { clients, lastPayload, process }
const transcriptionJobs = new Map();

// ownerKey (e.g. a music-video project id) -> running jobId, so a reloaded page can reattach.
const activeByOwner = new Map();

/** The running transcription job id started for `ownerKey`, or null. */
export function getActiveMidiTranscriptionJobId(ownerKey) {
  const jobId = activeByOwner.get(ownerKey);
  return jobId && transcriptionJobs.get(jobId) && !transcriptionJobs.get(jobId).settled ? jobId : null;
}

export const attachMidiTranscriptionSseClient = (jobId, res) => attachSse(transcriptionJobs, jobId, res);

/**
 * Cancel an in-flight transcription. Returns false if the job is unknown or
 * already finished. The flag (not just the SIGTERM) is what makes cancel
 * reliable in the windows where no child exists yet/anymore — before the
 * spawn (env still resolving) and between the child's exit and the result
 * landing — the run body checks it at both points.
 */
export function cancelMidiTranscription(jobId) {
  const job = transcriptionJobs.get(jobId);
  if (!job || job.settled) return false;
  job.cancelRequested = true;
  const proc = job.process;
  if (proc) killWithEscalation(proc, { label: 'muscriptor transcription', stillRunning: () => job.process === proc });
  return true;
}

/**
 * Kick off an audio → MIDI transcription. Returns `{ jobId, model }`
 * immediately; the sidecar runs detached and streams progress over SSE.
 * Terminal frames: `{ type: 'complete', filename, model, ...extra }` (where
 * `extra` is whatever the optional `onComplete` callback returns — the music
 * video route uses it to persist the pointer on the project and hand the
 * updated record to the client), `{ type: 'error', error }`, or
 * `{ type: 'canceled' }`. Throws (before returning a jobId) when the
 * MuScriptor venv isn't provisioned, so that surfaces as a real 503 with the
 * install hint.
 *
 * `audioPath` must already be resolved + validated by the caller (safeUnder
 * against the owning directory). `outputName` seeds the landed `.mid`
 * filename's human-readable prefix. `destDir` picks where the `.mid` lands —
 * uploads by default (the rounds path); the music video route passes
 * `PATHS.music` so the file federates to peers alongside the project's other
 * audio (the peer-sync asset manifest only ships known directories).
 */
export async function startMidiTranscription({ audioPath, outputName = 'transcription', model, onComplete, destDir = PATHS.uploads, ownerKey = null }) {
  // Gate on the actual import, not just the binary — a partial venv (binary
  // present, `muscriptor` not importable) must still 503 so the in-app
  // installer re-opens to repair it instead of failing later in the sidecar.
  if (!(await isMuscriptorRuntimeReady())) {
    throw new ServerError(MUSCRIPTOR_INSTALL_HINT, { status: 503, code: 'MIDI_RUNTIME_MISSING' });
  }
  const pythonPath = resolveMuscriptorPython();
  if (!existsSync(audioPath)) {
    throw new ServerError('Audio file not found', { status: 404, code: 'NOT_FOUND' });
  }
  const resolvedModel = resolveMuscriptorModel(model);
  const jobId = randomUUID();
  const tempOut = join(tmpdir(), `portos-midi-${jobId}.mid`);
  const job = { id: jobId, status: 'running', clients: [], process: null, cancelRequested: false, settled: false };
  transcriptionJobs.set(jobId, job);
  if (ownerKey) activeByOwner.set(ownerKey, jobId);
  console.log(`🎹 MIDI transcription ${shortId(jobId)} [${resolvedModel}] — ${audioPath}`);

  (async () => {
    try {
      const { bin, args } = buildMuscriptorArgs({ pythonPath, audioPath, outputPath: tempOut, model: resolvedModel });
      broadcastSse(job, { type: 'progress', stage: 'starting' });
      // MuScriptor's weights live in a gated HF repo (MuScriptor/muscriptor-*),
      // so pass the user's HF token through for the first download to
      // authenticate. Without an accepted license the sidecar 403s and we
      // classify that into a typed gated_repo frame below.
      const env = await hfChildEnv();
      // A cancel can land while the env was resolving (no child to SIGTERM yet)
      // — honor the flag before spawning anything.
      if (job.cancelRequested) {
        console.log(`🛑 MIDI transcription ${shortId(jobId)} cancelled before spawn`);
        broadcastSse(job, { type: 'canceled' });
        return;
      }
      // STAGE: lines become SSE progress frames (and pm2 log lines, so a stuck
      // first-run weight download is visible); onProcess tracks the live child
      // for cancelMidiTranscription's killWithEscalation.
      const result = await runSidecarProcess({
        bin, args, env,
        onProcess: (proc) => { job.process = proc; },
        onStage: (stage, detail, raw) => {
          console.log(`🎹 muscriptor ${raw}`);
          broadcastSse(job, { type: 'progress', stage, detail });
        },
      });

      // `job.cancelRequested` covers a cancel that arrived after the child
      // exited cleanly but before the result landed — don't persist it.
      if (result.canceled || job.cancelRequested) {
        console.log(`🛑 MIDI transcription ${shortId(jobId)} cancelled`);
        broadcastSse(job, { type: 'canceled' });
        return;
      }
      // A clean exit isn't enough — require a parsed RESULT line AND a
      // non-empty file on disk before landing the pointer, so a runtime-shape
      // drift can't attach a dangling/empty MIDI file.
      const parsed = result.ok ? parseSidecarResult(result.stdout) : null;
      const wroteFile = (statSync(tempOut, { throwIfNoEntry: false })?.size ?? 0) > 0;
      if (!result.ok || !parsed || !wroteFile) {
        const reason = !result.ok ? result.reason : (!wroteFile ? 'sidecar wrote no MIDI' : 'sidecar returned no result');
        const failure = classifyMidiFailure(reason, resolvedModel);
        const err = new Error(failure.error);
        if (failure.code) err.code = failure.code;
        if (failure.repo) err.repo = failure.repo;
        throw err;
      }

      broadcastSse(job, { type: 'progress', stage: 'importing' });
      // The .mid copy, the row that names it and the discard unlink are one
      // backup-admitted workflow (#9982): a snapshot never dumps a row for a
      // file its copy missed, nor copies a file whose row was just declined.
      const { filename, extra } = await withBackupAssetPublication(async () => {
        const imported = await importFileToDir(tempOut, `${outputName}.mid`, destDir);
        const result = (await onComplete?.({ filename: imported.filename, model: resolvedModel })) || {};
        if (result.discarded) await unlink(join(destDir, imported.filename)).catch(() => {});
        return { filename: imported.filename, extra: result };
      });

      // `onComplete` may decline the result (`discarded: true` — e.g. the music
      // video's audio source changed mid-run, so this .mid is of the OLD
      // track). Don't advertise a filename nothing points at: delete the
      // orphaned file and tell the client it was discarded, not "ready".
      if (extra.discarded) {
        console.log(`🗑️ MIDI transcription ${shortId(jobId)} discarded — ${extra.reason || 'stale result'}`);
        broadcastSse(job, { type: 'complete', discarded: true, model: resolvedModel });
        return;
      }

      console.log(`🎹 MIDI transcription ${shortId(jobId)} complete — ${filename}`);
      broadcastSse(job, { type: 'complete', filename, model: resolvedModel, ...extra });
    } catch (err) {
      console.error(`❌ MIDI transcription ${shortId(jobId)} failed: ${err?.message || err}`);
      // Carry the typed gated-repo fields (code/repo) through to the client so
      // it can open the license + token prompt; a plain failure ships as-is.
      const frame = { type: 'error', error: err?.message || String(err) };
      if (err?.code) frame.code = err.code;
      if (err?.repo) frame.repo = err.repo;
      broadcastSse(job, frame);
    } finally {
      job.settled = true;
      if (ownerKey && activeByOwner.get(ownerKey) === jobId) activeByOwner.delete(ownerKey);
      await unlink(tempOut).catch(() => {});
      closeJobAfterDelay(transcriptionJobs, jobId);
    }
  })();

  return { jobId, model: resolvedModel };
}