Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

10.1 KB
Raw Blame
/**
 * User-triggered lyric alignment. Vocal stems use known-text MMS_FA CTC;
 * legacy master-only callers retain Whisper until they consent to separation.
 * No models load at boot. Director cue boundaries and concurrent edits survive.
 */

import { mkdtemp, readFile, rm } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { ServerError } from '../../lib/errorHandler.js';
import { findFfmpeg, runFfmpegProcess } from '../../lib/ffmpeg.js';
import { musicVideoAudioAnalysisSchema } from '../../lib/musicVideoValidation.js';
import { getProject, updateProject } from './projects.js';
import {
  alignDirectorWords,
  lyricAlignFfmpegArgs,
  mergeTranscripts,
  vocalPcm,
  wavDurationSec,
  pickAlignmentPath,
  findSilentWords,
} from './lyricAlignCore.js';
import { relabelAnalysisSections } from './lyricMarkers.js';

async function decodeSourceToWav(sourcePath) {
  const ffmpeg = await findFfmpeg();
  if (!ffmpeg) {
    throw new ServerError('ffmpeg is required to align lyrics to the vocal.', { status: 500, code: 'FFMPEG_MISSING' });
  }
  const dir = await mkdtemp(join(tmpdir(), 'lyric-align-'));
  const outPath = join(dir, 'vocal.wav');
  try {
    const result = await runFfmpegProcess({ bin: ffmpeg, args: lyricAlignFfmpegArgs(sourcePath, outPath) });
    if (!result.ok) {
      console.error(`❌ Lyric alignment could not decode audio: ${result.reason}`);
      throw new ServerError('Could not decode the song for lyric alignment.', { status: 422, code: 'LYRIC_ALIGN_DECODE_FAILED' });
    }
    return await readFile(outPath);
  } finally {
    await rm(dir, { recursive: true, force: true }).catch(() => {});
  }
}

// Lazy: the runner pulls the download/spawn helpers only an alignment needs.
async function resolveTranscriber(onDownloadProgress) {
  const { resolveAlignmentTranscriber } = await import('./lyricTranscriber.js');
  return resolveAlignmentTranscriber({ onDownloadProgress });
}

const audioKey = (project) => `${project?.trackId ?? ''}\u0000${project?.uploadedAudioFilename ?? ''}\u0000${project?.vocalStemFilename ?? ''}`;
const textKey = (cues) => (cues || []).map((cue) => `${cue.id ?? ''}\u0000${cue.text ?? ''}`).join('\n');

function alignmentPrompt(cues) {
  return (cues || []).map((cue) => cue?.text || '').filter(Boolean).join('\n').slice(0, 800);
}

/** The analysis with its sections named after the lyric sheet, or null when nothing changes. */
function relabeledAnalysis(project, cues) {
  const analysis = project.audioAnalysis;
  if (!analysis || !Array.isArray(analysis.sections)) return null;
  const next = relabelAnalysisSections(analysis, cues, project.lyricMarkers || []);
  if (JSON.stringify(next.sections) === JSON.stringify(analysis.sections)) return null;
  const parsed = musicVideoAudioAnalysisSchema.safeParse(next);
  return parsed.success ? parsed.data : null;
}

// Measure the aligned word span, not an authored cue boundary that may be wider.
function compressedWords(words) {
  // CTC stores milliseconds; compare there so a span of exactly 0.15 s/word
  // cannot become compressed through floating-point subtraction.
  return words.length >= 3 && Math.round((words.at(-1).endSec - words[0].startSec) * 1000) < words.length * 150;
}

/**
 * Align one project's lyric cues. `cueId` re-aligns that line only: a timed
 * line uses its own slice; an untimed line uses full-song CTC context so a
 * repeated chorus keeps its own occurrence.
 */
export async function alignProjectLyrics(projectId, options = {}) {
  const onProgress = options.onProgress || (() => {});
  const checkCancel = () => {
    if (options.isCancelled?.()) throw Object.assign(new Error('cancelled'), { canceled: true });
  };
  const deps = {
    getProject,
    updateProject,
    resolveAudio: pickAlignmentPath,
    decodeAudio: decodeSourceToWav,
    resolveTranscriber,
    forceAlign: async (...args) => (await import('./lyricForcedAlign.js')).forceAlignLyrics(...args),
    separateVocals: async (...args) => (await import('./vocalSeparation.js')).separateProjectVocals(...args),
    ...options.deps,
  };
  const cueId = options.cueId || null;
  let project = await deps.getProject(projectId);
  if (!project) throw new ServerError('Project not found', { status: 404, code: 'NOT_FOUND' });
  const cues = Array.isArray(project.lyricCues) ? project.lyricCues : [];
  if (cues.length === 0) {
    throw new ServerError('Add lyric lines before aligning words.', { status: 400, code: 'NO_LYRICS' });
  }
  const cue = cueId ? cues.find((entry) => entry.id === cueId) : null;
  if (cueId && !cue) throw new ServerError('That lyric line is no longer on the project.', { status: 404, code: 'NOT_FOUND' });

  const expectedTextKey = textKey(cues);
  if (!project.vocalStemFilename && options.separateVocals === true) {
    onProgress({ stage: 'separating' });
    project = await deps.separateVocals(projectId, { onProgress, isCancelled: options.isCancelled });
    checkCancel();
  }
  const expectedAudioKey = audioKey(project);
  onProgress({ stage: 'decoding' });
  const { path: audioPath, source } = await deps.resolveAudio(project);
  const wav = await deps.decodeAudio(audioPath);
  checkCancel();
  const windowed = cue
    && typeof cue.startSec === 'number'
    && typeof cue.endSec === 'number'
    && cue.endSec > cue.startSec
    ? { startSec: cue.startSec, endSec: cue.endSec }
    : null;
  const region = windowed || { startSec: 0, endSec: null };
  const promptCues = windowed ? [cue] : cues;
  const prompt = alignmentPrompt(promptCues);
  let forcedWords = null;
  let recognized = [];
  let kind;
  if (source === 'vocal-stem') {
    onProgress({ stage: 'loading-model' });
    checkCancel();
    // Both explicit cue alignment and automatic repairs use the same runner,
    // decoded stem, cancellation and progress callbacks. Save only once below.
    const alignStemCues = (entries, window) => deps.forceAlign(wav, entries, {
      ...window, onProgress, isCancelled: options.isCancelled,
    });
    forcedWords = await alignStemCues(promptCues, region);
    if (!cueId) {
      const wholeSongWords = forcedWords;
      forcedWords = [...wholeSongWords];
      for (let index = 0; index < cues.length; index++) {
        if (!compressedWords(wholeSongWords[index])) continue;
        checkCancel();
        const startSec = cues[index - 1]?.endSec ?? wholeSongWords[index - 1]?.at(-1)?.endSec ?? 0;
        const endSec = cues[index + 1]?.startSec ?? wholeSongWords[index + 1]?.[0]?.startSec ?? wavDurationSec(wav);
        // Inconsistent authored neighbors cannot supply a useful repair window.
        if (!(endSec > startSec)) continue;
        const [retry] = await alignStemCues([cues[index]], { startSec, endSec });
        checkCancel();
        if (retry.length === wholeSongWords[index].length && !compressedWords(retry)) forcedWords[index] = retry;
      }
    }
    kind = 'MMS_FA';
  } else {
    onProgress({ stage: 'loading-model' });
    const transcriber = await deps.resolveTranscriber((download) => onProgress({ stage: 'downloading-model', percent: download.percent }));
    kind = transcriber.kind;
    try {
      checkCancel();
      onProgress({ stage: 'transcribing', current: 1, total: 1, percent: 0 });
      recognized = mergeTranscripts(await transcriber.transcribe(wav, { ...region, prompt }), [], promptCues.map((entry) => entry.text).join('\n'));
    } finally {
      await Promise.resolve(transcriber.release?.()).catch((err) => console.error(`❌ Could not stop the alignment runner: ${err.message}`));
    }
    if (!windowed && recognized.length === 0) {
      throw new ServerError('Speech-to-text heard no words in this audio. Check the song has vocals, then try Align words again.',
        { status: 422, code: 'LYRIC_ALIGN_EMPTY' });
    }
  }
  checkCancel();

  const fresh = await deps.getProject(projectId);
  if (!fresh || audioKey(fresh) !== expectedAudioKey) {
    throw new ServerError('The song changed while lyrics were aligning. Run Align words again.', { status: 409, code: 'MUSIC_VIDEO_AUDIO_CHANGED' });
  }
  const freshCues = Array.isArray(fresh.lyricCues) ? fresh.lyricCues : [];
  if (textKey(freshCues) !== expectedTextKey) {
    throw new ServerError('The lyric lines changed while they were aligning. Run Align words again.', { status: 409, code: 'LYRIC_ALIGN_TEXT_CHANGED' });
  }

  checkCancel();
  onProgress({ stage: 'saving', percent: 100 });
  const align = (entries) => {
    if (!forcedWords) return alignDirectorWords(entries, recognized).map((entry) => ({ ...entry, suspect: false }));
    return entries.map((entry, index) => {
      const words = forcedWords[index];
      return { ...entry, words, matched: words.length ? 1 : 0, suspect: compressedWords(words),
        startSec: entry.startSec ?? words[0]?.startSec ?? null,
        endSec: entry.endSec ?? words.at(-1)?.endSec ?? null };
    });
  };
  let nextCues;
  if (windowed) {
    const [aligned] = align([freshCues.find((entry) => entry.id === cue.id)]);
    nextCues = freshCues.map((entry) => (entry.id === cue.id ? { ...entry, ...aligned, id: entry.id } : entry));
  } else {
    const aligned = align(freshCues);
    nextCues = cueId ? freshCues.map((entry, index) => (entry.id === cueId ? aligned[index] : entry)) : aligned;
  }
  const matched = nextCues.reduce((sum, entry) => sum + (entry.words || []).filter((word) => word.conf === 'matched').length, 0);
  console.log(`🎤 Aligned lyric words from the ${source} via ${kind} (${matched} words matched)`);
  // Provenance is project-wide, so only a whole-song pass may change it; one
  // re-aligned line leaves the other lines' timings as stale as they were.
  const patch = { lyricCues: nextCues, ...(cueId ? {} : { lyricAlignSource: source }) };
  if (source === 'vocal-stem') {
    const silentWords = findSilentWords(nextCues, vocalPcm(wav));
    patch.lyricAlignSilentWords = silentWords.length;
    if (silentWords.some((word) => !cueId || word.cueId === cueId)) {
      throw new ServerError('Some lyric words aligned to silence. Check that the lyrics match the vocal, then re-align.',
        { status: 422, code: 'LYRIC_ALIGN_SILENT_WORDS' });
    }
  }
  if (!cueId) {
    const analysis = relabeledAnalysis(fresh, nextCues);
    if (analysis) patch.audioAnalysis = analysis;
  }
  return deps.updateProject(projectId, patch);
}