Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

22.5 KB
Raw Blame
/**
 * Qwen3-TTS Voice Fine-Tuning Service (#5381).
 *
 * Provides dataset readiness validation, explicit start, pushed progress
 * (`fineTuningEvents` → `voice:fine-tune:updated`), checkpoint generation,
 * audition samples, per-profile job listing, cancellation, and explicit
 * checkpoint promotion.
 *
 * Training is machine-local, optional, and never assumes the last checkpoint is best.
 * It starts only when the runner's probe names a real training adapter (the
 * official Qwen single-speaker recipe on bf16 CUDA). The runner publishes a
 * checkpoint only after it is sealed and reloads to render an audition, and
 * reports the digest-bound revision promotion records. Adapter-less legacy
 * checkpoints and checkpoints without that revision never promote.
 */

import { randomUUID } from 'node:crypto';
import { existsSync } from 'node:fs';
import { mkdir, readdir } from 'node:fs/promises';
import { dirname, isAbsolute, join, relative, sep } from 'node:path';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { spawn } from '../../lib/childProcess.js';
import { ServerError } from '../../lib/errorHandler.js';
import { atomicWrite, readJSONFileStrict } from '../../lib/fileUtils.js';
import { PATHS } from '../../lib/paths.js';
import { safeChildProcessOptions } from '../../lib/processEnv.js';
import { closeJobAfterDelay } from '../../lib/sseUtils.js';
import {
  DEFAULT_CLONE_MODEL,
  QWEN3_TTS_MODELS_DIR,
  QWEN3_TTS_RUNNER_SCRIPT,
  SUPPORTED_QWEN3_MODELS,
  getQwen3RuntimeStatus,
} from './qwen3TtsRuntime.js';
import {
  getVoiceProfileRequired,
  profileArtifactDirectory,
  promoteFineTunedProfile,
} from './profiles.js';
import { fineTuningEvents } from './fineTuningEvents.js';

// In-memory active training job map. Entries are evicted once terminal (see
// `finalizeJob`); the durable record is the `job.json` sidecar written beside
// the run's checkpoints, so a restart mid-training does not orphan them.
const activeJobs = new Map();

// Sidecar record written into `<profileDir>/fine-tune/<jobId>/`. It lives with
// the checkpoints it indexes so the record and its artifacts are deleted
// together, rather than in a separate store that could outlive them.
const JOB_RECORD_FILE = 'job.json';

// Job ids are minted with randomUUID; anything else never named a real run and
// must not reach `join`, where a separator or `..` would escape the profile's
// directory. Enforced here as well as in the route schema because this path is
// built from a caller-supplied id.
const JOB_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;

// A user cancel is recorded by `cancelFineTuningJob` and, when the abort reaches
// the child before the caller returns, by the child's terminal handler. One
// shape so whichever writes first leaves the same record.
const CANCELLED_OUTCOME = Object.freeze({ status: 'cancelled', error: 'Cancelled by user' });

// The official recipe fine-tunes the Base variants only.
const TRAINABLE_MODELS = new Set(SUPPORTED_QWEN3_MODELS.map((model) => model.id).filter((id) => id.endsWith('-Base')));
// Single-speaker recipe: one registered speaker name per checkpoint.
const TRAINING_SPEAKER = 'portos_voice';
const DATASET_MANIFEST_FILE = 'dataset.json';
const AUDIO_FILE_RE = /\.(wav|mp3|flac|m4a)$/i;
const STDERR_TAIL_BYTES = 8192;
// Training frames arrive every optimizer step; the UI only needs a progress
// tick, so step progress is pushed at most this often. Status changes and
// sealed checkpoints are always pushed immediately.
const PROGRESS_PUSH_INTERVAL_MS = 1000;
// The Voice Lab shows the newest runs; older sidecars stay on disk untouched.
const MAX_LISTED_JOBS = 20;
// A `running` sidecar with no live process is a run the server lost on
// restart: its child is gone, so it can neither progress nor be cancelled.
const INTERRUPTED_ERROR = 'Training stopped when the server restarted';
// The one reason a checkpoint is refused promotion; shown beside the disabled
// Promote control and returned as the 409 message.
const UNVERIFIED_CHECKPOINT_REASON = 'Checkpoint was not produced by a supported training adapter';

const jobRecordPath = (profileId, jobId) =>
  join(profileArtifactDirectory(profileId), 'fine-tune', jobId, JOB_RECORD_FILE);

// A record that parsed but is not shaped like a job (a truncated `{}`, a `null`,
// a hand-edited file) is not a usable recovery — it must not masquerade as a
// live job whose `checkpoints` a caller is about to `.find` over.
const isJobRecord = (record) => Boolean(
  record
  && typeof record === 'object'
  && typeof record.id === 'string'
  && typeof record.status === 'string'
  && Array.isArray(record.checkpoints)
);

// Drops the runtime-only handles (abort controller, child process, write chain,
// finalize latch) that cannot — and must not — be serialized.
const serializableJob = ({
  controller: _controller,
  child: _child,
  persistChain: _persistChain,
  finalized: _finalized,
  ...record
}) => record;

// Earlier runners wrote text placeholders named .safetensors. A checkpoint
// without its producing adapter and sealed revision never becomes a voice.
const promotionBlockedReason = (job, checkpoint) => (
  !job.trainingAdapter || typeof checkpoint.modelRevision !== 'string' ? UNVERIFIED_CHECKPOINT_REASON : null
);

// The audition WAV is written beside its checkpoint under the voice-profiles
// root, which is mounted at /data/voice-profiles.
const auditionUrl = (sampleWav) => {
  if (typeof sampleWav !== 'string') return null;
  const rel = relative(PATHS.voiceProfiles, sampleWav);
  if (!rel || rel.startsWith('..') || isAbsolute(rel)) return null;
  return `/data/voice-profiles/${rel.split(sep).map(encodeURIComponent).join('/')}`;
};

/**
 * The client view of a job: the durable record plus what the Voice Lab renders
 * per checkpoint — a playable audition URL and whether (and why not) it can be
 * promoted, decided by the same predicate `promoteCheckpoint` enforces.
 * `processActive` is true while the trainer process has not exited yet — a
 * cancelled run reads `cancelled` at once but may still hold the GPU until
 * then. Records read from disk never have a live process.
 */
const publicJob = (job) => {
  const record = serializableJob(job);
  return {
    ...record,
    processActive: job.finalized === false,
    checkpoints: record.checkpoints.map((checkpoint) => {
      const blockedReason = promotionBlockedReason(record, checkpoint);
      return {
        ...checkpoint,
        auditionUrl: auditionUrl(checkpoint.sampleWav),
        promotable: !blockedReason,
        promotionBlockedReason: blockedReason,
      };
    }),
  };
};

// Pushes the live state at the moment of the change, so frames arrive in the
// order the state moved and the last one is always current.
const publishJob = (jobState) => {
  fineTuningEvents.emit('updated', { profileId: jobState.profileId, job: publicJob(jobState) });
};

/**
 * Write the job record to disk. Writes are chained per job because `close` and
 * `error` can both fire for one child and each rewrites the same file.
 *
 * The runner seals a checkpoint before reporting it, so this write is the moment
 * a checkpoint becomes named. It takes backup admission: a snapshot that copied
 * the run directory before a new checkpoint landed must not capture a record
 * listing it afterwards. The runner's own writes are outside the process and
 * cannot be admitted, but nothing names them until this write (#9982).
 */
const persistJob = (jobState) => {
  // Capture the durable state when this write is queued. Otherwise a checkpoint
  // write that runs after a later terminal event serializes the mutable current
  // object and can race the terminal write out of order.
  const record = structuredClone(serializableJob(jobState));
  publishJob(jobState);
  jobState.persistChain = (jobState.persistChain || Promise.resolve())
    .then(() => withBackupAssetPublication(() => atomicWrite(jobRecordPath(jobState.profileId, jobState.id), record)))
    .catch((err) => console.error(`❌ Failed to persist fine-tune job ${jobState.id}: ${err.message}`));
  return jobState.persistChain;
};

/**
 * Resolve a job from memory, falling back to its on-disk record. Returns null
 * only when the record is genuinely absent — a present-but-unreadable record
 * throws instead, because reporting 404 would tell the operator their run is
 * gone while its checkpoints are still sitting on disk.
 */
async function loadJob(jobId, profileId) {
  const active = activeJobs.get(jobId);
  if (active) return active;
  if (!profileId || !JOB_ID_RE.test(jobId)) return null;
  const { ok, value } = await readJSONFileStrict(jobRecordPath(profileId, jobId), null);
  if (!ok || (value !== null && !isJobRecord(value))) {
    throw new ServerError('Fine-tuning job record is unreadable', {
      status: 500,
      code: 'JOB_RECORD_UNREADABLE',
    });
  }
  // Reported, not rewritten: the sidecar stays exactly as the run left it.
  if (value?.status === 'running') return { ...value, status: 'interrupted', error: value.error || INTERRUPTED_ERROR };
  return value;
}

/**
 * Persist the terminal state, then evict the in-memory entry after the SSE
 * grace window so a status poll racing completion still hits memory while the
 * map stays bounded. Runs at most once per job — `error` and `close` can both
 * fire for the same child.
 */
const finalizeJob = (jobState) => {
  if (jobState.finalized) return;
  jobState.finalized = true;
  persistJob(jobState);
  closeJobAfterDelay(activeJobs, jobState.id);
};

// Busy until the child has exited (`finalizeJob`), not merely until the status
// leaves `running`: a cancel flips the status before the aborted trainer has
// released the GPU.
const assertNoRunningJob = (profileId) => {
  for (const job of activeJobs.values()) {
    if (job.profileId === profileId && job.finalized === false) {
      throw new ServerError('A fine-tuning run is already in progress for this voice profile', {
        status: 409,
        code: 'FINE_TUNE_ALREADY_RUNNING',
      });
    }
  }
};

/**
 * Validate training dataset readiness for a given voice profile.
 */
export async function validateFineTuningDataset(profileId) {
  const profile = await getVoiceProfileRequired(profileId);
  const profileDir = profileArtifactDirectory(profile.id);
  const sourceDir = join(profileDir, 'source');

  const issues = [];
  let fileCount = 0;
  let transcriptsCount = 0;

  if (!existsSync(sourceDir)) {
    issues.push('No source audio directory found. Upload reference audio first.');
    return { ready: false, fileCount: 0, transcriptsCount: 0, issues };
  }

  const files = await readdir(sourceDir);
  const audioFiles = files.filter((f) => AUDIO_FILE_RE.test(f));
  fileCount = audioFiles.length;

  if (fileCount === 0) {
    issues.push('At least 1 clean audio recording is required for fine-tuning.');
  }

  for (const asset of profile.sourceAssets || []) {
    if (asset.transcript && asset.transcript.trim()) {
      transcriptsCount += 1;
    }
  }

  if (transcriptsCount === 0 && fileCount > 0) {
    issues.push('Source audio requires transcriptions for training dataset.');
  }

  const ready = issues.length === 0 && fileCount > 0;
  return {
    ready,
    fileCount,
    transcriptsCount,
    issues,
    sourceDir,
  };
}

/**
 * Pair each transcribed source asset with its recording. The first pair is the
 * speaker reference; the recipe recommends one reference for every sample.
 */
function buildTrainingDataset(profile, sourceDir) {
  const samples = (profile.sourceAssets || [])
    .filter((asset) => typeof asset.filename === 'string' && AUDIO_FILE_RE.test(asset.filename)
      && asset.filename === asset.filename.split(/[\\/]/).pop()
      && asset.transcript?.trim() && existsSync(join(sourceDir, asset.filename)))
    .map((asset) => ({ audio: join(sourceDir, asset.filename), text: asset.transcript.trim() }));
  if (samples.length === 0) {
    throw new ServerError('Dataset not ready: no transcribed source recording is available', {
      status: 400,
      code: 'DATASET_NOT_READY',
    });
  }
  return { speaker: TRAINING_SPEAKER, reference_audio: samples[0].audio, samples };
}

// A checkpoint frame is recorded only when the runner sealed and auditioned
// it inside this job's own directory.
const isPublishedCheckpoint = (event, outputDir) => (
  typeof event.checkpoint === 'string'
  && typeof event.checkpoint_path === 'string'
  && dirname(event.checkpoint_path) === outputDir
  && typeof event.model_revision === 'string'
  && /@[0-9a-f]{40}\+sha256\.[0-9a-f]{64}$/.test(event.model_revision)
);

/**
 * Start an explicit fine-tuning job for a voice profile.
 */
export async function startFineTuningJob({
  profileId,
  epochs = 5,
  checkpointInterval = 50,
  baseModel = DEFAULT_CLONE_MODEL,
} = {}) {
  const validation = await validateFineTuningDataset(profileId);
  if (!validation.ready) {
    throw new ServerError(`Dataset not ready: ${validation.issues.join('; ')}`, {
      status: 400,
      code: 'DATASET_NOT_READY',
    });
  }

  // Refuse before a job record or child exists. Inference readiness is not
  // training support; only a runner that names a real adapter may train.
  const runtime = await getQwen3RuntimeStatus();
  if (!runtime.ok || !runtime.pythonPath || !runtime.trainingAdapter) {
    throw new ServerError('Qwen3-TTS fine-tuning is unavailable: no supported training adapter is installed', {
      status: 503,
      code: 'QWEN3_TRAINING_UNAVAILABLE',
    });
  }
  if (!TRAINABLE_MODELS.has(baseModel)) {
    throw new ServerError('Qwen3-TTS fine-tuning supports only the Base models', {
      status: 400,
      code: 'QWEN3_TRAINING_UNSUPPORTED_MODEL',
    });
  }
  if (!runtime.models?.[baseModel]?.downloaded) {
    throw new ServerError('Download and verify the Qwen3-TTS base model before fine-tuning', {
      status: 409,
      code: 'QWEN3_MODEL_NOT_INSTALLED',
    });
  }

  const profile = await getVoiceProfileRequired(profileId);
  // Two full-parameter trainings of one voice would contend for the same GPU
  // and write competing checkpoints; refuse before any record or child exists.
  // Checked again after the awaits below, where nothing yields until the job
  // registers, so two overlapping starts cannot both pass.
  assertNoRunningJob(profile.id);
  const dataset = buildTrainingDataset(profile, validation.sourceDir);
  const jobId = randomUUID();
  const profileDir = profileArtifactDirectory(profile.id);
  const outputDir = join(profileDir, 'fine-tune', jobId);
  await mkdir(outputDir, { recursive: true });
  const datasetManifest = join(outputDir, DATASET_MANIFEST_FILE);
  await atomicWrite(datasetManifest, dataset);

  assertNoRunningJob(profile.id);
  const abortController = new AbortController();
  const jobState = {
    id: jobId,
    profileId: profile.id,
    universeId: profile.binding.universeId,
    characterId: profile.binding.characterId,
    status: 'running',
    progress: 0,
    step: 0,
    // Optimizer steps depend on the dataset; the runner reports the total.
    totalSteps: null,
    loss: null,
    checkpoints: [],
    outputDir,
    baseModel,
    trainingAdapter: runtime.trainingAdapter,
    startedAt: new Date().toISOString(),
    completedAt: null,
    error: null,
    controller: abortController,
    child: null,
    // Flipped by `finalizeJob` once the child has closed or errored.
    finalized: false,
  };

  activeJobs.set(jobId, jobState);
  await persistJob(jobState);

  const args = [
    QWEN3_TTS_RUNNER_SCRIPT,
    '--mode', 'fine-tune',
    '--models-dir', QWEN3_TTS_MODELS_DIR,
    '--dataset-manifest', datasetManifest,
    '--output-dir', outputDir,
    '--epochs', String(epochs),
    '--checkpoint-interval', String(checkpointInterval),
    '--model-id', baseModel,
  ];

  const child = spawn(runtime.pythonPath, args, safeChildProcessOptions({ signal: abortController.signal }));
  // Keep a handle on the child so a future shutdown hook can reach in-flight training.
  jobState.child = child;

  // Drain stderr so a long run cannot block on a full pipe; keep the tail for
  // the runner's structured (path-free) failure message.
  let stderrTail = '';
  child.stderr?.on('data', (chunk) => { stderrTail = (stderrTail + chunk.toString()).slice(-STDERR_TAIL_BYTES); });
  const runnerError = () => {
    try {
      const failure = JSON.parse(stderrTail.trim().split(/\r?\n/).at(-1));
      return typeof failure?.error === 'string' ? failure.error : null;
    } catch {
      return null;
    }
  };

  let lineBuffer = '';
  let lastProgressPush = 0;
  child.stdout.on('data', (chunk) => {
    lineBuffer += chunk.toString();
    const lines = lineBuffer.split('\n');
    lineBuffer = lines.pop(); // keep remainder

    for (const line of lines) {
      const trimmed = line.trim();
      if (!trimmed) continue;
      try {
        const event = JSON.parse(trimmed);
        if (event.stage === 'training') {
          jobState.step = event.step;
          jobState.totalSteps = event.total_steps || jobState.totalSteps;
          jobState.loss = event.loss;
          jobState.progress = event.progress;
          if (Date.now() - lastProgressPush >= PROGRESS_PUSH_INTERVAL_MS) {
            lastProgressPush = Date.now();
            publishJob(jobState);
          }
        } else if (event.stage === 'checkpoint' && isPublishedCheckpoint(event, outputDir)) {
          jobState.checkpoints.push({
            id: event.checkpoint,
            step: event.step,
            checkpointPath: event.checkpoint_path,
            sampleWav: event.sample_wav,
            loss: event.loss,
            modelRevision: event.model_revision,
            createdAt: new Date().toISOString(),
          });
          // Checkpoints are the promotable artifact, so index each one as it
          // lands — a restart mid-training must not orphan the ones already
          // written. Step/loss progress deliberately is not persisted: it
          // arrives every step and is worthless once the process is gone.
          persistJob(jobState);
        } else if (event.stage === 'completed') {
          jobState.status = 'completed';
          jobState.progress = 100;
          jobState.completedAt = new Date().toISOString();
          publishJob(jobState);
        }
      } catch {
        // non-JSON log line
      }
    }
  });

  // Only the FIRST terminal event may write the outcome. An aborted spawn fires
  // BOTH — `error` with an AbortError, then `close(null, 'SIGTERM')` — and
  // `cancelFineTuningJob` records 'cancelled' synchronously before either
  // arrives, so an unguarded `error` handler lands a user cancel on disk (and in
  // the next status poll) as 'failed'.
  const settle = (outcome) => {
    if (jobState.status === 'running') {
      Object.assign(jobState, outcome, { completedAt: new Date().toISOString() });
    }
    finalizeJob(jobState);
  };

  child.on('close', (code, signal) => {
    if (code === 0) return settle({ status: 'completed', progress: 100 });
    if (abortController.signal.aborted) return settle(CANCELLED_OUTCOME);
    // A killed child reports a null code — an OOM reap during a long training
    // run is the likely cause, so name the signal rather than "code null".
    settle({
      status: 'failed',
      error: code === null
        ? `Process terminated by signal ${signal}`
        : (runnerError() || `Process exited with code ${code}`),
    });
  });

  child.on('error', (err) => {
    settle(abortController.signal.aborted ? CANCELLED_OUTCOME : { status: 'failed', error: err.message });
  });

  return {
    jobId,
    profileId: profile.id,
    status: jobState.status,
    totalSteps: jobState.totalSteps,
    trainingAdapter: jobState.trainingAdapter,
    startedAt: jobState.startedAt,
    job: publicJob(jobState),
  };
}

/**
 * List a profile's fine-tuning runs, newest first, from their `job.json`
 * sidecars (live state for a run still in memory). This is how the Voice Lab
 * recovers a run after a reload or a restart.
 */
export async function listFineTuningJobs(profileId) {
  const profile = await getVoiceProfileRequired(profileId);
  const root = join(profileArtifactDirectory(profile.id), 'fine-tune');
  const entries = await readdir(root, { withFileTypes: true }).catch((err) => {
    if (err.code === 'ENOENT') return [];
    throw err;
  });
  const jobs = await Promise.all(entries
    .filter((entry) => entry.isDirectory() && JOB_ID_RE.test(entry.name))
    .map((entry) => loadJob(entry.name, profile.id).catch((err) => {
      // One corrupt sidecar must not hide every other run of this voice.
      console.warn(`⚠️ Skipping unreadable fine-tune job ${entry.name}: ${err.message}`);
      return null;
    })));
  return jobs
    .filter(Boolean)
    .map(publicJob)
    .sort((a, b) => String(b.startedAt || '').localeCompare(String(a.startedAt || '')))
    .slice(0, MAX_LISTED_JOBS);
}

/**
 * Get the current status and checkpoints of a fine-tuning job, from memory
 * while it is live and from its `job.json` sidecar afterwards.
 */
export async function getFineTuningJobStatus(jobId, profileId) {
  const job = await loadJob(jobId, profileId);
  if (!job) {
    throw new ServerError('Fine-tuning job not found', { status: 404, code: 'JOB_NOT_FOUND' });
  }
  return publicJob(job);
}

/**
 * Cancel an active fine-tuning job. With a `profileId`, a job belonging to a
 * different profile is reported as not found.
 */
export function cancelFineTuningJob(jobId, profileId) {
  const job = activeJobs.get(jobId);
  if (!job || (profileId && job.profileId !== profileId)) {
    throw new ServerError('Fine-tuning job not found', { status: 404, code: 'JOB_NOT_FOUND' });
  }
  if (job.status === 'running') {
    job.controller.abort();
    Object.assign(job, CANCELLED_OUTCOME, { completedAt: new Date().toISOString() });
    publishJob(job);
  }
  return { ok: true, jobId, status: job.status, job: publicJob(job) };
}

/**
 * Explicitly promote a selected checkpoint to the character's voice profile.
 */
export async function promoteCheckpoint({ profileId, jobId, checkpointId }) {
  const job = await loadJob(jobId, profileId);
  if (!job) {
    throw new ServerError('Fine-tuning job not found', { status: 404, code: 'JOB_NOT_FOUND' });
  }
  const ckpt = job.checkpoints.find((c) => c.id === checkpointId || String(c.step) === String(checkpointId));
  if (!ckpt) {
    throw new ServerError(`Checkpoint not found: ${checkpointId}`, { status: 404, code: 'CHECKPOINT_NOT_FOUND' });
  }
  const blockedReason = promotionBlockedReason(job, ckpt);
  if (blockedReason) {
    throw new ServerError(blockedReason, { status: 409, code: 'CHECKPOINT_UNVERIFIED' });
  }

  return promoteFineTunedProfile({
    profileId,
    universeId: job.universeId,
    characterId: job.characterId,
    checkpointPath: ckpt.checkpointPath,
    checkpointId: ckpt.id,
    modelRevision: ckpt.modelRevision,
    step: ckpt.step,
  });
}