Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

42.0 KB
Raw Blame
import { refuseRetiredTraining, withRetiredDataAccess, acquireRetiredDataAccess } from '../../lib/retiredLoraTraining.js';
/**
 * Retired trainer recovery only: observe/cancel a verified detached survivor,
 * finalize its output and keep historical records/checkpoint promotion readable.
 * New launch, resume and queue admission are permanently refused.
 */

import { spawn } from '../../lib/childProcess.js';
import { existsSync } from 'fs';
import { join, basename } from 'path';
import { platform } from 'os';
import { PATHS, ensureDir, shortId, copyFileGuarded, writeFileGuarded, rmGuarded } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';

import { reattachDetached, isReattachable } from '../../lib/detachedSpawn.js';
import { killWithEscalation } from '../../lib/killWithEscalation.js';
import { getImageModels } from '../../lib/mediaModels.js';

import { getSettings } from '../settings.js';
import { deleteLora, writeLoraSidecar } from '../loras.js';
import { getJob, mediaJobEvents } from '../mediaJobQueue/index.js';
import { updateDataset } from '../loraDatasets.js';
import { trainingEvents } from './events.js';
import { sleepDisplayForTraining, wakeDisplay } from './displayPower.js';

import { makeTrainingLineHandler } from './progress.js';
import { makeStallDetector } from './stallDetector.js';

import { claimHeavyLocalJob, adoptHeavyLocalJob } from '../../lib/heavyJobClaim.js';
import { classifyTrainingFailure } from './failure.js';
import { buildTrainedSidecar, trainedLoraFilename } from './sidecar.js';

import { listRunCheckpoints, listRunSamples, resolveCheckpointAdapterBuffer, selectDeployableCheckpoint } from './checkpoints.js';
import * as runsDb from './db.js';

export { listRuns, getRun, getRunRequired, deleteRun } from './db.js';
export { trainingEvents } from './events.js';

export const runDir = (runId) => join(PATHS.trainingRuns, runId);
export const runSamplesDir = (runId) => join(runDir(runId), 'samples');

// GPU lane serializes training with renders, so at most one trainer child
// exists at a time. Keyed by jobId anyway so a stale cancel can't kill a
// newer run.
let activeProcess = null;
let activeJobId = null;
const explicitlyCanceledTrainers = new WeakSet();

// Recovery still observes a surviving trainer for stalls.
const STALL_TICK_MS = 30_000;

export const cancel = (jobId) => {
  if (!activeProcess || (jobId && activeJobId !== jobId)) return false;
  const proc = activeProcess;
  explicitlyCanceledTrainers.add(proc);
  // Keep activeProcess set until 'close' clears it — the trainer may spend
  // a few seconds writing its cancel checkpoint. Escalate after 8s.
  killWithEscalation(proc, { label: 'training child', stillRunning: () => activeProcess === proc });
  return true;
};

export async function startTrainingRun() { refuseRetiredTraining(); }

export async function resumeTrainingRun() { refuseRetiredTraining(); }

const emitFailed = (jobId, error) => trainingEvents.emit('failed', { generationId: jobId, error });

// Collapse checkpoint records to one per `step`, preserving first-seen order
// (last value wins for a given step). Keeps run.artifacts.checkpoints unique
// when a #1332 re-attach replays already-persisted checkpoint lines.
const dedupeCheckpointsByStep = (checkpoints) => {
  const byStep = new Map();
  for (const c of checkpoints) byStep.set(c?.step, c);
  return [...byStep.values()];
};

const flipDatasetAfterRun = (run, { trained, loraFilename = null }) => {
  const datasetId = run?.datasetId;
  if (!datasetId) return Promise.resolve();
  // A run owns the dataset's training state only while the dataset still
  // points at the run's character. The dataset can be reassigned to a
  // different character mid-run (patchDataset resets it to draft); flipping
  // it here would otherwise mark the NEW character trained with the OLD
  // character's adapter, or clobber a fresh run's 'training' status. Skip the
  // flip when the dataset has moved on. Match on the full (universeId,
  // entryId) key the dataset store uses — a different universe can reuse the
  // same entryId, so entryId alone would falsely re-own a moved dataset.
  // Pre-reassignment runs predate the `character` snapshot guarantee, so a
  // missing entryId falls through (flip).
  const runEntryId = run?.character?.entryId || null;
  const runUniverseId = run?.character?.universeId || null;
  const runEntryKind = run?.character?.entryKind || 'characters';
  return updateDataset(datasetId, (current) => {
    const mismatch = runEntryId && (
      current.character?.entryId !== runEntryId
      || (runUniverseId && current.character?.universeId !== runUniverseId)
      || ((current.character?.entryKind || 'characters') !== runEntryKind)
    );
    if (mismatch) return null;
    return {
      ...current,
      status: trained ? 'trained' : 'draft',
      training: {
        ...current.training,
        ...(trained ? { loraFilename, completedAt: new Date().toISOString() } : {}),
      },
    };
  });
};

// When a trained LoRA artifact is deleted (DELETE /runs/:id?deleteLora=true),
// reset the owning dataset back to the untrained baseline so the training list
// stops advertising the subject as trained against a now-deleted file. Reset
// ONLY when the dataset still points at THIS run's adapter: same character
// (reuses flipDatasetAfterRun's ownership guard) AND the dataset's current
// `training.loraFilename` is exactly the file being deleted. A dataset that was
// retrained (different loraFilename) or reassigned to another character must
// keep its current state.
export const clearDatasetForDeletedLora = (run, deletedFilename) => {
  const datasetId = run?.datasetId;
  if (!datasetId || !deletedFilename) return Promise.resolve();
  const runEntryId = run?.character?.entryId || null;
  const runUniverseId = run?.character?.universeId || null;
  const runEntryKind = run?.character?.entryKind || 'characters';
  return updateDataset(datasetId, (current) => {
    const mismatch = runEntryId && (
      current.character?.entryId !== runEntryId
      || (runUniverseId && current.character?.universeId !== runUniverseId)
      || ((current.character?.entryKind || 'characters') !== runEntryKind)
    );
    if (mismatch || current.training?.loraFilename !== deletedFilename) return null;
    return { ...current, status: 'draft', training: {} };
  }).catch((err) => console.error(`❌ dataset reset after LoRA delete failed: ${err?.message}`));
};

/**
 * Delete a terminal run: its artifact dir, optionally its trained LoRA (and the
 * dataset's trained flag naming it), then the row. One backup admission spans
 * every step, so a snapshot never dumps a row naming samples or an adapter its
 * own file pass skipped (#9982).
 */
export async function deleteTrainingRun(runId, { withLora = false } = {}) {
  return withBackupAssetPublication(async () => {
    const run = await runsDb.getRunRequired(runId);
    if (['queued', 'running'].includes(run.status)) {
      throw new ServerError('Cancel the run before deleting it', { status: 409, code: 'RUN_ACTIVE' });
    }
    // Artifact dir is server-derived from the run id (uuid) — confined under
    // PATHS.trainingRuns by construction.
    await rmGuarded(runDir(run.id), { recursive: true, force: true });
    const loraFilename = run.output?.loraFilename;
    if (withLora && loraFilename) {
      await deleteLora(loraFilename).catch((err) => {
        console.warn(`⚠️ trained LoRA delete skipped: ${err?.message}`);
      });
      // Reset the owning dataset off 'trained' so it can't keep advertising the
      // subject as trained against the file just deleted.
      await clearDatasetForDeletedLora(run, loraFilename);
    }
    return runsDb.deleteRun(run.id);
  });
}

/**
 * Queue-worker entry — `mediaJobQueue.runJob` calls this for kind
 * 'training'. Resolves the trainer binary + args, spawns, parses the line
 * protocol into trainingEvents, and finalizes (LoRA registration or
 * failure classification) on close. Terminal status flows through the
 * queue's dispatcher; this function resolves once the observer attaches
 * (the queue awaits the terminal event separately).
 */
export async function runTraining({ jobId, runId, reattach = false, recoveryPermit = null }) {
  const fail = (message) => {
    console.error(`❌ training [${shortId(jobId)}] ${message}`);
    emitFailed(jobId, message);
  };

  const run = await runsDb.getRun(runId);
  if (!run) return fail(`run record missing: ${runId}`);
  const settings = await getSettings();
  const dir = runDir(runId);
  let heavyClaim = null;
  const retainOwnership = () => {
    maintenance.markCurrentUnsettled();
    maintenance.markResourceUnsettled('media', jobId);
  };
  let claimRelease;
  const releaseClaim = () => claimRelease ||= Promise.resolve().then(() => heavyClaim?.release()).catch(err => {
    retainOwnership();
    console.error(`❌ training [${shortId(jobId)}] claim release failed: ${err.message}`);
  });

  // Terminal failure BEFORE the child spawns: flip the run record to failed
  // AND release the dataset's `training` status, then emit the failed event.
  // Every pre-spawn exit funnels through here so none can leave the run row
  // stuck `running` (lingering until the next boot reconcile) or the dataset
  // stuck on its `training` chip.
  const failBeforeSpawn = async (message) => {
    if (reattach && await hasSurvivingTrainer(runId)) {
      retainOwnership();
      console.error(`❌ training [${shortId(jobId)}] observer unavailable; live trainer and artifacts preserved: ${message}`);
      return { preserved: true };
    }
    await releaseClaim();
    await runsDb.updateRun(runId, {
      status: 'failed', error: message, completedAt: new Date().toISOString(),
    }).catch(retainOwnership);
    await flipDatasetAfterRun(run, { trained: false }).catch(retainOwnership);
    return fail(message);
  };

  // Recovery attaches only to a trainer that already exists. The queue restores a run whose detached trainer
  // SURVIVED a hard server restart with `reattach: true`. The child reparented
  // to init and is still training (or finished during the downtime), so we tail
  // its existing control-dir output instead of spawning a second trainer — no
  // staging, no validation, no fresh spawn. wireProcLifecycle then drives the
  // exact same line-handling/finalize path as a normal spawn, so a run that
  // completed mid-restart still registers its LoRA instead of being discarded.
  //
  // The survivor already
  // holds the machine-wide accelerator claim from before the restart (handed
  // off to its PID pre-crash), so acquiring a NEW claim here would see that
  // live claim as a competing job and refuse it — failing every restart-
  // survived run outright. Adopt the existing claim instead of contending for
  // a fresh one.
  if (reattach) {
    const proc = await reattachDetached(join(dir, '.detached'));
    if (!proc) {
      // The survivor died (killed mid-run) with no RESULT to recover — fail the
      // run; its latest checkpoint stays retained for inspection, same as the
      // pre-#1332 reap path would have left it.
      return failBeforeSpawn('Trainer did not survive the restart — marking failed; artifacts are preserved.');
    }
    heavyClaim = (await adoptHeavyLocalJob({ kind: 'LoRA training', id: jobId, pid: proc.pid }))
      || (await claimHeavyLocalJob({ kind: 'LoRA training', id: jobId }));
    if (!heavyClaim.ok) return failBeforeSpawn(heavyClaim.message);
    // Only true on the claimHeavyLocalJob fallback (no matching on-disk claim
    // survived) — adoptHeavyLocalJob only ever returns a claim already
    // recorded against this exact pid.
    if (heavyClaim.holder?.pid !== proc.pid) await heavyClaim.handoffTo?.(proc.pid);
    console.log(`🔁 training [${shortId(jobId)}] re-attached to surviving trainer pid ${proc.pid} (run ${shortId(runId)})`);
    trainingEvents.emit('status', { generationId: jobId, message: 'Re-attached to trainer that survived a restart' });
    wireProcLifecycle(proc, { isReattach: true });
    return { observing: true };
  }

  refuseRetiredTraining();

  function wireProcLifecycle(proc, { isReattach = false } = {}) {
  const releaseDataAccess = acquireRetiredDataAccess();
  activeProcess = proc;
  activeJobId = jobId;
  void heavyClaim?.handoffTo?.(proc.pid)?.catch((err) => console.error(`❌ training [${shortId(jobId)}] claim handoff failed: ${err.message}`));

  // Keep the Mac awake for the duration of the training child (idle + system
  // sleep held off) — but DELIBERATELY let the *display* sleep. An active
  // display makes WindowServer contend for the GPU, which triggers the watchdog
  // kernel panic during heavy training (mlx #3267; see the incident doc). The
  // flags are `-is`, NOT `-dis`: `-d` would force the display awake and is what
  // was crashing the box. We then proactively sleep the display below so the
  // protection doesn't wait on the OS idle timeout. Best-effort — caffeinate
  // exits with the child.
  if (platform() === 'darwin' && proc.pid) {
    const caf = spawn('caffeinate', ['-is', '-w', String(proc.pid)], { stdio: 'ignore' });
    caf.on('error', () => {});
    caf.unref();
  }
  // Sleep the display now (Apple Silicon, opt-out via settings.loraTraining
  // .displaySleep) so sustained GPU training never competes with the display.
  // The close handler wakes it when the run finishes. Gated on proc.pid (like
  // the caffeinate spawn above) so a failed launch that emits 'error' without a
  // 'close' never sleeps a display nothing will wake. Re-sleeping a still-running
  // run on reattach is harmless (pmset displaysleepnow is idempotent).
  if (proc.pid) sleepDisplayForTraining(settings);

  // Debounced run-record progress mirror (~2s). Per-step DB writes from a
  // hot loop are the high-frequency-write anti-pattern; SSE is the live
  // channel, the row is the durable snapshot.
  let progressDirty = null;
  let progressTimer = null;
  // Persist whatever progress/artifacts are pending. Returns the write promise
  // so the close handler can AWAIT a final flush before finalize reads the run
  // record — otherwise the final checkpoint + last sample (which the collapse
  // guard and previewImageUrl both read from run.artifacts) can still be in the
  // debounce buffer, not on disk.
  const flushProgress = () => {
    if (progressTimer) { clearTimeout(progressTimer); progressTimer = null; }
    if (!progressDirty) return Promise.resolve();
    const flushing = progressDirty;
    progressDirty = null;
    // The trainer writes sample and checkpoint files in place before announcing
    // them, so only the row that first names them takes backup admission
    // (#9982); a progress-only flush never waits out a cut.
    const namesArtifacts = !!(flushing.checkpoints || flushing.samples);
    const write = () => runsDb.updateRun(runId, (current) => ({
      ...current,
      progress: { ...current.progress, ...flushing.progress },
      artifacts: {
        ...current.artifacts,
        // Dedupe on merge: a #1332 re-attach replays the survivor's whole log
        // from offset 0, re-firing onCheckpoint/onSample for artifacts already
        // persisted before the restart. Key checkpoints by step and samples by
        // filename (both unique) so a replay can't double-list them — last write
        // wins, which is identical for a true replay.
        ...(flushing.checkpoints ? { checkpoints: dedupeCheckpointsByStep([...current.artifacts.checkpoints, ...flushing.checkpoints]) } : {}),
        ...(flushing.samples ? { samples: [...new Set([...current.artifacts.samples, ...flushing.samples])] } : {}),
      },
    }));
    return (namesArtifacts ? withBackupAssetPublication(write) : write()).then(() => {
      if (namesArtifacts) trainingEvents.emit('checkpoints:changed', { runId });
    }).catch((err) => console.error(`❌ training [${shortId(jobId)}] progress persist failed: ${err?.message}`));
  };
  // Checkpoints/samples accumulate as arrays so two that land in one debounce
  // window (common in the final post-exit scan) both survive — a single-value
  // patch would let the later one clobber the earlier.
  const scheduleProgressPersist = (patch) => {
    progressDirty = progressDirty || {};
    if (patch.progress) progressDirty.progress = { ...progressDirty.progress, ...patch.progress };
    if (patch.checkpoint) (progressDirty.checkpoints ||= []).push(patch.checkpoint);
    if (patch.sample) (progressDirty.samples ||= []).push(patch.sample);
    if (progressTimer) return;
    progressTimer = setTimeout(() => { flushProgress(); }, 2000);
    progressTimer.unref?.();
  };

  const { handleLine, getState } = makeTrainingLineHandler({
    jobId,
    totalSteps: run.params.steps,
    emit: (event, payload) => {
      trainingEvents.emit(event, payload);
      if (event === 'progress' && typeof payload.step === 'number') {
        scheduleProgressPersist({ progress: { step: payload.step, totalSteps: payload.totalSteps } });
      }
    },
    onCheckpoint: (path, step, loss) => scheduleProgressPersist({
      checkpoint: { step, path: basename(path), loss: Number.isFinite(loss) ? loss : null },
      progress: { lastCheckpointStep: step },
    }),
    onSample: (path) => scheduleProgressPersist({ sample: basename(path) }),
    sampleUrl: (path) => `/api/lora-training/runs/${runId}/samples/${basename(path)}`,
  });

  // Phase-aware soft-hang stall watchdog (issue #1330). The queue's flat 30-min
  // idle watchdog only trips on TOTAL silence and can't see a soft GPU hang
  // mid-training (the python heartbeat/telemetry threads keep emitting lines, so
  // the flat watchdog never fires). This detector keys off the STAGE/STEP
  // protocol to apply a tight, step-rate-derived budget ONLY during
  // STAGE:training — load/encode/sampling/cooldown stay on the flat watchdog so
  // a legitimately slow phase is never false-killed. On a training-phase stall
  // we stop the wedged child and retain its newest checkpoint. Disable via settings.loraTraining.stallWatchdog = false.
  const stallWatchdogOn = (settings?.loraTraining?.stallWatchdog !== false);
  const stallDetector = makeStallDetector();
  let stallKilled = false; // set when the tick SIGKILLs — close handler reads it
  let stallTimer = null;
  const stopStallWatchdog = () => { if (stallTimer) { clearInterval(stallTimer); stallTimer = null; } };
  // The stall detector derives its budget from WALL-CLOCK gaps between STEP
  // lines. On a #1332 re-attach the survivor's whole log is replayed from
  // offset 0 in a few ms, so feeding those historical steps would record
  // ~0ms intervals and instantly exit the 20-min warmup into the tight floor —
  // false-killing a healthy slow-step run on its next live step. So on a
  // re-attach we withhold step observation until the tailer's one-time
  // 'replayed' signal (initial backlog drained); the detector then warms up
  // fresh on genuinely-live output, exactly as a fresh spawn would.
  let stallObserveLive = !isReattach;
  if (isReattach) proc.on('replayed', () => { stallObserveLive = true; });

  const makeSplitter = (stream) => {
    let buf = '';
    const safeLine = (text) => {
      // try/catch: this runs inside a child-process data/end callback — an
      // uncaught throw here would crash the server process.
      try {
        handleLine(text, stream);
        if (stallWatchdogOn && stallObserveLive) stallDetector.observe(text);
      } catch (err) {
        console.error(`❌ training [${shortId(jobId)}] line handler failed: ${err?.message}`);
      }
    };
    return {
      push: (chunk) => {
        buf += chunk.toString();
        let idx;
        while ((idx = buf.indexOf('\n')) >= 0) {
          safeLine(buf.slice(0, idx));
          buf = buf.slice(idx + 1);
        }
      },
      // Flush any trailing line that arrived without a final newline. The
      // trainers hard-exit via os._exit (teardown-hang defense), which can
      // truncate the pipe before the final `RESULT:{...}\n` newline flushes
      // — without this drain that line (the run's only success signal) is lost.
      flush: () => { if (buf) { safeLine(buf); buf = ''; } },
    };
  };
  const stdoutSplitter = makeSplitter('stdout');
  const stderrSplitter = makeSplitter('stderr');
  proc.stdout.on('data', stdoutSplitter.push);
  proc.stdout.on('end', stdoutSplitter.flush);
  proc.stderr.on('data', stderrSplitter.push);
  proc.stderr.on('end', stderrSplitter.flush);

  if (stallWatchdogOn) {
    stallTimer = setInterval(() => {
      // setInterval callback — runs outside the request lifecycle, so guard
      // against any throw (a crash here would take down the server process).
      try {
        if (stallKilled || activeProcess !== proc) return;
        const { stalled, idleMs, budgetMs } = stallDetector.checkStall();
        if (!stalled) return;
        stallKilled = true;
        stopStallWatchdog();
        console.log(`⏱️ training [${shortId(jobId)}] phase-aware watchdog: training stalled ${Math.round(idleMs / 1000)}s (budget ${Math.round(budgetMs / 1000)}s) — stopping stalled trainer; checkpoints preserved`);
        // Soft hang: SIGKILL straight away (SIGTERM relies on the GIL the hang
        // holds, so the 8s escalation in cancel() would just waste the window).
        // Direct PID kill through the already-owned detached handle.
        proc.kill('SIGKILL');
      } catch (err) {
        console.error(`❌ training [${shortId(jobId)}] stall watchdog tick failed: ${err?.message}`);
      }
    }, STALL_TICK_MS);
    stallTimer.unref?.();
  }

  let terminalStarted = false;
  let processError;
  proc.on('error', (err) => {
    if (terminalStarted) return;
    processError = err;
    // Detached handles can report failed signals while their trainer is live.
    // Preserve its cancel handle and claim until physical close.
    if (proc.pid) return;
    terminalStarted = true;
    stopStallWatchdog();
    if (activeProcess === proc) { activeProcess = null; activeJobId = null; }
    // A no-PID launch failure owns no running trainer.
    wakeDisplay(settings);
    // A launch failure after the run was marked `running` — a genuine spawn
    // error, including a detached observer losing its control directory.
    // Route through the same terminal cleanup as pre-spawn failures so the run
    // row doesn't stay stuck `running` and the dataset's `training` chip is
    // released; `fail()` alone only logs + emits. Async + guarded since this
    // runs outside the request lifecycle.
    Promise.resolve(failBeforeSpawn(`trainer spawn failed: ${err.message}`))
      .catch((e) => {
        retainOwnership();
        console.error(`❌ training [${shortId(jobId)}] failure cleanup failed: ${e?.message}`);
      });
  });

  proc.on('close', (code, signal) => {
    if (terminalStarted) return;
    terminalStarted = true;
    stopStallWatchdog();
    if (activeProcess === proc) { activeProcess = null; activeJobId = null; }
    // Flush the debounced progress (final checkpoint + last sample) BEFORE
    // finalize reads the run record — the collapse guard and previewImageUrl
    // both read run.artifacts. Async finalize wrapped so a rejection can't
    // escape the event handler (unhandled rejection kills the process on Node ≥15).
    // The trainer has exited; release its accelerator claim before finalizing.
    const settle = async () => {
      await releaseClaim();
      await flushProgress();
      if (processError) return failBeforeSpawn(`trainer process failed: ${processError.message}`);
      return finalizeTraining({ jobId, runId, code, signal, state: getState(), stallKilled, canceledByOperator: explicitlyCanceledTrainers.has(proc) });
    };
    const terminalWork = recoveryPermit ? recoveryPermit.run(settle) : settle();
    terminalWork
      .then(async () => { wakeDisplay(settings); await recoveryPermit?.finish(); })
      .catch((err) => {
        retainOwnership();
        // A finalize/flush rejection is still a terminal end with no auto-resume
        // enqueued, so wake the display here too — otherwise this error path
        // would leave it asleep (no .then ran). No-op unless we slept it.
        wakeDisplay(settings);
        console.error(`❌ training [${shortId(jobId)}] finalize failed: ${err?.message}`);
        fail(`finalize failed: ${err?.message}`);
      }).finally(releaseDataAccess);
  });
  } // end wireProcLifecycle
}

async function finalizeTraining({ jobId, runId, code, signal, state, stallKilled = false, canceledByOperator = false }) {
  const run = await runsDb.getRun(runId);
  const job = getJob(jobId);
  const canceled = !!job?.cancelRequested || canceledByOperator;

  // Run record vanished mid-training (direct DB edit / race — the DELETE
  // route blocks active runs, so this is defensive). Don't register a LoRA
  // with no run to anchor its sidecar lineage — that would leave an orphan
  // .safetensors in data/loras/. Just settle the job terminally.
  if (!run) {
    const msg = canceled ? 'Canceled' : `Run record vanished during finalize (exit ${code})`;
    console.error(`❌ training [${shortId(jobId)}] ${msg}`);
    trainingEvents.emit('failed', { generationId: jobId, error: msg });
    return;
  }

  if (code === 0 && state.result?.adapter_path) {
    const finalStep = Number.isInteger(state.result.steps) ? state.result.steps : (run.progress?.step ?? null);
    // Collapse guard: deploy the final adapter unless its preview diverged
    // (near-black/uniform), in which case fall back to the latest healthy
    // checkpoint. Loss is NOT used to pick — it was anti-correlated with
    // quality on the divergence run that motivated this (see checkpoints.js).
    const selection = await selectDeployableCheckpoint(run, state.result.adapter_path, finalStep);
    const filename = trainedLoraFilename({
      name: run?.name, characterName: run?.character?.name, runId,
    });
    // One backup admission from the adapter write through the run row and the
    // dataset flag that name it (#9982). The completion event fires after the
    // lease, so the queue's own admitted commit never borrows this one.
    const { sizeBytes } = await withBackupAssetPublication(async () => {
      const registered = await registerTrainedLora({
        run,
        buffer: selection.buffer,
        filename,
        result: state.result,
        previewImageUrl: selection.previewUrl,
        selectedStep: selection.step,
        autoSelected: selection.autoSelected,
      });
      await runsDb.updateRun(runId, {
        status: 'completed',
        completedAt: new Date().toISOString(),
        output: {
          loraFilename: filename,
          finalLoss: Number.isFinite(state.result.final_loss) ? state.result.final_loss : null,
          selectedCheckpointStep: selection.step,
          autoSelectedCheckpoint: selection.autoSelected,
        },
      });
      await flipDatasetAfterRun(run, { trained: true, loraFilename: filename });
      return registered;
    });
    if (selection.autoSelected) console.log(`⚠️ training [${shortId(jobId)}] ${selection.reason} (size ${sizeBytes ?? '?'}B)`);
    console.log(`✅ training [${shortId(jobId)}] complete — registered ${filename} @ step ${selection.step}`);
    trainingEvents.emit('completed', {
      generationId: jobId, runId, loraFilename: filename, filename,
    });
    return;
  }

  if (canceled) {
    // Queue's cancelRequested flips the failed event into a clean cancel;
    // record keeps the checkpoint lineage for inspection or explicit promotion.
    await runsDb.updateRun(runId, { status: 'canceled', completedAt: new Date().toISOString(), error: 'Canceled' });
    await flipDatasetAfterRun(run, { trained: false });
    trainingEvents.emit('failed', { generationId: jobId, error: 'Canceled' });
    return;
  }

  if (code === 0) {
    // Exited cleanly but emitted no parseable RESULT: line (success with a
    // result was handled above) — the trainer finished without producing an
    // adapter. Report the trainer's own USER_ERROR if it surfaced one,
    // otherwise a clear message instead of the confusing "exited with code 0".
    const message = state.userError?.message
      || 'Trainer exited cleanly but produced no adapter — check the dataset and run logs';
    await runsDb.updateRun(runId, {
      status: 'failed', completedAt: new Date().toISOString(),
      error: message, errorCode: state.userError?.kind || 'NO_RESULT',
    });
    await flipDatasetAfterRun(run, { trained: false });
    console.error(`❌ training [${shortId(jobId)}] no-result: ${message}`);
    trainingEvents.emit('failed', { generationId: jobId, error: message });
    return;
  }

  const { code: failCode, message, repo: failRepo = null } = classifyTrainingFailure({
    stderrTail: state.stderrTail, exitCode: code, signal, userError: state.userError,
  });
  await runsDb.updateRun(runId, {
    status: 'failed', completedAt: new Date().toISOString(), error: message, errorCode: failCode,
    // Gated-repo deep-link target for the UI banner (HF_AUTH only); null otherwise.
    errorRepo: failRepo,
  });
  await flipDatasetAfterRun(run, { trained: false });
  console.error(`❌ training [${shortId(jobId)}] ${failCode}: ${message}`);
  trainingEvents.emit('failed', { generationId: jobId, error: message, code: failCode, repo: failRepo });

}

/**
 * Write an adapter Buffer into data/loras/ as the registered trained LoRA and
 * emit its sidecar. Shared by finalize (collapse-guarded final) and manual
 * checkpoint promotion — both deploy a chosen adapter under one filename, so
 * promoting overwrites in place and the LoRA's identity in the picker is
 * stable across re-promotes.
 */
async function registerTrainedLora({
  run, buffer, filename, result = {}, previewImageUrl = null, selectedStep = null, autoSelected = false,
}) {
  await ensureDir(PATHS.loras);
  const dest = join(PATHS.loras, filename);
  await writeFileGuarded(dest, buffer);
  const sizeBytes = buffer.length; // bytes written === on-disk size; no stat round-trip
  const sidecar = buildTrainedSidecar({
    run, result, filename, previewImageUrl, sizeBytes, selectedStep, autoSelected,
  });
  await writeLoraSidecar(filename, sidecar);
  return { sizeBytes, sidecar };
}

/** Listable checkpoints (step, loss, preview, deployed flag) for a run. */
export async function listCheckpoints(runId) {
  const run = await runsDb.getRunRequired(runId);
  return { runId, runtime: run.runtime, checkpoints: listRunCheckpoints(run) };
}

/** Mid-training sample previews (step + url) for the live progress gallery. */
export async function listSamples(runId) {
  const run = await runsDb.getRunRequired(runId);
  return { runId, samples: listRunSamples(run) };
}

/**
 * Manually promote a checkpoint to be the deployed LoRA. Re-extracts that
 * checkpoint's adapter, registers it under the run's existing LoRA filename
 * (in place), and records the selected step on the run. Lets the user pick by
 * eye when the collapse guard's near-black veto wasn't enough (subtler
 * degradation — see the loss-is-misleading note in checkpoints.js).
 */
export async function promoteCheckpoint(runId, step) {
  return withRetiredDataAccess(() => promoteRetainedCheckpoint(runId, step));
}

async function promoteRetainedCheckpoint(runId, step) {
  const run = await runsDb.getRunRequired(runId);
  // Allow completed runs AND failed/canceled runs that saved at least one
  // checkpoint — promoting a partial checkpoint is a deliberate salvage (the
  // human clicks "Use this" on a specific preview), the same explicit intent
  // the completed-run picker relies on. Only an in-flight run is blocked: its
  // checkpoints are still moving, so "deploy this one" is ambiguous.
  if (['queued', 'running'].includes(run.status)) {
    throw new ServerError('Cancel the run before promoting a checkpoint', {
      status: 409, code: 'RUN_ACTIVE',
    });
  }
  const listed = listRunCheckpoints(run);
  const target = listed.find((c) => c.step === step);
  if (!target) {
    throw new ServerError(`No checkpoint at step ${step} for run ${runId}`, {
      status: 404, code: 'CHECKPOINT_NOT_FOUND',
    });
  }
  const buffer = await resolveCheckpointAdapterBuffer(run, step);
  const filename = run.output?.loraFilename
    || trainedLoraFilename({ name: run.name, characterName: run.character?.name, runId });
  // Keep trainedSteps pointing at the run's final step so the sidecar can note
  // "checkpoint @ step N" whenever the promoted step isn't the final one.
  const finalStep = Math.max(0, ...listed.map((c) => c.step), run.progress?.step || 0) || null;
  // Promotion replaces the deployed adapter in place, so the new bytes, the run
  // row and the dataset flag publish under one backup admission (#9982).
  await withBackupAssetPublication(async () => {
    await registerTrainedLora({
      run,
      buffer,
      filename,
      result: { steps: finalStep, final_loss: target.loss },
      previewImageUrl: target.previewUrl,
      selectedStep: step,
      autoSelected: false,
    });
    await runsDb.updateRun(runId, (current) => ({
      ...current,
      output: {
        ...current.output,
        loraFilename: filename,
        selectedCheckpointStep: step,
        autoSelectedCheckpoint: false,
      },
    }));
    await flipDatasetAfterRun(run, { trained: true, loraFilename: filename });
  });
  trainingEvents.emit('checkpoints:changed', { runId });
  console.log(`📌 training [${shortId(runId)}] promoted checkpoint step ${step} → ${filename}`);
  // If the promoted checkpoint had no preview (its step didn't land on the
  // sampleEvery cadence — most often the final step, e.g. 1188 with sampleEvery
  // 300), render one in the BACKGROUND from the just-registered LoRA so the
  // deployed card isn't blank. Fire-and-forget: promote stays snappy, and a
  // render failure (OOM, missing venv, server restart) must never fail the
  // promote — the LoRA is already deployed; the thumbnail is cosmetic.
  if (!target.previewUrl) {
    withRetiredDataAccess(() => ensureCheckpointPreview(run, step, filename)).catch((err) => {
      console.error(`⚠️ training [${shortId(runId)}] preview render for step ${step} failed: ${err?.message}`);
    });
  }
  return { loraFilename: filename, step };
}

// mflux preview filename for a step (matches stepFromSampleName's mflux regex in
// checkpoints.js, so listRunCheckpoints joins it to the checkpoint by step).
const previewSampleName = (step) => `${String(step).padStart(7, '0')}_preview_image_preview_1.png`;

/**
 * Render a neutral-prompt preview for a promoted checkpoint that has none, in the
 * background, and attach it to the run so the deployed-checkpoint card shows a
 * thumbnail. Best-effort by contract: every failure path logs and returns without
 * throwing (the caller already deployed the LoRA; this is purely cosmetic).
 *
 * Renders at the SAME neutral prompt + seed + dims the trainer uses for its
 * periodic samples (so it's visually comparable to the 600/900 thumbnails), via
 * the registered LoRA on its own 9b/4b base. Uses generateImage (writes into the
 * gallery dir + sidecar) then copies the PNG into the run's samples dir under the
 * step-joined name; the gallery copy is harmless and lets the user find it too.
 */
async function ensureCheckpointPreview(run, step, loraFilename) {
  const runId = run.id;
  const samplesDir = runSamplesDir(runId);
  const dest = join(samplesDir, previewSampleName(step));
  if (existsSync(dest)) return; // already has one (race / re-promote)

  // Resolve the inference model for this LoRA's variant. The registered base is
  // flux2-klein-<variant>; fall back to the run's baseModelId.
  const variant = run.fluxVariant || null;
  const models = getImageModels();
  const modelId = (variant && models.find((m) => m.id === `flux2-klein-${variant}`)?.id)
    || (models.find((m) => m.id === run.baseModelId)?.id)
    || run.baseModelId;
  if (!modelId) { console.error(`⚠️ training [${shortId(runId)}] no inference model for preview`); return; }

  const settings = await getSettings();
  const pythonPath = settings?.imageGen?.local?.pythonPath || null;
  const prompt = run.params?.samplePrompt || `${run.triggerWord} portrait, neutral background`;
  const seed = Number.isInteger(run.params?.seed) ? run.params.seed : 42;
  const res = run.params?.resolution || 768;

  const { generateImage } = await import('../imageGen/local.js');
  console.log(`🖼️ training [${shortId(runId)}] rendering preview for promoted step ${step} (${modelId})`);
  const result = await generateImage({
    pythonPath,
    prompt,
    modelId,
    width: res,
    height: res,
    steps: 20,
    seed,
    loraFilenames: [loraFilename],
    loraScales: [1.0],
    jobId: `lora-preview-${runId}-${step}`,
  });
  // generateImage writes PATHS.images/<filename> (filename = `${jobId}.png`).
  const renderedPath = join(PATHS.images, result?.filename || `lora-preview-${runId}-${step}.png`);
  if (!existsSync(renderedPath)) { console.error(`⚠️ training [${shortId(runId)}] preview render produced no file`); return; }

  // The copy and the row naming it are one backup-admitted publication (#9982);
  // the render above stays outside the lease.
  await withBackupAssetPublication(async () => {
    await ensureDir(samplesDir);
    await copyFileGuarded(renderedPath, dest);
    // Join is by step, sourced from artifacts.samples — append so listRunCheckpoints
    // picks it up. updateRun's function form merges against the freshest record.
    await runsDb.updateRun(runId, (current) => {
      const samples = current.artifacts?.samples || [];
      const name = previewSampleName(step);
      if (samples.includes(name)) return current;
      return { ...current, artifacts: { ...current.artifacts, samples: [...samples, name] } };
    });
  });
  trainingEvents.emit('checkpoint-preview', { generationId: run.jobId || runId, runId, step });
  trainingEvents.emit('checkpoints:changed', { runId });
  console.log(`🖼️ training [${shortId(runId)}] preview attached for step ${step}`);
}

/**
 * Boot probe used by the media-job queue's reconcile (#1332): does run `runId`
 * have a detached trainer worth re-attaching to (still alive, or finished but
 * its RESULT line never consumed) under its control dir? Lets the queue decide,
 * for a training job that was `running` at restart, whether to re-enqueue it for
 * re-attach or fail it — without the queue needing to know the run's on-disk
 * layout. Resolves false (never throws) when there's nothing to re-attach to.
 *
 * @param {string} runId
 * @returns {Promise<boolean>}
 */
export function hasSurvivingTrainer(runId) {
  if (!runId) return Promise.resolve(false);
  return isReattachable(join(runDir(runId), '.detached')).catch(() => false);
}

/**
 * Boot reconcile + terminal-state mirror. Called from server/index.js after
 * initMediaJobQueue(). Any run persisted as queued/running whose job isn't
 * live in the queue is marked failed (the queue does the same for its own
 * interrupted jobs — this keeps the two stores agreeing). Also subscribes
 * to mediaJobEvents so a queue-side cancel (user hits cancel while QUEUED,
 * which never reaches runTraining) still lands in the run record.
 *
 * A run whose detached trainer SURVIVED the restart is already re-enqueued for
 * re-attach by initMediaJobQueue (#1332) — its job is back in the queue as
 * queued/running, so the `!job || terminal` guard below skips it and the reap
 * path only touches runs that genuinely died.
 */
/** Postgres never accepted the session: missing database or no listener. */
const loraDatabaseUnreachable = (err) => {
  const code = err?.code;
  if (code === '3D000' || code === 'ECONNREFUSED' || code === 'ENOTFOUND' || code === 'ETIMEDOUT') return true;
  return /database ".+" does not exist/i.test(String(err?.message ?? ''));
};

export async function initLoraTraining() {
  const active = await runsDb.listActiveRuns().catch((err) => {
    // Unreachable database is a skip (hermetic smoke, Postgres down). A query
    // that reached the database and then failed is a reconcile failure.
    if (loraDatabaseUnreachable(err)) {
      console.warn(`⚠️ loraTraining boot reconcile skipped: ${err?.message}`);
      return [];
    }
    console.error(`❌ loraTraining boot reconcile failed: ${err?.message}`);
    return [];
  });
  for (const run of active) {
    const job = run.jobId ? getJob(run.jobId) : null;
    if (!job || ['failed', 'canceled', 'completed'].includes(job.status)) {
      // Upgrade never kills an orphan trainer. Keep its artifacts pinned until
      // the existing process finishes or the operator explicitly cancels it.
      if (await hasSurvivingTrainer(run.id)) {
        const jobId = run.jobId || run.id;
        const recoveryPermit = maintenance.recoverOwned('media', jobId);
        try {
          const observer = await recoveryPermit.run(() => runTraining({ jobId, runId: run.id, reattach: true, recoveryPermit }));
          if (observer?.observing) console.log(`🔁 Observing retired trainer ${shortId(run.id)} without restarting it`);
          else if (!observer?.preserved && !await hasSurvivingTrainer(run.id)) await recoveryPermit.finish();
        } catch (err) {
          // An uncertain observer must retain its recovery permit: the detached
          // process may still be writing, even when its database/claim is down.
          console.error(`❌ Retired trainer ${shortId(run.id)} could not be observed; artifacts stay pinned: ${err?.message}`);
        }
        continue;
      }
      await runsDb.updateRun(run.id, {
        status: job?.status === 'canceled' ? 'canceled' : 'failed', error: 'LoRA training retired; artifacts preserved', completedAt: new Date().toISOString(),
      }).catch(() => {});
      await flipDatasetAfterRun(run, { trained: false });
      console.log(`🧹 training run ${shortId(run.id)} retired; artifacts preserved`);
    }
  }

  // Queued-job cancels never reach runTraining, so mirror them here.
  mediaJobEvents.on('canceled', (job) => {
    if (job?.kind !== 'training' || !job?.params?.runId) return;
    runsDb.getRun(job.params.runId).then((run) => {
      if (!run || ['completed', 'failed', 'canceled'].includes(run.status)) return null;
      return runsDb.updateRun(run.id, {
        status: 'canceled', completedAt: new Date().toISOString(), error: job.error || 'Canceled',
      }).then(() => flipDatasetAfterRun(run, { trained: false }));
    }).catch((err) => console.error(`❌ training cancel mirror failed: ${err?.message}`));
  });
  console.log('🏋️ loraTraining initialized');
}