Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Checkpoint inspection + selection for trained LoRAs.
*
* A training run keeps every periodic checkpoint, but only one adapter is
* registered into `data/loras/` as the usable LoRA. mflux always hands back
* the FINAL step's adapter — which is exactly wrong when training diverges
* late (a recurring FLUX.2 failure: the loss keeps dropping while the sample
* image collapses to a near-black frame). Loss is therefore NOT a reliable
* "best checkpoint" signal — it was anti-correlated with quality on the run
* that motivated this module. So selection is two-pronged:
*
* 1. A cheap COLLAPSE GUARD at finalize that only vetoes the catastrophic
* case — a degenerate (near-black / near-uniform) final preview — and
* falls back to the latest non-degenerate checkpoint.
* 2. A MANUAL promote path (`promoteCheckpoint`) so the human can pick any
* checkpoint by eye from its preview thumbnail — the real safety net for
* subtler degradation the guard intentionally leaves alone.
*
* Runtime-aware: mflux checkpoints are `*_checkpoint.zip` bundles (the
* adapter is cracked out with lib/zipStream); the torch flux2 trainer writes
* `checkpoints/step-NNNNNN/pytorch_lora_weights.safetensors` directories.
*/
import { readFile } from 'fs/promises';
import { existsSync, statSync, readdirSync } from 'fs';
import { join, basename } from 'path';
import sharp from 'sharp';
import { PATHS } from '../../lib/fileUtils.js';
import { extractZipEntryToBuffer } from '../../lib/zipStream.js';
import { TRAINING_RUNTIMES } from './runtimes.js';
// Collapse thresholds — calibrated against a real divergence run: the
// black-frame final preview measured entropy 2.8 / max-channel-stdev 1.7,
// while every legitimate (even ugly) frame sat at entropy 5.5–7.1 / stdev
// 18–50. The wide gap means these vetoes never fire on a real image.
export const COLLAPSE_ENTROPY_MAX = 4.0;
export const COLLAPSE_STDEV_MAX = 6.0;
const runDirFor = (runId) => join(PATHS.trainingRuns, runId);
const samplesDirFor = (runId) => join(runDirFor(runId), 'samples');
/**
* mflux appends `_YYYYMMDD_HHMMSS` to its output dir when the configured one
* already exists (mirrors resolve_mflux_output in train_mflux_lora.py) — pick
* the newest real directory among the base name and its timestamped siblings.
*/
function resolveMfluxOutputDir(runId) {
const base = join(runDirFor(runId), 'mflux');
const dir = runDirFor(runId);
const candidates = [base];
if (existsSync(dir)) {
for (const name of readdirSync(dir)) {
if (name.startsWith('mflux_2') && statSync(join(dir, name)).isDirectory()) {
candidates.push(join(dir, name));
}
}
}
const present = candidates.filter((p) => existsSync(p) && statSync(p).isDirectory());
if (!present.length) return base;
return present.sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs)[0];
}
/** Parse the step a sample basename was rendered at (both runtime namings). */
function stepFromSampleName(name) {
// mflux: 0000250_preview_image_preview_1.png ; flux2: step-000250.png
const m = name.match(/^(\d+)_preview/) || name.match(/^step-(\d+)\.png$/);
return m ? Number(m[1]) : null;
}
/**
* Parse the boundary step a checkpoint artifact was saved at, from its
* basename. The mflux watcher tags its CHECKPOINT line with the POLL-TIME
* tqdm step, which can lag the actual save boundary by a few steps — so the
* recorded `step` may not match the boundary the preview filename embeds.
* Deriving the step from the filename (the same source `stepFromSampleName`
* uses) keeps the checkpoint↔preview join exact. Returns null if unparseable.
*/
function stepFromCheckpointName(name) {
// mflux: 0000250_checkpoint.zip ; flux2: step-000250
const m = String(name).match(/^(\d+)_checkpoint/) || String(name).match(/^step-(\d+)/);
return m ? Number(m[1]) : null;
}
/** Canonical (filename-derived) step for a recorded checkpoint, with fallback. */
const checkpointStep = (c) => stepFromCheckpointName(c.path) ?? c.step;
/** Map step → sample basename from the run's recorded sample artifacts. */
function sampleByStep(run) {
const map = new Map();
for (const s of run.artifacts?.samples || []) {
const step = stepFromSampleName(s);
if (step != null) map.set(step, s);
}
return map;
}
const sampleUrl = (runId, name) =>
name ? `/api/lora-training/runs/${runId}/samples/${name}` : null;
/**
* Listable view of a run's checkpoints — step, loss (captured at checkpoint
* time), preview thumbnail URL, and which one is currently deployed. Drives
* the manual checkpoint picker in the run UI.
*/
export function listRunCheckpoints(run) {
const samples = sampleByStep(run);
const deployedStep = run.output?.selectedCheckpointStep ?? null;
// Collapse the raw checkpoint records to one entry per (filename-derived)
// step before rendering cards. Two distinct artifacts of the run can pollute
// this list:
// - Duplicates: a MANUAL resume is a fresh job/wrapper/watcher with an empty
// `seen` set, so it re-emits checkpoints that already exist → the same
// step appears N times. Keep the entry that captured a real loss; tie-break
// on the latest record (later in append order).
// - The untrained step-0 checkpoint: mflux writes a `0000000_checkpoint.zip`
// from its num_iterations==0 monitoring block — the initial, zero-training
// adapter whose preview is the identical base-model frame and whose deploy
// is a no-op. Drop it so it can't masquerade as a "Use this" option.
const byStep = new Map();
for (const c of run.artifacts?.checkpoints || []) {
const step = checkpointStep(c);
if (step === 0) continue;
const existing = byStep.get(step);
// Replace unless the existing entry has a real loss and this one doesn't.
if (!existing || !(Number.isFinite(existing.loss) && !Number.isFinite(c.loss))) {
byStep.set(step, c);
}
}
return [...byStep.values()]
.map((c) => {
const step = checkpointStep(c);
const preview = samples.get(step) || null;
return {
step,
loss: Number.isFinite(c.loss) ? c.loss : null,
previewUrl: sampleUrl(run.id, preview),
hasPreview: !!preview,
deployed: deployedStep != null && step === deployedStep,
};
})
.sort((a, b) => a.step - b.step);
}
/**
* Every mid-training sample the run has recorded so far, as `{ step, url }`
* sorted by step. Unlike listRunCheckpoints this is keyed off the SAMPLE
* artifacts alone (samples are written on their own cadence, often denser than
* checkpoints) — it seeds the live progress gallery so a mid-run reload shows
* the full sample timeline, not just frames that arrived after re-subscribe.
*/
export function listRunSamples(run) {
return (run.artifacts?.samples || [])
.map((name) => ({ step: stepFromSampleName(name), url: sampleUrl(run.id, name) }))
.filter((s) => s.step != null)
.sort((a, b) => a.step - b.step);
}
/**
* Decide whether a preview image is a degenerate collapse (near-black /
* near-uniform). Conservative by design — only the catastrophic case. A
* missing/unreadable preview returns false (don't veto on analysis failure).
*/
export async function isPreviewCollapsed(previewPath) {
if (!previewPath || !existsSync(previewPath)) return false;
const stats = await sharp(previewPath).stats().catch(() => null);
if (!stats?.channels?.length) return false;
const rgb = stats.channels.slice(0, 3);
const maxStdev = Math.max(...rgb.map((c) => c.stdev));
const entropy = Number.isFinite(stats.entropy) ? stats.entropy : 0;
return entropy < COLLAPSE_ENTROPY_MAX || maxStdev < COLLAPSE_STDEV_MAX;
}
// Step zero-pad widths mirror each trainer's own naming: mflux writes
// NNNNNNN_checkpoint.zip (7 digits); the flux2 diffusers wrapper writes
// step-NNNNNN/ (6 digits). Used only as a fallback when the run record
// didn't capture the artifact's basename.
const pad7 = (n) => String(n).padStart(7, '0');
const pad6 = (n) => String(n).padStart(6, '0');
/**
* Resolve the latest on-disk checkpoint ARTIFACT (not its adapter) for a run,
* runtime-aware — the resume point a re-launched run hands the trainer:
* - mflux: the newest `*_checkpoint.zip` (full optimizer + adapter state) in
* the run's checkpoints dir → `mflux-train --resume <zip>`.
* - flux2: the highest `checkpoints/step-NNNNNN/` dir that carries an
* `optimizer.pt` resume bundle → `--resume-from <dir>`.
* A flux2 checkpoint dir that has only `pytorch_lora_weights.safetensors` (one
* written before optimizer-state resume existed) is NOT a resume point — the
* trainer can't restore optimizer state from it and would otherwise cold-start
* onto a random adapter. Such checkpoints are still listed/promotable via
* listRunCheckpoints + resolveCheckpointAdapterBuffer; they just can't be
* resumed. Reads the disk directly (the trainer may have written more
* checkpoints than the debounced run record captured before it was killed).
* Returns `{ step, path }` (absolute) or null when nothing resumable exists.
*/
export function resolveLatestCheckpointArtifact(run) {
// Highest-step entry whose name parses, among those a runtime-specific filter
// keeps. Returns { step, path } (absolute) or null.
const latest = (dir, keep) => {
if (!existsSync(dir)) return null;
return readdirSync(dir)
.filter(keep)
.map((n) => ({ step: stepFromCheckpointName(n) ?? -1, path: join(dir, n) }))
.filter((c) => c.step >= 0)
.sort((a, b) => b.step - a.step)[0] || null;
};
if (run.runtime === TRAINING_RUNTIMES.MFLUX) {
return latest(join(resolveMfluxOutputDir(run.id), 'checkpoints'), (n) => n.endsWith('.zip'));
}
const dir = join(runDirFor(run.id), 'checkpoints');
return latest(dir, (n) => existsSync(join(dir, n, 'optimizer.pt')));
}
/**
* Resolve a checkpoint's adapter weights to a Buffer, runtime-aware:
* - mflux: crack `*_adapter.safetensors` out of the step's checkpoint zip.
* - flux2: read `checkpoints/step-NNNNNN/pytorch_lora_weights.safetensors`.
* Throws a clear error if the checkpoint artifact is missing.
*/
export async function resolveCheckpointAdapterBuffer(run, step) {
// Match on the canonical (filename-derived) step so a promote request from
// listRunCheckpoints — which reports filename steps — finds the right
// artifact even when the recorded poll-time `step` drifted.
if (run.runtime === TRAINING_RUNTIMES.MFLUX) {
const recorded = (run.artifacts?.checkpoints || []).find((c) => checkpointStep(c) === step);
const zipName = recorded?.path || `${pad7(step)}_checkpoint.zip`;
const zipPath = join(resolveMfluxOutputDir(run.id), 'checkpoints', basename(zipName));
if (!existsSync(zipPath)) {
throw new Error(`checkpoint zip missing for step ${step} (${zipPath})`);
}
const adapter = await extractZipEntryToBuffer(
zipPath,
(name) => name.endsWith('_adapter.safetensors') && !name.includes('optimizer'),
);
if (!adapter) throw new Error(`no adapter found inside checkpoint zip for step ${step}`);
return adapter;
}
// flux2 torch runtime — diffusers save_lora_weights dir. Prefer the recorded
// checkpoint basename (symmetric with the mflux path) so a trainer naming
// change can't silently break resolution; fall back to the derived name.
const recorded = (run.artifacts?.checkpoints || []).find((c) => checkpointStep(c) === step);
const dirName = recorded?.path ? basename(recorded.path) : `step-${pad6(step)}`;
const file = join(runDirFor(run.id), 'checkpoints', dirName, 'pytorch_lora_weights.safetensors');
if (!existsSync(file)) {
throw new Error(`checkpoint adapter missing for step ${step} (${file})`);
}
return readFile(file);
}
/**
* Collapse guard for finalize: given the final adapter PortOS already
* extracted and the run's checkpoints, deploy the final unless its preview is
* a degenerate collapse — in which case walk back to the latest checkpoint
* with a healthy preview. Returns the adapter Buffer to register plus the
* selection metadata (which step, its preview, whether the guard overrode).
*
* `finalAdapterPath` is the wrapper-extracted adapter on disk (the final
* step). `finalStep` is its step. Falls back silently to the final adapter
* when there are no previews to judge (sampling disabled).
*/
export async function selectDeployableCheckpoint(run, finalAdapterPath, finalStep) {
const samples = sampleByStep(run);
const withPreview = listRunCheckpoints(run).filter((c) => c.hasPreview);
// `finalStep` is the trainer's reported end step, which IS the deployed
// adapter: for mflux it equals the final checkpoint's (filename-derived)
// step, so the picker's deployed flag matches a listed checkpoint; for flux2
// the final adapter is written separately AT finalStep (not as a checkpoint),
// and flux2 renders a sample at that step, so the proxy below still finds it.
// Health proxy: the final step's own preview, or — since mflux saves the
// final checkpoint a few steps PAST the last sampled step — the most recent.
const latest = withPreview.length ? withPreview[withPreview.length - 1] : null;
const finalPreviewName = samples.get(finalStep) || (latest ? samples.get(latest.step) : null);
const readFinal = async (reason = null) => ({
buffer: await readFile(finalAdapterPath),
step: finalStep,
previewUrl: sampleUrl(run.id, finalPreviewName),
autoSelected: false,
reason,
});
// Nothing to judge (sampling disabled) → trust the trainer's final adapter.
if (!finalPreviewName) return readFinal();
const finalCollapsed = await isPreviewCollapsed(join(samplesDirFor(run.id), finalPreviewName));
if (!finalCollapsed) return readFinal();
// Final diverged. Walk earlier checkpoints newest→oldest for a healthy one.
const earlier = withPreview
.filter((c) => c.step < finalStep)
.sort((a, b) => b.step - a.step);
for (const cand of earlier) {
const previewName = samples.get(cand.step);
const collapsed = await isPreviewCollapsed(join(samplesDirFor(run.id), previewName));
if (collapsed) continue;
const buffer = await resolveCheckpointAdapterBuffer(run, cand.step).catch(() => null);
if (!buffer) continue;
return {
buffer,
step: cand.step,
previewUrl: sampleUrl(run.id, previewName),
autoSelected: true,
reason: `final preview collapsed — auto-selected step ${cand.step}`,
};
}
// Every preview collapsed (or no earlier adapter recoverable) — keep final
// so the run still registers a (flawed) LoRA the user can inspect/replace.
return readFinal(`all previews collapsed — kept final step ${finalStep}`);
}