Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Retire LoRA training and add explicit data maintenance (#13536)

[._.]/cf8ea45a27a6
diff --git a/.env.example b/.env.example
index 0c0af1d7fe..0f497b5de0 100644
--- a/.env.example
+++ b/.env.example
@@ -298,7 +298,7 @@ PGPASSWORD=portos
# Terminate an idle queued Codex job after this many milliseconds (default: 1200000 / 20 minutes)
# MEDIA_JOB_WATCHDOG_CODEX_MS=1200000
-# Terminate an idle queued training job after this many milliseconds (default: 1800000 / 30 minutes)
+# Terminate an idle surviving retired trainer after this many milliseconds (default: 1800000 / 30 minutes)
# MEDIA_JOB_WATCHDOG_TRAINING_MS=1800000
# Terminate an idle queued audio job after this many milliseconds (default: 600000 / 10 minutes)
@@ -319,14 +319,6 @@ PGPASSWORD=portos
# LoRA warmup stall budget before step timing stabilizes (default: 1200000 / 20 minutes)
# LORA_TRAIN_STALL_WARMUP_BUDGET_MS=1200000
-# Automatic LoRA checkpoint resumes allowed after soft stalls (default: 2; 0 disables)
-# LORA_TRAIN_STALL_MAX_AUTO_RESUMES=2
-
-# Cap LoRA training quantization at this bit width (loraTraining/runtimes.js).
-# Only 8 or 4 are honoured; anything else is ignored and the memory-derived tier
-# stands. Set it to force a smaller quant on a machine whose auto-tier OOMs.
-# LORA_TRAIN_MAX_QUANT_BITS=4
-
# Ingredient catalog revisions retained per ingredient (default: 50)
# CATALOG_REVISION_RETENTION=50
diff --git a/client/src/App.jsx b/client/src/App.jsx
index 78378b948c..1b6573fdfc 100644
--- a/client/src/App.jsx
+++ b/client/src/App.jsx
@@ -227,7 +227,6 @@ const MEDIA_CREATIVE_DIRECTOR_PREFIX = /^\/media\/creative-director/;
const MEDIA_SPRITES_PREFIX = /^\/media\/sprites/;
const MEDIA_MUSIC_VIDEO_PREFIX = /^\/media\/music-video/;
const MEDIA_3D_PREFIX = /^\/media\/3d/;
-const MEDIA_TRAINING_PREFIX = /^\/media\/training/;
// The annotator lives under the Media shell (/media/annotate). A bare /annotate
// is what a bookmark, a typed URL, or a half-remembered path lands on — alias it
// rather than letting the catch-all swallow it (issue #3793).
@@ -408,6 +407,8 @@ export default function App() {
<Route path="voice/call-host" element={<VoiceCallHost />} />
<Route path="api-reference" element={<Navigate to="/api-reference/catalog" replace />} />
<Route path="api-reference/:tab" element={<ApiExplorer />} />
+ <Route path="models/training" element={<Navigate to="/settings/data-maintenance" replace />} />
+ <Route path="models/training/:recordId" element={<Navigate to="/settings/data-maintenance" replace />} />
<Route path="models" element={<Navigate to="/models/llms" replace />} />
{/* Runtimes was the LLMs page's default view until #7414 promoted it to
its own tab. A literal segment outranks `models/:tab/:recordId`, so
@@ -538,8 +539,8 @@ export default function App() {
<Route path="3d" element={<PrefixRedirect from={MEDIA_3D_PREFIX} to="/3d" />} />
<Route path="3d/:id" element={<PrefixRedirect from={MEDIA_3D_PREFIX} to="/3d" />} />
<Route path="loras" element={<RedirectWithSearch to="/models/loras" />} />
- <Route path="training" element={<PrefixRedirect from={MEDIA_TRAINING_PREFIX} to="/models/training" />} />
- <Route path="training/:datasetId" element={<PrefixRedirect from={MEDIA_TRAINING_PREFIX} to="/models/training" />} />
+ <Route path="training" element={<Navigate to="/settings/data-maintenance" replace />} />
+ <Route path="training/:datasetId" element={<Navigate to="/settings/data-maintenance" replace />} />
{/* Universes live at /universes (Create sidebar link). These
redirects keep legacy /media/universe-builder bookmarks working
after the MediaGen tab was removed. */}
diff --git a/client/src/components/dashboard/ActiveProcessingWidget.jsx b/client/src/components/dashboard/ActiveProcessingWidget.jsx
index f78cc4b671..fdc98277e2 100644
--- a/client/src/components/dashboard/ActiveProcessingWidget.jsx
+++ b/client/src/components/dashboard/ActiveProcessingWidget.jsx
@@ -56,7 +56,7 @@ function JobRow({ job, onCancel }) {
: kind === 'video' ? '/media/video'
: kind === 'image' ? '/media/image'
: kind === 'audio' ? '/media/history?type=audio'
- : kind === 'training' ? '/models/training' : '/media/history';
+ : kind === 'training' ? '/system-resources/queues' : '/media/history';
const Icon = kind === 'video' ? Film : kind === 'audio' ? AudioLines : kind === 'training' ? Cpu : ImageIcon;
const progress = Number.isFinite(job.progress) ? Math.max(0, Math.min(100, Math.round(job.progress * 100))) : null;
return (
diff --git a/client/src/components/loraTraining/CaptionModelPicker.jsx b/client/src/components/loraTraining/CaptionModelPicker.jsx
deleted file mode 100644
index 21b62e898b..0000000000
--- a/client/src/components/loraTraining/CaptionModelPicker.jsx
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Caption-model picker for the LoRA dataset workbench.
- *
- * Lists vision-capable installed local models (both Ollama + LM Studio, each
- * tagged with the provider that serves it) and lets the user pick which one
- * auto-captioning uses. The choice persists to `settings.loraTraining`
- * (captionProviderId/captionModel) so it sticks across runs and machines, and
- * is lifted to the parent via `onChange` so caption runs pass the explicit
- * provider+model. "Auto" defers to the server's vision-model auto-pick.
- */
-
-import { useEffect, useState } from 'react';
-import { Loader2, AlertTriangle } from 'lucide-react';
-import ListLoadError from '../ui/ListLoadError';
-import toast from '../ui/Toast';
-import { getVisionModels, getSettings, patchSettingsSlice } from '../../services/api';
-
-const AUTO = '__auto__';
-const keyFor = (m) => `${m.providerId}::${m.id}`;
-
-export default function CaptionModelPicker({ onChange }) {
- const [models, setModels] = useState(null); // null = loading; [] = none installed
- const [value, setValue] = useState(AUTO);
- const [saving, setSaving] = useState(false);
- const [loadError, setLoadError] = useState(null);
- const [attempt, setAttempt] = useState(0);
-
- // The model list and the saved choice seed the select together; if either
- // read fails the picker shows a Retry instead of a select that would read
- // "none installed" and report Auto for a saved model.
- useEffect(() => {
- let alive = true;
- setModels(null);
- setLoadError(null);
- Promise.all([
- getVisionModels({ silent: true }),
- getSettings({ silent: true }),
- ]).then(([vm, settings]) => {
- if (!alive) return;
- const list = vm?.models || [];
- setModels(list);
- const savedModel = settings?.loraTraining?.captionModel || null;
- const savedProvider = settings?.loraTraining?.captionProviderId || null;
- if (savedModel) {
- const match = list.find((m) => m.id === savedModel
- && (!savedProvider || m.providerId === savedProvider));
- const sel = match || { id: savedModel, providerId: savedProvider || 'lmstudio' };
- setValue(keyFor(sel));
- onChange?.({ providerId: sel.providerId, model: sel.id });
- } else {
- setValue(AUTO);
- onChange?.({ providerId: null, model: null });
- }
- }).catch((error) => {
- if (!alive) return;
- setLoadError(error instanceof Error ? error : new Error('Request failed'));
- });
- return () => { alive = false; };
- // onChange is a stable setter from the parent; reload only on retry.
- }, [attempt]);
-
- const handleChange = async (next) => {
- setValue(next);
- const picked = next === AUTO ? null : (models || []).find((m) => keyFor(m) === next) || null;
- onChange?.(picked ? { providerId: picked.providerId, model: picked.id } : { providerId: null, model: null });
- setSaving(true);
- // A slice patch, merged server-side into the stored `loraTraining`, so the
- // hand-set training knobs beside these two keys survive even when the
- // settings read above failed.
- try {
- await patchSettingsSlice('loraTraining', {
- captionProviderId: picked ? picked.providerId : null,
- captionModel: picked ? picked.id : null,
- }, { silent: true });
- } catch {
- toast.error('Could not save caption-model preference');
- } finally {
- setSaving(false);
- }
- };
-
- if (loadError) {
- return <ListLoadError what="vision models" error={loadError} onRetry={() => setAttempt((n) => n + 1)} />;
- }
-
- if (models === null) {
- return (
- <span className="text-xs text-gray-500 flex items-center gap-1">
- <Loader2 className="w-3 h-3 animate-spin" /> vision models…
- </span>
- );
- }
-
- return (
- <div className="flex items-center gap-1.5">
- <label htmlFor="lt-caption-model" className="text-xs text-gray-500">Caption model</label>
- <select
- id="lt-caption-model"
- value={value}
- onChange={(e) => handleChange(e.target.value)}
- disabled={saving}
- className="bg-port-bg border border-port-border rounded px-2 py-1 text-xs text-white max-w-[16rem] disabled:opacity-50"
- >
- <option value={AUTO}>Auto (first vision model)</option>
- {models.map((m) => (
- <option key={keyFor(m)} value={keyFor(m)}>{m.name} · {m.backend}</option>
- ))}
- </select>
- {saving && <Loader2 className="w-3 h-3 animate-spin text-gray-400" />}
- {!models.length && (
- <span className="text-[11px] text-port-warning flex items-center gap-1" title="Install a vision model (e.g. Qwen2.5-VL, LLaVA) from Models → LLMs">
- <AlertTriangle className="w-3 h-3" /> none installed
- </span>
- )}
- </div>
- );
-}
diff --git a/client/src/components/loraTraining/CaptionModelPicker.test.jsx b/client/src/components/loraTraining/CaptionModelPicker.test.jsx
deleted file mode 100644
index 42ac953dbd..0000000000
--- a/client/src/components/loraTraining/CaptionModelPicker.test.jsx
+++ /dev/null
@@ -1,40 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { act, fireEvent, render, screen } from '@testing-library/react';
-
-vi.mock('../ui/Toast', () => ({
- default: Object.assign(vi.fn(), { success: vi.fn(), error: vi.fn() }),
-}));
-const getVisionModels = vi.fn();
-const getSettings = vi.fn();
-vi.mock('../../services/api', () => ({
- getVisionModels: (...a) => getVisionModels(...a),
- getSettings: (...a) => getSettings(...a),
- patchSettingsSlice: vi.fn(),
-}));
-
-import CaptionModelPicker from './CaptionModelPicker';
-
-const renderPicker = async (onChange = vi.fn()) => {
- render(<CaptionModelPicker onChange={onChange} />);
- await act(async () => {});
- return onChange;
-};
-
-beforeEach(() => vi.clearAllMocks());
-
-describe('CaptionModelPicker', () => {
- it('shows a load error — not "none installed" or an Auto choice — when the model list fails', async () => {
- getVisionModels.mockRejectedValueOnce(new Error('Server unavailable'));
- getSettings.mockResolvedValue({ loraTraining: { captionModel: 'vision-a', captionProviderId: 'lmstudio' } });
- const onChange = await renderPicker();
- expect(screen.getByRole('alert')).toHaveTextContent('Could not load vision models — Server unavailable');
- expect(screen.queryByText('none installed')).toBeNull();
- expect(onChange).not.toHaveBeenCalled();
-
- getVisionModels.mockResolvedValueOnce({ models: [{ id: 'vision-a', providerId: 'lmstudio', name: 'Vision A', backend: 'LM Studio' }] });
- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });
- expect(screen.queryByRole('alert')).toBeNull();
- expect(screen.getByLabelText('Caption model')).toHaveValue('lmstudio::vision-a');
- expect(onChange).toHaveBeenCalledWith({ providerId: 'lmstudio', model: 'vision-a' });
- });
-});
diff --git a/client/src/components/loraTraining/CharacterLoraChip.jsx b/client/src/components/loraTraining/CharacterLoraChip.jsx
index d2d5f7e3c7..de5928f441 100644
--- a/client/src/components/loraTraining/CharacterLoraChip.jsx
+++ b/client/src/components/loraTraining/CharacterLoraChip.jsx
@@ -1,25 +1,13 @@
-/**
- * "Trained LoRA" chip + dataset entry point for a universe/catalog
- * character. Resolves lazily via /api/loras/by-character ({ silent: true }
- * — no link is a normal state, not an error) and offers a Dataset button
- * that find-or-creates the character's dataset and navigates to it.
- */
+/** Retained adapter link for catalog and universe subjects; never creates training data. */
-import { useState, useEffect, useCallback } from 'react';
-import { useNavigate, Link } from 'react-router';
-import { Sparkles, GraduationCap, Loader2 } from 'lucide-react';
-import { getCharacterLoras, createLoraDataset, listLoraDatasets } from '../../services/api';
+import { useCallback } from 'react';
+import { Link } from 'react-router';
+import { Sparkles } from 'lucide-react';
+import { getCharacterLoras } from '../../services/api';
import useListResource from '../../hooks/useListResource';
import ListLoadError from '../ui/ListLoadError';
-export default function CharacterLoraChip({ entryId, ingredientId, universeId, showDatasetButton = true }) {
- const navigate = useNavigate();
- // Existing dataset (looked up by ingredientId) — the link target on
- // surfaces like the catalog detail page that don't know the universe
- // entryId and therefore can't find-or-create.
- const [existingDataset, setExistingDataset] = useState(null);
- const [opening, setOpening] = useState(false);
-
+export default function CharacterLoraChip({ entryId, ingredientId }) {
const fetchLoras = useCallback(
() => getCharacterLoras({ entryId, ingredientId }, { silent: true }),
[entryId, ingredientId],
@@ -33,26 +21,6 @@ export default function CharacterLoraChip({ entryId, ingredientId, universeId, s
enabled: Boolean(entryId || ingredientId),
});
- const canCreate = !!(universeId && entryId);
- useEffect(() => {
- if (canCreate || (!entryId && !ingredientId)) return undefined;
- let active = true;
- listLoraDatasets(entryId ? { entryKind: 'characters', entryId } : { entryKind: 'characters', ingredientId })
- .then((list) => { if (active) setExistingDataset(list?.[0] || null); })
- .catch(err => console.warn('⚠️ Failed to load LoRA datasets: ' + err.message));
- return () => { active = false; };
- }, [canCreate, entryId, ingredientId]);
-
- const openDataset = async () => {
- setOpening(true);
- try {
- const dataset = await createLoraDataset({ universeId, entryId });
- navigate(`/models/training/${dataset.id}`);
- } finally {
- setOpening(false);
- }
- };
-
const lora = loras?.[0] || null;
return (
@@ -70,27 +38,6 @@ export default function CharacterLoraChip({ entryId, ingredientId, universeId, s
LoRA{lora.triggerWords?.[0] ? ` · ${lora.triggerWords[0]}` : ''}
</Link>
)}
- {showDatasetButton && canCreate && lorasState === 'ready' && (
- <button
- type="button"
- onClick={openDataset}
- disabled={opening}
- className="inline-flex items-center gap-1 px-2 py-0.5 rounded-full bg-port-card border border-port-border text-gray-400 hover:text-white disabled:opacity-50"
- title="Open the character's LoRA training dataset"
- >
- {opening ? <Loader2 className="w-3 h-3 animate-spin" /> : <GraduationCap className="w-3 h-3" />}
- {lora ? 'Dataset' : 'Train LoRA'}
- </button>
- )}
- {showDatasetButton && !canCreate && existingDataset && (
- <Link
- to={`/models/training/${existingDataset.id}`}
- className="inline-flex items-center gap-1 px-2 py-0.5 rounded-full bg-port-card border border-port-border text-gray-400 hover:text-white"
- title="Open the character's LoRA training dataset"
- >
- <GraduationCap className="w-3 h-3" /> Dataset
- </Link>
- )}
</div>
);
}
diff --git a/client/src/components/loraTraining/CharacterLoraChip.test.jsx b/client/src/components/loraTraining/CharacterLoraChip.test.jsx
index 5f30f2c9d7..5acee5f808 100644
--- a/client/src/components/loraTraining/CharacterLoraChip.test.jsx
+++ b/client/src/components/loraTraining/CharacterLoraChip.test.jsx
@@ -2,13 +2,9 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';
import { act, fireEvent, render, screen } from '@testing-library/react';
const getCharacterLoras = vi.fn();
-const createLoraDataset = vi.fn();
-const listLoraDatasets = vi.fn();
vi.mock('../../services/api', () => ({
getCharacterLoras: (...args) => getCharacterLoras(...args),
- createLoraDataset: (...args) => createLoraDataset(...args),
- listLoraDatasets: (...args) => listLoraDatasets(...args),
}));
const mockNavigate = vi.fn();
@@ -23,10 +19,9 @@ describe('CharacterLoraChip', () => {
beforeEach(() => {
vi.clearAllMocks();
getCharacterLoras.mockResolvedValue([]);
- listLoraDatasets.mockResolvedValue([]);
});
- it('shows a Retry — not "Train LoRA" — when the LoRA read fails, and shows "Train LoRA" after retry', async () => {
+ it('shows a Retry — not "Train LoRA" — when the LoRA read fails, and stays retired after retry', async () => {
getCharacterLoras.mockRejectedValueOnce(new Error('Server unavailable'));
render(
@@ -49,10 +44,10 @@ describe('CharacterLoraChip', () => {
});
expect(screen.queryByRole('alert')).toBeNull();
- expect(screen.getByRole('button', { name: /Train LoRA/i })).toBeInTheDocument();
+ expect(screen.queryByRole('button', { name: /Train LoRA/i })).toBeNull();
});
- it('renders LoRA chip and "Dataset" button only after a successful read', async () => {
+ it('preserves the inference adapter link without a training entry point', async () => {
getCharacterLoras.mockResolvedValueOnce([
{ filename: 'alice.safetensors', triggerWords: ['alice_lora'] },
]);
@@ -69,7 +64,7 @@ describe('CharacterLoraChip', () => {
expect(screen.queryByRole('alert')).toBeNull();
expect(screen.getByText('LoRA · alice_lora')).toBeInTheDocument();
- expect(screen.getByRole('button', { name: /Dataset/i })).toBeInTheDocument();
+ expect(screen.queryByRole('button', { name: /Dataset/i })).toBeNull();
expect(screen.queryByRole('button', { name: /Train LoRA/i })).toBeNull();
});
});
diff --git a/client/src/components/loraTraining/CheckpointPicker.jsx b/client/src/components/loraTraining/CheckpointPicker.jsx
deleted file mode 100644
index cdbf0e196e..0000000000
--- a/client/src/components/loraTraining/CheckpointPicker.jsx
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Manual checkpoint picker for a completed training run.
- *
- * A LoRA run keeps every periodic checkpoint, but only one is registered as
- * the usable LoRA. The trainer's FINAL step is often NOT the best — late
- * divergence (loss still dropping while the sample collapses to a black frame)
- * is a recurring FLUX.2 failure, and loss is no guide (it was anti-correlated
- * with quality on the run this was built for). So the user picks by eye from
- * the preview thumbnails; the server re-extracts that step's adapter and
- * overwrites the deployed LoRA in place.
- */
-
-import { useState, useCallback } from 'react';
-import useMounted from '../../hooks/useMounted';
-import useListResource from '../../hooks/useListResource';
-import ListLoadError from '../ui/ListLoadError';
-import { Layers, Loader2, CheckCircle2, AlertTriangle } from 'lucide-react';
-import toast from '../ui/Toast';
-import {
- listLoraTrainingCheckpoints,
- promoteLoraTrainingCheckpoint,
-} from '../../services/api';
-
-export default function CheckpointPicker({ run, onPromoted }) {
- const [expanded, setExpanded] = useState(false);
- // Stays true after the first expand so collapsing does not drop a loaded (or
- // failed, retryable) list back to a refetch.
- const [opened, setOpened] = useState(false);
- const [promotingStep, setPromotingStep] = useState(null);
- // Guards the background preview-poll setTimeout chain from firing after the
- // component unmounts (deferred-work-respects-unmount rule).
- const mountedRef = useMounted();
-
- const fetchCheckpoints = useCallback(
- () => listLoraTrainingCheckpoints(run.id, { silent: true }).then((res) => (Array.isArray(res?.checkpoints) ? res.checkpoints : [])),
- [run.id],
- );
- const { items: checkpoints, state, error, retry, setItems: setCheckpoints } = useListResource(fetchCheckpoints, { enabled: opened });
- const loading = state === 'loading';
-
- const toggle = () => {
- const next = !expanded;
- setExpanded(next);
- if (next) setOpened(true);
- };
-
- const promote = async (step) => {
- setPromotingStep(step);
- // If the promoted checkpoint has no thumbnail, the server renders one in the
- // background (its step wasn't on the sample cadence — e.g. the final step).
- // That render takes minutes, so poll a few times to surface it once ready
- // without a full page reload. (No socket event for this; bounded polling.)
- const lacksPreview = !checkpoints.find((c) => c.step === step)?.hasPreview;
- try {
- await promoteLoraTrainingCheckpoint(run.id, step, { silent: true });
- toast.success(`Promoted checkpoint @ step ${step}`);
- // Reactive: mark the chosen step deployed locally, then refetch + bubble
- // up so the parent's run record (loraFilename, selected step) refreshes.
- setCheckpoints((prev) => prev.map((c) => ({ ...c, deployed: c.step === step })));
- retry();
- onPromoted?.();
- if (lacksPreview) {
- // Backstop poll for the background-rendered preview (~5 min on 9B).
- // Each tick re-lists; stop once the step has a preview or attempts run out.
- let attempts = 0;
- const tick = () => {
- if (!mountedRef.current) return;
- attempts += 1;
- listLoraTrainingCheckpoints(run.id, { silent: true })
- .then((res) => {
- if (!mountedRef.current) return;
- const list = Array.isArray(res?.checkpoints) ? res.checkpoints : [];
- setCheckpoints(list);
- const ready = list.find((c) => c.step === step)?.hasPreview;
- if (!ready && attempts < 12) setTimeout(tick, 30000);
- })
- .catch(() => { if (mountedRef.current && attempts < 12) setTimeout(tick, 30000); });
- };
- setTimeout(tick, 30000);
- }
- } catch (err) {
- toast.error(`Promote failed: ${err?.message || 'unknown error'}`);
- } finally {
- setPromotingStep(null);
- }
- };
-
- const autoSelected = run.output?.autoSelectedCheckpoint;
-
- return (
- <div className="border-t border-port-border pt-2 mt-1">
- <button
- type="button"
- onClick={toggle}
- className="text-xs text-gray-300 hover:text-white flex items-center gap-1.5"
- aria-expanded={expanded}
- >
- <Layers className="w-3.5 h-3.5 text-port-accent" />
- Checkpoints{state === 'ready' ? ` (${checkpoints.length})` : ''}
- <span className="text-gray-500">{expanded ? '▲' : '▼'}</span>
- </button>
-
- {autoSelected && (
- <div className="mt-1.5 text-[11px] text-port-warning flex items-start gap-1">
- <AlertTriangle className="w-3 h-3 mt-0.5 shrink-0" />
- Final step diverged — auto-selected an earlier checkpoint. Review the previews and promote a better one if needed.
- </div>
- )}
-
- {expanded && (
- <div className="mt-2">
- {loading && (
- <div className="text-xs text-gray-400 flex items-center gap-1.5">
- <Loader2 className="w-3.5 h-3.5 animate-spin" /> Loading checkpoints…
- </div>
- )}
- {state === 'error' && <ListLoadError what="checkpoints" error={error} onRetry={retry} />}
- {state === 'ready' && checkpoints.length === 0 && (
- <div className="text-xs text-gray-500">No checkpoints recorded for this run.</div>
- )}
- {!loading && checkpoints.length > 0 && (
- <>
- <div className="text-[11px] text-gray-500 mb-1.5">
- Pick by the preview — loss is shown but is not a quality ranking.
- </div>
- <div className="grid grid-cols-2 sm:grid-cols-3 gap-2">
- {checkpoints.map((c) => (
- <div
- key={c.step}
- className={`rounded border p-1.5 flex flex-col gap-1 ${
- c.deployed ? 'border-port-success bg-port-success/10' : 'border-port-border bg-port-bg'
- }`}
- >
- {c.previewUrl ? (
- <img
- src={c.previewUrl}
- alt={`step ${c.step} preview`}
- className="w-full aspect-square object-cover rounded"
- loading="lazy"
- />
- ) : (
- <div className="w-full aspect-square rounded bg-port-card flex items-center justify-center text-[10px] text-gray-500">
- no preview
- </div>
- )}
- <div className="flex items-center justify-between text-[11px]">
- <span className="text-gray-300">step {c.step}</span>
- {c.loss != null && <span className="text-gray-500" title="training loss (not a quality score)">L {c.loss.toFixed(3)}</span>}
- </div>
- {c.deployed ? (
- <span className="text-[11px] text-port-success flex items-center gap-1">
- <CheckCircle2 className="w-3 h-3" /> Deployed
- </span>
- ) : (
- <button
- type="button"
- onClick={() => promote(c.step)}
- disabled={promotingStep != null}
- className="text-[11px] px-1.5 py-1 rounded bg-port-accent/20 text-port-accent hover:bg-port-accent/30 disabled:opacity-50 flex items-center justify-center gap-1"
- >
- {promotingStep === c.step ? <Loader2 className="w-3 h-3 animate-spin" /> : null}
- Use this
- </button>
- )}
- </div>
- ))}
- </div>
- </>
- )}
- </div>
- )}
- </div>
- );
-}
diff --git a/client/src/components/loraTraining/CheckpointPicker.test.jsx b/client/src/components/loraTraining/CheckpointPicker.test.jsx
deleted file mode 100644
index 681f990ed1..0000000000
--- a/client/src/components/loraTraining/CheckpointPicker.test.jsx
+++ /dev/null
@@ -1,40 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { act, fireEvent, render, screen } from '@testing-library/react';
-
-vi.mock('../ui/Toast', () => ({
- default: Object.assign(vi.fn(), { success: vi.fn(), error: vi.fn() }),
-}));
-const listLoraTrainingCheckpoints = vi.fn();
-vi.mock('../../services/api', () => ({
- listLoraTrainingCheckpoints: (...a) => listLoraTrainingCheckpoints(...a),
- promoteLoraTrainingCheckpoint: vi.fn(),
-}));
-
-import CheckpointPicker from './CheckpointPicker';
-
-const expand = async () => {
- render(<CheckpointPicker run={{ id: 'run-1', output: {} }} />);
- await act(async () => { fireEvent.click(screen.getByRole('button', { name: /Checkpoints/ })); });
-};
-
-beforeEach(() => vi.clearAllMocks());
-
-describe('CheckpointPicker', () => {
- it('shows a Retry — not "No checkpoints recorded" — when the list read fails, and lists them on retry', async () => {
- listLoraTrainingCheckpoints.mockRejectedValueOnce(new Error('Server unavailable'));
- await expand();
- expect(screen.getByRole('alert')).toHaveTextContent('Could not load checkpoints — Server unavailable');
- expect(screen.queryByText(/No checkpoints recorded/)).toBeNull();
-
- listLoraTrainingCheckpoints.mockResolvedValueOnce({ checkpoints: [{ step: 500, loss: 0.1, hasPreview: true, previewUrl: '/p.png' }] });
- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });
- expect(screen.queryByRole('alert')).toBeNull();
- expect(screen.getByText('step 500')).toBeInTheDocument();
- });
-
- it('says "No checkpoints recorded" only for a successful empty list', async () => {
- listLoraTrainingCheckpoints.mockResolvedValueOnce({ checkpoints: [] });
- await expand();
- expect(screen.getByText('No checkpoints recorded for this run.')).toBeInTheDocument();
- });
-});
diff --git a/client/src/components/loraTraining/DatasetImageGrid.jsx b/client/src/components/loraTraining/DatasetImageGrid.jsx
deleted file mode 100644
index d7449d478a..0000000000
--- a/client/src/components/loraTraining/DatasetImageGrid.jsx
+++ /dev/null
@@ -1,273 +0,0 @@
-/**
- * Dataset image grid — caption editing, per-image re-caption, delete.
- * Captions blur-save; local state updates reactively (no list refetch).
- */
-
-import { useState, useEffect, useCallback } from 'react';
-import {
- Loader2, RefreshCw, AlertTriangle, ChevronLeft, ChevronRight, X, Eye, EyeOff,
-} from 'lucide-react';
-import Modal from '../ui/Modal';
-import ConfirmButtonPair from '../ui/ConfirmButtonPair';
-import AutoSizeTextarea from '../ui/AutoSizeTextarea';
-import Pill from '../ui/Pill';
-import {
- updateLoraDatasetImageCaption,
- deleteLoraDatasetImage,
-} from '../../services/api';
-
-// Lightweight full-size preview for a dataset image. Kept page-local (the
-// rich MediaLightbox is bound to the media-gen `item` shape — prompts, clean/
-// regen actions, annotations — none of which a raw training image carries).
-// Navigates the ready-image subset with ← / → and Esc-closes via <Modal>.
-function DatasetImageLightbox({ datasetId, images, index, onIndexChange, onClose }) {
- const img = images[index];
- const hasPrev = index > 0;
- const hasNext = index < images.length - 1;
- useEffect(() => {
- const onKey = (e) => {
- if (e.key === 'ArrowLeft' && hasPrev) { e.preventDefault(); onIndexChange(index - 1); }
- if (e.key === 'ArrowRight' && hasNext) { e.preventDefault(); onIndexChange(index + 1); }
- };
- window.addEventListener('keydown', onKey);
- return () => window.removeEventListener('keydown', onKey);
- }, [index, hasPrev, hasNext, onIndexChange]);
- if (!img) return null;
- return (
- <Modal open onClose={onClose} size="none" backdropClassName="bg-black/90" ariaLabel="Training image preview">
- <div className="relative flex flex-col items-center gap-3">
- <button
- type="button"
- onClick={onClose}
- className="absolute -top-1 right-0 z-10 p-2 rounded-full bg-white text-black hover:bg-white/85 shadow-lg focus:outline-none focus:ring-2 focus:ring-port-accent min-h-[44px] min-w-[44px] flex items-center justify-center"
- aria-label="Close (Esc)"
- title="Close (Esc)"
- >
- <X className="w-4 h-4" />
- </button>
- <img
- src={`/data/lora-datasets/${datasetId}/images/${img.file}`}
- alt={img.caption || 'dataset image'}
- className="max-w-[92vw] max-h-[80vh] object-contain rounded-lg bg-black"
- />
- {img.caption && (
- <p className="max-w-[92vw] text-center text-sm text-gray-200 whitespace-pre-wrap">{img.caption}</p>
- )}
- {hasPrev && (
- <button
- type="button"
- onClick={() => onIndexChange(index - 1)}
- className="absolute left-2 md:-left-14 top-1/2 -translate-y-1/2 p-2.5 text-white/80 hover:text-white bg-black/50 hover:bg-black/70 backdrop-blur-sm rounded-full focus:outline-none focus:ring-2 focus:ring-port-accent"
- aria-label="Previous image"
- title="Previous (←)"
- >
- <ChevronLeft className="w-6 h-6" />
- </button>
- )}
- {hasNext && (
- <button
- type="button"
- onClick={() => onIndexChange(index + 1)}
- className="absolute right-2 md:-right-14 top-1/2 -translate-y-1/2 p-2.5 text-white/80 hover:text-white bg-black/50 hover:bg-black/70 backdrop-blur-sm rounded-full focus:outline-none focus:ring-2 focus:ring-port-accent"
- aria-label="Next image"
- title="Next (→)"
- >
- <ChevronRight className="w-6 h-6" />
- </button>
- )}
- </div>
- </Modal>
- );
-}
-
-// These chips sit ON TOP of an arbitrary training image, so they can't lean on
-// the theme's card/accent colors for contrast — a translucent tint over a bright
-// photo (or a light user theme) made the labels unreadable. Every chip carries
-// its own near-opaque dark plate (Pill `bare` + `bordered={false}`, so the ring
-// draws the edge); only the text hue distinguishes the source.
-const BADGE_BASE = 'max-w-full leading-tight font-medium bg-black/80 backdrop-blur-sm ring-1 shadow-sm';
-
-const SOURCE_BADGE = {
- generated: { label: 'generated', cls: 'text-blue-200 ring-blue-300/40' },
- upload: { label: 'upload', cls: 'text-emerald-200 ring-emerald-300/40' },
- 'refsheet-slice': { label: 'sheet crop', cls: 'text-purple-200 ring-purple-300/40' },
- gallery: { label: 'gallery', cls: 'text-sky-200 ring-sky-300/40' },
-};
-
-export default function DatasetImageGrid({
- dataset, onImagesChange, onRecaption, recaptioningId = null, captionBusy = false, draftResetToken = 0,
-}) {
- const [drafts, setDrafts] = useState({});
- // A bulk caption rewrite in the parent (e.g. strip-shared-fragments) replaces
- // dataset.images out from under any unsaved per-cell draft. Drop the drafts on
- // that signal so the next blur doesn't PATCH the superseded text back.
- useEffect(() => { setDrafts({}); }, [draftResetToken]);
- const [savingId, setSavingId] = useState(null);
- const [confirmingId, setConfirmingId] = useState(null);
- const [deletingId, setDeletingId] = useState(null);
- const [showCaptions, setShowCaptions] = useState(true);
- // Index into the ready-image subset that the lightbox is previewing (null = closed).
- const [previewIndex, setPreviewIndex] = useState(null);
-
- // Only ready images are previewable — rendering/failed cells have no file.
- const readyImages = dataset.images.filter((i) => i.status === 'ready');
- const openPreview = useCallback((img) => {
- const idx = readyImages.findIndex((i) => i.id === img.id);
- if (idx >= 0) setPreviewIndex(idx);
- }, [readyImages]);
-
- const saveCaption = async (img) => {
- const draft = drafts[img.id];
- if (draft === undefined || draft === img.caption) return;
- setSavingId(img.id);
- try {
- const updated = await updateLoraDatasetImageCaption(dataset.id, img.id, draft);
- onImagesChange((prev) => prev.map((i) => (i.id === img.id ? { ...i, ...updated } : i)));
- setDrafts((prev) => { const next = { ...prev }; delete next[img.id]; return next; });
- } finally {
- setSavingId(null);
- }
- };
-
- const removeImage = async (img) => {
- setDeletingId(img.id);
- try {
- await deleteLoraDatasetImage(dataset.id, img.id);
- onImagesChange((prev) => prev.filter((i) => i.id !== img.id));
- } finally {
- setDeletingId(null);
- setConfirmingId(null);
- }
- };
-
- if (!dataset.images.length) {
- return (
- <div className="bg-port-card border border-port-border rounded-lg p-8 text-center text-gray-400 text-sm">
- No images yet — generate reference renders, upload your own, or slice the character&apos;s reference sheet.
- </div>
- );
- }
-
- return (
- <div className="space-y-3">
- <div className="flex justify-end">
- <button
- type="button"
- onClick={() => setShowCaptions((v) => !v)}
- className="text-xs text-gray-400 hover:text-white flex items-center gap-1.5"
- title={showCaptions ? 'Hide captions — show only thumbnails' : 'Show caption editors'}
- aria-pressed={!showCaptions}
- >
- {showCaptions ? <EyeOff className="w-3.5 h-3.5" /> : <Eye className="w-3.5 h-3.5" />}
- {showCaptions ? 'Hide captions' : 'Show captions'}
- </button>
- </div>
- <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 items-start">
- {dataset.images.map((img) => {
- const badge = SOURCE_BADGE[img.source] || SOURCE_BADGE.upload;
- return (
- <div key={img.id} className="bg-port-card border border-port-border rounded-lg overflow-hidden flex flex-col">
- <div className="relative bg-port-bg">
- {img.status === 'rendering' ? (
- <div className="aspect-square flex flex-col items-center justify-center text-gray-400 gap-2">
- <Loader2 className="w-6 h-6 animate-spin" />
- <span className="text-xs">rendering…</span>
- </div>
- ) : img.status === 'failed' ? (
- <div className="aspect-square flex flex-col items-center justify-center text-port-error gap-2">
- <AlertTriangle className="w-6 h-6" />
- <span className="text-xs">render failed</span>
- </div>
- ) : (
- // Natural-height thumbnail: fixed column width, full relative
- // height so portrait/landscape crops aren't sliced by object-cover.
- <button
- type="button"
- onClick={() => openPreview(img)}
- className="block w-full cursor-zoom-in focus:outline-none focus:ring-2 focus:ring-port-accent"
- title="Click to preview full size"
- >
- <img
- src={`/data/lora-datasets/${dataset.id}/images/${img.file}`}
- alt={img.caption || 'dataset image'}
- className="block w-full h-auto"
- loading="lazy"
- />
- </button>
- )}
- {/* One wrapping row rather than two independently-anchored corners:
- a long view label ("front three-quarter") used to run under the
- source chip in a narrow column. */}
- <div className="absolute inset-x-2 top-2 flex flex-wrap items-start justify-between gap-1 pointer-events-none">
- <Pill size="xs" tone="bare" bordered={false} className={`${BADGE_BASE} ${badge.cls}`}>
- {badge.label}
- </Pill>
- {img.variation?.view && (
- <Pill size="xs" tone="bare" bordered={false} className={`${BADGE_BASE} text-gray-100 ring-white/25`}>
- {img.variation.view}
- </Pill>
- )}
- </div>
- </div>
- {showCaptions && (
- <div className="p-2 flex flex-col gap-2 flex-1">
- <AutoSizeTextarea
- value={drafts[img.id] ?? img.caption}
- onChange={(e) => setDrafts((prev) => ({ ...prev, [img.id]: e.target.value }))}
- onBlur={() => saveCaption(img)}
- placeholder={img.status === 'ready' ? 'Caption (must include the trigger word)…' : ''}
- disabled={img.status !== 'ready'}
- // Per-grid-cell field — a visible label would be noise, so the
- // accessible name comes from aria-label (the source + view give
- // a screen-reader user enough to tell cells apart).
- aria-label={`Caption for ${img.source}${img.variation?.view ? ` ${img.variation.view}` : ''} image`}
- className="w-full min-h-[4.5rem] bg-port-bg border border-port-border rounded p-2 text-xs text-gray-200 disabled:opacity-50"
- />
- <div className="flex items-center justify-between text-xs">
- <button
- type="button"
- onClick={() => onRecaption?.(img)}
- disabled={img.status !== 'ready' || captionBusy || recaptioningId === img.id}
- className="text-gray-400 hover:text-white flex items-center gap-1 disabled:opacity-50"
- title="Re-caption with the vision model"
- >
- {recaptioningId === img.id || savingId === img.id
- ? <Loader2 className="w-3.5 h-3.5 animate-spin" />
- : <RefreshCw className="w-3.5 h-3.5" />}
- Re-caption
- </button>
- {confirmingId === img.id ? (
- <ConfirmButtonPair
- prompt="Delete?"
- busy={deletingId === img.id}
- onConfirm={() => removeImage(img)}
- onCancel={() => setConfirmingId(null)}
- />
- ) : (
- <button
- type="button"
- onClick={() => setConfirmingId(img.id)}
- className="text-gray-500 hover:text-port-error"
- >
- Delete
- </button>
- )}
- </div>
- </div>
- )}
- </div>
- );
- })}
- </div>
- {previewIndex !== null && readyImages[previewIndex] && (
- <DatasetImageLightbox
- datasetId={dataset.id}
- images={readyImages}
- index={previewIndex}
- onIndexChange={setPreviewIndex}
- onClose={() => setPreviewIndex(null)}
- />
- )}
- </div>
- );
-}
diff --git a/client/src/components/loraTraining/GenerateBatchDialog.jsx b/client/src/components/loraTraining/GenerateBatchDialog.jsx
deleted file mode 100644
index 10524147c6..0000000000
--- a/client/src/components/loraTraining/GenerateBatchDialog.jsx
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Generate-batch dialog — count + optional variation-axis overrides
- * (views/expressions/outfits prefilled from the subject's canon when the
- * caller passes them). Submits to POST /api/lora-datasets/:id/generate.
- */
-
-import { useState } from 'react';
-import { Loader2, Wand2 } from 'lucide-react';
-import toast from '../ui/Toast';
-import Modal from '../ui/Modal';
-import { generateLoraDatasetImages } from '../../services/api';
-
-// Advisory dataset-quality targets, mirrored from server/lib/loraDataset.js
-// (RECOMMENDED_TRAINING_IMAGES / TRAINING_IMAGE_SWEET_SPOT_MAX). Kept local
-// so the batch default + sweet-spot copy share one source of truth with the
-// readiness coaching on LoraDatasetDetail. MAX_BATCH_IMAGES is this dialog's
-// own slider cap (matches server buildVariationMatrix's 40-tuple clamp).
-const RECOMMENDED_TRAINING_IMAGES = 20;
-const TRAINING_IMAGE_SWEET_SPOT_MAX = 30;
-const MAX_BATCH_IMAGES = 40;
-const subjectKind = (dataset) => dataset?.character?.entryKind || 'characters';
-
-const ChipToggleList = ({ idPrefix, label, options, selected, onToggle }) => (
- <div>
- <span className="block text-sm text-gray-400 mb-1">{label}</span>
- <div className="flex flex-wrap gap-1.5">
- {options.map((opt, i) => {
- const on = selected.includes(opt);
- return (
- <button
- key={`${idPrefix}-${i}`}
- type="button"
- onClick={() => onToggle(opt)}
- className={`text-xs px-2 py-1 rounded-full border ${on
- ? 'bg-port-accent/20 border-port-accent/40 text-port-accent'
- : 'bg-port-bg border-port-border text-gray-400 hover:text-white'}`}
- >
- {opt}
- </button>
- );
- })}
- </div>
- </div>
-);
-
-export default function GenerateBatchDialog({ dataset, expressionOptions = [], outfitOptions = [], onClose, onStarted }) {
- // Default to the recommended quality target rather than the bare minimum so
- // a one-click batch lands a strong dataset. See the tips block in
- // LoraDatasetDetail for the rationale (variety > volume; sweet spot range).
- const [count, setCount] = useState(RECOMMENDED_TRAINING_IMAGES);
- const [expressions, setExpressions] = useState(expressionOptions);
- const [outfits, setOutfits] = useState(outfitOptions);
- const [submitting, setSubmitting] = useState(false);
- const isCharacter = subjectKind(dataset) === 'characters';
-
- const toggle = (setter) => (value) =>
- setter((prev) => (prev.includes(value) ? prev.filter((v) => v !== value) : [...prev, value]));
-
- const submit = async () => {
- setSubmitting(true);
- try {
- const result = await generateLoraDatasetImages(dataset.id, {
- count,
- ...(expressions.length ? { expressions } : {}),
- ...(outfits.length ? { outfits } : {}),
- });
- toast.success(`Queued ${result.images.length} reference render${result.images.length === 1 ? '' : 's'} (${result.mode})`);
- onStarted(result);
- } finally {
- setSubmitting(false);
- }
- };
-
- return (
- <Modal
- open
- onClose={onClose}
- size="md"
- ariaLabelledBy="lt-generate-title"
- panelClassName="bg-port-card border border-port-border rounded-lg p-5"
- >
- <div className="space-y-4">
- <h2 id="lt-generate-title" className="text-base font-semibold text-white">
- Generate reference images — {dataset.character?.name}
- </h2>
- <p className="text-sm text-gray-400">
- Renders single-subject training images from the universe bible, varying view, composition,
- lighting, and context. Renders queue on the image lane; results stream into the dataset.
- </p>
- <div>
- <label htmlFor="lt-gen-count" className="block text-sm text-gray-400 mb-1">Images ({count})</label>
- <input
- id="lt-gen-count"
- type="range"
- min={1}
- max={MAX_BATCH_IMAGES}
- value={count}
- onChange={(e) => setCount(Number(e.target.value))}
- className="w-full"
- />
- <p className="text-xs text-gray-500 mt-1">
- Sweet spot ~{RECOMMENDED_TRAINING_IMAGES}–{TRAINING_IMAGE_SWEET_SPOT_MAX}. Fewer than 10 won&apos;t train;
- past ~{MAX_BATCH_IMAGES} mostly adds time and overfitting risk.
- </p>
- </div>
- {expressionOptions.length > 0 && (
- <ChipToggleList
- idPrefix="lt-exp"
- label={isCharacter ? 'Expressions' : 'Lighting'}
- options={expressionOptions}
- selected={expressions}
- onToggle={toggle(setExpressions)}
- />
- )}
- {outfitOptions.length > 0 && (
- <ChipToggleList
- idPrefix="lt-outfit"
- label={isCharacter ? 'Outfits' : 'Settings'}
- options={outfitOptions}
- selected={outfits}
- onToggle={toggle(setOutfits)}
- />
- )}
- <div className="flex justify-end gap-2">
- <button type="button" onClick={onClose} className="px-3 py-2 text-sm text-gray-400 hover:text-white">Cancel</button>
- <button
- type="button"
- onClick={submit}
- disabled={submitting}
- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"
- >
- {submitting ? <Loader2 className="w-4 h-4 animate-spin" /> : <Wand2 className="w-4 h-4" />}
- Generate {count}
- </button>
- </div>
- </div>
- </Modal>
- );
-}
diff --git a/client/src/components/loraTraining/ImportGalleryDialog.jsx b/client/src/components/loraTraining/ImportGalleryDialog.jsx
deleted file mode 100644
index 7d5aee55f9..0000000000
--- a/client/src/components/loraTraining/ImportGalleryDialog.jsx
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Import-from-gallery dialog for the LoRA dataset workbench.
- *
- * Multi-select picker over the local image gallery (GET /api/image-gen/gallery)
- * — the "choose images already in the system" path alongside upload/generate/
- * slice. Loads bounded, server-searched pages and accumulates a selection
- * across pages before importing them all in one POST. The server copies
- * each into the dataset (independent of the gallery original).
- */
-
-import { useCallback, useEffect, useRef, useState } from 'react';
-import { Search, X, RefreshCw, Loader2, Check } from 'lucide-react';
-import Modal from '../ui/Modal';
-import MediaCard from '../media/MediaCard';
-import toast from '../ui/Toast';
-import { normalizeImage } from '../media/normalize';
-import { listImageGalleryPage, importLoraDatasetGalleryImages } from '../../services/api';
-
-const MAX_IMPORT = 50;
-
-export default function ImportGalleryDialog({ dataset, onClose, onImported }) {
- const [items, setItems] = useState([]);
- const [loading, setLoading] = useState(false);
- const [nextOffset, setNextOffset] = useState(null);
- const [offset, setOffset] = useState(0);
- const requestId = useRef(0);
- const [error, setError] = useState(false);
- const [retry, setRetry] = useState(0);
- const [query, setQuery] = useState('');
- const [selected, setSelected] = useState([]); // ordered list of filenames
- const [importing, setImporting] = useState(false);
-
- useEffect(() => {
- const id = ++requestId.current;
- setLoading(true);
- setError(false);
- const timer = setTimeout(() => {
- listImageGalleryPage({ limit: 60, offset, q: query, hidden: false }, { silent: true })
- .then(page => {
- if (id !== requestId.current) return;
- const next = page.items.map(normalizeImage).filter(item => item.filename);
- setItems(prev => {
- const combined = offset === 0 ? next : [...prev, ...next];
- return [...new Map(combined.map(item => [item.filename, item])).values()];
- });
- const end = offset + page.items.length;
- setNextOffset(page.items.length && end < page.total ? end : null);
- })
- .catch(err => {
- if (id !== requestId.current) return;
- setError(true);
- toast.error(err.message || 'Failed to load gallery');
- })
- .finally(() => { if (id === requestId.current) setLoading(false); });
- }, query ? 250 : 0);
- return () => { clearTimeout(timer); requestId.current += 1; };
- }, [query, offset, retry]);
-
- const toggle = useCallback((filename) => {
- setSelected((prev) => {
- if (prev.includes(filename)) return prev.filter((f) => f !== filename);
- if (prev.length >= MAX_IMPORT) {
- toast.error(`Import up to ${MAX_IMPORT} at a time`);
- return prev;
- }
- return [...prev, filename];
- });
- }, []);
- const handleToggleCard = useCallback((item) => toggle(item.filename), [toggle]);
-
- const doImport = async () => {
- if (!selected.length) return;
- setImporting(true);
- try {
- const { images } = await importLoraDatasetGalleryImages(dataset.id, selected);
- toast.success(`Imported ${images.length} image${images.length === 1 ? '' : 's'} — caption them next`);
- onImported?.(images);
- } catch {
- // The api `request` helper already toasted the failure — swallow so the
- // un-awaited onClick doesn't surface an unhandled promise rejection.
- } finally {
- setImporting(false);
- }
- };
-
- return (
- <Modal
- open
- onClose={onClose}
- size="3xl"
- panelClassName="bg-port-card border border-port-border rounded-xl flex flex-col"
- ariaLabel="Import images from your gallery"
- >
- <div className="flex items-center justify-between gap-3 p-3 border-b border-port-border">
- <h2 className="text-sm font-medium text-white shrink-0">Import from gallery</h2>
- <div className="relative flex-1 max-w-md">
- <Search className="absolute left-2 top-1/2 -translate-y-1/2 w-3.5 h-3.5 text-gray-500" />
- <input
- type="text"
- value={query}
- onChange={(e) => { requestId.current += 1; setQuery(e.target.value); setOffset(0); setItems([]); setNextOffset(null); }}
- placeholder="Search prompt, model, seed, LoRA…"
- aria-label="Search gallery images"
- className="w-full pl-7 pr-7 py-1.5 text-xs bg-port-bg border border-port-border rounded-lg text-white placeholder-gray-500 focus:outline-none focus:border-port-accent"
- autoFocus
- />
- {query && (
- <button
- type="button"
- onClick={() => { requestId.current += 1; setQuery(''); setOffset(0); setItems([]); setNextOffset(null); }}
- className="absolute right-2 top-1/2 -translate-y-1/2 text-gray-500 hover:text-white"
- aria-label="Clear search"
- >
- <X className="w-3.5 h-3.5" />
- </button>
- )}
- </div>
- <button
- type="button"
- onClick={onClose}
- className="shrink-0 p-1.5 text-gray-400 hover:text-white rounded min-h-[44px] min-w-[44px] flex items-center justify-center"
- aria-label="Close"
- >
- <X className="w-4 h-4" />
- </button>
- </div>
-
- <div className="flex-1 overflow-y-auto p-3">
- {loading && offset === 0 ? (
- <div className="flex items-center justify-center gap-2 text-xs text-gray-400 py-10">
- <RefreshCw className="w-4 h-4 animate-spin" /> Loading gallery…
- </div>
- ) : items.length === 0 ? (
- <div className="text-xs text-gray-500 py-10 text-center">
- {error ? 'Gallery could not be loaded.' : query.trim() ? 'No images match your search.' : 'No images in your gallery yet.'}
- </div>
- ) : (
- <div className="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 gap-3">
- {items.map((item) => {
- const idx = selected.indexOf(item.filename);
- return (
- <MediaCard
- key={item.key}
- item={item}
- hideActions
- showCollectionMenu={false}
- selected={idx !== -1}
- selectionLabel={idx !== -1 ? String(idx + 1) : null}
- onClick={handleToggleCard}
- />
- );
- })}
- </div>
- )}
- {error && <button type="button" onClick={() => setRetry(n => n + 1)} className="w-full min-h-[44px] text-port-accent">Retry loading images</button>}
- {!error && nextOffset !== null && (
- <button type="button" disabled={loading} onClick={() => setOffset(nextOffset)}
- className="w-full min-h-[44px] mt-3 text-sm text-port-accent disabled:opacity-50">
- {loading ? 'Loading…' : 'Show more'}
- </button>
- )}
- </div>
-
- <div className="flex items-center justify-between gap-3 p-3 border-t border-port-border">
- <span className="text-xs text-gray-400">
- {selected.length ? `${selected.length} selected` : 'Click images to select'}
- </span>
- <div className="flex items-center gap-2">
- <button
- type="button"
- onClick={onClose}
- className="px-3 py-2 text-sm text-gray-400 hover:text-white"
- >
- Cancel
- </button>
- <button
- type="button"
- onClick={doImport}
- disabled={!selected.length || importing}
- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"
- >
- {importing ? <Loader2 className="w-4 h-4 animate-spin" /> : <Check className="w-4 h-4" />}
- Import {selected.length || ''}
- </button>
- </div>
- </div>
- </Modal>
- );
-}
diff --git a/client/src/components/loraTraining/ImportGalleryDialog.test.jsx b/client/src/components/loraTraining/ImportGalleryDialog.test.jsx
deleted file mode 100644
index f0fc5d414e..0000000000
--- a/client/src/components/loraTraining/ImportGalleryDialog.test.jsx
+++ /dev/null
@@ -1,44 +0,0 @@
-import { describe, it, expect, vi } from 'vitest';
-import { render, screen, fireEvent, waitFor } from '@testing-library/react';
-import ImportGalleryDialog from './ImportGalleryDialog';
-
-const api = vi.hoisted(() => ({
- listImageGalleryPage: vi.fn(),
- importLoraDatasetGalleryImages: vi.fn(),
-}));
-vi.mock('../../services/api', () => api);
-vi.mock('../media/MediaCard', () => ({
- default: ({ item, onClick }) => <button onClick={() => onClick(item)}>{item.filename}</button>,
-}));
-vi.mock('../ui/Modal', () => ({ default: ({ children }) => <div>{children}</div> }));
-
-describe('gallery import paging', () => {
- it('pages and searches on the server while retaining selected filenames', async () => {
- api.listImageGalleryPage.mockImplementation(async ({ offset, q }) => ({
- items: q ? [{ filename: 'found.png' }] : offset === 0 ? [{ filename: 'first.png' }]
- : offset === 1 ? [{ filename: 'first.png' }, { filename: 'second.png' }] : [],
- total: q ? 1 : 4, limit: 60, offset,
- }));
- api.importLoraDatasetGalleryImages.mockResolvedValue({ images: [] });
- const onImported = vi.fn();
- render(<ImportGalleryDialog dataset={{ id: 'dataset' }} onClose={vi.fn()} onImported={onImported} />);
- fireEvent.click(await screen.findByText('first.png'));
- fireEvent.click(screen.getByRole('button', { name: 'Show more' }));
- await screen.findByText('second.png');
- expect(screen.getAllByText('first.png')).toHaveLength(1);
- expect(api.listImageGalleryPage).toHaveBeenCalledWith(
- { limit: 60, offset: 1, q: '', hidden: false }, { silent: true });
- fireEvent.click(screen.getByRole('button', { name: 'Show more' }));
- await waitFor(() => expect(screen.queryByRole('button', { name: 'Show more' })).toBeNull());
- expect(api.listImageGalleryPage).toHaveBeenLastCalledWith(
- { limit: 60, offset: 3, q: '', hidden: false }, { silent: true });
- fireEvent.change(screen.getByLabelText('Search gallery images'), { target: { value: 'fox' } });
- await screen.findByText('found.png');
- expect(screen.queryByText('first.png')).toBeNull();
- expect(api.listImageGalleryPage).toHaveBeenLastCalledWith(
- { limit: 60, offset: 0, q: 'fox', hidden: false }, { silent: true });
- fireEvent.click(screen.getByRole('button', { name: 'Import 1' }));
- await waitFor(() => expect(onImported).toHaveBeenCalledWith([]));
- expect(api.importLoraDatasetGalleryImages).toHaveBeenCalledWith('dataset', ['first.png']);
- });
-});
diff --git a/client/src/components/loraTraining/LiveSampleGallery.jsx b/client/src/components/loraTraining/LiveSampleGallery.jsx
deleted file mode 100644
index c7135b1861..0000000000
--- a/client/src/components/loraTraining/LiveSampleGallery.jsx
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Live sample gallery for an in-flight training run.
- *
- * The trainer renders a preview image every `sampleEvery` steps (SAMPLE lines
- * on the wire). This surfaces that evolution as it happens: a large current
- * preview, a step-keyed thumbnail strip to scrub back through earlier samples,
- * a loss sparkline, and step/ETA metrics — so you can watch the LoRA learn (and
- * catch a late divergence) instead of waiting for the run to finish.
- *
- * Samples merge two sources so a mid-run reload still shows the full timeline:
- * - a one-shot seed from GET /runs/:id/samples (every sample persisted so far)
- * - live SSE frames carrying { currentImage, step } (samples since subscribe)
- * Live wins on a duplicate step. The loss series is live-only (the run record
- * keeps just the last step), so it builds up over the session.
- */
-
-import { useEffect, useMemo, useState } from 'react';
-import { ImageOff, Radio } from 'lucide-react';
-import { formatDurationMs } from '../../utils/formatters';
-import { useTimeTick } from '../../hooks/useTimeTick';
-import { lossSparklineGeometry } from '../../lib/lossSparkline';
-import { listLoraTrainingSamples } from '../../services/api';
-import useMounted from '../../hooks/useMounted';
-
-const SPARK_W = 240;
-const SPARK_H = 36;
-
-function LossSparkline({ series }) {
- const { points, last } = lossSparklineGeometry(series, { width: SPARK_W, height: SPARK_H });
- if (!points) return null;
- return (
- <div>
- <div className="flex items-center justify-between text-[11px] text-gray-500 mb-0.5">
- <span>Loss</span>
- <span className="text-gray-400">{last.toFixed(4)}</span>
- </div>
- <svg
- viewBox={`0 0 ${SPARK_W} ${SPARK_H}`}
- preserveAspectRatio="none"
- className="w-full h-9 bg-port-bg rounded border border-port-border"
- role="img"
- aria-label={`Training loss curve, latest ${last.toFixed(4)}`}
- >
- <polyline points={points} fill="none" stroke="#3b82f6" strokeWidth="1.5" vectorEffect="non-scaling-stroke" />
- </svg>
- </div>
- );
-}
-
-export default function LiveSampleGallery({ run, frames, progress, message }) {
- const [seed, setSeed] = useState([]);
- const [selectedStep, setSelectedStep] = useState(null);
- // Seconds-precision tick for the elapsed/ETA labels — shared, visibility-
- // aware timer (pauses when the tab is hidden) instead of a per-mount interval.
- const now = useTimeTick(1000);
- const mountedRef = useMounted();
-
- // One-shot seed of every sample persisted so far (survives a mid-run reload).
- useEffect(() => {
- if (!run?.id) return undefined;
- let active = true;
- listLoraTrainingSamples(run.id)
- .then((res) => { if (active && mountedRef.current) setSeed(Array.isArray(res?.samples) ? res.samples : []); })
- .catch(err => console.warn('⚠️ Failed to load training samples: ' + err.message));
- return () => { active = false; };
- }, [run?.id]);
-
- // Merge seed + live frames into a step-sorted, dedup'd sample list.
- const samples = useMemo(() => {
- const byStep = new Map();
- for (const s of seed) byStep.set(s.step, s.url);
- for (const f of frames) {
- if (typeof f.currentImage === 'string' && f.currentImage && typeof f.step === 'number') {
- byStep.set(f.step, f.currentImage);
- }
- }
- return [...byStep.entries()].map(([step, url]) => ({ step, url })).sort((a, b) => a.step - b.step);
- }, [seed, frames]);
-
- // Live loss series for the sparkline (progress frames carry numeric loss).
- const lossSeries = useMemo(() => {
- const out = [];
- for (const f of frames) {
- if (f.type === 'progress' && typeof f.loss === 'number' && typeof f.step === 'number') {
- out.push({ step: f.step, loss: f.loss });
- }
- }
- return out;
- }, [frames]);
-
- const latest = samples.length ? samples[samples.length - 1] : null;
- // `selectedStep == null` means "follow the latest sample"; a pinned step
- // shows that earlier sample until the user jumps back to live.
- const selected = (selectedStep != null && samples.find((s) => s.step === selectedStep)) || latest;
-
- const startedAt = run?.startedAt ? new Date(run.startedAt).getTime() : null;
- const elapsedMs = startedAt ? Math.max(0, now - startedAt) : null;
- const pct = typeof progress === 'number' ? Math.max(0, Math.min(1, progress)) : 0;
- const etaMs = elapsedMs != null && pct > 0.01 && pct < 1 ? (elapsedMs / pct) * (1 - pct) : null;
-
- return (
- <div className="space-y-2">
- <div className="flex flex-col sm:flex-row gap-3">
- {/* Current / selected preview */}
- <div className="sm:w-2/5 shrink-0">
- {selected ? (
- <div className="relative">
- <img
- src={selected.url}
- alt={`sample @ step ${selected.step}`}
- className="w-full aspect-square object-cover rounded border border-port-border"
- />
- <span className="absolute bottom-1 left-1 px-1.5 py-0.5 rounded bg-black/70 text-[11px] text-white">
- step {selected.step}
- </span>
- </div>
- ) : (
- <div className="w-full aspect-square rounded border border-port-border bg-port-bg flex flex-col items-center justify-center gap-1 text-gray-500">
- <ImageOff className="w-6 h-6" />
- <span className="text-[11px] text-center px-2">
- {(run?.params?.sampleEvery ?? 0) > 0
- ? `First preview renders at step ${run.params.sampleEvery}`
- : 'Sample previews disabled for this run'}
- </span>
- </div>
- )}
- </div>
-
- {/* Metrics + loss curve */}
- <div className="flex-1 min-w-0 space-y-2">
- <div className="text-xs text-gray-400 truncate">{message || 'Starting…'}</div>
- <div className="grid grid-cols-3 gap-2 text-center">
- <div className="bg-port-bg rounded border border-port-border py-1">
- <div className="text-sm text-white font-medium">{Math.round(pct * 100)}%</div>
- <div className="text-[10px] text-gray-500">progress</div>
- </div>
- <div className="bg-port-bg rounded border border-port-border py-1">
- <div className="text-sm text-white font-medium">{elapsedMs != null ? formatDurationMs(elapsedMs) : '—'}</div>
- <div className="text-[10px] text-gray-500">elapsed</div>
- </div>
- <div className="bg-port-bg rounded border border-port-border py-1">
- <div className="text-sm text-white font-medium">{etaMs != null ? `~${formatDurationMs(etaMs)}` : '—'}</div>
- <div className="text-[10px] text-gray-500">eta</div>
- </div>
- </div>
- <LossSparkline series={lossSeries} />
- </div>
- </div>
-
- {/* Thumbnail strip — scrub back through earlier samples */}
- {samples.length > 1 && (
- <div>
- <div className="flex items-center justify-between mb-1">
- <span className="text-[11px] text-gray-500">{samples.length} samples</span>
- {selectedStep != null && (
- <button
- type="button"
- onClick={() => setSelectedStep(null)}
- className="text-[11px] text-port-accent hover:underline flex items-center gap-1"
- >
- <Radio className="w-3 h-3" /> Jump to latest
- </button>
- )}
- </div>
- <div className="flex gap-1.5 overflow-x-auto pb-1">
- {samples.map((s) => {
- const active = selected && s.step === selected.step;
- return (
- <button
- key={s.step}
- type="button"
- onClick={() => setSelectedStep(selectedStep === s.step ? null : s.step)}
- title={`step ${s.step}`}
- className={`shrink-0 rounded border overflow-hidden ${active ? 'border-port-accent' : 'border-port-border'}`}
- >
- <img src={s.url} alt={`step ${s.step}`} className="w-14 h-14 object-cover" loading="lazy" />
- </button>
- );
- })}
- </div>
- </div>
- )}
- </div>
- );
-}
diff --git a/client/src/components/loraTraining/TrainingPanel.jsx b/client/src/components/loraTraining/TrainingPanel.jsx
deleted file mode 100644
index 6088ab1756..0000000000
--- a/client/src/components/loraTraining/TrainingPanel.jsx
+++ /dev/null
@@ -1,491 +0,0 @@
-/**
- * Training launch + live-run panel for a dataset.
- *
- * Launch gates on dataset readiness AND no in-flight trigger-word save
- * (the run reads the persisted trigger word, not the form). Live progress
- * streams over the run's media-job SSE; sample previews ride the 'preview'
- * frames. Completed runs surface the registered LoRA with a deep link.
- */
-
-import { useState, useEffect, useCallback } from 'react';
-import { Link } from 'react-router';
-import { Dumbbell, Loader2, Square, CheckCircle2, XCircle, Sparkles, RotateCcw, Moon, AlertTriangle, Eraser } from 'lucide-react';
-import toast from '../ui/Toast';
-import { useSseProgress } from '../../hooks/useSseProgress';
-import { useAsyncAction } from '../../hooks/useAsyncAction';
-import useListResource from '../../hooks/useListResource';
-import ListLoadError from '../ui/ListLoadError';
-import CheckpointPicker from './CheckpointPicker';
-import LiveSampleGallery from './LiveSampleGallery';
-import {
- getLoraTrainingStatus,
- listLoraTrainingRuns,
- startLoraTrainingRun,
- cancelLoraTrainingRun,
- resumeLoraTrainingRun,
- listImageModels,
- stripLoraDatasetSharedCaptionFragments,
-} from '../../services/api';
-
-const isActive = (run) => run && ['queued', 'running'].includes(run.status);
-
-// The status is a single object; wrapping it in a one-item list lets it ride
-// the same loading | ready | error resource as the lists, so a failed read is
-// never mistaken for "no engine installed".
-const fetchStatus = () => getLoraTrainingStatus({ silent: true }).then((s) => [s]);
-
-// FLUX.2 Klein only — mflux ≥0.17 dropped FLUX.1 training, and the torch
-// fallback trains the same bf16 Klein bases.
-const fetchTrainableModels = () => listImageModels({ silent: true })
- .then((list) => (Array.isArray(list) ? list : []).filter((m) => m.runner === 'flux2'));
-
-// Default to the 9B base — it captures identity better than 4B and trains
-// fine via 4-bit QLoRA (the server forces the panic-safe quant tier
-// regardless of which inference id is picked). Prefer the plain 9B id we
-// validated (flux2-klein-9b); avoid the explicit *-bf16 variant (the heavy
-// path that GPU-watchdog-panics this M5). Fall back to any 9B, then first.
-const defaultBaseModelId = (trainable) => {
- const nineB = trainable.filter((m) => /9b/i.test(m.id));
- const nonBf16 = nineB.find((m) => !/bf16/i.test(m.id));
- return (nonBf16 || nineB[0] || trainable[0])?.id || '';
-};
-
-export default function TrainingPanel({ dataset, readiness, triggerSaving, onRunFinished }) {
- const { items: statusItems, state: statusState, error: statusError, retry: retryStatus } = useListResource(fetchStatus);
- const status = statusItems[0] ?? null;
- const { items: models, state: modelsState, error: modelsError, retry: retryModels } = useListResource(fetchTrainableModels);
- const [baseModelId, setBaseModelId] = useState('');
- const [params, setParams] = useState(null);
- const [starting, setStarting] = useState(false);
- const [canceling, setCanceling] = useState(false);
- const [resuming, setResuming] = useState(false);
- // Server-detected caption identity-leak (409 CAPTION_IDENTITY_LEAK): the
- // fragments that repeat + the captioned-image total they're measured against.
- // Non-empty fragments → render the inline confirm row instead of queueing.
- const [captionLeak, setCaptionLeak] = useState({ fragments: [], total: 0 });
-
- useEffect(() => {
- if (status) setParams((prev) => prev || { ...status.defaults });
- }, [status]);
-
- useEffect(() => {
- if (modelsState === 'ready') setBaseModelId((prev) => prev || defaultBaseModelId(models));
- }, [models, modelsState]);
-
- // A reassigned dataset keeps its id but its old runs belong to the previous
- // subject. Re-fetch (and re-filter) whenever the subject identity moves,
- // not just the id, and show only runs matching the current (universeId,
- // entryId) so the panel + checkpoint picker can't surface/promote a stale
- // run from the prior subject.
- const charEntryId = dataset.character?.entryId;
- const charUniverseId = dataset.character?.universeId;
- const charEntryKind = dataset.character?.entryKind || 'characters';
- const fetchRuns = useCallback(() => (
- // Query by dataset id, then guard by the full subject identity. That covers
- // character/object/place reassignments without relying on the legacy
- // `character_id` mirror column.
- listLoraTrainingRuns({ datasetId: dataset.id, limit: 5 }, { silent: true }).then((runs) => (Array.isArray(runs) ? runs : []).filter(
- (r) => !r.character || (
- r.character.entryId === charEntryId
- && r.character.universeId === charUniverseId
- && (r.character.entryKind || 'characters') === charEntryKind
- ),
- ))
- ), [dataset.id, charEntryId, charEntryKind, charUniverseId]);
- const { items: runs, state: runsState, error: runsError, retry: retryRuns, setItems: setRuns } = useListResource(fetchRuns);
- const activeRun = runs.find(isActive) || null;
- const lastRun = runs.find((r) => !isActive(r)) || null;
- // Re-read in place so the panel does not flash back to the launch form. A
- // failed re-read drops to the resource's error state, which disables Start:
- // an unknown run state must never offer a second multi-hour launch.
- const refreshRuns = useCallback(() => {
- fetchRuns().then(setRuns).catch((err) => {
- console.warn('⚠️ Failed to refresh training runs: ' + err.message);
- retryRuns();
- });
- }, [fetchRuns, setRuns, retryRuns]);
-
- const sseUrl = activeRun ? `/api/lora-training/runs/${activeRun.id}/events` : null;
- const { frames, latest, closed } = useSseProgress(sseUrl, { enabled: !!activeRun });
-
- useEffect(() => {
- if (!closed || !activeRun) return;
- // Terminal frame (or stream loss) — re-read the run record for truth.
- refreshRuns();
- onRunFinished?.();
- }, [closed, activeRun, refreshRuns, onRunFinished]);
-
- // Engine pick mirrors the server: mflux (MLX) when its trainer is
- // installed, else the torch venv. Either being ready unblocks training.
- const engineReady = !!(status?.runtimes?.mflux?.ready || status?.runtimes?.flux2?.ready);
-
- // `silent: true` — this owns its own error surfacing: the expected
- // CAPTION_IDENTITY_LEAK 409 becomes the inline confirm row (not a toast), and
- // every other failure toasts here exactly once.
- const start = async ({ acknowledgeCaptionLeak = false } = {}) => {
- setStarting(true);
- try {
- await startLoraTrainingRun(
- { datasetId: dataset.id, baseModelId, params, acknowledgeCaptionLeak },
- { silent: true },
- );
- setCaptionLeak({ fragments: [], total: 0 });
- toast.success('Training queued');
- refreshRuns();
- } catch (err) {
- if (err?.code === 'CAPTION_IDENTITY_LEAK') {
- setCaptionLeak({ fragments: err.context?.sharedFragments || [], total: err.context?.total || 0 });
- } else {
- toast.error(err?.message || 'Failed to queue training');
- }
- } finally {
- setStarting(false);
- }
- };
-
- // From the leak row: strip the shared identity fragments server-side, sync the
- // parent's caption view, then re-launch (the gate now passes).
- const [stripAndQueue, strippingLeak] = useAsyncAction(async () => {
- const { removedFragments } = await stripLoraDatasetSharedCaptionFragments(dataset.id);
- if (removedFragments?.length) {
- toast.success(`Stripped ${removedFragments.length} shared identity fragment${removedFragments.length === 1 ? '' : 's'}`);
- }
- onRunFinished?.(); // parent reloads the dataset so captions + the lint banner refresh
- setCaptionLeak({ fragments: [], total: 0 });
- await start();
- }, { errorMessage: 'Failed to strip captions' });
-
- const cancel = async () => {
- if (!activeRun) return;
- setCanceling(true);
- try {
- await cancelLoraTrainingRun(activeRun.id);
- toast.success('Cancel requested — the trainer saves a checkpoint first');
- } finally {
- setCanceling(false);
- }
- };
-
- const resume = async () => {
- if (!lastRun) return;
- setResuming(true);
- try {
- const { fromStep } = await resumeLoraTrainingRun(lastRun.id);
- toast.success(`Resuming from checkpoint @ step ${fromStep}`);
- refreshRuns();
- } finally {
- setResuming(false);
- }
- };
-
- const setParam = (key, value) => setParams((prev) => ({ ...prev, [key]: value }));
-
- // A killed run keeps its checkpoints — surface the picker (view + salvage a
- // partial LoRA) whenever a non-completed run recorded one, and a Resume button
- // for any failed/canceled run. Gate Resume on status, NOT the recorded
- // checkpoint count: a crash can kill the run before the debounced record
- // persists its last checkpoint, yet the server resumes from disk — let its
- // 409 (NO_RESUMABLE_CHECKPOINT) handle the genuinely-empty case. Both runtimes
- // support true optimizer-state resume (mflux via --resume, FLUX.2 torch via
- // --resume-from), so this no longer filters by runtime.
- const lastRunCheckpoints = lastRun?.artifacts?.checkpoints?.length || 0;
- const canResume = lastRun && ['failed', 'canceled'].includes(lastRun.status);
-
- const disabledReason = !readiness?.trainable
- ? `Needs ${readiness?.required ?? 10} captioned images (have ${readiness?.captioned ?? 0})`
- : triggerSaving ? 'Saving trigger word…'
- : statusState === 'error' ? 'Could not read training status'
- : runsState === 'error' ? 'Could not read training runs'
- : modelsState === 'error' ? 'Could not load base models'
- : (statusState === 'loading' || runsState === 'loading') ? 'Checking training status…'
- : !baseModelId ? 'Pick a base model'
- : !engineReady ? 'No training engine — install mflux ≥0.17 or run scripts/setup-image-video.sh'
- : null;
-
- if (activeRun) {
- const progress = typeof latest?.progress === 'number' ? latest.progress : (activeRun.progress?.step || 0) / (activeRun.progress?.totalSteps || 1);
- return (
- <div className="bg-port-card border border-port-border rounded-lg p-4 space-y-3">
- <div className="flex items-center justify-between">
- <div className="flex items-center gap-2 text-white font-medium">
- <Loader2 className="w-4 h-4 animate-spin text-port-warning" />
- Training {activeRun.runtime} · {activeRun.baseModelId}
- </div>
- <button
- type="button"
- onClick={cancel}
- disabled={canceling}
- className="px-2 py-1 text-xs rounded bg-port-error/20 text-port-error hover:bg-port-error/30 flex items-center gap-1 disabled:opacity-50"
- >
- <Square className="w-3 h-3" /> {canceling ? 'Canceling…' : 'Cancel'}
- </button>
- </div>
- <div className="h-2 bg-port-bg rounded overflow-hidden">
- <div className="h-full bg-port-accent transition-all" style={{ width: `${Math.round(progress * 100)}%` }} />
- </div>
- {activeRun.status === 'queued' ? (
- <div className="text-xs text-gray-400">Queued (position {latest?.position ?? '…'})</div>
- ) : (
- <LiveSampleGallery run={activeRun} frames={frames} progress={progress} message={latest?.message} />
- )}
- </div>
- );
- }
-
- return (
- <div className="bg-port-card border border-port-border rounded-lg p-4 space-y-3">
- <div className="flex items-center gap-2 text-white font-medium">
- <Dumbbell className="w-4 h-4 text-port-accent" /> Train LoRA
- </div>
- {lastRun && (
- <div className="text-xs flex items-center gap-1.5">
- {lastRun.status === 'completed' ? (
- <>
- <CheckCircle2 className="w-3.5 h-3.5 text-port-success" />
- <span className="text-gray-300">
- Last run completed
- {lastRun.output?.loraFilename && (
- <> — <Link to="/models/loras" className="text-port-accent hover:underline inline-flex items-center gap-1"><Sparkles className="w-3 h-3" />{lastRun.output.loraFilename}</Link></>
- )}
- </span>
- </>
- ) : (
- <>
- <XCircle className="w-3.5 h-3.5 text-port-error" />
- <span className="text-gray-400" title={lastRun.error || ''}>Last run {lastRun.status}{lastRun.error ? ` — ${lastRun.error.slice(0, 120)}` : ''}</span>
- </>
- )}
- </div>
- )}
- {lastRun?.status === 'failed' && lastRun.errorCode === 'HF_AUTH' && (
- <div role="status" className="rounded-lg border border-port-warning/40 bg-port-warning/10 px-3 py-3 text-xs text-port-warning space-y-2">
- <div className="font-semibold text-sm">Model access required</div>
- <div className="text-port-warning/90">
- The training base model is gated on HuggingFace. Accept its license with the account
- that owns your stored token, then retry — no token change needed.
- </div>
- {lastRun.errorRepo && (
- <a
- href={`https://huggingface.co/${lastRun.errorRepo}`}
- target="_blank"
- rel="noreferrer"
- className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-lg bg-port-accent text-white text-xs font-medium hover:bg-port-accent/80"
- >
- Request access to {lastRun.errorRepo} ↗
- </a>
- )}
- </div>
- )}
- {canResume && (
- <div className="flex items-center justify-between gap-2 rounded-lg border border-port-accent/30 bg-port-accent/5 px-3 py-2">
- <span className="text-xs text-gray-300">
- {lastRunCheckpoints > 0
- ? `${lastRunCheckpoints} checkpoint${lastRunCheckpoints === 1 ? '' : 's'} saved — pick up where it stopped.`
- : 'Pick up from the last saved checkpoint.'}
- </span>
- <button
- type="button"
- onClick={resume}
- disabled={resuming}
- className="px-2.5 py-1.5 text-xs rounded bg-port-accent text-white hover:bg-port-accent/80 disabled:opacity-50 flex items-center gap-1.5 shrink-0"
- >
- {resuming ? <Loader2 className="w-3.5 h-3.5 animate-spin" /> : <RotateCcw className="w-3.5 h-3.5" />}
- Resume training
- </button>
- </div>
- )}
- {(lastRun?.status === 'completed' || lastRunCheckpoints > 0) && (
- <CheckpointPicker run={lastRun} onPromoted={refreshRuns} />
- )}
- {statusState === 'error' && <ListLoadError what="training status" error={statusError} onRetry={retryStatus} />}
- {runsState === 'error' && <ListLoadError what="training runs" error={runsError} onRetry={retryRuns} />}
- <div className="grid grid-cols-2 gap-3">
- <div className="col-span-2">
- {modelsState === 'error' ? (
- <ListLoadError what="base models" error={modelsError} onRetry={retryModels} />
- ) : (
- <>
- <label htmlFor="lt-base-model" className="block text-xs text-gray-400 mb-1">Base model</label>
- <select
- id="lt-base-model"
- value={baseModelId}
- onChange={(e) => setBaseModelId(e.target.value)}
- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"
- >
- {modelsState === 'loading' && <option value="">Loading models…</option>}
- {modelsState === 'ready' && !models.length && <option value="">No trainable models registered</option>}
- {models.map((m) => <option key={m.id} value={m.id}>{m.name || m.id}</option>)}
- </select>
- </>
- )}
- </div>
- {params && [
- ['steps', 'Steps', 10, 10000],
- ['rank', 'Rank', 1, 128],
- ['checkpointEvery', 'Checkpoint every', 0, 5000],
- ['sampleEvery', 'Sample every', 0, 5000],
- ].map(([key, label, min, max]) => (
- <div key={key}>
- <label htmlFor={`lt-param-${key}`} className="block text-xs text-gray-400 mb-1">{label}</label>
- <input
- id={`lt-param-${key}`}
- type="number"
- min={min}
- max={max}
- value={params[key] ?? ''}
- onChange={(e) => setParam(key, Number(e.target.value))}
- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"
- />
- </div>
- ))}
- {params && (
- <div>
- <label htmlFor="lt-param-resolution" className="block text-xs text-gray-400 mb-1">Resolution</label>
- <select
- id="lt-param-resolution"
- value={params.resolution ?? 512}
- onChange={(e) => setParam('resolution', Number(e.target.value))}
- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"
- >
- {[512, 768, 1024].map((r) => <option key={r} value={r}>{r}px</option>)}
- </select>
- </div>
- )}
- {params && (
- <div>
- <label htmlFor="lt-param-lr" className="block text-xs text-gray-400 mb-1">Learning rate</label>
- <input
- id="lt-param-lr"
- type="number"
- step="0.00001"
- min="0.000001"
- max="0.1"
- value={params.learningRate ?? ''}
- onChange={(e) => setParam('learningRate', Number(e.target.value))}
- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"
- />
- </div>
- )}
- {/* Frozen-base overrides (issue #1407 / #1321) — only feed the mflux
- config builder, so gate on the mflux engine being the one that'll
- run (the torch/FLUX2 path always trains the bf16 base and ignores
- these). "Auto" sends an explicit `null` (NOT absent): the server
- re-merges saved settings.loraTraining.defaults into the run, so an
- omitted key would silently keep a configured default — `null` is a
- deliberate clear that forces the memory-derived tier (still clamped
- by LORA_TRAIN_MAX_QUANT_BITS). buildMfluxTrainConfig treats a
- null/non-boolean as "no override". */}
- {params && status?.runtimes?.mflux?.ready && (
- <div>
- <label htmlFor="lt-param-baseQuant" className="block text-xs text-gray-400 mb-1">Base quant</label>
- <select
- id="lt-param-baseQuant"
- value={params.baseQuant ?? ''}
- onChange={(e) => setParam('baseQuant', e.target.value === '' ? null : Number(e.target.value))}
- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"
- >
- <option value="">Auto</option>
- <option value={16}>bf16 (heaviest)</option>
- <option value={8}>8-bit</option>
- <option value={4}>4-bit (lightest)</option>
- </select>
- </div>
- )}
- {params && status?.runtimes?.mflux?.ready && (
- <div>
- <label htmlFor="lt-param-lowRam" className="block text-xs text-gray-400 mb-1">Low RAM spill</label>
- <select
- id="lt-param-lowRam"
- value={params.lowRam === true ? 'on' : params.lowRam === false ? 'off' : ''}
- onChange={(e) => setParam('lowRam', e.target.value === '' ? null : e.target.value === 'on')}
- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"
- >
- <option value="">Auto</option>
- <option value="on">On</option>
- <option value="off">Off</option>
- </select>
- </div>
- )}
- </div>
- {status?.runtimes?.mflux?.ready && (
- <div className="flex items-start gap-2 rounded-lg border border-port-border bg-port-bg/50 px-3 py-2 text-xs text-gray-400">
- <Moon className="w-3.5 h-3.5 mt-0.5 shrink-0 text-port-accent" />
- <div className="space-y-1">
- <div className="text-gray-300">
- By default, this Mac's display is slept while training and woken when the run finishes.
- </div>
- <div>
- On Apple Silicon, an <span className="text-gray-300">active display can crash the whole machine</span>{' '}
- (GPU watchdog kernel panic) during heavy LoRA training — so PortOS puts the display to sleep for you
- and the system keeps running underneath. Big runs (9B, bf16) can take hours: start one and{' '}
- <span className="text-gray-300">let it run overnight</span>. Don't close the lid (that sleeps the
- whole Mac and pauses the run); just leave it be, or drive it over SSH. If you've set{' '}
- <code className="text-gray-300">loraTraining.displaySleep: false</code> (e.g. headless), PortOS leaves
- the display alone — keep it off the GPU yourself.
- </div>
- </div>
- </div>
- )}
- {captionLeak.fragments.length > 0 && (
- <div className="bg-port-warning/10 border border-port-warning/40 rounded-lg p-3 text-sm space-y-2">
- <div className="flex items-start gap-2">
- <AlertTriangle className="w-4 h-4 text-port-warning shrink-0 mt-0.5" />
- <p className="text-gray-200 min-w-0">
- <span className="font-medium text-port-warning">Identity is leaking into the captions.</span>{' '}
- These fragments repeat across most images, so the{' '}
- <span className="font-mono text-gray-300">{dataset.triggerWord || 'trigger'}</span>{' '}
- token would learn a generic subject. Strip them so the trigger absorbs the fixed identity, or train anyway.
- </p>
- </div>
- <div className="flex flex-wrap gap-1.5">
- {captionLeak.fragments.map((f) => (
- <span
- key={f.normalized}
- className="text-xs px-1.5 py-0.5 rounded bg-port-bg border border-port-border text-gray-300 font-mono"
- title={`in ${f.count} of ${captionLeak.total} captions`}
- >
- {f.fragment}<span className="text-gray-500"> ·{f.count}</span>
- </span>
- ))}
- </div>
- <div className="flex flex-wrap gap-2">
- <button
- type="button"
- onClick={stripAndQueue}
- disabled={strippingLeak || starting}
- className="px-3 py-1.5 text-xs rounded bg-port-warning/20 text-port-warning hover:bg-port-warning/30 flex items-center gap-2 disabled:opacity-50"
- >
- {strippingLeak ? <Loader2 className="w-3.5 h-3.5 animate-spin" /> : <Eraser className="w-3.5 h-3.5" />}
- Strip identity &amp; queue
- </button>
- <button
- type="button"
- onClick={() => start({ acknowledgeCaptionLeak: true })}
- disabled={strippingLeak || starting}
- className="px-3 py-1.5 text-xs rounded border border-port-border text-gray-300 hover:bg-port-bg flex items-center gap-2 disabled:opacity-50"
- >
- Train anyway
- </button>
- </div>
- </div>
- )}
- <div className="flex items-center justify-between gap-2">
- <span className={`text-xs ${!disabledReason && readiness?.quality === 'minimum' ? 'text-port-warning' : 'text-gray-500'}`}>
- {disabledReason
- || (readiness?.quality === 'minimum'
- ? `Trains on ${readiness?.captioned ?? 0} — ${Math.max(0, (readiness?.recommended ?? 20) - (readiness?.captioned ?? 0))} more recommended`
- : `Trains on ${readiness?.captioned ?? 0} captioned images`)}
- </span>
- <button
- type="button"
- onClick={() => start()}
- disabled={!!disabledReason || starting || strippingLeak}
- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"
- >
- {starting ? <Loader2 className="w-4 h-4 animate-spin" /> : <Dumbbell className="w-4 h-4" />}
- Train
- </button>
- </div>
- </div>
- );
-}
diff --git a/client/src/components/loraTraining/TrainingPanel.test.jsx b/client/src/components/loraTraining/TrainingPanel.test.jsx
deleted file mode 100644
index 749509c59e..0000000000
--- a/client/src/components/loraTraining/TrainingPanel.test.jsx
+++ /dev/null
@@ -1,109 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { act, fireEvent, render, screen } from '@testing-library/react';
-import { MemoryRouter } from 'react-router';
-
-vi.mock('../ui/Toast', () => ({
- default: Object.assign(vi.fn(), { success: vi.fn(), error: vi.fn() }),
-}));
-vi.mock('../../hooks/useSseProgress', () => ({
- useSseProgress: () => ({ frames: [], latest: null, closed: false }),
-}));
-vi.mock('./LiveSampleGallery', () => ({ default: () => <div>live samples</div> }));
-vi.mock('./CheckpointPicker', () => ({ default: () => null }));
-
-const getLoraTrainingStatus = vi.fn();
-const listLoraTrainingRuns = vi.fn();
-const listImageModels = vi.fn();
-vi.mock('../../services/api', () => ({
- getLoraTrainingStatus: (...a) => getLoraTrainingStatus(...a),
- listLoraTrainingRuns: (...a) => listLoraTrainingRuns(...a),
- listImageModels: (...a) => listImageModels(...a),
- startLoraTrainingRun: vi.fn(),
- cancelLoraTrainingRun: vi.fn(),
- resumeLoraTrainingRun: vi.fn(),
- stripLoraDatasetSharedCaptionFragments: vi.fn(),
-}));
-
-import TrainingPanel from './TrainingPanel';
-
-const READY_STATUS = { runtimes: { mflux: { ready: true } }, defaults: { steps: 1000, rank: 16 } };
-const MODELS = [{ id: 'flux2-klein-9b', name: 'Klein 9B', runner: 'flux2' }, { id: 'flux1-dev', runner: 'flux1' }];
-const READINESS = { trainable: true, captioned: 12, required: 10, recommended: 20, quality: 'good' };
-const ACTIVE_RUN = { id: 'run-1', status: 'running', runtime: 'mflux', baseModelId: 'flux2-klein-9b', progress: { step: 1, totalSteps: 10 } };
-
-const renderPanel = async () => {
- render(
- <MemoryRouter>
- <TrainingPanel dataset={{ id: 'ds-1' }} readiness={READINESS} triggerSaving={false} />
- </MemoryRouter>,
- );
- await act(async () => {});
-};
-
-const trainButton = () => screen.getByRole('button', { name: /Train$/ });
-
-beforeEach(() => {
- vi.clearAllMocks();
- getLoraTrainingStatus.mockResolvedValue(READY_STATUS);
- listLoraTrainingRuns.mockResolvedValue([]);
- listImageModels.mockResolvedValue(MODELS);
-});
-
-describe('TrainingPanel failed reads', () => {
- it('offers Start when every read succeeds', async () => {
- await renderPanel();
- expect(trainButton()).toBeEnabled();
- expect(screen.getByLabelText('Base model')).toHaveValue('flux2-klein-9b');
- });
-
- it('shows a Retry — not the mflux install message — when the status read fails, and recovers on retry', async () => {
- getLoraTrainingStatus.mockRejectedValueOnce(new Error('Gateway timeout'));
- await renderPanel();
- expect(screen.getByRole('alert')).toHaveTextContent('Could not load training status — Gateway timeout');
- expect(screen.getByText('Could not read training status')).toBeInTheDocument();
- expect(screen.queryByText(/No training engine/)).toBeNull();
- expect(trainButton()).toBeDisabled();
-
- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });
- expect(screen.queryByRole('alert')).toBeNull();
- expect(screen.getByLabelText('Steps')).toHaveValue(1000);
- expect(trainButton()).toBeEnabled();
- });
-
- it('still reports a missing engine when the status read succeeds with no ready runtime', async () => {
- getLoraTrainingStatus.mockResolvedValue({ runtimes: {}, defaults: {} });
- await renderPanel();
- expect(screen.getByText(/No training engine/)).toBeInTheDocument();
- expect(trainButton()).toBeDisabled();
- });
-
- it('disables Start with a Retry when the runs read fails, then shows the in-progress view for an active run', async () => {
- listLoraTrainingRuns.mockRejectedValueOnce(new Error('Server unavailable'));
- await renderPanel();
- expect(screen.getByRole('alert')).toHaveTextContent('Could not load training runs — Server unavailable');
- expect(trainButton()).toBeDisabled();
-
- listLoraTrainingRuns.mockResolvedValueOnce([ACTIVE_RUN]);
- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });
- expect(screen.getByText(/Training mflux/)).toBeInTheDocument();
- expect(screen.queryByRole('button', { name: /Train$/ })).toBeNull();
- });
-
- it('shows an error row — not "No trainable models registered" — when the models read fails', async () => {
- listImageModels.mockRejectedValueOnce(new Error('Server unavailable'));
- await renderPanel();
- expect(screen.getByRole('alert')).toHaveTextContent('Could not load base models — Server unavailable');
- expect(screen.queryByText('No trainable models registered')).toBeNull();
- expect(trainButton()).toBeDisabled();
-
- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });
- expect(screen.getByLabelText('Base model')).toHaveValue('flux2-klein-9b');
- expect(trainButton()).toBeEnabled();
- });
-
- it('says "No trainable models registered" only for a successful empty list', async () => {
- listImageModels.mockResolvedValue([{ id: 'flux1-dev', runner: 'flux1' }]);
- await renderPanel();
- expect(screen.getByText('No trainable models registered')).toBeInTheDocument();
- });
-});
diff --git a/client/src/components/loraTraining/UniverseCharacterPicker.jsx b/client/src/components/loraTraining/UniverseCharacterPicker.jsx
deleted file mode 100644
index a4b2bc4b03..0000000000
--- a/client/src/components/loraTraining/UniverseCharacterPicker.jsx
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Universe → bible-subject cascading select. Shared by the "New dataset" dialog
- * and the dataset-reassignment dialog so both pick a (universeId, entryKind, entryId)
- * pair through identical loading/empty states. The parent owns the selected
- * ids; this just renders the two <select>s and loads the option lists.
- *
- * When the universe changes the parent is responsible for clearing entryId
- * (a stale subject id from the previous universe must not survive) — the
- * `onUniverseChange` handler should reset it.
- */
-
-import { useCallback } from 'react';
-import useListResource from '../../hooks/useListResource';
-import ListLoadError from '../ui/ListLoadError';
-import { listUniverses, getUniverse } from '../../services/api';
-
-const ENTRY_KINDS = [
- { value: 'characters', label: 'Character', empty: 'No characters in this universe', pick: 'Pick a character' },
- { value: 'objects', label: 'Object', empty: 'No objects in this universe', pick: 'Pick an object' },
- { value: 'places', label: 'Place', empty: 'No places in this universe', pick: 'Pick a place' },
-];
-const fetchUniverses = () => listUniverses({ silent: true });
-const kindConfig = (entryKind) => ENTRY_KINDS.find((k) => k.value === entryKind) || ENTRY_KINDS[0];
-
-export default function UniverseCharacterPicker({
- universeId,
- entryKind = 'characters',
- entryId,
- onUniverseChange,
- onEntryKindChange = () => {},
- onEntryChange,
- idPrefix = 'ucp',
- disabledEntryIds = [],
-}) {
- const { items: universes, state: universesState, error: universesError, retry: retryUniverses } = useListResource(fetchUniverses);
- // One universe is a single object; a one-item list gives its read the same
- // loading | ready | error states, so a failed read never renders as an empty
- // subject list.
- const fetchUniverse = useCallback(() => getUniverse(universeId, { silent: true }).then((u) => [u || {}]), [universeId]);
- const { items: universeItems, state: universeState, error: universeError, retry: retryUniverse } = useListResource(fetchUniverse, { enabled: !!universeId });
- const entries = universeState === 'ready' && Array.isArray(universeItems[0]?.[entryKind]) ? universeItems[0][entryKind] : [];
-
- const disabled = new Set(disabledEntryIds);
- const kind = kindConfig(entryKind);
-
- return (
- <>
- <div>
- <label htmlFor={`${idPrefix}-universe`} className="block text-sm text-gray-400 mb-1">Universe</label>
- <select
- id={`${idPrefix}-universe`}
- value={universeId}
- onChange={(e) => onUniverseChange(e.target.value)}
- className="w-full bg-port-bg border border-port-border rounded px-3 py-2 text-white"
- >
- <option value="">{universesState === 'loading' ? 'Loading…' : 'Pick a universe'}</option>
- {universes.map((u) => <option key={u.id} value={u.id}>{u.name}</option>)}
- </select>
- {universesState === 'error' && <ListLoadError className="mt-1" what="universes" error={universesError} onRetry={retryUniverses} />}
- </div>
- {universeId && (
- <>
- <div>
- <label htmlFor={`${idPrefix}-entry-kind`} className="block text-sm text-gray-400 mb-1">Subject type</label>
- <select
- id={`${idPrefix}-entry-kind`}
- value={entryKind}
- onChange={(e) => onEntryKindChange(e.target.value)}
- className="w-full bg-port-bg border border-port-border rounded px-3 py-2 text-white"
- >
- {ENTRY_KINDS.map((k) => <option key={k.value} value={k.value}>{k.label}</option>)}
- </select>
- </div>
- <div>
- <label htmlFor={`${idPrefix}-entry`} className="block text-sm text-gray-400 mb-1">{kind.label}</label>
- <select
- id={`${idPrefix}-entry`}
- value={entryId}
- onChange={(e) => onEntryChange(e.target.value)}
- className="w-full bg-port-bg border border-port-border rounded px-3 py-2 text-white"
- >
- <option value="">
- {universeState === 'loading' ? 'Loading…' : universeState === 'error' ? 'Unavailable' : entries.length ? kind.pick : kind.empty}
- </option>
- {entries.map((entry) => (
- <option key={entry.id} value={entry.id} disabled={disabled.has(entry.id)}>
- {entry.name || entry.slugline || entry.id}{disabled.has(entry.id) ? ' (current)' : ''}
- </option>
- ))}
- </select>
- {universeState === 'error' && <ListLoadError className="mt-1" what="the universe" error={universeError} onRetry={retryUniverse} />}
- </div>
- </>
- )}
- </>
- );
-}
diff --git a/client/src/components/loraTraining/UniverseCharacterPicker.test.jsx b/client/src/components/loraTraining/UniverseCharacterPicker.test.jsx
deleted file mode 100644
index cfdfab8a3d..0000000000
--- a/client/src/components/loraTraining/UniverseCharacterPicker.test.jsx
+++ /dev/null
@@ -1,51 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { act, fireEvent, render, screen } from '@testing-library/react';
-
-const listUniverses = vi.fn();
-const getUniverse = vi.fn();
-vi.mock('../../services/api', () => ({
- listUniverses: (...a) => listUniverses(...a),
- getUniverse: (...a) => getUniverse(...a),
-}));
-
-import UniverseCharacterPicker from './UniverseCharacterPicker';
-
-const renderPicker = async () => {
- render(<UniverseCharacterPicker universeId="u1" entryKind="characters" entryId="" onUniverseChange={vi.fn()} onEntryChange={vi.fn()} />);
- await act(async () => {});
-};
-
-beforeEach(() => {
- vi.clearAllMocks();
- listUniverses.mockResolvedValue([{ id: 'u1', name: 'Example Universe' }]);
-});
-
-describe('UniverseCharacterPicker', () => {
- it('shows a Retry — not "No characters in this universe" — when the universe read fails, and lists them on retry', async () => {
- getUniverse.mockRejectedValueOnce(new Error('Server unavailable'));
- await renderPicker();
- expect(screen.getByRole('alert')).toHaveTextContent('Could not load the universe — Server unavailable');
- expect(screen.queryByText('No characters in this universe')).toBeNull();
-
- getUniverse.mockResolvedValueOnce({ characters: [{ id: 'c1', name: 'Alice' }] });
- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });
- expect(screen.queryByRole('alert')).toBeNull();
- expect(screen.getByRole('option', { name: 'Alice' })).toBeInTheDocument();
- });
-
- it('shows a Retry — not an empty universe picker — when the universe list read fails', async () => {
- listUniverses.mockRejectedValueOnce(new Error('Server unavailable'));
- getUniverse.mockResolvedValue({ characters: [] });
- await renderPicker();
- expect(screen.getByRole('alert')).toHaveTextContent('Could not load universes — Server unavailable');
-
- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });
- expect(screen.getByRole('option', { name: 'Example Universe' })).toBeInTheDocument();
- });
-
- it('says "No characters in this universe" only after a successful empty read', async () => {
- getUniverse.mockResolvedValue({ characters: [] });
- await renderPicker();
- expect(screen.getByText('No characters in this universe')).toBeInTheDocument();
- });
-});
diff --git a/client/src/components/media/MediaJobsQueue.jsx b/client/src/components/media/MediaJobsQueue.jsx
index 52f510bd9d..89323ec365 100644
--- a/client/src/components/media/MediaJobsQueue.jsx
+++ b/client/src/components/media/MediaJobsQueue.jsx
@@ -458,7 +458,7 @@ function TrainingJobDetail({ runId }) {
function JobRow({ job, onCancel, onRetry, onRunNow, onDelete }) {
const Icon = KIND_ICON[job.kind] || Film;
const canCancel = job.status === 'queued' || job.status === 'running';
- const canRetry = (job.status === 'failed' || job.status === 'canceled') && typeof onRetry === 'function';
+ const canRetry = job.kind !== 'training' && (job.status === 'failed' || job.status === 'canceled') && typeof onRetry === 'function';
// Delete only on terminal rows — live jobs go through Cancel.
const canDelete = (job.status === 'failed' || job.status === 'canceled' || job.status === 'completed')
&& typeof onDelete === 'function';
diff --git a/client/src/components/settings/DataMaintenanceTab.jsx b/client/src/components/settings/DataMaintenanceTab.jsx
new file mode 100644
index 0000000000..1da742280a
--- /dev/null
+++ b/client/src/components/settings/DataMaintenanceTab.jsx
@@ -0,0 +1,88 @@
+import { useEffect, useRef, useState } from 'react';
+import { Link } from 'react-router';
+import { RefreshCw, Trash2 } from 'lucide-react';
+import Modal from '../ui/Modal';
+import { request } from '../../services/apiCore';
+import { formatBytes, formatCount } from '../../utils/formatters';
+
+const api = (path = '', options = {}) => request(`/data-maintenance${path}`, { ...options, silent: true });
+
+export default function DataMaintenanceTab() {
+ const [items, setItems] = useState(null);
+ const [error, setError] = useState('');
+ const [notice, setNotice] = useState('');
+ const [preview, setPreview] = useState(null);
+ const [confirmation, setConfirmation] = useState('');
+ const [busy, setBusy] = useState(false);
+ const inFlight = useRef(false);
+ const generation = useRef(0);
+ useEffect(() => {
+ let active = true;
+ inFlight.current = true;
+ setBusy(true);
+ api().then(result => { if (active) setItems(result); }).catch(err => { if (active) setError(err.message); }).finally(() => { if (active) { inFlight.current = false; setBusy(false); } });
+ return () => { active = false; generation.current += 1; };
+ }, []);
+
+ async function act(work) {
+ if (inFlight.current) return;
+ inFlight.current = true;
+ setBusy(true);
+ setError('');
+ const current = generation.current;
+ try { await work(() => current === generation.current); }
+ catch (err) { if (current === generation.current) { setError(err.message); setPreview(null); setConfirmation(''); } }
+ finally { inFlight.current = false; if (current === generation.current) setBusy(false); }
+ }
+
+ function refresh() {
+ setPreview(null);
+ setConfirmation('');
+ setNotice('');
+ void act(async valid => { const result = await api(); if (valid()) setItems(result); });
+ }
+
+ function begin(operationId) {
+ void act(async valid => {
+ const result = await api('/preview', { method: 'POST', body: JSON.stringify({ operationId }) });
+ if (valid()) { setPreview(result); setConfirmation(''); }
+ });
+ }
+
+ function remove() {
+ if (!preview || confirmation !== preview.confirmation) return;
+ void act(async valid => {
+ const result = await api('/execute', { method: 'POST', body: JSON.stringify({ token: preview.token, confirmation }) });
+ if (!valid()) return;
+ setItems(previous => previous.map(item => item.id === result.operationId ? { ...item, status: result.status === 'deleted' ? 'absent' : result.status, bytes: result.remainingBytes, files: result.remainingFiles, actions: result.status === 'retained' ? ['delete'] : [] } : item));
+ setNotice(`Removed ${formatCount(result.removedFiles)} files (${formatBytes(result.removedBytes)}). Run history and installed adapters were retained.${result.status === 'retained' ? ' New files appeared during removal; refresh and preview them separately.' : result.status === 'unavailable' ? ' Inventory could not be read after removal; refresh to check remaining files.' : ''}`);
+ setPreview(null);
+ setConfirmation('');
+ });
+ }
+
+ return <div className="max-w-5xl space-y-4">
+ <div className="flex items-start justify-between gap-4">
+ <div><h2 className="text-lg font-semibold">Data Maintenance</h2><p className="text-sm text-port-text-muted">Inspect retained data from retired features and review supported maintenance actions.</p></div>
+ <button type="button" onClick={refresh} disabled={busy} className="flex items-center gap-2 px-3 py-2 border border-port-border rounded disabled:opacity-50"><RefreshCw className="w-4 h-4" />Refresh</button>
+ </div>
+ <p className="text-sm text-port-text-muted">LoRA training is retired. Existing files and configuration remain intact until you explicitly remove files here. There is no automatic conversion of training data to reference images. Use <Link to="/media/image" className="text-port-accent">reference-image generation</Link> for new work. <Link to="/system-resources/queues" className="text-port-accent">Review or cancel active media jobs</Link> before maintenance. Backup settings are unchanged; check your backups before removing unique data.</p>
+ {error && <div role="alert" className="text-port-error border border-port-error rounded p-3">{error}</div>}
+ {notice && <p role="status" className="text-port-success">{notice}</p>}
+ {!items && !error && <p role="status">Measuring retained data…</p>}
+ {items?.map(item => <section key={item.id} className="border border-port-border rounded-lg bg-port-card p-4 space-y-2">
+ <div className="flex flex-wrap items-center justify-between gap-3"><h3 className="font-medium">{item.label}</h3><span className="text-sm text-port-text-muted">{item.status}</span></div>
+ <div className="flex flex-wrap gap-x-5 text-sm"><code>{item.path}</code><span>{item.category}</span><span>{item.bytes === null ? 'Size unavailable' : formatBytes(item.bytes)} · {formatCount(item.files)} files</span></div>
+ <p className="text-sm text-port-text-muted">{item.warning}</p>
+ {item.error && <p role="alert" className="text-port-error">{item.error}</p>}
+ {item.id === 'shared-lora-adapters' && <Link to="/models/loras" className="text-port-accent text-sm">Manage installed adapters</Link>}
+ {item.actions.includes('delete') && <button type="button" disabled={busy} onClick={() => begin(item.id)} className="flex items-center gap-2 text-port-error border border-port-border rounded px-3 py-2 disabled:opacity-50"><Trash2 className="w-4 h-4" />Preview deletion</button>}
+ </section>)}
+ {preview && <Modal open panelClassName="bg-port-card border border-port-border rounded-lg shadow-xl" ariaLabel="Review permanent deletion" usePortal onClose={() => { if (!busy) { setPreview(null); setConfirmation(''); } }} closeOnEsc={!busy} closeOnBackdrop={!busy}>
+ <div className="p-5 space-y-4"><h3 className="font-semibold">Review permanent deletion</h3><p><code>{preview.path}</code> · {formatBytes(preview.bytes)} · {formatCount(preview.files)} files</p><p className="text-port-error">{preview.warning} This removal cannot be undone by PortOS.</p><p className="text-sm">Preview expires after five minutes. Changed files or active work require a new preview.</p>
+ <label htmlFor="maintenance-confirmation" className="block text-sm">Type <code>{preview.confirmation}</code> to confirm<input id="maintenance-confirmation" value={confirmation} onChange={event => setConfirmation(event.target.value)} disabled={busy} className="mt-2 w-full rounded border border-port-border bg-port-bg px-3 py-2" /></label>
+ <div className="flex justify-end gap-3"><button type="button" disabled={busy} onClick={() => { setPreview(null); setConfirmation(''); }}>Cancel</button><button type="button" onClick={remove} disabled={busy || confirmation !== preview.confirmation} className="rounded bg-port-error px-3 py-2 text-white disabled:opacity-50">{busy ? 'Removing…' : 'Permanently delete'}</button></div>
+ </div>
+ </Modal>}
+ </div>;
+}
diff --git a/client/src/components/settings/DataMaintenanceTab.test.jsx b/client/src/components/settings/DataMaintenanceTab.test.jsx
new file mode 100644
index 0000000000..f3e64db16a
--- /dev/null
+++ b/client/src/components/settings/DataMaintenanceTab.test.jsx
@@ -0,0 +1,58 @@
+import { beforeEach, expect, it, vi } from 'vitest';
+import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';
+import { MemoryRouter } from 'react-router';
+const api = vi.hoisted(() => vi.fn());
+vi.mock('../../services/apiCore', () => ({ request: api }));
+import DataMaintenanceTab from './DataMaintenanceTab';
+const rows = [{ id: 'retired-training-runs', label: 'Retired artifacts', path: 'data/training-runs', category: 'Unique checkpoints', status: 'retained', warning: 'Unique files', bytes: 17, files: 1, actions: ['delete'] }];
+const preview = { token: 'example-token', confirmation: 'DELETE training-runs', path: 'data/training-runs', bytes: 17, files: 1, warning: 'Unique files' };
+beforeEach(() => { api.mockReset(); api.mockResolvedValue(rows); });
+const mount = () => render(<MemoryRouter><DataMaintenanceTab /></MemoryRouter>);
+it('requires preview and exact typed confirmation, supports cancellation, prevents repeated clicks and updates inventory after success', async () => {
+ mount(); await screen.findByText('Retired artifacts');
+ api.mockResolvedValueOnce(preview);
+ fireEvent.click(screen.getByRole('button', { name: 'Preview deletion' }));
+ await screen.findByRole('dialog');
+ expect(screen.getByRole('button', { name: 'Permanently delete' })).toBeDisabled();
+ fireEvent.click(screen.getByRole('button', { name: 'Cancel' }));
+ expect(screen.queryByRole('dialog')).toBeNull();
+ expect(api).toHaveBeenCalledTimes(2);
+ api.mockResolvedValueOnce(preview);
+ fireEvent.click(screen.getByRole('button', { name: 'Preview deletion' }));
+ await screen.findByRole('dialog');
+ fireEvent.change(screen.getByRole('textbox'), { target: { value: 'DELETE training-runs' } });
+ let finish;
+ api.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; }));
+ fireEvent.click(screen.getByRole('button', { name: 'Permanently delete' }));
+ fireEvent.click(screen.getByRole('button', { name: 'Removing…' }));
+ expect(api).toHaveBeenCalledTimes(4);
+ await act(async () => { finish({ operationId: rows[0].id, removedBytes: 17, removedFiles: 1, status: 'deleted', remainingBytes: 0, remainingFiles: 0 }); });
+ expect(screen.queryByRole('dialog')).toBeNull();
+ expect(screen.getByText('absent')).toBeInTheDocument();
+ expect(screen.queryByRole('button', { name: 'Preview deletion' })).toBeNull();
+});
+it('surfaces stale-preview refusal, preserves the row, and permits explicit refresh', async () => {
+ mount(); await screen.findByText('Retired artifacts');
+ api.mockResolvedValueOnce(preview);
+ fireEvent.click(screen.getByRole('button', { name: 'Preview deletion' }));
+ await screen.findByRole('dialog');
+ fireEvent.change(screen.getByRole('textbox'), { target: { value: preview.confirmation } });
+ api.mockRejectedValueOnce(new Error('Files changed since preview'));
+ fireEvent.click(screen.getByRole('button', { name: 'Permanently delete' }));
+ expect(await screen.findByRole('alert')).toHaveTextContent('Files changed since preview');
+ expect(screen.getByText('retained')).toBeInTheDocument();
+ api.mockResolvedValueOnce(rows);
+ fireEvent.click(screen.getByRole('button', { name: 'Refresh' }));
+ await waitFor(() => expect(screen.queryByRole('alert')).toBeNull());
+});
+it('shows an unreadable inventory as unavailable instead of measured zero and blocks refresh during initial load', async () => {
+ let complete;
+ api.mockImplementationOnce(() => new Promise(resolve => { complete = resolve; }));
+ mount();
+ expect(screen.getByRole('button', { name: 'Refresh' })).toBeDisabled();
+ complete([{ ...rows[0], status: 'unavailable', bytes: null, files: null, error: 'Directory cannot be read', actions: [] }]);
+ expect(await screen.findByText(/Size unavailable/)).toBeInTheDocument();
+ expect(screen.getByRole('alert')).toHaveTextContent('Directory cannot be read');
+ expect(screen.queryByText('0 B')).toBeNull();
+ expect(screen.queryByRole('button', { name: 'Preview deletion' })).toBeNull();
+});
diff --git a/client/src/components/settings/ImageGenBackendSections.jsx b/client/src/components/settings/ImageGenBackendSections.jsx
index b39475246c..6d019f2aa8 100644
--- a/client/src/components/settings/ImageGenBackendSections.jsx
+++ b/client/src/components/settings/ImageGenBackendSections.jsx
@@ -205,7 +205,7 @@ export default function ImageGenBackendSections({ mediaTab, form, credentials })
<VideoKeyRow key={spec.id} spec={spec} row={credentials.rows[spec.id]} busy={credentials.busy} onSave={credentials.save} onClear={credentials.clear} />
))}
<div className="space-y-3">
- {RENDER_TARGET_OPTIONS.map(({ id, label, video }) => {
+ {RENDER_TARGET_OPTIONS.filter(option => !option.retired).map(({ id, label, video }) => {
const entry = renderDefaults[id] || {};
const pinnedMode = entry.imageMode && entry.imageMode !== RENDER_TARGET_BACKEND_AUTO ? entry.imageMode : '';
const pinnedVideoMode = normalizeRenderPinValue(entry.videoMode) || '';
diff --git a/client/src/components/settings/ImageGenTab.test.jsx b/client/src/components/settings/ImageGenTab.test.jsx
index 707f9f4282..265475e6a8 100644
--- a/client/src/components/settings/ImageGenTab.test.jsx
+++ b/client/src/components/settings/ImageGenTab.test.jsx
@@ -660,9 +660,9 @@ describe('ImageGenTab — Agy CLI section', () => {
it('grok pins never show a model input — its backend has no model knob (#3231)', async () => {
getSettings.mockResolvedValue({ imageGen: { mode: 'local' } });
await renderTab(['/media/image?mediaTab=defaults']);
- const select = await screen.findByLabelText('LoRA training datasets');
+ const select = await screen.findByLabelText('Sprite references & anchors');
fireEvent.change(select, { target: { value: 'grok' } });
- expect(screen.queryByLabelText('LoRA training datasets model')).toBeNull();
+ expect(screen.queryByLabelText('Sprite references & anchors model')).toBeNull();
});
it('names the fixed server-side image model as a read-only fact (#3231)', async () => {
@@ -794,3 +794,14 @@ describe('ImageGenTab local runtime card', () => {
});
vi.mock('../../services/apiRunwayCatalog.js', async (importOriginal) => ({ ...await importOriginal(), loadRunwayCatalog: vi.fn().mockResolvedValue(undefined) }));
+
+it('hides the retired training default and leaves its saved pin untouched while editing a supported surface', async () => {
+ getSettings.mockResolvedValue({ imageGen: { mode: 'local' }, renderDefaults: { 'lora-dataset': { imageMode: 'grok' } } });
+ await renderTab(['/media/image?mediaTab=defaults']);
+ const live = await screen.findByLabelText('Universe Bible & canon renders');
+ expect(screen.queryByLabelText('LoRA training datasets')).toBeNull();
+ fireEvent.change(live, { target: { value: 'grok' } });
+ fireEvent.click(screen.getByRole('button', { name: /^Save$/ }));
+ await awaitSave();
+ expect(sentSlices()).toEqual({ renderDefaults: { 'universe-bible': { imageMode: 'grok' } } });
+});
diff --git a/client/src/lib/README.md b/client/src/lib/README.md
index f434f01217..a000d9a4aa 100644
--- a/client/src/lib/README.md
+++ b/client/src/lib/README.md
@@ -97,7 +97,6 @@ grep -i "what you want to do" client/src/lib/README.md
| `imageCleaners.js` | Re-exports `resolveCleanersFromConfig` from the shared pure `server/lib/imageCleanDefaults.js` policy. Reads `{cleanC2PA, denoise}` off a per-mode settings record. `cleanC2PA` defaults are mode-aware (on for `codex` + `external` — the backends that emit C2PA chunks today — off otherwise, as an allow-list rather than a deny-list); `denoise` defaults off everywhere (lossy, opt-in only). |
| `reviewerModels.js` | `reviewerModelsFromDefaults` / `reviewerModelsToDefaults` / `reviewerEffortsFromDefaults` / `reviewerEffortsToDefaults` — adapt the Code Review Defaults' per-reviewer SCALARS (`codexModel`, `ollamaModel`, `claudeEffort`, …) to and from the token-keyed maps `ReviewerPicker`'s Model and Effort columns take. Mirrors `reviewerModelsFromDefaults` / `reviewerEffortsFromDefaults` in `server/lib/reviewerConfig.js`. Also preserves configured-provider model and effort pins in the additive `providerModels` / `providerEfforts` maps, keyed by `provider:<id>`; empty maps clear overrides without changing builtin scalars. |
| `reviewerPins.js` | Client mirror of the whole reviewer vocabulary — the roster (`REVIEWER_VALUES`), `REVIEWER_ALIASES`, `DEFAULT_REVIEWER(S)`, `normalizeReviewers`, the review-username pattern/cap (`MAX_REVIEW_USERNAMES` / `cleanReviewUsername` / `normalizeReviewUsernames`), `MAX_REVIEWER_MAX_ROUNDS`, `REVIEW_STOP_MODES` / `DEFAULT_REVIEW_STOP_MODE`, plus the per-reviewer pin vocabularies (`MODEL_SELECTABLE_REVIEWERS` / `EFFORT_SELECTABLE_REVIEWERS` / `REVIEWER_EFFORT_LEVELS` / `reviewerEffortLevels` / `normalizeReviewerSlug` / `sanitizeReviewerModelInput`) and their bounds. Dependency-free on purpose: the server suite imports it to pin the mirror against `server/lib/reviewerConfig.js`, so it must not reach for a client-only package. Re-exported by `components/cos/constants.js`. `isProviderReviewer` recognizes the shared `provider:<id>` identity alongside legacy harness slugs. |
-| `loraDatasetReadiness.js` | Re-export of `server/lib/loraDataset.js` (#8297) — `MIN_TRAINING_IMAGES` / `RECOMMENDED_TRAINING_IMAGES` / `TRAINING_IMAGE_SWEET_SPOT_MAX`, `captionHasTriggerWord`, `datasetQualityTier`, `computeDatasetReadiness`, `analyzeCaptionInvariants` (+ its `INVARIANT_SHARE_THRESHOLD` / `MIN_CAPTIONS_FOR_INVARIANT_ANALYSIS`). `LoraDatasetDetail.jsx`'s live advisory (readiness summary, Train gate, shared-caption-fragment warning) calls these directly instead of a page-local mirror, so it cannot drift from the server's authoritative training gate. |
| `loraEffect.js` | How the LoRA manager card renders an adapter-effect measurement (#4872). `LORA_EFFECT_STATUSES` is re-exported from `server/lib/loraEffect.js`; `LORA_EFFECT_BADGES` + `loraEffectBadge(status)` (only `zero` is styled as an error, because only `zero` refuses a render server-side) and `loraEffectDetail(report)` are client-only — the detail line returns `null` when the badge already says everything, where the server's `formatLoraEffect` always returns a sentence for its log. |
| `loraTriggers.js` | The picker's `+ trigger` append (#4665). `firstTriggerWord` / `promptHasTriggerWord` / `separatorFor` are re-exported from `server/lib/loraTriggers.js`, so the hint reads the rule the render enforces. `appendTriggerWords(prompt, words, effectivePrompt?)` is client-only: it adds ALL of a LoRA's words (the server only ever weaves the first) and judges presence against the composed prompt the page will actually submit. |
| `runnerFamilies.js` | Mirror of `server/lib/runners.js`, including the shared LTX + MiniMax H3 Ref2VA audio-to-video runtime predicate; `loraCompatKeysMatch` keeps image Test handoffs and the adapter picker on the same size/family compatibility rule. |
diff --git a/client/src/lib/imageGenModes.js b/client/src/lib/imageGenModes.js
index 23bdb2d681..ca94496840 100644
--- a/client/src/lib/imageGenModes.js
+++ b/client/src/lib/imageGenModes.js
@@ -159,7 +159,7 @@ export const RENDER_TARGET_OPTIONS = Object.freeze([
{ id: RENDER_TARGET.SPRITE_REFERENCE, label: 'Sprite references & anchors' },
{ id: RENDER_TARGET.PIPELINE_VISUAL, label: 'Pipeline visuals (storyboards, comics, covers)' },
{ id: RENDER_TARGET.MUSIC_VIDEO, label: 'Music Video scene frames & clips', video: true },
- { id: RENDER_TARGET.LORA_DATASET, label: 'LoRA training datasets' },
+ { id: RENDER_TARGET.LORA_DATASET, label: 'LoRA training datasets', retired: true },
{ id: RENDER_TARGET.CREATIVE_AGENT, label: 'Creative agent renders', video: true },
{ id: RENDER_TARGET.DECK, label: 'Deck card renders (playing cards & tarot)' },
]);
diff --git a/client/src/lib/index.js b/client/src/lib/index.js
index 35f4892235..2c2d2fa42c 100644
--- a/client/src/lib/index.js
+++ b/client/src/lib/index.js
@@ -9,7 +9,6 @@ export * from './canonPrompt.js';
export * from './cleanPlatePrompt.js';
export * from './composeStyledPrompt.js';
export * from './decks.js';
-export * from './loraDatasetReadiness.js';
export * from './personaTraitBlend.js';
export * from './scenePrompt.js';
export * from './seasonStructure.js';
diff --git a/client/src/lib/loraDatasetReadiness.js b/client/src/lib/loraDatasetReadiness.js
deleted file mode 100644
index 93c73224fc..0000000000
--- a/client/src/lib/loraDatasetReadiness.js
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * LoRA dataset training-readiness thresholds, trigger-word matching, and
- * caption-invariant analysis.
- *
- * Re-export of `server/lib/loraDataset.js` — the one definition of these
- * rules, imported rather than copied so the page-local advisory (which
- * updates the instant a caption is edited or an image deleted, without a
- * server round-trip — see `client/src/pages/LoraDatasetDetail.jsx`) cannot
- * drift from `computeDatasetReadiness` / `validateDatasetReady`, the
- * authoritative gate the server applies at train time (#8297). The server
- * module is dependency-free (`isPlainObject`, `escapeRegExp` — no Node
- * builtins), so it is safe to bundle into the browser build; see
- * `scripts/client-server-import-purity.test.js`.
- */
-export {
- MIN_TRAINING_IMAGES,
- RECOMMENDED_TRAINING_IMAGES,
- TRAINING_IMAGE_SWEET_SPOT_MAX,
- INVARIANT_SHARE_THRESHOLD,
- MIN_CAPTIONS_FOR_INVARIANT_ANALYSIS,
- isValidTriggerWord,
- captionHasTriggerWord,
- datasetQualityTier,
- computeDatasetReadiness,
- analyzeCaptionInvariants,
-} from '../../../server/lib/loraDataset.js';
diff --git a/client/src/lib/loraDatasetReadiness.test.js b/client/src/lib/loraDatasetReadiness.test.js
deleted file mode 100644
index 78c7a931a4..0000000000
--- a/client/src/lib/loraDatasetReadiness.test.js
+++ /dev/null
@@ -1,107 +0,0 @@
-// @vitest-environment node
-import { describe, it, expect } from 'vitest';
-import * as clientReadiness from './loraDatasetReadiness.js';
-import * as serverReadiness from '../../../server/lib/loraDataset.js';
-
-// Cross-runtime contract (#8297): `LoraDatasetDetail.jsx`'s live training
-// advisory (readiness summary, Train gate, shared-caption-fragment warning)
-// calls these helpers directly instead of a page-local mirror, so a page
-// caption/trigger-word edit can never show "Ready to train" while the
-// server's `validateDatasetReady` gate (which calls the same
-// `computeDatasetReadiness`/`analyzeCaptionInvariants`) would reject the run.
-// Two layers of protection:
-// 1. Reference identity — the client module is a straight re-export, so a
-// future edit that quietly swaps it for a duplicated implementation
-// (reintroducing the drift issue #8297 fixed) fails here immediately.
-// 2. Behavior on the shared fixture matrix (valid / boundary / non-
-// trainable), so the contract is pinned even if the re-export shape
-// changes later.
-const EXPORT_NAMES = [
- 'MIN_TRAINING_IMAGES',
- 'RECOMMENDED_TRAINING_IMAGES',
- 'TRAINING_IMAGE_SWEET_SPOT_MAX',
- 'INVARIANT_SHARE_THRESHOLD',
- 'MIN_CAPTIONS_FOR_INVARIANT_ANALYSIS',
- 'isValidTriggerWord',
- 'captionHasTriggerWord',
- 'datasetQualityTier',
- 'computeDatasetReadiness',
- 'analyzeCaptionInvariants',
-];
-
-const readyImage = (caption) => ({ status: 'ready', caption });
-const renderingImage = () => ({ status: 'rendering', caption: '' });
-
-// 10 ready, captioned images naming the trigger — exactly MIN_TRAINING_IMAGES.
-const boundaryImages = Array.from(
- { length: 10 },
- (_, i) => readyImage(`subj_x, pose ${i}, studio lighting`),
-);
-
-// Same images as boundaryImages, plus a few more so 4 of the captions repeat
-// an identical descriptive fragment ("white hair") — enough signal for
-// analyzeCaptionInvariants to flag it as a shared identity fragment.
-const invariantImages = [
- readyImage('subj_x, white hair, pose a'),
- readyImage('subj_x, white hair, pose b'),
- readyImage('subj_x, white hair, pose c'),
- readyImage('subj_x, white hair, pose d'),
- readyImage('subj_x, pose e'),
-];
-
-describe('loraDatasetReadiness (client re-export of server/lib/loraDataset.js)', () => {
- it('re-exports the exact same functions/constants the server owns — no page-local copy', () => {
- for (const name of EXPORT_NAMES) {
- expect(clientReadiness[name]).toBe(serverReadiness[name]);
- }
- });
-
- it('agrees with the server on a trainable dataset at the sweet spot (valid)', () => {
- const dataset = {
- triggerWord: 'subj_x',
- images: Array.from({ length: 22 }, (_, i) => readyImage(`subj_x, pose ${i}`)),
- };
- const clientResult = clientReadiness.computeDatasetReadiness(dataset);
- const serverResult = serverReadiness.computeDatasetReadiness(dataset);
- expect(clientResult).toEqual(serverResult);
- expect(clientResult).toMatchObject({ trainable: true, quality: 'good', captioned: 22 });
- });
-
- it('agrees with the server exactly at the MIN_TRAINING_IMAGES boundary', () => {
- const dataset = { triggerWord: 'subj_x', images: boundaryImages };
- const clientResult = clientReadiness.computeDatasetReadiness(dataset);
- const serverResult = serverReadiness.computeDatasetReadiness(dataset);
- expect(clientResult).toEqual(serverResult);
- expect(clientResult).toMatchObject({
- trainable: true,
- quality: 'minimum',
- captioned: clientReadiness.MIN_TRAINING_IMAGES,
- });
- });
-
- it('agrees with the server on a non-trainable dataset (missing trigger word)', () => {
- const dataset = { triggerWord: '', images: [...boundaryImages, renderingImage()] };
- const clientResult = clientReadiness.computeDatasetReadiness(dataset);
- const serverResult = serverReadiness.computeDatasetReadiness(dataset);
- expect(clientResult).toEqual(serverResult);
- // No trigger word configured: never trainable, quality never reports
- // 'good'/'minimum' even though 10 images are otherwise ready+captioned.
- expect(clientResult).toMatchObject({ trainable: false, quality: 'insufficient', rendering: 1 });
- });
-
- it('agrees with the server on caption-invariant fragment detection', () => {
- const clientResult = clientReadiness.analyzeCaptionInvariants(invariantImages, 'subj_x');
- const serverResult = serverReadiness.analyzeCaptionInvariants(invariantImages, 'subj_x');
- expect(clientResult).toEqual(serverResult);
- expect(clientResult.analyzable).toBe(true);
- expect(clientResult.sharedFragments.map((f) => f.normalized)).toContain('white hair');
- });
-
- it('agrees with the server on trigger-word token-boundary matching', () => {
- // "train" contains "ai" as a substring but not as a bounded token.
- expect(clientReadiness.captionHasTriggerWord('a subject in training gear', 'ai'))
- .toBe(serverReadiness.captionHasTriggerWord('a subject in training gear', 'ai'));
- expect(clientReadiness.captionHasTriggerWord('subj_x, wearing ai badge', 'ai'))
- .toBe(serverReadiness.captionHasTriggerWord('subj_x, wearing ai badge', 'ai'));
- });
-});
diff --git a/client/src/lib/navPresentation.js b/client/src/lib/navPresentation.js
index a8003f5da8..68b4a8b889 100644
--- a/client/src/lib/navPresentation.js
+++ b/client/src/lib/navPresentation.js
@@ -1,5 +1,6 @@
import { AudioLines,
Wrench,
+ Trash2,
Home,
Package,
FileText,
@@ -253,7 +254,6 @@ export const NAV_PRESENTATION = {
'/local-llm/playground': { icon: FlaskConical },
'/models/status': { icon: Activity },
'/models/subscriptions': { icon: CreditCard },
- '/models/training': { icon: GraduationCap },
'/settings/ai-assignments': { icon: Bot },
'/settings/api-access': { icon: Globe },
'/api-reference/catalog': { icon: Braces },
@@ -263,6 +263,7 @@ export const NAV_PRESENTATION = {
'/models/code-reviewers': { icon: ShieldCheck },
'/settings/database': { icon: Database },
'/settings/features': { icon: ListChecks },
+ '/settings/data-maintenance': { icon: Trash2 },
'/settings/general': { icon: Settings },
'/settings/mortalloom': { icon: Activity },
'/openclaw': { icon: MessagesSquare },
diff --git a/client/src/pages/LoraDatasetDetail.jsx b/client/src/pages/LoraDatasetDetail.jsx
deleted file mode 100644
index ba2d18ec43..0000000000
--- a/client/src/pages/LoraDatasetDetail.jsx
+++ /dev/null
@@ -1,703 +0,0 @@
-import { listSheetPointers } from '../lib/sheetPointers.js';
-/**
- * LoRA dataset workbench (/models/training/:recordId).
- *
- * Rendered by `pages/Models.jsx` inside the Models section shell (its TAB_DETAIL
- * map), not as a bare route — so it keeps the section header and tab bar the way
- * it used to under the Media Gen shell.
- *
- * Build reference material for one universe bible subject (generate via the image
- * queue, upload, slice the reference-sheet turnaround), caption it with
- * the vision model, then launch a training run. The dataset record is the
- * single source of truth; rendering images poll-refresh until they land.
- */
-
-import { useState, useEffect, useMemo, useCallback, useRef } from 'react';
-import { useParams, Link } from 'react-router';
-import {
- ArrowLeft, Loader2, Upload, Wand2, Scissors, Tags, RefreshCw, AlertTriangle, Images, Replace, Lightbulb, Eraser, XCircle,
-} from 'lucide-react';
-import toast from '../components/ui/Toast';
-import FilePickerButton from '../components/ui/FilePickerButton';
-import { IMAGE_ACCEPT } from '../utils/fileUpload';
-import Modal from '../components/ui/Modal';
-import { useSocketResource } from '../hooks/useSocketResource.js';
-import useSseJobSlot from '../hooks/useSseJobSlot';
-import DatasetImageGrid from '../components/loraTraining/DatasetImageGrid';
-import GenerateBatchDialog from '../components/loraTraining/GenerateBatchDialog';
-import TrainingPanel from '../components/loraTraining/TrainingPanel';
-import CaptionModelPicker from '../components/loraTraining/CaptionModelPicker';
-import ImportGalleryDialog from '../components/loraTraining/ImportGalleryDialog';
-import UniverseCharacterPicker from '../components/loraTraining/UniverseCharacterPicker';
-import {
- MIN_TRAINING_IMAGES,
- RECOMMENDED_TRAINING_IMAGES,
- TRAINING_IMAGE_SWEET_SPOT_MAX,
- isValidTriggerWord,
- computeDatasetReadiness,
- analyzeCaptionInvariants,
-} from '../lib/loraDatasetReadiness.js';
-import {
- getLoraDataset,
- getLoraDatasetVariationAxes,
- patchLoraDataset,
- uploadLoraDatasetImages,
- sliceLoraDatasetRefSheet,
- startLoraCaptionRun,
- cancelLoraCaptionRun,
- stripLoraDatasetSharedCaptionFragments,
- getUniverse,
-} from '../services/api';
-
-const DATASET_EVENTS = ['training:dataset:changed'];
-
-const SUBJECT_TYPE_LABEL = { characters: 'Character', objects: 'Object', places: 'Place' };
-const subjectKind = (dataset) => dataset?.character?.entryKind || 'characters';
-
-function SliceDialog({ dataset, onClose, onSliced }) {
- const [cols, setCols] = useState(3);
- const [rows, setRows] = useState(2);
- // Vision-auto detection is the default; the grid inputs are the fallback the
- // user reaches for when no vision model is installed or auto-detect mis-cuts.
- const [useVision, setUseVision] = useState(true);
- const [submitting, setSubmitting] = useState(false);
-
- const submit = async () => {
- setSubmitting(true);
- try {
- const result = await sliceLoraDatasetRefSheet(dataset.id, { cols, rows, useVision });
- const via = result.method === 'vision' ? 'auto-detected' : 'grid';
- toast.success(`Added ${result.images.length} ${via} crops from the reference sheet — prune any bad ones`);
- onSliced(result);
- } finally {
- setSubmitting(false);
- }
- };
-
- const numberInput = (id, label, value, set) => (
- <div>
- <label htmlFor={id} className="block text-xs text-gray-400 mb-1">{label}</label>
- <input
- id={id}
- type="number"
- min={1}
- max={6}
- value={value}
- onChange={(e) => set(Math.max(1, Math.min(6, Number(e.target.value) || 1)))}
- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"
- />
- </div>
- );
-
- return (
- <Modal
- open
- onClose={onClose}
- size="sm"
- ariaLabelledBy="lt-slice-title"
- panelClassName="bg-port-card border border-port-border rounded-lg p-5"
- >
- <div className="space-y-4">
- <h2 id="lt-slice-title" className="text-base font-semibold text-white">Slice reference sheet</h2>
- <p className="text-sm text-gray-400">
- Cuts the character&apos;s reference-sheet turnaround into individual training crops.
- Sheet layouts vary, so expect to delete cells that caught labels or palette swatches.
- </p>
- <label htmlFor="lt-slice-vision" className="flex items-start gap-2 text-sm text-gray-300">
- <input
- id="lt-slice-vision"
- type="checkbox"
- checked={useVision}
- onChange={(e) => setUseVision(e.target.checked)}
- className="mt-0.5"
- />
- <span>
- Auto-detect figures with a vision model
- <span className="block text-xs text-gray-500">
- Proposes a crop per figure instead of a rigid grid. Falls back to the grid below if no
- vision model is installed or detection finds nothing.
- </span>
- </span>
- </label>
- <div className={`grid grid-cols-2 gap-3 ${useVision ? 'opacity-50' : ''}`}>
- {numberInput('lt-slice-cols', 'Columns', cols, setCols)}
- {numberInput('lt-slice-rows', 'Rows', rows, setRows)}
- </div>
- <div className="flex justify-end gap-2">
- <button type="button" onClick={onClose} className="px-3 py-2 text-sm text-gray-400 hover:text-white">Cancel</button>
- <button
- type="button"
- onClick={submit}
- disabled={submitting}
- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"
- >
- {submitting ? <Loader2 className="w-4 h-4 animate-spin" /> : <Scissors className="w-4 h-4" />}
- {useVision ? 'Auto-slice' : `Slice ${cols}×${rows}`}
- </button>
- </div>
- </div>
- </Modal>
- );
-}
-
-function ReassignDialog({ dataset, onClose, onReassigned }) {
- // Default to the current assignment so the picker opens on the dataset's
- // own universe (subjects pre-load) and the user only changes what they want.
- const [universeId, setUniverseId] = useState(dataset.character.universeId);
- const [entryKind, setEntryKind] = useState(subjectKind(dataset));
- const [entryId, setEntryId] = useState(dataset.character.entryId);
- const [submitting, setSubmitting] = useState(false);
-
- const submit = async () => {
- setSubmitting(true);
- try {
- const next = await patchLoraDataset(dataset.id, { universeId, entryKind, entryId });
- toast.success(`Reassigned to ${next.character.name}`);
- onReassigned(next);
- } finally {
- setSubmitting(false);
- }
- };
-
- return (
- <Modal
- open
- onClose={onClose}
- size="sm"
- ariaLabelledBy="lt-reassign-title"
- panelClassName="bg-port-card border border-port-border rounded-lg p-5"
- >
- <div className="space-y-4">
- <h2 id="lt-reassign-title" className="text-base font-semibold text-white">Reassign subject</h2>
- <p className="text-sm text-gray-400">
- Move this dataset&apos;s images and trigger word to a different universe bible subject.
- The trigger word stays the same — edit it separately if the new subject needs its own token.
- </p>
- <UniverseCharacterPicker
- idPrefix="lt-reassign"
- universeId={universeId}
- entryKind={entryKind}
- entryId={entryId}
- onUniverseChange={(id) => { setUniverseId(id); setEntryId(''); }}
- onEntryKindChange={(kind) => { setEntryKind(kind); setEntryId(''); }}
- onEntryChange={setEntryId}
- />
- <div className="flex justify-end gap-2">
- <button type="button" onClick={onClose} className="px-3 py-2 text-sm text-gray-400 hover:text-white">Cancel</button>
- <button
- type="button"
- onClick={submit}
- disabled={!universeId || !entryId || submitting}
- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"
- >
- {submitting ? <Loader2 className="w-4 h-4 animate-spin" /> : <Replace className="w-4 h-4" />}
- Reassign
- </button>
- </div>
- </div>
- </Modal>
- );
-}
-
-export default function LoraDatasetDetail({ recordId }) {
- // Models renders this inside the section shell and passes the id from
- // `/models/training/:recordId`; `:datasetId` is the fallback for a direct mount.
- const params = useParams();
- const datasetId = recordId ?? params.datasetId;
- const { data: dataset, updateData: setDataset, error, refetch: refresh } = useSocketResource(
- () => getLoraDataset(datasetId),
- { events: DATASET_EVENTS, resourceKey: datasetId, matchesEvent: payload => payload?.datasetId === datasetId },
- );
- const loadError = error?.message;
- const [subject, setSubject] = useState(null);
- const [variationAxes, setVariationAxes] = useState(null);
- const [triggerDraft, setTriggerDraft] = useState(null);
- const [triggerSaving, setTriggerSaving] = useState(false);
- const [uploading, setUploading] = useState(false);
- const [showGenerate, setShowGenerate] = useState(false);
- const [showSlice, setShowSlice] = useState(false);
- const [showImport, setShowImport] = useState(false);
- const [showReassign, setShowReassign] = useState(false);
- // Chosen caption model { providerId, model } — null fields mean "let the
- // server auto-pick a vision model". Lifted from CaptionModelPicker so caption
- // runs pass the explicit selection.
- const [captionModel, setCaptionModel] = useState({ providerId: null, model: null });
- const [strippingFragments, setStrippingFragments] = useState(false);
- // Bumped after a bulk caption rewrite (strip) so the image grid drops any
- // unsaved caption drafts the rewrite superseded — otherwise a stale draft
- // blur-saves the old text back, undoing the strip.
- const [captionDraftResetToken, setCaptionDraftResetToken] = useState(0);
-
- // Pull the live canon subject once for variation-axis options + sheet link.
- useEffect(() => {
- if (!dataset?.character?.universeId) return undefined;
- let cancelled = false;
- getUniverse(dataset.character.universeId, { silent: true })
- .then((u) => {
- if (cancelled) return;
- const entries = Array.isArray(u?.[subjectKind(dataset)]) ? u[subjectKind(dataset)] : [];
- setSubject(entries.find((entry) => entry.id === dataset.character.entryId) || null);
- })
- .catch(() => {});
- return () => { cancelled = true; };
- }, [dataset?.character?.universeId, dataset?.character?.entryKind, dataset?.character?.entryId]);
-
- // Object/place variation axes (Lighting/Settings) live as server-side
- // constants in deriveVariationAxes — fetch them so the generate-batch
- // override chips render without mirroring the vocab client-side. Characters
- // seed their chips from live canon (expressions/wardrobes) instead.
- useEffect(() => {
- if (!datasetId || subjectKind(dataset) === 'characters') {
- setVariationAxes(null);
- return undefined;
- }
- // Clear first so a direct object↔place switch can't seed the generate
- // dialog from the previous kind's axes while the new fetch is in flight,
- // and cancel-on-cleanup so a late response can't overwrite the current one.
- let cancelled = false;
- setVariationAxes(null);
- getLoraDatasetVariationAxes(datasetId, { silent: true })
- .then((axes) => { if (!cancelled) setVariationAxes(axes); })
- .catch(() => { if (!cancelled) setVariationAxes(null); });
- return () => { cancelled = true; };
- }, [datasetId, dataset?.character?.entryKind]);
-
- // Readiness is derived from the live local images (not the server's snapshot
- // on `dataset.readiness`) so manual caption edits / deletes reflect in the
- // counts + Train gate immediately, without a refetch per keystroke.
- const readiness = useMemo(
- () => computeDatasetReadiness(dataset),
- [dataset?.images, dataset?.triggerWord],
- );
-
- // Caption-lint advisory: identity fragments repeated across most captions
- // (issue #1320). Derived live so it tracks manual caption edits without a
- // refetch; the "Strip" action recomputes server-side.
- const captionInvariants = useMemo(
- () => analyzeCaptionInvariants(dataset?.images, dataset?.triggerWord),
- [dataset?.images, dataset?.triggerWord],
- );
-
- const stripSharedFragments = async () => {
- setStrippingFragments(true);
- try {
- const { dataset: next, removedFragments, updatedImages } = await stripLoraDatasetSharedCaptionFragments(datasetId);
- setDataset(next);
- setCaptionDraftResetToken((n) => n + 1);
- if (removedFragments.length) {
- toast.success(`Stripped ${removedFragments.length} shared identity fragment${removedFragments.length === 1 ? '' : 's'} from ${updatedImages} caption${updatedImages === 1 ? '' : 's'}`);
- } else {
- toast.success('No shared identity fragments to strip');
- }
- } finally {
- setStrippingFragments(false);
- }
- };
-
- const renderingCount = readiness.rendering;
-
- // Caption run — one slot, driven by the SERVER's run (not the kickoff reply):
- // the dataset GET reports the in-flight run (`dataset.captionRun`), so a reload,
- // a second tab or a lost kickoff acknowledgement re-attaches to the same batch
- // instead of showing idle controls. The server emits per-image `error` progress
- // and a terminal frame carrying the failure tally + the last per-image reason
- // (refusal vs. an exhausted reasoning budget); surface it so a failure isn't silent.
- const toastCaptionFailure = (frame) => {
- const detail = frame.lastError ? ` ${frame.lastError}` : '';
- if (frame.type === 'error') {
- toast.error(frame.lastError || frame.message || frame.error || 'Captioning failed');
- return;
- }
- const noun = frame.failed === 1 ? 'image' : 'images';
- toast.error(frame.done > 0
- ? `Captioned ${frame.done}/${frame.total} — ${frame.failed} ${noun} failed.${detail || ' Re-caption individually or pick another vision model.'}`
- : `All ${frame.failed} ${noun} failed to caption.${detail || ' The vision model refused or returned nothing — try another vision model.'}`);
- };
- // Run ids already adopted from a dataset snapshot — a snapshot read before the
- // terminal refetch lands must not re-attach to the run that just settled.
- const adoptedCaptionRuns = useRef(new Set());
- const captionSlot = useSseJobSlot({
- startRequest: async (options) => {
- const run = await startLoraCaptionRun(datasetId, options);
- if (run.conflict) toast.info('A different caption run is already active on this dataset — showing its progress');
- return { jobId: run.runId };
- },
- eventsUrl: (runId) => `/api/lora-datasets/${datasetId}/caption-runs/${runId}/events`,
- cancelRequest: (runId, opts) => cancelLoraCaptionRun(datasetId, runId, opts),
- onKickoffSuccess: (runId) => adoptedCaptionRuns.current.add(runId),
- onSettled: () => refresh(),
- onComplete: (frame) => {
- if (frame.failed > 0) toastCaptionFailure(frame);
- else if (frame.done > 0) toast.success(`Captioned ${frame.done} image${frame.done === 1 ? '' : 's'}`);
- },
- onErrorFrame: (frame) => { toastCaptionFailure(frame); return true; },
- canceledMessage: 'Captioning canceled — captions already written are kept',
- lostConnectionMessage: 'Lost connection to the caption run — reload to re-attach',
- startErrorFallback: 'Failed to start captioning',
- });
- const activeCaptionRun = dataset?.captionRun;
- useEffect(() => {
- const runId = activeCaptionRun?.runId;
- if (!runId || adoptedCaptionRuns.current.has(runId)) return;
- if (captionSlot.attach(runId, null)) adoptedCaptionRuns.current.add(runId);
- }, [activeCaptionRun?.runId, captionSlot.active]);
- const captionBusy = captionSlot.active;
- const captionProgress = captionSlot.latest ?? activeCaptionRun;
- const canceling = captionProgress?.status === 'canceling';
-
- const onImagesChange = (mutate) => setDataset((prev) => (prev ? { ...prev, images: mutate(prev.images) } : prev));
-
- const saveTriggerWord = async () => {
- if (triggerDraft === null || triggerDraft === dataset.triggerWord) { setTriggerDraft(null); return; }
- if (!isValidTriggerWord(triggerDraft)) {
- toast.error('Trigger word must be 2-64 chars of a-z, 0-9, _');
- return;
- }
- setTriggerSaving(true);
- try {
- const next = await patchLoraDataset(datasetId, { triggerWord: triggerDraft });
- setDataset(next);
- setTriggerDraft(null);
- toast.success('Trigger word saved — re-caption images so captions carry the new token');
- } finally {
- setTriggerSaving(false);
- }
- };
-
- const onUpload = async (e) => {
- const files = e.target.files;
- if (!files?.length) return;
- setUploading(true);
- try {
- const { images } = await uploadLoraDatasetImages(datasetId, files);
- onImagesChange((prev) => [...prev, ...images]);
- refresh();
- toast.success(`Added ${images.length} image${images.length === 1 ? '' : 's'}`);
- } finally {
- setUploading(false);
- }
- };
-
- // Stable so CaptionModelPicker's run-once effect doesn't re-fire.
- const onCaptionModelChange = useCallback((sel) => setCaptionModel(sel), []);
-
- // Strip null provider/model so an "Auto" selection sends an empty body and
- // the server resolves a vision model itself.
- const captionOptions = (extra = {}) => ({
- ...extra,
- ...(captionModel.providerId ? { providerId: captionModel.providerId } : {}),
- ...(captionModel.model ? { model: captionModel.model } : {}),
- });
-
- // Caption only images that have no caption yet (overwrite: false). Re-caption
- // all overwrites every ready image's caption with the picked model — the way
- // to re-run the whole dataset through a newer/better VLM.
- const startCaption = (overwrite) => captionSlot.start(captionOptions({ overwrite }));
- const recaptionImage = (img) => captionSlot.start(captionOptions({ imageIds: [img.id], overwrite: true }));
- const captionAll = () => startCaption(false);
- const recaptionAll = () => startCaption(true);
-
- const expressionOptions = useMemo(
- () => (subjectKind(dataset) === 'characters'
- ? (subject?.expressions || []).map((e) => e?.name).filter(Boolean)
- : (variationAxes?.expressions || [])),
- [dataset, subject, variationAxes],
- );
- const outfitOptions = useMemo(
- () => (subjectKind(dataset) === 'characters'
- ? (subject?.wardrobes || []).map((w) => w?.name).filter(Boolean)
- : (variationAxes?.outfits || [])),
- [dataset, subject, variationAxes],
- );
- const hasReferenceSheet = listSheetPointers(subject).length > 0;
-
- if (loadError) {
- return (
- <div className="bg-port-card border border-port-border rounded-lg p-6 text-center text-gray-400">
- <AlertTriangle className="w-6 h-6 mx-auto mb-2 text-port-error" />
- {loadError} — <Link to="/models/training" className="text-port-accent hover:underline">back to datasets</Link>
- </div>
- );
- }
- if (!dataset) {
- return <div className="text-gray-400 flex items-center gap-2"><Loader2 className="w-4 h-4 animate-spin" /> Loading dataset…</div>;
- }
-
- return (
- <div className="@container/page min-w-0 space-y-4">
- <div className="flex items-start justify-between gap-3 flex-wrap">
- <div className="min-w-0">
- <Link to="/models/training" className="text-xs text-gray-400 hover:text-white flex items-center gap-1 mb-1">
- <ArrowLeft className="w-3 h-3" /> Datasets
- </Link>
- <div className="flex items-center gap-2 min-w-0">
- <h2 className="text-lg font-semibold text-white truncate">
- <Link
- to={`/universes/${dataset.character.universeId}`}
- className="hover:text-port-accent"
- title="Open in universe editor"
- >
- {dataset.character.name}
- </Link>
- </h2>
- <button
- type="button"
- onClick={() => setShowReassign(true)}
- disabled={dataset.status === 'training'}
- title={dataset.status === 'training'
- ? 'Cancel the in-progress training run before reassigning'
- : 'Reassign this dataset to a different universe subject'}
- className="shrink-0 text-xs text-gray-500 hover:text-port-accent flex items-center gap-1 disabled:opacity-40 disabled:hover:text-gray-500"
- >
- <Replace className="w-3.5 h-3.5" /> Reassign
- </button>
- </div>
- <div className="flex items-center gap-2 mt-1">
- <label htmlFor="lt-trigger" className="text-xs text-gray-500">Trigger word</label>
- <input
- id="lt-trigger"
- value={triggerDraft ?? dataset.triggerWord}
- onChange={(e) => setTriggerDraft(e.target.value)}
- onBlur={saveTriggerWord}
- disabled={triggerSaving}
- className="bg-port-bg border border-port-border rounded px-2 py-1 text-xs font-mono text-white w-56 disabled:opacity-50"
- />
- {triggerSaving && <Loader2 className="w-3 h-3 animate-spin text-gray-400" />}
- </div>
- <div className="text-xs text-gray-500 mt-1">
- {SUBJECT_TYPE_LABEL[subjectKind(dataset)] || 'Subject'} from the universe bible
- </div>
- </div>
- <div className="text-right text-sm">
- <div className={readiness.quality === 'good' ? 'text-port-success' : readiness.quality === 'minimum' ? 'text-port-warning' : 'text-gray-400'}>
- {readiness.ready ?? 0} images · {readiness.captioned ?? 0}/{readiness.required ?? 10} captioned
- {renderingCount ? ` · ${renderingCount} rendering` : ''}
- </div>
- <div className="text-xs text-gray-500">
- {readiness.quality === 'good'
- ? 'Ready to train — strong dataset'
- : readiness.quality === 'minimum'
- ? `Trainable now · ${Math.max(0, (readiness.recommended ?? 20) - (readiness.captioned ?? 0))} more for best quality`
- : (readiness.captioned ?? 0) >= (readiness.required ?? 10)
- ? 'Set a trigger word to train'
- : `Add + caption ${(readiness.required ?? 10) - (readiness.captioned ?? 0)} more to train`}
- </div>
- </div>
- </div>
-
- <div className="flex flex-wrap gap-2">
- <button
- type="button"
- onClick={() => setShowGenerate(true)}
- className="px-3 py-2 text-sm rounded bg-port-accent/20 text-port-accent hover:bg-port-accent/30 flex items-center gap-2"
- >
- <Wand2 className="w-4 h-4" /> Generate
- </button>
- <FilePickerButton
- accept={IMAGE_ACCEPT}
- multiple
- onChange={onUpload}
- disabled={uploading}
- ariaLabel="Upload dataset images"
- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2"
- >
- {uploading ? <Loader2 className="w-4 h-4 animate-spin" /> : <Upload className="w-4 h-4" />} Upload
- </FilePickerButton>
- <button
- type="button"
- onClick={() => setShowImport(true)}
- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2"
- >
- <Images className="w-4 h-4" /> From gallery
- </button>
- <button
- type="button"
- onClick={() => setShowSlice(true)}
- disabled={!hasReferenceSheet}
- title={hasReferenceSheet ? '' : 'Render a reference sheet in the universe editor first'}
- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2 disabled:opacity-50"
- >
- <Scissors className="w-4 h-4" /> Slice sheet
- </button>
- <button
- type="button"
- onClick={captionAll}
- disabled={captionBusy || !readiness.ready}
- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2 disabled:opacity-50"
- >
- {captionBusy ? <Loader2 className="w-4 h-4 animate-spin" /> : <Tags className="w-4 h-4" />}
- {captionBusy
- ? `Captioning ${captionProgress?.done ?? 0}/${captionProgress?.total ?? '…'}`
- : 'Caption all'}
- </button>
- <button
- type="button"
- onClick={recaptionAll}
- disabled={captionBusy || !readiness.ready}
- title="Re-caption every ready image with the selected model — overwrites all existing captions, including manual edits"
- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2 disabled:opacity-50"
- >
- <RefreshCw className="w-4 h-4" /> Re-caption all
- </button>
- <CaptionModelPicker onChange={onCaptionModelChange} />
- </div>
-
- {captionBusy && (
- <div
- role="status"
- className="bg-port-card border border-port-border rounded-lg p-3 flex flex-wrap items-center gap-3 text-sm"
- >
- <Loader2 className="w-4 h-4 animate-spin text-port-accent shrink-0" />
- <div className="min-w-0 flex-1">
- <div className="text-white">
- {canceling
- ? 'Canceling… finishing the vision call already in flight (its result is discarded)'
- : `Captioning ${captionProgress?.done ?? 0}/${captionProgress?.total ?? '…'}${captionProgress?.failed ? ` · ${captionProgress.failed} failed` : ''}`}
- </div>
- {activeCaptionRun?.model && (
- <div className="text-xs text-gray-400 truncate">{activeCaptionRun.provider} · {activeCaptionRun.model}</div>
- )}
- </div>
- <button
- type="button"
- onClick={captionSlot.cancel}
- disabled={!captionSlot.jobId || canceling}
- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2 disabled:opacity-50 min-h-[40px]"
- >
- <XCircle className="w-4 h-4" /> Cancel
- </button>
- </div>
- )}
-
- {captionInvariants.sharedFragments.length > 0 && (
- <div className="bg-port-warning/10 border border-port-warning/40 rounded-lg p-3 text-sm">
- <div className="flex items-start gap-2">
- <AlertTriangle className="w-4 h-4 text-port-warning shrink-0 mt-0.5" />
- <div className="min-w-0 flex-1 space-y-2">
- <p className="text-gray-200">
- <span className="font-medium text-port-warning">Identity is leaking into the captions.</span>
- {' '}
- These fragments repeat across most captions, so the model binds the look to the phrases instead of the
- {' '}
- <span className="font-mono text-gray-300">{dataset.triggerWord || 'trigger'}</span>
- {' '}
- token — the trigger then renders a generic subject. Keep only what changes shot-to-shot (pose, framing,
- setting) and let the trigger absorb the fixed identity.
- </p>
- <div className="flex flex-wrap gap-1.5">
- {captionInvariants.sharedFragments.map((f) => (
- <span
- key={f.normalized}
- className="text-xs px-1.5 py-0.5 rounded bg-port-bg border border-port-border text-gray-300 font-mono"
- title={`in ${f.count} of ${captionInvariants.total} captions`}
- >
- {f.fragment}
- <span className="text-gray-500"> ·{f.count}/{captionInvariants.total}</span>
- </span>
- ))}
- </div>
- <button
- type="button"
- onClick={stripSharedFragments}
- disabled={strippingFragments || captionBusy}
- className="px-3 py-1.5 text-xs rounded bg-port-warning/20 text-port-warning hover:bg-port-warning/30 flex items-center gap-2 disabled:opacity-50"
- title="Remove these shared fragments from every caption, keeping the trigger word and per-shot detail"
- >
- {strippingFragments ? <Loader2 className="w-3.5 h-3.5 animate-spin" /> : <Eraser className="w-3.5 h-3.5" />}
- Strip shared identity from all captions
- </button>
- </div>
- </div>
- </div>
- )}
-
- <details className="bg-port-card border border-port-border rounded-lg text-sm">
- <summary className="cursor-pointer select-none px-3 py-2 flex items-center gap-2 text-gray-300 hover:text-white">
- <Lightbulb className="w-4 h-4 text-port-warning shrink-0" />
- <span className="font-medium">Tips for a strong training dataset</span>
- <span className="text-xs text-gray-500">
- target ~{RECOMMENDED_TRAINING_IMAGES}–{TRAINING_IMAGE_SWEET_SPOT_MAX} images · {MIN_TRAINING_IMAGES} minimum
- </span>
- </summary>
- <div className="px-3 pb-3 pt-1 text-xs text-gray-400 space-y-2 border-t border-port-border/60">
- <p>
- Quality beats quantity. {MIN_TRAINING_IMAGES} images is the floor; ~{RECOMMENDED_TRAINING_IMAGES}–{TRAINING_IMAGE_SWEET_SPOT_MAX} sharp,
- varied shots is the sweet spot. Past ~50 you mostly add training time and overfitting risk —
- near-duplicate frames teach the model to memorize one setup instead of learning the subject.
- </p>
- <ul className="list-disc pl-4 space-y-1">
- <li><span className="text-gray-300">Vary the angle</span> — front, three-quarter, profile, and a back view.</li>
- <li><span className="text-gray-300">Vary the framing</span> — mix tight face close-ups (for likeness) with mid and full-body shots (for proportions).</li>
- <li><span className="text-gray-300">Vary pose, presentation &amp; expression</span> — action and rest for characters; display angles and use context for objects and places.</li>
- <li><span className="text-gray-300">Vary outfit, lighting &amp; background</span> — so the LoRA learns the subject, not one costume, key light, or backdrop.</li>
- <li><span className="text-gray-300">Keep it single-subject and on-model</span> — one clearly-visible subject per image, consistent identity, no clutter.</li>
- <li><span className="text-gray-300">Drop the weak ones</span> — blurry, occluded, or off-model frames hurt more than they help; avoid near-duplicates.</li>
- </ul>
- <p className="pt-1 border-t border-port-border/60">
- <span className="text-gray-300">Caption what changes shot-to-shot, not who the subject is.</span> The
- {' '}
- <span className="font-mono text-gray-400">{dataset.triggerWord || 'trigger word'}</span>
- {' '}
- token is what the LoRA binds the fixed identity to — describe pose, framing, expression, outfit, and setting
- in each caption, and leave invariant features (hair, eyes, skin, signature items) out. Repeating the same
- identity phrases in every caption teaches those words instead of the trigger, so the bare trigger then
- renders a generic subject. Captions are auto-prefixed with the trigger; this page warns when identity
- fragments repeat across most captions and offers a one-click strip.
- </p>
- </div>
- </details>
-
- <div className="grid grid-cols-1 @5xl/page:grid-cols-[minmax(0,1fr)_320px] gap-4 items-start">
- <DatasetImageGrid
- dataset={dataset}
- onImagesChange={onImagesChange}
- onRecaption={recaptionImage}
- recaptioningId={captionSlot.context?.imageIds?.length === 1 ? captionSlot.context.imageIds[0] : null}
- captionBusy={captionBusy}
- draftResetToken={captionDraftResetToken}
- />
- <TrainingPanel
- dataset={dataset}
- readiness={readiness}
- triggerSaving={triggerSaving}
- onRunFinished={refresh}
- />
- </div>
-
- {showGenerate && (
- <GenerateBatchDialog
- dataset={dataset}
- expressionOptions={expressionOptions}
- outfitOptions={outfitOptions}
- onClose={() => setShowGenerate(false)}
- onStarted={() => { setShowGenerate(false); refresh(); }}
- />
- )}
- {showSlice && (
- <SliceDialog
- dataset={dataset}
- onClose={() => setShowSlice(false)}
- onSliced={() => { setShowSlice(false); refresh(); }}
- />
- )}
- {showImport && (
- <ImportGalleryDialog
- dataset={dataset}
- onClose={() => setShowImport(false)}
- onImported={(images) => {
- setShowImport(false);
- if (images?.length) onImagesChange((prev) => [...prev, ...images]);
- refresh();
- }}
- />
- )}
- {showReassign && (
- <ReassignDialog
- dataset={dataset}
- onClose={() => setShowReassign(false)}
- onReassigned={(next) => { setShowReassign(false); setDataset(next); }}
- />
- )}
- </div>
- );
-}
diff --git a/client/src/pages/LoraDatasetDetail.test.jsx b/client/src/pages/LoraDatasetDetail.test.jsx
deleted file mode 100644
index 486aaf1d78..0000000000
--- a/client/src/pages/LoraDatasetDetail.test.jsx
+++ /dev/null
@@ -1,143 +0,0 @@
-import { act, cleanup, fireEvent, render, screen } from '@testing-library/react';
-import { MemoryRouter } from 'react-router';
-import { afterEach, expect, it, vi } from 'vitest';
-import LoraDatasetDetail from './LoraDatasetDetail';
-import { cancelLoraCaptionRun, getLoraDataset, startLoraCaptionRun } from '../services/api';
-import socket from '../services/socket';
-
-vi.mock('../services/api', () => ({
- getLoraDataset: vi.fn(), getUniverse: vi.fn(), startLoraCaptionRun: vi.fn(), cancelLoraCaptionRun: vi.fn(),
-}));
-vi.mock('../components/ui/Toast', () => ({
- default: Object.assign(vi.fn(), { success: vi.fn(), error: vi.fn(), info: vi.fn() }),
-}));
-vi.mock('../services/socket', () => {
- const handlers = new Map();
- return { default: {
- on: (event, handler) => { if (!handlers.has(event)) handlers.set(event, new Set()); handlers.get(event).add(handler); },
- off: (event, handler) => handlers.get(event)?.delete(handler),
- emit: (event, payload) => handlers.get(event)?.forEach(handler => handler(payload)),
- } };
-});
-// Controllable SSE stream: the test pushes frames; the page reads them through
-// the real useSseJobSlot, so adoption/cancel/settlement are exercised end to end.
-const sse = vi.hoisted(() => {
- const listeners = new Set();
- let value = { latest: null, closed: false, isOpen: true, frames: [] };
- return {
- urls: [],
- get: () => value,
- push: (frame) => { value = { latest: frame, closed: true, isOpen: false, frames: [frame] }; listeners.forEach(fn => fn()); },
- reset: () => { value = { latest: null, closed: false, isOpen: true, frames: [] }; sse.urls.length = 0; },
- subscribe: (fn) => { listeners.add(fn); return () => listeners.delete(fn); },
- };
-});
-vi.mock('../hooks/useSseProgress', async () => {
- const { useSyncExternalStore } = await import('react');
- return {
- isTerminalSseFrame: frame => ['complete', 'error', 'canceled'].includes(frame?.type),
- useSseProgress: (url) => {
- const value = useSyncExternalStore(sse.subscribe, sse.get);
- if (url) sse.urls.push(url);
- return url ? { ...value, latestUrl: url } : { latest: null, closed: false, isOpen: false, frames: [] };
- },
- };
-});
-vi.mock('../components/loraTraining/TrainingPanel', () => ({ default: () => null }));
-vi.mock('../components/loraTraining/CaptionModelPicker', () => ({ default: () => null }));
-vi.mock('../components/loraTraining/DatasetImageGrid', () => ({
- default: ({ dataset }) => <div>{dataset.images.map(image => <span key={image.id}>{image.status}</span>)}</div>,
-}));
-const dataset = (id, status = 'rendering') => ({
- id, character: { name: id, entryKind: 'characters' }, triggerWord: 'example',
- images: [{ id: 'image-1', status, file: 'example.png', caption: '' }], training: {},
-});
-afterEach(() => { cleanup(); vi.useRealTimers(); vi.clearAllMocks(); sse.reset(); });
-it('updates on scoped persistence events, reconnect and tab show without polling', async () => {
- vi.useFakeTimers();
- Object.defineProperty(document, 'visibilityState', { configurable: true, value: 'visible' });
- getLoraDataset.mockResolvedValue(dataset('example-a'));
- render(<MemoryRouter><LoraDatasetDetail recordId="example-a" /></MemoryRouter>);
- await act(async () => {});
- expect(screen.getByText('rendering')).toBeTruthy();
- await act(async () => { await vi.advanceTimersByTimeAsync(16000); });
- expect(getLoraDataset).toHaveBeenCalledTimes(1);
- await act(async () => { socket.emit('training:dataset:changed', { datasetId: 'other' }); });
- expect(getLoraDataset).toHaveBeenCalledTimes(1);
- getLoraDataset.mockResolvedValue(dataset('example-a', 'ready'));
- await act(async () => { socket.emit('training:dataset:changed', { datasetId: 'example-a' }); });
- expect(screen.getByText('ready')).toBeTruthy();
- expect(getLoraDataset).toHaveBeenCalledTimes(2);
- await act(async () => { socket.emit('connect'); });
- expect(getLoraDataset).toHaveBeenCalledTimes(3);
- await act(async () => {
- Object.defineProperty(document, 'visibilityState', { configurable: true, value: 'hidden' });
- document.dispatchEvent(new Event('visibilitychange'));
- socket.emit('training:dataset:changed', { datasetId: 'example-a' });
- });
- expect(getLoraDataset).toHaveBeenCalledTimes(3);
- getLoraDataset.mockResolvedValue(dataset('example-a', 'failed'));
- await act(async () => {
- Object.defineProperty(document, 'visibilityState', { configurable: true, value: 'visible' });
- document.dispatchEvent(new Event('visibilitychange'));
- });
- expect(screen.getByText('failed')).toBeTruthy();
- expect(getLoraDataset).toHaveBeenCalledTimes(4);
-});
-it('reads a changed route and ignores the previous response and events', async () => {
- let resolveOld;
- getLoraDataset.mockImplementation(id => id === 'example-a'
- ? new Promise(resolve => { resolveOld = resolve; }) : Promise.resolve(dataset(id, 'ready')));
- const view = render(<MemoryRouter><LoraDatasetDetail recordId="example-a" /></MemoryRouter>);
- await act(async () => {});
- view.rerender(<MemoryRouter><LoraDatasetDetail recordId="example-b" /></MemoryRouter>);
- await act(async () => {});
- await act(async () => { resolveOld(dataset('example-a')); });
- expect(screen.getByText('example-b')).toBeTruthy();
- expect(screen.queryByText('rendering')).toBeNull();
- await act(async () => { socket.emit('training:dataset:changed', { datasetId: 'example-a' }); });
- expect(getLoraDataset).toHaveBeenCalledTimes(2);
-});
-
-const readyDataset = (captionRun = null) => ({
- ...dataset('example-a', 'ready'),
- images: [{ id: 'image-1', status: 'ready', file: 'example.png', caption: '' }],
- readiness: {}, captionRun,
-});
-const activeRun = { runId: 'run-1', datasetId: 'example-a', status: 'running', provider: 'p', model: 'm', total: 3, done: 1, failed: 0 };
-
-it('re-attaches to the server caption run after a remount and cancels it from there', async () => {
- getLoraDataset.mockResolvedValue(readyDataset(activeRun));
- cancelLoraCaptionRun.mockResolvedValue({ canceled: true });
- const first = render(<MemoryRouter><LoraDatasetDetail recordId="example-a" /></MemoryRouter>);
- await act(async () => {});
- expect(screen.getByRole('button', { name: /Cancel/ })).toBeTruthy();
- first.unmount();
- // Reload / second tab: a fresh mount adopts the same run from the dataset read.
- render(<MemoryRouter><LoraDatasetDetail recordId="example-a" /></MemoryRouter>);
- await act(async () => {});
- expect(screen.getAllByText(/Captioning 1\/3/).length).toBeGreaterThan(0);
- expect(sse.urls.at(-1)).toBe('/api/lora-datasets/example-a/caption-runs/run-1/events');
- expect(screen.getByRole('button', { name: /^Captioning 1\/3/ }).disabled).toBe(true);
- expect(startLoraCaptionRun).not.toHaveBeenCalled();
-
- fireEvent.click(screen.getByRole('button', { name: /^Cancel/ }));
- expect(cancelLoraCaptionRun).toHaveBeenCalledWith('example-a', 'run-1', { silent: true });
- // The server's terminal canceled frame settles the UI and refetches the dataset.
- getLoraDataset.mockResolvedValue(readyDataset(null));
- await act(async () => { sse.push({ type: 'canceled', runId: 'run-1', done: 1, total: 3 }); });
- expect(screen.queryByRole('button', { name: /^Cancel/ })).toBeNull();
- expect(screen.getByRole('button', { name: /Caption all/ }).disabled).toBe(false);
- expect(getLoraDataset).toHaveBeenCalledTimes(3);
- expect(startLoraCaptionRun).not.toHaveBeenCalled();
-});
-
-it('attaches to the holder when a start reports a conflicting active run', async () => {
- getLoraDataset.mockResolvedValue(readyDataset(null));
- startLoraCaptionRun.mockResolvedValue({ runId: 'run-9', alreadyRunning: true, conflict: true, total: 3 });
- render(<MemoryRouter><LoraDatasetDetail recordId="example-a" /></MemoryRouter>);
- await act(async () => {});
- await act(async () => { fireEvent.click(screen.getByRole('button', { name: /Caption all/ })); });
- expect(sse.urls.at(-1)).toBe('/api/lora-datasets/example-a/caption-runs/run-9/events');
- expect(screen.getByRole('button', { name: /^Cancel/ })).toBeTruthy();
-});
diff --git a/client/src/pages/LoraTraining.jsx b/client/src/pages/LoraTraining.jsx
deleted file mode 100644
index 8e5fac1f0d..0000000000
--- a/client/src/pages/LoraTraining.jsx
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * LoRA Training — dataset list (/models/training).
- *
- * One dataset per universe bible subject. Cards show the subject, a thumb
- * strip, image/caption counts, and training status; "New dataset" walks
- * universe → character and lands on the dataset workbench.
- */
-
-import { useState } from 'react';
-import { Link, useNavigate } from 'react-router';
-import { GraduationCap, Plus, Loader2, Sparkles, Images } from 'lucide-react';
-import toast from '../components/ui/Toast';
-import Modal from '../components/ui/Modal';
-import ConfirmButtonPair from '../components/ui/ConfirmButtonPair';
-import UniverseCharacterPicker from '../components/loraTraining/UniverseCharacterPicker';
-import MediaJobsQueue from '../components/media/MediaJobsQueue';
-import ListLoadError from '../components/ui/ListLoadError';
-import useListResource from '../hooks/useListResource';
-import {
- listLoraDatasets,
- createLoraDataset,
- deleteLoraDataset,
-} from '../services/api';
-
-const loadLoraDatasets = () => listLoraDatasets({}, { silent: true });
-
-const STATUS_CHIP = {
- draft: 'bg-port-border/40 text-gray-300',
- training: 'bg-port-warning/20 text-port-warning',
- trained: 'bg-port-success/20 text-port-success',
-};
-const SUBJECT_TYPE_LABEL = { characters: 'Character', objects: 'Object', places: 'Place' };
-
-function NewDatasetDialog({ onClose, onCreated }) {
- const [universeId, setUniverseId] = useState('');
- const [entryKind, setEntryKind] = useState('characters');
- const [entryId, setEntryId] = useState('');
- const [creating, setCreating] = useState(false);
-
- const create = async () => {
- setCreating(true);
- try {
- const dataset = await createLoraDataset({ universeId, entryKind, entryId });
- onCreated(dataset);
- } finally {
- setCreating(false);
- }
- };
-
- return (
- <Modal
- open
- onClose={onClose}
- size="sm"
- ariaLabelledBy="lt-new-dataset-title"
- panelClassName="bg-port-card border border-port-border rounded-lg p-5"
- >
- <div className="space-y-4">
- <h2 id="lt-new-dataset-title" className="text-base font-semibold text-white">New training dataset</h2>
- <UniverseCharacterPicker
- idPrefix="lt-new"
- universeId={universeId}
- entryKind={entryKind}
- entryId={entryId}
- onUniverseChange={(id) => { setUniverseId(id); setEntryId(''); }}
- onEntryKindChange={(kind) => { setEntryKind(kind); setEntryId(''); }}
- onEntryChange={setEntryId}
- />
- <div className="flex justify-end gap-2">
- <button type="button" onClick={onClose} className="px-3 py-2 text-sm text-gray-400 hover:text-white">Cancel</button>
- <button
- type="button"
- onClick={create}
- disabled={!universeId || !entryId || creating}
- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"
- >
- {creating ? <Loader2 className="w-4 h-4 animate-spin" /> : <Plus className="w-4 h-4" />}
- Open dataset
- </button>
- </div>
- </div>
- </Modal>
- );
-}
-
-export default function LoraTraining() {
- const navigate = useNavigate();
- const datasetsResource = useListResource(loadLoraDatasets);
- const {
- items: datasets, state, error, retry, setItems: setDatasets,
- } = datasetsResource;
- const [showNew, setShowNew] = useState(false);
- const [deletingId, setDeletingId] = useState(null);
- const [confirmingId, setConfirmingId] = useState(null);
-
- const onDelete = async (id) => {
- setDeletingId(id);
- try {
- await deleteLoraDataset(id);
- setDatasets((prev) => (prev || []).filter((d) => d.id !== id));
- toast.success('Dataset deleted');
- } finally {
- setDeletingId(null);
- setConfirmingId(null);
- }
- };
-
- return (
- <div>
- <div className="flex items-center justify-between mb-4 flex-wrap gap-2">
- <div className="flex items-center gap-2">
- <GraduationCap className="w-5 h-5 text-port-accent" />
- <h2 className="text-lg font-semibold text-white">LoRA Training</h2>
- </div>
- <button
- type="button"
- onClick={() => setShowNew(true)}
- className="px-3 py-2 text-sm rounded bg-port-accent text-white flex items-center gap-2"
- >
- <Plus className="w-4 h-4" /> New dataset
- </button>
- </div>
-
- {state === 'loading' && (
- <div className="text-gray-400 flex items-center gap-2"><Loader2 className="w-4 h-4 animate-spin" /> Loading datasets…</div>
- )}
-
- {state === 'error' && (
- <ListLoadError what="datasets" error={error} onRetry={retry} />
- )}
-
- {state === 'ready' && datasets.length === 0 && (
- <div className="bg-port-card border border-port-border rounded-lg p-8 text-center text-gray-400">
- <Sparkles className="w-8 h-8 mx-auto mb-3 text-port-accent" />
- <p className="mb-1 text-white">Train a LoRA for consistent universe subjects.</p>
- <p className="text-sm">Pick a character, object, or place from a universe bible, build reference material, caption it, and train a FLUX adapter.</p>
- </div>
- )}
-
- {state === 'ready' && datasets.length > 0 && (
- <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
- {(datasets || []).map((d) => (
- <div key={d.id} className="bg-port-card border border-port-border rounded-lg p-4 flex flex-col gap-3">
- <div className="flex items-start justify-between gap-2">
- <Link to={`/models/training/${d.id}`} className="min-w-0">
- <div className="text-white font-medium truncate">{d.character?.name || 'Unnamed'}</div>
- <div className="text-xs text-gray-500 truncate">
- <span className="font-mono">{d.triggerWord}</span>
- <span className="mx-1">·</span>
- {SUBJECT_TYPE_LABEL[d.character?.entryKind || 'characters'] || 'Subject'}
- </div>
- </Link>
- <span className={`text-xs px-2 py-0.5 rounded-full shrink-0 ${STATUS_CHIP[d.status] || STATUS_CHIP.draft}`}>
- {d.status}
- </span>
- </div>
- {d.thumbnails?.length > 0 && (
- <Link to={`/models/training/${d.id}`} className="flex gap-1 overflow-hidden">
- {d.thumbnails.map((file) => (
- <img
- key={file}
- src={`/data/lora-datasets/${d.id}/images/${file}`}
- alt=""
- className="w-16 h-16 object-cover rounded border border-port-border"
- loading="lazy"
- />
- ))}
- </Link>
- )}
- <div className="flex items-center justify-between text-xs text-gray-400">
- <span className="flex items-center gap-1">
- <Images className="w-3.5 h-3.5" />
- {d.readiness?.ready ?? 0} images · {d.readiness?.captioned ?? 0} captioned
- {d.readiness?.rendering ? ` · ${d.readiness.rendering} rendering` : ''}
- </span>
- {confirmingId === d.id ? (
- <ConfirmButtonPair
- prompt="Delete?"
- busy={deletingId === d.id}
- onConfirm={() => onDelete(d.id)}
- onCancel={() => setConfirmingId(null)}
- />
- ) : (
- <button
- type="button"
- onClick={() => setConfirmingId(d.id)}
- className="text-gray-500 hover:text-port-error"
- >
- Delete
- </button>
- )}
- </div>
- </div>
- ))}
- </div>
- )}
-
- <div className="mt-6">
- <MediaJobsQueue kind="training" />
- </div>
-
- {showNew && (
- <NewDatasetDialog
- onClose={() => setShowNew(false)}
- onCreated={(dataset) => navigate(`/models/training/${dataset.id}`)}
- />
- )}
- </div>
- );
-}
diff --git a/client/src/pages/LoraTraining.test.jsx b/client/src/pages/LoraTraining.test.jsx
deleted file mode 100644
index 8303e1c03f..0000000000
--- a/client/src/pages/LoraTraining.test.jsx
+++ /dev/null
@@ -1,59 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { act, fireEvent, render, screen } from '@testing-library/react';
-import { MemoryRouter } from 'react-router';
-
-vi.mock('../services/api', () => ({
- listLoraDatasets: vi.fn(),
- createLoraDataset: vi.fn(),
- deleteLoraDataset: vi.fn(),
-}));
-vi.mock('../components/loraTraining/UniverseCharacterPicker', () => ({ default: () => null }));
-vi.mock('../components/media/MediaJobsQueue', () => ({ default: () => null }));
-vi.mock('../components/ui/Toast', () => ({ default: { error: vi.fn(), success: vi.fn() } }));
-
-import LoraTraining from './LoraTraining.jsx';
-import { listLoraDatasets } from '../services/api';
-
-describe('LoraTraining page', () => {
- beforeEach(() => {
- vi.clearAllMocks();
- });
-
- it('renders ListLoadError with retry on failed datasets load, and empty state on real empty list', async () => {
- listLoraDatasets.mockRejectedValueOnce(new Error('500 Internal Server Error'));
- render(
- <MemoryRouter>
- <LoraTraining />
- </MemoryRouter>
- );
- await act(async () => {});
-
- expect(screen.getByRole('alert')).toHaveTextContent('Could not load datasets — 500 Internal Server Error');
- expect(screen.getByRole('button', { name: 'Retry' })).toBeInTheDocument();
- expect(screen.queryByText('Train a LoRA for consistent universe subjects.')).not.toBeInTheDocument();
-
- listLoraDatasets.mockResolvedValueOnce([]);
- await act(async () => {
- fireEvent.click(screen.getByRole('button', { name: 'Retry' }));
- });
-
- expect(screen.queryByRole('alert')).not.toBeInTheDocument();
- expect(screen.getByText('Train a LoRA for consistent universe subjects.')).toBeInTheDocument();
- });
-
- it('renders datasets when load succeeds', async () => {
- listLoraDatasets.mockResolvedValueOnce([
- { id: 'ds-1', character: { name: 'Captain Nova' }, status: 'draft', imageCount: 12 },
- ]);
- render(
- <MemoryRouter>
- <LoraTraining />
- </MemoryRouter>
- );
- await act(async () => {});
-
- expect(screen.queryByRole('alert')).not.toBeInTheDocument();
- expect(screen.getByText('Captain Nova')).toBeInTheDocument();
- expect(screen.queryByText('Train a LoRA for consistent universe subjects.')).not.toBeInTheDocument();
- });
-});
diff --git a/client/src/pages/Loras.jsx b/client/src/pages/Loras.jsx
index 19ac686d54..aa59b2557d 100644
--- a/client/src/pages/Loras.jsx
+++ b/client/src/pages/Loras.jsx
@@ -1682,11 +1682,11 @@ function LoraCard({ lora, onDelete, onMeasured, deleting, deleteConfirm }) {
</div>
{/* Trained-in-PortOS lineage: character chip links back to the
- training dataset that produced this adapter. */}
+ retained training artifacts in Data Maintenance. */}
{lora.source === 'trained' && lora.character && (
<div className="mb-2">
<Link
- to={lora.trainedFromDatasetId ? `/models/training/${lora.trainedFromDatasetId}` : '/models/training'}
+ to="/settings/data-maintenance"
className="inline-flex items-center gap-1 text-[11px] px-2 py-0.5 rounded-full bg-port-success/15 text-port-success border border-port-success/30 hover:bg-port-success/25"
title="Trained in PortOS — open the training dataset"
>
diff --git a/client/src/pages/Models.jsx b/client/src/pages/Models.jsx
index 95c7b08de9..9e80bbfd7f 100644
--- a/client/src/pages/Models.jsx
+++ b/client/src/pages/Models.jsx
@@ -16,13 +16,11 @@ import { LocalLlmTab } from '../components/settings/LocalLlmTab';
import LocalLlmRuntimesView from '../components/settings/LocalLlmRuntimesView.jsx';
import { lazyWithReload } from '../utils/lazyWithReload';
-// The three panels moved in from the Media Gen tabs were each their own route
+// The media-model panels moved in from the Media Gen tabs were each their own route
// chunk before the move (#4728) — keep them split, or landing on Performance
// downloads the whole LoRA manager and HF-cache browser to render an
// assessments table. The small always-on tabs above stay static.
const Loras = lazyWithReload(() => import('./Loras'));
-const LoraTraining = lazyWithReload(() => import('./LoraTraining'));
-const LoraDatasetDetail = lazyWithReload(() => import('./LoraDatasetDetail'));
const MediaModels = lazyWithReload(() => import('./MediaModels'));
/**
@@ -43,7 +41,6 @@ const MediaModels = lazyWithReload(() => import('./MediaModels'));
* - **Providers** — configured AI provider connections and model catalogs.
* - **Status** — residency plus the downloaded-model inventory.
* - **Subscriptions** — which paid plans are on, their tier, price and spend.
- * - **Training** — LoRA fine-tuning datasets and runs.
* - **Usage** — provider quota and PortOS AI usage accounting.
*
* What deliberately stayed OUT is output rather than weights: Three.js Models is
@@ -69,21 +66,6 @@ const TAB_CONTENT = {
performance: LocalModelAssessments,
status: ModelStatusTab,
subscriptions: SubscriptionsTab,
- training: LoraTraining,
-};
-
-/**
- * Drill-downs rendered INSIDE the section shell, keyed by the tab that owns them.
- *
- * A tab's detail view is still that tab — the LoRA dataset workbench is Training
- * with one dataset open — so it keeps the section header and tab bar rather than
- * becoming a bare route that drops both. Under `/media` these pages got that for
- * free, because MediaGen was a layout route; declaring them here gives the same
- * result without every future detail view needing its own special case in
- * App.jsx.
- */
-const TAB_DETAIL = {
- training: LoraDatasetDetail,
};
export default function Models({ fixedTab, fixedRecordId } = {}) {
@@ -92,6 +74,7 @@ export default function Models({ fixedTab, fixedRecordId } = {}) {
const tab = fixedTab || params.tab;
const recordId = fixedRecordId || params.recordId;
const taskView = params.taskView || params.view || (fixedTab === 'performance' && params.assessmentKey ? 'results' : undefined);
+ if (tab === 'training') return <Navigate to="/settings/data-maintenance" replace />;
if (tab === 'llms' && recordId === 'jev') return <Navigate to={`/models/decision-classifiers/jev${taskView ? '/' + taskView : ''}`} replace />;
// An unknown slug lands on LLMs rather than rendering a blank page, matching
// the section's default destination in App and the primary navigation.
@@ -103,10 +86,6 @@ export default function Models({ fixedTab, fixedRecordId } = {}) {
const activeTab = tab && Object.hasOwn(TAB_CONTENT, tab) ? tab : null;
if (!activeTab) return <Navigate to="/models/llms" replace />;
- // A record id in the URL selects the tab's drill-down, when it has one. Tabs
- // without a detail component receive it as a focused sub-view id (LLMs uses
- // `library` and `abuse`); tabs that do not recognize it render their index.
- const DetailContent = recordId && Object.hasOwn(TAB_DETAIL, activeTab) ? TAB_DETAIL[activeTab] : null;
const TabContent = TAB_CONTENT[activeTab];
// The shell names the destination you are on (Code Reviewers, Performance),
// matching the section navigator. "Models" stays the area, not the page title.
@@ -126,9 +105,7 @@ export default function Models({ fixedTab, fixedRecordId } = {}) {
{/* Local boundary rather than the App-level one: a lazy tab must not blank
the section header and tab bar while its chunk loads. */}
<Suspense fallback={<PageSkeleton header="none" label="Loading models section" cards={3} sidebar={false} />}>
- {DetailContent
- ? <DetailContent recordId={recordId} />
- : <TabContent view={recordId} taskView={taskView} assessmentKey={params.assessmentKey} />}
+ <TabContent view={recordId} taskView={taskView} assessmentKey={params.assessmentKey} />
</Suspense>
</div>
</ModelsSectionLayout>
diff --git a/client/src/pages/Models.test.jsx b/client/src/pages/Models.test.jsx
index 12ed9400ea..a1c1beb6c4 100644
--- a/client/src/pages/Models.test.jsx
+++ b/client/src/pages/Models.test.jsx
@@ -36,9 +36,7 @@ vi.mock('../components/settings/CodeReviewersTab', () => ({ default: () => <div>
vi.mock('../components/models/ModelComparison', () => ({ default: () => <div>comparison panel</div> }));
vi.mock('../components/models/DecisionClassifiers', () => ({ default: ({ view }) => <div>classifiers panel {view}</div> }));
vi.mock('./Loras', () => ({ default: () => <div>loras panel</div> }));
-vi.mock('./LoraTraining', () => ({ default: () => <div>training panel</div> }));
vi.mock('./MediaModels', () => ({ default: () => <div>media models panel</div> }));
-vi.mock('./LoraDatasetDetail', () => ({ default: ({ recordId }) => <div>dataset workbench {recordId}</div> }));
import Models from './Models';
@@ -62,7 +60,6 @@ const PANEL_MARKER = {
performance: 'assessments panel',
status: 'status panel',
subscriptions: 'subscriptions panel',
- training: 'training panel',
};
// These destinations belong to Models in the sidebar but keep their own route
@@ -75,6 +72,7 @@ const renderAt = (path) => render(
<Routes>
<Route path="/models/performance/results/:assessmentKey" element={<Models fixedTab="performance" />} />
<Route path="/models/performance/:view" element={<Models fixedTab="performance" />} />
+ <Route path="/settings/data-maintenance" element={<div>Retired training maintenance</div>} />
<Route path="/models/:tab" element={<Models />} />
<Route path="/models/:tab/:recordId" element={<Models />} />
<Route path="/models/decision-classifiers/jev/:taskView" element={<Models fixedTab="decision-classifiers" fixedRecordId="jev" />} />
@@ -213,14 +211,10 @@ describe('Models — tab drill-downs', () => {
expect(screen.getByRole('tab', { name: view === 'library' ? 'Model Library' : 'Abuse Guard' })).toHaveAttribute('aria-selected', 'true');
});
- it('renders a tab detail view INSIDE the section shell, not as a bare page', async () => {
- // Under /media these pages kept the shell's chrome for free, because MediaGen
- // was a layout route. Registering the workbench as its own top-level route
- // would silently drop the section header and tab bar; asserting the tab bar is
- // still present (and still marks Training active) is what catches that.
- renderAt('/models/training/dataset-abc');
- expect(await screen.findByText('dataset workbench dataset-abc')).toBeInTheDocument();
- expect(screen.getByRole('tab', { name: 'Training' })).toHaveAttribute('aria-selected', 'true');
+ it.each(['/models/training', '/models/training/dataset-abc'])('redirects retired deep link %s to maintenance', async path => {
+ renderAt(path);
+ expect(await screen.findByText('Retired training maintenance')).toBeInTheDocument();
+ expect(screen.queryByRole('tab', { name: 'Training' })).toBeNull();
});
it('falls back to the tab index when the tab has no detail view', async () => {
diff --git a/client/src/pages/Settings.jsx b/client/src/pages/Settings.jsx
index e4ebdf8133..16634a1fb7 100644
--- a/client/src/pages/Settings.jsx
+++ b/client/src/pages/Settings.jsx
@@ -1,3 +1,4 @@
+import DataMaintenanceTab from '../components/settings/DataMaintenanceTab';
import { useParams, Navigate } from 'react-router';
import { Settings as SettingsIcon } from 'lucide-react';
import PageHeader from '../components/PageHeader';
@@ -44,6 +45,7 @@ export default function Settings() {
case 'api-access': return <ApiAccessTab />;
case 'autofixer': return <AutofixerTab />;
case 'backup': return <BackupTab />;
+ case 'data-maintenance': return <DataMaintenanceTab />;
case 'database': return <DatabaseTab />;
case 'credentials': return <CredentialsTab />;
case 'features': return <InstanceFeaturesTab />;
diff --git a/client/src/services/README.md b/client/src/services/README.md
index 1af68ed3a6..077acf1c26 100644
--- a/client/src/services/README.md
+++ b/client/src/services/README.md
@@ -106,7 +106,7 @@ toasts on throw). **Custom catch ⇒ `silent: true`** — otherwise toasts fire
| File | Purpose |
|---|---|
| `apiImageVideo.js` | Image-gen local backend extras; bounded image/mixed gallery pages, global facets and collection summaries, exact-reference `getGalleryImages`, one image's original-vs-cleaned variant set via `listImageVariants`, plus models, LoRAs, cancel and delete. |
-| `apiLoraTraining.js` | Character LoRA training — datasets (CRUD, upload, generate, slice, caption), training runs (start/list/cancel + status), character→LoRA link lookup. |
+| `apiLoraTraining.js` | Retained checkpoint inspection and character→adapter lookup. Retired training/dataset mutation wrappers are removed. |
| `apiMedia.js` | `importGalleryFile` dispatches image/video imports through the Prompt gallery upload endpoints; `assignImportedMedia` adds existing imports through idempotent bulk membership. Screenshots + media assets. Also owns the multi-file upload orchestration — `processScreenshotUploads` / `processAttachmentUploads` — moved from `utils/fileUpload.js` since they perform network I/O, not pure transforms. `utils/fileUpload.js` keeps only the pure helpers/constants and no longer re-exports these. |
| `apiMediaJobs.js` | Media generation job tracking + `refineMediaPrompt` / `promptFromMedia` (vision reverse-prompt). |
| `apiDecks.js` | Decks (playing-card / tarot designer) — deck + card CRUD, sample-design style analysis, universe casting + prompt generation (plus its SSE progress URL), single/batch card renders. |
diff --git a/client/src/services/apiLoraTraining.js b/client/src/services/apiLoraTraining.js
index 1d20107806..b875276630 100644
--- a/client/src/services/apiLoraTraining.js
+++ b/client/src/services/apiLoraTraining.js
@@ -1,148 +1,9 @@
-/**
- * LoRA training API — datasets (/api/lora-datasets), training
- * runs (/api/lora-training), and the character→LoRA link lookup
- * (/api/loras/by-character).
- */
-
+/** Retained checkpoint inspection and character-to-inference-adapter lookup. */
import { request } from './apiCore.js';
-// ---- Datasets ----
-
-export const listLoraDatasets = (filters = {}, options = {}) => {
- const params = new URLSearchParams();
- for (const key of ['universeId', 'entryKind', 'entryId', 'ingredientId']) {
- if (filters[key]) params.set(key, filters[key]);
- }
- const qs = params.toString();
- return request(`/lora-datasets${qs ? `?${qs}` : ''}`, options);
-};
-
-export const createLoraDataset = ({
- universeId, entryKind = 'characters', entryId, triggerWord,
-}, { silent = false } = {}) =>
- request('/lora-datasets', {
- method: 'POST',
- body: JSON.stringify({
- universeId, entryKind, entryId, ...(triggerWord ? { triggerWord } : {}),
- }),
- silent,
- });
-
-export const getLoraDataset = (id) => request(`/lora-datasets/${id}`);
-
-// Server-derived variation axes (expressions/outfits for characters; lighting/
-// settings for objects & places) — seeds the generate-batch override chips.
-export const getLoraDatasetVariationAxes = (id, options = {}) =>
- request(`/lora-datasets/${id}/variation-axes`, options);
-
-export const patchLoraDataset = (id, patch) =>
- request(`/lora-datasets/${id}`, { method: 'PATCH', body: JSON.stringify(patch) });
-
-export const deleteLoraDataset = (id) => request(`/lora-datasets/${id}`, { method: 'DELETE' });
-
-// `files` is a FileList/array — field names image1…image10 per the route.
-export const uploadLoraDatasetImages = (id, files) => {
- const form = new FormData();
- [...files].slice(0, 10).forEach((file, i) => form.append(`image${i + 1}`, file));
- return request(`/lora-datasets/${id}/images`, { method: 'POST', body: form });
-};
-
-// Import existing gallery images (basenames) into the dataset.
-export const importLoraDatasetGalleryImages = (id, filenames) =>
- request(`/lora-datasets/${id}/import-gallery`, {
- method: 'POST', body: JSON.stringify({ filenames }),
- });
-
-export const generateLoraDatasetImages = (id, options = {}) =>
- request(`/lora-datasets/${id}/generate`, { method: 'POST', body: JSON.stringify(options) });
-
-// `useVision` (default true) lets a vision model propose a bounding box per
-// figure; the fixed cols×rows grid is the fallback (and what `useVision: false`
-// forces). When omitted, the server auto-resolves a vision model.
-export const sliceLoraDatasetRefSheet = (id, { variant, cols, rows, useVision } = {}) =>
- request(`/lora-datasets/${id}/slice-reference-sheet`, {
- method: 'POST',
- body: JSON.stringify({
- ...(variant ? { variant } : {}),
- ...(cols ? { cols } : {}),
- ...(rows ? { rows } : {}),
- ...(useVision === false ? { useVision: false } : {}),
- }),
- });
-
-export const startLoraCaptionRun = (id, options = {}) =>
- request(`/lora-datasets/${id}/caption`, { method: 'POST', body: JSON.stringify(options) });
-
-// Run-scoped cancel: the server rejects (404) a run id that is not the dataset's
-// current caption run. The terminal `canceled` frame follows over SSE.
-export const cancelLoraCaptionRun = (id, runId, options = {}) =>
- request(`/lora-datasets/${id}/caption-runs/${runId}/cancel`, { method: 'POST', silent: options.silent });
-
-export const updateLoraDatasetImageCaption = (id, imageId, caption) =>
- request(`/lora-datasets/${id}/images/${imageId}`, {
- method: 'PATCH', body: JSON.stringify({ caption }),
- });
-
-export const deleteLoraDatasetImage = (id, imageId) =>
- request(`/lora-datasets/${id}/images/${imageId}`, { method: 'DELETE' });
-
-// Strip the identity fragments shared across most captions so the trigger token
-// learns the character (issue #1320). Returns { dataset, removedFragments, updatedImages }.
-export const stripLoraDatasetSharedCaptionFragments = (id) =>
- request(`/lora-datasets/${id}/strip-shared-fragments`, { method: 'POST' });
-
-// ---- Training runs ----
-
-export const getLoraTrainingStatus = (options = {}) => request('/lora-training/status', options);
-
-export const startLoraTrainingRun = ({ datasetId, baseModelId, name, params, acknowledgeCaptionLeak }, options = {}) =>
- request('/lora-training/runs', {
- method: 'POST',
- silent: options.silent,
- body: JSON.stringify({
- datasetId,
- baseModelId,
- ...(name ? { name } : {}),
- ...(params ? { params } : {}),
- ...(acknowledgeCaptionLeak ? { acknowledgeCaptionLeak: true } : {}),
- }),
- });
-
-export const listLoraTrainingRuns = (filters = {}, options = {}) => {
- const params = new URLSearchParams();
- for (const key of ['status', 'characterId', 'datasetId', 'limit']) {
- if (filters[key]) params.set(key, filters[key]);
- }
- const qs = params.toString();
- return request(`/lora-training/runs${qs ? `?${qs}` : ''}`, options);
-};
-
-export const cancelLoraTrainingRun = (runId) =>
- request(`/lora-training/runs/${runId}/cancel`, { method: 'POST' });
-
-// Resume a failed/canceled run from its latest checkpoint — continues the same
-// run (new job, same run id) so the trainer picks up its saved optimizer state.
-export const resumeLoraTrainingRun = (runId, { silent = false } = {}) =>
- request(`/lora-training/runs/${runId}/resume`, { method: 'POST', silent });
-
-// Checkpoints for a finished run — each with step, loss, preview thumbnail,
-// and which one is currently deployed. Drives the manual checkpoint picker.
export const listLoraTrainingCheckpoints = (runId, options = {}) =>
request(`/lora-training/runs/${runId}/checkpoints`, options);
-// Mid-training sample timeline (step + thumbnail URL) — seeds the live
-// progress gallery so a reload mid-run shows every sample rendered so far.
-export const listLoraTrainingSamples = (runId) =>
- request(`/lora-training/runs/${runId}/samples`);
-
-// Promote a checkpoint to be the deployed LoRA (re-extracts that step's
-// adapter and overwrites the run's registered .safetensors in place).
-export const promoteLoraTrainingCheckpoint = (runId, step, { silent = false } = {}) =>
- request(`/lora-training/runs/${runId}/promote-checkpoint`, {
- method: 'POST', body: JSON.stringify({ step }), silent,
- });
-
-// ---- Character → trained-LoRA link ----
export const getCharacterLoras = ({ entryId, ingredientId } = {}, { silent = false } = {}) => {
const params = new URLSearchParams();
diff --git a/docs/API.md b/docs/API.md
index 9757762db7..2397061091 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -819,7 +819,9 @@ Every mounted API prefix (see `server/index.js` for the authoritative list). Dom
| `/api/conflict-journal` | Sync conflict journal |
| `/api/importer` | Story importer |
| `/api/story-builder` | Story Builder |
-| `/api/loras`, `/api/lora-datasets`, `/api/lora-training` | LoRA management/training |
+| `/api/loras` | Supported inference adapter management |
+| `/api/lora-datasets`, `/api/lora-training` | Retained read/cancel/checkpoint operations; training launch/resume and dataset mutation return 410 |
+| `/api/data-maintenance` | Operator-authorized retired-data inventory, fresh preview and confirmed allowlisted removal; see [Data Maintenance](DATA_MAINTENANCE.md) |
| `/sdapi/v1` | AUTOMATIC1111-compatible image generation surface (gated by `settings.imageGen.expose.a1111`) |
| `/api/openclaw` | OpenClaw operator chat |
| `/api/rounds` | Rounds (music + Morse training) |
diff --git a/docs/DATA_MAINTENANCE.md b/docs/DATA_MAINTENANCE.md
new file mode 100644
index 0000000000..f334f16b7e
--- /dev/null
+++ b/docs/DATA_MAINTENANCE.md
@@ -0,0 +1,72 @@
+# Data Maintenance
+
+Open **Settings → Data Maintenance** (also available through the command palette
+and voice navigation) to inspect storage retained from retired features. Refresh
+measures current files. A failed or unsafe inventory is unavailable, never shown
+as zero bytes. The registry currently contains only LoRA retirement operations;
+archive retention and whole-checkout cleanup are separate workflows.
+
+LoRA training is retired. New training, dataset generation/captioning, resume and
+media-queue retries are refused. Old training URLs lead to Data Maintenance.
+Use the reference-image inputs supported by your chosen image model for new work.
+PortOS does not convert a training dataset or checkpoint into a reference-image
+workflow automatically, and the page offers no unsupported Upgrade action.
+
+## Retained storage
+
+- `data/lora-datasets`: unique source images, captions and dataset metadata.
+- `data/training-runs`: mixed storage, including potentially unique optimizer
+ checkpoints/adapters, training logs, caches and sample previews. Treat the
+ category as unique data rather than assuming every checkpoint is reproducible.
+- `data/loras`: installed adapters, including trained and imported adapters.
+ These remain supported inference inputs. Use **Models → LoRAs** to manage them.
+- Shared inference environments (mflux, MLX, torch/diffusers), media models and
+ Hugging Face caches remain supported. Retirement does not uninstall them.
+
+Saved training configuration and PostgreSQL history remain readable. No format
+migration, boot purge or automatic configuration rewrite is required. A queued
+trainer is retired without spawning; the run records the retirement. A verified
+trainer that already survived a restart is observed to completion without starting
+a new trainer. Its outputs are finalized normally. Automatic stall resumes are
+disabled. Existing cancellation controls remain available in **Active Queues**.
+A detached trainer whose queue record is missing is also observed through completion,
+and blocks maintenance until it exits. The retained run cancel endpoint
+(`POST /api/lora-training/runs/:id/cancel`) can cancel that observer-owned process;
+upgrade never kills it merely because the queue record is absent.
+
+## Explicit removal
+
+Copy unique files you want to keep and verify your own backup coverage first.
+Backup settings, including the existing LoRA Time Machine exclusion, are unchanged.
+The page is not a backup system.
+
+1. Preview a supported removal. The server allowlist determines its exact category
+ and path; callers cannot supply filesystem paths. The preview reports files,
+ measured size and the consequence of removing that category.
+2. Cancel to leave everything intact, or type the exact confirmation shown.
+3. Confirm within five minutes. Changed inventory, symlinks, an unreadable tree,
+ outstanding captioning, a surviving trainer or queued/running media work refuses
+ removal. Refresh and preview again after resolving the reason.
+
+Removal is permanent. There is no fake recovery/archive promise. A short admission
+gate prevents media submissions racing the removal. The server serializes removals,
+revalidates file identity, detaches the inspected directory and checks again before
+removing it under backup publication admission. Duplicate submissions with the same
+preview return the completed result while that preview remains valid. After a
+server restart, a new preview is required.
+
+On a filesystem failure, the UI reports an incomplete removal instead of success.
+The error names the category and any detached `.maintenance-…` recovery directory;
+inspect those locations before retrying. A partial filesystem failure can mean
+some files were already removed. No automatic retry or boot cleanup purges the
+remaining tree. PostgreSQL run history and installed inference adapters are never
+removed by these operations.
+
+## Extending the page
+
+`server/services/dataMaintenance.js` owns the small operation registry. A new
+operation must specify its fixed path, data category, preservation warning and
+actual supported action. It must account for writers and shared consumers, reuse
+fresh-preview/confirmation admission, and add a public-boundary synthetic fixture
+regression. Do not add arbitrary paths, speculative migrations, shared-runtime
+uninstallation or archive-cleanup operations to the registry.
diff --git a/docs/README.md b/docs/README.md
index 65580b79b4..a94efb704e 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -118,3 +118,5 @@ Comms & voice: [beeper](./features/beeper.md) · [messages-browser-send](./featu
- [Tool-free model delegation](SANDBOX_DELEGATION.md) — approved API workers, context packets, and advisory fidelity evaluation for Persistent Mind.
- [Runway model catalog](RUNWAY_MODELS.md) — supported operations, public schema source, caching, refresh and workflow limits.
+
+- [Data Maintenance](DATA_MAINTENANCE.md) — retained retired-feature storage, LoRA training retirement and explicit removal.
diff --git a/docs/STORAGE.md b/docs/STORAGE.md
index 0942854454..13691530fb 100644
--- a/docs/STORAGE.md
+++ b/docs/STORAGE.md
@@ -1284,3 +1284,5 @@ This also identifies additions made before a crash prevented the final journal
update. Never restore an old `usage.json` over newer usage to undo this import.
No migration or seed is required: this is a new optional ledger, absent until a
live completion or an explicit import publishes its first receipt.
+
+LoRA training is retired; its existing datasets, run artifacts, configuration and PostgreSQL history remain intact. See [Data Maintenance](DATA_MAINTENANCE.md) for inventory and explicit removal. Installed adapters and shared inference runtimes remain supported.
diff --git a/scripts/setup-image-video.sh b/scripts/setup-image-video.sh
index 5774a00f28..57a778bc93 100755
--- a/scripts/setup-image-video.sh
+++ b/scripts/setup-image-video.sh
@@ -170,11 +170,7 @@ INSTALL_VIDEO="${INSTALL_VIDEO:-$DEFAULT_INSTALL_VIDEO}"
if [[ "$INSTALL_MFLUX" == "1" ]]; then
# Install via pip --user so we don't pollute the system or require a venv.
# mflux comes with the mflux-generate CLI which the local image backend
- # spawns directly. Pin >=0.17: that's the first release with FLUX.2 LoRA
- # training (`mflux-train --config`, `flux2-klein-*` base models). Older
- # mflux (0.12.x) wants `--train-config` and has no flux2 models, so its
- # trainer dies with a bare "exited with code 2" — see train_mflux_lora.py.
- #
+ # spawns directly. The shared inference runtime remains supported.
echo "📦 Installing image generation packages (mflux + deps)..."
# VALIDATED TRIO (issue #1329) — pin the MLX/Metal backend, do NOT leave it a
# floor. The original three M5 Max GPU-watchdog kernel panics
@@ -192,7 +188,7 @@ if [[ "$INSTALL_MFLUX" == "1" ]]; then
MLX_PIN='mlx==0.31.2'
MLX_METAL_PIN='mlx-metal==0.31.2'
# mflux/mlx need Python 3.10+. The historical layout installs mflux with
- # `pip install --user` against $PYTHON_BIN, and PortOS finds `mflux-train`
+ # `pip install --user` against $PYTHON_BIN, and PortOS discovers the shared console scripts
# beside that interpreter. But when $PYTHON_BIN is too old (e.g. a system
# python3.8) or externally-managed (PEP 668) so the --user install fails or
# can't be imported, fall back to a dedicated venv at ~/.portos/venv-mflux —
diff --git a/scripts/train_flux2_lora.py b/scripts/train_flux2_lora.py
deleted file mode 100644
index 94edb57702..0000000000
--- a/scripts/train_flux2_lora.py
+++ /dev/null
@@ -1,459 +0,0 @@
-#!/usr/bin/env python3
-"""FLUX.2 Klein LoRA trainer — vendored, CUDA/CPU (PortOS).
-
-Trains a character LoRA on the bf16 Klein base (NEVER the SDNQ/int8
-quantized inference repos — no useful gradients through quant layers).
-The trained adapter loads onto quantized inference pipelines of the same
-size variant because the transformer hidden dims match.
-
-Two phases keep peak memory survivable on a consumer CUDA card (a 24 GB 3090
-has no headroom to spare). The split was written for Apple Silicon's unified
-memory, but `main()` now refuses MPS outright — see the guard below — so CUDA
-and CPU are the runtimes this actually serves:
-
- 1. STAGE:precompute-latents — encode every dataset image to VAE latents
- and every caption to Qwen3 text embeddings ONCE, then free the text
- encoder and move the VAE off-device. Only the transformer stays
- resident for the training loop.
- 2. STAGE:training — bf16 transformer + peft LoRA adapter on the attention
- projections, gradient checkpointing, AdamW, flow-matching loss
- (target = noise - latents, logit-normal timestep sampling — matches
- the FLUX family training recipes).
-
-Line protocol (parsed by server/services/loraTraining/progress.js):
- STEP:<cur>:<total>:<loss> CHECKPOINT:<path>:<step> SAMPLE:<path>:<step>
- STAGE:<name>[...heartbeat] STATUS:<msg> USER_ERROR:<kind>:<msg>
- RESULT:{"adapter_path": ..., "steps": N, "final_loss": F}
-
-SIGTERM → finish the current step, save a cancel checkpoint, exit 143.
-"""
-
-import argparse
-import gc
-import json
-import os
-import signal
-import sys
-from pathlib import Path
-
-os.environ.setdefault("PYTORCH_MPS_FAST_MATH", "1")
-
-import numpy as np
-import torch
-from PIL import Image
-
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-from _runner_common import ( # noqa: E402
- empty_device_cache,
- heartbeat,
- install_hf_error_handler,
- pick_device,
-)
-
-STOP_REQUESTED = False
-
-
-def _on_sigterm(_sig, _frame):
- global STOP_REQUESTED
- STOP_REQUESTED = True
- print("STATUS:cancel requested — finishing current step and checkpointing", flush=True)
-
-
-signal.signal(signal.SIGTERM, _on_sigterm)
-
-
-def log(msg: str) -> None:
- print(msg, flush=True)
-
-
-def parse_args():
- p = argparse.ArgumentParser(description="FLUX.2 Klein LoRA trainer")
- p.add_argument("--model-repo", required=True, help="bf16 base repo (black-forest-labs/FLUX.2-klein-4B|9B)")
- p.add_argument("--manifest", required=True, help="JSON manifest: { triggerWord, images: [{ path, caption }] }")
- p.add_argument("--output-dir", required=True)
- p.add_argument("--trigger-word", required=True)
- p.add_argument("--steps", type=int, default=1000)
- p.add_argument("--rank", type=int, default=16)
- p.add_argument("--lr", type=float, default=1e-4)
- p.add_argument("--resolution", type=int, default=512)
- p.add_argument("--checkpoint-every", type=int, default=250)
- p.add_argument("--sample-every", type=int, default=250)
- p.add_argument("--sample-prompt", default=None)
- p.add_argument("--seed", type=int, default=42)
- p.add_argument("--device", default="auto")
- p.add_argument("--resume-from", default=None,
- help="checkpoint dir to resume from (restores adapter + AdamW optimizer state + step offset from optimizer.pt)")
- return p.parse_args()
-
-
-def load_image_tensor(path: str, resolution: int) -> torch.Tensor:
- """Open → RGB → shortest-side resize → center crop → [-1, 1] CHW tensor."""
- img = Image.open(path).convert("RGB")
- w, h = img.size
- scale = resolution / min(w, h)
- img = img.resize((max(resolution, round(w * scale)), max(resolution, round(h * scale))), Image.LANCZOS)
- w, h = img.size
- left, top = (w - resolution) // 2, (h - resolution) // 2
- img = img.crop((left, top, left + resolution, top + resolution))
- t = torch.from_numpy(np.asarray(img)).float() / 127.5 - 1.0
- return t.permute(2, 0, 1).unsqueeze(0) # (1, 3, H, W)
-
-
-def lora_target_modules(transformer) -> list:
- """Collect attention-projection Linear module name suffixes that actually
- exist in this transformer — robust to naming drift across diffusers
- versions (to_q vs add_q_proj etc.)."""
- wanted = ("to_q", "to_k", "to_v", "to_out.0", "add_q_proj", "add_k_proj", "add_v_proj", "to_add_out")
- found = set()
- for name, module in transformer.named_modules():
- if not isinstance(module, torch.nn.Linear):
- continue
- if ".attn" not in name and "attention" not in name:
- continue
- for suffix in wanted:
- if name.endswith(suffix):
- found.add(suffix)
- if not found:
- raise RuntimeError("No attention projection Linears found to target with LoRA")
- return sorted(found)
-
-
-def save_checkpoint(pipe_cls, transformer, optimizer, out_dir: Path, step: int) -> Path:
- """Persist the adapter for inference/promotion AND a self-contained resume
- bundle (`optimizer.pt`) carrying the AdamW state, the PEFT adapter state, and
- the step counter — so `--resume-from` continues mid-run (correct optimizer
- momentum + starting-step offset) instead of warm-starting into a fresh
- `args.steps`-long loop (which would over-train and re-collide step numbers).
-
- The adapter goes into the bundle as its PEFT state dict (not via the diffusers
- loader) so resume can restore it into the SAME `default` adapter with
- `set_peft_model_state_dict` — round-tripping `get_peft_model_state_dict`
- exactly. Loading the `pytorch_lora_weights.safetensors` back through
- `pipe.load_lora_weights()` would instead inject a SECOND adapter
- (`get_adapter_name` returns `default_<n>` when one already exists), leaving the
- trained weights in `default_1` while export reads `default` — silently wrong.
- Saved on CPU; `load_state_dict` re-casts onto the fp32 on-device params."""
- from peft.utils import get_peft_model_state_dict
-
- ckpt_dir = out_dir / "checkpoints" / f"step-{step:06d}"
- ckpt_dir.mkdir(parents=True, exist_ok=True)
- state = get_peft_model_state_dict(transformer)
- pipe_cls.save_lora_weights(str(ckpt_dir), transformer_lora_layers=state)
- torch.save(
- {"optimizer": optimizer.state_dict(), "adapter": state, "step": step},
- ckpt_dir / "optimizer.pt",
- )
- (ckpt_dir / "state.json").write_text(json.dumps({"step": step}))
- return ckpt_dir
-
-
-def render_sample(pipe, transformer, embeds, text_ids, resolution, device, dtype, seed, out_path: Path):
- """Quick 8-step Euler flow-matching sample with the in-training adapter.
- Best-effort — callers wrap in try/except; a failed sample never fails
- the run."""
- latent_h = 2 * (resolution // (pipe.vae_scale_factor * 2))
- latent_w = 2 * (resolution // (pipe.vae_scale_factor * 2))
- channels = pipe.transformer.config.in_channels // 4 * 4 # packed channel count
- gen = torch.Generator(device="cpu").manual_seed(seed)
- raw = torch.randn((1, channels, latent_h // 2, latent_w // 2), generator=gen).to(device, dtype)
- latent_ids = pipe._prepare_latent_ids(raw).to(device)
- latents = pipe._pack_latents(raw)
- sigmas = torch.linspace(1.0, 0.0, 9)
- for i in range(8):
- sigma = sigmas[i]
- t = sigma.expand(1).to(device, dtype)
- pred = transformer(
- hidden_states=latents.to(dtype),
- timestep=t,
- guidance=None,
- encoder_hidden_states=embeds,
- txt_ids=text_ids,
- img_ids=latent_ids,
- return_dict=False,
- )[0]
- latents = latents - (sigmas[i] - sigmas[i + 1]).to(device, dtype) * pred
- unpacked = pipe._unpack_latents_with_ids(latents, latent_ids, latent_h // 2, latent_w // 2)
- bn_mean = pipe.vae.bn.running_mean.view(1, -1, 1, 1).to(unpacked.device, unpacked.dtype)
- bn_std = torch.sqrt(pipe.vae.bn.running_var.view(1, -1, 1, 1) + pipe.vae.config.batch_norm_eps).to(
- unpacked.device, unpacked.dtype
- )
- unpacked = unpacked * bn_std + bn_mean
- unpacked = pipe._unpatchify_latents(unpacked)
- # Decode on the training device, not wherever precompute parked the VAE.
- # Phase 1 deliberately leaves the VAE on CPU to keep it out of the training
- # footprint, but the pipeline is bf16 on an accelerator — and x86 has no
- # fast bf16 convolution path, so decoding there reads as a hang (a single
- # 512-px decode ran >18 min on CUDA/Windows before this moved it back).
- # The VAE is ~0.17 GB against a ~8 GB resident transformer, so borrowing the
- # device for one decode is cheap.
- # The move goes INSIDE the try: `.to()` walks parameters one at a time, so a
- # throw partway leaves the VAE straddling both devices. Outside the try the
- # `finally` would never run, and the next sample would read the now-`cuda`
- # resting place as `vae_home` — silently latching it on-device for the rest
- # of the run, on the card that just proved it has no headroom to spare.
- vae_home = next(pipe.vae.parameters()).device
- try:
- pipe.vae.to(device)
- image = pipe.vae.decode(unpacked.to(device, pipe.vae.dtype), return_dict=False)[0]
- finally:
- pipe.vae.to(vae_home)
- image = pipe.image_processor.postprocess(image, output_type="pil")[0]
- image.save(out_path)
-
-
-@install_hf_error_handler
-def main():
- args = parse_args()
- from diffusers import Flux2KleinPipeline
- from peft import LoraConfig
- from peft.utils import get_peft_model_state_dict
-
- device = pick_device(args.device)
- # Apple-Silicon guard: this torch/diffusers trainer can't complete a LoRA
- # backward on Metal. PyTorch's MPS backend has no `linear_backward` for the
- # FLUX.2 transformer's Linear layers (raises "mps_linear_backward:
- # unsupported weights data type" for both bf16 AND fp32 at the first
- # loss.backward()). The preferred Apple-Silicon runtime is mflux (MLX-native,
- # trains bf16 directly); this torch path is the fallback for CUDA/CPU boxes.
- # Fail fast BEFORE loading the ~16 GB base + minutes of latent precompute
- # rather than crashing at the first optimizer step. A forced `--device cpu`
- # is still allowed (fp32 CPU backward works, just slow).
- if device == "mps":
- print(
- "USER_ERROR:TRAINING_UNSUPPORTED_DEVICE:the torch FLUX.2 trainer can't train on "
- "Apple Silicon (Metal/MPS) — PyTorch has no MPS linear_backward for these layers. "
- "Use the mflux runtime instead: install mflux ≥0.17 via scripts/setup-image-video.sh. "
- "(Force --device cpu to train on CPU, but that is impractically slow.)",
- file=sys.stderr, flush=True,
- )
- sys.exit(1)
- dtype = torch.bfloat16 if device != "cpu" else torch.float32
- torch.manual_seed(args.seed)
-
- out_dir = Path(args.output_dir)
- (out_dir / "samples").mkdir(parents=True, exist_ok=True)
- (out_dir / "checkpoints").mkdir(parents=True, exist_ok=True)
-
- manifest = json.loads(Path(args.manifest).read_text())
- images = manifest.get("images") or []
- if not images:
- print("USER_ERROR:DATASET_ERROR:manifest contains no images", file=sys.stderr, flush=True)
- sys.exit(2)
-
- log("STAGE:load-pipeline")
- with heartbeat("load-pipeline"):
- pipe = Flux2KleinPipeline.from_pretrained(args.model_repo, torch_dtype=dtype)
-
- # Nothing in this pipeline is trainable until `add_adapter` introduces the
- # LoRA below — say so once, here, rather than leaving it to whatever
- # `from_pretrained` happened to return. This is the root-cause form of the
- # precompute leak described further down: `requires_grad` is a property of
- # the module, so clearing it makes the whole class of "inference forward
- # silently built an autograd graph" unreachable from EVERY call site,
- # present and future — where a `torch.no_grad()` block only protects the one
- # region someone remembered to wrap. Both encoders take grad-free inputs
- # (token ids; a tensor built from numpy), so this alone is sufficient; the
- # `no_grad` below is kept as the belt to this pair of braces.
- pipe.text_encoder.requires_grad_(False)
- pipe.vae.requires_grad_(False)
- pipe.transformer.requires_grad_(False)
-
- # ---- Phase 1: precompute text embeddings + image latents ----
- log("STAGE:precompute-latents")
- sample_prompt = args.sample_prompt or f"{args.trigger_word} portrait, neutral background"
- examples = []
- # `no_grad` is load-bearing here, not a micro-optimization — it is the
- # second half of the `requires_grad_(False)` pair above. Left unguarded with
- # trainable params, every encode builds and keeps a full autograd graph, and
- # the `.to("cpu")` cache copy inherits that `grad_fn` and pins it on-device:
- # ~2.6 GB of Qwen3 activations PER UNIQUE CAPTION on FLUX.2-klein-4B. A
- # 128 GB unified-memory Mac absorbs that silently; a 24 GB CUDA card spills
- # into driver shared memory (encode slows from 0.09 s to minutes) and then
- # aborts with OOM at `transformer.to(device)` below. Keeping both is
- # deliberate: `no_grad` additionally suppresses graph-building when an INPUT
- # carries a `grad_fn`, which clearing the params cannot.
- with torch.no_grad(), heartbeat("precompute-latents"):
- pipe.text_encoder.to(device)
- embed_cache = {}
- for entry in images + [{"caption": sample_prompt, "path": None}]:
- caption = entry["caption"]
- if caption not in embed_cache:
- embeds, text_ids = pipe.encode_prompt(caption, device=device)
- embed_cache[caption] = (embeds.to("cpu"), text_ids.to("cpu"))
- sample_embeds = embed_cache[sample_prompt]
- # Free the text encoder — the make-or-break memory move on every
- # backend: its ~8 GB has to be gone before the transformer moves on.
- pipe.text_encoder.to("cpu")
- pipe.text_encoder = None
- gc.collect()
- empty_device_cache(device)
-
- pipe.vae.to(device)
- gen = torch.Generator(device="cpu").manual_seed(args.seed)
- for i, entry in enumerate(images):
- pixel = load_image_tensor(entry["path"], args.resolution).to(device, pipe.vae.dtype)
- latents = pipe._encode_vae_image(pixel, generator=gen) # (1, C, H, W) normalized
- latent_ids = pipe._prepare_latent_ids(latents)
- packed = pipe._pack_latents(latents)
- embeds, text_ids = embed_cache[entry["caption"]]
- examples.append({
- "packed": packed.to("cpu"),
- "latent_ids": latent_ids.to("cpu"),
- "embeds": embeds,
- "text_ids": text_ids,
- })
- log(f"STATUS:encoded {i + 1}/{len(images)} dataset images")
- # VAE only needed again for samples — keep it, but off-device.
- pipe.vae.to("cpu")
- gc.collect()
- empty_device_cache(device)
-
- # ---- Phase 2: training ----
- log("STAGE:training")
- transformer = pipe.transformer # already frozen at load, with the encoders
- targets = lora_target_modules(transformer)
- log(f"STATUS:LoRA targets: {', '.join(targets)} (rank {args.rank})")
- # Always create the single trainable `default` adapter (gaussian init). On
- # resume its weights are overwritten in-place below from the checkpoint's
- # PEFT state dict — NOT via pipe.load_lora_weights(), which would inject a
- # second `default_<n>` adapter and leave training/export out of sync.
- transformer.add_adapter(LoraConfig(
- r=args.rank,
- lora_alpha=args.rank,
- init_lora_weights="gaussian",
- target_modules=targets,
- ))
- transformer.enable_gradient_checkpointing()
- transformer.to(device)
- transformer.train()
-
- trainable = [p for p in transformer.parameters() if p.requires_grad]
- # LoRA params train in fp32 for AdamW stability even when the base is bf16.
- for p in trainable:
- p.data = p.data.to(torch.float32)
- optimizer = torch.optim.AdamW(trainable, lr=args.lr, weight_decay=0.01)
-
- # Resume: restore the adapter weights, AdamW state, and a starting-step
- # offset from the checkpoint's optimizer.pt bundle, so the loop continues
- # toward the ORIGINAL total (range(start_step + 1, args.steps + 1)) with warm
- # momentum instead of re-running a fresh args.steps from a random adapter.
- # The adapter is loaded into the existing `default` adapter via
- # set_peft_model_state_dict (in-place — param identities, and therefore the
- # optimizer state alignment, are preserved). The server only hands us
- # checkpoints that carry optimizer.pt (resolveLatestCheckpointArtifact gates
- # on it); a checkpoint that predates optimizer-state resume is a hard error
- # rather than a silent cold-start onto the random gaussian adapter. (RNG
- # state is intentionally not restored; the per-step noise/timestep sequence
- # restarts from the seed, immaterial to quality — the load-bearing fix is
- # adapter + optimizer + step.)
- start_step = 0
- if args.resume_from:
- from peft.utils import set_peft_model_state_dict
-
- opt_file = Path(args.resume_from) / "optimizer.pt"
- if not opt_file.exists():
- print(
- f"USER_ERROR:TRAINING_FAILED:resume checkpoint has no optimizer state ({opt_file}) — "
- "it predates optimizer-state resume; start a fresh run",
- file=sys.stderr, flush=True,
- )
- sys.exit(1)
- ckpt = torch.load(opt_file, map_location="cpu")
- set_peft_model_state_dict(transformer, ckpt["adapter"], adapter_name="default")
- optimizer.load_state_dict(ckpt["optimizer"])
- start_step = int(ckpt.get("step", 0))
- log(f"STATUS:resumed adapter + optimizer state — continuing from step {start_step}")
- if start_step >= args.steps:
- log(f"STATUS:resume point (step {start_step}) already at/past target {args.steps} — nothing to train")
-
- order = list(range(len(examples)))
- rng = torch.Generator().manual_seed(args.seed)
- final_loss = None
- last_checkpoint = None
-
- for step in range(start_step + 1, args.steps + 1):
- if (step - 1) % len(order) == 0:
- order = torch.randperm(len(examples), generator=rng).tolist()
- ex = examples[order[(step - 1) % len(order)]]
- packed = ex["packed"].to(device, dtype)
- latent_ids = ex["latent_ids"].to(device)
- embeds = ex["embeds"].to(device, dtype)
- text_ids = ex["text_ids"].to(device)
-
- noise = torch.randn_like(packed)
- # Logit-normal timestep sampling (FLUX training recipe).
- sigma = torch.sigmoid(torch.randn((1,), generator=rng)).to(device, dtype)
- noisy = (1.0 - sigma) * packed + sigma * noise
- target = noise - packed
-
- pred = transformer(
- hidden_states=noisy,
- timestep=sigma.expand(1),
- guidance=None,
- encoder_hidden_states=embeds,
- txt_ids=text_ids,
- img_ids=latent_ids,
- return_dict=False,
- )[0]
- loss = torch.nn.functional.mse_loss(pred.float(), target.float())
- loss.backward()
- torch.nn.utils.clip_grad_norm_(trainable, 1.0)
- optimizer.step()
- optimizer.zero_grad(set_to_none=True)
-
- final_loss = float(loss.detach().item())
- log(f"STEP:{step}:{args.steps}:{final_loss:.4f}")
-
- if STOP_REQUESTED:
- ckpt = save_checkpoint(Flux2KleinPipeline, transformer, optimizer, out_dir, step)
- log(f"CHECKPOINT:{ckpt}:{step}")
- log("STATUS:canceled-checkpoint-saved")
- os._exit(143)
-
- if args.checkpoint_every > 0 and step % args.checkpoint_every == 0 and step < args.steps:
- ckpt = save_checkpoint(Flux2KleinPipeline, transformer, optimizer, out_dir, step)
- last_checkpoint = ckpt
- log(f"CHECKPOINT:{ckpt}:{step}")
-
- if args.sample_every > 0 and step % args.sample_every == 0:
- sample_path = out_dir / "samples" / f"step-{step:06d}.png"
- transformer.eval()
- try:
- render_sample(
- pipe, transformer, sample_embeds[0].to(device, dtype), sample_embeds[1].to(device),
- args.resolution, device, dtype, args.seed, sample_path,
- )
- log(f"SAMPLE:{sample_path}:{step}")
- except Exception as err: # noqa: BLE001 — samples are best-effort, never fail the run
- log(f"STATUS:sample render failed (training continues): {err}")
- finally:
- transformer.train()
- empty_device_cache(device)
-
- # ---- Finalize ----
- adapter_dir = out_dir / "adapter"
- adapter_dir.mkdir(parents=True, exist_ok=True)
- state = get_peft_model_state_dict(transformer)
- Flux2KleinPipeline.save_lora_weights(str(adapter_dir), transformer_lora_layers=state)
- adapter_path = adapter_dir / "pytorch_lora_weights.safetensors"
- if not adapter_path.exists():
- candidates = list(adapter_dir.glob("*.safetensors"))
- if not candidates:
- print("USER_ERROR:TRAINING_FAILED:adapter save produced no .safetensors", file=sys.stderr, flush=True)
- sys.exit(1)
- adapter_path = candidates[0]
- log("RESULT:" + json.dumps({
- "adapter_path": str(adapter_path),
- "steps": args.steps,
- "final_loss": final_loss,
- "last_checkpoint": str(last_checkpoint) if last_checkpoint else None,
- }))
- # Hard-exit defense against teardown hangs (mirrors the video runtimes).
- sys.stdout.flush()
- sys.stderr.flush()
- os._exit(0)
-
-
-if __name__ == "__main__":
- main()
diff --git a/scripts/train_flux2_lora_test.py b/scripts/train_flux2_lora_test.py
deleted file mode 100644
index 1efddd4c04..0000000000
--- a/scripts/train_flux2_lora_test.py
+++ /dev/null
@@ -1,157 +0,0 @@
-#!/usr/bin/env python3
-"""Standalone tests for train_flux2_lora.py's memory-discipline invariants.
-
-Run: python3 scripts/train_flux2_lora_test.py
-Exits non-zero if any check fails. Mirrors the runnable-test style of
-scripts/train_mflux_lora_test.py.
-
-These are AST checks on the source rather than behavioral tests, deliberately:
-exercising the real paths needs torch, a ~16 GB base download and a CUDA card,
-which no test suite can carry. So they pin *structural* invariants only — each
-one chosen because breaking it silently reintroduces a fault that made this
-trainer unusable on a 24 GB card, and because no reasonable refactor breaks it
-without also breaking the invariant. Background and measurements: issue #2786
-and docs/research/2026-08-05-torch-flux2-lora-cuda-validation.md.
-"""
-import ast
-import sys
-from pathlib import Path
-
-# The ✅/❌ prefixes are cp1252-unencodable on a Windows console, which would
-# abort the run before the first result printed. This trainer's whole point is
-# the CUDA path, and that is where Windows lives — so force UTF-8 on stdout.
-if hasattr(sys.stdout, "reconfigure"):
- sys.stdout.reconfigure(encoding="utf-8", errors="replace")
-
-SRC = Path(__file__).resolve().parent / "train_flux2_lora.py"
-TREE = ast.parse(SRC.read_text(encoding="utf-8"))
-
-FAILS = []
-
-
-def check(name, cond):
- print(("✅" if cond else "❌") + " " + name)
- if not cond:
- FAILS.append(name)
-
-
-def func(name):
- """The named top-level function, or exit — a missing function would make
- every check about it vacuously pass."""
- for node in ast.walk(TREE):
- if isinstance(node, ast.FunctionDef) and node.name == name:
- return node
- sys.exit(f"❌ {name}() not found in {SRC.name} — the file was restructured")
-
-
-def calls(node):
- """Every call under `node` as (dotted_name, Call), e.g. pipe.vae.to."""
- out = []
- for n in ast.walk(node):
- if isinstance(n, ast.Call):
- parts, f = [], n.func
- while isinstance(f, ast.Attribute):
- parts.append(f.attr)
- f = f.value
- if isinstance(f, ast.Name):
- parts.append(f.id)
- out.append((".".join(reversed(parts)), n))
- return out
-
-
-def names_in(call):
- """Identifiers passed to `call`, positionally or by keyword — so a check
- isn't defeated by `.to(device=device)` vs `.to(device)`."""
- args = list(call.args) + [k.value for k in call.keywords]
- return {a.id for a in args if isinstance(a, ast.Name)}
-
-
-main = func("main")
-main_calls = calls(main)
-
-# --- 1. inference-only modules are frozen at load ----------------------------
-# The root-cause form of the precompute leak: `requires_grad` is a property of
-# the module, so clearing it makes "an inference forward silently built an
-# autograd graph" unreachable from every call site, not just the wrapped one.
-frozen = {
- name.rsplit(".", 2)[-2]
- for name, c in main_calls
- if name.endswith(".requires_grad_")
- and c.args and isinstance(c.args[0], ast.Constant) and c.args[0].value is False
-}
-for module in ("text_encoder", "vae", "transformer"):
- check(f"main() freezes {module} with requires_grad_(False) at load "
- f"(root-cause guard for the ~2.6 GB/caption leak — issue #2786)",
- module in frozen)
-
-# --- 2. precompute's inference runs under torch.no_grad() --------------------
-# Belt to the braces above. Located by walking every no_grad block in main()
-# rather than by matching one `with` statement's shape, so nesting the guard or
-# extracting precompute into its own block keeps passing.
-no_grad_blocks = [
- n for n in ast.walk(main)
- if isinstance(n, ast.With) and any(
- isinstance(i.context_expr, ast.Call)
- and getattr(i.context_expr.func, "attr", None) == "no_grad"
- for i in n.items
- )
-]
-check("main() has a torch.no_grad() block", bool(no_grad_blocks))
-for fn in ("encode_prompt", "_encode_vae_image"):
- check(f"precompute's {fn} runs under torch.no_grad()",
- any(any(n.endswith(fn) for n, _ in calls(b)) for b in no_grad_blocks))
-
-# --- 3. render_sample decodes on the training device, and always restores ----
-# Precompute parks the VAE on CPU, but the pipeline is bf16 and x86 has no fast
-# bf16 conv path, so decoding there reads as a hang (>18 min for one 512-px
-# decode). The restore must be in a `finally`: the caller treats samples as
-# best-effort and swallows exceptions, so a throw mid-decode would otherwise
-# strand the VAE on-device for the rest of the run.
-rs = func("render_sample")
-decode_try = None
-for node in ast.walk(rs):
- if isinstance(node, ast.Try) and any(
- n.endswith("vae.decode") for n, _ in calls(node)
- ):
- decode_try = node
-
-check("render_sample wraps vae.decode in a try (issue #2786)", decode_try is not None)
-
-if decode_try:
- check("render_sample moves the VAE onto `device` inside that try "
- "(bf16 conv on CPU hangs — issue #2786)",
- any(n.endswith("vae.to") and "device" in names_in(c)
- for n, c in calls(ast.Module(body=decode_try.body, type_ignores=[]))))
- check("render_sample restores the VAE to `vae_home` in the try's finally",
- any(n.endswith("vae.to") and "vae_home" in names_in(c)
- for n, c in calls(ast.Module(body=decode_try.finalbody, type_ignores=[]))))
-
-# --- 4. cache clearing goes through the shared, device-agnostic helper -------
-# The original bug was an MPS-only `torch.mps.empty_cache()`, so a bare clear of
-# EITHER backend is the regression — banning only the MPS spelling would let the
-# mirror image straight back in.
-module_calls = [n for n, _ in calls(TREE)]
-check("no bare torch.mps/cuda.empty_cache anywhere in the module — "
- "clears route through _runner_common.empty_device_cache",
- not ({"torch.mps.empty_cache", "torch.cuda.empty_cache"} & set(module_calls)))
-check("the trainer actually calls empty_device_cache",
- "empty_device_cache" in module_calls)
-imported_from_common = any(
- isinstance(n, ast.ImportFrom) and n.module == "_runner_common"
- and any(a.name == "empty_device_cache" for a in n.names)
- for n in ast.walk(TREE)
-)
-redeclared_locally = any(
- isinstance(n, ast.FunctionDef) and n.name == "empty_device_cache"
- for n in ast.walk(TREE)
-)
-check("empty_device_cache comes from _runner_common, not a local re-declaration",
- imported_from_common and not redeclared_locally)
-
-print()
-if FAILS:
- print(f"❌ {len(FAILS)} check(s) failed:")
- for f in FAILS:
- print(f" - {f}")
- sys.exit(1)
-print("✅ all train_flux2_lora.py invariants hold")
diff --git a/scripts/train_mflux_lora.py b/scripts/train_mflux_lora.py
deleted file mode 100644
index 3a1264dbb3..0000000000
--- a/scripts/train_mflux_lora.py
+++ /dev/null
@@ -1,793 +0,0 @@
-#!/usr/bin/env python3
-"""mflux FLUX.2 LoRA training wrapper (PortOS).
-
-Wraps mflux's own trainer (`mflux-train`, mflux ≥0.17 — FLUX.1 training
-was removed upstream) in a subprocess and translates its native output
-into the PortOS trainer line protocol (see train_flux2_lora.py header /
-server/services/loraTraining/progress.js). We deliberately do NOT import
-mflux's training internals in-process — their API churns across versions;
-the CLI + config JSON is the stable surface (same wrap-don't-import
-strategy as generate_wan22.py).
-
-Verified against mflux 0.17.5:
- - CLI: `mflux-train --config <json>` (resume: `--resume <checkpoint.zip>`).
- - Progress: a tqdm bar over total iterations (epochs × images), written
- with carriage returns — the reader splits on \\r AND \\n.
- - Outputs under the config's checkpoint.output_path:
- checkpoints/NNNNNNN_checkpoint.zip (adapter + optimizer + config)
- preview/NNNNNNN_preview_*.png (when monitoring is enabled)
- loss/loss.html
- - The trained adapter inside each zip is `NNNNNNN_adapter.safetensors`
- with diffusers-style key naming (`transformer.<module>.lora_A.weight`),
- so the extracted file loads directly in diffusers flux2 pipelines.
-
-SIGTERM → forwarded to the mflux child; exit 143 after it stops (mflux
-checkpoints at save_frequency boundaries; mid-interval cancels keep the
-last saved checkpoint).
-
-Segmented training (watchdog-panic mitigation). When `--segment-steps N`
-is passed (>0), the run is broken into N-step segments instead of one
-sustained mflux process: train to a checkpoint boundary, **terminate the
-mflux child so its Metal/GPU context is fully torn down**, sleep
-`--cooldown-sec` to let the GPU clock down + the system settle, then
-`--resume` the newest checkpoint for the next segment. mflux resume reads
-`num_epochs` from the checkpoint (the config value is ignored on resume),
-so each segment continues toward the *same* total — segmentation is
-numerically equivalent to one continuous run, just with periodic
-full-process teardown. This directly attacks the macOS GPU
-watchdog-timeout kernel panics that hard-reboot the machine during long
-sustained mflux runs (see docs/research/2026-06-13-mflux-training-watchdog-panic.md):
-sustained GPU pressure never spans more than one segment, and any
-accumulated driver/thermal state resets between segments. `--segment-steps 0`
-(the default) restores the single-process behavior.
-"""
-
-import argparse
-import atexit
-import json
-import os
-import re
-import shutil
-import signal
-import subprocess
-import sys
-import threading
-import time
-import zipfile
-from collections import deque
-from datetime import datetime
-from pathlib import Path
-
-# macOS GPU-watchdog mitigation (mlx #3267). The Metal/AGX driver kills sustained
-# LoRA training under the GPU watchdog ("Impacting Interactivity" → process kill,
-# and on M5-class silicon escalating to a `watchdogd` kernel panic + hard reboot;
-# see docs/research/2026-06-13-mflux-training-watchdog-panic.md). The MLX
-# maintainer's confirmed workaround is to relax the AGX CDM context-store timeout
-# so a long command buffer isn't treated as a stuck/interactivity-impacting one.
-# It is read by the driver at Metal context creation, so it MUST be set before
-# mlx initializes — we set it here at module load, and since the mflux-train
-# child Popen (below) inherits this process env, the child gets it too. setdefault
-# lets an explicit override (or =0 to disable) still win. NOTE: per mlx #3267 the
-# kill is at the IOGPU layer *above* the process boundary, so process-teardown
-# segmentation alone does NOT prevent it — this env var attacks the actual cause.
-os.environ.setdefault("AGX_RELAX_CDM_CTXSTORE_TIMEOUT", "1")
-
-CHILD = None
-STOP_REQUESTED = False
-
-# How many of the trainer's recent non-progress output lines to retain and
-# replay to stderr on a non-zero exit (so the Node failure classifier sees the
-# real error, not just the exit code). See the loop in main().
-OUTPUT_TAIL_LINES = 40
-
-# Packages whose resolved versions form the LoRA-training stack fingerprint. The
-# macOS GPU watchdog kernel panics (docs/research/2026-06-13-mflux-training-watchdog-panic.md)
-# are version-specific to this trio + mlx-lm, so emitting them at startup makes
-# every run log and any future paniclog self-document the exact stack it ran on
-# (mirrors the video-gen runtime fingerprint — _runner_common.emit_runtime_fingerprint).
-RUNTIME_FINGERPRINT_PACKAGES = ["mflux", "mlx", "mlx-metal", "mlx-lm"]
-
-
-def log(msg: str) -> None:
- print(msg, flush=True)
-
-
-def format_runtime_fingerprint_line(fp: dict) -> str:
- """Render a build_runtime_fingerprint() dict to the one-line STATUS string
- the trainer logs at startup. Pure (no I/O) so it is unit-testable without a
- real mflux/mlx install. Absent packages are simply omitted from `versions`
- by the builder, so they never appear here."""
- versions = fp.get("versions") or {}
- parts = [f"{name} {ver}" for name, ver in versions.items()] or ["versions unavailable"]
- for field in ("chip", "os", "python"):
- val = fp.get(field)
- if val:
- parts.append(f"{field} {val}" if field == "python" else str(val))
- return "STATUS:runtime · " + " · ".join(parts)
-
-
-def emit_runtime_fingerprint() -> dict:
- """Log a one-line runtime fingerprint (mflux/mlx/mlx-metal/mlx-lm versions +
- Apple chip + macOS + python) at trainer startup. Best-effort: a fingerprint
- failure must never abort a training run, so all of it is guarded. Reuses the
- shared _runner_common.build_runtime_fingerprint so the trainer log and the
- video-gen /status probe never drift. Returns the dict it emitted (or {})."""
- try:
- sys.path.insert(0, str(Path(__file__).resolve().parent))
- from _runner_common import build_runtime_fingerprint # noqa: PLC0415
-
- fp = build_runtime_fingerprint("mflux-lora", RUNTIME_FINGERPRINT_PACKAGES)
- log(format_runtime_fingerprint_line(fp))
- return fp
- except Exception as err: # pragma: no cover - defensive; never abort a run
- log(f"STATUS:runtime fingerprint unavailable ({err})")
- return {}
-
-
-def _on_sigterm(_sig, _frame):
- global STOP_REQUESTED
- STOP_REQUESTED = True
- log("STATUS:cancel requested — stopping mflux trainer")
- if CHILD and CHILD.poll() is None:
- CHILD.terminate()
-
-
-signal.signal(signal.SIGTERM, _on_sigterm)
-
-
-def parse_args():
- p = argparse.ArgumentParser(description="mflux LoRA training wrapper")
- p.add_argument("--config", required=True, help="mflux-train config JSON")
- p.add_argument("--output-dir", required=True, help="PortOS run dir (samples/ lives here)")
- p.add_argument("--total-steps", type=int, default=1000,
- help="expected total iterations — segment planning + logging sanity")
- p.add_argument("--resume-checkpoint", default=None, help="checkpoint zip to resume from")
- p.add_argument("--segment-steps", type=int, default=0,
- help="0 = single sustained run; >0 = train in N-step segments, tearing "
- "down the GPU child + cooling down between each (watchdog-panic mitigation)")
- p.add_argument("--cooldown-sec", type=int, default=90,
- help="seconds to idle the GPU between segments (only with --segment-steps>0)")
- return p.parse_args()
-
-
-# Checkpoint zips are named NNNNNNN_checkpoint.zip where NNNNNNN is the
-# trainer's cumulative iteration count — the canonical "where did we get to".
-CKPT_RE = re.compile(r"(\d+)_checkpoint\.zip$")
-
-
-def _checkpoint_step(path) -> int:
- if not path:
- return 0
- m = CKPT_RE.search(Path(path).name)
- return int(m.group(1)) if m else 0
-
-
-def newest_checkpoint(configured_output: Path):
- """Newest checkpoint zip under the (possibly timestamp-suffixed) mflux
- output dir. mflux `--resume` writes back to the SAME output_path
- (new_folder=False), so checkpoints accumulate in one dir across segments;
- ranking by embedded step keeps the pick monotonic even if mtimes tie."""
- cdir = resolve_mflux_output(configured_output) / "checkpoints"
- if not cdir.is_dir():
- return None
- zips = list(cdir.glob("*_checkpoint.zip"))
- if not zips:
- return None
- return max(zips, key=lambda z: (_checkpoint_step(z), z.stat().st_mtime))
-
-
-def _terminate_child(timeout: float = 15.0) -> None:
- """SIGTERM the mflux child and reap it (escalate to SIGKILL if it lingers).
- Used at segment boundaries to fully release the Metal/GPU context."""
- global CHILD
- if CHILD and CHILD.poll() is None:
- CHILD.terminate()
- try:
- CHILD.wait(timeout=timeout)
- except subprocess.TimeoutExpired:
- CHILD.kill()
-
-
-# Cooldown heartbeat cadence. The mediaJobQueue idle watchdog resets only on
-# emitted runner output (progress.js emits 'activity' for every non-noise line),
-# so a silent cooldown longer than the watchdog window would get a healthy run
-# killed mid-pause. segmentCooldownSec is validated up to 3600s — above the 1800s
-# default watchdog — so the cooldown MUST keep the watchdog fed itself rather than
-# rely on the setting staying small.
-HEARTBEAT_INTERVAL_SEC = 30
-
-
-def interruptible_sleep(seconds: float, heartbeat_stage: str | None = None) -> None:
- """Sleep up to `seconds`, returning early if a SIGTERM cancel arrives — so
- the inter-segment cooldown never delays a user-requested stop. When
- `heartbeat_stage` is set, emit a `STAGE:<stage>:heartbeat:<elapsed>s`
- keep-alive every HEARTBEAT_INTERVAL_SEC so the idle watchdog sees activity
- during a long pause (the `:heartbeat:` form resets the watchdog without
- emitting a status message or changing the displayed stage)."""
- start = time.monotonic()
- end = start + seconds
- next_beat = start + HEARTBEAT_INTERVAL_SEC
- while not STOP_REQUESTED:
- now = time.monotonic()
- remaining = end - now
- if remaining <= 0:
- return
- if heartbeat_stage and now >= next_beat:
- log(f"STAGE:{heartbeat_stage}:heartbeat:{int(now - start)}s")
- next_beat = now + HEARTBEAT_INTERVAL_SEC
- time.sleep(min(1.0, remaining))
-
-
-def build_command(config_path: str, resume: str | None) -> list:
- """`mflux-train` console script next to THIS interpreter (no PATH
- ambiguity); fallback to the module behind it."""
- console = Path(sys.executable).parent / "mflux-train"
- if console.exists():
- cmd = [str(console)]
- else:
- probe = subprocess.run(
- [sys.executable, "-c",
- "import importlib.util as u; raise SystemExit(0 if u.find_spec('mflux.models.common.cli.train') else 1)"],
- capture_output=True,
- )
- if probe.returncode != 0:
- print(
- "USER_ERROR:MODULE_NOT_FOUND:mflux trainer not found (no mflux-train script and no "
- "mflux.models.common.cli.train module) — update mflux to ≥0.17",
- file=sys.stderr, flush=True,
- )
- sys.exit(2)
- cmd = [sys.executable, "-m", "mflux.models.common.cli.train"]
- if resume:
- cmd += ["--resume", resume]
- else:
- cmd += ["--config", config_path]
- return cmd
-
-
-# tqdm renders like ` 3%|▎ | 12/400 [00:30<16:20, 2.53s/it]`.
-TQDM_RE = re.compile(r"(\d+)\s*/\s*(\d+)\s*\[")
-# Belt-and-suspenders for any explicit "step N/M ... loss X" prose a future
-# mflux version might print.
-STEP_LOSS_RE = re.compile(r"[Ss]teps?[\s:=]+(\d+)\s*/\s*(\d+).*?[Ll]oss[\s:=]+([\d.eE+-]+)")
-NOISE_RE = re.compile(r"FutureWarning|UserWarning|DeprecationWarning", re.I)
-
-# We merge the mflux child's stderr into its stdout (so tqdm + prints stay in
-# order), which means the Node side tags every line `stream='stdout'` and the
-# JS failure classifier's stderr-tail stays empty for mflux runs. To keep
-# OOM / missing-module / gated-repo errors classifiable, sniff them out of the
-# merged stream and re-emit a structured USER_ERROR line (which the Node line
-# parser captures regardless of stream). Codes mirror failure.js.
-FATAL_PATTERNS = [
- ("OOM", re.compile(r"out of memory|Insufficient Memory|Metal.*out of memory|std::bad_alloc", re.I)),
- ("MODULE_NOT_FOUND", re.compile(r"ModuleNotFoundError|No module named", re.I)),
- ("HF_AUTH", re.compile(r"GatedRepoError|401 Client Error|is restricted|Repo.*is gated", re.I)),
-]
-# NOTE: argparse rejections (a too-old mflux that wants `--train-config` / has
-# no flux2 base models — the exact "exited with code 2" failure) are
-# deliberately NOT sniffed here. A USER_ERROR line short-circuits
-# classifyTrainingFailure on its *raw* text (failure.js), so sniffing the
-# argparse line would surface "unrecognized arguments: --config …" instead of
-# the actionable "upgrade mflux>=0.17" hint. Left unsniffed, the line rides the
-# non-zero-exit tail replay to stderr and failure.js's CLI_MISMATCH_RE renders
-# the actionable message — one source of truth for that text, on the JS side.
-
-
-def is_preview_progress_bar(line, cur, total, *, has_step_loss, preview_steps, training_total):
- """True if a matched tqdm bar belongs to the preview/sample IMAGE generation
- rather than the training loop.
-
- mflux runs `adapter.generate_preview_image` (→ `flux2.generate_image`) once
- before training (step 0) and again at every sample interval. That diffusion
- sampler prints its own tqdm bar whose total is the DENOISE step count
- (`config["steps"]`, e.g. 40) — identical in shape to the training bar except
- the total differs (training total = epochs × images). If we don't exclude it
- the wrapper both mislabels denoise ticks as training STEPs and lets them
- perturb `state["step"]`, which gates the segment-boundary kill (a preview bar
- reading 38/40 could spuriously satisfy a 150-step boundary).
-
- Discriminators, all required:
- - `has_step_loss` False — the explicit "step N/M … loss X" prose is emitted
- only by the training loop, never the sampler, so it's never a preview.
- - `preview_steps` known (caller passes None when steps == training_total,
- the degenerate tiny-dataset case where totals can't disambiguate).
- - the bar's total equals `preview_steps` and differs from `training_total`.
- """
- if has_step_loss or preview_steps is None:
- return False
- return total == preview_steps and (training_total is None or total != training_total)
-
-
-def resolve_mflux_output(configured: Path) -> Path:
- """mflux appends `_YYYYMMDD_HHMMSS` to checkpoint.output_path when the
- configured dir already exists (its new-run-folder behavior) — a killed
- previous attempt leaving the dir behind is enough to trigger it. Pick
- the newest candidate among the configured path and its timestamped
- siblings so the watcher + adapter discovery track where mflux actually
- wrote."""
- candidates = [p for p in [configured, *configured.parent.glob(f"{configured.name}_2*")] if p.is_dir()]
- if not candidates:
- return configured
- return max(candidates, key=lambda p: p.stat().st_mtime)
-
-
-class ArtifactWatcher(threading.Thread):
- """Poll mflux's output dir: new checkpoint zips → CHECKPOINT lines; new
- preview PNGs → copied into the run's samples/ (where the server's
- sample route serves from) + SAMPLE lines. The output dir is re-resolved
- each scan — it may materialize (possibly timestamp-suffixed) only after
- mflux finishes loading the model."""
-
- def __init__(self, configured_output: Path, samples_dir: Path, get_step):
- super().__init__(daemon=True)
- self.configured_output = configured_output
- self.samples_dir = samples_dir
- self.get_step = get_step
- self.seen = set()
- self.stop_event = threading.Event()
-
- def scan(self):
- output = resolve_mflux_output(self.configured_output)
- checkpoints_dir = output / "checkpoints"
- preview_dir = output / "preview"
- for f in sorted(checkpoints_dir.glob("*.zip")) if checkpoints_dir.exists() else []:
- if f in self.seen:
- continue
- self.seen.add(f)
- log(f"CHECKPOINT:{f}:{self.get_step()}")
- for f in sorted(preview_dir.glob("*.png")) if preview_dir.exists() else []:
- if f in self.seen:
- continue
- self.seen.add(f)
- dest = self.samples_dir / f.name
- try:
- shutil.copyfile(f, dest)
- log(f"SAMPLE:{dest}:{self.get_step()}")
- except OSError as err:
- log(f"STATUS:sample copy failed: {err}")
-
- def run(self):
- while not self.stop_event.wait(5.0):
- self.scan()
- self.scan() # final sweep after the child exits
-
-
-class TelemetrySidecar:
- """Optional GPU/thermal/power capture for crash forensics.
-
- macOS GPU watchdog-timeout kernel panics during sustained mflux training
- (see docs/TROUBLESHOOTING.md "GPU watchdog kernel panic" + the incident
- record under docs/research) hard-reboot the machine mid-run and leave NO
- application-level record of GPU temperature / power draw leading up to the
- hang. This streams `powermetrics` into <output_dir>/powermetrics.log so a
- post-crash investigation can tell a thermal/power fault from a driver hang.
-
- powermetrics requires root. We gate on a *non-interactive* `sudo -n -l`
- probe of powermetrics specifically: if passwordless sudo for that command
- isn't configured the sidecar no-ops with a single STATUS note and training
- proceeds unaffected — it never prompts or blocks.
- """
-
- SAMPLE_INTERVAL_MS = 5000
-
- def __init__(self, output_dir: Path):
- # Don't clobber a prior run's telemetry on resume — the post-crash
- # investigation reads it from this dir, and `powermetrics --output-file`
- # truncates. Roll to a timestamped name when the default already exists
- # so each launch keeps its own forensic log.
- base = output_dir / "powermetrics.log"
- if base.exists():
- base = output_dir / f"powermetrics.{datetime.now():%Y%m%d_%H%M%S}.log"
- self.log_path = base
- self.proc = None
-
- def start(self):
- if sys.platform != "darwin":
- return
- pm = shutil.which("powermetrics")
- if not pm:
- return
- # Probe whether *powermetrics specifically* is permitted under
- # passwordless sudo. The documented sudoers rule grants NOPASSWD for
- # powermetrics only, so a generic `sudo -n true` probe (which runs
- # /usr/bin/true) would fail for exactly the users who configured it
- # correctly. `sudo -n -l <cmd>` lists the permission without running
- # powermetrics and without prompting; non-zero means not allowed.
- # Telemetry is best-effort — a missing `sudo` must never crash training.
- try:
- probe = subprocess.run(["sudo", "-n", "-l", pm], capture_output=True)
- except OSError as err:
- log(f"STATUS:GPU telemetry unavailable ({err}); continuing without it")
- return
- if probe.returncode != 0:
- log("STATUS:GPU telemetry disabled — passwordless sudo for powermetrics not configured; "
- "see docs/TROUBLESHOOTING.md to enable thermal/power capture. Continuing without it.")
- return
- cmd = [
- "sudo", "-n", pm,
- "--samplers", "cpu_power,gpu_power,thermal",
- "-i", str(self.SAMPLE_INTERVAL_MS),
- "--output-file", str(self.log_path),
- ]
- try:
- self.proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
- except OSError as err:
- log(f"STATUS:GPU telemetry unavailable ({err}); continuing without it")
- self.proc = None
- return
- # Confirm powermetrics actually stayed up — a bad sampler, an OS that
- # dropped a sampler, or a perms issue makes it exit immediately, and we
- # must not report telemetry as enabled while the log stays empty.
- try:
- rc = self.proc.wait(timeout=0.5)
- log(f"STATUS:GPU telemetry failed to start (powermetrics exited {rc}); "
- "continuing without it")
- self.proc = None
- return
- except subprocess.TimeoutExpired:
- pass # still running after the grace window → healthy
- # Guarantee the root powermetrics child is reaped even if main() throws
- # between start() and the explicit stop() — unlike the daemon watcher
- # thread, this is a separate OS process that outlives the interpreter.
- # stop() is idempotent (poll() guard), so the later explicit call is safe.
- atexit.register(self.stop)
- log(f"STATUS:GPU telemetry → {self.log_path} (powermetrics @ {self.SAMPLE_INTERVAL_MS}ms)")
-
- def stop(self):
- if not self.proc or self.proc.poll() is not None:
- return
- # sudo forwards SIGTERM to powermetrics; escalate if it lingers.
- self.proc.terminate()
- try:
- self.proc.wait(timeout=5)
- except subprocess.TimeoutExpired:
- self.proc.kill()
-
-
-def find_adapter(checkpoints_dir: Path, output_dir: Path) -> Path | None:
- """Extract the lora adapter (`*_adapter.safetensors`, NOT the optimizer
- state) from the newest checkpoint zip."""
- zips = sorted(checkpoints_dir.glob("*.zip"), key=lambda f: f.stat().st_mtime)
- for z in reversed(zips):
- with zipfile.ZipFile(z) as zf:
- members = [m for m in zf.namelist()
- if m.endswith("_adapter.safetensors") and "optimizer" not in m]
- if not members:
- continue
- extract_dir = output_dir / "adapter"
- extract_dir.mkdir(parents=True, exist_ok=True)
- zf.extract(members[-1], extract_dir)
- return extract_dir / members[-1]
- return None
-
-
-def stream_lines(pipe):
- """Yield logical lines split on BOTH \\n and \\r — tqdm redraws its bar
- with carriage returns, so newline-only iteration would buffer the whole
- bar until completion."""
- buf = ""
- while True:
- chunk = pipe.read(256)
- if not chunk:
- if buf:
- yield buf
- return
- buf += chunk
- while True:
- cut = min((i for i in (buf.find("\n"), buf.find("\r")) if i >= 0), default=-1)
- if cut < 0:
- break
- yield buf[:cut]
- buf = buf[cut + 1:]
-
-
-def compute_effective_total(config: dict, fallback: int) -> int:
- """mflux's REAL iteration total = ceil(imageCount / batch_size) * num_epochs
- — identical to Iterator.total_number_of_steps(). The wrapper's --total-steps
- is the user's *requested* count, which mflux rounds up to whole epochs and
- can exceed (e.g. 600 requested with 240 images → round(600/240)=3 epochs →
- 720). Segment planning MUST use this real total, or the final segment spans
- from the last requested-total boundary all the way to the real end in one
- sustained process — exactly the unbounded GPU window this patch removes.
-
- Deterministic and known before launch (the config carries num_epochs +
- batch_size and the staged dataset dir). Falls back to the requested count if
- the config shape is unexpected; tqdm then corrects state["total"] live after
- the first segment anyway."""
- try:
- loop = config.get("training_loop", {}) or {}
- epochs = int(loop.get("num_epochs", 0))
- batch = int(loop.get("batch_size", 1)) or 1
- data_dir = config.get("data")
- # mflux auto-discovers NNNN.png + NNNN.txt training pairs; preview prompts
- # are .txt only, so *.png count is the image count it will encode.
- n_images = len(list(Path(data_dir).glob("*.png"))) if data_dir else 0
- if epochs > 0 and n_images > 0:
- batches_per_epoch = (n_images + batch - 1) // batch
- return batches_per_epoch * epochs
- except (ValueError, TypeError, OSError):
- pass
- return fallback
-
-
-def run_segment(cmd, *, segment_target, state, last_reported, output_tail,
- fatal_holder, configured_output, cooldown,
- preview_steps=None, training_total=None):
- """Run ONE mflux invocation and stream its output.
-
- Returns a dict with `reason`:
- - 'segment' — reached `segment_target`; the child was terminated after
- the boundary checkpoint was confirmed on disk. `checkpoint`
- holds the zip to resume the next segment from.
- - 'completed' — the child exited 0 on its own (training finished).
- - 'stopped' — a SIGTERM cancel arrived; caller should exit 143.
- - 'error' — the child exited non-zero; `code` carries the exit code.
-
- `segment_target=None` means "let mflux run to completion" (final/only
- segment) — no boundary kill is armed.
- """
- global CHILD
- # `cmd` is always list-form (see build_command) and `shell` stays at its
- # False default, so argv never reaches a shell — there is no injection
- # surface here. Scanners periodically flag this line anyway; an explicit
- # `shell=False` would be a no-op. See PR #2882 (closed).
- CHILD = subprocess.Popen(
- cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
- )
- if STOP_REQUESTED: # SIGTERM raced the spawn
- CHILD.terminate()
-
- in_training = False
- preview_logged = {"v": False} # one STAGE:sampling per contiguous preview run
- for raw in stream_lines(CHILD.stdout):
- line = raw.strip()
- if not line or NOISE_RE.search(line):
- continue
-
- # Surface the first fatal error in a structured form the Node
- # classifier can read (once — don't spam if it repeats in a traceback).
- if not fatal_holder["v"]:
- for code, pat in FATAL_PATTERNS:
- if pat.search(line):
- print(f"USER_ERROR:{code}:{line[:300]}", file=sys.stderr, flush=True)
- fatal_holder["v"] = True
- break
-
- step_loss_match = STEP_LOSS_RE.search(line)
- m = step_loss_match or TQDM_RE.search(line)
- if m:
- cur, total = int(m.group(1)), int(m.group(2))
- # Skip the preview/sample image generation's own tqdm bar so it isn't
- # mislabeled as a training step or allowed to trip the segment kill.
- if is_preview_progress_bar(
- line, cur, total,
- has_step_loss=step_loss_match is not None,
- preview_steps=preview_steps, training_total=training_total,
- ):
- if not in_training and not preview_logged["v"]:
- # Pre-training step-0 preview: surface a distinct stage so the
- # UI shows "sampling" instead of a frozen "load-model". Emit
- # once — the bar redraws every denoise tick.
- preview_logged["v"] = True
- log("STAGE:sampling")
- continue
- if total > 0 and cur >= 0:
- state["step"], state["total"] = cur, total
- if m.lastindex and m.lastindex >= 3:
- try:
- state["loss"] = float(m.group(3))
- except ValueError:
- pass
- if not in_training:
- in_training = True
- log("STAGE:training")
- # A real training step ends a preview run; re-arm so the NEXT
- # mid-training sampling phase (step 150, 300…) logs STAGE:sampling
- # again. Re-announce STAGE:training when we come back from one.
- elif preview_logged["v"]:
- log("STAGE:training")
- preview_logged["v"] = False
- # tqdm redraws constantly at the same step — only emit on change.
- if cur != last_reported["step"]:
- last_reported["step"] = cur
- log(f"STEP:{cur}:{total}:{state['loss'] if state['loss'] is not None else 'nan'}")
- # Segment boundary. Two reasons this gates on state["step"]:
- # 1. state["step"] is mflux's CUMULATIVE iteration count, not a
- # per-process counter — tqdm renders with
- # initial=num_iterations (restored from the checkpoint on
- # resume) over total_number_of_steps() (the cumulative
- # total), so the bar reads 150/600, 151/600… after a resume.
- # Comparing it to the cumulative `segment_target` is valid on
- # every segment, not just the first.
- # 2. Waiting until the trainer has stepped PAST the target is
- # the flush-safety signal: mflux's save() writes the zip
- # IN-PLACE and non-atomically (zipfile.ZipFile(zip_path,"w")
- # straight to the final path), and the next tqdm line only
- # appears after that synchronous save() returns. Do NOT
- # "simplify" this to terminate the moment
- # newest_checkpoint() >= segment_target — that would race a
- # half-written zip and resume from a truncated checkpoint.
- if segment_target is not None and state["step"] > segment_target:
- ck = newest_checkpoint(configured_output)
- if ck and _checkpoint_step(ck) >= segment_target:
- log(f"STATUS:segment checkpoint @ step {_checkpoint_step(ck)} saved — "
- f"tearing down GPU + cooling {cooldown}s (watchdog-panic mitigation)")
- _terminate_child()
- return {"reason": "segment", "checkpoint": ck, "code": 0}
- continue
-
- # Forward everything else (truncated) — keeps the JS idle watchdog
- # fed during model download/load phases and aids debugging.
- output_tail.append(line[:300])
- log(f"STATUS:{line[:300]}")
-
- code = CHILD.wait()
- if STOP_REQUESTED:
- return {"reason": "stopped", "code": 143}
- if code != 0:
- return {"reason": "error", "code": code}
- return {"reason": "completed", "checkpoint": newest_checkpoint(configured_output), "code": 0}
-
-
-def main():
- args = parse_args()
- # Self-document the exact numerical stack BEFORE anything can fail — the run
- # log + any watchdog paniclog then carry the mflux/mlx/mlx-metal trio inline.
- emit_runtime_fingerprint()
- # Self-document the GPU-watchdog mitigation too (mlx #3267) so a paniclog
- # shows whether the AGX timeout relaxation was active for the crashed run.
- log(f"STATUS:watchdog mitigation · AGX_RELAX_CDM_CTXSTORE_TIMEOUT={os.environ.get('AGX_RELAX_CDM_CTXSTORE_TIMEOUT', 'unset')}")
- output_dir = Path(args.output_dir)
- samples_dir = output_dir / "samples"
- samples_dir.mkdir(parents=True, exist_ok=True)
- config = json.loads(Path(args.config).read_text())
- configured_output = Path(config["checkpoint"]["output_path"])
-
- # mflux's real (epoch-rounded) total, not the requested --total-steps — see
- # compute_effective_total. Segment planning keys off this so the final
- # segment stays one save-interval wide even when epochs round the count up.
- total = max(1, compute_effective_total(config, args.total_steps))
- seg = max(0, args.segment_steps)
- cooldown = max(0, args.cooldown_sec)
- segmented = seg > 0
-
- # The preview/sample image generation prints its own tqdm bar whose total is
- # the denoise step count (`steps`), distinct from the training total. Pass
- # both to run_segment so it can tell a preview bar from a training bar. Guard
- # the rare degenerate case where steps == training total (tiny datasets):
- # then we can't disambiguate by total, so don't filter (preview_steps=None).
- preview_steps = config.get("steps")
- if not isinstance(preview_steps, int) or preview_steps <= 0 or preview_steps == total:
- preview_steps = None
-
- state = {"step": 0, "total": total, "loss": None}
- last_reported = {"step": -1}
-
- watcher = ArtifactWatcher(configured_output, samples_dir, lambda: state["step"])
- watcher.start()
-
- # Resolve the trainer command up front so a missing mflux exits before we
- # start telemetry (build_command exits early) — never leave an orphaned
- # root powermetrics process. One telemetry sidecar + one watcher span ALL
- # segments so the forensic log and artifact stream stay continuous.
- _ = build_command(args.config, args.resume_checkpoint)
- telemetry = TelemetrySidecar(output_dir)
- telemetry.start()
-
- fatal_holder = {"v": False}
- # Ring buffer of the trainer's recent non-progress output. mflux's stderr
- # is merged into its stdout (so tqdm stays in order), which means the Node
- # side tags every line `stream='stdout'` and the JS classifier's stderr
- # tail — its only window into *why* a run died — stays empty. On a non-zero
- # exit we replay this buffer to stderr so argparse errors, tracebacks, and
- # version-mismatch complaints actually reach the classifier instead of the
- # run failing with a bare "exited with code N".
- output_tail = deque(maxlen=OUTPUT_TAIL_LINES)
-
- resume = args.resume_checkpoint # None on a fresh run; a zip on resume
- final_code = 0
- segment_index = 0
- if segmented:
- log(f"STATUS:segmented training enabled — {seg}-step segments, {cooldown}s GPU cooldown between each")
- while True:
- start_step = _checkpoint_step(resume) if resume else 0
- # Plan against mflux's real epoch-rounded total. `total` is the
- # config-derived estimate (compute_effective_total); state["total"] is
- # the authoritative value tqdm reports once the first segment runs — take
- # the larger so the cutoff is right BOTH before the first tqdm line (fresh
- # run / resume-near-end use the config estimate) and after (tqdm corrects
- # any discrepancy). Only when the NEXT boundary would reach/exceed this
- # real total do we drop the boundary kill and run a final, run-to-
- # completion segment; otherwise every segment — including the last —
- # stays one save-interval wide, preserving the bounded GPU window. A
- # resume past the end just reloads, hits StopIteration, returns
- # 'completed' — correct, at the cost of one model load.
- effective_total = max(total, state["total"])
- segment_target = start_step + seg if segmented else None
- if segment_target is not None and segment_target >= effective_total:
- segment_target = None
-
- cmd = build_command(args.config, resume)
- log("STAGE:load-model")
- log(f"STATUS:launching {Path(cmd[0]).name}"
- + (f" (segment {segment_index + 1} from step {start_step})" if segmented else ""))
-
- # Each segment is a fresh mflux process — fatal-error detection (and the
- # decision to replay the stderr tail) must be per-invocation, so a
- # FATAL_PATTERN line matched in an earlier segment that still exited 0
- # can't suppress the tail replay for a later segment's real failure.
- fatal_holder["v"] = False
- result = run_segment(
- cmd, segment_target=segment_target, state=state, last_reported=last_reported,
- output_tail=output_tail, fatal_holder=fatal_holder,
- configured_output=configured_output, cooldown=cooldown,
- preview_steps=preview_steps, training_total=total,
- )
- segment_index += 1
-
- if result["reason"] == "stopped":
- break
- if result["reason"] == "error":
- final_code = result["code"] or 1
- break
- if result["reason"] == "completed":
- break
-
- # reason == 'segment' — cool down, then resume from the boundary checkpoint.
- resume = str(result["checkpoint"]) if result.get("checkpoint") else resume
- new_step = _checkpoint_step(resume)
- # No `new_step >= total` early-stop: a 'segment' result means we killed
- # at a non-final boundary (segment_target < total), so the next
- # iteration's segment_target crosses `total`, becomes the final
- # run-to-completion segment, and lets mflux finish its epoch-rounded
- # steps — rather than stopping at the approximate requested total.
- # Forward-progress guard: the boundary only fires on a checkpoint whose
- # step >= segment_target > start_step, so new_step should always advance.
- # If it somehow didn't (a stalled save, checkpoint renumbering), resuming
- # would re-run the identical segment forever — fail loudly instead.
- if new_step <= start_step:
- print(f"USER_ERROR:TRAINING_FAILED:segment made no checkpoint progress "
- f"(still at step {new_step}/{total}) — stopping to avoid a resume loop",
- file=sys.stderr, flush=True)
- final_code = 1
- break
- log(f"STATUS:segment {segment_index} complete at step {new_step}/{total} — cooling down {cooldown}s")
- interruptible_sleep(cooldown, heartbeat_stage="cooldown")
- if STOP_REQUESTED:
- break
-
- watcher.stop_event.set()
- watcher.join(timeout=10)
- telemetry.stop()
-
- if STOP_REQUESTED:
- log("STATUS:canceled — last saved checkpoint retained")
- sys.exit(143)
-
- if final_code != 0:
- # Replay the trainer's tail to stderr so the Node classifier sees the
- # real error. Skip when a structured USER_ERROR already pinned the
- # cause (its message is more actionable than raw tail lines).
- if not fatal_holder["v"]:
- for tail_line in output_tail:
- print(f"mflux: {tail_line}", file=sys.stderr, flush=True)
- print(f"❌ mflux trainer exited with code {final_code}", file=sys.stderr, flush=True)
- sys.exit(final_code or 1)
-
- checkpoints_dir = resolve_mflux_output(configured_output) / "checkpoints"
- adapter = find_adapter(checkpoints_dir, output_dir)
- if not adapter:
- print(
- "USER_ERROR:TRAINING_FAILED:mflux finished but no *_adapter.safetensors found in "
- f"{checkpoints_dir}", file=sys.stderr, flush=True,
- )
- sys.exit(1)
- log("RESULT:" + json.dumps({
- "adapter_path": str(adapter),
- "steps": state["step"] or args.total_steps,
- "final_loss": state["loss"],
- }))
-
-
-if __name__ == "__main__":
- main()
diff --git a/scripts/train_mflux_lora_test.py b/scripts/train_mflux_lora_test.py
deleted file mode 100644
index 99efd3cd35..0000000000
--- a/scripts/train_mflux_lora_test.py
+++ /dev/null
@@ -1,136 +0,0 @@
-#!/usr/bin/env python3
-"""Standalone tests for train_mflux_lora.py pure helpers (no pytest, no mflux).
-
-Run: ./data/python/venv/bin/python scripts/train_mflux_lora_test.py
-Exits non-zero on first failure. Mirrors the runnable-test style of
-scripts/generate_ltx2_teacache_test.py.
-"""
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-from train_mflux_lora import ( # noqa: E402
- is_preview_progress_bar,
- compute_effective_total,
- format_runtime_fingerprint_line,
- TQDM_RE,
- STEP_LOSS_RE,
-)
-
-FAILS = []
-
-
-def check(name, cond):
- print(("✅" if cond else "❌") + " " + name)
- if not cond:
- FAILS.append(name)
-
-
-# --- is_preview_progress_bar: the preview/sample bar vs the training bar -------
-# Scenario mirrors the live Freydis run: steps=40 (preview denoise), total=600.
-PREVIEW, TRAIN = 40, 600
-
-# A preview bar (total == preview_steps, != training_total, no step/loss prose).
-check(
- "preview bar (38/40) is detected as preview",
- is_preview_progress_bar("38/40 [00:19<00:01]", 38, 40,
- has_step_loss=False, preview_steps=PREVIEW, training_total=TRAIN),
-)
-check(
- "preview bar at completion (40/40) is detected as preview",
- is_preview_progress_bar("40/40 [00:20<00:00]", 40, 40,
- has_step_loss=False, preview_steps=PREVIEW, training_total=TRAIN),
-)
-
-# A training bar (total == training_total) is NOT a preview.
-check(
- "training bar (151/600) is NOT preview",
- not is_preview_progress_bar("151/600 [10:00<30:00]", 151, 600,
- has_step_loss=False, preview_steps=PREVIEW, training_total=TRAIN),
-)
-check(
- "training bar at a 40-ish step (40/600) is NOT preview (total differs)",
- not is_preview_progress_bar("40/600 [02:00<28:00]", 40, 600,
- has_step_loss=False, preview_steps=PREVIEW, training_total=TRAIN),
-)
-
-# Explicit "step N/M ... loss X" prose is training-only, never a preview, even
-# if its total coincidentally equals preview_steps.
-check(
- "explicit step/loss prose is never preview",
- not is_preview_progress_bar("step 40/40 loss 0.12", 40, 40,
- has_step_loss=True, preview_steps=PREVIEW, training_total=TRAIN),
-)
-
-# Degenerate case: caller passes preview_steps=None (steps == training_total on a
-# tiny dataset) → never filter, so no real training step is ever dropped.
-check(
- "preview_steps=None disables filtering",
- not is_preview_progress_bar("40/40 [..]", 40, 40,
- has_step_loss=False, preview_steps=None, training_total=40),
-)
-
-# preview_steps known but training_total unknown (None): filter on total match
-# alone — better to risk a missed preview tick than to mislabel, but we still
-# catch the common case.
-check(
- "training_total=None still filters an exact preview-total match",
- is_preview_progress_bar("12/40 [..]", 12, 40,
- has_step_loss=False, preview_steps=40, training_total=None),
-)
-
-# --- the regexes that feed it --------------------------------------------------
-m = TQDM_RE.search(" 3%|▎ | 12/400 [00:30<16:20, 2.53s/it]")
-check("TQDM_RE parses cur/total", bool(m) and m.group(1) == "12" and m.group(2) == "400")
-check("STEP_LOSS_RE matches explicit prose",
- bool(STEP_LOSS_RE.search("steps 40/40 loss 0.12")))
-check("STEP_LOSS_RE does NOT match a bare tqdm bar",
- STEP_LOSS_RE.search("38/40 [00:19<00:01]") is None)
-
-# --- compute_effective_total sanity (unchanged behavior) -----------------------
-# 25 images, batch 1, 24 epochs → 600 (the Freydis config).
-import tempfile, os # noqa: E402
-with tempfile.TemporaryDirectory() as d:
- for i in range(25):
- Path(d, f"{i:04d}.png").write_bytes(b"x")
- cfg = {"training_loop": {"num_epochs": 24, "batch_size": 1}, "data": d}
- check("compute_effective_total(25 img,24 ep,batch 1) == 600",
- compute_effective_total(cfg, 999) == 600)
-
-# Unexpected shape → falls back to the requested count.
-check("compute_effective_total falls back on bad config",
- compute_effective_total({}, 777) == 777)
-
-# --- format_runtime_fingerprint_line: one-line startup stack fingerprint -------
-# Full stack as the M5 panic reports capture it.
-full_fp = {
- "runtime": "mflux-lora",
- "versions": {"mflux": "0.17.5", "mlx": "0.30.6", "mlx-metal": "0.30.6", "mlx-lm": "0.29.1"},
- "chip": "Apple M5 Max",
- "os": "macOS-26.5.1-arm64-arm-64bit",
- "python": "3.11.9",
-}
-full_line = format_runtime_fingerprint_line(full_fp)
-check("fingerprint line is a single STATUS line",
- full_line.startswith("STATUS:runtime · ") and "\n" not in full_line)
-check("fingerprint line carries the mflux/mlx/mlx-metal/mlx-lm versions",
- all(s in full_line for s in ("mflux 0.17.5", "mlx 0.30.6", "mlx-metal 0.30.6", "mlx-lm 0.29.1")))
-check("fingerprint line carries chip/os/python",
- all(s in full_line for s in ("Apple M5 Max", "macOS-26.5.1-arm64-arm-64bit", "python 3.11.9")))
-
-# Absent packages omitted by the builder → line still renders, no crash, no empty
-# fragment for the missing pin.
-partial_line = format_runtime_fingerprint_line({"versions": {"mflux": "0.17.5"}, "chip": "Apple M5 Max"})
-check("partial fingerprint omits absent packages cleanly",
- "mflux 0.17.5" in partial_line and "mlx " not in partial_line and "· ·" not in partial_line)
-
-# Empty dict (everything failed to resolve) → still a single, non-throwing line.
-empty_line = format_runtime_fingerprint_line({})
-check("empty fingerprint degrades to a 'versions unavailable' line",
- empty_line == "STATUS:runtime · versions unavailable")
-
-print()
-if FAILS:
- print(f"❌ {len(FAILS)} failure(s): {FAILS}")
- sys.exit(1)
-print("✅ all train_mflux_lora helper tests passed")
diff --git a/server/hostControlCoverage.test.js b/server/hostControlCoverage.test.js
index 2cb38a9562..c0ac706dd3 100644
--- a/server/hostControlCoverage.test.js
+++ b/server/hostControlCoverage.test.js
@@ -657,17 +657,8 @@ const REVIEWED_OPEN_MUTATIONS = [
['DELETE /api/loops/:id', REMOVES_RECORD],
['POST /api/loops/:id/stop', REDUCES_EXECUTION],
// /api/lora-datasets
- ['POST /api/lora-datasets', RECORDS],
- ['DELETE /api/lora-datasets/:id', REMOVES_RECORD],
- ['PATCH /api/lora-datasets/:id', RECORDS],
['POST /api/lora-datasets/:id/caption-runs/:runId/cancel', REDUCES_EXECUTION],
- ['POST /api/lora-datasets/:id/images', RECORDS],
- ['DELETE /api/lora-datasets/:id/images/:imageId', REMOVES_RECORD],
- ['PATCH /api/lora-datasets/:id/images/:imageId', RECORDS],
- ['POST /api/lora-datasets/:id/import-gallery', IN_PROCESS_MEDIA],
- ['POST /api/lora-datasets/:id/strip-shared-fragments', RECORDS],
// /api/lora-training
- ['DELETE /api/lora-training/runs/:id', REMOVES_RECORD],
['POST /api/lora-training/runs/:id/cancel', REDUCES_EXECUTION],
// /api/loras
['DELETE /api/loras/:filename', REMOVES_RECORD],
diff --git a/server/index.js b/server/index.js
index 8fccaa84f2..7ca2799f63 100644
--- a/server/index.js
+++ b/server/index.js
@@ -181,6 +181,7 @@ import imageVideoModelsRoutes from './routes/imageVideoModels.js';
import lorasRoutes from './routes/loras.js';
import loraDatasetsRoutes from './routes/loraDatasets.js';
import loraTrainingRoutes from './routes/loraTraining.js';
+import dataMaintenanceRoutes from './routes/dataMaintenance.js';
import sdapiRoutes from './routes/sdapi.js';
import openclawRoutes from './routes/openclaw.js';
import sharingRoutes from './routes/sharing.js';
@@ -500,6 +501,7 @@ app.use('/api/importer', importerRoutes);
app.use('/api/story-builder', storyBuilderRoutes);
app.use('/api/image-video/models', imageVideoModelsRoutes);
app.use('/api/loras', lorasRoutes);
+app.use('/api/data-maintenance', dataMaintenanceRoutes);
app.use('/api/lora-datasets', loraDatasetsRoutes);
app.use('/api/lora-training', loraTrainingRoutes);
// AUTOMATIC1111-compatible surface for tailnet clients — gated by
diff --git a/server/lib/README.md b/server/lib/README.md
index b7e84c61a7..7d57fb12e3 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -31,6 +31,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `reconcileRetirementEvidence.js` | Matches exact checkout receipts and recovered-preservation receipts, preserves observation identity across delayed scans, and retains each unresolved checkout incarnation. Legacy observations keep their original generation timestamps. A recreated checkout at the same branch, path and head starts after every older receipt; verified historical preservation does not assert branch deletion. |
| `legacyRetirementTransition.js` | Freezes archive-era observations for evidence-based reconciliation and retains their original diagnoses and delivery/receipt proofs as audit history, without manufacturing deletion receipts or changing task outcomes. |
| `retirementCheckoutObservation.js` | Reads a checkout's filesystem identity and verifies its live branch/head before the schedule writer admits a recreated checkout's obligation. |
+| `retiredLoraTraining.js` | Retirement refusal and short-lived admission for retained training data access/removal. |
| `backupSharedAdmission.js` | Durable cross-process backup cuts and publication leases, with fail-closed owner recovery. |
| `runtimeFilePublication.js` | Admitted runtime recording updates with rollback of the prior file pair on failure. |
| `maintenanceAdmission.js` | Durable machine-local workflow drain, cross-process admission permits, continuation ownership and stale-resume fencing. |
diff --git a/server/lib/backupAssetOwners.js b/server/lib/backupAssetOwners.js
index 4f574ad803..7697a4c8b2 100644
--- a/server/lib/backupAssetOwners.js
+++ b/server/lib/backupAssetOwners.js
@@ -408,7 +408,7 @@ export const BACKUP_ASSET_OWNERS = Object.freeze([
// Explicit file cleanup can leave external references by design, but one
// lease keeps its single/bulk deletion out of a copy-then-dump snapshot.
id: 'operator-file-purge', status: 'admitted',
- modules: ['services/dataManager.js', 'routes/uploads.js', 'routes/attachments.js'],
+ modules: ['services/dataManager.js', 'services/dataMaintenance.js', 'routes/uploads.js', 'routes/attachments.js'],
},
{
// Snapshot JSON and its index entry are one pair. Deletion persists index
diff --git a/server/lib/hostControlRoutes.js b/server/lib/hostControlRoutes.js
index 7cbc46971e..f4dbf4c65c 100644
--- a/server/lib/hostControlRoutes.js
+++ b/server/lib/hostControlRoutes.js
@@ -1252,6 +1252,9 @@ export const HOST_CONTROL_ROUTES = Object.freeze([
// configured inference, like local-llm test and Pipeline TTS.
'POST /api/continuous-video',
'POST /api/lora-training/runs/:id/promote-checkpoint',
+ // Explicit retired-data removal uses the same operator authority as host maintenance.
+ 'POST /api/data-maintenance/preview',
+ 'POST /api/data-maintenance/execute',
'POST /api/music/supercollider/render',
'POST /api/media-jobs/holds/:holdId/resume',
'POST /api/voice/piper/fetch',
diff --git a/server/lib/hostControlRoutes.test.js b/server/lib/hostControlRoutes.test.js
index cf84853e03..554894e894 100644
--- a/server/lib/hostControlRoutes.test.js
+++ b/server/lib/hostControlRoutes.test.js
@@ -413,16 +413,8 @@ describe('auxiliary media mutation inventory (#9672)', () => {
// Reviewed as record CRUD, cancellation/pruning or deterministic work that
// never hands caller text to an agent. A new mutation needs a new decision.
const recordOrContained = [
- 'DELETE /api/lora-datasets/:id',
- 'DELETE /api/lora-datasets/:id/images/:imageId',
'DELETE /api/media-jobs/:id',
- 'PATCH /api/lora-datasets/:id',
- 'PATCH /api/lora-datasets/:id/images/:imageId',
- 'POST /api/lora-datasets',
'POST /api/lora-datasets/:id/caption-runs/:runId/cancel',
- 'POST /api/lora-datasets/:id/import-gallery',
- 'POST /api/lora-datasets/:id/images',
- 'POST /api/lora-datasets/:id/strip-shared-fragments',
'POST /api/media-jobs/:id/cancel',
'POST /api/media-jobs/cancel-queued',
];
diff --git a/server/lib/index.js b/server/lib/index.js
index 88dce78a64..0010b64f5e 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -977,3 +977,5 @@ export * from './issueClaimEvidence.js';
export * from './codeReviewBatches.js';
export * from './realityStarterV1.js';
export * from './universeStarters.js';
+
+export * from './retiredLoraTraining.js';
diff --git a/server/lib/localProviderRuntime.test.js b/server/lib/localProviderRuntime.test.js
index 4ba13682e3..8139d5e748 100644
--- a/server/lib/localProviderRuntime.test.js
+++ b/server/lib/localProviderRuntime.test.js
@@ -286,7 +286,6 @@ describe('shipped copy names a Models page by route, never by literal (#7419)',
const ALLOWLISTED_FILES = new Set([
'server/lib/navManifest.js',
'server/services/loraDatasetCaption.js',
- 'client/src/components/loraTraining/CaptionModelPicker.jsx',
]);
/**
diff --git a/server/lib/mediaValidation.js b/server/lib/mediaValidation.js
index 3494cf850c..10e4d65f1a 100644
--- a/server/lib/mediaValidation.js
+++ b/server/lib/mediaValidation.js
@@ -177,31 +177,23 @@ const loraTrainingParamsSchema = z.object({
// latent-cache spill. `null` is the form's "Auto": a deliberate clear that
// forces the deriveMfluxMemoryConfig tier even when a saved default exists
// (distinct from absent, which lets the saved default merge through). An
- // explicit value still cannot exceed the LORA_TRAIN_MAX_QUANT_BITS cap.
+ // values remain accepted as historical configuration after retirement.
baseQuant: z.union([z.literal(4), z.literal(8), z.literal(16)]).nullable().optional(),
lowRam: z.boolean().nullable().optional(),
});
-// LoRA training settings slice (`settings.loraTraining`) — vision-caption
-// provider pick + training parameter defaults. Code-level defaults live in
-// `services/loraTraining/runtimes.js` so an absent slice needs no migration.
+// Retired settings remain accepted for older installs and clients. They no
+// longer authorize launch/resume; recovery still honors display and stall settings.
export const loraTrainingConfigSchema = z.object({
// Both nullable — the caption-model picker clears them to null on "Auto"
// (defer to the server's vision-model auto-pick).
captionProviderId: z.string().max(128).nullable().optional(),
captionModel: z.string().max(256).nullable().optional(),
defaults: loraTrainingParamsSchema.optional(),
- // Segmented mflux training (watchdog-panic mitigation, default ON in
- // services/loraTraining/runtimes.js). Setting this false runs the trainer as
- // one sustained process again — flip it once a macOS/mflux update resolves
- // the GPU-driver hang. Cooldown is the GPU idle gap (seconds) between segments.
+ // Historical parameters are inert and preserved, not rewritten on upgrade.
segmentation: z.boolean().optional(),
segmentCooldownSec: z.number().int().min(0).max(3600).optional(),
- // Phase-aware soft-hang stall watchdog (issue #1330, default ON in
- // services/loraTraining/index.js). Detects a wedged GPU mid-training (steps
- // stop arriving within a step-rate-derived budget) and SIGKILLs + auto-resumes
- // from the newest checkpoint. Set false to fall back to only the flat 30-min
- // idle watchdog (e.g. if a future driver fix makes soft hangs impossible).
+ // Surviving trainers retain stall detection; automatic resumes are retired.
stallWatchdog: z.boolean().optional(),
// Auto display-sleep during training on Apple Silicon (default ON — the
// `!== false` read lives in services/loraTraining/displayPower.js
@@ -213,17 +205,6 @@ export const loraTrainingConfigSchema = z.object({
displaySleep: z.boolean().optional(),
});
-// POST /api/lora-training/runs — start a training run for a dataset.
-export const startTrainingRunSchema = z.object({
- datasetId: z.string().min(1).max(128),
- baseModelId: z.string().min(1).max(128),
- name: z.string().trim().max(120).optional(),
- params: loraTrainingParamsSchema.optional(),
- // Override the caption identity-leak gate (see validateDatasetReady) and train
- // anyway — the UI sends this from the explicit "Train anyway" action.
- acknowledgeCaptionLeak: z.boolean().optional(),
-});
-
// === Local LLM backends (Ollama / LM Studio) ===
export const localLlmBackendSchema = z.enum(['ollama', 'lmstudio']);
// modelId is passed positionally to the `lms` CLI (execFile, no shell) — reject
diff --git a/server/lib/navManifest.js b/server/lib/navManifest.js
index b19c3e1715..69931e47af 100644
--- a/server/lib/navManifest.js
+++ b/server/lib/navManifest.js
@@ -337,6 +337,7 @@ const RAW_NAV_COMMANDS = [
{ id: 'nav.settings.autofixer', path: '/settings/autofixer', label: 'Autofixer', section: 'Settings', tabId: 'autofixer', aliases: ['autofixer', 'settings-autofixer', 'auto-fixer'], keywords: ['crash', 'fix', 'pm2', 'repair', 'ai provider', 'restart'] },
{ id: 'nav.settings.backup', path: '/settings/backup', label: 'Backup', section: 'Settings', tabId: 'backup', aliases: ['backup', 'settings-backup'] },
{ id: 'nav.settings.credentials', path: '/settings/credentials', label: 'Credentials', section: 'Settings', tabId: 'credentials', aliases: ['settings-credentials', 'credentials', 'api-keys', 'tokens'], keywords: ['configured', 'unconfigured', 'env', 'huggingface', 'civitai', 'jira', 'datadog', 'telegram'] },
+ { id: 'nav.settings.data-maintenance', path: '/settings/data-maintenance', label: 'Data Maintenance', section: 'Settings', tabId: 'data-maintenance', previousPaths: ['/media/training', '/media/training/:datasetId', '/models/training', '/models/training/:recordId'], preservePreviousPathSuffix: false, aliases: ['data-maintenance', 'retired-data', 'lora-training', 'training', 'datasets', 'train-lora', 'character-lora'], keywords: ['cleanup', 'obsolete', 'storage', 'delete', 'upgrade', 'migration'] },
{ id: 'nav.settings.database', path: '/settings/database', label: 'Database', section: 'Settings', tabId: 'database', aliases: ['settings-database', 'database'] },
{ id: 'nav.settings.features', path: '/settings/features', label: 'Features', section: 'Settings', tabId: 'features', aliases: ['settings-features', 'instance-features', 'feature-usage'], keywords: ['enabled', 'disabled', 'instance', 'optional', 'participation', 'metrics', 'reminders'] },
{ id: 'nav.settings.general', path: '/settings/general', label: 'General', section: 'Settings', tabId: 'general', aliases: ['settings', 'settings-general', 'general'] },
@@ -373,7 +374,6 @@ const RAW_NAV_COMMANDS = [
// catalog page even though this page starts those servers too.
{ id: 'nav.models.llms-runtimes', path: '/models/llms-runtimes', label: 'Runtimes', section: 'Models', tabId: 'llms-runtimes', navGroup: 'Connect', previousPaths: ['/models/llms/runtimes'], aliases: ['runtimes', 'llm-runtimes', 'local-runtimes', 'model-runtimes', 'llama-server', 'llama-cpp', 'slotstream', 'mtplx'], keywords: ['llama.cpp', 'llama-server', 'mtplx', 'slotstream', 'start server', 'stop server', 'local server', 'model server', 'speculative decoding', 'draft model', 'checkpoint', 'idle release'] },
{ id: 'nav.media.loras', path: '/models/loras', label: 'LoRAs', section: 'Models', tabId: 'loras', navGroup: 'Library', previousPaths: ['/media/loras'], aliases: ['loras', 'lora', 'lora-manager', 'civitai'], keywords: ['lora', 'civitai', 'fine-tune', 'style adapter', 'realstagram', 'photoreal', 'flux lora'] },
- { id: 'nav.media.training', path: '/models/training', label: 'Training', section: 'Models', tabId: 'training', navGroup: 'Library', previousPaths: ['/media/training', '/media/training/:datasetId'], aliases: ['training', 'lora-training', 'train-lora', 'datasets', 'character-lora'], keywords: ['fine-tune', 'dataset', 'caption', 'dreambooth', 'character consistency', 'train', 'flux lora'] },
{ id: 'nav.media.models', path: '/models/media', label: 'Media', section: 'Models', tabId: 'media', navGroup: 'Library', previousPaths: ['/media/models', '/media-models'], aliases: ['media-models', 'image-models', 'video-models', 'huggingface'], keywords: ['hf cache', 'model storage', 'disk', 'add model', 'install model', 'custom model'] },
{ id: 'nav.models.performance', path: '/models/performance', label: 'Performance', section: 'Models', tabId: 'performance', navGroup: 'Evaluate', aliases: ['model-performance', 'performance', 'assessments', 'model-assessments', 'benchmark-models', 'tuning'], keywords: ['measure', 'assessment', 'benchmark', 'throughput', 'chars per second', 'ttft', 'context', 'tuning', 'llama.cpp', 'mtplx', 'vllm', 'which model', 'fastest model'] },
{ id: 'nav.models.performance.results', path: '/models/performance/results', label: 'Performance Results', section: 'Models', aliases: ['performance-results', 'assessment-results'], keywords: ['measured results', 'model evidence', 'selected evidence', 'rankings'] },
diff --git a/server/lib/navManifest.test.js b/server/lib/navManifest.test.js
index b445831487..e6e133a4be 100644
--- a/server/lib/navManifest.test.js
+++ b/server/lib/navManifest.test.js
@@ -167,7 +167,7 @@ describe('nav contract — generated section child navigation', () => {
expect(getSectionNavGroups('Models').map(({ label, tabs }) => [label, tabs.map((tab) => tab.label)])).toEqual([
['Connect', ['AI Services', 'Fallbacks', 'Harnesses', 'Providers', 'Runtimes']],
['Evaluate', ['Comparison', 'Performance', 'Playground']],
- ['Library', ['3D', 'Embeddings', 'LoRAs', 'Media', 'Model Library', 'Training']],
+ ['Library', ['3D', 'Embeddings', 'LoRAs', 'Media', 'Model Library']],
['Operate', ['Quota Burn', 'Status', 'Subscriptions', 'Usage']],
['Policies', ['Abuse Guard', 'Code Reviewers', 'Decision Classifiers']],
]);
@@ -176,7 +176,7 @@ describe('nav contract — generated section child navigation', () => {
it('resolves a promoted drill-down as itself instead of its LLM parent', () => {
expect(getSectionNavTabForPath('Models', '/models/llms/abuse')).toMatchObject({ id: 'abuse', label: 'Abuse Guard' });
expect(getNavCommandForPath('/models/decision-classifiers/jev')).toMatchObject({ path: '/models/decision-classifiers/jev', label: 'Jev' });
- expect(getSectionNavTabForPath('Models', '/models/training/dataset-abc')).toMatchObject({ id: 'training' });
+ expect(getSectionNavTabForPath('Models', '/models/training/dataset-abc')).toBeUndefined();
});
it.each([
@@ -381,12 +381,12 @@ describe('resolveNavCommand — fuzzy matching', () => {
});
it('resolves the moved model-management pages to their Models paths', () => {
- // Media models, LoRAs, LoRA training and embeddings moved out of Create and
- // Settings (#4728). Their command ids are unchanged (they are opaque and
- // persisted), so only these paths prove the move actually landed.
+ // Media models, LoRAs and embeddings moved out of Create and
+ // Settings (#4728); retired training now lands in maintenance. Active command ids are opaque and
+ // persisted, so paths prove the move actually landed.
expect(resolveNavCommand('media models')?.path).toBe('/models/media');
expect(resolveNavCommand('loras')?.path).toBe('/models/loras');
- expect(resolveNavCommand('lora training')?.path).toBe('/models/training');
+ expect(resolveNavCommand('lora training')?.path).toBe('/settings/data-maintenance');
expect(resolveNavCommand('embeddings')?.path).toBe('/models/embeddings');
expect(resolveNavCommand('prompt guard')?.path).toBe('/models/llms/abuse');
expect(resolveNavCommand('abuse-guard')?.path).toBe('/models/llms/abuse');
@@ -751,11 +751,12 @@ describe('nav coverage — every navigable App.jsx route has a manifest entry',
// segment was a deep link into one record, so its redirect has to carry that
// segment across — swapping the PrefixRedirect for a bare <Navigate to="/models/training">
// still points at the right page and would pass the check above, while every
- // bookmarked dataset quietly lands on the index instead.
+ // bookmarked record quietly lands on the index instead. Retired features may
+ // explicitly opt out because their destination no longer selects old records.
it('preserves the record id when a parameterized previous path redirects', () => {
const lossy = NAV_COMMANDS
- .flatMap((c) => (c.previousPaths || []).map((from) => ({ from, id: c.id })))
- .filter(({ from }) => from.split('/').some((seg) => seg.startsWith(':')))
+ .flatMap((c) => (c.previousPaths || []).map((from) => ({ from, id: c.id, preserveSuffix: c.preservePreviousPathSuffix !== false })))
+ .filter(({ from, preserveSuffix }) => preserveSuffix && from.split('/').some((seg) => seg.startsWith(':')))
.map(({ from, id }) => ({ from, id, hit: byFrom.get(from) }))
.filter(({ hit }) => !hit || !SUFFIX_PRESERVING_ELEMENTS.has(hit.element))
.map(({ from, id, hit }) => `${id}: ${from} forwards via ${hit?.element ?? 'NO REDIRECT'}, which drops the trailing segment`);
@@ -773,11 +774,13 @@ describe('nav coverage — every navigable App.jsx route has a manifest entry',
'/media/models', // #4728 — the rest of model management left Create/Settings/Dev Tools
'/media/loras',
'/media/training',
+ '/media/training/:datasetId',
+ '/models/training',
+ '/models/training/:recordId',
'/settings/embeddings',
'/system-resources/models',
// The dataset workbench was its own deep-linkable path, and a bookmark into
// one dataset breaks just as silently as the index.
- '/media/training/:datasetId',
// #7567 — /ai became /ai/presets and the Backend Connections drawer
// became the Services view; a stored pin or bookmark on either must land.
'/ai',
diff --git a/server/lib/pythonSetup.js b/server/lib/pythonSetup.js
index 4746fbbd31..852867054d 100644
--- a/server/lib/pythonSetup.js
+++ b/server/lib/pythonSetup.js
@@ -432,9 +432,8 @@ export function invalidateFlux2Health() {
// symmetry with the other resolvers.
const MFLUX_TRAIN_BIN = IS_WIN ? 'mflux-train.exe' : 'mflux-train';
-// True when `mflux-train` sits next to `pythonPath` (the same probe
-// loraTraining's isMfluxTrainAvailable uses; kept here so resolveMfluxPython is
-// self-contained). Null/empty path → false.
+// Historical console-script probe retained for shared image-runtime discovery.
+// Retirement never uninstalls or changes which existing venv is selected.
export const hasMfluxTrain = (pythonPath) =>
!!pythonPath && existsSync(join(dirname(pythonPath), MFLUX_TRAIN_BIN));
@@ -448,13 +447,12 @@ const MFLUX_VENV_CANDIDATES = IS_WIN
join(PATHS.data, 'python', 'venv-mflux', 'bin', 'python3'),
];
-// Resolve the Python whose `mflux-train` PortOS should spawn for MLX LoRA
-// training. Preference order: (1) an explicitly-configured image-gen Python
+// Resolve the existing shared mflux Python. Preserve the historical preference
+// order: (1) an explicitly-configured image-gen Python
// that actually ships mflux-train — the historical `pip --user` layout, so
// existing installs behave exactly as before; (2) the first dedicated
// ~/.portos/venv-mflux that ships it. When neither ships mflux-train, return
-// the configured path unchanged (or null) so the caller's isMfluxTrainAvailable
-// still reports the honest "not installed" state instead of a phantom venv path.
+// the configured path unchanged (or null) rather than inventing a venv path.
export function resolveMfluxPython(configuredPath = null) {
if (hasMfluxTrain(configuredPath)) return configuredPath;
for (const p of MFLUX_VENV_CANDIDATES) {
diff --git a/server/lib/retiredLoraTraining.js b/server/lib/retiredLoraTraining.js
new file mode 100644
index 0000000000..a9f3f8c1e8
--- /dev/null
+++ b/server/lib/retiredLoraTraining.js
@@ -0,0 +1,37 @@
+import { ServerError } from './errorHandler.js';
+
+export const TRAINING_RETIRED_MESSAGE = 'LoRA training is retired. Existing datasets, checkpoints and adapters are preserved. Use reference images for new generation; manage retained files in Settings → Data Maintenance.';
+
+export function refuseRetiredTraining() {
+ throw new ServerError(TRAINING_RETIRED_MESSAGE, { status: 410, code: 'LORA_TRAINING_RETIRED' });
+}
+
+let removingRetiredData = false;
+let retainedDataUsers = 0;
+export function assertRetiredDataAvailable() {
+ if (removingRetiredData) throw new ServerError('Data maintenance is removing retired inputs. Retry after it finishes.', { status: 409, code: 'DATA_MAINTENANCE_ACTIVE' });
+}
+export async function withRetiredDataRemoval(work) {
+ assertRetiredDataAvailable();
+ if (retainedDataUsers > 0) throw new ServerError('Retained training data is being updated or a checkpoint is being promoted. Retry after it finishes.', { status: 409, code: 'MAINTENANCE_BUSY' });
+ removingRetiredData = true;
+ try { return await work(); }
+ finally { removingRetiredData = false; }
+}
+
+export function acquireRetiredDataAccess() {
+ assertRetiredDataAvailable();
+ retainedDataUsers += 1;
+ let released = false;
+ return () => {
+ if (released) return;
+ released = true;
+ retainedDataUsers -= 1;
+ };
+}
+
+export async function withRetiredDataAccess(work) {
+ const release = acquireRetiredDataAccess();
+ try { return await work(); }
+ finally { release(); }
+}
diff --git a/server/routes/dataMaintenance.js b/server/routes/dataMaintenance.js
new file mode 100644
index 0000000000..ff51dd82df
--- /dev/null
+++ b/server/routes/dataMaintenance.js
@@ -0,0 +1,21 @@
+import { Router } from 'express';
+import { z } from 'zod';
+import { asyncHandler } from '../lib/errorHandler.js';
+import { validateRequest } from '../lib/validation.js';
+import { requireHostControl } from '../services/authGate.js';
+import { executeMaintenance, getMaintenanceInventory, previewMaintenance } from '../services/dataMaintenance.js';
+
+const router = Router();
+const previewSchema = z.object({ operationId: z.string().min(1).max(100) }).strict();
+const executeSchema = z.object({ token: z.string().uuid(), confirmation: z.string().max(150) }).strict();
+router.use(requireHostControl);
+router.get('/', asyncHandler(async (_req, res) => res.json(await getMaintenanceInventory())));
+router.post('/preview', asyncHandler(async (req, res) => {
+ const { operationId } = validateRequest(previewSchema, req.body);
+ res.json(await previewMaintenance(operationId));
+}));
+router.post('/execute', asyncHandler(async (req, res) => {
+ const body = validateRequest(executeSchema, req.body);
+ res.json(await executeMaintenance(body));
+}));
+export default router;
diff --git a/server/routes/dataMaintenance.test.js b/server/routes/dataMaintenance.test.js
new file mode 100644
index 0000000000..939d0b9bf1
--- /dev/null
+++ b/server/routes/dataMaintenance.test.js
@@ -0,0 +1,134 @@
+import { beforeEach, afterAll, describe, expect, it, vi } from 'vitest';
+import { mkdtempSync, mkdirSync, writeFileSync, existsSync, rmSync, symlinkSync, realpathSync, renameSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import express from 'express';
+import { request } from '../lib/testHelper.js';
+import { makePathsProxy } from '../lib/mockPathsDataRoot.js';
+const root = realpathSync(mkdtempSync(join(tmpdir(), 'maintenance-fixture-')));
+const state = vi.hoisted(() => ({ jobs: [], busy: false, caption: null, authorized: true, failRemoval: false, recreate: false }));
+vi.mock('../lib/fileUtils.js', async original => {
+ const actual = await original();
+ return { ...makePathsProxy(actual, { dataRoot: root }), rmGuarded: async (path, options) => {
+ if (state.failRemoval) throw new Error('Synthetic storage failure');
+ await actual.rmGuarded(path, options);
+ if (state.recreate) {
+ mkdirSync(join(root, 'training-runs'));
+ writeFileSync(join(root, 'training-runs/new.bin'), 'new writer');
+ }
+ } };
+});
+vi.mock('../services/mediaJobQueue/index.js', () => ({ listJobs: () => state.jobs }));
+vi.mock('../services/dataManagerBusy.js', () => ({ trainingRunsBusy: async () => ({ busy: state.busy }) }));
+vi.mock('../services/loraDatasetCaption.js', () => ({ getActiveCaptionRun: () => state.caption }));
+vi.mock('../lib/backupSnapshotBoundary.js', () => ({ withBackupAssetPublication: work => work() }));
+vi.mock('../services/authGate.js', () => ({ requireHostControl: (_req, res, next) => state.authorized ? next() : res.sendStatus(403) }));
+const { default: routes } = await import('./dataMaintenance.js');
+const app = express(); app.use(express.json()); app.use('/api/data-maintenance', routes);
+app.use((error, _req, res, _next) => res.status(error.status || 500).json({ error: error.message, code: error.code }));
+const preview = () => request(app).post('/api/data-maintenance/preview').send({ operationId: 'retired-training-runs' });
+const execute = p => request(app).post('/api/data-maintenance/execute').send({ token: p.token, confirmation: p.confirmation });
+beforeEach(() => {
+ rmSync(root, { recursive: true, force: true });
+ for (const path of ['training-runs/run-example', 'lora-datasets/example/images', 'loras']) mkdirSync(join(root, path), { recursive: true });
+ writeFileSync(join(root, 'training-runs/run-example/checkpoint.bin'), 'unique checkpoint');
+ writeFileSync(join(root, 'lora-datasets/example/images/source.png'), 'unique source');
+ writeFileSync(join(root, 'loras/example.safetensors'), 'shared adapter');
+ state.jobs = []; state.busy = false; state.caption = null; state.authorized = true; state.failRemoval = false; state.recreate = false;
+});
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+describe('allowlisted data maintenance HTTP workflow', () => {
+ it('previews without mutating, requires exact confirmation, removes only approved artifacts and replays idempotently', async () => {
+ const inventory = await request(app).get('/api/data-maintenance');
+ expect(inventory.body.find(row => row.id === 'retired-training-runs')).toMatchObject({ status: 'retained', files: 1, bytes: 17 });
+ const p = (await preview()).body;
+ expect(existsSync(join(root, 'training-runs/run-example/checkpoint.bin'))).toBe(true);
+ expect((await request(app).post('/api/data-maintenance/execute').send({ token: p.token, confirmation: 'DELETE' })).status).toBe(400);
+ const [one, two] = await Promise.all([execute(p), execute(p)]);
+ expect(one.status).toBe(200); expect(two.body).toEqual(one.body);
+ expect(existsSync(join(root, 'training-runs'))).toBe(false);
+ expect(existsSync(join(root, 'loras/example.safetensors'))).toBe(true);
+ expect(existsSync(join(root, 'lora-datasets/example/images/source.png'))).toBe(true);
+ });
+ it('rejects stale inventories and refuses a job that starts after preview', async () => {
+ const p = (await preview()).body;
+ writeFileSync(join(root, 'training-runs/run-example/new.bin'), 'new');
+ expect((await execute(p)).body.code).toBe('MAINTENANCE_STALE');
+ const fresh = (await preview()).body;
+ state.jobs = [{ status: 'running', kind: 'image' }];
+ expect((await execute(fresh)).body.code).toBe('MAINTENANCE_BUSY');
+ expect(existsSync(join(root, 'training-runs/run-example/new.bin'))).toBe(true);
+ });
+ it('refuses surviving trainers, captioning, arbitrary paths, shared assets, unknown previews and unauthorized callers', async () => {
+ state.busy = true; expect((await preview()).body.code).toBe('MAINTENANCE_BUSY');
+ state.busy = false; state.caption = { status: 'running' }; expect((await preview()).body.code).toBe('MAINTENANCE_BUSY');
+ state.caption = null;
+ for (const operationId of ['../loras', 'shared-lora-adapters', 'archives']) expect((await request(app).post('/api/data-maintenance/preview').send({ operationId })).status).toBe(404);
+ expect((await execute({ token: '00000000-0000-4000-8000-000000000001', confirmation: 'DELETE training-runs' })).body.code).toBe('MAINTENANCE_STALE');
+ state.authorized = false; expect((await preview()).status).toBe(403);
+ });
+ it('refuses links without following their targets', async () => {
+ symlinkSync(join(root, 'loras'), join(root, 'training-runs/run-example/link'));
+ expect((await preview()).body.code).toBe('MAINTENANCE_UNSAFE');
+ const rows = (await request(app).get('/api/data-maintenance')).body;
+ expect(rows.find(row => row.id === 'retired-training-runs').status).toBe('unavailable');
+ expect(existsSync(join(root, 'loras/example.safetensors'))).toBe(true);
+ });
+});
+
+it('refuses removal while checkpoint promotion or a retained dataset write owns the data', async () => {
+ const p = (await preview()).body;
+ const { withRetiredDataAccess } = await import('../lib/retiredLoraTraining.js');
+ let release;
+ const access = withRetiredDataAccess(() => new Promise(resolve => { release = resolve; }));
+ try {
+ expect((await execute(p)).body.code).toBe('MAINTENANCE_BUSY');
+ expect(existsSync(join(root, 'training-runs/run-example/checkpoint.bin'))).toBe(true);
+ } finally { release(); await access; }
+ expect((await execute(p)).status).toBe(200);
+});
+
+it('expires previews and rejects a replaced directory without removing its replacement', async () => {
+ const p = (await preview()).body;
+ const actualNow = Date.now;
+ vi.spyOn(Date, 'now').mockReturnValue(actualNow() + 6 * 60_000);
+ try { expect((await execute(p)).body.code).toBe('MAINTENANCE_STALE'); }
+ finally { vi.restoreAllMocks(); }
+ const fresh = (await preview()).body;
+ rmSync(join(root, 'training-runs'), { recursive: true });
+ mkdirSync(join(root, 'training-runs'));
+ writeFileSync(join(root, 'training-runs/new.bin'), 'replacement');
+ expect((await execute(fresh)).body.code).toBe('MAINTENANCE_STALE');
+ expect(existsSync(join(root, 'training-runs/new.bin'))).toBe(true);
+});
+
+it('reports filesystem failure without success and restores the untouched detached tree', async () => {
+ const p = (await preview()).body;
+ state.failRemoval = true;
+ const result = await execute(p);
+ expect(result.status).toBe(409);
+ expect(result.body.code).toBe('MAINTENANCE_INCOMPLETE');
+ expect(existsSync(join(root, 'training-runs/run-example/checkpoint.bin'))).toBe(true);
+});
+
+it('reports newly appearing files as retained instead of falsely reporting an empty category', async () => {
+ const p = (await preview()).body;
+ state.recreate = true;
+ const result = await execute(p);
+ expect(result.status).toBe(200);
+ expect(result.body).toMatchObject({ status: 'retained', remainingFiles: 1, remainingBytes: 10 });
+ expect(existsSync(join(root, 'training-runs/new.bin'))).toBe(true);
+});
+
+it('refuses a linked data root instead of following it into another location', async () => {
+ const moved = `${root}-moved`;
+ renameSync(root, moved);
+ try {
+ symlinkSync(moved, root, process.platform === 'win32' ? 'junction' : 'dir');
+ expect((await preview()).body.code).toBe('MAINTENANCE_UNSAFE');
+ expect(existsSync(join(moved, 'training-runs/run-example/checkpoint.bin'))).toBe(true);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ renameSync(moved, root);
+ }
+});
diff --git a/server/routes/hostControlAuxiliaryMedia.test.js b/server/routes/hostControlAuxiliaryMedia.test.js
index e1848b7085..0de98de33d 100644
--- a/server/routes/hostControlAuxiliaryMedia.test.js
+++ b/server/routes/hostControlAuxiliaryMedia.test.js
@@ -75,11 +75,11 @@ const operations = [
['post', '/api/media-jobs/example-job/retry', retryBody, effects.enqueue, 200],
['post', '/api/media-jobs/example-job/run-now', {}, effects.runNow, 200],
['post', '/api/lora-datasets/example-dataset/generate',
- { count: 1, poses: ['Example pose'], outfits: ['Example outfit'] }, effects.generate, 202],
+ { count: 1, poses: ['Example pose'], outfits: ['Example outfit'] }, effects.generate, 410],
['post', '/api/lora-datasets/example-dataset/slice-reference-sheet',
- { captionProviderId: 'example-vision-cli' }, effects.slice, 201],
+ { captionProviderId: 'example-vision-cli' }, effects.slice, 410],
['post', '/api/lora-datasets/example-dataset/caption',
- { providerId: 'example-vision-cli' }, effects.caption, 202],
+ { providerId: 'example-vision-cli' }, effects.caption, 410],
];
const appFor = (address = '192.0.2.10') => {
const app = express();
@@ -145,7 +145,7 @@ describe('auxiliary media agent dispatch authority (#9672)', () => {
const before = operation[3].mock.calls.length;
const response = await call(appFor(address), operation, headers);
expect(response.status, `${operation[1]}: ${JSON.stringify(response.body)}`).toBe(operation[4]);
- expect(operation[3]).toHaveBeenCalledTimes(before + 1);
+ expect(operation[3]).toHaveBeenCalledTimes(before + (operation[4] === 410 ? 0 : 1));
}
}
// Retry keeps the original agent backend while taking the new prompt.
diff --git a/server/routes/hostControlPreGuardTracksData.test.js b/server/routes/hostControlPreGuardTracksData.test.js
index 48fd35fb0a..6e4fddb772 100644
--- a/server/routes/hostControlPreGuardTracksData.test.js
+++ b/server/routes/hostControlPreGuardTracksData.test.js
@@ -98,10 +98,10 @@ vi.mock('../services/games/index.js', () => ({
}));
vi.mock('../services/dataManager.js', () => ({ archiveCategory: effects.archive }));
vi.mock('../services/loraTraining/index.js', () => ({
+ cancel: vi.fn(() => false),
getRunRequired: effects.getRun,
promoteCheckpoint: effects.promote,
}));
-vi.mock('../services/loraTraining/runtimes.js', () => ({ TRAINING_DEFAULTS: {} }));
vi.mock('../lib/pythonSetup.js', () => ({}));
vi.mock('../services/signalSync.js', () => ({ checkSetup: effects.signalSetup, runSync: effects.signalSync }));
vi.mock('../services/dataSync.js', () => ({ applyRemote: effects.applyRemote }));
diff --git a/server/routes/loraDatasets.caption.test.js b/server/routes/loraDatasets.caption.test.js
index e4fa1d9b01..e493d04192 100644
--- a/server/routes/loraDatasets.caption.test.js
+++ b/server/routes/loraDatasets.caption.test.js
@@ -74,3 +74,16 @@ describe('LoRA dataset caption-run routes', () => {
expect(res.status).toBe(404);
});
});
+
+it('refuses every retired dataset mutation before reaching a producer or deleting retained files', async () => {
+ for (const [method, path] of [
+ ['post', ''], ['patch', '/example'], ['delete', '/example'], ['post', '/example/images'],
+ ['post', '/example/import-gallery'], ['post', '/example/generate'], ['post', '/example/caption'],
+ ['post', '/example/slice-reference-sheet'], ['post', '/example/strip-shared-fragments'],
+ ['patch', '/example/images/image-example'], ['delete', '/example/images/image-example'],
+ ]) {
+ const response = await request(makeApp())[method](`/api/lora-datasets${path}`).send({});
+ expect(response.status).toBe(410);
+ expect(response.body.code).toBe('LORA_TRAINING_RETIRED');
+ }
+});
diff --git a/server/routes/loraDatasets.js b/server/routes/loraDatasets.js
index 4513739205..38988792ef 100644
--- a/server/routes/loraDatasets.js
+++ b/server/routes/loraDatasets.js
@@ -1,40 +1,27 @@
-/**
- * LoRA training dataset routes — `/api/lora-datasets`.
- *
- * CRUD + image upload (multipart) + batch generation + reference-sheet
- * slicing + vision captioning (SSE) for the per-subject training
- * datasets behind /models/training. Training-run launch lives at
- * `/api/lora-training` (routes/loraTraining.js); this surface ends at
- * "dataset ready to train".
- */
-
+import { refuseRetiredTraining } from '../lib/retiredLoraTraining.js';
+/** Read-only retired datasets and explicit cancellation of outstanding captions. */
import { Router } from 'express';
import { z } from 'zod';
import { asyncHandler, ServerError } from '../lib/errorHandler.js';
import { validateRequest, isPaginationRequested, paginateArray } from '../lib/validation.js';
-import { uploadFields } from '../lib/multipart.js';
-import {
- addUploadedImage,
- createDataset,
- deleteDataset,
- deleteImage,
- getDataset,
- importGalleryImages,
- listDatasets,
- patchDataset,
- reconcileRenderingImages,
- stripSharedCaptionFragments,
- updateImageCaption,
-} from '../services/loraDatasets.js';
-import { generateDatasetImages, getDatasetVariationAxes, sliceReferenceSheet } from '../services/loraDatasetGenerate.js';
-import {
- attachCaptionSseClient, cancelCaptionRun, getActiveCaptionRun, startCaptionRun,
-} from '../services/loraDatasetCaption.js';
+
+import { listDatasets, reconcileRenderingImages } from '../services/loraDatasets.js';
+
+import { attachCaptionSseClient, cancelCaptionRun, getActiveCaptionRun } from '../services/loraDatasetCaption.js';
import { LORA_DATASET_ENTRY_KINDS, computeDatasetReadiness } from '../lib/loraDataset.js';
const router = Router();
+// Keep history/image reads and explicit caption cancellation available. Legacy
+// clients cannot create, rewrite, render, caption, or delete training datasets.
+router.use((req, _res, next) => {
+ if (req.method !== 'GET' && !/^\/[^/]+\/caption-runs\/[^/]+\/cancel$/.test(req.path)) refuseRetiredTraining();
+ next();
+});
+
+router.post('/:id/generate', refuseRetiredTraining);
+router.post('/:id/caption', refuseRetiredTraining);
+router.post('/:id/slice-reference-sheet', refuseRetiredTraining);
-const triggerWordSchema = z.string().regex(/^[a-z0-9_]{2,64}$/, 'trigger word must be 2-64 chars of [a-z0-9_]');
const idSchema = z.string().min(1).max(128);
const entryKindSchema = z.enum(LORA_DATASET_ENTRY_KINDS);
@@ -51,18 +38,6 @@ router.get('/', asyncHandler(async (req, res) => {
res.json(paginateArray(datasets, req.query, { defaultLimit: 50, maxLimit: 500 }));
}));
-const createSchema = z.object({
- universeId: idSchema,
- entryKind: entryKindSchema.default('characters'),
- entryId: idSchema,
- triggerWord: triggerWordSchema.optional(),
-});
-router.post('/', asyncHandler(async (req, res) => {
- const body = validateRequest(createSchema, req.body);
- const { dataset, created } = await createDataset(body);
- res.status(created ? 201 : 200).json(dataset);
-}));
-
router.get('/:id', asyncHandler(async (req, res) => {
// Heal any images stuck in 'rendering' (server restart dropped the live
// completion hook) before returning — the grid then shows truth.
@@ -72,108 +47,6 @@ router.get('/:id', asyncHandler(async (req, res) => {
res.json({ ...dataset, readiness: computeDatasetReadiness(dataset), captionRun: getActiveCaptionRun(req.params.id) });
}));
-// Live variation axes (expressions/outfits for characters; lighting/settings
-// for objects & places) so the generate-batch dialog seeds its override chips
-// from the server vocab instead of duplicating the object/place axis constants.
-router.get('/:id/variation-axes', asyncHandler(async (req, res) => {
- res.json(await getDatasetVariationAxes(req.params.id));
-}));
-
-const patchSchema = z.object({
- triggerWord: triggerWordSchema.optional(),
- // Reassign the dataset to a different universe subject. Universe + entry id
- // must be present together; the service re-snapshots the subject and refuses
- // a collision with an existing dataset for that subject.
- universeId: idSchema.optional(),
- entryKind: entryKindSchema.optional(),
- entryId: idSchema.optional(),
-});
-router.patch('/:id', asyncHandler(async (req, res) => {
- const body = validateRequest(patchSchema, req.body);
- const dataset = await patchDataset(req.params.id, body);
- res.json({ ...dataset, readiness: computeDatasetReadiness(dataset) });
-}));
-
-router.delete('/:id', asyncHandler(async (req, res) => {
- res.json(await deleteDataset(req.params.id));
-}));
-
-// Image upload — up to 10 files per request on image1…image10 field names.
-const UPLOAD_FIELDS = Array.from({ length: 10 }, (_, i) => `image${i + 1}`);
-const ACCEPTED_MIME = new Set(['image/png', 'image/jpeg', 'image/webp']);
-const MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
-const datasetUploads = uploadFields(UPLOAD_FIELDS, {
- limits: { fileSize: MAX_UPLOAD_BYTES },
- fileFilter: (_req, file, cb) => cb(null, ACCEPTED_MIME.has((file.mimetype || '').toLowerCase())),
-});
-router.post('/:id/images', datasetUploads, asyncHandler(async (req, res) => {
- const staged = Object.values(req.files || {});
- if (!staged.length) {
- throw new ServerError('No image files in request (use field names image1…image10)', {
- status: 400, code: 'VALIDATION_ERROR',
- });
- }
- const entries = [];
- for (const file of staged) {
- entries.push(await addUploadedImage(req.params.id, {
- tmpPath: file.path, originalname: file.originalname,
- }));
- }
- res.status(201).json({ images: entries });
-}));
-
-// Import existing gallery images into the dataset by basename. The server
-// re-normalizes each through sharp (gallery PNGs are `<jobId>.png`).
-const importGallerySchema = z.object({
- filenames: z.array(z.string().max(256).regex(/^[^/\\]+\.png$/i, 'filename must be a gallery PNG basename')).min(1).max(50),
-});
-router.post('/:id/import-gallery', asyncHandler(async (req, res) => {
- const { filenames } = validateRequest(importGallerySchema, req.body);
- const images = await importGalleryImages(req.params.id, { filenames });
- res.status(201).json({ images });
-}));
-
-const generateSchema = z.object({
- count: z.number().int().min(1).max(40).default(12),
- views: z.array(z.string().max(120)).max(12).optional(),
- poses: z.array(z.string().max(120)).max(12).optional(),
- expressions: z.array(z.string().max(120)).max(12).optional(),
- outfits: z.array(z.string().max(120)).max(12).optional(),
- modelId: z.string().max(128).optional(),
-});
-router.post('/:id/generate', asyncHandler(async (req, res) => {
- const body = validateRequest(generateSchema, req.body);
- res.status(202).json(await generateDatasetImages(req.params.id, body));
-}));
-
-const sliceSchema = z.object({
- variant: z.string().max(64).default('standard'),
- cols: z.number().int().min(1).max(6).default(3),
- rows: z.number().int().min(1).max(6).default(2),
- // When true (default) a vision-LLM proposes a bounding box per figure and the
- // grid is the fallback; false forces the deterministic cols×rows grid.
- useVision: z.boolean().default(true),
- // Optional explicit vision model for the proposal pass — same provider/model
- // pair the caption picker uses; omitted = server auto-pick.
- captionProviderId: z.string().max(128).optional(),
- captionModel: z.string().max(256).optional(),
-});
-router.post('/:id/slice-reference-sheet', asyncHandler(async (req, res) => {
- const body = validateRequest(sliceSchema, req.body);
- res.status(201).json(await sliceReferenceSheet(req.params.id, body));
-}));
-
-const captionSchema = z.object({
- imageIds: z.array(idSchema).max(200).optional(),
- providerId: z.string().max(128).optional(),
- model: z.string().max(256).optional(),
- overwrite: z.boolean().default(false),
-});
-router.post('/:id/caption', asyncHandler(async (req, res) => {
- const body = validateRequest(captionSchema, req.body);
- res.status(202).json(await startCaptionRun(req.params.id, body));
-}));
-
router.get('/:id/caption-runs/:runId/events', asyncHandler(async (req, res) => {
if (!attachCaptionSseClient(req.params.id, req.params.runId, res)) {
throw new ServerError(`Caption run not found: ${req.params.runId}`, { status: 404, code: 'NOT_FOUND' });
@@ -187,21 +60,4 @@ router.post('/:id/caption-runs/:runId/cancel', asyncHandler(async (req, res) =>
res.status(202).json(cancelCaptionRun(req.params.id, req.params.runId));
}));
-// Bulk caption lint — strip the identity fragments shared across most captions
-// so the trigger token (not the caption phrases) learns the character. The
-// server recomputes the shared set itself; no request body.
-router.post('/:id/strip-shared-fragments', asyncHandler(async (req, res) => {
- res.json(await stripSharedCaptionFragments(req.params.id));
-}));
-
-const imagePatchSchema = z.object({ caption: z.string().max(2000) });
-router.patch('/:id/images/:imageId', asyncHandler(async (req, res) => {
- const { caption } = validateRequest(imagePatchSchema, req.body);
- res.json(await updateImageCaption(req.params.id, req.params.imageId, caption));
-}));
-
-router.delete('/:id/images/:imageId', asyncHandler(async (req, res) => {
- res.json(await deleteImage(req.params.id, req.params.imageId));
-}));
-
export default router;
diff --git a/server/routes/loraTraining.js b/server/routes/loraTraining.js
index d1574856f6..cf186bff11 100644
--- a/server/routes/loraTraining.js
+++ b/server/routes/loraTraining.js
@@ -1,160 +1,64 @@
-/**
- * LoRA training run routes — `/api/lora-training`.
- *
- * Run lifecycle (start/list/detail/cancel/delete), per-run SSE progress
- * (delegated to the media-job queue's stream for the run's jobId), sample
- * image serving, and a runtime-readiness status endpoint for the UI.
- */
-
+/** Retained run history and survivor controls; training launch is retired. */
import { Router } from 'express';
import { join } from 'path';
import { z } from 'zod';
import { asyncHandler, sendErrorResponse, ServerError } from '../lib/errorHandler.js';
-import { startTrainingRunSchema, validateRequest } from '../lib/validation.js';
+import { validateRequest } from '../lib/validation.js';
import { assertSafeFilename } from '../lib/fileUtils.js';
-import { resolveFlux2Python, isFlux2VenvHealthy, resolveMfluxPython } from '../lib/pythonSetup.js';
-import { getSettings } from '../services/settings.js';
+import { refuseRetiredTraining, TRAINING_RETIRED_MESSAGE } from '../lib/retiredLoraTraining.js';
import { attachSseClient, cancelJob } from '../services/mediaJobQueue/index.js';
-import {
- deleteTrainingRun,
- getRunRequired,
- isMfluxTrainAvailable,
- listCheckpoints,
- listRuns,
- listSamples,
- promoteCheckpoint,
- resumeTrainingRun,
- runSamplesDir,
- startTrainingRun,
-} from '../services/loraTraining/index.js';
-import { TRAINING_DEFAULTS } from '../services/loraTraining/runtimes.js';
+import { cancel as cancelTrainer, getRunRequired, listCheckpoints, listRuns, listSamples, promoteCheckpoint, runSamplesDir } from '../services/loraTraining/index.js';
const router = Router();
-
-// Engine readiness + defaults — drives the launch panel's enable/disable
-// state and the params form's initial values. Both engines train FLUX.2
-// Klein LoRAs; mflux (MLX-native) is preferred when its trainer ships
-// with the user's install, the torch venv is the fallback.
-router.get('/status', asyncHandler(async (_req, res) => {
- const settings = await getSettings();
- // Prefer the configured image-gen Python when it ships mflux-train, else the
- // auto-discovered dedicated ~/.portos/venv-mflux — so the panel reports mflux
- // ready even when the user never set pythonPath (fresh dedicated-venv install).
- const mfluxPython = resolveMfluxPython(settings?.imageGen?.local?.pythonPath || null);
- const flux2Python = resolveFlux2Python();
- res.json({
- runtimes: {
- mflux: { ready: isMfluxTrainAvailable(mfluxPython), pythonPath: mfluxPython },
- flux2: { ready: !!flux2Python && await isFlux2VenvHealthy(), venvPython: flux2Python },
- },
- defaults: { ...TRAINING_DEFAULTS, ...(settings?.loraTraining?.defaults || {}) },
- });
-}));
-
-router.post('/runs', asyncHandler(async (req, res) => {
- const body = validateRequest(startTrainingRunSchema, req.body);
- res.status(202).json(await startTrainingRun(body));
-}));
-
+router.use((req, _res, next) => {
+ if (req.method !== 'GET' && !(req.method === 'POST' && /^\/runs\/[^/]+\/(cancel|promote-checkpoint)$/.test(req.path))) refuseRetiredTraining();
+ next();
+});
+// Explicit tombstones preserve legacy route authorization and return 410.
+router.post('/runs', refuseRetiredTraining);
+router.post('/runs/:id/resume', refuseRetiredTraining);
+router.get('/status', (_req, res) => res.json({ retired: true, message: TRAINING_RETIRED_MESSAGE, runtimes: { mflux: { ready: false }, flux2: { ready: false } } }));
const listQuerySchema = z.object({
status: z.enum(['queued', 'running', 'completed', 'failed', 'canceled']).optional(),
- characterId: z.string().max(128).optional(),
- datasetId: z.string().max(128).optional(),
+ characterId: z.string().max(128).optional(), datasetId: z.string().max(128).optional(),
limit: z.coerce.number().int().min(1).max(500).optional(),
});
-router.get('/runs', asyncHandler(async (req, res) => {
- const query = validateRequest(listQuerySchema, req.query);
- res.json(await listRuns(query));
-}));
-
-router.get('/runs/:id', asyncHandler(async (req, res) => {
- res.json(await getRunRequired(req.params.id));
-}));
-
-// Live progress — the run's SSE stream IS the media-job stream. The queue
-// synthesizes a terminal frame for archived jobs, so late attaches after
-// completion still resolve.
+router.get('/runs', asyncHandler(async (req, res) => res.json(await listRuns(validateRequest(listQuerySchema, req.query)))));
+router.get('/runs/:id', asyncHandler(async (req, res) => res.json(await getRunRequired(req.params.id))));
router.get('/runs/:id/events', asyncHandler(async (req, res) => {
const run = await getRunRequired(req.params.id);
- if (!run.jobId || !attachSseClient(run.jobId, res)) {
- throw new ServerError(`No live stream for run ${req.params.id} (status: ${run.status})`, {
- status: 404, code: 'NOT_FOUND',
- });
- }
+ if (!run.jobId || !attachSseClient(run.jobId, res)) throw new ServerError('No live stream for this retained run', { status: 404, code: 'NOT_FOUND' });
}));
-
router.post('/runs/:id/cancel', asyncHandler(async (req, res) => {
const run = await getRunRequired(req.params.id);
- if (!run.jobId) {
- throw new ServerError('Run has no job to cancel', { status: 409, code: 'ALREADY_TERMINAL' });
- }
- const result = await cancelJob(run.jobId);
- if (!result.ok) {
- const status = result.code === 'NOT_FOUND' ? 404 : 409;
- throw new ServerError(result.error || 'Cancel failed', { status, code: result.code });
+ const result = run.jobId ? await cancelJob(run.jobId) : { ok: false, code: 'NOT_FOUND', error: 'Run has no queue job' };
+ if (!result.ok && result.code === 'NOT_FOUND' && cancelTrainer(run.jobId || run.id)) {
+ res.json({ ok: true, status: 'canceling', orphan: true });
+ return;
}
+ if (!result.ok) throw new ServerError(result.error || 'Cancel failed', { status: !run.jobId ? 409 : result.code === 'NOT_FOUND' ? 404 : 409, code: !run.jobId ? 'ALREADY_TERMINAL' : result.code });
res.json(result);
}));
-
-// Resume a failed/canceled run from its latest checkpoint — re-enqueues a job
-// against the SAME run id (mflux bakes the output path into the checkpoint, so
-// the resumed trainer must write back into the original run dir). 202-shaped.
-router.post('/runs/:id/resume', asyncHandler(async (req, res) => {
- await getRunRequired(req.params.id); // 404 if the run is unknown
- res.status(202).json(await resumeTrainingRun(req.params.id));
-}));
-
-const deleteRunQuerySchema = z.object({ deleteLora: z.enum(['true', 'false']).optional() });
-router.delete('/runs/:id', asyncHandler(async (req, res) => {
- const { deleteLora } = validateRequest(deleteRunQuerySchema, req.query);
- res.json(await deleteTrainingRun(req.params.id, { withLora: deleteLora === 'true' }));
-}));
-
-// Checkpoint picker: list every saved checkpoint with its step, loss, and
-// preview thumbnail so the user can promote one by eye. Loss is shown but is
-// NOT a quality ranking — it was anti-correlated with quality on the
-// divergence run this feature was built for (see checkpoints.js).
router.get('/runs/:id/checkpoints', asyncHandler(async (req, res) => {
- await getRunRequired(req.params.id); // 404 if the run is unknown
- res.json(await listCheckpoints(req.params.id));
+ await getRunRequired(req.params.id); res.json(await listCheckpoints(req.params.id));
}));
-
const promoteSchema = z.object({ step: z.coerce.number().int().min(0) });
router.post('/runs/:id/promote-checkpoint', asyncHandler(async (req, res) => {
await getRunRequired(req.params.id);
const { step } = validateRequest(promoteSchema, req.body);
res.json(await promoteCheckpoint(req.params.id, step));
}));
-
-// Mid-training sample timeline (step + thumbnail URL) — seeds the live
-// progress gallery so a reload mid-run shows every sample so far, not just
-// the ones streamed after re-subscribing. Registered before the
-// `/samples/:filename` serve route (which needs the extra path segment).
router.get('/runs/:id/samples', asyncHandler(async (req, res) => {
- await getRunRequired(req.params.id); // 404 if the run is unknown
- res.json(await listSamples(req.params.id));
+ await getRunRequired(req.params.id); res.json(await listSamples(req.params.id));
}));
-
router.get('/runs/:id/samples/:filename', asyncHandler(async (req, res) => {
const run = await getRunRequired(req.params.id);
assertSafeFilename(req.params.filename, { extensions: ['.png'], subject: 'sample filename' });
- // The sendFile callback fires outside asyncHandler's catch (the handler's
- // promise has already resolved), so a throw here would have nothing to bubble
- // to — build the standard envelope directly instead.
- res.sendFile(join(runSamplesDir(run.id), req.params.filename), (err) => {
+ res.sendFile(join(runSamplesDir(run.id), req.params.filename), err => {
if (!err) return;
- // Client cancelled (routine when the gallery unmounts an <img>) or the body
- // was already streaming when the read failed — nothing left to say, just
- // drop the socket rather than logging a false failure.
- if (err.code === 'ECONNABORTED' || res.headersSent) {
- res.destroy();
- return;
- }
- // A real I/O failure (EACCES/EIO) reads the same as a missing file to the
- // client — log the cause so it isn't invisible server-side.
+ if (err.code === 'ECONNABORTED' || res.headersSent) { res.destroy(); return; }
console.error(`❌ Sample send failed [${req.params.id}/${req.params.filename}]: ${err.message}`);
sendErrorResponse(res, new ServerError('Sample not found', { status: 404 }));
});
}));
-
export default router;
diff --git a/server/routes/loraTraining.test.js b/server/routes/loraTraining.test.js
index 9921de129d..d7b26b1da7 100644
--- a/server/routes/loraTraining.test.js
+++ b/server/routes/loraTraining.test.js
@@ -9,10 +9,10 @@ import { request } from '../lib/testHelper.js';
// stub every collaborator so the suite only exercises the HTTP layer.
const svc = vi.hoisted(() => ({ samplesDir: '' }));
vi.mock('../services/loraTraining/index.js', () => ({
+ cancel: vi.fn(() => false),
clearDatasetForDeletedLora: vi.fn(),
deleteRun: vi.fn(),
getRunRequired: vi.fn(async (id) => ({ id, status: 'completed' })),
- isMfluxTrainAvailable: () => false,
listCheckpoints: vi.fn(),
listRuns: vi.fn(),
listSamples: vi.fn(),
@@ -22,7 +22,6 @@ vi.mock('../services/loraTraining/index.js', () => ({
runSamplesDir: () => svc.samplesDir,
startTrainingRun: vi.fn(),
}));
-vi.mock('../services/loraTraining/runtimes.js', () => ({ TRAINING_DEFAULTS: {} }));
vi.mock('../services/settings.js', () => ({ getSettings: async () => ({}) }));
vi.mock('../services/mediaJobQueue/index.js', () => ({
attachSseClient: vi.fn(),
@@ -71,3 +70,35 @@ describe('lora-training sample serving', () => {
expect(typeof r.body.timestamp).toBe('number');
});
});
+
+it('retires launch, resume and legacy deletion while retaining history controls', async () => {
+ const app = makeApp();
+ for (const [method, path] of [['post', '/runs'], ['post', '/runs/example/resume'], ['delete', '/runs/example']]) {
+ const response = await request(app)[method](`/api/lora-training${path}`).send({});
+ expect(response.status).toBe(410);
+ expect(response.body.code).toBe('LORA_TRAINING_RETIRED');
+ }
+ const status = await request(app).get('/api/lora-training/status');
+ expect(status.body).toMatchObject({ retired: true, runtimes: { mflux: { ready: false }, flux2: { ready: false } } });
+});
+
+it('can explicitly cancel an observed orphan when the queue record is missing', async () => {
+ const { cancel } = await import('../services/loraTraining/index.js');
+ const { cancelJob } = await import('../services/mediaJobQueue/index.js');
+ const { getRunRequired } = await import('../services/loraTraining/index.js');
+ vi.mocked(getRunRequired).mockResolvedValueOnce({ id: 'example-run', jobId: 'example-job', status: 'running' });
+ vi.mocked(cancelJob).mockResolvedValueOnce({ ok: false, code: 'NOT_FOUND' });
+ vi.mocked(cancel).mockReturnValueOnce(true);
+ const response = await request(makeApp()).post('/api/lora-training/runs/example-run/cancel');
+ expect(response.status).toBe(200);
+ expect(response.body).toMatchObject({ ok: true, status: 'canceling', orphan: true });
+ expect(cancel).toHaveBeenCalledWith('example-job');
+});
+
+it('preserves the terminal-run cancellation contract when no queue job or observer exists', async () => {
+ const { getRunRequired } = await import('../services/loraTraining/index.js');
+ vi.mocked(getRunRequired).mockResolvedValueOnce({ id: 'example-ended', jobId: null, status: 'completed' });
+ const response = await request(makeApp()).post('/api/lora-training/runs/example-ended/cancel');
+ expect(response.status).toBe(409);
+ expect(response.body.code).toBe('ALREADY_TERMINAL');
+});
diff --git a/server/services/README.md b/server/services/README.md
index 8d4e82c0bf..17544de085 100644
--- a/server/services/README.md
+++ b/server/services/README.md
@@ -40,6 +40,7 @@ image-generation paths below are useful entry points when tracing render jobs.
| `creativeDirector/firstPassGen.js` | Gracefully gated first-pass portraits and scene frames. |
| `universeCharacterSheet.js` | Character reference-sheet render and completion workflow. |
| `loraDatasetGenerate.js` | Training-image render and dataset lifecycle. |
+| `dataMaintenance.js` | Allowlisted retired-data inventory, expiring previews and explicit serialized removal with active-work refusal. |
| `sprites/reference.js` | Sprite reference candidate rendering and provenance. |
| `deckRender.js` | Deck card image render jobs. |
| `fableLoom/production.js` | FableLoom production render planning and enqueueing. |
diff --git a/server/services/dataMaintenance.js b/server/services/dataMaintenance.js
new file mode 100644
index 0000000000..19196d7286
--- /dev/null
+++ b/server/services/dataMaintenance.js
@@ -0,0 +1,129 @@
+import { getNavPageForPath } from '../lib/navManifest.js';
+import { withRetiredDataRemoval } from '../lib/retiredLoraTraining.js';
+import { createHash, randomUUID } from 'node:crypto';
+import { lstat, readdir, realpath } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+import { PATHS, renameGuarded, rmGuarded } from '../lib/fileUtils.js';
+import { ServerError } from '../lib/errorHandler.js';
+import { createMutex } from '../lib/asyncMutex.js';
+import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
+import { listJobs } from './mediaJobQueue/index.js';
+import { trainingRunsBusy } from './dataManagerBusy.js';
+import { getActiveCaptionRun } from './loraDatasetCaption.js';
+
+// Operations own their paths. Neither the client nor a preview can supply one.
+export const MAINTENANCE_OPERATIONS = Object.freeze([
+ { id: 'retired-training-runs', directory: 'training-runs', label: 'Retired LoRA training artifacts', category: 'Unique checkpoints and regenerable scratch', warning: 'May contain the only copy of an adapter, optimizer checkpoint, logs or preview. PostgreSQL run history is retained but will point to missing artifacts.', deletable: true },
+ { id: 'retired-lora-datasets', directory: 'lora-datasets', label: 'Retired LoRA datasets', category: 'Unique source images and captions', warning: 'Source images, captions and dataset metadata may be irreplaceable. Copy anything you want to retain before deleting.', deletable: true },
+ { id: 'shared-lora-adapters', directory: 'loras', label: 'Installed LoRA adapters', category: 'Shared inference assets', warning: `Adapters still work in image and media generation. Manage individual adapters in ${getNavPageForPath('/models/loras')?.breadcrumb}. Shared Python runtimes and model caches remain supported.`, deletable: false },
+]);
+const fail = (message, code = 'MAINTENANCE_UNSAFE') => new ServerError(message, { status: 409, code });
+const missing = error => { if (error.code === 'ENOENT') return null; throw error; };
+const previews = new Map();
+const withLock = createMutex();
+const PREVIEW_TTL = 5 * 60_000;
+
+async function dataRootSnapshot() {
+ let cursor = resolve(PATHS.data);
+ for (;;) {
+ const info = await lstat(cursor);
+ if (info.isSymbolicLink()) throw fail('Data root contains a symbolic link; removal is unavailable.');
+ const parent = dirname(cursor);
+ if (parent === cursor) break;
+ cursor = parent;
+ }
+ // Native canonicalization also expands Windows short names/casing. A changed
+ // spelling is not evidence of a link; lstat above establishes that boundary.
+ const root = await realpath(PATHS.data);
+ const info = await lstat(root);
+ return { root, identity: [info.dev, info.ino] };
+}
+
+async function inspect(directory, logicalDirectory = directory) {
+ const { root, identity: rootIdentity } = await dataRootSnapshot();
+ const hash = createHash('sha256');
+ let identity = null;
+ let bytes = 0;
+ let files = 0;
+ async function walk(path, relative) {
+ const info = await lstat(path).catch(missing);
+ if (!info) return;
+ if (info.isSymbolicLink() || (!info.isDirectory() && !info.isFile())) throw fail('Unsupported file or symbolic link found; no files removed.');
+ if (relative === logicalDirectory) identity = [info.dev, info.ino];
+ hash.update(JSON.stringify([relative, info.dev, info.ino, info.mode, info.size, info.mtimeMs, relative === logicalDirectory ? null : info.ctimeMs]) + '\n');
+ if (info.isFile()) { bytes += info.size; files += 1; }
+ else for (const name of (await readdir(path)).sort()) await walk(join(path, name), `${relative}/${name}`);
+ }
+ await walk(join(root, directory), logicalDirectory);
+ return { canonicalRoot: root, rootIdentity, bytes, files, present: identity !== null, fingerprint: hash.digest('hex'), identity };
+}
+
+async function assertIdle() {
+ // Retired producers cannot start. Outstanding generation/caption requests may
+ // still own dataset inputs, including work admitted by the outgoing server.
+ if (listJobs().some(job => ['queued', 'running'].includes(job.status))) throw fail('Media work is queued or running. Wait for it to finish or cancel it explicitly.', 'MAINTENANCE_BUSY');
+ const busy = await trainingRunsBusy();
+ if (busy.busy !== false) throw fail(busy.reason || 'Training ownership could not be determined.', 'MAINTENANCE_BUSY');
+ const datasets = await readdir(join(PATHS.data, 'lora-datasets')).catch(error => missing(error) || []);
+ if (datasets.some(id => getActiveCaptionRun(id))) throw fail('Dataset captioning is still active; finish or cancel it first.', 'MAINTENANCE_BUSY');
+}
+
+export async function getMaintenanceInventory() {
+ return Promise.all(MAINTENANCE_OPERATIONS.map(async operation => {
+ try {
+ const { bytes, files, present } = await inspect(operation.directory);
+ return { ...operation, path: `data/${operation.directory}`, bytes, files, status: present ? 'retained' : 'absent', actions: operation.deletable && present ? ['delete'] : [] };
+ } catch (error) {
+ return { ...operation, path: `data/${operation.directory}`, bytes: null, files: null, status: 'unavailable', actions: [], error: error.code === 'MAINTENANCE_UNSAFE' ? error.message : 'Inventory could not be read. Check directory access before retrying.' };
+ }
+ }));
+}
+
+export async function previewMaintenance(id) {
+ const operation = MAINTENANCE_OPERATIONS.find(item => item.id === id && item.deletable);
+ if (!operation) throw new ServerError('Unsupported maintenance operation', { status: 404, code: 'NOT_FOUND' });
+ await assertIdle();
+ const inventory = await inspect(operation.directory);
+ if (!inventory.present) throw fail('Nothing remains to delete. Refresh inventory.', 'MAINTENANCE_STALE');
+ const now = Date.now();
+ for (const [key, value] of previews) if (value.expiresAt < now) previews.delete(key);
+ if (previews.size >= 100) throw fail('Too many outstanding previews. Wait for them to expire.');
+ const preview = { token: randomUUID(), operationId: id, path: `data/${operation.directory}`, warning: operation.warning, bytes: inventory.bytes, files: inventory.files, confirmation: `DELETE ${operation.directory}`, expiresAt: now + PREVIEW_TTL };
+ previews.set(preview.token, { ...preview, inventory, directory: operation.directory });
+ return preview;
+}
+
+export async function executeMaintenance({ token, confirmation }) {
+ return withLock(() => withBackupAssetPublication(() => withRetiredDataRemoval(async () => {
+ const preview = previews.get(token);
+ if (!preview || preview.expiresAt < Date.now()) throw fail('Preview expired. Refresh and preview again.', 'MAINTENANCE_STALE');
+ if (confirmation !== preview.confirmation) throw new ServerError('Confirmation must match the preview exactly', { status: 400, code: 'CONFIRMATION_REQUIRED' });
+ if (preview.result) return preview.result;
+ await assertIdle();
+ const current = await inspect(preview.directory);
+ if (current.fingerprint !== preview.inventory.fingerprint || JSON.stringify(current.rootIdentity) !== JSON.stringify(preview.inventory.rootIdentity)) throw fail('Files changed since preview. Refresh and preview again.', 'MAINTENANCE_STALE');
+ const source = join(current.canonicalRoot, preview.directory);
+ const stagedName = `.maintenance-${preview.token}`;
+ const staged = join(current.canonicalRoot, stagedName);
+ await renameGuarded(source, staged);
+ // Renaming first detaches the exact inspected directory from normal writers.
+ // Revalidate identity and contents before removing anything. On uncertainty,
+ // preserve the detached tree and report its recovery location.
+ try {
+ const detached = await inspect(stagedName, preview.directory);
+ if (JSON.stringify(detached.rootIdentity) !== JSON.stringify(current.rootIdentity) || JSON.stringify(detached.identity) !== JSON.stringify(current.identity) || detached.fingerprint !== current.fingerprint) throw fail('Files changed while preparing deletion.');
+ await assertIdle();
+ const finalRoot = await dataRootSnapshot();
+ if (finalRoot.root !== current.canonicalRoot || JSON.stringify(finalRoot.identity) !== JSON.stringify(current.rootIdentity)) throw fail('Data root changed while preparing deletion.');
+ if (await lstat(source).catch(missing)) throw fail('The source directory was recreated while preparing deletion.');
+ await rmGuarded(staged, { recursive: true, force: false });
+ } catch (error) {
+ const replacement = await lstat(source).catch(missing);
+ if (!replacement) await renameGuarded(staged, source).catch(() => {});
+ throw new ServerError(`Removal did not complete. Inspect data/${preview.directory} and data/${stagedName} before trying again.`, { status: 409, code: 'MAINTENANCE_INCOMPLETE', cause: error });
+ }
+ const remaining = await inspect(preview.directory).catch(() => null);
+ preview.result = { ok: true, operationId: preview.operationId, removedBytes: current.bytes, removedFiles: current.files, status: remaining === null ? 'unavailable' : remaining.present ? 'retained' : 'deleted', remainingBytes: remaining?.bytes ?? null, remainingFiles: remaining?.files ?? null };
+ return preview.result;
+ })));
+}
diff --git a/server/services/dataManager.busy.test.js b/server/services/dataManager.busy.test.js
index c3790b40ef..7192854efb 100644
--- a/server/services/dataManager.busy.test.js
+++ b/server/services/dataManager.busy.test.js
@@ -57,7 +57,6 @@ const busyWith = (reason) => () => Promise.resolve({ busy: true, reason });
describe('purgeCategory — CATEGORY_BUSY refusal', () => {
const cases = [
['image-clean-tmp', () => imageCleanTmpBusy, 'init-0000.png', 'an image render is in flight'],
- ['training-runs', () => trainingRunsBusy, 'run-0001', 'a trainer is running'],
['update-detached', () => updateDetachedBusy, 'pid', 'an update is running'],
];
@@ -74,10 +73,8 @@ describe('purgeCategory — CATEGORY_BUSY refusal', () => {
expect(existsSync(dataPath(key))).toBe(true);
});
- it('refuses a per-item purge while the category is busy', async () => {
- trainingRunsBusy.mockImplementation(busyWith('a trainer is running'));
- await expect(purgeCategory('training-runs', { subPath: 'run-0001' }))
- .rejects.toMatchObject({ status: 409, code: 'CATEGORY_BUSY' });
+ it('refuses legacy training purge even while idle; maintenance requires its own preview', async () => {
+ await expect(purgeCategory('training-runs')).rejects.toMatchObject({ status: 403 });
expect(existsSync(dataPath('training-runs', 'run-0001'))).toBe(true);
});
@@ -90,20 +87,20 @@ describe('purgeCategory — CATEGORY_BUSY refusal', () => {
// "Could not verify" and "nothing is running" must not collapse into the same
// answer — the purge is irreversible.
it('fails closed when the probe throws', async () => {
- trainingRunsBusy.mockImplementation(() => Promise.reject(new Error('queue unavailable')));
- await expect(purgeCategory('training-runs')).rejects.toMatchObject({ status: 409, code: 'CATEGORY_BUSY' });
- expect(existsSync(dataPath('training-runs', 'run-0001'))).toBe(true);
+ imageCleanTmpBusy.mockImplementation(() => Promise.reject(new Error('queue unavailable')));
+ await expect(purgeCategory('image-clean-tmp')).rejects.toMatchObject({ status: 409, code: 'CATEGORY_BUSY' });
+ expect(existsSync(dataPath('image-clean-tmp', 'init-0000.png'))).toBe(true);
});
it('fails closed when the probe returns a shape it cannot read', async () => {
- trainingRunsBusy.mockImplementation(() => Promise.resolve({ reason: 'no busy flag' }));
- await expect(purgeCategory('training-runs')).rejects.toMatchObject({ code: 'CATEGORY_BUSY' });
- expect(existsSync(dataPath('training-runs', 'run-0001'))).toBe(true);
+ imageCleanTmpBusy.mockImplementation(() => Promise.resolve({ reason: 'no busy flag' }));
+ await expect(purgeCategory('image-clean-tmp')).rejects.toMatchObject({ code: 'CATEGORY_BUSY' });
+ expect(existsSync(dataPath('image-clean-tmp', 'init-0000.png'))).toBe(true);
});
it('supplies a fallback reason when a busy probe names none', async () => {
- trainingRunsBusy.mockImplementation(() => Promise.resolve({ busy: true }));
- await expect(purgeCategory('training-runs')).rejects.toThrow(/Something is using data\/training-runs/);
+ imageCleanTmpBusy.mockImplementation(() => Promise.resolve({ busy: true }));
+ await expect(purgeCategory('image-clean-tmp')).rejects.toThrow(/Something is using data\/image-clean-tmp/);
});
});
diff --git a/server/services/dataManager.itemPurge.test.js b/server/services/dataManager.itemPurge.test.js
index 91a364f13b..cc58297e2c 100644
--- a/server/services/dataManager.itemPurge.test.js
+++ b/server/services/dataManager.itemPurge.test.js
@@ -151,8 +151,8 @@ describe('training-run source details', () => {
description: 'example-model · completed',
});
expect(detail.items.find(item => item.name === 'orphan-run')).toMatchObject({ sourceUnavailable: 'Run record no longer available' });
- await purgeCategory('training-runs', { subPath: 'example-run' });
- expect(existsSync(dataPath('training-runs', 'example-run'))).toBe(false);
+ await expect(purgeCategory('training-runs', { subPath: 'example-run' })).rejects.toMatchObject({ status: 403, code: 'CATEGORY_NOT_PURGEABLE' });
+ expect(existsSync(dataPath('training-runs', 'example-run'))).toBe(true);
expect(existsSync(dataPath('training-runs', 'orphan-run', 'checkpoint.bin'))).toBe(true);
});
diff --git a/server/services/dataManager.js b/server/services/dataManager.js
index 00f412a3ea..29d1594ea6 100644
--- a/server/services/dataManager.js
+++ b/server/services/dataManager.js
@@ -204,7 +204,7 @@ export const CATEGORIES = {
'telegram': { label: 'Telegram', description: 'Telegram bot data', archivable: true, deletable: true, purgeScope: 'category' },
'templates': { label: 'Visual Templates', description: 'Shipped layout assets used as render anchors', archivable: false, deletable: false },
'tools': { label: 'Tools', description: 'Tool execution data', archivable: true, deletable: true, purgeScope: 'category' },
- 'training-runs': { label: 'LoRA Training Runs', description: 'Training checkpoints, caches, and sample previews — the finished adapters live in LoRAs and survive a purge; run history in Postgres will point at missing artifacts', archivable: false, deletable: true, purgeScope: 'category', busyCheck: trainingRunsBusy },
+ 'training-runs': { label: 'LoRA Training Runs', description: 'Retired training checkpoints and previews; use Settings → Data Maintenance for a fresh preview and confirmed removal. May contain unique adapters.', archivable: false, deletable: false, busyCheck: trainingRunsBusy },
'universes': { label: 'Universes', description: 'Universe Builder records — bibles, canon, and style references', archivable: true, deletable: false },
'update-detached': { label: 'Update Control', description: 'Control files for a detached self-update run — safe to purge when no update is running', archivable: false, deletable: true, purgeScope: 'category', busyCheck: updateDetachedBusy },
'uploads': { label: 'Uploads', description: 'Files uploaded through the UI and referenced by records', archivable: true, deletable: false },
diff --git a/server/services/loraDatasets.js b/server/services/loraDatasets.js
index 92ad069322..ce08a573b9 100644
--- a/server/services/loraDatasets.js
+++ b/server/services/loraDatasets.js
@@ -1,3 +1,4 @@
+import { withRetiredDataAccess } from '../lib/retiredLoraTraining.js';
/**
* LoRA training datasets — collectionStore CRUD + image file management.
*
@@ -77,7 +78,7 @@ const requireDataset = async (id) => {
* lease through this record commit.
*/
export async function updateDataset(id, mutate) {
- return withBackupAssetPublication(() => loraDatasetStore.queueRecordWrite(id, async () => {
+ return withRetiredDataAccess(() => withBackupAssetPublication(() => loraDatasetStore.queueRecordWrite(id, async () => {
const current = await loraDatasetStore.loadOne(id);
if (!current) {
throw new ServerError(`LoRA dataset not found: ${id}`, { status: 404, code: 'NOT_FOUND' });
@@ -88,7 +89,7 @@ export async function updateDataset(id, mutate) {
await loraDatasetStore.saveOneNow(id, next);
trainingEvents.emit('dataset:changed', { datasetId: id });
return next;
- }));
+ })));
}
const summarize = (dataset) => ({
diff --git a/server/services/loraTraining/captionLeakStaging.test.js b/server/services/loraTraining/captionLeakStaging.test.js
deleted file mode 100644
index e803d26fef..0000000000
--- a/server/services/loraTraining/captionLeakStaging.test.js
+++ /dev/null
@@ -1,97 +0,0 @@
-import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest';
-import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';
-
-// runTraining takes the machine-wide heavy-local-job claim, whose path is
-// derived from PATHS.data at module load. Without this redirect the suite wrote
-// (and briefly held) the claim in the developer's LIVE data/ tree — a file that
-// gates their real local renders, stranded there whenever a case failed before
-// the release (#6176).
-vi.mock('../../lib/fileUtils.js', async (importOriginal) =>
- makePathsProxy(await importOriginal(), { dataRoot: () => lazyTempDataRoot('portos-caption-leak-') }));
-afterAll(cleanupTempDataRoots);
-
-// Regression guard for the caption-leak gate's SECOND call site. The gate is
-// re-checked when the queued job actually stages (runTraining), not just at
-// launch (startTrainingRun). The staging re-check must skip the gate ONLY for a
-// run that already opted past it — an explicit "Train anyway" (persisted as
-// run.captionLeakAcknowledged) or a resume (resumeCheckpoint set) — while still
-// re-checking an ordinary clean-at-launch run, so captions edited leaky WHILE
-// the run sat in the queue are caught instead of training silently.
-const getRunMock = vi.fn();
-const updateRunMock = vi.fn();
-const validateDatasetReadyMock = vi.fn();
-const updateDatasetMock = vi.fn();
-
-vi.mock('./db.js', () => ({
- getRun: (...a) => getRunMock(...a),
- getRunRequired: vi.fn(),
- updateRun: (...a) => updateRunMock(...a),
- listRuns: vi.fn(),
- deleteRun: vi.fn(),
-}));
-vi.mock('./dataset.js', () => ({ validateDatasetReady: (...a) => validateDatasetReadyMock(...a) }));
-vi.mock('../settings.js', () => ({ getSettings: vi.fn(async () => ({})) }));
-vi.mock('../loraDatasets.js', () => ({ updateDataset: (...a) => updateDatasetMock(...a) }));
-vi.mock('../mediaJobQueue/index.js', () => ({
- assertMediaQueueRoom: vi.fn(),
- enqueueJob: vi.fn(),
- getJob: vi.fn(),
- mediaJobEvents: { emit: vi.fn(), on: vi.fn() },
-}));
-
-const { runTraining } = await import('./index.js');
-
-// Mirrors the real gate: throw the leak 409 when the caller did NOT acknowledge,
-// otherwise return a dataset whose character MISMATCHES the run so runTraining
-// bails at the ownership check (before any real spawn). So a regressed staging
-// call that dropped the flag surfaces as a leak failure rather than the
-// reassignment one.
-const LEAK_MSG = 'Identity is leaking into the captions — "red cloak" repeat across most images';
-beforeEach(() => {
- vi.clearAllMocks();
- updateRunMock.mockResolvedValue();
- updateDatasetMock.mockResolvedValue();
- validateDatasetReadyMock.mockImplementation(async (_id, opts = {}) => {
- if (!opts.acknowledgeCaptionLeak) {
- const err = new Error(LEAK_MSG);
- err.status = 409;
- err.code = 'CAPTION_IDENTITY_LEAK';
- throw err;
- }
- return { dataset: { character: { entryId: 'other', universeId: 'u1' } }, manifest: { images: [] } };
- });
-});
-
-const failError = () =>
- updateRunMock.mock.calls.find(([, patch]) => patch?.status === 'failed')?.[1]?.error || '';
-
-describe('runTraining staging re-validation — caption-leak gate', () => {
- it('skips the gate for a run launched with "Train anyway" (captionLeakAcknowledged)', async () => {
- getRunMock.mockResolvedValue({
- id: 'run-1', datasetId: 'ds1', captionLeakAcknowledged: true,
- character: { entryId: 'c1', universeId: 'u1' },
- });
- await runTraining({ jobId: 'job-1', runId: 'run-1' });
- expect(validateDatasetReadyMock).toHaveBeenCalledWith('ds1', { acknowledgeCaptionLeak: true });
- expect(failError()).not.toMatch(/leaking/i); // bailed at ownership, not the leak gate
- });
-
- it('skips the gate when resuming (captions already trained once)', async () => {
- getRunMock.mockResolvedValue({
- id: 'run-1', datasetId: 'ds1', captionLeakAcknowledged: false,
- character: { entryId: 'c1', universeId: 'u1' },
- });
- await runTraining({ jobId: 'job-1', runId: 'run-1', resumeCheckpoint: '/ckpt/step-300' });
- expect(validateDatasetReadyMock).toHaveBeenCalledWith('ds1', { acknowledgeCaptionLeak: true });
- });
-
- it('STILL re-checks an ordinary queued run, catching captions edited leaky while queued', async () => {
- getRunMock.mockResolvedValue({
- id: 'run-1', datasetId: 'ds1', captionLeakAcknowledged: false,
- character: { entryId: 'c1', universeId: 'u1' },
- });
- await runTraining({ jobId: 'job-1', runId: 'run-1' });
- expect(validateDatasetReadyMock).toHaveBeenCalledWith('ds1', { acknowledgeCaptionLeak: false });
- expect(failError()).toMatch(/leaking/i); // the queued-time leak is caught, run fails
- });
-});
diff --git a/server/services/loraTraining/dataset.js b/server/services/loraTraining/dataset.js
deleted file mode 100644
index 22134e887c..0000000000
--- a/server/services/loraTraining/dataset.js
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Dataset readiness validation for training launch. Re-checked both at
- * route time (fast 409 before a run record exists) and at run start
- * (defense vs. the dataset being deleted/edited while queued).
- */
-
-import { access } from 'fs/promises';
-import { ServerError } from '../../lib/errorHandler.js';
-import {
- analyzeCaptionInvariants, captionHasTriggerWord, computeDatasetReadiness,
-} from '../../lib/loraDataset.js';
-import { getDataset, datasetImagePath, datasetImagesDir } from '../loraDatasets.js';
-
-/**
- * Load + validate the dataset, returning `{ dataset, manifest }` where
- * manifest is the trainer-facing image list:
- * `{ triggerWord, imagesDir, images: [{ file, path, caption }] }`.
- *
- * `acknowledgeCaptionLeak` skips the identity-leak gate (below) — set by the
- * resume path (the run already cleared it once) and by an explicit
- * "train anyway" from the UI.
- */
-export async function validateDatasetReady(datasetId, { acknowledgeCaptionLeak = false } = {}) {
- const dataset = await getDataset(datasetId);
- const readiness = computeDatasetReadiness(dataset);
- if (!readiness.trainable) {
- throw new ServerError(
- `Dataset is not ready to train — needs ≥${readiness.required} ready images captioned with the trigger word `
- + `(have ${readiness.captioned}/${readiness.required}${readiness.rendering ? `, ${readiness.rendering} still rendering` : ''})`,
- { status: 409, code: 'DATASET_NOT_READY' },
- );
- }
- // Backstop the caption omit-list: if the same identity fragments still repeat
- // across most captions, the look is binding to the phrases instead of the
- // trigger word — the LoRA renders a generic subject. Block (with the offending
- // fragments) rather than train silently, so the user strips them or opts in.
- if (!acknowledgeCaptionLeak) {
- const { total, sharedFragments } = analyzeCaptionInvariants(dataset.images, dataset.triggerWord);
- if (sharedFragments.length) {
- const preview = sharedFragments.slice(0, 6).map((f) => f.fragment).join(', ');
- throw new ServerError(
- `Identity is leaking into the captions — "${preview}"${sharedFragments.length > 6 ? '…' : ''} repeat across most images, `
- + 'so the trigger word would learn a generic subject. Strip the shared identity from the captions, or train anyway to override.',
- { status: 409, code: 'CAPTION_IDENTITY_LEAK', context: { sharedFragments, total } },
- );
- }
- }
- // Same token-boundary predicate as the readiness gate — share the helper so
- // the manifest can never include a caption the gate counted differently.
- const trainImages = dataset.images.filter(
- (img) => img.status === 'ready' && captionHasTriggerWord(img.caption, dataset.triggerWord),
- );
- // Every file must actually exist on disk — a missing file at trainer
- // start would fail minutes later with an opaque python traceback.
- for (const img of trainImages) {
- const path = datasetImagePath(datasetId, img.file);
- await access(path).catch(() => {
- throw new ServerError(
- `Dataset image missing on disk: ${img.file} — delete it from the dataset and retry`,
- { status: 409, code: 'DATASET_NOT_READY' },
- );
- });
- }
- return {
- dataset,
- manifest: {
- triggerWord: dataset.triggerWord,
- imagesDir: datasetImagesDir(datasetId),
- images: trainImages.map((img) => ({
- file: img.file,
- path: datasetImagePath(datasetId, img.file),
- caption: img.caption.trim(),
- })),
- },
- };
-}
diff --git a/server/services/loraTraining/dataset.test.js b/server/services/loraTraining/dataset.test.js
deleted file mode 100644
index 3076d554cc..0000000000
--- a/server/services/loraTraining/dataset.test.js
+++ /dev/null
@@ -1,67 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-
-// validateDatasetReady is a thin orchestration over the pure readiness/invariant
-// helpers + getDataset + an fs.access existence probe. Mock the data + fs deps so
-// the gate logic (leak detection, acknowledge override, resume skip) is testable
-// without a real dataset store or files on disk.
-vi.mock('../loraDatasets.js', () => ({
- getDataset: vi.fn(),
- datasetImagePath: vi.fn((id, file) => `/tmp/${id}/${file}`),
- datasetImagesDir: vi.fn((id) => `/tmp/${id}`),
-}));
-vi.mock('fs/promises', () => ({ access: vi.fn(async () => {}) }));
-
-const { getDataset } = await import('../loraDatasets.js');
-const { validateDatasetReady } = await import('./dataset.js');
-
-// 10 ready, trigger-prefixed images = trainable. `body(i)` lets a test make the
-// per-shot descriptive tail unique (no leak) or repeat a fragment (leak).
-const dataset = (body) => ({
- id: 'ds1',
- triggerWord: 'freydis',
- character: { entryId: 'c1', universeId: 'u1', entryKind: 'characters' },
- images: Array.from({ length: 10 }, (_, i) => ({
- id: `img${i}`, file: `${i}.png`, status: 'ready', caption: `freydis, ${body(i)}`,
- })),
-});
-
-beforeEach(() => vi.clearAllMocks());
-
-describe('validateDatasetReady caption-identity-leak gate', () => {
- it('passes (returns manifest) when captions share no identity fragment', async () => {
- getDataset.mockResolvedValue(dataset((i) => `pose ${i}, angle ${i}`));
- const { manifest } = await validateDatasetReady('ds1');
- expect(manifest.triggerWord).toBe('freydis');
- expect(manifest.images).toHaveLength(10);
- });
-
- it('throws 409 CAPTION_IDENTITY_LEAK with the shared fragments when identity repeats', async () => {
- getDataset.mockResolvedValue(dataset((i) => `red cloak, woven crown, pose ${i}`));
- await expect(validateDatasetReady('ds1')).rejects.toMatchObject({
- status: 409,
- code: 'CAPTION_IDENTITY_LEAK',
- context: {
- total: 10,
- sharedFragments: expect.arrayContaining([
- expect.objectContaining({ fragment: 'red cloak' }),
- expect.objectContaining({ fragment: 'woven crown' }),
- ]),
- },
- });
- });
-
- it('trains anyway when acknowledgeCaptionLeak overrides the gate', async () => {
- getDataset.mockResolvedValue(dataset((i) => `red cloak, woven crown, pose ${i}`));
- const { manifest } = await validateDatasetReady('ds1', { acknowledgeCaptionLeak: true });
- expect(manifest.images).toHaveLength(10);
- });
-
- it('still rejects an untrainable dataset before the leak check', async () => {
- getDataset.mockResolvedValue({
- id: 'ds1', triggerWord: 'freydis', character: { entryId: 'c1' }, images: [],
- });
- await expect(validateDatasetReady('ds1')).rejects.toMatchObject({
- status: 409, code: 'DATASET_NOT_READY',
- });
- });
-});
diff --git a/server/services/loraTraining/index.js b/server/services/loraTraining/index.js
index 1255d0d759..d4413bfb01 100644
--- a/server/services/loraTraining/index.js
+++ b/server/services/loraTraining/index.js
@@ -1,368 +1,69 @@
+import { refuseRetiredTraining, withRetiredDataAccess, acquireRetiredDataAccess } from '../../lib/retiredLoraTraining.js';
/**
- * LoRA training engine — run lifecycle + python trainer spawn.
- *
- * Routed through mediaJobQueue's GPU lane as `kind: 'training'` (training
- * shares the Metal/MLX runtime with renders, so serialization is correct).
- * The queue owns job status / SSE / watchdog / cancel escalation; this
- * module owns the run RECORD (PostgreSQL `lora_training_runs`), the child
- * process, artifact collection under `data/training-runs/<runId>/`, and
- * trained-LoRA registration into `data/loras/`.
- *
- * Spawn/cancel mirrors videoGen/local.js: SIGTERM → 8s SIGKILL escalation,
- * PYTHONUNBUFFERED, hfChildEnv, caffeinate on darwin.
- * The trainers checkpoint on SIGTERM, so a cancel keeps its progress.
+ * Retired trainer recovery only: observe/cancel a verified detached survivor,
+ * finalize its output and keep historical records/checkpoint promotion readable.
+ * New launch, resume and queue admission are permanently refused.
*/
import { spawn } from '../../lib/childProcess.js';
import { existsSync } from 'fs';
-import { join, basename, dirname } from 'path';
+import { join, basename } from 'path';
import { platform } from 'os';
-import { PATHS, ensureDir, atomicWrite, shortId, copyFileGuarded, writeFileGuarded, rmGuarded } from '../../lib/fileUtils.js';
+import { PATHS, ensureDir, shortId, copyFileGuarded, writeFileGuarded, rmGuarded } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
-import { v4 as uuidv4 } from '../../lib/uuid.js';
-import { hfChildEnv } from '../hfToken.js';
-import { spawnDetached, reapDetached, reattachDetached, isReattachable } from '../../lib/detachedSpawn.js';
+
+import { reattachDetached, isReattachable } from '../../lib/detachedSpawn.js';
import { killWithEscalation } from '../../lib/killWithEscalation.js';
import { getImageModels } from '../../lib/mediaModels.js';
-import { resolveFlux2Python, isFlux2VenvHealthy, resolveMfluxPython } from '../../lib/pythonSetup.js';
+
import { getSettings } from '../settings.js';
import { deleteLora, writeLoraSidecar } from '../loras.js';
-import { assertMediaQueueRoom, enqueueJob, getJob, mediaJobEvents } from '../mediaJobQueue/index.js';
+import { getJob, mediaJobEvents } from '../mediaJobQueue/index.js';
import { updateDataset } from '../loraDatasets.js';
import { trainingEvents } from './events.js';
import { sleepDisplayForTraining, wakeDisplay } from './displayPower.js';
-import {
- TRAINING_DEFAULTS,
- TRAINING_RUNTIMES,
- buildFlux2TrainArgs,
- buildMfluxTrainArgs,
- buildMfluxTrainConfig,
- resolveTrainingRuntime,
- MFLUX_DEFAULT_COOLDOWN_SEC,
-} from './runtimes.js';
+
import { makeTrainingLineHandler } from './progress.js';
import { makeStallDetector } from './stallDetector.js';
-import { prepareMemoryForTraining, gpuBlockersMessage, TRAINING_MIN_HEADROOM_GB } from './memoryPrep.js';
+
import { claimHeavyLocalJob, adoptHeavyLocalJob } from '../../lib/heavyJobClaim.js';
import { classifyTrainingFailure } from './failure.js';
import { buildTrainedSidecar, trainedLoraFilename } from './sidecar.js';
-import { validateDatasetReady } from './dataset.js';
-import {
- listRunCheckpoints,
- listRunSamples,
- resolveCheckpointAdapterBuffer,
- resolveLatestCheckpointArtifact,
- selectDeployableCheckpoint,
-} from './checkpoints.js';
+
+import { listRunCheckpoints, listRunSamples, resolveCheckpointAdapterBuffer, selectDeployableCheckpoint } from './checkpoints.js';
import * as runsDb from './db.js';
export { listRuns, getRun, getRunRequired, deleteRun } from './db.js';
export { trainingEvents } from './events.js';
-const TRAINER_SCRIPTS = {
- [TRAINING_RUNTIMES.MFLUX]: join(PATHS.root, 'scripts', 'train_mflux_lora.py'),
- [TRAINING_RUNTIMES.FLUX2]: join(PATHS.root, 'scripts', 'train_flux2_lora.py'),
-};
-
export const runDir = (runId) => join(PATHS.trainingRuns, runId);
export const runSamplesDir = (runId) => join(runDir(runId), 'samples');
-/**
- * mflux ships its trainer as the `mflux-train` console script — probe for
- * it next to the configured local-image-gen python (e.g.
- * /opt/miniconda3/bin/python3 → /opt/miniconda3/bin/mflux-train). Present
- * only on mflux ≥0.17 installs.
- */
-export const isMfluxTrainAvailable = (pythonPath) =>
- !!pythonPath && existsSync(join(dirname(pythonPath), 'mflux-train'));
-
// GPU lane serializes training with renders, so at most one trainer child
// exists at a time. Keyed by jobId anyway so a stale cancel can't kill a
// newer run.
let activeProcess = null;
let activeJobId = null;
+const explicitlyCanceledTrainers = new WeakSet();
-// Phase-aware stall watchdog (issue #1330): how many times a single run may be
-// auto-resumed after a *soft* GPU hang before we give up and let it stay failed.
-// Bounded so a config that wedges every segment can't resume-loop forever — the
-// user sees a failed run after N attempts and intervenes. Counted separately
-// from manual resumes (run.resume.autoCount). Env-overridable.
-const STALL_MAX_AUTO_RESUMES = (() => {
- const n = Number(process.env.LORA_TRAIN_STALL_MAX_AUTO_RESUMES);
- return Number.isFinite(n) && n >= 0 ? n : 2;
-})();
-// How often the watchdog polls checkStall while a trainer is running. Well
-// under the 90s floor budget so a stall is caught within ~one tick of the
-// budget elapsing (worst-case detection lag = budget + one tick).
+// Recovery still observes a surviving trainer for stalls.
const STALL_TICK_MS = 30_000;
export const cancel = (jobId) => {
if (!activeProcess || (jobId && activeJobId !== jobId)) return false;
const proc = activeProcess;
+ explicitlyCanceledTrainers.add(proc);
// Keep activeProcess set until 'close' clears it — the trainer may spend
// a few seconds writing its cancel checkpoint. Escalate after 8s.
killWithEscalation(proc, { label: 'training child', stillRunning: () => activeProcess === proc });
return true;
};
-/** Merge order: code defaults ← settings slice ← request params. */
-const mergeParams = (settings, requestParams = {}) => ({
- ...TRAINING_DEFAULTS,
- ...(settings?.loraTraining?.defaults || {}),
- ...requestParams,
-});
+export async function startTrainingRun() { refuseRetiredTraining(); }
-// A dataset belongs to a run only while it still points at the run's subject.
-// Match the full (universeId, entryId) key the dataset store uses — a different
-// universe can reuse the same entryId, so entryId alone would falsely re-own a
-// reassigned dataset. (flipDatasetAfterRun keeps its own missing-entryId
-// fallthrough for pre-reassignment runs that predate the character snapshot.)
-const sameCharacter = (a, b) =>
- a?.entryId === b?.entryId
- && a?.universeId === b?.universeId
- && (a?.entryKind || 'characters') === (b?.entryKind || 'characters');
-
-// Re-stamp a dataset as `training` with the run's new job/run ids, but only
-// while it still owns the dataset — the dataset can be reassigned to a different
-// character between validation and this stamp (patchDataset resets it to draft);
-// stamping a moved dataset would strand it in `training` forever (flipDatasetAfterRun
-// is character-guarded and would skip the un-flip). Used by both the fresh-launch
-// and resume paths.
-const stampDatasetTrainingStatus = (run, jobId) =>
- updateDataset(run.datasetId, (current) => {
- if (!sameCharacter(current.character, run.character)) return null;
- return {
- ...current,
- status: 'training',
- training: { ...current.training, lastJobId: jobId, lastRunId: run.id },
- };
- }).catch((err) => console.error(`❌ dataset training-status stamp failed: ${err?.message}`));
-
-/**
- * Route-facing run launcher. Validates dataset readiness + runtime health,
- * creates the run record, and enqueues the training job. Returns
- * `{ runId, jobId, position }` (202-shaped).
- */
-export async function startTrainingRun({
- datasetId, baseModelId, name = null, params = {}, acknowledgeCaptionLeak = false,
-}) {
- const settings = await getSettings();
- // Resolve the mflux trainer's Python: the configured image-gen Python when it
- // ships mflux-train (the `pip --user` layout), else an auto-discovered
- // dedicated ~/.portos/venv-mflux (setup-image-video.sh's fallback for a system
- // Python that can't host mflux). The resolved path threads into the job so
- // runTraining spawns the right interpreter without re-resolving.
- const pythonPath = resolveMfluxPython(settings?.imageGen?.local?.pythonPath || null);
- // Engine pick: prefer mflux's MLX trainer when the user's mflux install
- // ships it (Apple Silicon native, no second venv); fall back to the
- // torch/diffusers trainer in venv-flux2.
- const mlxAvailable = isMfluxTrainAvailable(pythonPath);
- const routing = resolveTrainingRuntime(baseModelId, getImageModels(), { mlxAvailable });
- const { dataset } = await validateDatasetReady(datasetId, { acknowledgeCaptionLeak });
-
- if (routing.runtime === TRAINING_RUNTIMES.FLUX2) {
- // The torch/diffusers fallback trainer can't train on Apple Silicon:
- // PyTorch's MPS backend has no `linear_backward` for the FLUX.2 transformer
- // Linear layers, so a LoRA backward dies at the first optimizer step
- // ("mps_linear_backward: unsupported weights data type", for both bf16 and
- // fp32 — validated on an M-series box, issue #1227). mflux (MLX-native) is
- // THE Apple-Silicon runtime. Refuse at request time so we never queue a run
- // that loads a ~16 GB base and precomputes latents for minutes before
- // crashing. `--device cpu` would work but is impractically slow, so the
- // actionable guidance is to install mflux. (Linux/CUDA installs are the
- // torch path's real target and are unaffected.)
- if (platform() === 'darwin') {
- throw new ServerError(
- 'LoRA training on Apple Silicon requires the mflux runtime — the torch fallback can\'t train on this Mac\'s GPU (Metal/MPS). '
- + 'Install mflux ≥0.17 in your local image-gen python (Settings → Image Gen) via `bash scripts/setup-image-video.sh`.',
- { status: 412, code: 'TRAINING_MPS_UNSUPPORTED' },
- );
- }
- const healthy = await isFlux2VenvHealthy();
- if (!healthy) {
- throw new ServerError(
- 'No training engine available — install mflux ≥0.17 in your local image-gen python (Settings → Image Gen) '
- + 'or set up the FLUX.2 venv via `bash scripts/setup-image-video.sh`',
- { status: 412, code: 'TRAINING_ENGINE_MISSING' },
- );
- }
- }
-
- // A full media queue (#8326) refuses the launch before the run row exists,
- // so the common case leaves no failed run behind; the enqueue below still
- // re-checks (and fails the row) if another producer filled the queue since.
- assertMediaQueueRoom(1);
- const mergedParams = mergeParams(settings, params);
- const runId = uuidv4();
- const run = {
- id: runId,
- jobId: null,
- status: 'queued',
- runtime: routing.runtime,
- baseModelId,
- fluxVariant: routing.variant,
- trainRepo: routing.trainRepo,
- mfluxModel: routing.mfluxModel,
- name: name || null,
- character: dataset.character,
- datasetId,
- triggerWord: dataset.triggerWord,
- // Persist whether the launch opted past the caption identity-leak gate, so
- // the staging re-validation can skip the gate for THIS run without also
- // skipping it for ordinary queued runs whose captions were edited (leaky)
- // while waiting in the queue.
- captionLeakAcknowledged: acknowledgeCaptionLeak === true,
- params: mergedParams,
- progress: { step: 0, totalSteps: mergedParams.steps, loss: null, lastCheckpointStep: null },
- artifacts: { dir: `training-runs/${runId}`, checkpoints: [], samples: [] },
- output: { loraFilename: null, finalLoss: null },
- error: null,
- createdAt: new Date().toISOString(),
- startedAt: null,
- completedAt: null,
- };
- await runsDb.createRun(run);
-
- // The run row already exists, so a refused admission (#8325) must fail it
- // rather than leave a 'queued' run no job will ever pick up.
- const queued = await enqueueJob({
- kind: 'training',
- owner: 'lora-training',
- params: {
- runId,
- runtime: routing.runtime,
- datasetId,
- characterId: dataset.character.entryId,
- characterName: dataset.character.name,
- triggerWord: dataset.triggerWord,
- baseModelId,
- steps: mergedParams.steps,
- rank: mergedParams.rank,
- pythonPath,
- },
- }).catch(async (err) => {
- await runsDb.updateRun(runId, { status: 'failed', error: err.message, completedAt: new Date().toISOString() });
- throw err;
- });
- await runsDb.updateRun(runId, { jobId: queued.jobId });
- await stampDatasetTrainingStatus(run, queued.jobId);
-
- console.log(`🏋️ Training run ${shortId(runId)} queued — ${routing.runtime}/${baseModelId} dataset=${shortId(datasetId)} job=${shortId(queued.jobId)}`);
- return { runId, jobId: queued.jobId, position: queued.position, status: 'queued' };
-}
-
-/**
- * Resume a failed/canceled run from its latest on-disk checkpoint. mflux bakes
- * the output path into the checkpoint zip and reads everything (config, dataset
- * paths, optimizer + adapter state, step counter) from `--resume <zip>` — so a
- * resume MUST re-run in the ORIGINAL run dir, not a fresh one, or the trainer
- * would write its new artifacts where the wrapper's watcher can't see them.
- * We therefore re-enqueue a new job against the SAME runId: new checkpoints and
- * samples append to the existing artifact arrays (the checkpoint picker shows
- * the full timeline across the resume), and finalize registers the LoRA as
- * usual. Progress may briefly read low if the trainer restarts its step bar at
- * the resume point — cosmetic; the durable record catches up on the next flush.
- */
-export async function resumeTrainingRun(runId, { auto = false } = {}) {
- const run = await runsDb.getRunRequired(runId);
- if (!['failed', 'canceled'].includes(run.status)) {
- throw new ServerError(`Can only resume a failed or canceled run (status: ${run.status})`, {
- status: 409, code: 'RUN_NOT_RESUMABLE',
- });
- }
- // A run can be marked failed (boot reconcile / cancel) while its detached
- // trainer is still alive — reap it before resuming so we never run two
- // trainers against the same checkpoint dir. Idempotent: a no-op when nothing
- // survives. Belt-and-suspenders with the boot-reconcile reap, since resume
- // can race the reconcile's SIGTERM grace window.
- const reaped = await reapDetached(join(runDir(runId), '.detached')).catch(() => ({ reaped: false }));
- if (reaped.reaped) console.log(`🧹 reaped surviving trainer pid ${reaped.pid} before resuming run ${shortId(runId)}`);
- // Both runtimes restore optimizer state + the step counter on resume, so
- // training picks up mid-run and finishes at the original total: mflux via
- // `mflux-train --resume <zip>`, and the torch FLUX.2 trainer via
- // `--resume-from <dir>` (restores the AdamW state from the checkpoint's
- // optimizer.pt and continues range(start_step + 1, steps + 1) — no
- // over-training, no checkpoint renumber collisions).
- const checkpoint = resolveLatestCheckpointArtifact(run);
- if (!checkpoint) {
- throw new ServerError(
- 'No checkpoint to resume from — the run was killed before its first checkpoint saved. Start a fresh run.',
- { status: 409, code: 'NO_RESUMABLE_CHECKPOINT' },
- );
- }
-
- // Re-validate dataset readiness + ownership exactly like a fresh launch — the
- // dataset may have been edited, deleted, or reassigned since the run failed.
- // Skip the caption-leak gate: this run already cleared it at first launch,
- // and resume shouldn't re-block on captions the user already accepted.
- const { dataset } = await validateDatasetReady(run.datasetId, { acknowledgeCaptionLeak: true });
- if (!sameCharacter(dataset.character, run.character)) {
- throw new ServerError('Dataset was reassigned to a different subject — start a fresh run.', {
- status: 409, code: 'DATASET_REASSIGNED',
- });
- }
-
- // Engine health: mirror startTrainingRun for the run's existing runtime —
- // including resolving the dedicated ~/.portos/venv-mflux, so a resume (manual
- // OR the stall-watchdog auto-resume) on a dedicated-venv install doesn't
- // falsely fail the availability check and thread a null interpreter into the
- // re-enqueued job.
- const settings = await getSettings();
- const pythonPath = resolveMfluxPython(settings?.imageGen?.local?.pythonPath || null);
- if (run.runtime === TRAINING_RUNTIMES.FLUX2) {
- if (!(await isFlux2VenvHealthy())) {
- throw new ServerError('FLUX.2 training venv is unavailable — run `bash scripts/setup-image-video.sh`', {
- status: 412, code: 'TRAINING_ENGINE_MISSING',
- });
- }
- } else if (!isMfluxTrainAvailable(pythonPath)) {
- throw new ServerError('mflux-train not found next to the configured python — update mflux (≥0.17)', {
- status: 412, code: 'TRAINING_ENGINE_MISSING',
- });
- }
-
- const queued = await enqueueJob({
- kind: 'training',
- owner: 'lora-training',
- params: {
- runId,
- runtime: run.runtime,
- datasetId: run.datasetId,
- characterId: run.character?.entryId,
- characterName: run.character?.name,
- triggerWord: run.triggerWord,
- baseModelId: run.baseModelId,
- steps: run.params?.steps,
- rank: run.params?.rank,
- pythonPath,
- resumeCheckpoint: checkpoint.path,
- },
- });
- await runsDb.updateRun(runId, (current) => ({
- ...current,
- status: 'queued',
- jobId: queued.jobId,
- error: null,
- errorCode: null,
- errorRepo: null,
- completedAt: null,
- resume: {
- count: (current.resume?.count || 0) + 1,
- // Auto (stall-watchdog) resumes are counted separately so the watchdog
- // can cap its own retries without a manual resume resetting the budget.
- autoCount: (current.resume?.autoCount || 0) + (auto ? 1 : 0),
- fromStep: checkpoint.step,
- resumedAt: new Date().toISOString(),
- lastReason: auto ? 'stall-watchdog' : 'manual',
- },
- }));
- await stampDatasetTrainingStatus(run, queued.jobId);
-
- console.log(`🏋️ Training run ${shortId(runId)} ${auto ? 'auto-resumed (stall watchdog)' : 'resumed'} from step ${checkpoint.step} — job=${shortId(queued.jobId)}`);
- return { runId, jobId: queued.jobId, position: queued.position, status: 'queued', fromStep: checkpoint.step };
-}
+export async function resumeTrainingRun() { refuseRetiredTraining(); }
const emitFailed = (jobId, error) => trainingEvents.emit('failed', { generationId: jobId, error });
@@ -467,10 +168,10 @@ export async function deleteTrainingRun(runId, { withLora = false } = {}) {
* 'training'. Resolves the trainer binary + args, spawns, parses the line
* protocol into trainingEvents, and finalizes (LoRA registration or
* failure classification) on close. Terminal status flows through the
- * queue's dispatcher; this function resolves once the child is spawned
+ * queue's dispatcher; this function resolves once the observer attaches
* (the queue awaits the terminal event separately).
*/
-export async function runTraining({ jobId, runId, pythonPath = null, resumeCheckpoint = null, reattach = false }) {
+export async function runTraining({ jobId, runId, reattach = false, recoveryPermit = null }) {
const fail = (message) => {
console.error(`❌ training [${shortId(jobId)}] ${message}`);
emitFailed(jobId, message);
@@ -497,6 +198,11 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck
// stuck `running` (lingering until the next boot reconcile) or the dataset
// stuck on its `training` chip.
const failBeforeSpawn = async (message) => {
+ if (reattach && await hasSurvivingTrainer(runId)) {
+ retainOwnership();
+ console.error(`❌ training [${shortId(jobId)}] observer unavailable; live trainer and artifacts preserved: ${message}`);
+ return { preserved: true };
+ }
await releaseClaim();
await runsDb.updateRun(runId, {
status: 'failed', error: message, completedAt: new Date().toISOString(),
@@ -505,7 +211,7 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck
return fail(message);
};
- // Boot re-attach (#1332): the queue re-enqueues a run whose detached trainer
+ // Recovery attaches only to a trainer that already exists. The queue restores a run whose detached trainer
// SURVIVED a hard server restart with `reattach: true`. The child reparented
// to init and is still training (or finished during the downtime), so we tail
// its existing control-dir output instead of spawning a second trainer — no
@@ -513,7 +219,7 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck
// exact same line-handling/finalize path as a normal spawn, so a run that
// completed mid-restart still registers its LoRA instead of being discarded.
//
- // This runs BEFORE the fresh claimHeavyLocalJob() below: the survivor already
+ // The survivor already
// holds the machine-wide accelerator claim from before the restart (handed
// off to its PID pre-crash), so acquiring a NEW claim here would see that
// live claim as a competing job and refuse it — failing every restart-
@@ -523,9 +229,9 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck
const proc = await reattachDetached(join(dir, '.detached'));
if (!proc) {
// The survivor died (killed mid-run) with no RESULT to recover — fail the
- // run; its latest checkpoint stays resumable manually/auto, same as the
+ // run; its latest checkpoint stays retained for inspection, same as the
// pre-#1332 reap path would have left it.
- return failBeforeSpawn('Trainer did not survive the restart — marking failed; resume from the latest checkpoint.');
+ return failBeforeSpawn('Trainer did not survive the restart — marking failed; artifacts are preserved.');
}
heavyClaim = (await adoptHeavyLocalJob({ kind: 'LoRA training', id: jobId, pid: proc.pid }))
|| (await claimHeavyLocalJob({ kind: 'LoRA training', id: jobId }));
@@ -537,191 +243,13 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck
console.log(`🔁 training [${shortId(jobId)}] re-attached to surviving trainer pid ${proc.pid} (run ${shortId(runId)})`);
trainingEvents.emit('status', { generationId: jobId, message: 'Re-attached to trainer that survived a restart' });
wireProcLifecycle(proc, { isReattach: true });
- return;
- }
-
- heavyClaim = await claimHeavyLocalJob({ kind: 'LoRA training', id: jobId });
- if (!heavyClaim.ok) return failBeforeSpawn(heavyClaim.message);
-
- // Re-validate — the dataset may have been edited/deleted while queued. Skip
- // the caption identity-leak gate ONLY for a run that already opted past it:
- // one launched with an explicit "Train anyway" (persisted on the record) or a
- // resume (its captions already trained once). An ordinary clean-at-launch run
- // is still re-checked here, so captions edited leaky while it sat in the queue
- // are caught instead of training silently.
- const skipCaptionLeakGate = run.captionLeakAcknowledged === true || resumeCheckpoint != null;
- let manifest;
- let dataset;
- try {
- ({ dataset, manifest } = await validateDatasetReady(
- run.datasetId,
- { acknowledgeCaptionLeak: skipCaptionLeakGate },
- ));
- } catch (err) {
- return failBeforeSpawn(err.message);
- }
- // Stage-time ownership check: if the dataset was reassigned to a different
- // subject after this run was queued, the run no longer owns it. Bail out
- // rather than training the moved dataset and registering a LoRA under the
- // run's now-stale subject. failBeforeSpawn's flipDatasetAfterRun is
- // character-guarded, so it won't disturb the reassigned dataset's state.
- // Match the full (universeId, entryId) key the dataset store uses elsewhere.
- if (!sameCharacter(dataset.character, run.character)) {
- return failBeforeSpawn('Dataset was reassigned to a different subject after this run was queued — cancel and retrain.');
- }
-
- // Reclaim unified memory before a GPU-heavy run, then gate on real headroom.
- // Training shares the unified-memory pool with resident LLMs and renders; an
- // oversubscribed run swap-thrashes and has coincided with GPU watchdog
- // reboots (docs/research/2026-06-13-mflux-training-watchdog-panic.md). We
- // unload resident models, measure what's actually free, and refuse to start
- // (rather than crash mid-run) when headroom is below the floor. The budget
- // also sizes the mflux quantize/low_ram tier below.
- const memReport = await prepareMemoryForTraining();
- // A GPU tenant the unload path cannot evict (today: the vLLM Qwen container)
- // makes the run unwinnable regardless of system-RAM headroom, so it is checked
- // BEFORE the budget gate below — that gate reports free system RAM, which says
- // nothing about VRAM on exactly the discrete-GPU host this matters on.
- if (memReport.blockers.length) return failBeforeSpawn(gpuBlockersMessage(memReport.blockers));
- if (memReport.unloaded.length) {
- console.log(`🧹 training [${shortId(jobId)}] freed ${memReport.unloaded.length} resident model(s): ${memReport.unloaded.join(', ')}`);
- }
- console.log(`🧮 training [${shortId(jobId)}] memory budget ${memReport.budgetGb.toFixed(0)} GB free of ${memReport.totalGb.toFixed(0)} GB total`);
- if (!Number.isFinite(memReport.budgetGb) || memReport.budgetGb < TRAINING_MIN_HEADROOM_GB) {
- // Fail safe: a non-finite budget (should never happen — both inputs are
- // finite) must REFUSE the run, not slip past `NaN < x` (always false).
- return failBeforeSpawn(`Not enough free memory to train safely — ${memReport.budgetGb.toFixed(1)} GB available, need ≥ ${TRAINING_MIN_HEADROOM_GB} GB. Stop other model servers or close apps and retry.`);
- }
-
- const checkpointsDir = join(dir, 'checkpoints');
- const samplesDir = join(dir, 'samples');
-
- let bin;
- let args;
- // Staging I/O (mkdir + copyFile/writeFile/atomicWrite) is wrapped: a throw
- // here — e.g. a dataset image deleted in the window after validateDatasetReady's
- // existence check (TOCTOU), or disk-full — would otherwise propagate to the
- // queue's catch (no crash) but leave the run record `running` forever.
- try {
- await ensureDir(checkpointsDir);
- await ensureDir(samplesDir);
-
- if (run.runtime === TRAINING_RUNTIMES.FLUX2) {
- bin = resolveFlux2Python();
- if (!bin) {
- return failBeforeSpawn('FLUX.2 python environment disappeared — re-run scripts/setup-image-video.sh');
- }
- const manifestPath = join(dir, 'manifest.json');
- await atomicWrite(manifestPath, {
- triggerWord: manifest.triggerWord,
- images: manifest.images.map((img) => ({ path: img.path, caption: img.caption })),
- });
- args = buildFlux2TrainArgs({
- scriptPath: TRAINER_SCRIPTS.flux2,
- trainRepo: run.trainRepo,
- manifestPath,
- runDir: dir,
- triggerWord: run.triggerWord,
- params: run.params,
- samplePrompt: run.params?.samplePrompt || null,
- resumeFrom: resumeCheckpoint,
- });
- } else {
- bin = pythonPath;
- if (!bin || !isMfluxTrainAvailable(bin)) {
- return failBeforeSpawn('mflux-train not found next to the configured python — update mflux (≥0.17) or set up the FLUX.2 venv');
- }
- // Stage the dataset in mflux's auto-discovery layout: NNNN.png +
- // NNNN.txt caption pairs, plus preview_1.txt for the periodic sample
- // render. mflux resolves everything from the config's `data` dir.
- const dataDir = join(dir, 'data');
- await ensureDir(dataDir);
- for (let i = 0; i < manifest.images.length; i += 1) {
- const stem = String(i + 1).padStart(4, '0');
- await copyFileGuarded(manifest.images[i].path, join(dataDir, `${stem}.png`));
- await writeFileGuarded(join(dataDir, `${stem}.txt`), `${manifest.images[i].caption}\n`);
- }
- if ((run.params?.sampleEvery ?? TRAINING_DEFAULTS.sampleEvery) > 0) {
- const samplePrompt = run.params?.samplePrompt || `${run.triggerWord} portrait, neutral background`;
- await writeFileGuarded(join(dataDir, 'preview_1.txt'), `${samplePrompt}\n`);
- }
- // output_path must NOT pre-exist — mflux appends a timestamp suffix to
- // an existing dir (its new_folder behavior), which would break the
- // wrapper's artifact watcher. mflux creates checkpoints/ + preview/
- // (+ loss/) inside it.
- const mfluxOutputDir = join(dir, 'mflux');
- const config = buildMfluxTrainConfig({
- params: run.params,
- variant: run.fluxVariant,
- mfluxModel: run.mfluxModel,
- dataDir,
- imageCount: manifest.images.length,
- outputDir: mfluxOutputDir,
- // Memory-derived quantize/low_ram (see deriveMfluxMemoryConfig) keyed
- // on the post-unload AVAILABLE budget, not raw RAM — a bf16 base +
- // in-RAM latent cache OOM-killed a 48 GB machine, and on a shared box
- // resident models eat the same pool, so the tier must track headroom.
- totalMemGb: memReport.budgetGb,
- });
- const configPath = join(dir, 'mflux-train.json');
- await atomicWrite(configPath, config);
- // Segmented training (watchdog-panic mitigation): default ON, globally
- // disable-able via settings.loraTraining.segmentation = false once a
- // macOS/mflux update fixes the underlying GPU-driver hang. Segment size
- // is the config's effective save_frequency so each segment ends exactly
- // on a checkpoint — no extra checkpoints, no lost steps on resume.
- const segCfg = settings?.loraTraining || {};
- const segmentationOn = segCfg.segmentation !== false;
- const cooldownSec = Number.isFinite(segCfg.segmentCooldownSec)
- ? segCfg.segmentCooldownSec : MFLUX_DEFAULT_COOLDOWN_SEC;
- args = buildMfluxTrainArgs({
- scriptPath: TRAINER_SCRIPTS.mflux,
- configPath,
- runDir: dir,
- totalSteps: run.params?.steps || TRAINING_DEFAULTS.steps,
- resumeCheckpoint,
- segmentSteps: segmentationOn ? config.checkpoint.save_frequency : 0,
- cooldownSec,
- });
- }
- } catch (err) {
- return failBeforeSpawn(`staging failed: ${err.message}`);
+ return { observing: true };
}
- await runsDb.updateRun(runId, { status: 'running', startedAt: new Date().toISOString() });
- trainingEvents.emit('status', { generationId: jobId, message: `Starting ${run.runtime} training (${run.params.steps} steps)` });
-
- const childEnv = await hfChildEnv({ PYTHONUNBUFFERED: '1' });
- delete childEnv.PYTHONPATH;
+ refuseRetiredTraining();
- console.log(`🏋️ training [${shortId(jobId)}] spawn ${basename(bin)} ${run.runtime} steps=${run.params.steps} rank=${run.params.rank} images=${manifest.images.length}${resumeCheckpoint ? ` resume=${basename(resumeCheckpoint)}` : ''}`);
- // `spawnDetached` double-forks the trainer so it reparents to init (PPID=1)
- // and leaves pm2's process tree entirely. A plain `detached: true` child only
- // gets its OWN process group — but pm2's TreeKill keys on PPID, not the group,
- // so the still-PPID-linked trainer was found and SIGINT'd anyway whenever
- // portos-server restarted (e.g. on the memory ceiling, which a long session
- // crosses routinely — observed twice: SIGINT/KeyboardInterrupt at the exact
- // second of a server restart, losing hours of GPU work). The detached trainer
- // streams stdout/stderr through on-disk log files under `<runDir>/.detached`
- // that the server tails. We still `proc.kill()` it directly by PID on cancel,
- // and the queue worker awaits the 'close' event for the run lifecycle. No
- // `cleanup` — those logs are the only copy of raw trainer stdout/stderr, kept
- // in the run dir for post-mortem and removed when the run dir is deleted.
- let proc;
- try {
- proc = await spawnDetached(bin, args, { env: childEnv, controlDir: join(dir, '.detached') });
- } catch (err) {
- return failBeforeSpawn(`trainer spawn failed: ${err.message}`);
- }
- wireProcLifecycle(proc);
-
- // Wire a freshly-spawned OR re-attached (#1332) trainer handle into the full
- // run lifecycle: live output streaming + progress mirror, the phase-aware
- // stall watchdog, and the close→finalize path that registers the LoRA. A
- // hoisted nested function so the early reattach branch (above) can call it too
- // — both paths share identical handling. Closes over jobId/runId/run/dir/
- // settings/fail/failBeforeSpawn.
function wireProcLifecycle(proc, { isReattach = false } = {}) {
+ const releaseDataAccess = acquireRetiredDataAccess();
activeProcess = proc;
activeJobId = jobId;
void heavyClaim?.handoffTo?.(proc.pid)?.catch((err) => console.error(`❌ training [${shortId(jobId)}] claim handoff failed: ${err.message}`));
@@ -821,8 +349,7 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck
// protocol to apply a tight, step-rate-derived budget ONLY during
// STAGE:training — load/encode/sampling/cooldown stay on the flat watchdog so
// a legitimately slow phase is never false-killed. On a training-phase stall
- // we SIGKILL the wedged child; the close handler auto-resumes from the newest
- // checkpoint. Disable via settings.loraTraining.stallWatchdog = false.
+ // we stop the wedged child and retain its newest checkpoint. Disable via settings.loraTraining.stallWatchdog = false.
const stallWatchdogOn = (settings?.loraTraining?.stallWatchdog !== false);
const stallDetector = makeStallDetector();
let stallKilled = false; // set when the tick SIGKILLs — close handler reads it
@@ -884,11 +411,10 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck
if (!stalled) return;
stallKilled = true;
stopStallWatchdog();
- console.log(`⏱️ training [${shortId(jobId)}] phase-aware watchdog: training stalled ${Math.round(idleMs / 1000)}s (budget ${Math.round(budgetMs / 1000)}s) — SIGKILL + auto-resume`);
+ console.log(`⏱️ training [${shortId(jobId)}] phase-aware watchdog: training stalled ${Math.round(idleMs / 1000)}s (budget ${Math.round(budgetMs / 1000)}s) — stopping stalled trainer; checkpoints preserved`);
// Soft hang: SIGKILL straight away (SIGTERM relies on the GIL the hang
// holds, so the 8s escalation in cancel() would just waste the window).
- // The newest checkpoint is the resume point; the close handler picks it
- // up. Direct PID kill via the detached handle.
+ // Direct PID kill through the already-owned detached handle.
proc.kill('SIGKILL');
} catch (err) {
console.error(`❌ training [${shortId(jobId)}] stall watchdog tick failed: ${err?.message}`);
@@ -911,7 +437,7 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck
// A no-PID launch failure owns no running trainer.
wakeDisplay(settings);
// A launch failure after the run was marked `running` — a genuine spawn
- // error, or (with spawnDetached) the control dir failing to create/clear.
+ // error, including a detached observer losing its control directory.
// Route through the same terminal cleanup as pre-spawn failures so the run
// row doesn't stay stuck `running` and the dataset's `training` chip is
// released; `fail()` alone only logs + emits. Async + guarded since this
@@ -932,22 +458,16 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck
// finalize reads the run record — the collapse guard and previewImageUrl
// both read run.artifacts. Async finalize wrapped so a rejection can't
// escape the event handler (unhandled rejection kills the process on Node ≥15).
- // The trainer has exited, so release before finalization can enqueue an
- // automatic checkpoint resume; that successor must acquire a fresh claim.
- releaseClaim()
- .then(() => flushProgress())
- .then(() => processError
- ? failBeforeSpawn(`trainer process failed: ${processError.message}`)
- : finalizeTraining({ jobId, runId, code, signal, state: getState(), stallKilled }))
- .then((resumed) => {
- // Wake the display now the run is over so the user sees the result —
- // but NOT when finalize actually enqueued a stall-watchdog auto-resume
- // (it re-spawns and re-sleeps the display for the next attempt). Gating
- // on the returned `resumed` flag, not raw `stallKilled`, ensures a
- // budget-exhausted or failed resume (terminal, no re-spawn) still wakes.
- // Best-effort, darwin-only.
- if (!resumed) wakeDisplay(settings);
- })
+ // The trainer has exited; release its accelerator claim before finalizing.
+ const settle = async () => {
+ await releaseClaim();
+ await flushProgress();
+ if (processError) return failBeforeSpawn(`trainer process failed: ${processError.message}`);
+ return finalizeTraining({ jobId, runId, code, signal, state: getState(), stallKilled, canceledByOperator: explicitlyCanceledTrainers.has(proc) });
+ };
+ const terminalWork = recoveryPermit ? recoveryPermit.run(settle) : settle();
+ terminalWork
+ .then(async () => { wakeDisplay(settings); await recoveryPermit?.finish(); })
.catch((err) => {
retainOwnership();
// A finalize/flush rejection is still a terminal end with no auto-resume
@@ -956,45 +476,15 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck
wakeDisplay(settings);
console.error(`❌ training [${shortId(jobId)}] finalize failed: ${err?.message}`);
fail(`finalize failed: ${err?.message}`);
- });
+ }).finally(releaseDataAccess);
});
} // end wireProcLifecycle
}
-/**
- * After a phase-aware-watchdog SIGKILL, auto-resume the run from its newest
- * checkpoint (issue #1330) — bounded by STALL_MAX_AUTO_RESUMES so a config that
- * wedges every segment can't resume-loop forever. Best-effort: any failure
- * leaves the run in its finalize-set `failed` state for manual resume. Runs
- * outside the request lifecycle (called from the close handler), so it owns its
- * own error handling rather than bubbling.
- *
- * Returns true ONLY when a resume was actually enqueued (a fresh trainer will
- * re-spawn). The close handler relies on this to decide whether to wake the
- * display: a budget-exhausted or failed resume does NOT re-spawn, so the display
- * must be woken just like any other terminal end — gating on raw `stallKilled`
- * would leave it asleep forever on those paths.
- */
-async function autoResumeAfterStall(runId, jobId) {
- const run = await runsDb.getRun(runId).catch(() => null);
- if (!run) return false;
- const autoCount = run.resume?.autoCount || 0;
- if (autoCount >= STALL_MAX_AUTO_RESUMES) {
- console.log(`⚠️ training [${shortId(jobId)}] stall auto-resume budget exhausted (${autoCount}/${STALL_MAX_AUTO_RESUMES}) — leaving run failed for manual resume`);
- return false;
- }
- // resumeTrainingRun requires a failed/canceled status + a resumable checkpoint;
- // finalize has already flipped the run to failed by the time this runs.
- return resumeTrainingRun(runId, { auto: true }).then(
- (res) => { console.log(`🔁 training [${shortId(jobId)}] auto-resumed run ${shortId(runId)} from step ${res.fromStep} (attempt ${autoCount + 1}/${STALL_MAX_AUTO_RESUMES})`); return true; },
- (err) => { console.error(`❌ training [${shortId(jobId)}] stall auto-resume failed: ${err?.message} — run stays failed for manual resume`); return false; },
- );
-}
-
-async function finalizeTraining({ jobId, runId, code, signal, state, stallKilled = false }) {
+async function finalizeTraining({ jobId, runId, code, signal, state, stallKilled = false, canceledByOperator = false }) {
const run = await runsDb.getRun(runId);
const job = getJob(jobId);
- const canceled = !!job?.cancelRequested;
+ const canceled = !!job?.cancelRequested || canceledByOperator;
// Run record vanished mid-training (direct DB edit / race — the DELETE
// route blocks active runs, so this is defensive). Don't register a LoRA
@@ -1053,7 +543,7 @@ async function finalizeTraining({ jobId, runId, code, signal, state, stallKilled
if (canceled) {
// Queue's cancelRequested flips the failed event into a clean cancel;
- // record keeps the checkpoint lineage for a future resume.
+ // record keeps the checkpoint lineage for inspection or explicit promotion.
await runsDb.updateRun(runId, { status: 'canceled', completedAt: new Date().toISOString(), error: 'Canceled' });
await flipDatasetAfterRun(run, { trained: false });
trainingEvents.emit('failed', { generationId: jobId, error: 'Canceled' });
@@ -1089,14 +579,6 @@ async function finalizeTraining({ jobId, runId, code, signal, state, stallKilled
console.error(`❌ training [${shortId(jobId)}] ${failCode}: ${message}`);
trainingEvents.emit('failed', { generationId: jobId, error: message, code: failCode, repo: failRepo });
- // Phase-aware watchdog SIGKILL (issue #1330): the run is now `failed`, so
- // resumeTrainingRun's status precondition is satisfied — auto-resume from the
- // newest checkpoint. Bounded by STALL_MAX_AUTO_RESUMES; awaited so the resume
- // (re-enqueue + run-record flip back to queued) completes within finalize.
- // Return whether a resume was actually enqueued so the close handler knows NOT
- // to wake the display in that case (the re-spawn re-sleeps it); every other
- // terminal path falls through to the implicit `undefined` (falsy) → wake.
- if (stallKilled) return autoResumeAfterStall(runId, jobId);
}
/**
@@ -1140,6 +622,10 @@ export async function listSamples(runId) {
* degradation — see the loss-is-misleading note in checkpoints.js).
*/
export async function promoteCheckpoint(runId, step) {
+ return withRetiredDataAccess(() => promoteRetainedCheckpoint(runId, step));
+}
+
+async function promoteRetainedCheckpoint(runId, step) {
const run = await runsDb.getRunRequired(runId);
// Allow completed runs AND failed/canceled runs that saved at least one
// checkpoint — promoting a partial checkpoint is a deliberate salvage (the
@@ -1196,7 +682,7 @@ export async function promoteCheckpoint(runId, step) {
// render failure (OOM, missing venv, server restart) must never fail the
// promote — the LoRA is already deployed; the thumbnail is cosmetic.
if (!target.previewUrl) {
- ensureCheckpointPreview(run, step, filename).catch((err) => {
+ withRetiredDataAccess(() => ensureCheckpointPreview(run, step, filename)).catch((err) => {
console.error(`⚠️ training [${shortId(runId)}] preview render for step ${step} failed: ${err?.message}`);
});
}
@@ -1327,17 +813,27 @@ export async function initLoraTraining() {
for (const run of active) {
const job = run.jobId ? getJob(run.jobId) : null;
if (!job || ['failed', 'canceled', 'completed'].includes(job.status)) {
- // The trainer is a detached child that survives a pm2 restart, so it may
- // STILL be running even though its in-memory job is gone. Reap it (clean
- // SIGTERM → checkpoint → SIGKILL) before marking the run failed/resumable,
- // so a resume can't spawn a second trainer into the same checkpoint dir.
- const reaped = await reapDetached(join(runDir(run.id), '.detached')).catch(() => ({ reaped: false }));
- if (reaped.reaped) console.log(`🧹 reaped surviving trainer pid ${reaped.pid} for run ${shortId(run.id)}`);
+ // Upgrade never kills an orphan trainer. Keep its artifacts pinned until
+ // the existing process finishes or the operator explicitly cancels it.
+ if (await hasSurvivingTrainer(run.id)) {
+ const jobId = run.jobId || run.id;
+ const recoveryPermit = maintenance.recoverOwned('media', jobId);
+ try {
+ const observer = await recoveryPermit.run(() => runTraining({ jobId, runId: run.id, reattach: true, recoveryPermit }));
+ if (observer?.observing) console.log(`🔁 Observing retired trainer ${shortId(run.id)} without restarting it`);
+ else if (!observer?.preserved && !await hasSurvivingTrainer(run.id)) await recoveryPermit.finish();
+ } catch (err) {
+ // An uncertain observer must retain its recovery permit: the detached
+ // process may still be writing, even when its database/claim is down.
+ console.error(`❌ Retired trainer ${shortId(run.id)} could not be observed; artifacts stay pinned: ${err?.message}`);
+ }
+ continue;
+ }
await runsDb.updateRun(run.id, {
- status: 'failed', error: 'interrupted by restart', completedAt: new Date().toISOString(),
+ status: job?.status === 'canceled' ? 'canceled' : 'failed', error: 'LoRA training retired; artifacts preserved', completedAt: new Date().toISOString(),
}).catch(() => {});
await flipDatasetAfterRun(run, { trained: false });
- console.log(`🧹 training run ${shortId(run.id)} marked failed (interrupted by restart)`);
+ console.log(`🧹 training run ${shortId(run.id)} retired; artifacts preserved`);
}
}
diff --git a/server/services/loraTraining/lifecycle.maintenance.test.js b/server/services/loraTraining/lifecycle.maintenance.test.js
index 7fbe2eccf5..c67701f98d 100644
--- a/server/services/loraTraining/lifecycle.maintenance.test.js
+++ b/server/services/loraTraining/lifecycle.maintenance.test.js
@@ -5,7 +5,7 @@ import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../li
const fixture = vi.hoisted(() => ({
admission: null, child: null, run: null, queueJob: null,
- release: vi.fn(), save: vi.fn(), saveDataset: vi.fn(),
+ release: vi.fn(), save: vi.fn(), saveDataset: vi.fn(), read: vi.fn(), adopt: vi.fn(), attach: vi.fn(), active: null,
}));
vi.mock('../../lib/fileUtils.js', async importOriginal =>
makePathsProxy(await importOriginal(), { dataRoot: () => lazyTempDataRoot('training-maintenance-') }));
@@ -18,15 +18,15 @@ vi.mock('../../lib/childProcess.js', async importOriginal => ({
...await importOriginal(), spawn: vi.fn(() => { throw new Error('Unexpected real child launch'); }),
}));
vi.mock('../../lib/detachedSpawn.js', () => ({
- reattachDetached: async () => fixture.child, spawnDetached: vi.fn(), reapDetached: vi.fn(), isReattachable: vi.fn(),
+ reattachDetached: (...args) => fixture.attach(...args), spawnDetached: vi.fn(), reapDetached: vi.fn(), isReattachable: vi.fn(),
}));
vi.mock('../../lib/heavyJobClaim.js', () => ({
- adoptHeavyLocalJob: async () => ({ ok: true, holder: { pid: fixture.child.pid },
- handoffTo: async () => {}, release: () => fixture.release() }),
+ adoptHeavyLocalJob: (...args) => fixture.adopt(...args),
claimHeavyLocalJob: vi.fn(),
}));
vi.mock('./db.js', () => ({
- getRun: async () => fixture.run,
+ getRun: (...args) => fixture.read(...args),
+ listActiveRuns: async () => fixture.active || [fixture.run],
updateRun: (...args) => fixture.save(...args), getRunRequired: vi.fn(), listRuns: vi.fn(), deleteRun: vi.fn(),
}));
vi.mock('../settings.js', () => ({ getSettings: async () => ({ loraTraining: { stallWatchdog: false } }) }));
@@ -36,8 +36,9 @@ vi.mock('../mediaJobQueue/index.js', () => ({
}));
vi.mock('./displayPower.js', () => ({ sleepDisplayForTraining: vi.fn(), wakeDisplay: vi.fn() }));
-const { runTraining, cancel, trainingEvents } = await import('./index.js');
+const { runTraining, initLoraTraining, cancel, trainingEvents } = await import('./index.js');
const { createMaintenanceAdmission } = await import('../../lib/maintenanceAdmission.js');
+const { isReattachable, spawnDetached, reapDetached } = await import('../../lib/detachedSpawn.js');
const { PATHS } = await import('../../lib/fileUtils.js');
let fixtureNumber = 0;
beforeEach(() => {
@@ -47,6 +48,12 @@ beforeEach(() => {
stdout: new EventEmitter(), stderr: new EventEmitter(), kill: vi.fn() });
fixture.run = { id: 'example-run', jobId: 'example-job', datasetId: 'example-dataset', params: { steps: 10 }, runtime: 'mflux' };
fixture.queueJob = {};
+ fixture.active = null;
+ fixture.read.mockReset().mockImplementation(async () => fixture.run);
+ fixture.attach.mockReset().mockImplementation(async () => fixture.child);
+ fixture.adopt.mockReset().mockImplementation(async () => ({ ok: true, holder: { pid: fixture.child.pid },
+ handoffTo: async () => {}, release: () => fixture.release() }));
+ vi.mocked(isReattachable).mockReset().mockResolvedValue(false);
fixture.release.mockReset().mockResolvedValue();
fixture.save.mockReset().mockResolvedValue();
fixture.saveDataset.mockReset().mockResolvedValue();
@@ -168,3 +175,51 @@ describe('training physical settlement under maintenance', () => {
}
});
});
+
+it('observes an orphan survivor without spawning/reaping and settles explicit cancellation in retained history', async () => {
+ fixture.queueJob = null;
+ fixture.run.status = 'running';
+ vi.mocked(isReattachable).mockResolvedValue(true);
+ await initLoraTraining();
+ expect(spawnDetached).not.toHaveBeenCalled();
+ expect(reapDetached).not.toHaveBeenCalled();
+ expect(fixture.child.listenerCount('close')).toBeGreaterThan(0);
+ const { withRetiredDataRemoval } = await import('../../lib/retiredLoraTraining.js');
+ await expect(withRetiredDataRemoval(async () => {})).rejects.toMatchObject({ code: 'MAINTENANCE_BUSY' });
+ expect(cancel('example-job')).toBe(true);
+ const originalSave = fixture.save.getMockImplementation();
+ let finishRecord;
+ fixture.save.mockImplementationOnce(() => new Promise(resolve => { finishRecord = resolve; }));
+ fixture.child.emit('close', 1, 'SIGTERM');
+ await vi.waitFor(() => expect(finishRecord).toBeTypeOf('function'));
+ await expect(withRetiredDataRemoval(async () => {})).rejects.toMatchObject({ code: 'MAINTENANCE_BUSY' });
+ finishRecord();
+ fixture.save.mockImplementation(originalSave);
+ await new Promise(resolve => setImmediate(resolve));
+ await expect(withRetiredDataRemoval(async () => 'settled')).resolves.toBe('settled');
+ await vi.waitFor(() => expect(fixture.save).toHaveBeenCalledWith('example-run', expect.objectContaining({ status: 'canceled' })));
+});
+
+
+it.each(['database', 'attach', 'claim', 'refused claim'])('preserves a live orphan after %s failure and continues boot reconciliation', async failure => {
+ fixture.queueJob = null;
+ fixture.run.status = 'running';
+ fixture.active = [fixture.run, { id: 'next-run', status: 'queued' }];
+ vi.mocked(isReattachable).mockImplementation(async path => path.includes('example-run'));
+ if (failure === 'database') fixture.read.mockRejectedValueOnce(new Error('Database unavailable'));
+ if (failure === 'attach') fixture.attach.mockRejectedValueOnce(new Error('Observer unavailable'));
+ if (failure === 'claim') fixture.adopt.mockRejectedValueOnce(new Error('Claim unavailable'));
+ if (failure === 'refused claim') fixture.adopt.mockResolvedValueOnce({ ok: false, message: 'Accelerator occupied' });
+ const { mediaJobEvents } = await import('../mediaJobQueue/index.js');
+ const before = mediaJobEvents.listenerCount('canceled');
+ await expect(initLoraTraining()).resolves.toBeUndefined();
+ expect(fixture.save).not.toHaveBeenCalledWith('example-run', expect.anything());
+ expect(fixture.save).toHaveBeenCalledWith('next-run', expect.objectContaining({ status: 'failed' }));
+ expect(mediaJobEvents.listenerCount('canceled')).toBe(before + 1);
+ expect(fixture.release).not.toHaveBeenCalled();
+ expect(fixture.admission.status().blockers).toEqual(expect.arrayContaining([
+ expect.objectContaining({ resource: 'example-job' }),
+ ]));
+ expect(spawnDetached).not.toHaveBeenCalled();
+ expect(reapDetached).not.toHaveBeenCalled();
+});
diff --git a/server/services/loraTraining/memoryPrep.js b/server/services/loraTraining/memoryPrep.js
deleted file mode 100644
index b9e02b5f15..0000000000
--- a/server/services/loraTraining/memoryPrep.js
+++ /dev/null
@@ -1,11 +0,0 @@
-/** Training-specific memory policy built on the shared local-memory preflight. */
-
-import { prepareLocalMemory } from '../localMemory.js';
-
-export { unloadResidentModels, getAvailableMemoryGb, isLocalBackendUrl, gpuBlockersMessage } from '../localMemory.js';
-
-// Below this floor even a small 4-bit training run risks swap-thrash. Larger
-// run sizing remains training-specific and still consumes `budgetGb` below.
-export const TRAINING_MIN_HEADROOM_GB = 24;
-
-export const prepareMemoryForTraining = prepareLocalMemory;
diff --git a/server/services/loraTraining/memoryPrep.test.js b/server/services/loraTraining/memoryPrep.test.js
deleted file mode 100644
index 107cbe9152..0000000000
--- a/server/services/loraTraining/memoryPrep.test.js
+++ /dev/null
@@ -1,162 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-
-// Hoisted mock state so the factory closures below can reach it.
-const h = vi.hoisted(() => ({
- ollamaLoaded: [],
- ollamaUnload: vi.fn(),
- ollamaUrl: 'http://localhost:11434',
- lmLoaded: [],
- lmUnload: vi.fn(),
- lmUrl: 'http://localhost:1234',
- platform: 'darwin',
- totalBytes: 128 * 2 ** 30,
- freeBytes: 8 * 2 ** 30,
- vmStat: { stdout: '' },
- vmStatThrows: false,
-}));
-
-vi.mock('../../services/ollamaManager.js', () => ({
- getLoadedModels: vi.fn(async () => h.ollamaLoaded),
- unloadModel: (...a) => h.ollamaUnload(...a),
- getBaseUrl: () => h.ollamaUrl,
-}));
-vi.mock('../../services/lmStudioManager.js', () => ({
- getLoadedModels: vi.fn(async () => h.lmLoaded),
- unloadModel: (...a) => h.lmUnload(...a),
- getBaseUrl: () => h.lmUrl,
-}));
-// Spread the real module: the preflight's own dependency graph reaches other
-// `os` exports (`homedir`, via the vLLM project-dir resolver), and a
-// listed-exports-only mock fails the whole file the moment it reaches one more.
-vi.mock('os', async (importOriginal) => ({
- ...(await importOriginal()),
- platform: () => h.platform,
- totalmem: () => h.totalBytes,
- freemem: () => h.freeBytes,
-}));
-vi.mock('../../lib/childProcess.js', () => ({
- execFile: (_cmd, cb) => {
- if (h.vmStatThrows) return cb(new Error('vm_stat missing'));
- return cb(null, h.vmStat);
- },
-}));
-
-const { unloadResidentModels, getAvailableMemoryGb, prepareMemoryForTraining, isLocalBackendUrl, TRAINING_MIN_HEADROOM_GB } =
- await import('./memoryPrep.js');
-
-// vm_stat with 16 KiB pages: free + inactive + speculative + purgeable pages
-// sum to a known GB total so the parser assertion is exact.
-const vmStatWith = (gb) => {
- const pages = (gb * 2 ** 30) / 16384;
- return `Mach Virtual Memory Statistics: (page size of 16384 bytes)
-Pages free: ${Math.floor(pages / 4)}.
-Pages active: 100000.
-Pages inactive: ${Math.floor(pages / 4)}.
-Pages speculative: ${Math.floor(pages / 4)}.
-Pages wired down: 50000.
-Pages purgeable: ${Math.ceil(pages / 4)}.
-`;
-};
-
-beforeEach(() => {
- h.ollamaLoaded = [];
- h.lmLoaded = [];
- h.ollamaUnload = vi.fn(async () => ({ unloaded: true }));
- h.lmUnload = vi.fn(async () => ({ success: true }));
- h.ollamaUrl = 'http://localhost:11434';
- h.lmUrl = 'http://localhost:1234';
- h.platform = 'darwin';
- h.totalBytes = 128 * 2 ** 30;
- h.freeBytes = 8 * 2 ** 30;
- h.vmStat = { stdout: vmStatWith(100) };
- h.vmStatThrows = false;
-});
-
-describe('unloadResidentModels', () => {
- it('unloads every resident ollama + LM Studio model and labels them', async () => {
- h.ollamaLoaded = [{ name: 'llama3:70b' }, { id: 'qwen2.5' }];
- h.lmLoaded = [{ id: 'mlx-community/big' }];
- const freed = await unloadResidentModels();
- expect(freed).toEqual(['ollama:llama3:70b', 'ollama:qwen2.5', 'lmstudio:mlx-community/big']);
- expect(h.ollamaUnload).toHaveBeenCalledTimes(2);
- expect(h.lmUnload).toHaveBeenCalledWith('mlx-community/big');
- });
-
- it('swallows unload failures and omits the model from the freed list', async () => {
- h.ollamaLoaded = [{ name: 'a' }, { name: 'b' }];
- h.ollamaUnload = vi.fn()
- .mockResolvedValueOnce({ unloaded: false, reason: 'not loaded' })
- .mockRejectedValueOnce(new Error('server down'));
- const freed = await unloadResidentModels();
- expect(freed).toEqual([]);
- });
-
- it('returns empty when nothing is resident', async () => {
- expect(await unloadResidentModels()).toEqual([]);
- });
-
- it('skips a remote backend — never evicts a peer machine\'s models', async () => {
- h.ollamaLoaded = [{ name: 'remote-llm' }];
- h.lmLoaded = [{ id: 'local-lm' }];
- h.ollamaUrl = 'http://10.0.0.5:11434'; // LAN peer — unloading frees no local memory
- h.lmUrl = 'http://127.0.0.1:1234'; // loopback — still eligible
- const freed = await unloadResidentModels();
- expect(freed).toEqual(['lmstudio:local-lm']);
- expect(h.ollamaUnload).not.toHaveBeenCalled();
- expect(h.lmUnload).toHaveBeenCalledWith('local-lm');
- });
-});
-
-describe('isLocalBackendUrl', () => {
- it('accepts loopback hosts (incl. 127.0.0.0/8 and IPv6)', () => {
- for (const u of [
- 'http://localhost:11434', 'http://127.0.0.1:1234', 'http://127.0.0.53:1234',
- 'http://0.0.0.0:1234', 'http://[::1]:1234',
- ]) expect(isLocalBackendUrl(u)).toBe(true);
- });
-
- it('rejects LAN/remote hosts and unparseable input', () => {
- for (const u of [
- 'http://10.0.0.5:11434', 'http://192.168.1.20:1234', 'http://peer.local:11434',
- 'http://example.com', '', null, undefined, 'not a url',
- ]) expect(isLocalBackendUrl(u)).toBe(false);
- });
-});
-
-describe('getAvailableMemoryGb', () => {
- it('parses vm_stat (free+inactive+speculative+purgeable) on darwin', async () => {
- h.vmStat = { stdout: vmStatWith(64) };
- expect(await getAvailableMemoryGb()).toBeCloseTo(64, 0);
- });
-
- it('falls back to freemem() when vm_stat fails', async () => {
- h.vmStatThrows = true;
- h.freeBytes = 12 * 2 ** 30;
- expect(await getAvailableMemoryGb()).toBeCloseTo(12, 5);
- });
-
- it('uses freemem() off darwin', async () => {
- h.platform = 'linux';
- h.freeBytes = 7 * 2 ** 30;
- expect(await getAvailableMemoryGb()).toBeCloseTo(7, 5);
- });
-});
-
-describe('prepareMemoryForTraining', () => {
- it('clamps the budget to physical RAM and reports freed models', async () => {
- h.ollamaLoaded = [{ name: 'm' }];
- h.totalBytes = 96 * 2 ** 30;
- h.vmStat = { stdout: vmStatWith(200) }; // bogus over-report → clamp to total
- const r = await prepareMemoryForTraining();
- expect(r.unloaded).toEqual(['ollama:m']);
- expect(r.budgetGb).toBe(96);
- expect(r.totalGb).toBe(96);
- });
-
- it('budget tracks the post-unload available pool when below total', async () => {
- h.vmStat = { stdout: vmStatWith(40) };
- const r = await prepareMemoryForTraining();
- expect(r.budgetGb).toBeCloseTo(40, 0);
- expect(r.budgetGb).toBeLessThan(TRAINING_MIN_HEADROOM_GB + 100);
- });
-});
diff --git a/server/services/loraTraining/mpsGuard.test.js b/server/services/loraTraining/mpsGuard.test.js
deleted file mode 100644
index e1be58c661..0000000000
--- a/server/services/loraTraining/mpsGuard.test.js
+++ /dev/null
@@ -1,71 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-
-// Guard: startTrainingRun must REFUSE the torch/diffusers fallback trainer on
-// Apple Silicon at request time. PyTorch's MPS backend has no linear_backward
-// for the FLUX.2 transformer layers, so a torch LoRA backward dies at the first
-// optimizer step — mflux (MLX) is the Apple-Silicon runtime (issue #1227). The
-// refusal must happen before any run record is created or job enqueued, so we
-// never load a ~16 GB base + precompute latents only to crash. On non-darwin
-// (the torch path's real CUDA/CPU target) the same request proceeds.
-let mockPlatform = 'darwin';
-vi.mock('os', async (importActual) => {
- const actual = await importActual();
- return { ...actual, platform: () => mockPlatform };
-});
-
-const createRunMock = vi.fn(async () => {});
-const updateRunMock = vi.fn(async () => {});
-const enqueueJobMock = vi.fn(async () => ({ jobId: 'job-1', position: 0 }));
-const validateDatasetReadyMock = vi.fn(async () => ({
- dataset: { character: { entryId: 'c1', universeId: 'u1', name: 'Test Subject' }, triggerWord: 'test_subject' },
- manifest: { images: [] },
-}));
-const isFlux2VenvHealthyMock = vi.fn(async () => true);
-
-vi.mock('./db.js', () => ({
- createRun: (...a) => createRunMock(...a),
- updateRun: (...a) => updateRunMock(...a),
- getRun: vi.fn(), getRunRequired: vi.fn(), listRuns: vi.fn(), deleteRun: vi.fn(),
-}));
-vi.mock('./dataset.js', () => ({ validateDatasetReady: (...a) => validateDatasetReadyMock(...a) }));
-vi.mock('../settings.js', () => ({ getSettings: vi.fn(async () => ({})) }));
-vi.mock('../loraDatasets.js', () => ({ updateDataset: vi.fn(async () => {}) }));
-vi.mock('../mediaJobQueue/index.js', () => ({
- assertMediaQueueRoom: vi.fn(),
- enqueueJob: (...a) => enqueueJobMock(...a),
- getJob: vi.fn(),
- mediaJobEvents: { emit: vi.fn(), on: vi.fn() },
-}));
-vi.mock('../../lib/pythonSetup.js', () => ({
- resolveFlux2Python: () => '/fake/venv-flux2/bin/python3',
- isFlux2VenvHealthy: (...a) => isFlux2VenvHealthyMock(...a),
- // No mflux-train anywhere → resolves to null, so routing falls through to the
- // torch (flux2) runtime the guard protects. Mirrors resolveMfluxPython(null).
- resolveMfluxPython: (p) => p || null,
-}));
-
-const { startTrainingRun } = await import('./index.js');
-
-// getSettings → {} means pythonPath is null, so isMfluxTrainAvailable(null) is
-// false and routing resolves to the torch (flux2) runtime — the branch the
-// guard protects. flux2-klein-4b is a real registry model.
-const BASE = 'flux2-klein-4b';
-
-beforeEach(() => { vi.clearAllMocks(); mockPlatform = 'darwin'; });
-
-describe('startTrainingRun — Apple Silicon torch-runtime guard', () => {
- it('refuses torch training on darwin before creating a run or queuing a job', async () => {
- await expect(startTrainingRun({ datasetId: 'ds1', baseModelId: BASE }))
- .rejects.toMatchObject({ code: 'TRAINING_MPS_UNSUPPORTED', status: 412 });
- expect(createRunMock).not.toHaveBeenCalled();
- expect(enqueueJobMock).not.toHaveBeenCalled();
- });
-
- it('does NOT fire the guard on non-darwin (torch path proceeds to queue)', async () => {
- mockPlatform = 'linux';
- const res = await startTrainingRun({ datasetId: 'ds1', baseModelId: BASE });
- expect(res).toMatchObject({ status: 'queued' });
- expect(createRunMock).toHaveBeenCalled();
- expect(enqueueJobMock).toHaveBeenCalled();
- });
-});
diff --git a/server/services/loraTraining/resume.test.js b/server/services/loraTraining/resume.test.js
index 493aabe866..8b19e9479d 100644
--- a/server/services/loraTraining/resume.test.js
+++ b/server/services/loraTraining/resume.test.js
@@ -1,124 +1,12 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-
-// Mock every external dependency resumeTrainingRun reaches so we can drive the
-// branch logic without a real DB, queue, or python venv. Keeping runtimes.js
-// real lets the test reference TRAINING_RUNTIMES.FLUX2 against the same source.
-const getRunRequiredMock = vi.fn();
-const updateRunMock = vi.fn();
-const resolveLatestCheckpointArtifactMock = vi.fn();
-const validateDatasetReadyMock = vi.fn();
-const enqueueJobMock = vi.fn();
-const updateDatasetMock = vi.fn();
-const isFlux2VenvHealthyMock = vi.fn();
-
-vi.mock('./db.js', () => ({
- getRunRequired: (...a) => getRunRequiredMock(...a),
- updateRun: (...a) => updateRunMock(...a),
- getRun: vi.fn(),
- listRuns: vi.fn(),
- deleteRun: vi.fn(),
-}));
-vi.mock('./checkpoints.js', () => ({
- resolveLatestCheckpointArtifact: (...a) => resolveLatestCheckpointArtifactMock(...a),
- listRunCheckpoints: vi.fn(),
- listRunSamples: vi.fn(),
- resolveCheckpointAdapterBuffer: vi.fn(),
- selectDeployableCheckpoint: vi.fn(),
-}));
-vi.mock('./dataset.js', () => ({ validateDatasetReady: (...a) => validateDatasetReadyMock(...a) }));
-vi.mock('../settings.js', () => ({ getSettings: vi.fn(async () => ({})) }));
-vi.mock('../../lib/pythonSetup.js', () => ({
- isFlux2VenvHealthy: (...a) => isFlux2VenvHealthyMock(...a),
- resolveFlux2Python: vi.fn(() => '/venv/python'),
- resolveMfluxPython: (p) => p || null,
-}));
-vi.mock('../mediaJobQueue/index.js', () => ({
- assertMediaQueueRoom: vi.fn(),
- enqueueJob: (...a) => enqueueJobMock(...a),
- getJob: vi.fn(),
- mediaJobEvents: { emit: vi.fn(), on: vi.fn() },
-}));
-vi.mock('../loraDatasets.js', () => ({ updateDataset: (...a) => updateDatasetMock(...a) }));
-
-const { TRAINING_RUNTIMES } = await import('./runtimes.js');
-const { resumeTrainingRun } = await import('./index.js');
-
-const flux2Run = (overrides = {}) => ({
- id: 'run-flux2',
- status: 'failed',
- runtime: TRAINING_RUNTIMES.FLUX2,
- datasetId: 'ds-1',
- character: { entryId: 'c1', universeId: 'u1' },
- triggerWord: 'kessa',
- baseModelId: 'm1',
- params: { steps: 600, rank: 16 },
- ...overrides,
-});
-
-describe('resumeTrainingRun — flux2 path', () => {
- beforeEach(() => {
- vi.clearAllMocks();
- updateRunMock.mockResolvedValue();
- updateDatasetMock.mockResolvedValue();
- validateDatasetReadyMock.mockResolvedValue({ dataset: { character: { entryId: 'c1', universeId: 'u1' } } });
- isFlux2VenvHealthyMock.mockResolvedValue(true);
- enqueueJobMock.mockReturnValue({ jobId: 'job-1', position: 0 });
- });
-
- it('enqueues a resume job for a failed flux2 run (no RESUME_UNSUPPORTED_RUNTIME)', async () => {
- getRunRequiredMock.mockResolvedValue(flux2Run());
- resolveLatestCheckpointArtifactMock.mockReturnValue({ step: 150, path: '/runs/run-flux2/checkpoints/step-000150' });
-
- const result = await resumeTrainingRun('run-flux2');
-
- expect(result).toMatchObject({ runId: 'run-flux2', jobId: 'job-1', status: 'queued', fromStep: 150 });
- expect(enqueueJobMock).toHaveBeenCalledTimes(1);
- const job = enqueueJobMock.mock.calls[0][0];
- expect(job.kind).toBe('training');
- expect(job.params.runtime).toBe(TRAINING_RUNTIMES.FLUX2);
- // The checkpoint dir flows through to the trainer's --resume-from.
- expect(job.params.resumeCheckpoint).toBe('/runs/run-flux2/checkpoints/step-000150');
- // The run record is flipped back to queued with the resume bookkeeping.
- expect(updateRunMock).toHaveBeenCalledWith('run-flux2', expect.any(Function));
- });
-
- it('still rejects a flux2 run with no on-disk checkpoint (NO_RESUMABLE_CHECKPOINT)', async () => {
- getRunRequiredMock.mockResolvedValue(flux2Run());
- resolveLatestCheckpointArtifactMock.mockReturnValue(null);
-
- await expect(resumeTrainingRun('run-flux2')).rejects.toMatchObject({ code: 'NO_RESUMABLE_CHECKPOINT' });
- expect(enqueueJobMock).not.toHaveBeenCalled();
- });
-
- it('refuses to resume a still-running flux2 run (RUN_NOT_RESUMABLE)', async () => {
- getRunRequiredMock.mockResolvedValue(flux2Run({ status: 'running' }));
-
- await expect(resumeTrainingRun('run-flux2')).rejects.toMatchObject({ code: 'RUN_NOT_RESUMABLE' });
- expect(enqueueJobMock).not.toHaveBeenCalled();
- });
-
- // Issue #1330: the phase-aware stall watchdog auto-resumes with { auto: true }.
- // The resume bookkeeping must increment autoCount (the watchdog's own budget,
- // separate from total count) and stamp lastReason so the cap is enforceable.
- it('stamps auto-resume bookkeeping (autoCount + lastReason) when auto:true', async () => {
- getRunRequiredMock.mockResolvedValue(flux2Run({ resume: { count: 1, autoCount: 1 } }));
- resolveLatestCheckpointArtifactMock.mockReturnValue({ step: 300, path: '/runs/run-flux2/checkpoints/step-000300' });
-
- await resumeTrainingRun('run-flux2', { auto: true });
-
- const updater = updateRunMock.mock.calls.find((c) => c[0] === 'run-flux2' && typeof c[1] === 'function')[1];
- const next = updater(flux2Run({ resume: { count: 1, autoCount: 1 } }));
- expect(next.resume).toMatchObject({ count: 2, autoCount: 2, fromStep: 300, lastReason: 'stall-watchdog' });
- });
-
- it('manual resume does not bump autoCount and stamps lastReason manual', async () => {
- getRunRequiredMock.mockResolvedValue(flux2Run({ resume: { count: 2, autoCount: 2 } }));
- resolveLatestCheckpointArtifactMock.mockReturnValue({ step: 300, path: '/runs/run-flux2/checkpoints/step-000300' });
-
- await resumeTrainingRun('run-flux2');
-
- const updater = updateRunMock.mock.calls.find((c) => c[0] === 'run-flux2' && typeof c[1] === 'function')[1];
- const next = updater(flux2Run({ resume: { count: 2, autoCount: 2 } }));
- expect(next.resume).toMatchObject({ count: 3, autoCount: 2, lastReason: 'manual' });
- });
+import { expect, it, vi } from 'vitest';
+vi.mock('./db.js', () => ({ getRun: vi.fn(), listRuns: vi.fn(), getRunRequired: vi.fn(), deleteRun: vi.fn(), updateRun: vi.fn() }));
+vi.mock('../mediaJobQueue/index.js', () => ({ getJob: vi.fn(), mediaJobEvents: { on: vi.fn() } }));
+vi.mock('../loraDatasets.js', () => ({ updateDataset: vi.fn() }));
+import { startTrainingRun, resumeTrainingRun } from './index.js';
+import { updateRun, getRunRequired } from './db.js';
+it('refuses fresh launch, manual resume and automatic resume without changing retained records', async () => {
+ await expect(startTrainingRun({ datasetId: 'example' })).rejects.toMatchObject({ status: 410, code: 'LORA_TRAINING_RETIRED' });
+ await expect(resumeTrainingRun('example')).rejects.toMatchObject({ status: 410, code: 'LORA_TRAINING_RETIRED' });
+ await expect(resumeTrainingRun('example', { auto: true })).rejects.toMatchObject({ status: 410, code: 'LORA_TRAINING_RETIRED' });
+ expect(updateRun).not.toHaveBeenCalled(); expect(getRunRequired).not.toHaveBeenCalled();
});
diff --git a/server/services/loraTraining/runtimes.js b/server/services/loraTraining/runtimes.js
index 25e85bea0d..f186d11abb 100644
--- a/server/services/loraTraining/runtimes.js
+++ b/server/services/loraTraining/runtimes.js
@@ -1,357 +1,2 @@
-/**
- * LoRA training runtime routing + argument/config builders. Pure — no I/O.
- *
- * Both runtimes train FLUX.2 Klein character LoRAs; they are two ENGINES
- * for the same output (verified against mflux 0.17.5, which removed
- * FLUX.1 training entirely):
- *
- * - 'mflux' — MLX-native training via mflux's `mflux-train` CLI,
- * wrapped by scripts/train_mflux_lora.py. Preferred on Apple Silicon
- * when the user's mflux install ships the trainer. Trains the
- * non-distilled `flux2-klein-base-{4b,9b}` models (mflux's documented
- * recommendation) and exports adapters with diffusers-style key
- * naming (`transformer.transformer_blocks.N...lora_A.weight`), so
- * the result loads in PortOS's diffusers-based flux2 renderer.
- * - 'flux2' — torch/diffusers+peft training via the vendored
- * scripts/train_flux2_lora.py in ~/.portos/venv-flux2. The fallback
- * when mflux's trainer is unavailable (non-Darwin installs, older
- * mflux). Trains against the bf16 base repos.
- *
- * Either way the trained LoRA is gated to its size variant via the
- * `flux2-4b` / `flux2-9b` compat key (hidden dims differ between sizes).
- */
-
-import { ServerError } from '../../lib/errorHandler.js';
-import { isFlux2, flux2VariantFromModel, FLUX2_BF16_BASE_REPOS } from '../../lib/runners.js';
-
+// Historical runtime identifiers used to decode retained checkpoints.
export const TRAINING_RUNTIMES = Object.freeze({ MFLUX: 'mflux', FLUX2: 'flux2' });
-
-// Defaults validated 2026-06-15 on a real-face character LoRA (44-image dataset,
-// FLUX.2 Klein 9B 4-bit): identity bound cleanly and fast, stable through the run.
-// History worth knowing before lowering these again: an earlier run "diverged"
-// (sample collapsed to black) and the defaults were cut to 600 steps / lr 5e-5 /
-// rank 16 — but the real cause was captions over-describing invariant identity
-// (it bound to the words, not the trigger), NOT the learning rate. With good
-// captions, lr 1e-4 + rank 32 + ~1200 steps converges faster and sharper without
-// collapse. 768 captures more facial detail than 512; checkpoint/sample cadence
-// gives the picker several healthy steps to choose from. The base model + quant
-// tier are chosen elsewhere (deriveMfluxMemoryConfig + the base-model picker,
-// which prefers 9B); these are the per-run hyperparameters the form seeds from.
-export const TRAINING_DEFAULTS = Object.freeze({
- steps: 1200,
- rank: 32,
- learningRate: 0.0001,
- resolution: 768,
- checkpointEvery: 300,
- sampleEvery: 300,
-});
-
-// bf16 training bases per FLUX.2 size variant (torch runtime). Inference may
-// run quantized repos; training resolves to these regardless of which flux2-*
-// model id the user picked. Same repos the image runner renders a LoRA against
-// (PEFT can't load onto quantized weights) — one source of truth in runners.js.
-export const FLUX2_TRAIN_REPOS = FLUX2_BF16_BASE_REPOS;
-
-// mflux model ids per size variant. mflux's flux2 README recommends the
-// non-distilled base models for training; the resulting adapter applies to
-// the distilled inference models of the same size (identical module names
-// and hidden dims).
-export const MFLUX_TRAIN_MODELS = Object.freeze({
- '4b': 'flux2-klein-base-4b',
- '9b': 'flux2-klein-base-9b',
-});
-
-// Crash-resilience floor: guarantee at least this many checkpoints across a
-// run regardless of the user's checkpointEvery. macOS GPU watchdog-timeout
-// kernel panics during sustained mflux training (see docs/TROUBLESHOOTING.md
-// "GPU watchdog kernel panic") hard-reboot the machine mid-run; without a
-// floor, checkpointEvery=0 ("only final save") or a very large interval means
-// such a crash discards the entire run. Capping the interval at
-// totalSteps/MIN_CHECKPOINTS bounds the worst-case loss to ~1/N of the run.
-export const MFLUX_MIN_CHECKPOINTS = 4;
-
-// Noise-schedule shape for mflux base-model training — mirrors the official
-// flux2 README example (steps 40, guidance 1.0, train the high-noise window
-// [25, 40)). These are INFERENCE-schedule steps, distinct from the training
-// duration (num_epochs × images).
-const MFLUX_SCHED = Object.freeze({ steps: 40, guidance: 1.0, timestepLow: 25, timestepHigh: 40 });
-
-/**
- * Memory-derived training knobs — mirrors the FFLF pixel-budget pattern
- * (videoGen/local.js): pure, with total RAM injected by the caller.
- * Training a bf16 base + in-RAM latent cache OOM-killed a 48 GB machine
- * during verification, so every tier trains QLoRA-style against an
- * on-the-fly-quantized base. The encoded dataset cache ALWAYS spills to
- * disk (`low_ram`): keeping it in RAM bought nothing but pushed a 128 GB box
- * to 122 GB used + 21 GB swap (the swap-thrash that coincided with GPU
- * watchdog reboots — see docs/research/2026-06-13-mflux-training-watchdog-panic.md);
- * disk-backing it costs only I/O, no training quality. Quantizing the FROZEN
- * base is the standard QLoRA recipe — the LoRA weights stay full precision.
- *
- * Default tier never auto-selects unquantized bf16 (issue #1321): even a
- * ≥96 GB box gets 8-bit QLoRA, not bf16. The bf16 9B path was pathologically
- * slow (couldn't complete a step in 14 min on an M5 Max 128 GB) AND the config
- * that GPU-watchdog-panicked the box three times
- * (docs/research/2026-06-13-mflux-training-watchdog-panic.md) — so it's opt-in
- * only, never the silent default. 8-bit QLoRA is the standard character-LoRA
- * recipe with minimal fidelity loss; a run that genuinely wants a heavier base
- * asks for it explicitly via `baseQuant: 16` (→ `quantize: null`).
- *
- * Callers feed the *available* memory budget (post-unload, see memoryPrep.js),
- * not raw RAM, so the tier reflects real headroom on a shared box.
- *
- * Override (`LORA_TRAIN_MAX_QUANT_BITS`): a hard per-install ceiling on the
- * frozen-base bit-width. Setting it to `4` forces every tier (and every per-run
- * opt-in) down to 4-bit; setting it to `8` clamps an explicit `baseQuant: 16`
- * bf16 opt-in back to 8-bit (the 8-bit default is already at/under the cap, so
- * it's a no-op there). Unset → the memory-derived tier plus whatever a run opts
- * into. Accepts 8 or 4; anything else is ignored.
- *
- * Per-run overrides (`overrides = { quantize?, low_ram? }`, issue #1321): a
- * single run can opt into a heavier or lighter frozen base than its memory tier
- * without a code change (exposed as the `baseQuant`/`lowRam` request params).
- * `quantize` present — including `null` (bf16) — replaces the derived tier;
- * `low_ram` boolean replaces the derived spill setting; absent keys keep the
- * memory-derived value. The `LORA_TRAIN_MAX_QUANT_BITS` cap still applies AFTER
- * the override, so a per-run opt-in can never exceed a box's hard safety
- * ceiling (the cap only ever makes the base lighter, never heavier).
- */
-export function deriveMfluxMemoryConfig(totalMemGb, overrides = {}) {
- const gb = Number.isFinite(totalMemGb) ? totalMemGb : 0; // unknown → most conservative
- // bf16 is never the auto default (issue #1321) — the top tier is 8-bit QLoRA,
- // not unquantized bf16, even at ≥96 GB (bf16 9B was pathologically slow and
- // GPU-watchdog-panicked the box). A run opts into bf16 explicitly via
- // `baseQuant: 16`. Tighter budgets step down to 4-bit.
- const result = gb >= 64
- ? { quantize: 8, low_ram: true }
- : { quantize: 4, low_ram: true };
- // Request override wins over the memory-derived tier. hasOwnProperty (not a
- // truthiness check) so an explicit `quantize: null` (bf16) is honored as a
- // deliberate clear, not mistaken for "absent".
- if (Object.prototype.hasOwnProperty.call(overrides, 'quantize')) {
- result.quantize = overrides.quantize;
- }
- if (typeof overrides.low_ram === 'boolean') {
- result.low_ram = overrides.low_ram;
- }
- const capRaw = Number.parseInt(process.env.LORA_TRAIN_MAX_QUANT_BITS ?? '', 10);
- if (capRaw === 8 || capRaw === 4) {
- // null (bf16) is "16+ bits" → always capped; an existing smaller quant
- // (more aggressive) is kept. Lower bits = smaller = stays. Applied last so
- // it clamps the post-override value, not just the memory tier.
- const curBits = result.quantize == null ? 16 : result.quantize;
- if (curBits > capRaw) result.quantize = capRaw;
- }
- return result;
-}
-
-/**
- * Resolve the runtime + repos for a base model id. `models` is the
- * getImageModels() registry; `mlxAvailable` says whether the user's mflux
- * install ships `mflux-train` (probed by the caller — this stays pure).
- */
-export function resolveTrainingRuntime(baseModelId, models, { mlxAvailable = false } = {}) {
- const entry = (models || []).find((m) => m.id === baseModelId);
- if (!entry) {
- throw new ServerError(`Unknown image model: ${baseModelId}`, {
- status: 400, code: 'TRAINING_UNSUPPORTED_MODEL',
- });
- }
- if (!isFlux2(entry)) {
- // mflux ≥0.17 removed FLUX.1 training ("Flux1 training is no longer
- // supported"), so dev/schnell and the diffusers families are all out.
- throw new ServerError(
- `LoRA training supports FLUX.2 Klein models only — not ${baseModelId}`,
- { status: 400, code: 'TRAINING_UNSUPPORTED_MODEL' },
- );
- }
- const variant = flux2VariantFromModel(entry);
- if (!variant || !FLUX2_TRAIN_REPOS[variant]) {
- throw new ServerError(
- `Cannot determine FLUX.2 size variant for ${baseModelId} — training needs a klein-4B/9B model`,
- { status: 400, code: 'TRAINING_UNSUPPORTED_MODEL' },
- );
- }
- return {
- runtime: mlxAvailable ? TRAINING_RUNTIMES.MFLUX : TRAINING_RUNTIMES.FLUX2,
- entry,
- variant,
- trainRepo: FLUX2_TRAIN_REPOS[variant],
- mfluxModel: MFLUX_TRAIN_MODELS[variant],
- };
-}
-
-const mergedParams = (params = {}) => ({ ...TRAINING_DEFAULTS, ...params });
-
-// The full attention + feed-forward target set from mflux's flux2 training
-// README, parameterized by rank and the variant's block counts. Klein 4B:
-// 5 double + 20 single blocks; 9B: 8 double + 24 single (counts from the
-// model configs; an over-long range would throw at injection, so these are
-// pinned per variant).
-const FLUX2_BLOCK_COUNTS = Object.freeze({
- '4b': { double: 5, single: 20 },
- '9b': { double: 8, single: 24 },
-});
-
-export function buildMfluxLoraTargets(variant, rank) {
- const counts = FLUX2_BLOCK_COUNTS[variant] || FLUX2_BLOCK_COUNTS['4b'];
- const doubleRange = { start: 0, end: counts.double };
- const singleRange = { start: 0, end: counts.single };
- const doublePaths = [
- 'attn.to_q', 'attn.to_k', 'attn.to_v', 'attn.to_out',
- 'attn.add_q_proj', 'attn.add_k_proj', 'attn.add_v_proj', 'attn.to_add_out',
- 'ff.linear_in', 'ff.linear_out', 'ff_context.linear_in', 'ff_context.linear_out',
- ];
- const singlePaths = ['attn.to_qkv_mlp_proj', 'attn.to_out'];
- return [
- ...doublePaths.map((p) => ({ module_path: `transformer_blocks.{block}.${p}`, blocks: doubleRange, rank })),
- ...singlePaths.map((p) => ({ module_path: `single_transformer_blocks.{block}.${p}`, blocks: singleRange, rank })),
- ];
-}
-
-/**
- * Build the mflux-train config JSON (mflux ≥0.17 schema — verified against
- * TrainingSpec.from_conf). Captions are NOT in the config: mflux
- * auto-discovers `NNNN.png` + `NNNN.txt` pairs under `data`, and preview
- * prompts come from `data/preview*.txt` — the run-staging step lays those
- * files out. "steps" in OUR params means total training iterations; mflux
- * counts epochs over the example set, so we translate.
- */
-export function buildMfluxTrainConfig({
- params = {},
- variant,
- mfluxModel,
- dataDir,
- imageCount,
- outputDir,
- totalMemGb = null,
-}) {
- const p = mergedParams(params);
- if (!Number.isInteger(imageCount) || imageCount < 1) {
- throw new ServerError('mflux train config needs at least one example image', {
- status: 400, code: 'VALIDATION_ERROR',
- });
- }
- const totalSteps = p.steps;
- const epochs = Math.max(1, Math.round(totalSteps / imageCount));
- // save_frequency must be > 0; 0/absent checkpointEvery → only the final
- // save (frequency = total). The crash-resilience floor then caps the
- // interval so a hard reboot mid-run loses at most ~1/MIN_CHECKPOINTS of it.
- const requestedSaveFrequency = p.checkpointEvery > 0 ? p.checkpointEvery : totalSteps;
- const maxSaveInterval = Math.max(1, Math.ceil(totalSteps / MFLUX_MIN_CHECKPOINTS));
- const saveFrequency = Math.min(requestedSaveFrequency, maxSaveInterval);
- // Per-run base-tier overrides (issue #1321): `baseQuant` lets a run pick the
- // frozen-base quant — 16 = unquantized bf16 (mflux `quantize: null`), 8/4 =
- // QLoRA bit-width — and `lowRam` toggles the on-disk latent-cache spill,
- // independent of the memory-derived default. mflux-only knobs (the torch
- // runtime always trains the bf16 base); absent → memory-derived tier.
- const memoryOverride = {};
- if (p.baseQuant != null) memoryOverride.quantize = p.baseQuant >= 16 ? null : p.baseQuant;
- if (typeof p.lowRam === 'boolean') memoryOverride.low_ram = p.lowRam;
- const memory = deriveMfluxMemoryConfig(totalMemGb, memoryOverride);
- return {
- model: mfluxModel,
- data: dataDir,
- seed: Number.isInteger(p.seed) ? p.seed : 42,
- steps: MFLUX_SCHED.steps,
- guidance: MFLUX_SCHED.guidance,
- quantize: memory.quantize,
- max_resolution: p.resolution,
- low_ram: memory.low_ram,
- training_loop: {
- num_epochs: epochs,
- batch_size: 1,
- timestep_low: MFLUX_SCHED.timestepLow,
- timestep_high: MFLUX_SCHED.timestepHigh,
- },
- optimizer: { name: 'AdamW', learning_rate: p.learningRate },
- checkpoint: { save_frequency: saveFrequency, output_path: outputDir },
- // monitoring omitted entirely when samples are off — mflux treats the
- // whole block as optional and skips previews/plots without it.
- ...(p.sampleEvery > 0 ? {
- monitoring: {
- preview_width: p.resolution,
- preview_height: p.resolution,
- plot_frequency: p.sampleEvery,
- generate_image_frequency: p.sampleEvery,
- },
- } : {}),
- lora_layers: { targets: buildMfluxLoraTargets(variant, p.rank) },
- };
-}
-
-// Default GPU cooldown (seconds) between training segments — long enough for
-// the GPU to clock down and the system to settle after a full Metal-context
-// teardown, short relative to a segment's compute. See the segmentation block
-// in scripts/train_mflux_lora.py and the watchdog-panic incident record.
-export const MFLUX_DEFAULT_COOLDOWN_SEC = 90;
-
-/**
- * Argv for the mflux wrapper script.
- *
- * `segmentSteps > 0` enables segmented training (watchdog-panic mitigation):
- * the wrapper trains one checkpoint interval, tears down the GPU child, cools
- * down `cooldownSec`, then resumes — so sustained GPU pressure never spans the
- * whole run. Pass `segmentSteps = the effective save_frequency` so each segment
- * ends exactly on a checkpoint. `segmentSteps = 0` keeps the single-process run.
- */
-export function buildMfluxTrainArgs({
- scriptPath,
- configPath,
- runDir,
- totalSteps,
- resumeCheckpoint = null,
- segmentSteps = 0,
- cooldownSec = MFLUX_DEFAULT_COOLDOWN_SEC,
-}) {
- const args = [
- scriptPath,
- '--config', configPath,
- '--output-dir', runDir,
- '--total-steps', String(totalSteps),
- ];
- if (segmentSteps > 0) {
- args.push('--segment-steps', String(segmentSteps), '--cooldown-sec', String(Math.max(0, cooldownSec)));
- }
- if (resumeCheckpoint) args.push('--resume-checkpoint', resumeCheckpoint);
- return args;
-}
-
-/** Argv for the vendored torch FLUX.2 trainer. */
-export function buildFlux2TrainArgs({
- scriptPath,
- trainRepo,
- manifestPath,
- runDir,
- triggerWord,
- params = {},
- samplePrompt = null,
- resumeFrom = null,
-}) {
- if (!trainRepo || !/^black-forest-labs\//.test(trainRepo)) {
- // Belt-and-suspenders: training against a quantized repo silently
- // produces garbage gradients — refuse anything but the bf16 bases.
- throw new ServerError(`FLUX.2 training requires a bf16 base repo (got: ${trainRepo})`, {
- status: 400, code: 'TRAINING_UNSUPPORTED_MODEL',
- });
- }
- const p = mergedParams(params);
- const args = [
- scriptPath,
- '--model-repo', trainRepo,
- '--manifest', manifestPath,
- '--output-dir', runDir,
- '--trigger-word', triggerWord,
- '--steps', String(p.steps),
- '--rank', String(p.rank),
- '--lr', String(p.learningRate),
- '--resolution', String(p.resolution),
- '--checkpoint-every', String(p.checkpointEvery),
- '--sample-every', String(p.sampleEvery),
- '--sample-prompt', samplePrompt || `${triggerWord} portrait, neutral background`,
- '--seed', String(Number.isInteger(p.seed) ? p.seed : 42),
- '--device', 'auto',
- ];
- if (resumeFrom) args.push('--resume-from', resumeFrom);
- return args;
-}
diff --git a/server/services/loraTraining/runtimes.test.js b/server/services/loraTraining/runtimes.test.js
deleted file mode 100644
index 0086cbd0f2..0000000000
--- a/server/services/loraTraining/runtimes.test.js
+++ /dev/null
@@ -1,301 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import {
- TRAINING_DEFAULTS,
- buildFlux2TrainArgs,
- buildMfluxLoraTargets,
- buildMfluxTrainArgs,
- buildMfluxTrainConfig,
- deriveMfluxMemoryConfig,
- resolveTrainingRuntime,
- MFLUX_DEFAULT_COOLDOWN_SEC,
-} from './runtimes.js';
-
-const MODELS = [
- { id: 'dev', name: 'Flux 1 Dev' },
- { id: 'schnell', name: 'Flux 1 Schnell' },
- { id: 'flux2-klein-4b', runner: 'flux2', repo: 'Disty0/FLUX.2-klein-4B-SDNQ-4bit-dynamic' },
- { id: 'flux2-klein-9b-bf16', runner: 'flux2', repo: 'black-forest-labs/FLUX.2-klein-9B' },
- { id: 'z-image-turbo', runner: 'z-image', repo: 'Tongyi/Z-Image-Turbo' },
-];
-
-describe('resolveTrainingRuntime', () => {
- it('routes flux2 models to mflux when its trainer is available', () => {
- const out = resolveTrainingRuntime('flux2-klein-4b', MODELS, { mlxAvailable: true });
- expect(out.runtime).toBe('mflux');
- expect(out.mfluxModel).toBe('flux2-klein-base-4b');
- expect(out.trainRepo).toBe('black-forest-labs/FLUX.2-klein-4B');
- expect(out.variant).toBe('4b');
- });
-
- it('falls back to the torch runtime without mflux', () => {
- const out = resolveTrainingRuntime('flux2-klein-9b-bf16', MODELS, { mlxAvailable: false });
- expect(out.runtime).toBe('flux2');
- expect(out.mfluxModel).toBe('flux2-klein-base-9b');
- expect(out.trainRepo).toBe('black-forest-labs/FLUX.2-klein-9B');
- });
-
- it('rejects FLUX.1 (training removed upstream), diffusers families, and unknown ids', () => {
- expect(() => resolveTrainingRuntime('dev', MODELS)).toThrow(/FLUX\.2 Klein models only/);
- expect(() => resolveTrainingRuntime('schnell', MODELS)).toThrow(/FLUX\.2 Klein models only/);
- expect(() => resolveTrainingRuntime('z-image-turbo', MODELS)).toThrow(/FLUX\.2 Klein models only/);
- expect(() => resolveTrainingRuntime('ghost', MODELS)).toThrow(/Unknown image model/);
- });
-});
-
-describe('buildMfluxLoraTargets', () => {
- it('pins block ranges per size variant (4b: 5+20, 9b: 8+24)', () => {
- const t4 = buildMfluxLoraTargets('4b', 16);
- const double4 = t4.find((t) => t.module_path === 'transformer_blocks.{block}.attn.to_q');
- const single4 = t4.find((t) => t.module_path === 'single_transformer_blocks.{block}.attn.to_qkv_mlp_proj');
- expect(double4.blocks).toEqual({ start: 0, end: 5 });
- expect(single4.blocks).toEqual({ start: 0, end: 20 });
- const t9 = buildMfluxLoraTargets('9b', 8);
- expect(t9.find((t) => t.module_path.startsWith('transformer_blocks')).blocks.end).toBe(8);
- expect(t9.find((t) => t.module_path.startsWith('single_transformer_blocks')).blocks.end).toBe(24);
- expect(t9.every((t) => t.rank === 8)).toBe(true);
- });
-});
-
-describe('buildMfluxTrainConfig', () => {
- const base = {
- variant: '4b',
- mfluxModel: 'flux2-klein-base-4b',
- dataDir: '/data/training-runs/r1/data',
- imageCount: 2,
- outputDir: '/data/training-runs/r1/mflux',
- };
-
- it('matches the mflux ≥0.17 schema and translates steps → epochs', () => {
- const config = buildMfluxTrainConfig({ ...base, params: { steps: 100 } });
- expect(config.model).toBe('flux2-klein-base-4b');
- expect(config.data).toBe(base.dataDir);
- expect(config.training_loop).toEqual({
- num_epochs: 50, batch_size: 1, timestep_low: 25, timestep_high: 40,
- });
- expect(config.steps).toBe(40); // noise-schedule steps, not training duration
- expect(config.guidance).toBe(1.0);
- expect(config.checkpoint.output_path).toBe(base.outputDir);
- expect(config.lora_layers.targets.length).toBeGreaterThan(10);
- // mflux rejects unknown keys like the legacy `examples`/`save` shapes.
- expect(config.examples).toBeUndefined();
- expect(config.save).toBeUndefined();
- });
-
- it('applies params and disables monitoring when samples are off', () => {
- const config = buildMfluxTrainConfig({
- ...base,
- params: { steps: 200, rank: 8, learningRate: 0.0002, resolution: 768, checkpointEvery: 0, sampleEvery: 0 },
- });
- expect(config.optimizer.learning_rate).toBe(0.0002);
- expect(config.max_resolution).toBe(768);
- // checkpointEvery:0 would mean "only the final save", but the crash-
- // resilience floor caps the interval at totalSteps/MIN_CHECKPOINTS (200/4).
- expect(config.checkpoint.save_frequency).toBe(50);
- expect(config.monitoring).toBeUndefined();
- expect(config.lora_layers.targets[0].rank).toBe(8);
- });
-
- it('caps save_frequency at totalSteps/MIN_CHECKPOINTS for crash resilience', () => {
- // A huge checkpointEvery (or 0 → "only final save") must still checkpoint
- // periodically so a mid-run hard reboot (GPU watchdog panic) keeps progress.
- const onlyFinal = buildMfluxTrainConfig({ ...base, params: { steps: 600, checkpointEvery: 0 } });
- expect(onlyFinal.checkpoint.save_frequency).toBe(150); // ceil(600/4)
- const tooLarge = buildMfluxTrainConfig({ ...base, params: { steps: 600, checkpointEvery: 9999 } });
- expect(tooLarge.checkpoint.save_frequency).toBe(150);
- // A reasonable user interval below the cap is left untouched.
- const reasonable = buildMfluxTrainConfig({ ...base, params: { steps: 600, checkpointEvery: 100 } });
- expect(reasonable.checkpoint.save_frequency).toBe(100);
- });
-
- it('enables monitoring with sample dimensions when sampleEvery > 0', () => {
- const config = buildMfluxTrainConfig({ ...base, params: { sampleEvery: 50, resolution: 512 } });
- expect(config.monitoring).toEqual({
- preview_width: 512, preview_height: 512, plot_frequency: 50, generate_image_frequency: 50,
- });
- });
-
- it('throws on an empty example set', () => {
- expect(() => buildMfluxTrainConfig({ ...base, imageCount: 0 })).toThrow(/at least one/);
- });
-
- it('derives quantize from the memory budget; latent cache always spills to disk', () => {
- // low_ram is true at every tier — an in-RAM latent cache bought nothing
- // but risked swap-thrash (the 128 GB / 21 GB-swap incident). bf16 is never
- // the auto default (issue #1321): even a 128 GB box gets 8-bit QLoRA, not
- // unquantized bf16. quantize still steps down to 4-bit on tighter budgets.
- expect(deriveMfluxMemoryConfig(128)).toEqual({ quantize: 8, low_ram: true });
- expect(deriveMfluxMemoryConfig(64)).toEqual({ quantize: 8, low_ram: true });
- expect(deriveMfluxMemoryConfig(48)).toEqual({ quantize: 4, low_ram: true });
- expect(deriveMfluxMemoryConfig(null)).toEqual({ quantize: 4, low_ram: true });
- const config = buildMfluxTrainConfig({ ...base, totalMemGb: 48 });
- expect(config.quantize).toBe(4);
- expect(config.low_ram).toBe(true);
- });
-
- it('LORA_TRAIN_MAX_QUANT_BITS caps the top tier (M5 bf16-panic mitigation)', () => {
- const prev = process.env.LORA_TRAIN_MAX_QUANT_BITS;
- try {
- // cap=8: already at the 8-bit default (no-op); smaller (4) is kept.
- process.env.LORA_TRAIN_MAX_QUANT_BITS = '8';
- expect(deriveMfluxMemoryConfig(128)).toEqual({ quantize: 8, low_ram: true });
- expect(deriveMfluxMemoryConfig(64)).toEqual({ quantize: 8, low_ram: true });
- expect(deriveMfluxMemoryConfig(48)).toEqual({ quantize: 4, low_ram: true });
- // cap=4: everything (default 8-bit and the 4-bit tier) forced to 4-bit.
- process.env.LORA_TRAIN_MAX_QUANT_BITS = '4';
- expect(deriveMfluxMemoryConfig(128)).toEqual({ quantize: 4, low_ram: true });
- expect(deriveMfluxMemoryConfig(64)).toEqual({ quantize: 4, low_ram: true });
- // garbage / unsupported values are ignored → memory-derived default (8-bit).
- process.env.LORA_TRAIN_MAX_QUANT_BITS = '16';
- expect(deriveMfluxMemoryConfig(128)).toEqual({ quantize: 8, low_ram: true });
- process.env.LORA_TRAIN_MAX_QUANT_BITS = 'banana';
- expect(deriveMfluxMemoryConfig(128)).toEqual({ quantize: 8, low_ram: true });
- } finally {
- if (prev === undefined) delete process.env.LORA_TRAIN_MAX_QUANT_BITS;
- else process.env.LORA_TRAIN_MAX_QUANT_BITS = prev;
- }
- });
-
- it('per-run overrides replace the memory-derived tier (issue #1321)', () => {
- // quantize override wins over the budget tier — heavier (bf16) on a 48 GB
- // box, or lighter than the box would otherwise pick.
- expect(deriveMfluxMemoryConfig(48, { quantize: null })).toEqual({ quantize: null, low_ram: true });
- expect(deriveMfluxMemoryConfig(128, { quantize: 4 })).toEqual({ quantize: 4, low_ram: true });
- // low_ram override flips the spill independently of quantize (128 GB → 8-bit default).
- expect(deriveMfluxMemoryConfig(128, { low_ram: false })).toEqual({ quantize: 8, low_ram: false });
- // Absent keys keep the memory-derived value; an empty override is a no-op.
- expect(deriveMfluxMemoryConfig(64, {})).toEqual({ quantize: 8, low_ram: true });
- // An explicit quantize: null (bf16) is honored as a deliberate clear, not
- // mistaken for "absent" — distinguishes intentional-empty from absent.
- expect(deriveMfluxMemoryConfig(48, { quantize: null }).quantize).toBeNull();
- });
-
- it('LORA_TRAIN_MAX_QUANT_BITS clamps a per-run override (cap is the ceiling)', () => {
- const prev = process.env.LORA_TRAIN_MAX_QUANT_BITS;
- try {
- // A run asks for bf16 (heaviest) but the install caps at 8-bit — the cap
- // wins, so a per-run opt-in can never breach the box's panic guard.
- process.env.LORA_TRAIN_MAX_QUANT_BITS = '8';
- expect(deriveMfluxMemoryConfig(48, { quantize: null })).toEqual({ quantize: 8, low_ram: true });
- // A run can still go LIGHTER than the cap.
- expect(deriveMfluxMemoryConfig(128, { quantize: 4 })).toEqual({ quantize: 4, low_ram: true });
- } finally {
- if (prev === undefined) delete process.env.LORA_TRAIN_MAX_QUANT_BITS;
- else process.env.LORA_TRAIN_MAX_QUANT_BITS = prev;
- }
- });
-
- it('maps baseQuant/lowRam request params into the mflux config (issue #1321)', () => {
- // baseQuant 16 → unquantized bf16 (mflux quantize: null), overriding the
- // null-budget tier (which would otherwise be 4-bit).
- const bf16 = buildMfluxTrainConfig({ ...base, params: { baseQuant: 16, lowRam: false } });
- expect(bf16.quantize).toBeNull();
- expect(bf16.low_ram).toBe(false);
- // baseQuant 8/4 map straight through as the QLoRA bit-width.
- expect(buildMfluxTrainConfig({ ...base, params: { baseQuant: 8 } }).quantize).toBe(8);
- expect(buildMfluxTrainConfig({ ...base, totalMemGb: 128, params: { baseQuant: 4 } }).quantize).toBe(4);
- // lowRam alone flips the spill at the config-builder level, leaving the
- // memory-derived quant (128 GB → 8-bit default) untouched.
- expect(buildMfluxTrainConfig({ ...base, totalMemGb: 128, params: { lowRam: false } }).low_ram).toBe(false);
- // No override → memory-derived tier is untouched (128 GB → 8-bit, never bf16; issue #1321).
- expect(buildMfluxTrainConfig({ ...base, totalMemGb: 128 }).quantize).toBe(8);
- });
-
- it('treats null baseQuant/lowRam ("Auto") as no override (issue #1407)', () => {
- // The form sends an explicit `null` for "Auto" (a deliberate clear that
- // beats a saved default), which must behave identically to absent: the
- // memory-derived tier wins. A null baseQuant must NOT be coerced to 0.
- const cfg = buildMfluxTrainConfig({ ...base, totalMemGb: 48, params: { baseQuant: null, lowRam: null } });
- expect(cfg.quantize).toBe(4); // 48 GB tier, not touched by the null clear
- expect(cfg.low_ram).toBe(true);
- });
-});
-
-describe('buildMfluxTrainArgs / buildFlux2TrainArgs', () => {
- it('builds the wrapper argv', () => {
- const args = buildMfluxTrainArgs({
- scriptPath: '/x/train_mflux_lora.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 500,
- });
- expect(args).toEqual([
- '/x/train_mflux_lora.py', '--config', '/r/cfg.json',
- '--output-dir', '/r', '--total-steps', '500',
- ]);
- });
-
- it('builds the flux2 argv with merged params', () => {
- const args = buildFlux2TrainArgs({
- scriptPath: '/x/train_flux2_lora.py',
- trainRepo: 'black-forest-labs/FLUX.2-klein-4B',
- manifestPath: '/r/manifest.json',
- runDir: '/r',
- triggerWord: 'kessa',
- params: { steps: 20, rank: 4 },
- });
- expect(args).toContain('--model-repo');
- expect(args[args.indexOf('--steps') + 1]).toBe('20');
- expect(args[args.indexOf('--rank') + 1]).toBe('4');
- expect(args[args.indexOf('--lr') + 1]).toBe(String(TRAINING_DEFAULTS.learningRate));
- expect(args[args.indexOf('--sample-prompt') + 1]).toContain('kessa');
- });
-
- it('appends the resume flag for each runtime when a checkpoint is given', () => {
- const mflux = buildMfluxTrainArgs({
- scriptPath: '/x/train_mflux_lora.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 500,
- resumeCheckpoint: '/r/mflux/checkpoints/0000150_checkpoint.zip',
- });
- expect(mflux[mflux.indexOf('--resume-checkpoint') + 1]).toBe('/r/mflux/checkpoints/0000150_checkpoint.zip');
-
- const flux2 = buildFlux2TrainArgs({
- scriptPath: '/x/train_flux2_lora.py',
- trainRepo: 'black-forest-labs/FLUX.2-klein-4B',
- manifestPath: '/r/manifest.json', runDir: '/r', triggerWord: 'kessa',
- resumeFrom: '/r/checkpoints/step-000150',
- });
- expect(flux2[flux2.indexOf('--resume-from') + 1]).toBe('/r/checkpoints/step-000150');
- });
-
- it('omits the resume flag on a fresh run', () => {
- const args = buildMfluxTrainArgs({
- scriptPath: '/x.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 500,
- });
- expect(args).not.toContain('--resume-checkpoint');
- });
-
- it('omits segment flags when segmentSteps is 0 (single-process run)', () => {
- const args = buildMfluxTrainArgs({
- scriptPath: '/x.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 500, segmentSteps: 0,
- });
- expect(args).not.toContain('--segment-steps');
- expect(args).not.toContain('--cooldown-sec');
- });
-
- it('emits segment + cooldown flags when segmentation is enabled', () => {
- const args = buildMfluxTrainArgs({
- scriptPath: '/x.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 600,
- segmentSteps: 150, cooldownSec: 120,
- });
- expect(args[args.indexOf('--segment-steps') + 1]).toBe('150');
- expect(args[args.indexOf('--cooldown-sec') + 1]).toBe('120');
- });
-
- it('defaults the cooldown when only segmentSteps is given', () => {
- const args = buildMfluxTrainArgs({
- scriptPath: '/x.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 600, segmentSteps: 150,
- });
- expect(args[args.indexOf('--cooldown-sec') + 1]).toBe(String(MFLUX_DEFAULT_COOLDOWN_SEC));
- });
-
- it('clamps a negative cooldown to 0', () => {
- const args = buildMfluxTrainArgs({
- scriptPath: '/x.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 600,
- segmentSteps: 150, cooldownSec: -5,
- });
- expect(args[args.indexOf('--cooldown-sec') + 1]).toBe('0');
- });
-
- it('refuses non-bf16 train repos', () => {
- expect(() => buildFlux2TrainArgs({
- scriptPath: '/x.py', trainRepo: 'Disty0/FLUX.2-klein-4B-SDNQ-4bit-dynamic',
- manifestPath: '/m.json', runDir: '/r', triggerWord: 'kessa',
- })).toThrow(/bf16/);
- });
-});
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.js
index 1641a688fd..e3d9fd20f4 100644
--- a/server/services/mediaJobQueue/index.js
+++ b/server/services/mediaJobQueue/index.js
@@ -1,3 +1,4 @@
+import { refuseRetiredTraining, TRAINING_RETIRED_MESSAGE, assertRetiredDataAvailable } from '../../lib/retiredLoraTraining.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
/**
* Media Job Queue — lane-aware FIFO for media generation jobs.
@@ -924,6 +925,13 @@ export async function initMediaJobQueue({ deferCompletionRecovery = false } = {}
}));
continue;
}
+ // Never dispatch a queued trainer after upgrade. A verified survivor below
+ // is observed to completion without launching a replacement process.
+ if (j.kind === 'training' && j.status === 'queued' && !j.params?.reattach) {
+ archive.push({ ...j, status: 'canceled', error: TRAINING_RETIRED_MESSAGE, completedAt: new Date().toISOString() });
+ restartedFailedIds.push(j.id);
+ continue;
+ }
if (j.status === 'running' && maintenance.held()) {
maintenance.recoverOwned('media', j.id);
if (j.kind === 'training' && j.params?.runId && await jobHasSurvivingTrainer(j.params.runId)) {
@@ -2040,6 +2048,7 @@ async function assertMusicVideoJobPolicy(kind, params) {
}
export async function enqueueJob({ kind, params, owner = null }) {
+ if (kind === 'training') refuseRetiredTraining();
await assertMusicVideoJobPolicy(kind, params);
if (params?.musicVideo?.sceneId) {
const { assertMusicVideoSceneReview } = await import('../musicVideo/productionReviewService.js');
@@ -2076,6 +2085,7 @@ export async function enqueueJob({ kind, params, owner = null }) {
return laneQueue.length + liveCount + 1;
})(),
};
+ assertRetiredDataAvailable();
job.admitting = true;
queue.push(job);
try {
diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.js
index 8b2f5a8801..2fe536c8ed 100644
--- a/server/services/mediaJobQueue/index.test.js
+++ b/server/services/mediaJobQueue/index.test.js
@@ -1832,16 +1832,26 @@ describe('Audio kind (#1928)', () => {
expect(params.remoteMedia.request.prompt).toBe('a harbour');
});
- it('leaves a training job carrying a stray marker on the local path', async () => {
- // Bypass probe for the normalization above: `training` has no federated
- // contract, so a marker on one is corrupt state — blanking its params would
- // destroy a real local run.
- const job = await mediaJobQueue.enqueueJob({
- kind: 'training',
- params: { runId: 'run-1', runtime: 'mflux', modelId: 'dev', remoteMedia: remoteImageMediaParams() },
+ it('rejects media admission during retired-data removal and permits it after settlement', async () => {
+ const { withRetiredDataRemoval } = await import('../../lib/retiredLoraTraining.js');
+ await withRetiredDataRemoval(async () => {
+ await expect(mediaJobQueue.enqueueJob({ kind: 'image', params: {} })).rejects.toMatchObject({ status: 409, code: 'DATA_MAINTENANCE_ACTIVE' });
});
+ const accepted = await mediaJobQueue.enqueueJob({ kind: 'image', params: {} });
+ expect(accepted.jobId).toBeTruthy();
+ });
+
+ it('retires persisted queued training without invoking a trainer or removing artifacts', async () => {
+ const id = '00000000-0000-4000-8000-000000000019';
+ writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ jobs: [{ id, kind: 'training', status: 'queued', params: { runId: 'example-retired' } }] }));
+ await importFresh();
+ await mediaJobQueue.initMediaJobQueue();
+ expect(mediaJobQueue.getJob(id)).toMatchObject({ status: 'canceled', error: expect.stringContaining('retired') });
+ expect(stubs.runTraining).not.toHaveBeenCalled();
+ });
- expect(mediaJobQueue.getJob(job.jobId).params.modelId).toBe('dev');
+ it('refuses retired training even when carrying a stray remote marker', async () => {
+ await expect(mediaJobQueue.enqueueJob({ kind: 'training', params: { runId: 'example', remoteMedia: remoteImageMediaParams() } })).rejects.toMatchObject({ status: 410, code: 'LORA_TRAINING_RETIRED' });
});
it('persists cancellation intent before signaling a running remote adapter', async () => {
@@ -2731,7 +2741,7 @@ describe('local video failure holds', () => {
remoteAdapter.mockRestore();
});
- it('lets other local models, image, audio, training, cloud and remote work pass a hold', async () => {
+ it('lets other local models, image, audio, cloud and remote work pass a hold', async () => {
const ids = await submitMany(4);
for (const id of ids.slice(0, 3)) { await tick(); await finish(id); }
const cloud = await submit('example-mlx', { mode: 'grok' });
@@ -2742,7 +2752,7 @@ describe('local video failure holds', () => {
expect(stubs.generateVideoRemote).toHaveBeenCalledWith(expect.objectContaining({ jobId: remote }));
expect(mediaJobQueue.getJob(other).status).toBe('running');
await finish(other, null);
- for (const [kind, emitter] of [['image', imageGenEvents], ['audio', audioGenEvents], ['training', (await import('../loraTraining/events.js')).trainingEvents]]) {
+ for (const [kind, emitter] of [['image', imageGenEvents], ['audio', audioGenEvents]]) {
stubs.runTraining.mockResolvedValue({});
const { jobId } = await mediaJobQueue.enqueueJob({ kind, params: {} });
await tick();
@@ -4236,7 +4246,6 @@ describe('mediaJobQueue per-kind runtime wiring', () => {
entry: 'runSharingCopy', bus: 'video', verb: 'Generating', window: WINDOWS.VIDEO },
{ kind: 'html-composition', params: { directory: 'compositions/example' }, entry: 'renderComposition', bus: 'video', verb: 'Generating', window: WINDOWS.IMAGE },
{ kind: 'image', params: {}, entry: 'generateImage', bus: 'image', verb: 'Generating', window: WINDOWS.IMAGE },
- { kind: 'training', params: {}, entry: 'runTraining', bus: 'training', verb: 'Training', window: WINDOWS.TRAINING },
{ kind: 'audio', params: {}, entry: 'generateAudio', bus: 'audio', verb: 'Generating', window: WINDOWS.AUDIO },
{ kind: 'supercollider', params: {}, entry: 'renderSuperCollider', bus: 'audio', verb: 'Generating', window: WINDOWS.IMAGE },
];
@@ -4247,8 +4256,8 @@ describe('mediaJobQueue per-kind runtime wiring', () => {
training: (await import('../loraTraining/events.js')).trainingEvents,
})[name];
- it('covers every declared job kind', () => {
- expect(KIND_CASES.map(c => c.kind).sort()).toEqual([...mediaJobQueue.JOB_KINDS].sort());
+ it('covers every newly dispatchable kind; historical training is recovery-only', () => {
+ expect(KIND_CASES.map(c => c.kind).sort()).toEqual(mediaJobQueue.JOB_KINDS.filter(kind => kind !== 'training').sort());
});
describe.each(KIND_CASES)('$kind', ({ kind, params, entry, bus, verb, window }) => {