Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Retire LoRA training and add explicit data maintenance (#13536)
[._.]/cf8ea45a27a6
diff --git a/.env.example b/.env.exampleindex 0c0af1d7fe..0f497b5de0 100644--- a/.env.example+++ b/.env.example@@ -298,7 +298,7 @@ PGPASSWORD=portos# Terminate an idle queued Codex job after this many milliseconds (default: 1200000 / 20 minutes)# MEDIA_JOB_WATCHDOG_CODEX_MS=1200000-# Terminate an idle queued training job after this many milliseconds (default: 1800000 / 30 minutes)+# Terminate an idle surviving retired trainer after this many milliseconds (default: 1800000 / 30 minutes)# MEDIA_JOB_WATCHDOG_TRAINING_MS=1800000# Terminate an idle queued audio job after this many milliseconds (default: 600000 / 10 minutes)@@ -319,14 +319,6 @@ PGPASSWORD=portos# LoRA warmup stall budget before step timing stabilizes (default: 1200000 / 20 minutes)# LORA_TRAIN_STALL_WARMUP_BUDGET_MS=1200000-# Automatic LoRA checkpoint resumes allowed after soft stalls (default: 2; 0 disables)-# LORA_TRAIN_STALL_MAX_AUTO_RESUMES=2--# Cap LoRA training quantization at this bit width (loraTraining/runtimes.js).-# Only 8 or 4 are honoured; anything else is ignored and the memory-derived tier-# stands. Set it to force a smaller quant on a machine whose auto-tier OOMs.-# LORA_TRAIN_MAX_QUANT_BITS=4-# Ingredient catalog revisions retained per ingredient (default: 50)# CATALOG_REVISION_RETENTION=50diff --git a/client/src/App.jsx b/client/src/App.jsxindex 78378b948c..1b6573fdfc 100644--- a/client/src/App.jsx+++ b/client/src/App.jsx@@ -227,7 +227,6 @@ const MEDIA_CREATIVE_DIRECTOR_PREFIX = /^\/media\/creative-director/;const MEDIA_SPRITES_PREFIX = /^\/media\/sprites/;const MEDIA_MUSIC_VIDEO_PREFIX = /^\/media\/music-video/;const MEDIA_3D_PREFIX = /^\/media\/3d/;-const MEDIA_TRAINING_PREFIX = /^\/media\/training/;// The annotator lives under the Media shell (/media/annotate). A bare /annotate// is what a bookmark, a typed URL, or a half-remembered path lands on — alias it// rather than letting the catch-all swallow it (issue #3793).@@ -408,6 +407,8 @@ export default function App() {<Route path="voice/call-host" element={<VoiceCallHost />} /><Route path="api-reference" element={<Navigate to="/api-reference/catalog" replace />} /><Route path="api-reference/:tab" element={<ApiExplorer />} />+ <Route path="models/training" element={<Navigate to="/settings/data-maintenance" replace />} />+ <Route path="models/training/:recordId" element={<Navigate to="/settings/data-maintenance" replace />} /><Route path="models" element={<Navigate to="/models/llms" replace />} />{/* Runtimes was the LLMs page's default view until #7414 promoted it toits own tab. A literal segment outranks `models/:tab/:recordId`, so@@ -538,8 +539,8 @@ export default function App() {<Route path="3d" element={<PrefixRedirect from={MEDIA_3D_PREFIX} to="/3d" />} /><Route path="3d/:id" element={<PrefixRedirect from={MEDIA_3D_PREFIX} to="/3d" />} /><Route path="loras" element={<RedirectWithSearch to="/models/loras" />} />- <Route path="training" element={<PrefixRedirect from={MEDIA_TRAINING_PREFIX} to="/models/training" />} />- <Route path="training/:datasetId" element={<PrefixRedirect from={MEDIA_TRAINING_PREFIX} to="/models/training" />} />+ <Route path="training" element={<Navigate to="/settings/data-maintenance" replace />} />+ <Route path="training/:datasetId" element={<Navigate to="/settings/data-maintenance" replace />} />{/* Universes live at /universes (Create sidebar link). Theseredirects keep legacy /media/universe-builder bookmarks workingafter the MediaGen tab was removed. */}diff --git a/client/src/components/dashboard/ActiveProcessingWidget.jsx b/client/src/components/dashboard/ActiveProcessingWidget.jsxindex f78cc4b671..fdc98277e2 100644--- a/client/src/components/dashboard/ActiveProcessingWidget.jsx+++ b/client/src/components/dashboard/ActiveProcessingWidget.jsx@@ -56,7 +56,7 @@ function JobRow({ job, onCancel }) {: kind === 'video' ? '/media/video': kind === 'image' ? '/media/image': kind === 'audio' ? '/media/history?type=audio'- : kind === 'training' ? '/models/training' : '/media/history';+ : kind === 'training' ? '/system-resources/queues' : '/media/history';const Icon = kind === 'video' ? Film : kind === 'audio' ? AudioLines : kind === 'training' ? Cpu : ImageIcon;const progress = Number.isFinite(job.progress) ? Math.max(0, Math.min(100, Math.round(job.progress * 100))) : null;return (diff --git a/client/src/components/loraTraining/CaptionModelPicker.jsx b/client/src/components/loraTraining/CaptionModelPicker.jsxdeleted file mode 100644index 21b62e898b..0000000000--- a/client/src/components/loraTraining/CaptionModelPicker.jsx+++ /dev/null@@ -1,117 +0,0 @@-/**- * Caption-model picker for the LoRA dataset workbench.- *- * Lists vision-capable installed local models (both Ollama + LM Studio, each- * tagged with the provider that serves it) and lets the user pick which one- * auto-captioning uses. The choice persists to `settings.loraTraining`- * (captionProviderId/captionModel) so it sticks across runs and machines, and- * is lifted to the parent via `onChange` so caption runs pass the explicit- * provider+model. "Auto" defers to the server's vision-model auto-pick.- */--import { useEffect, useState } from 'react';-import { Loader2, AlertTriangle } from 'lucide-react';-import ListLoadError from '../ui/ListLoadError';-import toast from '../ui/Toast';-import { getVisionModels, getSettings, patchSettingsSlice } from '../../services/api';--const AUTO = '__auto__';-const keyFor = (m) => `${m.providerId}::${m.id}`;--export default function CaptionModelPicker({ onChange }) {- const [models, setModels] = useState(null); // null = loading; [] = none installed- const [value, setValue] = useState(AUTO);- const [saving, setSaving] = useState(false);- const [loadError, setLoadError] = useState(null);- const [attempt, setAttempt] = useState(0);-- // The model list and the saved choice seed the select together; if either- // read fails the picker shows a Retry instead of a select that would read- // "none installed" and report Auto for a saved model.- useEffect(() => {- let alive = true;- setModels(null);- setLoadError(null);- Promise.all([- getVisionModels({ silent: true }),- getSettings({ silent: true }),- ]).then(([vm, settings]) => {- if (!alive) return;- const list = vm?.models || [];- setModels(list);- const savedModel = settings?.loraTraining?.captionModel || null;- const savedProvider = settings?.loraTraining?.captionProviderId || null;- if (savedModel) {- const match = list.find((m) => m.id === savedModel- && (!savedProvider || m.providerId === savedProvider));- const sel = match || { id: savedModel, providerId: savedProvider || 'lmstudio' };- setValue(keyFor(sel));- onChange?.({ providerId: sel.providerId, model: sel.id });- } else {- setValue(AUTO);- onChange?.({ providerId: null, model: null });- }- }).catch((error) => {- if (!alive) return;- setLoadError(error instanceof Error ? error : new Error('Request failed'));- });- return () => { alive = false; };- // onChange is a stable setter from the parent; reload only on retry.- }, [attempt]);-- const handleChange = async (next) => {- setValue(next);- const picked = next === AUTO ? null : (models || []).find((m) => keyFor(m) === next) || null;- onChange?.(picked ? { providerId: picked.providerId, model: picked.id } : { providerId: null, model: null });- setSaving(true);- // A slice patch, merged server-side into the stored `loraTraining`, so the- // hand-set training knobs beside these two keys survive even when the- // settings read above failed.- try {- await patchSettingsSlice('loraTraining', {- captionProviderId: picked ? picked.providerId : null,- captionModel: picked ? picked.id : null,- }, { silent: true });- } catch {- toast.error('Could not save caption-model preference');- } finally {- setSaving(false);- }- };-- if (loadError) {- return <ListLoadError what="vision models" error={loadError} onRetry={() => setAttempt((n) => n + 1)} />;- }-- if (models === null) {- return (- <span className="text-xs text-gray-500 flex items-center gap-1">- <Loader2 className="w-3 h-3 animate-spin" /> vision models…- </span>- );- }-- return (- <div className="flex items-center gap-1.5">- <label htmlFor="lt-caption-model" className="text-xs text-gray-500">Caption model</label>- <select- id="lt-caption-model"- value={value}- onChange={(e) => handleChange(e.target.value)}- disabled={saving}- className="bg-port-bg border border-port-border rounded px-2 py-1 text-xs text-white max-w-[16rem] disabled:opacity-50"- >- <option value={AUTO}>Auto (first vision model)</option>- {models.map((m) => (- <option key={keyFor(m)} value={keyFor(m)}>{m.name} · {m.backend}</option>- ))}- </select>- {saving && <Loader2 className="w-3 h-3 animate-spin text-gray-400" />}- {!models.length && (- <span className="text-[11px] text-port-warning flex items-center gap-1" title="Install a vision model (e.g. Qwen2.5-VL, LLaVA) from Models → LLMs">- <AlertTriangle className="w-3 h-3" /> none installed- </span>- )}- </div>- );-}diff --git a/client/src/components/loraTraining/CaptionModelPicker.test.jsx b/client/src/components/loraTraining/CaptionModelPicker.test.jsxdeleted file mode 100644index 42ac953dbd..0000000000--- a/client/src/components/loraTraining/CaptionModelPicker.test.jsx+++ /dev/null@@ -1,40 +0,0 @@-import { describe, it, expect, vi, beforeEach } from 'vitest';-import { act, fireEvent, render, screen } from '@testing-library/react';--vi.mock('../ui/Toast', () => ({- default: Object.assign(vi.fn(), { success: vi.fn(), error: vi.fn() }),-}));-const getVisionModels = vi.fn();-const getSettings = vi.fn();-vi.mock('../../services/api', () => ({- getVisionModels: (...a) => getVisionModels(...a),- getSettings: (...a) => getSettings(...a),- patchSettingsSlice: vi.fn(),-}));--import CaptionModelPicker from './CaptionModelPicker';--const renderPicker = async (onChange = vi.fn()) => {- render(<CaptionModelPicker onChange={onChange} />);- await act(async () => {});- return onChange;-};--beforeEach(() => vi.clearAllMocks());--describe('CaptionModelPicker', () => {- it('shows a load error — not "none installed" or an Auto choice — when the model list fails', async () => {- getVisionModels.mockRejectedValueOnce(new Error('Server unavailable'));- getSettings.mockResolvedValue({ loraTraining: { captionModel: 'vision-a', captionProviderId: 'lmstudio' } });- const onChange = await renderPicker();- expect(screen.getByRole('alert')).toHaveTextContent('Could not load vision models — Server unavailable');- expect(screen.queryByText('none installed')).toBeNull();- expect(onChange).not.toHaveBeenCalled();-- getVisionModels.mockResolvedValueOnce({ models: [{ id: 'vision-a', providerId: 'lmstudio', name: 'Vision A', backend: 'LM Studio' }] });- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });- expect(screen.queryByRole('alert')).toBeNull();- expect(screen.getByLabelText('Caption model')).toHaveValue('lmstudio::vision-a');- expect(onChange).toHaveBeenCalledWith({ providerId: 'lmstudio', model: 'vision-a' });- });-});diff --git a/client/src/components/loraTraining/CharacterLoraChip.jsx b/client/src/components/loraTraining/CharacterLoraChip.jsxindex d2d5f7e3c7..de5928f441 100644--- a/client/src/components/loraTraining/CharacterLoraChip.jsx+++ b/client/src/components/loraTraining/CharacterLoraChip.jsx@@ -1,25 +1,13 @@-/**- * "Trained LoRA" chip + dataset entry point for a universe/catalog- * character. Resolves lazily via /api/loras/by-character ({ silent: true }- * — no link is a normal state, not an error) and offers a Dataset button- * that find-or-creates the character's dataset and navigates to it.- */+/** Retained adapter link for catalog and universe subjects; never creates training data. */-import { useState, useEffect, useCallback } from 'react';-import { useNavigate, Link } from 'react-router';-import { Sparkles, GraduationCap, Loader2 } from 'lucide-react';-import { getCharacterLoras, createLoraDataset, listLoraDatasets } from '../../services/api';+import { useCallback } from 'react';+import { Link } from 'react-router';+import { Sparkles } from 'lucide-react';+import { getCharacterLoras } from '../../services/api';import useListResource from '../../hooks/useListResource';import ListLoadError from '../ui/ListLoadError';-export default function CharacterLoraChip({ entryId, ingredientId, universeId, showDatasetButton = true }) {- const navigate = useNavigate();- // Existing dataset (looked up by ingredientId) — the link target on- // surfaces like the catalog detail page that don't know the universe- // entryId and therefore can't find-or-create.- const [existingDataset, setExistingDataset] = useState(null);- const [opening, setOpening] = useState(false);-+export default function CharacterLoraChip({ entryId, ingredientId }) {const fetchLoras = useCallback(() => getCharacterLoras({ entryId, ingredientId }, { silent: true }),[entryId, ingredientId],@@ -33,26 +21,6 @@ export default function CharacterLoraChip({ entryId, ingredientId, universeId, senabled: Boolean(entryId || ingredientId),});- const canCreate = !!(universeId && entryId);- useEffect(() => {- if (canCreate || (!entryId && !ingredientId)) return undefined;- let active = true;- listLoraDatasets(entryId ? { entryKind: 'characters', entryId } : { entryKind: 'characters', ingredientId })- .then((list) => { if (active) setExistingDataset(list?.[0] || null); })- .catch(err => console.warn('⚠️ Failed to load LoRA datasets: ' + err.message));- return () => { active = false; };- }, [canCreate, entryId, ingredientId]);-- const openDataset = async () => {- setOpening(true);- try {- const dataset = await createLoraDataset({ universeId, entryId });- navigate(`/models/training/${dataset.id}`);- } finally {- setOpening(false);- }- };-const lora = loras?.[0] || null;return (@@ -70,27 +38,6 @@ export default function CharacterLoraChip({ entryId, ingredientId, universeId, sLoRA{lora.triggerWords?.[0] ? ` · ${lora.triggerWords[0]}` : ''}</Link>)}- {showDatasetButton && canCreate && lorasState === 'ready' && (- <button- type="button"- onClick={openDataset}- disabled={opening}- className="inline-flex items-center gap-1 px-2 py-0.5 rounded-full bg-port-card border border-port-border text-gray-400 hover:text-white disabled:opacity-50"- title="Open the character's LoRA training dataset"- >- {opening ? <Loader2 className="w-3 h-3 animate-spin" /> : <GraduationCap className="w-3 h-3" />}- {lora ? 'Dataset' : 'Train LoRA'}- </button>- )}- {showDatasetButton && !canCreate && existingDataset && (- <Link- to={`/models/training/${existingDataset.id}`}- className="inline-flex items-center gap-1 px-2 py-0.5 rounded-full bg-port-card border border-port-border text-gray-400 hover:text-white"- title="Open the character's LoRA training dataset"- >- <GraduationCap className="w-3 h-3" /> Dataset- </Link>- )}</div>);}diff --git a/client/src/components/loraTraining/CharacterLoraChip.test.jsx b/client/src/components/loraTraining/CharacterLoraChip.test.jsxindex 5f30f2c9d7..5acee5f808 100644--- a/client/src/components/loraTraining/CharacterLoraChip.test.jsx+++ b/client/src/components/loraTraining/CharacterLoraChip.test.jsx@@ -2,13 +2,9 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';import { act, fireEvent, render, screen } from '@testing-library/react';const getCharacterLoras = vi.fn();-const createLoraDataset = vi.fn();-const listLoraDatasets = vi.fn();vi.mock('../../services/api', () => ({getCharacterLoras: (...args) => getCharacterLoras(...args),- createLoraDataset: (...args) => createLoraDataset(...args),- listLoraDatasets: (...args) => listLoraDatasets(...args),}));const mockNavigate = vi.fn();@@ -23,10 +19,9 @@ describe('CharacterLoraChip', () => {beforeEach(() => {vi.clearAllMocks();getCharacterLoras.mockResolvedValue([]);- listLoraDatasets.mockResolvedValue([]);});- it('shows a Retry — not "Train LoRA" — when the LoRA read fails, and shows "Train LoRA" after retry', async () => {+ it('shows a Retry — not "Train LoRA" — when the LoRA read fails, and stays retired after retry', async () => {getCharacterLoras.mockRejectedValueOnce(new Error('Server unavailable'));render(@@ -49,10 +44,10 @@ describe('CharacterLoraChip', () => {});expect(screen.queryByRole('alert')).toBeNull();- expect(screen.getByRole('button', { name: /Train LoRA/i })).toBeInTheDocument();+ expect(screen.queryByRole('button', { name: /Train LoRA/i })).toBeNull();});- it('renders LoRA chip and "Dataset" button only after a successful read', async () => {+ it('preserves the inference adapter link without a training entry point', async () => {getCharacterLoras.mockResolvedValueOnce([{ filename: 'alice.safetensors', triggerWords: ['alice_lora'] },]);@@ -69,7 +64,7 @@ describe('CharacterLoraChip', () => {expect(screen.queryByRole('alert')).toBeNull();expect(screen.getByText('LoRA · alice_lora')).toBeInTheDocument();- expect(screen.getByRole('button', { name: /Dataset/i })).toBeInTheDocument();+ expect(screen.queryByRole('button', { name: /Dataset/i })).toBeNull();expect(screen.queryByRole('button', { name: /Train LoRA/i })).toBeNull();});});diff --git a/client/src/components/loraTraining/CheckpointPicker.jsx b/client/src/components/loraTraining/CheckpointPicker.jsxdeleted file mode 100644index cdbf0e196e..0000000000--- a/client/src/components/loraTraining/CheckpointPicker.jsx+++ /dev/null@@ -1,174 +0,0 @@-/**- * Manual checkpoint picker for a completed training run.- *- * A LoRA run keeps every periodic checkpoint, but only one is registered as- * the usable LoRA. The trainer's FINAL step is often NOT the best — late- * divergence (loss still dropping while the sample collapses to a black frame)- * is a recurring FLUX.2 failure, and loss is no guide (it was anti-correlated- * with quality on the run this was built for). So the user picks by eye from- * the preview thumbnails; the server re-extracts that step's adapter and- * overwrites the deployed LoRA in place.- */--import { useState, useCallback } from 'react';-import useMounted from '../../hooks/useMounted';-import useListResource from '../../hooks/useListResource';-import ListLoadError from '../ui/ListLoadError';-import { Layers, Loader2, CheckCircle2, AlertTriangle } from 'lucide-react';-import toast from '../ui/Toast';-import {- listLoraTrainingCheckpoints,- promoteLoraTrainingCheckpoint,-} from '../../services/api';--export default function CheckpointPicker({ run, onPromoted }) {- const [expanded, setExpanded] = useState(false);- // Stays true after the first expand so collapsing does not drop a loaded (or- // failed, retryable) list back to a refetch.- const [opened, setOpened] = useState(false);- const [promotingStep, setPromotingStep] = useState(null);- // Guards the background preview-poll setTimeout chain from firing after the- // component unmounts (deferred-work-respects-unmount rule).- const mountedRef = useMounted();-- const fetchCheckpoints = useCallback(- () => listLoraTrainingCheckpoints(run.id, { silent: true }).then((res) => (Array.isArray(res?.checkpoints) ? res.checkpoints : [])),- [run.id],- );- const { items: checkpoints, state, error, retry, setItems: setCheckpoints } = useListResource(fetchCheckpoints, { enabled: opened });- const loading = state === 'loading';-- const toggle = () => {- const next = !expanded;- setExpanded(next);- if (next) setOpened(true);- };-- const promote = async (step) => {- setPromotingStep(step);- // If the promoted checkpoint has no thumbnail, the server renders one in the- // background (its step wasn't on the sample cadence — e.g. the final step).- // That render takes minutes, so poll a few times to surface it once ready- // without a full page reload. (No socket event for this; bounded polling.)- const lacksPreview = !checkpoints.find((c) => c.step === step)?.hasPreview;- try {- await promoteLoraTrainingCheckpoint(run.id, step, { silent: true });- toast.success(`Promoted checkpoint @ step ${step}`);- // Reactive: mark the chosen step deployed locally, then refetch + bubble- // up so the parent's run record (loraFilename, selected step) refreshes.- setCheckpoints((prev) => prev.map((c) => ({ ...c, deployed: c.step === step })));- retry();- onPromoted?.();- if (lacksPreview) {- // Backstop poll for the background-rendered preview (~5 min on 9B).- // Each tick re-lists; stop once the step has a preview or attempts run out.- let attempts = 0;- const tick = () => {- if (!mountedRef.current) return;- attempts += 1;- listLoraTrainingCheckpoints(run.id, { silent: true })- .then((res) => {- if (!mountedRef.current) return;- const list = Array.isArray(res?.checkpoints) ? res.checkpoints : [];- setCheckpoints(list);- const ready = list.find((c) => c.step === step)?.hasPreview;- if (!ready && attempts < 12) setTimeout(tick, 30000);- })- .catch(() => { if (mountedRef.current && attempts < 12) setTimeout(tick, 30000); });- };- setTimeout(tick, 30000);- }- } catch (err) {- toast.error(`Promote failed: ${err?.message || 'unknown error'}`);- } finally {- setPromotingStep(null);- }- };-- const autoSelected = run.output?.autoSelectedCheckpoint;-- return (- <div className="border-t border-port-border pt-2 mt-1">- <button- type="button"- onClick={toggle}- className="text-xs text-gray-300 hover:text-white flex items-center gap-1.5"- aria-expanded={expanded}- >- <Layers className="w-3.5 h-3.5 text-port-accent" />- Checkpoints{state === 'ready' ? ` (${checkpoints.length})` : ''}- <span className="text-gray-500">{expanded ? '▲' : '▼'}</span>- </button>-- {autoSelected && (- <div className="mt-1.5 text-[11px] text-port-warning flex items-start gap-1">- <AlertTriangle className="w-3 h-3 mt-0.5 shrink-0" />- Final step diverged — auto-selected an earlier checkpoint. Review the previews and promote a better one if needed.- </div>- )}-- {expanded && (- <div className="mt-2">- {loading && (- <div className="text-xs text-gray-400 flex items-center gap-1.5">- <Loader2 className="w-3.5 h-3.5 animate-spin" /> Loading checkpoints…- </div>- )}- {state === 'error' && <ListLoadError what="checkpoints" error={error} onRetry={retry} />}- {state === 'ready' && checkpoints.length === 0 && (- <div className="text-xs text-gray-500">No checkpoints recorded for this run.</div>- )}- {!loading && checkpoints.length > 0 && (- <>- <div className="text-[11px] text-gray-500 mb-1.5">- Pick by the preview — loss is shown but is not a quality ranking.- </div>- <div className="grid grid-cols-2 sm:grid-cols-3 gap-2">- {checkpoints.map((c) => (- <div- key={c.step}- className={`rounded border p-1.5 flex flex-col gap-1 ${- c.deployed ? 'border-port-success bg-port-success/10' : 'border-port-border bg-port-bg'- }`}- >- {c.previewUrl ? (- <img- src={c.previewUrl}- alt={`step ${c.step} preview`}- className="w-full aspect-square object-cover rounded"- loading="lazy"- />- ) : (- <div className="w-full aspect-square rounded bg-port-card flex items-center justify-center text-[10px] text-gray-500">- no preview- </div>- )}- <div className="flex items-center justify-between text-[11px]">- <span className="text-gray-300">step {c.step}</span>- {c.loss != null && <span className="text-gray-500" title="training loss (not a quality score)">L {c.loss.toFixed(3)}</span>}- </div>- {c.deployed ? (- <span className="text-[11px] text-port-success flex items-center gap-1">- <CheckCircle2 className="w-3 h-3" /> Deployed- </span>- ) : (- <button- type="button"- onClick={() => promote(c.step)}- disabled={promotingStep != null}- className="text-[11px] px-1.5 py-1 rounded bg-port-accent/20 text-port-accent hover:bg-port-accent/30 disabled:opacity-50 flex items-center justify-center gap-1"- >- {promotingStep === c.step ? <Loader2 className="w-3 h-3 animate-spin" /> : null}- Use this- </button>- )}- </div>- ))}- </div>- </>- )}- </div>- )}- </div>- );-}diff --git a/client/src/components/loraTraining/CheckpointPicker.test.jsx b/client/src/components/loraTraining/CheckpointPicker.test.jsxdeleted file mode 100644index 681f990ed1..0000000000--- a/client/src/components/loraTraining/CheckpointPicker.test.jsx+++ /dev/null@@ -1,40 +0,0 @@-import { describe, it, expect, vi, beforeEach } from 'vitest';-import { act, fireEvent, render, screen } from '@testing-library/react';--vi.mock('../ui/Toast', () => ({- default: Object.assign(vi.fn(), { success: vi.fn(), error: vi.fn() }),-}));-const listLoraTrainingCheckpoints = vi.fn();-vi.mock('../../services/api', () => ({- listLoraTrainingCheckpoints: (...a) => listLoraTrainingCheckpoints(...a),- promoteLoraTrainingCheckpoint: vi.fn(),-}));--import CheckpointPicker from './CheckpointPicker';--const expand = async () => {- render(<CheckpointPicker run={{ id: 'run-1', output: {} }} />);- await act(async () => { fireEvent.click(screen.getByRole('button', { name: /Checkpoints/ })); });-};--beforeEach(() => vi.clearAllMocks());--describe('CheckpointPicker', () => {- it('shows a Retry — not "No checkpoints recorded" — when the list read fails, and lists them on retry', async () => {- listLoraTrainingCheckpoints.mockRejectedValueOnce(new Error('Server unavailable'));- await expand();- expect(screen.getByRole('alert')).toHaveTextContent('Could not load checkpoints — Server unavailable');- expect(screen.queryByText(/No checkpoints recorded/)).toBeNull();-- listLoraTrainingCheckpoints.mockResolvedValueOnce({ checkpoints: [{ step: 500, loss: 0.1, hasPreview: true, previewUrl: '/p.png' }] });- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });- expect(screen.queryByRole('alert')).toBeNull();- expect(screen.getByText('step 500')).toBeInTheDocument();- });-- it('says "No checkpoints recorded" only for a successful empty list', async () => {- listLoraTrainingCheckpoints.mockResolvedValueOnce({ checkpoints: [] });- await expand();- expect(screen.getByText('No checkpoints recorded for this run.')).toBeInTheDocument();- });-});diff --git a/client/src/components/loraTraining/DatasetImageGrid.jsx b/client/src/components/loraTraining/DatasetImageGrid.jsxdeleted file mode 100644index d7449d478a..0000000000--- a/client/src/components/loraTraining/DatasetImageGrid.jsx+++ /dev/null@@ -1,273 +0,0 @@-/**- * Dataset image grid — caption editing, per-image re-caption, delete.- * Captions blur-save; local state updates reactively (no list refetch).- */--import { useState, useEffect, useCallback } from 'react';-import {- Loader2, RefreshCw, AlertTriangle, ChevronLeft, ChevronRight, X, Eye, EyeOff,-} from 'lucide-react';-import Modal from '../ui/Modal';-import ConfirmButtonPair from '../ui/ConfirmButtonPair';-import AutoSizeTextarea from '../ui/AutoSizeTextarea';-import Pill from '../ui/Pill';-import {- updateLoraDatasetImageCaption,- deleteLoraDatasetImage,-} from '../../services/api';--// Lightweight full-size preview for a dataset image. Kept page-local (the-// rich MediaLightbox is bound to the media-gen `item` shape — prompts, clean/-// regen actions, annotations — none of which a raw training image carries).-// Navigates the ready-image subset with ← / → and Esc-closes via <Modal>.-function DatasetImageLightbox({ datasetId, images, index, onIndexChange, onClose }) {- const img = images[index];- const hasPrev = index > 0;- const hasNext = index < images.length - 1;- useEffect(() => {- const onKey = (e) => {- if (e.key === 'ArrowLeft' && hasPrev) { e.preventDefault(); onIndexChange(index - 1); }- if (e.key === 'ArrowRight' && hasNext) { e.preventDefault(); onIndexChange(index + 1); }- };- window.addEventListener('keydown', onKey);- return () => window.removeEventListener('keydown', onKey);- }, [index, hasPrev, hasNext, onIndexChange]);- if (!img) return null;- return (- <Modal open onClose={onClose} size="none" backdropClassName="bg-black/90" ariaLabel="Training image preview">- <div className="relative flex flex-col items-center gap-3">- <button- type="button"- onClick={onClose}- className="absolute -top-1 right-0 z-10 p-2 rounded-full bg-white text-black hover:bg-white/85 shadow-lg focus:outline-none focus:ring-2 focus:ring-port-accent min-h-[44px] min-w-[44px] flex items-center justify-center"- aria-label="Close (Esc)"- title="Close (Esc)"- >- <X className="w-4 h-4" />- </button>- <img- src={`/data/lora-datasets/${datasetId}/images/${img.file}`}- alt={img.caption || 'dataset image'}- className="max-w-[92vw] max-h-[80vh] object-contain rounded-lg bg-black"- />- {img.caption && (- <p className="max-w-[92vw] text-center text-sm text-gray-200 whitespace-pre-wrap">{img.caption}</p>- )}- {hasPrev && (- <button- type="button"- onClick={() => onIndexChange(index - 1)}- className="absolute left-2 md:-left-14 top-1/2 -translate-y-1/2 p-2.5 text-white/80 hover:text-white bg-black/50 hover:bg-black/70 backdrop-blur-sm rounded-full focus:outline-none focus:ring-2 focus:ring-port-accent"- aria-label="Previous image"- title="Previous (←)"- >- <ChevronLeft className="w-6 h-6" />- </button>- )}- {hasNext && (- <button- type="button"- onClick={() => onIndexChange(index + 1)}- className="absolute right-2 md:-right-14 top-1/2 -translate-y-1/2 p-2.5 text-white/80 hover:text-white bg-black/50 hover:bg-black/70 backdrop-blur-sm rounded-full focus:outline-none focus:ring-2 focus:ring-port-accent"- aria-label="Next image"- title="Next (→)"- >- <ChevronRight className="w-6 h-6" />- </button>- )}- </div>- </Modal>- );-}--// These chips sit ON TOP of an arbitrary training image, so they can't lean on-// the theme's card/accent colors for contrast — a translucent tint over a bright-// photo (or a light user theme) made the labels unreadable. Every chip carries-// its own near-opaque dark plate (Pill `bare` + `bordered={false}`, so the ring-// draws the edge); only the text hue distinguishes the source.-const BADGE_BASE = 'max-w-full leading-tight font-medium bg-black/80 backdrop-blur-sm ring-1 shadow-sm';--const SOURCE_BADGE = {- generated: { label: 'generated', cls: 'text-blue-200 ring-blue-300/40' },- upload: { label: 'upload', cls: 'text-emerald-200 ring-emerald-300/40' },- 'refsheet-slice': { label: 'sheet crop', cls: 'text-purple-200 ring-purple-300/40' },- gallery: { label: 'gallery', cls: 'text-sky-200 ring-sky-300/40' },-};--export default function DatasetImageGrid({- dataset, onImagesChange, onRecaption, recaptioningId = null, captionBusy = false, draftResetToken = 0,-}) {- const [drafts, setDrafts] = useState({});- // A bulk caption rewrite in the parent (e.g. strip-shared-fragments) replaces- // dataset.images out from under any unsaved per-cell draft. Drop the drafts on- // that signal so the next blur doesn't PATCH the superseded text back.- useEffect(() => { setDrafts({}); }, [draftResetToken]);- const [savingId, setSavingId] = useState(null);- const [confirmingId, setConfirmingId] = useState(null);- const [deletingId, setDeletingId] = useState(null);- const [showCaptions, setShowCaptions] = useState(true);- // Index into the ready-image subset that the lightbox is previewing (null = closed).- const [previewIndex, setPreviewIndex] = useState(null);-- // Only ready images are previewable — rendering/failed cells have no file.- const readyImages = dataset.images.filter((i) => i.status === 'ready');- const openPreview = useCallback((img) => {- const idx = readyImages.findIndex((i) => i.id === img.id);- if (idx >= 0) setPreviewIndex(idx);- }, [readyImages]);-- const saveCaption = async (img) => {- const draft = drafts[img.id];- if (draft === undefined || draft === img.caption) return;- setSavingId(img.id);- try {- const updated = await updateLoraDatasetImageCaption(dataset.id, img.id, draft);- onImagesChange((prev) => prev.map((i) => (i.id === img.id ? { ...i, ...updated } : i)));- setDrafts((prev) => { const next = { ...prev }; delete next[img.id]; return next; });- } finally {- setSavingId(null);- }- };-- const removeImage = async (img) => {- setDeletingId(img.id);- try {- await deleteLoraDatasetImage(dataset.id, img.id);- onImagesChange((prev) => prev.filter((i) => i.id !== img.id));- } finally {- setDeletingId(null);- setConfirmingId(null);- }- };-- if (!dataset.images.length) {- return (- <div className="bg-port-card border border-port-border rounded-lg p-8 text-center text-gray-400 text-sm">- No images yet — generate reference renders, upload your own, or slice the character's reference sheet.- </div>- );- }-- return (- <div className="space-y-3">- <div className="flex justify-end">- <button- type="button"- onClick={() => setShowCaptions((v) => !v)}- className="text-xs text-gray-400 hover:text-white flex items-center gap-1.5"- title={showCaptions ? 'Hide captions — show only thumbnails' : 'Show caption editors'}- aria-pressed={!showCaptions}- >- {showCaptions ? <EyeOff className="w-3.5 h-3.5" /> : <Eye className="w-3.5 h-3.5" />}- {showCaptions ? 'Hide captions' : 'Show captions'}- </button>- </div>- <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 items-start">- {dataset.images.map((img) => {- const badge = SOURCE_BADGE[img.source] || SOURCE_BADGE.upload;- return (- <div key={img.id} className="bg-port-card border border-port-border rounded-lg overflow-hidden flex flex-col">- <div className="relative bg-port-bg">- {img.status === 'rendering' ? (- <div className="aspect-square flex flex-col items-center justify-center text-gray-400 gap-2">- <Loader2 className="w-6 h-6 animate-spin" />- <span className="text-xs">rendering…</span>- </div>- ) : img.status === 'failed' ? (- <div className="aspect-square flex flex-col items-center justify-center text-port-error gap-2">- <AlertTriangle className="w-6 h-6" />- <span className="text-xs">render failed</span>- </div>- ) : (- // Natural-height thumbnail: fixed column width, full relative- // height so portrait/landscape crops aren't sliced by object-cover.- <button- type="button"- onClick={() => openPreview(img)}- className="block w-full cursor-zoom-in focus:outline-none focus:ring-2 focus:ring-port-accent"- title="Click to preview full size"- >- <img- src={`/data/lora-datasets/${dataset.id}/images/${img.file}`}- alt={img.caption || 'dataset image'}- className="block w-full h-auto"- loading="lazy"- />- </button>- )}- {/* One wrapping row rather than two independently-anchored corners:- a long view label ("front three-quarter") used to run under the- source chip in a narrow column. */}- <div className="absolute inset-x-2 top-2 flex flex-wrap items-start justify-between gap-1 pointer-events-none">- <Pill size="xs" tone="bare" bordered={false} className={`${BADGE_BASE} ${badge.cls}`}>- {badge.label}- </Pill>- {img.variation?.view && (- <Pill size="xs" tone="bare" bordered={false} className={`${BADGE_BASE} text-gray-100 ring-white/25`}>- {img.variation.view}- </Pill>- )}- </div>- </div>- {showCaptions && (- <div className="p-2 flex flex-col gap-2 flex-1">- <AutoSizeTextarea- value={drafts[img.id] ?? img.caption}- onChange={(e) => setDrafts((prev) => ({ ...prev, [img.id]: e.target.value }))}- onBlur={() => saveCaption(img)}- placeholder={img.status === 'ready' ? 'Caption (must include the trigger word)…' : ''}- disabled={img.status !== 'ready'}- // Per-grid-cell field — a visible label would be noise, so the- // accessible name comes from aria-label (the source + view give- // a screen-reader user enough to tell cells apart).- aria-label={`Caption for ${img.source}${img.variation?.view ? ` ${img.variation.view}` : ''} image`}- className="w-full min-h-[4.5rem] bg-port-bg border border-port-border rounded p-2 text-xs text-gray-200 disabled:opacity-50"- />- <div className="flex items-center justify-between text-xs">- <button- type="button"- onClick={() => onRecaption?.(img)}- disabled={img.status !== 'ready' || captionBusy || recaptioningId === img.id}- className="text-gray-400 hover:text-white flex items-center gap-1 disabled:opacity-50"- title="Re-caption with the vision model"- >- {recaptioningId === img.id || savingId === img.id- ? <Loader2 className="w-3.5 h-3.5 animate-spin" />- : <RefreshCw className="w-3.5 h-3.5" />}- Re-caption- </button>- {confirmingId === img.id ? (- <ConfirmButtonPair- prompt="Delete?"- busy={deletingId === img.id}- onConfirm={() => removeImage(img)}- onCancel={() => setConfirmingId(null)}- />- ) : (- <button- type="button"- onClick={() => setConfirmingId(img.id)}- className="text-gray-500 hover:text-port-error"- >- Delete- </button>- )}- </div>- </div>- )}- </div>- );- })}- </div>- {previewIndex !== null && readyImages[previewIndex] && (- <DatasetImageLightbox- datasetId={dataset.id}- images={readyImages}- index={previewIndex}- onIndexChange={setPreviewIndex}- onClose={() => setPreviewIndex(null)}- />- )}- </div>- );-}diff --git a/client/src/components/loraTraining/GenerateBatchDialog.jsx b/client/src/components/loraTraining/GenerateBatchDialog.jsxdeleted file mode 100644index 10524147c6..0000000000--- a/client/src/components/loraTraining/GenerateBatchDialog.jsx+++ /dev/null@@ -1,139 +0,0 @@-/**- * Generate-batch dialog — count + optional variation-axis overrides- * (views/expressions/outfits prefilled from the subject's canon when the- * caller passes them). Submits to POST /api/lora-datasets/:id/generate.- */--import { useState } from 'react';-import { Loader2, Wand2 } from 'lucide-react';-import toast from '../ui/Toast';-import Modal from '../ui/Modal';-import { generateLoraDatasetImages } from '../../services/api';--// Advisory dataset-quality targets, mirrored from server/lib/loraDataset.js-// (RECOMMENDED_TRAINING_IMAGES / TRAINING_IMAGE_SWEET_SPOT_MAX). Kept local-// so the batch default + sweet-spot copy share one source of truth with the-// readiness coaching on LoraDatasetDetail. MAX_BATCH_IMAGES is this dialog's-// own slider cap (matches server buildVariationMatrix's 40-tuple clamp).-const RECOMMENDED_TRAINING_IMAGES = 20;-const TRAINING_IMAGE_SWEET_SPOT_MAX = 30;-const MAX_BATCH_IMAGES = 40;-const subjectKind = (dataset) => dataset?.character?.entryKind || 'characters';--const ChipToggleList = ({ idPrefix, label, options, selected, onToggle }) => (- <div>- <span className="block text-sm text-gray-400 mb-1">{label}</span>- <div className="flex flex-wrap gap-1.5">- {options.map((opt, i) => {- const on = selected.includes(opt);- return (- <button- key={`${idPrefix}-${i}`}- type="button"- onClick={() => onToggle(opt)}- className={`text-xs px-2 py-1 rounded-full border ${on- ? 'bg-port-accent/20 border-port-accent/40 text-port-accent'- : 'bg-port-bg border-port-border text-gray-400 hover:text-white'}`}- >- {opt}- </button>- );- })}- </div>- </div>-);--export default function GenerateBatchDialog({ dataset, expressionOptions = [], outfitOptions = [], onClose, onStarted }) {- // Default to the recommended quality target rather than the bare minimum so- // a one-click batch lands a strong dataset. See the tips block in- // LoraDatasetDetail for the rationale (variety > volume; sweet spot range).- const [count, setCount] = useState(RECOMMENDED_TRAINING_IMAGES);- const [expressions, setExpressions] = useState(expressionOptions);- const [outfits, setOutfits] = useState(outfitOptions);- const [submitting, setSubmitting] = useState(false);- const isCharacter = subjectKind(dataset) === 'characters';-- const toggle = (setter) => (value) =>- setter((prev) => (prev.includes(value) ? prev.filter((v) => v !== value) : [...prev, value]));-- const submit = async () => {- setSubmitting(true);- try {- const result = await generateLoraDatasetImages(dataset.id, {- count,- ...(expressions.length ? { expressions } : {}),- ...(outfits.length ? { outfits } : {}),- });- toast.success(`Queued ${result.images.length} reference render${result.images.length === 1 ? '' : 's'} (${result.mode})`);- onStarted(result);- } finally {- setSubmitting(false);- }- };-- return (- <Modal- open- onClose={onClose}- size="md"- ariaLabelledBy="lt-generate-title"- panelClassName="bg-port-card border border-port-border rounded-lg p-5"- >- <div className="space-y-4">- <h2 id="lt-generate-title" className="text-base font-semibold text-white">- Generate reference images — {dataset.character?.name}- </h2>- <p className="text-sm text-gray-400">- Renders single-subject training images from the universe bible, varying view, composition,- lighting, and context. Renders queue on the image lane; results stream into the dataset.- </p>- <div>- <label htmlFor="lt-gen-count" className="block text-sm text-gray-400 mb-1">Images ({count})</label>- <input- id="lt-gen-count"- type="range"- min={1}- max={MAX_BATCH_IMAGES}- value={count}- onChange={(e) => setCount(Number(e.target.value))}- className="w-full"- />- <p className="text-xs text-gray-500 mt-1">- Sweet spot ~{RECOMMENDED_TRAINING_IMAGES}–{TRAINING_IMAGE_SWEET_SPOT_MAX}. Fewer than 10 won't train;- past ~{MAX_BATCH_IMAGES} mostly adds time and overfitting risk.- </p>- </div>- {expressionOptions.length > 0 && (- <ChipToggleList- idPrefix="lt-exp"- label={isCharacter ? 'Expressions' : 'Lighting'}- options={expressionOptions}- selected={expressions}- onToggle={toggle(setExpressions)}- />- )}- {outfitOptions.length > 0 && (- <ChipToggleList- idPrefix="lt-outfit"- label={isCharacter ? 'Outfits' : 'Settings'}- options={outfitOptions}- selected={outfits}- onToggle={toggle(setOutfits)}- />- )}- <div className="flex justify-end gap-2">- <button type="button" onClick={onClose} className="px-3 py-2 text-sm text-gray-400 hover:text-white">Cancel</button>- <button- type="button"- onClick={submit}- disabled={submitting}- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"- >- {submitting ? <Loader2 className="w-4 h-4 animate-spin" /> : <Wand2 className="w-4 h-4" />}- Generate {count}- </button>- </div>- </div>- </Modal>- );-}diff --git a/client/src/components/loraTraining/ImportGalleryDialog.jsx b/client/src/components/loraTraining/ImportGalleryDialog.jsxdeleted file mode 100644index 7d5aee55f9..0000000000--- a/client/src/components/loraTraining/ImportGalleryDialog.jsx+++ /dev/null@@ -1,189 +0,0 @@-/**- * Import-from-gallery dialog for the LoRA dataset workbench.- *- * Multi-select picker over the local image gallery (GET /api/image-gen/gallery)- * — the "choose images already in the system" path alongside upload/generate/- * slice. Loads bounded, server-searched pages and accumulates a selection- * across pages before importing them all in one POST. The server copies- * each into the dataset (independent of the gallery original).- */--import { useCallback, useEffect, useRef, useState } from 'react';-import { Search, X, RefreshCw, Loader2, Check } from 'lucide-react';-import Modal from '../ui/Modal';-import MediaCard from '../media/MediaCard';-import toast from '../ui/Toast';-import { normalizeImage } from '../media/normalize';-import { listImageGalleryPage, importLoraDatasetGalleryImages } from '../../services/api';--const MAX_IMPORT = 50;--export default function ImportGalleryDialog({ dataset, onClose, onImported }) {- const [items, setItems] = useState([]);- const [loading, setLoading] = useState(false);- const [nextOffset, setNextOffset] = useState(null);- const [offset, setOffset] = useState(0);- const requestId = useRef(0);- const [error, setError] = useState(false);- const [retry, setRetry] = useState(0);- const [query, setQuery] = useState('');- const [selected, setSelected] = useState([]); // ordered list of filenames- const [importing, setImporting] = useState(false);-- useEffect(() => {- const id = ++requestId.current;- setLoading(true);- setError(false);- const timer = setTimeout(() => {- listImageGalleryPage({ limit: 60, offset, q: query, hidden: false }, { silent: true })- .then(page => {- if (id !== requestId.current) return;- const next = page.items.map(normalizeImage).filter(item => item.filename);- setItems(prev => {- const combined = offset === 0 ? next : [...prev, ...next];- return [...new Map(combined.map(item => [item.filename, item])).values()];- });- const end = offset + page.items.length;- setNextOffset(page.items.length && end < page.total ? end : null);- })- .catch(err => {- if (id !== requestId.current) return;- setError(true);- toast.error(err.message || 'Failed to load gallery');- })- .finally(() => { if (id === requestId.current) setLoading(false); });- }, query ? 250 : 0);- return () => { clearTimeout(timer); requestId.current += 1; };- }, [query, offset, retry]);-- const toggle = useCallback((filename) => {- setSelected((prev) => {- if (prev.includes(filename)) return prev.filter((f) => f !== filename);- if (prev.length >= MAX_IMPORT) {- toast.error(`Import up to ${MAX_IMPORT} at a time`);- return prev;- }- return [...prev, filename];- });- }, []);- const handleToggleCard = useCallback((item) => toggle(item.filename), [toggle]);-- const doImport = async () => {- if (!selected.length) return;- setImporting(true);- try {- const { images } = await importLoraDatasetGalleryImages(dataset.id, selected);- toast.success(`Imported ${images.length} image${images.length === 1 ? '' : 's'} — caption them next`);- onImported?.(images);- } catch {- // The api `request` helper already toasted the failure — swallow so the- // un-awaited onClick doesn't surface an unhandled promise rejection.- } finally {- setImporting(false);- }- };-- return (- <Modal- open- onClose={onClose}- size="3xl"- panelClassName="bg-port-card border border-port-border rounded-xl flex flex-col"- ariaLabel="Import images from your gallery"- >- <div className="flex items-center justify-between gap-3 p-3 border-b border-port-border">- <h2 className="text-sm font-medium text-white shrink-0">Import from gallery</h2>- <div className="relative flex-1 max-w-md">- <Search className="absolute left-2 top-1/2 -translate-y-1/2 w-3.5 h-3.5 text-gray-500" />- <input- type="text"- value={query}- onChange={(e) => { requestId.current += 1; setQuery(e.target.value); setOffset(0); setItems([]); setNextOffset(null); }}- placeholder="Search prompt, model, seed, LoRA…"- aria-label="Search gallery images"- className="w-full pl-7 pr-7 py-1.5 text-xs bg-port-bg border border-port-border rounded-lg text-white placeholder-gray-500 focus:outline-none focus:border-port-accent"- autoFocus- />- {query && (- <button- type="button"- onClick={() => { requestId.current += 1; setQuery(''); setOffset(0); setItems([]); setNextOffset(null); }}- className="absolute right-2 top-1/2 -translate-y-1/2 text-gray-500 hover:text-white"- aria-label="Clear search"- >- <X className="w-3.5 h-3.5" />- </button>- )}- </div>- <button- type="button"- onClick={onClose}- className="shrink-0 p-1.5 text-gray-400 hover:text-white rounded min-h-[44px] min-w-[44px] flex items-center justify-center"- aria-label="Close"- >- <X className="w-4 h-4" />- </button>- </div>-- <div className="flex-1 overflow-y-auto p-3">- {loading && offset === 0 ? (- <div className="flex items-center justify-center gap-2 text-xs text-gray-400 py-10">- <RefreshCw className="w-4 h-4 animate-spin" /> Loading gallery…- </div>- ) : items.length === 0 ? (- <div className="text-xs text-gray-500 py-10 text-center">- {error ? 'Gallery could not be loaded.' : query.trim() ? 'No images match your search.' : 'No images in your gallery yet.'}- </div>- ) : (- <div className="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 gap-3">- {items.map((item) => {- const idx = selected.indexOf(item.filename);- return (- <MediaCard- key={item.key}- item={item}- hideActions- showCollectionMenu={false}- selected={idx !== -1}- selectionLabel={idx !== -1 ? String(idx + 1) : null}- onClick={handleToggleCard}- />- );- })}- </div>- )}- {error && <button type="button" onClick={() => setRetry(n => n + 1)} className="w-full min-h-[44px] text-port-accent">Retry loading images</button>}- {!error && nextOffset !== null && (- <button type="button" disabled={loading} onClick={() => setOffset(nextOffset)}- className="w-full min-h-[44px] mt-3 text-sm text-port-accent disabled:opacity-50">- {loading ? 'Loading…' : 'Show more'}- </button>- )}- </div>-- <div className="flex items-center justify-between gap-3 p-3 border-t border-port-border">- <span className="text-xs text-gray-400">- {selected.length ? `${selected.length} selected` : 'Click images to select'}- </span>- <div className="flex items-center gap-2">- <button- type="button"- onClick={onClose}- className="px-3 py-2 text-sm text-gray-400 hover:text-white"- >- Cancel- </button>- <button- type="button"- onClick={doImport}- disabled={!selected.length || importing}- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"- >- {importing ? <Loader2 className="w-4 h-4 animate-spin" /> : <Check className="w-4 h-4" />}- Import {selected.length || ''}- </button>- </div>- </div>- </Modal>- );-}diff --git a/client/src/components/loraTraining/ImportGalleryDialog.test.jsx b/client/src/components/loraTraining/ImportGalleryDialog.test.jsxdeleted file mode 100644index f0fc5d414e..0000000000--- a/client/src/components/loraTraining/ImportGalleryDialog.test.jsx+++ /dev/null@@ -1,44 +0,0 @@-import { describe, it, expect, vi } from 'vitest';-import { render, screen, fireEvent, waitFor } from '@testing-library/react';-import ImportGalleryDialog from './ImportGalleryDialog';--const api = vi.hoisted(() => ({- listImageGalleryPage: vi.fn(),- importLoraDatasetGalleryImages: vi.fn(),-}));-vi.mock('../../services/api', () => api);-vi.mock('../media/MediaCard', () => ({- default: ({ item, onClick }) => <button onClick={() => onClick(item)}>{item.filename}</button>,-}));-vi.mock('../ui/Modal', () => ({ default: ({ children }) => <div>{children}</div> }));--describe('gallery import paging', () => {- it('pages and searches on the server while retaining selected filenames', async () => {- api.listImageGalleryPage.mockImplementation(async ({ offset, q }) => ({- items: q ? [{ filename: 'found.png' }] : offset === 0 ? [{ filename: 'first.png' }]- : offset === 1 ? [{ filename: 'first.png' }, { filename: 'second.png' }] : [],- total: q ? 1 : 4, limit: 60, offset,- }));- api.importLoraDatasetGalleryImages.mockResolvedValue({ images: [] });- const onImported = vi.fn();- render(<ImportGalleryDialog dataset={{ id: 'dataset' }} onClose={vi.fn()} onImported={onImported} />);- fireEvent.click(await screen.findByText('first.png'));- fireEvent.click(screen.getByRole('button', { name: 'Show more' }));- await screen.findByText('second.png');- expect(screen.getAllByText('first.png')).toHaveLength(1);- expect(api.listImageGalleryPage).toHaveBeenCalledWith(- { limit: 60, offset: 1, q: '', hidden: false }, { silent: true });- fireEvent.click(screen.getByRole('button', { name: 'Show more' }));- await waitFor(() => expect(screen.queryByRole('button', { name: 'Show more' })).toBeNull());- expect(api.listImageGalleryPage).toHaveBeenLastCalledWith(- { limit: 60, offset: 3, q: '', hidden: false }, { silent: true });- fireEvent.change(screen.getByLabelText('Search gallery images'), { target: { value: 'fox' } });- await screen.findByText('found.png');- expect(screen.queryByText('first.png')).toBeNull();- expect(api.listImageGalleryPage).toHaveBeenLastCalledWith(- { limit: 60, offset: 0, q: 'fox', hidden: false }, { silent: true });- fireEvent.click(screen.getByRole('button', { name: 'Import 1' }));- await waitFor(() => expect(onImported).toHaveBeenCalledWith([]));- expect(api.importLoraDatasetGalleryImages).toHaveBeenCalledWith('dataset', ['first.png']);- });-});diff --git a/client/src/components/loraTraining/LiveSampleGallery.jsx b/client/src/components/loraTraining/LiveSampleGallery.jsxdeleted file mode 100644index c7135b1861..0000000000--- a/client/src/components/loraTraining/LiveSampleGallery.jsx+++ /dev/null@@ -1,185 +0,0 @@-/**- * Live sample gallery for an in-flight training run.- *- * The trainer renders a preview image every `sampleEvery` steps (SAMPLE lines- * on the wire). This surfaces that evolution as it happens: a large current- * preview, a step-keyed thumbnail strip to scrub back through earlier samples,- * a loss sparkline, and step/ETA metrics — so you can watch the LoRA learn (and- * catch a late divergence) instead of waiting for the run to finish.- *- * Samples merge two sources so a mid-run reload still shows the full timeline:- * - a one-shot seed from GET /runs/:id/samples (every sample persisted so far)- * - live SSE frames carrying { currentImage, step } (samples since subscribe)- * Live wins on a duplicate step. The loss series is live-only (the run record- * keeps just the last step), so it builds up over the session.- */--import { useEffect, useMemo, useState } from 'react';-import { ImageOff, Radio } from 'lucide-react';-import { formatDurationMs } from '../../utils/formatters';-import { useTimeTick } from '../../hooks/useTimeTick';-import { lossSparklineGeometry } from '../../lib/lossSparkline';-import { listLoraTrainingSamples } from '../../services/api';-import useMounted from '../../hooks/useMounted';--const SPARK_W = 240;-const SPARK_H = 36;--function LossSparkline({ series }) {- const { points, last } = lossSparklineGeometry(series, { width: SPARK_W, height: SPARK_H });- if (!points) return null;- return (- <div>- <div className="flex items-center justify-between text-[11px] text-gray-500 mb-0.5">- <span>Loss</span>- <span className="text-gray-400">{last.toFixed(4)}</span>- </div>- <svg- viewBox={`0 0 ${SPARK_W} ${SPARK_H}`}- preserveAspectRatio="none"- className="w-full h-9 bg-port-bg rounded border border-port-border"- role="img"- aria-label={`Training loss curve, latest ${last.toFixed(4)}`}- >- <polyline points={points} fill="none" stroke="#3b82f6" strokeWidth="1.5" vectorEffect="non-scaling-stroke" />- </svg>- </div>- );-}--export default function LiveSampleGallery({ run, frames, progress, message }) {- const [seed, setSeed] = useState([]);- const [selectedStep, setSelectedStep] = useState(null);- // Seconds-precision tick for the elapsed/ETA labels — shared, visibility-- // aware timer (pauses when the tab is hidden) instead of a per-mount interval.- const now = useTimeTick(1000);- const mountedRef = useMounted();-- // One-shot seed of every sample persisted so far (survives a mid-run reload).- useEffect(() => {- if (!run?.id) return undefined;- let active = true;- listLoraTrainingSamples(run.id)- .then((res) => { if (active && mountedRef.current) setSeed(Array.isArray(res?.samples) ? res.samples : []); })- .catch(err => console.warn('⚠️ Failed to load training samples: ' + err.message));- return () => { active = false; };- }, [run?.id]);-- // Merge seed + live frames into a step-sorted, dedup'd sample list.- const samples = useMemo(() => {- const byStep = new Map();- for (const s of seed) byStep.set(s.step, s.url);- for (const f of frames) {- if (typeof f.currentImage === 'string' && f.currentImage && typeof f.step === 'number') {- byStep.set(f.step, f.currentImage);- }- }- return [...byStep.entries()].map(([step, url]) => ({ step, url })).sort((a, b) => a.step - b.step);- }, [seed, frames]);-- // Live loss series for the sparkline (progress frames carry numeric loss).- const lossSeries = useMemo(() => {- const out = [];- for (const f of frames) {- if (f.type === 'progress' && typeof f.loss === 'number' && typeof f.step === 'number') {- out.push({ step: f.step, loss: f.loss });- }- }- return out;- }, [frames]);-- const latest = samples.length ? samples[samples.length - 1] : null;- // `selectedStep == null` means "follow the latest sample"; a pinned step- // shows that earlier sample until the user jumps back to live.- const selected = (selectedStep != null && samples.find((s) => s.step === selectedStep)) || latest;-- const startedAt = run?.startedAt ? new Date(run.startedAt).getTime() : null;- const elapsedMs = startedAt ? Math.max(0, now - startedAt) : null;- const pct = typeof progress === 'number' ? Math.max(0, Math.min(1, progress)) : 0;- const etaMs = elapsedMs != null && pct > 0.01 && pct < 1 ? (elapsedMs / pct) * (1 - pct) : null;-- return (- <div className="space-y-2">- <div className="flex flex-col sm:flex-row gap-3">- {/* Current / selected preview */}- <div className="sm:w-2/5 shrink-0">- {selected ? (- <div className="relative">- <img- src={selected.url}- alt={`sample @ step ${selected.step}`}- className="w-full aspect-square object-cover rounded border border-port-border"- />- <span className="absolute bottom-1 left-1 px-1.5 py-0.5 rounded bg-black/70 text-[11px] text-white">- step {selected.step}- </span>- </div>- ) : (- <div className="w-full aspect-square rounded border border-port-border bg-port-bg flex flex-col items-center justify-center gap-1 text-gray-500">- <ImageOff className="w-6 h-6" />- <span className="text-[11px] text-center px-2">- {(run?.params?.sampleEvery ?? 0) > 0- ? `First preview renders at step ${run.params.sampleEvery}`- : 'Sample previews disabled for this run'}- </span>- </div>- )}- </div>-- {/* Metrics + loss curve */}- <div className="flex-1 min-w-0 space-y-2">- <div className="text-xs text-gray-400 truncate">{message || 'Starting…'}</div>- <div className="grid grid-cols-3 gap-2 text-center">- <div className="bg-port-bg rounded border border-port-border py-1">- <div className="text-sm text-white font-medium">{Math.round(pct * 100)}%</div>- <div className="text-[10px] text-gray-500">progress</div>- </div>- <div className="bg-port-bg rounded border border-port-border py-1">- <div className="text-sm text-white font-medium">{elapsedMs != null ? formatDurationMs(elapsedMs) : '—'}</div>- <div className="text-[10px] text-gray-500">elapsed</div>- </div>- <div className="bg-port-bg rounded border border-port-border py-1">- <div className="text-sm text-white font-medium">{etaMs != null ? `~${formatDurationMs(etaMs)}` : '—'}</div>- <div className="text-[10px] text-gray-500">eta</div>- </div>- </div>- <LossSparkline series={lossSeries} />- </div>- </div>-- {/* Thumbnail strip — scrub back through earlier samples */}- {samples.length > 1 && (- <div>- <div className="flex items-center justify-between mb-1">- <span className="text-[11px] text-gray-500">{samples.length} samples</span>- {selectedStep != null && (- <button- type="button"- onClick={() => setSelectedStep(null)}- className="text-[11px] text-port-accent hover:underline flex items-center gap-1"- >- <Radio className="w-3 h-3" /> Jump to latest- </button>- )}- </div>- <div className="flex gap-1.5 overflow-x-auto pb-1">- {samples.map((s) => {- const active = selected && s.step === selected.step;- return (- <button- key={s.step}- type="button"- onClick={() => setSelectedStep(selectedStep === s.step ? null : s.step)}- title={`step ${s.step}`}- className={`shrink-0 rounded border overflow-hidden ${active ? 'border-port-accent' : 'border-port-border'}`}- >- <img src={s.url} alt={`step ${s.step}`} className="w-14 h-14 object-cover" loading="lazy" />- </button>- );- })}- </div>- </div>- )}- </div>- );-}diff --git a/client/src/components/loraTraining/TrainingPanel.jsx b/client/src/components/loraTraining/TrainingPanel.jsxdeleted file mode 100644index 6088ab1756..0000000000--- a/client/src/components/loraTraining/TrainingPanel.jsx+++ /dev/null@@ -1,491 +0,0 @@-/**- * Training launch + live-run panel for a dataset.- *- * Launch gates on dataset readiness AND no in-flight trigger-word save- * (the run reads the persisted trigger word, not the form). Live progress- * streams over the run's media-job SSE; sample previews ride the 'preview'- * frames. Completed runs surface the registered LoRA with a deep link.- */--import { useState, useEffect, useCallback } from 'react';-import { Link } from 'react-router';-import { Dumbbell, Loader2, Square, CheckCircle2, XCircle, Sparkles, RotateCcw, Moon, AlertTriangle, Eraser } from 'lucide-react';-import toast from '../ui/Toast';-import { useSseProgress } from '../../hooks/useSseProgress';-import { useAsyncAction } from '../../hooks/useAsyncAction';-import useListResource from '../../hooks/useListResource';-import ListLoadError from '../ui/ListLoadError';-import CheckpointPicker from './CheckpointPicker';-import LiveSampleGallery from './LiveSampleGallery';-import {- getLoraTrainingStatus,- listLoraTrainingRuns,- startLoraTrainingRun,- cancelLoraTrainingRun,- resumeLoraTrainingRun,- listImageModels,- stripLoraDatasetSharedCaptionFragments,-} from '../../services/api';--const isActive = (run) => run && ['queued', 'running'].includes(run.status);--// The status is a single object; wrapping it in a one-item list lets it ride-// the same loading | ready | error resource as the lists, so a failed read is-// never mistaken for "no engine installed".-const fetchStatus = () => getLoraTrainingStatus({ silent: true }).then((s) => [s]);--// FLUX.2 Klein only — mflux ≥0.17 dropped FLUX.1 training, and the torch-// fallback trains the same bf16 Klein bases.-const fetchTrainableModels = () => listImageModels({ silent: true })- .then((list) => (Array.isArray(list) ? list : []).filter((m) => m.runner === 'flux2'));--// Default to the 9B base — it captures identity better than 4B and trains-// fine via 4-bit QLoRA (the server forces the panic-safe quant tier-// regardless of which inference id is picked). Prefer the plain 9B id we-// validated (flux2-klein-9b); avoid the explicit *-bf16 variant (the heavy-// path that GPU-watchdog-panics this M5). Fall back to any 9B, then first.-const defaultBaseModelId = (trainable) => {- const nineB = trainable.filter((m) => /9b/i.test(m.id));- const nonBf16 = nineB.find((m) => !/bf16/i.test(m.id));- return (nonBf16 || nineB[0] || trainable[0])?.id || '';-};--export default function TrainingPanel({ dataset, readiness, triggerSaving, onRunFinished }) {- const { items: statusItems, state: statusState, error: statusError, retry: retryStatus } = useListResource(fetchStatus);- const status = statusItems[0] ?? null;- const { items: models, state: modelsState, error: modelsError, retry: retryModels } = useListResource(fetchTrainableModels);- const [baseModelId, setBaseModelId] = useState('');- const [params, setParams] = useState(null);- const [starting, setStarting] = useState(false);- const [canceling, setCanceling] = useState(false);- const [resuming, setResuming] = useState(false);- // Server-detected caption identity-leak (409 CAPTION_IDENTITY_LEAK): the- // fragments that repeat + the captioned-image total they're measured against.- // Non-empty fragments → render the inline confirm row instead of queueing.- const [captionLeak, setCaptionLeak] = useState({ fragments: [], total: 0 });-- useEffect(() => {- if (status) setParams((prev) => prev || { ...status.defaults });- }, [status]);-- useEffect(() => {- if (modelsState === 'ready') setBaseModelId((prev) => prev || defaultBaseModelId(models));- }, [models, modelsState]);-- // A reassigned dataset keeps its id but its old runs belong to the previous- // subject. Re-fetch (and re-filter) whenever the subject identity moves,- // not just the id, and show only runs matching the current (universeId,- // entryId) so the panel + checkpoint picker can't surface/promote a stale- // run from the prior subject.- const charEntryId = dataset.character?.entryId;- const charUniverseId = dataset.character?.universeId;- const charEntryKind = dataset.character?.entryKind || 'characters';- const fetchRuns = useCallback(() => (- // Query by dataset id, then guard by the full subject identity. That covers- // character/object/place reassignments without relying on the legacy- // `character_id` mirror column.- listLoraTrainingRuns({ datasetId: dataset.id, limit: 5 }, { silent: true }).then((runs) => (Array.isArray(runs) ? runs : []).filter(- (r) => !r.character || (- r.character.entryId === charEntryId- && r.character.universeId === charUniverseId- && (r.character.entryKind || 'characters') === charEntryKind- ),- ))- ), [dataset.id, charEntryId, charEntryKind, charUniverseId]);- const { items: runs, state: runsState, error: runsError, retry: retryRuns, setItems: setRuns } = useListResource(fetchRuns);- const activeRun = runs.find(isActive) || null;- const lastRun = runs.find((r) => !isActive(r)) || null;- // Re-read in place so the panel does not flash back to the launch form. A- // failed re-read drops to the resource's error state, which disables Start:- // an unknown run state must never offer a second multi-hour launch.- const refreshRuns = useCallback(() => {- fetchRuns().then(setRuns).catch((err) => {- console.warn('⚠️ Failed to refresh training runs: ' + err.message);- retryRuns();- });- }, [fetchRuns, setRuns, retryRuns]);-- const sseUrl = activeRun ? `/api/lora-training/runs/${activeRun.id}/events` : null;- const { frames, latest, closed } = useSseProgress(sseUrl, { enabled: !!activeRun });-- useEffect(() => {- if (!closed || !activeRun) return;- // Terminal frame (or stream loss) — re-read the run record for truth.- refreshRuns();- onRunFinished?.();- }, [closed, activeRun, refreshRuns, onRunFinished]);-- // Engine pick mirrors the server: mflux (MLX) when its trainer is- // installed, else the torch venv. Either being ready unblocks training.- const engineReady = !!(status?.runtimes?.mflux?.ready || status?.runtimes?.flux2?.ready);-- // `silent: true` — this owns its own error surfacing: the expected- // CAPTION_IDENTITY_LEAK 409 becomes the inline confirm row (not a toast), and- // every other failure toasts here exactly once.- const start = async ({ acknowledgeCaptionLeak = false } = {}) => {- setStarting(true);- try {- await startLoraTrainingRun(- { datasetId: dataset.id, baseModelId, params, acknowledgeCaptionLeak },- { silent: true },- );- setCaptionLeak({ fragments: [], total: 0 });- toast.success('Training queued');- refreshRuns();- } catch (err) {- if (err?.code === 'CAPTION_IDENTITY_LEAK') {- setCaptionLeak({ fragments: err.context?.sharedFragments || [], total: err.context?.total || 0 });- } else {- toast.error(err?.message || 'Failed to queue training');- }- } finally {- setStarting(false);- }- };-- // From the leak row: strip the shared identity fragments server-side, sync the- // parent's caption view, then re-launch (the gate now passes).- const [stripAndQueue, strippingLeak] = useAsyncAction(async () => {- const { removedFragments } = await stripLoraDatasetSharedCaptionFragments(dataset.id);- if (removedFragments?.length) {- toast.success(`Stripped ${removedFragments.length} shared identity fragment${removedFragments.length === 1 ? '' : 's'}`);- }- onRunFinished?.(); // parent reloads the dataset so captions + the lint banner refresh- setCaptionLeak({ fragments: [], total: 0 });- await start();- }, { errorMessage: 'Failed to strip captions' });-- const cancel = async () => {- if (!activeRun) return;- setCanceling(true);- try {- await cancelLoraTrainingRun(activeRun.id);- toast.success('Cancel requested — the trainer saves a checkpoint first');- } finally {- setCanceling(false);- }- };-- const resume = async () => {- if (!lastRun) return;- setResuming(true);- try {- const { fromStep } = await resumeLoraTrainingRun(lastRun.id);- toast.success(`Resuming from checkpoint @ step ${fromStep}`);- refreshRuns();- } finally {- setResuming(false);- }- };-- const setParam = (key, value) => setParams((prev) => ({ ...prev, [key]: value }));-- // A killed run keeps its checkpoints — surface the picker (view + salvage a- // partial LoRA) whenever a non-completed run recorded one, and a Resume button- // for any failed/canceled run. Gate Resume on status, NOT the recorded- // checkpoint count: a crash can kill the run before the debounced record- // persists its last checkpoint, yet the server resumes from disk — let its- // 409 (NO_RESUMABLE_CHECKPOINT) handle the genuinely-empty case. Both runtimes- // support true optimizer-state resume (mflux via --resume, FLUX.2 torch via- // --resume-from), so this no longer filters by runtime.- const lastRunCheckpoints = lastRun?.artifacts?.checkpoints?.length || 0;- const canResume = lastRun && ['failed', 'canceled'].includes(lastRun.status);-- const disabledReason = !readiness?.trainable- ? `Needs ${readiness?.required ?? 10} captioned images (have ${readiness?.captioned ?? 0})`- : triggerSaving ? 'Saving trigger word…'- : statusState === 'error' ? 'Could not read training status'- : runsState === 'error' ? 'Could not read training runs'- : modelsState === 'error' ? 'Could not load base models'- : (statusState === 'loading' || runsState === 'loading') ? 'Checking training status…'- : !baseModelId ? 'Pick a base model'- : !engineReady ? 'No training engine — install mflux ≥0.17 or run scripts/setup-image-video.sh'- : null;-- if (activeRun) {- const progress = typeof latest?.progress === 'number' ? latest.progress : (activeRun.progress?.step || 0) / (activeRun.progress?.totalSteps || 1);- return (- <div className="bg-port-card border border-port-border rounded-lg p-4 space-y-3">- <div className="flex items-center justify-between">- <div className="flex items-center gap-2 text-white font-medium">- <Loader2 className="w-4 h-4 animate-spin text-port-warning" />- Training {activeRun.runtime} · {activeRun.baseModelId}- </div>- <button- type="button"- onClick={cancel}- disabled={canceling}- className="px-2 py-1 text-xs rounded bg-port-error/20 text-port-error hover:bg-port-error/30 flex items-center gap-1 disabled:opacity-50"- >- <Square className="w-3 h-3" /> {canceling ? 'Canceling…' : 'Cancel'}- </button>- </div>- <div className="h-2 bg-port-bg rounded overflow-hidden">- <div className="h-full bg-port-accent transition-all" style={{ width: `${Math.round(progress * 100)}%` }} />- </div>- {activeRun.status === 'queued' ? (- <div className="text-xs text-gray-400">Queued (position {latest?.position ?? '…'})</div>- ) : (- <LiveSampleGallery run={activeRun} frames={frames} progress={progress} message={latest?.message} />- )}- </div>- );- }-- return (- <div className="bg-port-card border border-port-border rounded-lg p-4 space-y-3">- <div className="flex items-center gap-2 text-white font-medium">- <Dumbbell className="w-4 h-4 text-port-accent" /> Train LoRA- </div>- {lastRun && (- <div className="text-xs flex items-center gap-1.5">- {lastRun.status === 'completed' ? (- <>- <CheckCircle2 className="w-3.5 h-3.5 text-port-success" />- <span className="text-gray-300">- Last run completed- {lastRun.output?.loraFilename && (- <> — <Link to="/models/loras" className="text-port-accent hover:underline inline-flex items-center gap-1"><Sparkles className="w-3 h-3" />{lastRun.output.loraFilename}</Link></>- )}- </span>- </>- ) : (- <>- <XCircle className="w-3.5 h-3.5 text-port-error" />- <span className="text-gray-400" title={lastRun.error || ''}>Last run {lastRun.status}{lastRun.error ? ` — ${lastRun.error.slice(0, 120)}` : ''}</span>- </>- )}- </div>- )}- {lastRun?.status === 'failed' && lastRun.errorCode === 'HF_AUTH' && (- <div role="status" className="rounded-lg border border-port-warning/40 bg-port-warning/10 px-3 py-3 text-xs text-port-warning space-y-2">- <div className="font-semibold text-sm">Model access required</div>- <div className="text-port-warning/90">- The training base model is gated on HuggingFace. Accept its license with the account- that owns your stored token, then retry — no token change needed.- </div>- {lastRun.errorRepo && (- <a- href={`https://huggingface.co/${lastRun.errorRepo}`}- target="_blank"- rel="noreferrer"- className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-lg bg-port-accent text-white text-xs font-medium hover:bg-port-accent/80"- >- Request access to {lastRun.errorRepo} ↗- </a>- )}- </div>- )}- {canResume && (- <div className="flex items-center justify-between gap-2 rounded-lg border border-port-accent/30 bg-port-accent/5 px-3 py-2">- <span className="text-xs text-gray-300">- {lastRunCheckpoints > 0- ? `${lastRunCheckpoints} checkpoint${lastRunCheckpoints === 1 ? '' : 's'} saved — pick up where it stopped.`- : 'Pick up from the last saved checkpoint.'}- </span>- <button- type="button"- onClick={resume}- disabled={resuming}- className="px-2.5 py-1.5 text-xs rounded bg-port-accent text-white hover:bg-port-accent/80 disabled:opacity-50 flex items-center gap-1.5 shrink-0"- >- {resuming ? <Loader2 className="w-3.5 h-3.5 animate-spin" /> : <RotateCcw className="w-3.5 h-3.5" />}- Resume training- </button>- </div>- )}- {(lastRun?.status === 'completed' || lastRunCheckpoints > 0) && (- <CheckpointPicker run={lastRun} onPromoted={refreshRuns} />- )}- {statusState === 'error' && <ListLoadError what="training status" error={statusError} onRetry={retryStatus} />}- {runsState === 'error' && <ListLoadError what="training runs" error={runsError} onRetry={retryRuns} />}- <div className="grid grid-cols-2 gap-3">- <div className="col-span-2">- {modelsState === 'error' ? (- <ListLoadError what="base models" error={modelsError} onRetry={retryModels} />- ) : (- <>- <label htmlFor="lt-base-model" className="block text-xs text-gray-400 mb-1">Base model</label>- <select- id="lt-base-model"- value={baseModelId}- onChange={(e) => setBaseModelId(e.target.value)}- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"- >- {modelsState === 'loading' && <option value="">Loading models…</option>}- {modelsState === 'ready' && !models.length && <option value="">No trainable models registered</option>}- {models.map((m) => <option key={m.id} value={m.id}>{m.name || m.id}</option>)}- </select>- </>- )}- </div>- {params && [- ['steps', 'Steps', 10, 10000],- ['rank', 'Rank', 1, 128],- ['checkpointEvery', 'Checkpoint every', 0, 5000],- ['sampleEvery', 'Sample every', 0, 5000],- ].map(([key, label, min, max]) => (- <div key={key}>- <label htmlFor={`lt-param-${key}`} className="block text-xs text-gray-400 mb-1">{label}</label>- <input- id={`lt-param-${key}`}- type="number"- min={min}- max={max}- value={params[key] ?? ''}- onChange={(e) => setParam(key, Number(e.target.value))}- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"- />- </div>- ))}- {params && (- <div>- <label htmlFor="lt-param-resolution" className="block text-xs text-gray-400 mb-1">Resolution</label>- <select- id="lt-param-resolution"- value={params.resolution ?? 512}- onChange={(e) => setParam('resolution', Number(e.target.value))}- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"- >- {[512, 768, 1024].map((r) => <option key={r} value={r}>{r}px</option>)}- </select>- </div>- )}- {params && (- <div>- <label htmlFor="lt-param-lr" className="block text-xs text-gray-400 mb-1">Learning rate</label>- <input- id="lt-param-lr"- type="number"- step="0.00001"- min="0.000001"- max="0.1"- value={params.learningRate ?? ''}- onChange={(e) => setParam('learningRate', Number(e.target.value))}- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"- />- </div>- )}- {/* Frozen-base overrides (issue #1407 / #1321) — only feed the mflux- config builder, so gate on the mflux engine being the one that'll- run (the torch/FLUX2 path always trains the bf16 base and ignores- these). "Auto" sends an explicit `null` (NOT absent): the server- re-merges saved settings.loraTraining.defaults into the run, so an- omitted key would silently keep a configured default — `null` is a- deliberate clear that forces the memory-derived tier (still clamped- by LORA_TRAIN_MAX_QUANT_BITS). buildMfluxTrainConfig treats a- null/non-boolean as "no override". */}- {params && status?.runtimes?.mflux?.ready && (- <div>- <label htmlFor="lt-param-baseQuant" className="block text-xs text-gray-400 mb-1">Base quant</label>- <select- id="lt-param-baseQuant"- value={params.baseQuant ?? ''}- onChange={(e) => setParam('baseQuant', e.target.value === '' ? null : Number(e.target.value))}- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"- >- <option value="">Auto</option>- <option value={16}>bf16 (heaviest)</option>- <option value={8}>8-bit</option>- <option value={4}>4-bit (lightest)</option>- </select>- </div>- )}- {params && status?.runtimes?.mflux?.ready && (- <div>- <label htmlFor="lt-param-lowRam" className="block text-xs text-gray-400 mb-1">Low RAM spill</label>- <select- id="lt-param-lowRam"- value={params.lowRam === true ? 'on' : params.lowRam === false ? 'off' : ''}- onChange={(e) => setParam('lowRam', e.target.value === '' ? null : e.target.value === 'on')}- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"- >- <option value="">Auto</option>- <option value="on">On</option>- <option value="off">Off</option>- </select>- </div>- )}- </div>- {status?.runtimes?.mflux?.ready && (- <div className="flex items-start gap-2 rounded-lg border border-port-border bg-port-bg/50 px-3 py-2 text-xs text-gray-400">- <Moon className="w-3.5 h-3.5 mt-0.5 shrink-0 text-port-accent" />- <div className="space-y-1">- <div className="text-gray-300">- By default, this Mac's display is slept while training and woken when the run finishes.- </div>- <div>- On Apple Silicon, an <span className="text-gray-300">active display can crash the whole machine</span>{' '}- (GPU watchdog kernel panic) during heavy LoRA training — so PortOS puts the display to sleep for you- and the system keeps running underneath. Big runs (9B, bf16) can take hours: start one and{' '}- <span className="text-gray-300">let it run overnight</span>. Don't close the lid (that sleeps the- whole Mac and pauses the run); just leave it be, or drive it over SSH. If you've set{' '}- <code className="text-gray-300">loraTraining.displaySleep: false</code> (e.g. headless), PortOS leaves- the display alone — keep it off the GPU yourself.- </div>- </div>- </div>- )}- {captionLeak.fragments.length > 0 && (- <div className="bg-port-warning/10 border border-port-warning/40 rounded-lg p-3 text-sm space-y-2">- <div className="flex items-start gap-2">- <AlertTriangle className="w-4 h-4 text-port-warning shrink-0 mt-0.5" />- <p className="text-gray-200 min-w-0">- <span className="font-medium text-port-warning">Identity is leaking into the captions.</span>{' '}- These fragments repeat across most images, so the{' '}- <span className="font-mono text-gray-300">{dataset.triggerWord || 'trigger'}</span>{' '}- token would learn a generic subject. Strip them so the trigger absorbs the fixed identity, or train anyway.- </p>- </div>- <div className="flex flex-wrap gap-1.5">- {captionLeak.fragments.map((f) => (- <span- key={f.normalized}- className="text-xs px-1.5 py-0.5 rounded bg-port-bg border border-port-border text-gray-300 font-mono"- title={`in ${f.count} of ${captionLeak.total} captions`}- >- {f.fragment}<span className="text-gray-500"> ·{f.count}</span>- </span>- ))}- </div>- <div className="flex flex-wrap gap-2">- <button- type="button"- onClick={stripAndQueue}- disabled={strippingLeak || starting}- className="px-3 py-1.5 text-xs rounded bg-port-warning/20 text-port-warning hover:bg-port-warning/30 flex items-center gap-2 disabled:opacity-50"- >- {strippingLeak ? <Loader2 className="w-3.5 h-3.5 animate-spin" /> : <Eraser className="w-3.5 h-3.5" />}- Strip identity & queue- </button>- <button- type="button"- onClick={() => start({ acknowledgeCaptionLeak: true })}- disabled={strippingLeak || starting}- className="px-3 py-1.5 text-xs rounded border border-port-border text-gray-300 hover:bg-port-bg flex items-center gap-2 disabled:opacity-50"- >- Train anyway- </button>- </div>- </div>- )}- <div className="flex items-center justify-between gap-2">- <span className={`text-xs ${!disabledReason && readiness?.quality === 'minimum' ? 'text-port-warning' : 'text-gray-500'}`}>- {disabledReason- || (readiness?.quality === 'minimum'- ? `Trains on ${readiness?.captioned ?? 0} — ${Math.max(0, (readiness?.recommended ?? 20) - (readiness?.captioned ?? 0))} more recommended`- : `Trains on ${readiness?.captioned ?? 0} captioned images`)}- </span>- <button- type="button"- onClick={() => start()}- disabled={!!disabledReason || starting || strippingLeak}- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"- >- {starting ? <Loader2 className="w-4 h-4 animate-spin" /> : <Dumbbell className="w-4 h-4" />}- Train- </button>- </div>- </div>- );-}diff --git a/client/src/components/loraTraining/TrainingPanel.test.jsx b/client/src/components/loraTraining/TrainingPanel.test.jsxdeleted file mode 100644index 749509c59e..0000000000--- a/client/src/components/loraTraining/TrainingPanel.test.jsx+++ /dev/null@@ -1,109 +0,0 @@-import { describe, it, expect, vi, beforeEach } from 'vitest';-import { act, fireEvent, render, screen } from '@testing-library/react';-import { MemoryRouter } from 'react-router';--vi.mock('../ui/Toast', () => ({- default: Object.assign(vi.fn(), { success: vi.fn(), error: vi.fn() }),-}));-vi.mock('../../hooks/useSseProgress', () => ({- useSseProgress: () => ({ frames: [], latest: null, closed: false }),-}));-vi.mock('./LiveSampleGallery', () => ({ default: () => <div>live samples</div> }));-vi.mock('./CheckpointPicker', () => ({ default: () => null }));--const getLoraTrainingStatus = vi.fn();-const listLoraTrainingRuns = vi.fn();-const listImageModels = vi.fn();-vi.mock('../../services/api', () => ({- getLoraTrainingStatus: (...a) => getLoraTrainingStatus(...a),- listLoraTrainingRuns: (...a) => listLoraTrainingRuns(...a),- listImageModels: (...a) => listImageModels(...a),- startLoraTrainingRun: vi.fn(),- cancelLoraTrainingRun: vi.fn(),- resumeLoraTrainingRun: vi.fn(),- stripLoraDatasetSharedCaptionFragments: vi.fn(),-}));--import TrainingPanel from './TrainingPanel';--const READY_STATUS = { runtimes: { mflux: { ready: true } }, defaults: { steps: 1000, rank: 16 } };-const MODELS = [{ id: 'flux2-klein-9b', name: 'Klein 9B', runner: 'flux2' }, { id: 'flux1-dev', runner: 'flux1' }];-const READINESS = { trainable: true, captioned: 12, required: 10, recommended: 20, quality: 'good' };-const ACTIVE_RUN = { id: 'run-1', status: 'running', runtime: 'mflux', baseModelId: 'flux2-klein-9b', progress: { step: 1, totalSteps: 10 } };--const renderPanel = async () => {- render(- <MemoryRouter>- <TrainingPanel dataset={{ id: 'ds-1' }} readiness={READINESS} triggerSaving={false} />- </MemoryRouter>,- );- await act(async () => {});-};--const trainButton = () => screen.getByRole('button', { name: /Train$/ });--beforeEach(() => {- vi.clearAllMocks();- getLoraTrainingStatus.mockResolvedValue(READY_STATUS);- listLoraTrainingRuns.mockResolvedValue([]);- listImageModels.mockResolvedValue(MODELS);-});--describe('TrainingPanel failed reads', () => {- it('offers Start when every read succeeds', async () => {- await renderPanel();- expect(trainButton()).toBeEnabled();- expect(screen.getByLabelText('Base model')).toHaveValue('flux2-klein-9b');- });-- it('shows a Retry — not the mflux install message — when the status read fails, and recovers on retry', async () => {- getLoraTrainingStatus.mockRejectedValueOnce(new Error('Gateway timeout'));- await renderPanel();- expect(screen.getByRole('alert')).toHaveTextContent('Could not load training status — Gateway timeout');- expect(screen.getByText('Could not read training status')).toBeInTheDocument();- expect(screen.queryByText(/No training engine/)).toBeNull();- expect(trainButton()).toBeDisabled();-- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });- expect(screen.queryByRole('alert')).toBeNull();- expect(screen.getByLabelText('Steps')).toHaveValue(1000);- expect(trainButton()).toBeEnabled();- });-- it('still reports a missing engine when the status read succeeds with no ready runtime', async () => {- getLoraTrainingStatus.mockResolvedValue({ runtimes: {}, defaults: {} });- await renderPanel();- expect(screen.getByText(/No training engine/)).toBeInTheDocument();- expect(trainButton()).toBeDisabled();- });-- it('disables Start with a Retry when the runs read fails, then shows the in-progress view for an active run', async () => {- listLoraTrainingRuns.mockRejectedValueOnce(new Error('Server unavailable'));- await renderPanel();- expect(screen.getByRole('alert')).toHaveTextContent('Could not load training runs — Server unavailable');- expect(trainButton()).toBeDisabled();-- listLoraTrainingRuns.mockResolvedValueOnce([ACTIVE_RUN]);- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });- expect(screen.getByText(/Training mflux/)).toBeInTheDocument();- expect(screen.queryByRole('button', { name: /Train$/ })).toBeNull();- });-- it('shows an error row — not "No trainable models registered" — when the models read fails', async () => {- listImageModels.mockRejectedValueOnce(new Error('Server unavailable'));- await renderPanel();- expect(screen.getByRole('alert')).toHaveTextContent('Could not load base models — Server unavailable');- expect(screen.queryByText('No trainable models registered')).toBeNull();- expect(trainButton()).toBeDisabled();-- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });- expect(screen.getByLabelText('Base model')).toHaveValue('flux2-klein-9b');- expect(trainButton()).toBeEnabled();- });-- it('says "No trainable models registered" only for a successful empty list', async () => {- listImageModels.mockResolvedValue([{ id: 'flux1-dev', runner: 'flux1' }]);- await renderPanel();- expect(screen.getByText('No trainable models registered')).toBeInTheDocument();- });-});diff --git a/client/src/components/loraTraining/UniverseCharacterPicker.jsx b/client/src/components/loraTraining/UniverseCharacterPicker.jsxdeleted file mode 100644index a4b2bc4b03..0000000000--- a/client/src/components/loraTraining/UniverseCharacterPicker.jsx+++ /dev/null@@ -1,97 +0,0 @@-/**- * Universe → bible-subject cascading select. Shared by the "New dataset" dialog- * and the dataset-reassignment dialog so both pick a (universeId, entryKind, entryId)- * pair through identical loading/empty states. The parent owns the selected- * ids; this just renders the two <select>s and loads the option lists.- *- * When the universe changes the parent is responsible for clearing entryId- * (a stale subject id from the previous universe must not survive) — the- * `onUniverseChange` handler should reset it.- */--import { useCallback } from 'react';-import useListResource from '../../hooks/useListResource';-import ListLoadError from '../ui/ListLoadError';-import { listUniverses, getUniverse } from '../../services/api';--const ENTRY_KINDS = [- { value: 'characters', label: 'Character', empty: 'No characters in this universe', pick: 'Pick a character' },- { value: 'objects', label: 'Object', empty: 'No objects in this universe', pick: 'Pick an object' },- { value: 'places', label: 'Place', empty: 'No places in this universe', pick: 'Pick a place' },-];-const fetchUniverses = () => listUniverses({ silent: true });-const kindConfig = (entryKind) => ENTRY_KINDS.find((k) => k.value === entryKind) || ENTRY_KINDS[0];--export default function UniverseCharacterPicker({- universeId,- entryKind = 'characters',- entryId,- onUniverseChange,- onEntryKindChange = () => {},- onEntryChange,- idPrefix = 'ucp',- disabledEntryIds = [],-}) {- const { items: universes, state: universesState, error: universesError, retry: retryUniverses } = useListResource(fetchUniverses);- // One universe is a single object; a one-item list gives its read the same- // loading | ready | error states, so a failed read never renders as an empty- // subject list.- const fetchUniverse = useCallback(() => getUniverse(universeId, { silent: true }).then((u) => [u || {}]), [universeId]);- const { items: universeItems, state: universeState, error: universeError, retry: retryUniverse } = useListResource(fetchUniverse, { enabled: !!universeId });- const entries = universeState === 'ready' && Array.isArray(universeItems[0]?.[entryKind]) ? universeItems[0][entryKind] : [];-- const disabled = new Set(disabledEntryIds);- const kind = kindConfig(entryKind);-- return (- <>- <div>- <label htmlFor={`${idPrefix}-universe`} className="block text-sm text-gray-400 mb-1">Universe</label>- <select- id={`${idPrefix}-universe`}- value={universeId}- onChange={(e) => onUniverseChange(e.target.value)}- className="w-full bg-port-bg border border-port-border rounded px-3 py-2 text-white"- >- <option value="">{universesState === 'loading' ? 'Loading…' : 'Pick a universe'}</option>- {universes.map((u) => <option key={u.id} value={u.id}>{u.name}</option>)}- </select>- {universesState === 'error' && <ListLoadError className="mt-1" what="universes" error={universesError} onRetry={retryUniverses} />}- </div>- {universeId && (- <>- <div>- <label htmlFor={`${idPrefix}-entry-kind`} className="block text-sm text-gray-400 mb-1">Subject type</label>- <select- id={`${idPrefix}-entry-kind`}- value={entryKind}- onChange={(e) => onEntryKindChange(e.target.value)}- className="w-full bg-port-bg border border-port-border rounded px-3 py-2 text-white"- >- {ENTRY_KINDS.map((k) => <option key={k.value} value={k.value}>{k.label}</option>)}- </select>- </div>- <div>- <label htmlFor={`${idPrefix}-entry`} className="block text-sm text-gray-400 mb-1">{kind.label}</label>- <select- id={`${idPrefix}-entry`}- value={entryId}- onChange={(e) => onEntryChange(e.target.value)}- className="w-full bg-port-bg border border-port-border rounded px-3 py-2 text-white"- >- <option value="">- {universeState === 'loading' ? 'Loading…' : universeState === 'error' ? 'Unavailable' : entries.length ? kind.pick : kind.empty}- </option>- {entries.map((entry) => (- <option key={entry.id} value={entry.id} disabled={disabled.has(entry.id)}>- {entry.name || entry.slugline || entry.id}{disabled.has(entry.id) ? ' (current)' : ''}- </option>- ))}- </select>- {universeState === 'error' && <ListLoadError className="mt-1" what="the universe" error={universeError} onRetry={retryUniverse} />}- </div>- </>- )}- </>- );-}diff --git a/client/src/components/loraTraining/UniverseCharacterPicker.test.jsx b/client/src/components/loraTraining/UniverseCharacterPicker.test.jsxdeleted file mode 100644index cfdfab8a3d..0000000000--- a/client/src/components/loraTraining/UniverseCharacterPicker.test.jsx+++ /dev/null@@ -1,51 +0,0 @@-import { describe, it, expect, vi, beforeEach } from 'vitest';-import { act, fireEvent, render, screen } from '@testing-library/react';--const listUniverses = vi.fn();-const getUniverse = vi.fn();-vi.mock('../../services/api', () => ({- listUniverses: (...a) => listUniverses(...a),- getUniverse: (...a) => getUniverse(...a),-}));--import UniverseCharacterPicker from './UniverseCharacterPicker';--const renderPicker = async () => {- render(<UniverseCharacterPicker universeId="u1" entryKind="characters" entryId="" onUniverseChange={vi.fn()} onEntryChange={vi.fn()} />);- await act(async () => {});-};--beforeEach(() => {- vi.clearAllMocks();- listUniverses.mockResolvedValue([{ id: 'u1', name: 'Example Universe' }]);-});--describe('UniverseCharacterPicker', () => {- it('shows a Retry — not "No characters in this universe" — when the universe read fails, and lists them on retry', async () => {- getUniverse.mockRejectedValueOnce(new Error('Server unavailable'));- await renderPicker();- expect(screen.getByRole('alert')).toHaveTextContent('Could not load the universe — Server unavailable');- expect(screen.queryByText('No characters in this universe')).toBeNull();-- getUniverse.mockResolvedValueOnce({ characters: [{ id: 'c1', name: 'Alice' }] });- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });- expect(screen.queryByRole('alert')).toBeNull();- expect(screen.getByRole('option', { name: 'Alice' })).toBeInTheDocument();- });-- it('shows a Retry — not an empty universe picker — when the universe list read fails', async () => {- listUniverses.mockRejectedValueOnce(new Error('Server unavailable'));- getUniverse.mockResolvedValue({ characters: [] });- await renderPicker();- expect(screen.getByRole('alert')).toHaveTextContent('Could not load universes — Server unavailable');-- await act(async () => { fireEvent.click(screen.getByRole('button', { name: 'Retry' })); });- expect(screen.getByRole('option', { name: 'Example Universe' })).toBeInTheDocument();- });-- it('says "No characters in this universe" only after a successful empty read', async () => {- getUniverse.mockResolvedValue({ characters: [] });- await renderPicker();- expect(screen.getByText('No characters in this universe')).toBeInTheDocument();- });-});diff --git a/client/src/components/media/MediaJobsQueue.jsx b/client/src/components/media/MediaJobsQueue.jsxindex 52f510bd9d..89323ec365 100644--- a/client/src/components/media/MediaJobsQueue.jsx+++ b/client/src/components/media/MediaJobsQueue.jsx@@ -458,7 +458,7 @@ function TrainingJobDetail({ runId }) {function JobRow({ job, onCancel, onRetry, onRunNow, onDelete }) {const Icon = KIND_ICON[job.kind] || Film;const canCancel = job.status === 'queued' || job.status === 'running';- const canRetry = (job.status === 'failed' || job.status === 'canceled') && typeof onRetry === 'function';+ const canRetry = job.kind !== 'training' && (job.status === 'failed' || job.status === 'canceled') && typeof onRetry === 'function';// Delete only on terminal rows — live jobs go through Cancel.const canDelete = (job.status === 'failed' || job.status === 'canceled' || job.status === 'completed')&& typeof onDelete === 'function';diff --git a/client/src/components/settings/DataMaintenanceTab.jsx b/client/src/components/settings/DataMaintenanceTab.jsxnew file mode 100644index 0000000000..1da742280a--- /dev/null+++ b/client/src/components/settings/DataMaintenanceTab.jsx@@ -0,0 +1,88 @@+import { useEffect, useRef, useState } from 'react';+import { Link } from 'react-router';+import { RefreshCw, Trash2 } from 'lucide-react';+import Modal from '../ui/Modal';+import { request } from '../../services/apiCore';+import { formatBytes, formatCount } from '../../utils/formatters';++const api = (path = '', options = {}) => request(`/data-maintenance${path}`, { ...options, silent: true });++export default function DataMaintenanceTab() {+ const [items, setItems] = useState(null);+ const [error, setError] = useState('');+ const [notice, setNotice] = useState('');+ const [preview, setPreview] = useState(null);+ const [confirmation, setConfirmation] = useState('');+ const [busy, setBusy] = useState(false);+ const inFlight = useRef(false);+ const generation = useRef(0);+ useEffect(() => {+ let active = true;+ inFlight.current = true;+ setBusy(true);+ api().then(result => { if (active) setItems(result); }).catch(err => { if (active) setError(err.message); }).finally(() => { if (active) { inFlight.current = false; setBusy(false); } });+ return () => { active = false; generation.current += 1; };+ }, []);++ async function act(work) {+ if (inFlight.current) return;+ inFlight.current = true;+ setBusy(true);+ setError('');+ const current = generation.current;+ try { await work(() => current === generation.current); }+ catch (err) { if (current === generation.current) { setError(err.message); setPreview(null); setConfirmation(''); } }+ finally { inFlight.current = false; if (current === generation.current) setBusy(false); }+ }++ function refresh() {+ setPreview(null);+ setConfirmation('');+ setNotice('');+ void act(async valid => { const result = await api(); if (valid()) setItems(result); });+ }++ function begin(operationId) {+ void act(async valid => {+ const result = await api('/preview', { method: 'POST', body: JSON.stringify({ operationId }) });+ if (valid()) { setPreview(result); setConfirmation(''); }+ });+ }++ function remove() {+ if (!preview || confirmation !== preview.confirmation) return;+ void act(async valid => {+ const result = await api('/execute', { method: 'POST', body: JSON.stringify({ token: preview.token, confirmation }) });+ if (!valid()) return;+ setItems(previous => previous.map(item => item.id === result.operationId ? { ...item, status: result.status === 'deleted' ? 'absent' : result.status, bytes: result.remainingBytes, files: result.remainingFiles, actions: result.status === 'retained' ? ['delete'] : [] } : item));+ setNotice(`Removed ${formatCount(result.removedFiles)} files (${formatBytes(result.removedBytes)}). Run history and installed adapters were retained.${result.status === 'retained' ? ' New files appeared during removal; refresh and preview them separately.' : result.status === 'unavailable' ? ' Inventory could not be read after removal; refresh to check remaining files.' : ''}`);+ setPreview(null);+ setConfirmation('');+ });+ }++ return <div className="max-w-5xl space-y-4">+ <div className="flex items-start justify-between gap-4">+ <div><h2 className="text-lg font-semibold">Data Maintenance</h2><p className="text-sm text-port-text-muted">Inspect retained data from retired features and review supported maintenance actions.</p></div>+ <button type="button" onClick={refresh} disabled={busy} className="flex items-center gap-2 px-3 py-2 border border-port-border rounded disabled:opacity-50"><RefreshCw className="w-4 h-4" />Refresh</button>+ </div>+ <p className="text-sm text-port-text-muted">LoRA training is retired. Existing files and configuration remain intact until you explicitly remove files here. There is no automatic conversion of training data to reference images. Use <Link to="/media/image" className="text-port-accent">reference-image generation</Link> for new work. <Link to="/system-resources/queues" className="text-port-accent">Review or cancel active media jobs</Link> before maintenance. Backup settings are unchanged; check your backups before removing unique data.</p>+ {error && <div role="alert" className="text-port-error border border-port-error rounded p-3">{error}</div>}+ {notice && <p role="status" className="text-port-success">{notice}</p>}+ {!items && !error && <p role="status">Measuring retained data…</p>}+ {items?.map(item => <section key={item.id} className="border border-port-border rounded-lg bg-port-card p-4 space-y-2">+ <div className="flex flex-wrap items-center justify-between gap-3"><h3 className="font-medium">{item.label}</h3><span className="text-sm text-port-text-muted">{item.status}</span></div>+ <div className="flex flex-wrap gap-x-5 text-sm"><code>{item.path}</code><span>{item.category}</span><span>{item.bytes === null ? 'Size unavailable' : formatBytes(item.bytes)} · {formatCount(item.files)} files</span></div>+ <p className="text-sm text-port-text-muted">{item.warning}</p>+ {item.error && <p role="alert" className="text-port-error">{item.error}</p>}+ {item.id === 'shared-lora-adapters' && <Link to="/models/loras" className="text-port-accent text-sm">Manage installed adapters</Link>}+ {item.actions.includes('delete') && <button type="button" disabled={busy} onClick={() => begin(item.id)} className="flex items-center gap-2 text-port-error border border-port-border rounded px-3 py-2 disabled:opacity-50"><Trash2 className="w-4 h-4" />Preview deletion</button>}+ </section>)}+ {preview && <Modal open panelClassName="bg-port-card border border-port-border rounded-lg shadow-xl" ariaLabel="Review permanent deletion" usePortal onClose={() => { if (!busy) { setPreview(null); setConfirmation(''); } }} closeOnEsc={!busy} closeOnBackdrop={!busy}>+ <div className="p-5 space-y-4"><h3 className="font-semibold">Review permanent deletion</h3><p><code>{preview.path}</code> · {formatBytes(preview.bytes)} · {formatCount(preview.files)} files</p><p className="text-port-error">{preview.warning} This removal cannot be undone by PortOS.</p><p className="text-sm">Preview expires after five minutes. Changed files or active work require a new preview.</p>+ <label htmlFor="maintenance-confirmation" className="block text-sm">Type <code>{preview.confirmation}</code> to confirm<input id="maintenance-confirmation" value={confirmation} onChange={event => setConfirmation(event.target.value)} disabled={busy} className="mt-2 w-full rounded border border-port-border bg-port-bg px-3 py-2" /></label>+ <div className="flex justify-end gap-3"><button type="button" disabled={busy} onClick={() => { setPreview(null); setConfirmation(''); }}>Cancel</button><button type="button" onClick={remove} disabled={busy || confirmation !== preview.confirmation} className="rounded bg-port-error px-3 py-2 text-white disabled:opacity-50">{busy ? 'Removing…' : 'Permanently delete'}</button></div>+ </div>+ </Modal>}+ </div>;+}diff --git a/client/src/components/settings/DataMaintenanceTab.test.jsx b/client/src/components/settings/DataMaintenanceTab.test.jsxnew file mode 100644index 0000000000..f3e64db16a--- /dev/null+++ b/client/src/components/settings/DataMaintenanceTab.test.jsx@@ -0,0 +1,58 @@+import { beforeEach, expect, it, vi } from 'vitest';+import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';+import { MemoryRouter } from 'react-router';+const api = vi.hoisted(() => vi.fn());+vi.mock('../../services/apiCore', () => ({ request: api }));+import DataMaintenanceTab from './DataMaintenanceTab';+const rows = [{ id: 'retired-training-runs', label: 'Retired artifacts', path: 'data/training-runs', category: 'Unique checkpoints', status: 'retained', warning: 'Unique files', bytes: 17, files: 1, actions: ['delete'] }];+const preview = { token: 'example-token', confirmation: 'DELETE training-runs', path: 'data/training-runs', bytes: 17, files: 1, warning: 'Unique files' };+beforeEach(() => { api.mockReset(); api.mockResolvedValue(rows); });+const mount = () => render(<MemoryRouter><DataMaintenanceTab /></MemoryRouter>);+it('requires preview and exact typed confirmation, supports cancellation, prevents repeated clicks and updates inventory after success', async () => {+ mount(); await screen.findByText('Retired artifacts');+ api.mockResolvedValueOnce(preview);+ fireEvent.click(screen.getByRole('button', { name: 'Preview deletion' }));+ await screen.findByRole('dialog');+ expect(screen.getByRole('button', { name: 'Permanently delete' })).toBeDisabled();+ fireEvent.click(screen.getByRole('button', { name: 'Cancel' }));+ expect(screen.queryByRole('dialog')).toBeNull();+ expect(api).toHaveBeenCalledTimes(2);+ api.mockResolvedValueOnce(preview);+ fireEvent.click(screen.getByRole('button', { name: 'Preview deletion' }));+ await screen.findByRole('dialog');+ fireEvent.change(screen.getByRole('textbox'), { target: { value: 'DELETE training-runs' } });+ let finish;+ api.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; }));+ fireEvent.click(screen.getByRole('button', { name: 'Permanently delete' }));+ fireEvent.click(screen.getByRole('button', { name: 'Removing…' }));+ expect(api).toHaveBeenCalledTimes(4);+ await act(async () => { finish({ operationId: rows[0].id, removedBytes: 17, removedFiles: 1, status: 'deleted', remainingBytes: 0, remainingFiles: 0 }); });+ expect(screen.queryByRole('dialog')).toBeNull();+ expect(screen.getByText('absent')).toBeInTheDocument();+ expect(screen.queryByRole('button', { name: 'Preview deletion' })).toBeNull();+});+it('surfaces stale-preview refusal, preserves the row, and permits explicit refresh', async () => {+ mount(); await screen.findByText('Retired artifacts');+ api.mockResolvedValueOnce(preview);+ fireEvent.click(screen.getByRole('button', { name: 'Preview deletion' }));+ await screen.findByRole('dialog');+ fireEvent.change(screen.getByRole('textbox'), { target: { value: preview.confirmation } });+ api.mockRejectedValueOnce(new Error('Files changed since preview'));+ fireEvent.click(screen.getByRole('button', { name: 'Permanently delete' }));+ expect(await screen.findByRole('alert')).toHaveTextContent('Files changed since preview');+ expect(screen.getByText('retained')).toBeInTheDocument();+ api.mockResolvedValueOnce(rows);+ fireEvent.click(screen.getByRole('button', { name: 'Refresh' }));+ await waitFor(() => expect(screen.queryByRole('alert')).toBeNull());+});+it('shows an unreadable inventory as unavailable instead of measured zero and blocks refresh during initial load', async () => {+ let complete;+ api.mockImplementationOnce(() => new Promise(resolve => { complete = resolve; }));+ mount();+ expect(screen.getByRole('button', { name: 'Refresh' })).toBeDisabled();+ complete([{ ...rows[0], status: 'unavailable', bytes: null, files: null, error: 'Directory cannot be read', actions: [] }]);+ expect(await screen.findByText(/Size unavailable/)).toBeInTheDocument();+ expect(screen.getByRole('alert')).toHaveTextContent('Directory cannot be read');+ expect(screen.queryByText('0 B')).toBeNull();+ expect(screen.queryByRole('button', { name: 'Preview deletion' })).toBeNull();+});diff --git a/client/src/components/settings/ImageGenBackendSections.jsx b/client/src/components/settings/ImageGenBackendSections.jsxindex b39475246c..6d019f2aa8 100644--- a/client/src/components/settings/ImageGenBackendSections.jsx+++ b/client/src/components/settings/ImageGenBackendSections.jsx@@ -205,7 +205,7 @@ export default function ImageGenBackendSections({ mediaTab, form, credentials })<VideoKeyRow key={spec.id} spec={spec} row={credentials.rows[spec.id]} busy={credentials.busy} onSave={credentials.save} onClear={credentials.clear} />))}<div className="space-y-3">- {RENDER_TARGET_OPTIONS.map(({ id, label, video }) => {+ {RENDER_TARGET_OPTIONS.filter(option => !option.retired).map(({ id, label, video }) => {const entry = renderDefaults[id] || {};const pinnedMode = entry.imageMode && entry.imageMode !== RENDER_TARGET_BACKEND_AUTO ? entry.imageMode : '';const pinnedVideoMode = normalizeRenderPinValue(entry.videoMode) || '';diff --git a/client/src/components/settings/ImageGenTab.test.jsx b/client/src/components/settings/ImageGenTab.test.jsxindex 707f9f4282..265475e6a8 100644--- a/client/src/components/settings/ImageGenTab.test.jsx+++ b/client/src/components/settings/ImageGenTab.test.jsx@@ -660,9 +660,9 @@ describe('ImageGenTab — Agy CLI section', () => {it('grok pins never show a model input — its backend has no model knob (#3231)', async () => {getSettings.mockResolvedValue({ imageGen: { mode: 'local' } });await renderTab(['/media/image?mediaTab=defaults']);- const select = await screen.findByLabelText('LoRA training datasets');+ const select = await screen.findByLabelText('Sprite references & anchors');fireEvent.change(select, { target: { value: 'grok' } });- expect(screen.queryByLabelText('LoRA training datasets model')).toBeNull();+ expect(screen.queryByLabelText('Sprite references & anchors model')).toBeNull();});it('names the fixed server-side image model as a read-only fact (#3231)', async () => {@@ -794,3 +794,14 @@ describe('ImageGenTab local runtime card', () => {});vi.mock('../../services/apiRunwayCatalog.js', async (importOriginal) => ({ ...await importOriginal(), loadRunwayCatalog: vi.fn().mockResolvedValue(undefined) }));++it('hides the retired training default and leaves its saved pin untouched while editing a supported surface', async () => {+ getSettings.mockResolvedValue({ imageGen: { mode: 'local' }, renderDefaults: { 'lora-dataset': { imageMode: 'grok' } } });+ await renderTab(['/media/image?mediaTab=defaults']);+ const live = await screen.findByLabelText('Universe Bible & canon renders');+ expect(screen.queryByLabelText('LoRA training datasets')).toBeNull();+ fireEvent.change(live, { target: { value: 'grok' } });+ fireEvent.click(screen.getByRole('button', { name: /^Save$/ }));+ await awaitSave();+ expect(sentSlices()).toEqual({ renderDefaults: { 'universe-bible': { imageMode: 'grok' } } });+});diff --git a/client/src/lib/README.md b/client/src/lib/README.mdindex f434f01217..a000d9a4aa 100644--- a/client/src/lib/README.md+++ b/client/src/lib/README.md@@ -97,7 +97,6 @@ grep -i "what you want to do" client/src/lib/README.md| `imageCleaners.js` | Re-exports `resolveCleanersFromConfig` from the shared pure `server/lib/imageCleanDefaults.js` policy. Reads `{cleanC2PA, denoise}` off a per-mode settings record. `cleanC2PA` defaults are mode-aware (on for `codex` + `external` — the backends that emit C2PA chunks today — off otherwise, as an allow-list rather than a deny-list); `denoise` defaults off everywhere (lossy, opt-in only). || `reviewerModels.js` | `reviewerModelsFromDefaults` / `reviewerModelsToDefaults` / `reviewerEffortsFromDefaults` / `reviewerEffortsToDefaults` — adapt the Code Review Defaults' per-reviewer SCALARS (`codexModel`, `ollamaModel`, `claudeEffort`, …) to and from the token-keyed maps `ReviewerPicker`'s Model and Effort columns take. Mirrors `reviewerModelsFromDefaults` / `reviewerEffortsFromDefaults` in `server/lib/reviewerConfig.js`. Also preserves configured-provider model and effort pins in the additive `providerModels` / `providerEfforts` maps, keyed by `provider:<id>`; empty maps clear overrides without changing builtin scalars. || `reviewerPins.js` | Client mirror of the whole reviewer vocabulary — the roster (`REVIEWER_VALUES`), `REVIEWER_ALIASES`, `DEFAULT_REVIEWER(S)`, `normalizeReviewers`, the review-username pattern/cap (`MAX_REVIEW_USERNAMES` / `cleanReviewUsername` / `normalizeReviewUsernames`), `MAX_REVIEWER_MAX_ROUNDS`, `REVIEW_STOP_MODES` / `DEFAULT_REVIEW_STOP_MODE`, plus the per-reviewer pin vocabularies (`MODEL_SELECTABLE_REVIEWERS` / `EFFORT_SELECTABLE_REVIEWERS` / `REVIEWER_EFFORT_LEVELS` / `reviewerEffortLevels` / `normalizeReviewerSlug` / `sanitizeReviewerModelInput`) and their bounds. Dependency-free on purpose: the server suite imports it to pin the mirror against `server/lib/reviewerConfig.js`, so it must not reach for a client-only package. Re-exported by `components/cos/constants.js`. `isProviderReviewer` recognizes the shared `provider:<id>` identity alongside legacy harness slugs. |-| `loraDatasetReadiness.js` | Re-export of `server/lib/loraDataset.js` (#8297) — `MIN_TRAINING_IMAGES` / `RECOMMENDED_TRAINING_IMAGES` / `TRAINING_IMAGE_SWEET_SPOT_MAX`, `captionHasTriggerWord`, `datasetQualityTier`, `computeDatasetReadiness`, `analyzeCaptionInvariants` (+ its `INVARIANT_SHARE_THRESHOLD` / `MIN_CAPTIONS_FOR_INVARIANT_ANALYSIS`). `LoraDatasetDetail.jsx`'s live advisory (readiness summary, Train gate, shared-caption-fragment warning) calls these directly instead of a page-local mirror, so it cannot drift from the server's authoritative training gate. || `loraEffect.js` | How the LoRA manager card renders an adapter-effect measurement (#4872). `LORA_EFFECT_STATUSES` is re-exported from `server/lib/loraEffect.js`; `LORA_EFFECT_BADGES` + `loraEffectBadge(status)` (only `zero` is styled as an error, because only `zero` refuses a render server-side) and `loraEffectDetail(report)` are client-only — the detail line returns `null` when the badge already says everything, where the server's `formatLoraEffect` always returns a sentence for its log. || `loraTriggers.js` | The picker's `+ trigger` append (#4665). `firstTriggerWord` / `promptHasTriggerWord` / `separatorFor` are re-exported from `server/lib/loraTriggers.js`, so the hint reads the rule the render enforces. `appendTriggerWords(prompt, words, effectivePrompt?)` is client-only: it adds ALL of a LoRA's words (the server only ever weaves the first) and judges presence against the composed prompt the page will actually submit. || `runnerFamilies.js` | Mirror of `server/lib/runners.js`, including the shared LTX + MiniMax H3 Ref2VA audio-to-video runtime predicate; `loraCompatKeysMatch` keeps image Test handoffs and the adapter picker on the same size/family compatibility rule. |diff --git a/client/src/lib/imageGenModes.js b/client/src/lib/imageGenModes.jsindex 23bdb2d681..ca94496840 100644--- a/client/src/lib/imageGenModes.js+++ b/client/src/lib/imageGenModes.js@@ -159,7 +159,7 @@ export const RENDER_TARGET_OPTIONS = Object.freeze([{ id: RENDER_TARGET.SPRITE_REFERENCE, label: 'Sprite references & anchors' },{ id: RENDER_TARGET.PIPELINE_VISUAL, label: 'Pipeline visuals (storyboards, comics, covers)' },{ id: RENDER_TARGET.MUSIC_VIDEO, label: 'Music Video scene frames & clips', video: true },- { id: RENDER_TARGET.LORA_DATASET, label: 'LoRA training datasets' },+ { id: RENDER_TARGET.LORA_DATASET, label: 'LoRA training datasets', retired: true },{ id: RENDER_TARGET.CREATIVE_AGENT, label: 'Creative agent renders', video: true },{ id: RENDER_TARGET.DECK, label: 'Deck card renders (playing cards & tarot)' },]);diff --git a/client/src/lib/index.js b/client/src/lib/index.jsindex 35f4892235..2c2d2fa42c 100644--- a/client/src/lib/index.js+++ b/client/src/lib/index.js@@ -9,7 +9,6 @@ export * from './canonPrompt.js';export * from './cleanPlatePrompt.js';export * from './composeStyledPrompt.js';export * from './decks.js';-export * from './loraDatasetReadiness.js';export * from './personaTraitBlend.js';export * from './scenePrompt.js';export * from './seasonStructure.js';diff --git a/client/src/lib/loraDatasetReadiness.js b/client/src/lib/loraDatasetReadiness.jsdeleted file mode 100644index 93c73224fc..0000000000--- a/client/src/lib/loraDatasetReadiness.js+++ /dev/null@@ -1,26 +0,0 @@-/**- * LoRA dataset training-readiness thresholds, trigger-word matching, and- * caption-invariant analysis.- *- * Re-export of `server/lib/loraDataset.js` — the one definition of these- * rules, imported rather than copied so the page-local advisory (which- * updates the instant a caption is edited or an image deleted, without a- * server round-trip — see `client/src/pages/LoraDatasetDetail.jsx`) cannot- * drift from `computeDatasetReadiness` / `validateDatasetReady`, the- * authoritative gate the server applies at train time (#8297). The server- * module is dependency-free (`isPlainObject`, `escapeRegExp` — no Node- * builtins), so it is safe to bundle into the browser build; see- * `scripts/client-server-import-purity.test.js`.- */-export {- MIN_TRAINING_IMAGES,- RECOMMENDED_TRAINING_IMAGES,- TRAINING_IMAGE_SWEET_SPOT_MAX,- INVARIANT_SHARE_THRESHOLD,- MIN_CAPTIONS_FOR_INVARIANT_ANALYSIS,- isValidTriggerWord,- captionHasTriggerWord,- datasetQualityTier,- computeDatasetReadiness,- analyzeCaptionInvariants,-} from '../../../server/lib/loraDataset.js';diff --git a/client/src/lib/loraDatasetReadiness.test.js b/client/src/lib/loraDatasetReadiness.test.jsdeleted file mode 100644index 78c7a931a4..0000000000--- a/client/src/lib/loraDatasetReadiness.test.js+++ /dev/null@@ -1,107 +0,0 @@-// @vitest-environment node-import { describe, it, expect } from 'vitest';-import * as clientReadiness from './loraDatasetReadiness.js';-import * as serverReadiness from '../../../server/lib/loraDataset.js';--// Cross-runtime contract (#8297): `LoraDatasetDetail.jsx`'s live training-// advisory (readiness summary, Train gate, shared-caption-fragment warning)-// calls these helpers directly instead of a page-local mirror, so a page-// caption/trigger-word edit can never show "Ready to train" while the-// server's `validateDatasetReady` gate (which calls the same-// `computeDatasetReadiness`/`analyzeCaptionInvariants`) would reject the run.-// Two layers of protection:-// 1. Reference identity — the client module is a straight re-export, so a-// future edit that quietly swaps it for a duplicated implementation-// (reintroducing the drift issue #8297 fixed) fails here immediately.-// 2. Behavior on the shared fixture matrix (valid / boundary / non--// trainable), so the contract is pinned even if the re-export shape-// changes later.-const EXPORT_NAMES = [- 'MIN_TRAINING_IMAGES',- 'RECOMMENDED_TRAINING_IMAGES',- 'TRAINING_IMAGE_SWEET_SPOT_MAX',- 'INVARIANT_SHARE_THRESHOLD',- 'MIN_CAPTIONS_FOR_INVARIANT_ANALYSIS',- 'isValidTriggerWord',- 'captionHasTriggerWord',- 'datasetQualityTier',- 'computeDatasetReadiness',- 'analyzeCaptionInvariants',-];--const readyImage = (caption) => ({ status: 'ready', caption });-const renderingImage = () => ({ status: 'rendering', caption: '' });--// 10 ready, captioned images naming the trigger — exactly MIN_TRAINING_IMAGES.-const boundaryImages = Array.from(- { length: 10 },- (_, i) => readyImage(`subj_x, pose ${i}, studio lighting`),-);--// Same images as boundaryImages, plus a few more so 4 of the captions repeat-// an identical descriptive fragment ("white hair") — enough signal for-// analyzeCaptionInvariants to flag it as a shared identity fragment.-const invariantImages = [- readyImage('subj_x, white hair, pose a'),- readyImage('subj_x, white hair, pose b'),- readyImage('subj_x, white hair, pose c'),- readyImage('subj_x, white hair, pose d'),- readyImage('subj_x, pose e'),-];--describe('loraDatasetReadiness (client re-export of server/lib/loraDataset.js)', () => {- it('re-exports the exact same functions/constants the server owns — no page-local copy', () => {- for (const name of EXPORT_NAMES) {- expect(clientReadiness[name]).toBe(serverReadiness[name]);- }- });-- it('agrees with the server on a trainable dataset at the sweet spot (valid)', () => {- const dataset = {- triggerWord: 'subj_x',- images: Array.from({ length: 22 }, (_, i) => readyImage(`subj_x, pose ${i}`)),- };- const clientResult = clientReadiness.computeDatasetReadiness(dataset);- const serverResult = serverReadiness.computeDatasetReadiness(dataset);- expect(clientResult).toEqual(serverResult);- expect(clientResult).toMatchObject({ trainable: true, quality: 'good', captioned: 22 });- });-- it('agrees with the server exactly at the MIN_TRAINING_IMAGES boundary', () => {- const dataset = { triggerWord: 'subj_x', images: boundaryImages };- const clientResult = clientReadiness.computeDatasetReadiness(dataset);- const serverResult = serverReadiness.computeDatasetReadiness(dataset);- expect(clientResult).toEqual(serverResult);- expect(clientResult).toMatchObject({- trainable: true,- quality: 'minimum',- captioned: clientReadiness.MIN_TRAINING_IMAGES,- });- });-- it('agrees with the server on a non-trainable dataset (missing trigger word)', () => {- const dataset = { triggerWord: '', images: [...boundaryImages, renderingImage()] };- const clientResult = clientReadiness.computeDatasetReadiness(dataset);- const serverResult = serverReadiness.computeDatasetReadiness(dataset);- expect(clientResult).toEqual(serverResult);- // No trigger word configured: never trainable, quality never reports- // 'good'/'minimum' even though 10 images are otherwise ready+captioned.- expect(clientResult).toMatchObject({ trainable: false, quality: 'insufficient', rendering: 1 });- });-- it('agrees with the server on caption-invariant fragment detection', () => {- const clientResult = clientReadiness.analyzeCaptionInvariants(invariantImages, 'subj_x');- const serverResult = serverReadiness.analyzeCaptionInvariants(invariantImages, 'subj_x');- expect(clientResult).toEqual(serverResult);- expect(clientResult.analyzable).toBe(true);- expect(clientResult.sharedFragments.map((f) => f.normalized)).toContain('white hair');- });-- it('agrees with the server on trigger-word token-boundary matching', () => {- // "train" contains "ai" as a substring but not as a bounded token.- expect(clientReadiness.captionHasTriggerWord('a subject in training gear', 'ai'))- .toBe(serverReadiness.captionHasTriggerWord('a subject in training gear', 'ai'));- expect(clientReadiness.captionHasTriggerWord('subj_x, wearing ai badge', 'ai'))- .toBe(serverReadiness.captionHasTriggerWord('subj_x, wearing ai badge', 'ai'));- });-});diff --git a/client/src/lib/navPresentation.js b/client/src/lib/navPresentation.jsindex a8003f5da8..68b4a8b889 100644--- a/client/src/lib/navPresentation.js+++ b/client/src/lib/navPresentation.js@@ -1,5 +1,6 @@import { AudioLines,Wrench,+ Trash2,Home,Package,FileText,@@ -253,7 +254,6 @@ export const NAV_PRESENTATION = {'/local-llm/playground': { icon: FlaskConical },'/models/status': { icon: Activity },'/models/subscriptions': { icon: CreditCard },- '/models/training': { icon: GraduationCap },'/settings/ai-assignments': { icon: Bot },'/settings/api-access': { icon: Globe },'/api-reference/catalog': { icon: Braces },@@ -263,6 +263,7 @@ export const NAV_PRESENTATION = {'/models/code-reviewers': { icon: ShieldCheck },'/settings/database': { icon: Database },'/settings/features': { icon: ListChecks },+ '/settings/data-maintenance': { icon: Trash2 },'/settings/general': { icon: Settings },'/settings/mortalloom': { icon: Activity },'/openclaw': { icon: MessagesSquare },diff --git a/client/src/pages/LoraDatasetDetail.jsx b/client/src/pages/LoraDatasetDetail.jsxdeleted file mode 100644index ba2d18ec43..0000000000--- a/client/src/pages/LoraDatasetDetail.jsx+++ /dev/null@@ -1,703 +0,0 @@-import { listSheetPointers } from '../lib/sheetPointers.js';-/**- * LoRA dataset workbench (/models/training/:recordId).- *- * Rendered by `pages/Models.jsx` inside the Models section shell (its TAB_DETAIL- * map), not as a bare route — so it keeps the section header and tab bar the way- * it used to under the Media Gen shell.- *- * Build reference material for one universe bible subject (generate via the image- * queue, upload, slice the reference-sheet turnaround), caption it with- * the vision model, then launch a training run. The dataset record is the- * single source of truth; rendering images poll-refresh until they land.- */--import { useState, useEffect, useMemo, useCallback, useRef } from 'react';-import { useParams, Link } from 'react-router';-import {- ArrowLeft, Loader2, Upload, Wand2, Scissors, Tags, RefreshCw, AlertTriangle, Images, Replace, Lightbulb, Eraser, XCircle,-} from 'lucide-react';-import toast from '../components/ui/Toast';-import FilePickerButton from '../components/ui/FilePickerButton';-import { IMAGE_ACCEPT } from '../utils/fileUpload';-import Modal from '../components/ui/Modal';-import { useSocketResource } from '../hooks/useSocketResource.js';-import useSseJobSlot from '../hooks/useSseJobSlot';-import DatasetImageGrid from '../components/loraTraining/DatasetImageGrid';-import GenerateBatchDialog from '../components/loraTraining/GenerateBatchDialog';-import TrainingPanel from '../components/loraTraining/TrainingPanel';-import CaptionModelPicker from '../components/loraTraining/CaptionModelPicker';-import ImportGalleryDialog from '../components/loraTraining/ImportGalleryDialog';-import UniverseCharacterPicker from '../components/loraTraining/UniverseCharacterPicker';-import {- MIN_TRAINING_IMAGES,- RECOMMENDED_TRAINING_IMAGES,- TRAINING_IMAGE_SWEET_SPOT_MAX,- isValidTriggerWord,- computeDatasetReadiness,- analyzeCaptionInvariants,-} from '../lib/loraDatasetReadiness.js';-import {- getLoraDataset,- getLoraDatasetVariationAxes,- patchLoraDataset,- uploadLoraDatasetImages,- sliceLoraDatasetRefSheet,- startLoraCaptionRun,- cancelLoraCaptionRun,- stripLoraDatasetSharedCaptionFragments,- getUniverse,-} from '../services/api';--const DATASET_EVENTS = ['training:dataset:changed'];--const SUBJECT_TYPE_LABEL = { characters: 'Character', objects: 'Object', places: 'Place' };-const subjectKind = (dataset) => dataset?.character?.entryKind || 'characters';--function SliceDialog({ dataset, onClose, onSliced }) {- const [cols, setCols] = useState(3);- const [rows, setRows] = useState(2);- // Vision-auto detection is the default; the grid inputs are the fallback the- // user reaches for when no vision model is installed or auto-detect mis-cuts.- const [useVision, setUseVision] = useState(true);- const [submitting, setSubmitting] = useState(false);-- const submit = async () => {- setSubmitting(true);- try {- const result = await sliceLoraDatasetRefSheet(dataset.id, { cols, rows, useVision });- const via = result.method === 'vision' ? 'auto-detected' : 'grid';- toast.success(`Added ${result.images.length} ${via} crops from the reference sheet — prune any bad ones`);- onSliced(result);- } finally {- setSubmitting(false);- }- };-- const numberInput = (id, label, value, set) => (- <div>- <label htmlFor={id} className="block text-xs text-gray-400 mb-1">{label}</label>- <input- id={id}- type="number"- min={1}- max={6}- value={value}- onChange={(e) => set(Math.max(1, Math.min(6, Number(e.target.value) || 1)))}- className="w-full bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm text-white"- />- </div>- );-- return (- <Modal- open- onClose={onClose}- size="sm"- ariaLabelledBy="lt-slice-title"- panelClassName="bg-port-card border border-port-border rounded-lg p-5"- >- <div className="space-y-4">- <h2 id="lt-slice-title" className="text-base font-semibold text-white">Slice reference sheet</h2>- <p className="text-sm text-gray-400">- Cuts the character's reference-sheet turnaround into individual training crops.- Sheet layouts vary, so expect to delete cells that caught labels or palette swatches.- </p>- <label htmlFor="lt-slice-vision" className="flex items-start gap-2 text-sm text-gray-300">- <input- id="lt-slice-vision"- type="checkbox"- checked={useVision}- onChange={(e) => setUseVision(e.target.checked)}- className="mt-0.5"- />- <span>- Auto-detect figures with a vision model- <span className="block text-xs text-gray-500">- Proposes a crop per figure instead of a rigid grid. Falls back to the grid below if no- vision model is installed or detection finds nothing.- </span>- </span>- </label>- <div className={`grid grid-cols-2 gap-3 ${useVision ? 'opacity-50' : ''}`}>- {numberInput('lt-slice-cols', 'Columns', cols, setCols)}- {numberInput('lt-slice-rows', 'Rows', rows, setRows)}- </div>- <div className="flex justify-end gap-2">- <button type="button" onClick={onClose} className="px-3 py-2 text-sm text-gray-400 hover:text-white">Cancel</button>- <button- type="button"- onClick={submit}- disabled={submitting}- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"- >- {submitting ? <Loader2 className="w-4 h-4 animate-spin" /> : <Scissors className="w-4 h-4" />}- {useVision ? 'Auto-slice' : `Slice ${cols}×${rows}`}- </button>- </div>- </div>- </Modal>- );-}--function ReassignDialog({ dataset, onClose, onReassigned }) {- // Default to the current assignment so the picker opens on the dataset's- // own universe (subjects pre-load) and the user only changes what they want.- const [universeId, setUniverseId] = useState(dataset.character.universeId);- const [entryKind, setEntryKind] = useState(subjectKind(dataset));- const [entryId, setEntryId] = useState(dataset.character.entryId);- const [submitting, setSubmitting] = useState(false);-- const submit = async () => {- setSubmitting(true);- try {- const next = await patchLoraDataset(dataset.id, { universeId, entryKind, entryId });- toast.success(`Reassigned to ${next.character.name}`);- onReassigned(next);- } finally {- setSubmitting(false);- }- };-- return (- <Modal- open- onClose={onClose}- size="sm"- ariaLabelledBy="lt-reassign-title"- panelClassName="bg-port-card border border-port-border rounded-lg p-5"- >- <div className="space-y-4">- <h2 id="lt-reassign-title" className="text-base font-semibold text-white">Reassign subject</h2>- <p className="text-sm text-gray-400">- Move this dataset's images and trigger word to a different universe bible subject.- The trigger word stays the same — edit it separately if the new subject needs its own token.- </p>- <UniverseCharacterPicker- idPrefix="lt-reassign"- universeId={universeId}- entryKind={entryKind}- entryId={entryId}- onUniverseChange={(id) => { setUniverseId(id); setEntryId(''); }}- onEntryKindChange={(kind) => { setEntryKind(kind); setEntryId(''); }}- onEntryChange={setEntryId}- />- <div className="flex justify-end gap-2">- <button type="button" onClick={onClose} className="px-3 py-2 text-sm text-gray-400 hover:text-white">Cancel</button>- <button- type="button"- onClick={submit}- disabled={!universeId || !entryId || submitting}- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"- >- {submitting ? <Loader2 className="w-4 h-4 animate-spin" /> : <Replace className="w-4 h-4" />}- Reassign- </button>- </div>- </div>- </Modal>- );-}--export default function LoraDatasetDetail({ recordId }) {- // Models renders this inside the section shell and passes the id from- // `/models/training/:recordId`; `:datasetId` is the fallback for a direct mount.- const params = useParams();- const datasetId = recordId ?? params.datasetId;- const { data: dataset, updateData: setDataset, error, refetch: refresh } = useSocketResource(- () => getLoraDataset(datasetId),- { events: DATASET_EVENTS, resourceKey: datasetId, matchesEvent: payload => payload?.datasetId === datasetId },- );- const loadError = error?.message;- const [subject, setSubject] = useState(null);- const [variationAxes, setVariationAxes] = useState(null);- const [triggerDraft, setTriggerDraft] = useState(null);- const [triggerSaving, setTriggerSaving] = useState(false);- const [uploading, setUploading] = useState(false);- const [showGenerate, setShowGenerate] = useState(false);- const [showSlice, setShowSlice] = useState(false);- const [showImport, setShowImport] = useState(false);- const [showReassign, setShowReassign] = useState(false);- // Chosen caption model { providerId, model } — null fields mean "let the- // server auto-pick a vision model". Lifted from CaptionModelPicker so caption- // runs pass the explicit selection.- const [captionModel, setCaptionModel] = useState({ providerId: null, model: null });- const [strippingFragments, setStrippingFragments] = useState(false);- // Bumped after a bulk caption rewrite (strip) so the image grid drops any- // unsaved caption drafts the rewrite superseded — otherwise a stale draft- // blur-saves the old text back, undoing the strip.- const [captionDraftResetToken, setCaptionDraftResetToken] = useState(0);-- // Pull the live canon subject once for variation-axis options + sheet link.- useEffect(() => {- if (!dataset?.character?.universeId) return undefined;- let cancelled = false;- getUniverse(dataset.character.universeId, { silent: true })- .then((u) => {- if (cancelled) return;- const entries = Array.isArray(u?.[subjectKind(dataset)]) ? u[subjectKind(dataset)] : [];- setSubject(entries.find((entry) => entry.id === dataset.character.entryId) || null);- })- .catch(() => {});- return () => { cancelled = true; };- }, [dataset?.character?.universeId, dataset?.character?.entryKind, dataset?.character?.entryId]);-- // Object/place variation axes (Lighting/Settings) live as server-side- // constants in deriveVariationAxes — fetch them so the generate-batch- // override chips render without mirroring the vocab client-side. Characters- // seed their chips from live canon (expressions/wardrobes) instead.- useEffect(() => {- if (!datasetId || subjectKind(dataset) === 'characters') {- setVariationAxes(null);- return undefined;- }- // Clear first so a direct object↔place switch can't seed the generate- // dialog from the previous kind's axes while the new fetch is in flight,- // and cancel-on-cleanup so a late response can't overwrite the current one.- let cancelled = false;- setVariationAxes(null);- getLoraDatasetVariationAxes(datasetId, { silent: true })- .then((axes) => { if (!cancelled) setVariationAxes(axes); })- .catch(() => { if (!cancelled) setVariationAxes(null); });- return () => { cancelled = true; };- }, [datasetId, dataset?.character?.entryKind]);-- // Readiness is derived from the live local images (not the server's snapshot- // on `dataset.readiness`) so manual caption edits / deletes reflect in the- // counts + Train gate immediately, without a refetch per keystroke.- const readiness = useMemo(- () => computeDatasetReadiness(dataset),- [dataset?.images, dataset?.triggerWord],- );-- // Caption-lint advisory: identity fragments repeated across most captions- // (issue #1320). Derived live so it tracks manual caption edits without a- // refetch; the "Strip" action recomputes server-side.- const captionInvariants = useMemo(- () => analyzeCaptionInvariants(dataset?.images, dataset?.triggerWord),- [dataset?.images, dataset?.triggerWord],- );-- const stripSharedFragments = async () => {- setStrippingFragments(true);- try {- const { dataset: next, removedFragments, updatedImages } = await stripLoraDatasetSharedCaptionFragments(datasetId);- setDataset(next);- setCaptionDraftResetToken((n) => n + 1);- if (removedFragments.length) {- toast.success(`Stripped ${removedFragments.length} shared identity fragment${removedFragments.length === 1 ? '' : 's'} from ${updatedImages} caption${updatedImages === 1 ? '' : 's'}`);- } else {- toast.success('No shared identity fragments to strip');- }- } finally {- setStrippingFragments(false);- }- };-- const renderingCount = readiness.rendering;-- // Caption run — one slot, driven by the SERVER's run (not the kickoff reply):- // the dataset GET reports the in-flight run (`dataset.captionRun`), so a reload,- // a second tab or a lost kickoff acknowledgement re-attaches to the same batch- // instead of showing idle controls. The server emits per-image `error` progress- // and a terminal frame carrying the failure tally + the last per-image reason- // (refusal vs. an exhausted reasoning budget); surface it so a failure isn't silent.- const toastCaptionFailure = (frame) => {- const detail = frame.lastError ? ` ${frame.lastError}` : '';- if (frame.type === 'error') {- toast.error(frame.lastError || frame.message || frame.error || 'Captioning failed');- return;- }- const noun = frame.failed === 1 ? 'image' : 'images';- toast.error(frame.done > 0- ? `Captioned ${frame.done}/${frame.total} — ${frame.failed} ${noun} failed.${detail || ' Re-caption individually or pick another vision model.'}`- : `All ${frame.failed} ${noun} failed to caption.${detail || ' The vision model refused or returned nothing — try another vision model.'}`);- };- // Run ids already adopted from a dataset snapshot — a snapshot read before the- // terminal refetch lands must not re-attach to the run that just settled.- const adoptedCaptionRuns = useRef(new Set());- const captionSlot = useSseJobSlot({- startRequest: async (options) => {- const run = await startLoraCaptionRun(datasetId, options);- if (run.conflict) toast.info('A different caption run is already active on this dataset — showing its progress');- return { jobId: run.runId };- },- eventsUrl: (runId) => `/api/lora-datasets/${datasetId}/caption-runs/${runId}/events`,- cancelRequest: (runId, opts) => cancelLoraCaptionRun(datasetId, runId, opts),- onKickoffSuccess: (runId) => adoptedCaptionRuns.current.add(runId),- onSettled: () => refresh(),- onComplete: (frame) => {- if (frame.failed > 0) toastCaptionFailure(frame);- else if (frame.done > 0) toast.success(`Captioned ${frame.done} image${frame.done === 1 ? '' : 's'}`);- },- onErrorFrame: (frame) => { toastCaptionFailure(frame); return true; },- canceledMessage: 'Captioning canceled — captions already written are kept',- lostConnectionMessage: 'Lost connection to the caption run — reload to re-attach',- startErrorFallback: 'Failed to start captioning',- });- const activeCaptionRun = dataset?.captionRun;- useEffect(() => {- const runId = activeCaptionRun?.runId;- if (!runId || adoptedCaptionRuns.current.has(runId)) return;- if (captionSlot.attach(runId, null)) adoptedCaptionRuns.current.add(runId);- }, [activeCaptionRun?.runId, captionSlot.active]);- const captionBusy = captionSlot.active;- const captionProgress = captionSlot.latest ?? activeCaptionRun;- const canceling = captionProgress?.status === 'canceling';-- const onImagesChange = (mutate) => setDataset((prev) => (prev ? { ...prev, images: mutate(prev.images) } : prev));-- const saveTriggerWord = async () => {- if (triggerDraft === null || triggerDraft === dataset.triggerWord) { setTriggerDraft(null); return; }- if (!isValidTriggerWord(triggerDraft)) {- toast.error('Trigger word must be 2-64 chars of a-z, 0-9, _');- return;- }- setTriggerSaving(true);- try {- const next = await patchLoraDataset(datasetId, { triggerWord: triggerDraft });- setDataset(next);- setTriggerDraft(null);- toast.success('Trigger word saved — re-caption images so captions carry the new token');- } finally {- setTriggerSaving(false);- }- };-- const onUpload = async (e) => {- const files = e.target.files;- if (!files?.length) return;- setUploading(true);- try {- const { images } = await uploadLoraDatasetImages(datasetId, files);- onImagesChange((prev) => [...prev, ...images]);- refresh();- toast.success(`Added ${images.length} image${images.length === 1 ? '' : 's'}`);- } finally {- setUploading(false);- }- };-- // Stable so CaptionModelPicker's run-once effect doesn't re-fire.- const onCaptionModelChange = useCallback((sel) => setCaptionModel(sel), []);-- // Strip null provider/model so an "Auto" selection sends an empty body and- // the server resolves a vision model itself.- const captionOptions = (extra = {}) => ({- ...extra,- ...(captionModel.providerId ? { providerId: captionModel.providerId } : {}),- ...(captionModel.model ? { model: captionModel.model } : {}),- });-- // Caption only images that have no caption yet (overwrite: false). Re-caption- // all overwrites every ready image's caption with the picked model — the way- // to re-run the whole dataset through a newer/better VLM.- const startCaption = (overwrite) => captionSlot.start(captionOptions({ overwrite }));- const recaptionImage = (img) => captionSlot.start(captionOptions({ imageIds: [img.id], overwrite: true }));- const captionAll = () => startCaption(false);- const recaptionAll = () => startCaption(true);-- const expressionOptions = useMemo(- () => (subjectKind(dataset) === 'characters'- ? (subject?.expressions || []).map((e) => e?.name).filter(Boolean)- : (variationAxes?.expressions || [])),- [dataset, subject, variationAxes],- );- const outfitOptions = useMemo(- () => (subjectKind(dataset) === 'characters'- ? (subject?.wardrobes || []).map((w) => w?.name).filter(Boolean)- : (variationAxes?.outfits || [])),- [dataset, subject, variationAxes],- );- const hasReferenceSheet = listSheetPointers(subject).length > 0;-- if (loadError) {- return (- <div className="bg-port-card border border-port-border rounded-lg p-6 text-center text-gray-400">- <AlertTriangle className="w-6 h-6 mx-auto mb-2 text-port-error" />- {loadError} — <Link to="/models/training" className="text-port-accent hover:underline">back to datasets</Link>- </div>- );- }- if (!dataset) {- return <div className="text-gray-400 flex items-center gap-2"><Loader2 className="w-4 h-4 animate-spin" /> Loading dataset…</div>;- }-- return (- <div className="@container/page min-w-0 space-y-4">- <div className="flex items-start justify-between gap-3 flex-wrap">- <div className="min-w-0">- <Link to="/models/training" className="text-xs text-gray-400 hover:text-white flex items-center gap-1 mb-1">- <ArrowLeft className="w-3 h-3" /> Datasets- </Link>- <div className="flex items-center gap-2 min-w-0">- <h2 className="text-lg font-semibold text-white truncate">- <Link- to={`/universes/${dataset.character.universeId}`}- className="hover:text-port-accent"- title="Open in universe editor"- >- {dataset.character.name}- </Link>- </h2>- <button- type="button"- onClick={() => setShowReassign(true)}- disabled={dataset.status === 'training'}- title={dataset.status === 'training'- ? 'Cancel the in-progress training run before reassigning'- : 'Reassign this dataset to a different universe subject'}- className="shrink-0 text-xs text-gray-500 hover:text-port-accent flex items-center gap-1 disabled:opacity-40 disabled:hover:text-gray-500"- >- <Replace className="w-3.5 h-3.5" /> Reassign- </button>- </div>- <div className="flex items-center gap-2 mt-1">- <label htmlFor="lt-trigger" className="text-xs text-gray-500">Trigger word</label>- <input- id="lt-trigger"- value={triggerDraft ?? dataset.triggerWord}- onChange={(e) => setTriggerDraft(e.target.value)}- onBlur={saveTriggerWord}- disabled={triggerSaving}- className="bg-port-bg border border-port-border rounded px-2 py-1 text-xs font-mono text-white w-56 disabled:opacity-50"- />- {triggerSaving && <Loader2 className="w-3 h-3 animate-spin text-gray-400" />}- </div>- <div className="text-xs text-gray-500 mt-1">- {SUBJECT_TYPE_LABEL[subjectKind(dataset)] || 'Subject'} from the universe bible- </div>- </div>- <div className="text-right text-sm">- <div className={readiness.quality === 'good' ? 'text-port-success' : readiness.quality === 'minimum' ? 'text-port-warning' : 'text-gray-400'}>- {readiness.ready ?? 0} images · {readiness.captioned ?? 0}/{readiness.required ?? 10} captioned- {renderingCount ? ` · ${renderingCount} rendering` : ''}- </div>- <div className="text-xs text-gray-500">- {readiness.quality === 'good'- ? 'Ready to train — strong dataset'- : readiness.quality === 'minimum'- ? `Trainable now · ${Math.max(0, (readiness.recommended ?? 20) - (readiness.captioned ?? 0))} more for best quality`- : (readiness.captioned ?? 0) >= (readiness.required ?? 10)- ? 'Set a trigger word to train'- : `Add + caption ${(readiness.required ?? 10) - (readiness.captioned ?? 0)} more to train`}- </div>- </div>- </div>-- <div className="flex flex-wrap gap-2">- <button- type="button"- onClick={() => setShowGenerate(true)}- className="px-3 py-2 text-sm rounded bg-port-accent/20 text-port-accent hover:bg-port-accent/30 flex items-center gap-2"- >- <Wand2 className="w-4 h-4" /> Generate- </button>- <FilePickerButton- accept={IMAGE_ACCEPT}- multiple- onChange={onUpload}- disabled={uploading}- ariaLabel="Upload dataset images"- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2"- >- {uploading ? <Loader2 className="w-4 h-4 animate-spin" /> : <Upload className="w-4 h-4" />} Upload- </FilePickerButton>- <button- type="button"- onClick={() => setShowImport(true)}- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2"- >- <Images className="w-4 h-4" /> From gallery- </button>- <button- type="button"- onClick={() => setShowSlice(true)}- disabled={!hasReferenceSheet}- title={hasReferenceSheet ? '' : 'Render a reference sheet in the universe editor first'}- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2 disabled:opacity-50"- >- <Scissors className="w-4 h-4" /> Slice sheet- </button>- <button- type="button"- onClick={captionAll}- disabled={captionBusy || !readiness.ready}- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2 disabled:opacity-50"- >- {captionBusy ? <Loader2 className="w-4 h-4 animate-spin" /> : <Tags className="w-4 h-4" />}- {captionBusy- ? `Captioning ${captionProgress?.done ?? 0}/${captionProgress?.total ?? '…'}`- : 'Caption all'}- </button>- <button- type="button"- onClick={recaptionAll}- disabled={captionBusy || !readiness.ready}- title="Re-caption every ready image with the selected model — overwrites all existing captions, including manual edits"- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2 disabled:opacity-50"- >- <RefreshCw className="w-4 h-4" /> Re-caption all- </button>- <CaptionModelPicker onChange={onCaptionModelChange} />- </div>-- {captionBusy && (- <div- role="status"- className="bg-port-card border border-port-border rounded-lg p-3 flex flex-wrap items-center gap-3 text-sm"- >- <Loader2 className="w-4 h-4 animate-spin text-port-accent shrink-0" />- <div className="min-w-0 flex-1">- <div className="text-white">- {canceling- ? 'Canceling… finishing the vision call already in flight (its result is discarded)'- : `Captioning ${captionProgress?.done ?? 0}/${captionProgress?.total ?? '…'}${captionProgress?.failed ? ` · ${captionProgress.failed} failed` : ''}`}- </div>- {activeCaptionRun?.model && (- <div className="text-xs text-gray-400 truncate">{activeCaptionRun.provider} · {activeCaptionRun.model}</div>- )}- </div>- <button- type="button"- onClick={captionSlot.cancel}- disabled={!captionSlot.jobId || canceling}- className="px-3 py-2 text-sm rounded bg-port-card border border-port-border text-gray-300 hover:text-white flex items-center gap-2 disabled:opacity-50 min-h-[40px]"- >- <XCircle className="w-4 h-4" /> Cancel- </button>- </div>- )}-- {captionInvariants.sharedFragments.length > 0 && (- <div className="bg-port-warning/10 border border-port-warning/40 rounded-lg p-3 text-sm">- <div className="flex items-start gap-2">- <AlertTriangle className="w-4 h-4 text-port-warning shrink-0 mt-0.5" />- <div className="min-w-0 flex-1 space-y-2">- <p className="text-gray-200">- <span className="font-medium text-port-warning">Identity is leaking into the captions.</span>- {' '}- These fragments repeat across most captions, so the model binds the look to the phrases instead of the- {' '}- <span className="font-mono text-gray-300">{dataset.triggerWord || 'trigger'}</span>- {' '}- token — the trigger then renders a generic subject. Keep only what changes shot-to-shot (pose, framing,- setting) and let the trigger absorb the fixed identity.- </p>- <div className="flex flex-wrap gap-1.5">- {captionInvariants.sharedFragments.map((f) => (- <span- key={f.normalized}- className="text-xs px-1.5 py-0.5 rounded bg-port-bg border border-port-border text-gray-300 font-mono"- title={`in ${f.count} of ${captionInvariants.total} captions`}- >- {f.fragment}- <span className="text-gray-500"> ·{f.count}/{captionInvariants.total}</span>- </span>- ))}- </div>- <button- type="button"- onClick={stripSharedFragments}- disabled={strippingFragments || captionBusy}- className="px-3 py-1.5 text-xs rounded bg-port-warning/20 text-port-warning hover:bg-port-warning/30 flex items-center gap-2 disabled:opacity-50"- title="Remove these shared fragments from every caption, keeping the trigger word and per-shot detail"- >- {strippingFragments ? <Loader2 className="w-3.5 h-3.5 animate-spin" /> : <Eraser className="w-3.5 h-3.5" />}- Strip shared identity from all captions- </button>- </div>- </div>- </div>- )}-- <details className="bg-port-card border border-port-border rounded-lg text-sm">- <summary className="cursor-pointer select-none px-3 py-2 flex items-center gap-2 text-gray-300 hover:text-white">- <Lightbulb className="w-4 h-4 text-port-warning shrink-0" />- <span className="font-medium">Tips for a strong training dataset</span>- <span className="text-xs text-gray-500">- target ~{RECOMMENDED_TRAINING_IMAGES}–{TRAINING_IMAGE_SWEET_SPOT_MAX} images · {MIN_TRAINING_IMAGES} minimum- </span>- </summary>- <div className="px-3 pb-3 pt-1 text-xs text-gray-400 space-y-2 border-t border-port-border/60">- <p>- Quality beats quantity. {MIN_TRAINING_IMAGES} images is the floor; ~{RECOMMENDED_TRAINING_IMAGES}–{TRAINING_IMAGE_SWEET_SPOT_MAX} sharp,- varied shots is the sweet spot. Past ~50 you mostly add training time and overfitting risk —- near-duplicate frames teach the model to memorize one setup instead of learning the subject.- </p>- <ul className="list-disc pl-4 space-y-1">- <li><span className="text-gray-300">Vary the angle</span> — front, three-quarter, profile, and a back view.</li>- <li><span className="text-gray-300">Vary the framing</span> — mix tight face close-ups (for likeness) with mid and full-body shots (for proportions).</li>- <li><span className="text-gray-300">Vary pose, presentation & expression</span> — action and rest for characters; display angles and use context for objects and places.</li>- <li><span className="text-gray-300">Vary outfit, lighting & background</span> — so the LoRA learns the subject, not one costume, key light, or backdrop.</li>- <li><span className="text-gray-300">Keep it single-subject and on-model</span> — one clearly-visible subject per image, consistent identity, no clutter.</li>- <li><span className="text-gray-300">Drop the weak ones</span> — blurry, occluded, or off-model frames hurt more than they help; avoid near-duplicates.</li>- </ul>- <p className="pt-1 border-t border-port-border/60">- <span className="text-gray-300">Caption what changes shot-to-shot, not who the subject is.</span> The- {' '}- <span className="font-mono text-gray-400">{dataset.triggerWord || 'trigger word'}</span>- {' '}- token is what the LoRA binds the fixed identity to — describe pose, framing, expression, outfit, and setting- in each caption, and leave invariant features (hair, eyes, skin, signature items) out. Repeating the same- identity phrases in every caption teaches those words instead of the trigger, so the bare trigger then- renders a generic subject. Captions are auto-prefixed with the trigger; this page warns when identity- fragments repeat across most captions and offers a one-click strip.- </p>- </div>- </details>-- <div className="grid grid-cols-1 @5xl/page:grid-cols-[minmax(0,1fr)_320px] gap-4 items-start">- <DatasetImageGrid- dataset={dataset}- onImagesChange={onImagesChange}- onRecaption={recaptionImage}- recaptioningId={captionSlot.context?.imageIds?.length === 1 ? captionSlot.context.imageIds[0] : null}- captionBusy={captionBusy}- draftResetToken={captionDraftResetToken}- />- <TrainingPanel- dataset={dataset}- readiness={readiness}- triggerSaving={triggerSaving}- onRunFinished={refresh}- />- </div>-- {showGenerate && (- <GenerateBatchDialog- dataset={dataset}- expressionOptions={expressionOptions}- outfitOptions={outfitOptions}- onClose={() => setShowGenerate(false)}- onStarted={() => { setShowGenerate(false); refresh(); }}- />- )}- {showSlice && (- <SliceDialog- dataset={dataset}- onClose={() => setShowSlice(false)}- onSliced={() => { setShowSlice(false); refresh(); }}- />- )}- {showImport && (- <ImportGalleryDialog- dataset={dataset}- onClose={() => setShowImport(false)}- onImported={(images) => {- setShowImport(false);- if (images?.length) onImagesChange((prev) => [...prev, ...images]);- refresh();- }}- />- )}- {showReassign && (- <ReassignDialog- dataset={dataset}- onClose={() => setShowReassign(false)}- onReassigned={(next) => { setShowReassign(false); setDataset(next); }}- />- )}- </div>- );-}diff --git a/client/src/pages/LoraDatasetDetail.test.jsx b/client/src/pages/LoraDatasetDetail.test.jsxdeleted file mode 100644index 486aaf1d78..0000000000--- a/client/src/pages/LoraDatasetDetail.test.jsx+++ /dev/null@@ -1,143 +0,0 @@-import { act, cleanup, fireEvent, render, screen } from '@testing-library/react';-import { MemoryRouter } from 'react-router';-import { afterEach, expect, it, vi } from 'vitest';-import LoraDatasetDetail from './LoraDatasetDetail';-import { cancelLoraCaptionRun, getLoraDataset, startLoraCaptionRun } from '../services/api';-import socket from '../services/socket';--vi.mock('../services/api', () => ({- getLoraDataset: vi.fn(), getUniverse: vi.fn(), startLoraCaptionRun: vi.fn(), cancelLoraCaptionRun: vi.fn(),-}));-vi.mock('../components/ui/Toast', () => ({- default: Object.assign(vi.fn(), { success: vi.fn(), error: vi.fn(), info: vi.fn() }),-}));-vi.mock('../services/socket', () => {- const handlers = new Map();- return { default: {- on: (event, handler) => { if (!handlers.has(event)) handlers.set(event, new Set()); handlers.get(event).add(handler); },- off: (event, handler) => handlers.get(event)?.delete(handler),- emit: (event, payload) => handlers.get(event)?.forEach(handler => handler(payload)),- } };-});-// Controllable SSE stream: the test pushes frames; the page reads them through-// the real useSseJobSlot, so adoption/cancel/settlement are exercised end to end.-const sse = vi.hoisted(() => {- const listeners = new Set();- let value = { latest: null, closed: false, isOpen: true, frames: [] };- return {- urls: [],- get: () => value,- push: (frame) => { value = { latest: frame, closed: true, isOpen: false, frames: [frame] }; listeners.forEach(fn => fn()); },- reset: () => { value = { latest: null, closed: false, isOpen: true, frames: [] }; sse.urls.length = 0; },- subscribe: (fn) => { listeners.add(fn); return () => listeners.delete(fn); },- };-});-vi.mock('../hooks/useSseProgress', async () => {- const { useSyncExternalStore } = await import('react');- return {- isTerminalSseFrame: frame => ['complete', 'error', 'canceled'].includes(frame?.type),- useSseProgress: (url) => {- const value = useSyncExternalStore(sse.subscribe, sse.get);- if (url) sse.urls.push(url);- return url ? { ...value, latestUrl: url } : { latest: null, closed: false, isOpen: false, frames: [] };- },- };-});-vi.mock('../components/loraTraining/TrainingPanel', () => ({ default: () => null }));-vi.mock('../components/loraTraining/CaptionModelPicker', () => ({ default: () => null }));-vi.mock('../components/loraTraining/DatasetImageGrid', () => ({- default: ({ dataset }) => <div>{dataset.images.map(image => <span key={image.id}>{image.status}</span>)}</div>,-}));-const dataset = (id, status = 'rendering') => ({- id, character: { name: id, entryKind: 'characters' }, triggerWord: 'example',- images: [{ id: 'image-1', status, file: 'example.png', caption: '' }], training: {},-});-afterEach(() => { cleanup(); vi.useRealTimers(); vi.clearAllMocks(); sse.reset(); });-it('updates on scoped persistence events, reconnect and tab show without polling', async () => {- vi.useFakeTimers();- Object.defineProperty(document, 'visibilityState', { configurable: true, value: 'visible' });- getLoraDataset.mockResolvedValue(dataset('example-a'));- render(<MemoryRouter><LoraDatasetDetail recordId="example-a" /></MemoryRouter>);- await act(async () => {});- expect(screen.getByText('rendering')).toBeTruthy();- await act(async () => { await vi.advanceTimersByTimeAsync(16000); });- expect(getLoraDataset).toHaveBeenCalledTimes(1);- await act(async () => { socket.emit('training:dataset:changed', { datasetId: 'other' }); });- expect(getLoraDataset).toHaveBeenCalledTimes(1);- getLoraDataset.mockResolvedValue(dataset('example-a', 'ready'));- await act(async () => { socket.emit('training:dataset:changed', { datasetId: 'example-a' }); });- expect(screen.getByText('ready')).toBeTruthy();- expect(getLoraDataset).toHaveBeenCalledTimes(2);- await act(async () => { socket.emit('connect'); });- expect(getLoraDataset).toHaveBeenCalledTimes(3);- await act(async () => {- Object.defineProperty(document, 'visibilityState', { configurable: true, value: 'hidden' });- document.dispatchEvent(new Event('visibilitychange'));- socket.emit('training:dataset:changed', { datasetId: 'example-a' });- });- expect(getLoraDataset).toHaveBeenCalledTimes(3);- getLoraDataset.mockResolvedValue(dataset('example-a', 'failed'));- await act(async () => {- Object.defineProperty(document, 'visibilityState', { configurable: true, value: 'visible' });- document.dispatchEvent(new Event('visibilitychange'));- });- expect(screen.getByText('failed')).toBeTruthy();- expect(getLoraDataset).toHaveBeenCalledTimes(4);-});-it('reads a changed route and ignores the previous response and events', async () => {- let resolveOld;- getLoraDataset.mockImplementation(id => id === 'example-a'- ? new Promise(resolve => { resolveOld = resolve; }) : Promise.resolve(dataset(id, 'ready')));- const view = render(<MemoryRouter><LoraDatasetDetail recordId="example-a" /></MemoryRouter>);- await act(async () => {});- view.rerender(<MemoryRouter><LoraDatasetDetail recordId="example-b" /></MemoryRouter>);- await act(async () => {});- await act(async () => { resolveOld(dataset('example-a')); });- expect(screen.getByText('example-b')).toBeTruthy();- expect(screen.queryByText('rendering')).toBeNull();- await act(async () => { socket.emit('training:dataset:changed', { datasetId: 'example-a' }); });- expect(getLoraDataset).toHaveBeenCalledTimes(2);-});--const readyDataset = (captionRun = null) => ({- ...dataset('example-a', 'ready'),- images: [{ id: 'image-1', status: 'ready', file: 'example.png', caption: '' }],- readiness: {}, captionRun,-});-const activeRun = { runId: 'run-1', datasetId: 'example-a', status: 'running', provider: 'p', model: 'm', total: 3, done: 1, failed: 0 };--it('re-attaches to the server caption run after a remount and cancels it from there', async () => {- getLoraDataset.mockResolvedValue(readyDataset(activeRun));- cancelLoraCaptionRun.mockResolvedValue({ canceled: true });- const first = render(<MemoryRouter><LoraDatasetDetail recordId="example-a" /></MemoryRouter>);- await act(async () => {});- expect(screen.getByRole('button', { name: /Cancel/ })).toBeTruthy();- first.unmount();- // Reload / second tab: a fresh mount adopts the same run from the dataset read.- render(<MemoryRouter><LoraDatasetDetail recordId="example-a" /></MemoryRouter>);- await act(async () => {});- expect(screen.getAllByText(/Captioning 1\/3/).length).toBeGreaterThan(0);- expect(sse.urls.at(-1)).toBe('/api/lora-datasets/example-a/caption-runs/run-1/events');- expect(screen.getByRole('button', { name: /^Captioning 1\/3/ }).disabled).toBe(true);- expect(startLoraCaptionRun).not.toHaveBeenCalled();-- fireEvent.click(screen.getByRole('button', { name: /^Cancel/ }));- expect(cancelLoraCaptionRun).toHaveBeenCalledWith('example-a', 'run-1', { silent: true });- // The server's terminal canceled frame settles the UI and refetches the dataset.- getLoraDataset.mockResolvedValue(readyDataset(null));- await act(async () => { sse.push({ type: 'canceled', runId: 'run-1', done: 1, total: 3 }); });- expect(screen.queryByRole('button', { name: /^Cancel/ })).toBeNull();- expect(screen.getByRole('button', { name: /Caption all/ }).disabled).toBe(false);- expect(getLoraDataset).toHaveBeenCalledTimes(3);- expect(startLoraCaptionRun).not.toHaveBeenCalled();-});--it('attaches to the holder when a start reports a conflicting active run', async () => {- getLoraDataset.mockResolvedValue(readyDataset(null));- startLoraCaptionRun.mockResolvedValue({ runId: 'run-9', alreadyRunning: true, conflict: true, total: 3 });- render(<MemoryRouter><LoraDatasetDetail recordId="example-a" /></MemoryRouter>);- await act(async () => {});- await act(async () => { fireEvent.click(screen.getByRole('button', { name: /Caption all/ })); });- expect(sse.urls.at(-1)).toBe('/api/lora-datasets/example-a/caption-runs/run-9/events');- expect(screen.getByRole('button', { name: /^Cancel/ })).toBeTruthy();-});diff --git a/client/src/pages/LoraTraining.jsx b/client/src/pages/LoraTraining.jsxdeleted file mode 100644index 8e5fac1f0d..0000000000--- a/client/src/pages/LoraTraining.jsx+++ /dev/null@@ -1,210 +0,0 @@-/**- * LoRA Training — dataset list (/models/training).- *- * One dataset per universe bible subject. Cards show the subject, a thumb- * strip, image/caption counts, and training status; "New dataset" walks- * universe → character and lands on the dataset workbench.- */--import { useState } from 'react';-import { Link, useNavigate } from 'react-router';-import { GraduationCap, Plus, Loader2, Sparkles, Images } from 'lucide-react';-import toast from '../components/ui/Toast';-import Modal from '../components/ui/Modal';-import ConfirmButtonPair from '../components/ui/ConfirmButtonPair';-import UniverseCharacterPicker from '../components/loraTraining/UniverseCharacterPicker';-import MediaJobsQueue from '../components/media/MediaJobsQueue';-import ListLoadError from '../components/ui/ListLoadError';-import useListResource from '../hooks/useListResource';-import {- listLoraDatasets,- createLoraDataset,- deleteLoraDataset,-} from '../services/api';--const loadLoraDatasets = () => listLoraDatasets({}, { silent: true });--const STATUS_CHIP = {- draft: 'bg-port-border/40 text-gray-300',- training: 'bg-port-warning/20 text-port-warning',- trained: 'bg-port-success/20 text-port-success',-};-const SUBJECT_TYPE_LABEL = { characters: 'Character', objects: 'Object', places: 'Place' };--function NewDatasetDialog({ onClose, onCreated }) {- const [universeId, setUniverseId] = useState('');- const [entryKind, setEntryKind] = useState('characters');- const [entryId, setEntryId] = useState('');- const [creating, setCreating] = useState(false);-- const create = async () => {- setCreating(true);- try {- const dataset = await createLoraDataset({ universeId, entryKind, entryId });- onCreated(dataset);- } finally {- setCreating(false);- }- };-- return (- <Modal- open- onClose={onClose}- size="sm"- ariaLabelledBy="lt-new-dataset-title"- panelClassName="bg-port-card border border-port-border rounded-lg p-5"- >- <div className="space-y-4">- <h2 id="lt-new-dataset-title" className="text-base font-semibold text-white">New training dataset</h2>- <UniverseCharacterPicker- idPrefix="lt-new"- universeId={universeId}- entryKind={entryKind}- entryId={entryId}- onUniverseChange={(id) => { setUniverseId(id); setEntryId(''); }}- onEntryKindChange={(kind) => { setEntryKind(kind); setEntryId(''); }}- onEntryChange={setEntryId}- />- <div className="flex justify-end gap-2">- <button type="button" onClick={onClose} className="px-3 py-2 text-sm text-gray-400 hover:text-white">Cancel</button>- <button- type="button"- onClick={create}- disabled={!universeId || !entryId || creating}- className="px-3 py-2 text-sm rounded bg-port-accent text-white disabled:opacity-50 flex items-center gap-2"- >- {creating ? <Loader2 className="w-4 h-4 animate-spin" /> : <Plus className="w-4 h-4" />}- Open dataset- </button>- </div>- </div>- </Modal>- );-}--export default function LoraTraining() {- const navigate = useNavigate();- const datasetsResource = useListResource(loadLoraDatasets);- const {- items: datasets, state, error, retry, setItems: setDatasets,- } = datasetsResource;- const [showNew, setShowNew] = useState(false);- const [deletingId, setDeletingId] = useState(null);- const [confirmingId, setConfirmingId] = useState(null);-- const onDelete = async (id) => {- setDeletingId(id);- try {- await deleteLoraDataset(id);- setDatasets((prev) => (prev || []).filter((d) => d.id !== id));- toast.success('Dataset deleted');- } finally {- setDeletingId(null);- setConfirmingId(null);- }- };-- return (- <div>- <div className="flex items-center justify-between mb-4 flex-wrap gap-2">- <div className="flex items-center gap-2">- <GraduationCap className="w-5 h-5 text-port-accent" />- <h2 className="text-lg font-semibold text-white">LoRA Training</h2>- </div>- <button- type="button"- onClick={() => setShowNew(true)}- className="px-3 py-2 text-sm rounded bg-port-accent text-white flex items-center gap-2"- >- <Plus className="w-4 h-4" /> New dataset- </button>- </div>-- {state === 'loading' && (- <div className="text-gray-400 flex items-center gap-2"><Loader2 className="w-4 h-4 animate-spin" /> Loading datasets…</div>- )}-- {state === 'error' && (- <ListLoadError what="datasets" error={error} onRetry={retry} />- )}-- {state === 'ready' && datasets.length === 0 && (- <div className="bg-port-card border border-port-border rounded-lg p-8 text-center text-gray-400">- <Sparkles className="w-8 h-8 mx-auto mb-3 text-port-accent" />- <p className="mb-1 text-white">Train a LoRA for consistent universe subjects.</p>- <p className="text-sm">Pick a character, object, or place from a universe bible, build reference material, caption it, and train a FLUX adapter.</p>- </div>- )}-- {state === 'ready' && datasets.length > 0 && (- <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">- {(datasets || []).map((d) => (- <div key={d.id} className="bg-port-card border border-port-border rounded-lg p-4 flex flex-col gap-3">- <div className="flex items-start justify-between gap-2">- <Link to={`/models/training/${d.id}`} className="min-w-0">- <div className="text-white font-medium truncate">{d.character?.name || 'Unnamed'}</div>- <div className="text-xs text-gray-500 truncate">- <span className="font-mono">{d.triggerWord}</span>- <span className="mx-1">·</span>- {SUBJECT_TYPE_LABEL[d.character?.entryKind || 'characters'] || 'Subject'}- </div>- </Link>- <span className={`text-xs px-2 py-0.5 rounded-full shrink-0 ${STATUS_CHIP[d.status] || STATUS_CHIP.draft}`}>- {d.status}- </span>- </div>- {d.thumbnails?.length > 0 && (- <Link to={`/models/training/${d.id}`} className="flex gap-1 overflow-hidden">- {d.thumbnails.map((file) => (- <img- key={file}- src={`/data/lora-datasets/${d.id}/images/${file}`}- alt=""- className="w-16 h-16 object-cover rounded border border-port-border"- loading="lazy"- />- ))}- </Link>- )}- <div className="flex items-center justify-between text-xs text-gray-400">- <span className="flex items-center gap-1">- <Images className="w-3.5 h-3.5" />- {d.readiness?.ready ?? 0} images · {d.readiness?.captioned ?? 0} captioned- {d.readiness?.rendering ? ` · ${d.readiness.rendering} rendering` : ''}- </span>- {confirmingId === d.id ? (- <ConfirmButtonPair- prompt="Delete?"- busy={deletingId === d.id}- onConfirm={() => onDelete(d.id)}- onCancel={() => setConfirmingId(null)}- />- ) : (- <button- type="button"- onClick={() => setConfirmingId(d.id)}- className="text-gray-500 hover:text-port-error"- >- Delete- </button>- )}- </div>- </div>- ))}- </div>- )}-- <div className="mt-6">- <MediaJobsQueue kind="training" />- </div>-- {showNew && (- <NewDatasetDialog- onClose={() => setShowNew(false)}- onCreated={(dataset) => navigate(`/models/training/${dataset.id}`)}- />- )}- </div>- );-}diff --git a/client/src/pages/LoraTraining.test.jsx b/client/src/pages/LoraTraining.test.jsxdeleted file mode 100644index 8303e1c03f..0000000000--- a/client/src/pages/LoraTraining.test.jsx+++ /dev/null@@ -1,59 +0,0 @@-import { describe, it, expect, vi, beforeEach } from 'vitest';-import { act, fireEvent, render, screen } from '@testing-library/react';-import { MemoryRouter } from 'react-router';--vi.mock('../services/api', () => ({- listLoraDatasets: vi.fn(),- createLoraDataset: vi.fn(),- deleteLoraDataset: vi.fn(),-}));-vi.mock('../components/loraTraining/UniverseCharacterPicker', () => ({ default: () => null }));-vi.mock('../components/media/MediaJobsQueue', () => ({ default: () => null }));-vi.mock('../components/ui/Toast', () => ({ default: { error: vi.fn(), success: vi.fn() } }));--import LoraTraining from './LoraTraining.jsx';-import { listLoraDatasets } from '../services/api';--describe('LoraTraining page', () => {- beforeEach(() => {- vi.clearAllMocks();- });-- it('renders ListLoadError with retry on failed datasets load, and empty state on real empty list', async () => {- listLoraDatasets.mockRejectedValueOnce(new Error('500 Internal Server Error'));- render(- <MemoryRouter>- <LoraTraining />- </MemoryRouter>- );- await act(async () => {});-- expect(screen.getByRole('alert')).toHaveTextContent('Could not load datasets — 500 Internal Server Error');- expect(screen.getByRole('button', { name: 'Retry' })).toBeInTheDocument();- expect(screen.queryByText('Train a LoRA for consistent universe subjects.')).not.toBeInTheDocument();-- listLoraDatasets.mockResolvedValueOnce([]);- await act(async () => {- fireEvent.click(screen.getByRole('button', { name: 'Retry' }));- });-- expect(screen.queryByRole('alert')).not.toBeInTheDocument();- expect(screen.getByText('Train a LoRA for consistent universe subjects.')).toBeInTheDocument();- });-- it('renders datasets when load succeeds', async () => {- listLoraDatasets.mockResolvedValueOnce([- { id: 'ds-1', character: { name: 'Captain Nova' }, status: 'draft', imageCount: 12 },- ]);- render(- <MemoryRouter>- <LoraTraining />- </MemoryRouter>- );- await act(async () => {});-- expect(screen.queryByRole('alert')).not.toBeInTheDocument();- expect(screen.getByText('Captain Nova')).toBeInTheDocument();- expect(screen.queryByText('Train a LoRA for consistent universe subjects.')).not.toBeInTheDocument();- });-});diff --git a/client/src/pages/Loras.jsx b/client/src/pages/Loras.jsxindex 19ac686d54..aa59b2557d 100644--- a/client/src/pages/Loras.jsx+++ b/client/src/pages/Loras.jsx@@ -1682,11 +1682,11 @@ function LoraCard({ lora, onDelete, onMeasured, deleting, deleteConfirm }) {</div>{/* Trained-in-PortOS lineage: character chip links back to the- training dataset that produced this adapter. */}+ retained training artifacts in Data Maintenance. */}{lora.source === 'trained' && lora.character && (<div className="mb-2"><Link- to={lora.trainedFromDatasetId ? `/models/training/${lora.trainedFromDatasetId}` : '/models/training'}+ to="/settings/data-maintenance"className="inline-flex items-center gap-1 text-[11px] px-2 py-0.5 rounded-full bg-port-success/15 text-port-success border border-port-success/30 hover:bg-port-success/25"title="Trained in PortOS — open the training dataset">diff --git a/client/src/pages/Models.jsx b/client/src/pages/Models.jsxindex 95c7b08de9..9e80bbfd7f 100644--- a/client/src/pages/Models.jsx+++ b/client/src/pages/Models.jsx@@ -16,13 +16,11 @@ import { LocalLlmTab } from '../components/settings/LocalLlmTab';import LocalLlmRuntimesView from '../components/settings/LocalLlmRuntimesView.jsx';import { lazyWithReload } from '../utils/lazyWithReload';-// The three panels moved in from the Media Gen tabs were each their own route+// The media-model panels moved in from the Media Gen tabs were each their own route// chunk before the move (#4728) — keep them split, or landing on Performance// downloads the whole LoRA manager and HF-cache browser to render an// assessments table. The small always-on tabs above stay static.const Loras = lazyWithReload(() => import('./Loras'));-const LoraTraining = lazyWithReload(() => import('./LoraTraining'));-const LoraDatasetDetail = lazyWithReload(() => import('./LoraDatasetDetail'));const MediaModels = lazyWithReload(() => import('./MediaModels'));/**@@ -43,7 +41,6 @@ const MediaModels = lazyWithReload(() => import('./MediaModels'));* - **Providers** — configured AI provider connections and model catalogs.* - **Status** — residency plus the downloaded-model inventory.* - **Subscriptions** — which paid plans are on, their tier, price and spend.- * - **Training** — LoRA fine-tuning datasets and runs.* - **Usage** — provider quota and PortOS AI usage accounting.** What deliberately stayed OUT is output rather than weights: Three.js Models is@@ -69,21 +66,6 @@ const TAB_CONTENT = {performance: LocalModelAssessments,status: ModelStatusTab,subscriptions: SubscriptionsTab,- training: LoraTraining,-};--/**- * Drill-downs rendered INSIDE the section shell, keyed by the tab that owns them.- *- * A tab's detail view is still that tab — the LoRA dataset workbench is Training- * with one dataset open — so it keeps the section header and tab bar rather than- * becoming a bare route that drops both. Under `/media` these pages got that for- * free, because MediaGen was a layout route; declaring them here gives the same- * result without every future detail view needing its own special case in- * App.jsx.- */-const TAB_DETAIL = {- training: LoraDatasetDetail,};export default function Models({ fixedTab, fixedRecordId } = {}) {@@ -92,6 +74,7 @@ export default function Models({ fixedTab, fixedRecordId } = {}) {const tab = fixedTab || params.tab;const recordId = fixedRecordId || params.recordId;const taskView = params.taskView || params.view || (fixedTab === 'performance' && params.assessmentKey ? 'results' : undefined);+ if (tab === 'training') return <Navigate to="/settings/data-maintenance" replace />;if (tab === 'llms' && recordId === 'jev') return <Navigate to={`/models/decision-classifiers/jev${taskView ? '/' + taskView : ''}`} replace />;// An unknown slug lands on LLMs rather than rendering a blank page, matching// the section's default destination in App and the primary navigation.@@ -103,10 +86,6 @@ export default function Models({ fixedTab, fixedRecordId } = {}) {const activeTab = tab && Object.hasOwn(TAB_CONTENT, tab) ? tab : null;if (!activeTab) return <Navigate to="/models/llms" replace />;- // A record id in the URL selects the tab's drill-down, when it has one. Tabs- // without a detail component receive it as a focused sub-view id (LLMs uses- // `library` and `abuse`); tabs that do not recognize it render their index.- const DetailContent = recordId && Object.hasOwn(TAB_DETAIL, activeTab) ? TAB_DETAIL[activeTab] : null;const TabContent = TAB_CONTENT[activeTab];// The shell names the destination you are on (Code Reviewers, Performance),// matching the section navigator. "Models" stays the area, not the page title.@@ -126,9 +105,7 @@ export default function Models({ fixedTab, fixedRecordId } = {}) {{/* Local boundary rather than the App-level one: a lazy tab must not blankthe section header and tab bar while its chunk loads. */}<Suspense fallback={<PageSkeleton header="none" label="Loading models section" cards={3} sidebar={false} />}>- {DetailContent- ? <DetailContent recordId={recordId} />- : <TabContent view={recordId} taskView={taskView} assessmentKey={params.assessmentKey} />}+ <TabContent view={recordId} taskView={taskView} assessmentKey={params.assessmentKey} /></Suspense></div></ModelsSectionLayout>diff --git a/client/src/pages/Models.test.jsx b/client/src/pages/Models.test.jsxindex 12ed9400ea..a1c1beb6c4 100644--- a/client/src/pages/Models.test.jsx+++ b/client/src/pages/Models.test.jsx@@ -36,9 +36,7 @@ vi.mock('../components/settings/CodeReviewersTab', () => ({ default: () => <div>vi.mock('../components/models/ModelComparison', () => ({ default: () => <div>comparison panel</div> }));vi.mock('../components/models/DecisionClassifiers', () => ({ default: ({ view }) => <div>classifiers panel {view}</div> }));vi.mock('./Loras', () => ({ default: () => <div>loras panel</div> }));-vi.mock('./LoraTraining', () => ({ default: () => <div>training panel</div> }));vi.mock('./MediaModels', () => ({ default: () => <div>media models panel</div> }));-vi.mock('./LoraDatasetDetail', () => ({ default: ({ recordId }) => <div>dataset workbench {recordId}</div> }));import Models from './Models';@@ -62,7 +60,6 @@ const PANEL_MARKER = {performance: 'assessments panel',status: 'status panel',subscriptions: 'subscriptions panel',- training: 'training panel',};// These destinations belong to Models in the sidebar but keep their own route@@ -75,6 +72,7 @@ const renderAt = (path) => render(<Routes><Route path="/models/performance/results/:assessmentKey" element={<Models fixedTab="performance" />} /><Route path="/models/performance/:view" element={<Models fixedTab="performance" />} />+ <Route path="/settings/data-maintenance" element={<div>Retired training maintenance</div>} /><Route path="/models/:tab" element={<Models />} /><Route path="/models/:tab/:recordId" element={<Models />} /><Route path="/models/decision-classifiers/jev/:taskView" element={<Models fixedTab="decision-classifiers" fixedRecordId="jev" />} />@@ -213,14 +211,10 @@ describe('Models — tab drill-downs', () => {expect(screen.getByRole('tab', { name: view === 'library' ? 'Model Library' : 'Abuse Guard' })).toHaveAttribute('aria-selected', 'true');});- it('renders a tab detail view INSIDE the section shell, not as a bare page', async () => {- // Under /media these pages kept the shell's chrome for free, because MediaGen- // was a layout route. Registering the workbench as its own top-level route- // would silently drop the section header and tab bar; asserting the tab bar is- // still present (and still marks Training active) is what catches that.- renderAt('/models/training/dataset-abc');- expect(await screen.findByText('dataset workbench dataset-abc')).toBeInTheDocument();- expect(screen.getByRole('tab', { name: 'Training' })).toHaveAttribute('aria-selected', 'true');+ it.each(['/models/training', '/models/training/dataset-abc'])('redirects retired deep link %s to maintenance', async path => {+ renderAt(path);+ expect(await screen.findByText('Retired training maintenance')).toBeInTheDocument();+ expect(screen.queryByRole('tab', { name: 'Training' })).toBeNull();});it('falls back to the tab index when the tab has no detail view', async () => {diff --git a/client/src/pages/Settings.jsx b/client/src/pages/Settings.jsxindex e4ebdf8133..16634a1fb7 100644--- a/client/src/pages/Settings.jsx+++ b/client/src/pages/Settings.jsx@@ -1,3 +1,4 @@+import DataMaintenanceTab from '../components/settings/DataMaintenanceTab';import { useParams, Navigate } from 'react-router';import { Settings as SettingsIcon } from 'lucide-react';import PageHeader from '../components/PageHeader';@@ -44,6 +45,7 @@ export default function Settings() {case 'api-access': return <ApiAccessTab />;case 'autofixer': return <AutofixerTab />;case 'backup': return <BackupTab />;+ case 'data-maintenance': return <DataMaintenanceTab />;case 'database': return <DatabaseTab />;case 'credentials': return <CredentialsTab />;case 'features': return <InstanceFeaturesTab />;diff --git a/client/src/services/README.md b/client/src/services/README.mdindex 1af68ed3a6..077acf1c26 100644--- a/client/src/services/README.md+++ b/client/src/services/README.md@@ -106,7 +106,7 @@ toasts on throw). **Custom catch ⇒ `silent: true`** — otherwise toasts fire| File | Purpose ||---|---|| `apiImageVideo.js` | Image-gen local backend extras; bounded image/mixed gallery pages, global facets and collection summaries, exact-reference `getGalleryImages`, one image's original-vs-cleaned variant set via `listImageVariants`, plus models, LoRAs, cancel and delete. |-| `apiLoraTraining.js` | Character LoRA training — datasets (CRUD, upload, generate, slice, caption), training runs (start/list/cancel + status), character→LoRA link lookup. |+| `apiLoraTraining.js` | Retained checkpoint inspection and character→adapter lookup. Retired training/dataset mutation wrappers are removed. || `apiMedia.js` | `importGalleryFile` dispatches image/video imports through the Prompt gallery upload endpoints; `assignImportedMedia` adds existing imports through idempotent bulk membership. Screenshots + media assets. Also owns the multi-file upload orchestration — `processScreenshotUploads` / `processAttachmentUploads` — moved from `utils/fileUpload.js` since they perform network I/O, not pure transforms. `utils/fileUpload.js` keeps only the pure helpers/constants and no longer re-exports these. || `apiMediaJobs.js` | Media generation job tracking + `refineMediaPrompt` / `promptFromMedia` (vision reverse-prompt). || `apiDecks.js` | Decks (playing-card / tarot designer) — deck + card CRUD, sample-design style analysis, universe casting + prompt generation (plus its SSE progress URL), single/batch card renders. |diff --git a/client/src/services/apiLoraTraining.js b/client/src/services/apiLoraTraining.jsindex 1d20107806..b875276630 100644--- a/client/src/services/apiLoraTraining.js+++ b/client/src/services/apiLoraTraining.js@@ -1,148 +1,9 @@-/**- * LoRA training API — datasets (/api/lora-datasets), training- * runs (/api/lora-training), and the character→LoRA link lookup- * (/api/loras/by-character).- */-+/** Retained checkpoint inspection and character-to-inference-adapter lookup. */import { request } from './apiCore.js';-// ---- Datasets ------export const listLoraDatasets = (filters = {}, options = {}) => {- const params = new URLSearchParams();- for (const key of ['universeId', 'entryKind', 'entryId', 'ingredientId']) {- if (filters[key]) params.set(key, filters[key]);- }- const qs = params.toString();- return request(`/lora-datasets${qs ? `?${qs}` : ''}`, options);-};--export const createLoraDataset = ({- universeId, entryKind = 'characters', entryId, triggerWord,-}, { silent = false } = {}) =>- request('/lora-datasets', {- method: 'POST',- body: JSON.stringify({- universeId, entryKind, entryId, ...(triggerWord ? { triggerWord } : {}),- }),- silent,- });--export const getLoraDataset = (id) => request(`/lora-datasets/${id}`);--// Server-derived variation axes (expressions/outfits for characters; lighting/-// settings for objects & places) — seeds the generate-batch override chips.-export const getLoraDatasetVariationAxes = (id, options = {}) =>- request(`/lora-datasets/${id}/variation-axes`, options);--export const patchLoraDataset = (id, patch) =>- request(`/lora-datasets/${id}`, { method: 'PATCH', body: JSON.stringify(patch) });--export const deleteLoraDataset = (id) => request(`/lora-datasets/${id}`, { method: 'DELETE' });--// `files` is a FileList/array — field names image1…image10 per the route.-export const uploadLoraDatasetImages = (id, files) => {- const form = new FormData();- [...files].slice(0, 10).forEach((file, i) => form.append(`image${i + 1}`, file));- return request(`/lora-datasets/${id}/images`, { method: 'POST', body: form });-};--// Import existing gallery images (basenames) into the dataset.-export const importLoraDatasetGalleryImages = (id, filenames) =>- request(`/lora-datasets/${id}/import-gallery`, {- method: 'POST', body: JSON.stringify({ filenames }),- });--export const generateLoraDatasetImages = (id, options = {}) =>- request(`/lora-datasets/${id}/generate`, { method: 'POST', body: JSON.stringify(options) });--// `useVision` (default true) lets a vision model propose a bounding box per-// figure; the fixed cols×rows grid is the fallback (and what `useVision: false`-// forces). When omitted, the server auto-resolves a vision model.-export const sliceLoraDatasetRefSheet = (id, { variant, cols, rows, useVision } = {}) =>- request(`/lora-datasets/${id}/slice-reference-sheet`, {- method: 'POST',- body: JSON.stringify({- ...(variant ? { variant } : {}),- ...(cols ? { cols } : {}),- ...(rows ? { rows } : {}),- ...(useVision === false ? { useVision: false } : {}),- }),- });--export const startLoraCaptionRun = (id, options = {}) =>- request(`/lora-datasets/${id}/caption`, { method: 'POST', body: JSON.stringify(options) });--// Run-scoped cancel: the server rejects (404) a run id that is not the dataset's-// current caption run. The terminal `canceled` frame follows over SSE.-export const cancelLoraCaptionRun = (id, runId, options = {}) =>- request(`/lora-datasets/${id}/caption-runs/${runId}/cancel`, { method: 'POST', silent: options.silent });--export const updateLoraDatasetImageCaption = (id, imageId, caption) =>- request(`/lora-datasets/${id}/images/${imageId}`, {- method: 'PATCH', body: JSON.stringify({ caption }),- });--export const deleteLoraDatasetImage = (id, imageId) =>- request(`/lora-datasets/${id}/images/${imageId}`, { method: 'DELETE' });--// Strip the identity fragments shared across most captions so the trigger token-// learns the character (issue #1320). Returns { dataset, removedFragments, updatedImages }.-export const stripLoraDatasetSharedCaptionFragments = (id) =>- request(`/lora-datasets/${id}/strip-shared-fragments`, { method: 'POST' });--// ---- Training runs ------export const getLoraTrainingStatus = (options = {}) => request('/lora-training/status', options);--export const startLoraTrainingRun = ({ datasetId, baseModelId, name, params, acknowledgeCaptionLeak }, options = {}) =>- request('/lora-training/runs', {- method: 'POST',- silent: options.silent,- body: JSON.stringify({- datasetId,- baseModelId,- ...(name ? { name } : {}),- ...(params ? { params } : {}),- ...(acknowledgeCaptionLeak ? { acknowledgeCaptionLeak: true } : {}),- }),- });--export const listLoraTrainingRuns = (filters = {}, options = {}) => {- const params = new URLSearchParams();- for (const key of ['status', 'characterId', 'datasetId', 'limit']) {- if (filters[key]) params.set(key, filters[key]);- }- const qs = params.toString();- return request(`/lora-training/runs${qs ? `?${qs}` : ''}`, options);-};--export const cancelLoraTrainingRun = (runId) =>- request(`/lora-training/runs/${runId}/cancel`, { method: 'POST' });--// Resume a failed/canceled run from its latest checkpoint — continues the same-// run (new job, same run id) so the trainer picks up its saved optimizer state.-export const resumeLoraTrainingRun = (runId, { silent = false } = {}) =>- request(`/lora-training/runs/${runId}/resume`, { method: 'POST', silent });--// Checkpoints for a finished run — each with step, loss, preview thumbnail,-// and which one is currently deployed. Drives the manual checkpoint picker.export const listLoraTrainingCheckpoints = (runId, options = {}) =>request(`/lora-training/runs/${runId}/checkpoints`, options);-// Mid-training sample timeline (step + thumbnail URL) — seeds the live-// progress gallery so a reload mid-run shows every sample rendered so far.-export const listLoraTrainingSamples = (runId) =>- request(`/lora-training/runs/${runId}/samples`);--// Promote a checkpoint to be the deployed LoRA (re-extracts that step's-// adapter and overwrites the run's registered .safetensors in place).-export const promoteLoraTrainingCheckpoint = (runId, step, { silent = false } = {}) =>- request(`/lora-training/runs/${runId}/promote-checkpoint`, {- method: 'POST', body: JSON.stringify({ step }), silent,- });--// ---- Character → trained-LoRA link ----export const getCharacterLoras = ({ entryId, ingredientId } = {}, { silent = false } = {}) => {const params = new URLSearchParams();diff --git a/docs/API.md b/docs/API.mdindex 9757762db7..2397061091 100644--- a/docs/API.md+++ b/docs/API.md@@ -819,7 +819,9 @@ Every mounted API prefix (see `server/index.js` for the authoritative list). Dom| `/api/conflict-journal` | Sync conflict journal || `/api/importer` | Story importer || `/api/story-builder` | Story Builder |-| `/api/loras`, `/api/lora-datasets`, `/api/lora-training` | LoRA management/training |+| `/api/loras` | Supported inference adapter management |+| `/api/lora-datasets`, `/api/lora-training` | Retained read/cancel/checkpoint operations; training launch/resume and dataset mutation return 410 |+| `/api/data-maintenance` | Operator-authorized retired-data inventory, fresh preview and confirmed allowlisted removal; see [Data Maintenance](DATA_MAINTENANCE.md) || `/sdapi/v1` | AUTOMATIC1111-compatible image generation surface (gated by `settings.imageGen.expose.a1111`) || `/api/openclaw` | OpenClaw operator chat || `/api/rounds` | Rounds (music + Morse training) |diff --git a/docs/DATA_MAINTENANCE.md b/docs/DATA_MAINTENANCE.mdnew file mode 100644index 0000000000..f334f16b7e--- /dev/null+++ b/docs/DATA_MAINTENANCE.md@@ -0,0 +1,72 @@+# Data Maintenance++Open **Settings → Data Maintenance** (also available through the command palette+and voice navigation) to inspect storage retained from retired features. Refresh+measures current files. A failed or unsafe inventory is unavailable, never shown+as zero bytes. The registry currently contains only LoRA retirement operations;+archive retention and whole-checkout cleanup are separate workflows.++LoRA training is retired. New training, dataset generation/captioning, resume and+media-queue retries are refused. Old training URLs lead to Data Maintenance.+Use the reference-image inputs supported by your chosen image model for new work.+PortOS does not convert a training dataset or checkpoint into a reference-image+workflow automatically, and the page offers no unsupported Upgrade action.++## Retained storage++- `data/lora-datasets`: unique source images, captions and dataset metadata.+- `data/training-runs`: mixed storage, including potentially unique optimizer+ checkpoints/adapters, training logs, caches and sample previews. Treat the+ category as unique data rather than assuming every checkpoint is reproducible.+- `data/loras`: installed adapters, including trained and imported adapters.+ These remain supported inference inputs. Use **Models → LoRAs** to manage them.+- Shared inference environments (mflux, MLX, torch/diffusers), media models and+ Hugging Face caches remain supported. Retirement does not uninstall them.++Saved training configuration and PostgreSQL history remain readable. No format+migration, boot purge or automatic configuration rewrite is required. A queued+trainer is retired without spawning; the run records the retirement. A verified+trainer that already survived a restart is observed to completion without starting+a new trainer. Its outputs are finalized normally. Automatic stall resumes are+disabled. Existing cancellation controls remain available in **Active Queues**.+A detached trainer whose queue record is missing is also observed through completion,+and blocks maintenance until it exits. The retained run cancel endpoint+(`POST /api/lora-training/runs/:id/cancel`) can cancel that observer-owned process;+upgrade never kills it merely because the queue record is absent.++## Explicit removal++Copy unique files you want to keep and verify your own backup coverage first.+Backup settings, including the existing LoRA Time Machine exclusion, are unchanged.+The page is not a backup system.++1. Preview a supported removal. The server allowlist determines its exact category+ and path; callers cannot supply filesystem paths. The preview reports files,+ measured size and the consequence of removing that category.+2. Cancel to leave everything intact, or type the exact confirmation shown.+3. Confirm within five minutes. Changed inventory, symlinks, an unreadable tree,+ outstanding captioning, a surviving trainer or queued/running media work refuses+ removal. Refresh and preview again after resolving the reason.++Removal is permanent. There is no fake recovery/archive promise. A short admission+gate prevents media submissions racing the removal. The server serializes removals,+revalidates file identity, detaches the inspected directory and checks again before+removing it under backup publication admission. Duplicate submissions with the same+preview return the completed result while that preview remains valid. After a+server restart, a new preview is required.++On a filesystem failure, the UI reports an incomplete removal instead of success.+The error names the category and any detached `.maintenance-…` recovery directory;+inspect those locations before retrying. A partial filesystem failure can mean+some files were already removed. No automatic retry or boot cleanup purges the+remaining tree. PostgreSQL run history and installed inference adapters are never+removed by these operations.++## Extending the page++`server/services/dataMaintenance.js` owns the small operation registry. A new+operation must specify its fixed path, data category, preservation warning and+actual supported action. It must account for writers and shared consumers, reuse+fresh-preview/confirmation admission, and add a public-boundary synthetic fixture+regression. Do not add arbitrary paths, speculative migrations, shared-runtime+uninstallation or archive-cleanup operations to the registry.diff --git a/docs/README.md b/docs/README.mdindex 65580b79b4..a94efb704e 100644--- a/docs/README.md+++ b/docs/README.md@@ -118,3 +118,5 @@ Comms & voice: [beeper](./features/beeper.md) · [messages-browser-send](./featu- [Tool-free model delegation](SANDBOX_DELEGATION.md) — approved API workers, context packets, and advisory fidelity evaluation for Persistent Mind.- [Runway model catalog](RUNWAY_MODELS.md) — supported operations, public schema source, caching, refresh and workflow limits.++- [Data Maintenance](DATA_MAINTENANCE.md) — retained retired-feature storage, LoRA training retirement and explicit removal.diff --git a/docs/STORAGE.md b/docs/STORAGE.mdindex 0942854454..13691530fb 100644--- a/docs/STORAGE.md+++ b/docs/STORAGE.md@@ -1284,3 +1284,5 @@ This also identifies additions made before a crash prevented the final journalupdate. Never restore an old `usage.json` over newer usage to undo this import.No migration or seed is required: this is a new optional ledger, absent until alive completion or an explicit import publishes its first receipt.++LoRA training is retired; its existing datasets, run artifacts, configuration and PostgreSQL history remain intact. See [Data Maintenance](DATA_MAINTENANCE.md) for inventory and explicit removal. Installed adapters and shared inference runtimes remain supported.diff --git a/scripts/setup-image-video.sh b/scripts/setup-image-video.shindex 5774a00f28..57a778bc93 100755--- a/scripts/setup-image-video.sh+++ b/scripts/setup-image-video.sh@@ -170,11 +170,7 @@ INSTALL_VIDEO="${INSTALL_VIDEO:-$DEFAULT_INSTALL_VIDEO}"if [[ "$INSTALL_MFLUX" == "1" ]]; then# Install via pip --user so we don't pollute the system or require a venv.# mflux comes with the mflux-generate CLI which the local image backend- # spawns directly. Pin >=0.17: that's the first release with FLUX.2 LoRA- # training (`mflux-train --config`, `flux2-klein-*` base models). Older- # mflux (0.12.x) wants `--train-config` and has no flux2 models, so its- # trainer dies with a bare "exited with code 2" — see train_mflux_lora.py.- #+ # spawns directly. The shared inference runtime remains supported.echo "📦 Installing image generation packages (mflux + deps)..."# VALIDATED TRIO (issue #1329) — pin the MLX/Metal backend, do NOT leave it a# floor. The original three M5 Max GPU-watchdog kernel panics@@ -192,7 +188,7 @@ if [[ "$INSTALL_MFLUX" == "1" ]]; thenMLX_PIN='mlx==0.31.2'MLX_METAL_PIN='mlx-metal==0.31.2'# mflux/mlx need Python 3.10+. The historical layout installs mflux with- # `pip install --user` against $PYTHON_BIN, and PortOS finds `mflux-train`+ # `pip install --user` against $PYTHON_BIN, and PortOS discovers the shared console scripts# beside that interpreter. But when $PYTHON_BIN is too old (e.g. a system# python3.8) or externally-managed (PEP 668) so the --user install fails or# can't be imported, fall back to a dedicated venv at ~/.portos/venv-mflux —diff --git a/scripts/train_flux2_lora.py b/scripts/train_flux2_lora.pydeleted file mode 100644index 94edb57702..0000000000--- a/scripts/train_flux2_lora.py+++ /dev/null@@ -1,459 +0,0 @@-#!/usr/bin/env python3-"""FLUX.2 Klein LoRA trainer — vendored, CUDA/CPU (PortOS).--Trains a character LoRA on the bf16 Klein base (NEVER the SDNQ/int8-quantized inference repos — no useful gradients through quant layers).-The trained adapter loads onto quantized inference pipelines of the same-size variant because the transformer hidden dims match.--Two phases keep peak memory survivable on a consumer CUDA card (a 24 GB 3090-has no headroom to spare). The split was written for Apple Silicon's unified-memory, but `main()` now refuses MPS outright — see the guard below — so CUDA-and CPU are the runtimes this actually serves:-- 1. STAGE:precompute-latents — encode every dataset image to VAE latents- and every caption to Qwen3 text embeddings ONCE, then free the text- encoder and move the VAE off-device. Only the transformer stays- resident for the training loop.- 2. STAGE:training — bf16 transformer + peft LoRA adapter on the attention- projections, gradient checkpointing, AdamW, flow-matching loss- (target = noise - latents, logit-normal timestep sampling — matches- the FLUX family training recipes).--Line protocol (parsed by server/services/loraTraining/progress.js):- STEP:<cur>:<total>:<loss> CHECKPOINT:<path>:<step> SAMPLE:<path>:<step>- STAGE:<name>[...heartbeat] STATUS:<msg> USER_ERROR:<kind>:<msg>- RESULT:{"adapter_path": ..., "steps": N, "final_loss": F}--SIGTERM → finish the current step, save a cancel checkpoint, exit 143.-"""--import argparse-import gc-import json-import os-import signal-import sys-from pathlib import Path--os.environ.setdefault("PYTORCH_MPS_FAST_MATH", "1")--import numpy as np-import torch-from PIL import Image--sys.path.insert(0, str(Path(__file__).resolve().parent))-from _runner_common import ( # noqa: E402- empty_device_cache,- heartbeat,- install_hf_error_handler,- pick_device,-)--STOP_REQUESTED = False---def _on_sigterm(_sig, _frame):- global STOP_REQUESTED- STOP_REQUESTED = True- print("STATUS:cancel requested — finishing current step and checkpointing", flush=True)---signal.signal(signal.SIGTERM, _on_sigterm)---def log(msg: str) -> None:- print(msg, flush=True)---def parse_args():- p = argparse.ArgumentParser(description="FLUX.2 Klein LoRA trainer")- p.add_argument("--model-repo", required=True, help="bf16 base repo (black-forest-labs/FLUX.2-klein-4B|9B)")- p.add_argument("--manifest", required=True, help="JSON manifest: { triggerWord, images: [{ path, caption }] }")- p.add_argument("--output-dir", required=True)- p.add_argument("--trigger-word", required=True)- p.add_argument("--steps", type=int, default=1000)- p.add_argument("--rank", type=int, default=16)- p.add_argument("--lr", type=float, default=1e-4)- p.add_argument("--resolution", type=int, default=512)- p.add_argument("--checkpoint-every", type=int, default=250)- p.add_argument("--sample-every", type=int, default=250)- p.add_argument("--sample-prompt", default=None)- p.add_argument("--seed", type=int, default=42)- p.add_argument("--device", default="auto")- p.add_argument("--resume-from", default=None,- help="checkpoint dir to resume from (restores adapter + AdamW optimizer state + step offset from optimizer.pt)")- return p.parse_args()---def load_image_tensor(path: str, resolution: int) -> torch.Tensor:- """Open → RGB → shortest-side resize → center crop → [-1, 1] CHW tensor."""- img = Image.open(path).convert("RGB")- w, h = img.size- scale = resolution / min(w, h)- img = img.resize((max(resolution, round(w * scale)), max(resolution, round(h * scale))), Image.LANCZOS)- w, h = img.size- left, top = (w - resolution) // 2, (h - resolution) // 2- img = img.crop((left, top, left + resolution, top + resolution))- t = torch.from_numpy(np.asarray(img)).float() / 127.5 - 1.0- return t.permute(2, 0, 1).unsqueeze(0) # (1, 3, H, W)---def lora_target_modules(transformer) -> list:- """Collect attention-projection Linear module name suffixes that actually- exist in this transformer — robust to naming drift across diffusers- versions (to_q vs add_q_proj etc.)."""- wanted = ("to_q", "to_k", "to_v", "to_out.0", "add_q_proj", "add_k_proj", "add_v_proj", "to_add_out")- found = set()- for name, module in transformer.named_modules():- if not isinstance(module, torch.nn.Linear):- continue- if ".attn" not in name and "attention" not in name:- continue- for suffix in wanted:- if name.endswith(suffix):- found.add(suffix)- if not found:- raise RuntimeError("No attention projection Linears found to target with LoRA")- return sorted(found)---def save_checkpoint(pipe_cls, transformer, optimizer, out_dir: Path, step: int) -> Path:- """Persist the adapter for inference/promotion AND a self-contained resume- bundle (`optimizer.pt`) carrying the AdamW state, the PEFT adapter state, and- the step counter — so `--resume-from` continues mid-run (correct optimizer- momentum + starting-step offset) instead of warm-starting into a fresh- `args.steps`-long loop (which would over-train and re-collide step numbers).-- The adapter goes into the bundle as its PEFT state dict (not via the diffusers- loader) so resume can restore it into the SAME `default` adapter with- `set_peft_model_state_dict` — round-tripping `get_peft_model_state_dict`- exactly. Loading the `pytorch_lora_weights.safetensors` back through- `pipe.load_lora_weights()` would instead inject a SECOND adapter- (`get_adapter_name` returns `default_<n>` when one already exists), leaving the- trained weights in `default_1` while export reads `default` — silently wrong.- Saved on CPU; `load_state_dict` re-casts onto the fp32 on-device params."""- from peft.utils import get_peft_model_state_dict-- ckpt_dir = out_dir / "checkpoints" / f"step-{step:06d}"- ckpt_dir.mkdir(parents=True, exist_ok=True)- state = get_peft_model_state_dict(transformer)- pipe_cls.save_lora_weights(str(ckpt_dir), transformer_lora_layers=state)- torch.save(- {"optimizer": optimizer.state_dict(), "adapter": state, "step": step},- ckpt_dir / "optimizer.pt",- )- (ckpt_dir / "state.json").write_text(json.dumps({"step": step}))- return ckpt_dir--[email protected]_grad()-def render_sample(pipe, transformer, embeds, text_ids, resolution, device, dtype, seed, out_path: Path):- """Quick 8-step Euler flow-matching sample with the in-training adapter.- Best-effort — callers wrap in try/except; a failed sample never fails- the run."""- latent_h = 2 * (resolution // (pipe.vae_scale_factor * 2))- latent_w = 2 * (resolution // (pipe.vae_scale_factor * 2))- channels = pipe.transformer.config.in_channels // 4 * 4 # packed channel count- gen = torch.Generator(device="cpu").manual_seed(seed)- raw = torch.randn((1, channels, latent_h // 2, latent_w // 2), generator=gen).to(device, dtype)- latent_ids = pipe._prepare_latent_ids(raw).to(device)- latents = pipe._pack_latents(raw)- sigmas = torch.linspace(1.0, 0.0, 9)- for i in range(8):- sigma = sigmas[i]- t = sigma.expand(1).to(device, dtype)- pred = transformer(- hidden_states=latents.to(dtype),- timestep=t,- guidance=None,- encoder_hidden_states=embeds,- txt_ids=text_ids,- img_ids=latent_ids,- return_dict=False,- )[0]- latents = latents - (sigmas[i] - sigmas[i + 1]).to(device, dtype) * pred- unpacked = pipe._unpack_latents_with_ids(latents, latent_ids, latent_h // 2, latent_w // 2)- bn_mean = pipe.vae.bn.running_mean.view(1, -1, 1, 1).to(unpacked.device, unpacked.dtype)- bn_std = torch.sqrt(pipe.vae.bn.running_var.view(1, -1, 1, 1) + pipe.vae.config.batch_norm_eps).to(- unpacked.device, unpacked.dtype- )- unpacked = unpacked * bn_std + bn_mean- unpacked = pipe._unpatchify_latents(unpacked)- # Decode on the training device, not wherever precompute parked the VAE.- # Phase 1 deliberately leaves the VAE on CPU to keep it out of the training- # footprint, but the pipeline is bf16 on an accelerator — and x86 has no- # fast bf16 convolution path, so decoding there reads as a hang (a single- # 512-px decode ran >18 min on CUDA/Windows before this moved it back).- # The VAE is ~0.17 GB against a ~8 GB resident transformer, so borrowing the- # device for one decode is cheap.- # The move goes INSIDE the try: `.to()` walks parameters one at a time, so a- # throw partway leaves the VAE straddling both devices. Outside the try the- # `finally` would never run, and the next sample would read the now-`cuda`- # resting place as `vae_home` — silently latching it on-device for the rest- # of the run, on the card that just proved it has no headroom to spare.- vae_home = next(pipe.vae.parameters()).device- try:- pipe.vae.to(device)- image = pipe.vae.decode(unpacked.to(device, pipe.vae.dtype), return_dict=False)[0]- finally:- pipe.vae.to(vae_home)- image = pipe.image_processor.postprocess(image, output_type="pil")[0]- image.save(out_path)---@install_hf_error_handler-def main():- args = parse_args()- from diffusers import Flux2KleinPipeline- from peft import LoraConfig- from peft.utils import get_peft_model_state_dict-- device = pick_device(args.device)- # Apple-Silicon guard: this torch/diffusers trainer can't complete a LoRA- # backward on Metal. PyTorch's MPS backend has no `linear_backward` for the- # FLUX.2 transformer's Linear layers (raises "mps_linear_backward:- # unsupported weights data type" for both bf16 AND fp32 at the first- # loss.backward()). The preferred Apple-Silicon runtime is mflux (MLX-native,- # trains bf16 directly); this torch path is the fallback for CUDA/CPU boxes.- # Fail fast BEFORE loading the ~16 GB base + minutes of latent precompute- # rather than crashing at the first optimizer step. A forced `--device cpu`- # is still allowed (fp32 CPU backward works, just slow).- if device == "mps":- print(- "USER_ERROR:TRAINING_UNSUPPORTED_DEVICE:the torch FLUX.2 trainer can't train on "- "Apple Silicon (Metal/MPS) — PyTorch has no MPS linear_backward for these layers. "- "Use the mflux runtime instead: install mflux ≥0.17 via scripts/setup-image-video.sh. "- "(Force --device cpu to train on CPU, but that is impractically slow.)",- file=sys.stderr, flush=True,- )- sys.exit(1)- dtype = torch.bfloat16 if device != "cpu" else torch.float32- torch.manual_seed(args.seed)-- out_dir = Path(args.output_dir)- (out_dir / "samples").mkdir(parents=True, exist_ok=True)- (out_dir / "checkpoints").mkdir(parents=True, exist_ok=True)-- manifest = json.loads(Path(args.manifest).read_text())- images = manifest.get("images") or []- if not images:- print("USER_ERROR:DATASET_ERROR:manifest contains no images", file=sys.stderr, flush=True)- sys.exit(2)-- log("STAGE:load-pipeline")- with heartbeat("load-pipeline"):- pipe = Flux2KleinPipeline.from_pretrained(args.model_repo, torch_dtype=dtype)-- # Nothing in this pipeline is trainable until `add_adapter` introduces the- # LoRA below — say so once, here, rather than leaving it to whatever- # `from_pretrained` happened to return. This is the root-cause form of the- # precompute leak described further down: `requires_grad` is a property of- # the module, so clearing it makes the whole class of "inference forward- # silently built an autograd graph" unreachable from EVERY call site,- # present and future — where a `torch.no_grad()` block only protects the one- # region someone remembered to wrap. Both encoders take grad-free inputs- # (token ids; a tensor built from numpy), so this alone is sufficient; the- # `no_grad` below is kept as the belt to this pair of braces.- pipe.text_encoder.requires_grad_(False)- pipe.vae.requires_grad_(False)- pipe.transformer.requires_grad_(False)-- # ---- Phase 1: precompute text embeddings + image latents ----- log("STAGE:precompute-latents")- sample_prompt = args.sample_prompt or f"{args.trigger_word} portrait, neutral background"- examples = []- # `no_grad` is load-bearing here, not a micro-optimization — it is the- # second half of the `requires_grad_(False)` pair above. Left unguarded with- # trainable params, every encode builds and keeps a full autograd graph, and- # the `.to("cpu")` cache copy inherits that `grad_fn` and pins it on-device:- # ~2.6 GB of Qwen3 activations PER UNIQUE CAPTION on FLUX.2-klein-4B. A- # 128 GB unified-memory Mac absorbs that silently; a 24 GB CUDA card spills- # into driver shared memory (encode slows from 0.09 s to minutes) and then- # aborts with OOM at `transformer.to(device)` below. Keeping both is- # deliberate: `no_grad` additionally suppresses graph-building when an INPUT- # carries a `grad_fn`, which clearing the params cannot.- with torch.no_grad(), heartbeat("precompute-latents"):- pipe.text_encoder.to(device)- embed_cache = {}- for entry in images + [{"caption": sample_prompt, "path": None}]:- caption = entry["caption"]- if caption not in embed_cache:- embeds, text_ids = pipe.encode_prompt(caption, device=device)- embed_cache[caption] = (embeds.to("cpu"), text_ids.to("cpu"))- sample_embeds = embed_cache[sample_prompt]- # Free the text encoder — the make-or-break memory move on every- # backend: its ~8 GB has to be gone before the transformer moves on.- pipe.text_encoder.to("cpu")- pipe.text_encoder = None- gc.collect()- empty_device_cache(device)-- pipe.vae.to(device)- gen = torch.Generator(device="cpu").manual_seed(args.seed)- for i, entry in enumerate(images):- pixel = load_image_tensor(entry["path"], args.resolution).to(device, pipe.vae.dtype)- latents = pipe._encode_vae_image(pixel, generator=gen) # (1, C, H, W) normalized- latent_ids = pipe._prepare_latent_ids(latents)- packed = pipe._pack_latents(latents)- embeds, text_ids = embed_cache[entry["caption"]]- examples.append({- "packed": packed.to("cpu"),- "latent_ids": latent_ids.to("cpu"),- "embeds": embeds,- "text_ids": text_ids,- })- log(f"STATUS:encoded {i + 1}/{len(images)} dataset images")- # VAE only needed again for samples — keep it, but off-device.- pipe.vae.to("cpu")- gc.collect()- empty_device_cache(device)-- # ---- Phase 2: training ----- log("STAGE:training")- transformer = pipe.transformer # already frozen at load, with the encoders- targets = lora_target_modules(transformer)- log(f"STATUS:LoRA targets: {', '.join(targets)} (rank {args.rank})")- # Always create the single trainable `default` adapter (gaussian init). On- # resume its weights are overwritten in-place below from the checkpoint's- # PEFT state dict — NOT via pipe.load_lora_weights(), which would inject a- # second `default_<n>` adapter and leave training/export out of sync.- transformer.add_adapter(LoraConfig(- r=args.rank,- lora_alpha=args.rank,- init_lora_weights="gaussian",- target_modules=targets,- ))- transformer.enable_gradient_checkpointing()- transformer.to(device)- transformer.train()-- trainable = [p for p in transformer.parameters() if p.requires_grad]- # LoRA params train in fp32 for AdamW stability even when the base is bf16.- for p in trainable:- p.data = p.data.to(torch.float32)- optimizer = torch.optim.AdamW(trainable, lr=args.lr, weight_decay=0.01)-- # Resume: restore the adapter weights, AdamW state, and a starting-step- # offset from the checkpoint's optimizer.pt bundle, so the loop continues- # toward the ORIGINAL total (range(start_step + 1, args.steps + 1)) with warm- # momentum instead of re-running a fresh args.steps from a random adapter.- # The adapter is loaded into the existing `default` adapter via- # set_peft_model_state_dict (in-place — param identities, and therefore the- # optimizer state alignment, are preserved). The server only hands us- # checkpoints that carry optimizer.pt (resolveLatestCheckpointArtifact gates- # on it); a checkpoint that predates optimizer-state resume is a hard error- # rather than a silent cold-start onto the random gaussian adapter. (RNG- # state is intentionally not restored; the per-step noise/timestep sequence- # restarts from the seed, immaterial to quality — the load-bearing fix is- # adapter + optimizer + step.)- start_step = 0- if args.resume_from:- from peft.utils import set_peft_model_state_dict-- opt_file = Path(args.resume_from) / "optimizer.pt"- if not opt_file.exists():- print(- f"USER_ERROR:TRAINING_FAILED:resume checkpoint has no optimizer state ({opt_file}) — "- "it predates optimizer-state resume; start a fresh run",- file=sys.stderr, flush=True,- )- sys.exit(1)- ckpt = torch.load(opt_file, map_location="cpu")- set_peft_model_state_dict(transformer, ckpt["adapter"], adapter_name="default")- optimizer.load_state_dict(ckpt["optimizer"])- start_step = int(ckpt.get("step", 0))- log(f"STATUS:resumed adapter + optimizer state — continuing from step {start_step}")- if start_step >= args.steps:- log(f"STATUS:resume point (step {start_step}) already at/past target {args.steps} — nothing to train")-- order = list(range(len(examples)))- rng = torch.Generator().manual_seed(args.seed)- final_loss = None- last_checkpoint = None-- for step in range(start_step + 1, args.steps + 1):- if (step - 1) % len(order) == 0:- order = torch.randperm(len(examples), generator=rng).tolist()- ex = examples[order[(step - 1) % len(order)]]- packed = ex["packed"].to(device, dtype)- latent_ids = ex["latent_ids"].to(device)- embeds = ex["embeds"].to(device, dtype)- text_ids = ex["text_ids"].to(device)-- noise = torch.randn_like(packed)- # Logit-normal timestep sampling (FLUX training recipe).- sigma = torch.sigmoid(torch.randn((1,), generator=rng)).to(device, dtype)- noisy = (1.0 - sigma) * packed + sigma * noise- target = noise - packed-- pred = transformer(- hidden_states=noisy,- timestep=sigma.expand(1),- guidance=None,- encoder_hidden_states=embeds,- txt_ids=text_ids,- img_ids=latent_ids,- return_dict=False,- )[0]- loss = torch.nn.functional.mse_loss(pred.float(), target.float())- loss.backward()- torch.nn.utils.clip_grad_norm_(trainable, 1.0)- optimizer.step()- optimizer.zero_grad(set_to_none=True)-- final_loss = float(loss.detach().item())- log(f"STEP:{step}:{args.steps}:{final_loss:.4f}")-- if STOP_REQUESTED:- ckpt = save_checkpoint(Flux2KleinPipeline, transformer, optimizer, out_dir, step)- log(f"CHECKPOINT:{ckpt}:{step}")- log("STATUS:canceled-checkpoint-saved")- os._exit(143)-- if args.checkpoint_every > 0 and step % args.checkpoint_every == 0 and step < args.steps:- ckpt = save_checkpoint(Flux2KleinPipeline, transformer, optimizer, out_dir, step)- last_checkpoint = ckpt- log(f"CHECKPOINT:{ckpt}:{step}")-- if args.sample_every > 0 and step % args.sample_every == 0:- sample_path = out_dir / "samples" / f"step-{step:06d}.png"- transformer.eval()- try:- render_sample(- pipe, transformer, sample_embeds[0].to(device, dtype), sample_embeds[1].to(device),- args.resolution, device, dtype, args.seed, sample_path,- )- log(f"SAMPLE:{sample_path}:{step}")- except Exception as err: # noqa: BLE001 — samples are best-effort, never fail the run- log(f"STATUS:sample render failed (training continues): {err}")- finally:- transformer.train()- empty_device_cache(device)-- # ---- Finalize ----- adapter_dir = out_dir / "adapter"- adapter_dir.mkdir(parents=True, exist_ok=True)- state = get_peft_model_state_dict(transformer)- Flux2KleinPipeline.save_lora_weights(str(adapter_dir), transformer_lora_layers=state)- adapter_path = adapter_dir / "pytorch_lora_weights.safetensors"- if not adapter_path.exists():- candidates = list(adapter_dir.glob("*.safetensors"))- if not candidates:- print("USER_ERROR:TRAINING_FAILED:adapter save produced no .safetensors", file=sys.stderr, flush=True)- sys.exit(1)- adapter_path = candidates[0]- log("RESULT:" + json.dumps({- "adapter_path": str(adapter_path),- "steps": args.steps,- "final_loss": final_loss,- "last_checkpoint": str(last_checkpoint) if last_checkpoint else None,- }))- # Hard-exit defense against teardown hangs (mirrors the video runtimes).- sys.stdout.flush()- sys.stderr.flush()- os._exit(0)---if __name__ == "__main__":- main()diff --git a/scripts/train_flux2_lora_test.py b/scripts/train_flux2_lora_test.pydeleted file mode 100644index 1efddd4c04..0000000000--- a/scripts/train_flux2_lora_test.py+++ /dev/null@@ -1,157 +0,0 @@-#!/usr/bin/env python3-"""Standalone tests for train_flux2_lora.py's memory-discipline invariants.--Run: python3 scripts/train_flux2_lora_test.py-Exits non-zero if any check fails. Mirrors the runnable-test style of-scripts/train_mflux_lora_test.py.--These are AST checks on the source rather than behavioral tests, deliberately:-exercising the real paths needs torch, a ~16 GB base download and a CUDA card,-which no test suite can carry. So they pin *structural* invariants only — each-one chosen because breaking it silently reintroduces a fault that made this-trainer unusable on a 24 GB card, and because no reasonable refactor breaks it-without also breaking the invariant. Background and measurements: issue #2786-and docs/research/2026-08-05-torch-flux2-lora-cuda-validation.md.-"""-import ast-import sys-from pathlib import Path--# The ✅/❌ prefixes are cp1252-unencodable on a Windows console, which would-# abort the run before the first result printed. This trainer's whole point is-# the CUDA path, and that is where Windows lives — so force UTF-8 on stdout.-if hasattr(sys.stdout, "reconfigure"):- sys.stdout.reconfigure(encoding="utf-8", errors="replace")--SRC = Path(__file__).resolve().parent / "train_flux2_lora.py"-TREE = ast.parse(SRC.read_text(encoding="utf-8"))--FAILS = []---def check(name, cond):- print(("✅" if cond else "❌") + " " + name)- if not cond:- FAILS.append(name)---def func(name):- """The named top-level function, or exit — a missing function would make- every check about it vacuously pass."""- for node in ast.walk(TREE):- if isinstance(node, ast.FunctionDef) and node.name == name:- return node- sys.exit(f"❌ {name}() not found in {SRC.name} — the file was restructured")---def calls(node):- """Every call under `node` as (dotted_name, Call), e.g. pipe.vae.to."""- out = []- for n in ast.walk(node):- if isinstance(n, ast.Call):- parts, f = [], n.func- while isinstance(f, ast.Attribute):- parts.append(f.attr)- f = f.value- if isinstance(f, ast.Name):- parts.append(f.id)- out.append((".".join(reversed(parts)), n))- return out---def names_in(call):- """Identifiers passed to `call`, positionally or by keyword — so a check- isn't defeated by `.to(device=device)` vs `.to(device)`."""- args = list(call.args) + [k.value for k in call.keywords]- return {a.id for a in args if isinstance(a, ast.Name)}---main = func("main")-main_calls = calls(main)--# --- 1. inference-only modules are frozen at load -----------------------------# The root-cause form of the precompute leak: `requires_grad` is a property of-# the module, so clearing it makes "an inference forward silently built an-# autograd graph" unreachable from every call site, not just the wrapped one.-frozen = {- name.rsplit(".", 2)[-2]- for name, c in main_calls- if name.endswith(".requires_grad_")- and c.args and isinstance(c.args[0], ast.Constant) and c.args[0].value is False-}-for module in ("text_encoder", "vae", "transformer"):- check(f"main() freezes {module} with requires_grad_(False) at load "- f"(root-cause guard for the ~2.6 GB/caption leak — issue #2786)",- module in frozen)--# --- 2. precompute's inference runs under torch.no_grad() ---------------------# Belt to the braces above. Located by walking every no_grad block in main()-# rather than by matching one `with` statement's shape, so nesting the guard or-# extracting precompute into its own block keeps passing.-no_grad_blocks = [- n for n in ast.walk(main)- if isinstance(n, ast.With) and any(- isinstance(i.context_expr, ast.Call)- and getattr(i.context_expr.func, "attr", None) == "no_grad"- for i in n.items- )-]-check("main() has a torch.no_grad() block", bool(no_grad_blocks))-for fn in ("encode_prompt", "_encode_vae_image"):- check(f"precompute's {fn} runs under torch.no_grad()",- any(any(n.endswith(fn) for n, _ in calls(b)) for b in no_grad_blocks))--# --- 3. render_sample decodes on the training device, and always restores -----# Precompute parks the VAE on CPU, but the pipeline is bf16 and x86 has no fast-# bf16 conv path, so decoding there reads as a hang (>18 min for one 512-px-# decode). The restore must be in a `finally`: the caller treats samples as-# best-effort and swallows exceptions, so a throw mid-decode would otherwise-# strand the VAE on-device for the rest of the run.-rs = func("render_sample")-decode_try = None-for node in ast.walk(rs):- if isinstance(node, ast.Try) and any(- n.endswith("vae.decode") for n, _ in calls(node)- ):- decode_try = node--check("render_sample wraps vae.decode in a try (issue #2786)", decode_try is not None)--if decode_try:- check("render_sample moves the VAE onto `device` inside that try "- "(bf16 conv on CPU hangs — issue #2786)",- any(n.endswith("vae.to") and "device" in names_in(c)- for n, c in calls(ast.Module(body=decode_try.body, type_ignores=[]))))- check("render_sample restores the VAE to `vae_home` in the try's finally",- any(n.endswith("vae.to") and "vae_home" in names_in(c)- for n, c in calls(ast.Module(body=decode_try.finalbody, type_ignores=[]))))--# --- 4. cache clearing goes through the shared, device-agnostic helper --------# The original bug was an MPS-only `torch.mps.empty_cache()`, so a bare clear of-# EITHER backend is the regression — banning only the MPS spelling would let the-# mirror image straight back in.-module_calls = [n for n, _ in calls(TREE)]-check("no bare torch.mps/cuda.empty_cache anywhere in the module — "- "clears route through _runner_common.empty_device_cache",- not ({"torch.mps.empty_cache", "torch.cuda.empty_cache"} & set(module_calls)))-check("the trainer actually calls empty_device_cache",- "empty_device_cache" in module_calls)-imported_from_common = any(- isinstance(n, ast.ImportFrom) and n.module == "_runner_common"- and any(a.name == "empty_device_cache" for a in n.names)- for n in ast.walk(TREE)-)-redeclared_locally = any(- isinstance(n, ast.FunctionDef) and n.name == "empty_device_cache"- for n in ast.walk(TREE)-)-check("empty_device_cache comes from _runner_common, not a local re-declaration",- imported_from_common and not redeclared_locally)--print()-if FAILS:- print(f"❌ {len(FAILS)} check(s) failed:")- for f in FAILS:- print(f" - {f}")- sys.exit(1)-print("✅ all train_flux2_lora.py invariants hold")diff --git a/scripts/train_mflux_lora.py b/scripts/train_mflux_lora.pydeleted file mode 100644index 3a1264dbb3..0000000000--- a/scripts/train_mflux_lora.py+++ /dev/null@@ -1,793 +0,0 @@-#!/usr/bin/env python3-"""mflux FLUX.2 LoRA training wrapper (PortOS).--Wraps mflux's own trainer (`mflux-train`, mflux ≥0.17 — FLUX.1 training-was removed upstream) in a subprocess and translates its native output-into the PortOS trainer line protocol (see train_flux2_lora.py header /-server/services/loraTraining/progress.js). We deliberately do NOT import-mflux's training internals in-process — their API churns across versions;-the CLI + config JSON is the stable surface (same wrap-don't-import-strategy as generate_wan22.py).--Verified against mflux 0.17.5:- - CLI: `mflux-train --config <json>` (resume: `--resume <checkpoint.zip>`).- - Progress: a tqdm bar over total iterations (epochs × images), written- with carriage returns — the reader splits on \\r AND \\n.- - Outputs under the config's checkpoint.output_path:- checkpoints/NNNNNNN_checkpoint.zip (adapter + optimizer + config)- preview/NNNNNNN_preview_*.png (when monitoring is enabled)- loss/loss.html- - The trained adapter inside each zip is `NNNNNNN_adapter.safetensors`- with diffusers-style key naming (`transformer.<module>.lora_A.weight`),- so the extracted file loads directly in diffusers flux2 pipelines.--SIGTERM → forwarded to the mflux child; exit 143 after it stops (mflux-checkpoints at save_frequency boundaries; mid-interval cancels keep the-last saved checkpoint).--Segmented training (watchdog-panic mitigation). When `--segment-steps N`-is passed (>0), the run is broken into N-step segments instead of one-sustained mflux process: train to a checkpoint boundary, **terminate the-mflux child so its Metal/GPU context is fully torn down**, sleep-`--cooldown-sec` to let the GPU clock down + the system settle, then-`--resume` the newest checkpoint for the next segment. mflux resume reads-`num_epochs` from the checkpoint (the config value is ignored on resume),-so each segment continues toward the *same* total — segmentation is-numerically equivalent to one continuous run, just with periodic-full-process teardown. This directly attacks the macOS GPU-watchdog-timeout kernel panics that hard-reboot the machine during long-sustained mflux runs (see docs/research/2026-06-13-mflux-training-watchdog-panic.md):-sustained GPU pressure never spans more than one segment, and any-accumulated driver/thermal state resets between segments. `--segment-steps 0`-(the default) restores the single-process behavior.-"""--import argparse-import atexit-import json-import os-import re-import shutil-import signal-import subprocess-import sys-import threading-import time-import zipfile-from collections import deque-from datetime import datetime-from pathlib import Path--# macOS GPU-watchdog mitigation (mlx #3267). The Metal/AGX driver kills sustained-# LoRA training under the GPU watchdog ("Impacting Interactivity" → process kill,-# and on M5-class silicon escalating to a `watchdogd` kernel panic + hard reboot;-# see docs/research/2026-06-13-mflux-training-watchdog-panic.md). The MLX-# maintainer's confirmed workaround is to relax the AGX CDM context-store timeout-# so a long command buffer isn't treated as a stuck/interactivity-impacting one.-# It is read by the driver at Metal context creation, so it MUST be set before-# mlx initializes — we set it here at module load, and since the mflux-train-# child Popen (below) inherits this process env, the child gets it too. setdefault-# lets an explicit override (or =0 to disable) still win. NOTE: per mlx #3267 the-# kill is at the IOGPU layer *above* the process boundary, so process-teardown-# segmentation alone does NOT prevent it — this env var attacks the actual cause.-os.environ.setdefault("AGX_RELAX_CDM_CTXSTORE_TIMEOUT", "1")--CHILD = None-STOP_REQUESTED = False--# How many of the trainer's recent non-progress output lines to retain and-# replay to stderr on a non-zero exit (so the Node failure classifier sees the-# real error, not just the exit code). See the loop in main().-OUTPUT_TAIL_LINES = 40--# Packages whose resolved versions form the LoRA-training stack fingerprint. The-# macOS GPU watchdog kernel panics (docs/research/2026-06-13-mflux-training-watchdog-panic.md)-# are version-specific to this trio + mlx-lm, so emitting them at startup makes-# every run log and any future paniclog self-document the exact stack it ran on-# (mirrors the video-gen runtime fingerprint — _runner_common.emit_runtime_fingerprint).-RUNTIME_FINGERPRINT_PACKAGES = ["mflux", "mlx", "mlx-metal", "mlx-lm"]---def log(msg: str) -> None:- print(msg, flush=True)---def format_runtime_fingerprint_line(fp: dict) -> str:- """Render a build_runtime_fingerprint() dict to the one-line STATUS string- the trainer logs at startup. Pure (no I/O) so it is unit-testable without a- real mflux/mlx install. Absent packages are simply omitted from `versions`- by the builder, so they never appear here."""- versions = fp.get("versions") or {}- parts = [f"{name} {ver}" for name, ver in versions.items()] or ["versions unavailable"]- for field in ("chip", "os", "python"):- val = fp.get(field)- if val:- parts.append(f"{field} {val}" if field == "python" else str(val))- return "STATUS:runtime · " + " · ".join(parts)---def emit_runtime_fingerprint() -> dict:- """Log a one-line runtime fingerprint (mflux/mlx/mlx-metal/mlx-lm versions +- Apple chip + macOS + python) at trainer startup. Best-effort: a fingerprint- failure must never abort a training run, so all of it is guarded. Reuses the- shared _runner_common.build_runtime_fingerprint so the trainer log and the- video-gen /status probe never drift. Returns the dict it emitted (or {})."""- try:- sys.path.insert(0, str(Path(__file__).resolve().parent))- from _runner_common import build_runtime_fingerprint # noqa: PLC0415-- fp = build_runtime_fingerprint("mflux-lora", RUNTIME_FINGERPRINT_PACKAGES)- log(format_runtime_fingerprint_line(fp))- return fp- except Exception as err: # pragma: no cover - defensive; never abort a run- log(f"STATUS:runtime fingerprint unavailable ({err})")- return {}---def _on_sigterm(_sig, _frame):- global STOP_REQUESTED- STOP_REQUESTED = True- log("STATUS:cancel requested — stopping mflux trainer")- if CHILD and CHILD.poll() is None:- CHILD.terminate()---signal.signal(signal.SIGTERM, _on_sigterm)---def parse_args():- p = argparse.ArgumentParser(description="mflux LoRA training wrapper")- p.add_argument("--config", required=True, help="mflux-train config JSON")- p.add_argument("--output-dir", required=True, help="PortOS run dir (samples/ lives here)")- p.add_argument("--total-steps", type=int, default=1000,- help="expected total iterations — segment planning + logging sanity")- p.add_argument("--resume-checkpoint", default=None, help="checkpoint zip to resume from")- p.add_argument("--segment-steps", type=int, default=0,- help="0 = single sustained run; >0 = train in N-step segments, tearing "- "down the GPU child + cooling down between each (watchdog-panic mitigation)")- p.add_argument("--cooldown-sec", type=int, default=90,- help="seconds to idle the GPU between segments (only with --segment-steps>0)")- return p.parse_args()---# Checkpoint zips are named NNNNNNN_checkpoint.zip where NNNNNNN is the-# trainer's cumulative iteration count — the canonical "where did we get to".-CKPT_RE = re.compile(r"(\d+)_checkpoint\.zip$")---def _checkpoint_step(path) -> int:- if not path:- return 0- m = CKPT_RE.search(Path(path).name)- return int(m.group(1)) if m else 0---def newest_checkpoint(configured_output: Path):- """Newest checkpoint zip under the (possibly timestamp-suffixed) mflux- output dir. mflux `--resume` writes back to the SAME output_path- (new_folder=False), so checkpoints accumulate in one dir across segments;- ranking by embedded step keeps the pick monotonic even if mtimes tie."""- cdir = resolve_mflux_output(configured_output) / "checkpoints"- if not cdir.is_dir():- return None- zips = list(cdir.glob("*_checkpoint.zip"))- if not zips:- return None- return max(zips, key=lambda z: (_checkpoint_step(z), z.stat().st_mtime))---def _terminate_child(timeout: float = 15.0) -> None:- """SIGTERM the mflux child and reap it (escalate to SIGKILL if it lingers).- Used at segment boundaries to fully release the Metal/GPU context."""- global CHILD- if CHILD and CHILD.poll() is None:- CHILD.terminate()- try:- CHILD.wait(timeout=timeout)- except subprocess.TimeoutExpired:- CHILD.kill()---# Cooldown heartbeat cadence. The mediaJobQueue idle watchdog resets only on-# emitted runner output (progress.js emits 'activity' for every non-noise line),-# so a silent cooldown longer than the watchdog window would get a healthy run-# killed mid-pause. segmentCooldownSec is validated up to 3600s — above the 1800s-# default watchdog — so the cooldown MUST keep the watchdog fed itself rather than-# rely on the setting staying small.-HEARTBEAT_INTERVAL_SEC = 30---def interruptible_sleep(seconds: float, heartbeat_stage: str | None = None) -> None:- """Sleep up to `seconds`, returning early if a SIGTERM cancel arrives — so- the inter-segment cooldown never delays a user-requested stop. When- `heartbeat_stage` is set, emit a `STAGE:<stage>:heartbeat:<elapsed>s`- keep-alive every HEARTBEAT_INTERVAL_SEC so the idle watchdog sees activity- during a long pause (the `:heartbeat:` form resets the watchdog without- emitting a status message or changing the displayed stage)."""- start = time.monotonic()- end = start + seconds- next_beat = start + HEARTBEAT_INTERVAL_SEC- while not STOP_REQUESTED:- now = time.monotonic()- remaining = end - now- if remaining <= 0:- return- if heartbeat_stage and now >= next_beat:- log(f"STAGE:{heartbeat_stage}:heartbeat:{int(now - start)}s")- next_beat = now + HEARTBEAT_INTERVAL_SEC- time.sleep(min(1.0, remaining))---def build_command(config_path: str, resume: str | None) -> list:- """`mflux-train` console script next to THIS interpreter (no PATH- ambiguity); fallback to the module behind it."""- console = Path(sys.executable).parent / "mflux-train"- if console.exists():- cmd = [str(console)]- else:- probe = subprocess.run(- [sys.executable, "-c",- "import importlib.util as u; raise SystemExit(0 if u.find_spec('mflux.models.common.cli.train') else 1)"],- capture_output=True,- )- if probe.returncode != 0:- print(- "USER_ERROR:MODULE_NOT_FOUND:mflux trainer not found (no mflux-train script and no "- "mflux.models.common.cli.train module) — update mflux to ≥0.17",- file=sys.stderr, flush=True,- )- sys.exit(2)- cmd = [sys.executable, "-m", "mflux.models.common.cli.train"]- if resume:- cmd += ["--resume", resume]- else:- cmd += ["--config", config_path]- return cmd---# tqdm renders like ` 3%|▎ | 12/400 [00:30<16:20, 2.53s/it]`.-TQDM_RE = re.compile(r"(\d+)\s*/\s*(\d+)\s*\[")-# Belt-and-suspenders for any explicit "step N/M ... loss X" prose a future-# mflux version might print.-STEP_LOSS_RE = re.compile(r"[Ss]teps?[\s:=]+(\d+)\s*/\s*(\d+).*?[Ll]oss[\s:=]+([\d.eE+-]+)")-NOISE_RE = re.compile(r"FutureWarning|UserWarning|DeprecationWarning", re.I)--# We merge the mflux child's stderr into its stdout (so tqdm + prints stay in-# order), which means the Node side tags every line `stream='stdout'` and the-# JS failure classifier's stderr-tail stays empty for mflux runs. To keep-# OOM / missing-module / gated-repo errors classifiable, sniff them out of the-# merged stream and re-emit a structured USER_ERROR line (which the Node line-# parser captures regardless of stream). Codes mirror failure.js.-FATAL_PATTERNS = [- ("OOM", re.compile(r"out of memory|Insufficient Memory|Metal.*out of memory|std::bad_alloc", re.I)),- ("MODULE_NOT_FOUND", re.compile(r"ModuleNotFoundError|No module named", re.I)),- ("HF_AUTH", re.compile(r"GatedRepoError|401 Client Error|is restricted|Repo.*is gated", re.I)),-]-# NOTE: argparse rejections (a too-old mflux that wants `--train-config` / has-# no flux2 base models — the exact "exited with code 2" failure) are-# deliberately NOT sniffed here. A USER_ERROR line short-circuits-# classifyTrainingFailure on its *raw* text (failure.js), so sniffing the-# argparse line would surface "unrecognized arguments: --config …" instead of-# the actionable "upgrade mflux>=0.17" hint. Left unsniffed, the line rides the-# non-zero-exit tail replay to stderr and failure.js's CLI_MISMATCH_RE renders-# the actionable message — one source of truth for that text, on the JS side.---def is_preview_progress_bar(line, cur, total, *, has_step_loss, preview_steps, training_total):- """True if a matched tqdm bar belongs to the preview/sample IMAGE generation- rather than the training loop.-- mflux runs `adapter.generate_preview_image` (→ `flux2.generate_image`) once- before training (step 0) and again at every sample interval. That diffusion- sampler prints its own tqdm bar whose total is the DENOISE step count- (`config["steps"]`, e.g. 40) — identical in shape to the training bar except- the total differs (training total = epochs × images). If we don't exclude it- the wrapper both mislabels denoise ticks as training STEPs and lets them- perturb `state["step"]`, which gates the segment-boundary kill (a preview bar- reading 38/40 could spuriously satisfy a 150-step boundary).-- Discriminators, all required:- - `has_step_loss` False — the explicit "step N/M … loss X" prose is emitted- only by the training loop, never the sampler, so it's never a preview.- - `preview_steps` known (caller passes None when steps == training_total,- the degenerate tiny-dataset case where totals can't disambiguate).- - the bar's total equals `preview_steps` and differs from `training_total`.- """- if has_step_loss or preview_steps is None:- return False- return total == preview_steps and (training_total is None or total != training_total)---def resolve_mflux_output(configured: Path) -> Path:- """mflux appends `_YYYYMMDD_HHMMSS` to checkpoint.output_path when the- configured dir already exists (its new-run-folder behavior) — a killed- previous attempt leaving the dir behind is enough to trigger it. Pick- the newest candidate among the configured path and its timestamped- siblings so the watcher + adapter discovery track where mflux actually- wrote."""- candidates = [p for p in [configured, *configured.parent.glob(f"{configured.name}_2*")] if p.is_dir()]- if not candidates:- return configured- return max(candidates, key=lambda p: p.stat().st_mtime)---class ArtifactWatcher(threading.Thread):- """Poll mflux's output dir: new checkpoint zips → CHECKPOINT lines; new- preview PNGs → copied into the run's samples/ (where the server's- sample route serves from) + SAMPLE lines. The output dir is re-resolved- each scan — it may materialize (possibly timestamp-suffixed) only after- mflux finishes loading the model."""-- def __init__(self, configured_output: Path, samples_dir: Path, get_step):- super().__init__(daemon=True)- self.configured_output = configured_output- self.samples_dir = samples_dir- self.get_step = get_step- self.seen = set()- self.stop_event = threading.Event()-- def scan(self):- output = resolve_mflux_output(self.configured_output)- checkpoints_dir = output / "checkpoints"- preview_dir = output / "preview"- for f in sorted(checkpoints_dir.glob("*.zip")) if checkpoints_dir.exists() else []:- if f in self.seen:- continue- self.seen.add(f)- log(f"CHECKPOINT:{f}:{self.get_step()}")- for f in sorted(preview_dir.glob("*.png")) if preview_dir.exists() else []:- if f in self.seen:- continue- self.seen.add(f)- dest = self.samples_dir / f.name- try:- shutil.copyfile(f, dest)- log(f"SAMPLE:{dest}:{self.get_step()}")- except OSError as err:- log(f"STATUS:sample copy failed: {err}")-- def run(self):- while not self.stop_event.wait(5.0):- self.scan()- self.scan() # final sweep after the child exits---class TelemetrySidecar:- """Optional GPU/thermal/power capture for crash forensics.-- macOS GPU watchdog-timeout kernel panics during sustained mflux training- (see docs/TROUBLESHOOTING.md "GPU watchdog kernel panic" + the incident- record under docs/research) hard-reboot the machine mid-run and leave NO- application-level record of GPU temperature / power draw leading up to the- hang. This streams `powermetrics` into <output_dir>/powermetrics.log so a- post-crash investigation can tell a thermal/power fault from a driver hang.-- powermetrics requires root. We gate on a *non-interactive* `sudo -n -l`- probe of powermetrics specifically: if passwordless sudo for that command- isn't configured the sidecar no-ops with a single STATUS note and training- proceeds unaffected — it never prompts or blocks.- """-- SAMPLE_INTERVAL_MS = 5000-- def __init__(self, output_dir: Path):- # Don't clobber a prior run's telemetry on resume — the post-crash- # investigation reads it from this dir, and `powermetrics --output-file`- # truncates. Roll to a timestamped name when the default already exists- # so each launch keeps its own forensic log.- base = output_dir / "powermetrics.log"- if base.exists():- base = output_dir / f"powermetrics.{datetime.now():%Y%m%d_%H%M%S}.log"- self.log_path = base- self.proc = None-- def start(self):- if sys.platform != "darwin":- return- pm = shutil.which("powermetrics")- if not pm:- return- # Probe whether *powermetrics specifically* is permitted under- # passwordless sudo. The documented sudoers rule grants NOPASSWD for- # powermetrics only, so a generic `sudo -n true` probe (which runs- # /usr/bin/true) would fail for exactly the users who configured it- # correctly. `sudo -n -l <cmd>` lists the permission without running- # powermetrics and without prompting; non-zero means not allowed.- # Telemetry is best-effort — a missing `sudo` must never crash training.- try:- probe = subprocess.run(["sudo", "-n", "-l", pm], capture_output=True)- except OSError as err:- log(f"STATUS:GPU telemetry unavailable ({err}); continuing without it")- return- if probe.returncode != 0:- log("STATUS:GPU telemetry disabled — passwordless sudo for powermetrics not configured; "- "see docs/TROUBLESHOOTING.md to enable thermal/power capture. Continuing without it.")- return- cmd = [- "sudo", "-n", pm,- "--samplers", "cpu_power,gpu_power,thermal",- "-i", str(self.SAMPLE_INTERVAL_MS),- "--output-file", str(self.log_path),- ]- try:- self.proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)- except OSError as err:- log(f"STATUS:GPU telemetry unavailable ({err}); continuing without it")- self.proc = None- return- # Confirm powermetrics actually stayed up — a bad sampler, an OS that- # dropped a sampler, or a perms issue makes it exit immediately, and we- # must not report telemetry as enabled while the log stays empty.- try:- rc = self.proc.wait(timeout=0.5)- log(f"STATUS:GPU telemetry failed to start (powermetrics exited {rc}); "- "continuing without it")- self.proc = None- return- except subprocess.TimeoutExpired:- pass # still running after the grace window → healthy- # Guarantee the root powermetrics child is reaped even if main() throws- # between start() and the explicit stop() — unlike the daemon watcher- # thread, this is a separate OS process that outlives the interpreter.- # stop() is idempotent (poll() guard), so the later explicit call is safe.- atexit.register(self.stop)- log(f"STATUS:GPU telemetry → {self.log_path} (powermetrics @ {self.SAMPLE_INTERVAL_MS}ms)")-- def stop(self):- if not self.proc or self.proc.poll() is not None:- return- # sudo forwards SIGTERM to powermetrics; escalate if it lingers.- self.proc.terminate()- try:- self.proc.wait(timeout=5)- except subprocess.TimeoutExpired:- self.proc.kill()---def find_adapter(checkpoints_dir: Path, output_dir: Path) -> Path | None:- """Extract the lora adapter (`*_adapter.safetensors`, NOT the optimizer- state) from the newest checkpoint zip."""- zips = sorted(checkpoints_dir.glob("*.zip"), key=lambda f: f.stat().st_mtime)- for z in reversed(zips):- with zipfile.ZipFile(z) as zf:- members = [m for m in zf.namelist()- if m.endswith("_adapter.safetensors") and "optimizer" not in m]- if not members:- continue- extract_dir = output_dir / "adapter"- extract_dir.mkdir(parents=True, exist_ok=True)- zf.extract(members[-1], extract_dir)- return extract_dir / members[-1]- return None---def stream_lines(pipe):- """Yield logical lines split on BOTH \\n and \\r — tqdm redraws its bar- with carriage returns, so newline-only iteration would buffer the whole- bar until completion."""- buf = ""- while True:- chunk = pipe.read(256)- if not chunk:- if buf:- yield buf- return- buf += chunk- while True:- cut = min((i for i in (buf.find("\n"), buf.find("\r")) if i >= 0), default=-1)- if cut < 0:- break- yield buf[:cut]- buf = buf[cut + 1:]---def compute_effective_total(config: dict, fallback: int) -> int:- """mflux's REAL iteration total = ceil(imageCount / batch_size) * num_epochs- — identical to Iterator.total_number_of_steps(). The wrapper's --total-steps- is the user's *requested* count, which mflux rounds up to whole epochs and- can exceed (e.g. 600 requested with 240 images → round(600/240)=3 epochs →- 720). Segment planning MUST use this real total, or the final segment spans- from the last requested-total boundary all the way to the real end in one- sustained process — exactly the unbounded GPU window this patch removes.-- Deterministic and known before launch (the config carries num_epochs +- batch_size and the staged dataset dir). Falls back to the requested count if- the config shape is unexpected; tqdm then corrects state["total"] live after- the first segment anyway."""- try:- loop = config.get("training_loop", {}) or {}- epochs = int(loop.get("num_epochs", 0))- batch = int(loop.get("batch_size", 1)) or 1- data_dir = config.get("data")- # mflux auto-discovers NNNN.png + NNNN.txt training pairs; preview prompts- # are .txt only, so *.png count is the image count it will encode.- n_images = len(list(Path(data_dir).glob("*.png"))) if data_dir else 0- if epochs > 0 and n_images > 0:- batches_per_epoch = (n_images + batch - 1) // batch- return batches_per_epoch * epochs- except (ValueError, TypeError, OSError):- pass- return fallback---def run_segment(cmd, *, segment_target, state, last_reported, output_tail,- fatal_holder, configured_output, cooldown,- preview_steps=None, training_total=None):- """Run ONE mflux invocation and stream its output.-- Returns a dict with `reason`:- - 'segment' — reached `segment_target`; the child was terminated after- the boundary checkpoint was confirmed on disk. `checkpoint`- holds the zip to resume the next segment from.- - 'completed' — the child exited 0 on its own (training finished).- - 'stopped' — a SIGTERM cancel arrived; caller should exit 143.- - 'error' — the child exited non-zero; `code` carries the exit code.-- `segment_target=None` means "let mflux run to completion" (final/only- segment) — no boundary kill is armed.- """- global CHILD- # `cmd` is always list-form (see build_command) and `shell` stays at its- # False default, so argv never reaches a shell — there is no injection- # surface here. Scanners periodically flag this line anyway; an explicit- # `shell=False` would be a no-op. See PR #2882 (closed).- CHILD = subprocess.Popen(- cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,- )- if STOP_REQUESTED: # SIGTERM raced the spawn- CHILD.terminate()-- in_training = False- preview_logged = {"v": False} # one STAGE:sampling per contiguous preview run- for raw in stream_lines(CHILD.stdout):- line = raw.strip()- if not line or NOISE_RE.search(line):- continue-- # Surface the first fatal error in a structured form the Node- # classifier can read (once — don't spam if it repeats in a traceback).- if not fatal_holder["v"]:- for code, pat in FATAL_PATTERNS:- if pat.search(line):- print(f"USER_ERROR:{code}:{line[:300]}", file=sys.stderr, flush=True)- fatal_holder["v"] = True- break-- step_loss_match = STEP_LOSS_RE.search(line)- m = step_loss_match or TQDM_RE.search(line)- if m:- cur, total = int(m.group(1)), int(m.group(2))- # Skip the preview/sample image generation's own tqdm bar so it isn't- # mislabeled as a training step or allowed to trip the segment kill.- if is_preview_progress_bar(- line, cur, total,- has_step_loss=step_loss_match is not None,- preview_steps=preview_steps, training_total=training_total,- ):- if not in_training and not preview_logged["v"]:- # Pre-training step-0 preview: surface a distinct stage so the- # UI shows "sampling" instead of a frozen "load-model". Emit- # once — the bar redraws every denoise tick.- preview_logged["v"] = True- log("STAGE:sampling")- continue- if total > 0 and cur >= 0:- state["step"], state["total"] = cur, total- if m.lastindex and m.lastindex >= 3:- try:- state["loss"] = float(m.group(3))- except ValueError:- pass- if not in_training:- in_training = True- log("STAGE:training")- # A real training step ends a preview run; re-arm so the NEXT- # mid-training sampling phase (step 150, 300…) logs STAGE:sampling- # again. Re-announce STAGE:training when we come back from one.- elif preview_logged["v"]:- log("STAGE:training")- preview_logged["v"] = False- # tqdm redraws constantly at the same step — only emit on change.- if cur != last_reported["step"]:- last_reported["step"] = cur- log(f"STEP:{cur}:{total}:{state['loss'] if state['loss'] is not None else 'nan'}")- # Segment boundary. Two reasons this gates on state["step"]:- # 1. state["step"] is mflux's CUMULATIVE iteration count, not a- # per-process counter — tqdm renders with- # initial=num_iterations (restored from the checkpoint on- # resume) over total_number_of_steps() (the cumulative- # total), so the bar reads 150/600, 151/600… after a resume.- # Comparing it to the cumulative `segment_target` is valid on- # every segment, not just the first.- # 2. Waiting until the trainer has stepped PAST the target is- # the flush-safety signal: mflux's save() writes the zip- # IN-PLACE and non-atomically (zipfile.ZipFile(zip_path,"w")- # straight to the final path), and the next tqdm line only- # appears after that synchronous save() returns. Do NOT- # "simplify" this to terminate the moment- # newest_checkpoint() >= segment_target — that would race a- # half-written zip and resume from a truncated checkpoint.- if segment_target is not None and state["step"] > segment_target:- ck = newest_checkpoint(configured_output)- if ck and _checkpoint_step(ck) >= segment_target:- log(f"STATUS:segment checkpoint @ step {_checkpoint_step(ck)} saved — "- f"tearing down GPU + cooling {cooldown}s (watchdog-panic mitigation)")- _terminate_child()- return {"reason": "segment", "checkpoint": ck, "code": 0}- continue-- # Forward everything else (truncated) — keeps the JS idle watchdog- # fed during model download/load phases and aids debugging.- output_tail.append(line[:300])- log(f"STATUS:{line[:300]}")-- code = CHILD.wait()- if STOP_REQUESTED:- return {"reason": "stopped", "code": 143}- if code != 0:- return {"reason": "error", "code": code}- return {"reason": "completed", "checkpoint": newest_checkpoint(configured_output), "code": 0}---def main():- args = parse_args()- # Self-document the exact numerical stack BEFORE anything can fail — the run- # log + any watchdog paniclog then carry the mflux/mlx/mlx-metal trio inline.- emit_runtime_fingerprint()- # Self-document the GPU-watchdog mitigation too (mlx #3267) so a paniclog- # shows whether the AGX timeout relaxation was active for the crashed run.- log(f"STATUS:watchdog mitigation · AGX_RELAX_CDM_CTXSTORE_TIMEOUT={os.environ.get('AGX_RELAX_CDM_CTXSTORE_TIMEOUT', 'unset')}")- output_dir = Path(args.output_dir)- samples_dir = output_dir / "samples"- samples_dir.mkdir(parents=True, exist_ok=True)- config = json.loads(Path(args.config).read_text())- configured_output = Path(config["checkpoint"]["output_path"])-- # mflux's real (epoch-rounded) total, not the requested --total-steps — see- # compute_effective_total. Segment planning keys off this so the final- # segment stays one save-interval wide even when epochs round the count up.- total = max(1, compute_effective_total(config, args.total_steps))- seg = max(0, args.segment_steps)- cooldown = max(0, args.cooldown_sec)- segmented = seg > 0-- # The preview/sample image generation prints its own tqdm bar whose total is- # the denoise step count (`steps`), distinct from the training total. Pass- # both to run_segment so it can tell a preview bar from a training bar. Guard- # the rare degenerate case where steps == training total (tiny datasets):- # then we can't disambiguate by total, so don't filter (preview_steps=None).- preview_steps = config.get("steps")- if not isinstance(preview_steps, int) or preview_steps <= 0 or preview_steps == total:- preview_steps = None-- state = {"step": 0, "total": total, "loss": None}- last_reported = {"step": -1}-- watcher = ArtifactWatcher(configured_output, samples_dir, lambda: state["step"])- watcher.start()-- # Resolve the trainer command up front so a missing mflux exits before we- # start telemetry (build_command exits early) — never leave an orphaned- # root powermetrics process. One telemetry sidecar + one watcher span ALL- # segments so the forensic log and artifact stream stay continuous.- _ = build_command(args.config, args.resume_checkpoint)- telemetry = TelemetrySidecar(output_dir)- telemetry.start()-- fatal_holder = {"v": False}- # Ring buffer of the trainer's recent non-progress output. mflux's stderr- # is merged into its stdout (so tqdm stays in order), which means the Node- # side tags every line `stream='stdout'` and the JS classifier's stderr- # tail — its only window into *why* a run died — stays empty. On a non-zero- # exit we replay this buffer to stderr so argparse errors, tracebacks, and- # version-mismatch complaints actually reach the classifier instead of the- # run failing with a bare "exited with code N".- output_tail = deque(maxlen=OUTPUT_TAIL_LINES)-- resume = args.resume_checkpoint # None on a fresh run; a zip on resume- final_code = 0- segment_index = 0- if segmented:- log(f"STATUS:segmented training enabled — {seg}-step segments, {cooldown}s GPU cooldown between each")- while True:- start_step = _checkpoint_step(resume) if resume else 0- # Plan against mflux's real epoch-rounded total. `total` is the- # config-derived estimate (compute_effective_total); state["total"] is- # the authoritative value tqdm reports once the first segment runs — take- # the larger so the cutoff is right BOTH before the first tqdm line (fresh- # run / resume-near-end use the config estimate) and after (tqdm corrects- # any discrepancy). Only when the NEXT boundary would reach/exceed this- # real total do we drop the boundary kill and run a final, run-to-- # completion segment; otherwise every segment — including the last —- # stays one save-interval wide, preserving the bounded GPU window. A- # resume past the end just reloads, hits StopIteration, returns- # 'completed' — correct, at the cost of one model load.- effective_total = max(total, state["total"])- segment_target = start_step + seg if segmented else None- if segment_target is not None and segment_target >= effective_total:- segment_target = None-- cmd = build_command(args.config, resume)- log("STAGE:load-model")- log(f"STATUS:launching {Path(cmd[0]).name}"- + (f" (segment {segment_index + 1} from step {start_step})" if segmented else ""))-- # Each segment is a fresh mflux process — fatal-error detection (and the- # decision to replay the stderr tail) must be per-invocation, so a- # FATAL_PATTERN line matched in an earlier segment that still exited 0- # can't suppress the tail replay for a later segment's real failure.- fatal_holder["v"] = False- result = run_segment(- cmd, segment_target=segment_target, state=state, last_reported=last_reported,- output_tail=output_tail, fatal_holder=fatal_holder,- configured_output=configured_output, cooldown=cooldown,- preview_steps=preview_steps, training_total=total,- )- segment_index += 1-- if result["reason"] == "stopped":- break- if result["reason"] == "error":- final_code = result["code"] or 1- break- if result["reason"] == "completed":- break-- # reason == 'segment' — cool down, then resume from the boundary checkpoint.- resume = str(result["checkpoint"]) if result.get("checkpoint") else resume- new_step = _checkpoint_step(resume)- # No `new_step >= total` early-stop: a 'segment' result means we killed- # at a non-final boundary (segment_target < total), so the next- # iteration's segment_target crosses `total`, becomes the final- # run-to-completion segment, and lets mflux finish its epoch-rounded- # steps — rather than stopping at the approximate requested total.- # Forward-progress guard: the boundary only fires on a checkpoint whose- # step >= segment_target > start_step, so new_step should always advance.- # If it somehow didn't (a stalled save, checkpoint renumbering), resuming- # would re-run the identical segment forever — fail loudly instead.- if new_step <= start_step:- print(f"USER_ERROR:TRAINING_FAILED:segment made no checkpoint progress "- f"(still at step {new_step}/{total}) — stopping to avoid a resume loop",- file=sys.stderr, flush=True)- final_code = 1- break- log(f"STATUS:segment {segment_index} complete at step {new_step}/{total} — cooling down {cooldown}s")- interruptible_sleep(cooldown, heartbeat_stage="cooldown")- if STOP_REQUESTED:- break-- watcher.stop_event.set()- watcher.join(timeout=10)- telemetry.stop()-- if STOP_REQUESTED:- log("STATUS:canceled — last saved checkpoint retained")- sys.exit(143)-- if final_code != 0:- # Replay the trainer's tail to stderr so the Node classifier sees the- # real error. Skip when a structured USER_ERROR already pinned the- # cause (its message is more actionable than raw tail lines).- if not fatal_holder["v"]:- for tail_line in output_tail:- print(f"mflux: {tail_line}", file=sys.stderr, flush=True)- print(f"❌ mflux trainer exited with code {final_code}", file=sys.stderr, flush=True)- sys.exit(final_code or 1)-- checkpoints_dir = resolve_mflux_output(configured_output) / "checkpoints"- adapter = find_adapter(checkpoints_dir, output_dir)- if not adapter:- print(- "USER_ERROR:TRAINING_FAILED:mflux finished but no *_adapter.safetensors found in "- f"{checkpoints_dir}", file=sys.stderr, flush=True,- )- sys.exit(1)- log("RESULT:" + json.dumps({- "adapter_path": str(adapter),- "steps": state["step"] or args.total_steps,- "final_loss": state["loss"],- }))---if __name__ == "__main__":- main()diff --git a/scripts/train_mflux_lora_test.py b/scripts/train_mflux_lora_test.pydeleted file mode 100644index 99efd3cd35..0000000000--- a/scripts/train_mflux_lora_test.py+++ /dev/null@@ -1,136 +0,0 @@-#!/usr/bin/env python3-"""Standalone tests for train_mflux_lora.py pure helpers (no pytest, no mflux).--Run: ./data/python/venv/bin/python scripts/train_mflux_lora_test.py-Exits non-zero on first failure. Mirrors the runnable-test style of-scripts/generate_ltx2_teacache_test.py.-"""-import sys-from pathlib import Path--sys.path.insert(0, str(Path(__file__).resolve().parent))-from train_mflux_lora import ( # noqa: E402- is_preview_progress_bar,- compute_effective_total,- format_runtime_fingerprint_line,- TQDM_RE,- STEP_LOSS_RE,-)--FAILS = []---def check(name, cond):- print(("✅" if cond else "❌") + " " + name)- if not cond:- FAILS.append(name)---# --- is_preview_progress_bar: the preview/sample bar vs the training bar --------# Scenario mirrors the live Freydis run: steps=40 (preview denoise), total=600.-PREVIEW, TRAIN = 40, 600--# A preview bar (total == preview_steps, != training_total, no step/loss prose).-check(- "preview bar (38/40) is detected as preview",- is_preview_progress_bar("38/40 [00:19<00:01]", 38, 40,- has_step_loss=False, preview_steps=PREVIEW, training_total=TRAIN),-)-check(- "preview bar at completion (40/40) is detected as preview",- is_preview_progress_bar("40/40 [00:20<00:00]", 40, 40,- has_step_loss=False, preview_steps=PREVIEW, training_total=TRAIN),-)--# A training bar (total == training_total) is NOT a preview.-check(- "training bar (151/600) is NOT preview",- not is_preview_progress_bar("151/600 [10:00<30:00]", 151, 600,- has_step_loss=False, preview_steps=PREVIEW, training_total=TRAIN),-)-check(- "training bar at a 40-ish step (40/600) is NOT preview (total differs)",- not is_preview_progress_bar("40/600 [02:00<28:00]", 40, 600,- has_step_loss=False, preview_steps=PREVIEW, training_total=TRAIN),-)--# Explicit "step N/M ... loss X" prose is training-only, never a preview, even-# if its total coincidentally equals preview_steps.-check(- "explicit step/loss prose is never preview",- not is_preview_progress_bar("step 40/40 loss 0.12", 40, 40,- has_step_loss=True, preview_steps=PREVIEW, training_total=TRAIN),-)--# Degenerate case: caller passes preview_steps=None (steps == training_total on a-# tiny dataset) → never filter, so no real training step is ever dropped.-check(- "preview_steps=None disables filtering",- not is_preview_progress_bar("40/40 [..]", 40, 40,- has_step_loss=False, preview_steps=None, training_total=40),-)--# preview_steps known but training_total unknown (None): filter on total match-# alone — better to risk a missed preview tick than to mislabel, but we still-# catch the common case.-check(- "training_total=None still filters an exact preview-total match",- is_preview_progress_bar("12/40 [..]", 12, 40,- has_step_loss=False, preview_steps=40, training_total=None),-)--# --- the regexes that feed it ---------------------------------------------------m = TQDM_RE.search(" 3%|▎ | 12/400 [00:30<16:20, 2.53s/it]")-check("TQDM_RE parses cur/total", bool(m) and m.group(1) == "12" and m.group(2) == "400")-check("STEP_LOSS_RE matches explicit prose",- bool(STEP_LOSS_RE.search("steps 40/40 loss 0.12")))-check("STEP_LOSS_RE does NOT match a bare tqdm bar",- STEP_LOSS_RE.search("38/40 [00:19<00:01]") is None)--# --- compute_effective_total sanity (unchanged behavior) ------------------------# 25 images, batch 1, 24 epochs → 600 (the Freydis config).-import tempfile, os # noqa: E402-with tempfile.TemporaryDirectory() as d:- for i in range(25):- Path(d, f"{i:04d}.png").write_bytes(b"x")- cfg = {"training_loop": {"num_epochs": 24, "batch_size": 1}, "data": d}- check("compute_effective_total(25 img,24 ep,batch 1) == 600",- compute_effective_total(cfg, 999) == 600)--# Unexpected shape → falls back to the requested count.-check("compute_effective_total falls back on bad config",- compute_effective_total({}, 777) == 777)--# --- format_runtime_fingerprint_line: one-line startup stack fingerprint --------# Full stack as the M5 panic reports capture it.-full_fp = {- "runtime": "mflux-lora",- "versions": {"mflux": "0.17.5", "mlx": "0.30.6", "mlx-metal": "0.30.6", "mlx-lm": "0.29.1"},- "chip": "Apple M5 Max",- "os": "macOS-26.5.1-arm64-arm-64bit",- "python": "3.11.9",-}-full_line = format_runtime_fingerprint_line(full_fp)-check("fingerprint line is a single STATUS line",- full_line.startswith("STATUS:runtime · ") and "\n" not in full_line)-check("fingerprint line carries the mflux/mlx/mlx-metal/mlx-lm versions",- all(s in full_line for s in ("mflux 0.17.5", "mlx 0.30.6", "mlx-metal 0.30.6", "mlx-lm 0.29.1")))-check("fingerprint line carries chip/os/python",- all(s in full_line for s in ("Apple M5 Max", "macOS-26.5.1-arm64-arm-64bit", "python 3.11.9")))--# Absent packages omitted by the builder → line still renders, no crash, no empty-# fragment for the missing pin.-partial_line = format_runtime_fingerprint_line({"versions": {"mflux": "0.17.5"}, "chip": "Apple M5 Max"})-check("partial fingerprint omits absent packages cleanly",- "mflux 0.17.5" in partial_line and "mlx " not in partial_line and "· ·" not in partial_line)--# Empty dict (everything failed to resolve) → still a single, non-throwing line.-empty_line = format_runtime_fingerprint_line({})-check("empty fingerprint degrades to a 'versions unavailable' line",- empty_line == "STATUS:runtime · versions unavailable")--print()-if FAILS:- print(f"❌ {len(FAILS)} failure(s): {FAILS}")- sys.exit(1)-print("✅ all train_mflux_lora helper tests passed")diff --git a/server/hostControlCoverage.test.js b/server/hostControlCoverage.test.jsindex 2cb38a9562..c0ac706dd3 100644--- a/server/hostControlCoverage.test.js+++ b/server/hostControlCoverage.test.js@@ -657,17 +657,8 @@ const REVIEWED_OPEN_MUTATIONS = [['DELETE /api/loops/:id', REMOVES_RECORD],['POST /api/loops/:id/stop', REDUCES_EXECUTION],// /api/lora-datasets- ['POST /api/lora-datasets', RECORDS],- ['DELETE /api/lora-datasets/:id', REMOVES_RECORD],- ['PATCH /api/lora-datasets/:id', RECORDS],['POST /api/lora-datasets/:id/caption-runs/:runId/cancel', REDUCES_EXECUTION],- ['POST /api/lora-datasets/:id/images', RECORDS],- ['DELETE /api/lora-datasets/:id/images/:imageId', REMOVES_RECORD],- ['PATCH /api/lora-datasets/:id/images/:imageId', RECORDS],- ['POST /api/lora-datasets/:id/import-gallery', IN_PROCESS_MEDIA],- ['POST /api/lora-datasets/:id/strip-shared-fragments', RECORDS],// /api/lora-training- ['DELETE /api/lora-training/runs/:id', REMOVES_RECORD],['POST /api/lora-training/runs/:id/cancel', REDUCES_EXECUTION],// /api/loras['DELETE /api/loras/:filename', REMOVES_RECORD],diff --git a/server/index.js b/server/index.jsindex 8fccaa84f2..7ca2799f63 100644--- a/server/index.js+++ b/server/index.js@@ -181,6 +181,7 @@ import imageVideoModelsRoutes from './routes/imageVideoModels.js';import lorasRoutes from './routes/loras.js';import loraDatasetsRoutes from './routes/loraDatasets.js';import loraTrainingRoutes from './routes/loraTraining.js';+import dataMaintenanceRoutes from './routes/dataMaintenance.js';import sdapiRoutes from './routes/sdapi.js';import openclawRoutes from './routes/openclaw.js';import sharingRoutes from './routes/sharing.js';@@ -500,6 +501,7 @@ app.use('/api/importer', importerRoutes);app.use('/api/story-builder', storyBuilderRoutes);app.use('/api/image-video/models', imageVideoModelsRoutes);app.use('/api/loras', lorasRoutes);+app.use('/api/data-maintenance', dataMaintenanceRoutes);app.use('/api/lora-datasets', loraDatasetsRoutes);app.use('/api/lora-training', loraTrainingRoutes);// AUTOMATIC1111-compatible surface for tailnet clients — gated bydiff --git a/server/lib/README.md b/server/lib/README.mdindex b7e84c61a7..7d57fb12e3 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -31,6 +31,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `reconcileRetirementEvidence.js` | Matches exact checkout receipts and recovered-preservation receipts, preserves observation identity across delayed scans, and retains each unresolved checkout incarnation. Legacy observations keep their original generation timestamps. A recreated checkout at the same branch, path and head starts after every older receipt; verified historical preservation does not assert branch deletion. || `legacyRetirementTransition.js` | Freezes archive-era observations for evidence-based reconciliation and retains their original diagnoses and delivery/receipt proofs as audit history, without manufacturing deletion receipts or changing task outcomes. || `retirementCheckoutObservation.js` | Reads a checkout's filesystem identity and verifies its live branch/head before the schedule writer admits a recreated checkout's obligation. |+| `retiredLoraTraining.js` | Retirement refusal and short-lived admission for retained training data access/removal. || `backupSharedAdmission.js` | Durable cross-process backup cuts and publication leases, with fail-closed owner recovery. || `runtimeFilePublication.js` | Admitted runtime recording updates with rollback of the prior file pair on failure. || `maintenanceAdmission.js` | Durable machine-local workflow drain, cross-process admission permits, continuation ownership and stale-resume fencing. |diff --git a/server/lib/backupAssetOwners.js b/server/lib/backupAssetOwners.jsindex 4f574ad803..7697a4c8b2 100644--- a/server/lib/backupAssetOwners.js+++ b/server/lib/backupAssetOwners.js@@ -408,7 +408,7 @@ export const BACKUP_ASSET_OWNERS = Object.freeze([// Explicit file cleanup can leave external references by design, but one// lease keeps its single/bulk deletion out of a copy-then-dump snapshot.id: 'operator-file-purge', status: 'admitted',- modules: ['services/dataManager.js', 'routes/uploads.js', 'routes/attachments.js'],+ modules: ['services/dataManager.js', 'services/dataMaintenance.js', 'routes/uploads.js', 'routes/attachments.js'],},{// Snapshot JSON and its index entry are one pair. Deletion persists indexdiff --git a/server/lib/hostControlRoutes.js b/server/lib/hostControlRoutes.jsindex 7cbc46971e..f4dbf4c65c 100644--- a/server/lib/hostControlRoutes.js+++ b/server/lib/hostControlRoutes.js@@ -1252,6 +1252,9 @@ export const HOST_CONTROL_ROUTES = Object.freeze([// configured inference, like local-llm test and Pipeline TTS.'POST /api/continuous-video','POST /api/lora-training/runs/:id/promote-checkpoint',+ // Explicit retired-data removal uses the same operator authority as host maintenance.+ 'POST /api/data-maintenance/preview',+ 'POST /api/data-maintenance/execute','POST /api/music/supercollider/render','POST /api/media-jobs/holds/:holdId/resume','POST /api/voice/piper/fetch',diff --git a/server/lib/hostControlRoutes.test.js b/server/lib/hostControlRoutes.test.jsindex cf84853e03..554894e894 100644--- a/server/lib/hostControlRoutes.test.js+++ b/server/lib/hostControlRoutes.test.js@@ -413,16 +413,8 @@ describe('auxiliary media mutation inventory (#9672)', () => {// Reviewed as record CRUD, cancellation/pruning or deterministic work that// never hands caller text to an agent. A new mutation needs a new decision.const recordOrContained = [- 'DELETE /api/lora-datasets/:id',- 'DELETE /api/lora-datasets/:id/images/:imageId','DELETE /api/media-jobs/:id',- 'PATCH /api/lora-datasets/:id',- 'PATCH /api/lora-datasets/:id/images/:imageId',- 'POST /api/lora-datasets','POST /api/lora-datasets/:id/caption-runs/:runId/cancel',- 'POST /api/lora-datasets/:id/import-gallery',- 'POST /api/lora-datasets/:id/images',- 'POST /api/lora-datasets/:id/strip-shared-fragments','POST /api/media-jobs/:id/cancel','POST /api/media-jobs/cancel-queued',];diff --git a/server/lib/index.js b/server/lib/index.jsindex 88dce78a64..0010b64f5e 100644--- a/server/lib/index.js+++ b/server/lib/index.js@@ -977,3 +977,5 @@ export * from './issueClaimEvidence.js';export * from './codeReviewBatches.js';export * from './realityStarterV1.js';export * from './universeStarters.js';++export * from './retiredLoraTraining.js';diff --git a/server/lib/localProviderRuntime.test.js b/server/lib/localProviderRuntime.test.jsindex 4ba13682e3..8139d5e748 100644--- a/server/lib/localProviderRuntime.test.js+++ b/server/lib/localProviderRuntime.test.js@@ -286,7 +286,6 @@ describe('shipped copy names a Models page by route, never by literal (#7419)',const ALLOWLISTED_FILES = new Set(['server/lib/navManifest.js','server/services/loraDatasetCaption.js',- 'client/src/components/loraTraining/CaptionModelPicker.jsx',]);/**diff --git a/server/lib/mediaValidation.js b/server/lib/mediaValidation.jsindex 3494cf850c..10e4d65f1a 100644--- a/server/lib/mediaValidation.js+++ b/server/lib/mediaValidation.js@@ -177,31 +177,23 @@ const loraTrainingParamsSchema = z.object({// latent-cache spill. `null` is the form's "Auto": a deliberate clear that// forces the deriveMfluxMemoryConfig tier even when a saved default exists// (distinct from absent, which lets the saved default merge through). An- // explicit value still cannot exceed the LORA_TRAIN_MAX_QUANT_BITS cap.+ // values remain accepted as historical configuration after retirement.baseQuant: z.union([z.literal(4), z.literal(8), z.literal(16)]).nullable().optional(),lowRam: z.boolean().nullable().optional(),});-// LoRA training settings slice (`settings.loraTraining`) — vision-caption-// provider pick + training parameter defaults. Code-level defaults live in-// `services/loraTraining/runtimes.js` so an absent slice needs no migration.+// Retired settings remain accepted for older installs and clients. They no+// longer authorize launch/resume; recovery still honors display and stall settings.export const loraTrainingConfigSchema = z.object({// Both nullable — the caption-model picker clears them to null on "Auto"// (defer to the server's vision-model auto-pick).captionProviderId: z.string().max(128).nullable().optional(),captionModel: z.string().max(256).nullable().optional(),defaults: loraTrainingParamsSchema.optional(),- // Segmented mflux training (watchdog-panic mitigation, default ON in- // services/loraTraining/runtimes.js). Setting this false runs the trainer as- // one sustained process again — flip it once a macOS/mflux update resolves- // the GPU-driver hang. Cooldown is the GPU idle gap (seconds) between segments.+ // Historical parameters are inert and preserved, not rewritten on upgrade.segmentation: z.boolean().optional(),segmentCooldownSec: z.number().int().min(0).max(3600).optional(),- // Phase-aware soft-hang stall watchdog (issue #1330, default ON in- // services/loraTraining/index.js). Detects a wedged GPU mid-training (steps- // stop arriving within a step-rate-derived budget) and SIGKILLs + auto-resumes- // from the newest checkpoint. Set false to fall back to only the flat 30-min- // idle watchdog (e.g. if a future driver fix makes soft hangs impossible).+ // Surviving trainers retain stall detection; automatic resumes are retired.stallWatchdog: z.boolean().optional(),// Auto display-sleep during training on Apple Silicon (default ON — the// `!== false` read lives in services/loraTraining/displayPower.js@@ -213,17 +205,6 @@ export const loraTrainingConfigSchema = z.object({displaySleep: z.boolean().optional(),});-// POST /api/lora-training/runs — start a training run for a dataset.-export const startTrainingRunSchema = z.object({- datasetId: z.string().min(1).max(128),- baseModelId: z.string().min(1).max(128),- name: z.string().trim().max(120).optional(),- params: loraTrainingParamsSchema.optional(),- // Override the caption identity-leak gate (see validateDatasetReady) and train- // anyway — the UI sends this from the explicit "Train anyway" action.- acknowledgeCaptionLeak: z.boolean().optional(),-});-// === Local LLM backends (Ollama / LM Studio) ===export const localLlmBackendSchema = z.enum(['ollama', 'lmstudio']);// modelId is passed positionally to the `lms` CLI (execFile, no shell) — rejectdiff --git a/server/lib/navManifest.js b/server/lib/navManifest.jsindex b19c3e1715..69931e47af 100644--- a/server/lib/navManifest.js+++ b/server/lib/navManifest.js@@ -337,6 +337,7 @@ const RAW_NAV_COMMANDS = [{ id: 'nav.settings.autofixer', path: '/settings/autofixer', label: 'Autofixer', section: 'Settings', tabId: 'autofixer', aliases: ['autofixer', 'settings-autofixer', 'auto-fixer'], keywords: ['crash', 'fix', 'pm2', 'repair', 'ai provider', 'restart'] },{ id: 'nav.settings.backup', path: '/settings/backup', label: 'Backup', section: 'Settings', tabId: 'backup', aliases: ['backup', 'settings-backup'] },{ id: 'nav.settings.credentials', path: '/settings/credentials', label: 'Credentials', section: 'Settings', tabId: 'credentials', aliases: ['settings-credentials', 'credentials', 'api-keys', 'tokens'], keywords: ['configured', 'unconfigured', 'env', 'huggingface', 'civitai', 'jira', 'datadog', 'telegram'] },+ { id: 'nav.settings.data-maintenance', path: '/settings/data-maintenance', label: 'Data Maintenance', section: 'Settings', tabId: 'data-maintenance', previousPaths: ['/media/training', '/media/training/:datasetId', '/models/training', '/models/training/:recordId'], preservePreviousPathSuffix: false, aliases: ['data-maintenance', 'retired-data', 'lora-training', 'training', 'datasets', 'train-lora', 'character-lora'], keywords: ['cleanup', 'obsolete', 'storage', 'delete', 'upgrade', 'migration'] },{ id: 'nav.settings.database', path: '/settings/database', label: 'Database', section: 'Settings', tabId: 'database', aliases: ['settings-database', 'database'] },{ id: 'nav.settings.features', path: '/settings/features', label: 'Features', section: 'Settings', tabId: 'features', aliases: ['settings-features', 'instance-features', 'feature-usage'], keywords: ['enabled', 'disabled', 'instance', 'optional', 'participation', 'metrics', 'reminders'] },{ id: 'nav.settings.general', path: '/settings/general', label: 'General', section: 'Settings', tabId: 'general', aliases: ['settings', 'settings-general', 'general'] },@@ -373,7 +374,6 @@ const RAW_NAV_COMMANDS = [// catalog page even though this page starts those servers too.{ id: 'nav.models.llms-runtimes', path: '/models/llms-runtimes', label: 'Runtimes', section: 'Models', tabId: 'llms-runtimes', navGroup: 'Connect', previousPaths: ['/models/llms/runtimes'], aliases: ['runtimes', 'llm-runtimes', 'local-runtimes', 'model-runtimes', 'llama-server', 'llama-cpp', 'slotstream', 'mtplx'], keywords: ['llama.cpp', 'llama-server', 'mtplx', 'slotstream', 'start server', 'stop server', 'local server', 'model server', 'speculative decoding', 'draft model', 'checkpoint', 'idle release'] },{ id: 'nav.media.loras', path: '/models/loras', label: 'LoRAs', section: 'Models', tabId: 'loras', navGroup: 'Library', previousPaths: ['/media/loras'], aliases: ['loras', 'lora', 'lora-manager', 'civitai'], keywords: ['lora', 'civitai', 'fine-tune', 'style adapter', 'realstagram', 'photoreal', 'flux lora'] },- { id: 'nav.media.training', path: '/models/training', label: 'Training', section: 'Models', tabId: 'training', navGroup: 'Library', previousPaths: ['/media/training', '/media/training/:datasetId'], aliases: ['training', 'lora-training', 'train-lora', 'datasets', 'character-lora'], keywords: ['fine-tune', 'dataset', 'caption', 'dreambooth', 'character consistency', 'train', 'flux lora'] },{ id: 'nav.media.models', path: '/models/media', label: 'Media', section: 'Models', tabId: 'media', navGroup: 'Library', previousPaths: ['/media/models', '/media-models'], aliases: ['media-models', 'image-models', 'video-models', 'huggingface'], keywords: ['hf cache', 'model storage', 'disk', 'add model', 'install model', 'custom model'] },{ id: 'nav.models.performance', path: '/models/performance', label: 'Performance', section: 'Models', tabId: 'performance', navGroup: 'Evaluate', aliases: ['model-performance', 'performance', 'assessments', 'model-assessments', 'benchmark-models', 'tuning'], keywords: ['measure', 'assessment', 'benchmark', 'throughput', 'chars per second', 'ttft', 'context', 'tuning', 'llama.cpp', 'mtplx', 'vllm', 'which model', 'fastest model'] },{ id: 'nav.models.performance.results', path: '/models/performance/results', label: 'Performance Results', section: 'Models', aliases: ['performance-results', 'assessment-results'], keywords: ['measured results', 'model evidence', 'selected evidence', 'rankings'] },diff --git a/server/lib/navManifest.test.js b/server/lib/navManifest.test.jsindex b445831487..e6e133a4be 100644--- a/server/lib/navManifest.test.js+++ b/server/lib/navManifest.test.js@@ -167,7 +167,7 @@ describe('nav contract — generated section child navigation', () => {expect(getSectionNavGroups('Models').map(({ label, tabs }) => [label, tabs.map((tab) => tab.label)])).toEqual([['Connect', ['AI Services', 'Fallbacks', 'Harnesses', 'Providers', 'Runtimes']],['Evaluate', ['Comparison', 'Performance', 'Playground']],- ['Library', ['3D', 'Embeddings', 'LoRAs', 'Media', 'Model Library', 'Training']],+ ['Library', ['3D', 'Embeddings', 'LoRAs', 'Media', 'Model Library']],['Operate', ['Quota Burn', 'Status', 'Subscriptions', 'Usage']],['Policies', ['Abuse Guard', 'Code Reviewers', 'Decision Classifiers']],]);@@ -176,7 +176,7 @@ describe('nav contract — generated section child navigation', () => {it('resolves a promoted drill-down as itself instead of its LLM parent', () => {expect(getSectionNavTabForPath('Models', '/models/llms/abuse')).toMatchObject({ id: 'abuse', label: 'Abuse Guard' });expect(getNavCommandForPath('/models/decision-classifiers/jev')).toMatchObject({ path: '/models/decision-classifiers/jev', label: 'Jev' });- expect(getSectionNavTabForPath('Models', '/models/training/dataset-abc')).toMatchObject({ id: 'training' });+ expect(getSectionNavTabForPath('Models', '/models/training/dataset-abc')).toBeUndefined();});it.each([@@ -381,12 +381,12 @@ describe('resolveNavCommand — fuzzy matching', () => {});it('resolves the moved model-management pages to their Models paths', () => {- // Media models, LoRAs, LoRA training and embeddings moved out of Create and- // Settings (#4728). Their command ids are unchanged (they are opaque and- // persisted), so only these paths prove the move actually landed.+ // Media models, LoRAs and embeddings moved out of Create and+ // Settings (#4728); retired training now lands in maintenance. Active command ids are opaque and+ // persisted, so paths prove the move actually landed.expect(resolveNavCommand('media models')?.path).toBe('/models/media');expect(resolveNavCommand('loras')?.path).toBe('/models/loras');- expect(resolveNavCommand('lora training')?.path).toBe('/models/training');+ expect(resolveNavCommand('lora training')?.path).toBe('/settings/data-maintenance');expect(resolveNavCommand('embeddings')?.path).toBe('/models/embeddings');expect(resolveNavCommand('prompt guard')?.path).toBe('/models/llms/abuse');expect(resolveNavCommand('abuse-guard')?.path).toBe('/models/llms/abuse');@@ -751,11 +751,12 @@ describe('nav coverage — every navigable App.jsx route has a manifest entry',// segment was a deep link into one record, so its redirect has to carry that// segment across — swapping the PrefixRedirect for a bare <Navigate to="/models/training">// still points at the right page and would pass the check above, while every- // bookmarked dataset quietly lands on the index instead.+ // bookmarked record quietly lands on the index instead. Retired features may+ // explicitly opt out because their destination no longer selects old records.it('preserves the record id when a parameterized previous path redirects', () => {const lossy = NAV_COMMANDS- .flatMap((c) => (c.previousPaths || []).map((from) => ({ from, id: c.id })))- .filter(({ from }) => from.split('/').some((seg) => seg.startsWith(':')))+ .flatMap((c) => (c.previousPaths || []).map((from) => ({ from, id: c.id, preserveSuffix: c.preservePreviousPathSuffix !== false })))+ .filter(({ from, preserveSuffix }) => preserveSuffix && from.split('/').some((seg) => seg.startsWith(':'))).map(({ from, id }) => ({ from, id, hit: byFrom.get(from) })).filter(({ hit }) => !hit || !SUFFIX_PRESERVING_ELEMENTS.has(hit.element)).map(({ from, id, hit }) => `${id}: ${from} forwards via ${hit?.element ?? 'NO REDIRECT'}, which drops the trailing segment`);@@ -773,11 +774,13 @@ describe('nav coverage — every navigable App.jsx route has a manifest entry','/media/models', // #4728 — the rest of model management left Create/Settings/Dev Tools'/media/loras','/media/training',+ '/media/training/:datasetId',+ '/models/training',+ '/models/training/:recordId','/settings/embeddings','/system-resources/models',// The dataset workbench was its own deep-linkable path, and a bookmark into// one dataset breaks just as silently as the index.- '/media/training/:datasetId',// #7567 — /ai became /ai/presets and the Backend Connections drawer// became the Services view; a stored pin or bookmark on either must land.'/ai',diff --git a/server/lib/pythonSetup.js b/server/lib/pythonSetup.jsindex 4746fbbd31..852867054d 100644--- a/server/lib/pythonSetup.js+++ b/server/lib/pythonSetup.js@@ -432,9 +432,8 @@ export function invalidateFlux2Health() {// symmetry with the other resolvers.const MFLUX_TRAIN_BIN = IS_WIN ? 'mflux-train.exe' : 'mflux-train';-// True when `mflux-train` sits next to `pythonPath` (the same probe-// loraTraining's isMfluxTrainAvailable uses; kept here so resolveMfluxPython is-// self-contained). Null/empty path → false.+// Historical console-script probe retained for shared image-runtime discovery.+// Retirement never uninstalls or changes which existing venv is selected.export const hasMfluxTrain = (pythonPath) =>!!pythonPath && existsSync(join(dirname(pythonPath), MFLUX_TRAIN_BIN));@@ -448,13 +447,12 @@ const MFLUX_VENV_CANDIDATES = IS_WINjoin(PATHS.data, 'python', 'venv-mflux', 'bin', 'python3'),];-// Resolve the Python whose `mflux-train` PortOS should spawn for MLX LoRA-// training. Preference order: (1) an explicitly-configured image-gen Python+// Resolve the existing shared mflux Python. Preserve the historical preference+// order: (1) an explicitly-configured image-gen Python// that actually ships mflux-train — the historical `pip --user` layout, so// existing installs behave exactly as before; (2) the first dedicated// ~/.portos/venv-mflux that ships it. When neither ships mflux-train, return-// the configured path unchanged (or null) so the caller's isMfluxTrainAvailable-// still reports the honest "not installed" state instead of a phantom venv path.+// the configured path unchanged (or null) rather than inventing a venv path.export function resolveMfluxPython(configuredPath = null) {if (hasMfluxTrain(configuredPath)) return configuredPath;for (const p of MFLUX_VENV_CANDIDATES) {diff --git a/server/lib/retiredLoraTraining.js b/server/lib/retiredLoraTraining.jsnew file mode 100644index 0000000000..a9f3f8c1e8--- /dev/null+++ b/server/lib/retiredLoraTraining.js@@ -0,0 +1,37 @@+import { ServerError } from './errorHandler.js';++export const TRAINING_RETIRED_MESSAGE = 'LoRA training is retired. Existing datasets, checkpoints and adapters are preserved. Use reference images for new generation; manage retained files in Settings → Data Maintenance.';++export function refuseRetiredTraining() {+ throw new ServerError(TRAINING_RETIRED_MESSAGE, { status: 410, code: 'LORA_TRAINING_RETIRED' });+}++let removingRetiredData = false;+let retainedDataUsers = 0;+export function assertRetiredDataAvailable() {+ if (removingRetiredData) throw new ServerError('Data maintenance is removing retired inputs. Retry after it finishes.', { status: 409, code: 'DATA_MAINTENANCE_ACTIVE' });+}+export async function withRetiredDataRemoval(work) {+ assertRetiredDataAvailable();+ if (retainedDataUsers > 0) throw new ServerError('Retained training data is being updated or a checkpoint is being promoted. Retry after it finishes.', { status: 409, code: 'MAINTENANCE_BUSY' });+ removingRetiredData = true;+ try { return await work(); }+ finally { removingRetiredData = false; }+}++export function acquireRetiredDataAccess() {+ assertRetiredDataAvailable();+ retainedDataUsers += 1;+ let released = false;+ return () => {+ if (released) return;+ released = true;+ retainedDataUsers -= 1;+ };+}++export async function withRetiredDataAccess(work) {+ const release = acquireRetiredDataAccess();+ try { return await work(); }+ finally { release(); }+}diff --git a/server/routes/dataMaintenance.js b/server/routes/dataMaintenance.jsnew file mode 100644index 0000000000..ff51dd82df--- /dev/null+++ b/server/routes/dataMaintenance.js@@ -0,0 +1,21 @@+import { Router } from 'express';+import { z } from 'zod';+import { asyncHandler } from '../lib/errorHandler.js';+import { validateRequest } from '../lib/validation.js';+import { requireHostControl } from '../services/authGate.js';+import { executeMaintenance, getMaintenanceInventory, previewMaintenance } from '../services/dataMaintenance.js';++const router = Router();+const previewSchema = z.object({ operationId: z.string().min(1).max(100) }).strict();+const executeSchema = z.object({ token: z.string().uuid(), confirmation: z.string().max(150) }).strict();+router.use(requireHostControl);+router.get('/', asyncHandler(async (_req, res) => res.json(await getMaintenanceInventory())));+router.post('/preview', asyncHandler(async (req, res) => {+ const { operationId } = validateRequest(previewSchema, req.body);+ res.json(await previewMaintenance(operationId));+}));+router.post('/execute', asyncHandler(async (req, res) => {+ const body = validateRequest(executeSchema, req.body);+ res.json(await executeMaintenance(body));+}));+export default router;diff --git a/server/routes/dataMaintenance.test.js b/server/routes/dataMaintenance.test.jsnew file mode 100644index 0000000000..939d0b9bf1--- /dev/null+++ b/server/routes/dataMaintenance.test.js@@ -0,0 +1,134 @@+import { beforeEach, afterAll, describe, expect, it, vi } from 'vitest';+import { mkdtempSync, mkdirSync, writeFileSync, existsSync, rmSync, symlinkSync, realpathSync, renameSync } from 'node:fs';+import { join } from 'node:path';+import { tmpdir } from 'node:os';+import express from 'express';+import { request } from '../lib/testHelper.js';+import { makePathsProxy } from '../lib/mockPathsDataRoot.js';+const root = realpathSync(mkdtempSync(join(tmpdir(), 'maintenance-fixture-')));+const state = vi.hoisted(() => ({ jobs: [], busy: false, caption: null, authorized: true, failRemoval: false, recreate: false }));+vi.mock('../lib/fileUtils.js', async original => {+ const actual = await original();+ return { ...makePathsProxy(actual, { dataRoot: root }), rmGuarded: async (path, options) => {+ if (state.failRemoval) throw new Error('Synthetic storage failure');+ await actual.rmGuarded(path, options);+ if (state.recreate) {+ mkdirSync(join(root, 'training-runs'));+ writeFileSync(join(root, 'training-runs/new.bin'), 'new writer');+ }+ } };+});+vi.mock('../services/mediaJobQueue/index.js', () => ({ listJobs: () => state.jobs }));+vi.mock('../services/dataManagerBusy.js', () => ({ trainingRunsBusy: async () => ({ busy: state.busy }) }));+vi.mock('../services/loraDatasetCaption.js', () => ({ getActiveCaptionRun: () => state.caption }));+vi.mock('../lib/backupSnapshotBoundary.js', () => ({ withBackupAssetPublication: work => work() }));+vi.mock('../services/authGate.js', () => ({ requireHostControl: (_req, res, next) => state.authorized ? next() : res.sendStatus(403) }));+const { default: routes } = await import('./dataMaintenance.js');+const app = express(); app.use(express.json()); app.use('/api/data-maintenance', routes);+app.use((error, _req, res, _next) => res.status(error.status || 500).json({ error: error.message, code: error.code }));+const preview = () => request(app).post('/api/data-maintenance/preview').send({ operationId: 'retired-training-runs' });+const execute = p => request(app).post('/api/data-maintenance/execute').send({ token: p.token, confirmation: p.confirmation });+beforeEach(() => {+ rmSync(root, { recursive: true, force: true });+ for (const path of ['training-runs/run-example', 'lora-datasets/example/images', 'loras']) mkdirSync(join(root, path), { recursive: true });+ writeFileSync(join(root, 'training-runs/run-example/checkpoint.bin'), 'unique checkpoint');+ writeFileSync(join(root, 'lora-datasets/example/images/source.png'), 'unique source');+ writeFileSync(join(root, 'loras/example.safetensors'), 'shared adapter');+ state.jobs = []; state.busy = false; state.caption = null; state.authorized = true; state.failRemoval = false; state.recreate = false;+});+afterAll(() => rmSync(root, { recursive: true, force: true }));+describe('allowlisted data maintenance HTTP workflow', () => {+ it('previews without mutating, requires exact confirmation, removes only approved artifacts and replays idempotently', async () => {+ const inventory = await request(app).get('/api/data-maintenance');+ expect(inventory.body.find(row => row.id === 'retired-training-runs')).toMatchObject({ status: 'retained', files: 1, bytes: 17 });+ const p = (await preview()).body;+ expect(existsSync(join(root, 'training-runs/run-example/checkpoint.bin'))).toBe(true);+ expect((await request(app).post('/api/data-maintenance/execute').send({ token: p.token, confirmation: 'DELETE' })).status).toBe(400);+ const [one, two] = await Promise.all([execute(p), execute(p)]);+ expect(one.status).toBe(200); expect(two.body).toEqual(one.body);+ expect(existsSync(join(root, 'training-runs'))).toBe(false);+ expect(existsSync(join(root, 'loras/example.safetensors'))).toBe(true);+ expect(existsSync(join(root, 'lora-datasets/example/images/source.png'))).toBe(true);+ });+ it('rejects stale inventories and refuses a job that starts after preview', async () => {+ const p = (await preview()).body;+ writeFileSync(join(root, 'training-runs/run-example/new.bin'), 'new');+ expect((await execute(p)).body.code).toBe('MAINTENANCE_STALE');+ const fresh = (await preview()).body;+ state.jobs = [{ status: 'running', kind: 'image' }];+ expect((await execute(fresh)).body.code).toBe('MAINTENANCE_BUSY');+ expect(existsSync(join(root, 'training-runs/run-example/new.bin'))).toBe(true);+ });+ it('refuses surviving trainers, captioning, arbitrary paths, shared assets, unknown previews and unauthorized callers', async () => {+ state.busy = true; expect((await preview()).body.code).toBe('MAINTENANCE_BUSY');+ state.busy = false; state.caption = { status: 'running' }; expect((await preview()).body.code).toBe('MAINTENANCE_BUSY');+ state.caption = null;+ for (const operationId of ['../loras', 'shared-lora-adapters', 'archives']) expect((await request(app).post('/api/data-maintenance/preview').send({ operationId })).status).toBe(404);+ expect((await execute({ token: '00000000-0000-4000-8000-000000000001', confirmation: 'DELETE training-runs' })).body.code).toBe('MAINTENANCE_STALE');+ state.authorized = false; expect((await preview()).status).toBe(403);+ });+ it('refuses links without following their targets', async () => {+ symlinkSync(join(root, 'loras'), join(root, 'training-runs/run-example/link'));+ expect((await preview()).body.code).toBe('MAINTENANCE_UNSAFE');+ const rows = (await request(app).get('/api/data-maintenance')).body;+ expect(rows.find(row => row.id === 'retired-training-runs').status).toBe('unavailable');+ expect(existsSync(join(root, 'loras/example.safetensors'))).toBe(true);+ });+});++it('refuses removal while checkpoint promotion or a retained dataset write owns the data', async () => {+ const p = (await preview()).body;+ const { withRetiredDataAccess } = await import('../lib/retiredLoraTraining.js');+ let release;+ const access = withRetiredDataAccess(() => new Promise(resolve => { release = resolve; }));+ try {+ expect((await execute(p)).body.code).toBe('MAINTENANCE_BUSY');+ expect(existsSync(join(root, 'training-runs/run-example/checkpoint.bin'))).toBe(true);+ } finally { release(); await access; }+ expect((await execute(p)).status).toBe(200);+});++it('expires previews and rejects a replaced directory without removing its replacement', async () => {+ const p = (await preview()).body;+ const actualNow = Date.now;+ vi.spyOn(Date, 'now').mockReturnValue(actualNow() + 6 * 60_000);+ try { expect((await execute(p)).body.code).toBe('MAINTENANCE_STALE'); }+ finally { vi.restoreAllMocks(); }+ const fresh = (await preview()).body;+ rmSync(join(root, 'training-runs'), { recursive: true });+ mkdirSync(join(root, 'training-runs'));+ writeFileSync(join(root, 'training-runs/new.bin'), 'replacement');+ expect((await execute(fresh)).body.code).toBe('MAINTENANCE_STALE');+ expect(existsSync(join(root, 'training-runs/new.bin'))).toBe(true);+});++it('reports filesystem failure without success and restores the untouched detached tree', async () => {+ const p = (await preview()).body;+ state.failRemoval = true;+ const result = await execute(p);+ expect(result.status).toBe(409);+ expect(result.body.code).toBe('MAINTENANCE_INCOMPLETE');+ expect(existsSync(join(root, 'training-runs/run-example/checkpoint.bin'))).toBe(true);+});++it('reports newly appearing files as retained instead of falsely reporting an empty category', async () => {+ const p = (await preview()).body;+ state.recreate = true;+ const result = await execute(p);+ expect(result.status).toBe(200);+ expect(result.body).toMatchObject({ status: 'retained', remainingFiles: 1, remainingBytes: 10 });+ expect(existsSync(join(root, 'training-runs/new.bin'))).toBe(true);+});++it('refuses a linked data root instead of following it into another location', async () => {+ const moved = `${root}-moved`;+ renameSync(root, moved);+ try {+ symlinkSync(moved, root, process.platform === 'win32' ? 'junction' : 'dir');+ expect((await preview()).body.code).toBe('MAINTENANCE_UNSAFE');+ expect(existsSync(join(moved, 'training-runs/run-example/checkpoint.bin'))).toBe(true);+ } finally {+ rmSync(root, { recursive: true, force: true });+ renameSync(moved, root);+ }+});diff --git a/server/routes/hostControlAuxiliaryMedia.test.js b/server/routes/hostControlAuxiliaryMedia.test.jsindex e1848b7085..0de98de33d 100644--- a/server/routes/hostControlAuxiliaryMedia.test.js+++ b/server/routes/hostControlAuxiliaryMedia.test.js@@ -75,11 +75,11 @@ const operations = [['post', '/api/media-jobs/example-job/retry', retryBody, effects.enqueue, 200],['post', '/api/media-jobs/example-job/run-now', {}, effects.runNow, 200],['post', '/api/lora-datasets/example-dataset/generate',- { count: 1, poses: ['Example pose'], outfits: ['Example outfit'] }, effects.generate, 202],+ { count: 1, poses: ['Example pose'], outfits: ['Example outfit'] }, effects.generate, 410],['post', '/api/lora-datasets/example-dataset/slice-reference-sheet',- { captionProviderId: 'example-vision-cli' }, effects.slice, 201],+ { captionProviderId: 'example-vision-cli' }, effects.slice, 410],['post', '/api/lora-datasets/example-dataset/caption',- { providerId: 'example-vision-cli' }, effects.caption, 202],+ { providerId: 'example-vision-cli' }, effects.caption, 410],];const appFor = (address = '192.0.2.10') => {const app = express();@@ -145,7 +145,7 @@ describe('auxiliary media agent dispatch authority (#9672)', () => {const before = operation[3].mock.calls.length;const response = await call(appFor(address), operation, headers);expect(response.status, `${operation[1]}: ${JSON.stringify(response.body)}`).toBe(operation[4]);- expect(operation[3]).toHaveBeenCalledTimes(before + 1);+ expect(operation[3]).toHaveBeenCalledTimes(before + (operation[4] === 410 ? 0 : 1));}}// Retry keeps the original agent backend while taking the new prompt.diff --git a/server/routes/hostControlPreGuardTracksData.test.js b/server/routes/hostControlPreGuardTracksData.test.jsindex 48fd35fb0a..6e4fddb772 100644--- a/server/routes/hostControlPreGuardTracksData.test.js+++ b/server/routes/hostControlPreGuardTracksData.test.js@@ -98,10 +98,10 @@ vi.mock('../services/games/index.js', () => ({}));vi.mock('../services/dataManager.js', () => ({ archiveCategory: effects.archive }));vi.mock('../services/loraTraining/index.js', () => ({+ cancel: vi.fn(() => false),getRunRequired: effects.getRun,promoteCheckpoint: effects.promote,}));-vi.mock('../services/loraTraining/runtimes.js', () => ({ TRAINING_DEFAULTS: {} }));vi.mock('../lib/pythonSetup.js', () => ({}));vi.mock('../services/signalSync.js', () => ({ checkSetup: effects.signalSetup, runSync: effects.signalSync }));vi.mock('../services/dataSync.js', () => ({ applyRemote: effects.applyRemote }));diff --git a/server/routes/loraDatasets.caption.test.js b/server/routes/loraDatasets.caption.test.jsindex e4fa1d9b01..e493d04192 100644--- a/server/routes/loraDatasets.caption.test.js+++ b/server/routes/loraDatasets.caption.test.js@@ -74,3 +74,16 @@ describe('LoRA dataset caption-run routes', () => {expect(res.status).toBe(404);});});++it('refuses every retired dataset mutation before reaching a producer or deleting retained files', async () => {+ for (const [method, path] of [+ ['post', ''], ['patch', '/example'], ['delete', '/example'], ['post', '/example/images'],+ ['post', '/example/import-gallery'], ['post', '/example/generate'], ['post', '/example/caption'],+ ['post', '/example/slice-reference-sheet'], ['post', '/example/strip-shared-fragments'],+ ['patch', '/example/images/image-example'], ['delete', '/example/images/image-example'],+ ]) {+ const response = await request(makeApp())[method](`/api/lora-datasets${path}`).send({});+ expect(response.status).toBe(410);+ expect(response.body.code).toBe('LORA_TRAINING_RETIRED');+ }+});diff --git a/server/routes/loraDatasets.js b/server/routes/loraDatasets.jsindex 4513739205..38988792ef 100644--- a/server/routes/loraDatasets.js+++ b/server/routes/loraDatasets.js@@ -1,40 +1,27 @@-/**- * LoRA training dataset routes — `/api/lora-datasets`.- *- * CRUD + image upload (multipart) + batch generation + reference-sheet- * slicing + vision captioning (SSE) for the per-subject training- * datasets behind /models/training. Training-run launch lives at- * `/api/lora-training` (routes/loraTraining.js); this surface ends at- * "dataset ready to train".- */-+import { refuseRetiredTraining } from '../lib/retiredLoraTraining.js';+/** Read-only retired datasets and explicit cancellation of outstanding captions. */import { Router } from 'express';import { z } from 'zod';import { asyncHandler, ServerError } from '../lib/errorHandler.js';import { validateRequest, isPaginationRequested, paginateArray } from '../lib/validation.js';-import { uploadFields } from '../lib/multipart.js';-import {- addUploadedImage,- createDataset,- deleteDataset,- deleteImage,- getDataset,- importGalleryImages,- listDatasets,- patchDataset,- reconcileRenderingImages,- stripSharedCaptionFragments,- updateImageCaption,-} from '../services/loraDatasets.js';-import { generateDatasetImages, getDatasetVariationAxes, sliceReferenceSheet } from '../services/loraDatasetGenerate.js';-import {- attachCaptionSseClient, cancelCaptionRun, getActiveCaptionRun, startCaptionRun,-} from '../services/loraDatasetCaption.js';++import { listDatasets, reconcileRenderingImages } from '../services/loraDatasets.js';++import { attachCaptionSseClient, cancelCaptionRun, getActiveCaptionRun } from '../services/loraDatasetCaption.js';import { LORA_DATASET_ENTRY_KINDS, computeDatasetReadiness } from '../lib/loraDataset.js';const router = Router();+// Keep history/image reads and explicit caption cancellation available. Legacy+// clients cannot create, rewrite, render, caption, or delete training datasets.+router.use((req, _res, next) => {+ if (req.method !== 'GET' && !/^\/[^/]+\/caption-runs\/[^/]+\/cancel$/.test(req.path)) refuseRetiredTraining();+ next();+});++router.post('/:id/generate', refuseRetiredTraining);+router.post('/:id/caption', refuseRetiredTraining);+router.post('/:id/slice-reference-sheet', refuseRetiredTraining);-const triggerWordSchema = z.string().regex(/^[a-z0-9_]{2,64}$/, 'trigger word must be 2-64 chars of [a-z0-9_]');const idSchema = z.string().min(1).max(128);const entryKindSchema = z.enum(LORA_DATASET_ENTRY_KINDS);@@ -51,18 +38,6 @@ router.get('/', asyncHandler(async (req, res) => {res.json(paginateArray(datasets, req.query, { defaultLimit: 50, maxLimit: 500 }));}));-const createSchema = z.object({- universeId: idSchema,- entryKind: entryKindSchema.default('characters'),- entryId: idSchema,- triggerWord: triggerWordSchema.optional(),-});-router.post('/', asyncHandler(async (req, res) => {- const body = validateRequest(createSchema, req.body);- const { dataset, created } = await createDataset(body);- res.status(created ? 201 : 200).json(dataset);-}));-router.get('/:id', asyncHandler(async (req, res) => {// Heal any images stuck in 'rendering' (server restart dropped the live// completion hook) before returning — the grid then shows truth.@@ -72,108 +47,6 @@ router.get('/:id', asyncHandler(async (req, res) => {res.json({ ...dataset, readiness: computeDatasetReadiness(dataset), captionRun: getActiveCaptionRun(req.params.id) });}));-// Live variation axes (expressions/outfits for characters; lighting/settings-// for objects & places) so the generate-batch dialog seeds its override chips-// from the server vocab instead of duplicating the object/place axis constants.-router.get('/:id/variation-axes', asyncHandler(async (req, res) => {- res.json(await getDatasetVariationAxes(req.params.id));-}));--const patchSchema = z.object({- triggerWord: triggerWordSchema.optional(),- // Reassign the dataset to a different universe subject. Universe + entry id- // must be present together; the service re-snapshots the subject and refuses- // a collision with an existing dataset for that subject.- universeId: idSchema.optional(),- entryKind: entryKindSchema.optional(),- entryId: idSchema.optional(),-});-router.patch('/:id', asyncHandler(async (req, res) => {- const body = validateRequest(patchSchema, req.body);- const dataset = await patchDataset(req.params.id, body);- res.json({ ...dataset, readiness: computeDatasetReadiness(dataset) });-}));--router.delete('/:id', asyncHandler(async (req, res) => {- res.json(await deleteDataset(req.params.id));-}));--// Image upload — up to 10 files per request on image1…image10 field names.-const UPLOAD_FIELDS = Array.from({ length: 10 }, (_, i) => `image${i + 1}`);-const ACCEPTED_MIME = new Set(['image/png', 'image/jpeg', 'image/webp']);-const MAX_UPLOAD_BYTES = 25 * 1024 * 1024;-const datasetUploads = uploadFields(UPLOAD_FIELDS, {- limits: { fileSize: MAX_UPLOAD_BYTES },- fileFilter: (_req, file, cb) => cb(null, ACCEPTED_MIME.has((file.mimetype || '').toLowerCase())),-});-router.post('/:id/images', datasetUploads, asyncHandler(async (req, res) => {- const staged = Object.values(req.files || {});- if (!staged.length) {- throw new ServerError('No image files in request (use field names image1…image10)', {- status: 400, code: 'VALIDATION_ERROR',- });- }- const entries = [];- for (const file of staged) {- entries.push(await addUploadedImage(req.params.id, {- tmpPath: file.path, originalname: file.originalname,- }));- }- res.status(201).json({ images: entries });-}));--// Import existing gallery images into the dataset by basename. The server-// re-normalizes each through sharp (gallery PNGs are `<jobId>.png`).-const importGallerySchema = z.object({- filenames: z.array(z.string().max(256).regex(/^[^/\\]+\.png$/i, 'filename must be a gallery PNG basename')).min(1).max(50),-});-router.post('/:id/import-gallery', asyncHandler(async (req, res) => {- const { filenames } = validateRequest(importGallerySchema, req.body);- const images = await importGalleryImages(req.params.id, { filenames });- res.status(201).json({ images });-}));--const generateSchema = z.object({- count: z.number().int().min(1).max(40).default(12),- views: z.array(z.string().max(120)).max(12).optional(),- poses: z.array(z.string().max(120)).max(12).optional(),- expressions: z.array(z.string().max(120)).max(12).optional(),- outfits: z.array(z.string().max(120)).max(12).optional(),- modelId: z.string().max(128).optional(),-});-router.post('/:id/generate', asyncHandler(async (req, res) => {- const body = validateRequest(generateSchema, req.body);- res.status(202).json(await generateDatasetImages(req.params.id, body));-}));--const sliceSchema = z.object({- variant: z.string().max(64).default('standard'),- cols: z.number().int().min(1).max(6).default(3),- rows: z.number().int().min(1).max(6).default(2),- // When true (default) a vision-LLM proposes a bounding box per figure and the- // grid is the fallback; false forces the deterministic cols×rows grid.- useVision: z.boolean().default(true),- // Optional explicit vision model for the proposal pass — same provider/model- // pair the caption picker uses; omitted = server auto-pick.- captionProviderId: z.string().max(128).optional(),- captionModel: z.string().max(256).optional(),-});-router.post('/:id/slice-reference-sheet', asyncHandler(async (req, res) => {- const body = validateRequest(sliceSchema, req.body);- res.status(201).json(await sliceReferenceSheet(req.params.id, body));-}));--const captionSchema = z.object({- imageIds: z.array(idSchema).max(200).optional(),- providerId: z.string().max(128).optional(),- model: z.string().max(256).optional(),- overwrite: z.boolean().default(false),-});-router.post('/:id/caption', asyncHandler(async (req, res) => {- const body = validateRequest(captionSchema, req.body);- res.status(202).json(await startCaptionRun(req.params.id, body));-}));-router.get('/:id/caption-runs/:runId/events', asyncHandler(async (req, res) => {if (!attachCaptionSseClient(req.params.id, req.params.runId, res)) {throw new ServerError(`Caption run not found: ${req.params.runId}`, { status: 404, code: 'NOT_FOUND' });@@ -187,21 +60,4 @@ router.post('/:id/caption-runs/:runId/cancel', asyncHandler(async (req, res) =>res.status(202).json(cancelCaptionRun(req.params.id, req.params.runId));}));-// Bulk caption lint — strip the identity fragments shared across most captions-// so the trigger token (not the caption phrases) learns the character. The-// server recomputes the shared set itself; no request body.-router.post('/:id/strip-shared-fragments', asyncHandler(async (req, res) => {- res.json(await stripSharedCaptionFragments(req.params.id));-}));--const imagePatchSchema = z.object({ caption: z.string().max(2000) });-router.patch('/:id/images/:imageId', asyncHandler(async (req, res) => {- const { caption } = validateRequest(imagePatchSchema, req.body);- res.json(await updateImageCaption(req.params.id, req.params.imageId, caption));-}));--router.delete('/:id/images/:imageId', asyncHandler(async (req, res) => {- res.json(await deleteImage(req.params.id, req.params.imageId));-}));-export default router;diff --git a/server/routes/loraTraining.js b/server/routes/loraTraining.jsindex d1574856f6..cf186bff11 100644--- a/server/routes/loraTraining.js+++ b/server/routes/loraTraining.js@@ -1,160 +1,64 @@-/**- * LoRA training run routes — `/api/lora-training`.- *- * Run lifecycle (start/list/detail/cancel/delete), per-run SSE progress- * (delegated to the media-job queue's stream for the run's jobId), sample- * image serving, and a runtime-readiness status endpoint for the UI.- */-+/** Retained run history and survivor controls; training launch is retired. */import { Router } from 'express';import { join } from 'path';import { z } from 'zod';import { asyncHandler, sendErrorResponse, ServerError } from '../lib/errorHandler.js';-import { startTrainingRunSchema, validateRequest } from '../lib/validation.js';+import { validateRequest } from '../lib/validation.js';import { assertSafeFilename } from '../lib/fileUtils.js';-import { resolveFlux2Python, isFlux2VenvHealthy, resolveMfluxPython } from '../lib/pythonSetup.js';-import { getSettings } from '../services/settings.js';+import { refuseRetiredTraining, TRAINING_RETIRED_MESSAGE } from '../lib/retiredLoraTraining.js';import { attachSseClient, cancelJob } from '../services/mediaJobQueue/index.js';-import {- deleteTrainingRun,- getRunRequired,- isMfluxTrainAvailable,- listCheckpoints,- listRuns,- listSamples,- promoteCheckpoint,- resumeTrainingRun,- runSamplesDir,- startTrainingRun,-} from '../services/loraTraining/index.js';-import { TRAINING_DEFAULTS } from '../services/loraTraining/runtimes.js';+import { cancel as cancelTrainer, getRunRequired, listCheckpoints, listRuns, listSamples, promoteCheckpoint, runSamplesDir } from '../services/loraTraining/index.js';const router = Router();--// Engine readiness + defaults — drives the launch panel's enable/disable-// state and the params form's initial values. Both engines train FLUX.2-// Klein LoRAs; mflux (MLX-native) is preferred when its trainer ships-// with the user's install, the torch venv is the fallback.-router.get('/status', asyncHandler(async (_req, res) => {- const settings = await getSettings();- // Prefer the configured image-gen Python when it ships mflux-train, else the- // auto-discovered dedicated ~/.portos/venv-mflux — so the panel reports mflux- // ready even when the user never set pythonPath (fresh dedicated-venv install).- const mfluxPython = resolveMfluxPython(settings?.imageGen?.local?.pythonPath || null);- const flux2Python = resolveFlux2Python();- res.json({- runtimes: {- mflux: { ready: isMfluxTrainAvailable(mfluxPython), pythonPath: mfluxPython },- flux2: { ready: !!flux2Python && await isFlux2VenvHealthy(), venvPython: flux2Python },- },- defaults: { ...TRAINING_DEFAULTS, ...(settings?.loraTraining?.defaults || {}) },- });-}));--router.post('/runs', asyncHandler(async (req, res) => {- const body = validateRequest(startTrainingRunSchema, req.body);- res.status(202).json(await startTrainingRun(body));-}));-+router.use((req, _res, next) => {+ if (req.method !== 'GET' && !(req.method === 'POST' && /^\/runs\/[^/]+\/(cancel|promote-checkpoint)$/.test(req.path))) refuseRetiredTraining();+ next();+});+// Explicit tombstones preserve legacy route authorization and return 410.+router.post('/runs', refuseRetiredTraining);+router.post('/runs/:id/resume', refuseRetiredTraining);+router.get('/status', (_req, res) => res.json({ retired: true, message: TRAINING_RETIRED_MESSAGE, runtimes: { mflux: { ready: false }, flux2: { ready: false } } }));const listQuerySchema = z.object({status: z.enum(['queued', 'running', 'completed', 'failed', 'canceled']).optional(),- characterId: z.string().max(128).optional(),- datasetId: z.string().max(128).optional(),+ characterId: z.string().max(128).optional(), datasetId: z.string().max(128).optional(),limit: z.coerce.number().int().min(1).max(500).optional(),});-router.get('/runs', asyncHandler(async (req, res) => {- const query = validateRequest(listQuerySchema, req.query);- res.json(await listRuns(query));-}));--router.get('/runs/:id', asyncHandler(async (req, res) => {- res.json(await getRunRequired(req.params.id));-}));--// Live progress — the run's SSE stream IS the media-job stream. The queue-// synthesizes a terminal frame for archived jobs, so late attaches after-// completion still resolve.+router.get('/runs', asyncHandler(async (req, res) => res.json(await listRuns(validateRequest(listQuerySchema, req.query)))));+router.get('/runs/:id', asyncHandler(async (req, res) => res.json(await getRunRequired(req.params.id))));router.get('/runs/:id/events', asyncHandler(async (req, res) => {const run = await getRunRequired(req.params.id);- if (!run.jobId || !attachSseClient(run.jobId, res)) {- throw new ServerError(`No live stream for run ${req.params.id} (status: ${run.status})`, {- status: 404, code: 'NOT_FOUND',- });- }+ if (!run.jobId || !attachSseClient(run.jobId, res)) throw new ServerError('No live stream for this retained run', { status: 404, code: 'NOT_FOUND' });}));-router.post('/runs/:id/cancel', asyncHandler(async (req, res) => {const run = await getRunRequired(req.params.id);- if (!run.jobId) {- throw new ServerError('Run has no job to cancel', { status: 409, code: 'ALREADY_TERMINAL' });- }- const result = await cancelJob(run.jobId);- if (!result.ok) {- const status = result.code === 'NOT_FOUND' ? 404 : 409;- throw new ServerError(result.error || 'Cancel failed', { status, code: result.code });+ const result = run.jobId ? await cancelJob(run.jobId) : { ok: false, code: 'NOT_FOUND', error: 'Run has no queue job' };+ if (!result.ok && result.code === 'NOT_FOUND' && cancelTrainer(run.jobId || run.id)) {+ res.json({ ok: true, status: 'canceling', orphan: true });+ return;}+ if (!result.ok) throw new ServerError(result.error || 'Cancel failed', { status: !run.jobId ? 409 : result.code === 'NOT_FOUND' ? 404 : 409, code: !run.jobId ? 'ALREADY_TERMINAL' : result.code });res.json(result);}));--// Resume a failed/canceled run from its latest checkpoint — re-enqueues a job-// against the SAME run id (mflux bakes the output path into the checkpoint, so-// the resumed trainer must write back into the original run dir). 202-shaped.-router.post('/runs/:id/resume', asyncHandler(async (req, res) => {- await getRunRequired(req.params.id); // 404 if the run is unknown- res.status(202).json(await resumeTrainingRun(req.params.id));-}));--const deleteRunQuerySchema = z.object({ deleteLora: z.enum(['true', 'false']).optional() });-router.delete('/runs/:id', asyncHandler(async (req, res) => {- const { deleteLora } = validateRequest(deleteRunQuerySchema, req.query);- res.json(await deleteTrainingRun(req.params.id, { withLora: deleteLora === 'true' }));-}));--// Checkpoint picker: list every saved checkpoint with its step, loss, and-// preview thumbnail so the user can promote one by eye. Loss is shown but is-// NOT a quality ranking — it was anti-correlated with quality on the-// divergence run this feature was built for (see checkpoints.js).router.get('/runs/:id/checkpoints', asyncHandler(async (req, res) => {- await getRunRequired(req.params.id); // 404 if the run is unknown- res.json(await listCheckpoints(req.params.id));+ await getRunRequired(req.params.id); res.json(await listCheckpoints(req.params.id));}));-const promoteSchema = z.object({ step: z.coerce.number().int().min(0) });router.post('/runs/:id/promote-checkpoint', asyncHandler(async (req, res) => {await getRunRequired(req.params.id);const { step } = validateRequest(promoteSchema, req.body);res.json(await promoteCheckpoint(req.params.id, step));}));--// Mid-training sample timeline (step + thumbnail URL) — seeds the live-// progress gallery so a reload mid-run shows every sample so far, not just-// the ones streamed after re-subscribing. Registered before the-// `/samples/:filename` serve route (which needs the extra path segment).router.get('/runs/:id/samples', asyncHandler(async (req, res) => {- await getRunRequired(req.params.id); // 404 if the run is unknown- res.json(await listSamples(req.params.id));+ await getRunRequired(req.params.id); res.json(await listSamples(req.params.id));}));-router.get('/runs/:id/samples/:filename', asyncHandler(async (req, res) => {const run = await getRunRequired(req.params.id);assertSafeFilename(req.params.filename, { extensions: ['.png'], subject: 'sample filename' });- // The sendFile callback fires outside asyncHandler's catch (the handler's- // promise has already resolved), so a throw here would have nothing to bubble- // to — build the standard envelope directly instead.- res.sendFile(join(runSamplesDir(run.id), req.params.filename), (err) => {+ res.sendFile(join(runSamplesDir(run.id), req.params.filename), err => {if (!err) return;- // Client cancelled (routine when the gallery unmounts an <img>) or the body- // was already streaming when the read failed — nothing left to say, just- // drop the socket rather than logging a false failure.- if (err.code === 'ECONNABORTED' || res.headersSent) {- res.destroy();- return;- }- // A real I/O failure (EACCES/EIO) reads the same as a missing file to the- // client — log the cause so it isn't invisible server-side.+ if (err.code === 'ECONNABORTED' || res.headersSent) { res.destroy(); return; }console.error(`❌ Sample send failed [${req.params.id}/${req.params.filename}]: ${err.message}`);sendErrorResponse(res, new ServerError('Sample not found', { status: 404 }));});}));-export default router;diff --git a/server/routes/loraTraining.test.js b/server/routes/loraTraining.test.jsindex 9921de129d..d7b26b1da7 100644--- a/server/routes/loraTraining.test.js+++ b/server/routes/loraTraining.test.js@@ -9,10 +9,10 @@ import { request } from '../lib/testHelper.js';// stub every collaborator so the suite only exercises the HTTP layer.const svc = vi.hoisted(() => ({ samplesDir: '' }));vi.mock('../services/loraTraining/index.js', () => ({+ cancel: vi.fn(() => false),clearDatasetForDeletedLora: vi.fn(),deleteRun: vi.fn(),getRunRequired: vi.fn(async (id) => ({ id, status: 'completed' })),- isMfluxTrainAvailable: () => false,listCheckpoints: vi.fn(),listRuns: vi.fn(),listSamples: vi.fn(),@@ -22,7 +22,6 @@ vi.mock('../services/loraTraining/index.js', () => ({runSamplesDir: () => svc.samplesDir,startTrainingRun: vi.fn(),}));-vi.mock('../services/loraTraining/runtimes.js', () => ({ TRAINING_DEFAULTS: {} }));vi.mock('../services/settings.js', () => ({ getSettings: async () => ({}) }));vi.mock('../services/mediaJobQueue/index.js', () => ({attachSseClient: vi.fn(),@@ -71,3 +70,35 @@ describe('lora-training sample serving', () => {expect(typeof r.body.timestamp).toBe('number');});});++it('retires launch, resume and legacy deletion while retaining history controls', async () => {+ const app = makeApp();+ for (const [method, path] of [['post', '/runs'], ['post', '/runs/example/resume'], ['delete', '/runs/example']]) {+ const response = await request(app)[method](`/api/lora-training${path}`).send({});+ expect(response.status).toBe(410);+ expect(response.body.code).toBe('LORA_TRAINING_RETIRED');+ }+ const status = await request(app).get('/api/lora-training/status');+ expect(status.body).toMatchObject({ retired: true, runtimes: { mflux: { ready: false }, flux2: { ready: false } } });+});++it('can explicitly cancel an observed orphan when the queue record is missing', async () => {+ const { cancel } = await import('../services/loraTraining/index.js');+ const { cancelJob } = await import('../services/mediaJobQueue/index.js');+ const { getRunRequired } = await import('../services/loraTraining/index.js');+ vi.mocked(getRunRequired).mockResolvedValueOnce({ id: 'example-run', jobId: 'example-job', status: 'running' });+ vi.mocked(cancelJob).mockResolvedValueOnce({ ok: false, code: 'NOT_FOUND' });+ vi.mocked(cancel).mockReturnValueOnce(true);+ const response = await request(makeApp()).post('/api/lora-training/runs/example-run/cancel');+ expect(response.status).toBe(200);+ expect(response.body).toMatchObject({ ok: true, status: 'canceling', orphan: true });+ expect(cancel).toHaveBeenCalledWith('example-job');+});++it('preserves the terminal-run cancellation contract when no queue job or observer exists', async () => {+ const { getRunRequired } = await import('../services/loraTraining/index.js');+ vi.mocked(getRunRequired).mockResolvedValueOnce({ id: 'example-ended', jobId: null, status: 'completed' });+ const response = await request(makeApp()).post('/api/lora-training/runs/example-ended/cancel');+ expect(response.status).toBe(409);+ expect(response.body.code).toBe('ALREADY_TERMINAL');+});diff --git a/server/services/README.md b/server/services/README.mdindex 8d4e82c0bf..17544de085 100644--- a/server/services/README.md+++ b/server/services/README.md@@ -40,6 +40,7 @@ image-generation paths below are useful entry points when tracing render jobs.| `creativeDirector/firstPassGen.js` | Gracefully gated first-pass portraits and scene frames. || `universeCharacterSheet.js` | Character reference-sheet render and completion workflow. || `loraDatasetGenerate.js` | Training-image render and dataset lifecycle. |+| `dataMaintenance.js` | Allowlisted retired-data inventory, expiring previews and explicit serialized removal with active-work refusal. || `sprites/reference.js` | Sprite reference candidate rendering and provenance. || `deckRender.js` | Deck card image render jobs. || `fableLoom/production.js` | FableLoom production render planning and enqueueing. |diff --git a/server/services/dataMaintenance.js b/server/services/dataMaintenance.jsnew file mode 100644index 0000000000..19196d7286--- /dev/null+++ b/server/services/dataMaintenance.js@@ -0,0 +1,129 @@+import { getNavPageForPath } from '../lib/navManifest.js';+import { withRetiredDataRemoval } from '../lib/retiredLoraTraining.js';+import { createHash, randomUUID } from 'node:crypto';+import { lstat, readdir, realpath } from 'node:fs/promises';+import { dirname, join, resolve } from 'node:path';+import { PATHS, renameGuarded, rmGuarded } from '../lib/fileUtils.js';+import { ServerError } from '../lib/errorHandler.js';+import { createMutex } from '../lib/asyncMutex.js';+import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';+import { listJobs } from './mediaJobQueue/index.js';+import { trainingRunsBusy } from './dataManagerBusy.js';+import { getActiveCaptionRun } from './loraDatasetCaption.js';++// Operations own their paths. Neither the client nor a preview can supply one.+export const MAINTENANCE_OPERATIONS = Object.freeze([+ { id: 'retired-training-runs', directory: 'training-runs', label: 'Retired LoRA training artifacts', category: 'Unique checkpoints and regenerable scratch', warning: 'May contain the only copy of an adapter, optimizer checkpoint, logs or preview. PostgreSQL run history is retained but will point to missing artifacts.', deletable: true },+ { id: 'retired-lora-datasets', directory: 'lora-datasets', label: 'Retired LoRA datasets', category: 'Unique source images and captions', warning: 'Source images, captions and dataset metadata may be irreplaceable. Copy anything you want to retain before deleting.', deletable: true },+ { id: 'shared-lora-adapters', directory: 'loras', label: 'Installed LoRA adapters', category: 'Shared inference assets', warning: `Adapters still work in image and media generation. Manage individual adapters in ${getNavPageForPath('/models/loras')?.breadcrumb}. Shared Python runtimes and model caches remain supported.`, deletable: false },+]);+const fail = (message, code = 'MAINTENANCE_UNSAFE') => new ServerError(message, { status: 409, code });+const missing = error => { if (error.code === 'ENOENT') return null; throw error; };+const previews = new Map();+const withLock = createMutex();+const PREVIEW_TTL = 5 * 60_000;++async function dataRootSnapshot() {+ let cursor = resolve(PATHS.data);+ for (;;) {+ const info = await lstat(cursor);+ if (info.isSymbolicLink()) throw fail('Data root contains a symbolic link; removal is unavailable.');+ const parent = dirname(cursor);+ if (parent === cursor) break;+ cursor = parent;+ }+ // Native canonicalization also expands Windows short names/casing. A changed+ // spelling is not evidence of a link; lstat above establishes that boundary.+ const root = await realpath(PATHS.data);+ const info = await lstat(root);+ return { root, identity: [info.dev, info.ino] };+}++async function inspect(directory, logicalDirectory = directory) {+ const { root, identity: rootIdentity } = await dataRootSnapshot();+ const hash = createHash('sha256');+ let identity = null;+ let bytes = 0;+ let files = 0;+ async function walk(path, relative) {+ const info = await lstat(path).catch(missing);+ if (!info) return;+ if (info.isSymbolicLink() || (!info.isDirectory() && !info.isFile())) throw fail('Unsupported file or symbolic link found; no files removed.');+ if (relative === logicalDirectory) identity = [info.dev, info.ino];+ hash.update(JSON.stringify([relative, info.dev, info.ino, info.mode, info.size, info.mtimeMs, relative === logicalDirectory ? null : info.ctimeMs]) + '\n');+ if (info.isFile()) { bytes += info.size; files += 1; }+ else for (const name of (await readdir(path)).sort()) await walk(join(path, name), `${relative}/${name}`);+ }+ await walk(join(root, directory), logicalDirectory);+ return { canonicalRoot: root, rootIdentity, bytes, files, present: identity !== null, fingerprint: hash.digest('hex'), identity };+}++async function assertIdle() {+ // Retired producers cannot start. Outstanding generation/caption requests may+ // still own dataset inputs, including work admitted by the outgoing server.+ if (listJobs().some(job => ['queued', 'running'].includes(job.status))) throw fail('Media work is queued or running. Wait for it to finish or cancel it explicitly.', 'MAINTENANCE_BUSY');+ const busy = await trainingRunsBusy();+ if (busy.busy !== false) throw fail(busy.reason || 'Training ownership could not be determined.', 'MAINTENANCE_BUSY');+ const datasets = await readdir(join(PATHS.data, 'lora-datasets')).catch(error => missing(error) || []);+ if (datasets.some(id => getActiveCaptionRun(id))) throw fail('Dataset captioning is still active; finish or cancel it first.', 'MAINTENANCE_BUSY');+}++export async function getMaintenanceInventory() {+ return Promise.all(MAINTENANCE_OPERATIONS.map(async operation => {+ try {+ const { bytes, files, present } = await inspect(operation.directory);+ return { ...operation, path: `data/${operation.directory}`, bytes, files, status: present ? 'retained' : 'absent', actions: operation.deletable && present ? ['delete'] : [] };+ } catch (error) {+ return { ...operation, path: `data/${operation.directory}`, bytes: null, files: null, status: 'unavailable', actions: [], error: error.code === 'MAINTENANCE_UNSAFE' ? error.message : 'Inventory could not be read. Check directory access before retrying.' };+ }+ }));+}++export async function previewMaintenance(id) {+ const operation = MAINTENANCE_OPERATIONS.find(item => item.id === id && item.deletable);+ if (!operation) throw new ServerError('Unsupported maintenance operation', { status: 404, code: 'NOT_FOUND' });+ await assertIdle();+ const inventory = await inspect(operation.directory);+ if (!inventory.present) throw fail('Nothing remains to delete. Refresh inventory.', 'MAINTENANCE_STALE');+ const now = Date.now();+ for (const [key, value] of previews) if (value.expiresAt < now) previews.delete(key);+ if (previews.size >= 100) throw fail('Too many outstanding previews. Wait for them to expire.');+ const preview = { token: randomUUID(), operationId: id, path: `data/${operation.directory}`, warning: operation.warning, bytes: inventory.bytes, files: inventory.files, confirmation: `DELETE ${operation.directory}`, expiresAt: now + PREVIEW_TTL };+ previews.set(preview.token, { ...preview, inventory, directory: operation.directory });+ return preview;+}++export async function executeMaintenance({ token, confirmation }) {+ return withLock(() => withBackupAssetPublication(() => withRetiredDataRemoval(async () => {+ const preview = previews.get(token);+ if (!preview || preview.expiresAt < Date.now()) throw fail('Preview expired. Refresh and preview again.', 'MAINTENANCE_STALE');+ if (confirmation !== preview.confirmation) throw new ServerError('Confirmation must match the preview exactly', { status: 400, code: 'CONFIRMATION_REQUIRED' });+ if (preview.result) return preview.result;+ await assertIdle();+ const current = await inspect(preview.directory);+ if (current.fingerprint !== preview.inventory.fingerprint || JSON.stringify(current.rootIdentity) !== JSON.stringify(preview.inventory.rootIdentity)) throw fail('Files changed since preview. Refresh and preview again.', 'MAINTENANCE_STALE');+ const source = join(current.canonicalRoot, preview.directory);+ const stagedName = `.maintenance-${preview.token}`;+ const staged = join(current.canonicalRoot, stagedName);+ await renameGuarded(source, staged);+ // Renaming first detaches the exact inspected directory from normal writers.+ // Revalidate identity and contents before removing anything. On uncertainty,+ // preserve the detached tree and report its recovery location.+ try {+ const detached = await inspect(stagedName, preview.directory);+ if (JSON.stringify(detached.rootIdentity) !== JSON.stringify(current.rootIdentity) || JSON.stringify(detached.identity) !== JSON.stringify(current.identity) || detached.fingerprint !== current.fingerprint) throw fail('Files changed while preparing deletion.');+ await assertIdle();+ const finalRoot = await dataRootSnapshot();+ if (finalRoot.root !== current.canonicalRoot || JSON.stringify(finalRoot.identity) !== JSON.stringify(current.rootIdentity)) throw fail('Data root changed while preparing deletion.');+ if (await lstat(source).catch(missing)) throw fail('The source directory was recreated while preparing deletion.');+ await rmGuarded(staged, { recursive: true, force: false });+ } catch (error) {+ const replacement = await lstat(source).catch(missing);+ if (!replacement) await renameGuarded(staged, source).catch(() => {});+ throw new ServerError(`Removal did not complete. Inspect data/${preview.directory} and data/${stagedName} before trying again.`, { status: 409, code: 'MAINTENANCE_INCOMPLETE', cause: error });+ }+ const remaining = await inspect(preview.directory).catch(() => null);+ preview.result = { ok: true, operationId: preview.operationId, removedBytes: current.bytes, removedFiles: current.files, status: remaining === null ? 'unavailable' : remaining.present ? 'retained' : 'deleted', remainingBytes: remaining?.bytes ?? null, remainingFiles: remaining?.files ?? null };+ return preview.result;+ })));+}diff --git a/server/services/dataManager.busy.test.js b/server/services/dataManager.busy.test.jsindex c3790b40ef..7192854efb 100644--- a/server/services/dataManager.busy.test.js+++ b/server/services/dataManager.busy.test.js@@ -57,7 +57,6 @@ const busyWith = (reason) => () => Promise.resolve({ busy: true, reason });describe('purgeCategory — CATEGORY_BUSY refusal', () => {const cases = [['image-clean-tmp', () => imageCleanTmpBusy, 'init-0000.png', 'an image render is in flight'],- ['training-runs', () => trainingRunsBusy, 'run-0001', 'a trainer is running'],['update-detached', () => updateDetachedBusy, 'pid', 'an update is running'],];@@ -74,10 +73,8 @@ describe('purgeCategory — CATEGORY_BUSY refusal', () => {expect(existsSync(dataPath(key))).toBe(true);});- it('refuses a per-item purge while the category is busy', async () => {- trainingRunsBusy.mockImplementation(busyWith('a trainer is running'));- await expect(purgeCategory('training-runs', { subPath: 'run-0001' }))- .rejects.toMatchObject({ status: 409, code: 'CATEGORY_BUSY' });+ it('refuses legacy training purge even while idle; maintenance requires its own preview', async () => {+ await expect(purgeCategory('training-runs')).rejects.toMatchObject({ status: 403 });expect(existsSync(dataPath('training-runs', 'run-0001'))).toBe(true);});@@ -90,20 +87,20 @@ describe('purgeCategory — CATEGORY_BUSY refusal', () => {// "Could not verify" and "nothing is running" must not collapse into the same// answer — the purge is irreversible.it('fails closed when the probe throws', async () => {- trainingRunsBusy.mockImplementation(() => Promise.reject(new Error('queue unavailable')));- await expect(purgeCategory('training-runs')).rejects.toMatchObject({ status: 409, code: 'CATEGORY_BUSY' });- expect(existsSync(dataPath('training-runs', 'run-0001'))).toBe(true);+ imageCleanTmpBusy.mockImplementation(() => Promise.reject(new Error('queue unavailable')));+ await expect(purgeCategory('image-clean-tmp')).rejects.toMatchObject({ status: 409, code: 'CATEGORY_BUSY' });+ expect(existsSync(dataPath('image-clean-tmp', 'init-0000.png'))).toBe(true);});it('fails closed when the probe returns a shape it cannot read', async () => {- trainingRunsBusy.mockImplementation(() => Promise.resolve({ reason: 'no busy flag' }));- await expect(purgeCategory('training-runs')).rejects.toMatchObject({ code: 'CATEGORY_BUSY' });- expect(existsSync(dataPath('training-runs', 'run-0001'))).toBe(true);+ imageCleanTmpBusy.mockImplementation(() => Promise.resolve({ reason: 'no busy flag' }));+ await expect(purgeCategory('image-clean-tmp')).rejects.toMatchObject({ code: 'CATEGORY_BUSY' });+ expect(existsSync(dataPath('image-clean-tmp', 'init-0000.png'))).toBe(true);});it('supplies a fallback reason when a busy probe names none', async () => {- trainingRunsBusy.mockImplementation(() => Promise.resolve({ busy: true }));- await expect(purgeCategory('training-runs')).rejects.toThrow(/Something is using data\/training-runs/);+ imageCleanTmpBusy.mockImplementation(() => Promise.resolve({ busy: true }));+ await expect(purgeCategory('image-clean-tmp')).rejects.toThrow(/Something is using data\/image-clean-tmp/);});});diff --git a/server/services/dataManager.itemPurge.test.js b/server/services/dataManager.itemPurge.test.jsindex 91a364f13b..cc58297e2c 100644--- a/server/services/dataManager.itemPurge.test.js+++ b/server/services/dataManager.itemPurge.test.js@@ -151,8 +151,8 @@ describe('training-run source details', () => {description: 'example-model · completed',});expect(detail.items.find(item => item.name === 'orphan-run')).toMatchObject({ sourceUnavailable: 'Run record no longer available' });- await purgeCategory('training-runs', { subPath: 'example-run' });- expect(existsSync(dataPath('training-runs', 'example-run'))).toBe(false);+ await expect(purgeCategory('training-runs', { subPath: 'example-run' })).rejects.toMatchObject({ status: 403, code: 'CATEGORY_NOT_PURGEABLE' });+ expect(existsSync(dataPath('training-runs', 'example-run'))).toBe(true);expect(existsSync(dataPath('training-runs', 'orphan-run', 'checkpoint.bin'))).toBe(true);});diff --git a/server/services/dataManager.js b/server/services/dataManager.jsindex 00f412a3ea..29d1594ea6 100644--- a/server/services/dataManager.js+++ b/server/services/dataManager.js@@ -204,7 +204,7 @@ export const CATEGORIES = {'telegram': { label: 'Telegram', description: 'Telegram bot data', archivable: true, deletable: true, purgeScope: 'category' },'templates': { label: 'Visual Templates', description: 'Shipped layout assets used as render anchors', archivable: false, deletable: false },'tools': { label: 'Tools', description: 'Tool execution data', archivable: true, deletable: true, purgeScope: 'category' },- 'training-runs': { label: 'LoRA Training Runs', description: 'Training checkpoints, caches, and sample previews — the finished adapters live in LoRAs and survive a purge; run history in Postgres will point at missing artifacts', archivable: false, deletable: true, purgeScope: 'category', busyCheck: trainingRunsBusy },+ 'training-runs': { label: 'LoRA Training Runs', description: 'Retired training checkpoints and previews; use Settings → Data Maintenance for a fresh preview and confirmed removal. May contain unique adapters.', archivable: false, deletable: false, busyCheck: trainingRunsBusy },'universes': { label: 'Universes', description: 'Universe Builder records — bibles, canon, and style references', archivable: true, deletable: false },'update-detached': { label: 'Update Control', description: 'Control files for a detached self-update run — safe to purge when no update is running', archivable: false, deletable: true, purgeScope: 'category', busyCheck: updateDetachedBusy },'uploads': { label: 'Uploads', description: 'Files uploaded through the UI and referenced by records', archivable: true, deletable: false },diff --git a/server/services/loraDatasets.js b/server/services/loraDatasets.jsindex 92ad069322..ce08a573b9 100644--- a/server/services/loraDatasets.js+++ b/server/services/loraDatasets.js@@ -1,3 +1,4 @@+import { withRetiredDataAccess } from '../lib/retiredLoraTraining.js';/*** LoRA training datasets — collectionStore CRUD + image file management.*@@ -77,7 +78,7 @@ const requireDataset = async (id) => {* lease through this record commit.*/export async function updateDataset(id, mutate) {- return withBackupAssetPublication(() => loraDatasetStore.queueRecordWrite(id, async () => {+ return withRetiredDataAccess(() => withBackupAssetPublication(() => loraDatasetStore.queueRecordWrite(id, async () => {const current = await loraDatasetStore.loadOne(id);if (!current) {throw new ServerError(`LoRA dataset not found: ${id}`, { status: 404, code: 'NOT_FOUND' });@@ -88,7 +89,7 @@ export async function updateDataset(id, mutate) {await loraDatasetStore.saveOneNow(id, next);trainingEvents.emit('dataset:changed', { datasetId: id });return next;- }));+ })));}const summarize = (dataset) => ({diff --git a/server/services/loraTraining/captionLeakStaging.test.js b/server/services/loraTraining/captionLeakStaging.test.jsdeleted file mode 100644index e803d26fef..0000000000--- a/server/services/loraTraining/captionLeakStaging.test.js+++ /dev/null@@ -1,97 +0,0 @@-import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest';-import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';--// runTraining takes the machine-wide heavy-local-job claim, whose path is-// derived from PATHS.data at module load. Without this redirect the suite wrote-// (and briefly held) the claim in the developer's LIVE data/ tree — a file that-// gates their real local renders, stranded there whenever a case failed before-// the release (#6176).-vi.mock('../../lib/fileUtils.js', async (importOriginal) =>- makePathsProxy(await importOriginal(), { dataRoot: () => lazyTempDataRoot('portos-caption-leak-') }));-afterAll(cleanupTempDataRoots);--// Regression guard for the caption-leak gate's SECOND call site. The gate is-// re-checked when the queued job actually stages (runTraining), not just at-// launch (startTrainingRun). The staging re-check must skip the gate ONLY for a-// run that already opted past it — an explicit "Train anyway" (persisted as-// run.captionLeakAcknowledged) or a resume (resumeCheckpoint set) — while still-// re-checking an ordinary clean-at-launch run, so captions edited leaky WHILE-// the run sat in the queue are caught instead of training silently.-const getRunMock = vi.fn();-const updateRunMock = vi.fn();-const validateDatasetReadyMock = vi.fn();-const updateDatasetMock = vi.fn();--vi.mock('./db.js', () => ({- getRun: (...a) => getRunMock(...a),- getRunRequired: vi.fn(),- updateRun: (...a) => updateRunMock(...a),- listRuns: vi.fn(),- deleteRun: vi.fn(),-}));-vi.mock('./dataset.js', () => ({ validateDatasetReady: (...a) => validateDatasetReadyMock(...a) }));-vi.mock('../settings.js', () => ({ getSettings: vi.fn(async () => ({})) }));-vi.mock('../loraDatasets.js', () => ({ updateDataset: (...a) => updateDatasetMock(...a) }));-vi.mock('../mediaJobQueue/index.js', () => ({- assertMediaQueueRoom: vi.fn(),- enqueueJob: vi.fn(),- getJob: vi.fn(),- mediaJobEvents: { emit: vi.fn(), on: vi.fn() },-}));--const { runTraining } = await import('./index.js');--// Mirrors the real gate: throw the leak 409 when the caller did NOT acknowledge,-// otherwise return a dataset whose character MISMATCHES the run so runTraining-// bails at the ownership check (before any real spawn). So a regressed staging-// call that dropped the flag surfaces as a leak failure rather than the-// reassignment one.-const LEAK_MSG = 'Identity is leaking into the captions — "red cloak" repeat across most images';-beforeEach(() => {- vi.clearAllMocks();- updateRunMock.mockResolvedValue();- updateDatasetMock.mockResolvedValue();- validateDatasetReadyMock.mockImplementation(async (_id, opts = {}) => {- if (!opts.acknowledgeCaptionLeak) {- const err = new Error(LEAK_MSG);- err.status = 409;- err.code = 'CAPTION_IDENTITY_LEAK';- throw err;- }- return { dataset: { character: { entryId: 'other', universeId: 'u1' } }, manifest: { images: [] } };- });-});--const failError = () =>- updateRunMock.mock.calls.find(([, patch]) => patch?.status === 'failed')?.[1]?.error || '';--describe('runTraining staging re-validation — caption-leak gate', () => {- it('skips the gate for a run launched with "Train anyway" (captionLeakAcknowledged)', async () => {- getRunMock.mockResolvedValue({- id: 'run-1', datasetId: 'ds1', captionLeakAcknowledged: true,- character: { entryId: 'c1', universeId: 'u1' },- });- await runTraining({ jobId: 'job-1', runId: 'run-1' });- expect(validateDatasetReadyMock).toHaveBeenCalledWith('ds1', { acknowledgeCaptionLeak: true });- expect(failError()).not.toMatch(/leaking/i); // bailed at ownership, not the leak gate- });-- it('skips the gate when resuming (captions already trained once)', async () => {- getRunMock.mockResolvedValue({- id: 'run-1', datasetId: 'ds1', captionLeakAcknowledged: false,- character: { entryId: 'c1', universeId: 'u1' },- });- await runTraining({ jobId: 'job-1', runId: 'run-1', resumeCheckpoint: '/ckpt/step-300' });- expect(validateDatasetReadyMock).toHaveBeenCalledWith('ds1', { acknowledgeCaptionLeak: true });- });-- it('STILL re-checks an ordinary queued run, catching captions edited leaky while queued', async () => {- getRunMock.mockResolvedValue({- id: 'run-1', datasetId: 'ds1', captionLeakAcknowledged: false,- character: { entryId: 'c1', universeId: 'u1' },- });- await runTraining({ jobId: 'job-1', runId: 'run-1' });- expect(validateDatasetReadyMock).toHaveBeenCalledWith('ds1', { acknowledgeCaptionLeak: false });- expect(failError()).toMatch(/leaking/i); // the queued-time leak is caught, run fails- });-});diff --git a/server/services/loraTraining/dataset.js b/server/services/loraTraining/dataset.jsdeleted file mode 100644index 22134e887c..0000000000--- a/server/services/loraTraining/dataset.js+++ /dev/null@@ -1,76 +0,0 @@-/**- * Dataset readiness validation for training launch. Re-checked both at- * route time (fast 409 before a run record exists) and at run start- * (defense vs. the dataset being deleted/edited while queued).- */--import { access } from 'fs/promises';-import { ServerError } from '../../lib/errorHandler.js';-import {- analyzeCaptionInvariants, captionHasTriggerWord, computeDatasetReadiness,-} from '../../lib/loraDataset.js';-import { getDataset, datasetImagePath, datasetImagesDir } from '../loraDatasets.js';--/**- * Load + validate the dataset, returning `{ dataset, manifest }` where- * manifest is the trainer-facing image list:- * `{ triggerWord, imagesDir, images: [{ file, path, caption }] }`.- *- * `acknowledgeCaptionLeak` skips the identity-leak gate (below) — set by the- * resume path (the run already cleared it once) and by an explicit- * "train anyway" from the UI.- */-export async function validateDatasetReady(datasetId, { acknowledgeCaptionLeak = false } = {}) {- const dataset = await getDataset(datasetId);- const readiness = computeDatasetReadiness(dataset);- if (!readiness.trainable) {- throw new ServerError(- `Dataset is not ready to train — needs ≥${readiness.required} ready images captioned with the trigger word `- + `(have ${readiness.captioned}/${readiness.required}${readiness.rendering ? `, ${readiness.rendering} still rendering` : ''})`,- { status: 409, code: 'DATASET_NOT_READY' },- );- }- // Backstop the caption omit-list: if the same identity fragments still repeat- // across most captions, the look is binding to the phrases instead of the- // trigger word — the LoRA renders a generic subject. Block (with the offending- // fragments) rather than train silently, so the user strips them or opts in.- if (!acknowledgeCaptionLeak) {- const { total, sharedFragments } = analyzeCaptionInvariants(dataset.images, dataset.triggerWord);- if (sharedFragments.length) {- const preview = sharedFragments.slice(0, 6).map((f) => f.fragment).join(', ');- throw new ServerError(- `Identity is leaking into the captions — "${preview}"${sharedFragments.length > 6 ? '…' : ''} repeat across most images, `- + 'so the trigger word would learn a generic subject. Strip the shared identity from the captions, or train anyway to override.',- { status: 409, code: 'CAPTION_IDENTITY_LEAK', context: { sharedFragments, total } },- );- }- }- // Same token-boundary predicate as the readiness gate — share the helper so- // the manifest can never include a caption the gate counted differently.- const trainImages = dataset.images.filter(- (img) => img.status === 'ready' && captionHasTriggerWord(img.caption, dataset.triggerWord),- );- // Every file must actually exist on disk — a missing file at trainer- // start would fail minutes later with an opaque python traceback.- for (const img of trainImages) {- const path = datasetImagePath(datasetId, img.file);- await access(path).catch(() => {- throw new ServerError(- `Dataset image missing on disk: ${img.file} — delete it from the dataset and retry`,- { status: 409, code: 'DATASET_NOT_READY' },- );- });- }- return {- dataset,- manifest: {- triggerWord: dataset.triggerWord,- imagesDir: datasetImagesDir(datasetId),- images: trainImages.map((img) => ({- file: img.file,- path: datasetImagePath(datasetId, img.file),- caption: img.caption.trim(),- })),- },- };-}diff --git a/server/services/loraTraining/dataset.test.js b/server/services/loraTraining/dataset.test.jsdeleted file mode 100644index 3076d554cc..0000000000--- a/server/services/loraTraining/dataset.test.js+++ /dev/null@@ -1,67 +0,0 @@-import { describe, it, expect, vi, beforeEach } from 'vitest';--// validateDatasetReady is a thin orchestration over the pure readiness/invariant-// helpers + getDataset + an fs.access existence probe. Mock the data + fs deps so-// the gate logic (leak detection, acknowledge override, resume skip) is testable-// without a real dataset store or files on disk.-vi.mock('../loraDatasets.js', () => ({- getDataset: vi.fn(),- datasetImagePath: vi.fn((id, file) => `/tmp/${id}/${file}`),- datasetImagesDir: vi.fn((id) => `/tmp/${id}`),-}));-vi.mock('fs/promises', () => ({ access: vi.fn(async () => {}) }));--const { getDataset } = await import('../loraDatasets.js');-const { validateDatasetReady } = await import('./dataset.js');--// 10 ready, trigger-prefixed images = trainable. `body(i)` lets a test make the-// per-shot descriptive tail unique (no leak) or repeat a fragment (leak).-const dataset = (body) => ({- id: 'ds1',- triggerWord: 'freydis',- character: { entryId: 'c1', universeId: 'u1', entryKind: 'characters' },- images: Array.from({ length: 10 }, (_, i) => ({- id: `img${i}`, file: `${i}.png`, status: 'ready', caption: `freydis, ${body(i)}`,- })),-});--beforeEach(() => vi.clearAllMocks());--describe('validateDatasetReady caption-identity-leak gate', () => {- it('passes (returns manifest) when captions share no identity fragment', async () => {- getDataset.mockResolvedValue(dataset((i) => `pose ${i}, angle ${i}`));- const { manifest } = await validateDatasetReady('ds1');- expect(manifest.triggerWord).toBe('freydis');- expect(manifest.images).toHaveLength(10);- });-- it('throws 409 CAPTION_IDENTITY_LEAK with the shared fragments when identity repeats', async () => {- getDataset.mockResolvedValue(dataset((i) => `red cloak, woven crown, pose ${i}`));- await expect(validateDatasetReady('ds1')).rejects.toMatchObject({- status: 409,- code: 'CAPTION_IDENTITY_LEAK',- context: {- total: 10,- sharedFragments: expect.arrayContaining([- expect.objectContaining({ fragment: 'red cloak' }),- expect.objectContaining({ fragment: 'woven crown' }),- ]),- },- });- });-- it('trains anyway when acknowledgeCaptionLeak overrides the gate', async () => {- getDataset.mockResolvedValue(dataset((i) => `red cloak, woven crown, pose ${i}`));- const { manifest } = await validateDatasetReady('ds1', { acknowledgeCaptionLeak: true });- expect(manifest.images).toHaveLength(10);- });-- it('still rejects an untrainable dataset before the leak check', async () => {- getDataset.mockResolvedValue({- id: 'ds1', triggerWord: 'freydis', character: { entryId: 'c1' }, images: [],- });- await expect(validateDatasetReady('ds1')).rejects.toMatchObject({- status: 409, code: 'DATASET_NOT_READY',- });- });-});diff --git a/server/services/loraTraining/index.js b/server/services/loraTraining/index.jsindex 1255d0d759..d4413bfb01 100644--- a/server/services/loraTraining/index.js+++ b/server/services/loraTraining/index.js@@ -1,368 +1,69 @@+import { refuseRetiredTraining, withRetiredDataAccess, acquireRetiredDataAccess } from '../../lib/retiredLoraTraining.js';/**- * LoRA training engine — run lifecycle + python trainer spawn.- *- * Routed through mediaJobQueue's GPU lane as `kind: 'training'` (training- * shares the Metal/MLX runtime with renders, so serialization is correct).- * The queue owns job status / SSE / watchdog / cancel escalation; this- * module owns the run RECORD (PostgreSQL `lora_training_runs`), the child- * process, artifact collection under `data/training-runs/<runId>/`, and- * trained-LoRA registration into `data/loras/`.- *- * Spawn/cancel mirrors videoGen/local.js: SIGTERM → 8s SIGKILL escalation,- * PYTHONUNBUFFERED, hfChildEnv, caffeinate on darwin.- * The trainers checkpoint on SIGTERM, so a cancel keeps its progress.+ * Retired trainer recovery only: observe/cancel a verified detached survivor,+ * finalize its output and keep historical records/checkpoint promotion readable.+ * New launch, resume and queue admission are permanently refused.*/import { spawn } from '../../lib/childProcess.js';import { existsSync } from 'fs';-import { join, basename, dirname } from 'path';+import { join, basename } from 'path';import { platform } from 'os';-import { PATHS, ensureDir, atomicWrite, shortId, copyFileGuarded, writeFileGuarded, rmGuarded } from '../../lib/fileUtils.js';+import { PATHS, ensureDir, shortId, copyFileGuarded, writeFileGuarded, rmGuarded } from '../../lib/fileUtils.js';import { ServerError } from '../../lib/errorHandler.js';import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';import { maintenance } from '../../lib/maintenanceAdmission.js';-import { v4 as uuidv4 } from '../../lib/uuid.js';-import { hfChildEnv } from '../hfToken.js';-import { spawnDetached, reapDetached, reattachDetached, isReattachable } from '../../lib/detachedSpawn.js';++import { reattachDetached, isReattachable } from '../../lib/detachedSpawn.js';import { killWithEscalation } from '../../lib/killWithEscalation.js';import { getImageModels } from '../../lib/mediaModels.js';-import { resolveFlux2Python, isFlux2VenvHealthy, resolveMfluxPython } from '../../lib/pythonSetup.js';+import { getSettings } from '../settings.js';import { deleteLora, writeLoraSidecar } from '../loras.js';-import { assertMediaQueueRoom, enqueueJob, getJob, mediaJobEvents } from '../mediaJobQueue/index.js';+import { getJob, mediaJobEvents } from '../mediaJobQueue/index.js';import { updateDataset } from '../loraDatasets.js';import { trainingEvents } from './events.js';import { sleepDisplayForTraining, wakeDisplay } from './displayPower.js';-import {- TRAINING_DEFAULTS,- TRAINING_RUNTIMES,- buildFlux2TrainArgs,- buildMfluxTrainArgs,- buildMfluxTrainConfig,- resolveTrainingRuntime,- MFLUX_DEFAULT_COOLDOWN_SEC,-} from './runtimes.js';+import { makeTrainingLineHandler } from './progress.js';import { makeStallDetector } from './stallDetector.js';-import { prepareMemoryForTraining, gpuBlockersMessage, TRAINING_MIN_HEADROOM_GB } from './memoryPrep.js';+import { claimHeavyLocalJob, adoptHeavyLocalJob } from '../../lib/heavyJobClaim.js';import { classifyTrainingFailure } from './failure.js';import { buildTrainedSidecar, trainedLoraFilename } from './sidecar.js';-import { validateDatasetReady } from './dataset.js';-import {- listRunCheckpoints,- listRunSamples,- resolveCheckpointAdapterBuffer,- resolveLatestCheckpointArtifact,- selectDeployableCheckpoint,-} from './checkpoints.js';++import { listRunCheckpoints, listRunSamples, resolveCheckpointAdapterBuffer, selectDeployableCheckpoint } from './checkpoints.js';import * as runsDb from './db.js';export { listRuns, getRun, getRunRequired, deleteRun } from './db.js';export { trainingEvents } from './events.js';-const TRAINER_SCRIPTS = {- [TRAINING_RUNTIMES.MFLUX]: join(PATHS.root, 'scripts', 'train_mflux_lora.py'),- [TRAINING_RUNTIMES.FLUX2]: join(PATHS.root, 'scripts', 'train_flux2_lora.py'),-};-export const runDir = (runId) => join(PATHS.trainingRuns, runId);export const runSamplesDir = (runId) => join(runDir(runId), 'samples');-/**- * mflux ships its trainer as the `mflux-train` console script — probe for- * it next to the configured local-image-gen python (e.g.- * /opt/miniconda3/bin/python3 → /opt/miniconda3/bin/mflux-train). Present- * only on mflux ≥0.17 installs.- */-export const isMfluxTrainAvailable = (pythonPath) =>- !!pythonPath && existsSync(join(dirname(pythonPath), 'mflux-train'));-// GPU lane serializes training with renders, so at most one trainer child// exists at a time. Keyed by jobId anyway so a stale cancel can't kill a// newer run.let activeProcess = null;let activeJobId = null;+const explicitlyCanceledTrainers = new WeakSet();-// Phase-aware stall watchdog (issue #1330): how many times a single run may be-// auto-resumed after a *soft* GPU hang before we give up and let it stay failed.-// Bounded so a config that wedges every segment can't resume-loop forever — the-// user sees a failed run after N attempts and intervenes. Counted separately-// from manual resumes (run.resume.autoCount). Env-overridable.-const STALL_MAX_AUTO_RESUMES = (() => {- const n = Number(process.env.LORA_TRAIN_STALL_MAX_AUTO_RESUMES);- return Number.isFinite(n) && n >= 0 ? n : 2;-})();-// How often the watchdog polls checkStall while a trainer is running. Well-// under the 90s floor budget so a stall is caught within ~one tick of the-// budget elapsing (worst-case detection lag = budget + one tick).+// Recovery still observes a surviving trainer for stalls.const STALL_TICK_MS = 30_000;export const cancel = (jobId) => {if (!activeProcess || (jobId && activeJobId !== jobId)) return false;const proc = activeProcess;+ explicitlyCanceledTrainers.add(proc);// Keep activeProcess set until 'close' clears it — the trainer may spend// a few seconds writing its cancel checkpoint. Escalate after 8s.killWithEscalation(proc, { label: 'training child', stillRunning: () => activeProcess === proc });return true;};-/** Merge order: code defaults ← settings slice ← request params. */-const mergeParams = (settings, requestParams = {}) => ({- ...TRAINING_DEFAULTS,- ...(settings?.loraTraining?.defaults || {}),- ...requestParams,-});+export async function startTrainingRun() { refuseRetiredTraining(); }-// A dataset belongs to a run only while it still points at the run's subject.-// Match the full (universeId, entryId) key the dataset store uses — a different-// universe can reuse the same entryId, so entryId alone would falsely re-own a-// reassigned dataset. (flipDatasetAfterRun keeps its own missing-entryId-// fallthrough for pre-reassignment runs that predate the character snapshot.)-const sameCharacter = (a, b) =>- a?.entryId === b?.entryId- && a?.universeId === b?.universeId- && (a?.entryKind || 'characters') === (b?.entryKind || 'characters');--// Re-stamp a dataset as `training` with the run's new job/run ids, but only-// while it still owns the dataset — the dataset can be reassigned to a different-// character between validation and this stamp (patchDataset resets it to draft);-// stamping a moved dataset would strand it in `training` forever (flipDatasetAfterRun-// is character-guarded and would skip the un-flip). Used by both the fresh-launch-// and resume paths.-const stampDatasetTrainingStatus = (run, jobId) =>- updateDataset(run.datasetId, (current) => {- if (!sameCharacter(current.character, run.character)) return null;- return {- ...current,- status: 'training',- training: { ...current.training, lastJobId: jobId, lastRunId: run.id },- };- }).catch((err) => console.error(`❌ dataset training-status stamp failed: ${err?.message}`));--/**- * Route-facing run launcher. Validates dataset readiness + runtime health,- * creates the run record, and enqueues the training job. Returns- * `{ runId, jobId, position }` (202-shaped).- */-export async function startTrainingRun({- datasetId, baseModelId, name = null, params = {}, acknowledgeCaptionLeak = false,-}) {- const settings = await getSettings();- // Resolve the mflux trainer's Python: the configured image-gen Python when it- // ships mflux-train (the `pip --user` layout), else an auto-discovered- // dedicated ~/.portos/venv-mflux (setup-image-video.sh's fallback for a system- // Python that can't host mflux). The resolved path threads into the job so- // runTraining spawns the right interpreter without re-resolving.- const pythonPath = resolveMfluxPython(settings?.imageGen?.local?.pythonPath || null);- // Engine pick: prefer mflux's MLX trainer when the user's mflux install- // ships it (Apple Silicon native, no second venv); fall back to the- // torch/diffusers trainer in venv-flux2.- const mlxAvailable = isMfluxTrainAvailable(pythonPath);- const routing = resolveTrainingRuntime(baseModelId, getImageModels(), { mlxAvailable });- const { dataset } = await validateDatasetReady(datasetId, { acknowledgeCaptionLeak });-- if (routing.runtime === TRAINING_RUNTIMES.FLUX2) {- // The torch/diffusers fallback trainer can't train on Apple Silicon:- // PyTorch's MPS backend has no `linear_backward` for the FLUX.2 transformer- // Linear layers, so a LoRA backward dies at the first optimizer step- // ("mps_linear_backward: unsupported weights data type", for both bf16 and- // fp32 — validated on an M-series box, issue #1227). mflux (MLX-native) is- // THE Apple-Silicon runtime. Refuse at request time so we never queue a run- // that loads a ~16 GB base and precomputes latents for minutes before- // crashing. `--device cpu` would work but is impractically slow, so the- // actionable guidance is to install mflux. (Linux/CUDA installs are the- // torch path's real target and are unaffected.)- if (platform() === 'darwin') {- throw new ServerError(- 'LoRA training on Apple Silicon requires the mflux runtime — the torch fallback can\'t train on this Mac\'s GPU (Metal/MPS). '- + 'Install mflux ≥0.17 in your local image-gen python (Settings → Image Gen) via `bash scripts/setup-image-video.sh`.',- { status: 412, code: 'TRAINING_MPS_UNSUPPORTED' },- );- }- const healthy = await isFlux2VenvHealthy();- if (!healthy) {- throw new ServerError(- 'No training engine available — install mflux ≥0.17 in your local image-gen python (Settings → Image Gen) '- + 'or set up the FLUX.2 venv via `bash scripts/setup-image-video.sh`',- { status: 412, code: 'TRAINING_ENGINE_MISSING' },- );- }- }-- // A full media queue (#8326) refuses the launch before the run row exists,- // so the common case leaves no failed run behind; the enqueue below still- // re-checks (and fails the row) if another producer filled the queue since.- assertMediaQueueRoom(1);- const mergedParams = mergeParams(settings, params);- const runId = uuidv4();- const run = {- id: runId,- jobId: null,- status: 'queued',- runtime: routing.runtime,- baseModelId,- fluxVariant: routing.variant,- trainRepo: routing.trainRepo,- mfluxModel: routing.mfluxModel,- name: name || null,- character: dataset.character,- datasetId,- triggerWord: dataset.triggerWord,- // Persist whether the launch opted past the caption identity-leak gate, so- // the staging re-validation can skip the gate for THIS run without also- // skipping it for ordinary queued runs whose captions were edited (leaky)- // while waiting in the queue.- captionLeakAcknowledged: acknowledgeCaptionLeak === true,- params: mergedParams,- progress: { step: 0, totalSteps: mergedParams.steps, loss: null, lastCheckpointStep: null },- artifacts: { dir: `training-runs/${runId}`, checkpoints: [], samples: [] },- output: { loraFilename: null, finalLoss: null },- error: null,- createdAt: new Date().toISOString(),- startedAt: null,- completedAt: null,- };- await runsDb.createRun(run);-- // The run row already exists, so a refused admission (#8325) must fail it- // rather than leave a 'queued' run no job will ever pick up.- const queued = await enqueueJob({- kind: 'training',- owner: 'lora-training',- params: {- runId,- runtime: routing.runtime,- datasetId,- characterId: dataset.character.entryId,- characterName: dataset.character.name,- triggerWord: dataset.triggerWord,- baseModelId,- steps: mergedParams.steps,- rank: mergedParams.rank,- pythonPath,- },- }).catch(async (err) => {- await runsDb.updateRun(runId, { status: 'failed', error: err.message, completedAt: new Date().toISOString() });- throw err;- });- await runsDb.updateRun(runId, { jobId: queued.jobId });- await stampDatasetTrainingStatus(run, queued.jobId);-- console.log(`🏋️ Training run ${shortId(runId)} queued — ${routing.runtime}/${baseModelId} dataset=${shortId(datasetId)} job=${shortId(queued.jobId)}`);- return { runId, jobId: queued.jobId, position: queued.position, status: 'queued' };-}--/**- * Resume a failed/canceled run from its latest on-disk checkpoint. mflux bakes- * the output path into the checkpoint zip and reads everything (config, dataset- * paths, optimizer + adapter state, step counter) from `--resume <zip>` — so a- * resume MUST re-run in the ORIGINAL run dir, not a fresh one, or the trainer- * would write its new artifacts where the wrapper's watcher can't see them.- * We therefore re-enqueue a new job against the SAME runId: new checkpoints and- * samples append to the existing artifact arrays (the checkpoint picker shows- * the full timeline across the resume), and finalize registers the LoRA as- * usual. Progress may briefly read low if the trainer restarts its step bar at- * the resume point — cosmetic; the durable record catches up on the next flush.- */-export async function resumeTrainingRun(runId, { auto = false } = {}) {- const run = await runsDb.getRunRequired(runId);- if (!['failed', 'canceled'].includes(run.status)) {- throw new ServerError(`Can only resume a failed or canceled run (status: ${run.status})`, {- status: 409, code: 'RUN_NOT_RESUMABLE',- });- }- // A run can be marked failed (boot reconcile / cancel) while its detached- // trainer is still alive — reap it before resuming so we never run two- // trainers against the same checkpoint dir. Idempotent: a no-op when nothing- // survives. Belt-and-suspenders with the boot-reconcile reap, since resume- // can race the reconcile's SIGTERM grace window.- const reaped = await reapDetached(join(runDir(runId), '.detached')).catch(() => ({ reaped: false }));- if (reaped.reaped) console.log(`🧹 reaped surviving trainer pid ${reaped.pid} before resuming run ${shortId(runId)}`);- // Both runtimes restore optimizer state + the step counter on resume, so- // training picks up mid-run and finishes at the original total: mflux via- // `mflux-train --resume <zip>`, and the torch FLUX.2 trainer via- // `--resume-from <dir>` (restores the AdamW state from the checkpoint's- // optimizer.pt and continues range(start_step + 1, steps + 1) — no- // over-training, no checkpoint renumber collisions).- const checkpoint = resolveLatestCheckpointArtifact(run);- if (!checkpoint) {- throw new ServerError(- 'No checkpoint to resume from — the run was killed before its first checkpoint saved. Start a fresh run.',- { status: 409, code: 'NO_RESUMABLE_CHECKPOINT' },- );- }-- // Re-validate dataset readiness + ownership exactly like a fresh launch — the- // dataset may have been edited, deleted, or reassigned since the run failed.- // Skip the caption-leak gate: this run already cleared it at first launch,- // and resume shouldn't re-block on captions the user already accepted.- const { dataset } = await validateDatasetReady(run.datasetId, { acknowledgeCaptionLeak: true });- if (!sameCharacter(dataset.character, run.character)) {- throw new ServerError('Dataset was reassigned to a different subject — start a fresh run.', {- status: 409, code: 'DATASET_REASSIGNED',- });- }-- // Engine health: mirror startTrainingRun for the run's existing runtime —- // including resolving the dedicated ~/.portos/venv-mflux, so a resume (manual- // OR the stall-watchdog auto-resume) on a dedicated-venv install doesn't- // falsely fail the availability check and thread a null interpreter into the- // re-enqueued job.- const settings = await getSettings();- const pythonPath = resolveMfluxPython(settings?.imageGen?.local?.pythonPath || null);- if (run.runtime === TRAINING_RUNTIMES.FLUX2) {- if (!(await isFlux2VenvHealthy())) {- throw new ServerError('FLUX.2 training venv is unavailable — run `bash scripts/setup-image-video.sh`', {- status: 412, code: 'TRAINING_ENGINE_MISSING',- });- }- } else if (!isMfluxTrainAvailable(pythonPath)) {- throw new ServerError('mflux-train not found next to the configured python — update mflux (≥0.17)', {- status: 412, code: 'TRAINING_ENGINE_MISSING',- });- }-- const queued = await enqueueJob({- kind: 'training',- owner: 'lora-training',- params: {- runId,- runtime: run.runtime,- datasetId: run.datasetId,- characterId: run.character?.entryId,- characterName: run.character?.name,- triggerWord: run.triggerWord,- baseModelId: run.baseModelId,- steps: run.params?.steps,- rank: run.params?.rank,- pythonPath,- resumeCheckpoint: checkpoint.path,- },- });- await runsDb.updateRun(runId, (current) => ({- ...current,- status: 'queued',- jobId: queued.jobId,- error: null,- errorCode: null,- errorRepo: null,- completedAt: null,- resume: {- count: (current.resume?.count || 0) + 1,- // Auto (stall-watchdog) resumes are counted separately so the watchdog- // can cap its own retries without a manual resume resetting the budget.- autoCount: (current.resume?.autoCount || 0) + (auto ? 1 : 0),- fromStep: checkpoint.step,- resumedAt: new Date().toISOString(),- lastReason: auto ? 'stall-watchdog' : 'manual',- },- }));- await stampDatasetTrainingStatus(run, queued.jobId);-- console.log(`🏋️ Training run ${shortId(runId)} ${auto ? 'auto-resumed (stall watchdog)' : 'resumed'} from step ${checkpoint.step} — job=${shortId(queued.jobId)}`);- return { runId, jobId: queued.jobId, position: queued.position, status: 'queued', fromStep: checkpoint.step };-}+export async function resumeTrainingRun() { refuseRetiredTraining(); }const emitFailed = (jobId, error) => trainingEvents.emit('failed', { generationId: jobId, error });@@ -467,10 +168,10 @@ export async function deleteTrainingRun(runId, { withLora = false } = {}) {* 'training'. Resolves the trainer binary + args, spawns, parses the line* protocol into trainingEvents, and finalizes (LoRA registration or* failure classification) on close. Terminal status flows through the- * queue's dispatcher; this function resolves once the child is spawned+ * queue's dispatcher; this function resolves once the observer attaches* (the queue awaits the terminal event separately).*/-export async function runTraining({ jobId, runId, pythonPath = null, resumeCheckpoint = null, reattach = false }) {+export async function runTraining({ jobId, runId, reattach = false, recoveryPermit = null }) {const fail = (message) => {console.error(`❌ training [${shortId(jobId)}] ${message}`);emitFailed(jobId, message);@@ -497,6 +198,11 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck// stuck `running` (lingering until the next boot reconcile) or the dataset// stuck on its `training` chip.const failBeforeSpawn = async (message) => {+ if (reattach && await hasSurvivingTrainer(runId)) {+ retainOwnership();+ console.error(`❌ training [${shortId(jobId)}] observer unavailable; live trainer and artifacts preserved: ${message}`);+ return { preserved: true };+ }await releaseClaim();await runsDb.updateRun(runId, {status: 'failed', error: message, completedAt: new Date().toISOString(),@@ -505,7 +211,7 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheckreturn fail(message);};- // Boot re-attach (#1332): the queue re-enqueues a run whose detached trainer+ // Recovery attaches only to a trainer that already exists. The queue restores a run whose detached trainer// SURVIVED a hard server restart with `reattach: true`. The child reparented// to init and is still training (or finished during the downtime), so we tail// its existing control-dir output instead of spawning a second trainer — no@@ -513,7 +219,7 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck// exact same line-handling/finalize path as a normal spawn, so a run that// completed mid-restart still registers its LoRA instead of being discarded.//- // This runs BEFORE the fresh claimHeavyLocalJob() below: the survivor already+ // The survivor already// holds the machine-wide accelerator claim from before the restart (handed// off to its PID pre-crash), so acquiring a NEW claim here would see that// live claim as a competing job and refuse it — failing every restart-@@ -523,9 +229,9 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheckconst proc = await reattachDetached(join(dir, '.detached'));if (!proc) {// The survivor died (killed mid-run) with no RESULT to recover — fail the- // run; its latest checkpoint stays resumable manually/auto, same as the+ // run; its latest checkpoint stays retained for inspection, same as the// pre-#1332 reap path would have left it.- return failBeforeSpawn('Trainer did not survive the restart — marking failed; resume from the latest checkpoint.');+ return failBeforeSpawn('Trainer did not survive the restart — marking failed; artifacts are preserved.');}heavyClaim = (await adoptHeavyLocalJob({ kind: 'LoRA training', id: jobId, pid: proc.pid }))|| (await claimHeavyLocalJob({ kind: 'LoRA training', id: jobId }));@@ -537,191 +243,13 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheckconsole.log(`🔁 training [${shortId(jobId)}] re-attached to surviving trainer pid ${proc.pid} (run ${shortId(runId)})`);trainingEvents.emit('status', { generationId: jobId, message: 'Re-attached to trainer that survived a restart' });wireProcLifecycle(proc, { isReattach: true });- return;- }-- heavyClaim = await claimHeavyLocalJob({ kind: 'LoRA training', id: jobId });- if (!heavyClaim.ok) return failBeforeSpawn(heavyClaim.message);-- // Re-validate — the dataset may have been edited/deleted while queued. Skip- // the caption identity-leak gate ONLY for a run that already opted past it:- // one launched with an explicit "Train anyway" (persisted on the record) or a- // resume (its captions already trained once). An ordinary clean-at-launch run- // is still re-checked here, so captions edited leaky while it sat in the queue- // are caught instead of training silently.- const skipCaptionLeakGate = run.captionLeakAcknowledged === true || resumeCheckpoint != null;- let manifest;- let dataset;- try {- ({ dataset, manifest } = await validateDatasetReady(- run.datasetId,- { acknowledgeCaptionLeak: skipCaptionLeakGate },- ));- } catch (err) {- return failBeforeSpawn(err.message);- }- // Stage-time ownership check: if the dataset was reassigned to a different- // subject after this run was queued, the run no longer owns it. Bail out- // rather than training the moved dataset and registering a LoRA under the- // run's now-stale subject. failBeforeSpawn's flipDatasetAfterRun is- // character-guarded, so it won't disturb the reassigned dataset's state.- // Match the full (universeId, entryId) key the dataset store uses elsewhere.- if (!sameCharacter(dataset.character, run.character)) {- return failBeforeSpawn('Dataset was reassigned to a different subject after this run was queued — cancel and retrain.');- }-- // Reclaim unified memory before a GPU-heavy run, then gate on real headroom.- // Training shares the unified-memory pool with resident LLMs and renders; an- // oversubscribed run swap-thrashes and has coincided with GPU watchdog- // reboots (docs/research/2026-06-13-mflux-training-watchdog-panic.md). We- // unload resident models, measure what's actually free, and refuse to start- // (rather than crash mid-run) when headroom is below the floor. The budget- // also sizes the mflux quantize/low_ram tier below.- const memReport = await prepareMemoryForTraining();- // A GPU tenant the unload path cannot evict (today: the vLLM Qwen container)- // makes the run unwinnable regardless of system-RAM headroom, so it is checked- // BEFORE the budget gate below — that gate reports free system RAM, which says- // nothing about VRAM on exactly the discrete-GPU host this matters on.- if (memReport.blockers.length) return failBeforeSpawn(gpuBlockersMessage(memReport.blockers));- if (memReport.unloaded.length) {- console.log(`🧹 training [${shortId(jobId)}] freed ${memReport.unloaded.length} resident model(s): ${memReport.unloaded.join(', ')}`);- }- console.log(`🧮 training [${shortId(jobId)}] memory budget ${memReport.budgetGb.toFixed(0)} GB free of ${memReport.totalGb.toFixed(0)} GB total`);- if (!Number.isFinite(memReport.budgetGb) || memReport.budgetGb < TRAINING_MIN_HEADROOM_GB) {- // Fail safe: a non-finite budget (should never happen — both inputs are- // finite) must REFUSE the run, not slip past `NaN < x` (always false).- return failBeforeSpawn(`Not enough free memory to train safely — ${memReport.budgetGb.toFixed(1)} GB available, need ≥ ${TRAINING_MIN_HEADROOM_GB} GB. Stop other model servers or close apps and retry.`);- }-- const checkpointsDir = join(dir, 'checkpoints');- const samplesDir = join(dir, 'samples');-- let bin;- let args;- // Staging I/O (mkdir + copyFile/writeFile/atomicWrite) is wrapped: a throw- // here — e.g. a dataset image deleted in the window after validateDatasetReady's- // existence check (TOCTOU), or disk-full — would otherwise propagate to the- // queue's catch (no crash) but leave the run record `running` forever.- try {- await ensureDir(checkpointsDir);- await ensureDir(samplesDir);-- if (run.runtime === TRAINING_RUNTIMES.FLUX2) {- bin = resolveFlux2Python();- if (!bin) {- return failBeforeSpawn('FLUX.2 python environment disappeared — re-run scripts/setup-image-video.sh');- }- const manifestPath = join(dir, 'manifest.json');- await atomicWrite(manifestPath, {- triggerWord: manifest.triggerWord,- images: manifest.images.map((img) => ({ path: img.path, caption: img.caption })),- });- args = buildFlux2TrainArgs({- scriptPath: TRAINER_SCRIPTS.flux2,- trainRepo: run.trainRepo,- manifestPath,- runDir: dir,- triggerWord: run.triggerWord,- params: run.params,- samplePrompt: run.params?.samplePrompt || null,- resumeFrom: resumeCheckpoint,- });- } else {- bin = pythonPath;- if (!bin || !isMfluxTrainAvailable(bin)) {- return failBeforeSpawn('mflux-train not found next to the configured python — update mflux (≥0.17) or set up the FLUX.2 venv');- }- // Stage the dataset in mflux's auto-discovery layout: NNNN.png +- // NNNN.txt caption pairs, plus preview_1.txt for the periodic sample- // render. mflux resolves everything from the config's `data` dir.- const dataDir = join(dir, 'data');- await ensureDir(dataDir);- for (let i = 0; i < manifest.images.length; i += 1) {- const stem = String(i + 1).padStart(4, '0');- await copyFileGuarded(manifest.images[i].path, join(dataDir, `${stem}.png`));- await writeFileGuarded(join(dataDir, `${stem}.txt`), `${manifest.images[i].caption}\n`);- }- if ((run.params?.sampleEvery ?? TRAINING_DEFAULTS.sampleEvery) > 0) {- const samplePrompt = run.params?.samplePrompt || `${run.triggerWord} portrait, neutral background`;- await writeFileGuarded(join(dataDir, 'preview_1.txt'), `${samplePrompt}\n`);- }- // output_path must NOT pre-exist — mflux appends a timestamp suffix to- // an existing dir (its new_folder behavior), which would break the- // wrapper's artifact watcher. mflux creates checkpoints/ + preview/- // (+ loss/) inside it.- const mfluxOutputDir = join(dir, 'mflux');- const config = buildMfluxTrainConfig({- params: run.params,- variant: run.fluxVariant,- mfluxModel: run.mfluxModel,- dataDir,- imageCount: manifest.images.length,- outputDir: mfluxOutputDir,- // Memory-derived quantize/low_ram (see deriveMfluxMemoryConfig) keyed- // on the post-unload AVAILABLE budget, not raw RAM — a bf16 base +- // in-RAM latent cache OOM-killed a 48 GB machine, and on a shared box- // resident models eat the same pool, so the tier must track headroom.- totalMemGb: memReport.budgetGb,- });- const configPath = join(dir, 'mflux-train.json');- await atomicWrite(configPath, config);- // Segmented training (watchdog-panic mitigation): default ON, globally- // disable-able via settings.loraTraining.segmentation = false once a- // macOS/mflux update fixes the underlying GPU-driver hang. Segment size- // is the config's effective save_frequency so each segment ends exactly- // on a checkpoint — no extra checkpoints, no lost steps on resume.- const segCfg = settings?.loraTraining || {};- const segmentationOn = segCfg.segmentation !== false;- const cooldownSec = Number.isFinite(segCfg.segmentCooldownSec)- ? segCfg.segmentCooldownSec : MFLUX_DEFAULT_COOLDOWN_SEC;- args = buildMfluxTrainArgs({- scriptPath: TRAINER_SCRIPTS.mflux,- configPath,- runDir: dir,- totalSteps: run.params?.steps || TRAINING_DEFAULTS.steps,- resumeCheckpoint,- segmentSteps: segmentationOn ? config.checkpoint.save_frequency : 0,- cooldownSec,- });- }- } catch (err) {- return failBeforeSpawn(`staging failed: ${err.message}`);+ return { observing: true };}- await runsDb.updateRun(runId, { status: 'running', startedAt: new Date().toISOString() });- trainingEvents.emit('status', { generationId: jobId, message: `Starting ${run.runtime} training (${run.params.steps} steps)` });-- const childEnv = await hfChildEnv({ PYTHONUNBUFFERED: '1' });- delete childEnv.PYTHONPATH;+ refuseRetiredTraining();- console.log(`🏋️ training [${shortId(jobId)}] spawn ${basename(bin)} ${run.runtime} steps=${run.params.steps} rank=${run.params.rank} images=${manifest.images.length}${resumeCheckpoint ? ` resume=${basename(resumeCheckpoint)}` : ''}`);- // `spawnDetached` double-forks the trainer so it reparents to init (PPID=1)- // and leaves pm2's process tree entirely. A plain `detached: true` child only- // gets its OWN process group — but pm2's TreeKill keys on PPID, not the group,- // so the still-PPID-linked trainer was found and SIGINT'd anyway whenever- // portos-server restarted (e.g. on the memory ceiling, which a long session- // crosses routinely — observed twice: SIGINT/KeyboardInterrupt at the exact- // second of a server restart, losing hours of GPU work). The detached trainer- // streams stdout/stderr through on-disk log files under `<runDir>/.detached`- // that the server tails. We still `proc.kill()` it directly by PID on cancel,- // and the queue worker awaits the 'close' event for the run lifecycle. No- // `cleanup` — those logs are the only copy of raw trainer stdout/stderr, kept- // in the run dir for post-mortem and removed when the run dir is deleted.- let proc;- try {- proc = await spawnDetached(bin, args, { env: childEnv, controlDir: join(dir, '.detached') });- } catch (err) {- return failBeforeSpawn(`trainer spawn failed: ${err.message}`);- }- wireProcLifecycle(proc);-- // Wire a freshly-spawned OR re-attached (#1332) trainer handle into the full- // run lifecycle: live output streaming + progress mirror, the phase-aware- // stall watchdog, and the close→finalize path that registers the LoRA. A- // hoisted nested function so the early reattach branch (above) can call it too- // — both paths share identical handling. Closes over jobId/runId/run/dir/- // settings/fail/failBeforeSpawn.function wireProcLifecycle(proc, { isReattach = false } = {}) {+ const releaseDataAccess = acquireRetiredDataAccess();activeProcess = proc;activeJobId = jobId;void heavyClaim?.handoffTo?.(proc.pid)?.catch((err) => console.error(`❌ training [${shortId(jobId)}] claim handoff failed: ${err.message}`));@@ -821,8 +349,7 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck// protocol to apply a tight, step-rate-derived budget ONLY during// STAGE:training — load/encode/sampling/cooldown stay on the flat watchdog so// a legitimately slow phase is never false-killed. On a training-phase stall- // we SIGKILL the wedged child; the close handler auto-resumes from the newest- // checkpoint. Disable via settings.loraTraining.stallWatchdog = false.+ // we stop the wedged child and retain its newest checkpoint. Disable via settings.loraTraining.stallWatchdog = false.const stallWatchdogOn = (settings?.loraTraining?.stallWatchdog !== false);const stallDetector = makeStallDetector();let stallKilled = false; // set when the tick SIGKILLs — close handler reads it@@ -884,11 +411,10 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheckif (!stalled) return;stallKilled = true;stopStallWatchdog();- console.log(`⏱️ training [${shortId(jobId)}] phase-aware watchdog: training stalled ${Math.round(idleMs / 1000)}s (budget ${Math.round(budgetMs / 1000)}s) — SIGKILL + auto-resume`);+ console.log(`⏱️ training [${shortId(jobId)}] phase-aware watchdog: training stalled ${Math.round(idleMs / 1000)}s (budget ${Math.round(budgetMs / 1000)}s) — stopping stalled trainer; checkpoints preserved`);// Soft hang: SIGKILL straight away (SIGTERM relies on the GIL the hang// holds, so the 8s escalation in cancel() would just waste the window).- // The newest checkpoint is the resume point; the close handler picks it- // up. Direct PID kill via the detached handle.+ // Direct PID kill through the already-owned detached handle.proc.kill('SIGKILL');} catch (err) {console.error(`❌ training [${shortId(jobId)}] stall watchdog tick failed: ${err?.message}`);@@ -911,7 +437,7 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck// A no-PID launch failure owns no running trainer.wakeDisplay(settings);// A launch failure after the run was marked `running` — a genuine spawn- // error, or (with spawnDetached) the control dir failing to create/clear.+ // error, including a detached observer losing its control directory.// Route through the same terminal cleanup as pre-spawn failures so the run// row doesn't stay stuck `running` and the dataset's `training` chip is// released; `fail()` alone only logs + emits. Async + guarded since this@@ -932,22 +458,16 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheck// finalize reads the run record — the collapse guard and previewImageUrl// both read run.artifacts. Async finalize wrapped so a rejection can't// escape the event handler (unhandled rejection kills the process on Node ≥15).- // The trainer has exited, so release before finalization can enqueue an- // automatic checkpoint resume; that successor must acquire a fresh claim.- releaseClaim()- .then(() => flushProgress())- .then(() => processError- ? failBeforeSpawn(`trainer process failed: ${processError.message}`)- : finalizeTraining({ jobId, runId, code, signal, state: getState(), stallKilled }))- .then((resumed) => {- // Wake the display now the run is over so the user sees the result —- // but NOT when finalize actually enqueued a stall-watchdog auto-resume- // (it re-spawns and re-sleeps the display for the next attempt). Gating- // on the returned `resumed` flag, not raw `stallKilled`, ensures a- // budget-exhausted or failed resume (terminal, no re-spawn) still wakes.- // Best-effort, darwin-only.- if (!resumed) wakeDisplay(settings);- })+ // The trainer has exited; release its accelerator claim before finalizing.+ const settle = async () => {+ await releaseClaim();+ await flushProgress();+ if (processError) return failBeforeSpawn(`trainer process failed: ${processError.message}`);+ return finalizeTraining({ jobId, runId, code, signal, state: getState(), stallKilled, canceledByOperator: explicitlyCanceledTrainers.has(proc) });+ };+ const terminalWork = recoveryPermit ? recoveryPermit.run(settle) : settle();+ terminalWork+ .then(async () => { wakeDisplay(settings); await recoveryPermit?.finish(); }).catch((err) => {retainOwnership();// A finalize/flush rejection is still a terminal end with no auto-resume@@ -956,45 +476,15 @@ export async function runTraining({ jobId, runId, pythonPath = null, resumeCheckwakeDisplay(settings);console.error(`❌ training [${shortId(jobId)}] finalize failed: ${err?.message}`);fail(`finalize failed: ${err?.message}`);- });+ }).finally(releaseDataAccess);});} // end wireProcLifecycle}-/**- * After a phase-aware-watchdog SIGKILL, auto-resume the run from its newest- * checkpoint (issue #1330) — bounded by STALL_MAX_AUTO_RESUMES so a config that- * wedges every segment can't resume-loop forever. Best-effort: any failure- * leaves the run in its finalize-set `failed` state for manual resume. Runs- * outside the request lifecycle (called from the close handler), so it owns its- * own error handling rather than bubbling.- *- * Returns true ONLY when a resume was actually enqueued (a fresh trainer will- * re-spawn). The close handler relies on this to decide whether to wake the- * display: a budget-exhausted or failed resume does NOT re-spawn, so the display- * must be woken just like any other terminal end — gating on raw `stallKilled`- * would leave it asleep forever on those paths.- */-async function autoResumeAfterStall(runId, jobId) {- const run = await runsDb.getRun(runId).catch(() => null);- if (!run) return false;- const autoCount = run.resume?.autoCount || 0;- if (autoCount >= STALL_MAX_AUTO_RESUMES) {- console.log(`⚠️ training [${shortId(jobId)}] stall auto-resume budget exhausted (${autoCount}/${STALL_MAX_AUTO_RESUMES}) — leaving run failed for manual resume`);- return false;- }- // resumeTrainingRun requires a failed/canceled status + a resumable checkpoint;- // finalize has already flipped the run to failed by the time this runs.- return resumeTrainingRun(runId, { auto: true }).then(- (res) => { console.log(`🔁 training [${shortId(jobId)}] auto-resumed run ${shortId(runId)} from step ${res.fromStep} (attempt ${autoCount + 1}/${STALL_MAX_AUTO_RESUMES})`); return true; },- (err) => { console.error(`❌ training [${shortId(jobId)}] stall auto-resume failed: ${err?.message} — run stays failed for manual resume`); return false; },- );-}--async function finalizeTraining({ jobId, runId, code, signal, state, stallKilled = false }) {+async function finalizeTraining({ jobId, runId, code, signal, state, stallKilled = false, canceledByOperator = false }) {const run = await runsDb.getRun(runId);const job = getJob(jobId);- const canceled = !!job?.cancelRequested;+ const canceled = !!job?.cancelRequested || canceledByOperator;// Run record vanished mid-training (direct DB edit / race — the DELETE// route blocks active runs, so this is defensive). Don't register a LoRA@@ -1053,7 +543,7 @@ async function finalizeTraining({ jobId, runId, code, signal, state, stallKilledif (canceled) {// Queue's cancelRequested flips the failed event into a clean cancel;- // record keeps the checkpoint lineage for a future resume.+ // record keeps the checkpoint lineage for inspection or explicit promotion.await runsDb.updateRun(runId, { status: 'canceled', completedAt: new Date().toISOString(), error: 'Canceled' });await flipDatasetAfterRun(run, { trained: false });trainingEvents.emit('failed', { generationId: jobId, error: 'Canceled' });@@ -1089,14 +579,6 @@ async function finalizeTraining({ jobId, runId, code, signal, state, stallKilledconsole.error(`❌ training [${shortId(jobId)}] ${failCode}: ${message}`);trainingEvents.emit('failed', { generationId: jobId, error: message, code: failCode, repo: failRepo });- // Phase-aware watchdog SIGKILL (issue #1330): the run is now `failed`, so- // resumeTrainingRun's status precondition is satisfied — auto-resume from the- // newest checkpoint. Bounded by STALL_MAX_AUTO_RESUMES; awaited so the resume- // (re-enqueue + run-record flip back to queued) completes within finalize.- // Return whether a resume was actually enqueued so the close handler knows NOT- // to wake the display in that case (the re-spawn re-sleeps it); every other- // terminal path falls through to the implicit `undefined` (falsy) → wake.- if (stallKilled) return autoResumeAfterStall(runId, jobId);}/**@@ -1140,6 +622,10 @@ export async function listSamples(runId) {* degradation — see the loss-is-misleading note in checkpoints.js).*/export async function promoteCheckpoint(runId, step) {+ return withRetiredDataAccess(() => promoteRetainedCheckpoint(runId, step));+}++async function promoteRetainedCheckpoint(runId, step) {const run = await runsDb.getRunRequired(runId);// Allow completed runs AND failed/canceled runs that saved at least one// checkpoint — promoting a partial checkpoint is a deliberate salvage (the@@ -1196,7 +682,7 @@ export async function promoteCheckpoint(runId, step) {// render failure (OOM, missing venv, server restart) must never fail the// promote — the LoRA is already deployed; the thumbnail is cosmetic.if (!target.previewUrl) {- ensureCheckpointPreview(run, step, filename).catch((err) => {+ withRetiredDataAccess(() => ensureCheckpointPreview(run, step, filename)).catch((err) => {console.error(`⚠️ training [${shortId(runId)}] preview render for step ${step} failed: ${err?.message}`);});}@@ -1327,17 +813,27 @@ export async function initLoraTraining() {for (const run of active) {const job = run.jobId ? getJob(run.jobId) : null;if (!job || ['failed', 'canceled', 'completed'].includes(job.status)) {- // The trainer is a detached child that survives a pm2 restart, so it may- // STILL be running even though its in-memory job is gone. Reap it (clean- // SIGTERM → checkpoint → SIGKILL) before marking the run failed/resumable,- // so a resume can't spawn a second trainer into the same checkpoint dir.- const reaped = await reapDetached(join(runDir(run.id), '.detached')).catch(() => ({ reaped: false }));- if (reaped.reaped) console.log(`🧹 reaped surviving trainer pid ${reaped.pid} for run ${shortId(run.id)}`);+ // Upgrade never kills an orphan trainer. Keep its artifacts pinned until+ // the existing process finishes or the operator explicitly cancels it.+ if (await hasSurvivingTrainer(run.id)) {+ const jobId = run.jobId || run.id;+ const recoveryPermit = maintenance.recoverOwned('media', jobId);+ try {+ const observer = await recoveryPermit.run(() => runTraining({ jobId, runId: run.id, reattach: true, recoveryPermit }));+ if (observer?.observing) console.log(`🔁 Observing retired trainer ${shortId(run.id)} without restarting it`);+ else if (!observer?.preserved && !await hasSurvivingTrainer(run.id)) await recoveryPermit.finish();+ } catch (err) {+ // An uncertain observer must retain its recovery permit: the detached+ // process may still be writing, even when its database/claim is down.+ console.error(`❌ Retired trainer ${shortId(run.id)} could not be observed; artifacts stay pinned: ${err?.message}`);+ }+ continue;+ }await runsDb.updateRun(run.id, {- status: 'failed', error: 'interrupted by restart', completedAt: new Date().toISOString(),+ status: job?.status === 'canceled' ? 'canceled' : 'failed', error: 'LoRA training retired; artifacts preserved', completedAt: new Date().toISOString(),}).catch(() => {});await flipDatasetAfterRun(run, { trained: false });- console.log(`🧹 training run ${shortId(run.id)} marked failed (interrupted by restart)`);+ console.log(`🧹 training run ${shortId(run.id)} retired; artifacts preserved`);}}diff --git a/server/services/loraTraining/lifecycle.maintenance.test.js b/server/services/loraTraining/lifecycle.maintenance.test.jsindex 7fbe2eccf5..c67701f98d 100644--- a/server/services/loraTraining/lifecycle.maintenance.test.js+++ b/server/services/loraTraining/lifecycle.maintenance.test.js@@ -5,7 +5,7 @@ import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../liconst fixture = vi.hoisted(() => ({admission: null, child: null, run: null, queueJob: null,- release: vi.fn(), save: vi.fn(), saveDataset: vi.fn(),+ release: vi.fn(), save: vi.fn(), saveDataset: vi.fn(), read: vi.fn(), adopt: vi.fn(), attach: vi.fn(), active: null,}));vi.mock('../../lib/fileUtils.js', async importOriginal =>makePathsProxy(await importOriginal(), { dataRoot: () => lazyTempDataRoot('training-maintenance-') }));@@ -18,15 +18,15 @@ vi.mock('../../lib/childProcess.js', async importOriginal => ({...await importOriginal(), spawn: vi.fn(() => { throw new Error('Unexpected real child launch'); }),}));vi.mock('../../lib/detachedSpawn.js', () => ({- reattachDetached: async () => fixture.child, spawnDetached: vi.fn(), reapDetached: vi.fn(), isReattachable: vi.fn(),+ reattachDetached: (...args) => fixture.attach(...args), spawnDetached: vi.fn(), reapDetached: vi.fn(), isReattachable: vi.fn(),}));vi.mock('../../lib/heavyJobClaim.js', () => ({- adoptHeavyLocalJob: async () => ({ ok: true, holder: { pid: fixture.child.pid },- handoffTo: async () => {}, release: () => fixture.release() }),+ adoptHeavyLocalJob: (...args) => fixture.adopt(...args),claimHeavyLocalJob: vi.fn(),}));vi.mock('./db.js', () => ({- getRun: async () => fixture.run,+ getRun: (...args) => fixture.read(...args),+ listActiveRuns: async () => fixture.active || [fixture.run],updateRun: (...args) => fixture.save(...args), getRunRequired: vi.fn(), listRuns: vi.fn(), deleteRun: vi.fn(),}));vi.mock('../settings.js', () => ({ getSettings: async () => ({ loraTraining: { stallWatchdog: false } }) }));@@ -36,8 +36,9 @@ vi.mock('../mediaJobQueue/index.js', () => ({}));vi.mock('./displayPower.js', () => ({ sleepDisplayForTraining: vi.fn(), wakeDisplay: vi.fn() }));-const { runTraining, cancel, trainingEvents } = await import('./index.js');+const { runTraining, initLoraTraining, cancel, trainingEvents } = await import('./index.js');const { createMaintenanceAdmission } = await import('../../lib/maintenanceAdmission.js');+const { isReattachable, spawnDetached, reapDetached } = await import('../../lib/detachedSpawn.js');const { PATHS } = await import('../../lib/fileUtils.js');let fixtureNumber = 0;beforeEach(() => {@@ -47,6 +48,12 @@ beforeEach(() => {stdout: new EventEmitter(), stderr: new EventEmitter(), kill: vi.fn() });fixture.run = { id: 'example-run', jobId: 'example-job', datasetId: 'example-dataset', params: { steps: 10 }, runtime: 'mflux' };fixture.queueJob = {};+ fixture.active = null;+ fixture.read.mockReset().mockImplementation(async () => fixture.run);+ fixture.attach.mockReset().mockImplementation(async () => fixture.child);+ fixture.adopt.mockReset().mockImplementation(async () => ({ ok: true, holder: { pid: fixture.child.pid },+ handoffTo: async () => {}, release: () => fixture.release() }));+ vi.mocked(isReattachable).mockReset().mockResolvedValue(false);fixture.release.mockReset().mockResolvedValue();fixture.save.mockReset().mockResolvedValue();fixture.saveDataset.mockReset().mockResolvedValue();@@ -168,3 +175,51 @@ describe('training physical settlement under maintenance', () => {}});});++it('observes an orphan survivor without spawning/reaping and settles explicit cancellation in retained history', async () => {+ fixture.queueJob = null;+ fixture.run.status = 'running';+ vi.mocked(isReattachable).mockResolvedValue(true);+ await initLoraTraining();+ expect(spawnDetached).not.toHaveBeenCalled();+ expect(reapDetached).not.toHaveBeenCalled();+ expect(fixture.child.listenerCount('close')).toBeGreaterThan(0);+ const { withRetiredDataRemoval } = await import('../../lib/retiredLoraTraining.js');+ await expect(withRetiredDataRemoval(async () => {})).rejects.toMatchObject({ code: 'MAINTENANCE_BUSY' });+ expect(cancel('example-job')).toBe(true);+ const originalSave = fixture.save.getMockImplementation();+ let finishRecord;+ fixture.save.mockImplementationOnce(() => new Promise(resolve => { finishRecord = resolve; }));+ fixture.child.emit('close', 1, 'SIGTERM');+ await vi.waitFor(() => expect(finishRecord).toBeTypeOf('function'));+ await expect(withRetiredDataRemoval(async () => {})).rejects.toMatchObject({ code: 'MAINTENANCE_BUSY' });+ finishRecord();+ fixture.save.mockImplementation(originalSave);+ await new Promise(resolve => setImmediate(resolve));+ await expect(withRetiredDataRemoval(async () => 'settled')).resolves.toBe('settled');+ await vi.waitFor(() => expect(fixture.save).toHaveBeenCalledWith('example-run', expect.objectContaining({ status: 'canceled' })));+});+++it.each(['database', 'attach', 'claim', 'refused claim'])('preserves a live orphan after %s failure and continues boot reconciliation', async failure => {+ fixture.queueJob = null;+ fixture.run.status = 'running';+ fixture.active = [fixture.run, { id: 'next-run', status: 'queued' }];+ vi.mocked(isReattachable).mockImplementation(async path => path.includes('example-run'));+ if (failure === 'database') fixture.read.mockRejectedValueOnce(new Error('Database unavailable'));+ if (failure === 'attach') fixture.attach.mockRejectedValueOnce(new Error('Observer unavailable'));+ if (failure === 'claim') fixture.adopt.mockRejectedValueOnce(new Error('Claim unavailable'));+ if (failure === 'refused claim') fixture.adopt.mockResolvedValueOnce({ ok: false, message: 'Accelerator occupied' });+ const { mediaJobEvents } = await import('../mediaJobQueue/index.js');+ const before = mediaJobEvents.listenerCount('canceled');+ await expect(initLoraTraining()).resolves.toBeUndefined();+ expect(fixture.save).not.toHaveBeenCalledWith('example-run', expect.anything());+ expect(fixture.save).toHaveBeenCalledWith('next-run', expect.objectContaining({ status: 'failed' }));+ expect(mediaJobEvents.listenerCount('canceled')).toBe(before + 1);+ expect(fixture.release).not.toHaveBeenCalled();+ expect(fixture.admission.status().blockers).toEqual(expect.arrayContaining([+ expect.objectContaining({ resource: 'example-job' }),+ ]));+ expect(spawnDetached).not.toHaveBeenCalled();+ expect(reapDetached).not.toHaveBeenCalled();+});diff --git a/server/services/loraTraining/memoryPrep.js b/server/services/loraTraining/memoryPrep.jsdeleted file mode 100644index b9e02b5f15..0000000000--- a/server/services/loraTraining/memoryPrep.js+++ /dev/null@@ -1,11 +0,0 @@-/** Training-specific memory policy built on the shared local-memory preflight. */--import { prepareLocalMemory } from '../localMemory.js';--export { unloadResidentModels, getAvailableMemoryGb, isLocalBackendUrl, gpuBlockersMessage } from '../localMemory.js';--// Below this floor even a small 4-bit training run risks swap-thrash. Larger-// run sizing remains training-specific and still consumes `budgetGb` below.-export const TRAINING_MIN_HEADROOM_GB = 24;--export const prepareMemoryForTraining = prepareLocalMemory;diff --git a/server/services/loraTraining/memoryPrep.test.js b/server/services/loraTraining/memoryPrep.test.jsdeleted file mode 100644index 107cbe9152..0000000000--- a/server/services/loraTraining/memoryPrep.test.js+++ /dev/null@@ -1,162 +0,0 @@-import { describe, it, expect, vi, beforeEach } from 'vitest';--// Hoisted mock state so the factory closures below can reach it.-const h = vi.hoisted(() => ({- ollamaLoaded: [],- ollamaUnload: vi.fn(),- ollamaUrl: 'http://localhost:11434',- lmLoaded: [],- lmUnload: vi.fn(),- lmUrl: 'http://localhost:1234',- platform: 'darwin',- totalBytes: 128 * 2 ** 30,- freeBytes: 8 * 2 ** 30,- vmStat: { stdout: '' },- vmStatThrows: false,-}));--vi.mock('../../services/ollamaManager.js', () => ({- getLoadedModels: vi.fn(async () => h.ollamaLoaded),- unloadModel: (...a) => h.ollamaUnload(...a),- getBaseUrl: () => h.ollamaUrl,-}));-vi.mock('../../services/lmStudioManager.js', () => ({- getLoadedModels: vi.fn(async () => h.lmLoaded),- unloadModel: (...a) => h.lmUnload(...a),- getBaseUrl: () => h.lmUrl,-}));-// Spread the real module: the preflight's own dependency graph reaches other-// `os` exports (`homedir`, via the vLLM project-dir resolver), and a-// listed-exports-only mock fails the whole file the moment it reaches one more.-vi.mock('os', async (importOriginal) => ({- ...(await importOriginal()),- platform: () => h.platform,- totalmem: () => h.totalBytes,- freemem: () => h.freeBytes,-}));-vi.mock('../../lib/childProcess.js', () => ({- execFile: (_cmd, cb) => {- if (h.vmStatThrows) return cb(new Error('vm_stat missing'));- return cb(null, h.vmStat);- },-}));--const { unloadResidentModels, getAvailableMemoryGb, prepareMemoryForTraining, isLocalBackendUrl, TRAINING_MIN_HEADROOM_GB } =- await import('./memoryPrep.js');--// vm_stat with 16 KiB pages: free + inactive + speculative + purgeable pages-// sum to a known GB total so the parser assertion is exact.-const vmStatWith = (gb) => {- const pages = (gb * 2 ** 30) / 16384;- return `Mach Virtual Memory Statistics: (page size of 16384 bytes)-Pages free: ${Math.floor(pages / 4)}.-Pages active: 100000.-Pages inactive: ${Math.floor(pages / 4)}.-Pages speculative: ${Math.floor(pages / 4)}.-Pages wired down: 50000.-Pages purgeable: ${Math.ceil(pages / 4)}.-`;-};--beforeEach(() => {- h.ollamaLoaded = [];- h.lmLoaded = [];- h.ollamaUnload = vi.fn(async () => ({ unloaded: true }));- h.lmUnload = vi.fn(async () => ({ success: true }));- h.ollamaUrl = 'http://localhost:11434';- h.lmUrl = 'http://localhost:1234';- h.platform = 'darwin';- h.totalBytes = 128 * 2 ** 30;- h.freeBytes = 8 * 2 ** 30;- h.vmStat = { stdout: vmStatWith(100) };- h.vmStatThrows = false;-});--describe('unloadResidentModels', () => {- it('unloads every resident ollama + LM Studio model and labels them', async () => {- h.ollamaLoaded = [{ name: 'llama3:70b' }, { id: 'qwen2.5' }];- h.lmLoaded = [{ id: 'mlx-community/big' }];- const freed = await unloadResidentModels();- expect(freed).toEqual(['ollama:llama3:70b', 'ollama:qwen2.5', 'lmstudio:mlx-community/big']);- expect(h.ollamaUnload).toHaveBeenCalledTimes(2);- expect(h.lmUnload).toHaveBeenCalledWith('mlx-community/big');- });-- it('swallows unload failures and omits the model from the freed list', async () => {- h.ollamaLoaded = [{ name: 'a' }, { name: 'b' }];- h.ollamaUnload = vi.fn()- .mockResolvedValueOnce({ unloaded: false, reason: 'not loaded' })- .mockRejectedValueOnce(new Error('server down'));- const freed = await unloadResidentModels();- expect(freed).toEqual([]);- });-- it('returns empty when nothing is resident', async () => {- expect(await unloadResidentModels()).toEqual([]);- });-- it('skips a remote backend — never evicts a peer machine\'s models', async () => {- h.ollamaLoaded = [{ name: 'remote-llm' }];- h.lmLoaded = [{ id: 'local-lm' }];- h.ollamaUrl = 'http://10.0.0.5:11434'; // LAN peer — unloading frees no local memory- h.lmUrl = 'http://127.0.0.1:1234'; // loopback — still eligible- const freed = await unloadResidentModels();- expect(freed).toEqual(['lmstudio:local-lm']);- expect(h.ollamaUnload).not.toHaveBeenCalled();- expect(h.lmUnload).toHaveBeenCalledWith('local-lm');- });-});--describe('isLocalBackendUrl', () => {- it('accepts loopback hosts (incl. 127.0.0.0/8 and IPv6)', () => {- for (const u of [- 'http://localhost:11434', 'http://127.0.0.1:1234', 'http://127.0.0.53:1234',- 'http://0.0.0.0:1234', 'http://[::1]:1234',- ]) expect(isLocalBackendUrl(u)).toBe(true);- });-- it('rejects LAN/remote hosts and unparseable input', () => {- for (const u of [- 'http://10.0.0.5:11434', 'http://192.168.1.20:1234', 'http://peer.local:11434',- 'http://example.com', '', null, undefined, 'not a url',- ]) expect(isLocalBackendUrl(u)).toBe(false);- });-});--describe('getAvailableMemoryGb', () => {- it('parses vm_stat (free+inactive+speculative+purgeable) on darwin', async () => {- h.vmStat = { stdout: vmStatWith(64) };- expect(await getAvailableMemoryGb()).toBeCloseTo(64, 0);- });-- it('falls back to freemem() when vm_stat fails', async () => {- h.vmStatThrows = true;- h.freeBytes = 12 * 2 ** 30;- expect(await getAvailableMemoryGb()).toBeCloseTo(12, 5);- });-- it('uses freemem() off darwin', async () => {- h.platform = 'linux';- h.freeBytes = 7 * 2 ** 30;- expect(await getAvailableMemoryGb()).toBeCloseTo(7, 5);- });-});--describe('prepareMemoryForTraining', () => {- it('clamps the budget to physical RAM and reports freed models', async () => {- h.ollamaLoaded = [{ name: 'm' }];- h.totalBytes = 96 * 2 ** 30;- h.vmStat = { stdout: vmStatWith(200) }; // bogus over-report → clamp to total- const r = await prepareMemoryForTraining();- expect(r.unloaded).toEqual(['ollama:m']);- expect(r.budgetGb).toBe(96);- expect(r.totalGb).toBe(96);- });-- it('budget tracks the post-unload available pool when below total', async () => {- h.vmStat = { stdout: vmStatWith(40) };- const r = await prepareMemoryForTraining();- expect(r.budgetGb).toBeCloseTo(40, 0);- expect(r.budgetGb).toBeLessThan(TRAINING_MIN_HEADROOM_GB + 100);- });-});diff --git a/server/services/loraTraining/mpsGuard.test.js b/server/services/loraTraining/mpsGuard.test.jsdeleted file mode 100644index e1be58c661..0000000000--- a/server/services/loraTraining/mpsGuard.test.js+++ /dev/null@@ -1,71 +0,0 @@-import { describe, it, expect, vi, beforeEach } from 'vitest';--// Guard: startTrainingRun must REFUSE the torch/diffusers fallback trainer on-// Apple Silicon at request time. PyTorch's MPS backend has no linear_backward-// for the FLUX.2 transformer layers, so a torch LoRA backward dies at the first-// optimizer step — mflux (MLX) is the Apple-Silicon runtime (issue #1227). The-// refusal must happen before any run record is created or job enqueued, so we-// never load a ~16 GB base + precompute latents only to crash. On non-darwin-// (the torch path's real CUDA/CPU target) the same request proceeds.-let mockPlatform = 'darwin';-vi.mock('os', async (importActual) => {- const actual = await importActual();- return { ...actual, platform: () => mockPlatform };-});--const createRunMock = vi.fn(async () => {});-const updateRunMock = vi.fn(async () => {});-const enqueueJobMock = vi.fn(async () => ({ jobId: 'job-1', position: 0 }));-const validateDatasetReadyMock = vi.fn(async () => ({- dataset: { character: { entryId: 'c1', universeId: 'u1', name: 'Test Subject' }, triggerWord: 'test_subject' },- manifest: { images: [] },-}));-const isFlux2VenvHealthyMock = vi.fn(async () => true);--vi.mock('./db.js', () => ({- createRun: (...a) => createRunMock(...a),- updateRun: (...a) => updateRunMock(...a),- getRun: vi.fn(), getRunRequired: vi.fn(), listRuns: vi.fn(), deleteRun: vi.fn(),-}));-vi.mock('./dataset.js', () => ({ validateDatasetReady: (...a) => validateDatasetReadyMock(...a) }));-vi.mock('../settings.js', () => ({ getSettings: vi.fn(async () => ({})) }));-vi.mock('../loraDatasets.js', () => ({ updateDataset: vi.fn(async () => {}) }));-vi.mock('../mediaJobQueue/index.js', () => ({- assertMediaQueueRoom: vi.fn(),- enqueueJob: (...a) => enqueueJobMock(...a),- getJob: vi.fn(),- mediaJobEvents: { emit: vi.fn(), on: vi.fn() },-}));-vi.mock('../../lib/pythonSetup.js', () => ({- resolveFlux2Python: () => '/fake/venv-flux2/bin/python3',- isFlux2VenvHealthy: (...a) => isFlux2VenvHealthyMock(...a),- // No mflux-train anywhere → resolves to null, so routing falls through to the- // torch (flux2) runtime the guard protects. Mirrors resolveMfluxPython(null).- resolveMfluxPython: (p) => p || null,-}));--const { startTrainingRun } = await import('./index.js');--// getSettings → {} means pythonPath is null, so isMfluxTrainAvailable(null) is-// false and routing resolves to the torch (flux2) runtime — the branch the-// guard protects. flux2-klein-4b is a real registry model.-const BASE = 'flux2-klein-4b';--beforeEach(() => { vi.clearAllMocks(); mockPlatform = 'darwin'; });--describe('startTrainingRun — Apple Silicon torch-runtime guard', () => {- it('refuses torch training on darwin before creating a run or queuing a job', async () => {- await expect(startTrainingRun({ datasetId: 'ds1', baseModelId: BASE }))- .rejects.toMatchObject({ code: 'TRAINING_MPS_UNSUPPORTED', status: 412 });- expect(createRunMock).not.toHaveBeenCalled();- expect(enqueueJobMock).not.toHaveBeenCalled();- });-- it('does NOT fire the guard on non-darwin (torch path proceeds to queue)', async () => {- mockPlatform = 'linux';- const res = await startTrainingRun({ datasetId: 'ds1', baseModelId: BASE });- expect(res).toMatchObject({ status: 'queued' });- expect(createRunMock).toHaveBeenCalled();- expect(enqueueJobMock).toHaveBeenCalled();- });-});diff --git a/server/services/loraTraining/resume.test.js b/server/services/loraTraining/resume.test.jsindex 493aabe866..8b19e9479d 100644--- a/server/services/loraTraining/resume.test.js+++ b/server/services/loraTraining/resume.test.js@@ -1,124 +1,12 @@-import { describe, it, expect, vi, beforeEach } from 'vitest';--// Mock every external dependency resumeTrainingRun reaches so we can drive the-// branch logic without a real DB, queue, or python venv. Keeping runtimes.js-// real lets the test reference TRAINING_RUNTIMES.FLUX2 against the same source.-const getRunRequiredMock = vi.fn();-const updateRunMock = vi.fn();-const resolveLatestCheckpointArtifactMock = vi.fn();-const validateDatasetReadyMock = vi.fn();-const enqueueJobMock = vi.fn();-const updateDatasetMock = vi.fn();-const isFlux2VenvHealthyMock = vi.fn();--vi.mock('./db.js', () => ({- getRunRequired: (...a) => getRunRequiredMock(...a),- updateRun: (...a) => updateRunMock(...a),- getRun: vi.fn(),- listRuns: vi.fn(),- deleteRun: vi.fn(),-}));-vi.mock('./checkpoints.js', () => ({- resolveLatestCheckpointArtifact: (...a) => resolveLatestCheckpointArtifactMock(...a),- listRunCheckpoints: vi.fn(),- listRunSamples: vi.fn(),- resolveCheckpointAdapterBuffer: vi.fn(),- selectDeployableCheckpoint: vi.fn(),-}));-vi.mock('./dataset.js', () => ({ validateDatasetReady: (...a) => validateDatasetReadyMock(...a) }));-vi.mock('../settings.js', () => ({ getSettings: vi.fn(async () => ({})) }));-vi.mock('../../lib/pythonSetup.js', () => ({- isFlux2VenvHealthy: (...a) => isFlux2VenvHealthyMock(...a),- resolveFlux2Python: vi.fn(() => '/venv/python'),- resolveMfluxPython: (p) => p || null,-}));-vi.mock('../mediaJobQueue/index.js', () => ({- assertMediaQueueRoom: vi.fn(),- enqueueJob: (...a) => enqueueJobMock(...a),- getJob: vi.fn(),- mediaJobEvents: { emit: vi.fn(), on: vi.fn() },-}));-vi.mock('../loraDatasets.js', () => ({ updateDataset: (...a) => updateDatasetMock(...a) }));--const { TRAINING_RUNTIMES } = await import('./runtimes.js');-const { resumeTrainingRun } = await import('./index.js');--const flux2Run = (overrides = {}) => ({- id: 'run-flux2',- status: 'failed',- runtime: TRAINING_RUNTIMES.FLUX2,- datasetId: 'ds-1',- character: { entryId: 'c1', universeId: 'u1' },- triggerWord: 'kessa',- baseModelId: 'm1',- params: { steps: 600, rank: 16 },- ...overrides,-});--describe('resumeTrainingRun — flux2 path', () => {- beforeEach(() => {- vi.clearAllMocks();- updateRunMock.mockResolvedValue();- updateDatasetMock.mockResolvedValue();- validateDatasetReadyMock.mockResolvedValue({ dataset: { character: { entryId: 'c1', universeId: 'u1' } } });- isFlux2VenvHealthyMock.mockResolvedValue(true);- enqueueJobMock.mockReturnValue({ jobId: 'job-1', position: 0 });- });-- it('enqueues a resume job for a failed flux2 run (no RESUME_UNSUPPORTED_RUNTIME)', async () => {- getRunRequiredMock.mockResolvedValue(flux2Run());- resolveLatestCheckpointArtifactMock.mockReturnValue({ step: 150, path: '/runs/run-flux2/checkpoints/step-000150' });-- const result = await resumeTrainingRun('run-flux2');-- expect(result).toMatchObject({ runId: 'run-flux2', jobId: 'job-1', status: 'queued', fromStep: 150 });- expect(enqueueJobMock).toHaveBeenCalledTimes(1);- const job = enqueueJobMock.mock.calls[0][0];- expect(job.kind).toBe('training');- expect(job.params.runtime).toBe(TRAINING_RUNTIMES.FLUX2);- // The checkpoint dir flows through to the trainer's --resume-from.- expect(job.params.resumeCheckpoint).toBe('/runs/run-flux2/checkpoints/step-000150');- // The run record is flipped back to queued with the resume bookkeeping.- expect(updateRunMock).toHaveBeenCalledWith('run-flux2', expect.any(Function));- });-- it('still rejects a flux2 run with no on-disk checkpoint (NO_RESUMABLE_CHECKPOINT)', async () => {- getRunRequiredMock.mockResolvedValue(flux2Run());- resolveLatestCheckpointArtifactMock.mockReturnValue(null);-- await expect(resumeTrainingRun('run-flux2')).rejects.toMatchObject({ code: 'NO_RESUMABLE_CHECKPOINT' });- expect(enqueueJobMock).not.toHaveBeenCalled();- });-- it('refuses to resume a still-running flux2 run (RUN_NOT_RESUMABLE)', async () => {- getRunRequiredMock.mockResolvedValue(flux2Run({ status: 'running' }));-- await expect(resumeTrainingRun('run-flux2')).rejects.toMatchObject({ code: 'RUN_NOT_RESUMABLE' });- expect(enqueueJobMock).not.toHaveBeenCalled();- });-- // Issue #1330: the phase-aware stall watchdog auto-resumes with { auto: true }.- // The resume bookkeeping must increment autoCount (the watchdog's own budget,- // separate from total count) and stamp lastReason so the cap is enforceable.- it('stamps auto-resume bookkeeping (autoCount + lastReason) when auto:true', async () => {- getRunRequiredMock.mockResolvedValue(flux2Run({ resume: { count: 1, autoCount: 1 } }));- resolveLatestCheckpointArtifactMock.mockReturnValue({ step: 300, path: '/runs/run-flux2/checkpoints/step-000300' });-- await resumeTrainingRun('run-flux2', { auto: true });-- const updater = updateRunMock.mock.calls.find((c) => c[0] === 'run-flux2' && typeof c[1] === 'function')[1];- const next = updater(flux2Run({ resume: { count: 1, autoCount: 1 } }));- expect(next.resume).toMatchObject({ count: 2, autoCount: 2, fromStep: 300, lastReason: 'stall-watchdog' });- });-- it('manual resume does not bump autoCount and stamps lastReason manual', async () => {- getRunRequiredMock.mockResolvedValue(flux2Run({ resume: { count: 2, autoCount: 2 } }));- resolveLatestCheckpointArtifactMock.mockReturnValue({ step: 300, path: '/runs/run-flux2/checkpoints/step-000300' });-- await resumeTrainingRun('run-flux2');-- const updater = updateRunMock.mock.calls.find((c) => c[0] === 'run-flux2' && typeof c[1] === 'function')[1];- const next = updater(flux2Run({ resume: { count: 2, autoCount: 2 } }));- expect(next.resume).toMatchObject({ count: 3, autoCount: 2, lastReason: 'manual' });- });+import { expect, it, vi } from 'vitest';+vi.mock('./db.js', () => ({ getRun: vi.fn(), listRuns: vi.fn(), getRunRequired: vi.fn(), deleteRun: vi.fn(), updateRun: vi.fn() }));+vi.mock('../mediaJobQueue/index.js', () => ({ getJob: vi.fn(), mediaJobEvents: { on: vi.fn() } }));+vi.mock('../loraDatasets.js', () => ({ updateDataset: vi.fn() }));+import { startTrainingRun, resumeTrainingRun } from './index.js';+import { updateRun, getRunRequired } from './db.js';+it('refuses fresh launch, manual resume and automatic resume without changing retained records', async () => {+ await expect(startTrainingRun({ datasetId: 'example' })).rejects.toMatchObject({ status: 410, code: 'LORA_TRAINING_RETIRED' });+ await expect(resumeTrainingRun('example')).rejects.toMatchObject({ status: 410, code: 'LORA_TRAINING_RETIRED' });+ await expect(resumeTrainingRun('example', { auto: true })).rejects.toMatchObject({ status: 410, code: 'LORA_TRAINING_RETIRED' });+ expect(updateRun).not.toHaveBeenCalled(); expect(getRunRequired).not.toHaveBeenCalled();});diff --git a/server/services/loraTraining/runtimes.js b/server/services/loraTraining/runtimes.jsindex 25e85bea0d..f186d11abb 100644--- a/server/services/loraTraining/runtimes.js+++ b/server/services/loraTraining/runtimes.js@@ -1,357 +1,2 @@-/**- * LoRA training runtime routing + argument/config builders. Pure — no I/O.- *- * Both runtimes train FLUX.2 Klein character LoRAs; they are two ENGINES- * for the same output (verified against mflux 0.17.5, which removed- * FLUX.1 training entirely):- *- * - 'mflux' — MLX-native training via mflux's `mflux-train` CLI,- * wrapped by scripts/train_mflux_lora.py. Preferred on Apple Silicon- * when the user's mflux install ships the trainer. Trains the- * non-distilled `flux2-klein-base-{4b,9b}` models (mflux's documented- * recommendation) and exports adapters with diffusers-style key- * naming (`transformer.transformer_blocks.N...lora_A.weight`), so- * the result loads in PortOS's diffusers-based flux2 renderer.- * - 'flux2' — torch/diffusers+peft training via the vendored- * scripts/train_flux2_lora.py in ~/.portos/venv-flux2. The fallback- * when mflux's trainer is unavailable (non-Darwin installs, older- * mflux). Trains against the bf16 base repos.- *- * Either way the trained LoRA is gated to its size variant via the- * `flux2-4b` / `flux2-9b` compat key (hidden dims differ between sizes).- */--import { ServerError } from '../../lib/errorHandler.js';-import { isFlux2, flux2VariantFromModel, FLUX2_BF16_BASE_REPOS } from '../../lib/runners.js';-+// Historical runtime identifiers used to decode retained checkpoints.export const TRAINING_RUNTIMES = Object.freeze({ MFLUX: 'mflux', FLUX2: 'flux2' });--// Defaults validated 2026-06-15 on a real-face character LoRA (44-image dataset,-// FLUX.2 Klein 9B 4-bit): identity bound cleanly and fast, stable through the run.-// History worth knowing before lowering these again: an earlier run "diverged"-// (sample collapsed to black) and the defaults were cut to 600 steps / lr 5e-5 /-// rank 16 — but the real cause was captions over-describing invariant identity-// (it bound to the words, not the trigger), NOT the learning rate. With good-// captions, lr 1e-4 + rank 32 + ~1200 steps converges faster and sharper without-// collapse. 768 captures more facial detail than 512; checkpoint/sample cadence-// gives the picker several healthy steps to choose from. The base model + quant-// tier are chosen elsewhere (deriveMfluxMemoryConfig + the base-model picker,-// which prefers 9B); these are the per-run hyperparameters the form seeds from.-export const TRAINING_DEFAULTS = Object.freeze({- steps: 1200,- rank: 32,- learningRate: 0.0001,- resolution: 768,- checkpointEvery: 300,- sampleEvery: 300,-});--// bf16 training bases per FLUX.2 size variant (torch runtime). Inference may-// run quantized repos; training resolves to these regardless of which flux2-*-// model id the user picked. Same repos the image runner renders a LoRA against-// (PEFT can't load onto quantized weights) — one source of truth in runners.js.-export const FLUX2_TRAIN_REPOS = FLUX2_BF16_BASE_REPOS;--// mflux model ids per size variant. mflux's flux2 README recommends the-// non-distilled base models for training; the resulting adapter applies to-// the distilled inference models of the same size (identical module names-// and hidden dims).-export const MFLUX_TRAIN_MODELS = Object.freeze({- '4b': 'flux2-klein-base-4b',- '9b': 'flux2-klein-base-9b',-});--// Crash-resilience floor: guarantee at least this many checkpoints across a-// run regardless of the user's checkpointEvery. macOS GPU watchdog-timeout-// kernel panics during sustained mflux training (see docs/TROUBLESHOOTING.md-// "GPU watchdog kernel panic") hard-reboot the machine mid-run; without a-// floor, checkpointEvery=0 ("only final save") or a very large interval means-// such a crash discards the entire run. Capping the interval at-// totalSteps/MIN_CHECKPOINTS bounds the worst-case loss to ~1/N of the run.-export const MFLUX_MIN_CHECKPOINTS = 4;--// Noise-schedule shape for mflux base-model training — mirrors the official-// flux2 README example (steps 40, guidance 1.0, train the high-noise window-// [25, 40)). These are INFERENCE-schedule steps, distinct from the training-// duration (num_epochs × images).-const MFLUX_SCHED = Object.freeze({ steps: 40, guidance: 1.0, timestepLow: 25, timestepHigh: 40 });--/**- * Memory-derived training knobs — mirrors the FFLF pixel-budget pattern- * (videoGen/local.js): pure, with total RAM injected by the caller.- * Training a bf16 base + in-RAM latent cache OOM-killed a 48 GB machine- * during verification, so every tier trains QLoRA-style against an- * on-the-fly-quantized base. The encoded dataset cache ALWAYS spills to- * disk (`low_ram`): keeping it in RAM bought nothing but pushed a 128 GB box- * to 122 GB used + 21 GB swap (the swap-thrash that coincided with GPU- * watchdog reboots — see docs/research/2026-06-13-mflux-training-watchdog-panic.md);- * disk-backing it costs only I/O, no training quality. Quantizing the FROZEN- * base is the standard QLoRA recipe — the LoRA weights stay full precision.- *- * Default tier never auto-selects unquantized bf16 (issue #1321): even a- * ≥96 GB box gets 8-bit QLoRA, not bf16. The bf16 9B path was pathologically- * slow (couldn't complete a step in 14 min on an M5 Max 128 GB) AND the config- * that GPU-watchdog-panicked the box three times- * (docs/research/2026-06-13-mflux-training-watchdog-panic.md) — so it's opt-in- * only, never the silent default. 8-bit QLoRA is the standard character-LoRA- * recipe with minimal fidelity loss; a run that genuinely wants a heavier base- * asks for it explicitly via `baseQuant: 16` (→ `quantize: null`).- *- * Callers feed the *available* memory budget (post-unload, see memoryPrep.js),- * not raw RAM, so the tier reflects real headroom on a shared box.- *- * Override (`LORA_TRAIN_MAX_QUANT_BITS`): a hard per-install ceiling on the- * frozen-base bit-width. Setting it to `4` forces every tier (and every per-run- * opt-in) down to 4-bit; setting it to `8` clamps an explicit `baseQuant: 16`- * bf16 opt-in back to 8-bit (the 8-bit default is already at/under the cap, so- * it's a no-op there). Unset → the memory-derived tier plus whatever a run opts- * into. Accepts 8 or 4; anything else is ignored.- *- * Per-run overrides (`overrides = { quantize?, low_ram? }`, issue #1321): a- * single run can opt into a heavier or lighter frozen base than its memory tier- * without a code change (exposed as the `baseQuant`/`lowRam` request params).- * `quantize` present — including `null` (bf16) — replaces the derived tier;- * `low_ram` boolean replaces the derived spill setting; absent keys keep the- * memory-derived value. The `LORA_TRAIN_MAX_QUANT_BITS` cap still applies AFTER- * the override, so a per-run opt-in can never exceed a box's hard safety- * ceiling (the cap only ever makes the base lighter, never heavier).- */-export function deriveMfluxMemoryConfig(totalMemGb, overrides = {}) {- const gb = Number.isFinite(totalMemGb) ? totalMemGb : 0; // unknown → most conservative- // bf16 is never the auto default (issue #1321) — the top tier is 8-bit QLoRA,- // not unquantized bf16, even at ≥96 GB (bf16 9B was pathologically slow and- // GPU-watchdog-panicked the box). A run opts into bf16 explicitly via- // `baseQuant: 16`. Tighter budgets step down to 4-bit.- const result = gb >= 64- ? { quantize: 8, low_ram: true }- : { quantize: 4, low_ram: true };- // Request override wins over the memory-derived tier. hasOwnProperty (not a- // truthiness check) so an explicit `quantize: null` (bf16) is honored as a- // deliberate clear, not mistaken for "absent".- if (Object.prototype.hasOwnProperty.call(overrides, 'quantize')) {- result.quantize = overrides.quantize;- }- if (typeof overrides.low_ram === 'boolean') {- result.low_ram = overrides.low_ram;- }- const capRaw = Number.parseInt(process.env.LORA_TRAIN_MAX_QUANT_BITS ?? '', 10);- if (capRaw === 8 || capRaw === 4) {- // null (bf16) is "16+ bits" → always capped; an existing smaller quant- // (more aggressive) is kept. Lower bits = smaller = stays. Applied last so- // it clamps the post-override value, not just the memory tier.- const curBits = result.quantize == null ? 16 : result.quantize;- if (curBits > capRaw) result.quantize = capRaw;- }- return result;-}--/**- * Resolve the runtime + repos for a base model id. `models` is the- * getImageModels() registry; `mlxAvailable` says whether the user's mflux- * install ships `mflux-train` (probed by the caller — this stays pure).- */-export function resolveTrainingRuntime(baseModelId, models, { mlxAvailable = false } = {}) {- const entry = (models || []).find((m) => m.id === baseModelId);- if (!entry) {- throw new ServerError(`Unknown image model: ${baseModelId}`, {- status: 400, code: 'TRAINING_UNSUPPORTED_MODEL',- });- }- if (!isFlux2(entry)) {- // mflux ≥0.17 removed FLUX.1 training ("Flux1 training is no longer- // supported"), so dev/schnell and the diffusers families are all out.- throw new ServerError(- `LoRA training supports FLUX.2 Klein models only — not ${baseModelId}`,- { status: 400, code: 'TRAINING_UNSUPPORTED_MODEL' },- );- }- const variant = flux2VariantFromModel(entry);- if (!variant || !FLUX2_TRAIN_REPOS[variant]) {- throw new ServerError(- `Cannot determine FLUX.2 size variant for ${baseModelId} — training needs a klein-4B/9B model`,- { status: 400, code: 'TRAINING_UNSUPPORTED_MODEL' },- );- }- return {- runtime: mlxAvailable ? TRAINING_RUNTIMES.MFLUX : TRAINING_RUNTIMES.FLUX2,- entry,- variant,- trainRepo: FLUX2_TRAIN_REPOS[variant],- mfluxModel: MFLUX_TRAIN_MODELS[variant],- };-}--const mergedParams = (params = {}) => ({ ...TRAINING_DEFAULTS, ...params });--// The full attention + feed-forward target set from mflux's flux2 training-// README, parameterized by rank and the variant's block counts. Klein 4B:-// 5 double + 20 single blocks; 9B: 8 double + 24 single (counts from the-// model configs; an over-long range would throw at injection, so these are-// pinned per variant).-const FLUX2_BLOCK_COUNTS = Object.freeze({- '4b': { double: 5, single: 20 },- '9b': { double: 8, single: 24 },-});--export function buildMfluxLoraTargets(variant, rank) {- const counts = FLUX2_BLOCK_COUNTS[variant] || FLUX2_BLOCK_COUNTS['4b'];- const doubleRange = { start: 0, end: counts.double };- const singleRange = { start: 0, end: counts.single };- const doublePaths = [- 'attn.to_q', 'attn.to_k', 'attn.to_v', 'attn.to_out',- 'attn.add_q_proj', 'attn.add_k_proj', 'attn.add_v_proj', 'attn.to_add_out',- 'ff.linear_in', 'ff.linear_out', 'ff_context.linear_in', 'ff_context.linear_out',- ];- const singlePaths = ['attn.to_qkv_mlp_proj', 'attn.to_out'];- return [- ...doublePaths.map((p) => ({ module_path: `transformer_blocks.{block}.${p}`, blocks: doubleRange, rank })),- ...singlePaths.map((p) => ({ module_path: `single_transformer_blocks.{block}.${p}`, blocks: singleRange, rank })),- ];-}--/**- * Build the mflux-train config JSON (mflux ≥0.17 schema — verified against- * TrainingSpec.from_conf). Captions are NOT in the config: mflux- * auto-discovers `NNNN.png` + `NNNN.txt` pairs under `data`, and preview- * prompts come from `data/preview*.txt` — the run-staging step lays those- * files out. "steps" in OUR params means total training iterations; mflux- * counts epochs over the example set, so we translate.- */-export function buildMfluxTrainConfig({- params = {},- variant,- mfluxModel,- dataDir,- imageCount,- outputDir,- totalMemGb = null,-}) {- const p = mergedParams(params);- if (!Number.isInteger(imageCount) || imageCount < 1) {- throw new ServerError('mflux train config needs at least one example image', {- status: 400, code: 'VALIDATION_ERROR',- });- }- const totalSteps = p.steps;- const epochs = Math.max(1, Math.round(totalSteps / imageCount));- // save_frequency must be > 0; 0/absent checkpointEvery → only the final- // save (frequency = total). The crash-resilience floor then caps the- // interval so a hard reboot mid-run loses at most ~1/MIN_CHECKPOINTS of it.- const requestedSaveFrequency = p.checkpointEvery > 0 ? p.checkpointEvery : totalSteps;- const maxSaveInterval = Math.max(1, Math.ceil(totalSteps / MFLUX_MIN_CHECKPOINTS));- const saveFrequency = Math.min(requestedSaveFrequency, maxSaveInterval);- // Per-run base-tier overrides (issue #1321): `baseQuant` lets a run pick the- // frozen-base quant — 16 = unquantized bf16 (mflux `quantize: null`), 8/4 =- // QLoRA bit-width — and `lowRam` toggles the on-disk latent-cache spill,- // independent of the memory-derived default. mflux-only knobs (the torch- // runtime always trains the bf16 base); absent → memory-derived tier.- const memoryOverride = {};- if (p.baseQuant != null) memoryOverride.quantize = p.baseQuant >= 16 ? null : p.baseQuant;- if (typeof p.lowRam === 'boolean') memoryOverride.low_ram = p.lowRam;- const memory = deriveMfluxMemoryConfig(totalMemGb, memoryOverride);- return {- model: mfluxModel,- data: dataDir,- seed: Number.isInteger(p.seed) ? p.seed : 42,- steps: MFLUX_SCHED.steps,- guidance: MFLUX_SCHED.guidance,- quantize: memory.quantize,- max_resolution: p.resolution,- low_ram: memory.low_ram,- training_loop: {- num_epochs: epochs,- batch_size: 1,- timestep_low: MFLUX_SCHED.timestepLow,- timestep_high: MFLUX_SCHED.timestepHigh,- },- optimizer: { name: 'AdamW', learning_rate: p.learningRate },- checkpoint: { save_frequency: saveFrequency, output_path: outputDir },- // monitoring omitted entirely when samples are off — mflux treats the- // whole block as optional and skips previews/plots without it.- ...(p.sampleEvery > 0 ? {- monitoring: {- preview_width: p.resolution,- preview_height: p.resolution,- plot_frequency: p.sampleEvery,- generate_image_frequency: p.sampleEvery,- },- } : {}),- lora_layers: { targets: buildMfluxLoraTargets(variant, p.rank) },- };-}--// Default GPU cooldown (seconds) between training segments — long enough for-// the GPU to clock down and the system to settle after a full Metal-context-// teardown, short relative to a segment's compute. See the segmentation block-// in scripts/train_mflux_lora.py and the watchdog-panic incident record.-export const MFLUX_DEFAULT_COOLDOWN_SEC = 90;--/**- * Argv for the mflux wrapper script.- *- * `segmentSteps > 0` enables segmented training (watchdog-panic mitigation):- * the wrapper trains one checkpoint interval, tears down the GPU child, cools- * down `cooldownSec`, then resumes — so sustained GPU pressure never spans the- * whole run. Pass `segmentSteps = the effective save_frequency` so each segment- * ends exactly on a checkpoint. `segmentSteps = 0` keeps the single-process run.- */-export function buildMfluxTrainArgs({- scriptPath,- configPath,- runDir,- totalSteps,- resumeCheckpoint = null,- segmentSteps = 0,- cooldownSec = MFLUX_DEFAULT_COOLDOWN_SEC,-}) {- const args = [- scriptPath,- '--config', configPath,- '--output-dir', runDir,- '--total-steps', String(totalSteps),- ];- if (segmentSteps > 0) {- args.push('--segment-steps', String(segmentSteps), '--cooldown-sec', String(Math.max(0, cooldownSec)));- }- if (resumeCheckpoint) args.push('--resume-checkpoint', resumeCheckpoint);- return args;-}--/** Argv for the vendored torch FLUX.2 trainer. */-export function buildFlux2TrainArgs({- scriptPath,- trainRepo,- manifestPath,- runDir,- triggerWord,- params = {},- samplePrompt = null,- resumeFrom = null,-}) {- if (!trainRepo || !/^black-forest-labs\//.test(trainRepo)) {- // Belt-and-suspenders: training against a quantized repo silently- // produces garbage gradients — refuse anything but the bf16 bases.- throw new ServerError(`FLUX.2 training requires a bf16 base repo (got: ${trainRepo})`, {- status: 400, code: 'TRAINING_UNSUPPORTED_MODEL',- });- }- const p = mergedParams(params);- const args = [- scriptPath,- '--model-repo', trainRepo,- '--manifest', manifestPath,- '--output-dir', runDir,- '--trigger-word', triggerWord,- '--steps', String(p.steps),- '--rank', String(p.rank),- '--lr', String(p.learningRate),- '--resolution', String(p.resolution),- '--checkpoint-every', String(p.checkpointEvery),- '--sample-every', String(p.sampleEvery),- '--sample-prompt', samplePrompt || `${triggerWord} portrait, neutral background`,- '--seed', String(Number.isInteger(p.seed) ? p.seed : 42),- '--device', 'auto',- ];- if (resumeFrom) args.push('--resume-from', resumeFrom);- return args;-}diff --git a/server/services/loraTraining/runtimes.test.js b/server/services/loraTraining/runtimes.test.jsdeleted file mode 100644index 0086cbd0f2..0000000000--- a/server/services/loraTraining/runtimes.test.js+++ /dev/null@@ -1,301 +0,0 @@-import { describe, it, expect } from 'vitest';-import {- TRAINING_DEFAULTS,- buildFlux2TrainArgs,- buildMfluxLoraTargets,- buildMfluxTrainArgs,- buildMfluxTrainConfig,- deriveMfluxMemoryConfig,- resolveTrainingRuntime,- MFLUX_DEFAULT_COOLDOWN_SEC,-} from './runtimes.js';--const MODELS = [- { id: 'dev', name: 'Flux 1 Dev' },- { id: 'schnell', name: 'Flux 1 Schnell' },- { id: 'flux2-klein-4b', runner: 'flux2', repo: 'Disty0/FLUX.2-klein-4B-SDNQ-4bit-dynamic' },- { id: 'flux2-klein-9b-bf16', runner: 'flux2', repo: 'black-forest-labs/FLUX.2-klein-9B' },- { id: 'z-image-turbo', runner: 'z-image', repo: 'Tongyi/Z-Image-Turbo' },-];--describe('resolveTrainingRuntime', () => {- it('routes flux2 models to mflux when its trainer is available', () => {- const out = resolveTrainingRuntime('flux2-klein-4b', MODELS, { mlxAvailable: true });- expect(out.runtime).toBe('mflux');- expect(out.mfluxModel).toBe('flux2-klein-base-4b');- expect(out.trainRepo).toBe('black-forest-labs/FLUX.2-klein-4B');- expect(out.variant).toBe('4b');- });-- it('falls back to the torch runtime without mflux', () => {- const out = resolveTrainingRuntime('flux2-klein-9b-bf16', MODELS, { mlxAvailable: false });- expect(out.runtime).toBe('flux2');- expect(out.mfluxModel).toBe('flux2-klein-base-9b');- expect(out.trainRepo).toBe('black-forest-labs/FLUX.2-klein-9B');- });-- it('rejects FLUX.1 (training removed upstream), diffusers families, and unknown ids', () => {- expect(() => resolveTrainingRuntime('dev', MODELS)).toThrow(/FLUX\.2 Klein models only/);- expect(() => resolveTrainingRuntime('schnell', MODELS)).toThrow(/FLUX\.2 Klein models only/);- expect(() => resolveTrainingRuntime('z-image-turbo', MODELS)).toThrow(/FLUX\.2 Klein models only/);- expect(() => resolveTrainingRuntime('ghost', MODELS)).toThrow(/Unknown image model/);- });-});--describe('buildMfluxLoraTargets', () => {- it('pins block ranges per size variant (4b: 5+20, 9b: 8+24)', () => {- const t4 = buildMfluxLoraTargets('4b', 16);- const double4 = t4.find((t) => t.module_path === 'transformer_blocks.{block}.attn.to_q');- const single4 = t4.find((t) => t.module_path === 'single_transformer_blocks.{block}.attn.to_qkv_mlp_proj');- expect(double4.blocks).toEqual({ start: 0, end: 5 });- expect(single4.blocks).toEqual({ start: 0, end: 20 });- const t9 = buildMfluxLoraTargets('9b', 8);- expect(t9.find((t) => t.module_path.startsWith('transformer_blocks')).blocks.end).toBe(8);- expect(t9.find((t) => t.module_path.startsWith('single_transformer_blocks')).blocks.end).toBe(24);- expect(t9.every((t) => t.rank === 8)).toBe(true);- });-});--describe('buildMfluxTrainConfig', () => {- const base = {- variant: '4b',- mfluxModel: 'flux2-klein-base-4b',- dataDir: '/data/training-runs/r1/data',- imageCount: 2,- outputDir: '/data/training-runs/r1/mflux',- };-- it('matches the mflux ≥0.17 schema and translates steps → epochs', () => {- const config = buildMfluxTrainConfig({ ...base, params: { steps: 100 } });- expect(config.model).toBe('flux2-klein-base-4b');- expect(config.data).toBe(base.dataDir);- expect(config.training_loop).toEqual({- num_epochs: 50, batch_size: 1, timestep_low: 25, timestep_high: 40,- });- expect(config.steps).toBe(40); // noise-schedule steps, not training duration- expect(config.guidance).toBe(1.0);- expect(config.checkpoint.output_path).toBe(base.outputDir);- expect(config.lora_layers.targets.length).toBeGreaterThan(10);- // mflux rejects unknown keys like the legacy `examples`/`save` shapes.- expect(config.examples).toBeUndefined();- expect(config.save).toBeUndefined();- });-- it('applies params and disables monitoring when samples are off', () => {- const config = buildMfluxTrainConfig({- ...base,- params: { steps: 200, rank: 8, learningRate: 0.0002, resolution: 768, checkpointEvery: 0, sampleEvery: 0 },- });- expect(config.optimizer.learning_rate).toBe(0.0002);- expect(config.max_resolution).toBe(768);- // checkpointEvery:0 would mean "only the final save", but the crash-- // resilience floor caps the interval at totalSteps/MIN_CHECKPOINTS (200/4).- expect(config.checkpoint.save_frequency).toBe(50);- expect(config.monitoring).toBeUndefined();- expect(config.lora_layers.targets[0].rank).toBe(8);- });-- it('caps save_frequency at totalSteps/MIN_CHECKPOINTS for crash resilience', () => {- // A huge checkpointEvery (or 0 → "only final save") must still checkpoint- // periodically so a mid-run hard reboot (GPU watchdog panic) keeps progress.- const onlyFinal = buildMfluxTrainConfig({ ...base, params: { steps: 600, checkpointEvery: 0 } });- expect(onlyFinal.checkpoint.save_frequency).toBe(150); // ceil(600/4)- const tooLarge = buildMfluxTrainConfig({ ...base, params: { steps: 600, checkpointEvery: 9999 } });- expect(tooLarge.checkpoint.save_frequency).toBe(150);- // A reasonable user interval below the cap is left untouched.- const reasonable = buildMfluxTrainConfig({ ...base, params: { steps: 600, checkpointEvery: 100 } });- expect(reasonable.checkpoint.save_frequency).toBe(100);- });-- it('enables monitoring with sample dimensions when sampleEvery > 0', () => {- const config = buildMfluxTrainConfig({ ...base, params: { sampleEvery: 50, resolution: 512 } });- expect(config.monitoring).toEqual({- preview_width: 512, preview_height: 512, plot_frequency: 50, generate_image_frequency: 50,- });- });-- it('throws on an empty example set', () => {- expect(() => buildMfluxTrainConfig({ ...base, imageCount: 0 })).toThrow(/at least one/);- });-- it('derives quantize from the memory budget; latent cache always spills to disk', () => {- // low_ram is true at every tier — an in-RAM latent cache bought nothing- // but risked swap-thrash (the 128 GB / 21 GB-swap incident). bf16 is never- // the auto default (issue #1321): even a 128 GB box gets 8-bit QLoRA, not- // unquantized bf16. quantize still steps down to 4-bit on tighter budgets.- expect(deriveMfluxMemoryConfig(128)).toEqual({ quantize: 8, low_ram: true });- expect(deriveMfluxMemoryConfig(64)).toEqual({ quantize: 8, low_ram: true });- expect(deriveMfluxMemoryConfig(48)).toEqual({ quantize: 4, low_ram: true });- expect(deriveMfluxMemoryConfig(null)).toEqual({ quantize: 4, low_ram: true });- const config = buildMfluxTrainConfig({ ...base, totalMemGb: 48 });- expect(config.quantize).toBe(4);- expect(config.low_ram).toBe(true);- });-- it('LORA_TRAIN_MAX_QUANT_BITS caps the top tier (M5 bf16-panic mitigation)', () => {- const prev = process.env.LORA_TRAIN_MAX_QUANT_BITS;- try {- // cap=8: already at the 8-bit default (no-op); smaller (4) is kept.- process.env.LORA_TRAIN_MAX_QUANT_BITS = '8';- expect(deriveMfluxMemoryConfig(128)).toEqual({ quantize: 8, low_ram: true });- expect(deriveMfluxMemoryConfig(64)).toEqual({ quantize: 8, low_ram: true });- expect(deriveMfluxMemoryConfig(48)).toEqual({ quantize: 4, low_ram: true });- // cap=4: everything (default 8-bit and the 4-bit tier) forced to 4-bit.- process.env.LORA_TRAIN_MAX_QUANT_BITS = '4';- expect(deriveMfluxMemoryConfig(128)).toEqual({ quantize: 4, low_ram: true });- expect(deriveMfluxMemoryConfig(64)).toEqual({ quantize: 4, low_ram: true });- // garbage / unsupported values are ignored → memory-derived default (8-bit).- process.env.LORA_TRAIN_MAX_QUANT_BITS = '16';- expect(deriveMfluxMemoryConfig(128)).toEqual({ quantize: 8, low_ram: true });- process.env.LORA_TRAIN_MAX_QUANT_BITS = 'banana';- expect(deriveMfluxMemoryConfig(128)).toEqual({ quantize: 8, low_ram: true });- } finally {- if (prev === undefined) delete process.env.LORA_TRAIN_MAX_QUANT_BITS;- else process.env.LORA_TRAIN_MAX_QUANT_BITS = prev;- }- });-- it('per-run overrides replace the memory-derived tier (issue #1321)', () => {- // quantize override wins over the budget tier — heavier (bf16) on a 48 GB- // box, or lighter than the box would otherwise pick.- expect(deriveMfluxMemoryConfig(48, { quantize: null })).toEqual({ quantize: null, low_ram: true });- expect(deriveMfluxMemoryConfig(128, { quantize: 4 })).toEqual({ quantize: 4, low_ram: true });- // low_ram override flips the spill independently of quantize (128 GB → 8-bit default).- expect(deriveMfluxMemoryConfig(128, { low_ram: false })).toEqual({ quantize: 8, low_ram: false });- // Absent keys keep the memory-derived value; an empty override is a no-op.- expect(deriveMfluxMemoryConfig(64, {})).toEqual({ quantize: 8, low_ram: true });- // An explicit quantize: null (bf16) is honored as a deliberate clear, not- // mistaken for "absent" — distinguishes intentional-empty from absent.- expect(deriveMfluxMemoryConfig(48, { quantize: null }).quantize).toBeNull();- });-- it('LORA_TRAIN_MAX_QUANT_BITS clamps a per-run override (cap is the ceiling)', () => {- const prev = process.env.LORA_TRAIN_MAX_QUANT_BITS;- try {- // A run asks for bf16 (heaviest) but the install caps at 8-bit — the cap- // wins, so a per-run opt-in can never breach the box's panic guard.- process.env.LORA_TRAIN_MAX_QUANT_BITS = '8';- expect(deriveMfluxMemoryConfig(48, { quantize: null })).toEqual({ quantize: 8, low_ram: true });- // A run can still go LIGHTER than the cap.- expect(deriveMfluxMemoryConfig(128, { quantize: 4 })).toEqual({ quantize: 4, low_ram: true });- } finally {- if (prev === undefined) delete process.env.LORA_TRAIN_MAX_QUANT_BITS;- else process.env.LORA_TRAIN_MAX_QUANT_BITS = prev;- }- });-- it('maps baseQuant/lowRam request params into the mflux config (issue #1321)', () => {- // baseQuant 16 → unquantized bf16 (mflux quantize: null), overriding the- // null-budget tier (which would otherwise be 4-bit).- const bf16 = buildMfluxTrainConfig({ ...base, params: { baseQuant: 16, lowRam: false } });- expect(bf16.quantize).toBeNull();- expect(bf16.low_ram).toBe(false);- // baseQuant 8/4 map straight through as the QLoRA bit-width.- expect(buildMfluxTrainConfig({ ...base, params: { baseQuant: 8 } }).quantize).toBe(8);- expect(buildMfluxTrainConfig({ ...base, totalMemGb: 128, params: { baseQuant: 4 } }).quantize).toBe(4);- // lowRam alone flips the spill at the config-builder level, leaving the- // memory-derived quant (128 GB → 8-bit default) untouched.- expect(buildMfluxTrainConfig({ ...base, totalMemGb: 128, params: { lowRam: false } }).low_ram).toBe(false);- // No override → memory-derived tier is untouched (128 GB → 8-bit, never bf16; issue #1321).- expect(buildMfluxTrainConfig({ ...base, totalMemGb: 128 }).quantize).toBe(8);- });-- it('treats null baseQuant/lowRam ("Auto") as no override (issue #1407)', () => {- // The form sends an explicit `null` for "Auto" (a deliberate clear that- // beats a saved default), which must behave identically to absent: the- // memory-derived tier wins. A null baseQuant must NOT be coerced to 0.- const cfg = buildMfluxTrainConfig({ ...base, totalMemGb: 48, params: { baseQuant: null, lowRam: null } });- expect(cfg.quantize).toBe(4); // 48 GB tier, not touched by the null clear- expect(cfg.low_ram).toBe(true);- });-});--describe('buildMfluxTrainArgs / buildFlux2TrainArgs', () => {- it('builds the wrapper argv', () => {- const args = buildMfluxTrainArgs({- scriptPath: '/x/train_mflux_lora.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 500,- });- expect(args).toEqual([- '/x/train_mflux_lora.py', '--config', '/r/cfg.json',- '--output-dir', '/r', '--total-steps', '500',- ]);- });-- it('builds the flux2 argv with merged params', () => {- const args = buildFlux2TrainArgs({- scriptPath: '/x/train_flux2_lora.py',- trainRepo: 'black-forest-labs/FLUX.2-klein-4B',- manifestPath: '/r/manifest.json',- runDir: '/r',- triggerWord: 'kessa',- params: { steps: 20, rank: 4 },- });- expect(args).toContain('--model-repo');- expect(args[args.indexOf('--steps') + 1]).toBe('20');- expect(args[args.indexOf('--rank') + 1]).toBe('4');- expect(args[args.indexOf('--lr') + 1]).toBe(String(TRAINING_DEFAULTS.learningRate));- expect(args[args.indexOf('--sample-prompt') + 1]).toContain('kessa');- });-- it('appends the resume flag for each runtime when a checkpoint is given', () => {- const mflux = buildMfluxTrainArgs({- scriptPath: '/x/train_mflux_lora.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 500,- resumeCheckpoint: '/r/mflux/checkpoints/0000150_checkpoint.zip',- });- expect(mflux[mflux.indexOf('--resume-checkpoint') + 1]).toBe('/r/mflux/checkpoints/0000150_checkpoint.zip');-- const flux2 = buildFlux2TrainArgs({- scriptPath: '/x/train_flux2_lora.py',- trainRepo: 'black-forest-labs/FLUX.2-klein-4B',- manifestPath: '/r/manifest.json', runDir: '/r', triggerWord: 'kessa',- resumeFrom: '/r/checkpoints/step-000150',- });- expect(flux2[flux2.indexOf('--resume-from') + 1]).toBe('/r/checkpoints/step-000150');- });-- it('omits the resume flag on a fresh run', () => {- const args = buildMfluxTrainArgs({- scriptPath: '/x.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 500,- });- expect(args).not.toContain('--resume-checkpoint');- });-- it('omits segment flags when segmentSteps is 0 (single-process run)', () => {- const args = buildMfluxTrainArgs({- scriptPath: '/x.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 500, segmentSteps: 0,- });- expect(args).not.toContain('--segment-steps');- expect(args).not.toContain('--cooldown-sec');- });-- it('emits segment + cooldown flags when segmentation is enabled', () => {- const args = buildMfluxTrainArgs({- scriptPath: '/x.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 600,- segmentSteps: 150, cooldownSec: 120,- });- expect(args[args.indexOf('--segment-steps') + 1]).toBe('150');- expect(args[args.indexOf('--cooldown-sec') + 1]).toBe('120');- });-- it('defaults the cooldown when only segmentSteps is given', () => {- const args = buildMfluxTrainArgs({- scriptPath: '/x.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 600, segmentSteps: 150,- });- expect(args[args.indexOf('--cooldown-sec') + 1]).toBe(String(MFLUX_DEFAULT_COOLDOWN_SEC));- });-- it('clamps a negative cooldown to 0', () => {- const args = buildMfluxTrainArgs({- scriptPath: '/x.py', configPath: '/r/cfg.json', runDir: '/r', totalSteps: 600,- segmentSteps: 150, cooldownSec: -5,- });- expect(args[args.indexOf('--cooldown-sec') + 1]).toBe('0');- });-- it('refuses non-bf16 train repos', () => {- expect(() => buildFlux2TrainArgs({- scriptPath: '/x.py', trainRepo: 'Disty0/FLUX.2-klein-4B-SDNQ-4bit-dynamic',- manifestPath: '/m.json', runDir: '/r', triggerWord: 'kessa',- })).toThrow(/bf16/);- });-});diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.jsindex 1641a688fd..e3d9fd20f4 100644--- a/server/services/mediaJobQueue/index.js+++ b/server/services/mediaJobQueue/index.js@@ -1,3 +1,4 @@+import { refuseRetiredTraining, TRAINING_RETIRED_MESSAGE, assertRetiredDataAvailable } from '../../lib/retiredLoraTraining.js';import { maintenance } from '../../lib/maintenanceAdmission.js';/*** Media Job Queue — lane-aware FIFO for media generation jobs.@@ -924,6 +925,13 @@ export async function initMediaJobQueue({ deferCompletionRecovery = false } = {}}));continue;}+ // Never dispatch a queued trainer after upgrade. A verified survivor below+ // is observed to completion without launching a replacement process.+ if (j.kind === 'training' && j.status === 'queued' && !j.params?.reattach) {+ archive.push({ ...j, status: 'canceled', error: TRAINING_RETIRED_MESSAGE, completedAt: new Date().toISOString() });+ restartedFailedIds.push(j.id);+ continue;+ }if (j.status === 'running' && maintenance.held()) {maintenance.recoverOwned('media', j.id);if (j.kind === 'training' && j.params?.runId && await jobHasSurvivingTrainer(j.params.runId)) {@@ -2040,6 +2048,7 @@ async function assertMusicVideoJobPolicy(kind, params) {}export async function enqueueJob({ kind, params, owner = null }) {+ if (kind === 'training') refuseRetiredTraining();await assertMusicVideoJobPolicy(kind, params);if (params?.musicVideo?.sceneId) {const { assertMusicVideoSceneReview } = await import('../musicVideo/productionReviewService.js');@@ -2076,6 +2085,7 @@ export async function enqueueJob({ kind, params, owner = null }) {return laneQueue.length + liveCount + 1;})(),};+ assertRetiredDataAvailable();job.admitting = true;queue.push(job);try {diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.jsindex 8b2f5a8801..2fe536c8ed 100644--- a/server/services/mediaJobQueue/index.test.js+++ b/server/services/mediaJobQueue/index.test.js@@ -1832,16 +1832,26 @@ describe('Audio kind (#1928)', () => {expect(params.remoteMedia.request.prompt).toBe('a harbour');});- it('leaves a training job carrying a stray marker on the local path', async () => {- // Bypass probe for the normalization above: `training` has no federated- // contract, so a marker on one is corrupt state — blanking its params would- // destroy a real local run.- const job = await mediaJobQueue.enqueueJob({- kind: 'training',- params: { runId: 'run-1', runtime: 'mflux', modelId: 'dev', remoteMedia: remoteImageMediaParams() },+ it('rejects media admission during retired-data removal and permits it after settlement', async () => {+ const { withRetiredDataRemoval } = await import('../../lib/retiredLoraTraining.js');+ await withRetiredDataRemoval(async () => {+ await expect(mediaJobQueue.enqueueJob({ kind: 'image', params: {} })).rejects.toMatchObject({ status: 409, code: 'DATA_MAINTENANCE_ACTIVE' });});+ const accepted = await mediaJobQueue.enqueueJob({ kind: 'image', params: {} });+ expect(accepted.jobId).toBeTruthy();+ });++ it('retires persisted queued training without invoking a trainer or removing artifacts', async () => {+ const id = '00000000-0000-4000-8000-000000000019';+ writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ jobs: [{ id, kind: 'training', status: 'queued', params: { runId: 'example-retired' } }] }));+ await importFresh();+ await mediaJobQueue.initMediaJobQueue();+ expect(mediaJobQueue.getJob(id)).toMatchObject({ status: 'canceled', error: expect.stringContaining('retired') });+ expect(stubs.runTraining).not.toHaveBeenCalled();+ });- expect(mediaJobQueue.getJob(job.jobId).params.modelId).toBe('dev');+ it('refuses retired training even when carrying a stray remote marker', async () => {+ await expect(mediaJobQueue.enqueueJob({ kind: 'training', params: { runId: 'example', remoteMedia: remoteImageMediaParams() } })).rejects.toMatchObject({ status: 410, code: 'LORA_TRAINING_RETIRED' });});it('persists cancellation intent before signaling a running remote adapter', async () => {@@ -2731,7 +2741,7 @@ describe('local video failure holds', () => {remoteAdapter.mockRestore();});- it('lets other local models, image, audio, training, cloud and remote work pass a hold', async () => {+ it('lets other local models, image, audio, cloud and remote work pass a hold', async () => {const ids = await submitMany(4);for (const id of ids.slice(0, 3)) { await tick(); await finish(id); }const cloud = await submit('example-mlx', { mode: 'grok' });@@ -2742,7 +2752,7 @@ describe('local video failure holds', () => {expect(stubs.generateVideoRemote).toHaveBeenCalledWith(expect.objectContaining({ jobId: remote }));expect(mediaJobQueue.getJob(other).status).toBe('running');await finish(other, null);- for (const [kind, emitter] of [['image', imageGenEvents], ['audio', audioGenEvents], ['training', (await import('../loraTraining/events.js')).trainingEvents]]) {+ for (const [kind, emitter] of [['image', imageGenEvents], ['audio', audioGenEvents]]) {stubs.runTraining.mockResolvedValue({});const { jobId } = await mediaJobQueue.enqueueJob({ kind, params: {} });await tick();@@ -4236,7 +4246,6 @@ describe('mediaJobQueue per-kind runtime wiring', () => {entry: 'runSharingCopy', bus: 'video', verb: 'Generating', window: WINDOWS.VIDEO },{ kind: 'html-composition', params: { directory: 'compositions/example' }, entry: 'renderComposition', bus: 'video', verb: 'Generating', window: WINDOWS.IMAGE },{ kind: 'image', params: {}, entry: 'generateImage', bus: 'image', verb: 'Generating', window: WINDOWS.IMAGE },- { kind: 'training', params: {}, entry: 'runTraining', bus: 'training', verb: 'Training', window: WINDOWS.TRAINING },{ kind: 'audio', params: {}, entry: 'generateAudio', bus: 'audio', verb: 'Generating', window: WINDOWS.AUDIO },{ kind: 'supercollider', params: {}, entry: 'renderSuperCollider', bus: 'audio', verb: 'Generating', window: WINDOWS.IMAGE },];@@ -4247,8 +4256,8 @@ describe('mediaJobQueue per-kind runtime wiring', () => {training: (await import('../loraTraining/events.js')).trainingEvents,})[name];- it('covers every declared job kind', () => {- expect(KIND_CASES.map(c => c.kind).sort()).toEqual([...mediaJobQueue.JOB_KINDS].sort());+ it('covers every newly dispatchable kind; historical training is recovery-only', () => {+ expect(KIND_CASES.map(c => c.kind).sort()).toEqual(mediaJobQueue.JOB_KINDS.filter(kind => kind !== 'training').sort());});describe.each(KIND_CASES)('$kind', ({ kind, params, entry, bus, verb, window }) => {