Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: show diffusion training preparation progress (#15337)
Daniel Hanbafc1e84dbc6
diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.pyindex 618266ef171..28f7ae49b6c 100644--- a/studio/backend/core/training/diffusion_training_service.py+++ b/studio/backend/core/training/diffusion_training_service.py@@ -329,6 +329,7 @@ def _idle_state() -> dict[str, Any]:"audio_loss": None,"num_images": None,"in_model_load": False,+ "preparation": None,"output_dir": None,"lora_path": None,# The optional second (EMA-averaged) adapter, when ema_decay was enabled.@@ -810,6 +811,9 @@ class DiffusionTrainingService:self._state.update(active = False,status = "error",+ # A worker killed mid-load / mid-preparation sends no terminal event to clear these.+ in_model_load = False,+ preparation = None,message = self._unexpected_exit_message(proc),updated_at = time.time(),)@@ -965,11 +969,16 @@ class DiffusionTrainingService:s = self._states["updated_at"] = time.time()if etype == "model_load_started":- s.update(in_model_load = True, status = "running", message = "Loading base model...")+ s.update(+ in_model_load = True,+ preparation = None,+ status = "running",+ message = "Loading base model...",+ )if ev.get("num_images") is not None:s["num_images"] = ev.get("num_images")elif etype == "model_load_completed":- s.update(in_model_load = False, message = "Training...")+ s.update(in_model_load = False, preparation = None, message = "Training...")elif etype == "preparing":# Long precompute such as the VAE latent cache, surfaced so the UI does not look stalled.done, total = ev.get("done"), ev.get("total")@@ -977,6 +986,7 @@ class DiffusionTrainingService:s.update(status = "running",in_model_load = True,+ preparation = {"stage": ev.get("stage"), "done": done, "total": total},message = (f"Preparing ({stage} {done}/{total})..."if done is not None and total is not None@@ -1038,6 +1048,8 @@ class DiffusionTrainingService:s.update(status = "running",step = ev.get("step", s["step"]),+ preparation = None,+ in_model_load = False,total_steps = ev.get("total_steps", s["total_steps"]),loss = loss,avg_loss = avg_loss,@@ -1075,6 +1087,7 @@ class DiffusionTrainingService:active = False,in_model_load = False,status = "stopped" if ev.get("stopped") else "completed",+ preparation = None,output_dir = ev.get("output_dir"),lora_path = ev.get("lora_path"),ema_path = ev.get("ema_path"),@@ -1108,6 +1121,7 @@ class DiffusionTrainingService:active = False,in_model_load = False,status = "error",+ preparation = None,message = str(ev.get("message", "error")),)diff --git a/studio/backend/models/training.py b/studio/backend/models/training.pyindex 6412427b800..af412ee1262 100644--- a/studio/backend/models/training.py+++ b/studio/backend/models/training.py@@ -1205,6 +1205,12 @@ class DiffusionSampleImage(BaseModel):seed: Optional[int] = None+class DiffusionTrainingPreparation(BaseModel):+ stage: Optional[str] = None+ done: Optional[int] = None+ total: Optional[int] = None++class DiffusionTrainingStatusResponse(BaseModel):"""A snapshot of the current diffusion training job (or idle)."""@@ -1224,6 +1230,7 @@ class DiffusionTrainingStatusResponse(BaseModel):audio_loss: Optional[float] = Nonenum_images: Optional[int] = Nonein_model_load: bool = False+ preparation: Optional[DiffusionTrainingPreparation] = Noneoutput_dir: Optional[str] = Nonelora_path: Optional[str] = None# The second, EMA-averaged adapter written in the run's ema subdir when ema_decay was enabled.diff --git a/studio/backend/tests/test_diffusion_training.py b/studio/backend/tests/test_diffusion_training.pyindex a09cac343a1..efc27906b01 100644--- a/studio/backend/tests/test_diffusion_training.py+++ b/studio/backend/tests/test_diffusion_training.py@@ -324,6 +324,41 @@ def test_apply_event_transitions():assert svc.status()["status"] == "error" and svc.status()["message"] == "boom"+def test_preparation_counts_survive_status_polling_and_clear_for_training():+ svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 48, "total": 60})+ assert svc.status()["preparation"] == {"stage": "cache_latents", "done": 48, "total": 60}+ assert svc.status()["in_model_load"] is True+ svc._apply_event({"type": "model_load_completed"})+ assert svc.status()["preparation"] is None+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 60, "total": 60})+ svc._apply_event({"type": "progress", "step": 1, "total_steps": 30, "loss": 0.4})+ assert svc.status()["preparation"] is None+ assert svc.status()["in_model_load"] is False++[email protected]("event", [{"type": "complete"}, {"type": "error", "message": "boom"}])+def test_terminal_events_clear_preparation_counts(event):+ svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 48, "total": 60})+ svc._apply_event(event)+ assert svc.status()["preparation"] is None+++def _dies_while_preparing(*, event_queue, stop_queue, config):+ event_queue.put({"type": "model_load_started", "num_images": 60})+ event_queue.put({"type": "preparing", "stage": "cache_latents", "done": 12, "total": 60})+++def test_worker_dying_mid_preparation_clears_preparation_counts():+ svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _dies_while_preparing)+ svc.start(dict(_CFG))+ st = _wait_status(svc, "error")+ assert st["status"] == "error"+ assert st["preparation"] is None+ assert st["in_model_load"] is False++def test_the_joint_losses_reach_status_and_history():"""video_loss and audio_loss must reach status and history, index-aligned with steps."""svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)@@ -2062,6 +2097,18 @@ def test_complete_event_records_family_and_catalog():assert st["catalog_path"] == "/loras/w.safetensors"+def test_status_route_returns_preparation_counts(client):+ # The response model filters unknown keys, so the counts reach the UI only if it declares them.+ client._fake.status_extra = {+ "in_model_load": True,+ "preparation": {"stage": "cache_latents", "done": 48, "total": 60},+ }+ body = client.get("/api/train/diffusion/status").json()+ assert body["preparation"] == {"stage": "cache_latents", "done": 48, "total": 60}+ client._fake.status_extra = {"preparation": None}+ assert client.get("/api/train/diffusion/status").json()["preparation"] is None++def test_status_route_nests_metric_history(client):client._fake.status_extra = {"metric_steps": [1, 2],diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.tsindex 9d3262ff63f..0da5b5f9ad5 100644--- a/studio/frontend/src/features/images/api.ts+++ b/studio/frontend/src/features/images/api.ts@@ -688,6 +688,7 @@ export interface DiffusionTrainingStatus {grad_norm?: number | null;num_images: number | null;in_model_load: boolean;+ preparation?: { stage?: string | null; done?: number | null; total?: number | null } | null;output_dir: string | null;lora_path: string | null;ema_path?: string | null;diff --git a/studio/frontend/src/features/images/train/diffusion-train-panel.tsx b/studio/frontend/src/features/images/train/diffusion-train-panel.tsxindex d6085c5b6ec..c93a30d19fa 100644--- a/studio/frontend/src/features/images/train/diffusion-train-panel.tsx+++ b/studio/frontend/src/features/images/train/diffusion-train-panel.tsx@@ -32,6 +32,7 @@ import {AlertDialogTitle,} from "@/components/ui/alert-dialog";import { Badge } from "@/components/ui/badge";+import { Progress } from "@/components/ui/progress";import { Button } from "@/components/ui/button";import { InfoHint } from "@/components/ui/info-hint";import { useScrollFades } from "@/hooks/use-scroll-fades";@@ -116,6 +117,7 @@ import {resolveDiffusionTrainingBase,} from "./diffusion-train-deploy";import { resolveDiffusionTrainingFacts } from "./diffusion-train-family-facts";+import { diffusionTrainingProgress } from "./diffusion-train-progress";import { type LrScheduler, lrSchedulePreset } from "./diffusion-train-lr-schedule";// Fallback list for an older backend whose /info reports none, in popularity order.@@ -689,10 +691,7 @@ export function DiffusionTrainPanel({status?.status === "stopped" &&Boolean(status?.lora_path) &&status.job_id !== dismissedJobId;- const pct =- status && status.total_steps > 0- ? Math.min(100, Math.round((status.step / status.total_steps) * 100))- : 0;+ const runProgress = diffusionTrainingProgress(status);const stopRequested = running && stopRequestedLocal;@@ -2184,21 +2183,20 @@ export function DiffusionTrainPanel({<div className="flex flex-col gap-3"><div className="flex items-center justify-between"><span className="text-sm font-semibold capitalize">- {status?.status === "completed" ? "Training complete \u{1F389}" : status?.status}+ {runProgress.title}</span><span className="text-xs text-muted-foreground">- {(status?.total_steps ?? 0) > 0- ? `${status?.step}/${status?.total_steps} steps`- : ""}+ {runProgress.count}</span></div>- <div className="h-2 w-full overflow-hidden rounded-full bg-border">- <div- className="h-full bg-primary transition-all"- style={{ width: `${pct}%` }}- />- </div>- <div className="grid grid-cols-2 gap-3 @min-[440px]:grid-cols-4">+ <Progress+ className="h-2"+ value={runProgress.percent ?? undefined}+ indeterminate={runProgress.percent === null}+ aria-label={runProgress.title}+ />+ {!runProgress.preparing && (+ <div className="grid grid-cols-2 gap-3 @min-[440px]:grid-cols-4"><Statlabel="Loss"value={status?.loss != null ? status.loss.toFixed(4) : "-"}@@ -2223,9 +2221,12 @@ export function DiffusionTrainPanel({: "-"}/>- </div>- {status?.message && (- <p className="text-ui-11 text-muted-foreground">{status.message}</p>+ </div>+ )}+ {runProgress.detail && (+ <p role="status" className="text-ui-11 text-muted-foreground">+ {runProgress.detail}+ </p>)}{running && (<Button@@ -2345,7 +2346,9 @@ export function DiffusionTrainPanel({</div>)}- <DiffusionCharts lossHistory={lossHistory} gradNormHistory={gradNormHistory} />+ {!runProgress.preparing && lossHistory.length > 0 && (+ <DiffusionCharts lossHistory={lossHistory} gradNormHistory={gradNormHistory} />+ )}{status?.job_id ? (<DiffusionSamples jobId={status.job_id} samples={status.samples} />) : null}diff --git a/studio/frontend/src/features/images/train/diffusion-train-progress.ts b/studio/frontend/src/features/images/train/diffusion-train-progress.tsnew file mode 100644index 00000000000..25471c5dbef--- /dev/null+++ b/studio/frontend/src/features/images/train/diffusion-train-progress.ts@@ -0,0 +1,107 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import type { DiffusionTrainingStatus } from "../api.ts";++type Status = Pick<+ DiffusionTrainingStatus,+ | "active"+ | "status"+ | "step"+ | "total_steps"+ | "in_model_load"+ | "message"+ | "preparation"+>;++const PHASE_MESSAGE =+ /^(?:Loading base model\.\.\.|Training\.\.\.|Preparing \(.*\)\.\.\.)$/;+const TERMINAL_STATUSES = new Set(["completed", "stopped", "error"]);+// Older backends only carry the preparation counts in the status text.+const LEGACY_PREPARING = /Preparing \(([^()]+?) (\d+)\/(\d+)\)/;++function backendNote(message: string | null | undefined): string | null {+ return message && !PHASE_MESSAGE.test(message) ? message : null;+}++export function diffusionTrainingProgress(status: Status | null) {+ const running = status?.active === true || status?.status === "running";+ // A stopped / failed run can still report active while its worker exits; not a restart.+ const terminal = TERMINAL_STATUSES.has(status?.status ?? "");+ const preparing =+ running &&+ !terminal &&+ (status?.in_model_load === true || (status?.step ?? 0) <= 0);+ if (preparing) {+ const note = backendNote(status?.message);+ const legacy = status?.message?.match(LEGACY_PREPARING);+ const prep =+ status?.preparation ??+ (legacy+ ? {+ stage: legacy[1].replaceAll(" ", "_"),+ done: Number(legacy[2]),+ total: Number(legacy[3]),+ }+ : null);+ if (prep) {+ const { done, total } = prep;+ const known =+ Number.isFinite(done) &&+ Number.isFinite(total) &&+ (done as number) >= 0 &&+ (total as number) > 0;+ // MiniMax-H3 encodes every caption in one go (n/n) and only then caches latents; that is not done yet.+ const finished =+ known &&+ (done as number) >= (total as number) &&+ prep.stage !== "encode_prompts";+ return {+ preparing: true,+ title: finished ? "Training data prepared" : "Preparing training data",+ detail:+ note ??+ (finished+ ? "Finishing model setup before training starts."+ : "Preparing images and captions so the model can train on them."),+ count: known+ ? `${Math.min(done as number, total as number)} / ${total} prepared`+ : null,+ percent: known+ ? Math.min(100, ((done as number) / (total as number)) * 100)+ : null,+ };+ }+ return {+ preparing: true,+ title: status?.in_model_load+ ? "Loading training model"+ : "Starting training",+ detail:+ note ??+ (status?.in_model_load+ ? "Getting the model ready before training starts."+ : "Preparing the first training step. Compilation can take longer on the first run."),+ count: null,+ percent: null,+ };+ }+ return {+ preparing: false,+ title:+ status?.status === "completed"+ ? "Training complete"+ : running && !terminal+ ? "Training"+ : (status?.status ?? ""),+ detail: status?.message ?? null,+ count:+ status && status.total_steps > 0+ ? `${status.step} / ${status.total_steps} steps`+ : null,+ percent:+ status && status.total_steps > 0+ ? Math.max(0, Math.min(100, (status.step / status.total_steps) * 100))+ : 0,+ };+}diff --git a/studio/frontend/tests/diffusion-train-progress.test.ts b/studio/frontend/tests/diffusion-train-progress.test.tsnew file mode 100644index 00000000000..287240144dd--- /dev/null+++ b/studio/frontend/tests/diffusion-train-progress.test.ts@@ -0,0 +1,156 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";+import { diffusionTrainingProgress } from "../src/features/images/train/diffusion-train-progress.ts";++const running = {+ active: true,+ status: "running",+ step: 0,+ total_steps: 100,+ in_model_load: true,+ message: "Loading base model...",+};++test("image preparation advances its own bar before any training step", () => {+ const progress = diffusionTrainingProgress({+ ...running,+ preparation: { stage: "cache_latents", done: 48, total: 60 },+ });+ assert.equal(progress.title, "Preparing training data");+ assert.equal(progress.percent, 80);+ assert.equal(progress.count, "48 / 60 prepared");+ assert.equal(progress.preparing, true);+});++test("60/60 means data is prepared, not that training is complete", () => {+ const progress = diffusionTrainingProgress({+ ...running,+ preparation: { stage: "cache_latents", done: 60, total: 60 },+ });+ assert.equal(progress.title, "Training data prepared");+ assert.equal(progress.percent, 100);+ assert.match(progress.detail!, /before training starts/);+ assert.equal(progress.preparing, true);+});++test("unknown model-loading and first-step durations remain indeterminate", () => {+ assert.equal(diffusionTrainingProgress(running).percent, null);+ const firstStep = diffusionTrainingProgress({+ ...running,+ in_model_load: false,+ });+ assert.equal(firstStep.title, "Starting training");+ assert.equal(firstStep.percent, null);+ assert.match(firstStep.detail!, /Compilation/);+});++test("the first training step switches back to optimization progress", () => {+ const progress = diffusionTrainingProgress({+ ...running,+ step: 1,+ in_model_load: false,+ });+ assert.equal(progress.title, "Training");+ assert.equal(progress.percent, 1);+ assert.equal(progress.count, "1 / 100 steps");+ assert.equal(progress.preparing, false);+});++test("older backends' preparation messages still move the bar", () => {+ const progress = diffusionTrainingProgress({+ ...running,+ message: "Preparing (cache latents 48/60)...",+ });+ assert.equal(progress.percent, 80);+ assert.equal(progress.count, "48 / 60 prepared");+});++test("unreadable preparation counts never fabricate progress", () => {+ for (const preparation of [+ { done: 0, total: 0 },+ { done: -1, total: 60 },+ { done: NaN, total: 60 },+ { done: 4, total: Infinity },+ ]) {+ assert.equal(+ diffusionTrainingProgress({ ...running, preparation }).percent,+ null,+ );+ }+});++test("stopped and failed runs do not keep an animated preparation bar", () => {+ for (const status of ["stopped", "error"]) {+ const progress = diffusionTrainingProgress({+ ...running,+ active: false,+ status,+ });+ assert.equal(progress.preparing, false);+ assert.equal(progress.percent, 0);+ }+});++test("resuming a positive-step run still shows its preparation stage", () => {+ const progress = diffusionTrainingProgress({+ ...running,+ step: 20,+ preparation: { stage: "cache_latents", done: 30, total: 60 },+ });+ assert.equal(progress.preparing, true);+ assert.equal(progress.percent, 50);+});++test("warnings and stop requests stay visible while preparing", () => {+ const stopping =+ "Cancel requested; finishing the current step (no adapter will be saved)...";+ const prep = diffusionTrainingProgress({+ ...running,+ message: stopping,+ preparation: { stage: "cache_latents", done: 12, total: 60 },+ });+ assert.equal(prep.detail, stopping);+ assert.equal(prep.percent, 20);+ const warning = "torch.compile disabled (eager fallback): boom";+ const firstStep = diffusionTrainingProgress({+ ...running,+ in_model_load: false,+ message: warning,+ });+ assert.equal(firstStep.detail, warning);+ assert.match(+ diffusionTrainingProgress({+ ...running,+ message: "Preparing (cache latents 12/60)...",+ }).detail!,+ /Preparing images and captions/,+ );+});++test("a run stopped mid-preparation does not read as starting while its worker exits", () => {+ const progress = diffusionTrainingProgress({+ active: true,+ status: "stopped",+ step: 0,+ total_steps: 0,+ in_model_load: false,+ message: "Stopped (no adapter saved).",+ preparation: null,+ });+ assert.equal(progress.preparing, false);+ assert.equal(progress.title, "stopped");+ assert.equal(progress.percent, 0);+ assert.equal(progress.detail, "Stopped (no adapter saved).");+});++test("encoded captions alone do not read as prepared training data", () => {+ const progress = diffusionTrainingProgress({+ ...running,+ preparation: { stage: "encode_prompts", done: 12, total: 12 },+ });+ assert.equal(progress.title, "Preparing training data");+ assert.equal(progress.percent, 100);+});