Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: show diffusion training preparation progress (#15337)

Daniel Hanbafc1e84dbc6
diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.py
index 618266ef171..28f7ae49b6c 100644
--- a/studio/backend/core/training/diffusion_training_service.py
+++ b/studio/backend/core/training/diffusion_training_service.py
@@ -329,6 +329,7 @@ def _idle_state() -> dict[str, Any]:
"audio_loss": None,
"num_images": None,
"in_model_load": False,
+ "preparation": None,
"output_dir": None,
"lora_path": None,
# The optional second (EMA-averaged) adapter, when ema_decay was enabled.
@@ -810,6 +811,9 @@ class DiffusionTrainingService:
self._state.update(
active = False,
status = "error",
+ # A worker killed mid-load / mid-preparation sends no terminal event to clear these.
+ in_model_load = False,
+ preparation = None,
message = self._unexpected_exit_message(proc),
updated_at = time.time(),
)
@@ -965,11 +969,16 @@ class DiffusionTrainingService:
s = self._state
s["updated_at"] = time.time()
if etype == "model_load_started":
- s.update(in_model_load = True, status = "running", message = "Loading base model...")
+ s.update(
+ in_model_load = True,
+ preparation = None,
+ status = "running",
+ message = "Loading base model...",
+ )
if ev.get("num_images") is not None:
s["num_images"] = ev.get("num_images")
elif etype == "model_load_completed":
- s.update(in_model_load = False, message = "Training...")
+ s.update(in_model_load = False, preparation = None, message = "Training...")
elif etype == "preparing":
# Long precompute such as the VAE latent cache, surfaced so the UI does not look stalled.
done, total = ev.get("done"), ev.get("total")
@@ -977,6 +986,7 @@ class DiffusionTrainingService:
s.update(
status = "running",
in_model_load = True,
+ preparation = {"stage": ev.get("stage"), "done": done, "total": total},
message = (
f"Preparing ({stage} {done}/{total})..."
if done is not None and total is not None
@@ -1038,6 +1048,8 @@ class DiffusionTrainingService:
s.update(
status = "running",
step = ev.get("step", s["step"]),
+ preparation = None,
+ in_model_load = False,
total_steps = ev.get("total_steps", s["total_steps"]),
loss = loss,
avg_loss = avg_loss,
@@ -1075,6 +1087,7 @@ class DiffusionTrainingService:
active = False,
in_model_load = False,
status = "stopped" if ev.get("stopped") else "completed",
+ preparation = None,
output_dir = ev.get("output_dir"),
lora_path = ev.get("lora_path"),
ema_path = ev.get("ema_path"),
@@ -1108,6 +1121,7 @@ class DiffusionTrainingService:
active = False,
in_model_load = False,
status = "error",
+ preparation = None,
message = str(ev.get("message", "error")),
)
diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py
index 6412427b800..af412ee1262 100644
--- a/studio/backend/models/training.py
+++ b/studio/backend/models/training.py
@@ -1205,6 +1205,12 @@ class DiffusionSampleImage(BaseModel):
seed: Optional[int] = None
+class DiffusionTrainingPreparation(BaseModel):
+ stage: Optional[str] = None
+ done: Optional[int] = None
+ total: Optional[int] = None
+
+
class DiffusionTrainingStatusResponse(BaseModel):
"""A snapshot of the current diffusion training job (or idle)."""
@@ -1224,6 +1230,7 @@ class DiffusionTrainingStatusResponse(BaseModel):
audio_loss: Optional[float] = None
num_images: Optional[int] = None
in_model_load: bool = False
+ preparation: Optional[DiffusionTrainingPreparation] = None
output_dir: Optional[str] = None
lora_path: Optional[str] = None
# The second, EMA-averaged adapter written in the run's ema subdir when ema_decay was enabled.
diff --git a/studio/backend/tests/test_diffusion_training.py b/studio/backend/tests/test_diffusion_training.py
index a09cac343a1..efc27906b01 100644
--- a/studio/backend/tests/test_diffusion_training.py
+++ b/studio/backend/tests/test_diffusion_training.py
@@ -324,6 +324,41 @@ def test_apply_event_transitions():
assert svc.status()["status"] == "error" and svc.status()["message"] == "boom"
+def test_preparation_counts_survive_status_polling_and_clear_for_training():
+ svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 48, "total": 60})
+ assert svc.status()["preparation"] == {"stage": "cache_latents", "done": 48, "total": 60}
+ assert svc.status()["in_model_load"] is True
+ svc._apply_event({"type": "model_load_completed"})
+ assert svc.status()["preparation"] is None
+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 60, "total": 60})
+ svc._apply_event({"type": "progress", "step": 1, "total_steps": 30, "loss": 0.4})
+ assert svc.status()["preparation"] is None
+ assert svc.status()["in_model_load"] is False
+
+
[email protected]("event", [{"type": "complete"}, {"type": "error", "message": "boom"}])
+def test_terminal_events_clear_preparation_counts(event):
+ svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 48, "total": 60})
+ svc._apply_event(event)
+ assert svc.status()["preparation"] is None
+
+
+def _dies_while_preparing(*, event_queue, stop_queue, config):
+ event_queue.put({"type": "model_load_started", "num_images": 60})
+ event_queue.put({"type": "preparing", "stage": "cache_latents", "done": 12, "total": 60})
+
+
+def test_worker_dying_mid_preparation_clears_preparation_counts():
+ svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _dies_while_preparing)
+ svc.start(dict(_CFG))
+ st = _wait_status(svc, "error")
+ assert st["status"] == "error"
+ assert st["preparation"] is None
+ assert st["in_model_load"] is False
+
+
def test_the_joint_losses_reach_status_and_history():
"""video_loss and audio_loss must reach status and history, index-aligned with steps."""
svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
@@ -2062,6 +2097,18 @@ def test_complete_event_records_family_and_catalog():
assert st["catalog_path"] == "/loras/w.safetensors"
+def test_status_route_returns_preparation_counts(client):
+ # The response model filters unknown keys, so the counts reach the UI only if it declares them.
+ client._fake.status_extra = {
+ "in_model_load": True,
+ "preparation": {"stage": "cache_latents", "done": 48, "total": 60},
+ }
+ body = client.get("/api/train/diffusion/status").json()
+ assert body["preparation"] == {"stage": "cache_latents", "done": 48, "total": 60}
+ client._fake.status_extra = {"preparation": None}
+ assert client.get("/api/train/diffusion/status").json()["preparation"] is None
+
+
def test_status_route_nests_metric_history(client):
client._fake.status_extra = {
"metric_steps": [1, 2],
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts
index 9d3262ff63f..0da5b5f9ad5 100644
--- a/studio/frontend/src/features/images/api.ts
+++ b/studio/frontend/src/features/images/api.ts
@@ -688,6 +688,7 @@ export interface DiffusionTrainingStatus {
grad_norm?: number | null;
num_images: number | null;
in_model_load: boolean;
+ preparation?: { stage?: string | null; done?: number | null; total?: number | null } | null;
output_dir: string | null;
lora_path: string | null;
ema_path?: string | null;
diff --git a/studio/frontend/src/features/images/train/diffusion-train-panel.tsx b/studio/frontend/src/features/images/train/diffusion-train-panel.tsx
index d6085c5b6ec..c93a30d19fa 100644
--- a/studio/frontend/src/features/images/train/diffusion-train-panel.tsx
+++ b/studio/frontend/src/features/images/train/diffusion-train-panel.tsx
@@ -32,6 +32,7 @@ import {
AlertDialogTitle,
} from "@/components/ui/alert-dialog";
import { Badge } from "@/components/ui/badge";
+import { Progress } from "@/components/ui/progress";
import { Button } from "@/components/ui/button";
import { InfoHint } from "@/components/ui/info-hint";
import { useScrollFades } from "@/hooks/use-scroll-fades";
@@ -116,6 +117,7 @@ import {
resolveDiffusionTrainingBase,
} from "./diffusion-train-deploy";
import { resolveDiffusionTrainingFacts } from "./diffusion-train-family-facts";
+import { diffusionTrainingProgress } from "./diffusion-train-progress";
import { type LrScheduler, lrSchedulePreset } from "./diffusion-train-lr-schedule";
// Fallback list for an older backend whose /info reports none, in popularity order.
@@ -689,10 +691,7 @@ export function DiffusionTrainPanel({
status?.status === "stopped" &&
Boolean(status?.lora_path) &&
status.job_id !== dismissedJobId;
- const pct =
- status && status.total_steps > 0
- ? Math.min(100, Math.round((status.step / status.total_steps) * 100))
- : 0;
+ const runProgress = diffusionTrainingProgress(status);
const stopRequested = running && stopRequestedLocal;
@@ -2184,21 +2183,20 @@ export function DiffusionTrainPanel({
<div className="flex flex-col gap-3">
<div className="flex items-center justify-between">
<span className="text-sm font-semibold capitalize">
- {status?.status === "completed" ? "Training complete \u{1F389}" : status?.status}
+ {runProgress.title}
</span>
<span className="text-xs text-muted-foreground">
- {(status?.total_steps ?? 0) > 0
- ? `${status?.step}/${status?.total_steps} steps`
- : ""}
+ {runProgress.count}
</span>
</div>
- <div className="h-2 w-full overflow-hidden rounded-full bg-border">
- <div
- className="h-full bg-primary transition-all"
- style={{ width: `${pct}%` }}
- />
- </div>
- <div className="grid grid-cols-2 gap-3 @min-[440px]:grid-cols-4">
+ <Progress
+ className="h-2"
+ value={runProgress.percent ?? undefined}
+ indeterminate={runProgress.percent === null}
+ aria-label={runProgress.title}
+ />
+ {!runProgress.preparing && (
+ <div className="grid grid-cols-2 gap-3 @min-[440px]:grid-cols-4">
<Stat
label="Loss"
value={status?.loss != null ? status.loss.toFixed(4) : "-"}
@@ -2223,9 +2221,12 @@ export function DiffusionTrainPanel({
: "-"
}
/>
- </div>
- {status?.message && (
- <p className="text-ui-11 text-muted-foreground">{status.message}</p>
+ </div>
+ )}
+ {runProgress.detail && (
+ <p role="status" className="text-ui-11 text-muted-foreground">
+ {runProgress.detail}
+ </p>
)}
{running && (
<Button
@@ -2345,7 +2346,9 @@ export function DiffusionTrainPanel({
</div>
)}
- <DiffusionCharts lossHistory={lossHistory} gradNormHistory={gradNormHistory} />
+ {!runProgress.preparing && lossHistory.length > 0 && (
+ <DiffusionCharts lossHistory={lossHistory} gradNormHistory={gradNormHistory} />
+ )}
{status?.job_id ? (
<DiffusionSamples jobId={status.job_id} samples={status.samples} />
) : null}
diff --git a/studio/frontend/src/features/images/train/diffusion-train-progress.ts b/studio/frontend/src/features/images/train/diffusion-train-progress.ts
new file mode 100644
index 00000000000..25471c5dbef
--- /dev/null
+++ b/studio/frontend/src/features/images/train/diffusion-train-progress.ts
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import type { DiffusionTrainingStatus } from "../api.ts";
+
+type Status = Pick<
+ DiffusionTrainingStatus,
+ | "active"
+ | "status"
+ | "step"
+ | "total_steps"
+ | "in_model_load"
+ | "message"
+ | "preparation"
+>;
+
+const PHASE_MESSAGE =
+ /^(?:Loading base model\.\.\.|Training\.\.\.|Preparing \(.*\)\.\.\.)$/;
+const TERMINAL_STATUSES = new Set(["completed", "stopped", "error"]);
+// Older backends only carry the preparation counts in the status text.
+const LEGACY_PREPARING = /Preparing \(([^()]+?) (\d+)\/(\d+)\)/;
+
+function backendNote(message: string | null | undefined): string | null {
+ return message && !PHASE_MESSAGE.test(message) ? message : null;
+}
+
+export function diffusionTrainingProgress(status: Status | null) {
+ const running = status?.active === true || status?.status === "running";
+ // A stopped / failed run can still report active while its worker exits; not a restart.
+ const terminal = TERMINAL_STATUSES.has(status?.status ?? "");
+ const preparing =
+ running &&
+ !terminal &&
+ (status?.in_model_load === true || (status?.step ?? 0) <= 0);
+ if (preparing) {
+ const note = backendNote(status?.message);
+ const legacy = status?.message?.match(LEGACY_PREPARING);
+ const prep =
+ status?.preparation ??
+ (legacy
+ ? {
+ stage: legacy[1].replaceAll(" ", "_"),
+ done: Number(legacy[2]),
+ total: Number(legacy[3]),
+ }
+ : null);
+ if (prep) {
+ const { done, total } = prep;
+ const known =
+ Number.isFinite(done) &&
+ Number.isFinite(total) &&
+ (done as number) >= 0 &&
+ (total as number) > 0;
+ // MiniMax-H3 encodes every caption in one go (n/n) and only then caches latents; that is not done yet.
+ const finished =
+ known &&
+ (done as number) >= (total as number) &&
+ prep.stage !== "encode_prompts";
+ return {
+ preparing: true,
+ title: finished ? "Training data prepared" : "Preparing training data",
+ detail:
+ note ??
+ (finished
+ ? "Finishing model setup before training starts."
+ : "Preparing images and captions so the model can train on them."),
+ count: known
+ ? `${Math.min(done as number, total as number)} / ${total} prepared`
+ : null,
+ percent: known
+ ? Math.min(100, ((done as number) / (total as number)) * 100)
+ : null,
+ };
+ }
+ return {
+ preparing: true,
+ title: status?.in_model_load
+ ? "Loading training model"
+ : "Starting training",
+ detail:
+ note ??
+ (status?.in_model_load
+ ? "Getting the model ready before training starts."
+ : "Preparing the first training step. Compilation can take longer on the first run."),
+ count: null,
+ percent: null,
+ };
+ }
+ return {
+ preparing: false,
+ title:
+ status?.status === "completed"
+ ? "Training complete"
+ : running && !terminal
+ ? "Training"
+ : (status?.status ?? ""),
+ detail: status?.message ?? null,
+ count:
+ status && status.total_steps > 0
+ ? `${status.step} / ${status.total_steps} steps`
+ : null,
+ percent:
+ status && status.total_steps > 0
+ ? Math.max(0, Math.min(100, (status.step / status.total_steps) * 100))
+ : 0,
+ };
+}
diff --git a/studio/frontend/tests/diffusion-train-progress.test.ts b/studio/frontend/tests/diffusion-train-progress.test.ts
new file mode 100644
index 00000000000..287240144dd
--- /dev/null
+++ b/studio/frontend/tests/diffusion-train-progress.test.ts
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import { diffusionTrainingProgress } from "../src/features/images/train/diffusion-train-progress.ts";
+
+const running = {
+ active: true,
+ status: "running",
+ step: 0,
+ total_steps: 100,
+ in_model_load: true,
+ message: "Loading base model...",
+};
+
+test("image preparation advances its own bar before any training step", () => {
+ const progress = diffusionTrainingProgress({
+ ...running,
+ preparation: { stage: "cache_latents", done: 48, total: 60 },
+ });
+ assert.equal(progress.title, "Preparing training data");
+ assert.equal(progress.percent, 80);
+ assert.equal(progress.count, "48 / 60 prepared");
+ assert.equal(progress.preparing, true);
+});
+
+test("60/60 means data is prepared, not that training is complete", () => {
+ const progress = diffusionTrainingProgress({
+ ...running,
+ preparation: { stage: "cache_latents", done: 60, total: 60 },
+ });
+ assert.equal(progress.title, "Training data prepared");
+ assert.equal(progress.percent, 100);
+ assert.match(progress.detail!, /before training starts/);
+ assert.equal(progress.preparing, true);
+});
+
+test("unknown model-loading and first-step durations remain indeterminate", () => {
+ assert.equal(diffusionTrainingProgress(running).percent, null);
+ const firstStep = diffusionTrainingProgress({
+ ...running,
+ in_model_load: false,
+ });
+ assert.equal(firstStep.title, "Starting training");
+ assert.equal(firstStep.percent, null);
+ assert.match(firstStep.detail!, /Compilation/);
+});
+
+test("the first training step switches back to optimization progress", () => {
+ const progress = diffusionTrainingProgress({
+ ...running,
+ step: 1,
+ in_model_load: false,
+ });
+ assert.equal(progress.title, "Training");
+ assert.equal(progress.percent, 1);
+ assert.equal(progress.count, "1 / 100 steps");
+ assert.equal(progress.preparing, false);
+});
+
+test("older backends' preparation messages still move the bar", () => {
+ const progress = diffusionTrainingProgress({
+ ...running,
+ message: "Preparing (cache latents 48/60)...",
+ });
+ assert.equal(progress.percent, 80);
+ assert.equal(progress.count, "48 / 60 prepared");
+});
+
+test("unreadable preparation counts never fabricate progress", () => {
+ for (const preparation of [
+ { done: 0, total: 0 },
+ { done: -1, total: 60 },
+ { done: NaN, total: 60 },
+ { done: 4, total: Infinity },
+ ]) {
+ assert.equal(
+ diffusionTrainingProgress({ ...running, preparation }).percent,
+ null,
+ );
+ }
+});
+
+test("stopped and failed runs do not keep an animated preparation bar", () => {
+ for (const status of ["stopped", "error"]) {
+ const progress = diffusionTrainingProgress({
+ ...running,
+ active: false,
+ status,
+ });
+ assert.equal(progress.preparing, false);
+ assert.equal(progress.percent, 0);
+ }
+});
+
+test("resuming a positive-step run still shows its preparation stage", () => {
+ const progress = diffusionTrainingProgress({
+ ...running,
+ step: 20,
+ preparation: { stage: "cache_latents", done: 30, total: 60 },
+ });
+ assert.equal(progress.preparing, true);
+ assert.equal(progress.percent, 50);
+});
+
+test("warnings and stop requests stay visible while preparing", () => {
+ const stopping =
+ "Cancel requested; finishing the current step (no adapter will be saved)...";
+ const prep = diffusionTrainingProgress({
+ ...running,
+ message: stopping,
+ preparation: { stage: "cache_latents", done: 12, total: 60 },
+ });
+ assert.equal(prep.detail, stopping);
+ assert.equal(prep.percent, 20);
+ const warning = "torch.compile disabled (eager fallback): boom";
+ const firstStep = diffusionTrainingProgress({
+ ...running,
+ in_model_load: false,
+ message: warning,
+ });
+ assert.equal(firstStep.detail, warning);
+ assert.match(
+ diffusionTrainingProgress({
+ ...running,
+ message: "Preparing (cache latents 12/60)...",
+ }).detail!,
+ /Preparing images and captions/,
+ );
+});
+
+test("a run stopped mid-preparation does not read as starting while its worker exits", () => {
+ const progress = diffusionTrainingProgress({
+ active: true,
+ status: "stopped",
+ step: 0,
+ total_steps: 0,
+ in_model_load: false,
+ message: "Stopped (no adapter saved).",
+ preparation: null,
+ });
+ assert.equal(progress.preparing, false);
+ assert.equal(progress.title, "stopped");
+ assert.equal(progress.percent, 0);
+ assert.equal(progress.detail, "Stopped (no adapter saved).");
+});
+
+test("encoded captions alone do not read as prepared training data", () => {
+ const progress = diffusionTrainingProgress({
+ ...running,
+ preparation: { stage: "encode_prompts", done: 12, total: 12 },
+ });
+ assert.equal(progress.title, "Preparing training data");
+ assert.equal(progress.percent, 100);
+});