Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Report measured model loading and 4-bit conversion during image training (#15536)

Lee Jacksonb48b16e28c8b
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index f5794a61baf..6274b36fd54 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -37,6 +37,7 @@ from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any, Callable, Optional
+from core.training.diffusion_load_progress import report_model_load_progress
from core._torchao_stub import is_stubbed, torch_is_rocm
from core.training.diffusion_train_common import (
AUTO_FLOW_SHIFT_FAMILIES,
@@ -2395,7 +2396,11 @@ def run_dit_lora_training(
cfg.resume_from_checkpoint, identity = identity, target_steps = cfg.train_steps
)
_emit(
- on_event, "model_load_started", num_images = len(pairs), fetch_repo = training_fetch_repo(cfg)
+ on_event,
+ "model_load_started",
+ stage = "conditioning",
+ num_images = len(pairs),
+ fetch_repo = training_fetch_repo(cfg),
)
if _check_stop():
out_dir = Path(cfg.output_dir).expanduser()
@@ -2512,15 +2517,22 @@ def _train_dit(
if caption_embeds is not None and sample_texts:
# Warm conditioning skips the VAE and encoders, but previews need the VAE and their own embeddings. RNG-isolated:
# a run without previews never makes this load, so it must not move the training streams.
- _emit(on_event, "model_load_started", fetch_repo = training_fetch_repo(cfg))
+ _emit(
+ on_event,
+ "model_load_started",
+ stage = "conditioning",
+ fetch_repo = training_fetch_repo(cfg),
+ )
with isolated_sampling(device, False):
- pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)
+ with report_model_load_progress(on_event):
+ pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)
encoded = _encode_prompts_cached(spec, pipe, sample_texts, device, pcache)
sample_embeds = dict(zip(sample_texts, encoded))
_free_text_encoders(pipe)
if caption_embeds is None:
latent_cache = None
- pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)
+ with report_model_load_progress(on_event):
+ pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)
extra = [t for t in sample_texts if t not in to_encode]
encoded = _encode_prompts_cached(spec, pipe, to_encode + extra, device, pcache)
caption_embeds = {cap: emb for cap, emb in zip(to_encode, encoded)}
@@ -2586,9 +2598,17 @@ def _train_dit(
variant_rng = random.Random(cfg.seed + 1)
# Transformer loads last, at resolved precision: nf4 by default, 'auto' picks from free VRAM.
- _emit(on_event, "model_load_started", fetch_repo = training_fetch_repo(cfg))
base_precision = _resolve_base_precision(cfg, spec, device)
- transformer = spec.load_transformer(fetch_cfg, device, weight_dtype, base_precision)
+ _emit(
+ on_event,
+ "model_load_started",
+ stage = "quantizing"
+ if base_precision == "nf4" and not repo_is_prequantized(fetch_cfg.base_model)
+ else "transformer",
+ fetch_repo = training_fetch_repo(cfg),
+ )
+ with report_model_load_progress(on_event):
+ transformer = spec.load_transformer(fetch_cfg, device, weight_dtype, base_precision)
base_is_bnb = base_precision == "nf4"
transformer.requires_grad_(False)
diff --git a/studio/backend/core/training/diffusion_load_progress.py b/studio/backend/core/training/diffusion_load_progress.py
new file mode 100644
index 00000000000..e7a36e9878b
--- /dev/null
+++ b/studio/backend/core/training/diffusion_load_progress.py
@@ -0,0 +1,173 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Loader progress for the spawned diffusion training worker: diffusers has no per-weight callback, so observe
+tensor placement / quantizer calls and library tqdm bars, crediting only successful returns; hooks always restored."""
+
+from contextlib import contextmanager
+from functools import wraps
+import inspect
+import threading
+import time
+from weakref import WeakKeyDictionary
+
+
+@contextmanager
+def report_model_load_progress(on_event):
+ if on_event is None:
+ yield
+ return
+
+ try:
+ from diffusers.models import model_loading_utils as loading
+ from diffusers.models import modeling_utils
+ from diffusers.utils import logging as diffusion_logging
+ except ImportError:
+ yield
+ return
+ from transformers.utils import logging as transformer_logging
+
+ try:
+ from transformers import core_model_loading
+ except ImportError:
+ core_model_loading = None
+ from tqdm.auto import tqdm
+
+ lock = threading.RLock()
+ counters = WeakKeyDictionary()
+ last = [None, 0.0]
+
+ def model_label(model):
+ return (
+ "Image encoder weights"
+ if "Autoencoder" in type(model).__name__
+ else "Training model weights"
+ )
+
+ def emit(done, total, unit, label):
+ if not total or total <= 0:
+ return
+ with lock:
+ identity = (unit, label, total)
+ now = time.monotonic()
+ if identity == last[0] and done not in (0, total) and now - last[1] < 0.2:
+ return
+ last[:] = [identity, now]
+ on_event(
+ dict(type = "model_load_progress", done = done, total = total, unit = unit, label = label)
+ )
+
+ def completed(model, name):
+ with lock:
+ counter = counters.get(model)
+ if counter is None or name not in counter[0]:
+ return
+ counter[1].add(name)
+ emit(len(counter[1]), len(counter[0]), "tensors", model_label(model))
+
+ original_set = getattr(loading, "set_module_tensor_to_device", None)
+ original_meta = getattr(loading, "load_model_dict_into_meta", None)
+ if not callable(original_set) or not callable(original_meta):
+ yield
+ return
+ meta_signature = inspect.signature(original_meta)
+ if "model" not in meta_signature.parameters:
+ yield
+ return
+
+ @wraps(original_set)
+ def place(module, tensor_name, *args, **kwargs):
+ result = original_set(module, tensor_name, *args, **kwargs)
+ completed(module, tensor_name)
+ return result
+
+ class QuantizerProgress:
+ def __init__(self, quantizer):
+ self.quantizer = quantizer
+
+ def __getattr__(self, name):
+ return getattr(self.quantizer, name)
+
+ def create_quantized_param(self, model, value, name, *args, **kwargs):
+ result = self.quantizer.create_quantized_param(model, value, name, *args, **kwargs)
+ completed(model, name)
+ return result
+
+ @wraps(original_meta)
+ def load_meta(*args, **kwargs):
+ bound = meta_signature.bind(*args, **kwargs)
+ model = bound.arguments["model"]
+ with lock:
+ if model not in counters:
+ # Snapshot names before quantization adds checkpoint metadata to state_dict.
+ counters[model] = (set(model.state_dict()), set())
+ names, done = counters[model]
+ emit(len(done), len(names), "tensors", model_label(model))
+ quantizer = bound.arguments.get("hf_quantizer")
+ if quantizer is not None:
+ bound.arguments["hf_quantizer"] = QuantizerProgress(quantizer)
+ return original_meta(*bound.args, **bound.kwargs)
+
+ class CountedTqdm(tqdm):
+ def __init__(self, *args, unit, label, **kwargs):
+ self.progress_unit, self.progress_label = unit, label
+ # Count even when quiet logging disabled the bar; display() is muted and leave=False stops close() writing "\n".
+ kwargs["disable"], kwargs["leave"] = False, False
+ super().__init__(*args, **kwargs)
+ emit(self.n, self.total, unit, label)
+
+ def display(self, *args, **kwargs):
+ pass
+
+ def update(self, n = 1):
+ result = super().update(n)
+ emit(self.n, self.total, self.progress_unit, self.progress_label)
+ return result
+
+ def __iter__(self):
+ try:
+ for item in self.iterable:
+ yield item
+ # Credited on resume: an item whose load raised is never counted.
+ self.update(1)
+ finally:
+ self.close()
+
+ class ProgressFactory:
+ def __init__(self, original):
+ self.original = original
+
+ def __getattr__(self, name):
+ return getattr(self.original, name)
+
+ def __call__(self, *args, **kwargs):
+ desc = kwargs.get("desc", "")
+ if desc.startswith("Loading pipeline components"):
+ return CountedTqdm(*args, unit = "components", label = "Model components", **kwargs)
+ if desc == "Loading weights":
+ return CountedTqdm(*args, unit = "tensors", label = "Text encoder weights", **kwargs)
+ if desc == "Loading checkpoint shards":
+ return CountedTqdm(*args, unit = "shards", label = "Checkpoint shards", **kwargs)
+ return self.original(*args, **kwargs)
+
+ patched = []
+
+ def patch(module, name, replacement):
+ original = getattr(module, name)
+ patched.append((module, name, original))
+ setattr(module, name, replacement)
+
+ try:
+ patch(loading, "set_module_tensor_to_device", place)
+ patch(loading, "load_model_dict_into_meta", load_meta)
+ if getattr(modeling_utils, "load_model_dict_into_meta", None) is original_meta:
+ patch(modeling_utils, "load_model_dict_into_meta", load_meta)
+ patch(diffusion_logging, "tqdm", ProgressFactory(diffusion_logging.tqdm))
+ patch(transformer_logging, "tqdm", ProgressFactory(transformer_logging.tqdm))
+ # Transformers 5 imports tqdm by name.
+ if core_model_loading is not None and hasattr(core_model_loading, "tqdm"):
+ patch(core_model_loading, "tqdm", ProgressFactory(core_model_loading.tqdm))
+ yield
+ finally:
+ for module, name, original in reversed(patched):
+ setattr(module, name, original)
diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.py
index 19249afa5cc..883839a0e3b 100644
--- a/studio/backend/core/training/diffusion_lora_trainer.py
+++ b/studio/backend/core/training/diffusion_lora_trainer.py
@@ -418,9 +418,12 @@ def run_diffusion_lora_training(
)
return str(out_dir)
- pipe = StableDiffusionXLPipeline.from_pretrained(
- cfg.base_model, torch_dtype = weight_dtype, token = cfg.hf_token, add_watermarker = False
- )
+ from core.training.diffusion_load_progress import report_model_load_progress
+
+ with report_model_load_progress(on_event):
+ pipe = StableDiffusionXLPipeline.from_pretrained(
+ cfg.base_model, torch_dtype = weight_dtype, token = cfg.hf_token, add_watermarker = False
+ )
unet, vae = pipe.unet, pipe.vae
tokenizers = [pipe.tokenizer, pipe.tokenizer_2]
text_encoders = [pipe.text_encoder, pipe.text_encoder_2]
diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.py
index d9c1414a385..9961cb41457 100644
--- a/studio/backend/core/training/diffusion_training_service.py
+++ b/studio/backend/core/training/diffusion_training_service.py
@@ -359,6 +359,8 @@ def _idle_state() -> dict[str, Any]:
"num_images": None,
"in_model_load": False,
"preparation": None,
+ "model_load_stage": None,
+ "model_load_progress": None,
"download_progress": None,
"output_dir": None,
"lora_path": None,
@@ -786,6 +788,8 @@ class DiffusionTrainingService:
status = "stopped",
in_model_load = False,
preparation = None,
+ model_load_stage = None,
+ model_load_progress = None,
download_progress = None,
message = "Stopped while loading model files; training had not started.",
updated_at = time.time(),
@@ -919,6 +923,8 @@ class DiffusionTrainingService:
# A worker killed mid-load / mid-preparation sends no terminal event to clear these.
in_model_load = False,
preparation = None,
+ model_load_stage = None,
+ model_load_progress = None,
download_progress = None,
message = self._unexpected_exit_message(proc),
updated_at = time.time(),
@@ -1089,12 +1095,36 @@ class DiffusionTrainingService:
s.update(
in_model_load = True,
preparation = None,
+ model_load_stage = None,
+ model_load_progress = None,
download_progress = None,
status = "running",
message = "Loading base model...",
)
+ s["model_load_stage"] = (
+ ev.get("stage")
+ if ev.get("stage") in ("conditioning", "transformer", "quantizing")
+ else None
+ )
if ev.get("num_images") is not None:
s["num_images"] = ev.get("num_images")
+ elif etype == "model_load_progress":
+ done, total = ev.get("done"), ev.get("total")
+ if (
+ s.get("in_model_load")
+ and s.get("preparation") is None
+ and type(done) is int
+ and type(total) is int
+ and 0 <= done <= total
+ and total > 0
+ and ev.get("unit") in ("tensors", "components", "shards")
+ ):
+ s["model_load_progress"] = {
+ "done": done,
+ "total": total,
+ "unit": ev["unit"],
+ "label": str(ev.get("label", "Model weights"))[:200],
+ }
elif etype == "model_download_progress":
if s.get("in_model_load") and s.get("preparation") is None:
progress = ev.get("download_progress")
@@ -1115,10 +1145,14 @@ class DiffusionTrainingService:
s.update(
in_model_load = False,
preparation = None,
+ model_load_stage = None,
+ model_load_progress = None,
download_progress = None,
message = "Training...",
)
elif etype == "preparing":
+ s["model_load_stage"] = None
+ s["model_load_progress"] = None
# Long precompute such as the VAE latent cache, surfaced so the UI does not look stalled.
done, total = ev.get("done"), ev.get("total")
stage = str(ev.get("stage", "prepare")).replace("_", " ")
@@ -1189,6 +1223,8 @@ class DiffusionTrainingService:
status = "running",
step = ev.get("step", s["step"]),
preparation = None,
+ model_load_stage = None,
+ model_load_progress = None,
download_progress = None,
in_model_load = False,
total_steps = ev.get("total_steps", s["total_steps"]),
@@ -1229,6 +1265,8 @@ class DiffusionTrainingService:
in_model_load = False,
status = "stopped" if ev.get("stopped") else "completed",
preparation = None,
+ model_load_stage = None,
+ model_load_progress = None,
download_progress = None,
output_dir = ev.get("output_dir"),
lora_path = ev.get("lora_path"),
@@ -1264,6 +1302,8 @@ class DiffusionTrainingService:
in_model_load = False,
status = "error",
preparation = None,
+ model_load_stage = None,
+ model_load_progress = None,
download_progress = None,
message = str(ev.get("message", "error")),
)
diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py
index 0381f644812..1fb07245726 100644
--- a/studio/backend/models/training.py
+++ b/studio/backend/models/training.py
@@ -1222,6 +1222,13 @@ class DiffusionSampleImage(BaseModel):
seed: Optional[int] = None
+class DiffusionModelLoadProgress(BaseModel):
+ done: int = Field(ge = 0)
+ total: int = Field(gt = 0)
+ unit: Literal["tensors", "components", "shards"]
+ label: str
+
+
class DiffusionTrainingPreparation(BaseModel):
stage: Optional[str] = None
done: Optional[int] = None
@@ -1255,6 +1262,8 @@ class DiffusionTrainingStatusResponse(BaseModel):
num_images: Optional[int] = None
in_model_load: bool = False
preparation: Optional[DiffusionTrainingPreparation] = None
+ model_load_stage: Optional[Literal["conditioning", "transformer", "quantizing"]] = None
+ model_load_progress: Optional[DiffusionModelLoadProgress] = None
download_progress: Optional[DiffusionTrainingDownloadProgress] = None
output_dir: Optional[str] = None
lora_path: Optional[str] = None
diff --git a/studio/backend/tests/test_diffusion_load_progress.py b/studio/backend/tests/test_diffusion_load_progress.py
new file mode 100644
index 00000000000..0c6ee67215f
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_load_progress.py
@@ -0,0 +1,179 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import itertools
+import sys
+from types import ModuleType
+
+import pytest
+
+from core.training.diffusion_load_progress import report_model_load_progress
+
+
+def diffusers_available():
+ pytest.importorskip("diffusers")
+
+
+def test_real_sharded_model_counts_loaded_tensors_and_preserves_weights(
+ tmp_path, monkeypatch, diffusers_available
+):
+ import torch
+ from diffusers import FluxTransformer2DModel
+ from diffusers.models import model_loading_utils
+
+ ticks = itertools.count()
+ monkeypatch.setattr("core.training.diffusion_load_progress.time.monotonic", lambda: next(ticks))
+ model = FluxTransformer2DModel(
+ patch_size = 1,
+ in_channels = 4,
+ num_layers = 1,
+ num_single_layers = 1,
+ attention_head_dim = 8,
+ num_attention_heads = 2,
+ joint_attention_dim = 16,
+ pooled_projection_dim = 8,
+ axes_dims_rope = (2, 2, 4),
+ )
+ model.save_pretrained(tmp_path, max_shard_size = "5KB")
+ expected = model.state_dict()
+ events = []
+ original = model_loading_utils.load_model_dict_into_meta
+ with report_model_load_progress(events.append):
+ loaded = FluxTransformer2DModel.from_pretrained(tmp_path, low_cpu_mem_usage = True)
+ assert model_loading_utils.load_model_dict_into_meta is original
+ weights = [e for e in events if e["unit"] == "tensors"]
+ assert weights[0]["done"] == 0
+ assert weights[-1]["done"] == weights[-1]["total"] == len(expected)
+ assert {e["total"] for e in weights} == {len(expected)}
+ assert [e["done"] for e in weights] == sorted(e["done"] for e in weights)
+ assert any(0 < e["done"] < e["total"] for e in weights)
+ for key, value in expected.items():
+ assert torch.equal(loaded.state_dict()[key], value), key
+
+
[email protected]("fail", [False, True])
+def test_quantized_tensor_is_counted_only_after_success_and_hooks_restore(
+ fail, diffusers_available
+):
+ import torch
+ from diffusers.models import model_loading_utils as loading
+
+ model = torch.nn.Linear(3, 2, device = "meta")
+ values = torch.nn.Linear(3, 2).state_dict()
+ original_set, original_meta = (
+ loading.set_module_tensor_to_device,
+ loading.load_model_dict_into_meta,
+ )
+ events = []
+
+ class Quantizer:
+ pre_quantized = False
+
+ def check_if_quantized_param(self, model, value, name, state, **kwargs):
+ return name == "weight"
+
+ def create_quantized_param(self, model, value, name, device, *args, **kwargs):
+ assert events[-1]["done"] == 0 # No credit for merely starting conversion.
+ if fail:
+ raise RuntimeError("conversion failed")
+ original_set(model, name, device, value = value)
+
+ try:
+ with report_model_load_progress(events.append):
+ loading.load_model_dict_into_meta(
+ model,
+ values,
+ dtype = torch.float32,
+ device_map = {"": "cpu"},
+ hf_quantizer = Quantizer(),
+ )
+ except RuntimeError as exc:
+ assert fail and str(exc) == "conversion failed"
+ else:
+ assert not fail
+ assert loading.set_module_tensor_to_device is original_set
+ assert loading.load_model_dict_into_meta is original_meta
+ assert events[-1]["done"] == (0 if fail else 2)
+ assert events[-1]["total"] == 2
+
+
+def test_component_counter_works_when_library_bars_are_disabled(diffusers_available):
+ from diffusers.utils import logging
+
+ original = logging.tqdm
+ enabled = logging.is_progress_bar_enabled()
+ logging.disable_progress_bar()
+ events = []
+ try:
+ with report_model_load_progress(events.append):
+ bar = logging.tqdm(
+ ["encoder", "vae"], desc = "Loading pipeline components...", disable = True
+ )
+ iterator = iter(bar)
+ assert next(iterator) == "encoder"
+ assert events[-1]["done"] == 0
+ assert next(iterator) == "vae"
+ iterator.close() # The unfinished VAE must not be credited.
+ assert bar.n == 1
+ assert not any(e["done"] == 2 for e in events)
+ finally:
+ if enabled:
+ logging.enable_progress_bar()
+ assert logging.tqdm is original
+
+
+def test_no_callback_does_not_import_diffusers(monkeypatch):
+ monkeypatch.setitem(sys.modules, "diffusers.models", None)
+ with report_model_load_progress(None):
+ pass
+
+
+def test_unsupported_loader_api_falls_back_without_blocking_training(
+ monkeypatch, diffusers_available
+):
+ from diffusers.models import model_loading_utils as loading
+
+ original = loading.load_model_dict_into_meta
+ monkeypatch.delattr(loading, "set_module_tensor_to_device")
+ events = []
+ with report_model_load_progress(events.append):
+ assert loading.load_model_dict_into_meta is original
+ assert events == []
+
+
+ "missing", ["package", "stub", "model_loading_utils", "modeling_utils", "logging"]
+)
+def test_optional_diffusers_imports_do_not_block_training(monkeypatch, missing):
+ # Isolate from cached real submodules so the stub case also runs on ML hosts.
+ for name in tuple(sys.modules):
+ if name == "diffusers" or name.startswith("diffusers."):
+ monkeypatch.delitem(sys.modules, name)
+ diffusers = ModuleType("diffusers")
+ monkeypatch.setitem(sys.modules, "diffusers", None if missing == "package" else diffusers)
+ if missing not in ("package", "stub"):
+ diffusers.__path__ = []
+ for name, members in (
+ ("models", ("model_loading_utils", "modeling_utils")),
+ ("utils", ("logging",)),
+ ):
+ module = ModuleType(f"diffusers.{name}")
+ module.__path__ = []
+ monkeypatch.setitem(sys.modules, module.__name__, module)
+ setattr(diffusers, name, module)
+ for member in members:
+ if member != missing:
+ setattr(module, member, ModuleType(f"{module.__name__}.{member}"))
+
+ events = []
+ with report_model_load_progress(events.append):
+ events.append("loader ran")
+ assert events == ["loader ran"]
+
+ # Only observer imports are optional; failures from the loader must propagate.
+ failure = ImportError("model dependency missing")
+ with pytest.raises(ImportError) as caught:
+ with report_model_load_progress(events.append):
+ raise failure
+ assert caught.value is failure
diff --git a/studio/backend/tests/test_diffusion_training.py b/studio/backend/tests/test_diffusion_training.py
index a05e1e1f74e..55e7a3dd925 100644
--- a/studio/backend/tests/test_diffusion_training.py
+++ b/studio/backend/tests/test_diffusion_training.py
@@ -2653,3 +2653,91 @@ def test_model_download_progress_rejects_unknown_byte_counts(progress):
svc._apply_event({"type": "model_load_started"})
svc._apply_event({"type": "model_download_progress", "download_progress": progress})
assert svc.status()["download_progress"] is None
+
+
+def test_model_loading_stage_follows_phase_boundaries():
+ svc = DiffusionTrainingService()
+ assert svc.status()["model_load_stage"] is None
+ svc._apply_event({"type": "model_load_started", "stage": "conditioning"})
+ assert svc.status()["model_load_stage"] == "conditioning"
+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 5, "total": 5})
+ assert svc.status()["model_load_stage"] is None
+ svc._apply_event({"type": "model_load_started", "stage": "quantizing"})
+ assert svc.status()["model_load_stage"] == "quantizing"
+ assert svc.status()["preparation"] is None
+ svc._apply_event({"type": "model_load_completed"})
+ assert svc.status()["model_load_stage"] is None
+ assert svc.status()["in_model_load"] is False
+ svc._apply_event({"type": "model_load_started", "stage": "transformer"})
+ assert svc.status()["model_load_stage"] == "transformer"
+ # Trainers that do not emit a stage retain the generic loading fallback.
+ svc._apply_event({"type": "model_load_started"})
+ assert svc.status()["model_load_stage"] is None
+
+
+def test_route_preserves_measured_loading_and_preparation_fields(client):
+ svc = DiffusionTrainingService()
+ client._fake.status = svc.status
+ svc._apply_event({"type": "model_load_started", "stage": "quantizing"})
+ svc._apply_event(
+ {
+ "type": "model_load_progress",
+ "done": 12,
+ "total": 40,
+ "unit": "tensors",
+ "label": "FluxTransformer2DModel",
+ }
+ )
+ response = client.get("/api/train/diffusion/status")
+ assert response.status_code == 200
+ status = response.json()
+ assert status["model_load_stage"] == "quantizing"
+ assert status["model_load_progress"] == {
+ "done": 12,
+ "total": 40,
+ "unit": "tensors",
+ "label": "FluxTransformer2DModel",
+ }
+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 2, "total": 5})
+ status = client.get("/api/train/diffusion/status").json()
+ assert status["model_load_progress"] is None
+ assert status["preparation"] == {"stage": "cache_latents", "done": 2, "total": 5}
+ svc._apply_event({"type": "model_load_started", "stage": "transformer"})
+ status = client.get("/api/train/diffusion/status").json()
+ assert status["preparation"] is None
+ assert status["model_load_progress"] is None
+ svc._apply_event({"type": "model_load_completed"})
+ assert client.get("/api/train/diffusion/status").json()["model_load_stage"] is None
+
+
+ "done,total,unit",
+ [
+ (-1, 2, "tensors"),
+ (3, 2, "tensors"),
+ (0, 0, "tensors"),
+ (1, 2, "seconds"),
+ (float("nan"), 2, "tensors"),
+ ],
+)
+def test_invalid_model_load_counts_are_ignored(done, total, unit):
+ svc = DiffusionTrainingService()
+ svc._apply_event({"type": "model_load_started"})
+ svc._apply_event({"type": "model_load_progress", "done": done, "total": total, "unit": unit})
+ assert svc.status()["model_load_progress"] is None
+
+
+ "terminal", ["complete", "error", "progress", "preparing", "model_load_completed"]
+)
+def test_measured_loading_clears_when_leaving_load_phase(terminal):
+ svc = DiffusionTrainingService()
+ svc._apply_event({"type": "model_load_started", "stage": "quantizing"})
+ event = {"type": "model_load_progress", "done": 1, "total": 2, "unit": "tensors"}
+ svc._apply_event(event)
+ assert svc.status()["model_load_progress"]["done"] == 1
+ svc._apply_event({"type": terminal})
+ assert svc.status()["model_load_progress"] is None
+ assert svc.status()["model_load_stage"] is None
+ svc._apply_event(event)
+ assert svc.status()["model_load_progress"] is None
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts
index e9451414633..f9e118c6dea 100644
--- a/studio/frontend/src/features/images/api.ts
+++ b/studio/frontend/src/features/images/api.ts
@@ -688,6 +688,13 @@ export interface DiffusionTrainingStatus {
grad_norm?: number | null;
num_images: number | null;
in_model_load: boolean;
+ model_load_stage?: "conditioning" | "transformer" | "quantizing" | null;
+ model_load_progress?: {
+ done: number;
+ total: number;
+ unit: "tensors" | "components" | "shards";
+ label: string;
+ } | null;
preparation?: { stage?: string | null; done?: number | null; total?: number | null } | null;
download_progress?: { bytes_done: number; bytes_total: number; label?: string; scope?: "active_files" } | null;
output_dir: string | null;
diff --git a/studio/frontend/src/features/images/train/diffusion-train-progress.ts b/studio/frontend/src/features/images/train/diffusion-train-progress.ts
index 49544cf60af..5b81d1e3b49 100644
--- a/studio/frontend/src/features/images/train/diffusion-train-progress.ts
+++ b/studio/frontend/src/features/images/train/diffusion-train-progress.ts
@@ -13,8 +13,13 @@ type Status = Pick<
| "message"
| "preparation"
| "download_progress"
+ | "model_load_stage"
+ | "model_load_progress"
>;
+const QUANTIZATION_NOTE =
+ "Downloaded files are reused; 4-bit conversion runs in memory each time.";
+
const PHASE_MESSAGE =
/^(?:Loading base model\.\.\.|Training\.\.\.|Preparing \(.*\)\.\.\.)$/;
const TERMINAL_STATUSES = new Set(["completed", "stopped", "error"]);
@@ -78,6 +83,30 @@ export function diffusionTrainingProgress(status: Status | null) {
: null,
};
}
+ const load = status?.model_load_progress;
+ const stage = status?.model_load_stage;
+ // A retained tensor count is stale while the download manager shows a transfer.
+ const downloading = status?.download_progress != null;
+ const measured = status?.in_model_load && !downloading && load &&
+ Number.isFinite(load.done) && Number.isFinite(load.total) &&
+ load.done >= 0 && load.total > 0 && load.done <= load.total;
+ if (status?.in_model_load && !downloading && (measured || stage)) {
+ return {
+ preparing: true,
+ title: stage === "quantizing" ? "Loading model and converting to 4-bit"
+ : stage === "conditioning" ? "Loading text encoders and image encoder"
+ : "Loading training model weights",
+ detail: note ?? (measured
+ ? stage === "quantizing"
+ ? `${load.done.toLocaleString()} of ${load.total.toLocaleString()} ${load.unit === "tensors" ? "weight tensors" : load.unit} processed. ${QUANTIZATION_NOTE}`
+ : `${load.label} · ${load.done.toLocaleString()} / ${load.total.toLocaleString()} ${load.unit} loaded.`
+ : stage === "quantizing"
+ ? QUANTIZATION_NOTE
+ : "Waiting for the model loader to report progress. Training has not started yet."),
+ count: measured ? `${Math.floor(load.done / load.total * 100)}%` : null,
+ percent: measured ? load.done / load.total * 100 : null,
+ };
+ }
return {
preparing: true,
title: status?.in_model_load
diff --git a/studio/frontend/tests/diffusion-train-progress.test.ts b/studio/frontend/tests/diffusion-train-progress.test.ts
index 81c5dd25da4..baeaf7d8f4c 100644
--- a/studio/frontend/tests/diffusion-train-progress.test.ts
+++ b/studio/frontend/tests/diffusion-train-progress.test.ts
@@ -183,3 +183,43 @@ test("unknown download totals never fabricate a percentage", () => {
download_progress: { bytes_done: 1, bytes_total: total } }).percent, null);
}
});
+
+
+test("successful tensor loading gives measured progress and explains 4-bit conversion", () => {
+ const progress = diffusionTrainingProgress({ ...running,
+ model_load_stage: "quantizing",
+ model_load_progress: { done: 28, total: 62, unit: "tensors", label: "Training model weights" },
+ });
+ assert.equal(progress.title, "Loading model and converting to 4-bit");
+ assert.equal(progress.percent, 28 / 62 * 100);
+ assert.equal(progress.count, "45%");
+ assert.equal(progress.detail,
+ "28 of 62 weight tensors processed. Downloaded files are reused; 4-bit conversion runs in memory each time.");
+});
+
+test("loading counts yield to downloads, preparation, training and terminal states", () => {
+ const loaded = { ...running, model_load_stage: "quantizing" as const,
+ model_load_progress: { done: 28, total: 62, unit: "tensors" as const, label: "Weights" },
+ };
+ assert.equal(diffusionTrainingProgress({ ...loaded,
+ download_progress: { bytes_done: 1, bytes_total: 10 },
+ }).percent, null);
+ assert.equal(diffusionTrainingProgress({ ...loaded,
+ preparation: { stage: "cache_latents", done: 2, total: 5 },
+ }).percent, 40);
+ assert.equal(diffusionTrainingProgress({ ...loaded, in_model_load: false, step: 10 }).percent, 10);
+ assert.equal(diffusionTrainingProgress({ ...loaded, status: "stopped" }).preparing, false);
+});
+
+test("unmeasured and invalid loader counts never fabricate a percentage or hide warnings", () => {
+ for (const [done, total] of [[0, 0], [-1, 20], [21, 20], [NaN, 20], [1, Infinity]]) {
+ const progress = diffusionTrainingProgress({ ...running, model_load_stage: "conditioning",
+ model_load_progress: { done, total, unit: "tensors", label: "Weights" },
+ });
+ assert.equal(progress.percent, null);
+ }
+ const message = "Cancel requested; finishing the current step...";
+ assert.equal(diffusionTrainingProgress({ ...running, message,
+ model_load_stage: "quantizing",
+ }).detail, message);
+});