Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Report measured model loading and 4-bit conversion during image training (#15536)
Lee Jacksonb48b16e28c8b
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.pyindex f5794a61baf..6274b36fd54 100644--- a/studio/backend/core/training/diffusion_dit_trainer.py+++ b/studio/backend/core/training/diffusion_dit_trainer.py@@ -37,6 +37,7 @@ from dataclasses import dataclass, replacefrom pathlib import Pathfrom typing import Any, Callable, Optional+from core.training.diffusion_load_progress import report_model_load_progressfrom core._torchao_stub import is_stubbed, torch_is_rocmfrom core.training.diffusion_train_common import (AUTO_FLOW_SHIFT_FAMILIES,@@ -2395,7 +2396,11 @@ def run_dit_lora_training(cfg.resume_from_checkpoint, identity = identity, target_steps = cfg.train_steps)_emit(- on_event, "model_load_started", num_images = len(pairs), fetch_repo = training_fetch_repo(cfg)+ on_event,+ "model_load_started",+ stage = "conditioning",+ num_images = len(pairs),+ fetch_repo = training_fetch_repo(cfg),)if _check_stop():out_dir = Path(cfg.output_dir).expanduser()@@ -2512,15 +2517,22 @@ def _train_dit(if caption_embeds is not None and sample_texts:# Warm conditioning skips the VAE and encoders, but previews need the VAE and their own embeddings. RNG-isolated:# a run without previews never makes this load, so it must not move the training streams.- _emit(on_event, "model_load_started", fetch_repo = training_fetch_repo(cfg))+ _emit(+ on_event,+ "model_load_started",+ stage = "conditioning",+ fetch_repo = training_fetch_repo(cfg),+ )with isolated_sampling(device, False):- pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)+ with report_model_load_progress(on_event):+ pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)encoded = _encode_prompts_cached(spec, pipe, sample_texts, device, pcache)sample_embeds = dict(zip(sample_texts, encoded))_free_text_encoders(pipe)if caption_embeds is None:latent_cache = None- pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)+ with report_model_load_progress(on_event):+ pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)extra = [t for t in sample_texts if t not in to_encode]encoded = _encode_prompts_cached(spec, pipe, to_encode + extra, device, pcache)caption_embeds = {cap: emb for cap, emb in zip(to_encode, encoded)}@@ -2586,9 +2598,17 @@ def _train_dit(variant_rng = random.Random(cfg.seed + 1)# Transformer loads last, at resolved precision: nf4 by default, 'auto' picks from free VRAM.- _emit(on_event, "model_load_started", fetch_repo = training_fetch_repo(cfg))base_precision = _resolve_base_precision(cfg, spec, device)- transformer = spec.load_transformer(fetch_cfg, device, weight_dtype, base_precision)+ _emit(+ on_event,+ "model_load_started",+ stage = "quantizing"+ if base_precision == "nf4" and not repo_is_prequantized(fetch_cfg.base_model)+ else "transformer",+ fetch_repo = training_fetch_repo(cfg),+ )+ with report_model_load_progress(on_event):+ transformer = spec.load_transformer(fetch_cfg, device, weight_dtype, base_precision)base_is_bnb = base_precision == "nf4"transformer.requires_grad_(False)diff --git a/studio/backend/core/training/diffusion_load_progress.py b/studio/backend/core/training/diffusion_load_progress.pynew file mode 100644index 00000000000..e7a36e9878b--- /dev/null+++ b/studio/backend/core/training/diffusion_load_progress.py@@ -0,0 +1,173 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Loader progress for the spawned diffusion training worker: diffusers has no per-weight callback, so observe+tensor placement / quantizer calls and library tqdm bars, crediting only successful returns; hooks always restored."""++from contextlib import contextmanager+from functools import wraps+import inspect+import threading+import time+from weakref import WeakKeyDictionary+++@contextmanager+def report_model_load_progress(on_event):+ if on_event is None:+ yield+ return++ try:+ from diffusers.models import model_loading_utils as loading+ from diffusers.models import modeling_utils+ from diffusers.utils import logging as diffusion_logging+ except ImportError:+ yield+ return+ from transformers.utils import logging as transformer_logging++ try:+ from transformers import core_model_loading+ except ImportError:+ core_model_loading = None+ from tqdm.auto import tqdm++ lock = threading.RLock()+ counters = WeakKeyDictionary()+ last = [None, 0.0]++ def model_label(model):+ return (+ "Image encoder weights"+ if "Autoencoder" in type(model).__name__+ else "Training model weights"+ )++ def emit(done, total, unit, label):+ if not total or total <= 0:+ return+ with lock:+ identity = (unit, label, total)+ now = time.monotonic()+ if identity == last[0] and done not in (0, total) and now - last[1] < 0.2:+ return+ last[:] = [identity, now]+ on_event(+ dict(type = "model_load_progress", done = done, total = total, unit = unit, label = label)+ )++ def completed(model, name):+ with lock:+ counter = counters.get(model)+ if counter is None or name not in counter[0]:+ return+ counter[1].add(name)+ emit(len(counter[1]), len(counter[0]), "tensors", model_label(model))++ original_set = getattr(loading, "set_module_tensor_to_device", None)+ original_meta = getattr(loading, "load_model_dict_into_meta", None)+ if not callable(original_set) or not callable(original_meta):+ yield+ return+ meta_signature = inspect.signature(original_meta)+ if "model" not in meta_signature.parameters:+ yield+ return++ @wraps(original_set)+ def place(module, tensor_name, *args, **kwargs):+ result = original_set(module, tensor_name, *args, **kwargs)+ completed(module, tensor_name)+ return result++ class QuantizerProgress:+ def __init__(self, quantizer):+ self.quantizer = quantizer++ def __getattr__(self, name):+ return getattr(self.quantizer, name)++ def create_quantized_param(self, model, value, name, *args, **kwargs):+ result = self.quantizer.create_quantized_param(model, value, name, *args, **kwargs)+ completed(model, name)+ return result++ @wraps(original_meta)+ def load_meta(*args, **kwargs):+ bound = meta_signature.bind(*args, **kwargs)+ model = bound.arguments["model"]+ with lock:+ if model not in counters:+ # Snapshot names before quantization adds checkpoint metadata to state_dict.+ counters[model] = (set(model.state_dict()), set())+ names, done = counters[model]+ emit(len(done), len(names), "tensors", model_label(model))+ quantizer = bound.arguments.get("hf_quantizer")+ if quantizer is not None:+ bound.arguments["hf_quantizer"] = QuantizerProgress(quantizer)+ return original_meta(*bound.args, **bound.kwargs)++ class CountedTqdm(tqdm):+ def __init__(self, *args, unit, label, **kwargs):+ self.progress_unit, self.progress_label = unit, label+ # Count even when quiet logging disabled the bar; display() is muted and leave=False stops close() writing "\n".+ kwargs["disable"], kwargs["leave"] = False, False+ super().__init__(*args, **kwargs)+ emit(self.n, self.total, unit, label)++ def display(self, *args, **kwargs):+ pass++ def update(self, n = 1):+ result = super().update(n)+ emit(self.n, self.total, self.progress_unit, self.progress_label)+ return result++ def __iter__(self):+ try:+ for item in self.iterable:+ yield item+ # Credited on resume: an item whose load raised is never counted.+ self.update(1)+ finally:+ self.close()++ class ProgressFactory:+ def __init__(self, original):+ self.original = original++ def __getattr__(self, name):+ return getattr(self.original, name)++ def __call__(self, *args, **kwargs):+ desc = kwargs.get("desc", "")+ if desc.startswith("Loading pipeline components"):+ return CountedTqdm(*args, unit = "components", label = "Model components", **kwargs)+ if desc == "Loading weights":+ return CountedTqdm(*args, unit = "tensors", label = "Text encoder weights", **kwargs)+ if desc == "Loading checkpoint shards":+ return CountedTqdm(*args, unit = "shards", label = "Checkpoint shards", **kwargs)+ return self.original(*args, **kwargs)++ patched = []++ def patch(module, name, replacement):+ original = getattr(module, name)+ patched.append((module, name, original))+ setattr(module, name, replacement)++ try:+ patch(loading, "set_module_tensor_to_device", place)+ patch(loading, "load_model_dict_into_meta", load_meta)+ if getattr(modeling_utils, "load_model_dict_into_meta", None) is original_meta:+ patch(modeling_utils, "load_model_dict_into_meta", load_meta)+ patch(diffusion_logging, "tqdm", ProgressFactory(diffusion_logging.tqdm))+ patch(transformer_logging, "tqdm", ProgressFactory(transformer_logging.tqdm))+ # Transformers 5 imports tqdm by name.+ if core_model_loading is not None and hasattr(core_model_loading, "tqdm"):+ patch(core_model_loading, "tqdm", ProgressFactory(core_model_loading.tqdm))+ yield+ finally:+ for module, name, original in reversed(patched):+ setattr(module, name, original)diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.pyindex 19249afa5cc..883839a0e3b 100644--- a/studio/backend/core/training/diffusion_lora_trainer.py+++ b/studio/backend/core/training/diffusion_lora_trainer.py@@ -418,9 +418,12 @@ def run_diffusion_lora_training()return str(out_dir)- pipe = StableDiffusionXLPipeline.from_pretrained(- cfg.base_model, torch_dtype = weight_dtype, token = cfg.hf_token, add_watermarker = False- )+ from core.training.diffusion_load_progress import report_model_load_progress++ with report_model_load_progress(on_event):+ pipe = StableDiffusionXLPipeline.from_pretrained(+ cfg.base_model, torch_dtype = weight_dtype, token = cfg.hf_token, add_watermarker = False+ )unet, vae = pipe.unet, pipe.vaetokenizers = [pipe.tokenizer, pipe.tokenizer_2]text_encoders = [pipe.text_encoder, pipe.text_encoder_2]diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.pyindex d9c1414a385..9961cb41457 100644--- a/studio/backend/core/training/diffusion_training_service.py+++ b/studio/backend/core/training/diffusion_training_service.py@@ -359,6 +359,8 @@ def _idle_state() -> dict[str, Any]:"num_images": None,"in_model_load": False,"preparation": None,+ "model_load_stage": None,+ "model_load_progress": None,"download_progress": None,"output_dir": None,"lora_path": None,@@ -786,6 +788,8 @@ class DiffusionTrainingService:status = "stopped",in_model_load = False,preparation = None,+ model_load_stage = None,+ model_load_progress = None,download_progress = None,message = "Stopped while loading model files; training had not started.",updated_at = time.time(),@@ -919,6 +923,8 @@ class DiffusionTrainingService:# A worker killed mid-load / mid-preparation sends no terminal event to clear these.in_model_load = False,preparation = None,+ model_load_stage = None,+ model_load_progress = None,download_progress = None,message = self._unexpected_exit_message(proc),updated_at = time.time(),@@ -1089,12 +1095,36 @@ class DiffusionTrainingService:s.update(in_model_load = True,preparation = None,+ model_load_stage = None,+ model_load_progress = None,download_progress = None,status = "running",message = "Loading base model...",)+ s["model_load_stage"] = (+ ev.get("stage")+ if ev.get("stage") in ("conditioning", "transformer", "quantizing")+ else None+ )if ev.get("num_images") is not None:s["num_images"] = ev.get("num_images")+ elif etype == "model_load_progress":+ done, total = ev.get("done"), ev.get("total")+ if (+ s.get("in_model_load")+ and s.get("preparation") is None+ and type(done) is int+ and type(total) is int+ and 0 <= done <= total+ and total > 0+ and ev.get("unit") in ("tensors", "components", "shards")+ ):+ s["model_load_progress"] = {+ "done": done,+ "total": total,+ "unit": ev["unit"],+ "label": str(ev.get("label", "Model weights"))[:200],+ }elif etype == "model_download_progress":if s.get("in_model_load") and s.get("preparation") is None:progress = ev.get("download_progress")@@ -1115,10 +1145,14 @@ class DiffusionTrainingService:s.update(in_model_load = False,preparation = None,+ model_load_stage = None,+ model_load_progress = None,download_progress = None,message = "Training...",)elif etype == "preparing":+ s["model_load_stage"] = None+ s["model_load_progress"] = None# Long precompute such as the VAE latent cache, surfaced so the UI does not look stalled.done, total = ev.get("done"), ev.get("total")stage = str(ev.get("stage", "prepare")).replace("_", " ")@@ -1189,6 +1223,8 @@ class DiffusionTrainingService:status = "running",step = ev.get("step", s["step"]),preparation = None,+ model_load_stage = None,+ model_load_progress = None,download_progress = None,in_model_load = False,total_steps = ev.get("total_steps", s["total_steps"]),@@ -1229,6 +1265,8 @@ class DiffusionTrainingService:in_model_load = False,status = "stopped" if ev.get("stopped") else "completed",preparation = None,+ model_load_stage = None,+ model_load_progress = None,download_progress = None,output_dir = ev.get("output_dir"),lora_path = ev.get("lora_path"),@@ -1264,6 +1302,8 @@ class DiffusionTrainingService:in_model_load = False,status = "error",preparation = None,+ model_load_stage = None,+ model_load_progress = None,download_progress = None,message = str(ev.get("message", "error")),)diff --git a/studio/backend/models/training.py b/studio/backend/models/training.pyindex 0381f644812..1fb07245726 100644--- a/studio/backend/models/training.py+++ b/studio/backend/models/training.py@@ -1222,6 +1222,13 @@ class DiffusionSampleImage(BaseModel):seed: Optional[int] = None+class DiffusionModelLoadProgress(BaseModel):+ done: int = Field(ge = 0)+ total: int = Field(gt = 0)+ unit: Literal["tensors", "components", "shards"]+ label: str++class DiffusionTrainingPreparation(BaseModel):stage: Optional[str] = Nonedone: Optional[int] = None@@ -1255,6 +1262,8 @@ class DiffusionTrainingStatusResponse(BaseModel):num_images: Optional[int] = Nonein_model_load: bool = Falsepreparation: Optional[DiffusionTrainingPreparation] = None+ model_load_stage: Optional[Literal["conditioning", "transformer", "quantizing"]] = None+ model_load_progress: Optional[DiffusionModelLoadProgress] = Nonedownload_progress: Optional[DiffusionTrainingDownloadProgress] = Noneoutput_dir: Optional[str] = Nonelora_path: Optional[str] = Nonediff --git a/studio/backend/tests/test_diffusion_load_progress.py b/studio/backend/tests/test_diffusion_load_progress.pynew file mode 100644index 00000000000..0c6ee67215f--- /dev/null+++ b/studio/backend/tests/test_diffusion_load_progress.py@@ -0,0 +1,179 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import itertools+import sys+from types import ModuleType++import pytest++from core.training.diffusion_load_progress import report_model_load_progress+++def diffusers_available():+ pytest.importorskip("diffusers")+++def test_real_sharded_model_counts_loaded_tensors_and_preserves_weights(+ tmp_path, monkeypatch, diffusers_available+):+ import torch+ from diffusers import FluxTransformer2DModel+ from diffusers.models import model_loading_utils++ ticks = itertools.count()+ monkeypatch.setattr("core.training.diffusion_load_progress.time.monotonic", lambda: next(ticks))+ model = FluxTransformer2DModel(+ patch_size = 1,+ in_channels = 4,+ num_layers = 1,+ num_single_layers = 1,+ attention_head_dim = 8,+ num_attention_heads = 2,+ joint_attention_dim = 16,+ pooled_projection_dim = 8,+ axes_dims_rope = (2, 2, 4),+ )+ model.save_pretrained(tmp_path, max_shard_size = "5KB")+ expected = model.state_dict()+ events = []+ original = model_loading_utils.load_model_dict_into_meta+ with report_model_load_progress(events.append):+ loaded = FluxTransformer2DModel.from_pretrained(tmp_path, low_cpu_mem_usage = True)+ assert model_loading_utils.load_model_dict_into_meta is original+ weights = [e for e in events if e["unit"] == "tensors"]+ assert weights[0]["done"] == 0+ assert weights[-1]["done"] == weights[-1]["total"] == len(expected)+ assert {e["total"] for e in weights} == {len(expected)}+ assert [e["done"] for e in weights] == sorted(e["done"] for e in weights)+ assert any(0 < e["done"] < e["total"] for e in weights)+ for key, value in expected.items():+ assert torch.equal(loaded.state_dict()[key], value), key++[email protected]("fail", [False, True])+def test_quantized_tensor_is_counted_only_after_success_and_hooks_restore(+ fail, diffusers_available+):+ import torch+ from diffusers.models import model_loading_utils as loading++ model = torch.nn.Linear(3, 2, device = "meta")+ values = torch.nn.Linear(3, 2).state_dict()+ original_set, original_meta = (+ loading.set_module_tensor_to_device,+ loading.load_model_dict_into_meta,+ )+ events = []++ class Quantizer:+ pre_quantized = False++ def check_if_quantized_param(self, model, value, name, state, **kwargs):+ return name == "weight"++ def create_quantized_param(self, model, value, name, device, *args, **kwargs):+ assert events[-1]["done"] == 0 # No credit for merely starting conversion.+ if fail:+ raise RuntimeError("conversion failed")+ original_set(model, name, device, value = value)++ try:+ with report_model_load_progress(events.append):+ loading.load_model_dict_into_meta(+ model,+ values,+ dtype = torch.float32,+ device_map = {"": "cpu"},+ hf_quantizer = Quantizer(),+ )+ except RuntimeError as exc:+ assert fail and str(exc) == "conversion failed"+ else:+ assert not fail+ assert loading.set_module_tensor_to_device is original_set+ assert loading.load_model_dict_into_meta is original_meta+ assert events[-1]["done"] == (0 if fail else 2)+ assert events[-1]["total"] == 2+++def test_component_counter_works_when_library_bars_are_disabled(diffusers_available):+ from diffusers.utils import logging++ original = logging.tqdm+ enabled = logging.is_progress_bar_enabled()+ logging.disable_progress_bar()+ events = []+ try:+ with report_model_load_progress(events.append):+ bar = logging.tqdm(+ ["encoder", "vae"], desc = "Loading pipeline components...", disable = True+ )+ iterator = iter(bar)+ assert next(iterator) == "encoder"+ assert events[-1]["done"] == 0+ assert next(iterator) == "vae"+ iterator.close() # The unfinished VAE must not be credited.+ assert bar.n == 1+ assert not any(e["done"] == 2 for e in events)+ finally:+ if enabled:+ logging.enable_progress_bar()+ assert logging.tqdm is original+++def test_no_callback_does_not_import_diffusers(monkeypatch):+ monkeypatch.setitem(sys.modules, "diffusers.models", None)+ with report_model_load_progress(None):+ pass+++def test_unsupported_loader_api_falls_back_without_blocking_training(+ monkeypatch, diffusers_available+):+ from diffusers.models import model_loading_utils as loading++ original = loading.load_model_dict_into_meta+ monkeypatch.delattr(loading, "set_module_tensor_to_device")+ events = []+ with report_model_load_progress(events.append):+ assert loading.load_model_dict_into_meta is original+ assert events == []+++ "missing", ["package", "stub", "model_loading_utils", "modeling_utils", "logging"]+)+def test_optional_diffusers_imports_do_not_block_training(monkeypatch, missing):+ # Isolate from cached real submodules so the stub case also runs on ML hosts.+ for name in tuple(sys.modules):+ if name == "diffusers" or name.startswith("diffusers."):+ monkeypatch.delitem(sys.modules, name)+ diffusers = ModuleType("diffusers")+ monkeypatch.setitem(sys.modules, "diffusers", None if missing == "package" else diffusers)+ if missing not in ("package", "stub"):+ diffusers.__path__ = []+ for name, members in (+ ("models", ("model_loading_utils", "modeling_utils")),+ ("utils", ("logging",)),+ ):+ module = ModuleType(f"diffusers.{name}")+ module.__path__ = []+ monkeypatch.setitem(sys.modules, module.__name__, module)+ setattr(diffusers, name, module)+ for member in members:+ if member != missing:+ setattr(module, member, ModuleType(f"{module.__name__}.{member}"))++ events = []+ with report_model_load_progress(events.append):+ events.append("loader ran")+ assert events == ["loader ran"]++ # Only observer imports are optional; failures from the loader must propagate.+ failure = ImportError("model dependency missing")+ with pytest.raises(ImportError) as caught:+ with report_model_load_progress(events.append):+ raise failure+ assert caught.value is failurediff --git a/studio/backend/tests/test_diffusion_training.py b/studio/backend/tests/test_diffusion_training.pyindex a05e1e1f74e..55e7a3dd925 100644--- a/studio/backend/tests/test_diffusion_training.py+++ b/studio/backend/tests/test_diffusion_training.py@@ -2653,3 +2653,91 @@ def test_model_download_progress_rejects_unknown_byte_counts(progress):svc._apply_event({"type": "model_load_started"})svc._apply_event({"type": "model_download_progress", "download_progress": progress})assert svc.status()["download_progress"] is None+++def test_model_loading_stage_follows_phase_boundaries():+ svc = DiffusionTrainingService()+ assert svc.status()["model_load_stage"] is None+ svc._apply_event({"type": "model_load_started", "stage": "conditioning"})+ assert svc.status()["model_load_stage"] == "conditioning"+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 5, "total": 5})+ assert svc.status()["model_load_stage"] is None+ svc._apply_event({"type": "model_load_started", "stage": "quantizing"})+ assert svc.status()["model_load_stage"] == "quantizing"+ assert svc.status()["preparation"] is None+ svc._apply_event({"type": "model_load_completed"})+ assert svc.status()["model_load_stage"] is None+ assert svc.status()["in_model_load"] is False+ svc._apply_event({"type": "model_load_started", "stage": "transformer"})+ assert svc.status()["model_load_stage"] == "transformer"+ # Trainers that do not emit a stage retain the generic loading fallback.+ svc._apply_event({"type": "model_load_started"})+ assert svc.status()["model_load_stage"] is None+++def test_route_preserves_measured_loading_and_preparation_fields(client):+ svc = DiffusionTrainingService()+ client._fake.status = svc.status+ svc._apply_event({"type": "model_load_started", "stage": "quantizing"})+ svc._apply_event(+ {+ "type": "model_load_progress",+ "done": 12,+ "total": 40,+ "unit": "tensors",+ "label": "FluxTransformer2DModel",+ }+ )+ response = client.get("/api/train/diffusion/status")+ assert response.status_code == 200+ status = response.json()+ assert status["model_load_stage"] == "quantizing"+ assert status["model_load_progress"] == {+ "done": 12,+ "total": 40,+ "unit": "tensors",+ "label": "FluxTransformer2DModel",+ }+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 2, "total": 5})+ status = client.get("/api/train/diffusion/status").json()+ assert status["model_load_progress"] is None+ assert status["preparation"] == {"stage": "cache_latents", "done": 2, "total": 5}+ svc._apply_event({"type": "model_load_started", "stage": "transformer"})+ status = client.get("/api/train/diffusion/status").json()+ assert status["preparation"] is None+ assert status["model_load_progress"] is None+ svc._apply_event({"type": "model_load_completed"})+ assert client.get("/api/train/diffusion/status").json()["model_load_stage"] is None+++ "done,total,unit",+ [+ (-1, 2, "tensors"),+ (3, 2, "tensors"),+ (0, 0, "tensors"),+ (1, 2, "seconds"),+ (float("nan"), 2, "tensors"),+ ],+)+def test_invalid_model_load_counts_are_ignored(done, total, unit):+ svc = DiffusionTrainingService()+ svc._apply_event({"type": "model_load_started"})+ svc._apply_event({"type": "model_load_progress", "done": done, "total": total, "unit": unit})+ assert svc.status()["model_load_progress"] is None+++ "terminal", ["complete", "error", "progress", "preparing", "model_load_completed"]+)+def test_measured_loading_clears_when_leaving_load_phase(terminal):+ svc = DiffusionTrainingService()+ svc._apply_event({"type": "model_load_started", "stage": "quantizing"})+ event = {"type": "model_load_progress", "done": 1, "total": 2, "unit": "tensors"}+ svc._apply_event(event)+ assert svc.status()["model_load_progress"]["done"] == 1+ svc._apply_event({"type": terminal})+ assert svc.status()["model_load_progress"] is None+ assert svc.status()["model_load_stage"] is None+ svc._apply_event(event)+ assert svc.status()["model_load_progress"] is Nonediff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.tsindex e9451414633..f9e118c6dea 100644--- a/studio/frontend/src/features/images/api.ts+++ b/studio/frontend/src/features/images/api.ts@@ -688,6 +688,13 @@ export interface DiffusionTrainingStatus {grad_norm?: number | null;num_images: number | null;in_model_load: boolean;+ model_load_stage?: "conditioning" | "transformer" | "quantizing" | null;+ model_load_progress?: {+ done: number;+ total: number;+ unit: "tensors" | "components" | "shards";+ label: string;+ } | null;preparation?: { stage?: string | null; done?: number | null; total?: number | null } | null;download_progress?: { bytes_done: number; bytes_total: number; label?: string; scope?: "active_files" } | null;output_dir: string | null;diff --git a/studio/frontend/src/features/images/train/diffusion-train-progress.ts b/studio/frontend/src/features/images/train/diffusion-train-progress.tsindex 49544cf60af..5b81d1e3b49 100644--- a/studio/frontend/src/features/images/train/diffusion-train-progress.ts+++ b/studio/frontend/src/features/images/train/diffusion-train-progress.ts@@ -13,8 +13,13 @@ type Status = Pick<| "message"| "preparation"| "download_progress"+ | "model_load_stage"+ | "model_load_progress">;+const QUANTIZATION_NOTE =+ "Downloaded files are reused; 4-bit conversion runs in memory each time.";+const PHASE_MESSAGE =/^(?:Loading base model\.\.\.|Training\.\.\.|Preparing \(.*\)\.\.\.)$/;const TERMINAL_STATUSES = new Set(["completed", "stopped", "error"]);@@ -78,6 +83,30 @@ export function diffusionTrainingProgress(status: Status | null) {: null,};}+ const load = status?.model_load_progress;+ const stage = status?.model_load_stage;+ // A retained tensor count is stale while the download manager shows a transfer.+ const downloading = status?.download_progress != null;+ const measured = status?.in_model_load && !downloading && load &&+ Number.isFinite(load.done) && Number.isFinite(load.total) &&+ load.done >= 0 && load.total > 0 && load.done <= load.total;+ if (status?.in_model_load && !downloading && (measured || stage)) {+ return {+ preparing: true,+ title: stage === "quantizing" ? "Loading model and converting to 4-bit"+ : stage === "conditioning" ? "Loading text encoders and image encoder"+ : "Loading training model weights",+ detail: note ?? (measured+ ? stage === "quantizing"+ ? `${load.done.toLocaleString()} of ${load.total.toLocaleString()} ${load.unit === "tensors" ? "weight tensors" : load.unit} processed. ${QUANTIZATION_NOTE}`+ : `${load.label} · ${load.done.toLocaleString()} / ${load.total.toLocaleString()} ${load.unit} loaded.`+ : stage === "quantizing"+ ? QUANTIZATION_NOTE+ : "Waiting for the model loader to report progress. Training has not started yet."),+ count: measured ? `${Math.floor(load.done / load.total * 100)}%` : null,+ percent: measured ? load.done / load.total * 100 : null,+ };+ }return {preparing: true,title: status?.in_model_loaddiff --git a/studio/frontend/tests/diffusion-train-progress.test.ts b/studio/frontend/tests/diffusion-train-progress.test.tsindex 81c5dd25da4..baeaf7d8f4c 100644--- a/studio/frontend/tests/diffusion-train-progress.test.ts+++ b/studio/frontend/tests/diffusion-train-progress.test.ts@@ -183,3 +183,43 @@ test("unknown download totals never fabricate a percentage", () => {download_progress: { bytes_done: 1, bytes_total: total } }).percent, null);}});+++test("successful tensor loading gives measured progress and explains 4-bit conversion", () => {+ const progress = diffusionTrainingProgress({ ...running,+ model_load_stage: "quantizing",+ model_load_progress: { done: 28, total: 62, unit: "tensors", label: "Training model weights" },+ });+ assert.equal(progress.title, "Loading model and converting to 4-bit");+ assert.equal(progress.percent, 28 / 62 * 100);+ assert.equal(progress.count, "45%");+ assert.equal(progress.detail,+ "28 of 62 weight tensors processed. Downloaded files are reused; 4-bit conversion runs in memory each time.");+});++test("loading counts yield to downloads, preparation, training and terminal states", () => {+ const loaded = { ...running, model_load_stage: "quantizing" as const,+ model_load_progress: { done: 28, total: 62, unit: "tensors" as const, label: "Weights" },+ };+ assert.equal(diffusionTrainingProgress({ ...loaded,+ download_progress: { bytes_done: 1, bytes_total: 10 },+ }).percent, null);+ assert.equal(diffusionTrainingProgress({ ...loaded,+ preparation: { stage: "cache_latents", done: 2, total: 5 },+ }).percent, 40);+ assert.equal(diffusionTrainingProgress({ ...loaded, in_model_load: false, step: 10 }).percent, 10);+ assert.equal(diffusionTrainingProgress({ ...loaded, status: "stopped" }).preparing, false);+});++test("unmeasured and invalid loader counts never fabricate a percentage or hide warnings", () => {+ for (const [done, total] of [[0, 0], [-1, 20], [21, 20], [NaN, 20], [1, Infinity]]) {+ const progress = diffusionTrainingProgress({ ...running, model_load_stage: "conditioning",+ model_load_progress: { done, total, unit: "tensors", label: "Weights" },+ });+ assert.equal(progress.percent, null);+ }+ const message = "Cancel requested; finishing the current step...";+ assert.equal(diffusionTrainingProgress({ ...running, message,+ model_load_stage: "quantizing",+ }).detail, message);+});