Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: use shared download manager for diffusion training (#15476)

Daniel Hanc3df55b602f1
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index add1cf39964..c964414cdb4 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -54,6 +54,7 @@ from core.training.diffusion_train_common import (
_apply_perf_flags,
_assert_trusted_base_model,
_emit,
+ training_fetch_repo,
_latent_cache_forced,
_latent_cache_over_budget,
_plan_cache_variants,
@@ -2393,7 +2394,9 @@ def run_dit_lora_training(
preflight_resume(
cfg.resume_from_checkpoint, identity = identity, target_steps = cfg.train_steps
)
- _emit(on_event, "model_load_started", num_images = len(pairs))
+ _emit(
+ on_event, "model_load_started", num_images = len(pairs), fetch_repo = training_fetch_repo(cfg)
+ )
if _check_stop():
out_dir = Path(cfg.output_dir).expanduser()
_emit(
@@ -2509,6 +2512,7 @@ def _train_dit(
if caption_embeds is not None and sample_texts:
# Warm conditioning skips the VAE and encoders, but previews need the VAE and their own embeddings. RNG-isolated:
# a run without previews never makes this load, so it must not move the training streams.
+ _emit(on_event, "model_load_started", fetch_repo = training_fetch_repo(cfg))
with isolated_sampling(device, False):
pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)
encoded = _encode_prompts_cached(spec, pipe, sample_texts, device, pcache)
@@ -2582,6 +2586,7 @@ def _train_dit(
variant_rng = random.Random(cfg.seed + 1)
# Transformer loads last, at resolved precision: nf4 by default, 'auto' picks from free VRAM.
+ _emit(on_event, "model_load_started", fetch_repo = training_fetch_repo(cfg))
base_precision = _resolve_base_precision(cfg, spec, device)
transformer = spec.load_transformer(fetch_cfg, device, weight_dtype, base_precision)
base_is_bnb = base_precision == "nf4"
diff --git a/studio/backend/core/training/diffusion_download_progress.py b/studio/backend/core/training/diffusion_download_progress.py
new file mode 100644
index 00000000000..40554550ead
--- /dev/null
+++ b/studio/backend/core/training/diffusion_download_progress.py
@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Observe only this training child's active byte bars; never scan unrelated cached repositories."""
+
+from __future__ import annotations
+
+import math
+import threading
+from typing import Any, Callable, Iterable
+
+from utils.account_context import account_thread
+
+
+def _bytes(value: Any) -> int | None:
+ try:
+ return int(value) if math.isfinite(float(value)) and float(value) >= 0 else None
+ except (TypeError, ValueError, OverflowError):
+ return None
+
+
+def download_snapshot(bars: Iterable[Any]) -> dict[str, Any] | None:
+ entries = []
+ for bar in bars:
+ if getattr(bar, "disable", False) or getattr(bar, "unit", None) != "B":
+ continue
+ done, total = _bytes(getattr(bar, "n", None)), _bytes(getattr(bar, "total", None))
+ if done is None or total is None or total <= 0:
+ continue
+ label = str(getattr(bar, "desc", "") or "Model files")[:200]
+ entries.append((label, min(done, total), total))
+ if not entries:
+ return None
+ # Network and reconstruction bars cover the SAME bytes: count only reconstruction.
+ reconstructed = [entry for entry in entries if "reconstruct" in entry[0].lower()]
+ transfers = [entry for entry in entries if entry[0].lower() == "downloading bytes"]
+ selected = reconstructed or transfers or entries
+ return {
+ "bytes_done": sum(entry[1] for entry in selected),
+ "bytes_total": sum(entry[2] for entry in selected),
+ "label": selected[0][0] if len(selected) == 1 else f"{len(selected)} model files",
+ "scope": "active_files",
+ }
+
+
+def _active_downloads() -> dict[str, Any] | None:
+ from tqdm.std import tqdm
+ with tqdm.get_lock():
+ bars = list(tqdm._instances)
+ return download_snapshot(bars)
+
+
+class TrainingDownloadProgress:
+ """Forward trainer events, adding byte progress only during its model-loading phase."""
+
+ def __init__(
+ self,
+ emit: Callable[[dict], None],
+ sample = _active_downloads,
+ interval: float = 0.5,
+ ):
+ self._emit = emit
+ self._sample = sample
+ self._interval = interval
+ self._lock = threading.Lock()
+ self._stop = threading.Event()
+ self._active = False
+ self._generation = 0
+ self._last = None
+ self._thread = None
+
+ def report(self, event: dict) -> None:
+ with self._lock:
+ kind = event.get("type")
+ if kind == "model_load_started":
+ self._active = True
+ self._generation += 1
+ self._last = None
+ elif kind in ("model_load_completed", "preparing", "progress", "complete", "error"):
+ self._active = False
+ self._generation += 1
+ self._last = None
+ self._emit(event)
+
+ def poll(self) -> None:
+ with self._lock:
+ if not self._active:
+ return
+ generation = self._generation
+ progress = self._sample()
+ with self._lock:
+ if not self._active or generation != self._generation or progress == self._last:
+ return
+ self._last = progress
+ self._emit({"type": "model_download_progress", "download_progress": progress})
+
+ def _run(self) -> None:
+ while not self._stop.wait(self._interval):
+ try:
+ self.poll()
+ except Exception: # noqa: BLE001 -- progress must never stop a training run
+ pass
+
+ def __enter__(self):
+ self._thread = account_thread(target = self._run, daemon = True)
+ self._thread.start()
+ return self.report
+
+ def __exit__(self, *_exc):
+ with self._lock:
+ self._active = False
+ self._generation += 1
+ self._stop.set()
+ self._thread.join(timeout = 2)
diff --git a/studio/backend/core/training/diffusion_h3_trainer.py b/studio/backend/core/training/diffusion_h3_trainer.py
index ef63fb8a770..a222e395b5e 100644
--- a/studio/backend/core/training/diffusion_h3_trainer.py
+++ b/studio/backend/core/training/diffusion_h3_trainer.py
@@ -78,6 +78,7 @@ from core.training.diffusion_train_common import (
_apply_perf_flags,
_assert_trusted_base_model,
_emit,
+ training_fetch_repo,
_latent_cache_forced,
_latent_cache_over_budget,
_restore_perf_flags,
@@ -437,7 +438,9 @@ def run_h3_lora_training(
cfg.data_dir, instance_prompt = cfg.instance_prompt, caption_column = cfg.caption_column
)
cfg = replace(cfg, train_steps = resolve_train_steps(cfg, len(pairs)), num_epochs = 0)
- _emit(on_event, "model_load_started", num_images = len(pairs))
+ _emit(
+ on_event, "model_load_started", num_images = len(pairs), fetch_repo = training_fetch_repo(cfg)
+ )
if _check_stop():
out_dir = Path(cfg.output_dir).expanduser()
_emit(
@@ -553,6 +556,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
"through one set of linears, so the activation range is not the per-family range "
"the fp8 filter was measured against. Use nf4, int8 or bf16."
)
+ _emit(on_event, "model_load_started", fetch_repo = training_fetch_repo(cfg))
transformer = _load_transformer(cfg, device, base_precision)
transformer.requires_grad_(False)
# DEFAULT_LORA_TARGETS here means "unset", so the family's own regex wins.
diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.py
index b3b58995e1a..19249afa5cc 100644
--- a/studio/backend/core/training/diffusion_lora_trainer.py
+++ b/studio/backend/core/training/diffusion_lora_trainer.py
@@ -56,6 +56,7 @@ from core.training.diffusion_train_common import ( # noqa: F401
_config_from_dict,
_CONFIG_ALIASES,
_emit,
+ training_fetch_repo,
resolve_train_device,
_latent_cache_forced,
_latent_cache_over_budget,
@@ -395,7 +396,12 @@ def run_diffusion_lora_training(
"GPUs only, so expect slow steps and previews."
),
)
- _emit(on_event, "model_load_started", num_images = len(pairs))
+ _emit(
+ on_event,
+ "model_load_started",
+ num_images = len(pairs),
+ fetch_repo = training_fetch_repo(cfg),
+ )
# Honour a stop requested before the (slow) base model load.
if _check_stop():
@@ -956,7 +962,10 @@ def run_diffusion_training_process(*, event_queue: Any, stop_queue: Any, config:
# normalized() validates the family; the registry sends DiT families to their own trainer.
cfg = _config_from_dict(config).normalized()
trainer = get_trainer(cfg.resolved_family)
- trainer(cfg, on_event = on_event, should_stop = should_stop)
+ from .diffusion_download_progress import TrainingDownloadProgress
+
+ with TrainingDownloadProgress(on_event) as report:
+ trainer(cfg, on_event = report, should_stop = should_stop)
except Exception as exc: # noqa: BLE001 -- surfaced to the parent as an error event
# Emit both keys: the diffusion service reads ``message``, the generic worker reads ``error``.
event_queue.put(
diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.py
index 0634ea15b8f..43a5c5e5882 100644
--- a/studio/backend/core/training/diffusion_train_common.py
+++ b/studio/backend/core/training/diffusion_train_common.py
@@ -1632,6 +1632,14 @@ def discover_training_pairs(
)
+def training_fetch_repo(cfg: Any) -> Optional[str]:
+ """The repository the trainer downloads: DiT families load the resolved mirror, others the base model."""
+ family = (getattr(cfg, "resolved_family", None) or "sdxl").strip().lower()
+ if family in _DIT_TRAIN_FAMILIES:
+ return getattr(cfg, "fetch_base_model", None) or cfg.base_model
+ return cfg.base_model
+
+
def _emit(on_event: Optional[EventCb], type_: str, **kw: Any) -> None:
if on_event is not None:
on_event({"type": type_, "ts": time.time(), **kw})
diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.py
index 28f7ae49b6c..9ac2211eb0b 100644
--- a/studio/backend/core/training/diffusion_training_service.py
+++ b/studio/backend/core/training/diffusion_training_service.py
@@ -42,6 +42,7 @@ import uuid
from pathlib import Path
from typing import Any, Callable, Optional
from utils.paths.path_utils import drop_appledouble_metadata
+from hub.services import training_downloads
# Spawn (not fork): a fresh interpreter so parent CUDA/torch state never leaks into the trainer.
_CTX = mp.get_context("spawn")
@@ -61,7 +62,38 @@ def _finite_or_none(value: Any) -> Optional[float]:
return f if math.isfinite(f) else None
-def _run_diffusion_child(*, event_queue: Any, stop_queue: Any, config: dict) -> None:
+class _DownloadPhaseQueue:
+ """Publish the real child phase before its status message reaches the parent."""
+
+ def __init__(self, queue, phase):
+ self.queue = queue
+ self.phase = phase
+
+ def put(self, event):
+ kind = event.get("type")
+ if kind == "model_load_started" or kind in (
+ "model_load_completed",
+ "preparing",
+ "progress",
+ "complete",
+ "error",
+ ):
+ flag, lock = self.phase
+ with lock:
+ if kind == "model_load_started":
+ flag.set()
+ else:
+ flag.clear()
+ self.queue.put(event)
+
+
+def _run_diffusion_child(
+ *,
+ event_queue: Any,
+ stop_queue: Any,
+ config: dict,
+ download_phase = None,
+) -> None:
# Fresh spawned interpreter: re-apply network injections inside the secret scrub and before the trainer imports diffusers.
from utils.native_tls import activate_native_tls
from utils.happy_eyeballs import activate_happy_eyeballs
@@ -69,17 +101,23 @@ def _run_diffusion_child(*, event_queue: Any, stop_queue: Any, config: dict) ->
activate_native_tls()
activate_happy_eyeballs()
- # Imported lazily so this module (and the route layer) stays torch-free at import.
- from .diffusion_lora_trainer import run_diffusion_training_process
-
- # Skips LogConfig.setup_logging, so this installs the Hub default; diffusers setup is in the trainer.
+ # Restore only Unsloth's own quiet default before Hub caches it at import.
try:
- from loggers.config import quiet_third_party_progress_bars
+ from loggers.config import keep_progress_bars_countable, quiet_third_party_progress_bars
quiet_third_party_progress_bars()
- except Exception: # noqa: BLE001 - never let log tidying stop a training run
+ keep_progress_bars_countable()
+ except Exception: # noqa: BLE001 -- progress setup must never stop training
pass
- run_diffusion_training_process(event_queue = event_queue, stop_queue = stop_queue, config = config)
+ from .diffusion_lora_trainer import run_diffusion_training_process
+
+ run_diffusion_training_process(
+ event_queue = _DownloadPhaseQueue(event_queue, download_phase)
+ if download_phase
+ else event_queue,
+ stop_queue = stop_queue,
+ config = config,
+ )
def _default_target(
@@ -88,6 +126,7 @@ def _default_target(
stop_queue: Any,
config: dict,
unsloth_stderr_mirror_path: Optional[str] = None,
+ download_phase = None,
) -> None:
# First in the child, before torch: bind to parent death and scrub the native path secret; token policy precedes the returning account branch.
if unsloth_stderr_mirror_path:
@@ -121,6 +160,7 @@ def _default_target(
event_queue = event_queue,
stop_queue = stop_queue,
config = config,
+ **({"download_phase": download_phase} if download_phase is not None else {}),
)
return
from utils.native_path_leases import run_without_native_path_secret
@@ -130,6 +170,7 @@ def _default_target(
event_queue = event_queue,
stop_queue = stop_queue,
config = config,
+ **({"download_phase": download_phase} if download_phase is not None else {}),
)
@@ -330,6 +371,7 @@ def _idle_state() -> dict[str, Any]:
"num_images": None,
"in_model_load": False,
"preparation": None,
+ "download_progress": None,
"output_dir": None,
"lora_path": None,
# The optional second (EMA-averaged) adapter, when ema_decay was enabled.
@@ -492,6 +534,9 @@ class DiffusionTrainingService:
self._persist_interim = False
# Only the first stop reaches the child, so only the first may set the parent's disposition.
self._stop_signalled = False
+ self._download_key: Optional[str] = None
+ self._download_repo: Optional[str] = None
+ self._download_phase = None
# GPU load admissions in flight; same two-sided rule as dataset mutations.
self._gpu_admissions = 0
self._proc: Any = None
@@ -630,6 +675,9 @@ class DiffusionTrainingService:
}
# Test targets take only the three job kwargs.
if self._target is _default_target:
+ self._download_phase = (self._ctx.Event(), self._ctx.Lock())
+ self._download_phase[0].set()
+ child_kwargs["download_phase"] = self._download_phase
self._open_worker_stderr_capture()
if self._stderr_capture is not None:
from utils.native_path_leases import STDERR_MIRROR_KWARG
@@ -639,11 +687,23 @@ class DiffusionTrainingService:
kwargs = child_kwargs,
daemon = True,
)
+ from .diffusion_train_common import training_fetch_repo
+
+ self._download_repo = (
+ training_fetch_repo(normalized_cfg)
+ or config.get("base_model")
+ or config.get("model_name")
+ )
+ self._download_key = training_downloads.claim(self, job_id, self._download_repo)
# Keep the lease secret out of the child's env, as other orchestrators do.
from utils.native_path_leases import native_path_secret_removed_for_child_start
- with native_path_secret_removed_for_child_start():
- self._proc.start()
+ try:
+ with native_path_secret_removed_for_child_start():
+ self._proc.start()
+ except BaseException:
+ training_downloads.settle(self._download_key)
+ raise
try:
from utils.process_lifetime import adopt_pid
adopt_pid(self._proc.pid)
@@ -656,6 +716,7 @@ class DiffusionTrainingService:
active = True,
job_id = job_id,
status = "running",
+ in_model_load = True,
message = "Starting diffusion LoRA training...",
base_model = config.get("base_model") or config.get("model_name"),
started_at = now,
@@ -679,14 +740,70 @@ class DiffusionTrainingService:
@_invalidates_gpu_memory("diffusion training stop")
@job_control
- def stop(self, save: bool = True) -> bool:
+ def stop(
+ self,
+ save: bool = True,
+ *,
+ job_id: Optional[str] = None,
+ download_only: bool = False,
+ download_generation: Optional[int] = None,
+ ) -> bool:
"""Request a clean stop: the trainer finishes the current step, then either saves
a partial adapter (``save=True``, the default) or discards the run (``save=False``,
matching the LLM trainer's cancel). Returns True if a stop was signalled, False if
nothing was running."""
- with self._lock:
+ with (
+ self._lock,
+ self._download_phase[1] if self._download_phase else contextlib.nullcontext(),
+ ):
+ # Check under the signalling lock: never cancel a different run or its training.
+ if job_id is not None and self._state.get("job_id") != job_id:
+ return False
+ if download_generation is not None:
+ from hub.utils.download_registry import get_models_registry
+ if (
+ not self._download_key
+ or get_models_registry().current_generation(self._download_key)
+ != download_generation
+ ):
+ return False
+ if download_only and (
+ job_id is None
+ or not self._state.get("in_model_load")
+ or self._state.get("preparation") is not None
+ or self._state.get("status") != "running"
+ or (self._download_phase is not None and not self._download_phase[0].is_set())
+ ):
+ return False
if self._proc is None or not self._proc.is_alive() or self._stop_queue is None:
return False
+ loading = self._state.get("in_model_load") and self._state.get("preparation") is None
+ if self._download_phase is not None:
+ loading = loading and self._download_phase[0].is_set()
+ if loading and hasattr(self._proc, "terminate"):
+ # Native Xet ignores the cooperative queue; no optimizer runs yet, so terminate the worker.
+ self._proc.terminate()
+ self._proc.join(timeout = 2.0)
+ if self._proc.is_alive() and hasattr(self._proc, "kill"):
+ self._proc.kill()
+ self._proc.join(timeout = 2.0)
+ if self._proc.is_alive():
+ raise RuntimeError(
+ "The training download worker has not stopped yet. Try stopping it again."
+ )
+ self._stop_signalled = True
+ self._discard_requested = not save
+ self._state.update(
+ active = False,
+ status = "stopped",
+ in_model_load = False,
+ preparation = None,
+ download_progress = None,
+ message = "Stopped while loading model files; training had not started.",
+ updated_at = time.time(),
+ )
+ training_downloads.settle(self._download_key, "cancelled")
+ return True
if self._stop_signalled:
# Child consumes only the FIRST signal; a later stop-without-saving must not set a parent discard, which would delete checkpoints the child kept.
return True
@@ -814,10 +931,20 @@ class DiffusionTrainingService:
# A worker killed mid-load / mid-preparation sends no terminal event to clear these.
in_model_load = False,
preparation = None,
+ download_progress = None,
message = self._unexpected_exit_message(proc),
updated_at = time.time(),
)
discarding = self._discard_requested
+ training_downloads.settle(
+ self._download_key,
+ "cancelled"
+ if self._state.get("status") == "stopped"
+ else "error"
+ if self._state.get("status") == "error"
+ else "idle",
+ self._state.get("message"),
+ )
if discarding:
self._apply_discard_intent()
_ = drained
@@ -966,19 +1093,43 @@ class DiffusionTrainingService:
with self._lock:
if proc is not None and self._proc is not proc:
return
+ if self._stop_signalled and self._state.get("status") == "stopped":
+ return
s = self._state
s["updated_at"] = time.time()
if etype == "model_load_started":
s.update(
in_model_load = True,
preparation = None,
+ download_progress = None,
status = "running",
message = "Loading base model...",
)
if ev.get("num_images") is not None:
s["num_images"] = ev.get("num_images")
+ elif etype == "model_download_progress":
+ if s.get("in_model_load") and s.get("preparation") is None:
+ progress = ev.get("download_progress")
+ progress = progress if isinstance(progress, dict) else {}
+ done = _finite_or_none(progress.get("bytes_done"))
+ total = _finite_or_none(progress.get("bytes_total"))
+ s["download_progress"] = (
+ {
+ "bytes_done": int(min(done, total)),
+ "bytes_total": int(total),
+ "label": str(progress.get("label", "Model files"))[:200],
+ "scope": "active_files",
+ }
+ if done is not None and done >= 0 and total is not None and total > 0
+ else None
+ )
elif etype == "model_load_completed":
- s.update(in_model_load = False, preparation = None, message = "Training...")
+ s.update(
+ in_model_load = False,
+ preparation = None,
+ download_progress = None,
+ message = "Training...",
+ )
elif etype == "preparing":
# Long precompute such as the VAE latent cache, surfaced so the UI does not look stalled.
done, total = ev.get("done"), ev.get("total")
@@ -987,6 +1138,7 @@ class DiffusionTrainingService:
status = "running",
in_model_load = True,
preparation = {"stage": ev.get("stage"), "done": done, "total": total},
+ download_progress = None,
message = (
f"Preparing ({stage} {done}/{total})..."
if done is not None and total is not None
@@ -1049,6 +1201,7 @@ class DiffusionTrainingService:
status = "running",
step = ev.get("step", s["step"]),
preparation = None,
+ download_progress = None,
in_model_load = False,
total_steps = ev.get("total_steps", s["total_steps"]),
loss = loss,
@@ -1088,6 +1241,7 @@ class DiffusionTrainingService:
in_model_load = False,
status = "stopped" if ev.get("stopped") else "completed",
preparation = None,
+ download_progress = None,
output_dir = ev.get("output_dir"),
lora_path = ev.get("lora_path"),
ema_path = ev.get("ema_path"),
@@ -1122,8 +1276,33 @@ class DiffusionTrainingService:
in_model_load = False,
status = "error",
preparation = None,
+ download_progress = None,
message = str(ev.get("message", "error")),
)
+ if etype == "model_load_started":
+ try:
+ self._download_key = training_downloads.claim(
+ self,
+ s.get("job_id"),
+ ev.get("fetch_repo")
+ or self._download_repo
+ or s.get("base_model")
+ or self._config.get("base_model"),
+ )
+ except RuntimeError as exc:
+ s.update(status = "error", message = str(exc), in_model_load = False)
+ if self._proc is not None and hasattr(self._proc, "terminate"):
+ self._proc.terminate()
+ elif etype in ("preparing", "model_load_completed", "progress", "complete", "error"):
+ training_downloads.settle(
+ self._download_key,
+ "error"
+ if etype == "error"
+ else "cancelled"
+ if s.get("status") == "stopped"
+ else "idle",
+ s.get("message") if etype == "error" else None,
+ )
_service: Optional[DiffusionTrainingService] = None
diff --git a/studio/backend/hub/schemas/downloads.py b/studio/backend/hub/schemas/downloads.py
index 6207009a568..1006b1cf292 100644
--- a/studio/backend/hub/schemas/downloads.py
+++ b/studio/backend/hub/schemas/downloads.py
@@ -152,6 +152,7 @@ class TransportStatusResponse(BaseModel):
class DownloadProgressResponse(BaseModel):
+ progress_scope: Optional[Literal["active_files"]] = None
downloaded_bytes: int
# Finalized-blob bytes only, so a partial is not mistaken for finished.
completed_bytes: int = 0
diff --git a/studio/backend/hub/services/models/downloads.py b/studio/backend/hub/services/models/downloads.py
index 219f73fd22e..d6166d292b4 100644
--- a/studio/backend/hub/services/models/downloads.py
+++ b/studio/backend/hub/services/models/downloads.py
@@ -562,6 +562,11 @@ async def cancel_download_model_response(body: CancelDownloadRequest):
detail = f"Invalid gguf_variant: {variant!r}",
)
key = _download_job_key(repo_id, variant)
+ from hub.services import training_downloads
+
+ if training_downloads.is_training_variant(variant):
+ state = await asyncio.to_thread(training_downloads.cancel, key, body.generation)
+ return {"job_key": key, "state": state}
if load_downloads.is_load_owned(_registry, key):
raise HTTPException(
status_code = 409,
@@ -787,6 +792,11 @@ async def get_gguf_download_progress_response(
hf_token: Optional[str] = None,
) -> dict:
"""Return download progress for a specific GGUF variant."""
+ from hub.services import training_downloads
+
+ if training_downloads.is_training_variant(variant):
+ key = _download_job_key(repo_id, variant)
+ return await asyncio.to_thread(training_downloads.progress, key)
try:
repo_id, variant = await _audio_cpp_target(repo_id, variant, hf_token)
except HTTPException:
diff --git a/studio/backend/hub/services/training_downloads.py b/studio/backend/hub/services/training_downloads.py
new file mode 100644
index 00000000000..b76af5dbd24
--- /dev/null
+++ b/studio/backend/hub/services/training_downloads.py
@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Training-owned transfers in the same registry and API as Hub downloads.
+
+The trainer owns its subprocess, so cancellation must stop that exact loading
+worker rather than enqueue a signal behind a blocking native Xet call. Progress
+describes current files only; leaving a load phase never verifies a full repo.
+"""
+
+from __future__ import annotations
+
+import weakref
+import re
+from typing import Optional
+
+from hub.services import download_lifecycle
+from hub.utils import download_registry
+from hub.utils.paths import is_valid_repo_id
+
+OWNER = "training"
+PREFIX = "@training-"
+_services: weakref.WeakValueDictionary = weakref.WeakValueDictionary()
+
+
+def is_training_variant(variant: Optional[str]) -> bool:
+ return bool(re.fullmatch(r"@training-[0-9a-f]{32}", variant or ""))
+
+
+def claim(service, job_id: str, repo_id: Optional[str]) -> Optional[str]:
+ if not repo_id or not is_valid_repo_id(repo_id) or not is_training_variant(f"{PREFIX}{job_id}"):
+ return None
+ registry = download_registry.get_models_registry()
+ variant = f"{PREFIX}{job_id}"
+ key = download_registry.normalize_job_key(f"{repo_id}::{variant}")
+ metadata = registry.get_job_metadata(key)
+ if (
+ metadata is not None
+ and metadata.owner == OWNER
+ and registry.get_job(key).state == "running"
+ ):
+ return key
+ accepted, state = registry.claim(
+ key,
+ download_registry.TRANSPORT_AUTO,
+ repo_type = "model",
+ repo_id = repo_id,
+ variant = variant,
+ owner = OWNER,
+ )
+ if not accepted:
+ raise RuntimeError(f"Model files are already in use by another download ({state}).")
+ download_lifecycle.record_download_account(registry, key)
+ _services[key] = service
+ return key
+
+
+def settle(
+ key: Optional[str],
+ outcome: str = "idle",
+ error: Optional[str] = None,
+) -> None:
+ if key is None:
+ return
+ registry = download_registry.get_models_registry()
+ metadata = registry.get_job_metadata(key)
+ if metadata is None or metadata.owner != OWNER:
+ return
+ if outcome == "idle":
+ registry.release_owned(key, OWNER)
+ else:
+ registry.set_job(key, outcome, error)
+ _services.pop(key, None)
+
+
+def cancel(key: str, generation: Optional[int]) -> str:
+ registry = download_registry.get_models_registry()
+ download_lifecycle.require_download_account(registry, key)
+ if generation is None or generation != registry.current_generation(key):
+ return registry.get_job(key).state
+ metadata = registry.get_job_metadata(key)
+ service = _services.get(key)
+ if metadata is None or metadata.owner != OWNER or service is None:
+ return registry.get_job(key).state
+ job_id = metadata.variant[len(PREFIX) :]
+ service.stop(save = False, job_id = job_id, download_only = True, download_generation = generation)
+ return registry.get_job(key).state
+
+
+def progress(key: str) -> dict:
+ registry = download_registry.get_models_registry()
+ download_lifecycle.require_download_account(registry, key)
+ service = _services.get(key)
+ snapshot = service.status() if service is not None else {}
+ metadata = registry.get_job_metadata(key)
+ same_job = metadata is not None and metadata.variant == f"{PREFIX}{snapshot.get('job_id')}"
+ reading = (
+ snapshot.get("download_progress") if same_job and snapshot.get("in_model_load") else None
+ )
+ done = reading.get("bytes_done", 0) if reading else 0
+ total = reading.get("bytes_total", 0) if reading else 0
+ return {
+ "downloaded_bytes": done,
+ "expected_bytes": total,
+ "completed_bytes": 0,
+ "complete_on_disk": False,
+ "progress": done / total if total else 0,
+ "cache_path": None,
+ "cache_measured": True,
+ "target_present": bool(reading),
+ "progress_scope": "active_files",
+ }
diff --git a/studio/backend/loggers/config.py b/studio/backend/loggers/config.py
index 936e5408f44..bf4f2edea3a 100644
--- a/studio/backend/loggers/config.py
+++ b/studio/backend/loggers/config.py
@@ -289,6 +289,9 @@ def _redirect_every_bar_output() -> None:
def _quiet_init(self, *args, **kwargs):
kwargs.setdefault("file", _NullStream())
+ # disable=None means TTY detection; our stream is not a TTY but the UI needs counts.
+ if kwargs.get("disable") is None:
+ kwargs["disable"] = False
original_init(self, *args, **kwargs)
bar_cls.__init__ = _quiet_init
@@ -307,6 +310,15 @@ def keep_progress_bars_countable() -> None:
return
_redirect_every_bar_output()
allow_progress_bars()
+ # Hub may already be imported: undo its cached copy of OUR default, never an operator opt-out.
+ for name in ("huggingface_hub.constants", "huggingface_hub.utils.tqdm"):
+ module = sys.modules.get(name)
+ if module is not None and getattr(module, "HF_HUB_DISABLE_PROGRESS_BARS", None) is True:
+ module.HF_HUB_DISABLE_PROGRESS_BARS = None
+ bars = sys.modules.get("huggingface_hub.utils.tqdm")
+ enable = getattr(bars, "enable_progress_bars", None)
+ if callable(enable):
+ enable()
def quiet_bar_kwargs() -> dict:
diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py
index af412ee1262..5be9f5e9862 100644
--- a/studio/backend/models/training.py
+++ b/studio/backend/models/training.py
@@ -1211,6 +1211,13 @@ class DiffusionTrainingPreparation(BaseModel):
total: Optional[int] = None
+class DiffusionTrainingDownloadProgress(BaseModel):
+ bytes_done: int = Field(default = 0, ge = 0)
+ bytes_total: int = Field(default = 0, ge = 0)
+ label: str = "Model files"
+ scope: str = "active_files"
+
+
class DiffusionTrainingStatusResponse(BaseModel):
"""A snapshot of the current diffusion training job (or idle)."""
@@ -1231,6 +1238,7 @@ class DiffusionTrainingStatusResponse(BaseModel):
num_images: Optional[int] = None
in_model_load: bool = False
preparation: Optional[DiffusionTrainingPreparation] = None
+ download_progress: Optional[DiffusionTrainingDownloadProgress] = None
output_dir: Optional[str] = None
lora_path: Optional[str] = None
# The second, EMA-averaged adapter written in the run's ema subdir when ema_decay was enabled.
diff --git a/studio/backend/tests/test_diffusion_download_progress.py b/studio/backend/tests/test_diffusion_download_progress.py
new file mode 100644
index 00000000000..70fcf6205ea
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_download_progress.py
@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from types import SimpleNamespace
+
+import pytest
+
+from core.training.diffusion_download_progress import TrainingDownloadProgress, download_snapshot
+
+
+def bar(
+ done = 4,
+ total = 10,
+ desc = "weights.safetensors",
+ unit = "B",
+ disable = False,
+):
+ return SimpleNamespace(n = done, total = total, desc = desc, unit = unit, disable = disable)
+
+
+def test_download_snapshot_counts_only_active_bytes():
+ snapshot = download_snapshot([bar(), bar(1, 100, unit = "it"), bar(disable = True)])
+ assert snapshot == {
+ "bytes_done": 4,
+ "bytes_total": 10,
+ "label": "weights.safetensors",
+ "scope": "active_files",
+ }
+ assert download_snapshot([bar(0)])["bytes_done"] == 0
+
+
+def test_parallel_file_downloads_share_one_current_batch():
+ snapshot = download_snapshot([bar(), bar(6, 20, desc = "encoder.safetensors")])
+ assert snapshot["bytes_done"] == 10
+ assert snapshot["bytes_total"] == 30
+ assert snapshot["label"] == "2 model files"
+
+
+def test_xet_transfer_and_reconstruction_are_not_counted_twice():
+ snapshot = download_snapshot(
+ [
+ bar(10, 10, "Downloading bytes"),
+ bar(4, 10, "Reconstructing (incomplete total...)"),
+ ]
+ )
+ assert snapshot["bytes_done"] == 4
+ assert snapshot["bytes_total"] == 10
+
+
[email protected]("done, total", [(-1, 10), (float("nan"), 10), (1, float("inf")), (1, 0)])
+def test_invalid_or_unknown_byte_counts_stay_indeterminate(done, total):
+ assert download_snapshot([bar(done, total)]) is None
+
+
+def test_download_progress_clears_when_file_bars_close():
+ events = []
+ current = [bar()]
+ monitor = TrainingDownloadProgress(events.append, sample = lambda: download_snapshot(current))
+ monitor.poll()
+ assert events == []
+ monitor.report({"type": "model_load_started"})
+ monitor.poll()
+ monitor.poll()
+ assert len(events) == 2
+ current.clear()
+ monitor.poll()
+ assert events[-1] == {"type": "model_download_progress", "download_progress": None}
+
+
+ "phase", ["preparing", "model_load_completed", "progress", "complete", "error"]
+)
+def test_byte_progress_cannot_replace_later_training_phases(phase):
+ events = []
+ monitor = None
+
+ def sample():
+ monitor.report({"type": phase})
+ return download_snapshot([bar()])
+
+ monitor = TrainingDownloadProgress(events.append, sample = sample)
+ monitor.report({"type": "model_load_started"})
+ monitor.poll()
+ assert [event["type"] for event in events] == ["model_load_started", phase]
+
+
+def test_monitor_thread_is_stopped_on_a_trainer_error():
+ monitor = TrainingDownloadProgress(lambda _ev: None, interval = 3600)
+ with pytest.raises(ValueError):
+ with monitor as report:
+ report({"type": "model_load_started"})
+ raise ValueError("trainer failed")
+ assert not monitor._thread.is_alive()
diff --git a/studio/backend/tests/test_diffusion_h3_trainer.py b/studio/backend/tests/test_diffusion_h3_trainer.py
index 4fe539404ff..0078d60c0d1 100644
--- a/studio/backend/tests/test_diffusion_h3_trainer.py
+++ b/studio/backend/tests/test_diffusion_h3_trainer.py
@@ -1460,7 +1460,12 @@ def _stub_training_stack(monkeypatch):
monkeypatch.setitem(sys.modules, name, module)
-def _h3_cache_run(monkeypatch, *, num_clips: int):
+def _h3_cache_run(
+ monkeypatch,
+ *,
+ num_clips: int,
+ on_event = None,
+):
"""Drive _train_h3 through phase 2 with a sentinel _load_transformer, so reaching it is observable."""
import torch
@@ -1504,12 +1509,32 @@ def _h3_cache_run(monkeypatch, *, num_clips: int):
random.Random(0),
"cpu",
torch.float32,
- lambda *_a, **_k: None,
+ on_event or (lambda *_a, **_k: None),
lambda: False,
lambda: True,
)
+def test_h3_resumes_model_download_progress_after_latent_preparation(monkeypatch):
+ from core.training.diffusion_download_progress import TrainingDownloadProgress
+
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_FORCE_LATENT_CACHE", "1")
+ events = []
+ monitor = TrainingDownloadProgress(
+ events.append,
+ sample = lambda: {"bytes_done": 1, "bytes_total": 2, "scope": "active_files"},
+ )
+ monitor.report({"type": "model_load_started"})
+ with pytest.raises(RuntimeError, match = "reached phase 3"):
+ _h3_cache_run(monkeypatch, num_clips = 1, on_event = monitor.report)()
+ monitor.poll()
+ assert any(event["type"] == "preparing" for event in events)
+ assert [event["type"] for event in events[-2:]] == [
+ "model_load_started",
+ "model_download_progress",
+ ]
+
+
def test_the_h3_latent_cache_is_size_gated_like_the_image_trainers(monkeypatch):
"""H3 must size-gate its latent cache against the host-memory budget, as the image trainers do."""
from core.training import diffusion_train_common as common
diff --git a/studio/backend/tests/test_diffusion_training.py b/studio/backend/tests/test_diffusion_training.py
index efc27906b01..a05e1e1f74e 100644
--- a/studio/backend/tests/test_diffusion_training.py
+++ b/studio/backend/tests/test_diffusion_training.py
@@ -2613,3 +2613,43 @@ def test_llm_start_admission_fails_open_without_a_diffusion_stack(monkeypatch):
monkeypatch.setitem(sys.modules, "core.training.diffusion_training_service", None)
with tr._diffusion_gpu_admission():
pass
+
+
+def test_status_route_preserves_model_download_bytes(client):
+ progress = {
+ "bytes_done": 4,
+ "bytes_total": 10,
+ "label": "model.safetensors",
+ "scope": "active_files",
+ }
+ client._fake.status_extra = {"in_model_load": True, "download_progress": progress}
+ assert client.get("/api/train/diffusion/status").json()["download_progress"] == progress
+
+
+def test_model_download_progress_clears_before_preparation():
+ svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
+ svc._apply_event({"type": "model_load_started"})
+ event = {
+ "type": "model_download_progress",
+ "download_progress": {"bytes_done": 4, "bytes_total": 10},
+ }
+ svc._apply_event(event)
+ assert svc.status()["download_progress"]["bytes_done"] == 4
+ svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 3, "total": 10})
+ svc._apply_event(event)
+ assert svc.status()["download_progress"] is None
+ assert svc.status()["preparation"]["done"] == 3
+ svc._apply_event({"type": "model_load_started"})
+ svc._apply_event(event)
+ assert svc.status()["preparation"] is None
+ assert svc.status()["download_progress"]["bytes_done"] == 4
+
+
+ "progress", [None, "invalid", {"bytes_done": float("nan"), "bytes_total": 10}]
+)
+def test_model_download_progress_rejects_unknown_byte_counts(progress):
+ svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
+ svc._apply_event({"type": "model_load_started"})
+ svc._apply_event({"type": "model_download_progress", "download_progress": progress})
+ assert svc.status()["download_progress"] is None
diff --git a/studio/backend/tests/test_diffusion_training_download_phases.py b/studio/backend/tests/test_diffusion_training_download_phases.py
new file mode 100644
index 00000000000..462123af905
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_training_download_phases.py
@@ -0,0 +1,104 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import contextlib
+import random
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+from core.training import diffusion_dit_trainer as dit
+from core.training.diffusion_download_progress import TrainingDownloadProgress
+from core.training.diffusion_train_common import DiffusionLoraConfig
+
+
[email protected]("warm_preview", [False, True])
+def test_dit_resumes_download_progress_after_preparation(monkeypatch, tmp_path, warm_preview):
+ # Warm route can still need encoders for preview prompts.
+ monkeypatch.setitem(
+ sys.modules, "diffusers", SimpleNamespace(FlowMatchEulerDiscreteScheduler = object)
+ )
+ monkeypatch.setitem(
+ sys.modules, "diffusers.optimization", SimpleNamespace(get_scheduler = object)
+ )
+ monkeypatch.setitem(
+ sys.modules, "diffusers.training_utils", SimpleNamespace(cast_training_params = object)
+ )
+ monkeypatch.setitem(sys.modules, "peft", SimpleNamespace(LoraConfig = object))
+ monkeypatch.setitem(
+ sys.modules, "peft.utils", SimpleNamespace(get_peft_model_state_dict = object)
+ )
+ monkeypatch.setattr("loggers.config.quiet_third_party_progress_bars", lambda: None)
+ monkeypatch.setattr(dit, "plan_image_canvases", lambda *_a: (None, False))
+ monkeypatch.setattr(dit, "_plan_cache_variants", lambda *_a, **_k: object())
+ monkeypatch.setattr(dit, "_free_text_encoders", lambda *_a: None)
+ monkeypatch.setattr(dit, "_encode_prompts_cached", lambda *_a: [object()])
+ monkeypatch.setattr(dit, "_resolve_base_precision", lambda *_a: "bf16")
+ monkeypatch.setattr(dit, "isolated_sampling", lambda *_a: contextlib.nullcontext())
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_NO_LATENT_CACHE", raising = False)
+ events = []
+ monitor = TrainingDownloadProgress(
+ events.append,
+ sample = lambda: {"bytes_done": 1, "bytes_total": 2, "scope": "active_files"},
+ )
+ monitor.report({"type": "model_load_started"})
+
+ class ReachedDownload(Exception):
+ pass
+
+ def download(*_args):
+ monitor.poll()
+ assert any(event["type"] == "preparing" for event in events)
+ assert [event["type"] for event in events[-2:]] == [
+ "model_load_started",
+ "model_download_progress",
+ ]
+ raise ReachedDownload
+
+ vae = SimpleNamespace(to = lambda *_a: None)
+ spec = SimpleNamespace(
+ family = "flux.1",
+ lora_targets = ("to_q",),
+ decode_latents = warm_preview,
+ load_conditioners = download if warm_preview else lambda *_a: (SimpleNamespace(), vae),
+ load_transformer = download,
+ )
+ cfg = DiffusionLoraConfig(
+ base_model = "unsloth/FLUX.1-dev",
+ data_dir = str(tmp_path),
+ output_dir = str(tmp_path / "out"),
+ cache_latents = True,
+ cond_cache_dir = str(tmp_path / "cache") if warm_preview else None,
+ )
+ if warm_preview:
+ monkeypatch.setattr(
+ dit, "plan_samples", lambda *_a: SimpleNamespace(encode_texts = ["preview"])
+ )
+ monkeypatch.setattr(
+ "core.training.diffusion_train_extras.source_revision", lambda *_a: "local"
+ )
+ monkeypatch.setattr(dit, "PersistentConditioningCache", lambda *_a: object())
+ monkeypatch.setattr(
+ dit, "_load_warm_conditioning", lambda *_a: ({"caption": object()}, [object()])
+ )
+ else:
+
+ def cache(*args, **kwargs):
+ args[6]({"type": "preparing", "stage": "cache_latents", "done": 1, "total": 1})
+ return [object()]
+
+ monkeypatch.setattr(dit, "_build_latent_cache", cache)
+ with pytest.raises(ReachedDownload):
+ dit._train_dit(
+ cfg,
+ spec,
+ [(str(tmp_path / "image.png"), "caption")],
+ random.Random(0),
+ "cpu",
+ None,
+ monitor.report,
+ lambda: False,
+ lambda: True,
+ None,
+ )
diff --git a/studio/backend/tests/test_third_party_progress_bars.py b/studio/backend/tests/test_third_party_progress_bars.py
index 7d400bd38c4..f69c57034f5 100644
--- a/studio/backend/tests/test_third_party_progress_bars.py
+++ b/studio/backend/tests/test_third_party_progress_bars.py
@@ -345,3 +345,26 @@ def test_the_shared_dataset_loader_quiets_the_bars_it_just_imported():
assert body.index("from datasets import load_dataset") < body.index(
"quiet_third_party_progress_bars()"
)
+
+
+def test_countable_worker_bars_restore_the_hub_default_cached_before_bootstrap(monkeypatch):
+ import importlib
+
+ hub_bars = importlib.import_module("huggingface_hub.utils.tqdm")
+ hub_constants = importlib.import_module("huggingface_hub.constants")
+ monkeypatch.setattr(log_config, "_BARS_RESTORED", False)
+ monkeypatch.setenv(_HUB, "1")
+ monkeypatch.setenv(log_config._PROGRESS_BARS_DEFAULTED, "1")
+ monkeypatch.setattr(hub_constants, "HF_HUB_DISABLE_PROGRESS_BARS", True)
+ monkeypatch.setattr(hub_bars, "HF_HUB_DISABLE_PROGRESS_BARS", True)
+ monkeypatch.setattr(hub_bars, "progress_bar_states", {"_global": False})
+ log_config.keep_progress_bars_countable()
+ assert not hub_bars.are_progress_bars_disabled()
+ with hub_bars.tqdm(total = 10, unit = "B", desc = "training weights", disable = None) as progress:
+ progress.update(4)
+ from core.training.diffusion_download_progress import download_snapshot
+
+ snapshot = download_snapshot([progress])
+ assert snapshot["bytes_done"] == 4
+ with hub_bars.tqdm(total = 10, unit = "B", disable = True) as progress:
+ assert progress.disable is True
diff --git a/studio/backend/tests/test_training_downloads.py b/studio/backend/tests/test_training_downloads.py
new file mode 100644
index 00000000000..4770a5c8de7
--- /dev/null
+++ b/studio/backend/tests/test_training_downloads.py
@@ -0,0 +1,284 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import asyncio
+import queue
+import threading
+import time
+
+import pytest
+
+from core.training.diffusion_training_service import DiffusionTrainingService
+from hub.schemas.downloads import CancelDownloadRequest
+from hub.services import download_lifecycle, training_downloads
+from hub.services.models import downloads
+from hub.utils import download_registry
+
+
+class BlockingDownload:
+ def __init__(self):
+ self.done = threading.Event()
+ self.thread = threading.Thread(target = self.done.wait, daemon = True)
+ self.thread.start()
+ self.terminations = 0
+
+ def is_alive(self):
+ return self.thread.is_alive()
+
+ def terminate(self):
+ self.terminations += 1
+ self.done.set()
+
+ def join(self, timeout):
+ self.thread.join(timeout)
+
+
+def rig(monkeypatch):
+ registry = download_registry.DownloadRegistry()
+ monkeypatch.setattr(download_registry, "get_models_registry", lambda: registry)
+ monkeypatch.setattr(downloads, "_registry", registry)
+ service = DiffusionTrainingService()
+ service._proc = BlockingDownload()
+ service._stop_queue = queue.Queue()
+ service._state.update(
+ job_id = "a" * 32,
+ active = True,
+ status = "running",
+ base_model = "unsloth/FLUX.1-dev",
+ in_model_load = True,
+ )
+ service._download_key = training_downloads.claim(service, "a" * 32, "unsloth/FLUX.1-dev")
+ yield service, registry, service._download_key
+ service._proc.terminate()
+ service._proc.join(1)
+
+
+def test_metadata_stage_is_discoverable_and_cancellable_through_main_manager(rig):
+ service, registry, key = rig
+ rows = download_lifecycle.active_download_refs(registry, None, with_variant = True)
+ assert [(r.repo_id, r.owner, r.variant) for r in rows] == [
+ ("unsloth/FLUX.1-dev", "training", "@training-" + "a" * 32)
+ ]
+ reading = asyncio.run(
+ downloads.get_gguf_download_progress_response(rows[0].repo_id, rows[0].variant)
+ )
+ assert reading["expected_bytes"] == 0
+ assert reading["progress_scope"] == "active_files"
+ result = asyncio.run(
+ downloads.cancel_download_model_response(
+ CancelDownloadRequest(
+ repo_id = rows[0].repo_id,
+ gguf_variant = rows[0].variant,
+ generation = rows[0].generation,
+ )
+ )
+ )
+ assert result["state"] == "cancelled"
+ assert not service._proc.is_alive()
+ assert service._stop_queue.empty(), "blocked Xet never reads this queue"
+ assert service.status()["status"] == "stopped"
+ assert registry.get_job(key).state == "cancelled"
+
+
+def test_current_files_never_claim_the_entire_model_is_installed(rig):
+ service, registry, key = rig
+ service._apply_event(
+ {
+ "type": "model_download_progress",
+ "download_progress": {
+ "bytes_done": 10,
+ "bytes_total": 10,
+ "label": "Reconstructing (incomplete total...)",
+ },
+ }
+ )
+ reading = training_downloads.progress(key)
+ assert (reading["downloaded_bytes"], reading["expected_bytes"]) == (10, 10)
+ assert reading["complete_on_disk"] is False
+ assert reading["completed_bytes"] == 0
+ service._apply_event({"type": "preparing", "stage": "cache_latents", "done": 1, "total": 60})
+ assert registry.get_job(key).state == "idle"
+ assert training_downloads.progress(key)["expected_bytes"] == 0
+ assert not download_lifecycle.active_download_refs(registry, None, with_variant = True)
+
+
[email protected]("change", ["job", "phase", "generation", "unbound"])
+def test_a_stale_download_cancel_never_stops_training_or_a_replacement(rig, change):
+ service, registry, key = rig
+ generation = registry.current_generation(key)
+ if change == "job":
+ service._state["job_id"] = "b" * 32
+ elif change == "phase":
+ service._state.update(in_model_load = False, step = 1)
+ elif change == "generation":
+ generation += 1
+ else:
+ generation = None
+ training_downloads.cancel(key, generation)
+ assert service._proc.is_alive()
+ assert service._proc.terminations == 0
+ assert service._stop_queue.empty()
+
+
+def test_training_step_stop_remains_cooperative_and_preserves_save_choice(rig):
+ service, _registry, _key = rig
+ service._state.update(in_model_load = False, step = 1)
+ assert service.stop(save = True)
+ assert service._stop_queue.get_nowait() is True
+ assert service._proc.is_alive()
+ assert service._discard_requested is False
+
+
+def test_real_child_phase_wins_over_a_delayed_parent_loading_event(rig):
+ from core.training.diffusion_training_service import _DownloadPhaseQueue
+
+ service, _registry, _key = rig
+ phase = (threading.Event(), threading.Lock())
+ phase[0].set()
+ service._download_phase = phase
+ pending = queue.Queue()
+ events = _DownloadPhaseQueue(pending, phase)
+ events.put({"type": "model_load_completed"})
+ assert not phase[0].is_set()
+ assert service._state["in_model_load"] is True
+ assert service.stop(save = False, job_id = "a" * 32, download_only = True) is False
+ assert service.stop(save = True) is True
+ assert (
+ service._proc.is_alive()
+ ), "optimizer/preparation cannot be terminated using stale parent status"
+ assert service._stop_queue.get_nowait() is True
+
+
+def test_buffered_events_cannot_resurrect_a_killed_download(rig):
+ service, registry, key = rig
+ assert service.stop(save = False)
+ service._apply_event({"type": "model_load_started"}, proc = service._proc)
+ service._apply_event(
+ {
+ "type": "model_download_progress",
+ "download_progress": {"bytes_done": 1, "bytes_total": 2},
+ }
+ )
+ assert service.status()["status"] == "stopped"
+ assert service.status()["download_progress"] is None
+ assert registry.get_job(key).state == "cancelled"
+
+
+def test_later_file_loads_get_a_new_generation_without_completing_the_repo(rig):
+ service, registry, key = rig
+ generation = registry.current_generation(key)
+ service._apply_event({"type": "preparing", "stage": "cache_latents", "done": 60, "total": 60})
+ service._apply_event({"type": "model_load_started"})
+ assert registry.current_generation(key) > generation
+ assert registry.get_job(key).state == "running"
+ training_downloads.cancel(key, generation)
+ assert service._proc.is_alive()
+
+
+def test_registry_account_is_checked_before_reading_or_cancelling(rig, monkeypatch):
+ _service, registry, key = rig
+ from fastapi import HTTPException
+
+ monkeypatch.setattr(download_lifecycle, "download_belongs_to_account", lambda *_args: False)
+ for action in [
+ lambda: training_downloads.progress(key),
+ lambda: training_downloads.cancel(key, registry.current_generation(key)),
+ ]:
+ with pytest.raises(HTTPException) as error:
+ action()
+ assert error.value.status_code == 404
+
+
+_GATED = "black-forest-labs/FLUX.1-dev"
+_MIRROR = "unsloth/FLUX.1-dev"
+
+
+def _loading_target(*, event_queue, stop_queue, config):
+ from core.training.diffusion_lora_trainer import _config_from_dict
+ from core.training.diffusion_train_common import _emit, training_fetch_repo
+
+ cfg = _config_from_dict(config).normalized()
+ _emit(event_queue.put, "model_load_started", fetch_repo = training_fetch_repo(cfg))
+ stop_queue.get(timeout = 5.0)
+
+
+class _ThreadProc:
+ def __init__(self, target, kwargs, daemon):
+ self.thread = threading.Thread(target = target, kwargs = kwargs, daemon = daemon)
+ self.pid = None
+
+ def start(self):
+ self.thread.start()
+
+ def is_alive(self):
+ return self.thread.is_alive()
+
+
+class _FakeCtx:
+ Queue = queue.Queue
+ Process = _ThreadProc
+
+
+def start_rig(monkeypatch, tmp_path, healthy_diffusers):
+ import core.training.diffusion_training_service as dts
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_NO_MIRROR", raising = False)
+ monkeypatch.setattr(dts, "_runs_dir", lambda: tmp_path)
+ registry = download_registry.DownloadRegistry()
+ monkeypatch.setattr(download_registry, "get_models_registry", lambda: registry)
+ service = DiffusionTrainingService(ctx = _FakeCtx(), target = _loading_target)
+ config = {"base_model": _GATED, "data_dir": "d", "output_dir": str(tmp_path), "train_steps": 2}
+ yield service, registry, config
+ if service._stop_queue is not None:
+ service._stop_queue.put(True)
+ if service._pump is not None:
+ service._pump.join(5)
+
+
+def _hub_snapshot(registry, repo_id):
+ accepted, _ = registry.claim(
+ repo_id, download_registry.TRANSPORT_AUTO, repo_type = "model", repo_id = repo_id
+ )
+ assert accepted
+
+
+def test_tokenless_gated_base_claims_the_mirror_it_fetches(start_rig):
+ from core.training.diffusion_lora_trainer import _config_from_dict
+ from core.training.diffusion_train_common import training_fetch_repo
+
+ service, registry, config = start_rig
+ assert training_fetch_repo(_config_from_dict(config).normalized()) == _MIRROR
+ job_id = service.start(config)
+ key = service._download_key
+ assert registry.get_job_metadata(key).repo_id == _MIRROR
+ assert service.status()["base_model"] == _GATED
+ deadline = time.time() + 3
+ while not service.status().get("in_model_load") and time.time() < deadline:
+ time.sleep(0.01)
+ assert service._download_key == key
+ assert registry.get_job_metadata(key).variant == f"@training-{job_id}"
+
+
+def test_canonical_snapshot_download_does_not_block_a_mirror_fetch(start_rig):
+ service, registry, config = start_rig
+ _hub_snapshot(registry, _GATED)
+ service.start(config)
+ assert registry.get_job_metadata(service._download_key).repo_id == _MIRROR
+
+
+def test_mirror_snapshot_download_blocks_the_start(start_rig):
+ service, registry, config = start_rig
+ _hub_snapshot(registry, _MIRROR)
+ with pytest.raises(RuntimeError, match = "already in use"):
+ service.start(config)
+
+
+def test_model_load_reclaim_uses_the_reported_fetch_repo(start_rig):
+ service, registry, config = start_rig
+ service._state.update(job_id = "b" * 32, base_model = _GATED)
+ _hub_snapshot(registry, _GATED)
+ service._apply_event({"type": "model_load_started", "fetch_repo": _MIRROR})
+ assert service.status()["status"] != "error"
+ assert registry.get_job_metadata(service._download_key).repo_id == _MIRROR
diff --git a/studio/frontend/src/features/chat/components/model-load-status.tsx b/studio/frontend/src/features/chat/components/model-load-status.tsx
index 94841110f97..88b49dd3d52 100644
--- a/studio/frontend/src/features/chat/components/model-load-status.tsx
+++ b/studio/frontend/src/features/chat/components/model-load-status.tsx
@@ -6,6 +6,13 @@ import { Spinner } from "@/components/ui/spinner";
import { Button } from "@/components/ui/button";
import { cn } from "@/lib/utils";
+export const MODEL_LOAD_TOAST_CLASSNAMES = {
+ toast: "chat-model-load-toast items-center gap-2.5",
+ content: "gap-0.5 flex-1 min-w-0",
+ title: "leading-5",
+ description: "mt-0 w-full",
+};
+
type ModelLoadDescriptionProps = {
title?: string | null;
message?: string | null;
diff --git a/studio/frontend/src/features/hub/download-manager/api.ts b/studio/frontend/src/features/hub/download-manager/api.ts
index 39ee534db9e..59a792934e0 100644
--- a/studio/frontend/src/features/hub/download-manager/api.ts
+++ b/studio/frontend/src/features/hub/download-manager/api.ts
@@ -63,6 +63,7 @@ export interface DownloadStartResult {
}
export interface ActiveModelDownload {
+ owner?: string | null;
repo_id?: string;
variant: string | null;
transport?: TransportMode | null;
@@ -118,6 +119,7 @@ export interface TransportStatus {
}
export interface DownloadProgressResponse {
+ progress_scope?: "active_files" | null;
downloaded_bytes: number;
completed_bytes: number;
complete_on_disk: boolean;
diff --git a/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx b/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx
index 74349b7ef4b..19fddc2516b 100644
--- a/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx
+++ b/studio/frontend/src/features/hub/download-manager/download-manager-panel.tsx
@@ -40,6 +40,8 @@ import {
} from "./pending-starts";
import { assetLabel } from "./required-assets";
import { queuedStagedEntries, useStagedDownloadQueues } from "./staged-download-queue";
+import { isTrainingDownloadVariant } from "./training-downloads";
+import { useTrainingDownloads } from "./use-training-downloads";
type RowSubject = Pick<
ManagedDownload,
@@ -112,6 +114,7 @@ const REQUIRED_ASSET_NOTE =
"Required to run this model. Downloaded once, shared across compatible variants.";
function variantSuffix(job: RowSubject): string {
+ if (isTrainingDownloadVariant(job.variant)) return " · Training model";
if (job.variant?.startsWith("@")) {
return ` · ${
isRequiredAssetJob(job)
@@ -137,6 +140,9 @@ function RowTitle({ job }: { job: RowSubject }) {
}
function RowDetail({ job }: { job: RowSubject }) {
+ if (isTrainingDownloadVariant(job.variant)) {
+ return <div className="text-ui-10p5 text-muted-foreground">Current files; additional model files may follow.</div>;
+ }
return job.presentation ? (
<div className="truncate text-ui-10p5 text-muted-foreground">
{job.presentation.filename}
@@ -255,7 +261,7 @@ function DownloadRow({
bytesPerSec={job.bytesPerSec}
cancelling={job.state === "cancelling"}
etaSeconds={job.etaSeconds}
- activity={job.activity}
+ activity={job.activity ?? (isTrainingDownloadVariant(job.variant) ? "Preparing training model files…" : undefined)}
/>
) : null}
{!starting && job.details?.length ? (
@@ -283,6 +289,7 @@ export function DownloadManagerPanel({
const pathname = useRouterState({ select: (s) => s.location.pathname });
const enabled = canUseDownloadManager(pathname);
useEngines(enabled, true);
+ useTrainingDownloads(enabled);
useHubDownloadQueue();
const [collapsed, setCollapsed] = useState(false);
diff --git a/studio/frontend/src/features/hub/download-manager/download-manager-types.ts b/studio/frontend/src/features/hub/download-manager/download-manager-types.ts
index f28493c177d..61fc218e1fd 100644
--- a/studio/frontend/src/features/hub/download-manager/download-manager-types.ts
+++ b/studio/frontend/src/features/hub/download-manager/download-manager-types.ts
@@ -168,6 +168,7 @@ export interface JobRuntime {
}
export interface ProgressLike {
+ progress_scope?: "active_files" | null;
downloaded_bytes: number;
completed_bytes?: number;
complete_on_disk?: boolean;
diff --git a/studio/frontend/src/features/hub/download-manager/progress-reconcile.ts b/studio/frontend/src/features/hub/download-manager/progress-reconcile.ts
index fc248684d22..55dc877a631 100644
--- a/studio/frontend/src/features/hub/download-manager/progress-reconcile.ts
+++ b/studio/frontend/src/features/hub/download-manager/progress-reconcile.ts
@@ -47,6 +47,19 @@ export function resolveProgressUpdate(
fraction: number;
madeProgress: boolean;
} {
+ if (progressResp.progress_scope === "active_files") {
+ // Current Xet files, not a repo plan: batches reset counters (even to zero).
+ const expected = Math.max(0, finiteReading(progressResp.expected_bytes));
+ const downloadedBytes = expected > 0
+ ? Math.min(expected, Math.max(0, finiteReading(progressResp.downloaded_bytes)))
+ : 0;
+ return {
+ expected, downloadedBytes, measuredTransfer: expected > 0,
+ completedBytes: 0, completeOnDisk: false,
+ fraction: expected > 0 ? Math.min(MAX_PROGRESS_FRACTION, downloadedBytes / expected) : 0,
+ madeProgress: expected !== job.expectedBytes || downloadedBytes !== job.downloadedBytes,
+ };
+ }
const reported = finiteReading(progressResp.expected_bytes);
const isGgufVariantJob =
job.kind === DOWNLOAD_KIND.MODEL && job.variant !== null;
diff --git a/studio/frontend/src/features/hub/download-manager/training-downloads.ts b/studio/frontend/src/features/hub/download-manager/training-downloads.ts
new file mode 100644
index 00000000000..85fef1b3ba7
--- /dev/null
+++ b/studio/frontend/src/features/hub/download-manager/training-downloads.ts
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import type { ActiveModelDownload } from "./api.ts";
+
+export function isTrainingDownloadVariant(
+ variant: string | null | undefined,
+): boolean {
+ return /^@training-[0-9a-f]{32}$/.test(variant ?? "");
+}
+
+/** Only real server-owned jobs are adopted; a selected model alone is no download. */
+export function trainingDownloads(downloads: readonly ActiveModelDownload[]) {
+ return downloads.filter(
+ (job) =>
+ job.owner === "training" &&
+ isTrainingDownloadVariant(job.variant) &&
+ Boolean(job.repo_id?.trim()) &&
+ Number.isSafeInteger(job.generation) &&
+ (job.state === "running" || job.state === "cancelling"),
+ );
+}
+
+/** Drop late responses after a session switch or global-view teardown. */
+export class TrainingDownloadDiscovery {
+ private epoch = 0;
+ private disposed = false;
+ private abort = new AbortController();
+ private ports: {
+ fetch: (signal: AbortSignal) => Promise<readonly ActiveModelDownload[]>;
+ session: () => string;
+ adopt: (job: ActiveModelDownload) => void;
+ };
+
+ constructor(ports: TrainingDownloadDiscovery["ports"]) {
+ this.ports = ports;
+ }
+
+ async poll(): Promise<boolean> {
+ const epoch = this.epoch;
+ const session = this.ports.session();
+ const current = () =>
+ !this.disposed &&
+ epoch === this.epoch &&
+ session === this.ports.session();
+ try {
+ const jobs = await this.ports.fetch(this.abort.signal);
+ if (current())
+ for (const job of trainingDownloads(jobs)) this.ports.adopt(job);
+ } catch {
+ /* Transient failures retain regular manager rows and their own polling. */
+ }
+ return current();
+ }
+
+ reset(): void {
+ this.epoch += 1;
+ this.abort.abort();
+ this.abort = new AbortController();
+ }
+
+ dispose(): void {
+ this.disposed = true;
+ this.reset();
+ }
+}
diff --git a/studio/frontend/src/features/hub/download-manager/use-training-downloads.tsx b/studio/frontend/src/features/hub/download-manager/use-training-downloads.tsx
new file mode 100644
index 00000000000..459beca070e
--- /dev/null
+++ b/studio/frontend/src/features/hub/download-manager/use-training-downloads.tsx
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { useEffect } from "react";
+import {
+ AUTH_SESSION_CLEARED_EVENT,
+ AUTH_SESSION_MARK_KEY,
+ AUTH_SESSION_STORED_EVENT,
+ getAuthSessionEpoch,
+} from "@/features/auth/session";
+import {
+ ModelLoadDescription,
+ MODEL_LOAD_TOAST_CLASSNAMES,
+} from "@/features/chat/components/model-load-status";
+import { toast } from "@/lib/toast";
+import { formatBytes } from "../lib/format";
+import { getAllActiveModelDownloads } from "./api";
+import { DOWNLOAD_KIND } from "./constants";
+import {
+ getState,
+ jobKeyOf,
+ removeJob,
+ useDownloadManagerStore,
+} from "./download-manager-state";
+import { adoptJob } from "./poll-loop";
+import {
+ isTrainingDownloadVariant,
+ TrainingDownloadDiscovery,
+} from "./training-downloads";
+
+/** Discover registry jobs globally; the regular manager owns polling and cancellation. */
+export function useTrainingDownloads(enabled: boolean): void {
+ useEffect(() => {
+ if (!enabled) return;
+ let disposed = false;
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const session = () =>
+ `${getAuthSessionEpoch()}:${localStorage.getItem(AUTH_SESSION_MARK_KEY)}`;
+ const notices = new Set<string>();
+ const dismissed = new Set<string>();
+ const clearNotices = () => {
+ for (const id of notices) toast.dismiss(id);
+ notices.clear();
+ dismissed.clear();
+ };
+ const render = () => {
+ const live = new Set<string>();
+ for (const job of Object.values(getState().jobs)) {
+ if (!isTrainingDownloadVariant(job.variant)) continue;
+ const id = `training-download:${job.key}`;
+ if (job.state !== "running" && job.state !== "cancelling") continue;
+ live.add(id);
+ if (dismissed.has(id)) continue;
+ notices.add(id);
+ const known = job.expectedBytes > 0;
+ toast(null, {
+ id,
+ duration: Infinity,
+ closeButton: true,
+ classNames: MODEL_LOAD_TOAST_CLASSNAMES,
+ description: (
+ <ModelLoadDescription
+ title={
+ job.state === "cancelling"
+ ? "Stopping training download…"
+ : known
+ ? `Downloading ${job.repoId.split("/").at(-1)}…`
+ : `Preparing ${job.repoId.split("/").at(-1)} download…`
+ }
+ message="Preparing model files for training…"
+ progressPercent={known ? job.fraction * 100 : null}
+ progressLabel={
+ known
+ ? `${formatBytes(job.downloadedBytes)} of ${formatBytes(job.expectedBytes)} · current files`
+ : null
+ }
+ />
+ ),
+ onDismiss: () => {
+ dismissed.add(id);
+ notices.delete(id);
+ },
+ });
+ }
+ for (const id of notices) {
+ if (!live.has(id)) {
+ toast.dismiss(id);
+ notices.delete(id);
+ }
+ }
+ };
+ const unsubscribe = useDownloadManagerStore.subscribe(render);
+ const discovery = new TrainingDownloadDiscovery({
+ fetch: (signal) => getAllActiveModelDownloads(signal, { fresh: true }),
+ session,
+ adopt: (job) => {
+ const repoId = job.repo_id!.trim();
+ const key = jobKeyOf(DOWNLOAD_KIND.MODEL, repoId, job.variant);
+ const existing = getState().jobs[key];
+ if (
+ existing &&
+ existing.serverGeneration === job.generation &&
+ (existing.state === "running" || existing.state === "cancelling")
+ )
+ return;
+ adoptJob(
+ {
+ kind: DOWNLOAD_KIND.MODEL,
+ repoId,
+ variant: job.variant,
+ expectedBytes: 0,
+ },
+ job.generation,
+ job.state,
+ );
+ },
+ });
+ const poll = async () => {
+ if ((await discovery.poll()) && !disposed)
+ timer = setTimeout(() => void poll(), 1500);
+ };
+ const reset = () => {
+ discovery.reset();
+ clearTimeout(timer);
+ for (const job of Object.values(getState().jobs)) {
+ if (isTrainingDownloadVariant(job.variant)) removeJob(job.key);
+ }
+ clearNotices();
+ void poll();
+ };
+ const storage = (event: StorageEvent) => {
+ if (event.key === AUTH_SESSION_MARK_KEY) reset();
+ };
+ window.addEventListener(AUTH_SESSION_CLEARED_EVENT, reset);
+ window.addEventListener(AUTH_SESSION_STORED_EVENT, reset);
+ window.addEventListener("storage", storage);
+ render();
+ void poll();
+ return () => {
+ disposed = true;
+ discovery.dispose();
+ clearTimeout(timer);
+ unsubscribe();
+ clearNotices();
+ window.removeEventListener(AUTH_SESSION_CLEARED_EVENT, reset);
+ window.removeEventListener(AUTH_SESSION_STORED_EVENT, reset);
+ window.removeEventListener("storage", storage);
+ };
+ }, [enabled]);
+}
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts
index 0da5b5f9ad5..1d3c8c81271 100644
--- a/studio/frontend/src/features/images/api.ts
+++ b/studio/frontend/src/features/images/api.ts
@@ -689,6 +689,7 @@ export interface DiffusionTrainingStatus {
num_images: number | null;
in_model_load: boolean;
preparation?: { stage?: string | null; done?: number | null; total?: number | null } | null;
+ download_progress?: { bytes_done: number; bytes_total: number; label?: string; scope?: "active_files" } | null;
output_dir: string | null;
lora_path: string | null;
ema_path?: string | null;
diff --git a/studio/frontend/src/features/images/train/diffusion-train-progress.ts b/studio/frontend/src/features/images/train/diffusion-train-progress.ts
index 25471c5dbef..46dfe61f38c 100644
--- a/studio/frontend/src/features/images/train/diffusion-train-progress.ts
+++ b/studio/frontend/src/features/images/train/diffusion-train-progress.ts
@@ -12,6 +12,7 @@ type Status = Pick<
| "in_model_load"
| "message"
| "preparation"
+ | "download_progress"
>;
const PHASE_MESSAGE =
diff --git a/studio/frontend/src/lib/use-diffusion-pick-toast.tsx b/studio/frontend/src/lib/use-diffusion-pick-toast.tsx
index 8a32981e3a7..f0ca010c81c 100644
--- a/studio/frontend/src/lib/use-diffusion-pick-toast.tsx
+++ b/studio/frontend/src/lib/use-diffusion-pick-toast.tsx
@@ -4,6 +4,7 @@
import { useEffect, useState } from "react";
import { ModelLoadDescription } from "@/features/chat";
+import { MODEL_LOAD_TOAST_CLASSNAMES } from "@/features/chat/components/model-load-status";
import { formatBytes, type StagedDownloadProgress } from "@/features/hub";
import {
createPickToast,
@@ -12,13 +13,6 @@ import {
type PickToastProgress,
} from "@/lib/diffusion-pick-toast";
-const PICK_TOAST_CLASSNAMES = {
- toast: "chat-model-load-toast items-center gap-2.5",
- content: "gap-0.5 flex-1 min-w-0",
- title: "leading-5",
- description: "mt-0 w-full",
-};
-
function describePick(phase: PickToastPhase, progress: PickToastProgress | null, modelName?: string) {
if (phase === "downloading") {
return (
@@ -70,7 +64,7 @@ function describePick(phase: PickToastPhase, progress: PickToastProgress | null,
export function useDiffusionPickToast(): PickToast {
const [pickToast] = useState(() =>
- createPickToast({ describe: describePick, classNames: PICK_TOAST_CLASSNAMES }),
+ createPickToast({ describe: describePick, classNames: MODEL_LOAD_TOAST_CLASSNAMES }),
);
useEffect(() => () => pickToast.dismissAll(), [pickToast]);
return pickToast;
diff --git a/studio/frontend/tests/diffusion-train-progress.test.ts b/studio/frontend/tests/diffusion-train-progress.test.ts
index 287240144dd..889754a63fe 100644
--- a/studio/frontend/tests/diffusion-train-progress.test.ts
+++ b/studio/frontend/tests/diffusion-train-progress.test.ts
@@ -154,3 +154,31 @@ test("encoded captions alone do not read as prepared training data", () => {
assert.equal(progress.title, "Preparing training data");
assert.equal(progress.percent, 100);
});
+
+
+test("download bytes belong to the global manager, not the inline preparation bar", () => {
+ const progress = diffusionTrainingProgress({ ...running, download_progress: {
+ bytes_done: 2 * 1024 ** 3, bytes_total: 4 * 1024 ** 3, label: "model.safetensors",
+ } });
+ assert.equal(progress.title, "Loading training model");
+ assert.equal(progress.percent, null);
+ assert.equal(progress.count, null);
+});
+
+test("download progress yields to preparation and actual training", () => {
+ const download_progress = { bytes_done: 1, bytes_total: 10 };
+ const prep = diffusionTrainingProgress({ ...running, download_progress,
+ preparation: { stage: "cache_latents", done: 8, total: 10 } });
+ assert.equal(prep.title, "Preparing training data");
+ assert.equal(prep.percent, 80);
+ const train = diffusionTrainingProgress({ ...running, in_model_load: false, step: 20, download_progress });
+ assert.equal(train.title, "Training");
+ assert.equal(train.percent, 20);
+});
+
+test("unknown download totals never fabricate a percentage", () => {
+ for (const total of [0, NaN, Infinity]) {
+ assert.equal(diffusionTrainingProgress({ ...running,
+ download_progress: { bytes_done: 1, bytes_total: total } }).percent, null);
+ }
+});
diff --git a/studio/frontend/tests/training-downloads.test.ts b/studio/frontend/tests/training-downloads.test.ts
new file mode 100644
index 00000000000..ef3994d0d02
--- /dev/null
+++ b/studio/frontend/tests/training-downloads.test.ts
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+ TrainingDownloadDiscovery,
+ trainingDownloads,
+} from "../src/features/hub/download-manager/training-downloads.ts";
+import { registerBundlerResolver } from "./helpers/kit.ts";
+import type { ActiveModelDownload } from "../src/features/hub/download-manager/api.ts";
+import type { ManagedDownload } from "../src/features/hub/download-manager/download-manager-types.ts";
+
+registerBundlerResolver();
+const { resolveProgressUpdate } = await import(
+ "../src/features/hub/download-manager/progress-reconcile.ts"
+);
+
+const job: ActiveModelDownload = {
+ repo_id: "unsloth/FLUX.1-dev",
+ variant: `@training-${"a".repeat(32)}`,
+ owner: "training",
+ generation: 4,
+ state: "running",
+};
+
+test("discovery uses real registry identity and never adopts another subsystem's jobs", () => {
+ assert.deepEqual(
+ trainingDownloads([
+ job,
+ { ...job, owner: "load" },
+ { ...job, variant: "Q4_K_M" },
+ { ...job, state: "complete" },
+ { ...job, generation: undefined },
+ ]),
+ [job],
+ );
+});
+
+test("global discovery adopts a live training download after navigating or reloading", async () => {
+ const adopted: ActiveModelDownload[] = [];
+ const discovery = new TrainingDownloadDiscovery({
+ fetch: async () => [job],
+ session: () => "account-a",
+ adopt: (row) => adopted.push(row),
+ });
+ assert.equal(await discovery.poll(), true);
+ assert.deepEqual(adopted, [job]);
+ discovery.dispose();
+});
+
+test("session changes and teardown discard in-flight adoption responses", async () => {
+ for (const action of ["switch", "reset", "dispose"]) {
+ let session = "account-a";
+ let resolve!: (rows: ActiveModelDownload[]) => void;
+ const adopted: ActiveModelDownload[] = [];
+ const discovery = new TrainingDownloadDiscovery({
+ fetch: () =>
+ new Promise((done) => {
+ resolve = done;
+ }),
+ session: () => session,
+ adopt: (row) => adopted.push(row),
+ });
+ const polling = discovery.poll();
+ if (action === "switch") session = "account-b";
+ else if (action === "reset") discovery.reset();
+ else discovery.dispose();
+ resolve([job]);
+ assert.equal(await polling, false);
+ assert.deepEqual(adopted, []);
+ discovery.dispose();
+ }
+});
+
+test("current file batches reset truthfully instead of reusing an earlier total or percentage", () => {
+ const previous = {
+ kind: "model",
+ variant: job.variant,
+ expectedBytes: 22,
+ downloadedBytes: 13,
+ completedBytes: 0,
+ fraction: 0.6,
+ } as ManagedDownload;
+ const next = resolveProgressUpdate(previous, {
+ progress_scope: "active_files",
+ downloaded_bytes: 1,
+ expected_bytes: 4,
+ progress: 0.25,
+ complete_on_disk: true,
+ completed_bytes: 4,
+ });
+ assert.equal(next.downloadedBytes, 1);
+ assert.equal(next.expected, 4);
+ assert.equal(next.fraction, 0.25);
+ assert.equal(next.completeOnDisk, false);
+ assert.equal(next.completedBytes, 0);
+ const gap = resolveProgressUpdate(previous, {
+ progress_scope: "active_files",
+ downloaded_bytes: 0,
+ expected_bytes: 0,
+ progress: 0,
+ });
+ assert.equal(gap.expected, 0);
+ assert.equal(gap.downloadedBytes, 0);
+ assert.equal(gap.fraction, 0);
+});