Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: overlap model checkpoint downloads with dataset preparation (#15483)
Daniel Hane4251c58e625
diff --git a/studio/backend/core/training/model_weight_prefetch.py b/studio/backend/core/training/model_weight_prefetch.pynew file mode 100644index 00000000000..e2d351a0958--- /dev/null+++ b/studio/backend/core/training/model_weight_prefetch.py@@ -0,0 +1,214 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Opportunistic checkpoint I/O; model loading and dataset preparation stay on the main thread."""++from __future__ import annotations++import json+import os+import re+import threading+import time+from contextvars import copy_context+from pathlib import Path++from loggers import get_logger+from utils.account_context import account_thread++logger = get_logger(__name__)+_SHARD = re.compile(r"model-[0-9]+-of-[0-9]+\.safetensors\Z")+++def prefetch_repo(+ config,+ trainer,+ *,+ load_in_4bit,+ capabilities,+ offline = False,+ formatting_can_fork = False,+):+ """Only routes whose ordinary root checkpoint can be resolved without loading model code."""+ name = config.get("model_name", "")+ if (+ offline+ or formatting_can_fork+ or trainer.should_stop+ or not isinstance(name, str)+ or not re.fullmatch(r"[\w.-]+/[\w.-]+", name)+ or Path(name).expanduser().exists()+ or config.get("training_type", "LoRA/QLoRA") != "LoRA/QLoRA"+ or any(+ config.get(key)+ for key in (+ "model_snapshot_path",+ "model_revision",+ "resume_from_checkpoint",+ "require_exact_resume_resources",+ "require_exact_model_resource",+ "trust_remote_code",+ "is_embedding",+ "is_decision",+ "is_dataset_audio",+ )+ )+ or getattr(trainer, "is_audio", False)+ or getattr(trainer, "is_audio_vlm", False)+ or getattr(trainer, "_audio_type", None)+ or getattr(trainer, "diffusion_lm_profile", None)+ or not getattr(trainer, "_audio_type_known", False)+ or capabilities is None+ ):+ return None+ allow_bnb = getattr(capabilities, "ALLOW_BITSANDBYTES", None)+ allow_prequant = getattr(capabilities, "ALLOW_PREQUANTIZED_MODELS", None)+ if not isinstance(allow_bnb, bool) or allow_prequant is not True:+ # The loader can strip/remap prequants on unsupported stacks; do not speculate.+ return None+ from utils.models.unsloth_mirror import mirror_lookup_available, unsloth_public_mirror++ if not mirror_lookup_available():+ return None+ quantized = load_in_4bit() if callable(load_in_4bit) else load_in_4bit+ return unsloth_public_mirror(name, bool(quantized and allow_bnb)) or name+++def checkpoint_files(files, read_json):+ """Pick one standard safetensors representation, never all weights/quantizations in a repo."""+ files = set(files)+ if "adapter_config.json" in files or "config.json" not in files:+ return []+ config = read_json("config.json")+ if (+ not isinstance(config, dict)+ or config.get("auto_map")+ or config.get("base_model_name_or_path")+ ):+ return []+ if "model.safetensors" in files:+ return ["model.safetensors"]+ index_name = "model.safetensors.index.json"+ if index_name not in files:+ return []+ index = read_json(index_name)+ weights = index.get("weight_map") if isinstance(index, dict) else None+ if not isinstance(weights, dict) or not weights:+ return []+ if any(not isinstance(name, str) for name in weights.values()):+ return []+ shards = set(weights.values())+ if len(shards) > 1024 or any(+ not isinstance(name, str) or not _SHARD.fullmatch(name) or name not in files+ for name in shards+ ):+ return []+ return sorted(shards)+++class ModelWeightPrefetch:+ def __init__(self, repo, *, token, cache_dir, emit, should_stop):+ self.repo, self.token, self.cache_dir = repo, token, str(cache_dir)+ self._emit, self._should_stop = emit, should_stop+ self._cancel = threading.Event()+ self._closed = threading.Event()+ self._lock = threading.Lock()+ self._thread = None+ self._watchdog_stop = None++ def _send(self, kind, **kwargs):+ with self._lock:+ if not self._closed.is_set():+ self._emit({"type": kind, **kwargs, "ts": time.time()})++ def _stopped(self):+ return self._cancel.is_set() or self._should_stop()++ def _download(self):+ from huggingface_hub import HfApi, hf_hub_download+ from utils.hf_xet_fallback import start_watchdog++ info = HfApi().model_info(self.repo, token = self.token)+ if self._stopped():+ return+ commit = info.sha+ if not isinstance(commit, str) or not re.fullmatch(r"[a-fA-F0-9]{40}", commit):+ return+ self._send("model_load_resolved", repo_id = self.repo)+ self._watchdog_stop = start_watchdog(+ repo_ids = [self.repo],+ on_stall = lambda message: self._send("stall", message = message),+ xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",+ )++ def fetch(filename):+ if self._stopped():+ raise InterruptedError("Training model prefetch cancelled")+ return hf_hub_download(+ repo_id = self.repo,+ filename = filename,+ revision = commit,+ token = self.token,+ cache_dir = self.cache_dir,+ )++ def read_json(filename):+ path = Path(fetch(filename))+ if path.stat().st_size > 5 * 1024**2:+ raise ValueError(+ "Unfamiliar checkpoint metadata; leave selection to the model loader"+ )+ return json.loads(path.read_text(encoding = "utf-8"))++ files = checkpoint_files([sibling.rfilename for sibling in info.siblings or ()], read_json)+ for filename in files:+ fetch(filename)++ def _run(self):+ try:+ if not self._stopped():+ self._download()+ except Exception as error: # noqa: BLE001 -- the authoritative loader handles failures later+ logger.info("Model checkpoint prefetch skipped for %s: %s", self.repo, error)+ finally:+ if self._watchdog_stop is not None:+ self._watchdog_stop.set()++ def start(self):+ # Account identity and other scopes (cache/download/security) must survive the new thread.+ context = copy_context()+ self._send("model_load_started")+ self._thread = account_thread(target = lambda: context.run(self._run), daemon = True)+ self._thread.start()++ def finish(self, *, wait):+ try:+ while wait and self._thread.is_alive() and not self._should_stop():+ self._thread.join(timeout = 0.2)+ finally:+ self._cancel.set()+ if self._watchdog_stop is not None:+ self._watchdog_stop.set()+ # A Hub call still exiting on Stop/error must not emit into the next phase.+ with self._lock:+ self._closed.set()+ self._emit({"type": "model_load_completed", "ts": time.time()})+++def prepare_dataset_with_prefetch(load_dataset, prefetch):+ """Do not consume streaming rows, copy datasets, or move tokenizer/GPU work into a thread."""+ if prefetch is None:+ return load_dataset()+ try:+ prefetch.start()+ except Exception as error: # noqa: BLE001 -- an optimization cannot prevent the normal load+ logger.info("Could not start model checkpoint prefetch: %s", error)+ prefetch.finish(wait = False)+ return load_dataset()+ ready = False+ try:+ result = load_dataset()+ ready = result is not None and result[0] is not None+ return result+ finally:+ prefetch.finish(wait = ready)diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 027ea83f941..bda8d024131 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -234,6 +234,38 @@ def _effective_training_load_in_4bit(return effective_training_load_in_4bit(config, model_load_target, hf_token)+def _training_weight_prefetch(config, trainer, hf_token, event_queue):+ """Resolve only the ordinary SFT checkpoint; uncertain routes keep the existing sequential load."""+ try:+ from .model_weight_prefetch import ModelWeightPrefetch, prefetch_repo+ from utils.hf_cache_settings import get_hf_cache_paths+ from utils.hardware import dataset_map_num_proc++ repo = prefetch_repo(+ config,+ trainer,+ load_in_4bit = lambda: _effective_training_load_in_4bit(+ config, _resolve_cached_model_load_name(config), hf_token+ ),+ capabilities = sys.modules.get("unsloth.device_type"),+ offline = _model_offline_mode_enabled(),+ formatting_can_fork = (+ not config.get("dataset_streaming", False) and (dataset_map_num_proc() or 1) > 1+ ),+ )+ if repo is not None:+ return ModelWeightPrefetch(+ repo,+ token = hf_token,+ cache_dir = get_hf_cache_paths().hub_cache,+ emit = event_queue.put,+ should_stop = lambda: trainer.should_stop,+ )+ except Exception as error: # noqa: BLE001+ logger.info("Skipping model checkpoint prefetch: %s", error)+ return None++def _drop_model_pin(config: dict) -> str:config["model_snapshot_path"] = Nonereturn config["model_name"]@@ -4880,7 +4912,12 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->return_send_status(event_queue, "Loading and formatting dataset...")- dataset, eval_dataset = _load_training_dataset()+ from .model_weight_prefetch import prepare_dataset_with_prefetch++ dataset, eval_dataset = prepare_dataset_with_prefetch(+ _load_training_dataset,+ _training_weight_prefetch(config, trainer, hf_token, event_queue),+ )if dataset is None or trainer.should_stop:if trainer.should_stop:diff --git a/studio/backend/tests/test_training_model_weight_prefetch.py b/studio/backend/tests/test_training_model_weight_prefetch.pynew file mode 100644index 00000000000..0ecc83d3b7e--- /dev/null+++ b/studio/backend/tests/test_training_model_weight_prefetch.py@@ -0,0 +1,358 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import json+import sys+import threading+from contextvars import ContextVar+from types import SimpleNamespace++import pytest++from core.training import model_weight_prefetch as prefetch+from utils.account_context import AccountContext, current_account_id, run_as+++def test_checkpoint_selection_chooses_single_safetensors_without_other_formats():+ files = [+ "config.json",+ "model.safetensors",+ "model.safetensors.index.json",+ "pytorch_model.bin",+ "q4.gguf",+ ]+ reads = []+ assert prefetch.checkpoint_files(files, lambda name: reads.append(name) or {}) == [+ "model.safetensors"+ ]+ assert reads == ["config.json"]+++def test_checkpoint_selection_uses_only_exact_index_shards():+ first, second = "model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors"+ metadata = {+ "config.json": {},+ "model.safetensors.index.json": {"weight_map": {"a": second, "b": first, "c": first}},+ }+ assert prefetch.checkpoint_files(+ [*metadata, first, second, "other.safetensors"], metadata.__getitem__+ ) == [first, second]+++ "shard",+ [+ "../secret.safetensors",+ "/model.safetensors",+ "folder/model-1-of-1.safetensors",+ "model-1-of-1.bin",+ 123,+ ["bad"],+ ],+)+def test_unfamiliar_index_paths_are_not_downloaded(shard):+ metadata = {"config.json": {}, "model.safetensors.index.json": {"weight_map": {"a": shard}}}+ assert prefetch.checkpoint_files(metadata, metadata.__getitem__) == []+++ "config",+ [{"auto_map": {"AutoModel": "custom.Model"}}, {"base_model_name_or_path": "other/base"}],+)+def test_custom_and_adapter_configs_are_left_to_loader(config):+ assert (+ prefetch.checkpoint_files(["config.json", "model.safetensors"], lambda _name: config) == []+ )+ assert (+ prefetch.checkpoint_files(+ ["adapter_config.json", "config.json", "model.safetensors"],+ lambda _name: pytest.fail("adapter metadata must not be fetched"),+ )+ == []+ )+++def mirror(monkeypatch):+ import utils.models.unsloth_mirror as mapping++ seen = []+ monkeypatch.setattr(mapping, "mirror_lookup_available", lambda: True)+ monkeypatch.setattr(+ mapping,+ "unsloth_public_mirror",+ lambda name, quant: seen.append((name, quant))+ or ("unsloth/model-bnb-4bit" if quant else "unsloth/model"),+ )+ return seen+++def test_prefetch_resolves_the_effective_quantization(mirror):+ trainer = SimpleNamespace(should_stop = False, _audio_type_known = True)+ capabilities = SimpleNamespace(ALLOW_BITSANDBYTES = True, ALLOW_PREQUANTIZED_MODELS = True)+ config = {"model_name": "upstream/model", "dataset_streaming": True}+ assert (+ prefetch.prefetch_repo(config, trainer, load_in_4bit = True, capabilities = capabilities)+ == "unsloth/model-bnb-4bit"+ )+ assert (+ prefetch.prefetch_repo(config, trainer, load_in_4bit = False, capabilities = capabilities)+ == "unsloth/model"+ )+ capabilities.ALLOW_BITSANDBYTES = False+ assert (+ prefetch.prefetch_repo(config, trainer, load_in_4bit = True, capabilities = capabilities)+ == "unsloth/model"+ )+ assert mirror == [+ ("upstream/model", True),+ ("upstream/model", False),+ ("upstream/model", False),+ ]+++ "patch",+ [+ {"model_revision": "sha"},+ {"model_snapshot_path": "snapshot"},+ {"resume_from_checkpoint": "checkpoint"},+ {"require_exact_model_resource": True},+ {"require_exact_resume_resources": True},+ {"trust_remote_code": True},+ {"training_type": "Full Finetuning"},+ {"is_dataset_audio": True},+ {"is_decision": True},+ {"is_embedding": True},+ {"model_name": "/local/model"},+ ],+)+def test_pinned_and_uncertain_jobs_do_not_prefetch(patch, mirror):+ trainer = SimpleNamespace(should_stop = False, _audio_type_known = True)+ capabilities = SimpleNamespace(ALLOW_BITSANDBYTES = True, ALLOW_PREQUANTIZED_MODELS = True)+ assert (+ prefetch.prefetch_repo(+ {"model_name": "upstream/model", **patch},+ trainer,+ load_in_4bit = True,+ capabilities = capabilities,+ )+ is None+ )+ assert mirror == []+++def test_unproven_runtime_and_offline_jobs_do_not_prefetch(mirror):+ trainer = SimpleNamespace(should_stop = False, _audio_type_known = True)+ config = {"model_name": "upstream/model"}+ for capabilities in (+ None,+ SimpleNamespace(),+ SimpleNamespace(ALLOW_BITSANDBYTES = True, ALLOW_PREQUANTIZED_MODELS = False),+ ):+ assert (+ prefetch.prefetch_repo(config, trainer, load_in_4bit = True, capabilities = capabilities)+ is None+ )+ assert (+ prefetch.prefetch_repo(+ config, trainer, load_in_4bit = True, capabilities = object(), offline = True+ )+ is None+ )+ assert mirror == []+++ "num_proc, streaming, expected",+ [(None, False, True), (1, False, True), (4, False, False), (4, True, True)],+)+def test_worker_uses_prefetch_only_without_forking_preparation(+ monkeypatch, tmp_path, mirror, num_proc, streaming, expected+):+ from core.training import worker+ import utils.hardware+ import utils.hf_cache_settings++ capabilities = SimpleNamespace(ALLOW_BITSANDBYTES = True, ALLOW_PREQUANTIZED_MODELS = True)+ monkeypatch.setitem(sys.modules, "unsloth.device_type", capabilities)+ monkeypatch.setattr(worker, "_model_offline_mode_enabled", lambda: False)+ resolved = []+ monkeypatch.setattr(+ worker, "_effective_training_load_in_4bit", lambda *_a: resolved.append(True) or False+ )+ monkeypatch.setattr(utils.hardware, "dataset_map_num_proc", lambda: num_proc)+ monkeypatch.setattr(+ utils.hf_cache_settings, "get_hf_cache_paths", lambda: SimpleNamespace(hub_cache = tmp_path)+ )+ job = worker._training_weight_prefetch(+ {"model_name": "upstream/model", "dataset_streaming": streaming},+ SimpleNamespace(should_stop = False, _audio_type_known = True),+ False,+ SimpleNamespace(put = lambda _ev: None),+ )+ assert (job is not None) is expected+ assert bool(resolved) is expected+ if job:+ assert job.repo == "unsloth/model"+ assert job.token is False+ assert job.cache_dir == str(tmp_path)+++def fake_hub(monkeypatch, tmp_path, *, download):+ commit = "a" * 40+ info = SimpleNamespace(+ sha = commit,+ siblings = [+ SimpleNamespace(rfilename = name)+ for name in ("config.json", "model.safetensors", "pytorch_model.bin")+ ],+ )+ calls = []++ def fetch(**kwargs):+ calls.append(kwargs)+ path = tmp_path / kwargs["filename"]+ if kwargs["filename"] == "config.json":+ path.write_text(json.dumps({}))+ else:+ download()+ path.write_bytes(b"weights")+ return str(path)++ watchdog = threading.Event()+ monkeypatch.setitem(+ sys.modules,+ "huggingface_hub",+ SimpleNamespace(+ HfApi = lambda: SimpleNamespace(model_info = lambda *a, **kw: info), hf_hub_download = fetch+ ),+ )+ monkeypatch.setitem(+ sys.modules, "utils.hf_xet_fallback", SimpleNamespace(start_watchdog = lambda **_kw: watchdog)+ )+ return calls, watchdog+++def test_model_io_overlaps_dataset_preparation_and_keeps_context(monkeypatch, tmp_path):+ entered, finish = threading.Event(), threading.Event()+ custom = ContextVar("prefetch_test_context", default = "unset")+ observed = []++ def download():+ observed.append((current_account_id(), custom.get(), threading.current_thread().ident))+ entered.set()+ assert finish.wait(3)++ calls, watchdog = fake_hub(monkeypatch, tmp_path, download = download)+ events = []+ account = AccountContext("test-account", "test")+ context_token = custom.set("captured")+ main_thread = threading.current_thread().ident++ class LazyDataset:+ def __iter__(self):+ pytest.fail("startup must not consume streaming rows")++ dataset = LazyDataset()++ def load_dataset():+ assert entered.wait(3), "weight I/O must already be in progress during dataset preparation"+ assert not watchdog.is_set()+ finish.set()+ return dataset, None++ def run():+ job = prefetch.ModelWeightPrefetch(+ "unsloth/model",+ token = False,+ cache_dir = tmp_path / "account-cache",+ emit = events.append,+ should_stop = lambda: False,+ )+ assert prefetch.prepare_dataset_with_prefetch(load_dataset, job) == (dataset, None)+ assert (+ not job._thread.is_alive()+ ), "actual model placement starts after background I/O finishes"++ try:+ run_as(account, run)+ finally:+ custom.reset(context_token)+ finish.set()+ assert observed == [("test-account", "captured", observed[0][2])]+ assert observed[0][2] != main_thread+ assert [call["filename"] for call in calls] == ["config.json", "model.safetensors"]+ assert all(+ call["revision"] == "a" * 40+ and call["token"] is False+ and call["cache_dir"] == str(tmp_path / "account-cache")+ for call in calls+ )+ assert [event["type"] for event in events] == [+ "model_load_started",+ "model_load_resolved",+ "model_load_completed",+ ]+ assert watchdog.is_set()++[email protected]("outcome", ["error", "empty", "stop"])+def test_failed_or_stopped_dataset_does_not_wait_for_download(monkeypatch, tmp_path, outcome):+ entered, release = threading.Event(), threading.Event()+ calls, watchdog = fake_hub(+ monkeypatch, tmp_path, download = lambda: (entered.set(), release.wait(3))+ )+ events = []+ stopped = []+ job = prefetch.ModelWeightPrefetch(+ "unsloth/model",+ token = "token",+ cache_dir = tmp_path,+ emit = events.append,+ should_stop = lambda: bool(stopped),+ )++ def load_dataset():+ assert entered.wait(3)+ if outcome == "error":+ raise ValueError("invalid dataset")+ if outcome == "stop":+ stopped.append(True)+ return (None, None) if outcome == "empty" else (object(), None)++ try:+ if outcome == "error":+ with pytest.raises(ValueError, match = "invalid dataset"):+ prefetch.prepare_dataset_with_prefetch(load_dataset, job)+ else:+ prefetch.prepare_dataset_with_prefetch(load_dataset, job)+ assert job._thread.is_alive(), "an in-flight Hub call must not block cancellation"+ assert watchdog.is_set()+ assert events[-1]["type"] == "model_load_completed"+ before = list(events)+ job._send("stall", message = "late")+ assert events == before+ finally:+ release.set()+ job._thread.join(timeout = 3)+ assert not job._thread.is_alive()+++def test_prefetch_failure_leaves_dataset_result_authoritative(monkeypatch, tmp_path):+ fake_hub(+ monkeypatch,+ tmp_path,+ download = lambda: (_ for _ in ()).throw(OSError("temporary Hub error")),+ )+ events = []+ dataset = object()+ job = prefetch.ModelWeightPrefetch(+ "unsloth/model",+ token = None,+ cache_dir = tmp_path,+ emit = events.append,+ should_stop = lambda: False,+ )+ assert prefetch.prepare_dataset_with_prefetch(lambda: (dataset, None), job) == (dataset, None)+ assert events[-1]["type"] == "model_load_completed"