Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: overlap model checkpoint downloads with dataset preparation (#15483)

Daniel Hane4251c58e625
diff --git a/studio/backend/core/training/model_weight_prefetch.py b/studio/backend/core/training/model_weight_prefetch.py
new file mode 100644
index 00000000000..e2d351a0958
--- /dev/null
+++ b/studio/backend/core/training/model_weight_prefetch.py
@@ -0,0 +1,214 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Opportunistic checkpoint I/O; model loading and dataset preparation stay on the main thread."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import threading
+import time
+from contextvars import copy_context
+from pathlib import Path
+
+from loggers import get_logger
+from utils.account_context import account_thread
+
+logger = get_logger(__name__)
+_SHARD = re.compile(r"model-[0-9]+-of-[0-9]+\.safetensors\Z")
+
+
+def prefetch_repo(
+ config,
+ trainer,
+ *,
+ load_in_4bit,
+ capabilities,
+ offline = False,
+ formatting_can_fork = False,
+):
+ """Only routes whose ordinary root checkpoint can be resolved without loading model code."""
+ name = config.get("model_name", "")
+ if (
+ offline
+ or formatting_can_fork
+ or trainer.should_stop
+ or not isinstance(name, str)
+ or not re.fullmatch(r"[\w.-]+/[\w.-]+", name)
+ or Path(name).expanduser().exists()
+ or config.get("training_type", "LoRA/QLoRA") != "LoRA/QLoRA"
+ or any(
+ config.get(key)
+ for key in (
+ "model_snapshot_path",
+ "model_revision",
+ "resume_from_checkpoint",
+ "require_exact_resume_resources",
+ "require_exact_model_resource",
+ "trust_remote_code",
+ "is_embedding",
+ "is_decision",
+ "is_dataset_audio",
+ )
+ )
+ or getattr(trainer, "is_audio", False)
+ or getattr(trainer, "is_audio_vlm", False)
+ or getattr(trainer, "_audio_type", None)
+ or getattr(trainer, "diffusion_lm_profile", None)
+ or not getattr(trainer, "_audio_type_known", False)
+ or capabilities is None
+ ):
+ return None
+ allow_bnb = getattr(capabilities, "ALLOW_BITSANDBYTES", None)
+ allow_prequant = getattr(capabilities, "ALLOW_PREQUANTIZED_MODELS", None)
+ if not isinstance(allow_bnb, bool) or allow_prequant is not True:
+ # The loader can strip/remap prequants on unsupported stacks; do not speculate.
+ return None
+ from utils.models.unsloth_mirror import mirror_lookup_available, unsloth_public_mirror
+
+ if not mirror_lookup_available():
+ return None
+ quantized = load_in_4bit() if callable(load_in_4bit) else load_in_4bit
+ return unsloth_public_mirror(name, bool(quantized and allow_bnb)) or name
+
+
+def checkpoint_files(files, read_json):
+ """Pick one standard safetensors representation, never all weights/quantizations in a repo."""
+ files = set(files)
+ if "adapter_config.json" in files or "config.json" not in files:
+ return []
+ config = read_json("config.json")
+ if (
+ not isinstance(config, dict)
+ or config.get("auto_map")
+ or config.get("base_model_name_or_path")
+ ):
+ return []
+ if "model.safetensors" in files:
+ return ["model.safetensors"]
+ index_name = "model.safetensors.index.json"
+ if index_name not in files:
+ return []
+ index = read_json(index_name)
+ weights = index.get("weight_map") if isinstance(index, dict) else None
+ if not isinstance(weights, dict) or not weights:
+ return []
+ if any(not isinstance(name, str) for name in weights.values()):
+ return []
+ shards = set(weights.values())
+ if len(shards) > 1024 or any(
+ not isinstance(name, str) or not _SHARD.fullmatch(name) or name not in files
+ for name in shards
+ ):
+ return []
+ return sorted(shards)
+
+
+class ModelWeightPrefetch:
+ def __init__(self, repo, *, token, cache_dir, emit, should_stop):
+ self.repo, self.token, self.cache_dir = repo, token, str(cache_dir)
+ self._emit, self._should_stop = emit, should_stop
+ self._cancel = threading.Event()
+ self._closed = threading.Event()
+ self._lock = threading.Lock()
+ self._thread = None
+ self._watchdog_stop = None
+
+ def _send(self, kind, **kwargs):
+ with self._lock:
+ if not self._closed.is_set():
+ self._emit({"type": kind, **kwargs, "ts": time.time()})
+
+ def _stopped(self):
+ return self._cancel.is_set() or self._should_stop()
+
+ def _download(self):
+ from huggingface_hub import HfApi, hf_hub_download
+ from utils.hf_xet_fallback import start_watchdog
+
+ info = HfApi().model_info(self.repo, token = self.token)
+ if self._stopped():
+ return
+ commit = info.sha
+ if not isinstance(commit, str) or not re.fullmatch(r"[a-fA-F0-9]{40}", commit):
+ return
+ self._send("model_load_resolved", repo_id = self.repo)
+ self._watchdog_stop = start_watchdog(
+ repo_ids = [self.repo],
+ on_stall = lambda message: self._send("stall", message = message),
+ xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",
+ )
+
+ def fetch(filename):
+ if self._stopped():
+ raise InterruptedError("Training model prefetch cancelled")
+ return hf_hub_download(
+ repo_id = self.repo,
+ filename = filename,
+ revision = commit,
+ token = self.token,
+ cache_dir = self.cache_dir,
+ )
+
+ def read_json(filename):
+ path = Path(fetch(filename))
+ if path.stat().st_size > 5 * 1024**2:
+ raise ValueError(
+ "Unfamiliar checkpoint metadata; leave selection to the model loader"
+ )
+ return json.loads(path.read_text(encoding = "utf-8"))
+
+ files = checkpoint_files([sibling.rfilename for sibling in info.siblings or ()], read_json)
+ for filename in files:
+ fetch(filename)
+
+ def _run(self):
+ try:
+ if not self._stopped():
+ self._download()
+ except Exception as error: # noqa: BLE001 -- the authoritative loader handles failures later
+ logger.info("Model checkpoint prefetch skipped for %s: %s", self.repo, error)
+ finally:
+ if self._watchdog_stop is not None:
+ self._watchdog_stop.set()
+
+ def start(self):
+ # Account identity and other scopes (cache/download/security) must survive the new thread.
+ context = copy_context()
+ self._send("model_load_started")
+ self._thread = account_thread(target = lambda: context.run(self._run), daemon = True)
+ self._thread.start()
+
+ def finish(self, *, wait):
+ try:
+ while wait and self._thread.is_alive() and not self._should_stop():
+ self._thread.join(timeout = 0.2)
+ finally:
+ self._cancel.set()
+ if self._watchdog_stop is not None:
+ self._watchdog_stop.set()
+ # A Hub call still exiting on Stop/error must not emit into the next phase.
+ with self._lock:
+ self._closed.set()
+ self._emit({"type": "model_load_completed", "ts": time.time()})
+
+
+def prepare_dataset_with_prefetch(load_dataset, prefetch):
+ """Do not consume streaming rows, copy datasets, or move tokenizer/GPU work into a thread."""
+ if prefetch is None:
+ return load_dataset()
+ try:
+ prefetch.start()
+ except Exception as error: # noqa: BLE001 -- an optimization cannot prevent the normal load
+ logger.info("Could not start model checkpoint prefetch: %s", error)
+ prefetch.finish(wait = False)
+ return load_dataset()
+ ready = False
+ try:
+ result = load_dataset()
+ ready = result is not None and result[0] is not None
+ return result
+ finally:
+ prefetch.finish(wait = ready)
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 027ea83f941..bda8d024131 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -234,6 +234,38 @@ def _effective_training_load_in_4bit(
return effective_training_load_in_4bit(config, model_load_target, hf_token)
+def _training_weight_prefetch(config, trainer, hf_token, event_queue):
+ """Resolve only the ordinary SFT checkpoint; uncertain routes keep the existing sequential load."""
+ try:
+ from .model_weight_prefetch import ModelWeightPrefetch, prefetch_repo
+ from utils.hf_cache_settings import get_hf_cache_paths
+ from utils.hardware import dataset_map_num_proc
+
+ repo = prefetch_repo(
+ config,
+ trainer,
+ load_in_4bit = lambda: _effective_training_load_in_4bit(
+ config, _resolve_cached_model_load_name(config), hf_token
+ ),
+ capabilities = sys.modules.get("unsloth.device_type"),
+ offline = _model_offline_mode_enabled(),
+ formatting_can_fork = (
+ not config.get("dataset_streaming", False) and (dataset_map_num_proc() or 1) > 1
+ ),
+ )
+ if repo is not None:
+ return ModelWeightPrefetch(
+ repo,
+ token = hf_token,
+ cache_dir = get_hf_cache_paths().hub_cache,
+ emit = event_queue.put,
+ should_stop = lambda: trainer.should_stop,
+ )
+ except Exception as error: # noqa: BLE001
+ logger.info("Skipping model checkpoint prefetch: %s", error)
+ return None
+
+
def _drop_model_pin(config: dict) -> str:
config["model_snapshot_path"] = None
return config["model_name"]
@@ -4880,7 +4912,12 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
return
_send_status(event_queue, "Loading and formatting dataset...")
- dataset, eval_dataset = _load_training_dataset()
+ from .model_weight_prefetch import prepare_dataset_with_prefetch
+
+ dataset, eval_dataset = prepare_dataset_with_prefetch(
+ _load_training_dataset,
+ _training_weight_prefetch(config, trainer, hf_token, event_queue),
+ )
if dataset is None or trainer.should_stop:
if trainer.should_stop:
diff --git a/studio/backend/tests/test_training_model_weight_prefetch.py b/studio/backend/tests/test_training_model_weight_prefetch.py
new file mode 100644
index 00000000000..0ecc83d3b7e
--- /dev/null
+++ b/studio/backend/tests/test_training_model_weight_prefetch.py
@@ -0,0 +1,358 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import json
+import sys
+import threading
+from contextvars import ContextVar
+from types import SimpleNamespace
+
+import pytest
+
+from core.training import model_weight_prefetch as prefetch
+from utils.account_context import AccountContext, current_account_id, run_as
+
+
+def test_checkpoint_selection_chooses_single_safetensors_without_other_formats():
+ files = [
+ "config.json",
+ "model.safetensors",
+ "model.safetensors.index.json",
+ "pytorch_model.bin",
+ "q4.gguf",
+ ]
+ reads = []
+ assert prefetch.checkpoint_files(files, lambda name: reads.append(name) or {}) == [
+ "model.safetensors"
+ ]
+ assert reads == ["config.json"]
+
+
+def test_checkpoint_selection_uses_only_exact_index_shards():
+ first, second = "model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors"
+ metadata = {
+ "config.json": {},
+ "model.safetensors.index.json": {"weight_map": {"a": second, "b": first, "c": first}},
+ }
+ assert prefetch.checkpoint_files(
+ [*metadata, first, second, "other.safetensors"], metadata.__getitem__
+ ) == [first, second]
+
+
+ "shard",
+ [
+ "../secret.safetensors",
+ "/model.safetensors",
+ "folder/model-1-of-1.safetensors",
+ "model-1-of-1.bin",
+ 123,
+ ["bad"],
+ ],
+)
+def test_unfamiliar_index_paths_are_not_downloaded(shard):
+ metadata = {"config.json": {}, "model.safetensors.index.json": {"weight_map": {"a": shard}}}
+ assert prefetch.checkpoint_files(metadata, metadata.__getitem__) == []
+
+
+ "config",
+ [{"auto_map": {"AutoModel": "custom.Model"}}, {"base_model_name_or_path": "other/base"}],
+)
+def test_custom_and_adapter_configs_are_left_to_loader(config):
+ assert (
+ prefetch.checkpoint_files(["config.json", "model.safetensors"], lambda _name: config) == []
+ )
+ assert (
+ prefetch.checkpoint_files(
+ ["adapter_config.json", "config.json", "model.safetensors"],
+ lambda _name: pytest.fail("adapter metadata must not be fetched"),
+ )
+ == []
+ )
+
+
+def mirror(monkeypatch):
+ import utils.models.unsloth_mirror as mapping
+
+ seen = []
+ monkeypatch.setattr(mapping, "mirror_lookup_available", lambda: True)
+ monkeypatch.setattr(
+ mapping,
+ "unsloth_public_mirror",
+ lambda name, quant: seen.append((name, quant))
+ or ("unsloth/model-bnb-4bit" if quant else "unsloth/model"),
+ )
+ return seen
+
+
+def test_prefetch_resolves_the_effective_quantization(mirror):
+ trainer = SimpleNamespace(should_stop = False, _audio_type_known = True)
+ capabilities = SimpleNamespace(ALLOW_BITSANDBYTES = True, ALLOW_PREQUANTIZED_MODELS = True)
+ config = {"model_name": "upstream/model", "dataset_streaming": True}
+ assert (
+ prefetch.prefetch_repo(config, trainer, load_in_4bit = True, capabilities = capabilities)
+ == "unsloth/model-bnb-4bit"
+ )
+ assert (
+ prefetch.prefetch_repo(config, trainer, load_in_4bit = False, capabilities = capabilities)
+ == "unsloth/model"
+ )
+ capabilities.ALLOW_BITSANDBYTES = False
+ assert (
+ prefetch.prefetch_repo(config, trainer, load_in_4bit = True, capabilities = capabilities)
+ == "unsloth/model"
+ )
+ assert mirror == [
+ ("upstream/model", True),
+ ("upstream/model", False),
+ ("upstream/model", False),
+ ]
+
+
+ "patch",
+ [
+ {"model_revision": "sha"},
+ {"model_snapshot_path": "snapshot"},
+ {"resume_from_checkpoint": "checkpoint"},
+ {"require_exact_model_resource": True},
+ {"require_exact_resume_resources": True},
+ {"trust_remote_code": True},
+ {"training_type": "Full Finetuning"},
+ {"is_dataset_audio": True},
+ {"is_decision": True},
+ {"is_embedding": True},
+ {"model_name": "/local/model"},
+ ],
+)
+def test_pinned_and_uncertain_jobs_do_not_prefetch(patch, mirror):
+ trainer = SimpleNamespace(should_stop = False, _audio_type_known = True)
+ capabilities = SimpleNamespace(ALLOW_BITSANDBYTES = True, ALLOW_PREQUANTIZED_MODELS = True)
+ assert (
+ prefetch.prefetch_repo(
+ {"model_name": "upstream/model", **patch},
+ trainer,
+ load_in_4bit = True,
+ capabilities = capabilities,
+ )
+ is None
+ )
+ assert mirror == []
+
+
+def test_unproven_runtime_and_offline_jobs_do_not_prefetch(mirror):
+ trainer = SimpleNamespace(should_stop = False, _audio_type_known = True)
+ config = {"model_name": "upstream/model"}
+ for capabilities in (
+ None,
+ SimpleNamespace(),
+ SimpleNamespace(ALLOW_BITSANDBYTES = True, ALLOW_PREQUANTIZED_MODELS = False),
+ ):
+ assert (
+ prefetch.prefetch_repo(config, trainer, load_in_4bit = True, capabilities = capabilities)
+ is None
+ )
+ assert (
+ prefetch.prefetch_repo(
+ config, trainer, load_in_4bit = True, capabilities = object(), offline = True
+ )
+ is None
+ )
+ assert mirror == []
+
+
+ "num_proc, streaming, expected",
+ [(None, False, True), (1, False, True), (4, False, False), (4, True, True)],
+)
+def test_worker_uses_prefetch_only_without_forking_preparation(
+ monkeypatch, tmp_path, mirror, num_proc, streaming, expected
+):
+ from core.training import worker
+ import utils.hardware
+ import utils.hf_cache_settings
+
+ capabilities = SimpleNamespace(ALLOW_BITSANDBYTES = True, ALLOW_PREQUANTIZED_MODELS = True)
+ monkeypatch.setitem(sys.modules, "unsloth.device_type", capabilities)
+ monkeypatch.setattr(worker, "_model_offline_mode_enabled", lambda: False)
+ resolved = []
+ monkeypatch.setattr(
+ worker, "_effective_training_load_in_4bit", lambda *_a: resolved.append(True) or False
+ )
+ monkeypatch.setattr(utils.hardware, "dataset_map_num_proc", lambda: num_proc)
+ monkeypatch.setattr(
+ utils.hf_cache_settings, "get_hf_cache_paths", lambda: SimpleNamespace(hub_cache = tmp_path)
+ )
+ job = worker._training_weight_prefetch(
+ {"model_name": "upstream/model", "dataset_streaming": streaming},
+ SimpleNamespace(should_stop = False, _audio_type_known = True),
+ False,
+ SimpleNamespace(put = lambda _ev: None),
+ )
+ assert (job is not None) is expected
+ assert bool(resolved) is expected
+ if job:
+ assert job.repo == "unsloth/model"
+ assert job.token is False
+ assert job.cache_dir == str(tmp_path)
+
+
+def fake_hub(monkeypatch, tmp_path, *, download):
+ commit = "a" * 40
+ info = SimpleNamespace(
+ sha = commit,
+ siblings = [
+ SimpleNamespace(rfilename = name)
+ for name in ("config.json", "model.safetensors", "pytorch_model.bin")
+ ],
+ )
+ calls = []
+
+ def fetch(**kwargs):
+ calls.append(kwargs)
+ path = tmp_path / kwargs["filename"]
+ if kwargs["filename"] == "config.json":
+ path.write_text(json.dumps({}))
+ else:
+ download()
+ path.write_bytes(b"weights")
+ return str(path)
+
+ watchdog = threading.Event()
+ monkeypatch.setitem(
+ sys.modules,
+ "huggingface_hub",
+ SimpleNamespace(
+ HfApi = lambda: SimpleNamespace(model_info = lambda *a, **kw: info), hf_hub_download = fetch
+ ),
+ )
+ monkeypatch.setitem(
+ sys.modules, "utils.hf_xet_fallback", SimpleNamespace(start_watchdog = lambda **_kw: watchdog)
+ )
+ return calls, watchdog
+
+
+def test_model_io_overlaps_dataset_preparation_and_keeps_context(monkeypatch, tmp_path):
+ entered, finish = threading.Event(), threading.Event()
+ custom = ContextVar("prefetch_test_context", default = "unset")
+ observed = []
+
+ def download():
+ observed.append((current_account_id(), custom.get(), threading.current_thread().ident))
+ entered.set()
+ assert finish.wait(3)
+
+ calls, watchdog = fake_hub(monkeypatch, tmp_path, download = download)
+ events = []
+ account = AccountContext("test-account", "test")
+ context_token = custom.set("captured")
+ main_thread = threading.current_thread().ident
+
+ class LazyDataset:
+ def __iter__(self):
+ pytest.fail("startup must not consume streaming rows")
+
+ dataset = LazyDataset()
+
+ def load_dataset():
+ assert entered.wait(3), "weight I/O must already be in progress during dataset preparation"
+ assert not watchdog.is_set()
+ finish.set()
+ return dataset, None
+
+ def run():
+ job = prefetch.ModelWeightPrefetch(
+ "unsloth/model",
+ token = False,
+ cache_dir = tmp_path / "account-cache",
+ emit = events.append,
+ should_stop = lambda: False,
+ )
+ assert prefetch.prepare_dataset_with_prefetch(load_dataset, job) == (dataset, None)
+ assert (
+ not job._thread.is_alive()
+ ), "actual model placement starts after background I/O finishes"
+
+ try:
+ run_as(account, run)
+ finally:
+ custom.reset(context_token)
+ finish.set()
+ assert observed == [("test-account", "captured", observed[0][2])]
+ assert observed[0][2] != main_thread
+ assert [call["filename"] for call in calls] == ["config.json", "model.safetensors"]
+ assert all(
+ call["revision"] == "a" * 40
+ and call["token"] is False
+ and call["cache_dir"] == str(tmp_path / "account-cache")
+ for call in calls
+ )
+ assert [event["type"] for event in events] == [
+ "model_load_started",
+ "model_load_resolved",
+ "model_load_completed",
+ ]
+ assert watchdog.is_set()
+
+
[email protected]("outcome", ["error", "empty", "stop"])
+def test_failed_or_stopped_dataset_does_not_wait_for_download(monkeypatch, tmp_path, outcome):
+ entered, release = threading.Event(), threading.Event()
+ calls, watchdog = fake_hub(
+ monkeypatch, tmp_path, download = lambda: (entered.set(), release.wait(3))
+ )
+ events = []
+ stopped = []
+ job = prefetch.ModelWeightPrefetch(
+ "unsloth/model",
+ token = "token",
+ cache_dir = tmp_path,
+ emit = events.append,
+ should_stop = lambda: bool(stopped),
+ )
+
+ def load_dataset():
+ assert entered.wait(3)
+ if outcome == "error":
+ raise ValueError("invalid dataset")
+ if outcome == "stop":
+ stopped.append(True)
+ return (None, None) if outcome == "empty" else (object(), None)
+
+ try:
+ if outcome == "error":
+ with pytest.raises(ValueError, match = "invalid dataset"):
+ prefetch.prepare_dataset_with_prefetch(load_dataset, job)
+ else:
+ prefetch.prepare_dataset_with_prefetch(load_dataset, job)
+ assert job._thread.is_alive(), "an in-flight Hub call must not block cancellation"
+ assert watchdog.is_set()
+ assert events[-1]["type"] == "model_load_completed"
+ before = list(events)
+ job._send("stall", message = "late")
+ assert events == before
+ finally:
+ release.set()
+ job._thread.join(timeout = 3)
+ assert not job._thread.is_alive()
+
+
+def test_prefetch_failure_leaves_dataset_result_authoritative(monkeypatch, tmp_path):
+ fake_hub(
+ monkeypatch,
+ tmp_path,
+ download = lambda: (_ for _ in ()).throw(OSError("temporary Hub error")),
+ )
+ events = []
+ dataset = object()
+ job = prefetch.ModelWeightPrefetch(
+ "unsloth/model",
+ token = None,
+ cache_dir = tmp_path,
+ emit = events.append,
+ should_stop = lambda: False,
+ )
+ assert prefetch.prepare_dataset_with_prefetch(lambda: (dataset, None), job) == (dataset, None)
+ assert events[-1]["type"] == "model_load_completed"