Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fetch up to four Hub files at once over HTTP when partials resume (#15781)
Daniel Han98b37c2b6732
diff --git a/studio/backend/hub/services/download_lifecycle.py b/studio/backend/hub/services/download_lifecycle.pyindex a21331d2934..05b21807715 100644--- a/studio/backend/hub/services/download_lifecycle.py+++ b/studio/backend/hub/services/download_lifecycle.py@@ -1124,7 +1124,7 @@ def _start_stall_watchdog(logger,on_stall: Callable[[str], None],):- """Kill *proc* if its download stops making byte-level progress. Returns a stop event, or ``None`` when no watchdog could be started. SIGKILL rather than a polite signal: the worker traps SIGTERM and exits 130 ("cancelled"), which would record a user cancel and skip the recovery ladder, while an untrapped kill lands as "error", which is what triggers the retry. What survives the kill depends on the transport the retry picks: over HTTP the writer is sequential and the partial genuinely resumes, while over XET it does not, since hf_xet reconstructs a file from offset zero rather than resuming it, so the recovery worker's own ``prepare_cache_for_transport(..., "xet", ...)`` purges the partial on startup and re-fetches the in-flight file. Every file already finalized into a blob is kept either way, and the worker runs ``snapshot_download(max_workers=1)``, so at most one file is ever replayed."""+ """Kill *proc* if its download stops making byte-level progress. Returns a stop event, or ``None`` when no watchdog could be started. SIGKILL rather than a polite signal: the worker traps SIGTERM and exits 130 ("cancelled"), which would record a user cancel and skip the recovery ladder, while an untrapped kill lands as "error", which is what triggers the retry. What survives the kill depends on the transport the retry picks: over HTTP the writer is sequential and the partial genuinely resumes, while over XET it does not, since hf_xet reconstructs a file from offset zero rather than resuming it, so the recovery worker's own ``prepare_cache_for_transport(..., "xet", ...)`` purges the partial on startup and re-fetches the in-flight file. Every file already finalized into a blob is kept either way, and the worker fetches one file at a time over XET (HTTP overlaps files only where every partial resumes), so at most one file is ever replayed."""try:from utils.hf_xet_fallback import start_watchdogexcept Exception as exc: # noqa: BLE001 - degraded unsloth_zoo: keep the old behaviour@@ -1189,7 +1189,7 @@ def _start_stall_watchdog(child_pid = proc.pid,# Scope the measurement to partials this worker holds open; otherwise the shared helper stays repo-wide, child_pid does nothing, and two concurrent same-transport GGUF variants of one repo reset each other's stall timer. The DATA clock only: before its first byte a variant is still covered by the repo-wide peer-progress check.watch_new_partials_only = True,- # The shared 90s zero-byte default assumes a single-file download whose pre-byte phase is one HEAD; snapshot_download(max_workers=1) does a model_info lookup plus one sequential HEAD per file, which for an already-cached repo is the entire job with no byte written.+ # The shared 90s zero-byte default assumes a single-file download whose pre-byte phase is one HEAD; snapshot_download does a model_info lookup plus one HEAD per file (sequential over XET), which for an already-cached repo is the entire job with no byte written.connect_timeout = 600.0,xet_disabled = False,)diff --git a/studio/backend/hub/workers/hf_download.py b/studio/backend/hub/workers/hf_download.pyindex b921dc7432b..6d913658f3b 100644--- a/studio/backend/hub/workers/hf_download.py+++ b/studio/backend/hub/workers/hf_download.py@@ -1,8 +1,8 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""HF download subprocess (SIGKILL stops chunk threads); resume needs single-stream writes, enforced-by HF_HUB_DISABLE_XET=1, HF_HUB_ENABLE_HF_TRANSFER=0 (http), max_workers=1, purge of unprovable+"""HF download subprocess (SIGKILL stops chunk threads); resume needs single-stream writes per file, enforced+by HF_HUB_DISABLE_XET=1, HF_HUB_ENABLE_HF_TRANSFER=0 (http), one writer per blob, purge of unprovable.incomplete blobs."""from __future__ import annotations@@ -38,11 +38,26 @@ from hub.utils.gguf_plan import (sibling_sha256,)from hub.utils.state_dir import RepoType-from hub.utils.resumable_partials import restore_resumable_partials+from hub.utils.resumable_partials import (+ LAST_STOCK_RESUMABLE_VERSION,+ _hub_version,+ restore_resumable_partials,+)# Put huggingface_hub's 1.17 HTTP writer back: the SIGKILL then restart loop reads .incomplete for its resume offset, and 1.18+ leaves nothing to read._PARTIALS_RESUMABLE = restore_resumable_partials()+# Each file is still one sequential writer under huggingface_hub's per-blob lock, so overlapping files only hides request latency and per-connection caps.+_HTTP_FILE_WORKERS = 4+++def _file_workers(mode: str) -> int:+ """``snapshot_download(max_workers=...)``. Xet stays serial: hf_xet fans out inside a file and a killed Xet file restarts from zero. HTTP overlaps only where a killed partial resumes: the stock writer up to 1.17, past it only the writer restored at import."""+ version = _hub_version()+ resumes = _PARTIALS_RESUMABLE or (bool(version) and version <= LAST_STOCK_RESUMABLE_VERSION)+ return _HTTP_FILE_WORKERS if mode == "http" and resumes else 1++# typing.Union, not `str | bool | None`: an alias is evaluated on import and PEP 604 raises below 3.10.HfTokenArg = Union[str, bool, None]@@ -669,7 +684,7 @@ def _download_snapshot(repo_id = repo_id,token = _hf_token_arg(hf_token),ignore_patterns = ignore_patterns,- max_workers = 1,+ max_workers = _file_workers(mode),tqdm_class = tqdm_class,)if info is None:@@ -818,7 +833,7 @@ def _download_gguf_variant(repo_id = repo_id,token = _hf_token_arg(hf_token),allow_patterns = targets,- max_workers = 1,+ max_workers = _file_workers(mode),tqdm_class = tqdm_class,)_verify_completed_download(@@ -919,7 +934,7 @@ def _download_scoped_snapshot(repo_id = repo_id,token = _hf_token_arg(hf_token),allow_patterns = files,- max_workers = 1,+ max_workers = _file_workers(mode),tqdm_class = tqdm_class,)if info is None:@@ -992,7 +1007,7 @@ def _download_dataset("repo_id": repo_id,"token": _hf_token_arg(hf_token),"repo_type": "dataset",- "max_workers": 1,+ "max_workers": _file_workers(mode),"tqdm_class": tqdm_class,}if isinstance(commit_hash, str) and commit_hash.strip():diff --git a/studio/backend/tests/test_hf_download_file_concurrency.py b/studio/backend/tests/test_hf_download_file_concurrency.pynew file mode 100644index 00000000000..fef066091a1--- /dev/null+++ b/studio/backend/tests/test_hf_download_file_concurrency.py@@ -0,0 +1,226 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The HTTP download worker overlaps independent files and still writes the same bytes; Xet stays serial."""++from __future__ import annotations++import ast+import hashlib+import json+import os+import subprocess+import sys+import threading+import time+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++from hub.workers import hf_download++_WORKER = _BACKEND / "hub" / "workers" / "hf_download.py"+_COMMIT = "c" * 40+++def test_xet_and_unknown_transports_stay_serial():+ assert hf_download._file_workers("xet") == 1+ assert hf_download._file_workers("auto") == 1+++ "version, restored, workers",+ [+ ((0, 36), False, 4), # stock writer appends to the shared partial+ ((1, 17), False, 4),+ ((1, 33), True, 4),+ ((1, 33), False, 1), # stock 1.18+ writer: a killed partial cannot resume+ ((), False, 1), # unreadable version proves neither writer+ ],+)+def test_http_overlaps_only_when_partials_resume(monkeypatch, version, restored, workers):+ monkeypatch.setattr(hf_download, "_hub_version", lambda: version)+ monkeypatch.setattr(hf_download, "_PARTIALS_RESUMABLE", restored)+ assert hf_download._file_workers("http") == workers+ assert hf_download._file_workers("xet") == 1+++def test_every_snapshot_download_call_uses_the_transport_budget():+ tree = ast.parse(_WORKER.read_text(encoding = "utf-8"))+ seen = 0+ for node in ast.walk(tree):+ values = []+ if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "snapshot_download":+ values = [kw.value for kw in node.keywords if kw.arg == "max_workers"]+ if not values:+ # The dataset path passes a kwargs dict; checked below.+ assert any(kw.arg is None for kw in node.keywords)+ continue+ elif isinstance(node, ast.Dict):+ values = [+ v+ for k, v in zip(node.keys, node.values)+ if isinstance(k, ast.Constant) and k.value == "max_workers"+ ]+ for value in values:+ seen += 1+ assert ast.unparse(value) == "_file_workers(mode)"+ assert seen == 4+++def _fixture() -> dict:+ files = {}+ for i in range(6):+ files[f"small_{i}.json"] = json.dumps({"i": i}).encode() * 20+ for i in range(3):+ files[f"model-0000{i + 1}-of-00003.safetensors"] = (+ hashlib.sha256(str(i).encode()).digest() * 4096+ )+ return files+++def _serve(files: dict, stats: dict):+ def etag(path, data):+ if path.endswith(".safetensors"):+ return hashlib.sha256(data).hexdigest()+ return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest()++ class Handler(BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"++ def log_message(self, *a):+ pass++ def _reply(self, head: bool):+ path = self.path.split("?", 1)[0]+ if path.startswith("/api/models/") and "/tree/" in path:+ # huggingface_hub >= 1.x lists the tree before fetching a snapshot.+ tree = []+ for name, data in files.items():+ entry = {+ "type": "file",+ "path": name,+ "size": len(data),+ "oid": etag(name, data),+ }+ if name.endswith(".safetensors"):+ entry["lfs"] = {+ "oid": etag(name, data),+ "size": len(data),+ "pointerSize": 130,+ }+ tree.append(entry)+ body = json.dumps(tree).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ if not head:+ self.wfile.write(body)+ return+ if path.startswith("/api/models/"):+ siblings = []+ for name, data in files.items():+ entry = {"rfilename": name, "size": len(data), "blobId": etag(name, data)}+ if name.endswith(".safetensors"):+ entry["lfs"] = {+ "sha256": etag(name, data),+ "size": len(data),+ "pointerSize": 130,+ }+ siblings.append(entry)+ body = json.dumps(+ {"id": "fake/repo", "sha": _COMMIT, "siblings": siblings, "private": False}+ ).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ if not head:+ self.wfile.write(body)+ return+ name = path.split("/resolve/", 1)[-1].split("/", 1)[-1]+ data = files.get(name)+ if data is None:+ self.send_response(404)+ self.send_header("Content-Length", "0")+ self.end_headers()+ return+ tag = etag(name, data)+ self.send_response(200)+ self.send_header("X-Repo-Commit", _COMMIT)+ self.send_header("ETag", f'"{tag}"')+ if name.endswith(".safetensors"):+ self.send_header("X-Linked-Etag", f'"{tag}"')+ self.send_header("X-Linked-Size", str(len(data)))+ self.send_header("Content-Length", str(len(data)))+ self.end_headers()+ if head:+ return+ with stats["lock"]:+ stats["gets"][name] = stats["gets"].get(name, 0) + 1+ stats["live"] += 1+ stats["peak"] = max(stats["peak"], stats["live"])+ try:+ # Hold the connection long enough that overlapping fetches are observable.+ time.sleep(0.3)+ self.wfile.write(data)+ finally:+ with stats["lock"]:+ stats["live"] -= 1++ def do_GET(self):+ self._reply(False)++ def do_HEAD(self):+ self._reply(True)++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ server.daemon_threads = True+ threading.Thread(target = server.serve_forever, daemon = True).start()+ return server+++ not __import__("hub.utils.hf_cache_state", fromlist = ["x"]).hf_partials_are_resumable(),+ reason = "partials do not resume on this cache filesystem, so the worker stays serial",+)+def test_http_worker_overlaps_files_and_writes_identical_bytes(tmp_path):+ files = _fixture()+ stats = {"lock": threading.Lock(), "gets": {}, "live": 0, "peak": 0}+ server = _serve(files, stats)+ try:+ env = dict(os.environ)+ env.pop("HF_TOKEN", None)+ env.update(+ HF_ENDPOINT = f"http://127.0.0.1:{server.server_address[1]}",+ HF_HOME = str(tmp_path / "hf"),+ HF_HUB_CACHE = str(tmp_path / "hf" / "hub"),+ UNSLOTH_STUDIO_HOME = str(tmp_path / "studio"),+ HF_HUB_DISABLE_XET = "1",+ HF_HUB_ENABLE_HF_TRANSFER = "0",+ HF_HUB_DISABLE_TELEMETRY = "1",+ HF_HUB_OFFLINE = "0",+ )+ proc = subprocess.run(+ [sys.executable, str(_WORKER), "--repo-id", "fake/repo", "--transport", "http"],+ cwd = str(_BACKEND),+ env = env,+ capture_output = True,+ text = True,+ timeout = 120,+ )+ finally:+ server.shutdown()+ assert proc.returncode == 0, proc.stderr[-2000:]+ snapshot = tmp_path / "hf" / "hub" / "models--fake--repo" / "snapshots" / _COMMIT+ for name, data in files.items():+ assert (snapshot / name).read_bytes() == data+ assert stats["gets"] == {name: 1 for name in files}+ assert stats["peak"] >= 2+ assert not list((tmp_path / "hf" / "hub" / "models--fake--repo" / "blobs").glob("*.incomplete"))