Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion/video: share the Hub reachability verdict across cached-load calls (#15059)
Daniel Hand497ab25048d
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 170f9a70cd0..95cb17b1b13 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -56,6 +56,7 @@ from .diffusion_content import (whole_pipeline_gguf_family,)from .diffusion_gguf_pipeline import load_whole_pipeline_gguf, whole_pipeline_gguf_resident_mib+from . import hub_reachabilityfrom .diffusion_families import (SMALL_DIT_BF16_WHEN_RESIDENT,small_dit_rule_excludes,@@ -1357,7 +1358,7 @@ def _assert_base_repo_accessible(return status in (401, 403)try:- gated = getattr(HfApi().model_info(repo, token = hf_token), "gated", None)+ gated = getattr(hub_reachability.model_info(HfApi(), repo, token = hf_token), "gated", None)except GatedRepoError: # a gated repo can also withhold its metadataif _already_downloaded():return other_root_snapshot@@ -1505,12 +1506,17 @@ def _te_quant_for_supplied_encoders(requested: Optional[str]) -> Optional[str]:def _hub_file_size(repo_id: str, filename: str, hf_token: Optional[str]) -> int:+ # get_paths_info takes no timeout.+ if hub_reachability.hub_unreachable():+ return 0try:from huggingface_hub import HfApifor info in HfApi().get_paths_info(repo_id, [filename], token = hf_token):+ hub_reachability.note_hub_success()return int(getattr(info, "size", 0) or 0)- except Exception: # noqa: BLE001 - a size is a progress-bar hint, never a refusal- pass+ hub_reachability.note_hub_success()+ except Exception as exc: # noqa: BLE001 - a size is a progress-bar hint, never a refusal+ hub_reachability.note_hub_failure(exc)return 0@@ -4235,12 +4241,15 @@ class DiffusionBackend:# re-pulls the whole multi-GB file inside the load lock, after eviction, where unload cannot preempt it and# progress already reported 100%.cache_dir = hub_cache_dir()+ offline_ok = local_files_only or hub_reachability.hub_unreachable()+ cached_here = False# Best-effort: an unreadable cache raises here, and letting that escape would abort a load that only had to# download. Any failure falls through to the pinned download below.try:- if not isinstance(+ cached_here = isinstance(try_to_load_from_cache(repo_id, gguf_filename, cache_dir = cache_dir), str- ):+ )+ if not cached_here:# Same other-root reuse as the pre-quant checkpoint, reached THROUGH that root rather than returned# raw, so the ref still resolves and a republished GGUF is picked up; the blob is reused, and offline# the cached pointer comes back anyway.@@ -4252,7 +4261,7 @@ class DiffusionBackend:gguf_filename,token = hf_token,cache_dir = None,- local_files_only = local_files_only,+ local_files_only = offline_ok,)except Exception: # noqa: BLE001 - revalidation is a bonus, never a new failurereturn elsewhere@@ -4263,7 +4272,7 @@ class DiffusionBackend:gguf_filename,token = hf_token,cache_dir = cache_dir,- local_files_only = local_files_only,+ local_files_only = local_files_only or (cached_here and offline_ok),)def _dense_quant_prefetch_needed(@@ -5594,20 +5603,51 @@ class DiffusionBackend:# fallback, never the override: it fires only when the estimate came back empty, since the metadata that# fills it is the same call whose failure earned the escape. Without it the load 401s with every byte# already on disk.- kwargs["_base_local_dir"] = (- self._prefetch_files(+ # Where diffusers' own "Couldn't connect" fallback lands, minus its untimed revision / model_info calls.+ forced_offline = False+ if not local_files_only and _pick_cached_while_hub_unreachable(+ fam, kind, kwargs["repo_id"], kwargs.get("gguf_filename"), base, fetch_base+ ):+ logger.info(+ "diffusion.hub_unreachable: %s is cached and the Hub did not connect moments ago; loading "+ "from the cache",kwargs["repo_id"],- kwargs.get("gguf_filename"),- base,- base_files,- kwargs.get("hf_token"),- cancel_event = cancel_event,- fetch_base = fetch_base,- local_files_only = local_files_only,)- or base_snapshot- )- self.load_pipeline(**kwargs)+ forced_offline = True++ def _prefetch_and_load(offline: bool) -> None:+ kwargs["local_files_only"] = offline+ kwargs["_base_local_dir"] = (+ self._prefetch_files(+ kwargs["repo_id"],+ kwargs.get("gguf_filename"),+ base,+ base_files,+ kwargs.get("hf_token"),+ cancel_event = cancel_event,+ fetch_base = fetch_base,+ local_files_only = offline,+ )+ or base_snapshot+ )+ self.load_pipeline(**kwargs)++ try:+ _prefetch_and_load(local_files_only or forced_offline)+ except OSError as exc:+ # Only the manifest is checked up front: an interrupted download still needs the online load.+ if (+ not forced_offline+ or self._load_token != token+ or (cancel_event is not None and cancel_event.is_set())+ ):+ raise+ logger.info(+ "diffusion.hub_unreachable: %s is not fully cached (%s); retrying online",+ kwargs["repo_id"],+ exc,+ )+ _prefetch_and_load(False)with self._load_cancel_lock:# Only clear the marker if this load is still current (a superseder has its own token).if self._load_token == token:@@ -6200,7 +6240,13 @@ class DiffusionBackend:from huggingface_hub.errors import RepositoryNotFoundErrortry:- info = HfApi(token = hf_token or None).model_info(source.location, files_metadata = True)+ # Decides what loads: always asked, only bounded.+ info = hub_reachability.model_info(+ HfApi(token = hf_token or None),+ source.location,+ files_metadata = True,+ skip_when_unreachable = False,+ )except RepositoryNotFoundError as exc:# 401/403/404 on the repo = no hosted checkpoint, not a partial listing (GatedRepoError subclasses this).logger.info(@@ -6360,7 +6406,9 @@ class DiffusionBackend:if kind == "pipeline":def _pipeline_listing(metadata_repo: str) -> tuple[Any, Any, list[Any]]:- info = api.model_info(metadata_repo, files_metadata = True, token = hf_token)+ info = hub_reachability.model_info(+ api, metadata_repo, files_metadata = True, token = hf_token+ )components = _pipeline_components_from_index(metadata_repo,info,@@ -6429,7 +6477,9 @@ class DiffusionBackend:_record_revision(revisions_out, metadata_repo, info)return total, base_filesif gguf_filename and not Path(repo_id).expanduser().exists():- info = api.model_info(repo_id, files_metadata = True, token = hf_token)+ info = hub_reachability.model_info(+ api, repo_id, files_metadata = True, token = hf_token+ )gguf_bytes = sum(s.size or 0 for s in info.siblings if s.rfilename == gguf_filename)total += gguf_bytesif sizes_out is not None:@@ -6466,7 +6516,9 @@ class DiffusionBackend:)def _base_listing(listing_repo: str) -> list[Any]:- info = api.model_info(listing_repo, files_metadata = True, token = hf_token)+ info = hub_reachability.model_info(+ api, listing_repo, files_metadata = True, token = hf_token+ )# Under the repo it describes: a mirror's commit says nothing about a cached upstream snapshot._record_revision(revisions_out, listing_repo, info)return [s for s in info.siblings if not _listing_extra(listing_repo, s.rfilename)]@@ -7067,7 +7119,10 @@ class DiffusionBackend:"""The current Hub commit for a mirror repo, or None when metadata is unavailable."""try:from huggingface_hub import HfApi- return getattr(HfApi().model_info(repo_id, token = hf_token), "sha", None) or None+ return (+ getattr(hub_reachability.model_info(HfApi(), repo_id, token = hf_token), "sha", None)+ or None+ )except Exception: # noqa: BLE001 -- no SHA means conservative stagingreturn None@@ -9390,7 +9445,10 @@ class DiffusionBackend:# config is a REPO ID, and diffusers forwards this into the load_config() that resolves# it, so without the flag this branch reaches the Hub on a load nobody asked for. The# pipeline assembly below was already guarded; this call was not.- "local_files_only": local_files_only,+ "local_files_only": local_files_only+ or _cached_while_hub_unreachable(+ fetch_base, "transformer/config.json"+ ),# config is the family base's (2511); 2509 / original Edit lack its zero_cond_t**transformer_config_overrides_for(fam, gguf_filename, repo_id, display_repo_id, base@@ -14706,6 +14764,48 @@ def _remember_companion_base(repo_id: str, base: str) -> None:logger.debug("diffusion.companion_link_record_failed: %s", exc)+def _cached_while_hub_unreachable(repo_id: Any, filename: str) -> bool:+ """Hub just failed to connect AND ``filename`` is cached: ``local_files_only=True`` then returns what the online+ call would after its HEAD timed out. False on any doubt, so a first download still asks."""+ if not hub_reachability.hub_unreachable():+ return False+ try:+ if not isinstance(repo_id, str) or Path(repo_id).expanduser().exists():+ return False+ from huggingface_hub import try_to_load_from_cache+ return isinstance(try_to_load_from_cache(repo_id, filename, cache_dir = hub_cache_dir()), str)+ except Exception: # noqa: BLE001+ return False+++def _pick_cached_while_hub_unreachable(+ fam: Any,+ kind: str,+ repo_id: str,+ gguf_filename: Optional[str],+ base: Optional[str],+ fetch_base: Optional[str],+) -> bool:+ """Base manifest cached (or local) and, for a GGUF / single-file pick, the checkpoint too."""+ if not hub_reachability.hub_unreachable():+ return False+ try:+ manifest = _base_manifest_name(fam)+ if not (+ (bool(base) and _is_local_path(base))+ or any(+ _cached_while_hub_unreachable(repo, manifest)+ for repo in dict.fromkeys(r for r in (fetch_base, base) if r)+ )+ ):+ return False+ if kind != "pipeline" and gguf_filename and not _is_local_path(repo_id):+ return _cached_while_hub_unreachable(repo_id, gguf_filename)+ return True+ except Exception: # noqa: BLE001 - any doubt keeps the online load+ return False++def _hf_base_model(repo_id: str, hf_token: Optional[str]) -> Optional[str]:"""The diffusers base repo from a GGUF repo's ``base_model`` tag, or None. Lets one family entrycover every variant (Turbo/full, schnell/dev, the 2512 Qwen revision). Skipped for local@@ -14714,7 +14814,14 @@ def _hf_base_model(repo_id: str, hf_token: Optional[str]) -> Optional[str]:return Nonetry:from huggingface_hub import HfApi- meta = HfApi().model_info(repo_id, token = hf_token).cardData or {}++ # Picks the base: never skipped, or a recovered network would get the family default.+ meta = (+ hub_reachability.model_info(+ HfApi(), repo_id, token = hf_token, skip_when_unreachable = False+ ).cardData+ or {}+ )except Exception: # noqa: BLE001 - best-effort; fall back to the family defaultreturn Nonebase = meta.get("base_model")diff --git a/studio/backend/core/inference/diffusion_compat.py b/studio/backend/core/inference/diffusion_compat.pyindex b012dd8b1be..ee97ea62989 100644--- a/studio/backend/core/inference/diffusion_compat.py+++ b/studio/backend/core/inference/diffusion_compat.py@@ -157,6 +157,10 @@ def _hub_revision(repo_id: str, gguf_filename: str, hf_token: Optional[str]) ->One HEAD, no body: the caller only needs to know whether the local copy is still the currentone, and an offline or erroring host must leave today's verdict alone."""+ from core.inference import hub_reachability++ if hub_reachability.hub_unreachable():+ return None # the Hub failed to connect moments ago: unanswerable, without another timeouttry:from huggingface_hub import get_hf_file_metadata, hf_hub_urlmeta = get_hf_file_metadata(@@ -164,8 +168,10 @@ def _hub_revision(repo_id: str, gguf_filename: str, hf_token: Optional[str]) ->token = hf_token,timeout = _HEADER_TIMEOUT_SECONDS,)- except Exception: # noqa: BLE001 — a revision we cannot read is not a verdict+ except Exception as exc: # noqa: BLE001 — a revision we cannot read is not a verdict+ hub_reachability.note_hub_failure(exc)return None+ hub_reachability.note_hub_success()return getattr(meta, "commit_hash", None) or Nonediff --git a/studio/backend/core/inference/diffusion_prequant_cache.py b/studio/backend/core/inference/diffusion_prequant_cache.pyindex 637deb9e536..f173c95f89c 100644--- a/studio/backend/core/inference/diffusion_prequant_cache.py+++ b/studio/backend/core/inference/diffusion_prequant_cache.py@@ -35,6 +35,8 @@ from contextlib import contextmanagerfrom pathlib import Pathfrom typing import Any, Iterable, Optional+from . import hub_reachability as _reach+ENV_MODE = "UNSLOTH_DIFFUSION_PREQUANT_CACHE"ENV_DIR = "UNSLOTH_DIFFUSION_PREQUANT_CACHE_DIR"ENV_MAX_GB = "UNSLOTH_DIFFUSION_PREQUANT_CACHE_MAX_GB"@@ -585,37 +587,10 @@ _REMOTE_TTL_SECONDS = 300.0_REMOTE_TIMEOUT_SECONDS = 5.0_REMOTE_MEMO: dict[tuple, tuple] = {}_GONE = "gone"-# Endpoint -> monotonic time its connection failed; later probes answer "unknown" for the TTL instead of timing out.-_UNREACHABLE: dict[str, float] = {}---def _connection_failed(exc: BaseException) -> bool:- """Connect / timeout failure only; HTTP answers are per repo and proxy errors are not cache fallbacks in HF."""- try:- import httpx- if isinstance(exc, httpx.ProxyError):- return False- if isinstance(exc, (httpx.ConnectError, httpx.TimeoutException)):- return True- except ImportError:- pass- try:- import requests- if isinstance(exc, requests.exceptions.ProxyError):- return False- if isinstance(exc, (requests.exceptions.ConnectionError, requests.exceptions.Timeout)):- return True- except ImportError:- pass- return isinstance(exc, (OSError, TimeoutError)) and not hasattr(exc, "response")---def _endpoint() -> str:- try:- from huggingface_hub import constants- return str(constants.ENDPOINT)- except Exception: # noqa: BLE001- return ""+# Shared hub_reachability verdict: after one failure, lookups answer "unknown" (local snapshot stands) for the TTL.+_UNREACHABLE = _reach._UNREACHABLE+_connection_failed = _reach.connection_failed+_endpoint = _reach.endpointdef _hub_coords(path: Any) -> Optional[tuple[str, str, str, str]]:@@ -669,10 +644,8 @@ def _remote_etag(if hit is not None and time.monotonic() - hit[0] < _REMOTE_TTL_SECONDS:return hit[1]endpoint = _endpoint()- with _lock:- down_at = _UNREACHABLE.get(endpoint)- if down_at is not None and time.monotonic() - down_at < _REMOTE_TTL_SECONDS:- return None+ if _reach.hub_unreachable(endpoint):+ return None # the Hub did not answer moments ago: unanswerable, without another timeouttry:from huggingface_hub import get_hf_file_metadata, hf_hub_urlfrom huggingface_hub.utils import EntryNotFoundError@@ -693,10 +666,9 @@ def _remote_etag(except EntryNotFoundError:value = _GONEexcept Exception as exc: # noqa: BLE001 - offline, gated without a token, rate limited: unanswerable- if _connection_failed(exc):- with _lock:- _UNREACHABLE[endpoint] = time.monotonic()+ _reach.note_hub_failure(exc, endpoint)return None+ _reach.note_hub_success(endpoint)with _lock:_UNREACHABLE.pop(endpoint, None)_REMOTE_MEMO[memo] = (time.monotonic(), value)diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.pyindex 64b687466bd..5bebbf854ad 100644--- a/studio/backend/core/inference/diffusion_te_prequant.py+++ b/studio/backend/core/inference/diffusion_te_prequant.py@@ -1975,7 +1975,10 @@ def te_prequant_hub_files(found[component] = [(mirrored[0], os.path.getsize(mirrored[1]))]continuetry:- info = api.model_info(source.location, files_metadata = True)+ from .hub_reachability import always_asked++ # Picks the encoder: always asked, only bounded.+ info = always_asked(api).model_info(source.location, files_metadata = True)except Exception as exc: # noqa: BLE001 -- unavailable pre-cast means the dense encoder_warn(logger, f"hub_files:{source.location}", exc)continuediff --git a/studio/backend/core/inference/hub_reachability.py b/studio/backend/core/inference/hub_reachability.pynew file mode 100644index 00000000000..04eb1ba572d--- /dev/null+++ b/studio/backend/core/inference/hub_reachability.py@@ -0,0 +1,147 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Per-process verdict on whether the Hub endpoint answers. After one transport failure, cached-load Hub calls use+their fallback for ``TTL_SECONDS`` instead of each waiting out a timeout. HTTP answers never set the verdict.+"""++from __future__ import annotations++import threading+import time+from typing import Any, Optional++# Matches the prequant freshness memo.+TTL_SECONDS = 300.0+METADATA_TIMEOUT_SECONDS = 10.0++_UNREACHABLE: dict[str, float] = {}+_lock = threading.Lock()+++class HubUnreachableError(ConnectionError):+ """A ``ConnectionError`` so existing transport-failure handling takes it unchanged."""+++def connection_failed(exc: BaseException) -> bool:+ """Not proxy errors: huggingface_hub raises those instead of falling back to the cache."""+ if isinstance(exc, HubUnreachableError):+ return True+ try:+ import httpx+ if isinstance(exc, httpx.ProxyError):+ return False+ if isinstance(exc, (httpx.ConnectError, httpx.TimeoutException)):+ return True+ except ImportError:+ pass+ try:+ import requests+ if isinstance(exc, requests.exceptions.ProxyError):+ return False+ if isinstance(exc, (requests.exceptions.ConnectionError, requests.exceptions.Timeout)):+ return True+ except ImportError:+ pass+ return isinstance(exc, (OSError, TimeoutError)) and not hasattr(exc, "response")+++def endpoint() -> str:+ try:+ from huggingface_hub import constants+ return str(constants.ENDPOINT)+ except Exception: # noqa: BLE001+ return ""+++def hub_unreachable(at: Optional[str] = None) -> bool:+ key = endpoint() if at is None else at+ with _lock:+ down_at = _UNREACHABLE.get(key)+ return down_at is not None and time.monotonic() - down_at < TTL_SECONDS+++def note_hub_failure(exc: BaseException, at: Optional[str] = None) -> bool:+ if not connection_failed(exc):+ return False+ if isinstance(exc, HubUnreachableError):+ return True # a skipped call: the verdict that caused it stays dated at the real failure+ key = endpoint() if at is None else at+ with _lock:+ _UNREACHABLE[key] = time.monotonic()+ return True+++def note_hub_success(at: Optional[str] = None) -> None:+ key = endpoint() if at is None else at+ with _lock:+ _UNREACHABLE.pop(key, None)+++def _answered(exc: BaseException) -> bool:+ return getattr(exc, "response", None) is not None+++def model_info(+ api: Any,+ repo_id: str,+ *,+ skip_when_unreachable: bool = True,+ **kwargs: Any,+) -> Any:+ """``skip_when_unreachable=False`` is for an answer that decides WHAT loads: always asked, only bounded."""+ at = endpoint()+ if skip_when_unreachable and hub_unreachable(at):+ raise HubUnreachableError(+ f"{at or 'the Hub'} did not connect within the last {int(TTL_SECONDS)} s"+ )+ if _is_hf_api(api):+ # Duck-typed clients (test doubles) may not take ``timeout``.+ kwargs.setdefault("timeout", METADATA_TIMEOUT_SECONDS)+ try:+ info = api.model_info(repo_id, **kwargs)+ except Exception as exc:+ if not note_hub_failure(exc, at) and _answered(exc):+ note_hub_success(at)+ raise+ note_hub_success(at)+ return info+++def _is_hf_api(api: Any) -> bool:+ try:+ # Defining module: tests may swap the package attribute for a double.+ from huggingface_hub.hf_api import HfApi+ return isinstance(api, HfApi)+ except Exception: # noqa: BLE001+ return False+++class GuardedApi:+ def __init__(+ self,+ api: Any,+ skip_when_unreachable: bool = True,+ ):+ self._api = api+ self._skip = skip_when_unreachable++ def model_info(self, repo_id: str, *args: Any, **kwargs: Any) -> Any:+ if args:+ return self._api.model_info(repo_id, *args, **kwargs)+ return model_info(self._api, repo_id, skip_when_unreachable = self._skip, **kwargs)++ def __getattr__(self, name: str) -> Any:+ return getattr(self._api, name)+++def guarded(api: Any, *, skip_when_unreachable: bool = True) -> Any:+ if api is None or isinstance(api, GuardedApi):+ return api+ return GuardedApi(api, skip_when_unreachable)+++def always_asked(api: Any) -> Any:+ if api is None:+ return None+ return GuardedApi(api._api if isinstance(api, GuardedApi) else api, skip_when_unreachable = False)diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 6e50ab5f17c..7bdd98e9930 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -61,6 +61,7 @@ from .diffusion_attention import (select_attention_backend,)from .diffusion_flow_shift import apply_comfy_flow_shift+from . import hub_reachabilityfrom .diffusion_comfy_quant import (comfy_fp8_backend,comfy_int8_backend,@@ -5511,7 +5512,11 @@ class VideoBackend:# Skipped wholesale offline: model_info is a Hub call, and the number it produces is the size of a download# that is not going to happen -- every file below either resolves from the cache or fails the load. total# stays 0, which load_progress already reads as "no estimate" and reports as a bare phase.- api = HfApi(token = hf_token or None) if not local_files_only else None+ api = (+ hub_reachability.guarded(HfApi(token = hf_token or None))+ if not local_files_only+ else None+ )for repo, wanted in requests:if api is None:break@@ -6495,6 +6500,7 @@ class VideoBackend:way: the base entry drops the dense ``text_encoder/`` shards whenever this resolves, sowithout it the artifact that replaces them is in no entry at all and the disk preflightpasses on a volume that cannot hold the 27 GB it is about to pull."""+ api = hub_reachability.always_asked(api) # picks the conditioner: bounded, never skippedfrom .video_minimax_h3_te import h3_te_quant_filename, h3_te_quant_sourcefilename = h3_te_quant_filename(scheme)@@ -6716,6 +6722,9 @@ class VideoBackend:"""``(repo_id, [(rfilename, size)])`` for every hosted denoiser artifact, or ``(None, [])``, so preflightcounts the checkpoint that replaces the dropped dense shards. The header probe asks with ``api``'s owntoken, so a gated artifact is judged by the same credentials that would stage it."""+ api = hub_reachability.always_asked(+ api+ ) # picks the denoiser source: bounded, never skippedtoken = getattr(api, "token", None)sources = VideoBackend._denoiser_prequant_source_list(fam,@@ -7087,7 +7096,7 @@ class VideoBackend:from huggingface_hub import HfApitotal = 0- api = HfApi(token = hf_token or None)+ api = hub_reachability.guarded(HfApi(token = hf_token or None))if skip_te_override is not None:# The encoder fetches already ran: plan around what they actually staged.skip_te_components = tuple(skip_te_override)@@ -7290,7 +7299,7 @@ class VideoBackend:return requiredtry:- api = HfApi(token = hf_token or None)+ api = hub_reachability.guarded(HfApi(token = hf_token or None))listings = _OneCommitListings(api)if gguf_filename and not Path(repo_id).expanduser().exists():listing = _video_listing_repo(repo_id)@@ -7574,7 +7583,7 @@ class VideoBackend:required_total = 0checkpoint_bytes = 0try:- api = HfApi(token = hf_token or None)+ api = hub_reachability.guarded(HfApi(token = hf_token or None))for repo, filename in wanted:if Path(repo).expanduser().exists():continue@@ -7823,7 +7832,8 @@ class VideoBackend:# Listed from the mirror (byte identical, so the same names and sizes), then the fetch repo is settled# against the EXACT names: an upstream snapshot that already holds every one of them keeps being used.listing = _video_listing_repo(base)- info = HfApi(token = hf_token or None).model_info(+ info = hub_reachability.model_info(+ HfApi(token = hf_token or None),listing,files_metadata = True,# The plan's commit describes the listing only when the plan reads the listing repo; an upstream kept fordiff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex 643a6c2ac57..6450655e580 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -117,6 +117,19 @@ def _no_real_mxc_drive_aliases(monkeypatch):monkeypatch.setenv("UNSLOTH_MXC_DRIVE_ALIAS", "0")[email protected](autouse = True)+def _forget_hub_unreachable_verdict():+ # Per-process Hub verdict must not leak between tests.+ def _forget():+ reach = sys.modules.get("core.inference.hub_reachability")+ if reach is not None:+ reach._UNREACHABLE.clear()++ _forget()+ yield+ _forget()++@pytest.fixture(autouse = True)def _forget_load_budget_snapshots():def _forget():diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex 34e9c6ba97e..d1442e23ea1 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -4690,6 +4690,52 @@ def _run_te_load(monkeypatch, on_load, **run_kwargs):return backend+def _forced_offline_load(monkeypatch, *, caller_offline):+ from core.inference import diffusion as dmod+ from huggingface_hub.errors import LocalEntryNotFoundError++ _stub_te_run_load(+ monkeypatch, base = "Qwen/Qwen-Image-2.1", base_bytes = 0, cache = {}, file_bytes = {}+ )+ monkeypatch.setattr(dmod, "_pick_cached_while_hub_unreachable", lambda *a, **k: True)+ prefetched, loaded = [], []+ monkeypatch.setattr(+ DiffusionBackend,+ "_prefetch_files",+ lambda self, *a, **k: prefetched.append(k["local_files_only"]),+ )++ def load(self, **kw):+ loaded.append(kw["local_files_only"])+ if kw["local_files_only"]:+ raise LocalEntryNotFoundError("transformer shard not in the cache")++ monkeypatch.setattr(DiffusionBackend, "load_pipeline", load)+ backend = DiffusionBackend()+ backend._loading = _LoadingState(repo_id = "Qwen/Qwen-Image-2.1", base_repo = None)+ backend._load_token = 7+ backend._run_load(+ _load_token = 7,+ repo_id = "Qwen/Qwen-Image-2.1",+ model_kind = "pipeline",+ local_files_only = caller_offline,+ )+ return backend, prefetched, loaded+++def test_a_partly_cached_load_forced_offline_retries_online(monkeypatch):+ # Manifest cached, a shard missing (interrupted download), Hub down moments ago: the online load still runs.+ backend, prefetched, loaded = _forced_offline_load(monkeypatch, caller_offline = False)+ assert prefetched == [True, False] and loaded == [True, False]+ assert backend.load_progress()["phase"] != "error"+++def test_a_caller_offline_load_is_not_retried_online(monkeypatch):+ backend, prefetched, loaded = _forced_offline_load(monkeypatch, caller_offline = True)+ assert prefetched == [True] and loaded == [True]+ assert backend.load_progress()["phase"] == "error"++def test_run_load_reports_the_pre_cast_encoder_download_as_downloading(monkeypatch):# GGUF + base cached; a cached fp8 sibling in the encoder repo must not stand in for the int8 file.cache = {"unsloth/Qwen-Image-2.1-GGUF": 700, "Qwen/Qwen-Image-2.1": 300, _TE_REPO: 950}@@ -10269,6 +10315,9 @@ def test_download_plan_drops_a_repo_the_fallback_root_holds_whole(monkeypatch, t},)monkeypatch.setattr("core.inference.diffusion._resolve_base_repo", lambda *a, **k: base)+ # The speech-pick revision HEAD is not part of this rule; unstubbed it fails to connect, and that failure would+ # (correctly) stand for the whole endpoint and skip the stubbed listings below.+ monkeypatch.setattr("core.inference.diffusion_compat._hub_revision", lambda *a, **k: None)_no_dense_prefetch(monkeypatch)_all_cached(monkeypatch)for name, size in _FLUX_BASE_SIBLINGS_BY_NAME.items():diff --git a/studio/backend/tests/test_hub_reachability.py b/studio/backend/tests/test_hub_reachability.pynew file mode 100644index 00000000000..ffb6ecb8a5d--- /dev/null+++ b/studio/backend/tests/test_hub_reachability.py@@ -0,0 +1,286 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Shared "Hub did not connect" verdict. Requests use huggingface_hub's HTTP stack on a mocked transport."""++from __future__ import annotations++import types+from pathlib import Path++import pytest++import core.inference.diffusion as D+import core.inference.diffusion_compat as compat+import core.inference.diffusion_prequant_cache as C+from core.inference import hub_reachability as R++huggingface_hub = pytest.importorskip("huggingface_hub")+if (+ getattr(huggingface_hub, "set_client_factory", None) is None+): # pragma: no cover - requests-era huggingface_hub+ pytest.skip("needs huggingface_hub's httpx client factory", allow_module_level = True)+import httpx # noqa: E402++REPO = "org/cached-pipe"++[email protected](autouse = True)+def _fresh_verdict(monkeypatch):+ monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", False)+ R._UNREACHABLE.clear()+ C._REMOTE_MEMO.clear()+ yield+ R._UNREACHABLE.clear()+++def _transport(handler):+ """Route huggingface_hub's client through ``handler``; returns the list of (method, url, read timeout)."""+ seen: list = []++ def wrapped(request):+ seen.append(+ (request.method, str(request.url), request.extensions.get("timeout", {}).get("read"))+ )+ return handler(request)++ huggingface_hub.set_client_factory(lambda: httpx.Client(transport = httpx.MockTransport(wrapped)))+ return seen+++def blackholed():+ """Accepts and never answers: every request times out."""++ def handler(request):+ raise httpx.ReadTimeout("timed out", request = request)++ try:+ yield _transport(handler)+ finally:+ from huggingface_hub.utils._http import default_client_factory+ huggingface_hub.set_client_factory(default_client_factory)+++def answering():+ """A healthy Hub: ``model_info`` answers a small open repo, a HEAD answers with a commit."""++ def handler(request):+ if request.method == "HEAD":+ return httpx.Response(+ 200,+ headers = {+ "x-repo-commit": "a" * 40,+ "etag": '"' + "b" * 64 + '"',+ "content-length": "4",+ },+ request = request,+ )+ return httpx.Response(+ 200,+ json = {+ "id": REPO,+ "sha": "a" * 40,+ "gated": False,+ "cardData": {"base_model": "org/base"},+ "siblings": [{"rfilename": "model_index.json", "size": 4}],+ },+ request = request,+ )++ try:+ yield _transport(handler)+ finally:+ from huggingface_hub.utils._http import default_client_factory+ huggingface_hub.set_client_factory(default_client_factory)+++def _cache(tmp_path: Path, repo: str, files: dict[str, bytes]) -> Path:+ """A minimal Hub cache entry (refs/main + snapshot files) for ``repo``; returns the cache root."""+ root = tmp_path / "hub"+ entry = root / ("models--" + repo.replace("/", "--"))+ commit = "c" * 40+ (entry / "refs").mkdir(parents = True, exist_ok = True)+ (entry / "refs" / "main").write_text(commit)+ for name, data in files.items():+ path = entry / "snapshots" / commit / name+ path.parent.mkdir(parents = True, exist_ok = True)+ path.write_bytes(data)+ return root+++# ------------------------------------------------------------------------------------------------------ model_info+def test_metadata_calls_share_one_timeout_on_a_blackholed_hub(blackholed):+ """Gated check, mirror SHA, size estimate: one request between them, not one wait each."""+ for _ in range(2):+ assert D._assert_base_repo_accessible(REPO, None) is None # fail open, as before+ assert D.DiffusionBackend._current_sha(REPO, None) is None+ failures: list = []+ assert D.DiffusionBackend._estimate_download_bytes(+ REPO, None, REPO, None, kind = "pipeline", failures_out = failures+ ) == (0, [])+ assert len(failures) == 1 # the estimate still reports itself partial+ assert len(blackholed) == 1, blackholed+++def test_model_info_is_bounded(blackholed):+ """huggingface_hub's default client has no timeout, so a dropped packet hung the call indefinitely."""+ D.DiffusionBackend._current_sha(REPO, None)+ assert blackholed[0][2] == R.METADATA_TIMEOUT_SECONDS+++def test_lookups_that_pick_what_loads_are_bounded_but_never_skipped(blackholed):+ """Lookups that decide WHICH weights load are still asked, only with a timeout."""+ R.note_hub_failure(httpx.ConnectError("refused"))+ assert D._hf_base_model(REPO, None) is None+ source = types.SimpleNamespace(kind = "repo", location = "org/prequant", filename = "x.safetensors")+ with pytest.raises(Exception):+ D.DiffusionBackend._prequant_source_hub_entry(source, None)+ asked = [s for s in blackholed if "/api/models/" in s[1]]+ assert len(asked) == 2, blackholed+ assert all(s[2] == R.METADATA_TIMEOUT_SECONDS for s in asked)+++def test_an_answering_hub_is_asked_every_time_with_unchanged_answers(answering):+ """No verdict when the Hub answers: every call still goes out and returns what the Hub said."""+ for _ in range(2):+ assert D._hf_base_model(REPO, None) == "org/base"+ assert D.DiffusionBackend._current_sha(REPO, None) == "a" * 40+ assert D._assert_base_repo_accessible(REPO, None) is None+ assert len(answering) == 6+ assert not R._UNREACHABLE+++def test_an_http_error_is_an_answer_not_an_unreachable_hub():+ """404 / 401 / 5xx come from a reachable Hub: they never silence the next call."""++ def handler(request):+ return httpx.Response(503, request = request)++ seen = _transport(handler)+ try:+ for _ in range(2):+ assert D.DiffusionBackend._current_sha(REPO, None) is None+ finally:+ from huggingface_hub.utils._http import default_client_factory+ huggingface_hub.set_client_factory(default_client_factory)+ assert len([s for s in seen if "/api/models/" in s[1]]) >= 2+ assert not R._UNREACHABLE+++def test_the_verdict_expires_and_a_success_clears_it(blackholed, monkeypatch):+ D.DiffusionBackend._current_sha(REPO, None)+ assert R.hub_unreachable()+ R._UNREACHABLE[R.endpoint()] -= R.TTL_SECONDS + 1+ assert not R.hub_unreachable()+ R.note_hub_failure(httpx.ConnectError("refused"))+ assert R.hub_unreachable()+ R.note_hub_success()+ assert not R.hub_unreachable()+++def test_a_gated_uncached_base_keeps_todays_answers(monkeypatch):+ """An answering Hub that says gated still refuses an uncached base."""+ monkeypatch.setattr("huggingface_hub.try_to_load_from_cache", lambda *a, **k: None)+ R.note_hub_failure(httpx.ConnectError("refused"))+ assert D._assert_base_repo_accessible("org/gated", None) is None+ R.note_hub_success()++ def handler(request):+ return httpx.Response(+ 401, headers = {"X-Error-Code": "GatedRepo", "X-Error-Message": "gated"}, request = request+ )++ seen = _transport(handler)+ try:+ with pytest.raises(ValueError, match = "org/gated"):+ D._assert_base_repo_accessible("org/gated", None)+ finally:+ from huggingface_hub.utils._http import default_client_factory+ huggingface_hub.set_client_factory(default_client_factory)+ assert seen and not R._UNREACHABLE+++def test_a_duck_typed_client_gets_its_call_unchanged():+ """Test doubles and wrappers that do not take ``timeout`` keep working."""+ calls = []++ class _Api:+ def model_info(+ self,+ repo_id,+ files_metadata = False,+ ):+ calls.append((repo_id, files_metadata))+ return "info"++ assert R.guarded(_Api()).model_info("a/b", files_metadata = True) == "info"+ assert calls == [("a/b", True)]+ R.note_hub_failure(httpx.ConnectError("refused"))+ with pytest.raises(ConnectionError):+ R.guarded(_Api()).model_info("a/b", files_metadata = True)+ assert len(calls) == 1+++# ------------------------------------------------------------------------------------------------- cache fallbacks+def test_resolve_gguf_reads_a_cached_checkpoint_off_disk_while_the_hub_is_down(+ tmp_path, monkeypatch+):+ root = _cache(tmp_path, "org/gguf", {"model-Q4_K.gguf": b"GGUF"})+ monkeypatch.setattr(D, "hub_cache_dir", lambda: str(root))+ calls = []+ real = huggingface_hub.hf_hub_download++ def spy(*a, **k):+ calls.append(k.get("local_files_only"))+ return real(*a, **k)++ monkeypatch.setattr(huggingface_hub, "hf_hub_download", spy)+ R.note_hub_failure(httpx.ConnectError("refused"))+ path = D.DiffusionBackend._resolve_gguf_path(None, "org/gguf", "model-Q4_K.gguf", None)+ assert Path(path).read_bytes() == b"GGUF"+ assert calls == [True]+++def test_resolve_gguf_still_downloads_an_uncached_checkpoint(tmp_path, monkeypatch):+ """A first download is never refused off a verdict: the Hub may be back."""+ root = tmp_path / "hub"+ monkeypatch.setattr(D, "hub_cache_dir", lambda: str(root))+ calls = []+ monkeypatch.setattr(+ huggingface_hub,+ "hf_hub_download",+ lambda *a, **k: calls.append(k.get("local_files_only")) or "x",+ )+ R.note_hub_failure(httpx.ConnectError("refused"))+ assert D.DiffusionBackend._resolve_gguf_path(None, "org/gguf", "model-Q4_K.gguf", None) == "x"+ assert calls == [False]+++def test_compat_revision_probe_is_skipped_while_the_hub_is_down(blackholed):+ assert compat._hub_revision("org/gguf", "model-Q4_K.gguf", None) is None+ assert compat._hub_revision("org/gguf", "model-Q4_K.gguf", None) is None+ assert len(blackholed) == 1+++def test_a_cached_pick_loads_off_the_cache_only_while_the_hub_is_down(tmp_path, monkeypatch):+ root = _cache(tmp_path, REPO, {"model_index.json": b"{}", "transformer/config.json": b"{}"})+ monkeypatch.setattr(D, "hub_cache_dir", lambda: str(root))+ fam = types.SimpleNamespace(name = "flux.1")+ pick = (fam, "pipeline", REPO, None, REPO, REPO)+ assert (+ D._pick_cached_while_hub_unreachable(*pick) is False+ ) # Hub fine: the online load is untouched+ R.note_hub_failure(httpx.ConnectError("refused"))+ assert D._pick_cached_while_hub_unreachable(*pick) is True+ assert D._cached_while_hub_unreachable(REPO, "transformer/config.json") is True+ # Nothing cached: a first download keeps asking the Hub.+ assert (+ D._pick_cached_while_hub_unreachable(fam, "pipeline", "org/new", None, "org/new", "org/new")+ is False+ )+ # A GGUF pick needs its checkpoint cached too, not only the base.+ assert (+ D._pick_cached_while_hub_unreachable(fam, "gguf", "org/gguf", "m.gguf", REPO, REPO) is False+ )