Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion/video: share the Hub reachability verdict across cached-load calls (#15059)

Daniel Hand497ab25048d
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 170f9a70cd0..95cb17b1b13 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -56,6 +56,7 @@ from .diffusion_content import (
whole_pipeline_gguf_family,
)
from .diffusion_gguf_pipeline import load_whole_pipeline_gguf, whole_pipeline_gguf_resident_mib
+from . import hub_reachability
from .diffusion_families import (
SMALL_DIT_BF16_WHEN_RESIDENT,
small_dit_rule_excludes,
@@ -1357,7 +1358,7 @@ def _assert_base_repo_accessible(
return status in (401, 403)
try:
- gated = getattr(HfApi().model_info(repo, token = hf_token), "gated", None)
+ gated = getattr(hub_reachability.model_info(HfApi(), repo, token = hf_token), "gated", None)
except GatedRepoError: # a gated repo can also withhold its metadata
if _already_downloaded():
return other_root_snapshot
@@ -1505,12 +1506,17 @@ def _te_quant_for_supplied_encoders(requested: Optional[str]) -> Optional[str]:
def _hub_file_size(repo_id: str, filename: str, hf_token: Optional[str]) -> int:
+ # get_paths_info takes no timeout.
+ if hub_reachability.hub_unreachable():
+ return 0
try:
from huggingface_hub import HfApi
for info in HfApi().get_paths_info(repo_id, [filename], token = hf_token):
+ hub_reachability.note_hub_success()
return int(getattr(info, "size", 0) or 0)
- except Exception: # noqa: BLE001 - a size is a progress-bar hint, never a refusal
- pass
+ hub_reachability.note_hub_success()
+ except Exception as exc: # noqa: BLE001 - a size is a progress-bar hint, never a refusal
+ hub_reachability.note_hub_failure(exc)
return 0
@@ -4235,12 +4241,15 @@ class DiffusionBackend:
# re-pulls the whole multi-GB file inside the load lock, after eviction, where unload cannot preempt it and
# progress already reported 100%.
cache_dir = hub_cache_dir()
+ offline_ok = local_files_only or hub_reachability.hub_unreachable()
+ cached_here = False
# Best-effort: an unreadable cache raises here, and letting that escape would abort a load that only had to
# download. Any failure falls through to the pinned download below.
try:
- if not isinstance(
+ cached_here = isinstance(
try_to_load_from_cache(repo_id, gguf_filename, cache_dir = cache_dir), str
- ):
+ )
+ if not cached_here:
# Same other-root reuse as the pre-quant checkpoint, reached THROUGH that root rather than returned
# raw, so the ref still resolves and a republished GGUF is picked up; the blob is reused, and offline
# the cached pointer comes back anyway.
@@ -4252,7 +4261,7 @@ class DiffusionBackend:
gguf_filename,
token = hf_token,
cache_dir = None,
- local_files_only = local_files_only,
+ local_files_only = offline_ok,
)
except Exception: # noqa: BLE001 - revalidation is a bonus, never a new failure
return elsewhere
@@ -4263,7 +4272,7 @@ class DiffusionBackend:
gguf_filename,
token = hf_token,
cache_dir = cache_dir,
- local_files_only = local_files_only,
+ local_files_only = local_files_only or (cached_here and offline_ok),
)
def _dense_quant_prefetch_needed(
@@ -5594,20 +5603,51 @@ class DiffusionBackend:
# fallback, never the override: it fires only when the estimate came back empty, since the metadata that
# fills it is the same call whose failure earned the escape. Without it the load 401s with every byte
# already on disk.
- kwargs["_base_local_dir"] = (
- self._prefetch_files(
+ # Where diffusers' own "Couldn't connect" fallback lands, minus its untimed revision / model_info calls.
+ forced_offline = False
+ if not local_files_only and _pick_cached_while_hub_unreachable(
+ fam, kind, kwargs["repo_id"], kwargs.get("gguf_filename"), base, fetch_base
+ ):
+ logger.info(
+ "diffusion.hub_unreachable: %s is cached and the Hub did not connect moments ago; loading "
+ "from the cache",
kwargs["repo_id"],
- kwargs.get("gguf_filename"),
- base,
- base_files,
- kwargs.get("hf_token"),
- cancel_event = cancel_event,
- fetch_base = fetch_base,
- local_files_only = local_files_only,
)
- or base_snapshot
- )
- self.load_pipeline(**kwargs)
+ forced_offline = True
+
+ def _prefetch_and_load(offline: bool) -> None:
+ kwargs["local_files_only"] = offline
+ kwargs["_base_local_dir"] = (
+ self._prefetch_files(
+ kwargs["repo_id"],
+ kwargs.get("gguf_filename"),
+ base,
+ base_files,
+ kwargs.get("hf_token"),
+ cancel_event = cancel_event,
+ fetch_base = fetch_base,
+ local_files_only = offline,
+ )
+ or base_snapshot
+ )
+ self.load_pipeline(**kwargs)
+
+ try:
+ _prefetch_and_load(local_files_only or forced_offline)
+ except OSError as exc:
+ # Only the manifest is checked up front: an interrupted download still needs the online load.
+ if (
+ not forced_offline
+ or self._load_token != token
+ or (cancel_event is not None and cancel_event.is_set())
+ ):
+ raise
+ logger.info(
+ "diffusion.hub_unreachable: %s is not fully cached (%s); retrying online",
+ kwargs["repo_id"],
+ exc,
+ )
+ _prefetch_and_load(False)
with self._load_cancel_lock:
# Only clear the marker if this load is still current (a superseder has its own token).
if self._load_token == token:
@@ -6200,7 +6240,13 @@ class DiffusionBackend:
from huggingface_hub.errors import RepositoryNotFoundError
try:
- info = HfApi(token = hf_token or None).model_info(source.location, files_metadata = True)
+ # Decides what loads: always asked, only bounded.
+ info = hub_reachability.model_info(
+ HfApi(token = hf_token or None),
+ source.location,
+ files_metadata = True,
+ skip_when_unreachable = False,
+ )
except RepositoryNotFoundError as exc:
# 401/403/404 on the repo = no hosted checkpoint, not a partial listing (GatedRepoError subclasses this).
logger.info(
@@ -6360,7 +6406,9 @@ class DiffusionBackend:
if kind == "pipeline":
def _pipeline_listing(metadata_repo: str) -> tuple[Any, Any, list[Any]]:
- info = api.model_info(metadata_repo, files_metadata = True, token = hf_token)
+ info = hub_reachability.model_info(
+ api, metadata_repo, files_metadata = True, token = hf_token
+ )
components = _pipeline_components_from_index(
metadata_repo,
info,
@@ -6429,7 +6477,9 @@ class DiffusionBackend:
_record_revision(revisions_out, metadata_repo, info)
return total, base_files
if gguf_filename and not Path(repo_id).expanduser().exists():
- info = api.model_info(repo_id, files_metadata = True, token = hf_token)
+ info = hub_reachability.model_info(
+ api, repo_id, files_metadata = True, token = hf_token
+ )
gguf_bytes = sum(s.size or 0 for s in info.siblings if s.rfilename == gguf_filename)
total += gguf_bytes
if sizes_out is not None:
@@ -6466,7 +6516,9 @@ class DiffusionBackend:
)
def _base_listing(listing_repo: str) -> list[Any]:
- info = api.model_info(listing_repo, files_metadata = True, token = hf_token)
+ info = hub_reachability.model_info(
+ api, listing_repo, files_metadata = True, token = hf_token
+ )
# Under the repo it describes: a mirror's commit says nothing about a cached upstream snapshot.
_record_revision(revisions_out, listing_repo, info)
return [s for s in info.siblings if not _listing_extra(listing_repo, s.rfilename)]
@@ -7067,7 +7119,10 @@ class DiffusionBackend:
"""The current Hub commit for a mirror repo, or None when metadata is unavailable."""
try:
from huggingface_hub import HfApi
- return getattr(HfApi().model_info(repo_id, token = hf_token), "sha", None) or None
+ return (
+ getattr(hub_reachability.model_info(HfApi(), repo_id, token = hf_token), "sha", None)
+ or None
+ )
except Exception: # noqa: BLE001 -- no SHA means conservative staging
return None
@@ -9390,7 +9445,10 @@ class DiffusionBackend:
# config is a REPO ID, and diffusers forwards this into the load_config() that resolves
# it, so without the flag this branch reaches the Hub on a load nobody asked for. The
# pipeline assembly below was already guarded; this call was not.
- "local_files_only": local_files_only,
+ "local_files_only": local_files_only
+ or _cached_while_hub_unreachable(
+ fetch_base, "transformer/config.json"
+ ),
# config is the family base's (2511); 2509 / original Edit lack its zero_cond_t
**transformer_config_overrides_for(
fam, gguf_filename, repo_id, display_repo_id, base
@@ -14706,6 +14764,48 @@ def _remember_companion_base(repo_id: str, base: str) -> None:
logger.debug("diffusion.companion_link_record_failed: %s", exc)
+def _cached_while_hub_unreachable(repo_id: Any, filename: str) -> bool:
+ """Hub just failed to connect AND ``filename`` is cached: ``local_files_only=True`` then returns what the online
+ call would after its HEAD timed out. False on any doubt, so a first download still asks."""
+ if not hub_reachability.hub_unreachable():
+ return False
+ try:
+ if not isinstance(repo_id, str) or Path(repo_id).expanduser().exists():
+ return False
+ from huggingface_hub import try_to_load_from_cache
+ return isinstance(try_to_load_from_cache(repo_id, filename, cache_dir = hub_cache_dir()), str)
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _pick_cached_while_hub_unreachable(
+ fam: Any,
+ kind: str,
+ repo_id: str,
+ gguf_filename: Optional[str],
+ base: Optional[str],
+ fetch_base: Optional[str],
+) -> bool:
+ """Base manifest cached (or local) and, for a GGUF / single-file pick, the checkpoint too."""
+ if not hub_reachability.hub_unreachable():
+ return False
+ try:
+ manifest = _base_manifest_name(fam)
+ if not (
+ (bool(base) and _is_local_path(base))
+ or any(
+ _cached_while_hub_unreachable(repo, manifest)
+ for repo in dict.fromkeys(r for r in (fetch_base, base) if r)
+ )
+ ):
+ return False
+ if kind != "pipeline" and gguf_filename and not _is_local_path(repo_id):
+ return _cached_while_hub_unreachable(repo_id, gguf_filename)
+ return True
+ except Exception: # noqa: BLE001 - any doubt keeps the online load
+ return False
+
+
def _hf_base_model(repo_id: str, hf_token: Optional[str]) -> Optional[str]:
"""The diffusers base repo from a GGUF repo's ``base_model`` tag, or None. Lets one family entry
cover every variant (Turbo/full, schnell/dev, the 2512 Qwen revision). Skipped for local
@@ -14714,7 +14814,14 @@ def _hf_base_model(repo_id: str, hf_token: Optional[str]) -> Optional[str]:
return None
try:
from huggingface_hub import HfApi
- meta = HfApi().model_info(repo_id, token = hf_token).cardData or {}
+
+ # Picks the base: never skipped, or a recovered network would get the family default.
+ meta = (
+ hub_reachability.model_info(
+ HfApi(), repo_id, token = hf_token, skip_when_unreachable = False
+ ).cardData
+ or {}
+ )
except Exception: # noqa: BLE001 - best-effort; fall back to the family default
return None
base = meta.get("base_model")
diff --git a/studio/backend/core/inference/diffusion_compat.py b/studio/backend/core/inference/diffusion_compat.py
index b012dd8b1be..ee97ea62989 100644
--- a/studio/backend/core/inference/diffusion_compat.py
+++ b/studio/backend/core/inference/diffusion_compat.py
@@ -157,6 +157,10 @@ def _hub_revision(repo_id: str, gguf_filename: str, hf_token: Optional[str]) ->
One HEAD, no body: the caller only needs to know whether the local copy is still the current
one, and an offline or erroring host must leave today's verdict alone."""
+ from core.inference import hub_reachability
+
+ if hub_reachability.hub_unreachable():
+ return None # the Hub failed to connect moments ago: unanswerable, without another timeout
try:
from huggingface_hub import get_hf_file_metadata, hf_hub_url
meta = get_hf_file_metadata(
@@ -164,8 +168,10 @@ def _hub_revision(repo_id: str, gguf_filename: str, hf_token: Optional[str]) ->
token = hf_token,
timeout = _HEADER_TIMEOUT_SECONDS,
)
- except Exception: # noqa: BLE001 — a revision we cannot read is not a verdict
+ except Exception as exc: # noqa: BLE001 — a revision we cannot read is not a verdict
+ hub_reachability.note_hub_failure(exc)
return None
+ hub_reachability.note_hub_success()
return getattr(meta, "commit_hash", None) or None
diff --git a/studio/backend/core/inference/diffusion_prequant_cache.py b/studio/backend/core/inference/diffusion_prequant_cache.py
index 637deb9e536..f173c95f89c 100644
--- a/studio/backend/core/inference/diffusion_prequant_cache.py
+++ b/studio/backend/core/inference/diffusion_prequant_cache.py
@@ -35,6 +35,8 @@ from contextlib import contextmanager
from pathlib import Path
from typing import Any, Iterable, Optional
+from . import hub_reachability as _reach
+
ENV_MODE = "UNSLOTH_DIFFUSION_PREQUANT_CACHE"
ENV_DIR = "UNSLOTH_DIFFUSION_PREQUANT_CACHE_DIR"
ENV_MAX_GB = "UNSLOTH_DIFFUSION_PREQUANT_CACHE_MAX_GB"
@@ -585,37 +587,10 @@ _REMOTE_TTL_SECONDS = 300.0
_REMOTE_TIMEOUT_SECONDS = 5.0
_REMOTE_MEMO: dict[tuple, tuple] = {}
_GONE = "gone"
-# Endpoint -> monotonic time its connection failed; later probes answer "unknown" for the TTL instead of timing out.
-_UNREACHABLE: dict[str, float] = {}
-
-
-def _connection_failed(exc: BaseException) -> bool:
- """Connect / timeout failure only; HTTP answers are per repo and proxy errors are not cache fallbacks in HF."""
- try:
- import httpx
- if isinstance(exc, httpx.ProxyError):
- return False
- if isinstance(exc, (httpx.ConnectError, httpx.TimeoutException)):
- return True
- except ImportError:
- pass
- try:
- import requests
- if isinstance(exc, requests.exceptions.ProxyError):
- return False
- if isinstance(exc, (requests.exceptions.ConnectionError, requests.exceptions.Timeout)):
- return True
- except ImportError:
- pass
- return isinstance(exc, (OSError, TimeoutError)) and not hasattr(exc, "response")
-
-
-def _endpoint() -> str:
- try:
- from huggingface_hub import constants
- return str(constants.ENDPOINT)
- except Exception: # noqa: BLE001
- return ""
+# Shared hub_reachability verdict: after one failure, lookups answer "unknown" (local snapshot stands) for the TTL.
+_UNREACHABLE = _reach._UNREACHABLE
+_connection_failed = _reach.connection_failed
+_endpoint = _reach.endpoint
def _hub_coords(path: Any) -> Optional[tuple[str, str, str, str]]:
@@ -669,10 +644,8 @@ def _remote_etag(
if hit is not None and time.monotonic() - hit[0] < _REMOTE_TTL_SECONDS:
return hit[1]
endpoint = _endpoint()
- with _lock:
- down_at = _UNREACHABLE.get(endpoint)
- if down_at is not None and time.monotonic() - down_at < _REMOTE_TTL_SECONDS:
- return None
+ if _reach.hub_unreachable(endpoint):
+ return None # the Hub did not answer moments ago: unanswerable, without another timeout
try:
from huggingface_hub import get_hf_file_metadata, hf_hub_url
from huggingface_hub.utils import EntryNotFoundError
@@ -693,10 +666,9 @@ def _remote_etag(
except EntryNotFoundError:
value = _GONE
except Exception as exc: # noqa: BLE001 - offline, gated without a token, rate limited: unanswerable
- if _connection_failed(exc):
- with _lock:
- _UNREACHABLE[endpoint] = time.monotonic()
+ _reach.note_hub_failure(exc, endpoint)
return None
+ _reach.note_hub_success(endpoint)
with _lock:
_UNREACHABLE.pop(endpoint, None)
_REMOTE_MEMO[memo] = (time.monotonic(), value)
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index 64b687466bd..5bebbf854ad 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -1975,7 +1975,10 @@ def te_prequant_hub_files(
found[component] = [(mirrored[0], os.path.getsize(mirrored[1]))]
continue
try:
- info = api.model_info(source.location, files_metadata = True)
+ from .hub_reachability import always_asked
+
+ # Picks the encoder: always asked, only bounded.
+ info = always_asked(api).model_info(source.location, files_metadata = True)
except Exception as exc: # noqa: BLE001 -- unavailable pre-cast means the dense encoder
_warn(logger, f"hub_files:{source.location}", exc)
continue
diff --git a/studio/backend/core/inference/hub_reachability.py b/studio/backend/core/inference/hub_reachability.py
new file mode 100644
index 00000000000..04eb1ba572d
--- /dev/null
+++ b/studio/backend/core/inference/hub_reachability.py
@@ -0,0 +1,147 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Per-process verdict on whether the Hub endpoint answers. After one transport failure, cached-load Hub calls use
+their fallback for ``TTL_SECONDS`` instead of each waiting out a timeout. HTTP answers never set the verdict.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any, Optional
+
+# Matches the prequant freshness memo.
+TTL_SECONDS = 300.0
+METADATA_TIMEOUT_SECONDS = 10.0
+
+_UNREACHABLE: dict[str, float] = {}
+_lock = threading.Lock()
+
+
+class HubUnreachableError(ConnectionError):
+ """A ``ConnectionError`` so existing transport-failure handling takes it unchanged."""
+
+
+def connection_failed(exc: BaseException) -> bool:
+ """Not proxy errors: huggingface_hub raises those instead of falling back to the cache."""
+ if isinstance(exc, HubUnreachableError):
+ return True
+ try:
+ import httpx
+ if isinstance(exc, httpx.ProxyError):
+ return False
+ if isinstance(exc, (httpx.ConnectError, httpx.TimeoutException)):
+ return True
+ except ImportError:
+ pass
+ try:
+ import requests
+ if isinstance(exc, requests.exceptions.ProxyError):
+ return False
+ if isinstance(exc, (requests.exceptions.ConnectionError, requests.exceptions.Timeout)):
+ return True
+ except ImportError:
+ pass
+ return isinstance(exc, (OSError, TimeoutError)) and not hasattr(exc, "response")
+
+
+def endpoint() -> str:
+ try:
+ from huggingface_hub import constants
+ return str(constants.ENDPOINT)
+ except Exception: # noqa: BLE001
+ return ""
+
+
+def hub_unreachable(at: Optional[str] = None) -> bool:
+ key = endpoint() if at is None else at
+ with _lock:
+ down_at = _UNREACHABLE.get(key)
+ return down_at is not None and time.monotonic() - down_at < TTL_SECONDS
+
+
+def note_hub_failure(exc: BaseException, at: Optional[str] = None) -> bool:
+ if not connection_failed(exc):
+ return False
+ if isinstance(exc, HubUnreachableError):
+ return True # a skipped call: the verdict that caused it stays dated at the real failure
+ key = endpoint() if at is None else at
+ with _lock:
+ _UNREACHABLE[key] = time.monotonic()
+ return True
+
+
+def note_hub_success(at: Optional[str] = None) -> None:
+ key = endpoint() if at is None else at
+ with _lock:
+ _UNREACHABLE.pop(key, None)
+
+
+def _answered(exc: BaseException) -> bool:
+ return getattr(exc, "response", None) is not None
+
+
+def model_info(
+ api: Any,
+ repo_id: str,
+ *,
+ skip_when_unreachable: bool = True,
+ **kwargs: Any,
+) -> Any:
+ """``skip_when_unreachable=False`` is for an answer that decides WHAT loads: always asked, only bounded."""
+ at = endpoint()
+ if skip_when_unreachable and hub_unreachable(at):
+ raise HubUnreachableError(
+ f"{at or 'the Hub'} did not connect within the last {int(TTL_SECONDS)} s"
+ )
+ if _is_hf_api(api):
+ # Duck-typed clients (test doubles) may not take ``timeout``.
+ kwargs.setdefault("timeout", METADATA_TIMEOUT_SECONDS)
+ try:
+ info = api.model_info(repo_id, **kwargs)
+ except Exception as exc:
+ if not note_hub_failure(exc, at) and _answered(exc):
+ note_hub_success(at)
+ raise
+ note_hub_success(at)
+ return info
+
+
+def _is_hf_api(api: Any) -> bool:
+ try:
+ # Defining module: tests may swap the package attribute for a double.
+ from huggingface_hub.hf_api import HfApi
+ return isinstance(api, HfApi)
+ except Exception: # noqa: BLE001
+ return False
+
+
+class GuardedApi:
+ def __init__(
+ self,
+ api: Any,
+ skip_when_unreachable: bool = True,
+ ):
+ self._api = api
+ self._skip = skip_when_unreachable
+
+ def model_info(self, repo_id: str, *args: Any, **kwargs: Any) -> Any:
+ if args:
+ return self._api.model_info(repo_id, *args, **kwargs)
+ return model_info(self._api, repo_id, skip_when_unreachable = self._skip, **kwargs)
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self._api, name)
+
+
+def guarded(api: Any, *, skip_when_unreachable: bool = True) -> Any:
+ if api is None or isinstance(api, GuardedApi):
+ return api
+ return GuardedApi(api, skip_when_unreachable)
+
+
+def always_asked(api: Any) -> Any:
+ if api is None:
+ return None
+ return GuardedApi(api._api if isinstance(api, GuardedApi) else api, skip_when_unreachable = False)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 6e50ab5f17c..7bdd98e9930 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -61,6 +61,7 @@ from .diffusion_attention import (
select_attention_backend,
)
from .diffusion_flow_shift import apply_comfy_flow_shift
+from . import hub_reachability
from .diffusion_comfy_quant import (
comfy_fp8_backend,
comfy_int8_backend,
@@ -5511,7 +5512,11 @@ class VideoBackend:
# Skipped wholesale offline: model_info is a Hub call, and the number it produces is the size of a download
# that is not going to happen -- every file below either resolves from the cache or fails the load. total
# stays 0, which load_progress already reads as "no estimate" and reports as a bare phase.
- api = HfApi(token = hf_token or None) if not local_files_only else None
+ api = (
+ hub_reachability.guarded(HfApi(token = hf_token or None))
+ if not local_files_only
+ else None
+ )
for repo, wanted in requests:
if api is None:
break
@@ -6495,6 +6500,7 @@ class VideoBackend:
way: the base entry drops the dense ``text_encoder/`` shards whenever this resolves, so
without it the artifact that replaces them is in no entry at all and the disk preflight
passes on a volume that cannot hold the 27 GB it is about to pull."""
+ api = hub_reachability.always_asked(api) # picks the conditioner: bounded, never skipped
from .video_minimax_h3_te import h3_te_quant_filename, h3_te_quant_source
filename = h3_te_quant_filename(scheme)
@@ -6716,6 +6722,9 @@ class VideoBackend:
"""``(repo_id, [(rfilename, size)])`` for every hosted denoiser artifact, or ``(None, [])``, so preflight
counts the checkpoint that replaces the dropped dense shards. The header probe asks with ``api``'s own
token, so a gated artifact is judged by the same credentials that would stage it."""
+ api = hub_reachability.always_asked(
+ api
+ ) # picks the denoiser source: bounded, never skipped
token = getattr(api, "token", None)
sources = VideoBackend._denoiser_prequant_source_list(
fam,
@@ -7087,7 +7096,7 @@ class VideoBackend:
from huggingface_hub import HfApi
total = 0
- api = HfApi(token = hf_token or None)
+ api = hub_reachability.guarded(HfApi(token = hf_token or None))
if skip_te_override is not None:
# The encoder fetches already ran: plan around what they actually staged.
skip_te_components = tuple(skip_te_override)
@@ -7290,7 +7299,7 @@ class VideoBackend:
return required
try:
- api = HfApi(token = hf_token or None)
+ api = hub_reachability.guarded(HfApi(token = hf_token or None))
listings = _OneCommitListings(api)
if gguf_filename and not Path(repo_id).expanduser().exists():
listing = _video_listing_repo(repo_id)
@@ -7574,7 +7583,7 @@ class VideoBackend:
required_total = 0
checkpoint_bytes = 0
try:
- api = HfApi(token = hf_token or None)
+ api = hub_reachability.guarded(HfApi(token = hf_token or None))
for repo, filename in wanted:
if Path(repo).expanduser().exists():
continue
@@ -7823,7 +7832,8 @@ class VideoBackend:
# Listed from the mirror (byte identical, so the same names and sizes), then the fetch repo is settled
# against the EXACT names: an upstream snapshot that already holds every one of them keeps being used.
listing = _video_listing_repo(base)
- info = HfApi(token = hf_token or None).model_info(
+ info = hub_reachability.model_info(
+ HfApi(token = hf_token or None),
listing,
files_metadata = True,
# The plan's commit describes the listing only when the plan reads the listing repo; an upstream kept for
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 643a6c2ac57..6450655e580 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -117,6 +117,19 @@ def _no_real_mxc_drive_aliases(monkeypatch):
monkeypatch.setenv("UNSLOTH_MXC_DRIVE_ALIAS", "0")
[email protected](autouse = True)
+def _forget_hub_unreachable_verdict():
+ # Per-process Hub verdict must not leak between tests.
+ def _forget():
+ reach = sys.modules.get("core.inference.hub_reachability")
+ if reach is not None:
+ reach._UNREACHABLE.clear()
+
+ _forget()
+ yield
+ _forget()
+
+
@pytest.fixture(autouse = True)
def _forget_load_budget_snapshots():
def _forget():
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 34e9c6ba97e..d1442e23ea1 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -4690,6 +4690,52 @@ def _run_te_load(monkeypatch, on_load, **run_kwargs):
return backend
+def _forced_offline_load(monkeypatch, *, caller_offline):
+ from core.inference import diffusion as dmod
+ from huggingface_hub.errors import LocalEntryNotFoundError
+
+ _stub_te_run_load(
+ monkeypatch, base = "Qwen/Qwen-Image-2.1", base_bytes = 0, cache = {}, file_bytes = {}
+ )
+ monkeypatch.setattr(dmod, "_pick_cached_while_hub_unreachable", lambda *a, **k: True)
+ prefetched, loaded = [], []
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_prefetch_files",
+ lambda self, *a, **k: prefetched.append(k["local_files_only"]),
+ )
+
+ def load(self, **kw):
+ loaded.append(kw["local_files_only"])
+ if kw["local_files_only"]:
+ raise LocalEntryNotFoundError("transformer shard not in the cache")
+
+ monkeypatch.setattr(DiffusionBackend, "load_pipeline", load)
+ backend = DiffusionBackend()
+ backend._loading = _LoadingState(repo_id = "Qwen/Qwen-Image-2.1", base_repo = None)
+ backend._load_token = 7
+ backend._run_load(
+ _load_token = 7,
+ repo_id = "Qwen/Qwen-Image-2.1",
+ model_kind = "pipeline",
+ local_files_only = caller_offline,
+ )
+ return backend, prefetched, loaded
+
+
+def test_a_partly_cached_load_forced_offline_retries_online(monkeypatch):
+ # Manifest cached, a shard missing (interrupted download), Hub down moments ago: the online load still runs.
+ backend, prefetched, loaded = _forced_offline_load(monkeypatch, caller_offline = False)
+ assert prefetched == [True, False] and loaded == [True, False]
+ assert backend.load_progress()["phase"] != "error"
+
+
+def test_a_caller_offline_load_is_not_retried_online(monkeypatch):
+ backend, prefetched, loaded = _forced_offline_load(monkeypatch, caller_offline = True)
+ assert prefetched == [True] and loaded == [True]
+ assert backend.load_progress()["phase"] == "error"
+
+
def test_run_load_reports_the_pre_cast_encoder_download_as_downloading(monkeypatch):
# GGUF + base cached; a cached fp8 sibling in the encoder repo must not stand in for the int8 file.
cache = {"unsloth/Qwen-Image-2.1-GGUF": 700, "Qwen/Qwen-Image-2.1": 300, _TE_REPO: 950}
@@ -10269,6 +10315,9 @@ def test_download_plan_drops_a_repo_the_fallback_root_holds_whole(monkeypatch, t
},
)
monkeypatch.setattr("core.inference.diffusion._resolve_base_repo", lambda *a, **k: base)
+ # The speech-pick revision HEAD is not part of this rule; unstubbed it fails to connect, and that failure would
+ # (correctly) stand for the whole endpoint and skip the stubbed listings below.
+ monkeypatch.setattr("core.inference.diffusion_compat._hub_revision", lambda *a, **k: None)
_no_dense_prefetch(monkeypatch)
_all_cached(monkeypatch)
for name, size in _FLUX_BASE_SIBLINGS_BY_NAME.items():
diff --git a/studio/backend/tests/test_hub_reachability.py b/studio/backend/tests/test_hub_reachability.py
new file mode 100644
index 00000000000..ffb6ecb8a5d
--- /dev/null
+++ b/studio/backend/tests/test_hub_reachability.py
@@ -0,0 +1,286 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Shared "Hub did not connect" verdict. Requests use huggingface_hub's HTTP stack on a mocked transport."""
+
+from __future__ import annotations
+
+import types
+from pathlib import Path
+
+import pytest
+
+import core.inference.diffusion as D
+import core.inference.diffusion_compat as compat
+import core.inference.diffusion_prequant_cache as C
+from core.inference import hub_reachability as R
+
+huggingface_hub = pytest.importorskip("huggingface_hub")
+if (
+ getattr(huggingface_hub, "set_client_factory", None) is None
+): # pragma: no cover - requests-era huggingface_hub
+ pytest.skip("needs huggingface_hub's httpx client factory", allow_module_level = True)
+import httpx # noqa: E402
+
+REPO = "org/cached-pipe"
+
+
[email protected](autouse = True)
+def _fresh_verdict(monkeypatch):
+ monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", False)
+ R._UNREACHABLE.clear()
+ C._REMOTE_MEMO.clear()
+ yield
+ R._UNREACHABLE.clear()
+
+
+def _transport(handler):
+ """Route huggingface_hub's client through ``handler``; returns the list of (method, url, read timeout)."""
+ seen: list = []
+
+ def wrapped(request):
+ seen.append(
+ (request.method, str(request.url), request.extensions.get("timeout", {}).get("read"))
+ )
+ return handler(request)
+
+ huggingface_hub.set_client_factory(lambda: httpx.Client(transport = httpx.MockTransport(wrapped)))
+ return seen
+
+
+def blackholed():
+ """Accepts and never answers: every request times out."""
+
+ def handler(request):
+ raise httpx.ReadTimeout("timed out", request = request)
+
+ try:
+ yield _transport(handler)
+ finally:
+ from huggingface_hub.utils._http import default_client_factory
+ huggingface_hub.set_client_factory(default_client_factory)
+
+
+def answering():
+ """A healthy Hub: ``model_info`` answers a small open repo, a HEAD answers with a commit."""
+
+ def handler(request):
+ if request.method == "HEAD":
+ return httpx.Response(
+ 200,
+ headers = {
+ "x-repo-commit": "a" * 40,
+ "etag": '"' + "b" * 64 + '"',
+ "content-length": "4",
+ },
+ request = request,
+ )
+ return httpx.Response(
+ 200,
+ json = {
+ "id": REPO,
+ "sha": "a" * 40,
+ "gated": False,
+ "cardData": {"base_model": "org/base"},
+ "siblings": [{"rfilename": "model_index.json", "size": 4}],
+ },
+ request = request,
+ )
+
+ try:
+ yield _transport(handler)
+ finally:
+ from huggingface_hub.utils._http import default_client_factory
+ huggingface_hub.set_client_factory(default_client_factory)
+
+
+def _cache(tmp_path: Path, repo: str, files: dict[str, bytes]) -> Path:
+ """A minimal Hub cache entry (refs/main + snapshot files) for ``repo``; returns the cache root."""
+ root = tmp_path / "hub"
+ entry = root / ("models--" + repo.replace("/", "--"))
+ commit = "c" * 40
+ (entry / "refs").mkdir(parents = True, exist_ok = True)
+ (entry / "refs" / "main").write_text(commit)
+ for name, data in files.items():
+ path = entry / "snapshots" / commit / name
+ path.parent.mkdir(parents = True, exist_ok = True)
+ path.write_bytes(data)
+ return root
+
+
+# ------------------------------------------------------------------------------------------------------ model_info
+def test_metadata_calls_share_one_timeout_on_a_blackholed_hub(blackholed):
+ """Gated check, mirror SHA, size estimate: one request between them, not one wait each."""
+ for _ in range(2):
+ assert D._assert_base_repo_accessible(REPO, None) is None # fail open, as before
+ assert D.DiffusionBackend._current_sha(REPO, None) is None
+ failures: list = []
+ assert D.DiffusionBackend._estimate_download_bytes(
+ REPO, None, REPO, None, kind = "pipeline", failures_out = failures
+ ) == (0, [])
+ assert len(failures) == 1 # the estimate still reports itself partial
+ assert len(blackholed) == 1, blackholed
+
+
+def test_model_info_is_bounded(blackholed):
+ """huggingface_hub's default client has no timeout, so a dropped packet hung the call indefinitely."""
+ D.DiffusionBackend._current_sha(REPO, None)
+ assert blackholed[0][2] == R.METADATA_TIMEOUT_SECONDS
+
+
+def test_lookups_that_pick_what_loads_are_bounded_but_never_skipped(blackholed):
+ """Lookups that decide WHICH weights load are still asked, only with a timeout."""
+ R.note_hub_failure(httpx.ConnectError("refused"))
+ assert D._hf_base_model(REPO, None) is None
+ source = types.SimpleNamespace(kind = "repo", location = "org/prequant", filename = "x.safetensors")
+ with pytest.raises(Exception):
+ D.DiffusionBackend._prequant_source_hub_entry(source, None)
+ asked = [s for s in blackholed if "/api/models/" in s[1]]
+ assert len(asked) == 2, blackholed
+ assert all(s[2] == R.METADATA_TIMEOUT_SECONDS for s in asked)
+
+
+def test_an_answering_hub_is_asked_every_time_with_unchanged_answers(answering):
+ """No verdict when the Hub answers: every call still goes out and returns what the Hub said."""
+ for _ in range(2):
+ assert D._hf_base_model(REPO, None) == "org/base"
+ assert D.DiffusionBackend._current_sha(REPO, None) == "a" * 40
+ assert D._assert_base_repo_accessible(REPO, None) is None
+ assert len(answering) == 6
+ assert not R._UNREACHABLE
+
+
+def test_an_http_error_is_an_answer_not_an_unreachable_hub():
+ """404 / 401 / 5xx come from a reachable Hub: they never silence the next call."""
+
+ def handler(request):
+ return httpx.Response(503, request = request)
+
+ seen = _transport(handler)
+ try:
+ for _ in range(2):
+ assert D.DiffusionBackend._current_sha(REPO, None) is None
+ finally:
+ from huggingface_hub.utils._http import default_client_factory
+ huggingface_hub.set_client_factory(default_client_factory)
+ assert len([s for s in seen if "/api/models/" in s[1]]) >= 2
+ assert not R._UNREACHABLE
+
+
+def test_the_verdict_expires_and_a_success_clears_it(blackholed, monkeypatch):
+ D.DiffusionBackend._current_sha(REPO, None)
+ assert R.hub_unreachable()
+ R._UNREACHABLE[R.endpoint()] -= R.TTL_SECONDS + 1
+ assert not R.hub_unreachable()
+ R.note_hub_failure(httpx.ConnectError("refused"))
+ assert R.hub_unreachable()
+ R.note_hub_success()
+ assert not R.hub_unreachable()
+
+
+def test_a_gated_uncached_base_keeps_todays_answers(monkeypatch):
+ """An answering Hub that says gated still refuses an uncached base."""
+ monkeypatch.setattr("huggingface_hub.try_to_load_from_cache", lambda *a, **k: None)
+ R.note_hub_failure(httpx.ConnectError("refused"))
+ assert D._assert_base_repo_accessible("org/gated", None) is None
+ R.note_hub_success()
+
+ def handler(request):
+ return httpx.Response(
+ 401, headers = {"X-Error-Code": "GatedRepo", "X-Error-Message": "gated"}, request = request
+ )
+
+ seen = _transport(handler)
+ try:
+ with pytest.raises(ValueError, match = "org/gated"):
+ D._assert_base_repo_accessible("org/gated", None)
+ finally:
+ from huggingface_hub.utils._http import default_client_factory
+ huggingface_hub.set_client_factory(default_client_factory)
+ assert seen and not R._UNREACHABLE
+
+
+def test_a_duck_typed_client_gets_its_call_unchanged():
+ """Test doubles and wrappers that do not take ``timeout`` keep working."""
+ calls = []
+
+ class _Api:
+ def model_info(
+ self,
+ repo_id,
+ files_metadata = False,
+ ):
+ calls.append((repo_id, files_metadata))
+ return "info"
+
+ assert R.guarded(_Api()).model_info("a/b", files_metadata = True) == "info"
+ assert calls == [("a/b", True)]
+ R.note_hub_failure(httpx.ConnectError("refused"))
+ with pytest.raises(ConnectionError):
+ R.guarded(_Api()).model_info("a/b", files_metadata = True)
+ assert len(calls) == 1
+
+
+# ------------------------------------------------------------------------------------------------- cache fallbacks
+def test_resolve_gguf_reads_a_cached_checkpoint_off_disk_while_the_hub_is_down(
+ tmp_path, monkeypatch
+):
+ root = _cache(tmp_path, "org/gguf", {"model-Q4_K.gguf": b"GGUF"})
+ monkeypatch.setattr(D, "hub_cache_dir", lambda: str(root))
+ calls = []
+ real = huggingface_hub.hf_hub_download
+
+ def spy(*a, **k):
+ calls.append(k.get("local_files_only"))
+ return real(*a, **k)
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", spy)
+ R.note_hub_failure(httpx.ConnectError("refused"))
+ path = D.DiffusionBackend._resolve_gguf_path(None, "org/gguf", "model-Q4_K.gguf", None)
+ assert Path(path).read_bytes() == b"GGUF"
+ assert calls == [True]
+
+
+def test_resolve_gguf_still_downloads_an_uncached_checkpoint(tmp_path, monkeypatch):
+ """A first download is never refused off a verdict: the Hub may be back."""
+ root = tmp_path / "hub"
+ monkeypatch.setattr(D, "hub_cache_dir", lambda: str(root))
+ calls = []
+ monkeypatch.setattr(
+ huggingface_hub,
+ "hf_hub_download",
+ lambda *a, **k: calls.append(k.get("local_files_only")) or "x",
+ )
+ R.note_hub_failure(httpx.ConnectError("refused"))
+ assert D.DiffusionBackend._resolve_gguf_path(None, "org/gguf", "model-Q4_K.gguf", None) == "x"
+ assert calls == [False]
+
+
+def test_compat_revision_probe_is_skipped_while_the_hub_is_down(blackholed):
+ assert compat._hub_revision("org/gguf", "model-Q4_K.gguf", None) is None
+ assert compat._hub_revision("org/gguf", "model-Q4_K.gguf", None) is None
+ assert len(blackholed) == 1
+
+
+def test_a_cached_pick_loads_off_the_cache_only_while_the_hub_is_down(tmp_path, monkeypatch):
+ root = _cache(tmp_path, REPO, {"model_index.json": b"{}", "transformer/config.json": b"{}"})
+ monkeypatch.setattr(D, "hub_cache_dir", lambda: str(root))
+ fam = types.SimpleNamespace(name = "flux.1")
+ pick = (fam, "pipeline", REPO, None, REPO, REPO)
+ assert (
+ D._pick_cached_while_hub_unreachable(*pick) is False
+ ) # Hub fine: the online load is untouched
+ R.note_hub_failure(httpx.ConnectError("refused"))
+ assert D._pick_cached_while_hub_unreachable(*pick) is True
+ assert D._cached_while_hub_unreachable(REPO, "transformer/config.json") is True
+ # Nothing cached: a first download keeps asking the Hub.
+ assert (
+ D._pick_cached_while_hub_unreachable(fam, "pipeline", "org/new", None, "org/new", "org/new")
+ is False
+ )
+ # A GGUF pick needs its checkpoint cached too, not only the base.
+ assert (
+ D._pick_cached_while_hub_unreachable(fam, "gguf", "org/gguf", "m.gguf", REPO, REPO) is False
+ )