Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

42.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Shared backend utilities."""

import os
import structlog
import threading
from contextvars import ContextVar
import time
from loggers import get_logger
from contextlib import contextmanager
from pathlib import Path
from typing import Optional, Union
import shutil
import tempfile
from utils.paths.path_utils import is_appledouble_metadata
from hub.utils.hf_errors import modelscope_missing
from . import auth_safe

AuthSafeRedirectHandler = auth_safe.AuthSafeRedirectHandler
auth_safe_open = auth_safe.auth_safe_open


logger = get_logger(__name__)


# ── Offline / HF-cache helpers ──────────────────────────────────
# Offline loads must never touch the network (DNS-dead sessions hang on hub retries).
_HF_OFFLINE_TRUE_VALUES = frozenset({"1", "true", "yes", "on"})


def hf_env_offline() -> bool:
    """True when HF_HUB_OFFLINE or TRANSFORMERS_OFFLINE asks for offline mode. TRANSFORMERS_OFFLINE counts too (the hub reads only HF_HUB_OFFLINE), as does an open force_hf_offline window: hf_environment_restored_for_spawn briefly puts the user's values back, and an env-only check on another thread would then read "online"."""
    if force_hf_offline_active():
        return True
    for var in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"):
        if os.environ.get(var, "").strip().lower() in _HF_OFFLINE_TRUE_VALUES:
            return True
    return False


def package_installs_offline() -> bool:
    """True when installs must stay off the network: an operator-set HF offline, or UV_OFFLINE
    (install_wheel's pip fallback would reach the index anyway). A Studio auto-set HF offline
    says nothing about GitHub or PyPI, so only UV_OFFLINE counts."""
    from utils.wheel_utils import hf_offline_auto_set, uv_offline

    if hf_offline_auto_set():
        return uv_offline()
    return hf_env_offline() or uv_offline()


def anonymous_and_offline(hf_token, *, repo_id: Optional[str] = None) -> bool:
    """token=False denies auth, not the cache: offline, a downloaded private repo resolves from disk. With repo_id, the cached-read rule applies, not a refusal."""
    from hub.utils.hf_tokens import cached_read_refused, is_anonymous

    if not is_anonymous(hf_token):
        return False
    if not hf_env_offline():
        # The per-reader gates answer the online question; this guard is the offline one.
        return False
    if repo_id is None:
        return True
    # is_cached is True because these routes read whatever the cache holds; only authorization is left, which offline resolves against the disk.
    return cached_read_refused(
        hf_token,
        repo_id = repo_id,
        is_cached = lambda: True,
        offline = True,
    )


def canonical_model_repo_id(model_name: str) -> str:
    """Normalize a Hugging Face model repository ID selected in Unsloth."""
    return model_name.strip()


def hf_endpoint_url() -> str:
    """Configured hub endpoint, scheme-normalised. Mirror users point this elsewhere."""
    endpoint = (os.environ.get("HF_ENDPOINT") or "").strip() or "https://huggingface.co"
    return endpoint if "://" in endpoint else "https://" + endpoint


def hf_endpoint_host() -> str:
    """Host of the configured endpoint; probing huggingface.co would misjudge a mirror."""
    try:
        from urllib.parse import urlparse
        return urlparse(hf_endpoint_url()).hostname or "huggingface.co"
    except Exception:
        return "huggingface.co"


def _stdlib_proxy_for_url(url: str) -> Optional[str]:
    """requests' proxy selection rebuilt on the stdlib, for installs without requests. huggingface_hub 1.x dropped requests, so importing requests.utils raises there and we would report "no proxy" on a machine that has one, forcing a working proxy-only setup offline. getproxies covers the same sources, incl. macOS sysconf and the Windows registry."""
    from urllib.parse import urlparse
    from urllib.request import getproxies, proxy_bypass

    parsed = urlparse(url)
    host = parsed.hostname
    if not host:
        return None
    try:
        if proxy_bypass(host):
            return None
    except Exception:
        pass
    proxies = {k.lower(): v for k, v in getproxies().items()}
    scheme = (parsed.scheme or "https").lower()
    # select_proxy order: scheme://host, then scheme, then the all catch-all.
    for key in (f"{scheme}://{host}", scheme, "all"):
        value = proxies.get(key)
        if value:
            return value
    return None


def hf_proxy_for_endpoint(endpoint: Optional[str] = None) -> Optional[str]:
    """Return the Hub client's proxy choice, including ALL_PROXY and NO_PROXY rules."""
    url = endpoint or hf_endpoint_url()
    try:
        from requests.utils import get_environ_proxies, select_proxy
        return select_proxy(url, get_environ_proxies(url))
    except ImportError:
        # No requests (huggingface_hub 1.x); fall back rather than go blind.
        pass
    except Exception:
        return None
    try:
        return _stdlib_proxy_for_url(url)
    except Exception:
        return None


def hf_proxy_usable_by_urllib(proxy: Optional[str]) -> bool:
    """True when urllib can route through this proxy. urllib speaks only http/https, so a socks5:// proxy makes urlopen raise "unknown url type", which reads as no egress even though the Hub client reaches the hub through it."""
    if not proxy:
        return True
    from urllib.parse import urlparse

    scheme = urlparse(proxy if "://" in proxy else "http://" + proxy).scheme.lower()
    return scheme in ("http", "https")


def hf_proxy_configured() -> bool:
    """True when egress goes through a proxy: it resolves the hub host, so local DNS proves nothing about reachability and must not declare the hub offline."""
    return hf_proxy_for_endpoint() is not None


def call_with_deadline(
    fn,
    timeout_s: float,
    *,
    name: str = "deadline-call",
):
    """Run `fn()` on a daemon thread; raise TimeoutError if it outlives `timeout_s`. For network work that is bounded on paper but not in practice: a connect timeout applies per address, so a host whose leading addresses blackhole pays it once for each. A timed-out worker is abandoned, not stopped, and holds the callable until the kernel gives up, so keep this to short work. The callable's own exception is re-raised rather than swallowed, which stops a deadline turning a bug into an apparent dead network."""
    import contextvars

    outcome: dict = {}
    # Log context is per-thread: without the copy, fn()'s own logging loses the request fields it carries when the same call runs inline.
    context = contextvars.copy_context()

    def _run() -> None:
        try:
            outcome["value"] = context.run(fn)
        except BaseException as exc:  # noqa: BLE001 - re-raised below, in the caller
            outcome["error"] = exc

    t = threading.Thread(target = _run, daemon = True, name = name)
    t.start()
    t.join(timeout_s)
    if t.is_alive():
        raise TimeoutError(f"call did not finish within {timeout_s}s")
    if "error" in outcome:
        raise outcome["error"]
    return outcome.get("value")


def dns_host_dead(host: str, timeout: float = 2.0) -> bool:
    """True only when host definitively does not resolve. Daemon thread, so a wedged resolver cannot block past the deadline and socket.setdefaulttimeout is left alone. getaddrinfo, not gethostbyname: the latter is IPv4-only and would call an AAAA-only mirror or an IPv6 literal dead. A missed deadline is inconclusive, not dead: slow-but-working DNS (cold cache, DNSSEC, a fresh VPN) resolves past 2s and this shortcut skips the fail-open probe, so calling it dead would strand a working machine for a whole job, while a truly wedged resolver is still caught by the caller's HEAD probe hanging on the same lookup."""
    result: list = [None]

    def _probe() -> None:
        import socket as _socket
        try:
            _socket.getaddrinfo(host, None)
            result[0] = False
        except Exception:
            result[0] = True

    t = threading.Thread(target = _probe, daemon = True)
    t.start()
    t.join(timeout)
    return False if result[0] is None else result[0]


def hf_connect_target(endpoint: Optional[str] = None):
    """(host, port) egress actually has to reach: the proxy when one applies, else the endpoint."""
    from urllib.parse import urlparse

    url = endpoint or hf_endpoint_url()
    parsed = urlparse(url)
    default_port = 443 if parsed.scheme == "https" else 80
    try:
        proxy = hf_proxy_for_endpoint(url)
        if proxy:
            p = urlparse(proxy if "://" in proxy else "http://" + proxy)
            # An https:// proxy with no explicit port listens on 443, not 80.
            return p.hostname, p.port or (443 if p.scheme == "https" else 80)
    except Exception:
        pass
    return parsed.hostname, parsed.port or default_port


def hf_tcp_reachable(timeout: float = 3.0, endpoint: Optional[str] = None) -> bool:
    """True when a TCP connection to the hub (or its proxy) can be established. Separates "no egress" from "slow to answer": a loaded server still handshakes promptly, a blackholed route times out. A refusal counts as reachable, since something answered."""
    import socket as _socket

    host, port = hf_connect_target(endpoint)
    if not host:
        return True
    try:
        with _socket.create_connection((host, port), timeout = timeout):
            return True
    except ConnectionRefusedError:
        return True
    except OSError:
        return False
    except Exception:
        return True


def hf_dns_dead(timeout: float = 2.0) -> bool:
    """Fast offline shortcut: the endpoint's host does not resolve and no proxy applies. False whenever a proxy is configured, so proxy-only setups fall through to the real reachability probe instead of being wrongly declared offline."""
    if hf_proxy_configured():
        return False
    return dns_host_dead(hf_endpoint_host(), timeout)


# One load makes many hub calls, so the verdict is shared briefly. Kept short in BOTH directions: a stale "reachable" misses the plug being pulled, and a stale "unreachable" sends a load to the cache after the user reconnected.
_HF_REACHABILITY_TTL_S = 5.0
_hf_reachability: Optional[tuple] = None
_hf_reachability_lock = threading.Lock()


def _reachability_fresh(entry) -> bool:
    """True while a cached (timestamp, unreachable) verdict may still be reused."""
    return entry is not None and (time.monotonic() - entry[0]) < _HF_REACHABILITY_TTL_S


def hf_probe_disabled() -> bool:
    """True when UNSLOTH_OFFLINE_PROBE opts out of the reachability probe."""
    return os.environ.get("UNSLOTH_OFFLINE_PROBE", "1").strip().lower() in {
        "0",
        "false",
        "no",
        "off",
    }


# The memo above expires on wall-clock, right between requests and wrong inside one: a slow request outlives the TTL, so its later guards re-probe and can disagree with the first.
_hf_reachability_pin: "ContextVar[Optional[list]]" = ContextVar("hf_reachability_pin", default = None)


@contextmanager
def pinned_hf_reachability(verdict: Optional[bool] = None):
    """Hold one Hub reachability verdict for this block, pinned lazily on first ask; pass `verdict` if already probed."""
    token = _hf_reachability_pin.set([] if verdict is None else [verdict])
    try:
        yield
    finally:
        _hf_reachability_pin.reset(token)


def _pin_reachability(verdict: bool) -> bool:
    """Record the first verdict this request reaches, so every later read returns it."""
    pinned = _hf_reachability_pin.get()
    if pinned is not None and not pinned:
        pinned.append(verdict)
    return verdict


def hf_reachability_memo() -> Optional[bool]:
    """The pinned or memoised verdict while still usable, else None. Lets a caller skip a cheaper-but-still-slow shortcut it has already effectively run: one request opens several guards, and repeating a 2s DNS lookup per guard adds up. Lock-free like force_hf_offline_active: the tuple read is atomic."""
    pinned = _hf_reachability_pin.get()
    if pinned:
        return pinned[0]
    cached = _hf_reachability
    if not _reachability_fresh(cached):
        return None
    # Answering from the memo produces a verdict, and a warm memo is the usual first one.
    return _pin_reachability(cached[1])


def reset_hf_reachability_cache() -> None:
    """Drop the memoised verdict so the next call re-probes (tests, network changes)."""
    global _hf_reachability
    with _hf_reachability_lock:
        _hf_reachability = None


def hf_unreachable(timeout: int = 3) -> bool:
    """True when the HF endpoint is unreachable, memoised for _HF_REACHABILITY_TTL_S. DNS resolving does not mean the Hub is reachable: a live router with the WAN down, a captive portal or a stale DNS cache all answer lookups while every request then burns huggingface_hub's retry backoff. Bounded and proxy-aware, as the export path already does; UNSLOTH_OFFLINE_PROBE=0 disables it. Fails open, so an unavailable probe reports reachable and the load decides as it does today."""
    if hf_probe_disabled():
        # A declination, not a verdict, so it does not pin. UNSLOTH_OFFLINE_PROBE turns off the TCP probe and not DNS, and pinning here answers for the DNS shortcut too: guards 2..N would read "reachable" and never look again. Leaving it open costs no lookup, since a dead one returns above this.
        return False

    pinned = _hf_reachability_pin.get()
    if pinned:
        return pinned[0]

    global _hf_reachability
    cached = _hf_reachability
    if _reachability_fresh(cached):
        return _pin_reachability(cached[1])

    with _hf_reachability_lock:
        cached = _hf_reachability
        if _reachability_fresh(cached):
            return _pin_reachability(cached[1])
        try:
            from utils.transformers_version import hf_endpoint_unreachable

            # Both flags off for the same reason: an ambiguous answer must not force offline.
            unreachable = hf_endpoint_unreachable(
                timeout,
                gateway_errors_offline = False,
                proxy_timeouts_offline = False,
            )
        except Exception:
            unreachable = False
        _hf_reachability = (time.monotonic(), unreachable)
        return _pin_reachability(unreachable)


def _reset_hf_sessions() -> None:
    """Drop cached hub sessions so they remount with the current offline adapter."""
    try:
        from huggingface_hub.utils import _http

        for name in ("_get_session_from_cache", "get_session"):
            cache_clear = getattr(getattr(_http, name, None), "cache_clear", None)
            if cache_clear is not None:
                cache_clear()
        reset = getattr(_http, "reset_sessions", None)
        if reset is not None:
            reset()
    except Exception:
        pass


# Process-global, so nested/concurrent loads refcount rather than restore out from under each other.
_force_offline_depth = 0
_force_offline_saved: list = []
_force_offline_saved_env: dict = {}
# Spawn contexts can nest while holding this lock through Process.start().
_force_offline_lock = threading.RLock()

_OFFLINE_ENV_KEYS = ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE")
_OFFLINE_CONSTANTS = (
    ("huggingface_hub.constants", ("HF_HUB_OFFLINE",)),
    ("transformers.utils.hub", ("_is_offline_mode", "OFFLINE")),
)


def force_hf_offline_active() -> bool:
    """True while a force_hf_offline window is open anywhere in this process. Lets a concurrent caller tell our forced offline apart from one the user set, so it takes its own reference instead of no-opping and losing offline when the first exits. Lock-free: hf_environment_restored_for_spawn holds the lock across Process.start(), and blocking for that window would stall the operation the guard protects; the int read is atomic and the depth rises only after env and constants are already offline."""
    return _force_offline_depth > 0


def force_hf_offline_state() -> tuple[bool, bool]:
    """Return guard ownership and env presence under one lock."""
    with _force_offline_lock:
        return _force_offline_depth > 0, "HF_HUB_OFFLINE" in os.environ


def _restore_saved_offline_env(environment) -> None:
    """Apply the user's pre-guard offline intent to a child environment mapping."""
    for key in _OFFLINE_ENV_KEYS:
        value = _force_offline_saved_env.get(key)
        if value is None:
            environment.pop(key, None)
        else:
            environment[key] = value
    # Hub ignores TRANSFORMERS_OFFLINE, so preserve that user intent in children.
    if (
        "HF_HUB_OFFLINE" not in environment
        and str(environment.get("TRANSFORMERS_OFFLINE", "")).strip().lower()
        in _HF_OFFLINE_TRUE_VALUES
    ):
        environment["HF_HUB_OFFLINE"] = "1"


def hf_environment_for_spawn() -> dict[str, str]:
    """Copy the environment without scoped offline values."""
    return hf_environment_scrubbed(os.environ)


def hf_environment_scrubbed(base) -> dict[str, str]:
    """Copy an env mapping with our scoped offline values replaced by the user's intent. A caller that captured os.environ itself would otherwise hand a child the HF_HUB_OFFLINE=1 we set for one operation, and the child would stay cache-only for life."""
    with _force_offline_lock:
        environment = dict(base)
        if _force_offline_depth > 0:
            _restore_saved_offline_env(environment)
        return environment


@contextmanager
def hf_environment_restored_for_spawn():
    """Restore user offline values while multiprocessing snapshots ``os.environ``."""
    with _force_offline_lock:
        if _force_offline_depth == 0:
            yield
            return

        missing = object()
        forced_environment = {key: os.environ.get(key, missing) for key in _OFFLINE_ENV_KEYS}
        _restore_saved_offline_env(os.environ)
        try:
            yield
        finally:
            for key, value in forced_environment.items():
                if value is missing:
                    os.environ.pop(key, None)
                else:
                    os.environ[key] = value


@contextmanager
def force_hf_offline():
    """Force HF offline for this block, in-process. Env vars alone are too late once running: huggingface_hub and transformers read their offline constants at import and sessions cache a non-offline adapter, so flip the constants and rebuild the sessions to make hub calls fail fast. All restored on exit."""
    global _force_offline_depth, _force_offline_saved, _force_offline_saved_env
    import importlib

    with _force_offline_lock:
        if _force_offline_depth == 0:
            saved: list = []
            saved_env: dict = {}
            # Snapshot constants BEFORE forcing the env, else a module imported inside the window reads the "1".
            for mod_name, attrs in _OFFLINE_CONSTANTS:
                try:
                    mod = importlib.import_module(mod_name)
                except Exception:
                    continue
                for attr in attrs:
                    if hasattr(mod, attr):
                        saved.append((mod, attr, getattr(mod, attr)))
            for key in _OFFLINE_ENV_KEYS:
                saved_env[key] = os.environ.get(key)
                os.environ[key] = "1"
            for mod, attr, _ in saved:
                try:
                    setattr(mod, attr, True)
                except Exception:
                    pass
            _force_offline_saved = saved
            _force_offline_saved_env = saved_env
            _reset_hf_sessions()
        _force_offline_depth += 1
    try:
        yield
    finally:
        with _force_offline_lock:
            _force_offline_depth -= 1
            if _force_offline_depth == 0:
                for mod, attr, val in _force_offline_saved:
                    try:
                        setattr(mod, attr, val)
                    except Exception:
                        pass
                _force_offline_saved = []
                for key, val in _force_offline_saved_env.items():
                    if val is None:
                        os.environ.pop(key, None)
                    else:
                        os.environ[key] = val
                _force_offline_saved_env = {}
                _reset_hf_sessions()


def st_repo_id_candidates(model_name: str) -> list:
    """Repo ids a Sentence-Transformers load may resolve model_name to; a slashless name also resolves under the sentence-transformers/ namespace, so both are candidates."""
    name = (model_name or "").strip().strip("/")
    if not name:
        return []
    candidates = [name]
    if "/" not in name:
        candidates.append(f"sentence-transformers/{name}")
    return candidates


def _expand_path(raw: str) -> Path:
    """Expand ~ and $VARS as huggingface_hub does, so the gate resolves the loader's dir."""
    return Path(os.path.expandvars(os.path.expanduser(raw)))


def _hf_cache_roots() -> list:
    """Cache roots to search for a model's local snapshot, most-authoritative first. The app's selected hub cache (set via /settings) is searched first: after a no-restart cache switch the process env is stale, yet the loader reads the selected cache via ``cache_folder=active_hf_hub_cache()``, so the snapshot and offline security lookups must match where it actually loads. The env precedence (SENTENCE_TRANSFORMERS_HOME, HF_HUB_CACHE, HF_HOME/hub, ~/.cache/huggingface/hub) follows so a copy still in a previous cache resolves."""
    roots: list = []
    seen: set = set()

    def _add(path) -> None:
        if path is None:
            return
        expanded = _expand_path(str(path))
        key = str(expanded)
        if key not in seen:
            seen.add(key)
            roots.append(expanded)

    try:
        from utils.hf_cache_settings import get_hf_cache_paths
        _add(get_hf_cache_paths().hub_cache)
    except Exception:
        pass

    if st_home := os.environ.get("SENTENCE_TRANSFORMERS_HOME"):
        _add(st_home)
    if hub := (os.environ.get("HF_HUB_CACHE") or os.environ.get("HUGGINGFACE_HUB_CACHE")):
        _add(hub)
    if hf_home := os.environ.get("HF_HOME"):
        _add(_expand_path(hf_home) / "hub")
    if not roots:
        _add(Path.home() / ".cache" / "huggingface" / "hub")
    return roots


ST_WEIGHT_SUFFIXES = (".safetensors", ".bin")


def is_st_weight_name(basename: str) -> bool:
    """Whether a filename is a checkpoint SentenceTransformer can load. ``.bin`` is the loose one: ``tokenizer.bin`` shares the extension with real weights. Shared so the resolver's plan and the loader's cache check cannot disagree about what counts as a checkpoint."""
    name = basename.lower()
    for suffix in ST_WEIGHT_SUFFIXES:
        if not name.endswith(suffix):
            continue
        if suffix == ".bin":
            return name.startswith(("pytorch_model", "model", "adapter_model", "consolidated"))
        return True
    return False


def cached_st_source(model_name: str) -> Optional[tuple]:
    """``(repo id, snapshot dir)`` whose cache holds ST-loadable weights, complete. Alias-aware, and it reports WHICH candidate matched: a slashless name caches under ``sentence-transformers/``, so the literal id names a repo that usually does not exist, and a stale literal cache entry is not the directory that supplied the weights. Completeness comes from ``hf_cache_snapshot_is_loadable`` on that same candidate: ST weights alone are satisfied by the first finalized shard of a transfer still in flight."""
    for candidate in st_repo_id_candidates(model_name):
        # Exactly this candidate: the alias-expanding lookup answers a literal slashless name with the namespaced snapshot, pairing a directory with a repo id that supplied nothing.
        snapshot = hf_cache_snapshot_dir_for_repo(candidate)
        if snapshot is None:
            continue
        try:
            if not any(is_st_weight_name(p.name) and p.is_file() for p in snapshot.rglob("*")):
                continue
        except OSError:
            continue
        # This snapshot, not whatever the alias-expanding lookup would find: with several cache roots those differ, and a complete namespaced copy in one would vouch for the partial literal copy in another that gets loaded.
        if snapshot_is_loadable(snapshot, candidate):
            return (candidate, snapshot)
    return None


def cached_st_repo(model_name: str) -> Optional[str]:
    """Repo id whose cached snapshot holds complete ST-loadable weights."""
    source = cached_st_source(model_name)
    return source[0] if source else None


def snapshot_has_st_weights(model_name: str) -> bool:
    """Whether ``model_name`` has a complete cached checkpoint ST can open. ``hf_cache_snapshot_is_loadable`` counts ``.gguf``, which is right for the llama backend and wrong wherever SentenceTransformer is the loader; this pairs it with the ST-specific file family so both hold."""
    return cached_st_source(model_name) is not None


def _repo_folder_name(repo_id: str) -> str:
    """The cache directory name huggingface_hub gives ``repo_id``."""
    try:
        from huggingface_hub.file_download import repo_folder_name
    except Exception:
        return "models--" + repo_id.replace("/", "--")
    return repo_folder_name(repo_id = repo_id, repo_type = "model")


def _snapshot_in_repo_dir(repo_dir: Path) -> Optional[Path]:
    """``repo_dir``'s main-revision snapshot, or None. ``repo_dir`` is a cache entry the caller already picked, so no repo id is re-derived from its name."""
    try:
        ref = repo_dir / "refs" / "main"
        if not ref.is_file():
            return None
        commit = ref.read_text(encoding = "utf-8").strip()
        if not commit:
            return None
        snapshot = repo_dir / "snapshots" / commit
        return snapshot if snapshot.is_dir() else None
    # UnicodeDecodeError is a ValueError, not an OSError: a torn refs file must keep meaning "not cached here".
    except (OSError, UnicodeDecodeError):
        return None


def _snapshot_in_root(cache_root: Path, repo_id: str) -> Optional[Path]:
    """``repo_id``'s main-revision snapshot under exactly ``cache_root``, or None."""
    try:
        return _snapshot_in_repo_dir(cache_root / _repo_folder_name(repo_id))
    except (OSError, UnicodeDecodeError):
        return None


def hf_cache_snapshot_dir_for_repo(repo_id: str) -> Optional[Path]:
    """Snapshot dir for exactly ``repo_id``, with no alias expansion. ``hf_cache_snapshot_dir`` answers "is this model cached anywhere", trying the ST alias, so asking it about a literal slashless name can return the namespaced snapshot. A caller that has to report WHICH repo supplied the weights needs this one instead, or it pairs the alias's directory with the literal id and sends verification at a repo that does not exist."""
    for cache_root in _hf_cache_roots():
        snapshot = _snapshot_in_root(cache_root, repo_id)
        if snapshot is not None:
            return snapshot
    return None


def hf_cache_snapshot_dir(model_name: str) -> Optional[Path]:
    """Active local snapshot dir for model_name's main revision, or None if not cached. Reads refs/main then snapshots/<commit>; no network. Tries the ST alias for slashless names."""
    for cache_root in _hf_cache_roots():
        for repo_id in st_repo_id_candidates(model_name):
            snapshot = _snapshot_in_root(cache_root, repo_id)
            if snapshot is not None:
                return snapshot
    return None


# A weight file plus a config distinguishes a real cached model from a metadata-only partial cache.
_LOADABLE_WEIGHT_SUFFIXES = frozenset({".safetensors", ".bin", ".gguf", ".pt", ".pth", ".ckpt"})


def checkpoint_directory_is_complete(root: Path, weights = None) -> bool:
    """Whether ``root`` holds a whole checkpoint, shards and declared modules alike. Shared by the Hub-cache check and the local-path one so a directory is judged the same way however it got there: a single shard of a two-shard family, or a module ``modules.json`` declares and the directory does not have, is a torn checkpoint that SentenceTransformer fails to open at the first index. ``weights`` is the already-scanned weight list when the caller has one."""
    from hub.utils.inventory_scan import snapshot_holds_a_complete_payload

    if weights is None:
        weights = [
            path
            for path in root.rglob("*")
            if path.suffix.lower() in _LOADABLE_WEIGHT_SUFFIXES
            and path.is_file()
            and not is_appledouble_metadata(path)
        ]
    # SentenceTransformer modules may keep their own transformer checkpoint below 0_Transformer/. Validate every module subtree that carries weights; config-only modules such as Pooling need no weight family of their own.
    if (root / "modules.json").is_file():
        import json
        from pathlib import PurePosixPath

        try:
            modules = json.loads((root / "modules.json").read_text(encoding = "utf-8"))
        except (OSError, UnicodeDecodeError, ValueError):
            return False
        roots = []
        for module in modules if isinstance(modules, list) else []:
            value = module.get("path") if isinstance(module, dict) else None
            if not isinstance(value, str) or "\\" in value:
                continue
            relative = PurePosixPath(value or ".")
            if relative.is_absolute() or ".." in relative.parts:
                continue
            module_root = root.joinpath(*relative.parts)
            # A declared module the directory lacks entirely is a torn checkpoint whatever the others hold; existence is the whole test, since config-only modules have no weight family.
            if module_root != root and not module_root.is_dir():
                return False
            if any(path == module_root or module_root in path.parents for path in weights):
                roots.append(module_root)
        if roots:
            return all(snapshot_holds_a_complete_payload(r, quants = False) for r in roots)
    return snapshot_holds_a_complete_payload(root, quants = False)


def hf_cache_snapshot_is_loadable(model_name: str) -> bool:
    """True when the cached snapshot can satisfy a cache-only transformer load. App-managed downloads are checked against their exact manifest; imported or legacy caches without one fall back to the same weight-family/index scanner used by Hub inventory, so one shard of a cancelled checkpoint is not enough. No network."""
    snapshot = hf_cache_snapshot_dir(model_name)
    if snapshot is None:
        return False
    return snapshot_is_loadable(snapshot, model_name)


def active_hf_cache_loadable_snapshot(repo_id: str) -> Optional[Path]:
    """Return a loadable snapshot from the active cache only, or None."""
    try:
        from utils.hf_cache_settings import active_hf_hub_cache
        snapshot = _snapshot_in_root(_expand_path(active_hf_hub_cache()), repo_id)
    except Exception:
        return None
    return snapshot if snapshot is not None and snapshot_is_loadable(snapshot, repo_id) else None


def active_hf_cache_repo_spelling(repo_id: str) -> Optional[str]:
    """Hub is case-insensitive but cache dirs are not: return the cached spelling, since another casing re-downloads the checkpoint; None if not cached."""
    if active_hf_cache_loadable_snapshot(repo_id) is not None:
        return repo_id
    try:
        from utils.hf_cache_settings import active_hf_hub_cache

        exact = _repo_folder_name(repo_id)
        wanted = exact.lower()
        prefix = len("models--")
        for repo_dir in _expand_path(active_hf_hub_cache()).iterdir():
            if repo_dir.name.lower() != wanted or repo_dir.name == exact:
                continue
            # Repo ids cannot contain '--', so the dir name maps back to exactly one id.
            variant = repo_dir.name[prefix:].replace("--", "/")
            snapshot = _snapshot_in_repo_dir(repo_dir)
            if snapshot is not None and snapshot_is_loadable(snapshot, variant):
                return variant
    except Exception:
        return None
    return None


def snapshot_is_loadable(snapshot, model_name: str) -> bool:
    """``hf_cache_snapshot_is_loadable`` for a snapshot the caller already has. A caller that picked a specific directory has to have THAT one judged: the lookup above expands the ST alias within each cache root while an exact per-repo lookup walks the roots for one id, so with several roots configured the two can land on different snapshots and the verdict would belong to a directory nobody is going to load."""
    try:
        has_config = (snapshot / "config.json").is_file() or (snapshot / "modules.json").is_file()
        if not has_config:
            return False

        weights = []
        for path in snapshot.rglob("*"):
            if path.suffix.lower() not in _LOADABLE_WEIGHT_SUFFIXES or not path.is_file():
                continue
            if not is_appledouble_metadata(path):
                weights.append(path)
        if not weights:
            return False

        # A managed full-snapshot transfer records its exact expected files before downloading. A cancel marker or unfinished blob is conclusive even when config.json and the first finalized shard already exist.
        repo_dir = snapshot.parent.parent
        hub_cache = repo_dir.parent
        repo_id = model_name
        try:
            from huggingface_hub.file_download import repo_folder_name
            for candidate in st_repo_id_candidates(model_name):
                if repo_folder_name(repo_id = candidate, repo_type = "model") == repo_dir.name:
                    repo_id = candidate
                    break
        except Exception:
            pass
        from hub.utils import download_manifest
        from hub.utils.hf_cache_state import snapshot_has_broken_symlinks

        if download_manifest.has_cancel_marker("model", repo_id, None, hub_cache = hub_cache):
            return False
        manifest = download_manifest.read_manifest("model", repo_id, None, hub_cache = hub_cache)
        if manifest is not None:
            # This exact full-snapshot plan is stronger evidence than an unrelated .incomplete blob left under the repository by another revision or scoped GGUF job.
            return download_manifest.verify_against_disk(manifest, snapshot).ok
        # Judge THIS snapshot's own links, not every blob in the shared cache directory, or a stray .incomplete from another revision condemns a model that is fully present.
        if snapshot_has_broken_symlinks(snapshot):
            return False

        return checkpoint_directory_is_complete(snapshot, weights)
    except OSError:
        return False
    except Exception:
        # Completeness is a safety property here: an unprovable partial must keep the pending marker so the loader cannot silently reach the network.
        return False


# ── Client-safe error helpers ───────────────────────────────────
_METAL_QUEUE_DEAD_MARKERS = ("gpu timeout", "submissionsignored")


def is_metal_queue_dead(error: Union[Exception, str]) -> bool:
    """Watchdog kill or the refusal after it; not mlx's ``Command buffer execution failed:``
    wrapper, which also carries a recoverable ``Insufficient Memory``."""
    text = str(error).lower()
    return any(marker in text for marker in _METAL_QUEUE_DEAD_MARKERS)


# Never return raw exception text to clients; log server-side, return generic.
def safe_error_detail(error: Exception, fallback: str = "An internal error occurred") -> str:
    """Mid-stream llama-server errors are user-facing, but non-streaming paths would reduce them to the fallback. Lazy import keeps utils free of core at import."""
    try:
        from core.inference.stream_errors import LlamaStreamError  # noqa: PLC0415

        if isinstance(error, LlamaStreamError) and error.friendly:
            return error.friendly
        # Same reason: a context refusal is built for the user and names what to change.
        from core.inference.context_refusal import ContextBudgetExceeded  # noqa: PLC0415

        if isinstance(error, ContextBudgetExceeded):
            return str(error)
    except Exception:  # noqa: BLE001 -- fall through to the generic mapping below
        pass
    text = str(error).lower()
    # Before the connection test, which "GPU Timeout Error" would match.
    if is_metal_queue_dead(error):
        return "The GPU stopped responding. Reload the model to recover."
    if (
        "out of memory" in text
        or "cuda error" in text
        or "unable to allocate" in text
        or "insufficient memory" in text
    ):
        return "Ran out of memory. Try a smaller model or shorter input."
    if (
        isinstance(error, (ConnectionError, TimeoutError))
        or "connection" in text
        or "timed out" in text
        or "timeout" in text
    ):
        return "Could not reach an upstream service. Please try again."
    return fallback


def safe_curated_detail(error: Exception, fallback: str = "An internal error occurred") -> str:
    """Client-safe text for curated domain/validation exceptions. Keeps the message (paths stripped) instead of a generic fallback; for known exception types only (use ``safe_error_detail`` for generic ``Exception``)."""
    from utils.native_path_leases import redact_native_paths

    msg = redact_native_paths(str(error)).strip()
    return msg or fallback


def log_and_http_error(
    error: Exception,
    status_code: int,
    public_message: str,
    *,
    event: str = "request_failed",
    log = None,
    headers: Optional[dict] = None,
):
    """Logs the full error server-side; HTTPException detail carries only public_message, never raw text."""
    from fastapi import HTTPException

    # A 4xx is a normal outcome the caller handles. One warning line and no traceback: at error with exc_info, one generation buried the log under 54 rejected saves. 5xx keeps the traceback, and exc_info works for structlog too.
    emitter = log or logger
    if 400 <= status_code < 500:
        emitter.warning(f"{event}: {error}")
    else:
        emitter.error(f"{event}: {error}", exc_info = error)
    return HTTPException(status_code = status_code, detail = public_message, headers = headers)


@contextmanager
def without_hf_auth():
    """Temporarily disable HuggingFace authentication within a with block, so the code runs without cached tokens."""
    saved_env = {}
    env_vars = ["HF_TOKEN", "HUGGINGFACE_HUB_TOKEN", "HF_HOME"]
    for var in env_vars:
        if var in os.environ:
            saved_env[var] = os.environ[var]
            del os.environ[var]

    saved_disable = os.environ.get("HF_HUB_DISABLE_IMPLICIT_TOKEN")
    os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"

    token_files = []
    token_locations = [
        Path.home() / ".cache" / "huggingface" / "token",
        Path.home() / ".huggingface" / "token",
    ]

    for token_loc in token_locations:
        if token_loc.exists():
            temp = tempfile.NamedTemporaryFile(delete = False)
            temp.close()
            shutil.move(str(token_loc), temp.name)
            token_files.append((token_loc, temp.name))

    try:
        yield
    finally:
        for original, temp in token_files:
            try:
                original.parent.mkdir(parents = True, exist_ok = True)
                shutil.move(temp, str(original))
            except Exception as e:
                logger.error(f"Failed to restore token {original}: {e}")

        for var, value in saved_env.items():
            os.environ[var] = value

        if saved_disable is not None:
            os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = saved_disable
        else:
            os.environ.pop("HF_HUB_DISABLE_IMPLICIT_TOKEN", None)


def _is_repo_level_401(error: BaseException, response) -> bool:
    """A 401 the Hub gave for the repo, not the credential: anonymous reads of a private or
    missing repo get one too. Refusals of the token itself name it in X-Error-Message."""
    if type(error).__name__ != "RepositoryNotFoundError":
        return False
    from hub.utils.hf_tokens import hub_token_rejections

    rejections = hub_token_rejections()
    if rejections is not None and rejections.refused:
        return False
    try:
        reason = str(response.headers.get("X-Error-Message") or "").lower()
    except Exception:
        reason = ""
    return not any(marker in reason for marker in ("token", "credential"))


def is_hf_authentication_error(error: Exception) -> bool:
    """Return whether an exception chain contains a definitive HF auth failure."""
    seen: set[int] = set()
    current: BaseException | None = error
    while current is not None and id(current) not in seen:
        seen.add(id(current))
        response = getattr(current, "response", None)
        status = getattr(response, "status_code", None)
        try:
            if (
                status is not None
                and int(status) == 401
                and not _is_repo_level_401(current, response)
            ):
                return True
        except (TypeError, ValueError):
            pass
        message = str(current).lower()
        if "invalid user token" in message or "invalid hf token" in message:
            return True
        current = current.__cause__ or current.__context__
    return False


def format_error_message(error: Exception, model_name: str) -> str:
    """Format a user-friendly error message for common load issues, such as a missing modelscope package."""
    missing = modelscope_missing(error)
    if missing:
        return missing

    error_str = str(error).lower()
    model_short = model_name.split("/")[-1] if "/" in model_name else model_name

    if "repository not found" in error_str or "404" in error_str:
        return f"Model '{model_short}' not found. Check the model name."

    if "401" in error_str or "unauthorized" in error_str:
        return f"Authentication failed for '{model_short}'. Please provide a valid HF token."

    if "gated" in error_str or "access to model" in error_str:
        return f"Model '{model_short}' requires authentication. Please provide a valid HF token."

    if "invalid user token" in error_str:
        return "Invalid HF token. Please check your token and try again."

    if (
        "out of memory" in error_str
        or "out of device memory" in error_str
        or "out_of_device_memory" in error_str
        or "out_of_host_memory" in error_str
        or "not enough memory" in error_str
        or "cannot allocate memory" in error_str
        or "memory allocation failed" in error_str
        or "cublas_status_alloc_failed" in error_str
        or ("cuda error" in error_str and "alloc" in error_str)
        or ("xpu" in error_str and ("alloc" in error_str or "memory" in error_str))
        or isinstance(error, MemoryError)
        or ("mlx" in error_str and ("memory" in error_str or "allocate" in error_str))
    ):
        # Resolve get_device() at call time so tests that monkey-patch it after import see the patch.
        from utils.hardware import get_device

        device = get_device()
        device_label = {
            "cuda": "GPU",
            "xpu": "Intel GPU",
            "mlx": "Apple Silicon GPU",
            "cpu": "system",
        }.get(device.value, "GPU")
        return f"Not enough {device_label} memory to load '{model_short}'. Try a smaller model or free memory."

    return str(error)