Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

152 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Automatic transformers version switching.

Some newer model architectures (Ministral-3, GLM-4.7-Flash, Qwen3-30B-A3B MoE,
tiny_qwen3_moe) require transformers>=5.3.0, while Gemma 4 models require a
newer 5.x sidecar.  Dense NemotronH models (e.g. NVIDIA-Nemotron-3-Nano-4B) use
MLP layers that only transformers>=5.10 can parse natively, so they go on the
5.10 sidecar too.  Everything else runs on the ambient default that ships with
Unsloth (TRANSFORMERS_DEFAULT_VERSION).

Two separate target directories are maintained:
  - .venv_t5_530/  — transformers 5.3.0 (Ministral-3, GLM, Qwen3 MoE, etc.)
  - .venv_t5_550/  — transformers 5.5.0 (Gemma 4)
  - .venv_t5_510/  — transformers 5.10.2 (Gemma 4 Unified / 12B)

When loading a LoRA adapter with a custom name, we resolve the base model from
``adapter_config.json`` and check *that* against the model list.

Strategy:
  Training and inference run in subprocesses that activate the correct version
  via sys.path (prepending the appropriate .venv_t5_*/ directory). See:
    - core/training/worker.py
    - core/inference/worker.py

  For export (still in-process), ensure_transformers_version() does a lightweight
  sys.path swap using the same directories pre-installed by setup.sh.
"""

import ast
import importlib
import importlib.util
import json
import hashlib
import structlog
from loggers import get_logger
import errno
import contextlib
import contextvars
import os
from concurrent.futures import Future
import re
import shutil
import stat
import subprocess
import sys
import sysconfig
import tarfile
import tempfile
import urllib.request
import threading
import time
from pathlib import Path

from hub.utils.hf_tokens import (
    ANONYMOUS_CACHE_IDENTITY,
    HfTokenArg,
    apply_token_to_child_env,
    cache_reads_authorized,
    is_anonymous,
    qualify_cache_identity,
)
from utils.native_path_leases import child_env_without_native_path_secret
from utils.prebuilt.update_flow import resolves_into_studio_app_tree
from utils.native_tls import inline_gate_source, vendor_dir
from utils.happy_eyeballs import inline_activation_source
from utils.child_stdio import utf8_child_env
from utils.hf_cache_settings import get_hf_cache_paths
from utils.subprocess_compat import (
    windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs,
)

# Safe at module scope, unlike utils.models below: utils.training_runs is stdlib-only, so it
# cannot pin a transformers version into sys.modules before the sidecar is activated.
from utils.training_runs import base_model_from_run_dir_name

logger = get_logger(__name__)


_OFFLINE_TRUE_VALUES = {"1", "true", "yes", "on"}


def _env_offline() -> bool:
    """An open force_hf_offline window counts too, since a spawn briefly restores the user's env values."""
    try:
        from utils.utils import force_hf_offline_active
        if force_hf_offline_active():
            return True
    except Exception:
        pass
    return (
        os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _OFFLINE_TRUE_VALUES
        or os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _OFFLINE_TRUE_VALUES
    )


def _hf_raw_url(model_name: str, filename: str) -> str:
    """Uses /resolve rather than /raw, because a mirror may serve only the download route."""
    from utils.utils import hf_endpoint_url
    return f"{hf_endpoint_url().rstrip('/')}/{model_name}/resolve/main/{filename}"


def _hf_proxy_opener(url: str):
    """Opener matching the Hub client proxy so probe and reads agree on egress; None for socks proxies, which urllib cannot speak."""
    import urllib.parse
    import urllib.request

    try:
        from utils.utils import (
            AuthSafeRedirectHandler,
            hf_proxy_for_endpoint,
            hf_proxy_usable_by_urllib,
        )

        proxy = hf_proxy_for_endpoint(url)
        scheme = urllib.parse.urlparse(url).scheme or "https"
        if proxy:
            if not hf_proxy_usable_by_urllib(proxy):
                return None
            return urllib.request.build_opener(
                urllib.request.ProxyHandler({scheme: proxy}), AuthSafeRedirectHandler()
            )
        if any(urllib.request.getproxies().get(key) for key in (scheme, "all")):
            # Force a direct opener: urllib's coarser NO_PROXY parsing could route through the proxy.
            return urllib.request.build_opener(
                urllib.request.ProxyHandler({}), AuthSafeRedirectHandler()
            )
    except Exception:
        pass
    return None


_hf_json_ahead: dict[tuple[str, str | None], Future] = {}


def read_activation_files_ahead(model_name: str, hf_token: str | None) -> None:
    """Start activation's raw Hub reads together so TLS handshakes overlap; each answer serves one read."""
    if not _is_canonical_repo_id(model_name) or _env_offline():
        return
    from utils.paths import is_local_path

    if is_local_path(model_name):
        return

    def fill(future: Future, url: str) -> None:
        try:
            future.set_result(_read_hf_json(url, hf_token))
        except BaseException as exc:
            future.set_exception(exc)

    for filename in ("adapter_config.json", "tokenizer_config.json", "config.json"):
        url = _hf_raw_url(model_name, filename)
        future = Future()
        _hf_json_ahead[(url, hf_token)] = future
        threading.Thread(target = fill, args = (future, url), daemon = True).start()


def _hf_json(url: str, hf_token: str | None):
    ahead = _hf_json_ahead.pop((url, hf_token), None)
    return _read_hf_json(url, hf_token) if ahead is None else ahead.result()


def _read_hf_json(url: str, hf_token: str | None):
    """GET a JSON file from the Hub, once more without the token if the Hub refuses it (#11551)."""
    import urllib.request

    from hub.utils.hf_tokens import call_with_anonymous_retry

    def read(token):
        headers = {"User-Agent": "unsloth-studio"}
        if isinstance(token, str) and token:
            headers["Authorization"] = f"Bearer {token}"
        with _hf_urlopen(urllib.request.Request(url, headers = headers), timeout = 10) as resp:
            return json.loads(resp.read().decode())

    # No token sends none here (not the ambient one).
    return call_with_anonymous_retry(read, hf_token or False)


def _hf_urlopen(req, timeout: int):
    """``urlopen`` through the same proxy huggingface_hub would use for this request,
    with redirects that cannot carry the Authorization header off-origin."""
    from utils.utils import auth_safe_open

    opener = _hf_proxy_opener(req.full_url)
    if opener is not None:
        return opener.open(req, timeout = timeout)
    return auth_safe_open(req, timeout = timeout)


def hf_endpoint_unreachable(
    timeout: int = 3,
    *,
    gateway_errors_offline: bool = True,
    proxy_timeouts_offline: bool = True,
) -> bool:
    """HEAD probe to the HF endpoint, True if unreachable; no ML imports, so safe before activation."""
    import ssl
    import threading
    import urllib.error
    import urllib.request

    # Shared normaliser so an empty/whitespace HF_ENDPOINT falls back to the default hub.
    try:
        from utils.utils import hf_endpoint_url
        endpoint = hf_endpoint_url()
    except Exception:
        endpoint = (os.environ.get("HF_ENDPOINT") or "").strip() or "https://huggingface.co"
        if "://" not in endpoint:
            endpoint = "https://" + endpoint

    # Pin the Hub client's proxy choice: the default urllib opener ignores all_proxy.
    try:
        from utils.utils import hf_proxy_for_endpoint, hf_proxy_usable_by_urllib
        proxy = hf_proxy_for_endpoint(endpoint)
        # urllib cannot speak socks5, so its instant failure is not proof of no egress.
        if proxy and not hf_proxy_usable_by_urllib(proxy):
            return False
    except Exception:
        pass
    opener = _hf_proxy_opener(endpoint)
    _open = opener.open if opener is not None else urllib.request.urlopen

    result = {"online": False, "timed_out": False}

    def _probe():
        try:
            req = urllib.request.Request(endpoint, method = "HEAD")
            with _open(req, timeout = timeout):
                result["online"] = True
        except urllib.error.HTTPError as exc:
            # Gateway errors usually mean the hub is down; lifetime callers pass False so a 503 cannot strand.
            result["online"] = (
                True if not gateway_errors_offline else exc.code not in (502, 503, 504)
            )
        except urllib.error.URLError as exc:
            # TLS failures and refused/reset mean the wire answered; a blackhole raises gaierror instead.
            if isinstance(exc.reason, (ssl.SSLError, ConnectionError)):
                result["online"] = True
            elif isinstance(exc.reason, TimeoutError):
                # Resolved below, off-thread, so the extra probe cannot outrun the join.
                result["timed_out"] = True
            elif isinstance(exc.reason, OSError):
                result["online"] = False  # gaierror / network unreachable: a real answer
            else:
                # A string reason ("no host given") is client-side, not an egress answer.
                result["online"] = True
        except ssl.SSLError:
            result["online"] = True
        except TimeoutError:
            result["timed_out"] = True
        except ConnectionError:
            # An OSError from getresponse() arrives raw; a momentary reset must not read as no egress.
            result["online"] = True
        except OSError:
            result["online"] = False
        except Exception:
            # Bad endpoint/proxy, or a bug here. Not a network answer, so fail open.
            result["online"] = True

    t = threading.Thread(target = _probe, daemon = True)
    t.start()
    t.join(timeout + 1)
    if t.is_alive():
        # Behind a proxy a hang is ambiguous (each phase may fit the timeout while the total passes the join), so lifetime callers fail open; direct, it means offline.
        try:
            from utils.utils import hf_proxy_configured
            if hf_proxy_configured():
                return proxy_timeouts_offline
        except Exception:
            pass
        return True
    if result["timed_out"]:
        # A slow server still completes the TCP handshake; a blackholed route does not.
        try:
            from utils.utils import hf_proxy_configured, hf_tcp_reachable
            if hf_proxy_configured():
                # Through a proxy the handshake only proves the proxy is up, not the hub; lifetime callers fail open on this ambiguous result.
                return proxy_timeouts_offline
            return not hf_tcp_reachable(min(timeout, 2.0), endpoint)
        except Exception:
            return True
    return not result["online"]


def _safe_is_file(p: Path) -> bool:
    """``p.is_file()`` returning False instead of raising on a bad path."""
    try:
        return p.is_file()
    except (OSError, ValueError):
        return False


def _safe_is_dir(p: Path) -> bool:
    """``p.is_dir()`` returning False instead of raising on a bad path."""
    try:
        return p.is_dir()
    except (OSError, ValueError):
        return False


# Lowercase substrings that need transformers 5.3.0.
TRANSFORMERS_5_MODEL_SUBSTRINGS: tuple[str, ...] = (
    "ministral-3-",  # Ministral-3-{3,8,14}B-{Instruct,Reasoning,Base}-2512
    "glm-4.7-flash",  # GLM-4.7-Flash
    "qwen3-30b-a3b",  # Qwen3-30B-A3B-Instruct-2507 and variants
    "qwen3.5",  # Qwen3.5 family (35B-A3B, etc.)
    "qwen3-next",  # Qwen3-Next and variants
    "tiny_qwen3_moe",  # imdatta0/tiny_qwen3_moe_2.8B_0.7B
    "lfm2.5-vl-450m",  # LiquidAI/LFM2.5-VL-450M
)

# Lowercase substrings for models that require transformers 5.10.x (checked first).
TRANSFORMERS_510_MODEL_SUBSTRINGS: tuple[str, ...] = (
    "gemma-4-12b",  # Gemma 4 Unified 12B
    "gemma4-12b",
)

# Lowercase substrings for models that require the transformers 5.5 sidecar.
TRANSFORMERS_550_MODEL_SUBSTRINGS: tuple[str, ...] = (
    "gemma-4",  # Gemma-4 (E2B-it, E4B-it, 31B-it, 26B-A4B-it)
    "gemma4",  # Gemma-4 alternate naming
    "qwen3.6",
    "kimi-k3",
    "kimik3",
    "locate-anything",
    "locateanything",
    "diffusion-gemma",
    "diffusiongemma",
    "higgs-tts-2",
    "higgs-audio-v2",
    "higgs-audio-v3-tts",
)

# Architecture classes / model_type values requiring transformers 5.10.x (via config.json).
_TRANSFORMERS_510_ARCHITECTURES: set[str] = {
    "Gemma4UnifiedForConditionalGeneration",
    "Gemma4AssistantForCausalLM",
    "Gemma4UnifiedAssistantForCausalLM",
}
_TRANSFORMERS_510_MODEL_TYPES: set[str] = {
    "gemma4_unified",
    "gemma4_assistant",
    "gemma4_unified_assistant",
}

# DeepSeek-OCR remote code subclasses transformers 4.x DeepseekV2 modules (DeepseekV2MoE, removed in 5.x)
# and its config fails 5.x validation (kv_lora_rank: null), so it only loads on 4.57.x (#8732).
_TRANSFORMERS_457_ARCHITECTURES: frozenset[str] = frozenset(
    {"DeepseekOCRForCausalLM", "DeepseekOCR2ForCausalLM"}
)

# Architecture classes / model_type values requiring transformers 5.5.0 (via config.json).
_TRANSFORMERS_550_ARCHITECTURES: set[str] = {
    "DiffusionGemmaForBlockDiffusion",
    # Remote-code text-diffusion LMs: their tokenizers are 5.x-only, and 5.3.0 lacks helpers their code imports
    # (Nemotron-Labs-Diffusion needs transformers.utils.generic.check_model_inputs). Matched by architecture,
    # since Nemotron-Labs-Diffusion shares model_type "nemotron" with the autoregressive Nemotron models.
    "LLaDAModelLM",
    "LLaDAMoEModel",
    "LLaDAMoEModelLM",
    "DreamModel",
    "NemotronLabsDiffusionModel",
    "LLaDA2MoeModelLM",
    "SDARForCausalLM",
    "Gemma4ForConditionalGeneration",
    "KimiK3ForConditionalGeneration",
    "LocateAnythingForConditionalGeneration",
    "HiggsAudioV2ForConditionalGeneration",
    "HiggsMultimodalQwen3ForConditionalGeneration",
}
_TRANSFORMERS_550_MODEL_TYPES: set[str] = {
    "diffusion_gemma",
    "gemma4",
    "kimi_k3",
    "locateanything",
    "higgs_audio_v2",
    "higgs_multimodal_qwen3",
}

# Architecture classes / model_type values requiring transformers 5.3.0 (via config.json).
_TRANSFORMERS_530_ARCHITECTURES: set[str] = {
    "Qwen3_5ForCausalLM",
    "Qwen3_5ForConditionalGeneration",
    "Qwen3_5MoeForCausalLM",
    "Qwen3_5MoeForConditionalGeneration",
    "Qwen3MoeForCausalLM",
    "Qwen3NextForCausalLM",
    "Glm4MoeLiteForCausalLM",
    "Lfm2MoeForCausalLM",
    "Lfm2VlForConditionalGeneration",
}
_TRANSFORMERS_530_MODEL_TYPES: set[str] = {
    "qwen3_5",
    "qwen3_5_text",
    "qwen3_5_moe",
    "qwen3_5_moe_text",
    "qwen3_moe",
    "qwen3_next",
    "glm4_moe_lite",
    "lfm2_moe",
    "lfm2_vl",
}

# Tokenizer classes that only exist in transformers>=5.x.
_TRANSFORMERS_5_TOKENIZER_CLASSES: set[str] = {
    "TokenizersBackend",
}

# Keyed on token hash so unauthed misses cannot poison authed lookups; offline misses not cached.
_tokenizer_class_cache: dict[tuple[str, str | None], bool] = {}
_config_json_cache: dict[tuple[str, str | None], dict | None] = {}
_config_needs_510_cache: dict[tuple[str, str | None], bool] = {}
_config_needs_550_cache: dict[tuple[str, str | None], bool] = {}
_config_needs_530_cache: dict[tuple[str, str | None], bool] = {}
_config_needs_457_cache: dict[tuple[str, str | None], bool] = {}

# Keyed by local config.json signature so overwritten checkpoints re-probe, not Hub sha; no huggingface_hub import before sidecar venv activates.
_probe_tier_cache: dict[str, str] = {}

TRANSFORMERS_510_VERSION = "5.10.2"
TRANSFORMERS_550_VERSION = "5.5.0"
TRANSFORMERS_530_VERSION = "5.3.0"
TRANSFORMERS_457_VERSION = "4.57.6"
TRANSFORMERS_DEFAULT_VERSION = "5.5.0" if sys.version_info >= (3, 10) else "4.57.6"
# Backwards-compat alias for the highest 5.x tier; prefer TRANSFORMERS_510_VERSION, TRANSFORMERS_550_VERSION or TRANSFORMERS_530_VERSION.
TRANSFORMERS_5_VERSION = TRANSFORMERS_510_VERSION

# Pre-installed directories - created by setup.sh / setup.ps1.
from utils.paths.storage_roots import studio_root as _studio_root  # noqa: E402

_VENV_T5_530_DIR = str(_studio_root() / ".venv_t5_530")
_VENV_T5_550_DIR = str(_studio_root() / ".venv_t5_550")
_VENV_T5_510_DIR = str(_studio_root() / ".venv_t5_510")
# Built at runtime on the first DeepSeek-OCR load; not pre-installed by setup.
_VENV_T4_457_DIR = str(_studio_root() / ".venv_t4_457")
# Backwards-compat alias
_VENV_T5_DIR = _VENV_T5_550_DIR

# llm-compressor-main shadow for FP8/FP4 export of newer-transformers models; like .venv_t5_* but also shadows compressed-tensors; --no-deps.
_VENV_LLMCOMPRESSOR_DIR = str(_studio_root() / ".venv_llmcompressor")

# User-consented "latest transformers" sidecar (utils/transformers_latest.py); pinned version in a marker file.
_VENV_T5_LATEST_DIR = str(_studio_root() / ".venv_t5_latest")
_LATEST_PIN_MARKER = ".unsloth_pinned_transformers"

# Tier precedence: higher rank wins in _higher_tier. "latest" outranks every fixed tier.
_TIER_RANK = {"default": 0, "457": 0.5, "530": 1, "550": 2, "510": 3, "latest": 4}


def _higher_tier(a: str, b: str) -> str:
    return a if _TIER_RANK.get(a, 0) >= _TIER_RANK.get(b, 0) else b


def get_transformers_activation_tier(model_name: str, hf_token: str | None = None) -> str:
    if _is_lora_adapter_dir(Path(model_name)):
        resolved = _resolve_base_model(model_name)
    else:
        resolved = _remote_lora_base(model_name, hf_token = hf_token) or model_name
    tier = get_transformers_tier(resolved, hf_token)
    if model_name != resolved and _safe_is_file(Path(model_name) / "config.json"):
        tier = _higher_tier(tier, get_transformers_tier(model_name, hf_token))
    return tier


def activate_transformers_for_subprocess(model_name: str, hf_token: str | None = None) -> None:
    """Must run before ML imports; token forwarded for gated repos; full checkpoints tier by local config.json, since _name_or_path may be unreachable."""
    tier = get_transformers_activation_tier(model_name, hf_token)

    if tier == "latest":
        pinned = latest_venv_pinned_version()
        if pinned is None or not _ensure_venv_t5_latest_exists():
            raise RuntimeError(
                f"Cannot activate the latest-transformers sidecar: "
                f".venv_t5_latest missing or unpinned at {_VENV_T5_LATEST_DIR}"
            )
        if _VENV_T5_LATEST_DIR not in sys.path:
            sys.path.insert(0, _VENV_T5_LATEST_DIR)
        logger.info(
            "Prepended transformers %s venv to sys.path from %s "
            "(path only; the loaded version is confirmed later by "
            "'Subprocess loaded transformers ...' on first import)",
            pinned,
            _VENV_T5_LATEST_DIR,
        )
        _pp = os.environ.get("PYTHONPATH", "")
        os.environ["PYTHONPATH"] = _VENV_T5_LATEST_DIR + (os.pathsep + _pp if _pp else "")
    elif tier == "510":
        if not _ensure_venv_t5_510_exists():
            raise RuntimeError(
                f"Cannot activate transformers {TRANSFORMERS_510_VERSION}: "
                f".venv_t5_510 missing at {_VENV_T5_510_DIR}"
            )
        if _VENV_T5_510_DIR not in sys.path:
            sys.path.insert(0, _VENV_T5_510_DIR)
        logger.info(
            "Prepended transformers %s venv to sys.path from %s "
            "(path only; the loaded version is confirmed later by "
            "'Subprocess loaded transformers ...' on first import)",
            TRANSFORMERS_510_VERSION,
            _VENV_T5_510_DIR,
        )
        _pp = os.environ.get("PYTHONPATH", "")
        os.environ["PYTHONPATH"] = _VENV_T5_510_DIR + (os.pathsep + _pp if _pp else "")
    elif tier == "550":
        if not _ensure_venv_t5_550_exists():
            raise RuntimeError(
                f"Cannot activate transformers {TRANSFORMERS_550_VERSION}: "
                f".venv_t5_550 missing at {_VENV_T5_550_DIR}"
            )
        if _VENV_T5_550_DIR not in sys.path:
            sys.path.insert(0, _VENV_T5_550_DIR)
        logger.info(
            "Prepended transformers %s venv to sys.path from %s "
            "(path only; the loaded version is confirmed later by "
            "'Subprocess loaded transformers ...' on first import)",
            TRANSFORMERS_550_VERSION,
            _VENV_T5_550_DIR,
        )
        _pp = os.environ.get("PYTHONPATH", "")
        os.environ["PYTHONPATH"] = _VENV_T5_550_DIR + (os.pathsep + _pp if _pp else "")
    elif tier == "457":
        if not _ensure_venv_t4_457_exists():
            raise RuntimeError(
                f"Cannot activate transformers {TRANSFORMERS_457_VERSION}: "
                f".venv_t4_457 missing at {_VENV_T4_457_DIR}"
            )
        if _VENV_T4_457_DIR not in sys.path:
            sys.path.insert(0, _VENV_T4_457_DIR)
        logger.info(
            "Prepended transformers %s venv to sys.path from %s",
            TRANSFORMERS_457_VERSION,
            _VENV_T4_457_DIR,
        )
        _pp = os.environ.get("PYTHONPATH", "")
        os.environ["PYTHONPATH"] = _VENV_T4_457_DIR + (os.pathsep + _pp if _pp else "")
    elif tier == "530":
        if not _ensure_venv_t5_530_exists():
            raise RuntimeError(
                f"Cannot activate transformers 5.3.0: "
                f".venv_t5_530 missing at {_VENV_T5_530_DIR}"
            )
        if _VENV_T5_530_DIR not in sys.path:
            sys.path.insert(0, _VENV_T5_530_DIR)
        logger.info(
            "Prepended transformers %s venv to sys.path from %s "
            "(path only; the loaded version is confirmed later by "
            "'Subprocess loaded transformers ...' on first import)",
            TRANSFORMERS_530_VERSION,
            _VENV_T5_530_DIR,
        )
        _pp = os.environ.get("PYTHONPATH", "")
        os.environ["PYTHONPATH"] = _VENV_T5_530_DIR + (os.pathsep + _pp if _pp else "")
    else:
        logger.info(
            "Using default transformers (%s) for %s", TRANSFORMERS_DEFAULT_VERSION, model_name
        )


def latest_tier_active_for(model_name: str, hf_token: str | None = None) -> bool:
    """Mirrors worker routing; never raises, and any resolution failure returns False."""
    try:
        # No consented sidecar pin means nothing routes to latest; return before any resolution.
        if latest_venv_pinned_version() is None:
            return False
        return get_transformers_activation_tier(model_name, hf_token) == "latest"
    except Exception:
        return False


def _has_adapter_weights(path: Path) -> bool:
    """True if *path* holds LoRA adapter weight files (``adapter_model.*``)."""
    try:
        return any(path.glob("adapter_model*.safetensors")) or any(path.glob("adapter_model*.bin"))
    except OSError:
        return False


def _is_lora_adapter_dir(path: Path) -> bool:
    """True if *path* is a local LoRA dir (adapter_config.json or adapter_model-only
    weights). Import-light so it can run during subprocess activation."""
    try:
        if not path.is_dir():
            return False
        return (path / "adapter_config.json").is_file() or _has_adapter_weights(path)
    except OSError:
        return False


def _is_same_path(value: str, local_path: Path) -> bool:
    """True if *value* resolves to *local_path* (relative/absolute/symlink)."""
    if value == str(local_path):
        return True
    try:
        return os.path.realpath(value) == os.path.realpath(str(local_path))
    except OSError:
        return False


def recorded_local_base(model_name) -> "tuple[str | None, bool]":
    """(base, needs_hub) from local files; needs_hub only if get_base_model_from_lora could answer or the read failed; a self-reference is not a base."""
    root = Path(model_name)
    try:
        adapter_cfg = _safe_is_file(root / "adapter_config.json")
        if adapter_cfg:
            with open(root / "adapter_config.json", encoding = "utf-8-sig") as f:
                base = json.load(f).get("base_model_name_or_path")
            if base:
                return base, False
        if _safe_is_file(root / "config.json"):
            with open(root / "config.json", encoding = "utf-8-sig") as f:
                cfg = json.load(f)
            for _key in ("model_name", "_name_or_path"):
                base = cfg.get(_key)
                if isinstance(base, str) and base and not _is_same_path(base, root):
                    return base, False
        # Only reachable without a Hub call when there is no adapter_config.json.
        base = base_model_from_run_dir_name(root.name)
        if base and not adapter_cfg and _has_adapter_weights(root):
            return base, False
        return None, adapter_cfg
    except Exception:
        return None, True


def _resolve_base_model(model_name: str) -> str:
    """Heavier lookup only for real local dirs, since Hub ids would log noisy warnings."""
    local_path = Path(model_name)
    adapter_cfg_path = local_path / "adapter_config.json"
    if _safe_is_file(adapter_cfg_path):
        try:
            with open(adapter_cfg_path, encoding = "utf-8-sig") as f:
                cfg = json.load(f)
            base = cfg.get("base_model_name_or_path")
            if base:
                logger.info(
                    "Resolved LoRA adapter '%s' → base model '%s'",
                    model_name,
                    base,
                )
                return base
        except Exception as exc:
            logger.debug("Could not read %s: %s", adapter_cfg_path, exc)

    # --- config.json fallback (LoRA and full fine-tune) ---
    config_json_path = local_path / "config.json"
    if _safe_is_file(config_json_path):
        try:
            with open(config_json_path, encoding = "utf-8-sig") as f:
                cfg = json.load(f)
            # Unsloth writes model_name, HF writes _name_or_path; skip a self-reference.
            for _key in ("model_name", "_name_or_path"):
                base = cfg.get(_key)
                if isinstance(base, str) and base and not _is_same_path(base, local_path):
                    logger.info(
                        "Resolved checkpoint '%s' → base model '%s' (via config.json)",
                        model_name,
                        base,
                    )
                    return base
        except Exception as exc:
            logger.debug("Could not read %s: %s", config_json_path, exc)

    # Gate on adapter_config.json: utils.models imports transformers before the sidecar activates.
    if _safe_is_file(adapter_cfg_path):
        try:
            from utils.models import get_base_model_from_lora
            base = get_base_model_from_lora(model_name)
            if base:
                logger.info(
                    "Resolved LoRA adapter '%s' → base model '%s' "
                    "(via get_base_model_from_lora)",
                    model_name,
                    base,
                )
                return base
        except Exception as exc:
            logger.debug(
                "get_base_model_from_lora failed for '%s': %s",
                model_name,
                exc,
            )

    # adapter_model-only LoRA: no config, so parse the unsloth_<model>_<timestamp> dir name.
    base = base_model_from_run_dir_name(local_path.name)
    if base and _has_adapter_weights(local_path):
        logger.info(
            "Resolved adapter-only LoRA '%s' → base model '%s' (via directory name)",
            model_name,
            base,
        )
        return base

    return model_name


def _token_cache_key(model_name: str, hf_token: HfTokenArg) -> tuple[str, str | None]:
    """Anonymous and UI-session callers get own slots: unauthed misses must not poison authed reads, and an API caller must not read a UI-session result."""
    import hashlib

    if is_anonymous(hf_token):
        return (model_name, ANONYMOUS_CACHE_IDENTITY)
    if not hf_token:
        return (model_name, None)
    digest = hashlib.sha256(hf_token.encode()).hexdigest()[:16]
    return (model_name, qualify_cache_identity(hf_token, digest))


def _is_canonical_repo_id(model_name: str) -> bool:
    """True for a canonical ``owner/repo`` Hub id (not a local or relative path)."""
    return bool(
        model_name
        and model_name.count("/") == 1
        and model_name[0] not in "/.~"
        and "\\" not in model_name
    )


def _adapter_base_from_hf_cache(model_name: str) -> str | None:
    """Stdlib-only hub cache read (no huggingface_hub import), so a cached LoRA resolves offline."""
    if not _is_canonical_repo_id(model_name):
        return None
    # HF_HUB_CACHE env is stale after a no-restart /settings switch; use the DB-backed path.
    hub = str(get_hf_cache_paths().hub_cache)
    repo_dir = Path(hub) / ("models--" + model_name.replace("/", "--"))
    candidates = []
    ref_main = repo_dir / "refs" / "main"

    def _mtime(p: Path) -> float:
        try:
            return p.stat().st_mtime
        except OSError:
            return 0.0

    try:
        if ref_main.is_file():
            candidates.append(
                repo_dir
                / "snapshots"
                / ref_main.read_text(encoding = "utf-8").strip()
                / "adapter_config.json"
            )
        candidates += sorted(
            repo_dir.glob("snapshots/*/adapter_config.json"), key = _mtime, reverse = True
        )
        for cfg_path in candidates:
            if cfg_path.is_file():
                base = json.loads(cfg_path.read_text(encoding = "utf-8-sig")).get(
                    "base_model_name_or_path"
                )
                return base or None
    except Exception as exc:
        logger.debug("HF cache adapter_config.json lookup failed for '%s': %s", model_name, exc)
    return None


_remote_lora_base_answers: "contextvars.ContextVar[dict | None]" = contextvars.ContextVar(
    "_remote_lora_base_answers", default = None
)


@contextlib.contextmanager
def remote_lora_base_scope():
    """Within one load only: reuses definitive answers (a read config or a 404), never a failure; re-entrant."""
    if _remote_lora_base_answers.get() is not None:
        yield
        return
    reset = _remote_lora_base_answers.set({})
    try:
        yield
    finally:
        _remote_lora_base_answers.reset(reset)


def _remote_lora_base(model_name: str, hf_token: str | None = None) -> str | None:
    """Raw HTTP so the base is known before ML import; offline or transient failure reads hub cache; definitive 404 returns None, not a stale base."""
    if not _is_canonical_repo_id(model_name):
        return None
    try:
        from utils.paths import is_local_path
        if is_local_path(model_name):
            return None  # an existing relative path is a local checkpoint, not a Hub repo
    except Exception:
        pass
    if _env_offline():
        return _adapter_base_from_hf_cache(model_name)

    import urllib.error
    import urllib.request

    url = _hf_raw_url(model_name, "adapter_config.json")
    answers = _remote_lora_base_answers.get()
    if not isinstance(hf_token, (str, bool, type(None))):
        answers = None
    key = (url, hf_token)
    if answers is not None and key in answers:
        base = answers[key]
        if base:
            logger.info("Resolved remote LoRA adapter '%s' → base model '%s'", model_name, base)
        return base
    try:
        cfg = _hf_json(url, hf_token)
        base = cfg.get("base_model_name_or_path")
        if base:
            logger.info("Resolved remote LoRA adapter '%s' → base model '%s'", model_name, base)
        if answers is not None:
            answers[key] = base or None
        return base or None
    except urllib.error.HTTPError as exc:
        if exc.code == 404:
            if answers is not None:
                answers[key] = None
            return None  # definitively not a LoRA; do not serve a stale cached base
        logger.debug("adapter_config.json fetch failed for '%s': %s", model_name, exc)
        return _adapter_base_from_hf_cache(model_name)
    except Exception as exc:
        logger.debug("No remote adapter_config.json for '%s': %s", model_name, exc)
        return _adapter_base_from_hf_cache(model_name)


def _check_tokenizer_config_needs_v5(model_name: str, hf_token: str | None = None) -> bool:
    """Cached per model and token so an anonymous miss cannot poison an authed read; errors fail open."""
    cache_key = _token_cache_key(model_name, hf_token)
    if cache_key in _tokenizer_class_cache:
        return _tokenizer_class_cache[cache_key]

    # --- Check local tokenizer_config.json first ---
    local_path = Path(model_name)
    local_tc = local_path / "tokenizer_config.json"
    if _safe_is_file(local_tc):
        try:
            with open(local_tc, encoding = "utf-8-sig") as f:
                data = json.load(f)
            tokenizer_class = data.get("tokenizer_class", "")
            result = tokenizer_class in _TRANSFORMERS_5_TOKENIZER_CLASSES
            if result:
                logger.info(
                    "Local check: %s uses tokenizer_class=%s (requires transformers 5.x)",
                    model_name,
                    tokenizer_class,
                )
            _tokenizer_class_cache[cache_key] = result
            return result
        except Exception as exc:
            logger.debug("Could not read %s: %s", local_tc, exc)

    # Local checkpoint without the file yet: don't fetch as a Hub id or cache the miss.
    if _safe_is_dir(local_path):
        return False

    # Offline: skip the 10s fetch (fail open). Don't cache this assumed negative.
    if _env_offline():
        return False

    # --- Fall back to fetching from HuggingFace ---
    import urllib.error
    import urllib.request

    url = _hf_raw_url(model_name, "tokenizer_config.json")
    try:
        data = _hf_json(url, hf_token)
        tokenizer_class = data.get("tokenizer_class", "")
        result = tokenizer_class in _TRANSFORMERS_5_TOKENIZER_CLASSES
        if result:
            logger.info(
                "Dynamic check: %s uses tokenizer_class=%s (requires transformers 5.x)",
                model_name,
                tokenizer_class,
            )
        _tokenizer_class_cache[cache_key] = result
        return result
    except urllib.error.HTTPError as exc:
        # 401/403/404 are normal misses; anything else suggests a mirror that does not proxy /resolve/.
        if exc.code in (401, 403, 404):
            logger.debug(
                "tokenizer_config.json not readable for '%s' at %s: %s", model_name, url, exc
            )
        else:
            logger.warning(
                "HTTP %s fetching tokenizer_config.json for '%s' from %s; "
                "if HF_ENDPOINT is set to a mirror, verify it proxies /resolve/ paths",
                exc.code,
                model_name,
                url,
            )
        _tokenizer_class_cache[cache_key] = False
        return False
    except urllib.error.URLError as exc:
        logger.warning(
            "Connection error fetching tokenizer_config.json for '%s' from %s: %s; "
            "if HF_ENDPOINT is set to a mirror, verify it is reachable",
            model_name,
            url,
            exc,
        )
        _tokenizer_class_cache[cache_key] = False
        return False
    except Exception as exc:
        logger.debug("Could not fetch tokenizer_config.json for '%s': %s", model_name, exc)
        _tokenizer_class_cache[cache_key] = False
        return False


def _safe_mtime(path: Path) -> float:
    try:
        return path.stat().st_mtime
    except OSError:
        return 0.0


def _config_json_from_hf_cache(model_name: str) -> dict | None:
    """Stdlib-only, so tier detection never imports huggingface_hub before the sidecar venv activates."""
    if not model_name or model_name.count("/") != 1 or model_name[0] in "/.~" or "\\" in model_name:
        return None
    # HF_HUB_CACHE env is stale after a no-restart /settings switch; use the DB-backed path.
    hub = str(get_hf_cache_paths().hub_cache)
    repo_dir = Path(hub) / ("models--" + model_name.replace("/", "--"))
    candidates = []
    ref_main = repo_dir / "refs" / "main"
    try:
        if ref_main.is_file():
            candidates.append(
                repo_dir
                / "snapshots"
                / ref_main.read_text(encoding = "utf-8").strip()
                / "config.json"
            )
        # No refs/main (commit-pinned downloads): newest snapshot by mtime, not a stale first SHA.
        candidates += sorted(
            repo_dir.glob("snapshots/*/config.json"), key = _safe_mtime, reverse = True
        )
        for cfg_path in candidates:
            if cfg_path.is_file():
                with open(cfg_path, encoding = "utf-8-sig") as f:
                    return json.load(f)
    except Exception as exc:
        logger.debug("HF cache config.json lookup failed for '%s': %s", model_name, exc)
    return None


def _load_config_json(model_name: str, hf_token: str | None = None) -> dict | None:
    """Consults the HF hub cache only offline or after a failed fetch, so an online read is never stale."""
    cache_key = _token_cache_key(model_name, hf_token)
    local_cfg = Path(model_name) / "config.json"
    if cache_key in _config_json_cache:
        # A hit predates the 60 s auth TTL, so a revoked explicit token must re-fetch; local config paths are exempt.
        if (
            not isinstance(hf_token, str)
            or _safe_is_file(local_cfg)
            or _safe_is_dir(Path(model_name))
            or cache_reads_authorized(hf_token, repo_id = model_name)
        ):
            return _config_json_cache[cache_key]

    if _safe_is_file(local_cfg):
        try:
            with open(local_cfg, encoding = "utf-8-sig") as f:
                cfg = json.load(f)
            _config_json_cache[cache_key] = cfg
            return cfg
        except Exception as exc:
            logger.debug("Could not read %s: %s", local_cfg, exc)
            _config_json_cache[cache_key] = None
            return None

    # Local checkpoint without the file yet: don't fetch as a Hub id or cache the miss.
    if _safe_is_dir(Path(model_name)):
        return None

    # Hub cache reads skip authorization, so a caller denied the ambient credential gets none.
    cache_denied = not cache_reads_authorized(hf_token, repo_id = model_name)

    if _env_offline():
        # Offline: tier from hub cache; cache hits but never misses (so online reads still fetch); an unverified explicit token is denied here.
        if cache_denied:
            return None
        cfg = _config_json_from_hf_cache(model_name)
        # Memo only ambient/anonymous reads: an untimed memo outlives 60 s token grants.
        if cfg is not None and not isinstance(hf_token, str):
            _config_json_cache[cache_key] = cfg
        return cfg

    import urllib.error
    import urllib.request

    url = _hf_raw_url(model_name, "config.json")
    try:
        cfg = _hf_json(url, hf_token)
        _config_json_cache[cache_key] = cfg
        return cfg
    except urllib.error.HTTPError as exc:
        # 401/403/404 is definitive: never serve another caller's cached private metadata.
        if exc.code in (401, 403, 404):
            logger.debug("config.json access denied for '%s': %s", model_name, exc)
            return None
        # 5xx: debug here hides a broken mirror behind a later transformers crash.
        logger.warning(
            "HTTP %s fetching config.json for '%s' from %s; "
            "if HF_ENDPOINT is set to a mirror, verify it proxies /resolve/ paths",
            exc.code,
            model_name,
            url,
        )
        # Transient: serve hub cache uncached so the next call retries; never private data of another caller.
        return None if cache_denied else _config_json_from_hf_cache(model_name)
    except Exception as exc:
        logger.debug("Could not fetch config.json for '%s': %s", model_name, exc)
        # Transient: serve the hub cache uncached so the next call retries the network.
        return None if cache_denied else _config_json_from_hf_cache(model_name)


def _config_json_is_definitive(model_name: str, hf_token: str | None = None) -> bool:
    """True if the last ``_load_config_json`` read for this model+token was cached
    (definitive), not a transient fallback (not stored, so callers re-check next call)."""
    return _token_cache_key(model_name, hf_token) in _config_json_cache


def _config_matches_tier(cfg: dict, architectures: set[str], model_types: set[str]) -> bool:
    # Defensive: a malformed config may carry non-string values (e.g. list model_type).
    archs = cfg.get("architectures")
    if isinstance(archs, (list, tuple)) and any(a in architectures for a in archs):
        return True
    mt = cfg.get("model_type")
    return isinstance(mt, str) and mt in model_types


def _config_needs_550(cfg: dict) -> bool:
    return _config_matches_tier(
        cfg,
        _TRANSFORMERS_550_ARCHITECTURES,
        _TRANSFORMERS_550_MODEL_TYPES,
    )


_NESTED_CONFIG_KEYS = ("llm_config", "text_config", "language_config", "thinker_config")


def _nemotron_h_needs_mlp_support(cfg: dict) -> bool:
    """NemotronH dash layers need transformers 5.10+ (5.3/5.5 raise KeyError) and nested llm_config/text_config of VL wrappers are checked too."""
    if not isinstance(cfg, dict):
        return False
    if cfg.get("model_type") == "nemotron_h":
        pattern = cfg.get("hybrid_override_pattern")
        if isinstance(pattern, str) and "-" in pattern:
            return True
        block_types = cfg.get("layers_block_type")
        if isinstance(block_types, (list, tuple)) and "mlp" in block_types:
            return True
    return any(_nemotron_h_needs_mlp_support(cfg.get(key)) for key in _NESTED_CONFIG_KEYS)


def _config_needs_510(cfg: dict) -> bool:
    if _config_matches_tier(
        cfg,
        _TRANSFORMERS_510_ARCHITECTURES,
        _TRANSFORMERS_510_MODEL_TYPES,
    ):
        return True
    return _nemotron_h_needs_mlp_support(cfg)


def _config_needs_457(cfg: dict) -> bool:
    # macOS runs these through MLX, which needs the 5.x stack.
    if sys.platform == "darwin":
        return False
    return _config_matches_tier(cfg, _TRANSFORMERS_457_ARCHITECTURES, set())


def _config_needs_530(cfg: dict) -> bool:
    return _config_matches_tier(
        cfg,
        _TRANSFORMERS_530_ARCHITECTURES,
        _TRANSFORMERS_530_MODEL_TYPES,
    )


def _check_config_needs_550(model_name: str, hf_token: str | None = None) -> bool:
    """Cached only for a definitive read, so a transient failure retries; errors return False."""
    cache_key = _token_cache_key(model_name, hf_token)
    if cache_key in _config_needs_550_cache:
        return _config_needs_550_cache[cache_key]

    cfg = _load_config_json(model_name, hf_token)
    result = bool(cfg) and _config_needs_550(cfg)
    if result:
        logger.info(
            "config.json check: %s needs transformers %s (architectures=%s, model_type=%s)",
            model_name,
            TRANSFORMERS_550_VERSION,
            cfg.get("architectures", []),
            cfg.get("model_type"),
        )
    if _config_json_is_definitive(model_name, hf_token):
        _config_needs_550_cache[cache_key] = result
    return result


def _check_config_needs_530(model_name: str, hf_token: str | None = None) -> bool:
    """Cached only for a definitive read, so a transient failure retries; errors return False."""
    cache_key = _token_cache_key(model_name, hf_token)
    if cache_key in _config_needs_530_cache:
        return _config_needs_530_cache[cache_key]

    cfg = _load_config_json(model_name, hf_token)
    result = bool(cfg) and _config_needs_530(cfg)
    if result:
        logger.info(
            "config.json check: %s needs transformers %s (architectures=%s, model_type=%s)",
            model_name,
            TRANSFORMERS_530_VERSION,
            cfg.get("architectures", []),
            cfg.get("model_type"),
        )
    if _config_json_is_definitive(model_name, hf_token):
        _config_needs_530_cache[cache_key] = result
    return result


def _check_config_needs_457(model_name: str, hf_token: str | None = None) -> bool:
    """True if ``config.json`` is a DeepSeek-OCR checkpoint (cached like the 5.x checks)."""
    cache_key = _token_cache_key(model_name, hf_token)
    if cache_key in _config_needs_457_cache:
        return _config_needs_457_cache[cache_key]

    cfg = _load_config_json(model_name, hf_token)
    result = bool(cfg) and _config_needs_457(cfg)
    if _config_json_is_definitive(model_name, hf_token):
        _config_needs_457_cache[cache_key] = result
    return result


def _check_config_needs_510(model_name: str, hf_token: str | None = None) -> bool:
    """Gemma 4 Unified / 12B check of config.json; caches definitive reads per (model, token)."""
    cache_key = _token_cache_key(model_name, hf_token)
    if cache_key in _config_needs_510_cache:
        return _config_needs_510_cache[cache_key]

    cfg = _load_config_json(model_name, hf_token)
    result = bool(cfg) and _config_needs_510(cfg)
    if result:
        logger.info(
            "config.json check: %s needs transformers %s (architectures=%s, model_type=%s)",
            model_name,
            TRANSFORMERS_510_VERSION,
            cfg.get("architectures", []),
            cfg.get("model_type"),
        )
    if _config_json_is_definitive(model_name, hf_token):
        _config_needs_510_cache[cache_key] = result
    return result


def _config_saved_by_transformers_5(cfg: dict | None) -> bool:
    """True if ``config.json``'s ``transformers_version`` is >= 5. Only a cheap "worth
    probing" hint (the saving version, not the minimum to load); the default-first probe
    decides the actual tier."""
    if not isinstance(cfg, dict):
        return False
    ver = cfg.get("transformers_version")
    if not isinstance(ver, str):
        return False
    try:
        return int(ver.strip().split(".", 1)[0]) >= 5
    except ValueError:
        return False


def _release_tuple(ver) -> tuple[int, int, int] | None:
    """``"5.19.0.dev0"`` -> ``(5, 19, 0)``; a dev build counts as its release."""
    m = re.match(r"\s*(\d+)\.(\d+)(?:\.(\d+))?", ver) if isinstance(ver, str) else None
    return (int(m.group(1)), int(m.group(2)), int(m.group(3) or 0)) if m else None


def _remote_code_saving_version(cfg: dict) -> str | None:
    """Top-level ``transformers_version``, else the newest one on a nested sub-config."""
    ver = cfg.get("transformers_version")
    if isinstance(ver, str):
        return ver
    nested = [
        sub.get("transformers_version")
        for sub in cfg.values()
        if isinstance(sub, dict) and _release_tuple(sub.get("transformers_version"))
    ]
    return max(nested, key = _release_tuple, default = None)


def _remote_code_min_version(cfg: dict | None) -> tuple[int, int, int] | None:
    """Saving transformers 5.x version of a custom-code (``auto_map`` AutoConfig) config, else None."""
    if not isinstance(cfg, dict):
        return None
    auto_map = cfg.get("auto_map")
    if not isinstance(auto_map, dict) or "AutoConfig" not in auto_map:
        return None
    v = _release_tuple(_remote_code_saving_version(cfg))
    if v is None or v[0] < 5:
        return None
    # Patch releases keep remote-code APIs: 5.5.4 still fits the 5.5.0 default.
    return (v[0], v[1], 0)


def _remote_code_tier(cfg: dict | None) -> str | None:
    """Default if new enough, else the oldest sidecar at or above the saving version; None if none is."""
    required = _remote_code_min_version(cfg)
    if required is None:
        return None
    tiers = [
        ("default", TRANSFORMERS_DEFAULT_VERSION),
        ("530", TRANSFORMERS_530_VERSION),
        ("550", TRANSFORMERS_550_VERSION),
        ("510", TRANSFORMERS_510_VERSION),
    ]
    if not _latest_tier_disabled():
        tiers.append(("latest", latest_venv_pinned_version()))
    fits = [
        (tier != "default", v, _TIER_RANK[tier], tier)
        for tier, ver in tiers
        if (v := _release_tuple(ver)) is not None and v >= required
    ]
    return min(fits)[3] if fits else None


def _route_remote_code(
    model_name: str,
    cfg: dict | None,
    above: str | None = None,
) -> str | None:
    """Tier for a custom-code config; None falls through, also when ``above`` already satisfies it."""
    required = _remote_code_min_version(cfg)
    if required is None or (above is not None and required <= _release_tuple(above)):
        return None
    tier = _remote_code_tier(cfg)
    if tier is None:
        tier = "latest" if (not _latest_tier_disabled() and latest_venv_pinned_version()) else "510"
    logger.info(
        "Transformers tier %s selected for %s (custom code saved by transformers %s)",
        tier,
        model_name,
        _remote_code_saving_version(cfg),
    )
    return tier


def _cached_config_json(model_name: str, hf_token: str | None) -> dict | None:
    """Cached config.json only, no fetch; a miss just skips the version-field probe."""
    return _config_json_cache.get(_token_cache_key(model_name, hf_token))


# AST-parse each sidecar CONFIG_MAPPING_NAMES (no import, network or exec); pick lowest tier shipping the type; only upgrades default, never lowers.
_config_mapping_cache: dict[str, frozenset[str]] = {}


def _latest_tier_disabled() -> bool:
    """Kill switch shared with transformers_latest; roll back without deleting sidecar files."""
    return os.environ.get("UNSLOTH_STUDIO_NO_LATEST_TRANSFORMERS", "").strip().lower() in (
        "1",
        "true",
        "yes",
        "on",
    )


# Failed lazy repairs back off so a broken sidecar can't turn every routing call into pip.
_latest_repair_failed_at: float = 0.0
_LATEST_REPAIR_BACKOFF_SECS = 5 * 60


# Marks RECORD-scan damage for the cheap predicate; inside the sidecar so a swap clears it.
_LATEST_REPAIR_MARKER = ".unsloth_sidecar_repair_needed"


def _latest_repair_marker_path() -> Path:
    return Path(_VENV_T5_LATEST_DIR) / _LATEST_REPAIR_MARKER


def _latest_repair_requested() -> bool:
    """A single isfile check, so the routing predicate can carry it; the kill switch ignores the marker."""
    if _sidecar_file_check_disabled():
        # The kill switch must ignore the marker, else a false positive withholds the sidecar for the backoff.
        return False
    try:
        return _latest_repair_marker_path().is_file()
    except OSError:
        return False


def _request_latest_repair() -> None:
    """Record that the sidecar needs a repair the caller could not perform."""
    try:
        _latest_repair_marker_path().write_text(
            json.dumps({"pid": os.getpid(), "at": time.time()}), encoding = "utf-8"
        )
    except OSError:
        pass


def _clear_latest_repair_request() -> None:
    """Drop the flag once a scan finds the sidecar healthy, so a request left by a
    lost race costs one extra scan instead of looping forever."""
    try:
        _latest_repair_marker_path().unlink()
    except OSError:
        pass


def _latest_sidecar_intact() -> bool:
    """Pin, transformers dir and pinned packages only (hot path); False if pin gone, so a cached latest mapping is dropped; sub-file damage caught at activation."""
    pin = _latest_pin_data()
    if pin is None:
        return False
    if _latest_repair_requested():
        return False
    return _venv_dir_is_valid(_VENV_T5_LATEST_DIR, tuple(pin["packages"]))


def _latest_sidecar_undamaged() -> bool:
    """Full RECORD scan, so it runs only on a config-mapping cache miss, not on the hot path."""
    pin = _latest_pin_data()
    if pin is None:
        return False
    return _venv_dir_is_valid_and_undamaged(_VENV_T5_LATEST_DIR, tuple(pin["packages"]))


def _overlay_transformers_dir(tier: str) -> str | None:
    """transformers source dir for a tier, located without importing it."""
    global _latest_repair_failed_at
    if tier != "default":
        # latest requires a valid pin and the kill switch off.
        if tier == "latest" and (_latest_tier_disabled() or latest_venv_pinned_version() is None):
            return None
        root = {
            "530": _VENV_T5_530_DIR,
            "550": _VENV_T5_550_DIR,
            "510": _VENV_T5_510_DIR,
            "latest": _VENV_T5_LATEST_DIR,
        }.get(tier)
        src = os.path.join(root, "transformers") if root else None
        if src and tier == "latest":
            # A broken pinned sidecar must self-heal or latest-only models fail. Repair under swap reservation; back off after failure so routing doesn't hammer pip.
            heal_due = time.time() - _latest_repair_failed_at >= _LATEST_REPAIR_BACKOFF_SECS
            # RECORD scan only when a repair could follow, on config-mapping cache miss; never in backoff, where every call re-pays it for an answer it cannot act on.
            broken = not _latest_sidecar_intact() or (heal_due and not _latest_sidecar_undamaged())
            repaired = False
            if broken and heal_due:
                if _ensure_venv_t5_latest_exists():
                    _latest_repair_failed_at = 0.0
                    repaired = True
                else:
                    # The marker keeps the sidecar withheld during backoff; else it would read as healthy.
                    _latest_repair_failed_at = time.time()
            if broken and not repaired:
                # Still broken: treat the overlay as unavailable rather than route models to a tier whose worker activation fails; older tiers keep loading.
                return None
        return src if src and _safe_is_dir(Path(src)) else None
    # Default is the base 4.x transformers; find_spec may resolve to a 5.x sidecar already on sys.path, so skip any .venv_t5_* / llmcompressor overlay dir.
    sidecars = tuple(
        os.path.abspath(d) + os.sep
        for d in (
            _VENV_T5_530_DIR,
            _VENV_T5_550_DIR,
            _VENV_T5_510_DIR,
            _VENV_T5_LATEST_DIR,
            _VENV_LLMCOMPRESSOR_DIR,
        )
    )
    candidates = []
    try:
        spec = importlib.util.find_spec("transformers")
        if spec and spec.origin:
            candidates.append(os.path.dirname(spec.origin))
    except Exception:
        pass
    candidates += [os.path.join(e, "transformers") for e in sys.path if e]
    for c in candidates:
        if _safe_is_dir(Path(c)) and not os.path.abspath(c).startswith(sidecars):
            return c
    return None


def _mapping_first_keys(value: ast.AST) -> set[str]:
    """First keys of a dict literal, or of an OrderedDict(...)/dict(...)/.update(...)
    built from 2-tuple lists and **{...} unpacking."""

    def keys_of(node):
        if isinstance(node, ast.Dict):
            return list(node.keys)
        if isinstance(node, (ast.List, ast.Tuple)):
            return [
                el.elts[0] for el in node.elts if isinstance(el, (ast.Tuple, ast.List)) and el.elts
            ]
        return []

    nodes = keys_of(value)
    if isinstance(value, ast.Call):
        for a in value.args:
            nodes += keys_of(a)
        for kw in value.keywords:  # **{...} unpacking has kw.arg is None
            if kw.arg is None:
                nodes += keys_of(kw.value)
    return {n.value for n in nodes if isinstance(n, ast.Constant) and isinstance(n.value, str)}


def _model_types_from_source(source: str) -> set[str]:
    """Parses the AST only, never executes source; covers CONFIG_MAPPING_NAMES.update() as well."""
    keys: set[str] = set()
    tree = ast.parse(source)
    for node in ast.walk(tree):
        if isinstance(node, ast.Assign) and any(
            isinstance(t, ast.Name) and t.id == "CONFIG_MAPPING_NAMES" for t in node.targets
        ):
            keys |= _mapping_first_keys(node.value)
        elif isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):
            fn = node.value.func
            if (
                isinstance(fn, ast.Attribute)
                and fn.attr == "update"
                and isinstance(fn.value, ast.Name)
                and fn.value.id == "CONFIG_MAPPING_NAMES"
            ):
                keys |= _mapping_first_keys(node.value)
    return keys


def _config_model_types(tier: str) -> frozenset[str]:
    """model_type keys in a tier's CONFIG_MAPPING_NAMES (5.10 moved it to auto_mappings.py)."""
    # Kill switch beats the cache: a stale mapping must not route latest-only models.
    if tier == "latest" and _latest_tier_disabled():
        return frozenset()
    cached = _config_mapping_cache.get(tier)
    if cached is not None:
        # A cached latest mapping can outlive a deleted sidecar or one that lost a package; drop it so routing re-resolves and self-heals, not to a broken tier.
        if tier != "latest" or _latest_sidecar_intact():
            return cached
        _config_mapping_cache.pop("latest", None)
    tdir = _overlay_transformers_dir(tier)
    if tdir is None:
        return frozenset()  # overlay not provisioned yet; do not cache so a later call re-reads
    keys: set[str] = set()
    for rel in ("models/auto/configuration_auto.py", "models/auto/auto_mappings.py"):
        path = Path(tdir) / rel
        if not _safe_is_file(path):
            continue
        try:
            keys |= _model_types_from_source(path.read_text(encoding = "utf-8"))
        except Exception:
            continue
    result = frozenset(keys)
    _config_mapping_cache[tier] = result
    return result


def _model_types_from_config(cfg: dict) -> list[str]:
    """All model_types in the config, primary first; nested types matter for routing too, since
    wrappers build sub-configs via CONFIG_MAPPING."""
    seen: list[str] = []

    def add(value):
        if isinstance(value, str) and value and value not in seen:
            seen.append(value)

    add(cfg.get("model_type"))
    for key in _NESTED_CONFIG_KEYS:
        sub = cfg.get(key)
        if isinstance(sub, dict):
            add(sub.get("model_type"))
    for value in cfg.values():
        if isinstance(value, dict):
            add(value.get("model_type"))
    return seen


def _lowest_tier_for(model_type: str) -> str | None:
    for tier in sorted(_TIER_RANK, key = _TIER_RANK.get):
        if model_type in _config_model_types(tier):
            return tier
    return None


def _tier_from_config_mapping(cfg: dict) -> str | None:
    """Lowest tier loading every model_type in cfg (None if primary unknown); nested type can raise it, but unknown nested never vetoes: no tier could load it."""
    types = _model_types_from_config(cfg)
    if not types:
        return None
    best = _lowest_tier_for(types[0])
    if best is None:
        return None
    for model_type in types[1:]:
        tier = _lowest_tier_for(model_type)
        if tier is not None and _TIER_RANK[tier] > _TIER_RANK[best]:
            best = tier
    return best


def _raise_tier_for_nested(cfg: dict | None, tier: str) -> str:
    """Raise-only for nested sub-configs that need a newer sidecar; never lowers a fast-path tier, so Qwen3.6 overrides keep theirs; failure keeps the tier."""
    if not isinstance(cfg, dict):
        return tier
    try:
        mapped = _tier_from_config_mapping(cfg)
        if mapped is not None and _TIER_RANK.get(mapped, 0) > _TIER_RANK.get(tier, 0):
            return mapped
    except Exception:
        pass
    return tier


# When cheap signals only say "some 5.x", parse config.json with each sidecar, lowest first.
_PROBE_TIER_ORDER = ("530", "550", "510")
_PROBE_TIMEOUT_SECS = 60

# config.json-only parse in a sidecar: built-in parser, no repo code, no weights, exit 0 = parses.
_PROBE_CONFIG_SCRIPT = (
    r"""
import sys, os
os.environ["TOKENIZERS_PARALLELISM"] = "false"
# This -c child cannot import backend modules, and AutoConfig.from_pretrained
# below may download from the Hub, so it carries the gate as generated source.
# repr() the path: a Windows backslash or a space must survive into the source.
_TRUSTSTORE_VENDOR = """
    + repr(vendor_dir())
    + "\n"
    + inline_gate_source()
    + inline_activation_source()
    + r"""
target_dir, model_name = sys.argv[1], sys.argv[2]
if target_dir:  # empty = probe the ambient (default-tier) transformers, no sidecar prepend
    sys.path.insert(0, target_dir)
try:
    from transformers import AutoConfig
    AutoConfig.from_pretrained(model_name, trust_remote_code=False)
    sys.exit(0)
except Exception as exc:
    # stderr encoding may not be UTF-8 (e.g. cp1252 on Windows); write bytes so a
    # non-ASCII error message cannot itself raise UnicodeEncodeError.
    sys.stderr.buffer.write((type(exc).__name__ + ": " + str(exc)).encode("utf-8", "replace"))
    sys.exit(1)
"""
)

# stderr fragments meaning "couldn't fetch/auth", NOT "needs a newer parser".
_PROBE_TRANSIENT_MARKERS = (
    "ConnectionError",
    "HTTPError",
    "Timeout",
    "Max retries",
    "Temporary failure",
    "GatedRepoError",
    "RepositoryNotFoundError",
    "LocalEntryNotFoundError",
    "OfflineModeIsEnabled",
    "401",
    "403",
    "404",
)


def _stderr_is_transient(err: str) -> bool:
    return any(marker in err for marker in _PROBE_TRANSIENT_MARKERS)


def _probe_tier_venvs():
    """tier -> (target_dir, ensure_fn), a function so the later _ensure_* defs resolve. The
    ``default`` entry (empty target_dir = ambient default) is only probed with include_default."""
    return {
        "default": ("", lambda: True),
        "530": (_VENV_T5_530_DIR, _ensure_venv_t5_530_exists),
        "550": (_VENV_T5_550_DIR, _ensure_venv_t5_550_exists),
        "510": (_VENV_T5_510_DIR, _ensure_venv_t5_510_exists),
        "latest": (_VENV_T5_LATEST_DIR, _ensure_venv_t5_latest_exists),
    }


def _probe_tier_order() -> tuple[str, ...]:
    """Sidecar probe order. The consented "latest" sidecar joins only once it is
    provisioned (pin marker present): an absent optional tier must not flip the probe's
    skipped-tier bookkeeping, keeping pre-latest behavior byte-identical."""
    if not _latest_tier_disabled() and latest_venv_pinned_version() is not None:
        return _PROBE_TIER_ORDER + ("latest",)
    return _PROBE_TIER_ORDER


def _probe_autoconfig(target_dir: str, model_name: str, hf_token: str | None) -> bool | None:
    """True if config.json parses, False on parse/version failure, None on transient error (not cached)."""
    env = get_hf_cache_paths().child_env(child_env_without_native_path_secret())
    # The probe reads the implicit HF_TOKEN env, so grant or scrub here, not via argv.
    apply_token_to_child_env(env, hf_token)
    if _env_offline():
        env["HF_HUB_OFFLINE"] = "1"
        env["TRANSFORMERS_OFFLINE"] = "1"
    try:
        result = subprocess.run(
            [sys.executable, "-c", _PROBE_CONFIG_SCRIPT, target_dir, model_name],
            capture_output = True,
            text = True,
            encoding = "utf-8",
            errors = "replace",
            timeout = _PROBE_TIMEOUT_SECS,
            env = utf8_child_env(env),
            **_windows_hidden_subprocess_kwargs(),
        )
    except subprocess.TimeoutExpired:
        logger.warning("AutoConfig probe timed out for '%s' in %s", model_name, target_dir)
        return None
    except Exception as exc:
        logger.warning("AutoConfig probe could not spawn for '%s': %s", model_name, exc)
        return None
    if result.returncode == 0:
        return True
    err = (result.stderr or "").strip()
    if _stderr_is_transient(err):
        logger.warning("AutoConfig probe transient failure for '%s': %s", model_name, err)
        return None
    logger.info("AutoConfig probe parse failure for '%s' in %s: %s", model_name, target_dir, err)
    return False


def _probe_cache_key(model_name: str) -> str:
    """Local checkpoints also key on config.json size and mtime (overwritten in place); remote ids key by name only, as a Hub lookup would pin the wrong env."""
    try:
        config_path = (Path(model_name) / "config.json").resolve()
        st = config_path.stat()
    except OSError:
        return model_name
    return f"{config_path}\0{st.st_size}:{st.st_mtime_ns}"


# Spawned workers start with the dict above empty, so local-dir verdicts also persist on disk.
_PROBE_DISK_CACHE_NAME = "transformers_tier_probe.json"
_PROBE_DISK_CACHE_MAX = 256


def _local_config_sha(model_name: str) -> str | None:
    """sha256 of a local dir's config.json, or None for a Hub id / unreadable file."""
    try:
        path = Path(model_name)
        if not path.is_dir():
            return None
        return hashlib.sha256((path / "config.json").read_bytes()).hexdigest()
    except (OSError, ValueError):
        return None


def _tier_transformers_version(target_dir: str) -> str | None:
    """Installed transformers version in a sidecar dir (dist-info), or the ambient one for
    ``""`` (importlib.metadata, without importing transformers)."""
    try:
        if not target_dir:
            import importlib.metadata as md
            return md.version("transformers")
        for entry in sorted(Path(target_dir).glob("transformers-*.dist-info")):
            meta = entry / "METADATA"
            if meta.is_file():
                for line in meta.read_text(encoding = "utf-8", errors = "replace").splitlines():
                    if line.startswith("Version:"):
                        return line.split(":", 1)[1].strip()
                    if not line.strip():
                        break
    except Exception:
        return None
    return None


def _probe_disk_cache_path() -> Path | None:
    try:
        from utils.paths.storage_roots import cache_root
        return cache_root() / _PROBE_DISK_CACHE_NAME
    except Exception:
        return None


def _probe_disk_read() -> dict:
    path = _probe_disk_cache_path()
    if path is None:
        return {}
    try:
        data = json.loads(path.read_text(encoding = "utf-8"))
    except (OSError, ValueError):
        return {}
    return data if isinstance(data, dict) else {}


def _probe_disk_lookup(disk_key: str, venvs: dict) -> str | None:
    entry = _probe_disk_read().get(disk_key)
    if not isinstance(entry, dict):
        return None
    tier, walked = entry.get("tier"), entry.get("walked")
    if not isinstance(tier, str) or not isinstance(walked, list) or not walked:
        return None
    for item in walked:
        if not (isinstance(item, list) and len(item) == 2 and item[0] in venvs):
            return None
        if _tier_transformers_version(venvs[item[0]][0]) != item[1]:
            return None
    return tier


def _probe_disk_store(disk_key: str, tier: str, walked: list) -> None:
    path = _probe_disk_cache_path()
    if path is None:
        return
    tmp = None
    try:
        data = _probe_disk_read()
        data.pop(disk_key, None)
        data[disk_key] = {"tier": tier, "walked": walked}
        while len(data) > _PROBE_DISK_CACHE_MAX:
            data.pop(next(iter(data)))
        path.parent.mkdir(parents = True, exist_ok = True)
        fd, tmp = tempfile.mkstemp(prefix = ".tier_probe_", suffix = ".tmp", dir = str(path.parent))
        with os.fdopen(fd, "w", encoding = "utf-8") as f:
            json.dump(data, f)
        os.replace(tmp, path)
        tmp = None
    except Exception as exc:
        logger.debug("Could not persist tier probe verdict: %s", exc)
    finally:
        if tmp is not None:
            with contextlib.suppress(OSError):
                os.unlink(tmp)


def _cached_tier_available(venvs: dict, tier: str) -> bool:
    """A disk hit must pass the tier's ensure_fn, as a live probe does."""
    try:
        return bool(venvs[tier][1]())
    except Exception:
        return False


def _probe_tier(
    model_name: str,
    hf_token: str | None,
    reason: str,
    *,
    include_default: bool = False,
    floor: str = "530",
) -> str:
    """Lowest tier whose parser loads config; uncertain results use floor, uncached. No Hub sha: it imports huggingface_hub before sidecar is on sys.path."""
    if os.environ.get("UNSLOTH_DISABLE_TIER_PROBE", "").lower() in ("1", "true", "yes", "on"):
        return floor
    key = _probe_cache_key(model_name)
    # Key by probe mode: a 'default' result must not be reused for floor='530' callers.
    if include_default or floor != "530":
        key = f"{key}\0floor={floor}:def={int(include_default)}"

    def _latest_unusable() -> bool:
        return (
            _latest_tier_disabled()
            or _latest_repair_requested()
            or latest_venv_pinned_version() is None
        )

    if key in _probe_tier_cache:
        cached = _probe_tier_cache[key]
        # Do not reuse a cached latest if disabled, damaged or unpinned (a deleted pin drops the repair marker); else workers spawn into a failing tier. Re-probe.
        if cached != "latest" or not _latest_unusable():
            return cached

    venvs = _probe_tier_venvs()
    sidecar_order = _probe_tier_order()
    order = (("default",) + sidecar_order) if include_default else sidecar_order

    config_sha = _local_config_sha(model_name)
    disk_key = None
    if config_sha is not None:
        disk_key = f"{config_sha}:floor={floor}:def={int(include_default)}:order={','.join(order)}"
        disk_tier = _probe_disk_lookup(disk_key, venvs)
        if (
            disk_tier is not None
            and (disk_tier != "latest" or not _latest_unusable())
            and _cached_tier_available(venvs, disk_tier)
        ):
            _probe_tier_cache[key] = disk_tier
            logger.info(
                "Transformers tier %s selected for %s (cached AutoConfig probe; %s)",
                disk_tier,
                model_name,
                reason,
            )
            return disk_tier
    walked: list = []

    def _cache(tier: str, *, skipped: bool) -> str:
        # Don't pin a result that depended on a skipped lower tier; re-probe next call.
        if not skipped:
            _probe_tier_cache[key] = tier
            if (
                disk_key is not None
                and all(v is not None for _, v in walked)
                and _local_config_sha(model_name) == config_sha
            ):
                _probe_disk_store(disk_key, tier, walked)
        return tier

    probed_count = 0
    skipped_any = False
    for tier in order:
        target_dir, ensure_fn = venvs[tier]
        try:
            available = ensure_fn()
        except Exception:
            available = False
        if not available:
            skipped_any = True
            continue
        probed_count += 1
        if disk_key is not None:
            walked.append([tier, _tier_transformers_version(target_dir)])
        ok = _probe_autoconfig(target_dir, model_name, hf_token)
        if ok is True:
            logger.info(
                "Transformers tier %s selected for %s (AutoConfig probe; %s)",
                tier,
                model_name,
                reason,
            )
            return _cache(tier, skipped = skipped_any)
        if ok is None:
            logger.info("Tier probe inconclusive for %s (%s); using %s", model_name, reason, floor)
            return floor  # transient: retry next load

    # Cache a miss only when every tier was probed; a skipped sidecar means an incomplete environment, so retry uncached.
    if skipped_any or probed_count == 0:
        logger.info(
            "Tier probe incomplete for %s (%s); using %s (uncached)", model_name, reason, floor
        )
        return floor
    logger.info(
        "Transformers tier %s selected for %s (AutoConfig probe found no higher tier; %s)",
        floor,
        model_name,
        reason,
    )
    return _cache(floor, skipped = False)


def _norm_separators(s: str) -> str:
    """Collapse ``_``/whitespace to ``-`` (underscore aliases) but keep ``.`` so a
    version dot (``qwen3.5``) isn't conflated with a size separator (``Qwen3-5B``)."""
    return "".join("-" if ch in "_ \t" else ch for ch in s)


def _looks_like_hf_id(value: str) -> bool:
    """True if value looks like a Hub id (org/name); an existing path wins, as in transformers."""
    if not value or not value.strip():
        return False
    if os.path.isabs(value) or value.startswith((".", "~")) or "\\" in value:
        return False
    if os.path.exists(value):
        return False
    return value.count("/") <= 1


def _tier_from_name(name: str) -> tuple[str, str] | None:
    """Dot versions match only dot/underscore forms, never hyphens, so Qwen3-6B size names stay put."""
    lowered = name.lower()
    norm = _norm_separators(lowered)
    dotted = lowered.replace("_", ".")
    if "assistant" in lowered and ("gemma-4" in norm or "gemma4" in norm):
        return "510", "gemma-4 assistant variant"
    for substrings, tier in (
        (TRANSFORMERS_510_MODEL_SUBSTRINGS, "510"),
        (TRANSFORMERS_550_MODEL_SUBSTRINGS, "550"),
        (TRANSFORMERS_5_MODEL_SUBSTRINGS, "530"),
    ):
        for s in substrings:
            if "." in s:
                if s in lowered or s in dotted:
                    return tier, s
            elif s in lowered or _norm_separators(s) in norm:
                return tier, s
    return None


def _higher_tier_name_override(name_hint: str | None) -> str | None:
    """510/550 tier if *name_hint* names a higher-tier model, else ``None``. Qwen3.6
    reuses Qwen3.5 config ids but needs the 5.5 sidecar, so a name hint overrides 530."""
    if not name_hint:
        return None
    hint = _tier_from_name(name_hint)
    return hint[0] if hint is not None and hint[0] in ("510", "550") else None


def get_transformers_tier(
    model_name: str,
    hf_token: str | None = None,
    probe: bool = True,
) -> str:
    """Returns 510, 550, 530 or default; local config.json is checked before name heuristics (dir-name false positives); probe=False skips sidecar subprocesses."""
    tier = _native_transformers_tier(model_name, hf_token, probe)
    # Raise only; the name fast path fetched no config, so a real load (probe=True) reads it here.
    if probe and tier != "default":
        cfg = _load_config_json(model_name, hf_token)
    else:
        cfg = _cached_config_json(model_name, hf_token)
    required = _remote_code_min_version(cfg)
    current = _release_tuple(_tier_version(tier))
    if required is not None and current is not None and current < required:
        return _route_remote_code(model_name, cfg) or tier
    return tier


def _tier_version(tier: str) -> str | None:
    return {
        "default": TRANSFORMERS_DEFAULT_VERSION,
        "457": TRANSFORMERS_457_VERSION,
        "530": TRANSFORMERS_530_VERSION,
        "550": TRANSFORMERS_550_VERSION,
        "510": TRANSFORMERS_510_VERSION,
    }.get(tier) or (latest_venv_pinned_version() if tier == "latest" else None)


def _native_transformers_tier(model_name: str, hf_token: str | None, probe: bool) -> str:
    local_cfg = Path(model_name) / "config.json"
    if _safe_is_file(local_cfg):
        cfg = _load_config_json(model_name, hf_token)
        if cfg is not None:
            if _config_needs_457(cfg):
                logger.info(
                    "Transformers tier 457 selected for %s (local config.json check)", model_name
                )
                return "457"
            if _config_needs_510(cfg):
                tier = _raise_tier_for_nested(cfg, "510")
                logger.info(
                    "Transformers tier %s selected for %s (local config.json check)",
                    tier,
                    model_name,
                )
                return tier
            if _config_needs_550(cfg):
                tier = _raise_tier_for_nested(cfg, "550")
                logger.info(
                    "Transformers tier %s selected for %s (local config.json check)",
                    tier,
                    model_name,
                )
                return tier
            if _config_needs_530(cfg):
                # Qwen3.6 reuses Qwen3.5 ids but needs 5.5 by name; only a Hub id or folder name may override 530, so a stale local _name_or_path cannot flip it.
                base = _resolve_base_model(model_name)
                hint_src = (
                    base
                    if (base != model_name and _looks_like_hf_id(base))
                    else Path(model_name).name
                )
                override = _higher_tier_name_override(hint_src)
                if override is not None:
                    override = _raise_tier_for_nested(cfg, override)
                    logger.info(
                        "Transformers tier %s selected for %s (name overrides 530 config)",
                        override,
                        model_name,
                    )
                    return override
                tier = _raise_tier_for_nested(cfg, "530")
                logger.info(
                    "Transformers tier %s selected for %s (local config.json check)",
                    tier,
                    model_name,
                )
                return tier
            # A resolved local dir recurses; a Hub id uses name rules only (no network).
            resolved = _resolve_base_model(model_name)
            if resolved != model_name:
                if _safe_is_dir(Path(resolved)):
                    tier = get_transformers_tier(resolved, hf_token, probe = probe)
                    if tier != "default":
                        logger.info(
                            "Transformers tier %s selected for %s (resolved local path: %s)",
                            tier,
                            model_name,
                            resolved,
                        )
                        return tier
                elif _looks_like_hf_id(resolved):
                    result = _tier_from_name(resolved)
                    if result is not None:
                        tier, match = result
                        logger.info(
                            "Transformers tier %s selected for %s (resolved HF ID: %s, match: %s)",
                            tier,
                            model_name,
                            resolved,
                            match,
                        )
                        return tier
            static = _tier_from_config_mapping(cfg)
            if static is not None and static != "default":
                logger.info(
                    "Transformers tier %s selected for %s (config mapping: model_type absent below)",
                    static,
                    model_name,
                )
                return static
            # The fall-through's 530 floor fits these; routing here would lower them to default.
            remote_code = _route_remote_code(model_name, cfg, above = TRANSFORMERS_530_VERSION)
            if remote_code is not None:
                return remote_code
            local_tc = Path(model_name) / "tokenizer_config.json"
            if _safe_is_file(local_tc) and _check_tokenizer_config_needs_v5(model_name, hf_token):
                if not probe:
                    return "530"
                return _probe_tier(model_name, hf_token, "local tokenizer needs 5.x")
            if _config_saved_by_transformers_5(cfg):
                if not probe:
                    return "530"  # cheap 5.x hint; the real path resolves the exact tier
                tier = _probe_tier(
                    model_name,
                    hf_token,
                    "local config saved by transformers 5.x",
                    include_default = True,
                    floor = "default",
                )
                if tier != "default":
                    return tier
            logger.info(
                "Transformers tier default (%s) selected for %s (local config.json no match)",
                TRANSFORMERS_DEFAULT_VERSION,
                model_name,
            )
            return "default"

    # --- Fast substring checks (no I/O) ------------------------------------
    result = _tier_from_name(model_name)
    if result is not None:
        tier, match = result
        # With a latest sidecar pinned, a fixed-tier name can still carry a latest-only model_type; check config only when pinned (keeps pre-latest path I/O-free).
        if latest_venv_pinned_version() is not None:
            tier = _raise_tier_for_nested(_load_config_json(model_name, hf_token), tier)
        logger.info(
            "Transformers tier %s selected for %s (substring match: %s)",
            tier,
            model_name,
            match,
        )
        return tier

    # --- Slow config fallbacks (network for HF IDs; authenticated with hf_token) --------
    if _check_config_needs_457(model_name, hf_token):
        logger.info("Transformers tier 457 selected for %s (config.json check)", model_name)
        return "457"
    if _check_config_needs_510(model_name, hf_token):
        tier = _raise_tier_for_nested(_load_config_json(model_name, hf_token), "510")
        logger.info("Transformers tier %s selected for %s (config.json check)", tier, model_name)
        return tier
    if _check_config_needs_550(model_name, hf_token):
        tier = _raise_tier_for_nested(_load_config_json(model_name, hf_token), "550")
        logger.info("Transformers tier %s selected for %s (config.json check)", tier, model_name)
        return tier
    if _check_config_needs_530(model_name, hf_token):
        # Qwen3.6 reuses Qwen3.5 config ids but needs 5.5 by name; honor a real Hub-id hint from _name_or_path before selecting 530.
        remote_cfg = _load_config_json(model_name, hf_token) or {}
        base = remote_cfg.get("_name_or_path") or remote_cfg.get("model_name")
        override = _higher_tier_name_override(
            base if isinstance(base, str) and base != model_name else None
        )
        if override is not None:
            override = _raise_tier_for_nested(remote_cfg, override)
            logger.info(
                "Transformers tier %s selected for %s (name overrides 530 config)",
                override,
                model_name,
            )
            return override
        tier = _raise_tier_for_nested(remote_cfg, "530")
        logger.info("Transformers tier %s selected for %s (config.json check)", tier, model_name)
        return tier
    # Not the cache-only reader, so a hub-cache config during an outage still counts.
    remote_cfg = _load_config_json(model_name, hf_token)
    if remote_cfg is not None:
        static = _tier_from_config_mapping(remote_cfg)
        if static is not None and static != "default":
            logger.info(
                "Transformers tier %s selected for %s (config mapping: model_type absent below)",
                static,
                model_name,
            )
            return static
        remote_code = _route_remote_code(model_name, remote_cfg, above = TRANSFORMERS_530_VERSION)
        if remote_code is not None:
            return remote_code
    if _check_tokenizer_config_needs_v5(model_name, hf_token):
        if not probe:
            return "530"
        return _probe_tier(model_name, hf_token, "tokenizer needs 5.x")

    if _config_saved_by_transformers_5(_cached_config_json(model_name, hf_token)):
        if not probe:
            return "530"  # cheap 5.x hint; the real path resolves the exact tier
        tier = _probe_tier(
            model_name,
            hf_token,
            "config saved by transformers 5.x",
            include_default = True,
            floor = "default",
        )
        if tier != "default":
            return tier

    logger.info(
        "Transformers tier default (%s) selected for %s (no match)",
        TRANSFORMERS_DEFAULT_VERSION,
        model_name,
    )
    return "default"


def needs_transformers_5(model_name: str) -> bool:
    """Return True if *model_name* requires any transformers 5.x version.

    Convenience wrapper around :func:`get_transformers_tier`. Passes ``probe=False`` so a
    log-only parent caller never spawns sidecar probes (the worker re-resolves the exact
    tier with ``probe=True`` on the real activation path).
    """
    return get_transformers_tier(model_name, probe = False) not in ("default", "457")


# --- Version switching (in-process, used only by export) ---


def _get_in_memory_version() -> str | None:
    """Return the transformers version currently loaded in this process."""
    tf = sys.modules.get("transformers")
    if tf is not None:
        return getattr(tf, "__version__", None)
    return None


# All top-level prefixes that hold references to transformers internals.
_PURGE_PREFIXES = (
    "transformers",
    "huggingface_hub",
    "unsloth",
    "unsloth_zoo",
    "peft",
    "trl",
    "accelerate",
    "auto_gptq",
    # bitsandbytes is excluded: its torch.library ops survive a purge and re-register crashes.
    "utils.models",
    "core.training",
    "core.inference",
    "core.export",
)


def _purge_modules() -> int:
    """Remove all cached modules for transformers and its dependents; returns the number of modules purged."""
    importlib.invalidate_caches()
    to_remove = [
        k
        for k in list(sys.modules.keys())
        if any(k == p or k.startswith(p + ".") for p in _PURGE_PREFIXES)
    ]
    for key in to_remove:
        del sys.modules[key]
    return len(to_remove)


_VENV_T5_530_PACKAGES = (
    f"transformers=={TRANSFORMERS_530_VERSION}",
    "huggingface_hub==1.8.0",
    "hf_xet==1.4.2",
    "tiktoken",
)

_VENV_T5_510_PACKAGES = (
    f"transformers=={TRANSFORMERS_510_VERSION}",
    "huggingface_hub==1.8.0",
    "hf_xet==1.4.2",
    "tiktoken",
)

_VENV_T5_550_PACKAGES = (
    f"transformers=={TRANSFORMERS_550_VERSION}",
    "huggingface_hub==1.8.0",
    "hf_xet==1.4.2",
    "tiktoken",
)

# Backwards-compat alias
_VENV_T5_PACKAGES = _VENV_T5_550_PACKAGES

# transformers 4.57.6 refuses to import next to huggingface_hub 1.x.
_VENV_T4_457_PACKAGES = (
    f"transformers=={TRANSFORMERS_457_VERSION}",
    "huggingface_hub==0.36.2",
)

# Setup installs tiktoken best-effort, so the runtime must agree or it deletes the sidecar.
_OPTIONAL_SIDECAR_PACKAGES = frozenset({"tiktoken"})


def _sidecar_package_is_optional(pkg_spec: str) -> bool:
    return (
        pkg_spec.split("==", 1)[0].strip().lower().replace("_", "-") in _OPTIONAL_SIDECAR_PACKAGES
    )


_SIDECAR_FILE_CHECK_ENV = "UNSLOTH_SKIP_SIDECAR_FILE_CHECK"


def _sidecar_file_check_disabled() -> bool:
    """Escape hatch for the file scan (a false positive costs a several-hundred-MB reinstall); it must cover the repair marker or it lags a backoff window."""
    return os.environ.get(_SIDECAR_FILE_CHECK_ENV, "").strip().lower() in (
        "1",
        "true",
        "yes",
        "on",
    )


def _sidecar_scan(venv_dir: str, limit: int = 3) -> tuple[list[str], bool]:
    """inconclusive means a row was unreadable, so empty findings mean nothing proven, not healthy."""
    return _sidecar_scan_impl(venv_dir, limit)


# Mirrored from unsloth_cli/_studio_deps.py (backend never imports the CLI). Keep in sync.
_SHARED_NON_RUNTIME_ROOTS = frozenset(
    (
        "test",
        "tests",
        "doc",
        "docs",
        "example",
        "examples",
        "benchmark",
        "benchmarks",
        "sample",
        "samples",
        "scripts",
    )
)
_INSTALLER_REWRITTEN_NAMES = frozenset(("package-lock.json",))
# Untagged and pypy/graalpy/debug names are deliberately unmatched: report nothing, not a guess.
_EXT_VERSION_TAG_RE = re.compile(r"\.(?:cpython-|cp)(\d{2,}t?)\b")
# abi3 segfaults free-threaded builds (fine on GIL); no installer writes abi3 there, so reporting it cannot loop: reinstall fetches the cp<ver>t wheel.
_ABI3_EXT_RE = re.compile(r"\.abi3\.(?:so|pyd)$")
_CURRENT_EXT_TAG = "{}{}{}".format(
    sys.version_info.major,
    sys.version_info.minor,
    "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "",
)


def _sidecar_damaged_files(venv_dir: str, limit: int = 3) -> list[str]:
    """RECORD files gone or short; use _sidecar_scan to tell an empty result from an unreadable disk."""
    return _sidecar_scan_impl(venv_dir, limit)[0]


def _sidecar_scan_impl(venv_dir: str, limit: int = 3) -> tuple[list[str], bool]:
    """Checks RECORD rows vs disk (version checks pass a truncated sidecar); fails open. Sizes only for single-owned paths; a larger file is a collision, not damage."""
    import csv
    import io

    inconclusive = False
    if _sidecar_file_check_disabled():
        return [], False
    root = Path(venv_dir)
    entries: list[tuple] = []
    owners: dict[str, int] = {}
    try:
        dist_infos = sorted(root.glob("*.dist-info"))
    except OSError:
        return [], True
    for di in dist_infos:
        name = di.name.split("-")[0]
        # Absent is fine; present, its RECORD is held to the same standard.
        try:
            record = (di / "RECORD").read_text(encoding = "utf-8", errors = "replace")
        except FileNotFoundError:
            # No RECORD says nothing about damage; some installs legitimately have none.
            continue
        except OSError:
            # Present but unreadable: a real gap, so the result cannot be called clean.
            inconclusive = True
            continue
        try:
            rows = list(csv.reader(io.StringIO(record)))
        except csv.Error:
            inconclusive = True
            continue
        for row in rows:
            rel = row[0] if row else ""
            # A trailing slash is a directory entry, which has nothing to check.
            if not rel or rel.endswith("/"):
                continue
            # Installer metadata is rewritten in place and drifts; .pyc is regenerated.
            if ".dist-info/" in rel or ".egg-info/" in rel or rel.endswith(".pyc"):
                continue
            # Console scripts can't be checked in a flat --target tree, and believing them would fail closed on a healthy sidecar; never imported.
            parts = tuple(p for p in rel.replace("\\", "/").split("/") if p)
            if (
                rel.startswith("/")
                or (len(rel) > 1 and rel[1] == ":")
                or ".." in parts
                or (parts and parts[0] in ("bin", "Scripts"))
            ):
                continue
            target = root / rel
            key = os.path.normcase(str(target))
            # Before the filter: a dropped row still owns the path it claims.
            owners[key] = owners.get(key, 0) + 1
            # Top-level dirs several wheels write into, so one uninstall deletes another's files; covers what we ship too. See _shared_non_runtime in _studio_deps.
            if len(parts) > 1 and parts[0] in _SHARED_NON_RUNTIME_ROOTS:
                continue
            # The size field is optional; keep the row so a deletion is still detected.
            recorded: int | None = None
            # Installers rewrite these in place, so only check existence.
            if len(row) >= 3 and row[2] and parts[-1] not in _INSTALLER_REWRITTEN_NAMES:
                try:
                    recorded = int(row[2])
                except ValueError:
                    recorded = None
            entries.append((name, rel, recorded, target, key))

    found: list[str] = []
    for name, rel, recorded, target, key in entries:
        try:
            info = target.stat()
        except (FileNotFoundError, NotADirectoryError):
            # Shared ownership cannot explain a missing file; NotADirectoryError is damage too.
            found.append(f"{name}: {rel} is missing")
        except OSError:
            # EIO/ESTALE/EACCES are inconclusive, not damage; skip the row but do not certify clean.
            inconclusive = True
            continue
        else:
            if not stat.S_ISREG(info.st_mode):
                # A directory in place of a module imports as something else; its st_size can pass the check.
                found.append(f"{name}: {rel} is not a regular file")
            elif owners[key] == 1 and recorded is not None and info.st_size < recorded:
                found.append(f"{name}: {rel} is {info.st_size} bytes, expected {recorded}")
            elif rel.endswith((".so", ".pyd")):
                # Python upgrades break tagged extensions; check the basename only, since a tagged parent dir must not trigger a wipe of hundreds of MB.
                base = rel.replace("\\", "/").rsplit("/", 1)[-1]
                m = _EXT_VERSION_TAG_RE.search(base)
                if m and m.group(1) != _CURRENT_EXT_TAG:
                    found.append(
                        f"{name}: {rel} targets cp{m.group(1)}, interpreter is cp{_CURRENT_EXT_TAG}"
                    )
                elif m is None and _CURRENT_EXT_TAG.endswith("t") and _ABI3_EXT_RE.search(base):
                    found.append(
                        f"{name}: {rel} is a stable-ABI build, which free-threaded "
                        f"cp{_CURRENT_EXT_TAG} cannot load"
                    )
        if len(found) >= limit:
            return found, inconclusive
    return found, inconclusive


def _venv_dir_is_valid(venv_dir: str, packages: tuple[str, ...]) -> bool:
    """Package-level check only; repair callers want _venv_dir_is_valid_and_undamaged instead."""
    if not os.path.isdir(venv_dir) or not os.listdir(venv_dir):
        return False
    for pkg_spec in packages:
        parts = pkg_spec.split("==")
        pkg_name = parts[0]
        pkg_version = parts[1] if len(parts) > 1 else None
        pkg_name_norm = pkg_name.replace("-", "_")
        if not any(
            (Path(venv_dir) / d).is_dir() for d in (pkg_name_norm, pkg_name_norm.replace("_", "-"))
        ):
            if _sidecar_package_is_optional(pkg_spec):
                continue
            return False
        # Unpinned packages: existence is enough.
        if pkg_version is None:
            continue
        dist_info_found = False
        for di in Path(venv_dir).glob(f"{pkg_name_norm}-*.dist-info"):
            metadata = di / "METADATA"
            if not metadata.is_file():
                continue
            for line in metadata.read_text(errors = "replace", encoding = "utf-8").splitlines():
                if line.startswith("Version:"):
                    installed_ver = line.split(":", 1)[1].strip()
                    if installed_ver != pkg_version:
                        logger.warning(
                            "%s has %s==%s but need %s -- venv will be wiped and reinstalled",
                            venv_dir,
                            pkg_name,
                            installed_ver,
                            pkg_version,
                        )
                        return False
                    dist_info_found = True
                    break
            if dist_info_found:
                break
        if not dist_info_found:
            return False
    return True


def _venv_dir_is_valid_and_undamaged(venv_dir: str, packages: tuple[str, ...]) -> bool:
    """Adds the RECORD-against-disk scan; only for callers that wipe and reinstall on a False result."""
    return _venv_dir_health(venv_dir, packages)[0]


def _venv_dir_health(venv_dir: str, packages: tuple[str, ...]) -> tuple[bool, bool]:
    """conclusive is False when a row was unreadable, so undamaged then means unproven, not healthy."""
    if not _venv_dir_is_valid(venv_dir, packages):
        return False, True
    damaged, inconclusive = _sidecar_scan(venv_dir)
    if damaged:
        logger.warning(
            "%s has damaged files -- venv will be wiped and reinstalled: %s",
            venv_dir,
            "; ".join(damaged),
        )
        return False, True
    return True, not inconclusive


def _venv_t5_is_valid() -> bool:
    """Backwards-compat: check the Gemma 4 sidecar venv."""
    return _venv_dir_is_valid(_VENV_T5_550_DIR, _VENV_T5_550_PACKAGES)


def _install_to_dir(pkg: str, target_dir: str) -> bool:
    """Install a single package into *target_dir*, preferring uv then pip."""
    pkg = _install_source(pkg)
    # Try uv first (faster) if on PATH -- do NOT install uv at runtime.
    if shutil.which("uv"):
        result = subprocess.run(
            [
                "uv",
                "pip",
                "install",
                "--python",
                sys.executable,
                "--target",
                target_dir,
                "--no-deps",
                "--upgrade",
                pkg,
            ],
            stdout = subprocess.PIPE,
            stderr = subprocess.STDOUT,
            text = True,
            encoding = "utf-8",
            errors = "replace",
            env = utf8_child_env(
                get_hf_cache_paths().child_env(child_env_without_native_path_secret())
            ),
            **_windows_hidden_subprocess_kwargs(),
        )
        if result.returncode == 0:
            return True
        logger.warning("uv install of %s failed", pkg)
    if _runtime_repair_is_offline() and not _pip_is_configured_offline():
        # pip has no offline mode except a local wheelhouse (PIP_NO_INDEX + PIP_FIND_LINKS).
        logger.warning(
            "%s not installed: the session is offline and pip would use the network", pkg
        )
        return False
    logger.warning("installing %s with pip", pkg)
    result = subprocess.run(
        [
            sys.executable,
            "-m",
            "pip",
            "install",
            "--target",
            target_dir,
            "--no-deps",
            "--upgrade",
            pkg,
        ],
        stdout = subprocess.PIPE,
        stderr = subprocess.STDOUT,
        text = True,
        encoding = "utf-8",
        errors = "replace",
        env = utf8_child_env(get_hf_cache_paths().child_env(child_env_without_native_path_secret())),
        **_windows_hidden_subprocess_kwargs(),
    )
    if result.returncode != 0:
        logger.error("install failed:\n%s", result.stdout)
        return False
    return True


# setup scripts refuse unmarked dirs, so a runtime rebuild must restore this marker.
_STUDIO_OWNED_MARKER = ".unsloth-studio-owned"


def _mark_studio_owned(venv_dir: str) -> None:
    try:
        Path(venv_dir, _STUDIO_OWNED_MARKER).touch()
    except OSError:
        # Best effort, as the shell does: a missing marker costs a clear error, never a wrong deletion.
        pass


# An unavailable wheel is asked for once per session.
_OPTIONAL_TOP_UP_ATTEMPTED: set[tuple[str, str]] = set()


def _optional_package_absent(venv_dir: str, pkg_spec: str) -> bool:
    """Present only once its dist-info has landed; a payload directory alone is an interrupted top-up."""
    name = pkg_spec.split("==")[0].replace("-", "_")
    root = Path(venv_dir)
    if not any((root / d / "__init__.py").is_file() for d in (name, name.replace("_", "-"))):
        return True
    return not any(
        (root / entry / "RECORD").is_file() for entry in _dist_info_entries(venv_dir, name)
    )


def _optional_package_entries(venv_dir: str, pkg_spec: str) -> list[str]:
    """Every top-level entry of the sidecar the optional package's wheel owns."""
    stem = pkg_spec.split("==")[0].lower().replace("-", "_")
    try:
        entries = os.listdir(venv_dir)
    except OSError:
        return []
    return [
        entry
        for entry in entries
        if entry.lower().replace("-", "_") == stem
        or entry.lower().replace("-", "_").startswith((stem + "_", stem + "."))
    ]


def _optional_package_partly_there(venv_dir: str, pkg_spec: str) -> bool:
    """Payload without dist-info, left by an interrupted top-up, sits ahead of site-packages."""
    return _optional_package_absent(venv_dir, pkg_spec) and bool(
        _optional_package_entries(venv_dir, pkg_spec)
    )


def _remove_optional_remnants(venv_dir: str, pkg_spec: str) -> bool:
    """Removes what a failed install left (payload and dist-info); False if a remnant stays, so the caller must treat the dir as unusable."""
    root = Path(venv_dir)

    def _owned() -> list[str]:
        return _optional_package_entries(venv_dir, pkg_spec)

    for entry in _owned():
        path = root / entry
        if path.is_dir() and not path.is_symlink():
            shutil.rmtree(path, ignore_errors = True)
        elif path.exists() or path.is_symlink():
            try:
                path.unlink()
            except OSError:
                pass
    left = _owned()
    if left:
        logger.warning(
            "%s could not be removed from %s after a failed install of %s; the directory is not usable",
            ", ".join(sorted(left)[:5]),
            venv_dir,
            pkg_spec,
        )
    return not left


def _remove_recordless_dist_infos(venv_dir: str, pkg_spec: str) -> None:
    """Drop every `<pkg>-*.dist-info` with no RECORD: metadata uv cannot uninstall."""
    name = pkg_spec.split("==")[0]
    root = Path(venv_dir)
    for entry in _dist_info_entries(venv_dir, name):
        if not (root / entry / "RECORD").is_file():
            shutil.rmtree(root / entry, ignore_errors = True)


def _dist_info_entries(venv_dir: str, name: str) -> list[str]:
    wanted = name.lower().replace("-", "_")
    try:
        entries = os.listdir(venv_dir)
    except OSError:
        return []
    return [
        entry
        for entry in entries
        if entry.endswith(".dist-info")
        and entry[: -len(".dist-info")].rsplit("-", 1)[0].lower().replace("-", "_") == wanted
    ]


# Recorded beside the sidecar so every worker sees them; retried later in case it was the network.
_OPTIONAL_TOP_UP_FAILED = ".optional-top-up-failed.json"
_OPTIONAL_TOP_UP_RETRY_SECONDS = 6 * 60 * 60.0


def _top_up_failure_path(venv_dir: str) -> str:
    return os.path.join(venv_dir, _OPTIONAL_TOP_UP_FAILED)


def _read_top_up_failures(venv_dir: str) -> dict:
    try:
        with open(_top_up_failure_path(venv_dir), encoding = "utf-8") as fh:
            data = json.load(fh)
    except (OSError, ValueError):
        return {}
    return data if isinstance(data, dict) else {}


def _write_top_up_failures(venv_dir: str, failures: dict) -> None:
    path = _top_up_failure_path(venv_dir)
    try:
        if not failures:
            if os.path.exists(path):
                os.unlink(path)
            return
        tmp = path + ".tmp"
        with open(tmp, "w", encoding = "utf-8") as fh:
            json.dump(failures, fh)
        os.replace(tmp, path)
    except OSError:
        pass


def _top_up_failed_recently(venv_dir: str, pkg: str) -> bool:
    when = _read_top_up_failures(venv_dir).get(pkg)
    if not isinstance(when, (int, float)):
        return False
    age = time.time() - when
    return 0 <= age < _OPTIONAL_TOP_UP_RETRY_SECONDS


def _record_top_up_outcome(venv_dir: str, pkg: str, ok: bool) -> None:
    failures = _read_top_up_failures(venv_dir)
    if ok:
        if pkg not in failures:
            return
        failures.pop(pkg, None)
    else:
        failures[pkg] = time.time()
    _write_top_up_failures(venv_dir, failures)


def _top_up_optional_packages(venv_dir: str, packages: tuple[str, ...]) -> bool:
    """Installs to a scratch dir and renames in, dist-info last, so no RECORD precedes its files."""
    usable = True
    # Offline installs can only miss and the miss is remembered, unless pip has a local wheelhouse.
    offline = _env_offline() or (_runtime_repair_is_offline() and not _pip_is_configured_offline())
    for pkg in packages:
        if not _sidecar_package_is_optional(pkg):
            continue
        if offline:
            # A payload without its dist-info counts as absent yet would shadow the ambient copy.
            if _optional_package_partly_there(venv_dir, pkg):
                with _optional_top_up_lock(venv_dir) as held:
                    if held and _optional_package_partly_there(venv_dir, pkg):
                        logger.warning(
                            "%s: removing the partial %s an interrupted add left", venv_dir, pkg
                        )
                        if not _remove_optional_remnants(venv_dir, pkg):
                            usable = False
                    elif not held:
                        usable = False
            continue
        # The staging cleanup never reaches a recordless dist-info, and metadata still answers it.
        _remove_recordless_dist_infos(venv_dir, pkg)
        if not _optional_package_absent(venv_dir, pkg):
            continue
        key = (os.path.normcase(os.path.abspath(venv_dir)), pkg)
        if key in _OPTIONAL_TOP_UP_ATTEMPTED:
            continue
        _OPTIONAL_TOP_UP_ATTEMPTED.add(key)
        with _optional_top_up_lock(venv_dir) as held:
            if not held:
                logger.warning(
                    "%s: another process held the top-up lock too long; left as is", venv_dir
                )
                if _optional_package_partly_there(venv_dir, pkg):
                    usable = False
                continue
            if not _optional_package_absent(venv_dir, pkg):
                continue
            if _top_up_failed_recently(venv_dir, pkg):
                logger.info(
                    "%s: a recent attempt to add %s failed; not retrying yet", venv_dir, pkg
                )
                continue
            logger.info("Adding %s to %s (optional package missing) ...", pkg, venv_dir)
            ok = _stage_optional_package(pkg, venv_dir)
            _record_top_up_outcome(venv_dir, pkg, ok)
            if not ok:
                logger.warning(
                    "%s could not be added to %s; continuing without it (Qwen tokenizers may fail)",
                    pkg,
                    venv_dir,
                )
                if _optional_package_partly_there(venv_dir, pkg):
                    usable = False
    return usable


def _stage_optional_package(pkg: str, venv_dir: str) -> bool:
    """Install *pkg* beside the sidecar, then move its entries in, dist-info last."""
    staging = os.path.join(venv_dir, ".top-up-staging")
    shutil.rmtree(staging, ignore_errors = True)
    try:
        os.makedirs(staging, exist_ok = True)
        if not _install_to_dir(pkg, staging):
            # A partial payload would shadow the ambient copy for the whole backoff.
            _remove_optional_remnants(venv_dir, pkg)
            return False
        entries = sorted(os.listdir(staging), key = lambda name: name.endswith(".dist-info"))
        # uv cannot uninstall a recordless dist-info and metadata answers whichever it meets first.
        for name in entries:
            if not name.endswith(".dist-info"):
                continue
            project = name[: -len(".dist-info")].rsplit("-", 1)[0]
            for stale in _dist_info_entries(venv_dir, project):
                if stale != name:
                    shutil.rmtree(os.path.join(venv_dir, stale), ignore_errors = True)
        for name in entries:
            source = os.path.join(staging, name)
            target = os.path.join(venv_dir, name)
            if os.path.isdir(target) and not os.path.islink(target):
                shutil.rmtree(target, ignore_errors = True)
            elif os.path.lexists(target):
                os.unlink(target)
            os.replace(source, target)
        return True
    except OSError as exc:
        logger.warning("staging %s into %s failed: %s", pkg, venv_dir, exc)
        _remove_optional_remnants(venv_dir, pkg)
        return False
    finally:
        shutil.rmtree(staging, ignore_errors = True)


_OPTIONAL_TOP_UP_LOCK = ".optional-top-up.lock"


_OPTIONAL_TOP_UP_WAIT_SECONDS = 120.0


# Errnos meaning a peer holds the lock; others mean the fs cannot lock at all.
_LOCK_CONTENDED_ERRNOS = frozenset(
    {errno.EACCES, errno.EAGAIN, errno.EWOULDBLOCK, errno.EINTR, errno.EDEADLK}
)


def _close_quietly(handle) -> None:
    """Drop the handle. Never raises: a close that fails must not fail an activation."""
    if handle is None:
        return
    try:
        handle.close()
    except OSError:
        pass


@contextlib.contextmanager
def _file_lock(path: str, wait_seconds: float):
    """A cross-process lock on *path*, waited for up to *wait_seconds*. Yields True when
    this process holds it, False when another kept it past the bound (or it cannot be
    taken at all, read as "someone else's turn" rather than a reason to write unguarded)."""
    handle = None
    try:
        handle = open(path, "a+b")
        deadline = time.monotonic() + wait_seconds
        while True:
            try:
                if sys.platform == "win32":
                    import msvcrt
                    handle.seek(0)
                    msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
                else:
                    import fcntl
                    fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
                break
            except OSError as exc:
                # Only contention is worth waiting out; a non-locking mount answers at once.
                if exc.errno not in _LOCK_CONTENDED_ERRNOS:
                    raise
                if time.monotonic() >= deadline:
                    raise
                time.sleep(0.25)
    except (OSError, ImportError):
        # ImportError too: no fcntl or msvcrt must not kill an activation.
        _close_quietly(handle)
        yield False
        return
    try:
        yield True
    finally:
        try:
            if sys.platform == "win32":
                import msvcrt
                handle.seek(0)
                msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
            else:
                import fcntl
                fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
        except (OSError, ImportError):
            pass
        _close_quietly(handle)


_REBUILD_LOCK_DIR = ".sidecar-locks"


def _rebuild_lock_path(venv_dir: str) -> str:
    """The tier's lock file, in a directory beside the sidecars rather than beside the
    tier itself: the sidecar scans (and the tests' sibling listings) key on the tier's
    name as a prefix, and a lock file has to stay once taken."""
    base = venv_dir.rstrip("/\\")
    parent, stem = os.path.split(base)
    lock_dir = os.path.join(parent or ".", _REBUILD_LOCK_DIR)
    try:
        os.makedirs(lock_dir, exist_ok = True)
    except OSError:
        pass
    return os.path.join(lock_dir, stem + ".lock")


# A worker that finds another mid-rebuild waits rather than building a second copy.
_REBUILD_WAIT_SECONDS = 15 * 60.0


@contextlib.contextmanager
def _optional_top_up_lock(venv_dir: str):
    """Serializes optional top-ups across processes; yields False if the lock is held past the bound or cannot be taken at all."""
    with _file_lock(
        os.path.join(venv_dir, _OPTIONAL_TOP_UP_LOCK), _OPTIONAL_TOP_UP_WAIT_SECONDS
    ) as held:
        yield held


_UV_OFFLINE_TRUE_VALUES = _OFFLINE_TRUE_VALUES | {"t", "y"}


_PIP_TRUE_VALUES = frozenset({"1", "true", "t", "yes", "y", "on"})

# pip's precedence: the environment (":env:" in `pip config list`) over [install] over [global].
_PIP_SETTING_SCOPES = (":env:", "install", "global")


def _pip_effective_settings() -> dict[str, str] | None:
    """Pip's config via `pip config list` (config files and PIP_* env); None if pip cannot answer."""
    try:
        result = subprocess.run(
            [sys.executable, "-m", "pip", "config", "list"],
            stdout = subprocess.PIPE,
            stderr = subprocess.DEVNULL,
            text = True,
            encoding = "utf-8",
            errors = "replace",
            timeout = 60,
            env = utf8_child_env(
                get_hf_cache_paths().child_env(child_env_without_native_path_secret())
            ),
            **_windows_hidden_subprocess_kwargs(),
        )
    except (OSError, subprocess.SubprocessError):
        return None
    if result.returncode != 0:
        return None
    settings: dict[str, str] = {}
    for line in result.stdout.splitlines():
        key, sep, raw = line.partition("=")
        if not sep:
            continue
        raw = raw.strip()
        try:
            value = ast.literal_eval(raw)
        except (ValueError, SyntaxError):
            value = raw.strip("'\"")
        settings[key.strip()] = str(value)
    return settings


def _pip_setting(settings: dict[str, str], name: str) -> str | None:
    for scope in _PIP_SETTING_SCOPES:
        value = settings.get(f"{scope}.{name}")
        if value is not None:
            return value
    return None


def _is_local_wheelhouse_dir(entry: str) -> bool:
    """A --find-links entry that can only hand pip local files: a directory, as a path
    or a file:// URL. A URL is the network; a FILE (an HTML index) is parsed for links
    and those may point at the network as well."""
    lowered = entry.lower()
    if lowered.startswith("file://"):
        from urllib.parse import urlparse
        from urllib.request import url2pathname

        parsed = urlparse(entry)
        path = parsed.path
        # file://server/share is a UNC share: the host is part of the path.
        if parsed.netloc and parsed.netloc.lower() != "localhost":
            path = "//" + parsed.netloc + path
        entry = url2pathname(path)
    elif "://" in lowered:
        return False
    return os.path.isdir(os.path.expanduser(entry))


def _pip_is_configured_offline() -> bool:
    """Only a local wheelhouse counts (--no-index, local --find-links): a URL may reach the network."""
    settings = _pip_effective_settings()
    if settings is None:
        settings = {}
        for name, variable in (("no-index", "PIP_NO_INDEX"), ("find-links", "PIP_FIND_LINKS")):
            if variable in os.environ:
                settings[f":env:.{name}"] = os.environ[variable]
    # pip's own boolean spellings (strtobool): 1/true/t/yes/y/on.
    no_index = (_pip_setting(settings, "no-index") or "").strip().lower() in _PIP_TRUE_VALUES
    if not no_index:
        return False
    entries = (_pip_setting(settings, "find-links") or "").split()
    if not entries:
        return False
    return all(_is_local_wheelhouse_dir(entry) for entry in entries)


def _runtime_repair_is_offline() -> bool:
    """Only UV_OFFLINE counts, not the HF offline switches; under it a repair would wipe a tree it cannot rebuild."""
    return os.environ.get("UV_OFFLINE", "").strip().lower() in _UV_OFFLINE_TRUE_VALUES


def _sidecar_has_content(venv_dir: str) -> bool:
    """True if anything besides the ownership marker is present; a marker-only dir is a first install."""
    try:
        return any(name != _STUDIO_OWNED_MARKER for name in os.listdir(venv_dir))
    except (FileNotFoundError, NotADirectoryError):
        return False
    except OSError:
        # Unreadable is not empty: the caller's next move on "empty" is to delete the tree.
        return True


_OFFLINE_STAGING_SUFFIX = ".offline-staging-"
_OFFLINE_RETIRED_SUFFIX = ".offline-old-"


def _sidecar_siblings(venv_dir: str, suffix: str) -> list[str]:
    """`<venv_dir><suffix>*` beside the sidecar, oldest first by modification time."""
    base = venv_dir.rstrip("/\\")
    parent, stem = os.path.split(base)
    try:
        names = os.listdir(parent or ".")
    except OSError:
        return []
    # Only `<stem><suffix><pid>` with our marker: callers delete or rename what this returns.
    found = [
        os.path.join(parent, n)
        for n in names
        if n.startswith(stem + suffix)
        and n[len(stem + suffix) :].isdigit()
        and os.path.isfile(os.path.join(parent, n, _STUDIO_OWNED_MARKER))
    ]

    def modified(path: str) -> float:
        # Another worker can remove a sibling between the listing and this read.
        try:
            return os.path.getmtime(path)
        except OSError:
            return 0.0

    return sorted(found, key = modified)


def _recover_retired_sidecar(venv_dir: str) -> None:
    """Restores a retired tree if the live path is empty; otherwise retired copies are removed."""
    retired = _sidecar_siblings(venv_dir, _OFFLINE_RETIRED_SUFFIX)
    if not retired:
        return
    if os.path.isdir(venv_dir) and _sidecar_has_content(venv_dir):
        for old in retired:
            shutil.rmtree(old, ignore_errors = True)
        return
    newest = retired[-1]
    shutil.rmtree(venv_dir, ignore_errors = True)
    try:
        os.rename(newest, venv_dir)
        logger.warning("restored %s from %s (an earlier swap was interrupted)", venv_dir, newest)
    except OSError as exc:
        logger.warning("could not restore %s from %s: %s", venv_dir, newest, exc)
        return
    for old in retired[:-1]:
        shutil.rmtree(old, ignore_errors = True)


def _ensure_venv_dir(venv_dir: str, packages: tuple[str, ...], label: str) -> bool:
    """Ensure *venv_dir* exists with all *packages*. Install if missing."""
    if _venv_dir_is_valid_and_undamaged(venv_dir, packages):
        # A live tree with content makes retired copies leftovers to sweep.
        _recover_retired_sidecar(venv_dir)
        return _top_up_optional_packages(venv_dir, packages)

    # One repair per tier across processes, else the loser can delete the winner's tree.
    with _file_lock(_rebuild_lock_path(venv_dir), _REBUILD_WAIT_SECONDS) as held:
        if not held:
            # Another process is still building it: deferred, this activation goes without.
            logger.warning(
                "%s: the rebuild lock was not obtained; leaving the repair to the process holding it",
                venv_dir,
            )
            return False
        _recover_retired_sidecar(venv_dir)
        if _venv_dir_is_valid_and_undamaged(venv_dir, packages):
            logger.info("%s at %s was completed by another process", label, venv_dir)
            return _top_up_optional_packages(venv_dir, packages)
        # Offline, never repair in place (it deletes a serving tree); build beside it from the uv cache only, not pip, then swap.
        if _runtime_repair_is_offline() and _sidecar_has_content(venv_dir):
            return _repair_offline_beside(venv_dir, packages, label)
        return _rebuild_venv_dir(venv_dir, packages, label)


def _rebuild_venv_dir(venv_dir: str, packages: tuple[str, ...], label: str) -> bool:
    """Wipe *venv_dir* and install every package into it; the caller holds the tier's lock."""
    logger.warning("%s not found or incomplete at %s -- installing at runtime", label, venv_dir)
    # Docker links sidecars into the app tree and rmtree refuses symlinks; repair the target if ours.
    if os.path.islink(venv_dir) and resolves_into_studio_app_tree(Path(venv_dir)):
        venv_dir = os.path.realpath(venv_dir)
    shutil.rmtree(venv_dir, ignore_errors = True)
    os.makedirs(venv_dir, exist_ok = True)
    _mark_studio_owned(venv_dir)
    total = len(packages)
    for idx, pkg in enumerate(packages, start = 1):
        logger.info("Installing %s (%d/%d) into %s ...", pkg, idx, total, venv_dir)
        if not _install_to_dir(pkg, venv_dir):
            if _sidecar_package_is_optional(pkg):
                logger.warning(
                    "%s could not be installed into %s; continuing without it (Qwen tokenizers may fail)",
                    pkg,
                    venv_dir,
                )
                # Only absent is safe: a half-copied payload fails at tokenization.
                if _remove_optional_remnants(venv_dir, pkg):
                    # Recorded so later activations and workers skip the same doomed install.
                    _record_top_up_outcome(venv_dir, pkg, False)
                    _OPTIONAL_TOP_UP_ATTEMPTED.add(
                        (os.path.normcase(os.path.abspath(venv_dir)), pkg)
                    )
                    continue
            # A partial tree would count as usable later, unless another process rebuilt it meanwhile.
            if _venv_dir_is_valid_and_undamaged(venv_dir, packages):
                logger.info("%s at %s was completed by another process", label, venv_dir)
                return True
            shutil.rmtree(venv_dir, ignore_errors = True)
            return False
    logger.info("Installed %s to %s", label, venv_dir)
    return True


def _drop_offline_staging(staging: str) -> None:
    """The staging tree and the per-process rebuild lock _ensure_venv_dir took for it."""
    shutil.rmtree(staging, ignore_errors = True)
    try:
        os.unlink(_rebuild_lock_path(staging))
    except OSError:
        pass


def _repair_offline_beside(venv_dir: str, packages: tuple[str, ...], label: str) -> bool:
    """Rebuild *venv_dir* from uv's cache into a staging directory beside it and swap
    only once every package landed; a cold cache leaves the tree exactly as it was."""
    base = venv_dir.rstrip("/\\")
    # Per process: a shared staging path would have one worker deleting the other's build.
    staging = f"{base}{_OFFLINE_STAGING_SUFFIX}{os.getpid()}"
    retired = f"{base}{_OFFLINE_RETIRED_SUFFIX}{os.getpid()}"
    _drop_offline_staging(staging)
    # Staging trees of processes long gone (a kill mid-build); an hour is beyond any build here.
    for stale in _sidecar_siblings(venv_dir, _OFFLINE_STAGING_SUFFIX):
        try:
            if stale != staging and time.time() - os.path.getmtime(stale) > 3600:
                shutil.rmtree(stale, ignore_errors = True)
        except OSError:
            pass
    # An empty directory takes the ordinary path; a failure removes the staging tree.
    if not _ensure_venv_dir(staging, packages, label):
        _drop_offline_staging(staging)
        logger.warning(
            "%s not found or incomplete at %s, and this session is offline with no cached "
            "copy to rebuild it from -- left as is until the next online update",
            label,
            venv_dir,
        )
        return False
    # Only a tree passing the activation's own predicate replaces the live one.
    if not _venv_dir_is_valid_and_undamaged(staging, packages):
        _drop_offline_staging(staging)
        logger.warning("the offline rebuild of %s did not validate; %s left as is", label, venv_dir)
        return False
    try:
        shutil.rmtree(retired, ignore_errors = True)
        os.rename(venv_dir, retired)
        try:
            os.rename(staging, venv_dir)
        except OSError:
            if not os.path.isdir(venv_dir) and os.path.isdir(retired):
                os.rename(retired, venv_dir)
            raise
    except OSError as exc:
        logger.warning("could not swap the offline rebuild of %s into %s: %s", label, venv_dir, exc)
        _drop_offline_staging(staging)
        return False
    shutil.rmtree(retired, ignore_errors = True)
    # The staging tree is live now; the lock taken for its build goes with the staging name.
    try:
        os.unlink(_rebuild_lock_path(staging))
    except OSError:
        pass
    logger.info("Rebuilt %s at %s offline, from the cache", label, venv_dir)
    return True


def _ensure_venv_t5_530_exists() -> bool:
    """Ensure .venv_t5_530/ exists with transformers 5.3.0."""
    return _ensure_venv_dir(_VENV_T5_530_DIR, _VENV_T5_530_PACKAGES, "transformers 5.3.0")


def _ensure_venv_t5_550_exists() -> bool:
    """Ensure .venv_t5_550/ exists with transformers 5.5.0."""
    return _ensure_venv_dir(
        _VENV_T5_550_DIR,
        _VENV_T5_550_PACKAGES,
        f"transformers {TRANSFORMERS_550_VERSION}",
    )


def _ensure_venv_t5_510_exists() -> bool:
    """Ensure .venv_t5_510/ exists with transformers 5.10.x."""
    return _ensure_venv_dir(
        _VENV_T5_510_DIR,
        _VENV_T5_510_PACKAGES,
        f"transformers {TRANSFORMERS_510_VERSION}",
    )


def _ensure_venv_t4_457_exists() -> bool:
    return _ensure_venv_dir(
        _VENV_T4_457_DIR,
        _VENV_T4_457_PACKAGES,
        f"transformers {TRANSFORMERS_457_VERSION}",
    )


def _ensure_venv_t5_exists() -> bool:
    """Backwards-compat: ensure the Gemma 4 5.5 sidecar venv exists."""
    return _ensure_venv_t5_550_exists()


# PEP 440-ish release strings only (guards the pip install spec against injection).
_LATEST_VERSION_RE = r"[0-9]+(\.[0-9]+)*((a|b|rc)[0-9]+)?(\.post[0-9]+)?(\.dev[0-9]+)?"


def _is_valid_version_string(version: str) -> bool:
    import re
    return isinstance(version, str) and re.fullmatch(_LATEST_VERSION_RE, version) is not None


# Only plain recipe package specs may come from the on-disk pin; anything else is rebuilt.
_PIN_SPEC_RE = re.compile(r"^[A-Za-z0-9_.-]+(==[A-Za-z0-9_.+-]+)?$")
_PIN_ALLOWED_NAMES = frozenset(
    {
        "transformers",
        "huggingface_hub",
        "huggingface-hub",
        "hf_xet",
        "hf-xet",
        "tiktoken",
        "tokenizers",
        "safetensors",
    }
)


def _is_safe_pin_spec(spec: str) -> bool:
    if not _PIN_SPEC_RE.match(spec):
        return False
    name = spec.split("==", 1)[0].lower().replace("_", "-")
    return name in {n.replace("_", "-") for n in _PIN_ALLOWED_NAMES}


def _recover_stranded_latest_sidecar() -> None:
    """Restores a stranded .old sidecar only when no live dir exists and no swap is in flight."""
    live = Path(_VENV_T5_LATEST_DIR)
    retired = Path(_VENV_T5_LATEST_DIR + ".old")
    try:
        if live.exists() or not retired.is_dir() or sidecar_swap_in_progress():
            return
        os.rename(retired, live)
        logger.info("Recovered .venv_t5_latest from a stranded .old after a failed swap")
    except OSError:
        pass


def _latest_pin_data() -> dict | None:
    """Accepts a plain version string from older writers, expanding it with the default package set."""
    _recover_stranded_latest_sidecar()
    marker = Path(_VENV_T5_LATEST_DIR) / _LATEST_PIN_MARKER
    try:
        if not marker.is_file():
            return None
        raw = marker.read_text(encoding = "utf-8").strip()
    except Exception:
        return None
    try:
        data = json.loads(raw)
    except ValueError:
        data = raw
    if isinstance(data, str):
        if not _is_valid_version_string(data):
            return None
        return {"version": data, "packages": list(_venv_t5_latest_packages(data))}
    if not isinstance(data, dict):
        return None
    version = data.get("version")
    if not _is_valid_version_string(version):
        return None
    packages = data.get("packages")
    if not (
        isinstance(packages, list)
        and packages
        and all(isinstance(p, str) and _is_safe_pin_spec(p) for p in packages)
    ):
        # The pin is user-writable on disk: malformed specs never reach pip, rebuild instead.
        packages = list(_venv_t5_latest_packages(version))
    return {"version": version, "packages": packages}


def latest_venv_pinned_version() -> str | None:
    """Exact transformers version pinned in .venv_t5_latest's marker, or None if the
    sidecar was never provisioned (or the marker is unreadable/invalid)."""
    data = _latest_pin_data()
    return data["version"] if data else None


def _venv_t5_latest_packages(version: str, extra_packages: tuple[str, ...] = ()) -> tuple[str, ...]:
    """Package set for the latest sidecar; mirrors the fixed .venv_t5_* sidecars.
    *extra_packages* carries dep-compat shadows (e.g. a newer tokenizers) computed by
    utils.transformers_latest before install."""
    base = (
        f"transformers=={version}",
        "huggingface_hub==1.8.0",
        "hf_xet==1.4.2",
        "tiktoken",
    )
    overridden = {_pin_spec_name(p) for p in extra_packages}
    return tuple(p for p in base if _pin_spec_name(p) not in overridden) + tuple(extra_packages)


def _pin_spec_name(spec: str) -> str:
    # PEP 503 name normalization, so huggingface_hub and huggingface-hub collide.
    return re.sub(r"[-_.]+", "-", re.split(r"[<>=!~ @;\[]", spec, maxsplit = 1)[0]).lower()


# PyPI never hosts a transformers .devN build, so a dev pin means main (the zip needs no git).
_TRANSFORMERS_MAIN_ARCHIVE = (
    "transformers @ https://github.com/huggingface/transformers/archive/{ref}.zip"
)
# The main commit the consented install checked; installs are serialized by the swap reservation.
_main_archive_commit: str | None = None


@contextlib.contextmanager
def transformers_main_at(commit: str):
    """Install transformers main from *commit* (not the moving branch) inside this block."""
    global _main_archive_commit
    previous, _main_archive_commit = _main_archive_commit, commit
    try:
        yield
    finally:
        _main_archive_commit = previous


def _install_source(pkg: str) -> str:
    if re.fullmatch(r"transformers==[0-9.]+\.dev[0-9]+", pkg):
        return _TRANSFORMERS_MAIN_ARCHIVE.format(ref = _main_archive_commit or "refs/heads/main")
    return pkg


# One reservation for any .venv_t5_latest swap, checked by training and export starts so no worker spawns mid-swap; a lock file keeps worker repairs visible.
_sidecar_swap_lock = threading.Lock()
_sidecar_swap_active = False
_sidecar_swap_token: str | None = None
_sidecar_swap_kind: str | None = None
# An install is minutes; a lock this old is a crashed owner, not a live swap.
_SWAP_LOCK_STALE_SECS = 2 * 60 * 60


def _swap_lock_path() -> Path:
    return Path(_VENV_T5_LATEST_DIR + ".swaplock")


def _pid_alive(pid) -> bool:
    if not isinstance(pid, int) or pid <= 0:
        return False
    try:
        import psutil
        return psutil.pid_exists(pid)
    except Exception:
        pass
    if os.name == "nt":
        # os.kill(pid, 0) on Windows sends CTRL_C_EVENT; probe via OpenProcess instead.
        try:
            import ctypes
            from ctypes import wintypes

            kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
            kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
            kernel32.OpenProcess.restype = wintypes.HANDLE
            # PROCESS_QUERY_LIMITED_INFORMATION: minimal right, granted across integrity levels.
            handle = kernel32.OpenProcess(0x1000, False, pid)
            if handle:
                kernel32.CloseHandle(handle)
                return True
            # ERROR_ACCESS_DENIED means the process exists but we may not query it.
            return ctypes.get_last_error() == 5
        except Exception:
            return False
    try:
        os.kill(pid, 0)
        return True
    except OSError:
        return False
    except Exception:
        return False


def _swap_lock_is_stale(path: Path) -> bool:
    """Stale only when the recorded PID is provably dead; an unreadable PID falls back to the age cutoff."""
    try:
        age = time.time() - path.stat().st_mtime
    except OSError:
        return False
    data = _read_swap_lock(path) or {}
    pid = data.get("pid")
    if not isinstance(pid, int) or pid <= 0:
        return age > _SWAP_LOCK_STALE_SECS
    return not _pid_alive(pid)


class SidecarSwapInProgress(RuntimeError):
    """A worker start lost the race to a .venv_t5_latest install/repair; retryable."""


def _read_swap_lock(path: Path) -> dict | None:
    try:
        data = json.loads(path.read_text(encoding = "utf-8-sig"))
        return data if isinstance(data, dict) else {}
    except FileNotFoundError:
        return None
    except OSError:
        return {}
    except Exception:
        return {}


def try_begin_sidecar_swap(kind: str = "install") -> bool:
    """Reserve the sidecar swap window; False if one is held here or, via the lock file, by any
    worker subprocess. kind is install (consented) or repair (lazy venv repair)."""
    global _sidecar_swap_active, _sidecar_swap_token, _sidecar_swap_kind
    with _sidecar_swap_lock:
        if _sidecar_swap_active:
            return False
        token = f"{os.getpid()}-{time.time_ns()}"
        path = _swap_lock_path()
        try:
            path.parent.mkdir(parents = True, exist_ok = True)
        except OSError:
            pass
        for attempt in range(2):
            try:
                fd = os.open(str(path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
                break
            except FileExistsError:
                if attempt or not _swap_lock_is_stale(path):
                    return False
                try:
                    path.unlink()
                except OSError:
                    return False
            except OSError:
                # Lock file not creatable (odd filesystem): fall back to the process-local reservation.
                fd = None
                break
        if fd is not None:
            try:
                with os.fdopen(fd, "w", encoding = "utf-8") as f:
                    f.write(
                        json.dumps(
                            {"pid": os.getpid(), "at": time.time(), "token": token, "kind": kind}
                        )
                    )
            except OSError:
                pass
        _sidecar_swap_active = True
        _sidecar_swap_token = token
        _sidecar_swap_kind = kind
        return True


def end_sidecar_swap() -> None:
    """Release the reservation taken by :func:`try_begin_sidecar_swap`."""
    global _sidecar_swap_active, _sidecar_swap_token, _sidecar_swap_kind
    with _sidecar_swap_lock:
        if _sidecar_swap_active:
            # Only remove our own file: a superseded stale reservation must not drop the new owner's lock.
            path = _swap_lock_path()
            data = _read_swap_lock(path)
            if data is not None and data.get("token", _sidecar_swap_token) == _sidecar_swap_token:
                try:
                    path.unlink()
                except OSError:
                    pass
        _sidecar_swap_active = False
        _sidecar_swap_token = None
        _sidecar_swap_kind = None


def sidecar_swap_in_progress() -> bool:
    """True while a .venv_t5_latest install or repair holds the reservation,
    in this process or any other Unsloth process (lock file)."""
    return sidecar_swap_kind() is not None


def sidecar_swap_kind() -> str | None:
    """The active reservation's kind ("install" / "repair"), or None when idle.
    Lets guards that rely on the install route's own abort-on-active-worker
    checks keep refusing for repairs, which have no such checks."""
    with _sidecar_swap_lock:
        if _sidecar_swap_active:
            return _sidecar_swap_kind or "install"
    path = _swap_lock_path()
    try:
        if not path.is_file() or _swap_lock_is_stale(path):
            return None
    except OSError:
        return None
    data = _read_swap_lock(path) or {}
    kind = data.get("kind")
    return kind if kind in ("install", "repair") else "install"


def _stage_and_swap_latest_venv(
    version: str,
    packages: tuple[str, ...],
    before_swap = None,
) -> bool:
    """Builds the new sidecar beside the live one and swaps only when complete; before_swap runs just before the swap, and if it raises the old one stays."""
    staging = _VENV_T5_LATEST_DIR + ".staging"
    retired = _VENV_T5_LATEST_DIR + ".old"
    shutil.rmtree(staging, ignore_errors = True)
    try:
        if not _ensure_venv_dir(staging, packages, f"transformers {version} (latest)"):
            # No exception, so the except cleanup below never runs; drop the partial dir.
            shutil.rmtree(staging, ignore_errors = True)
            return False
        (Path(staging) / _LATEST_PIN_MARKER).write_text(
            json.dumps({"version": version, "packages": list(packages)}), encoding = "utf-8"
        )
        if before_swap is not None:
            before_swap()
        shutil.rmtree(retired, ignore_errors = True)
        if os.path.isdir(_VENV_T5_LATEST_DIR):
            os.rename(_VENV_T5_LATEST_DIR, retired)
        try:
            os.rename(staging, _VENV_T5_LATEST_DIR)
        except OSError:
            # Restore the previous sidecar if the final swap fails.
            if not os.path.isdir(_VENV_T5_LATEST_DIR) and os.path.isdir(retired):
                os.rename(retired, _VENV_T5_LATEST_DIR)
            raise
    except Exception as exc:
        logger.error("Could not provision transformers %s into .venv_t5_latest: %s", version, exc)
        shutil.rmtree(staging, ignore_errors = True)
        return False
    shutil.rmtree(retired, ignore_errors = True)
    # CONFIG_MAPPING_NAMES may have changed: drop the cached key set.
    _config_mapping_cache.pop("latest", None)
    logger.info("Provisioned .venv_t5_latest with transformers %s", version)
    return True


def _workers_active_for_repair() -> bool:
    """Best-effort: any parent-visible chat/training/export worker alive. Never
    raises; unavailable backends (worker subprocess, early startup) count idle."""
    try:
        from core.training import get_training_backend
        if get_training_backend().is_training_active():
            return True
    except Exception:
        pass
    try:
        from core.export import get_export_backend

        _export = get_export_backend()
        if _export.is_export_active():
            return True
        _alive = getattr(_export, "is_worker_alive", None)
        if callable(_alive) and _alive():
            return True
    except Exception:
        pass
    try:
        from core.inference import get_inference_backend

        backend = get_inference_backend()
        if getattr(backend, "active_model_name", None):
            return True
        # An in-flight load counts too: its worker spawns moments later.
        if getattr(backend, "loading_models", None):
            return True
        _alive = getattr(backend, "is_worker_alive", None)
        if callable(_alive) and _alive():
            return True
    except Exception:
        pass
    return False


def _ensure_venv_t5_latest_exists() -> bool:
    """Never installs without a pin: no marker returns False, leaving routing and probing unchanged."""
    pin = _latest_pin_data()
    if pin is None:
        return False
    version = pin["version"]
    packages = tuple(pin["packages"])
    undamaged, conclusive = _venv_dir_health(_VENV_T5_LATEST_DIR, packages)
    if undamaged and (conclusive or not _latest_repair_requested()):
        # A clean conclusive scan retires a stale repair request, else self-heal loops forever.
        if conclusive:
            _clear_latest_repair_request()
        # Activation is where a missing tiktoken gets topped up.
        return _top_up_optional_packages(_VENV_T5_LATEST_DIR, packages)
    # Flag before any bailout so routing withholds the damaged sidecar; a repair clears it.
    _request_latest_repair()
    if ".dev" in version:
        # A rebuild would install main code nobody consented to; broken -> the dialog offers it again.
        logger.warning(
            ".venv_t5_latest (transformers %s from main) is incomplete; load the model again "
            "to reinstall it.",
            version,
        )
        return False
    if _env_offline():
        logger.warning(
            ".venv_t5_latest (transformers %s) is incomplete and offline mode is set; "
            "cannot repair it.",
            version,
        )
        return False
    # Repairs are parent-only: a worker child cannot see live siblings using the sidecar.
    try:
        import multiprocessing as _mp
        if _mp.parent_process() is not None:
            logger.warning(
                ".venv_t5_latest is incomplete; repairs run in the parent process. "
                "Retry after the parent repairs the sidecar."
            )
            return False
    except Exception:
        pass
    # Same stage-and-swap and reservation as install; training/export starts (sidecar_swap_in_progress) wait; a failed repair keeps the pin.
    if not try_begin_sidecar_swap(kind = "repair"):
        logger.warning(
            "Cannot repair .venv_t5_latest: another sidecar install or repair is in progress."
        )
        return False
    try:
        # Check workers under the reservation: worker starts set their markers BEFORE rechecking it, so one side always sees the other.
        if _workers_active_for_repair():
            logger.warning(
                "Cannot repair .venv_t5_latest: active chat/training/export workers "
                "may be importing from it. Retry when they are idle."
            )
            return False
        return _stage_and_swap_latest_venv(version, packages)
    finally:
        end_sidecar_swap()


def ensure_latest_transformers_venv(
    version: str,
    extra_packages: tuple[str, ...] = (),
    before_swap = None,
) -> bool:
    """Opt-in install; the pin marker is written after the --target install so it survives restarts."""
    if not _is_valid_version_string(version):
        logger.error("Refusing to install invalid transformers version %r", version)
        return False
    if _env_offline():
        logger.warning(
            "Cannot install transformers %s: HF/transformers offline mode is set.", version
        )
        return False
    packages = _venv_t5_latest_packages(version, extra_packages)
    pin = _latest_pin_data()
    if (
        pin is not None
        # A consented main install always rebuilds: main gains architectures without a .devN bump.
        and _main_archive_commit is None
        and pin["version"] == version
        and tuple(pin["packages"]) == packages
        and _venv_dir_is_valid_and_undamaged(_VENV_T5_LATEST_DIR, packages)
    ):
        return _top_up_optional_packages(_VENV_T5_LATEST_DIR, packages)
    return _stage_and_swap_latest_venv(version, packages, before_swap = before_swap)


# Exact pins, bump deliberately; validated to FP8-quantize Qwen3.5 / Gemma-4 / Llama.
_LLMC_MAIN_TRANSFORMERS = "5.10.2"
_LLMC_MAIN_SHA = "973c9c539a84dd9efaf74e115ede5ca419704c18"
_LLMC_MAIN_COMPRESSED_TENSORS = "0.17.2a20260702"
# Content digest of GitHub's source archive of _LLMC_MAIN_SHA (_archive_content_digest), and
# `git describe` of that commit. GitHub keeps an archive's contents stable, not its bytes.
_LLMC_MAIN_ARCHIVE_DIGEST = "63ccb4f99833e3b24602f18a1e386e6c1127c1a84f7fc4e8ff56336c929138f8"
_LLMC_MAIN_DESCRIBE = "0.12.0-40"
# --no-deps keeps torch untouched; pins are the full runtime llm-compressor main needs.
_VENV_LLMCOMPRESSOR_SPECS = (
    f"transformers=={_LLMC_MAIN_TRANSFORMERS}",
    f"compressed-tensors=={_LLMC_MAIN_COMPRESSED_TENSORS}",
    "huggingface-hub==1.21.0",
    "hf-xet==1.5.1",
    "tokenizers==0.22.2",
    "safetensors==0.8.0",
    "accelerate==1.14.0",
    "datasets==5.0.0",
    "pydantic==2.13.4",
    "pydantic-core==2.46.4",
    "typing-inspection==0.4.2",
    "loguru==0.7.3",
    "pyyaml==6.0.3",
    "nvidia-ml-py==13.610.43",
    "pillow==12.3.0",
    "auto-round==0.13.1",
    "regex==2026.6.28",
)
# Fingerprint of the pin set; bump the trailing schema version to force a rebuild on layout changes.
_LLMC_SHADOW_FINGERPRINT = (
    f"{_LLMC_MAIN_SHA}|{_LLMC_MAIN_TRANSFORMERS}|{_LLMC_MAIN_COMPRESSED_TENSORS}|schema=1"
)
_LLMC_SHADOW_MARKER = ".unsloth_llmc_fingerprint"


def _llmcompressor_main_disabled() -> bool:
    """True if the operator forbids the llm-compressor-main shadow (air-gapped / locked-down)."""
    return os.environ.get("UNSLOTH_DISABLE_LLMCOMPRESSOR_MAIN", "").strip().lower() in {
        "1",
        "true",
        "yes",
        "on",
    }


def _llmcompressor_shadow_is_valid() -> bool:
    """True if the shadow dir exists with a marker matching the current pin fingerprint."""
    marker = Path(_VENV_LLMCOMPRESSOR_DIR) / _LLMC_SHADOW_MARKER
    try:
        return (
            marker.is_file()
            and marker.read_text(encoding = "utf-8").strip() == _LLMC_SHADOW_FINGERPRINT
        )
    except Exception:
        return False


def _ensure_venv_llmcompressor_exists() -> bool:
    """Installs with --no-deps into a --target dir so the workspace torch is never touched."""
    if _llmcompressor_shadow_is_valid():
        return True
    if _llmcompressor_main_disabled():
        logger.warning(
            "llm-compressor-main shadow needed but UNSLOTH_DISABLE_LLMCOMPRESSOR_MAIN is set; "
            "compressed export of newer-transformers models will fail fast."
        )
        return False
    if _env_offline():
        logger.warning(
            "llm-compressor-main shadow missing and HF/offline mode is set; cannot provision it."
        )
        return False

    logger.warning(
        "Provisioning llm-compressor-main shadow at %s (one-time, ~a few hundred MB, no torch) ...",
        _VENV_LLMCOMPRESSOR_DIR,
    )
    shutil.rmtree(_VENV_LLMCOMPRESSOR_DIR, ignore_errors = True)
    os.makedirs(_VENV_LLMCOMPRESSOR_DIR, exist_ok = True)
    with tempfile.TemporaryDirectory(
        dir = os.path.dirname(_VENV_LLMCOMPRESSOR_DIR), prefix = ".llmc-src-"
    ) as tmp:
        try:
            source = _download_llmcompressor_source(Path(tmp))
        except Exception as exc:  # network, checksum, bad archive
            logger.error("Cannot provision llm-compressor-main shadow: %s", exc)
            return False
        return _install_llmcompressor_shadow(source)


def _download_llmcompressor_source(parent: Path) -> Path:
    """Writes .git_archival.txt for the pinned commit (sha256-checked tarball, no git needed); without it archive builds silently drop files and version."""
    from utils.llama_cpp_source import safe_extract_tar

    url = f"https://github.com/vllm-project/llm-compressor/archive/{_LLMC_MAIN_SHA}.tar.gz"
    archive = parent / "llm-compressor.tar.gz"
    headers = {"User-Agent": "unsloth-studio"}
    opener = _hf_proxy_opener(url)  # honours ALL_PROXY, which plain urlopen ignores for https
    from utils.utils import hf_proxy_for_endpoint, hf_proxy_usable_by_urllib

    proxy = hf_proxy_for_endpoint(url)
    if opener is None and not hf_proxy_usable_by_urllib(proxy):
        # A socks proxy urllib cannot speak: go through it as the Hub client does, never direct.
        import httpx
        try:
            with httpx.stream("GET", url, headers = headers, follow_redirects = True, timeout = 300) as r:
                r.raise_for_status()
                with open(archive, "wb") as out:
                    for chunk in r.iter_bytes(1 << 20):
                        out.write(chunk)
        except ImportError as exc:  # httpx needs socksio for socks
            raise RuntimeError(
                f"the configured proxy {proxy} needs SOCKS support: install socksio "
                "(pip install socksio) or set HTTPS_PROXY to an http(s) proxy"
            ) from exc
    else:
        request = urllib.request.Request(url, headers = headers)
        with (opener.open if opener else urllib.request.urlopen)(request, timeout = 300) as response:
            with open(archive, "wb") as out:
                shutil.copyfileobj(response, out, 1 << 20)
    if _archive_content_digest(archive) != _LLMC_MAIN_ARCHIVE_DIGEST:
        raise RuntimeError(f"{url} failed its content checksum")
    safe_extract_tar(archive, parent)
    source = parent / f"llm-compressor-{_LLMC_MAIN_SHA}"
    (source / ".git_archival.txt").write_text(
        f"node: {_LLMC_MAIN_SHA}\ndescribe-name: {_LLMC_MAIN_DESCRIBE}-g{_LLMC_MAIN_SHA[:8]}\n",
        encoding = "utf-8",
    )
    return source


def _archive_content_digest(archive: Path) -> str:
    """sha256 of each file's path and contents; compression, order and root dir are ignored."""
    entries = []
    with tarfile.open(archive, "r:gz") as tar:
        for member in tar:
            rel = member.name.split("/", 1)[1] if "/" in member.name else ""
            if member.isfile():
                file_digest = hashlib.sha256()
                handle = tar.extractfile(member)
                while chunk := handle.read(1 << 20):
                    file_digest.update(chunk)
                entries.append(f"f {rel} {file_digest.hexdigest()}")
            elif member.issym() or member.islnk():
                entries.append(f"l {rel} {member.linkname}")
    digest = hashlib.sha256()
    for entry in sorted(entries):
        digest.update(entry.encode("utf-8") + b"\n")
    return digest.hexdigest()


def _install_llmcompressor_shadow(source: Path) -> bool:
    # Prefer uv then pip; every spec at once, --no-deps, prereleases allowed (compressed-tensors).
    base = [
        "--target",
        _VENV_LLMCOMPRESSOR_DIR,
        "--no-deps",
        "--prerelease=allow",
        str(source),
        *_VENV_LLMCOMPRESSOR_SPECS,
    ]
    cmds = []
    if shutil.which("uv"):
        cmds.append(["uv", "pip", "install", "--python", sys.executable, *base])
    cmds.append(
        [
            sys.executable,
            "-m",
            "pip",
            "install",
            *[a for a in base if a != "--prerelease=allow"],
            "--pre",
        ]
    )

    last_out = ""
    for cmd in cmds:
        result = subprocess.run(
            cmd,
            stdout = subprocess.PIPE,
            stderr = subprocess.STDOUT,
            text = True,
            encoding = "utf-8",
            errors = "replace",
            env = utf8_child_env(
                get_hf_cache_paths().child_env(child_env_without_native_path_secret())
            ),
            **_windows_hidden_subprocess_kwargs(),
        )
        last_out = result.stdout or ""
        if result.returncode == 0:
            try:
                (Path(_VENV_LLMCOMPRESSOR_DIR) / _LLMC_SHADOW_MARKER).write_text(
                    _LLMC_SHADOW_FINGERPRINT, encoding = "utf-8"
                )
            except Exception:
                pass
            logger.info("Provisioned llm-compressor-main shadow at %s", _VENV_LLMCOMPRESSOR_DIR)
            return True
        logger.warning("llm-compressor-main shadow install failed with %s; trying next", cmd[0])

    logger.error(
        "Failed to provision llm-compressor-main shadow (spec: llmcompressor@%s). Output:\n%s",
        _LLMC_MAIN_SHA,
        last_out[-4000:],
    )
    return False


def llmcompressor_shadow_pythonpath(*, allow_provision: bool = False) -> str | None:
    """Return the llm-compressor-main shadow's sys.path entry, provisioning a missing one only with
    allow_provision (the user's consent), or None (disabled, offline, not consented, failed)."""
    if _llmcompressor_main_disabled():
        return None
    if _llmcompressor_shadow_is_valid():
        return _VENV_LLMCOMPRESSOR_DIR
    if not allow_provision:
        return None
    if _ensure_venv_llmcompressor_exists():
        return _VENV_LLMCOMPRESSOR_DIR
    return None


def _activate_venv(venv_dir: str, label: str) -> None:
    """Prepend *venv_dir* to sys.path, purge stale modules, reimport."""
    if venv_dir not in sys.path:
        sys.path.insert(0, venv_dir)
        logger.info("Prepended %s to sys.path", venv_dir)

    count = _purge_modules()
    logger.info("Purged %d cached modules", count)

    import transformers

    logger.info("Loaded transformers %s (%s)", transformers.__version__, label)


def _deactivate_5x() -> None:
    """Remove all .venv_t5_*/ dirs from sys.path, purge stale modules, reimport."""
    for d in (_VENV_T5_530_DIR, _VENV_T5_550_DIR, _VENV_T5_510_DIR, _VENV_T5_LATEST_DIR):
        while d in sys.path:
            sys.path.remove(d)
    logger.info("Removed venv_t5 dirs from sys.path")

    count = _purge_modules()
    logger.info("Purged %d cached modules", count)

    import transformers

    logger.info("Reverted to transformers %s", transformers.__version__)


def ensure_transformers_version(model_name: str) -> None:
    """Export path only; training/inference use subprocess isolation instead. Switches sys.path to the matching .venv_t5_* sidecar, purges modules."""
    if _is_lora_adapter_dir(Path(model_name)):
        resolved = _resolve_base_model(model_name)
    else:
        # A remote adapter's tier is its BASE model's (see activation above).
        resolved = _remote_lora_base(model_name) or model_name
    tier = get_transformers_tier(resolved)
    if model_name != resolved and _safe_is_file(Path(model_name) / "config.json"):
        # Gate on a real local config.json: a checkpoint may carry config the base does not surface; path names alone must not upgrade a plain adapter.
        tier = _higher_tier(tier, get_transformers_tier(model_name))

    if tier == "latest":
        pinned = latest_venv_pinned_version()
        if pinned is None:
            raise RuntimeError(
                f"Cannot activate the latest-transformers sidecar: "
                f"no pin marker at {_VENV_T5_LATEST_DIR}"
            )
        target_version = pinned
        venv_dir = _VENV_T5_LATEST_DIR
        ensure_fn = _ensure_venv_t5_latest_exists
    elif tier == "510":
        target_version = TRANSFORMERS_510_VERSION
        venv_dir = _VENV_T5_510_DIR
        ensure_fn = _ensure_venv_t5_510_exists
    elif tier == "550":
        target_version = TRANSFORMERS_550_VERSION
        venv_dir = _VENV_T5_550_DIR
        ensure_fn = _ensure_venv_t5_550_exists
    elif tier == "530":
        target_version = TRANSFORMERS_530_VERSION
        venv_dir = _VENV_T5_530_DIR
        ensure_fn = _ensure_venv_t5_530_exists
    else:
        target_version = TRANSFORMERS_DEFAULT_VERSION
        venv_dir = None
        ensure_fn = None

    target_major = int(target_version.split(".")[0])

    in_memory = _get_in_memory_version()

    logger.info(
        "Version check for '%s' (resolved: '%s'): need=%s, in_memory=%s",
        model_name,
        resolved,
        target_version,
        in_memory,
    )

    # --- Already correct? ---
    if in_memory is not None:
        if in_memory == target_version:
            logger.info(
                "transformers %s already loaded — correct for '%s'",
                in_memory,
                model_name,
            )
            return
        # Different 5.x -> need to switch (e.g. 5.3.0 loaded but need 5.10.x).
        in_memory_major = int(in_memory.split(".")[0])
        if in_memory_major == target_major and venv_dir is None:
            # Both are default (4.x) - close enough.
            logger.info(
                "transformers %s already loaded — correct for '%s'",
                in_memory,
                model_name,
            )
            return

    # --- Switch version ---
    if venv_dir is not None:
        # First remove any other 5.x venv from sys.path.
        _deactivate_5x()
        if not ensure_fn():
            raise RuntimeError(
                f"Cannot activate transformers {target_version}: " f"venv missing at {venv_dir}"
            )
        logger.info("Activating transformers %s…", target_version)
        _activate_venv(venv_dir, f"transformers {target_version}")
    else:
        logger.info("Reverting to default transformers %s…", TRANSFORMERS_DEFAULT_VERSION)
        _deactivate_5x()

    final = _get_in_memory_version()
    logger.info("✓ transformers version is now %s", final)