Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

31.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Checks whether PyPI's latest or GitHub main transformers ships a missing model_type, with no
auth, code execution or trust_remote_code; each fetch has a wall-clock budget so a hung network
never blocks loading; UNSLOTH_STUDIO_NO_LATEST_TRANSFORMERS=1 or offline env vars make checks
return None."""

import contextlib
import json
import os
import re
import threading
import time
from pathlib import Path

from loggers import get_logger
from utils.paths.storage_roots import studio_root as _studio_root
from utils.transformers_version import (
    _env_offline,
    _load_config_json,
    _model_types_from_source,
    _tier_from_config_mapping,
    _config_model_types,
    _NESTED_CONFIG_KEYS,
    _release_tuple,
    _remote_code_min_version,
    _remote_code_saving_version,
    _remote_code_tier,
    _TIER_RANK,
    _model_types_from_config,
    _TRANSFORMERS_510_MODEL_TYPES,
    _TRANSFORMERS_530_MODEL_TYPES,
    _TRANSFORMERS_550_MODEL_TYPES,
    ensure_latest_transformers_venv,
    latest_venv_pinned_version,
)

logger = get_logger(__name__)

_PYPI_JSON_URL = "https://pypi.org/pypi/transformers/json"
_RAW_URL = (
    "https://raw.githubusercontent.com/huggingface/transformers/{ref}"
    "/src/transformers/models/auto/{name}"
)
_AUTO_FILES = ("configuration_auto.py", "auto_mappings.py")

_FETCH_TIMEOUT_SECONDS = 5.0
_FETCH_RETRIES = 1
# urlopen's timeout bounds each read, not the transfer, so the transfer gets its own budget.
_FETCH_DEADLINE_SECONDS = 2 * _FETCH_TIMEOUT_SECONDS
# Worst case is the budget plus one blocked socket read; the deadline is checked between reads.
_FETCH_ATTEMPT_SECONDS = _FETCH_DEADLINE_SECONDS + _FETCH_TIMEOUT_SECONDS
_READ_CHUNK_BYTES = 1 << 16
_CACHE_TTL_SECONDS = 24 * 60 * 60
_FAILURE_BACKOFF_SECONDS = 300

_CACHE_FILE_NAME = "transformers_latest_check.json"
# 2 adds main_version: a schema-1 file would hide the main install until its TTL.
_SNAPSHOT_SCHEMA = 2

# Snapshot: {"schema", "fetched_at", "pypi_version", "pypi_model_types", "main_model_types",
# "main_checked", "main_version"}.
# Install-in-progress state lives in utils.transformers_version (the sidecar swap reservation).
_lock = threading.Lock()
_memory_snapshot: dict | None = None
_last_failure_at: float = 0.0
_is_fetching: bool = False
# Set when no refresh runs; concurrent callers wait on it rather than report no answer.
_fetch_done: threading.Event = threading.Event()
_fetch_done.set()
# Body, or None if still running at deadline; read1 returns what has arrived, since resp.read() has no bound against a drip-fed body.
_REFRESH_URL_COUNT = 1 + 2 * len(_AUTO_FILES)
_INFLIGHT_WAIT_SECONDS = _REFRESH_URL_COUNT * (1 + _FETCH_RETRIES) * _FETCH_ATTEMPT_SECONDS + 5.0

_TRUE_VALUES = {"1", "true", "yes", "on"}


def _disabled() -> bool:
    """True if the operator disabled the latest-transformers check entirely."""
    return (
        os.environ.get("UNSLOTH_STUDIO_NO_LATEST_TRANSFORMERS", "").strip().lower() in _TRUE_VALUES
    )


def _cache_file() -> Path:
    return _studio_root() / "cache" / _CACHE_FILE_NAME


# Sentinel for HTTP 404 (absent at ref), distinct from transient failures.
_FETCH_MISSING = "__unsloth_fetch_missing__"


def _read_within(resp, deadline: float) -> str | None:
    """Body of resp, or None if still running at deadline; resp.read() has no bound against a
    drip-fed body, while read1 returns what has arrived so the budget is checked as it comes in."""
    read1 = getattr(resp, "read1", None)
    if read1 is None:
        # A file-like that hands back the whole body in one go has nothing to dribble.
        return resp.read().decode("utf-8", "replace")
    chunks: list[bytes] = []
    while True:
        if time.monotonic() >= deadline:
            return None
        chunk = read1(_READ_CHUNK_BYTES)
        if not chunk:
            return b"".join(chunks).decode("utf-8", "replace")
        chunks.append(chunk)


def _fetch_text(url: str) -> str | None:
    """Transient fetch or parse failure in either file fails the lookup, so a partial map is never cached; a 404 is skipped; an empty result is a failure."""
    import urllib.error
    import urllib.request

    for attempt in range(1 + _FETCH_RETRIES):
        deadline = time.monotonic() + _FETCH_DEADLINE_SECONDS
        try:
            req = urllib.request.Request(url, headers = {"User-Agent": "unsloth-studio"})
            with urllib.request.urlopen(req, timeout = _FETCH_TIMEOUT_SECONDS) as resp:
                body = _read_within(resp, deadline)
            if body is not None:
                return body
            logger.debug(
                "Fetch (attempt %d) for %s outran its %.1fs budget",
                attempt + 1,
                url,
                _FETCH_DEADLINE_SECONDS,
            )
        except urllib.error.HTTPError as exc:
            if exc.code == 404:
                return _FETCH_MISSING
            logger.debug("Fetch failed (attempt %d) for %s: %s", attempt + 1, url, exc)
        except Exception as exc:
            logger.debug("Fetch failed (attempt %d) for %s: %s", attempt + 1, url, exc)
    return None


def _fetch_latest_pypi_version() -> str | None:
    """Latest transformers release version from PyPI's unauthenticated JSON API."""
    body = _fetch_text(_PYPI_JSON_URL)
    if body is None or body == _FETCH_MISSING:
        return None
    try:
        version = json.loads(body).get("info", {}).get("version")
    except Exception as exc:
        logger.debug("Could not parse PyPI JSON: %s", exc)
        return None
    return version if isinstance(version, str) and version else None


def _fetch_remote_model_types(ref: str) -> frozenset[str] | None:
    """None on failure; PyPI version and tagged mapping required; main is best-effort, and main_checked False marks dev-only as unknown, not unsupported."""
    keys: set[str] = set()
    fetched_any = False
    for name in _AUTO_FILES:
        source = _fetch_text(_RAW_URL.format(ref = ref, name = name))
        if source is None:
            return None
        if source == _FETCH_MISSING:
            continue
        fetched_any = True
        try:
            keys |= _model_types_from_source(source)
        except Exception as exc:
            logger.debug("Could not parse %s at %s: %s", name, ref, exc)
            return None
    if not fetched_any or not keys:
        return None
    return frozenset(keys)


def _load_snapshot_file() -> dict | None:
    """Persisted snapshot from disk, or None (missing/corrupt/old schema)."""
    try:
        with open(_cache_file(), encoding = "utf-8") as f:
            data = json.load(f)
    except Exception:
        return None
    if not isinstance(data, dict) or data.get("schema") != _SNAPSHOT_SCHEMA:
        return None
    if not isinstance(data.get("fetched_at"), (int, float)):
        return None
    if not isinstance(data.get("pypi_version"), str):
        return None
    for key in ("pypi_model_types", "main_model_types"):
        value = data.get(key)
        if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
            return None
    return data


def _save_snapshot_file(snapshot: dict) -> None:
    """Atomic best-effort write (tmp + os.replace, Windows-safe); failures only log."""
    path = _cache_file()
    tmp = path.with_name(path.name + ".tmp")
    try:
        path.parent.mkdir(parents = True, exist_ok = True)
        tmp.write_text(json.dumps(snapshot), encoding = "utf-8")
        os.replace(tmp, path)
    except Exception as exc:
        logger.debug("Could not persist %s: %s", path, exc)
        try:
            tmp.unlink(missing_ok = True)
        except Exception:
            pass


def _snapshot_is_fresh(snapshot: dict | None) -> bool:
    if snapshot is None:
        return False
    # A failed main lookup (mappings or version) would otherwise hide the main install all day.
    ttl = (
        _CACHE_TTL_SECONDS
        if snapshot.get("main_checked") and snapshot.get("main_version")
        else _FAILURE_BACKOFF_SECONDS
    )
    return (time.time() - float(snapshot.get("fetched_at", 0))) < ttl


def _refresh_snapshot() -> dict | None:
    """Concurrent callers wait on the in-flight fetch, since an empty answer would let Start proceed; the network refresh runs outside the lock."""
    version = _fetch_latest_pypi_version()
    if version is None:
        return None
    pypi_types = _fetch_remote_model_types(f"v{version}")
    if pypi_types is None:
        return None
    main_types = _fetch_remote_model_types("main")
    main_version = _fetch_main_version() if main_types is not None else None
    return {
        "schema": _SNAPSHOT_SCHEMA,
        "fetched_at": time.time(),
        "pypi_version": version,
        "pypi_model_types": sorted(pypi_types),
        "main_model_types": sorted(main_types) if main_types is not None else [],
        "main_checked": main_types is not None,
        "main_version": main_version,
    }


def _get_snapshot() -> dict | None:
    """A concurrent caller waits on the in-flight fetch, since an empty answer would let a start proceed."""
    global _memory_snapshot, _last_failure_at, _is_fetching, _fetch_done
    with _lock:
        if _snapshot_is_fresh(_memory_snapshot):
            return _memory_snapshot
        disk = _load_snapshot_file()
        if _snapshot_is_fresh(disk):
            _memory_snapshot = disk
            return disk
        if _disabled() or _env_offline():
            return None
        if time.time() - _last_failure_at < _FAILURE_BACKOFF_SECONDS:
            return None
        if _is_fetching:
            in_flight = _fetch_done
        else:
            in_flight = None
            _is_fetching = True
            _fetch_done = done = threading.Event()
    if in_flight is not None:
        # Mirrors unsloth_zoo _remap_unsloth_bnb_hub_id_for_mlx: unsloth/ bnb Hub ids only, never local paths; MLX quantizes those, so transformers never sees them.
        while not in_flight.wait(_INFLIGHT_WAIT_SECONDS):
            with _lock:
                if not _is_fetching or _fetch_done is not in_flight:
                    break
            logger.debug("Still waiting on the in-flight transformers support refresh")
        with _lock:
            return _memory_snapshot if _snapshot_is_fresh(_memory_snapshot) else None
    fresh = None
    try:
        fresh = _refresh_snapshot()
    finally:
        with _lock:
            _is_fetching = False
            done.set()
            if fresh is None:
                _last_failure_at = time.time()
            else:
                _memory_snapshot = fresh
    if fresh is None:
        # A stale positive could offer a version PyPI no longer serves; be strict.
        return None
    _save_snapshot_file(fresh)
    return fresh


def clear_caches() -> None:
    """Test helper: drop the in-memory snapshot, failure backoff, and busy flags."""
    global _memory_snapshot, _last_failure_at, _is_fetching
    with _lock:
        _memory_snapshot = None
        _last_failure_at = 0.0
        _is_fetching = False
        _fetch_done.set()
    from utils.transformers_version import end_sidecar_swap

    end_sidecar_swap()


# What unsloth_zoo strips to reach a bnb repo's full-precision base.
_MLX_BNB_SUFFIXES = ("-unsloth-bnb-4bit", "-bnb-4bit")


def _is_bitsandbytes_config(cfg: dict | None) -> bool:
    """Whether *cfg* describes a bitsandbytes-quantized checkpoint."""
    if not isinstance(cfg, dict):
        return False
    quant = cfg.get("quantization_config")
    return isinstance(quant, dict) and quant.get("quant_method") == "bitsandbytes"


def _mlx_swaps_bnb_repo_for_its_base(model_name: str) -> bool:
    """On MLX an upgrade cannot help, except third-party or local bnb repos: mlx-lm cannot read bnb, so transformers builds them (unsloth bnb-4bit swap to base)."""
    if not isinstance(model_name, str) or not model_name.startswith("unsloth/"):
        return False
    if os.path.exists(model_name):
        return False
    return model_name.endswith(_MLX_BNB_SUFFIXES)


def _architecture_cannot_come_from_transformers(
    model_name: str = "", cfg: dict | None = None
) -> bool:
    """Whether newest transformers ships model_type: dict of pypi_version, supported_in_pypi, supported_in_main; None if unavailable (caller keeps current behavior)."""
    try:
        from utils.hardware import DeviceType, get_device
        if get_device() != DeviceType.MLX:
            return False
    except Exception:
        return False
    if _is_bitsandbytes_config(cfg) and not _mlx_swaps_bnb_repo_for_its_base(model_name):
        return False
    return True


def latest_transformers_supports(model_type: str) -> dict | None:
    """Fires only when model_type is in no installed overlay or tier table (load would fail); never raises; off via UNSLOTH_STUDIO_NO_LATEST_TRANSFORMERS."""
    if not isinstance(model_type, str) or not model_type:
        return None
    if _disabled() or _env_offline():
        return None
    snapshot = _get_snapshot()
    if snapshot is None:
        return None
    return {
        "pypi_version": snapshot["pypi_version"],
        "supported_in_pypi": model_type in set(snapshot["pypi_model_types"]),
        "supported_in_main": model_type in set(snapshot["main_model_types"]),
        "main_version": snapshot.get("main_version"),
    }


# model_types the hardcoded tier tables already route; never remote-check these.
def _hardcoded_model_types() -> frozenset[str]:
    return frozenset(
        _TRANSFORMERS_530_MODEL_TYPES
        | _TRANSFORMERS_550_MODEL_TYPES
        | _TRANSFORMERS_510_MODEL_TYPES
    )


# Pinned to 5.5 for its mlx-vlm processor, which has no model for the type: prompt even before the sidecar exists.
_TIER_ROUTED_WITHOUT_MODEL = frozenset({"diffusion_gemma"})


# Nested model_types the wrapper config never builds (Cosmos3-Edge's projector_config), so they must not veto an offer.
_INERT_NESTED_MODEL_TYPES = frozenset({"cosmos3_edge_projector"})


def _hardcoded_tier_ships(model_type: str) -> bool:
    """A hardcoded tier counts as handling *model_type* unless that sidecar is installed and its
    CONFIG_MAPPING lacks it: DiffusionGemma is routed to 5.5 for the mlx-vlm processor, but 5.5 has
    no DiffusionGemma model, so a CUDA load needs the latest-transformers prompt."""
    if model_type not in _hardcoded_model_types() or model_type in _TIER_ROUTED_WITHOUT_MODEL:
        return False
    for tier, types in (
        ("530", _TRANSFORMERS_530_MODEL_TYPES),
        ("550", _TRANSFORMERS_550_MODEL_TYPES),
        ("510", _TRANSFORMERS_510_MODEL_TYPES),
    ):
        if model_type in types:
            installed = _config_model_types(tier)
            return not installed or model_type in installed
    return True


def _remote_code_upgrade(model_name: str, cfg: dict) -> dict | None:
    """Offer latest transformers when it meets a custom-code config's saving version and no installed tier does."""
    required = _remote_code_min_version(cfg)
    if required is None or _remote_code_tier(cfg) is not None:
        return None
    snapshot = _get_snapshot()
    if snapshot is None:
        return None
    pypi = _release_tuple(snapshot["pypi_version"])
    main = _release_tuple(snapshot.get("main_version"))
    in_pypi = pypi is not None and pypi >= required
    # Main ships as the next release's .dev0, so a dev build of the required release counts.
    offerable = in_pypi or (main is not None and main >= required)
    if not offerable:
        return None
    logger.info(
        "Model %s has custom code saved by transformers %s, newer than every installed tier "
        "(latest PyPI %s)",
        model_name,
        _remote_code_saving_version(cfg),
        snapshot["pypi_version"],
    )
    types = _model_types_from_config(cfg)
    return {
        "model_type": types[0] if types else str(cfg.get("model_type") or ""),
        "pypi_version": snapshot["pypi_version"],
        "supported_in_pypi": in_pypi,
        "supported_in_main": True,
        "main_version": snapshot.get("main_version"),
    }


def check_upgrade_for_model(model_name: str, hf_token: str | None = None) -> dict | None:
    """Fires only when model_type is in no installed overlay or tier table, i.e. the load would fail."""
    try:
        if _disabled() or _env_offline():
            return None
        cfg = _load_config_json(model_name, hf_token)
        if not isinstance(cfg, dict):
            return None
        if _architecture_cannot_come_from_transformers(model_name, cfg):
            return None
        # Custom code loads from the repo, so only the saving version matters, not the mapping.
        if _remote_code_min_version(cfg) is not None:
            return _remote_code_upgrade(model_name, cfg)
        candidates = _model_types_from_config(cfg)
        if not candidates:
            return None
        # Without a readable base mapping every type looks brand new; bail out.
        if not _config_model_types("default"):
            return None
        missing = [
            candidate
            for candidate in candidates
            if not _hardcoded_tier_ships(candidate)
            and not any(candidate in _config_model_types(tier) for tier in _TIER_RANK)
            and candidate not in _INERT_NESTED_MODEL_TYPES
        ]
        if not missing:
            return None
        # Latest must load EVERY missing type (wrappers build nested sub-configs) or the load fails.
        supports = [latest_transformers_supports(candidate) for candidate in missing]
        if any(
            s is None or not (s["supported_in_pypi"] or s["supported_in_main"]) for s in supports
        ):
            return None
        # Offer PyPI only if it ships every missing type; a main-only type means dev-only.
        model_type = missing[0]
        supported_in_pypi = all(s["supported_in_pypi"] for s in supports)
        supported_in_main = all(s["supported_in_pypi"] or s["supported_in_main"] for s in supports)
        logger.info(
            "Model %s has model_type=%s unknown to every installed transformers "
            "(latest PyPI %s: %s, main: %s)",
            model_name,
            model_type,
            supports[0]["pypi_version"],
            "supported" if supported_in_pypi else "unsupported",
            "supported" if supported_in_main else "unsupported",
        )
        return {
            "model_type": model_type,
            "pypi_version": supports[0]["pypi_version"],
            "supported_in_pypi": supported_in_pypi,
            "supported_in_main": supported_in_main,
            "main_version": supports[0].get("main_version"),
        }
    except Exception as exc:
        logger.debug("Latest-transformers check failed for '%s': %s", model_name, exc)
        return None


# Sidecars install --no-deps: unmet shadowable deps become --target pins, others block.
_SHADOWABLE_DEPS = frozenset({"tokenizers", "safetensors"})
# Provided by the sidecar recipe; checked against its pin, not the base env.
_SIDECAR_PROVIDED = {"huggingface-hub": "1.8.0", "hf-xet": "1.4.2"}
# CLI-only; never imported at runtime in Unsloth's workers.
_IGNORED_DEPS = frozenset({"typer"})


def _canonical_dep_name(name: str) -> str:
    return name.lower().replace("_", "-")


_MAIN_RAW = "https://raw.githubusercontent.com/huggingface/transformers/main"


def _fetch_main_version(ref: str = "main") -> str | None:
    """``__version__`` on transformers *ref* (a ``.devN`` string), or None."""
    from utils.transformers_version import _is_valid_version_string

    body = _fetch_text(f"{_MAIN_RAW.rsplit('/', 1)[0]}/{ref}/src/transformers/__init__.py")
    if body is None or body == _FETCH_MISSING:
        return None
    match = re.search(r'^__version__\s*=\s*"([^"]+)"', body, re.MULTILINE)
    if (
        match is None
        or ".dev" not in match.group(1)
        or not _is_valid_version_string(match.group(1))
    ):
        return None
    return match.group(1)


_MAIN_REFS = "https://github.com/huggingface/transformers.git/info/refs?service=git-upload-pack"


def _resolve_main_commit() -> str | None:
    """The commit main points at, from git's ref advertisement (not the rate-limited API)."""
    body = _fetch_text(_MAIN_REFS)
    if body is None or body == _FETCH_MISSING:
        return None
    match = re.search(r"([0-9a-f]{40}) refs/heads/main(?:\x00|\n)", body)
    return match.group(1) if match else None


def _fetch_main_requires() -> list[str] | None:
    """Core install_requires of transformers main (at the commit being installed), from setup.py."""
    from utils import transformers_version as tv

    ref = tv._main_archive_commit or "main"
    body = _fetch_text(f"{_MAIN_RAW.rsplit('/', 1)[0]}/{ref}/setup.py")
    if body is None or body == _FETCH_MISSING:
        return None
    deps_block = re.search(r"^_deps = \[(.*?)^\]", body, re.MULTILINE | re.DOTALL)
    install_block = re.search(r"^install_requires = \[(.*?)^\]", body, re.MULTILINE | re.DOTALL)
    if deps_block is None or install_block is None:
        return None
    deps = {}
    for spec in re.findall(r'"([^"]+)"', deps_block.group(1)):
        deps[re.split(r"[<>=!~ \[;]", spec, maxsplit = 1)[0]] = spec
    names = re.findall(r'deps\["([^"]+)"\]', install_block.group(1))
    if not names or any(name not in deps for name in names):
        return None
    return [deps[name] for name in names]


def _fetch_requires_dist(version: str) -> list[str] | None:
    """Core (marker-free, non-extra) requires_dist of transformers *version* from PyPI."""
    body = _fetch_text(f"https://pypi.org/pypi/transformers/{version}/json")
    if body is None or body == _FETCH_MISSING:
        return None
    try:
        reqs = json.loads(body).get("info", {}).get("requires_dist")
    except Exception:
        return None
    if not isinstance(reqs, list):
        return None
    return [r for r in reqs if isinstance(r, str)]


def _resolve_exact_version(name: str, specifier) -> str | None:
    """Newest PyPI release of *name* satisfying *specifier* (exact pin for the shadow)."""
    body = _fetch_text(f"https://pypi.org/pypi/{name}/json")
    if body is None or body == _FETCH_MISSING:
        return None
    try:
        from packaging.version import InvalidVersion, Version

        releases = json.loads(body).get("releases", {})
        best = None
        for candidate in releases:
            try:
                parsed = Version(candidate)
            except InvalidVersion:
                continue
            if parsed.is_prerelease or not specifier.contains(candidate):
                continue
            if best is None or parsed > Version(best):
                best = candidate
        return best
    except Exception as exc:
        logger.debug("Could not resolve an exact %s version: %s", name, exc)
        return None


def compat_plan(version: str) -> tuple[tuple[str, ...], list[str]]:
    """Only current PyPI latest, so clients cannot pin a version; before_swap runs after staging, before live swap; reserved = caller holds the swap reservation."""
    reqs = _fetch_main_requires() if ".dev" in version else _fetch_requires_dist(version)
    if reqs is None:
        return (), ["dependency metadata for this release (could not be fetched from PyPI; retry)"]
    try:
        from importlib.metadata import PackageNotFoundError
        from importlib.metadata import version as _installed_version
        from packaging.requirements import InvalidRequirement, Requirement
    except Exception:
        return (), []
    extras: list[str] = []
    blockers: list[str] = []
    for raw in reqs:
        try:
            req = Requirement(raw)
        except InvalidRequirement:
            continue
        if req.extras or (req.marker is not None and not req.marker.evaluate()):
            continue
        name = _canonical_dep_name(req.name)
        if name in _IGNORED_DEPS:
            continue
        if name in _SIDECAR_PROVIDED:
            if not req.specifier.contains(_SIDECAR_PROVIDED[name], prereleases = True):
                # studio.txt already ships a newer hub: pin the base env's hub + hf-xet pair instead.
                base_pins = _base_env_sidecar_pins()
                base_name_pin = next((p for p in base_pins if p.startswith(f"{name}==")), None)
                if base_name_pin and req.specifier.contains(
                    base_name_pin.split("==", 1)[1], prereleases = True
                ):
                    extras.extend(p for p in base_pins if p not in extras)
                else:
                    blockers.append(raw)
            continue
        try:
            installed = _installed_version(req.name)
        except PackageNotFoundError:
            installed = None
        if installed is not None and req.specifier.contains(installed, prereleases = True):
            continue
        if name in _SHADOWABLE_DEPS:
            exact = _resolve_exact_version(name, req.specifier)
            if exact is None:
                blockers.append(raw)
            else:
                extras.append(f"{name}=={exact}")
        else:
            blockers.append(raw)
    return tuple(extras), blockers


def _base_env_sidecar_pins() -> list[str]:
    """Exact pins of the base env's own huggingface-hub and hf-xet (empty if either is missing)."""
    from importlib.metadata import PackageNotFoundError
    from importlib.metadata import version as _installed_version

    pins = []
    for name in _SIDECAR_PROVIDED:
        try:
            pins.append(f"{name}=={_installed_version(name)}")
        except PackageNotFoundError:
            return []
    return pins


def is_install_in_progress() -> bool:
    """True while a latest-transformers install or lazy repair holds the sidecar swap; training and
    export check it so a fresh worker never activates mid-swap."""
    from utils.transformers_version import sidecar_swap_in_progress
    return sidecar_swap_in_progress()


def install_latest_transformers(
    version: str,
    before_swap = None,
    reserved: bool = False,
) -> dict:
    """Consented install of the latest transformers sidecar; returns a structured result.

    Guards: the requested *version* must match the current PyPI latest from the (cached)
    snapshot, so a client cannot pin an arbitrary package version through this endpoint.
    On success ``.venv_t5_latest`` is provisioned and pinned; routing then resolves the
    new tier automatically on this and every future start. *before_swap* is forwarded
    to the stage-and-swap: it runs only after the staged install succeeded, right
    before the live sidecar is replaced. *reserved* means the caller already holds the
    sidecar swap reservation (the install route takes it before waiting on the
    inference lifecycle gate, so worker starts see it for the whole window).
    """
    from utils.transformers_version import end_sidecar_swap, try_begin_sidecar_swap

    if not reserved and not try_begin_sidecar_swap():
        return {
            "success": False,
            "version": version,
            "message": "A transformers installation is already in progress.",
        }
    try:
        return _install_latest_transformers_locked(version, before_swap = before_swap)
    finally:
        if not reserved:
            end_sidecar_swap()


def _install_latest_transformers_locked(version: str, before_swap = None) -> dict:
    """Body of install_latest_transformers; runs with the in-progress flag held."""
    if _disabled():
        return {
            "success": False,
            "version": version,
            "message": "Latest-transformers installs are disabled "
            "(UNSLOTH_STUDIO_NO_LATEST_TRANSFORMERS).",
        }
    if _env_offline():
        return {
            "success": False,
            "version": version,
            "message": "Cannot install: Unsloth is in offline mode.",
        }
    # Re-verify against a live snapshot (a release may land within the TTL).
    global _memory_snapshot
    snapshot = _refresh_snapshot()
    if snapshot is not None:
        with _lock:
            _memory_snapshot = snapshot
        _save_snapshot_file(snapshot)
    else:
        snapshot = _get_snapshot()
    if snapshot is None:
        return {
            "success": False,
            "version": version,
            "message": "Could not verify the latest transformers release on PyPI.",
        }
    from_main = (
        version != snapshot["pypi_version"]
        and snapshot.get("main_checked")
        and version == snapshot.get("main_version")
    )
    if version != snapshot["pypi_version"] and not from_main:
        return {
            "success": False,
            "version": version,
            "message": f"Requested version {version!r} is not the latest transformers "
            f"release ({snapshot['pypi_version']}).",
            # Retry uses these instead of re-sending the stale version forever.
            "latest_version": snapshot["pypi_version"],
            "latest_main_version": snapshot.get("main_version"),
        }
    main_commit = current_main = None
    if from_main:
        # Check the version at one commit and install that commit, so main cannot move between.
        main_commit = _resolve_main_commit()
        current_main = _fetch_main_version(main_commit) if main_commit else None
    if from_main and current_main != version:
        return {
            "success": False,
            "version": version,
            "message": f"transformers main is no longer {version}; retry to install the current main."
            if current_main
            else "Could not check transformers main; retry.",
            "latest_version": snapshot["pypi_version"],
            "latest_main_version": current_main,
        }
    from utils.transformers_version import transformers_main_at

    # Requirements and archive both come from the pinned commit.
    with transformers_main_at(main_commit) if main_commit else contextlib.nullcontext():
        extra_packages, blockers = compat_plan(version)
        if blockers:
            return {
                "success": False,
                "version": version,
                "message": "Cannot install transformers "
                f"{version}: this environment does not satisfy {', '.join(blockers)}. "
                "An Unsloth update is required first.",
            }
        installed = ensure_latest_transformers_venv(
            version, extra_packages, before_swap = before_swap
        )
    if not installed:
        return {
            "success": False,
            "version": version,
            "message": f"Installing transformers {version} failed; see the Unsloth logs.",
        }
    _invalidate_capability_caches()
    return {
        "success": True,
        "version": version,
        "message": f"Installed transformers {version}{' (main)' if from_main else ''} into the latest sidecar "
        f"(pinned: {latest_venv_pinned_version()}).",
    }


def _invalidate_capability_caches():
    """Drop caches built before the new sidecar existed (tier probes, latest model_type mapping,
    vision detection); a raw-heuristic False may now defer to the sidecar AutoConfig probe."""
    try:
        from utils import transformers_version as tv
        tv._probe_tier_cache.clear()
        tv._config_mapping_cache.pop("latest", None)
    except Exception:
        pass
    try:
        from utils.models import model_config as mc
        mc._vision_detection_cache.clear()
    except Exception:
        pass