Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

33.4 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Imports the ML stack on a background thread, started last in main.py's lifespan so it overlaps
serving; idempotent, never fatal, never half-initialised; torch endpoints block on get_device()
until hardware stage ends, so async handlers must use asyncio.to_thread."""

from __future__ import annotations

import importlib
import importlib.machinery
import os
import sys
import threading
import time
from contextlib import contextmanager
from functools import partial, wraps
from importlib._bootstrap import _ModuleLockManager
from typing import Collection, Optional

from loggers import get_logger

logger = get_logger(__name__)

DISABLE_ENV_VAR = "UNSLOTH_STUDIO_DISABLE_TORCH_WARM"

_start_lock = threading.Lock()
_thread: Optional[threading.Thread] = None
# Detection epoch of the live warm: "already warmed this lifespan" vs. one whose lifespan ended.
_thread_epoch: Optional[int] = None
_status: dict = {"started": False, "finished": False, "stages": {}}


def _is_extension_module(name: str) -> bool:
    """True if sys.modules[name] is a compiled extension, not Python source."""
    module = sys.modules.get(name)
    origin = getattr(getattr(module, "__spec__", None), "origin", None) or getattr(
        module, "__file__", None
    )
    if not isinstance(origin, str):
        return False
    return origin.endswith(tuple(importlib.machinery.EXTENSION_SUFFIXES))


_DATASETS_ARROW_EXTENSION_TYPES = tuple(
    f"datasets.features.features.Array{dimensions}DExtensionType" for dimensions in range(2, 6)
)


def _clear_external_import_state(package: str) -> list[str]:
    """Undo native registrations made by a pure-Python module before it failed."""
    if package != "datasets":
        return []
    pyarrow = sys.modules.get("pyarrow")
    unregister = getattr(pyarrow, "unregister_extension_type", None)
    if unregister is None:
        return []
    cleared: list[str] = []
    for type_name in _DATASETS_ARROW_EXTENSION_TYPES:
        try:
            unregister(type_name)
        except KeyError:
            continue
        cleared.append(type_name)
    if cleared:
        logger.warning(
            "unregistered %d PyArrow extension type(s) left by the failed %s "
            "import so its modules can be executed again",
            len(cleared),
            package,
        )
    return cleared


def _synchronize_with_imports(fn):
    """Run cleanup under the same per-module lock used by CPython imports."""

    @wraps(fn)
    def synchronized(package: str):
        with _ModuleLockManager(package):
            return fn(package)

    return synchronized


@_synchronize_with_imports
def purge_partial_import(package: str) -> list:
    """Drop the submodules a failed package import left behind in sys.modules.

    When ``package/__init__.py`` raises, CPython evicts only the parent and keeps every submodule it executed, so the next import re-runs ``__init__`` with each ``from .x import y`` served from that cache: the package imports "successfully" while missing pieces (#7580).

    Acts only on that exact signature (parent gone, submodules present), so a concurrent still-running import is left alone; returns what it removed.

    Declines when any submodule is a loaded C extension: evicting one re-runs its module init, and pybind11 answers a duplicate type registration with std::terminate. Known native registries populated by pure-Python modules are reset only after every stale module has been removed and no importer has republished the parent.
    """
    if package in sys.modules:
        return []
    prefix = package + "."
    stale = [name for name in list(sys.modules) if name.startswith(prefix)]
    if package in sys.modules:
        logger.info(
            "not purging %s: another importer republished it while collecting its "
            "leftovers, so that import owns them now",
            package,
        )
        return []
    compiled = sorted(name for name in stale if _is_extension_module(name))
    if compiled:
        logger.warning(
            "not purging %s: %d of its submodule(s) are loaded C extensions and "
            "re-importing one aborts the process (%s). The next import will reuse "
            "the cached submodules and may be missing attributes.",
            package,
            len(compiled),
            ", ".join(compiled[:4]),
        )
        return []
    # Track what actually went: a partway bail must not report a clean slate that never happened.
    removed = []
    for name in stale:
        # Same race, per pop: bail the moment the parent is back.
        if package in sys.modules:
            logger.warning(
                "stopped purging %s partway: another importer republished it. The "
                "submodules already removed will be re-executed by that import.",
                package,
            )
            break
        if sys.modules.pop(name, None) is not None:
            removed.append(name)
    fully_purged = package not in sys.modules and not any(name in sys.modules for name in stale)
    if fully_purged:
        _clear_external_import_state(package)
    if removed:
        logger.warning(
            "purged %d half-imported %s submodule(s) so the next import re-runs clean: %s",
            len(removed),
            package,
            ", ".join(sorted(removed)[:8]),
        )
    return removed


# Stage -> package to purge on failure. inference_backend is absent: it imports nothing.
_STAGE_PACKAGE = {
    "hardware": "torch",
    "transformers": "transformers",
    "datasets": "datasets",
}

# Hold the import lock across import and cleanup, or a queued importer sees stale submodules.
_BARE_IMPORT_STAGES = frozenset({"datasets"})


@contextmanager
def _held_import_lock(name: str, package: Optional[str]):
    """Hold ``package``'s import lock for a bare-import stage; a no-op for the rest."""
    if package is None or name not in _BARE_IMPORT_STAGES:
        yield
        return
    with _ModuleLockManager(package):
        yield


def _run_stage(name: str, fn) -> None:
    package = _STAGE_PACKAGE.get(name)
    started = time.perf_counter()
    with _held_import_lock(name, package):
        try:
            fn()
        except BaseException as exc:  # noqa: BLE001 - a warm failure must be visible, not fatal
            _status["stages"][name] = {"ok": False, "error": repr(exc)}
            # warning, not debug: the stage stays cold and the first request pays for it.
            logger.warning("torch warm stage %r failed: %r", name, exc)
            if package:
                purge_partial_import(package)
        else:
            _status["stages"][name] = {
                "ok": True,
                "seconds": round(time.perf_counter() - started, 3),
            }


def _warm_hardware(epoch: Optional[int] = None) -> None:
    from utils.hardware import ensure_hardware_detected
    ensure_hardware_detected(epoch)


def _warm_transformers() -> None:
    from utils.models.model_config import _detection_sets
    _detection_sets()
    # The sets may skip the import now; keep warming it for the first request that loads a model.
    try:
        importlib.import_module("transformers.models.auto.modeling_auto")
    except Exception as exc:  # noqa: BLE001
        logger.debug("transformers registry import failed in the warm: %r", exc)


def _warm_datasets() -> None:
    # Independent of hardware detection; datasets may import shared libraries such as torch.
    importlib.import_module("datasets")


# Unsloth Zoo is loaded by utils.hf_xet_fallback only when a Hub operation needs it.
def _warm_inference_backend() -> None:
    from core.inference import get_inference_backend

    get_inference_backend()
    # Must precede _prime_nvlink_topology so the first dynamo import is single-threaded.
    ensure_dynamo_imported()
    _prime_nvlink_topology()


def _prime_nvlink_topology() -> Optional[threading.Thread]:
    """Runs off the load path, fire and forget (its timeouts would delay every stage), and caches success only: an early miss keeps P2P off."""

    def _probe() -> None:
        try:
            from core.inference.llama_cpp import LlamaCppBackend

            # Opted out, so the answer could never be used; the load path skips it too.
            if os.environ.get("UNSLOTH_DISABLE_DC_TUNING") == "1":
                return
            if LlamaCppBackend._p2p_user_opted_out():
                return
            if LlamaCppBackend._effective_gpu_count() < 2:
                return
            if not LlamaCppBackend._all_selected_gpus_match(
                LlamaCppBackend._NVLINK_FABRIC_GPU_RE, None
            ):
                return
            LlamaCppBackend.prime_nvlink_topology()
        except Exception as e:  # noqa: BLE001 -- a warm miss costs latency, never correctness
            logger.debug("NVLink topology prime skipped: %r", e)

    worker = threading.Thread(target = _probe, daemon = True, name = "nvlink-topology-prime")
    worker.start()
    return worker


_dynamo_lock = threading.Lock()
_dynamo_done = False

# Kill switch for the request-side gates only; the gates in front of `import diffusers` stay on.
DYNAMO_GATE_DISABLE_ENV_VAR = "UNSLOTH_STUDIO_DISABLE_DYNAMO_IMPORT_GATE"
DYNAMO_GATE_TIMEOUT_ENV_VAR = "UNSLOTH_STUDIO_DYNAMO_IMPORT_GATE_TIMEOUT"
_DEFAULT_DYNAMO_GATE_TIMEOUT_S = 600.0


def _dynamo_gate_timeout() -> float:
    try:
        value = float(os.environ.get(DYNAMO_GATE_TIMEOUT_ENV_VAR, _DEFAULT_DYNAMO_GATE_TIMEOUT_S))
    except ValueError:
        return _DEFAULT_DYNAMO_GATE_TIMEOUT_S
    return min(value, threading.TIMEOUT_MAX) if value > 0 else _DEFAULT_DYNAMO_GATE_TIMEOUT_S


def gate_torch_stack_import(reason: str, log = None) -> bool:
    """A racing torch._dynamo import can deadlock on package locks and leave a half-built module."""
    if _dynamo_done:
        _prime_cuda_driver()
        return True
    if os.environ.get(DYNAMO_GATE_DISABLE_ENV_VAR) == "1":
        _prime_cuda_driver()  # the kill switch skips only the dynamo wait
        return False
    imported = ensure_dynamo_imported(
        log = log or logger, reason = reason, timeout = _dynamo_gate_timeout()
    )
    _prime_cuda_driver()
    return imported


def _prime_cuda_driver() -> None:
    """``cuInit`` off the GIL before the gated importer probes CUDA. See utils/cuda_driver_prime.py."""
    try:
        from utils.cuda_driver_prime import prime_cuda_driver  # noqa: PLC0415
        prime_cuda_driver()
    except Exception as exc:  # noqa: BLE001 -- never fatal; torch initialises as before
        logger.debug("cuda driver prime skipped: %r", exc)


def ensure_dynamo_imported(
    log = None,
    reason: Optional[str] = None,
    timeout: Optional[float] = None,
) -> bool:
    """Finishes torch._dynamo on one thread; it is lazy, so an early read sees a half-initialized module."""
    global _dynamo_done
    if _dynamo_done:
        return True
    if not _dynamo_lock.acquire(blocking = False):
        waited = time.perf_counter()
        if log is not None:
            log.info(
                "%s: waiting for another thread to finish importing torch._dynamo",
                reason or "torch import",
            )
        if not _dynamo_lock.acquire(timeout = -1 if timeout is None else timeout):
            (log or logger).warning(
                "%s: gave up waiting for the torch._dynamo import after %.0fs; continuing "
                "without it (set %s=1 to skip this wait)",
                reason or "torch import",
                time.perf_counter() - waited,
                DYNAMO_GATE_DISABLE_ENV_VAR,
            )
            return False
        if log is not None:
            log.info(
                "%s: torch._dynamo ready after waiting %.1fs",
                reason or "torch import",
                time.perf_counter() - waited,
            )
    try:
        if _dynamo_done:
            return True
        try:
            import torch  # noqa: PLC0415
            import torch._dynamo  # noqa: PLC0415
            import torch._dynamo.utils  # noqa: F401, PLC0415

            # Check the attribute: a cached submodule may be unbound on its parent (the broken state).
            if getattr(torch._dynamo, "utils", None) is None:
                return False
        except Exception as exc:  # noqa: BLE001 -- no torch, or a dynamo that cannot import
            logger.debug("torch._dynamo warm skipped: %r", exc)
            return False
        _dynamo_done = True
        return True
    finally:
        _dynamo_lock.release()


def _heal_kernels_conflict(log) -> None:
    """A kernels install transformers cannot import fails every diffusers import: repair or hide it first."""
    try:
        from utils.kernels_compat import heal_kernels_conflict
    except Exception:  # noqa: BLE001
        return
    heal_kernels_conflict(log or logger)


def close_dynamo_import_window(log) -> bool:
    """Call before importing diffusers, itself a dynamo importer that cannot recover a lost race; wrap the import of this module too."""
    _heal_kernels_conflict(log)
    imported = ensure_dynamo_imported(log = log, reason = "diffusers import")
    # `import diffusers` probes CUDA at module level (diffusers.utils.torch_utils) with the GIL held.
    _prime_cuda_driver()
    # Not nested in the dynamo gate: the background import it waits on reaches torch._dynamo.
    claim_media_import_window(log)
    if imported:
        return True
    log.warning(
        "torch._dynamo is not importable in this process; "
        "if this load fails on a dynamo import, restart Unsloth"
    )
    return False


# diffusers and peft are circular: concurrent importers get a partially initialized module.
_media_import_lock = threading.Lock()
_media_import_claimed = False
_media_import_owner: Optional[int] = None


@contextmanager
def background_media_import():
    """Hold the window for background work; yields False once a load has claimed it (skip then)."""
    global _media_import_owner
    _heal_kernels_conflict(None)
    with _media_import_lock:
        _media_import_owner = threading.get_ident()
        try:
            yield not _media_import_claimed
        finally:
            _media_import_owner = None


def claim_media_import_window(log = None) -> bool:
    """Wait out a background media import in flight, then keep later ones off. False on timeout."""
    global _media_import_claimed
    if _media_import_claimed or _media_import_owner == threading.get_ident():
        return True
    if not _media_import_lock.acquire(blocking = False):
        waited = time.perf_counter()
        if log is not None:
            log.info("diffusers import: waiting for the background diffusers import to finish")
        if not _media_import_lock.acquire(timeout = _dynamo_gate_timeout()):
            (log or logger).warning(
                "diffusers import: gave up waiting for the background diffusers import after %.0fs",
                time.perf_counter() - waited,
            )
            return False
        if log is not None:
            log.info(
                "diffusers import: background import finished after waiting %.1fs",
                time.perf_counter() - waited,
            )
    try:
        _media_import_claimed = True
        return True
    finally:
        _media_import_lock.release()


@contextmanager
def hold_media_import_window(reason: str, log = None):
    """Hold the window around a foreground import of the torch compile stack; an expired wait still imports."""
    global _media_import_owner
    if (
        os.environ.get(DYNAMO_GATE_DISABLE_ENV_VAR) == "1"
        or _media_import_owner == threading.get_ident()
    ):
        yield
        return
    held = _media_import_lock.acquire(blocking = False)
    if not held:
        waited = time.perf_counter()
        (log or logger).info("%s: waiting for a background media import to finish", reason)
        held = _media_import_lock.acquire(timeout = _dynamo_gate_timeout())
        if held:
            (log or logger).info(
                "%s: background media import finished after waiting %.1fs",
                reason,
                time.perf_counter() - waited,
            )
        else:
            (log or logger).warning(
                "%s: gave up waiting for the background media import after %.0fs; importing unguarded",
                reason,
                time.perf_counter() - waited,
            )
    if not held:
        yield
        return
    _media_import_owner = threading.get_ident()
    try:
        yield
    finally:
        _media_import_owner = None
        _media_import_lock.release()


_STAGES = (
    ("hardware", _warm_hardware),
    ("inference_backend", _warm_inference_backend),
    ("transformers", _warm_transformers),
    ("datasets", _warm_datasets),
)


def _selected_stage_names(features: Collection[str]) -> set[str]:
    """Dependencies are shared, but an unselected feature never gets its own warm stage."""
    selected = set(features)
    stages = set()
    if selected.intersection({"hardware", "text", "media", "quantization"}):
        stages.add("hardware")
    if selected.intersection({"text", "media", "quantization"}):
        stages.add("inference_backend")  # Dynamo must finish before any compile-stack consumer.
    if "text" in selected:
        stages.add("transformers")
    if "datasets" in selected:
        stages.add("datasets")
    return stages


START_GATE_MAX_WAIT_S = 2.0


def _wait_for_start_gate(gate: Optional[threading.Event]) -> None:
    if gate is None:
        return
    waited = time.perf_counter()
    opened = gate.wait(START_GATE_MAX_WAIT_S)
    logger.info(
        "torch warm leaving its start gate after %.0fms (%s)",
        (time.perf_counter() - waited) * 1000,
        "first response sent" if opened else "no request yet",
    )


def _warm(
    epoch: Optional[int] = None,
    gate: Optional[threading.Event] = None,
    features: Optional[frozenset[str]] = None,
) -> None:
    _wait_for_start_gate(gate)
    started = time.perf_counter()
    if epoch is None:
        epoch = _detection_epoch()
    # None preserves explicit legacy callers; startup always supplies its immutable selection.
    stages = None if features is None else _selected_stage_names(features)
    warm_quantization = features is None or "quantization" in features
    # The scope binds the epoch so a mid-stage shutdown discards this pass, not republishes DEVICE.
    with _owning_epoch(epoch):
        for name, fn in _STAGES:
            if stages is not None and name not in stages:
                continue
            if epoch is not None and _detection_epoch() != epoch:
                # Before the first stage too: a shutdown between the epoch read and start().
                logger.info("torch warm stopped before %s: its lifespan ended", name)
                return
            # Only the real stage takes the epoch; a patched _STAGES entry is called bare.
            _run_stage(name, partial(fn, epoch) if fn is _warm_hardware else fn)
            if epoch is not None and _detection_epoch() != epoch:
                # Later stages reach get_device(), republishing DEVICE after teardown.
                logger.info("torch warm stopped after %s: its lifespan ended", name)
                return
            if name == "inference_backend" and warm_quantization:
                # Dynamo is ready and this lifespan still owns the work.
                _kick_early_quant_probe()
    _status["finished"] = True
    _status["seconds"] = round(time.perf_counter() - started, 3)
    logger.info("torch warm finished in %.1fms", (time.perf_counter() - started) * 1000)


@contextmanager
def _owning_epoch(epoch: Optional[int]):
    """hardware.owning_detection_epoch(), a no-op when hardware is not importable: a --no-torch host still runs the warm and each stage reports its own absence."""
    try:
        from utils.hardware import hardware as _hw
        scope = _hw.owning_detection_epoch(epoch)
    except Exception:
        yield
        return
    with scope:
        yield


def _detection_epoch() -> Optional[int]:
    """The current detection epoch, or None if hardware is not importable."""
    try:
        from utils.hardware import hardware as _hw
        return _hw.current_detection_epoch()
    except Exception:
        return None


def _warm_after(
    previous: threading.Thread,
    epoch: Optional[int],
    gate: Optional[threading.Event] = None,
    features: Optional[frozenset[str]] = None,
) -> None:
    """Wait out a retired warm, then warm for ``epoch``. One importer at a time."""
    previous.join()
    _warm(epoch, gate, features)


def start_background_warm(
    gate: Optional[threading.Event] = None, *, features: Optional[Collection[str]] = None
) -> bool:
    """Warm selected features; omitting features preserves an explicit caller's full warm.

    A finished thread from an earlier lifespan does not count; shutdown cleared the state it warmed.
    ``gate``: reports started now, runs stages once set or after START_GATE_MAX_WAIT_S (utils/first_response_gate.py)."""
    global _thread
    if os.environ.get(DISABLE_ENV_VAR) == "1":
        return False
    selection = None if features is None else frozenset(features)
    if selection is not None and not _selected_stage_names(selection):
        return False
    global _thread_epoch
    # Epoch read before start(): reading it in the child would adopt the post-shutdown one.
    epoch = _detection_epoch()
    with _start_lock:
        target, args = _warm, (epoch, gate, selection)
        if _thread is not None:
            if _thread_epoch is not None and epoch == _thread_epoch:
                return False
            if _thread.is_alive():
                # Stale but mid-stage: hand off to a successor that joins it first, keeping one importer.
                target, args = _warm_after, (_thread, epoch, gate, selection)
            else:
                _clear_finished_warm_locked()
        _thread = threading.Thread(
            target = target,
            args = args,
            daemon = True,
            name = "torch-warm",
        )
        _thread_epoch = epoch
        _status["started"] = True
        _thread.start()
        return True


def reset_background_warm() -> bool:
    """Resets so a later lifespan can warm afresh (shutdown cleared the state); declines while the previous warm still runs, so two never share imports."""
    with _start_lock:
        thread = _thread
        if thread is not None and thread.is_alive():
            return False
        _clear_finished_warm_locked()
        return True


def _clear_finished_warm_locked() -> None:
    """Drop the finished warm and its status. Caller holds ``_start_lock``."""
    global _thread, _thread_epoch
    _thread = None
    _thread_epoch = None
    _status["started"] = False
    _status["finished"] = False
    _status["stages"] = {}
    _status.pop("seconds", None)


DIFFUSERS_PREWARM_DISABLE_ENV_VAR = "UNSLOTH_STUDIO_DISABLE_DIFFUSERS_PREWARM"
DIFFUSERS_PREWARM_MODELS_ENV_VAR = "UNSLOTH_STUDIO_DIFFUSERS_PREWARM_MODELS"
_DIFFUSERS_PREWARM_MODEL_MODULES = ("diffusers.models.transformers",)

# Must equal the task ids of the catalog, else an empty index reads as no models and the gate refuses forever; pinned by test_diffusers_prewarm.py.
_VIDEO_TASK = "text-to-video"
_MEDIA_PREWARM_TASKS = ("text-to-image", _VIDEO_TASK)

_diffusers_prewarm_lock = threading.Lock()
_diffusers_prewarmed = False


def _a_local_model_would_load_through_diffusers() -> bool:
    """A GGUF routed to sd.cpp imports no diffusers, so routing matters as well as presence."""
    from core.inference.diffusion_engine_router import (  # noqa: PLC0415
        ENGINE_DIFFUSERS,
        predict_engine,
    )
    from core.inference.media_locality import detected_image_family  # noqa: PLC0415
    from core.inference.media_model_index import (  # noqa: PLC0415
        available_media_model_ids,
        resolve_local_media_model,
    )

    # The media index is per account and this thread has none, so it answers for the owner.
    for task in _MEDIA_PREWARM_TASKS:
        for model_id in available_media_model_ids(task):
            pick = resolve_local_media_model(model_id, task = task)
            if pick is None:
                continue
            kind = pick.model_kind or ("gguf" if pick.gguf_filename else None)
            if kind != "gguf":
                return True  # only a GGUF can go native, by either backend
            if task == _VIDEO_TASK:
                # The image resolver cannot answer for video; see _is_native_video_pick.
                if _is_native_video_pick(pick):
                    continue
                return True
            family = detected_image_family(pick)
            if family is None:
                return True  # unknown family: diffusers is where the load would land
            if predict_engine(family, model_kind = "gguf") == ENGINE_DIFFUSERS:
                return True
    return False


def _is_native_video_pick(pick) -> bool:
    """The one video load that returns before its diffusers import; all other video loads reach it."""
    from core.inference.video_families import detect_video_family  # noqa: PLC0415
    from core.inference.video_minimax_h3 import is_h3_native  # noqa: PLC0415

    gguf = getattr(pick, "gguf_filename", None)
    for base in (pick.model_path, pick.model_id):
        if not base:
            continue
        # Same order and pair as video.py's _detect_load_family; the token may be only in the filename.
        for needle in (base, f"{base}/{gguf}" if gguf else None):
            if not needle:
                continue
            try:
                family = detect_video_family(needle)
            except Exception:  # noqa: BLE001 -- a probe failure must not decide "native"
                continue
            if family is not None:
                return bool(is_h3_native(family, "gguf"))
    # Deliberately skips reading general.architecture from a renamed GGUF: file IO on boot.
    return False


EARLY_PROBE_ENV_VAR = "UNSLOTH_DIFFUSION_PROBE_EARLY"


def _early_quant_probe() -> None:
    try:
        from core.inference.diffusion_probe_cache import has_file  # noqa: PLC0415 - stdlib only
        if has_file():
            # A later start reads the persisted table; importing the probe module here would only slow the warm.
            return
        if not _a_local_model_would_load_through_diffusers():
            return
    except Exception as exc:  # noqa: BLE001 -- a gate that cannot answer means skip, not crash
        logger.debug("early quant smoke probe skipped: %r", exc)
        return
    _prewarm_quant_probe()


def _kick_early_quant_probe() -> Optional[threading.Thread]:
    """Start the quant smoke probe's child during the warm instead of after the diffusers import (a load posted right
    after the warm waited ~3 s for it). ``UNSLOTH_DIFFUSION_PROBE_EARLY=0`` keeps the old timing."""
    if os.environ.get(EARLY_PROBE_ENV_VAR, "").strip().lower() in ("0", "false", "no", "off"):
        return None
    thread = threading.Thread(target = _early_quant_probe, name = "early-quant-probe", daemon = True)
    thread.start()
    return thread


def _prewarm_quant_probe() -> None:
    """Run the 4-5 s quant smoke probe here instead of at the first image load; never fatal."""
    try:
        from core.inference.diffusion_transformer_quant import prewarm_probe_table  # noqa: PLC0415
        started = time.perf_counter()
        if prewarm_probe_table():
            logger.info(
                "quant smoke probe prewarmed in %.0fms; the first quantised load skips it",
                (time.perf_counter() - started) * 1000,
            )
    except Exception as exc:  # noqa: BLE001 -- the load path probes again and reports
        logger.debug("quant smoke probe prewarm skipped: %r", exc)


def prewarm_quant_probe_if_image_models_exist() -> None:
    """Fallback for a selected quantization check, including when early probing is disabled."""
    if os.environ.get(DISABLE_ENV_VAR) == "1":
        return
    try:
        if not _a_local_model_would_load_through_diffusers():
            return
    except Exception as exc:
        logger.debug("quant smoke probe gate unavailable: %r", exc)
        return
    # Unlike the early hint, validate any persisted table against the current stack.
    _prewarm_quant_probe()


def prewarm_diffusers_if_image_models_exist(*, warm_quantization: bool = True) -> bool:
    """Runs after the warm joins so it cannot delay a stage; skipped once a load claims the window; opt out with UNSLOTH_STUDIO_DISABLE_DIFFUSERS_PREWARM=1."""
    global _diffusers_prewarmed
    if _diffusers_prewarmed:
        return False
    if os.environ.get(DIFFUSERS_PREWARM_DISABLE_ENV_VAR) == "1":
        return False
    # join_background_warm() is True when no worker ran, so check the disable var here too.
    if os.environ.get(DISABLE_ENV_VAR) == "1":
        return False
    with _diffusers_prewarm_lock, background_media_import() as window_open:
        if _diffusers_prewarmed:
            return False
        if not window_open:
            logger.debug("diffusers prewarm skipped: a load is importing diffusers")
            return False
        try:
            if not _a_local_model_would_load_through_diffusers():
                # Not latched: a model downloaded later lets the next lifespan reconsider.
                logger.debug("diffusers prewarm skipped: no local model routes to diffusers")
                return False
        except Exception as exc:  # noqa: BLE001 -- a gate that cannot answer means skip, not crash
            logger.debug("diffusers prewarm gate unavailable: %r", exc)
            return False

        try:
            # On Windows ROCm diffusers reaches xformers/torchao, which need these distributed stubs.
            from core._torchao_stub import (  # noqa: PLC0415
                alias_torchao_fqn_to_config,
                hide_xformers_built_for_another_torch,
                install_torchao_windows_rocm_stub,
                install_xformers_windows_rocm_stub,
            )
            from core.inference.diffusion_torchao_patches import (  # noqa: PLC0415
                install_torchao_int_mm_patch,
            )

            install_xformers_windows_rocm_stub()
            hide_xformers_built_for_another_torch()
            install_torchao_windows_rocm_stub()
            alias_torchao_fqn_to_config()
            install_torchao_int_mm_patch()
        except Exception as exc:  # noqa: BLE001 -- importing unprotected is the hazard; skip
            logger.debug("diffusers prewarm skipped: stubs unavailable: %r", exc)
            return False

        started = time.perf_counter()
        # The try goes INSIDE each `with`, else a waiter can republish the package before the purge.
        with _ModuleLockManager("diffusers"):
            try:
                import diffusers  # noqa: F401, PLC0415
            except Exception as exc:  # noqa: BLE001 -- the load path imports it again and reports
                logger.debug("diffusers prewarm skipped: %r", exc)
                purge_partial_import("diffusers")
                return False

        # Never nest in the scope above: CPython takes the child lock first, so nesting inverts order.
        with _ModuleLockManager("diffusers.hooks"):
            try:
                import diffusers.hooks  # noqa: F401, PLC0415
            except Exception as exc:  # noqa: BLE001 -- the load path imports it again and reports
                logger.debug("diffusers prewarm skipped: %r", exc)
                # Purge the hook submodules that ran, or a later import rebuilds from them.
                purge_partial_import("diffusers.hooks")
                return False

        # The model classes too (1.5-3.7 s of the first load); own scope for the same lock-order reason as above.
        if os.environ.get(DIFFUSERS_PREWARM_MODELS_ENV_VAR, "").strip().lower() not in (
            "0",
            "false",
            "no",
            "off",
        ):
            for module_name in _DIFFUSERS_PREWARM_MODEL_MODULES:
                with _ModuleLockManager(module_name):
                    try:
                        importlib.import_module(module_name)
                    except Exception as exc:  # noqa: BLE001 -- the load path imports it again and reports
                        logger.debug("diffusers model prewarm of %s skipped: %r", module_name, exc)
                        purge_partial_import(module_name)
                        break

        # Outside both locks (it imports nothing under diffusers); diffusers hard-codes _tqdm_active = True, so its bars would draw into structlog output.
        try:
            from loggers.config import quiet_third_party_progress_bars  # noqa: PLC0415
            quiet_third_party_progress_bars()
        except Exception as exc:  # noqa: BLE001 -- cosmetic only
            logger.debug("quieting third-party progress bars failed: %r", exc)

        _diffusers_prewarmed = True
        logger.info(
            "diffusers prewarmed in %.0fms; the first image load skips that import",
            (time.perf_counter() - started) * 1000,
        )
    # Outside the window: a child-process probe, not an import. Startup opts into it separately.
    if warm_quantization:
        _prewarm_quant_probe()
    return True


def warm_status() -> dict:
    """Snapshot of the warm for diagnostics and tests."""
    return {
        "started": _status["started"],
        "finished": _status["finished"],
        "alive": bool(_thread is not None and _thread.is_alive()),
        "stages": dict(_status["stages"]),
        "seconds": _status.get("seconds"),
    }


def join_background_warm(timeout: Optional[float] = None) -> bool:
    """Wait for the warm thread. Returns True if it is done (or never ran)."""
    thread = _thread
    if thread is None:
        return True
    thread.join(timeout)
    return not thread.is_alive()