Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""One engine per load: diffusers on CUDA/ROCm/XPU, native sd.cpp on CPU or MPS when enabled; MLX
auto takes plain text-to-image on Apple Silicon, else diffusers MPS; explicit picks refused,
never swapped; env override wins over a per-load pick."""
from __future__ import annotations
import functools
import importlib.util
import os
import re
import platform
import sys
import threading
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Any, Callable, Optional
from core.inference.diffusion_device import resolve_diffusion_device_target
from core.inference.diffusion_families import (
DiffusionFamily,
family_mlx_supported,
family_pipeline_available,
family_sd_cpp_supported,
modular_image_family_names,
pipeline_available_family_names,
)
from core.inference.sd_cpp_backend import (
_card_lookup_inventory,
_install_allowed,
_managed_tree_in_use,
_pin_moved,
_server_binary_runnable,
ensure_sd_cpp_binary,
ensure_sd_server_binary,
note_unlaunchable_accelerator_build,
off_torch_build_mismatch,
preferred_accelerator,
sd_cpp_binary_runs_family,
sd_cpp_vulkan_fallback_enabled,
usable_or_recorded_failure,
)
from core.inference.sd_cpp_engine import (
ENGINE_DIFFUSERS,
ENGINE_MLX,
ENGINE_SD_CPP,
SdCppEngine,
select_diffusion_engine,
)
from loggers import get_logger
logger = get_logger(__name__)
_DISABLE_TOKENS = frozenset({"0", "off", "false", "no"})
_ENABLE_TOKENS = frozenset({"1", "on", "true", "yes"})
# Force-native GPU loads only: without it the installer defaults to "cpu" and runs on CPU.
_INSTALL_ACCELERATOR = {"rocm": "rocm", "cuda": "cuda", "xpu": "vulkan"}
def _install_accelerator_for(backend: str) -> str:
return _INSTALL_ACCELERATOR.get(backend, "auto")
_OFF_TORCH_VENDOR_ACCELERATOR = {"nvidia": "cuda"}
_off_torch_warned: set[str] = set()
@dataclass(frozen = True)
class OffTorchDevice:
"""A card torch cannot see. ``index`` is nvidia-smi's row, never a torch ordinal."""
vendor: str
index: int
accelerator: str
@property
def label(self) -> str:
return f"{self.vendor}:{self.index}"
def child_env(self) -> dict[str, str]:
# nvidia-smi numbers cards in PCI order; CUDA's default is fastest-first.
return {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": str(self.index)}
def _warn_off_torch_once(raw: str, message: str, *args: Any) -> None:
if raw in _off_torch_warned:
return
_off_torch_warned.add(raw)
logger.warning(message, *args)
def _physical_inventory() -> dict:
return _card_lookup_inventory() or {}
def off_torch_sd_cpp_device(backend: Optional[str] = None) -> Optional[OffTorchDevice]:
"""The card ``UNSLOTH_DIFFUSION_SD_CPP_DEVICE`` asks native image generation to run on, when
torch cannot drive it (an NVIDIA card beside a ROCm torch). None keeps today's routing: unset,
unparseable, a vendor torch already serves, or an inventory that answered without that card."""
raw = os.environ.get("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "").strip().lower()
if not raw:
return None
vendor, _, index_text = raw.partition(":")
vendor = vendor.strip()
accelerator = _OFF_TORCH_VENDOR_ACCELERATOR.get(vendor)
try:
index = int(index_text) if index_text.strip() else 0
except ValueError:
index = -1
if accelerator is None or index < 0:
_warn_off_torch_once(
raw,
"UNSLOTH_DIFFUSION_SD_CPP_DEVICE=%r is not understood (expected nvidia or "
"nvidia:<index>); ignoring it",
raw,
)
return None
if backend is None:
backend = resolve_diffusion_device_target().backend
if _INSTALL_ACCELERATOR.get(backend) == accelerator:
return None
inventory = _physical_inventory()
unanswered = set(inventory.get("unanswered") or ())
# Per vendor: an unanswered AMD probe says nothing about NVIDIA. Unknown naming nobody = cold cache.
if vendor not in unanswered and not (inventory.get("unknown") and not unanswered):
present = any(
isinstance(device, dict)
and device.get("vendor") == vendor
and device.get("index") == index
for device in inventory.get("devices") or ()
)
if not present:
_warn_off_torch_once(
raw,
"UNSLOTH_DIFFUSION_SD_CPP_DEVICE=%s: this host lists no such card; ignoring it",
raw,
)
return None
return OffTorchDevice(vendor = vendor, index = index, accelerator = accelerator)
# Hosts the pinned release ships a Vulkan sd.cpp bundle for.
_VULKAN_SD_CPP_PLATFORMS = frozenset({("linux", "x86_64"), ("win32", "amd64")})
@functools.lru_cache(maxsize = 1)
def _vulkan_loader_loads() -> bool:
"""Whether the Vulkan loader library opens here. The Vulkan build links it, so without it
neither sd-cli nor sd-server would start."""
import ctypes
try:
if sys.platform == "win32":
ctypes.WinDLL("vulkan-1.dll")
else:
ctypes.CDLL("libvulkan.so.1")
return True
except Exception: # noqa: BLE001
return False
def _masked_vendors() -> set:
"""Vendors a set runtime mask applies to (HIP honours CUDA_VISIBLE_DEVICES too). Vulkan reads
none of them, so a card hidden by one stays hidden. Unreadable counts as every vendor."""
try:
from utils.hardware.hardware import _VISIBILITY_MASK_VENDORS, _ze_affinity_mask
masked = set()
for var, vendors in _VISIBILITY_MASK_VENDORS.items():
value = _ze_affinity_mask() if var == "ZE_AFFINITY_MASK" else os.environ.get(var)
if value is not None:
masked |= vendors
return masked
except Exception: # noqa: BLE001
return {"amd", "intel", "nvidia"}
# PCI vendors of the cards this route renders on.
_RENDER_NODE_VENDORS = {"0x1002": "amd", "0x8086": "intel"}
def _accessible_render_vendors(vendors) -> set:
"""Which of ``vendors`` have a readable and writable /dev/dri render node. A container can list
a card in sysfs without mounting its node, and Vulkan then sees no device. Empty when another
vendor's GPU (virtio, VMware) also has an openable node, since unpinned sd.cpp could take it first."""
import glob
found = set()
for node in glob.glob("/dev/dri/renderD*"):
accessible = os.access(node, os.R_OK | os.W_OK)
try:
with open(
f"/sys/class/drm/{os.path.basename(node)}/device/vendor", encoding = "utf-8"
) as f:
pci_vendor = f.read().strip().lower()
except OSError:
# An openable node of unknown vendor could be the device sd.cpp picks first.
if accessible:
return set()
continue
vendor = _RENDER_NODE_VENDORS.get(pci_vendor)
if vendor is None:
if accessible:
return set()
continue
if vendor in vendors and accessible:
found.add(vendor)
return found
def _reachable_intel_arc_render_node() -> bool:
"""True when this process can open the render node of an Intel Arc itself (PCI class per node), not
only some Intel node: beside a UHD iGPU, a container may mount the iGPU's node alone."""
import glob
from utils.hardware.hardware import _intel_pci_device_is_xpu_class
for node in glob.glob("/dev/dri/renderD*"):
if not os.access(node, os.R_OK | os.W_OK):
continue
device = f"/sys/class/drm/{os.path.basename(node)}/device"
try:
with open(f"{device}/vendor", encoding = "utf-8") as f:
if _RENDER_NODE_VENDORS.get(f.read().strip().lower()) != "intel":
continue
except OSError:
continue
try:
if _intel_pci_device_is_xpu_class(device) is True:
return True
except Exception: # noqa: BLE001 -- unknown class: not proof the Arc is reachable
continue
return False
def _vulkan_gpu_for_cpu_torch() -> bool:
"""True when torch is CPU-only but the host has a GPU the Vulkan sd.cpp build can render on:
an AMD card, an Intel Arc, or any Intel GPU once the llama.cpp backend is set to Vulkan. Never
beside NVIDIA, which Vulkan could enumerate first, nor under a mask Vulkan does not read (#13666)."""
if not sd_cpp_vulkan_fallback_enabled():
return False
if (sys.platform, platform.machine().lower()) not in _VULKAN_SD_CPP_PLATFORMS:
return False
# ggml reads this one itself; anything but a list of ordinals selects nothing or throws. Its
# ordinals index the raw Vulkan list, so it never vouches for which card a mixed host gets.
vk_mask = os.environ.get("GGML_VK_VISIBLE_DEVICES")
if vk_mask is not None and not re.fullmatch(r"\s*\d+(?:[\s,]+\d+)*[\s,]*", vk_mask):
return False
inventory = _physical_inventory()
if inventory.get("unknown") or "nvidia" in (inventory.get("unanswered") or ()):
return False
devices = [d for d in inventory.get("devices") or () if isinstance(d, dict)]
if any(d.get("vendor") == "nvidia" for d in devices) or _nvidia_adapter_present():
return False
from utils.prebuilt.llama_backend import environment_backend_override
# A non-Arc iGPU can render slower than the CPU build, so only an explicit Vulkan request takes it.
vulkan_requested = (
environment_backend_override(
os.environ.get("UNSLOTH_LLAMA_CPP_BACKEND"), os.environ.get("UNSLOTH_FORCE_VULKAN")
)
== "vulkan"
)
def _renders(device: dict) -> bool:
if device.get("vendor") == "amd":
return True
if device.get("vendor") != "intel":
return False
if vulkan_requested:
return True
# Linux carries the PCI class, the Windows registry only the name.
if "xpu_class" in device:
return device.get("xpu_class") is True
return "arc" in str(device.get("name") or "").lower().replace("(tm)", " ").split()
gpus = [d for d in devices if d.get("vendor") in ("amd", "intel")]
if sys.platform != "win32":
# Vulkan only sees cards whose render node this process can open (a container may mount
# one vendor's), and the node and the driver must belong to the same vendor.
reachable = _accessible_render_vendors({d.get("vendor") for d in gpus})
gpus = [d for d in gpus if d.get("vendor") in reachable]
masked = _masked_vendors()
if not gpus or any(d.get("vendor") in masked for d in gpus):
return False
rendering = [d for d in gpus if _renders(d)]
if len(rendering) < len(gpus):
# Unpinned sd.cpp takes a discrete GPU before an iGPU (get_default_backend_name); a mask's ordinals may name the iGPU.
if vk_mask is not None or not rendering:
return False
if any(d.get("vendor") != "intel" for d in gpus if not _renders(d)):
return False
# The vendor-level node check cannot tell an Arc's node from the iGPU's: an unopenable Arc vouches for nothing.
if sys.platform != "win32" and not _reachable_intel_arc_render_node():
rendering = [d for d in rendering if d.get("vendor") != "intel"]
if not rendering:
return False
# The driver must belong to a card that renders: beside an AMD card, an Intel-only ICD drives just the iGPU.
rendering_vendors = {d.get("vendor") for d in rendering}
return _hardware_vulkan_driver(rendering_vendors) and _vulkan_loader_loads()
# The ICD manifests that drive each vendor's cards (RADV as radeon_icd or radv_icd, AMDVLK /
# Adrenalin, ANV); a software rasteriser or another vendor's driver lists none of them.
_VENDOR_ICD_NEEDLES = {"amd": ("radeon", "radv", "amd"), "intel": ("intel",)}
def _hardware_vulkan_driver(vendors) -> bool:
"""Whether the loader, under its active filters, would load a driver for one of vendors; Windows
registers drivers the manifest walk may not read, so an empty walk there is no evidence."""
try:
from utils.hardware.amd import _loadable_icd_manifests, _vulkan_icd_manifest_paths
if not _vulkan_icd_manifest_paths():
# A driver filter the walk cannot apply may leave the loader with none.
filtered = os.environ.get("VK_LOADER_DRIVERS_SELECT") or os.environ.get(
"VK_LOADER_DRIVERS_DISABLE"
)
return sys.platform == "win32" and not filtered
needles = [n for vendor in vendors for n in _VENDOR_ICD_NEEDLES.get(vendor, ())]
return any(
any(needle in os.path.basename(str(path)).lower() for needle in needles)
for path in _loadable_icd_manifests()
)
except Exception: # noqa: BLE001
return False
def _nvidia_adapter_present() -> bool:
"""An NVIDIA card nvidia-smi cannot see (Nouveau, no proprietary driver, no nvidia-smi) is still
a Vulkan device. Unanswerable counts as present."""
if sys.platform == "win32":
try:
from utils.hardware.hardware import (
_windows_amd_adapter_records_by_luid,
_windows_live_adapter_names,
)
records = _windows_amd_adapter_records_by_luid(0x10DE, distinguish_failure = True)
if records is None:
return True
if not records:
return False
# The registry outlives the hardware, so a record only counts if the live scan agrees.
live = _windows_live_adapter_names()
return live is None or any("nvidia" in name.lower() for name in live)
except Exception: # noqa: BLE001
return True
import glob
for vendor_file in glob.glob("/sys/class/drm/card*/device/vendor"):
try:
with open(vendor_file, encoding = "utf-8") as f:
if f.read().strip().lower() == "0x10de":
return True
except OSError:
continue
return False
_CUDA_SD_CPP_PLATFORMS = frozenset({("linux", "x86_64"), ("win32", "amd64")})
def _cuda_gpu_for_cpu_torch() -> bool:
"""No GPU torch beside an NVIDIA card llama.cpp's CUDA build runs on (proves the driver, #16803)."""
# Gated with the Vulkan route: the CPU rung below a failed CUDA build is the same ladder.
if not sd_cpp_vulkan_fallback_enabled():
return False
if (sys.platform, platform.machine().lower()) not in _CUDA_SD_CPP_PLATFORMS:
return False
mask = os.environ.get("CUDA_VISIBLE_DEVICES")
if mask is not None and mask.strip() in ("", "-1"):
return False
from core.inference.sd_cpp_engine import llama_cpp_cuda_major
if not llama_cpp_cuda_major():
return False
inventory = _physical_inventory()
return any(
isinstance(device, dict) and device.get("vendor") == "nvidia"
for device in inventory.get("devices") or ()
)
def image_install_accelerator(backend: str) -> str:
"""The sd.cpp build the IMAGE engine wants: the off-torch card's when one is set, else torch's,
else Vulkan for an eligible AMD or Intel GPU, or CUDA beside an NVIDIA card llama.cpp already
renders on. Video keeps ``_install_accelerator_for``, since torch places its loads."""
off_torch = off_torch_sd_cpp_device(backend)
if off_torch is not None:
return off_torch.accelerator
if backend == "cpu" and _vulkan_gpu_for_cpu_torch():
return "vulkan"
if backend == "cpu" and _cuda_gpu_for_cpu_torch():
return "cuda"
return _install_accelerator_for(backend)
# The engine the current load committed to, and why a non-native choice was made. Mutated only under _lock.
_lock = threading.Lock()
# Serializes a whole engine switch (check -> unload -> publish); _lock alone is released during the slow unload().
_transition_lock = threading.Lock()
_active_engine_name: str = ENGINE_DIFFUSERS
_fallback_reason: Optional[str] = None
# Runs under _transition_lock just before a switch unloads the active engine, and may raise to refuse it.
before_engine_switch: ContextVar[Optional[Callable[[], None]]] = ContextVar(
"before_engine_switch", default = None
)
_mlx_fallback_reason: Optional[str] = None
# The explicit engine the active engine's last load asked for (diffusers | sd_cpp), None when it was left to auto.
_engine_pick: Optional[str] = None
_MLX_MIN_MACOS = (14, 0)
MLX_OFF = "off"
MLX_AUTO = "auto"
MLX_FORCE = "force"
def mlx_mode() -> str:
"""``UNSLOTH_DIFFUSION_MLX``: unset / ``auto`` routes per request, ``1`` forces MLX, else off."""
raw = os.environ.get("UNSLOTH_DIFFUSION_MLX", "auto").strip().lower()
if raw in ("", "auto"):
return MLX_AUTO
if raw in _ENABLE_TOKENS:
return MLX_FORCE
return MLX_OFF
def _mlx_host_refusal() -> Optional[str]:
if sys.platform != "darwin":
return "the MLX engine runs on macOS only"
if platform.machine().lower() not in ("arm64", "aarch64"):
return "the MLX engine needs Apple Silicon"
release = platform.mac_ver()[0]
try:
version = tuple(int(part) for part in release.split(".")[:2])
except ValueError:
version = ()
if not version or version < _MLX_MIN_MACOS:
return f"the MLX engine needs macOS 14 or newer (found {release or 'unknown'})"
return None
# mflux's own darwin mlx range plus the light deps install_python_stack.py installs beside it (mflux goes in --no-deps).
_MLX_RUNTIME_REQUIREMENTS: tuple[str, ...] = (
"mlx>=0.32.0,<0.33.0",
# Pillow 13.0.0 breaks every mflux import (PIL._typing), so it is refused here, not at load.
"pillow>=12.3.0,<13",
"piexif>=1.1.3,<2",
"platformdirs>=4,<5",
"toml>=0.10.2,<1",
)
def _installed_version(name: str) -> Optional[str]:
try:
from importlib.metadata import PackageNotFoundError, version
return version(name)
except (PackageNotFoundError, ValueError):
return None
def _mlx_runtime_refusal() -> Optional[str]:
"""find_spec and distribution metadata only: never import mlx / mflux here."""
missing = [name for name in ("mlx", "mflux") if importlib.util.find_spec(name) is None]
if missing:
return f"{' and '.join(missing)} not installed"
from core.inference.mlx_diffusion_convert import MFLUX_VERSION
try:
from packaging.requirements import Requirement
for spec in (f"mflux=={MFLUX_VERSION}", *_MLX_RUNTIME_REQUIREMENTS):
requirement = Requirement(spec)
installed = _installed_version(requirement.name)
if installed is None or not requirement.specifier.contains(installed, prereleases = True):
return f"the MLX engine needs {spec} (found {installed or 'none'})"
except Exception as exc: # noqa: BLE001 -- no packaging, or a version it cannot parse
return f"could not verify the MLX runtime versions ({exc})"
return None
def has_active_loras(loras: Any) -> bool:
"""Every loader drops weight-0 entries (``diffusion_lora.resolve_specs``, sd.cpp's ``active_loras``),
so routing must ignore them too. An unreadable weight counts as active so a real adapter never hits MLX."""
for entry in loras or ():
weight = getattr(entry, "weight", None)
if weight is None:
try:
_lid, weight = entry
except (TypeError, ValueError):
return True
try:
if float(weight) != 0:
return True
except (TypeError, ValueError):
return True
return False
def mlx_selection(
fam: Optional[DiffusionFamily],
*,
model_kind: Optional[str],
gguf_filename: Optional[str],
backend: Optional[str] = None,
engine: Optional[str] = None,
with_loras: bool = False,
load_blocker: Optional[str] = None,
) -> tuple[bool, Optional[str]]:
"""``(use_mlx, refusal)``. Shared by selection and ``predict_engine`` so the download plan and the load
cannot disagree about MLX. ``engine`` is the load's own pick (``/images/load``'s ``engine`` field).
``with_loras`` says the load applies LoRA adapters: a family whose MLX build cannot keeps the engine it would have
had without MLX (in every mode) instead of reaching the MLX LoRA refusal.
``load_blocker`` (``mlx_load_blocker``) names a load option MLX cannot honour: in ``auto`` it keeps the load on
diffusers; a forced MLX engine refuses that load in the route instead, as before."""
mode = mlx_mode()
if mode == MLX_OFF:
return False, None
if mode == MLX_AUTO and _mlx_host_refusal():
return False, None
forced = os.environ.get("UNSLOTH_DIFFUSION_ENGINE", "auto").strip().lower()
if forced in (ENGINE_DIFFUSERS, ENGINE_SD_CPP):
return False, f"UNSLOTH_DIFFUSION_ENGINE={forced} overrides UNSLOTH_DIFFUSION_MLX"
if engine in (ENGINE_DIFFUSERS, ENGINE_SD_CPP):
return False, f"the load asked for the {engine} engine"
if mode == MLX_AUTO and _engine_config()[2]:
return False, "UNSLOTH_DIFFUSION_SD_CPP_MPS=1 asks for the native engine on the Mac GPU"
if model_kind and model_kind != "gguf":
return False, f"the MLX engine loads GGUF picks only (got {model_kind})"
if fam is None or not family_mlx_supported(fam):
name = getattr(fam, "name", None) or "unknown"
return False, f"family '{name}' has no MLX implementation"
if with_loras:
from core.inference.mlx_diffusion_routing import mlx_capabilities
if not mlx_capabilities(fam.name).loras:
return False, f"the MLX engine does not apply LoRA adapters to {fam.name} yet"
from core.inference.mlx_diffusion_convert import quant_convertible
convertible, why = quant_convertible(gguf_filename)
if not convertible:
return False, why or "GGUF quant not convertible to MLX"
host = _mlx_host_refusal()
if host:
return False, host
if backend is None:
backend = resolve_diffusion_device_target().backend
if backend not in ("mps", "cpu"):
return False, f"device backend '{backend}' is not Apple MPS"
runtime = _mlx_runtime_refusal()
if runtime:
return False, runtime
if mode == MLX_AUTO and load_blocker:
return False, f"{load_blocker} (diffusers serves this load)"
return True, None
def mlx_load_blocker(
*,
component_files: Any = None,
transformer_quant: Optional[str] = None,
text_encoder_quant: Optional[str] = None,
) -> Optional[str]:
from core.inference.mlx_diffusion_routing import load_blocker
return load_blocker(
component_files = component_files,
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
)
def _refuse_lora_fallback(
fam: DiffusionFamily,
choice: str,
*,
with_loras: bool,
model_kind: Optional[str],
gguf_filename: Optional[str],
engine: Optional[str],
) -> None:
if not with_loras or mlx_mode() != MLX_FORCE:
return
if not mlx_selection(fam, model_kind = model_kind, gguf_filename = gguf_filename, engine = engine)[0]:
return
from .diffusion_lora import supports_lora
if supports_lora(engine = choice, family = fam.name, model_kind = "gguf", transformer_quant = None):
return
via = "the native sd.cpp engine" if choice == ENGINE_SD_CPP else "the diffusers engine"
raise ValueError(
"LoRA adapters are not supported on the MLX engine yet, and this load would fall back to "
f"{via}, which cannot apply them here either. Load it without LoRA adapters."
)
def _note_mlx_refusal(reason: Optional[str]) -> None:
global _mlx_fallback_reason
with _lock:
_mlx_fallback_reason = reason
if reason:
if mlx_mode() == MLX_FORCE:
logger.warning("UNSLOTH_DIFFUSION_MLX=1 but the MLX engine is not used: %s", reason)
else:
logger.info("MLX engine not used for this load: %s", reason)
# Values of ``/images/load``'s ``engine`` field. "auto" (or absent) keeps the host's own choice.
ENGINE_CHOICES: tuple[str, ...] = ("auto", ENGINE_DIFFUSERS, ENGINE_SD_CPP)
def env_forced_engine() -> Optional[str]:
"""The engine UNSLOTH_DIFFUSION_ENGINE forces, else None."""
forced = os.environ.get("UNSLOTH_DIFFUSION_ENGINE", "auto").strip().lower()
return forced if forced in (ENGINE_DIFFUSERS, ENGINE_SD_CPP) else None
def _requested_engine(engine: Optional[str]) -> Optional[str]:
"""The load's own pick when it names an engine and the env does not force one."""
pick = (engine or "auto").strip().lower()
if pick not in ENGINE_CHOICES:
raise ValueError(f"engine must be one of {', '.join(ENGINE_CHOICES)} (got {engine!r}).")
if pick == "auto" or env_forced_engine() is not None:
return None
return pick
def _engine_config(engine: Optional[str] = None) -> tuple[str, str, bool]:
forced = os.environ.get("UNSLOTH_DIFFUSION_ENGINE", "auto").strip().lower()
requested = _requested_engine(engine)
if requested is not None:
forced = requested
sd_cpp = os.environ.get("UNSLOTH_DIFFUSION_SD_CPP", "auto").strip().lower()
mps = os.environ.get("UNSLOTH_DIFFUSION_SD_CPP_MPS", "0").strip().lower() in _ENABLE_TOKENS
return forced, sd_cpp, mps
def engine_for(name: str) -> Any:
"""The engine object a name refers to, WITHOUT activating it: activating unloads the resident
model, so /images/load's gated-repo preflight needs the pending engine before the switch."""
if name == ENGINE_SD_CPP:
from core.inference.sd_cpp_backend import get_sd_cpp_backend
return get_sd_cpp_backend()
if name == ENGINE_MLX:
if mlx_mode() == MLX_AUTO:
from core.inference.mlx_diffusion_routing import get_routed_mlx_engine
return get_routed_mlx_engine()
from core.inference.mlx_diffusion_backend import get_mlx_diffusion_backend
return get_mlx_diffusion_backend()
from core.inference.diffusion import get_diffusion_backend
return get_diffusion_backend()
def get_active_diffusion_engine() -> Any:
"""The engine object the active selection points at (defaults to diffusers)."""
return engine_for(_active_engine_name)
# (module, singleton attribute) of every image engine, read from sys.modules only.
_ENGINE_SINGLETONS = (
("core.inference.diffusion", "_diffusion_backend"),
("core.inference.sd_cpp_backend", "_sd_cpp_backend"),
("core.inference.mlx_diffusion_backend", "_mlx_diffusion_backend"),
)
def cancel_generation_for_account(account_id: str) -> bool:
"""Stop an in-flight image generation owned by ``account_id``; True when one was signalled.
Engines come from ``sys.modules`` (no import, no construction); all are checked because a
deselected engine can still be draining."""
cancelled = False
for module_name, attribute in _ENGINE_SINGLETONS:
module = sys.modules.get(module_name)
engine = getattr(module, attribute, None) if module is not None else None
if engine is None or engine._active_generate_account != account_id:
continue
# cancel_generate rechecks the owner under its lock.
if engine.cancel_generate(expected_account = account_id):
cancelled = True
routing = sys.modules.get("core.inference.mlx_diffusion_routing")
routed = getattr(routing, "_routed_mlx_engine", None) if routing is not None else None
if routed is not None and routed._active_generate_account == account_id:
if routed.cancel_generate(expected_account = account_id):
cancelled = True
return cancelled
def retire_load_for_account(account_id: str) -> bool:
"""Tear down an in-flight image load ``account_id`` started; True when one was found."""
from hub.services.models.account_access import retire_media_load
retired = False
for module_name, attribute in _ENGINE_SINGLETONS:
module = sys.modules.get(module_name)
engine = getattr(module, attribute, None) if module is not None else None
if retire_media_load("diffusion", account_id, engine):
retired = True
return retired
def active_engine_name() -> str:
return _active_engine_name
def _activate(name: str, reason: Optional[str]) -> Any:
global _active_engine_name, _fallback_reason
with _transition_lock:
# Unload the old engine first (else it leaks VRAM), outside _lock since unload is slow.
engine_to_unload = None
old_name = None
with _lock:
if name != _active_engine_name:
engine_to_unload = get_active_diffusion_engine()
old_name = _active_engine_name
else:
_fallback_reason = reason if name == ENGINE_DIFFUSERS else None
check = before_engine_switch.get() if engine_to_unload is not None else None
if check is not None:
check()
if engine_to_unload is not None:
# A launching FLUX.3 upsampler warm start holds the old engine's generation slot the unload waits on.
_upsampler = sys.modules.get("core.inference.flux3_prompt_upsampler")
if _upsampler is not None:
try:
_upsampler.cancel_warm_start()
except Exception as exc: # noqa: BLE001 - the start times out on its own
logger.warning("could not cancel a prompt upsampler warm start: %s", exc)
# Publish the new engine only AFTER the old one unloads: the evictor unloads
# get_active_diffusion_engine(), so flipping the name first would let a concurrent acquire_for evict the
# new (empty) engine while the old model frees VRAM.
try:
engine_to_unload.unload()
except Exception as exc:
# Do not publish after a failed teardown: the old model would leak, hidden from the evictor.
logger.error("failed to unload previous engine %s: %s", old_name, exc)
raise RuntimeError(
f"Could not switch the diffusion engine to {name}: unloading the current "
f"{old_name} model failed ({exc}). The current model is still loaded; "
"unload it and try again."
) from exc
with _lock:
_active_engine_name = name
_fallback_reason = reason if name == ENGINE_DIFFUSERS else None
if name == ENGINE_SD_CPP:
logger.info("diffusion engine: sd_cpp")
elif name == ENGINE_MLX:
logger.info("diffusion engine: mlx")
else:
logger.info("diffusion engine: diffusers (%s)", reason or "selected")
return get_active_diffusion_engine()
def begin_load_on(expected_engine: Any, start: Callable[[], Any]) -> Any:
"""Re-checks the engine under the transition lock; a second load can switch engines in the start gap."""
with _transition_lock:
if expected_engine is not get_active_diffusion_engine():
raise RuntimeError(
"The diffusion engine changed while this load was starting. Retry the load."
)
return start()
def _selected_card(gpu_ordinal) -> Optional[str]:
"""The card at an already RESOLVED ordinal, or ``None``, meaning every record applies. Never
re-derived from the id list: free-VRAM ranking can name a different card the second time."""
if gpu_ordinal is None:
return None
try:
from core.inference.sd_cpp_backend import selected_card_identity
return selected_card_identity(gpu_ordinal)
except Exception: # noqa: BLE001
return None
# A started load's pick, committed to ``_engine_pick`` only once that load's state is published: (pick, engine, the
# engine's state when the load started).
_pending_engine_pick: Optional[tuple[Optional[str], Any, Any]] = None
def _settle_engine_pick_locked() -> None:
"""Commit the pending pick once its engine holds a NEW loaded state. A background load that fails before it
publishes keeps the old state resident, and with it the old pick; one that tore the old state down leaves nothing
loaded to label, and a later load replaces the pending entry."""
global _engine_pick, _pending_engine_pick
if _pending_engine_pick is None:
return
pick, engine, before = _pending_engine_pick
current = loaded_state_of(engine)
if current is not None and current is not before:
_engine_pick = pick
_pending_engine_pick = None
_NO_SNAPSHOT = object()
def loaded_state_of(engine: Any) -> Any:
"""The engine's published load state (identity only), for ``note_engine_pick``'s before-load snapshot."""
return getattr(engine, "_state", None)
def note_engine_pick(
engine: Optional[str],
loaded_by: Any = None,
before: Any = _NO_SNAPSHOT,
) -> None:
"""Record the load's engine pick for the status provenance. The load route calls it once ``loaded_by`` (the
engine) has accepted the load, inside ``begin_load_on``'s transition lock, so a pick refused by admission,
preflight or the arbiter never relabels the resident model and concurrent loads register in order. With an engine
the pick is committed only when that load lands (``_settle_engine_pick_locked``); without one, at once.
``before`` is ``loaded_state_of(loaded_by)`` taken before the load's worker started: a cached load can publish
before ``begin_load`` returns, and a snapshot taken after it would wait for a state that never comes."""
global _engine_pick, _pending_engine_pick
pick = _requested_engine(engine)
with _lock:
_settle_engine_pick_locked()
if loaded_by is None:
_engine_pick = pick
_pending_engine_pick = None
else:
prior = loaded_state_of(loaded_by) if before is _NO_SNAPSHOT else before
_pending_engine_pick = (pick, loaded_by, prior)
_settle_engine_pick_locked()
def select_and_activate_engine(
fam: DiffusionFamily,
*,
hf_token: Optional[str] = None,
model_kind: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
before_fallback: Optional[Callable[[], None]] = None,
gguf_filename: Optional[str] = None,
engine: Optional[str] = None,
with_loras: bool = False,
mlx_blocker: Optional[str] = None,
) -> Any:
"""Pick + activate the engine for loading ``fam`` on this host; return the engine.
``engine`` is the load's own pick (auto | diffusers | sd_cpp). An explicit one the host or the pick cannot honour
raises ValueError BEFORE anything is unloaded (``refuse_engine_pick``); UNSLOTH_DIFFUSION_ENGINE wins over it.
``before_fallback`` may raise to refuse a diffusers fallback before it unloads the resident model.
Falls back to diffusers (recording a reason) when the native route is disabled, the device has
a usable GPU, MPS is not enabled, the family has no native asset, or the binary is unavailable
-- always BEFORE the slow load, so a fallback never strands a half-native load.
On an Apple Silicon Mac the MLX engine is tried first unless UNSLOTH_DIFFUSION_MLX=0 (``mlx_selection``, which
needs ``gguf_filename`` to judge the quant, and ``mlx_blocker`` for a load option MLX cannot honour); a refusal is
recorded and the choice below runs unchanged.
"""
requested = _requested_engine(engine)
refuse_engine_pick(fam, engine = engine, model_kind = model_kind)
use_mlx, mlx_refusal = mlx_selection(
fam,
model_kind = model_kind,
gguf_filename = gguf_filename,
engine = requested,
with_loras = with_loras,
load_blocker = mlx_blocker,
)
_note_mlx_refusal(mlx_refusal)
if use_mlx:
return _activate(ENGINE_MLX, None)
if model_kind and model_kind != "gguf":
return _activate(ENGINE_DIFFUSERS, f"non-GGUF load ({model_kind}) requires diffusers")
forced, sd_cpp_pref, mps_enabled = _engine_config(engine)
if forced == ENGINE_DIFFUSERS:
return _activate(
ENGINE_DIFFUSERS,
"selected for this load"
if requested == ENGINE_DIFFUSERS
else "forced (UNSLOTH_DIFFUSION_ENGINE=diffusers)",
)
lora_guard = functools.partial(
_refuse_lora_fallback,
fam,
with_loras = with_loras,
model_kind = model_kind,
gguf_filename = gguf_filename,
engine = requested,
)
prefer_native = forced == ENGINE_SD_CPP
if sd_cpp_pref in _DISABLE_TOKENS and not prefer_native:
lora_guard(ENGINE_DIFFUSERS)
return _activate(ENGINE_DIFFUSERS, "native engine disabled (UNSLOTH_DIFFUSION_SD_CPP=0)")
if native_gguf_only(fam, model_kind):
prefer_native = True
target = resolve_diffusion_device_target()
backend = target.backend
off_torch = off_torch_sd_cpp_device(backend)
if off_torch is not None:
# A card torch cannot see is only reachable natively, and torch's ordinals name other cards.
prefer_native = True
gpu_ordinal = None
# CPU always native-eligible; MPS only when enabled; a GPU backend never, unless forced
policy_eligible = backend == "cpu" or (backend == "mps" and mps_enabled) or prefer_native
fam_ok = family_sd_cpp_supported(fam)
binary = None
server_binary = None
incapable_build = False
if policy_eligible and fam_ok:
# Once, so server and CLI cannot disagree.
selected_card = _selected_card(gpu_ordinal)
install_accelerator = preferred_accelerator(
image_install_accelerator(backend), selected_card
)
if off_torch is not None:
install_accelerator = preferred_accelerator(off_torch.accelerator, selected_card)
# Offline an ensure returns the condemned ROCm build; a deferred upgrade keeps native.
upgrade_is_deferred = _managed_tree_in_use() and _install_allowed()
def _accept(candidate):
if candidate and upgrade_is_deferred:
return candidate
wrong_build = off_torch_build_mismatch(off_torch, candidate)
if wrong_build:
logger.warning(
"%s is the %s sd.cpp build; %s needs %s",
candidate,
wrong_build,
off_torch.label,
off_torch.accelerator,
)
return None
return usable_or_recorded_failure(candidate, install_accelerator, selected_card)
# A CPU torch has nothing else on the card: a GPU build that cannot start steps down now.
for _rung in range(2):
server_binary = _accept(
ensure_sd_server_binary(
allow_install = _install_allowed(),
accelerator = install_accelerator,
)
)
unlaunchable_server: Optional[str] = None
if server_binary and not _server_binary_runnable(server_binary):
logger.warning(
"sd-server at %s is present but not runnable; not using it", server_binary
)
# Held for the single recorder below.
unlaunchable_server = server_binary
server_binary = None
# Probe runnability first, else a non-runnable binary fails inside the background load.
binary = _accept(
ensure_sd_cpp_binary(
allow_install = _install_allowed() and server_binary is None,
accelerator = install_accelerator,
)
)
unlaunchable_cli: Optional[str] = None
if binary and SdCppEngine(binary = binary).version() is None:
logger.warning("sd-cli at %s is present but not runnable; not using it", binary)
unlaunchable_cli = binary
binary = None
if (
binary is None
and server_binary is None
and (unlaunchable_cli or unlaunchable_server)
):
# One strike per bundle, only when neither executable runs.
note_unlaunchable_accelerator_build(
unlaunchable_cli or unlaunchable_server, card = selected_card
)
stepped = preferred_accelerator(image_install_accelerator(backend), selected_card)
if (
backend == "cpu"
and off_torch is None
and _rung == 0
and stepped != install_accelerator
):
logger.warning(
"the %s sd.cpp build cannot start here; using the %s build",
install_accelerator,
stepped,
)
install_accelerator = stepped
continue
break
# Runnable is not capable: an old build is never upgraded and fails late on new families.
for name, candidate in (("sd-server", server_binary), ("sd-cli", binary)):
if candidate and not sd_cpp_binary_runs_family(candidate, fam):
incapable_build = True
logger.warning(
"%s at %s predates %s support; using diffusers. Reinstall the native engine "
"(delete the managed stable-diffusion.cpp install, or point SD_CLI_PATH at a "
"build from after upstream added it) to use it here",
name,
candidate,
fam.name,
)
if name == "sd-server":
server_binary = None
else:
binary = None
native_available = bool(binary or server_binary) and policy_eligible and fam_ok
choice = select_diffusion_engine(
backend, native_available = native_available, prefer_native = prefer_native
)
lora_guard(choice)
if choice == ENGINE_SD_CPP:
return _activate(ENGINE_SD_CPP, None)
# Explain the diffusers choice for status/telemetry.
if not policy_eligible:
reason = f"GPU backend '{backend}' uses diffusers"
elif not fam_ok:
reason = f"family '{fam.name}' has no native sd.cpp asset mapping"
elif incapable_build:
reason = f"the installed sd.cpp build predates '{fam.name}' support"
elif not (binary or server_binary):
reason = "native sd.cpp binary unavailable"
else:
reason = "diffusers selected"
if off_torch is not None:
reason = f"{reason}; UNSLOTH_DIFFUSION_SD_CPP_DEVICE={off_torch.label} not honoured"
logger.warning("image load stays on torch's device: %s", reason)
if requested == ENGINE_SD_CPP or native_gguf_only(fam, model_kind):
# Refuse before anything unloads rather than silently fall back to diffusers.
raise ValueError(f"The native sd.cpp engine cannot load this model here: {reason}.")
if before_fallback is not None:
before_fallback()
return _activate(ENGINE_DIFFUSERS, reason)
def native_gguf_only(fam: Optional[DiffusionFamily], model_kind: Optional[str]) -> bool:
"""A GGUF of a family whose transformer-only files load natively only (``native_transformer_files_only``)."""
return model_kind == "gguf" and bool(getattr(fam, "native_transformer_files_only", False))
def refuse_engine_pick(
fam: Optional[DiffusionFamily], *, engine: Optional[str], model_kind: Optional[str]
) -> None:
"""ValueError for an explicit engine pick that cannot serve this load, checked without any side effect. The
binary-dependent native refusal comes later, from selection itself, still before anything is unloaded."""
requested = _requested_engine(engine)
if requested == ENGINE_SD_CPP:
if model_kind and model_kind != "gguf":
raise ValueError("The native sd.cpp engine loads GGUF models only.")
# Only the administrator's UNSLOTH_DIFFUSION_ENGINE=sd_cpp overrides the disable, and that pin already made
# ``requested`` None above.
if _engine_config()[1] in _DISABLE_TOKENS:
raise ValueError(
"The native sd.cpp engine is disabled on this server (UNSLOTH_DIFFUSION_SD_CPP=0)."
)
if fam is not None and not family_sd_cpp_supported(fam):
raise ValueError(
f"The native sd.cpp engine has no support for the {fam.name} model family."
)
elif requested == ENGINE_DIFFUSERS:
if native_gguf_only(fam, model_kind):
raise ValueError(
f"Diffusers cannot load a {fam.name} GGUF; use the Auto or native engine, or the pipeline repo."
)
if fam is not None and not family_pipeline_available(fam):
raise ValueError(
f"Diffusers cannot build the {fam.name} model family with the installed version; "
"use the Auto or native engine."
)
def native_binary_installed(
*, gpu_ordinal: Optional[int] = None, fam: Optional[DiffusionFamily] = None
) -> bool:
"""Answers whether a runnable sd.cpp binary exists on disk, never installing one, unlike the
prediction."""
backend = resolve_diffusion_device_target().backend
off_torch = off_torch_sd_cpp_device(backend)
if off_torch is not None:
gpu_ordinal = None
selected_card = _selected_card(gpu_ordinal)
install_accelerator = preferred_accelerator(image_install_accelerator(backend), selected_card)
def _usable(candidate):
if off_torch_build_mismatch(off_torch, candidate):
return None
return usable_or_recorded_failure(candidate, install_accelerator, selected_card)
server_binary = _usable(
ensure_sd_server_binary(allow_install = False, accelerator = install_accelerator)
)
if (
server_binary
and _server_binary_runnable(server_binary)
and (fam is None or sd_cpp_binary_runs_family(server_binary, fam))
):
return True
binary = _usable(ensure_sd_cpp_binary(allow_install = False, accelerator = install_accelerator))
if fam is not None and binary and not sd_cpp_binary_runs_family(binary, fam):
return False
return bool(binary and SdCppEngine(binary = binary).version() is not None)
def _resident_build_upgrades_on_load(gpu_ordinal: Optional[int]) -> bool:
backend = resolve_diffusion_device_target().backend
if off_torch_sd_cpp_device(backend) is not None:
gpu_ordinal = None
accelerator = preferred_accelerator(
image_install_accelerator(backend), _selected_card(gpu_ordinal)
)
resident = ensure_sd_server_binary(
allow_install = False, accelerator = accelerator
) or ensure_sd_cpp_binary(allow_install = False, accelerator = accelerator)
return bool(resident) and _pin_moved(resident, accelerator)
def predict_engine(
fam: DiffusionFamily,
*,
model_kind: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
gguf_filename: Optional[str] = None,
engine: Optional[str] = None,
with_loras: bool = False,
mlx_blocker: Optional[str] = None,
) -> str:
"""The engine a load of ``fam`` would select on this host, WITHOUT any side effect. ``engine`` is the load's own
pick, as in ``select_and_activate_engine``; a pick selection would refuse raises the same ValueError here.
Same policy as ``select_and_activate_engine`` -- and it has to be, because the download plan
is built from it: the two engines need different files (sharded diffusers components vs
sd-cli's single-file VAE + text encoders), so a plan built for the wrong one stages GB the
load never opens and then fetches the right files inline, outside the manager.
It differs from selection in exactly two ways, both deliberate. Nothing is activated (staging
a download must not unload the resident model), and the binary is only LOCATED, never
installed -- but an absent binary on a host where install is allowed still counts as
available, because that is what the load will do. Getting that wrong the other way (planning
diffusers for the very first native load, when no binary is on disk yet) would mispredict the
common case: a fresh CPU host.
"""
requested = _requested_engine(engine)
# The plan must refuse what the load will refuse, rather than plan (and stage) for the other engine.
refuse_engine_pick(fam, engine = engine, model_kind = model_kind)
if mlx_selection(
fam,
model_kind = model_kind,
gguf_filename = gguf_filename,
engine = requested,
with_loras = with_loras,
load_blocker = mlx_blocker,
)[0]:
return ENGINE_MLX
if model_kind and model_kind != "gguf":
return ENGINE_DIFFUSERS
forced, sd_cpp_pref, mps_enabled = _engine_config(engine)
if forced == ENGINE_DIFFUSERS:
return ENGINE_DIFFUSERS
lora_guard = functools.partial(
_refuse_lora_fallback,
fam,
with_loras = with_loras,
model_kind = model_kind,
gguf_filename = gguf_filename,
engine = requested,
)
prefer_native = forced == ENGINE_SD_CPP
if sd_cpp_pref in _DISABLE_TOKENS and not prefer_native:
lora_guard(ENGINE_DIFFUSERS)
return ENGINE_DIFFUSERS
if native_gguf_only(fam, model_kind):
prefer_native = True
backend = resolve_diffusion_device_target().backend
if off_torch_sd_cpp_device(backend) is not None:
prefer_native = True
gpu_ordinal = None
policy_eligible = backend == "cpu" or (backend == "mps" and mps_enabled) or prefer_native
if not (policy_eligible and family_sd_cpp_supported(fam)):
lora_guard(ENGINE_DIFFUSERS)
return ENGINE_DIFFUSERS
# A permitted install counts as available only where selection would actually perform one.
# ``ensure_*_binary`` keeps a runnable build of the right accelerator untouched, so a resident
# build that cannot run this family is never upgraded away, and predicting native for it would
# stage sd-cli's companions for a load the router sends to diffusers. With nothing resident (or
# a moved pin) the install lands on the pinned prebuilt, which is current by definition.
native_available = native_binary_installed(gpu_ordinal = gpu_ordinal, fam = fam) or (
_install_allowed()
and (
not native_binary_installed(gpu_ordinal = gpu_ordinal)
or _resident_build_upgrades_on_load(gpu_ordinal)
)
)
choice = select_diffusion_engine(
backend, native_available = native_available, prefer_native = prefer_native
)
if (
requested == ENGINE_SD_CPP or native_gguf_only(fam, model_kind)
) and choice != ENGINE_SD_CPP:
# Selection raises the same refusal; planning diffusers here would stage GB for a load that cannot run.
why = (
"native sd.cpp binary unavailable"
if requested == ENGINE_SD_CPP
else f"no usable sd.cpp build here supports '{fam.name}'"
)
raise ValueError(f"The native sd.cpp engine cannot load this model here: {why}.")
lora_guard(choice)
return choice
def family_buildable_here(
fam: Optional[DiffusionFamily],
*,
model_kind: Optional[str],
gguf_filename: Optional[str] = None,
provisional: bool = False,
) -> bool:
"""True when THIS host can actually build ``fam`` for a ``model_kind`` load, by either engine.
``provisional`` (listing before detection) answers engine predictions True so detection never starts.
The two engines need different things. diffusers instantiates ``fam.pipeline_class``, which the
newer families ship only in a newer diffusers -- and packaging still allows an older one on
Python 3.9, whose ceiling predates them. The native sd.cpp engine assembles the same GGUF from
single-file assets and imports no pipeline class at all, so a supported family loads there on a
diffusers that has never heard of it.
A family is therefore unbuildable only when NEITHER engine can do it, and both one-sided answers
are wrong: gating on the diffusers class alone refuses (and hides) a GGUF the native engine
serves fine, on the very hosts the native engine exists for; not gating at all advertises a pick
that can only fail. The listing routes and ``validate_load_request`` share this one predicate so
the picker and the loader cannot disagree.
Cheap on an ordinary host: the engine prediction is reached only when the class is missing.
``predict_engine`` activates nothing and installs nothing."""
if fam is None:
return False
def predicts(engine_name: str, **kwargs) -> bool:
if provisional:
return True
try:
return predict_engine(fam, model_kind = "gguf", **kwargs) == engine_name
except Exception: # noqa: BLE001 -- refused or unprobeable: not buildable by that engine
return False
if native_gguf_only(fam, model_kind):
return predicts(ENGINE_SD_CPP)
if family_pipeline_available(fam):
return True
if (
model_kind == "gguf"
and gguf_filename
and family_mlx_supported(fam)
and predicts(ENGINE_MLX, gguf_filename = gguf_filename)
):
return True
# only a GGUF can go native, and only for a family with the single-file assets sd.cpp needs
if model_kind != "gguf" or not family_sd_cpp_supported(fam):
return False
return predicts(ENGINE_SD_CPP)
@functools.cache
def _supported_family_capabilities() -> tuple[str, ...]:
return tuple(pipeline_available_family_names())
def annotate_status(status: dict[str, Any]) -> dict[str, Any]:
"""Tag a backend status dict with the active engine + any fallback reason."""
out = dict(status)
out["engine"] = _active_engine_name
out["fallback_reason"] = _fallback_reason
out["mlx_fallback_reason"] = _mlx_fallback_reason
out["supported_families"] = list(_supported_family_capabilities())
# The family override picker classes an opaque folder as a modular pipeline only for these (FLUX.3).
out["modular_families"] = list(modular_image_family_names(_supported_family_capabilities()))
# FLUX.3 prompt upsampling, by the loaded family (pure: no binary or cache probe on a status poll).
from .flux3_prompt_upsampler import family_capabilities
out.update(family_capabilities(out.get("family") if out.get("loaded") else None))
out.update(engine_choice_status(out))
return out
def engine_choice_status(status: dict[str, Any]) -> dict[str, Any]:
"""What the Images page needs to offer the engine choice for the loaded pick: the env pin (which wins over a pick),
whether the native engine can serve it (a GGUF of a family with sd.cpp assets; the binary is checked on reload, not
on a status poll), and whether reloading it there would enable LoRA where this engine cannot apply one."""
from .diffusion_lora import supports_lora
fam = None
if status.get("loaded") and status.get("model_kind") == "gguf" and status.get("family"):
try:
from .diffusion_families import detect_family
fam = detect_family("", override = status.get("family"))
except Exception: # noqa: BLE001 -- an unknown family reads as not eligible
fam = None
# Separate text-encoder / VAE files load on diffusers only; a native reload of this build is refused.
# UNSLOTH_DIFFUSION_SD_CPP=0 refuses a native pick (``refuse_engine_pick``) unless the server pins sd_cpp. The
# MPS knob does not apply: an explicit pick is native-eligible on any backend.
native_disabled = (
_engine_config()[1] in _DISABLE_TOKENS and env_forced_engine() != ENGINE_SD_CPP
)
native_eligible = (
fam is not None
and not native_disabled
and family_sd_cpp_supported(fam)
and not status.get("component_files")
)
lora_via_native = bool(
native_eligible
and status.get("engine") != ENGINE_SD_CPP
and not status.get("supports_lora")
and supports_lora(
engine = ENGINE_SD_CPP, family = fam.name, model_kind = "gguf", transformer_quant = None
)
)
out: dict[str, Any] = {
"engine_forced": env_forced_engine(),
"native_engine_eligible": native_eligible,
"lora_via_native": lora_via_native,
}
if status.get("loaded") and status.get("engine"):
# The Advanced Engine select's provenance, in the shape of the other resolved controls.
resolved = dict(status.get("resolved") or {})
resolved["engine"] = _engine_resolved(status.get("engine"))
out["resolved"] = resolved
return out
def _engine_resolved(active: str) -> dict[str, Any]:
forced = env_forced_engine()
with _lock:
_settle_engine_pick_locked()
committed = _engine_pick
pick = None if forced else committed
if pick:
reason = "selected for this load"
elif forced:
reason = f"pinned by UNSLOTH_DIFFUSION_ENGINE={forced}"
elif active == ENGINE_SD_CPP:
reason = "native sd.cpp engine chosen for this host"
elif active == ENGINE_MLX:
reason = (
"MLX engine (UNSLOTH_DIFFUSION_MLX=1)"
if mlx_mode() == MLX_FORCE
else "MLX engine on Apple Silicon; diffusers serves what MLX cannot"
)
else:
reason = _fallback_reason or "diffusers selected"
return {
"value": active,
"requested": pick,
"source": "explicit" if pick else "auto",
"status": "applied",
"reason": reason,
}
def active_status() -> dict[str, Any]:
"""The active engine's status, annotated with which engine + any fallback reason."""
return annotate_status(get_active_diffusion_engine().status())