Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio (AMD): run native image generation on an NVIDIA card next to ROCm torch (#12252)

Leo Borcherding1a08bd9618c6
diff --git a/studio/backend/core/inference/diffusion_engine_router.py b/studio/backend/core/inference/diffusion_engine_router.py
index 059f44e1169..abf0f9f7ed8 100644
--- a/studio/backend/core/inference/diffusion_engine_router.py
+++ b/studio/backend/core/inference/diffusion_engine_router.py
@@ -16,6 +16,7 @@ Env knobs:
UNSLOTH_DIFFUSION_SD_CPP=auto|0|1 enable/disable the native route
UNSLOTH_DIFFUSION_SD_CPP_MPS=0|1 allow native on Apple MPS (default off)
UNSLOTH_DIFFUSION_SD_CPP_INSTALL=auto|0|1 allow lazy binary install (in sd_cpp_backend)
+ UNSLOTH_DIFFUSION_SD_CPP_DEVICE=nvidia[:N] run native on a card torch cannot see
"""
from __future__ import annotations
@@ -23,6 +24,7 @@ from __future__ import annotations
import os
import sys
import threading
+from dataclasses import dataclass
from typing import Any, Callable, Optional
from core.inference.diffusion_device import resolve_diffusion_device_target
@@ -32,12 +34,14 @@ from core.inference.diffusion_families import (
family_sd_cpp_supported,
)
from core.inference.sd_cpp_backend import (
+ _card_lookup_inventory,
_install_allowed,
_managed_tree_in_use,
_server_binary_runnable,
ensure_sd_cpp_binary,
ensure_sd_server_binary,
note_unlaunchable_accelerator_build,
+ off_torch_build_mismatch,
preferred_accelerator,
sd_cpp_binary_runs_family,
usable_or_recorded_failure,
@@ -65,6 +69,91 @@ def _install_accelerator_for(backend: str) -> str:
return _INSTALL_ACCELERATOR.get(backend, "auto")
+_OFF_TORCH_VENDOR_ACCELERATOR = {"nvidia": "cuda"}
+_off_torch_warned: set[str] = set()
+
+
+@dataclass(frozen = True)
+class OffTorchDevice:
+ """A card torch cannot see. ``index`` is nvidia-smi's row, never a torch ordinal."""
+
+ vendor: str
+ index: int
+ accelerator: str
+
+ @property
+ def label(self) -> str:
+ return f"{self.vendor}:{self.index}"
+
+ def child_env(self) -> dict[str, str]:
+ # nvidia-smi numbers cards in PCI order; CUDA's default is fastest-first.
+ return {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": str(self.index)}
+
+
+def _warn_off_torch_once(raw: str, message: str, *args: Any) -> None:
+ if raw in _off_torch_warned:
+ return
+ _off_torch_warned.add(raw)
+ logger.warning(message, *args)
+
+
+def _physical_inventory() -> dict:
+ return _card_lookup_inventory() or {}
+
+
+def off_torch_sd_cpp_device(backend: Optional[str] = None) -> Optional[OffTorchDevice]:
+ """The card ``UNSLOTH_DIFFUSION_SD_CPP_DEVICE`` asks native image generation to run on, when
+ torch cannot drive it (an NVIDIA card beside a ROCm torch). None keeps today's routing: unset,
+ unparseable, a vendor torch already serves, or an inventory that answered without that card."""
+ raw = os.environ.get("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "").strip().lower()
+ if not raw:
+ return None
+ vendor, _, index_text = raw.partition(":")
+ vendor = vendor.strip()
+ accelerator = _OFF_TORCH_VENDOR_ACCELERATOR.get(vendor)
+ try:
+ index = int(index_text) if index_text.strip() else 0
+ except ValueError:
+ index = -1
+ if accelerator is None or index < 0:
+ _warn_off_torch_once(
+ raw,
+ "UNSLOTH_DIFFUSION_SD_CPP_DEVICE=%r is not understood (expected nvidia or "
+ "nvidia:<index>); ignoring it",
+ raw,
+ )
+ return None
+ if backend is None:
+ backend = resolve_diffusion_device_target().backend
+ if _INSTALL_ACCELERATOR.get(backend) == accelerator:
+ return None
+ inventory = _physical_inventory()
+ unanswered = set(inventory.get("unanswered") or ())
+ # Per vendor: an unanswered AMD probe says nothing about NVIDIA. Unknown naming nobody = cold cache.
+ if vendor not in unanswered and not (inventory.get("unknown") and not unanswered):
+ present = any(
+ isinstance(device, dict)
+ and device.get("vendor") == vendor
+ and device.get("index") == index
+ for device in inventory.get("devices") or ()
+ )
+ if not present:
+ _warn_off_torch_once(
+ raw,
+ "UNSLOTH_DIFFUSION_SD_CPP_DEVICE=%s: this host lists no such card; ignoring it",
+ raw,
+ )
+ return None
+ return OffTorchDevice(vendor = vendor, index = index, accelerator = accelerator)
+
+
+def image_install_accelerator(backend: str) -> str:
+ """The sd.cpp build the IMAGE engine wants: the off-torch card's when one is set, else torch's.
+ Video keeps ``_install_accelerator_for``, since torch places its loads."""
+ off_torch = off_torch_sd_cpp_device(backend)
+ return off_torch.accelerator if off_torch is not None else _install_accelerator_for(backend)
+
+
# The engine the current load committed to, and why a non-native choice was made. Mutated only under _lock.
_lock = threading.Lock()
# Serializes a whole engine switch (check -> unload -> publish); _lock alone is released during the slow unload().
@@ -213,8 +302,10 @@ def select_and_activate_engine(
hf_token: Optional[str] = None,
model_kind: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
+ before_fallback: Optional[Callable[[], None]] = None,
) -> Any:
"""Pick + activate the engine for loading ``fam`` on this host; return the engine.
+ ``before_fallback`` may raise to refuse a diffusers fallback before it unloads the resident model.
Falls back to diffusers (recording a reason) when the native route is disabled, the device has
a usable GPU, MPS is not enabled, the family has no native asset, or the binary is unavailable
@@ -234,6 +325,11 @@ def select_and_activate_engine(
target = resolve_diffusion_device_target()
backend = target.backend
+ off_torch = off_torch_sd_cpp_device(backend)
+ if off_torch is not None:
+ # A card torch cannot see is only reachable natively, and torch's ordinals name other cards.
+ prefer_native = True
+ gpu_ordinal = None
# CPU always native-eligible; MPS only when enabled; a GPU backend never, unless forced
policy_eligible = backend == "cpu" or (backend == "mps" and mps_enabled) or prefer_native
fam_ok = family_sd_cpp_supported(fam)
@@ -247,6 +343,8 @@ def select_and_activate_engine(
install_accelerator = preferred_accelerator(
_install_accelerator_for(backend), selected_card
)
+ if off_torch is not None:
+ install_accelerator = preferred_accelerator(off_torch.accelerator, selected_card)
# Probe the resident sd-server FIRST (the backend prefers it): a server-only install must still route to
# native and should not pay an sd-cli download. Install the accelerator-matched build so a forced-native GPU
# load gets the GPU server.
@@ -257,6 +355,16 @@ def select_and_activate_engine(
def _accept(candidate):
if candidate and upgrade_is_deferred:
return candidate
+ wrong_build = off_torch_build_mismatch(off_torch, candidate)
+ if wrong_build:
+ logger.warning(
+ "%s is the %s sd.cpp build; %s needs %s",
+ candidate,
+ wrong_build,
+ off_torch.label,
+ off_torch.accelerator,
+ )
+ return None
return usable_or_recorded_failure(candidate, install_accelerator, selected_card)
server_binary = _accept(
@@ -334,6 +442,11 @@ def select_and_activate_engine(
reason = "native sd.cpp binary unavailable"
else:
reason = "diffusers selected"
+ if off_torch is not None:
+ reason = f"{reason}; UNSLOTH_DIFFUSION_SD_CPP_DEVICE={off_torch.label} not honoured"
+ logger.warning("image load stays on torch's device: %s", reason)
+ if before_fallback is not None:
+ before_fallback()
return _activate(ENGINE_DIFFUSERS, reason)
@@ -352,14 +465,20 @@ def native_binary_installed(
VAE and text encoder for a load that goes to diffusers. Without ``fam`` the question is the
older one, "is there a binary at all".
"""
+ backend = resolve_diffusion_device_target().backend
+ off_torch = off_torch_sd_cpp_device(backend)
+ if off_torch is not None:
+ gpu_ordinal = None
selected_card = _selected_card(gpu_ordinal)
- install_accelerator = preferred_accelerator(
- _install_accelerator_for(resolve_diffusion_device_target().backend), selected_card
- )
- server_binary = usable_or_recorded_failure(
- ensure_sd_server_binary(allow_install = False, accelerator = install_accelerator),
- install_accelerator,
- selected_card,
+ install_accelerator = preferred_accelerator(image_install_accelerator(backend), selected_card)
+
+ def _usable(candidate):
+ if off_torch_build_mismatch(off_torch, candidate):
+ return None
+ return usable_or_recorded_failure(candidate, install_accelerator, selected_card)
+
+ server_binary = _usable(
+ ensure_sd_server_binary(allow_install = False, accelerator = install_accelerator)
)
if (
server_binary
@@ -367,11 +486,7 @@ def native_binary_installed(
and (fam is None or sd_cpp_binary_runs_family(server_binary, fam))
):
return True
- binary = usable_or_recorded_failure(
- ensure_sd_cpp_binary(allow_install = False, accelerator = install_accelerator),
- install_accelerator,
- selected_card,
- )
+ binary = _usable(ensure_sd_cpp_binary(allow_install = False, accelerator = install_accelerator))
if fam is not None and binary and not sd_cpp_binary_runs_family(binary, fam):
return False
return bool(binary and SdCppEngine(binary = binary).version() is not None)
@@ -408,6 +523,9 @@ def predict_engine(
return ENGINE_DIFFUSERS
backend = resolve_diffusion_device_target().backend
+ if off_torch_sd_cpp_device(backend) is not None:
+ prefer_native = True
+ gpu_ordinal = None
policy_eligible = backend == "cpu" or (backend == "mps" and mps_enabled) or prefer_native
if not (policy_eligible and family_sd_cpp_supported(fam)):
return ENGINE_DIFFUSERS
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index ff734f4299b..530d68db8dc 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -1273,6 +1273,29 @@ def accelerator_runtime_failed(accelerator: Optional[str], card: Optional[str] =
return _record_diverts(_stored_accelerator_runtime_failures().get(klass), fingerprint, card)
+def off_torch_build_mismatch(off_torch: Any, binary: Optional[str]) -> Optional[str]:
+ """Why ``binary`` is not provably the off-torch card's build, else None. Any other build ignores
+ CUDA_VISIBLE_DEVICES (Vulkan) or reads it as HIP's mask (ROCm), so it would run on torch's cards
+ past the arbiter and the training guard; an unrecorded one (SD_CLI_PATH) could be either."""
+ if off_torch is None or not binary:
+ return None
+ klass = _installed_accelerator_of(binary)
+ if klass != _accelerator_class_of(off_torch.accelerator):
+ return klass or "unrecorded"
+ return None
+
+
+def _refuse_off_torch_build_mismatch(off_torch: Any, binary: Optional[str]) -> None:
+ klass = off_torch_build_mismatch(off_torch, binary)
+ if klass:
+ raise RuntimeError(
+ f"UNSLOTH_DIFFUSION_SD_CPP_DEVICE={off_torch.label} needs the "
+ f"{off_torch.accelerator} stable-diffusion.cpp build, but the installed one is "
+ f"{klass}. Let the {off_torch.accelerator} build install, or unset the setting, "
+ "then load again."
+ )
+
+
def usable_or_recorded_failure(
binary,
requested,
@@ -1946,6 +1969,12 @@ class _SdState:
physical_gpu_id: Optional[int] = None
# This load's card, as the failure record names cards.
selected_card: Optional[str] = None
+ # UNSLOTH_DIFFUSION_SD_CPP_DEVICE placement: its label, and the env every spawn gets to open only that card.
+ off_torch_device: Optional[str] = None
+ child_env: tuple[tuple[str, str], ...] = ()
+
+ def spawn_env(self) -> Optional[dict[str, str]]:
+ return dict(self.child_env) or None
def _offload_with_device_pin_impl(
@@ -2027,6 +2056,7 @@ class _SdLoading:
expected_bytes: int = 0
downloaded_bytes: int = 0
error: Optional[str] = None
+ off_torch_device: Optional[str] = None
@dataclass
@@ -2179,6 +2209,16 @@ class SdCppDiffusionBackend:
def is_loaded(self) -> bool:
return self._state is not None
+ @property
+ def runs_off_torch_device(self) -> bool:
+ """Everything resident or loading sits on a card torch cannot see. A pending load counts, or
+ training would cancel it; a torch-placed resident beside it still has to be freed."""
+ loading = getattr(self, "_loading", None)
+ if loading is not None and loading.error is not None:
+ loading = None
+ held = [x for x in (self._state, loading) if x is not None]
+ return bool(held) and all(getattr(x, "off_torch_device", None) for x in held)
+
def _loading_card_store(self) -> threading.local:
"""Lazily, so an instance built with ``__new__`` (the unit-test seam) still answers."""
store = getattr(self, "_loading_cards", None)
@@ -2258,9 +2298,9 @@ class SdCppDiffusionBackend:
``preferred_accelerator`` is applied here so all four call sites agree, or
``_accelerator_changed`` would reinstall over what the others chose."""
- from core.inference.diffusion_engine_router import _install_accelerator_for
+ from core.inference.diffusion_engine_router import image_install_accelerator
return preferred_accelerator(
- _install_accelerator_for(getattr(resolve_diffusion_device_target(), "backend", "cpu")),
+ image_install_accelerator(getattr(resolve_diffusion_device_target(), "backend", "cpu")),
card,
)
@@ -2433,6 +2473,11 @@ class SdCppDiffusionBackend:
"""Validate, then fetch assets on a daemon thread. Returns at once."""
# Empty/whitespace token = "no token"; "" verbatim breaks the anonymous fallback.
hf_token = hf_token.strip() if hf_token and hf_token.strip() else None
+ from core.inference.diffusion_engine_router import off_torch_sd_cpp_device
+
+ off_torch = off_torch_sd_cpp_device()
+ if off_torch is not None:
+ gpu_ids, gpu_ordinal = None, None
# Same fallback the diffusers and video backends take: the route ranks the selection and passes the winner,
# but a direct caller (an MCP client, a test, a plugin) hands over gpu_ids alone, and without this the native
# engine is the one engine that would drop the pick silently. Re-ranked only when nobody has, so a
@@ -2504,6 +2549,7 @@ class SdCppDiffusionBackend:
if kind != "diffusion_model"
)
),
+ off_torch_device = off_torch.label if off_torch is not None else None,
)
account_thread(
@@ -2519,6 +2565,7 @@ class SdCppDiffusionBackend:
memory_mode = memory_mode,
speed_mode = speed_mode,
gpu_ordinal = gpu_ordinal,
+ off_torch = off_torch,
_load_token = token,
_cancel_event = cancel_event,
),
@@ -2542,6 +2589,7 @@ class SdCppDiffusionBackend:
memory_mode: Optional[str] = None,
speed_mode: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
+ off_torch: Any = None,
_load_token: int,
_cancel_event: Optional[threading.Event] = None,
) -> None:
@@ -2668,7 +2716,12 @@ class SdCppDiffusionBackend:
llm_vision = paths.get("llm_vision"),
qwen2vl = paths.get("qwen2vl"),
)
- device = resolve_diffusion_device_target().device
+ device = (
+ off_torch.accelerator
+ if off_torch is not None
+ else resolve_diffusion_device_target().device
+ )
+ spawn_env = off_torch.child_env() if off_torch is not None else None
# Honor speed everywhere; offload only off-CPU (on CPU weights are resident, so the flags are no-ops)
offload: tuple[str, ...] = ()
if device != "cpu":
@@ -2758,6 +2811,7 @@ class SdCppDiffusionBackend:
"load again."
)
else:
+ _refuse_off_torch_build_mismatch(off_torch, server_binary)
server = SdCppServer(server_binary)
# Published INSIDE the claim: _tree_in_use reads _pending_server, so this is the handover
# from "a reader holds the tree" to "a starting server does", with no gap between them.
@@ -2794,6 +2848,7 @@ class SdCppDiffusionBackend:
),
native_speed = native_speed,
threads = _default_threads(),
+ env = spawn_env,
)
started_ok = True
except SdCppCancelled:
@@ -2853,6 +2908,8 @@ class SdCppDiffusionBackend:
"The stable-diffusion.cpp binary was replaced by an install for a "
"different accelerator while this model was loading. Try the load again."
)
+ if mode == "oneshot":
+ _refuse_off_torch_build_mismatch(off_torch, getattr(engine, "binary", None))
committed_offload_flags = tuple(
_offload_with_device_pin_impl(
offload,
@@ -2882,6 +2939,7 @@ class SdCppDiffusionBackend:
# Only the one-shot path needs to carry it: it re-resolves sd-cli per image, long after this
# decision, and has nothing else to check the answer against.
sd_accelerator = engine_accelerator if mode == "oneshot" else None,
+ # Never on an off-torch card: the parent-visible ids are torch's, so CUDA0 would name one of them.
physical_gpu_id = (
_resolved_server_physical_gpu_id(
server_binary,
@@ -2889,10 +2947,12 @@ class SdCppDiffusionBackend:
gpu_ordinal,
committed_offload_flags,
)
- if mode == "server"
+ if mode == "server" and off_torch is None
else None
),
selected_card = self._loading_card,
+ off_torch_device = off_torch.label if off_torch is not None else None,
+ child_env = tuple(sorted((spawn_env or {}).items())),
)
superseded = False
orphan: Optional[SdCppServer] = None
@@ -2922,7 +2982,9 @@ class SdCppDiffusionBackend:
"sd_cpp.loaded: repo=%s gguf=%s device=%s mode=%s speed=%s offload_flags=%s",
state.repo_id,
state.gguf_filename,
- state.device,
+ f"{state.device} ({state.off_torch_device}, outside torch)"
+ if state.off_torch_device
+ else state.device,
state.mode,
state.native_speed,
without_device_backend_flags(state.offload_flags) or "none",
@@ -3822,6 +3884,7 @@ class SdCppDiffusionBackend:
offload = without_device_backend_flags(state.offload_flags),
native_speed = state.native_speed,
threads = state.threads,
+ env = state.spawn_env(),
extra_args = list(CPU_BACKEND_FLAGS),
)
except Exception: # noqa: BLE001 -- the original abort is the more useful error
@@ -3940,6 +4003,7 @@ class SdCppDiffusionBackend:
native_speed = state.native_speed,
threads = state.threads,
extra_args = extra_args or None,
+ env = state.spawn_env(),
on_log = self._on_log,
cancel_event = cancel,
)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 2898bc8e4da..0b025394620 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -41487,6 +41487,28 @@ def _training_is_active() -> bool:
return _diffusion_training_active()
+def _unload_native_video_sharing_the_sd_cpp_tree() -> None:
+ """An off-torch image load skips the arbiter, a native H3 model's only evictor, yet may reinstall
+ the managed sd.cpp tree that model runs from."""
+ from core.inference import gpu_arbiter
+ from core.inference.video import get_video_backend
+
+ from core.inference.video_minimax_h3 import H3_GGUF_REPO
+
+ video = get_video_backend()
+ # engine is published at commit; a downloading H3 load shows only in _loading, from begin_load on.
+ loading = getattr(video, "_loading", None)
+ h3_loading = (
+ loading is not None
+ and loading.error is None
+ and H3_GGUF_REPO in (getattr(loading, "asset_repos", None) or ())
+ )
+ if h3_loading or video.status().get("engine") == "sd_cpp":
+ logger.info("Unloading the native video model: the Images load replaces its sd.cpp build")
+ video.unload()
+ gpu_arbiter.release(gpu_arbiter.VIDEO)
+
+
def _guard_diffusion_load_against_training() -> None:
"""Refuse loading an image model while a training run is active. Unlike chat,
a diffusion pipeline's VRAM can't be cheaply estimated before the load, so the
@@ -41836,6 +41858,7 @@ async def load_diffusion_model_gated(
annotate_status,
begin_load_on,
engine_for,
+ off_torch_sd_cpp_device,
predict_engine,
select_and_activate_engine,
)
@@ -41869,8 +41892,11 @@ async def load_diffusion_model_gated(
model_kind = kind,
base_repo = request.base_repo,
)
+ # Off-torch native shares no VRAM with training or chat; re-settled after selection (diffusers lands on torch's card).
+ off_torch = await asyncio.to_thread(off_torch_sd_cpp_device)
# Refuse while training is running: a multi-GB pipeline would compete with the training subprocess for VRAM.
- _guard_diffusion_load_against_training()
+ if off_torch is None:
+ _guard_diffusion_load_against_training()
# Take the GPU from chat only on a non-CPU device: gate on the device, not the engine name.
# Pure resolve, so it can run before selection, which the refusal below has to precede.
device = await asyncio.to_thread(lambda: resolve_diffusion_device_target().device)
@@ -41907,6 +41933,9 @@ async def load_diffusion_model_gated(
)
except Exception: # noqa: BLE001 -- a probe failure must not refuse a loadable pick
pending_name = None
+ # Before selection: switching to diffusers would unload the resident off-torch model, then refuse.
+ if off_torch is not None and pending_name != ENGINE_SD_CPP:
+ _guard_diffusion_load_against_training()
# Same bar, same reason, for an EXPLICIT precision this host can never honor. begin_load
# makes the identical network-free check, but it runs inside acquire_for -- which evicts
# chat under the arbiter lock BEFORE the register callback -- and after selection, which
@@ -41958,6 +41987,8 @@ async def load_diffusion_model_gated(
# begin_load signals whatever generation is running, so guard on every device, not just GPU.
require_no_foreign_generations()
+ if off_torch is not None and pending_name == ENGINE_SD_CPP:
+ await asyncio.to_thread(_unload_native_video_sharing_the_sd_cpp_tree)
# Pick the engine for this host (diffusers on GPU, native sd.cpp otherwise), installing sd-cli if needed, BEFORE evicting chat.
engine = await asyncio.to_thread(
functools.partial(
@@ -41967,6 +41998,10 @@ async def load_diffusion_model_gated(
model_kind = kind,
# The ordinal resolved above: re-resolving re-ranks by free VRAM and can pick another card.
gpu_ordinal = gpu_ordinal,
+ # A fallback lands on torch's card: refuse it during training before it unloads anything.
+ before_fallback = (
+ _guard_diffusion_load_against_training if off_torch is not None else None
+ ),
)
)
# predict_engine is selection's read-only twin: it never installs, so a host whose sd-cli
@@ -42002,6 +42037,11 @@ async def load_diffusion_model_gated(
repo_id = request.model_path,
base_repo = request.base_repo,
)
+ if off_torch is not None:
+ if activated == ENGINE_SD_CPP:
+ needs_gpu = False
+ else:
+ _guard_diffusion_load_against_training()
def _start_engine_load():
# Recorded HERE for the reason the video route gives: inside the admitted callback,
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index e61e464f0d1..d6358496093 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -1892,12 +1892,15 @@ async def start_training(
# The ACTIVE engine, not the diffusers singleton: on a native (sd_cpp) selection the diffusers backend
# reports unloaded while the native engine still holds state.
diffusion = get_active_diffusion_engine()
- if diffusion.is_loaded:
- logger.info(
- "Unloading diffusion (Images) model to free GPU memory for training"
- )
- diffusion.unload()
- gpu_arbiter.release(gpu_arbiter.DIFFUSION)
+ if getattr(diffusion, "runs_off_torch_device", False) is True:
+ logger.info("Keeping the Images model: it runs outside torch's GPUs")
+ else:
+ if diffusion.is_loaded:
+ logger.info(
+ "Unloading diffusion (Images) model to free GPU memory for training"
+ )
+ diffusion.unload()
+ gpu_arbiter.release(gpu_arbiter.DIFFUSION)
except Exception as e:
logger.warning("Could not unload diffusion model for training: %s", e)
@@ -2910,10 +2913,13 @@ def _free_gpu_for_diffusion_training() -> None:
# The ACTIVE engine, not the diffusers singleton: on a native (sd_cpp) selection the resident sd-server still
# holds the GPU, so unloading only the singleton is a no-op.
diffusion = get_active_diffusion_engine()
- if diffusion.is_loaded:
- logger.info("Unloading resident Images pipeline to free GPU memory for training")
- diffusion.unload()
- gpu_arbiter.release(gpu_arbiter.DIFFUSION)
+ if getattr(diffusion, "runs_off_torch_device", False) is True:
+ logger.info("Keeping the Images model: it runs outside torch's GPUs")
+ else:
+ if diffusion.is_loaded:
+ logger.info("Unloading resident Images pipeline to free GPU memory for training")
+ diffusion.unload()
+ gpu_arbiter.release(gpu_arbiter.DIFFUSION)
except Exception as e: # noqa: BLE001
logger.warning("Could not unload Images pipeline for diffusion training: %s", e)
diff --git a/studio/backend/routes/video.py b/studio/backend/routes/video.py
index 1391a08c02d..018fb3c4664 100644
--- a/studio/backend/routes/video.py
+++ b/studio/backend/routes/video.py
@@ -438,6 +438,14 @@ async def load_video_model_gated(
# begin_load signals whatever generation is running, so guard on every device.
require_no_foreign_generations()
+ from core.inference.video_minimax_h3 import is_h3_native
+
+ if is_h3_native(fam, kind):
+ from core.inference.diffusion_engine_router import get_active_diffusion_engine
+ images = get_active_diffusion_engine()
+ if getattr(images, "runs_off_torch_device", False) is True:
+ # No arbiter owner evicts it, yet it holds the one managed sd.cpp tree H3 installs into.
+ await asyncio.to_thread(images.unload)
if device != "cpu":
# Register the in-flight load UNDER the arbiter lock: otherwise a competing acquire in that gap evicts VIDEO
# before the load is marked, finds nothing to cancel, and both allocate at once. The training admission wraps
diff --git a/studio/backend/tests/test_sd_cpp_backend.py b/studio/backend/tests/test_sd_cpp_backend.py
index fdd29e5c6ba..439edf60696 100644
--- a/studio/backend/tests/test_sd_cpp_backend.py
+++ b/studio/backend/tests/test_sd_cpp_backend.py
@@ -191,7 +191,9 @@ class _FakeServer:
native_speed = None,
threads = None,
extra_args = None,
+ env = None,
):
+ self.env = env
self.started = dict(
files = files,
vae_format = vae_format,
diff --git a/studio/backend/tests/test_sd_cpp_off_torch_device.py b/studio/backend/tests/test_sd_cpp_off_torch_device.py
new file mode 100644
index 00000000000..dd4d093c5f5
--- /dev/null
+++ b/studio/backend/tests/test_sd_cpp_off_torch_device.py
@@ -0,0 +1,547 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""UNSLOTH_DIFFUSION_SD_CPP_DEVICE: native image generation on a card torch cannot see.
+
+The host these pin is a ROCm torch beside an NVIDIA card (R9700 + RTX 3080). Every hardware answer
+is stubbed: the torch backend, the physical inventory and the binaries, so no test asks the machine
+it runs on."""
+
+from __future__ import annotations
+
+import inspect
+from types import SimpleNamespace
+
+import pytest
+
+from core.inference import diffusion_engine_router as r
+from core.inference import sd_cpp_backend
+from core.inference.diffusion_families import detect_family
+from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS, ENGINE_SD_CPP
+
+_ENVS = (
+ "UNSLOTH_DIFFUSION_ENGINE",
+ "UNSLOTH_DIFFUSION_SD_CPP",
+ "UNSLOTH_DIFFUSION_SD_CPP_MPS",
+ "UNSLOTH_DIFFUSION_SD_CPP_INSTALL",
+ "UNSLOTH_DIFFUSION_SD_CPP_DEVICE",
+)
+
+_MIXED_HOST = {
+ "unknown": False,
+ "unanswered": [],
+ "devices": [
+ {"vendor": "nvidia", "index": 0, "name": "NVIDIA GeForce RTX 3080"},
+ {"vendor": "amd", "index": 0, "name": "AMD Radeon AI PRO R9700"},
+ ],
+}
+
+
[email protected](autouse = True)
+def _pinned_host(monkeypatch):
+ for e in _ENVS:
+ monkeypatch.delenv(e, raising = False)
+ monkeypatch.setattr(r, "_off_torch_warned", set())
+ monkeypatch.setattr(
+ r,
+ "resolve_diffusion_device_target",
+ lambda: SimpleNamespace(backend = "rocm", device = "cuda"),
+ )
+ monkeypatch.setattr(r, "_physical_inventory", lambda: _MIXED_HOST)
+ monkeypatch.setattr(sd_cpp_backend, "_installed_accelerator_of", lambda _b: "cuda")
+ monkeypatch.setattr(
+ r,
+ "get_active_diffusion_engine",
+ lambda: SimpleNamespace(
+ status = lambda: {"loaded": False, "repo_id": None}, unload = lambda: None
+ ),
+ )
+ monkeypatch.setattr(r, "ensure_sd_server_binary", lambda **_: None)
+ monkeypatch.setattr(r, "_server_binary_runnable", lambda *_a, **_k: True)
+ monkeypatch.setattr(r, "SdCppEngine", lambda **_: SimpleNamespace(version = lambda: "sd-cli v0"))
+ saved_engine = r._active_engine_name
+ saved_reason = r._fallback_reason
+ try:
+ yield
+ finally:
+ r._active_engine_name = saved_engine
+ r._fallback_reason = saved_reason
+
+
+def _record_cli_requests(monkeypatch, path = "/opt/sd/sd-cli"):
+ asked: list[str] = []
+
+ def _ensure(**kwargs):
+ asked.append(kwargs["accelerator"])
+ return path
+
+ monkeypatch.setattr(r, "ensure_sd_cpp_binary", _ensure)
+ return asked
+
+
+def test_unset_is_today():
+ assert r.off_torch_sd_cpp_device() is None
+
+
+def test_nvidia_on_a_rocm_torch_names_the_card(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ device = r.off_torch_sd_cpp_device()
+ assert device == r.OffTorchDevice(vendor = "nvidia", index = 0, accelerator = "cuda")
+ assert device.child_env() == {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": "0"}
+
+
+def test_a_cuda_torch_already_drives_nvidia(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia:0")
+ assert r.off_torch_sd_cpp_device("cuda") is None
+
+
[email protected]("raw", ["amd", "nvidia:x", "nvidia:-1", "intel:0"])
+def test_values_it_cannot_act_on_are_ignored(monkeypatch, raw):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", raw)
+ assert r.off_torch_sd_cpp_device() is None
+
+
+def test_a_card_the_inventory_does_not_list_is_ignored(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia:1")
+ assert r.off_torch_sd_cpp_device() is None
+
+
+ "inventory",
+ [
+ {"unknown": True, "devices": []},
+ {"unknown": False, "unanswered": ["nvidia"], "devices": []},
+ ],
+)
+def test_an_unanswered_probe_trusts_the_explicit_setting(monkeypatch, inventory):
+ monkeypatch.setattr(r, "_physical_inventory", lambda: inventory)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia:1")
+ assert r.off_torch_sd_cpp_device().index == 1
+
+
+def test_another_vendors_unanswered_probe_does_not_vouch_for_the_card(monkeypatch):
+ inventory = {
+ "unknown": True,
+ "unanswered": ["amd"],
+ "devices": [{"vendor": "nvidia", "index": 0, "name": "NVIDIA GeForce RTX 3080"}],
+ }
+ monkeypatch.setattr(r, "_physical_inventory", lambda: inventory)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia:1")
+ assert r.off_torch_sd_cpp_device() is None
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia:0")
+ assert r.off_torch_sd_cpp_device().index == 0
+
+
+def test_only_the_image_path_changes_build(monkeypatch):
+ """Video still installs by torch's backend: its loads are placed by torch."""
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ assert r.image_install_accelerator("rocm") == "cuda"
+ assert r._install_accelerator_for("rocm") == "rocm"
+
+
+def test_mixed_host_with_the_setting_runs_cuda_sd_cpp(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ asked = _record_cli_requests(monkeypatch)
+ r.select_and_activate_engine(detect_family("z-image"))
+ assert r.active_engine_name() == ENGINE_SD_CPP
+ assert asked == ["cuda"]
+
+
+def test_same_host_without_the_setting_stays_on_diffusers(monkeypatch):
+ asked = _record_cli_requests(monkeypatch)
+ r.select_and_activate_engine(detect_family("z-image"))
+ assert r.active_engine_name() == ENGINE_DIFFUSERS
+ assert "uses diffusers" in (r.active_status()["fallback_reason"] or "")
+ assert asked == []
+
+
+def test_torchs_ordinal_is_not_carried_to_the_other_vendor(monkeypatch):
+ """gpu_ordinal 1 is an AMD card here; naming it to a CUDA build would pick a different card."""
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ _record_cli_requests(monkeypatch)
+ seen: list = []
+ monkeypatch.setattr(r, "_selected_card", lambda ordinal: seen.append(ordinal))
+ r.select_and_activate_engine(detect_family("z-image"), gpu_ordinal = 1)
+ assert seen == [None]
+
+
+def test_no_cuda_build_falls_back_and_says_so(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ _record_cli_requests(monkeypatch, path = None)
+ r.select_and_activate_engine(detect_family("z-image"))
+ assert r.active_engine_name() == ENGINE_DIFFUSERS
+ reason = r.active_status()["fallback_reason"] or ""
+ assert "binary unavailable" in reason
+ assert "UNSLOTH_DIFFUSION_SD_CPP_DEVICE=nvidia:0 not honoured" in reason
+
+
+def test_a_family_without_native_assets_is_not_rerouted(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ asked = _record_cli_requests(monkeypatch)
+ r.select_and_activate_engine(detect_family("sdxl"))
+ assert r.active_engine_name() == ENGINE_DIFFUSERS
+ assert asked == []
+
+
+def test_explicit_opt_outs_still_win(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ _record_cli_requests(monkeypatch)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP", "0")
+ r.select_and_activate_engine(detect_family("z-image"))
+ assert r.active_engine_name() == ENGINE_DIFFUSERS
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SD_CPP")
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ENGINE", "diffusers")
+ r.select_and_activate_engine(detect_family("z-image"))
+ assert r.active_engine_name() == ENGINE_DIFFUSERS
+
+
+def test_prediction_agrees_with_selection(monkeypatch):
+ """The download plan is built from the prediction, so it has to follow the setting too."""
+ _record_cli_requests(monkeypatch)
+ fam = detect_family("z-image")
+ assert r.predict_engine(fam, model_kind = "gguf") == ENGINE_DIFFUSERS
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ assert r.predict_engine(fam, model_kind = "gguf", gpu_ordinal = 1) == ENGINE_SD_CPP
+
+
+def _state(**overrides):
+ fields = dict(
+ repo_id = "unsloth/Z-Image-Turbo-GGUF",
+ base_repo = "Tongyi-MAI/Z-Image-Turbo",
+ family = detect_family("z-image"),
+ device = "cuda",
+ files = SimpleNamespace(),
+ )
+ fields.update(overrides)
+ return sd_cpp_backend._SdState(**fields)
+
+
+def test_a_torch_placed_load_spawns_with_the_inherited_env():
+ assert _state().spawn_env() is None
+
+
+def test_an_off_torch_load_pins_its_card_at_spawn():
+ env = r.OffTorchDevice(vendor = "nvidia", index = 1, accelerator = "cuda").child_env()
+ state = _state(off_torch_device = "nvidia:1", child_env = tuple(sorted(env.items())))
+ assert state.spawn_env() == {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": "1"}
+
+
+def test_every_spawn_site_passes_the_pin():
+ load = inspect.getsource(sd_cpp_backend.SdCppDiffusionBackend._run_load)
+ assert "env = spawn_env," in load
+ assert 'if mode == "server" and off_torch is None' in load
+ restart = inspect.getsource(sd_cpp_backend.SdCppDiffusionBackend._restart_server_on_cpu_backend)
+ assert "env = state.spawn_env()," in restart
+ oneshot = inspect.getsource(sd_cpp_backend.SdCppDiffusionBackend._generate_oneshot)
+ assert "env = state.spawn_env()," in oneshot
+
+
+def test_the_resident_flag_the_trainer_reads():
+ backend = sd_cpp_backend.SdCppDiffusionBackend.__new__(sd_cpp_backend.SdCppDiffusionBackend)
+ backend._state = None
+ assert backend.runs_off_torch_device is False
+ backend._state = _state()
+ assert backend.runs_off_torch_device is False
+ backend._state = _state(off_torch_device = "nvidia:0")
+ assert backend.runs_off_torch_device is True
+
+
+def test_the_load_route_skips_the_arbiter_only_once_native_is_active():
+ import routes.inference as inference_routes
+
+ source = inspect.getsource(inference_routes)
+ start = source.index("off_torch = await asyncio.to_thread(off_torch_sd_cpp_device)")
+ window = source[start : start + 25000]
+ assert "if off_torch is None:\n _guard_diffusion_load_against_training()" in window
+ settle = window.index("if off_torch is not None:\n if activated == ENGINE_SD_CPP:")
+ assert window.index("activated = active_engine_name()") < settle
+ assert "needs_gpu = False" in window[settle : settle + 300]
+ assert "_guard_diffusion_load_against_training()" in window[settle : settle + 400]
+ assert settle < window.index("if needs_gpu:")
+
+
+def test_training_keeps_an_off_torch_images_model():
+ import routes.training as training_routes
+ source = inspect.getsource(training_routes)
+ assert 'getattr(diffusion, "runs_off_torch_device", False) is True' in source
+
+
+def _drive_load_route(
+ monkeypatch,
+ *,
+ predicted,
+ training_active,
+ video_engine = None,
+ video_loading = None,
+):
+ """Run the real image-load route with every hardware and engine answer stubbed; returns the
+ route calls in order, plus what the route raised."""
+ import asyncio
+
+ from fastapi import HTTPException
+
+ from core.inference import diffusion, diffusion_compat, diffusion_device
+ from core.inference import gpu_arbiter as arb
+ from hub.services.models import account_access
+ from models.inference import DiffusionLoadRequest
+ from routes import inference as route
+
+ calls: list[str] = []
+ backend = SimpleNamespace(
+ validate_load_request = lambda *_a, **_k: detect_family("z-image"),
+ preflight_base_access = lambda *_a, **_k: None,
+ assert_precision_available = lambda *_a, **_k: None,
+ begin_load = lambda *_a, **_k: calls.append("begin_load") or {},
+ )
+
+ async def _no_ordinal(*_a, **_k):
+ return None
+
+ def _guard():
+ calls.append("training_guard")
+ if training_active:
+ raise HTTPException(status_code = 409, detail = "training")
+
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ monkeypatch.setattr(arb, "_owner", None)
+ monkeypatch.setattr(diffusion, "get_diffusion_backend", lambda: backend)
+ monkeypatch.setattr(
+ diffusion_device,
+ "resolve_diffusion_device_target",
+ lambda *_a, **_k: SimpleNamespace(backend = "rocm", device = "cuda"),
+ )
+ monkeypatch.setattr(r, "predict_engine", lambda *_a, **_k: predicted)
+ monkeypatch.setattr(r, "engine_for", lambda *_a: backend)
+ monkeypatch.setattr(
+ r,
+ "select_and_activate_engine",
+ lambda *_a, **_k: calls.append("select_and_activate") or backend,
+ )
+ monkeypatch.setattr(r, "active_engine_name", lambda: predicted)
+ monkeypatch.setattr(r, "begin_load_on", lambda _e, start: start())
+ monkeypatch.setattr(diffusion_compat, "assert_pick_is_not_speech", lambda *_a, **_k: None)
+ monkeypatch.setattr(route, "_guard_diffusion_load_against_training", _guard)
+ monkeypatch.setattr(route, "_selected_gpu_ordinal", _no_ordinal)
+ monkeypatch.setattr(route, "_assert_native_precision_unset", lambda **_k: None)
+ monkeypatch.setattr(route, "_repo_is_in_the_hub_cache", lambda *_a: True)
+ monkeypatch.setattr(account_access, "admit_media_load", lambda _k, fn, *_a: fn())
+ monkeypatch.setattr(account_access, "note_resident_components", lambda *_a, **_k: None)
+
+ monkeypatch.setattr(
+ "core.inference.video.get_video_backend",
+ lambda: SimpleNamespace(
+ status = lambda: {"engine": video_engine},
+ unload = lambda: calls.append("video_unload"),
+ _loading = video_loading,
+ ),
+ )
+ monkeypatch.setattr(arb, "release", lambda owner: calls.append(f"release:{owner}"))
+
+ request = DiffusionLoadRequest(model_path = "org/image", gguf_filename = "model.gguf")
+ try:
+ asyncio.run(route.load_diffusion_model_gated(request, "tester"))
+ except HTTPException as exc:
+ return calls, exc
+ return calls, None
+
+
+def test_a_load_predicted_for_torch_is_refused_during_training_before_the_engine_switch(
+ monkeypatch,
+):
+ """A refused load must not unload the resident model: activating diffusers would do exactly that."""
+ calls, raised = _drive_load_route(monkeypatch, predicted = ENGINE_DIFFUSERS, training_active = True)
+ assert raised is not None and raised.status_code == 409
+ assert "select_and_activate" not in calls
+
+
+def test_an_off_torch_native_load_is_admitted_during_training(monkeypatch):
+ calls, raised = _drive_load_route(monkeypatch, predicted = ENGINE_SD_CPP, training_active = True)
+ assert raised is None, raised
+ assert "training_guard" not in calls
+ assert calls[-1] == "begin_load"
+
+
+def test_diffusion_training_keeps_an_off_torch_images_model(monkeypatch):
+ import routes.training as training_routes
+ from core.inference import gpu_arbiter
+
+ unloaded: list[str] = []
+ resident = SimpleNamespace(
+ runs_off_torch_device = True,
+ is_loaded = True,
+ unload = lambda: unloaded.append("images"),
+ )
+ monkeypatch.setattr(r, "get_active_diffusion_engine", lambda: resident)
+ monkeypatch.setattr(gpu_arbiter, "release", lambda owner: unloaded.append(f"release:{owner}"))
+ monkeypatch.setattr(
+ "core.inference.video.get_video_backend",
+ lambda: SimpleNamespace(status = lambda: {"loaded": False}, unload = lambda: None),
+ )
+ monkeypatch.setattr("routes.training_vram.summarize_resident_chat", lambda: {"any": False})
+ training_routes._free_gpu_for_diffusion_training()
+ assert "images" not in unloaded
+ assert f"release:{gpu_arbiter.DIFFUSION}" not in unloaded
+
+ resident.runs_off_torch_device = False
+ training_routes._free_gpu_for_diffusion_training()
+ assert "images" in unloaded
+
+
[email protected]("installed", ["rocm", "vulkan", "cpu", None])
+def test_a_leftover_build_of_another_accelerator_is_not_treated_as_off_torch(
+ monkeypatch, installed
+):
+ """Offline, installs off or no CUDA asset: the ensure returns the tree's build, which reads
+ CUDA_VISIBLE_DEVICES as HIP's mask (ROCm) or ignores it, landing on torch's card unguarded."""
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ _record_cli_requests(monkeypatch)
+ monkeypatch.setattr(sd_cpp_backend, "_installed_accelerator_of", lambda _b: installed)
+ r.select_and_activate_engine(detect_family("z-image"))
+ assert r.active_engine_name() == ENGINE_DIFFUSERS
+ assert "not honoured" in (r.active_status()["fallback_reason"] or "")
+
+
+def test_the_cuda_build_is_still_accepted(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ _record_cli_requests(monkeypatch)
+ monkeypatch.setattr(sd_cpp_backend, "_installed_accelerator_of", lambda _b: "cuda")
+ r.select_and_activate_engine(detect_family("z-image"))
+ assert r.active_engine_name() == ENGINE_SD_CPP
+
+
+def test_the_load_refuses_to_spawn_another_accelerators_build(monkeypatch):
+ device = r.OffTorchDevice(vendor = "nvidia", index = 0, accelerator = "cuda")
+ monkeypatch.setattr(sd_cpp_backend, "_installed_accelerator_of", lambda _b: "rocm")
+ with pytest.raises(RuntimeError, match = "needs the cuda"):
+ sd_cpp_backend._refuse_off_torch_build_mismatch(device, "/opt/sd/sd-server")
+ # An unrecorded build (SD_SERVER_PATH) cannot be shown to be CUDA either.
+ monkeypatch.setattr(sd_cpp_backend, "_installed_accelerator_of", lambda _b: None)
+ with pytest.raises(RuntimeError, match = "unrecorded"):
+ sd_cpp_backend._refuse_off_torch_build_mismatch(device, "/opt/sd/sd-server")
+ # Torch-placed loads are untouched.
+ sd_cpp_backend._refuse_off_torch_build_mismatch(None, "/opt/sd/sd-server")
+
+
+def test_an_in_flight_off_torch_load_counts_as_off_torch():
+ backend = sd_cpp_backend.SdCppDiffusionBackend.__new__(sd_cpp_backend.SdCppDiffusionBackend)
+ backend._state = None
+ backend._loading = sd_cpp_backend._SdLoading(
+ repo_id = "org/m", base_repo = "org/m", off_torch_device = "nvidia:0"
+ )
+ assert backend.runs_off_torch_device is True
+ # A torch-placed resident beside it still has to be freed.
+ backend._state = _state()
+ assert backend.runs_off_torch_device is False
+ # A failed load holds nothing.
+ backend._state = None
+ backend._loading.error = "boom"
+ assert backend.runs_off_torch_device is False
+ backend._loading = sd_cpp_backend._SdLoading(repo_id = "org/m", base_repo = "org/m")
+ assert backend.runs_off_torch_device is False
+
+
[email protected]("install_allowed", [False, True])
+def test_prediction_filters_a_leftover_build_like_selection(monkeypatch, install_allowed):
+ """A wrong prediction skips the route's pre-selection training guard and stages the wrong files."""
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_INSTALL", "1" if install_allowed else "0")
+ _record_cli_requests(monkeypatch)
+ monkeypatch.setattr(r, "ensure_sd_server_binary", lambda **_: "/opt/sd/sd-server")
+ monkeypatch.setattr(sd_cpp_backend, "_installed_accelerator_of", lambda _b: "rocm")
+ predicted = r.predict_engine(detect_family("z-image"), model_kind = "gguf")
+ # Installs allowed: the load replaces the ROCm build with the CUDA one, so native is right.
+ assert predicted == (ENGINE_SD_CPP if install_allowed else ENGINE_DIFFUSERS)
+ if not install_allowed:
+ r.select_and_activate_engine(detect_family("z-image"))
+ assert r.active_engine_name() == predicted
+
+
+ "family, kind, evicted", [("minimax-h3", "gguf", True), ("ltx-2", "pipeline", False)]
+)
+def test_a_native_video_load_frees_the_off_torch_images_engine(monkeypatch, family, kind, evicted):
+ """An off-torch image server holds the managed sd.cpp tree without owning DIFFUSION."""
+ import asyncio
+
+ from core.inference import diffusion_compat, diffusion_device, video
+ from hub.services.models import account_access
+ from models.inference import VideoLoadRequest
+ from routes import video as video_route
+
+ unloaded: list[str] = []
+ images = SimpleNamespace(runs_off_torch_device = True, unload = lambda: unloaded.append("images"))
+ backend = SimpleNamespace(
+ validate_load_request = lambda *_a, **_k: SimpleNamespace(name = family, base_repo = None),
+ begin_load = lambda *_a, **_k: {},
+ )
+
+ async def _no_ordinal(*_a, **_k):
+ return None
+
+ monkeypatch.setattr(r, "get_active_diffusion_engine", lambda: images)
+ monkeypatch.setattr(video, "get_video_backend", lambda: backend)
+ monkeypatch.setattr(video, "assert_video_precision_available", lambda *_a, **_k: None)
+ monkeypatch.setattr(video, "resolve_video_model_kind", lambda *_a, **_k: kind, raising = False)
+ monkeypatch.setattr(
+ video_route, "resolve_video_model_kind", lambda *_a, **_k: kind, raising = False
+ )
+ monkeypatch.setattr(
+ diffusion_device,
+ "resolve_diffusion_device_target",
+ lambda *_a, **_k: SimpleNamespace(device = "cpu", backend = "cpu"),
+ )
+ monkeypatch.setattr(diffusion_compat, "assert_pick_is_not_speech", lambda *_a, **_k: None)
+ monkeypatch.setattr(video_route, "_guard_video_load_against_training", lambda: None)
+ monkeypatch.setattr(video_route, "_selected_gpu_ordinal", _no_ordinal)
+ monkeypatch.setattr(account_access, "admit_media_load", lambda _k, fn, *_a: fn())
+ monkeypatch.setattr(account_access, "note_resident_components", lambda *_a, **_k: None)
+
+ request = VideoLoadRequest(model_path = "org/video", gguf_filename = "model.gguf")
+ asyncio.run(video_route.load_video_model_gated(request, "tester"))
+ assert unloaded == (["images"] if evicted else [])
+
+
+ "video_engine, evicted", [("sd_cpp", True), ("diffusers", False), (None, False)]
+)
+def test_an_off_torch_image_load_frees_a_native_video_model(monkeypatch, video_engine, evicted):
+ """The install it may run replaces the build a native H3 model was loaded from."""
+ calls, raised = _drive_load_route(
+ monkeypatch, predicted = ENGINE_SD_CPP, training_active = False, video_engine = video_engine
+ )
+ assert raised is None, raised
+ assert ("video_unload" in calls) is evicted
+ assert calls.index("select_and_activate") > (calls.index("video_unload") if evicted else -1)
+
+
+def test_a_refused_fallback_keeps_the_resident_engine(monkeypatch):
+ """A native prediction whose selection then falls back must be refused before _activate unloads."""
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_DEVICE", "nvidia")
+ _record_cli_requests(monkeypatch, path = None)
+ activated: list = []
+ monkeypatch.setattr(r, "_activate", lambda *a, **_k: activated.append(a))
+
+ def _refuse():
+ raise RuntimeError("training is running")
+
+ with pytest.raises(RuntimeError, match = "training"):
+ r.select_and_activate_engine(detect_family("z-image"), before_fallback = _refuse)
+ assert activated == []
+ # Without a guard the fallback still activates as before.
+ r.select_and_activate_engine(detect_family("z-image"))
+ assert activated and activated[-1][0] == ENGINE_DIFFUSERS
+
+
+ "repos, evicted", [(("unsloth/MiniMax-H3-GGUF",), True), (("org/ltx",), False)]
+)
+def test_an_in_flight_native_video_load_is_cancelled_too(monkeypatch, repos, evicted):
+ from core.inference import video_minimax_h3
+
+ h3_repos = (video_minimax_h3.H3_GGUF_REPO,) if evicted else repos
+ loading = SimpleNamespace(error = None, asset_repos = h3_repos)
+ calls, raised = _drive_load_route(
+ monkeypatch, predicted = ENGINE_SD_CPP, training_active = False, video_loading = loading
+ )
+ assert raised is None, raised
+ assert ("video_unload" in calls) is evicted