Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio native diffusion: run the text encoders on a second CUDA card when one card cannot hold the load (#17276)
Daniel Hanc38dc70a2280
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.pyindex 3800fa952c0..cabd73978f7 100644--- a/studio/backend/core/inference/sd_cpp_args.py+++ b/studio/backend/core/inference/sd_cpp_args.py@@ -710,26 +710,37 @@ def is_device_oom_text(text: Optional[str]) -> bool:def device_backend_flags(- device_name: Optional[str], offload: Optional[list[str]] = None+ device_name: Optional[str],+ offload: Optional[list[str]] = None,+ *,+ te_device: Optional[str] = None,+ vae_device: Optional[str] = None,) -> list[str]:"""Pins only modules the offload policy left on GPU, else sd.cpp uses ordinal 0 on mixed boxes; a later device flag would undo --clip-on-cpu."""if not device_name:return []flags = offload or []- te = "cpu" if "--clip-on-cpu" in flags else device_name- vae = "cpu" if "--vae-on-cpu" in flags else device_name+ te = "cpu" if "--clip-on-cpu" in flags else (te_device or device_name)+ vae = "cpu" if "--vae-on-cpu" in flags else (vae_device or device_name)return ["--backend", f"diffusion={device_name},te={te},vae={vae}"]def without_device_backend_flags(flags: Sequence[str]) -> list[str]:- """Strips every --backend pair: sd.cpp concatenates repeats, so a CPU restart needs the pin gone; status and recipe read empty flags as no offload."""+ """Strips every --backend pair: sd.cpp concatenates repeats, so a CPU restart needs the pin gone; status and recipe read empty flags as no offload.+ A two-card block split's ``--auto-fit`` / ``--max-vram`` belong to its pin and go with it."""+ items = list(flags)+ split = any(+ f == "--backend" and i + 1 < len(items) and "&" in str(items[i + 1])+ for i, f in enumerate(items)+ )+ paired = ("--backend", "--auto-fit", "--max-vram") if split else ("--backend",)out: list[str] = []skip = False- for flag in flags:+ for flag in items:if skip:skip = Falsecontinue- if flag == "--backend":+ if flag in paired:skip = Truecontinueout.append(flag)diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex bae0715f870..0a5d3bd62a7 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -41,7 +41,14 @@ from core.inference.diffusion_device import (auto_cuda_ordinal_for_load,resolve_diffusion_device_target,resolve_selected_cuda_ordinal,+ resolve_selected_cuda_ordinals,)+from core.inference.diffusion_multigpu import (+ AUTOMATIC_ORDINALS,+ multigpu_all_enabled,+ with_load_selection,+)+from core.inference.sd_cpp_multigpu import split_for_loadfrom core.inference.diffusion_families import (krea2_raw_checkpoint,krea2_sd_cpp_flow_shift,@@ -623,6 +630,14 @@ def sd_cpp_graph_cut_options(binary: Optional[str]) -> frozenset[str]:return frozenset(flag for flag in ("--max-vram", "--stream-layers") if flag in text)+def _layer_split_supported(binary: Optional[str]) -> bool:+ """Whether ``binary`` takes a per-device ``--max-vram`` and ``--auto-fit off`` (a two-card block split)."""+ if not binary:+ return False+ text = _sd_cpp_probe_output(binary, "--help")+ return text is not None and "--max-vram" in text and "--auto-fit" in text++def sd_cpp_supports_sage_attn(binary: Optional[str]) -> bool:"""fails closed because sd-cli rejects unknown flags (u13b9d92 predates --sage-attn)."""if not binary:@@ -2909,25 +2924,33 @@ class _SdState:child_env: tuple[tuple[str, str], ...] = ()# The user picked the GPU (a gpu_ids pool or UNSLOTH_DIFFUSION_SD_CPP_DEVICE): never moved to CPU behind their back.gpu_explicit: bool = False+ multi_gpu: Optional[dict[str, Any]] = None+ split_physical_gpu_id: Optional[int] = Nonedef spawn_env(self) -> Optional[dict[str, str]]:return dict(self.child_env) or None@propertydef placed_ordinal(self) -> Optional[int]:- """Torch ordinal of ``physical_gpu_id``; without it card-sharing helpers (upsampler) split over all GPUs."""- if self.physical_gpu_id is None or self.off_torch_device is not None:+ """Torch ordinal of ``physical_gpu_id`` (a split's diffusion card); without it card-sharing helpers (upsampler) split over all GPUs."""+ physical = (+ self.physical_gpu_id if self.physical_gpu_id is not None else self.split_physical_gpu_id+ )+ if physical is None or self.off_torch_device is not None:return Nonetry:from utils.hardware import get_parent_visible_gpu_idsvisible = [int(i) for i in get_parent_visible_gpu_ids()]- return visible.index(int(self.physical_gpu_id))+ return visible.index(int(physical))except Exception: # noqa: BLE001return Nonedef _offload_with_device_pin_impl(- offload: tuple[str, ...] | list[str], binary: Optional[str], ordinal: Optional[int]+ offload: tuple[str, ...] | list[str],+ binary: Optional[str],+ ordinal: Optional[int],+ split: Any = None,) -> list[str]:"""``offload`` plus the ``--backend`` pin for whichever build is about to run it."""flags = list(offload)@@ -2938,15 +2961,67 @@ def _offload_with_device_pin_impl(# `Vulkan0` is not a physical index; pin by card name instead.selected_name, selected_position = physical_card_name(ordinal)device_name = sd_cpp_device_named(binary, selected_name, position = selected_position)+ if split is not None and device_name is not None and int(split.primary) == int(ordinal):+ second = sd_cpp_device_name_for_ordinal(binary, int(split.secondary))+ if second is not None and second != device_name:+ layer = bool(split.layer) and _layer_split_supported(binary)+ pinned = device_backend_flags(+ f"{device_name}&{second}" if layer else device_name,+ flags,+ te_device = second,+ vae_device = second if split.vae_on_secondary else None,+ )+ if layer:+ # auto-fit would demand the whole model on each listed card; the budgets size sd.cpp's block cut+ budgets = ",".join(+ f"{name}={int(mib) / 1024:.2f}"+ for name, mib in zip((device_name, second), split.budgets_mib)+ )+ pinned += ["--auto-fit", "off", "--max-vram", budgets]+ return [*flags, *pinned]+ logger.info(+ "sd_cpp.multigpu: one card (cuda:%s %s)",+ split.secondary,+ "has no device name in this build"+ if second is None+ else f"resolves to {second} as well",+ )return [*flags, *device_backend_flags(device_name, flags)]+def _split_physical_gpu_id(ordinal: int) -> Optional[int]:+ """Physical id of torch ordinal ``ordinal`` (a split's diffusion card), or None."""+ try:+ from utils.hardware import get_parent_visible_gpu_ids+ visible = [int(i) for i in get_parent_visible_gpu_ids()]+ return visible[int(ordinal)] if 0 <= int(ordinal) < len(visible) else None+ except Exception: # noqa: BLE001 -- unknown keeps the helpers on every card, as before+ return None+++def _backend_devices(committed_flags: tuple[str, ...]) -> set[str]:+ """Every device a committed ``--backend`` assignment names (``diffusion=CUDA0,te=CUDA1`` -> CUDA0, CUDA1)."""+ out: set[str] = set()+ for index, flag in enumerate(committed_flags[:-1]):+ if flag != "--backend":+ continue+ for part in str(committed_flags[index + 1]).split(","):+ _key, sep, value = part.partition("=")+ for name in (value if sep else part).split("&"):+ name = name.strip()+ if name and name.lower() != "cpu":+ out.add(name.lower())+ return out++def _resolved_server_physical_gpu_id(binary: Optional[str], device: str, ordinal: Optional[int], committed_flags: tuple[str, ...]) -> Optional[int]:"""Physical id for a provably single-device resident sd-server."""if device != "cuda" or "--offload-to-cpu" in committed_flags:return None+ if len(_backend_devices(committed_flags)) > 1:+ return Nonetry:from utils.hardware import get_parent_visible_gpu_idsvisible = [int(i) for i in get_parent_visible_gpu_ids()]@@ -3518,6 +3593,16 @@ class SdCppDiffusionBackend:if auto_ordinal:# Automatic ranks the torch cards too, after validation, unless sd.cpp runs on one torch cannot see.gpu_ordinal = auto_cuda_ordinal_for_load()+ gpu_ordinals: Any = None+ if (+ off_torch is None+ and multigpu_all_enabled()+ and resolve_diffusion_device_target().device == "cuda"+ ):+ if gpu_ids and len(set(gpu_ids)) > 1:+ gpu_ordinals = resolve_selected_cuda_ordinals(gpu_ids)+ elif not gpu_ids:+ gpu_ordinals = AUTOMATIC_ORDINALSself._loading_family = fambase = resolve_base_repo(fam, base_repo)@@ -3585,12 +3670,14 @@ class SdCppDiffusionBackend:off_torch = off_torch,_load_token = token,_cancel_event = cancel_event,+ **({"_gpu_ordinals": gpu_ordinals} if gpu_ordinals else {}),),daemon = True,).start()return self.status()@_invalidates_gpu_memory("sd.cpp load")+ @with_load_selectiondef _run_load(self,*,@@ -3793,6 +3880,32 @@ class SdCppDiffusionBackend:def _ordinal() -> Optional[int]:return selected_ordinal if device == "cuda" else None+ split_memo: dict[tuple, Any] = {}++ def _split() -> Any:+ key = (device, _ordinal(), tuple(offload), _running()[1])+ if key not in split_memo:+ split_memo[key] = (+ split_for_load(+ files,+ primary = _ordinal(),+ offload = offload,+ layer_ok = _layer_split_supported(_running()[1]),+ logger = logger,+ )+ if device == "cuda" and off_torch is None+ else None+ )+ return split_memo[key]++ def _split_arg() -> tuple:+ split = _split()+ return () if split is None else (split,)++ def _pin_ordinal() -> Optional[int]:+ split = _split()+ return int(split.primary) if split is not None else _ordinal()+native_speed = _native_speed_for(speed_mode)# Teardown and commit run under _generate_lock, taken only now: abort and wait for a generation begun during the download, so no stale image persists.@@ -3877,12 +3990,12 @@ class SdCppDiffusionBackend:started_ok = Falsetry:server_offload = _offload_with_device_pin_impl(- offload, server_binary, _ordinal()+ offload, server_binary, _pin_ordinal(), *_split_arg())spawn_env = _sd_cpp_spawn_env(off_torch,device,- _ordinal(),+ _pin_ordinal(),offload,server_offload,_diffusion_weight_types,@@ -3959,14 +4072,15 @@ class SdCppDiffusionBackend:_offload_with_device_pin_impl(offload,server_binary if mode == "server" else getattr(engine, "binary", None),- _ordinal(),+ _pin_ordinal(),+ *_split_arg(),))# Recomputed for what is committed: a deferred install or the one-shot fallback can move placement.spawn_env = _sd_cpp_spawn_env(off_torch,device,- _ordinal(),+ _pin_ordinal(),offload,committed_offload_flags,_diffusion_weight_types,@@ -4011,6 +4125,18 @@ class SdCppDiffusionBackend:else None),selected_card = self._loading_card,+ multi_gpu = (+ _split().describe()+ if _split() is not None+ and len(_backend_devices(committed_offload_flags)) > 1+ else None+ ),+ split_physical_gpu_id = (+ _split_physical_gpu_id(_split().primary)+ if _split() is not None+ and len(_backend_devices(committed_offload_flags)) > 1+ else None+ ),off_torch_device = off_torch.label if off_torch is not None else None,child_env = tuple(sorted((spawn_env or {}).items())),gpu_explicit = gpu_explicit,@@ -5003,6 +5129,8 @@ class SdCppDiffusionBackend:without_device_backend_flags(state.offload_flags)),physical_gpu_id = None,+ multi_gpu = None,+ split_physical_gpu_id = None,)self._hand_over_replacement(state, server, old_server)# The old server's stop removed its scratch dir, and the staged adapters with it.@@ -5592,6 +5720,12 @@ class SdCppDiffusionBackend:"workflows": workflows,"conditioning": conditioning,**_native_hires_status(state),+ # not after a restart onto the CPU backend, whose argv has no device pin+ **(+ {"multi_gpu": state.multi_gpu}+ if state.multi_gpu is not None and not self._cpu_backend_forced+ else {}+ ),}diff --git a/studio/backend/core/inference/sd_cpp_multigpu.py b/studio/backend/core/inference/sd_cpp_multigpu.pynew file mode 100644index 00000000000..d452066de14--- /dev/null+++ b/studio/backend/core/inference/sd_cpp_multigpu.py@@ -0,0 +1,201 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Two-card sd-cli split via per-module ``--backend``; sized from nvidia-smi so Studio opens no CUDA context."""++from __future__ import annotations++import dataclasses+import json+import os+from typing import Any, Callable, Optional, Sequence++# Working memory beside the weights; denoise matches the graph-cut budget's 2 GiB.+DENOISE_HEADROOM_MIB = 2048+ENCODER_HEADROOM_MIB = 1024+VAE_DECODE_HEADROOM_MIB = 1536+# Held back from free MiB: the child's CUDA context (~550 MiB on a T4) plus sd.cpp auto-fit's 512 MiB.+CARD_RESERVE_MIB = 1536++[email protected](frozen = True)+class SdCppCardSplit:+ primary: int+ secondary: int+ vae_on_secondary: bool+ reason: str+ # the diffusion model's blocks across both cards (sd.cpp's own layer split), each card held to its budget+ layer: bool = False+ budgets_mib: tuple[int, int] = (0, 0)++ def describe(self) -> dict[str, Any]:+ diffusion = f"cuda:{self.primary}" + (f"+cuda:{self.secondary}" if self.layer else "")+ return {+ "chosen": "layer_split" if self.layer else "companion_split",+ "primary": f"cuda:{self.primary}",+ "secondary": f"cuda:{self.secondary}",+ "placement": {+ "diffusion_model": diffusion,+ "text_encoders": f"cuda:{self.secondary}",+ "vae": f"cuda:{self.secondary if self.vae_on_secondary else self.primary}",+ },+ "summary": self.reason,+ }+++def plan_sd_cpp_split(+ *,+ diffusion_mib: int,+ text_encoder_mib: int,+ vae_mib: int,+ primary: int,+ primary_limit_mib: Optional[int],+ secondary_limits_mib: dict[int, Optional[int]],+ layer_ok: bool = False,+) -> tuple[Optional[SdCppCardSplit], str]:+ """``(split, reason)``. Splits only when one card cannot hold every module; with ``layer_ok`` blocks are cut across both."""+ if primary_limit_mib is None:+ return None, f"free memory of cuda:{primary} is unknown"+ if text_encoder_mib <= 0:+ return None, "no text encoder to move"+ whole = diffusion_mib + text_encoder_mib + vae_mib + DENOISE_HEADROOM_MIB+ if whole <= primary_limit_mib:+ return (+ None,+ f"not needed: every module fits cuda:{primary} ({whole} of {primary_limit_mib} MiB)",+ )+ limits = {+ int(o): int(v)+ for o, v in secondary_limits_mib.items()+ if v is not None and int(o) != primary+ }+ if not limits:+ return None, "no second card with readable free memory"+ best = max(limits, key = lambda o: (limits[o], -o))+ # each card's room for diffusion blocks; the secondary's is counted after the encoders, the VAE and their working memory+ primary_room = primary_limit_mib - DENOISE_HEADROOM_MIB+ secondary_room = (+ limits[best]+ - text_encoder_mib+ - vae_mib+ - DENOISE_HEADROOM_MIB+ - max(ENCODER_HEADROOM_MIB, VAE_DECODE_HEADROOM_MIB)+ )+ if (+ layer_ok+ and diffusion_mib + DENOISE_HEADROOM_MIB > primary_limit_mib+ and primary_room > 0+ and secondary_room > 0+ and diffusion_mib <= primary_room + secondary_room+ ):+ # no card holds the diffusion model resident: sd.cpp cuts its blocks in proportion to each card's budget+ return (+ SdCppCardSplit(+ primary = int(primary),+ secondary = best,+ vae_on_secondary = vae_mib > 0,+ layer = True,+ budgets_mib = (int(primary_limit_mib), int(limits[best])),+ reason = (+ f"multi-GPU: diffusion model ({diffusion_mib} MiB) split across cuda:{primary} ({primary_room} MiB "+ f"room) and cuda:{best} ({secondary_room} MiB beside the text encoders); one card needs {whole} MiB"+ ),+ ),+ "",+ )+ with_vae = text_encoder_mib + vae_mib + max(ENCODER_HEADROOM_MIB, VAE_DECODE_HEADROOM_MIB)+ encoders = text_encoder_mib + ENCODER_HEADROOM_MIB+ options = ((with_vae, True), (encoders, False)) if vae_mib > 0 else ((encoders, False),)+ for need, vae_moves in options:+ if need > limits[best]:+ continue+ on_primary = diffusion_mib + (0 if vae_moves else vae_mib) + DENOISE_HEADROOM_MIB+ moved = "text encoders + VAE" if vae_moves else "text encoders"+ return (+ SdCppCardSplit(+ primary = int(primary),+ secondary = best,+ vae_on_secondary = vae_moves,+ reason = (+ f"multi-GPU: diffusion model on cuda:{primary} ({on_primary} of {primary_limit_mib} MiB), "+ f"{moved} on cuda:{best} ({need} of {limits[best]} MiB); one card needs {whole} MiB"+ ),+ ),+ "",+ )+ return None, f"text encoders need {encoders} MiB, cuda:{best} has {limits[best]} MiB"+++def file_mib(path: Optional[str]) -> int:+ """MiB of a model file; a ``*.index.json`` counts its shards. 0 when unreadable."""+ if not path:+ return 0+ try:+ if str(path).endswith(".index.json"):+ with open(path, encoding = "utf-8") as f:+ shards = set((json.load(f).get("weight_map") or {}).values())+ base = os.path.dirname(str(path))+ return sum(os.path.getsize(os.path.join(base, s)) for s in shards) >> 20+ return os.path.getsize(str(path)) >> 20+ except (OSError, ValueError, TypeError, AttributeError):+ return 0+++TEXT_ENCODER_FIELDS: tuple[str, ...] = ("clip_l", "clip_g", "t5xxl", "llm", "llm_vision", "qwen2vl")+++def card_limit_mib(ordinal: int) -> Optional[int]:+ """Free MiB minus max(CARD_RESERVE_MIB, VRAM Budget reserve), via nvidia-smi (no CUDA context here). None when unreadable."""+ from .diffusion_device import smi_free_total_mib+ from .diffusion_memory import _user_budget_reserve_mib++ free, total = smi_free_total_mib(int(ordinal))+ if free is None:+ return None+ reserve = max(CARD_RESERVE_MIB, int(_user_budget_reserve_mib(total or free) or 0))+ return max(0, int(free) - reserve)+++def split_for_load(+ files: Any,+ *,+ primary: Optional[int],+ offload: Sequence[str],+ layer_ok: bool = False,+ limit_of: Optional[Callable[[int], Optional[int]]] = None,+ logger: Any = None,+) -> Optional[SdCppCardSplit]:+ """Split for this load, else None. Only without offload flags; balanced / low_vram / cpu_offload stay one card."""+ from .diffusion_multigpu import current_selection, multigpu_all_enabled++ selection = current_selection()+ if selection is None or not multigpu_all_enabled() or not selection.secondaries:+ return None+ if primary is None and selection.automatic:+ primary = selection.primary+ if primary is None or int(primary) != int(selection.primary):+ return None+ limit_of = limit_of or card_limit_mib+ if list(offload):+ if logger is not None:+ logger.info("sd_cpp.multigpu: one card (explicit offload %s)", " ".join(offload))+ return None+ from .diffusion_memory import vram_budget_scope++ limits = vram_budget_scope("image", load = True)(+ lambda: {int(o): limit_of(int(o)) for o in (int(primary), *selection.secondaries)}+ )()+ split, why = plan_sd_cpp_split(+ diffusion_mib = file_mib(getattr(files, "diffusion_model", None)),+ text_encoder_mib = sum(file_mib(getattr(files, f, None)) for f in TEXT_ENCODER_FIELDS),+ vae_mib = file_mib(getattr(files, "vae", None)),+ primary = int(primary),+ primary_limit_mib = limits.get(int(primary)),+ secondary_limits_mib = {int(o): limits.get(int(o)) for o in selection.secondaries},+ layer_ok = layer_ok,+ )+ if logger is not None:+ logger.info(+ "sd_cpp.multigpu: %s", split.reason if split is not None else f"one card ({why})"+ )+ return splitdiff --git a/studio/backend/tests/test_sd_cpp_multigpu.py b/studio/backend/tests/test_sd_cpp_multigpu.pynew file mode 100644index 00000000000..6d5002c35f3--- /dev/null+++ b/studio/backend/tests/test_sd_cpp_multigpu.py@@ -0,0 +1,414 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The native (sd.cpp) engine on two CUDA cards, CPU only: begin_load carries the multi-card selection, the split+decision, the two-card ``--backend`` assignment the server gets, status, and every way it stays one card (kill+switch, one card, explicit offload, a load that fits one card)."""++from __future__ import annotations++import os+import sys+import time+import types+from pathlib import Path++import pytest++from core.inference import sd_cpp_backend as bk+from core.inference.diffusion_families import detect_family+from core.inference.sd_cpp_backend import SdCppDiffusionBackend++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++from test_sd_cpp_backend import _FakeServer # noqa: E402++GIB = 1024+++def _m():+ from core.inference import sd_cpp_multigpu+ return sd_cpp_multigpu+++# An idle Kaggle T4: 14,900 MiB free, less the 1.5 GiB reserve.+T4_LIMIT = 13_364++[email protected](autouse = True)+def _clean_env(monkeypatch):+ for name in ("UNSLOTH_DIFFUSION_MULTIGPU", "UNSLOTH_DIFFUSION_MULTIGPU_ALL"):+ monkeypatch.delenv(name, raising = False)+++def _plan(**kw):+ args = dict(+ diffusion_mib = 12_247,+ text_encoder_mib = 4_465,+ vae_mib = 242,+ primary = 0,+ primary_limit_mib = T4_LIMIT,+ secondary_limits_mib = {1: T4_LIMIT},+ )+ args.update(kw)+ return _m().plan_sd_cpp_split(**args)+++def test_a_load_that_does_not_fit_one_card_moves_the_encoders_to_the_second():+ # Qwen-Image Q4_K_M on 2x T4: 12.0 GiB DiT + 4.4 GiB Qwen2.5-VL + VAE does not fit one card resident.+ split, why = _plan()+ assert split is not None and why == ""+ assert (split.primary, split.secondary, split.vae_on_secondary) == (0, 1, True)+ assert split.describe()["placement"] == {+ "diffusion_model": "cuda:0",+ "text_encoders": "cuda:1",+ "vae": "cuda:1",+ }+++def test_the_vae_stays_with_the_diffusion_model_when_the_second_card_has_no_decode_room():+ split, _ = _plan(secondary_limits_mib = {1: 5_600})+ assert split is not None and split.vae_on_secondary is False+++def test_one_card_whenever_one_card_holds_everything_or_nothing_fits():+ assert _plan(primary_limit_mib = 40_000)[0] is None+ assert "not needed" in _plan(primary_limit_mib = 40_000)[1]+ assert _plan(secondary_limits_mib = {1: 2_000})[0] is None+ assert _plan(secondary_limits_mib = {1: None})[0] is None+ assert _plan(primary_limit_mib = None)[0] is None+ assert _plan(text_encoder_mib = 0)[0] is None+ split, _ = _plan(vae_mib = 0)+ assert split is not None and split.vae_on_secondary is False+++def test_the_roomiest_secondary_is_chosen():+ split, _ = _plan(secondary_limits_mib = {1: 7_000, 2: 9_000, 3: 9_000})+ assert split.secondary == 2+++def test_file_sizes_count_index_shards(tmp_path):+ (tmp_path / "a.safetensors").write_bytes(b"\0" * (3 << 20))+ (tmp_path / "b.safetensors").write_bytes(b"\0" * (2 << 20))+ index = tmp_path / "model.safetensors.index.json"+ index.write_text(+ '{"weight_map": {"x": "a.safetensors", "y": "b.safetensors", "z": "a.safetensors"}}'+ )+ assert _m().file_mib(str(index)) == 5+ assert _m().file_mib(str(tmp_path / "a.safetensors")) == 3+ assert _m().file_mib(str(tmp_path / "missing.gguf")) == 0+ assert _m().file_mib(None) == 0+++def _names(monkeypatch):+ monkeypatch.setattr(+ bk, "_sd_cpp_probe_output", lambda *_a: "CUDA0\tTesla T4\nCUDA1\tTesla T4\nCPU\tXeon\n"+ )+ monkeypatch.setattr(bk, "physical_card_name", lambda ordinal: (None, None))+++def test_the_pin_names_a_card_per_module(monkeypatch):+ _names(monkeypatch)+ split = _m().SdCppCardSplit(primary = 0, secondary = 1, vae_on_secondary = False, reason = "r")+ assert bk._offload_with_device_pin_impl([], "/sd/sd-cli", 0, split) == [+ "--backend",+ "diffusion=CUDA0,te=CUDA1,vae=CUDA0",+ ]+ both = _m().SdCppCardSplit(primary = 0, secondary = 1, vae_on_secondary = True, reason = "r")+ assert bk._offload_with_device_pin_impl([], "/sd/sd-cli", 0, both) == [+ "--backend",+ "diffusion=CUDA0,te=CUDA1,vae=CUDA1",+ ]+ # without a split (and for a split of another primary) the one-card pin is byte-identical to before+ for extra in (+ (),+ (None,),+ (_m().SdCppCardSplit(primary = 1, secondary = 0, vae_on_secondary = False, reason = "r"),),+ ):+ assert bk._offload_with_device_pin_impl([], "/sd/sd-cli", 0, *extra) == [+ "--backend",+ "diffusion=CUDA0,te=CUDA0,vae=CUDA0",+ ]+++def test_a_two_card_server_is_not_charged_to_one_card(monkeypatch):+ _names(monkeypatch)+ monkeypatch.setattr("utils.hardware.get_parent_visible_gpu_ids", lambda: [0, 1])+ one = ("--backend", "diffusion=CUDA0,te=CUDA0,vae=CUDA0")+ two = ("--backend", "diffusion=CUDA0,te=CUDA1,vae=CUDA0")+ assert bk._resolved_server_physical_gpu_id("/sd/sd-server", "cuda", 0, one) == 0+ assert bk._resolved_server_physical_gpu_id("/sd/sd-server", "cuda", 0, two) is None+ assert bk._backend_devices(two) == {"cuda0", "cuda1"}+ assert bk._backend_devices(("--backend", "diffusion=CUDA0&CUDA1,te=cpu,vae=CUDA0")) == {+ "cuda0",+ "cuda1",+ }+++def _begin(monkeypatch, gpu_ids):+ seen: dict = {}+ monkeypatch.setattr(+ "core.inference.diffusion_engine_router.off_torch_sd_cpp_device", lambda *a, **k: None+ )+ monkeypatch.setattr(+ bk, "resolve_diffusion_device_target", lambda **kw: types.SimpleNamespace(device = "cuda")+ )+ monkeypatch.setattr(bk, "resolve_selected_cuda_ordinal", lambda ids: min(ids))+ monkeypatch.setattr(bk, "resolve_selected_cuda_ordinals", lambda ids: sorted(set(ids)))+ monkeypatch.setattr(bk, "auto_cuda_ordinal_for_load", lambda: 0)+ b = SdCppDiffusionBackend()+ monkeypatch.setattr(b, "_run_load", lambda **kw: seen.update(kw, done = True))+ b.begin_load(+ repo_id = "unsloth/Qwen-Image-GGUF", gguf_filename = "qwen-image-Q4_K_M.gguf", gpu_ids = gpu_ids+ )+ deadline = time.monotonic() + 5+ while "done" not in seen and time.monotonic() < deadline:+ time.sleep(0.01)+ return seen+++def test_begin_load_carries_the_multi_card_selection(monkeypatch):+ assert _begin(monkeypatch, [0, 1])["_gpu_ordinals"] == [0, 1]+ assert _begin(monkeypatch, None)["_gpu_ordinals"] == "automatic"+++def test_one_card_and_the_kill_switch_carry_no_selection(monkeypatch):+ assert "_gpu_ordinals" not in _begin(monkeypatch, [1])+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MULTIGPU_ALL", "0")+ assert "_gpu_ordinals" not in _begin(monkeypatch, [0, 1])+ assert "_gpu_ordinals" not in _begin(monkeypatch, None)+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MULTIGPU_ALL", "1")+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MULTIGPU", "0")+ assert "_gpu_ordinals" not in _begin(monkeypatch, [0, 1])+++def _sparse(path: Path, mib: int) -> str:+ with open(path, "wb") as f:+ f.truncate(mib << 20)+ return str(path)+++def _server_load(+ monkeypatch,+ tmp_path,+ *,+ gpu_ordinals,+ memory_mode = None,+ limits = None,+ gpu_ordinal = 0,+):+ _names(monkeypatch)+ cards = limits or {0: (14_900, 15_360), 1: (14_900, 15_360)}+ monkeypatch.setattr(+ "core.inference.diffusion_device.smi_free_total_mib",+ lambda o: cards.get(int(o), (None, None)),+ )+ fam = detect_family("qwen-image")+ servers: list = []+ monkeypatch.setattr(bk, "find_sd_server_binary", lambda: "/x/sd-server")+ monkeypatch.setattr(bk, "_server_binary_runnable", lambda *_a, **_k: True)+ monkeypatch.setattr(+ bk, "SdCppServer", lambda binary: (servers.append(_FakeServer(binary)), servers[-1])[1]+ )+ paths = {+ "diffusion_model": _sparse(tmp_path / "qwen-image-Q4_K_M.gguf", 12 * GIB),+ "vae": _sparse(tmp_path / "vae.safetensors", 250),+ "qwen2vl": _sparse(tmp_path / "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", 4 * GIB + 400),+ }+ b = SdCppDiffusionBackend()+ monkeypatch.setattr(b, "_asset_specs", lambda *a, **k: [])+ monkeypatch.setattr(b, "_set_expected_bytes", lambda *a, **k: None)+ monkeypatch.setattr(b, "_fetch_assets", lambda *a, **k: dict(paths))+ monkeypatch.setattr(+ bk, "resolve_diffusion_device_target", lambda: types.SimpleNamespace(device = "cuda")+ )+ b._load_token = 1+ kwargs = dict(+ repo_id = "unsloth/Qwen-Image-GGUF",+ gguf_filename = "qwen-image-Q4_K_M.gguf",+ base = fam.base_repo,+ fam = fam,+ hf_token = None,+ _load_token = 1,+ gpu_ordinal = gpu_ordinal,+ memory_mode = memory_mode,+ )+ if gpu_ordinals is not None:+ kwargs["_gpu_ordinals"] = gpu_ordinals+ b._run_load(**kwargs)+ return b, servers+++def _backend_arg(flags):+ flags = list(flags or ())+ return flags[flags.index("--backend") + 1] if "--backend" in flags else None+++def test_a_two_card_load_starts_the_server_with_the_encoders_on_the_second_card(+ monkeypatch, tmp_path+):+ b, servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = [0, 1])+ assert _backend_arg(servers[-1].started.get("offload")) == "diffusion=CUDA0,te=CUDA1,vae=CUDA1"+ assert _backend_arg(b._state.offload_flags) == "diffusion=CUDA0,te=CUDA1,vae=CUDA1"+ assert b._state.physical_gpu_id is None+ status = b.status()+ assert status["multi_gpu"]["placement"]["text_encoders"] == "cuda:1"+ assert status["offload_policy"] == "none"+++ "case",+ ["one_card", "kill_switch", "explicit_offload", "fits_one_card"],+)+def test_every_one_card_case_keeps_main_argv(monkeypatch, tmp_path, case):+ ordinals = [0, 1]+ memory_mode, limits = None, None+ if case == "one_card":+ ordinals = None+ elif case == "kill_switch":+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MULTIGPU_ALL", "0")+ elif case == "explicit_offload":+ memory_mode = "balanced"+ elif case == "fits_one_card":+ limits = {0: (48_000, 49_152), 1: (48_000, 49_152)}+ b, servers = _server_load(+ monkeypatch, tmp_path, gpu_ordinals = ordinals, memory_mode = memory_mode, limits = limits+ )+ assert _backend_arg(servers[-1].started.get("offload")) == "diffusion=CUDA0,te=CUDA0,vae=CUDA0"+ assert "multi_gpu" not in b.status()+ if case != "explicit_offload":+ ref, ref_servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = None)+ assert servers[-1].started.get("offload") == ref_servers[-1].started.get("offload")+ assert b._state.offload_flags == ref._state.offload_flags+++def test_keep_multiple_models_loaded_keeps_one_card(monkeypatch, tmp_path):+ monkeypatch.setattr("core.inference.diffusion_multigpu.models_kept_loaded", lambda: True)+ b, servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = [0, 1])+ assert _backend_arg(servers[-1].started.get("offload")) == "diffusion=CUDA0,te=CUDA0,vae=CUDA0"+ assert os.environ.get("UNSLOTH_DIFFUSION_MULTIGPU_ALL") is None+++def test_automatic_with_an_unpinned_card_still_splits_and_pins_the_primary(monkeypatch, tmp_path):+ # identical idle cards: Automatic's ranker leaves the load unpinned, the selection's primary is cuda:0+ monkeypatch.setattr("core.inference.diffusion_device.auto_cuda_ranking_possible", lambda: True)+ monkeypatch.setattr(+ "core.inference.diffusion_device.resolve_diffusion_device_target",+ lambda **kw: types.SimpleNamespace(device = "cuda"),+ )+ monkeypatch.setattr("torch.cuda.current_device", lambda: 0)+ monkeypatch.setattr("torch.cuda.device_count", lambda: 2)+ b, servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = "automatic", gpu_ordinal = None)+ assert _backend_arg(servers[-1].started.get("offload")) == "diffusion=CUDA0,te=CUDA1,vae=CUDA1"+ assert b.status()["multi_gpu"]["primary"] == "cuda:0"+++def test_helpers_keep_to_the_split_diffusion_card_and_status_follows_a_cpu_restart(+ monkeypatch, tmp_path+):+ monkeypatch.setattr("utils.hardware.get_parent_visible_gpu_ids", lambda: [2, 3])+ b, _servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = [0, 1])+ assert b._state.physical_gpu_id is None+ assert b._state.placed_ordinal == 0+ assert "multi_gpu" in b.status()+ b._cpu_backend_forced = True+ assert "multi_gpu" not in b.status()+++def test_a_diffusion_model_no_card_holds_is_cut_across_both(monkeypatch):+ split, _ = _plan(layer_ok = True)+ assert split.layer is True and split.vae_on_secondary is True+ assert split.budgets_mib == (T4_LIMIT, T4_LIMIT)+ assert split.describe()["placement"]["diffusion_model"] == "cuda:0+cuda:1"+ # a model that fits the primary keeps the companion split; without a capable build there is no block split+ assert _plan(diffusion_mib = 9_000, layer_ok = True)[0].layer is False+ assert _plan(layer_ok = False)[0].layer is False+ assert _plan(diffusion_mib = 22_000, layer_ok = True)[0].layer is False+ # the secondary's room is what is left beside the encoders and the VAE: no blocks fit there, no block split+ crowded, _ = _plan(+ primary_limit_mib = 20_000,+ secondary_limits_mib = {1: 14_000},+ diffusion_mib = 23_000,+ text_encoder_mib = 10_000,+ vae_mib = 250,+ layer_ok = True,+ )+ assert crowded is None or crowded.layer is False+++def test_the_block_split_pin_lists_both_cards_and_their_budgets(monkeypatch):+ monkeypatch.setattr(+ bk,+ "_sd_cpp_probe_output",+ lambda _b, *args: (+ " --max-vram budget\n --auto-fit on|off\n"+ if args and args[0] == "--help"+ else "CUDA0\tTesla T4\nCUDA1\tTesla T4\n"+ ),+ )+ monkeypatch.setattr(bk, "physical_card_name", lambda ordinal: (None, None))+ split = _m().SdCppCardSplit(+ primary = 0,+ secondary = 1,+ vae_on_secondary = True,+ reason = "r",+ layer = True,+ budgets_mib = (13_876, 13_876),+ )+ flags = bk._offload_with_device_pin_impl([], "/sd/sd-server", 0, split)+ assert flags == [+ "--backend",+ "diffusion=CUDA0&CUDA1,te=CUDA1,vae=CUDA1",+ "--auto-fit",+ "off",+ "--max-vram",+ "CUDA0=13.55,CUDA1=13.55",+ ]+ from core.inference.sd_cpp_args import without_device_backend_flags++ # the CPU restart and status drop the whole pin; a one-card pin keeps any other --max-vram+ assert without_device_backend_flags(["--diffusion-fa", *flags]) == ["--diffusion-fa"]+ assert without_device_backend_flags(+ ["--max-vram", "-1", "--backend", "diffusion=CUDA0,te=CUDA0,vae=CUDA0"]+ ) == ["--max-vram", "-1"]+ monkeypatch.setattr(+ bk, "_sd_cpp_probe_output", lambda _b, *args: "CUDA0\tTesla T4\nCUDA1\tTesla T4\n"+ )+ assert bk._offload_with_device_pin_impl([], "/sd/sd-server", 0, split) == [+ "--backend",+ "diffusion=CUDA0,te=CUDA1,vae=CUDA1",+ ]+++def test_a_two_card_load_cuts_the_model_across_both_cards_on_a_capable_build(monkeypatch, tmp_path):+ monkeypatch.setattr(bk, "_layer_split_supported", lambda binary: True)+ monkeypatch.setattr("utils.hardware.get_parent_visible_gpu_ids", lambda: [0, 1])+ b, servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = [0, 1])+ started = list(servers[-1].started.get("offload"))+ assert _backend_arg(started) == "diffusion=CUDA0&CUDA1,te=CUDA1,vae=CUDA1"+ assert started[started.index("--auto-fit") + 1] == "off"+ assert "--max-vram" in started+ status = b.status()+ assert status["multi_gpu"]["chosen"] == "layer_split"+ assert status["offload_policy"] == "none" and status["cpu_offload"] is False+ assert b._state.physical_gpu_id is None and b._state.placed_ordinal == 0+++def test_a_model_sd_cpp_would_stream_on_one_card_is_cut_across_both(monkeypatch, tmp_path):+ # 2x T4 under Automatic: nvidia-smi shows 15,360 MiB free, the sd-server's CUDA context leaves 14,806 and auto-fit+ # budgets 14,294, 1 MiB short of the 12,247 MiB model plus its 2 GiB reserve: priced as one card, it streams.+ monkeypatch.setattr(bk, "_layer_split_supported", lambda binary: True)+ monkeypatch.setattr("utils.hardware.get_parent_visible_gpu_ids", lambda: [0, 1])+ b, servers = _server_load(+ monkeypatch,+ tmp_path,+ gpu_ordinals = [0, 1],+ limits = {0: (15_360, 15_360), 1: (15_360, 15_360)},+ )+ assert (+ _backend_arg(servers[-1].started.get("offload"))+ == "diffusion=CUDA0&CUDA1,te=CUDA1,vae=CUDA1"+ )+ assert b.status()["multi_gpu"]["chosen"] == "layer_split"