Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio native diffusion: run the text encoders on a second CUDA card when one card cannot hold the load (#17276)

Daniel Hanc38dc70a2280
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.py
index 3800fa952c0..cabd73978f7 100644
--- a/studio/backend/core/inference/sd_cpp_args.py
+++ b/studio/backend/core/inference/sd_cpp_args.py
@@ -710,26 +710,37 @@ def is_device_oom_text(text: Optional[str]) -> bool:
def device_backend_flags(
- device_name: Optional[str], offload: Optional[list[str]] = None
+ device_name: Optional[str],
+ offload: Optional[list[str]] = None,
+ *,
+ te_device: Optional[str] = None,
+ vae_device: Optional[str] = None,
) -> list[str]:
"""Pins only modules the offload policy left on GPU, else sd.cpp uses ordinal 0 on mixed boxes; a later device flag would undo --clip-on-cpu."""
if not device_name:
return []
flags = offload or []
- te = "cpu" if "--clip-on-cpu" in flags else device_name
- vae = "cpu" if "--vae-on-cpu" in flags else device_name
+ te = "cpu" if "--clip-on-cpu" in flags else (te_device or device_name)
+ vae = "cpu" if "--vae-on-cpu" in flags else (vae_device or device_name)
return ["--backend", f"diffusion={device_name},te={te},vae={vae}"]
def without_device_backend_flags(flags: Sequence[str]) -> list[str]:
- """Strips every --backend pair: sd.cpp concatenates repeats, so a CPU restart needs the pin gone; status and recipe read empty flags as no offload."""
+ """Strips every --backend pair: sd.cpp concatenates repeats, so a CPU restart needs the pin gone; status and recipe read empty flags as no offload.
+ A two-card block split's ``--auto-fit`` / ``--max-vram`` belong to its pin and go with it."""
+ items = list(flags)
+ split = any(
+ f == "--backend" and i + 1 < len(items) and "&" in str(items[i + 1])
+ for i, f in enumerate(items)
+ )
+ paired = ("--backend", "--auto-fit", "--max-vram") if split else ("--backend",)
out: list[str] = []
skip = False
- for flag in flags:
+ for flag in items:
if skip:
skip = False
continue
- if flag == "--backend":
+ if flag in paired:
skip = True
continue
out.append(flag)
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index bae0715f870..0a5d3bd62a7 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -41,7 +41,14 @@ from core.inference.diffusion_device import (
auto_cuda_ordinal_for_load,
resolve_diffusion_device_target,
resolve_selected_cuda_ordinal,
+ resolve_selected_cuda_ordinals,
)
+from core.inference.diffusion_multigpu import (
+ AUTOMATIC_ORDINALS,
+ multigpu_all_enabled,
+ with_load_selection,
+)
+from core.inference.sd_cpp_multigpu import split_for_load
from core.inference.diffusion_families import (
krea2_raw_checkpoint,
krea2_sd_cpp_flow_shift,
@@ -623,6 +630,14 @@ def sd_cpp_graph_cut_options(binary: Optional[str]) -> frozenset[str]:
return frozenset(flag for flag in ("--max-vram", "--stream-layers") if flag in text)
+def _layer_split_supported(binary: Optional[str]) -> bool:
+ """Whether ``binary`` takes a per-device ``--max-vram`` and ``--auto-fit off`` (a two-card block split)."""
+ if not binary:
+ return False
+ text = _sd_cpp_probe_output(binary, "--help")
+ return text is not None and "--max-vram" in text and "--auto-fit" in text
+
+
def sd_cpp_supports_sage_attn(binary: Optional[str]) -> bool:
"""fails closed because sd-cli rejects unknown flags (u13b9d92 predates --sage-attn)."""
if not binary:
@@ -2909,25 +2924,33 @@ class _SdState:
child_env: tuple[tuple[str, str], ...] = ()
# The user picked the GPU (a gpu_ids pool or UNSLOTH_DIFFUSION_SD_CPP_DEVICE): never moved to CPU behind their back.
gpu_explicit: bool = False
+ multi_gpu: Optional[dict[str, Any]] = None
+ split_physical_gpu_id: Optional[int] = None
def spawn_env(self) -> Optional[dict[str, str]]:
return dict(self.child_env) or None
@property
def placed_ordinal(self) -> Optional[int]:
- """Torch ordinal of ``physical_gpu_id``; without it card-sharing helpers (upsampler) split over all GPUs."""
- if self.physical_gpu_id is None or self.off_torch_device is not None:
+ """Torch ordinal of ``physical_gpu_id`` (a split's diffusion card); without it card-sharing helpers (upsampler) split over all GPUs."""
+ physical = (
+ self.physical_gpu_id if self.physical_gpu_id is not None else self.split_physical_gpu_id
+ )
+ if physical is None or self.off_torch_device is not None:
return None
try:
from utils.hardware import get_parent_visible_gpu_ids
visible = [int(i) for i in get_parent_visible_gpu_ids()]
- return visible.index(int(self.physical_gpu_id))
+ return visible.index(int(physical))
except Exception: # noqa: BLE001
return None
def _offload_with_device_pin_impl(
- offload: tuple[str, ...] | list[str], binary: Optional[str], ordinal: Optional[int]
+ offload: tuple[str, ...] | list[str],
+ binary: Optional[str],
+ ordinal: Optional[int],
+ split: Any = None,
) -> list[str]:
"""``offload`` plus the ``--backend`` pin for whichever build is about to run it."""
flags = list(offload)
@@ -2938,15 +2961,67 @@ def _offload_with_device_pin_impl(
# `Vulkan0` is not a physical index; pin by card name instead.
selected_name, selected_position = physical_card_name(ordinal)
device_name = sd_cpp_device_named(binary, selected_name, position = selected_position)
+ if split is not None and device_name is not None and int(split.primary) == int(ordinal):
+ second = sd_cpp_device_name_for_ordinal(binary, int(split.secondary))
+ if second is not None and second != device_name:
+ layer = bool(split.layer) and _layer_split_supported(binary)
+ pinned = device_backend_flags(
+ f"{device_name}&{second}" if layer else device_name,
+ flags,
+ te_device = second,
+ vae_device = second if split.vae_on_secondary else None,
+ )
+ if layer:
+ # auto-fit would demand the whole model on each listed card; the budgets size sd.cpp's block cut
+ budgets = ",".join(
+ f"{name}={int(mib) / 1024:.2f}"
+ for name, mib in zip((device_name, second), split.budgets_mib)
+ )
+ pinned += ["--auto-fit", "off", "--max-vram", budgets]
+ return [*flags, *pinned]
+ logger.info(
+ "sd_cpp.multigpu: one card (cuda:%s %s)",
+ split.secondary,
+ "has no device name in this build"
+ if second is None
+ else f"resolves to {second} as well",
+ )
return [*flags, *device_backend_flags(device_name, flags)]
+def _split_physical_gpu_id(ordinal: int) -> Optional[int]:
+ """Physical id of torch ordinal ``ordinal`` (a split's diffusion card), or None."""
+ try:
+ from utils.hardware import get_parent_visible_gpu_ids
+ visible = [int(i) for i in get_parent_visible_gpu_ids()]
+ return visible[int(ordinal)] if 0 <= int(ordinal) < len(visible) else None
+ except Exception: # noqa: BLE001 -- unknown keeps the helpers on every card, as before
+ return None
+
+
+def _backend_devices(committed_flags: tuple[str, ...]) -> set[str]:
+ """Every device a committed ``--backend`` assignment names (``diffusion=CUDA0,te=CUDA1`` -> CUDA0, CUDA1)."""
+ out: set[str] = set()
+ for index, flag in enumerate(committed_flags[:-1]):
+ if flag != "--backend":
+ continue
+ for part in str(committed_flags[index + 1]).split(","):
+ _key, sep, value = part.partition("=")
+ for name in (value if sep else part).split("&"):
+ name = name.strip()
+ if name and name.lower() != "cpu":
+ out.add(name.lower())
+ return out
+
+
def _resolved_server_physical_gpu_id(
binary: Optional[str], device: str, ordinal: Optional[int], committed_flags: tuple[str, ...]
) -> Optional[int]:
"""Physical id for a provably single-device resident sd-server."""
if device != "cuda" or "--offload-to-cpu" in committed_flags:
return None
+ if len(_backend_devices(committed_flags)) > 1:
+ return None
try:
from utils.hardware import get_parent_visible_gpu_ids
visible = [int(i) for i in get_parent_visible_gpu_ids()]
@@ -3518,6 +3593,16 @@ class SdCppDiffusionBackend:
if auto_ordinal:
# Automatic ranks the torch cards too, after validation, unless sd.cpp runs on one torch cannot see.
gpu_ordinal = auto_cuda_ordinal_for_load()
+ gpu_ordinals: Any = None
+ if (
+ off_torch is None
+ and multigpu_all_enabled()
+ and resolve_diffusion_device_target().device == "cuda"
+ ):
+ if gpu_ids and len(set(gpu_ids)) > 1:
+ gpu_ordinals = resolve_selected_cuda_ordinals(gpu_ids)
+ elif not gpu_ids:
+ gpu_ordinals = AUTOMATIC_ORDINALS
self._loading_family = fam
base = resolve_base_repo(fam, base_repo)
@@ -3585,12 +3670,14 @@ class SdCppDiffusionBackend:
off_torch = off_torch,
_load_token = token,
_cancel_event = cancel_event,
+ **({"_gpu_ordinals": gpu_ordinals} if gpu_ordinals else {}),
),
daemon = True,
).start()
return self.status()
@_invalidates_gpu_memory("sd.cpp load")
+ @with_load_selection
def _run_load(
self,
*,
@@ -3793,6 +3880,32 @@ class SdCppDiffusionBackend:
def _ordinal() -> Optional[int]:
return selected_ordinal if device == "cuda" else None
+ split_memo: dict[tuple, Any] = {}
+
+ def _split() -> Any:
+ key = (device, _ordinal(), tuple(offload), _running()[1])
+ if key not in split_memo:
+ split_memo[key] = (
+ split_for_load(
+ files,
+ primary = _ordinal(),
+ offload = offload,
+ layer_ok = _layer_split_supported(_running()[1]),
+ logger = logger,
+ )
+ if device == "cuda" and off_torch is None
+ else None
+ )
+ return split_memo[key]
+
+ def _split_arg() -> tuple:
+ split = _split()
+ return () if split is None else (split,)
+
+ def _pin_ordinal() -> Optional[int]:
+ split = _split()
+ return int(split.primary) if split is not None else _ordinal()
+
native_speed = _native_speed_for(speed_mode)
# Teardown and commit run under _generate_lock, taken only now: abort and wait for a generation begun during the download, so no stale image persists.
@@ -3877,12 +3990,12 @@ class SdCppDiffusionBackend:
started_ok = False
try:
server_offload = _offload_with_device_pin_impl(
- offload, server_binary, _ordinal()
+ offload, server_binary, _pin_ordinal(), *_split_arg()
)
spawn_env = _sd_cpp_spawn_env(
off_torch,
device,
- _ordinal(),
+ _pin_ordinal(),
offload,
server_offload,
_diffusion_weight_types,
@@ -3959,14 +4072,15 @@ class SdCppDiffusionBackend:
_offload_with_device_pin_impl(
offload,
server_binary if mode == "server" else getattr(engine, "binary", None),
- _ordinal(),
+ _pin_ordinal(),
+ *_split_arg(),
)
)
# Recomputed for what is committed: a deferred install or the one-shot fallback can move placement.
spawn_env = _sd_cpp_spawn_env(
off_torch,
device,
- _ordinal(),
+ _pin_ordinal(),
offload,
committed_offload_flags,
_diffusion_weight_types,
@@ -4011,6 +4125,18 @@ class SdCppDiffusionBackend:
else None
),
selected_card = self._loading_card,
+ multi_gpu = (
+ _split().describe()
+ if _split() is not None
+ and len(_backend_devices(committed_offload_flags)) > 1
+ else None
+ ),
+ split_physical_gpu_id = (
+ _split_physical_gpu_id(_split().primary)
+ if _split() is not None
+ and len(_backend_devices(committed_offload_flags)) > 1
+ else None
+ ),
off_torch_device = off_torch.label if off_torch is not None else None,
child_env = tuple(sorted((spawn_env or {}).items())),
gpu_explicit = gpu_explicit,
@@ -5003,6 +5129,8 @@ class SdCppDiffusionBackend:
without_device_backend_flags(state.offload_flags)
),
physical_gpu_id = None,
+ multi_gpu = None,
+ split_physical_gpu_id = None,
)
self._hand_over_replacement(state, server, old_server)
# The old server's stop removed its scratch dir, and the staged adapters with it.
@@ -5592,6 +5720,12 @@ class SdCppDiffusionBackend:
"workflows": workflows,
"conditioning": conditioning,
**_native_hires_status(state),
+ # not after a restart onto the CPU backend, whose argv has no device pin
+ **(
+ {"multi_gpu": state.multi_gpu}
+ if state.multi_gpu is not None and not self._cpu_backend_forced
+ else {}
+ ),
}
diff --git a/studio/backend/core/inference/sd_cpp_multigpu.py b/studio/backend/core/inference/sd_cpp_multigpu.py
new file mode 100644
index 00000000000..d452066de14
--- /dev/null
+++ b/studio/backend/core/inference/sd_cpp_multigpu.py
@@ -0,0 +1,201 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Two-card sd-cli split via per-module ``--backend``; sized from nvidia-smi so Studio opens no CUDA context."""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+from typing import Any, Callable, Optional, Sequence
+
+# Working memory beside the weights; denoise matches the graph-cut budget's 2 GiB.
+DENOISE_HEADROOM_MIB = 2048
+ENCODER_HEADROOM_MIB = 1024
+VAE_DECODE_HEADROOM_MIB = 1536
+# Held back from free MiB: the child's CUDA context (~550 MiB on a T4) plus sd.cpp auto-fit's 512 MiB.
+CARD_RESERVE_MIB = 1536
+
+
[email protected](frozen = True)
+class SdCppCardSplit:
+ primary: int
+ secondary: int
+ vae_on_secondary: bool
+ reason: str
+ # the diffusion model's blocks across both cards (sd.cpp's own layer split), each card held to its budget
+ layer: bool = False
+ budgets_mib: tuple[int, int] = (0, 0)
+
+ def describe(self) -> dict[str, Any]:
+ diffusion = f"cuda:{self.primary}" + (f"+cuda:{self.secondary}" if self.layer else "")
+ return {
+ "chosen": "layer_split" if self.layer else "companion_split",
+ "primary": f"cuda:{self.primary}",
+ "secondary": f"cuda:{self.secondary}",
+ "placement": {
+ "diffusion_model": diffusion,
+ "text_encoders": f"cuda:{self.secondary}",
+ "vae": f"cuda:{self.secondary if self.vae_on_secondary else self.primary}",
+ },
+ "summary": self.reason,
+ }
+
+
+def plan_sd_cpp_split(
+ *,
+ diffusion_mib: int,
+ text_encoder_mib: int,
+ vae_mib: int,
+ primary: int,
+ primary_limit_mib: Optional[int],
+ secondary_limits_mib: dict[int, Optional[int]],
+ layer_ok: bool = False,
+) -> tuple[Optional[SdCppCardSplit], str]:
+ """``(split, reason)``. Splits only when one card cannot hold every module; with ``layer_ok`` blocks are cut across both."""
+ if primary_limit_mib is None:
+ return None, f"free memory of cuda:{primary} is unknown"
+ if text_encoder_mib <= 0:
+ return None, "no text encoder to move"
+ whole = diffusion_mib + text_encoder_mib + vae_mib + DENOISE_HEADROOM_MIB
+ if whole <= primary_limit_mib:
+ return (
+ None,
+ f"not needed: every module fits cuda:{primary} ({whole} of {primary_limit_mib} MiB)",
+ )
+ limits = {
+ int(o): int(v)
+ for o, v in secondary_limits_mib.items()
+ if v is not None and int(o) != primary
+ }
+ if not limits:
+ return None, "no second card with readable free memory"
+ best = max(limits, key = lambda o: (limits[o], -o))
+ # each card's room for diffusion blocks; the secondary's is counted after the encoders, the VAE and their working memory
+ primary_room = primary_limit_mib - DENOISE_HEADROOM_MIB
+ secondary_room = (
+ limits[best]
+ - text_encoder_mib
+ - vae_mib
+ - DENOISE_HEADROOM_MIB
+ - max(ENCODER_HEADROOM_MIB, VAE_DECODE_HEADROOM_MIB)
+ )
+ if (
+ layer_ok
+ and diffusion_mib + DENOISE_HEADROOM_MIB > primary_limit_mib
+ and primary_room > 0
+ and secondary_room > 0
+ and diffusion_mib <= primary_room + secondary_room
+ ):
+ # no card holds the diffusion model resident: sd.cpp cuts its blocks in proportion to each card's budget
+ return (
+ SdCppCardSplit(
+ primary = int(primary),
+ secondary = best,
+ vae_on_secondary = vae_mib > 0,
+ layer = True,
+ budgets_mib = (int(primary_limit_mib), int(limits[best])),
+ reason = (
+ f"multi-GPU: diffusion model ({diffusion_mib} MiB) split across cuda:{primary} ({primary_room} MiB "
+ f"room) and cuda:{best} ({secondary_room} MiB beside the text encoders); one card needs {whole} MiB"
+ ),
+ ),
+ "",
+ )
+ with_vae = text_encoder_mib + vae_mib + max(ENCODER_HEADROOM_MIB, VAE_DECODE_HEADROOM_MIB)
+ encoders = text_encoder_mib + ENCODER_HEADROOM_MIB
+ options = ((with_vae, True), (encoders, False)) if vae_mib > 0 else ((encoders, False),)
+ for need, vae_moves in options:
+ if need > limits[best]:
+ continue
+ on_primary = diffusion_mib + (0 if vae_moves else vae_mib) + DENOISE_HEADROOM_MIB
+ moved = "text encoders + VAE" if vae_moves else "text encoders"
+ return (
+ SdCppCardSplit(
+ primary = int(primary),
+ secondary = best,
+ vae_on_secondary = vae_moves,
+ reason = (
+ f"multi-GPU: diffusion model on cuda:{primary} ({on_primary} of {primary_limit_mib} MiB), "
+ f"{moved} on cuda:{best} ({need} of {limits[best]} MiB); one card needs {whole} MiB"
+ ),
+ ),
+ "",
+ )
+ return None, f"text encoders need {encoders} MiB, cuda:{best} has {limits[best]} MiB"
+
+
+def file_mib(path: Optional[str]) -> int:
+ """MiB of a model file; a ``*.index.json`` counts its shards. 0 when unreadable."""
+ if not path:
+ return 0
+ try:
+ if str(path).endswith(".index.json"):
+ with open(path, encoding = "utf-8") as f:
+ shards = set((json.load(f).get("weight_map") or {}).values())
+ base = os.path.dirname(str(path))
+ return sum(os.path.getsize(os.path.join(base, s)) for s in shards) >> 20
+ return os.path.getsize(str(path)) >> 20
+ except (OSError, ValueError, TypeError, AttributeError):
+ return 0
+
+
+TEXT_ENCODER_FIELDS: tuple[str, ...] = ("clip_l", "clip_g", "t5xxl", "llm", "llm_vision", "qwen2vl")
+
+
+def card_limit_mib(ordinal: int) -> Optional[int]:
+ """Free MiB minus max(CARD_RESERVE_MIB, VRAM Budget reserve), via nvidia-smi (no CUDA context here). None when unreadable."""
+ from .diffusion_device import smi_free_total_mib
+ from .diffusion_memory import _user_budget_reserve_mib
+
+ free, total = smi_free_total_mib(int(ordinal))
+ if free is None:
+ return None
+ reserve = max(CARD_RESERVE_MIB, int(_user_budget_reserve_mib(total or free) or 0))
+ return max(0, int(free) - reserve)
+
+
+def split_for_load(
+ files: Any,
+ *,
+ primary: Optional[int],
+ offload: Sequence[str],
+ layer_ok: bool = False,
+ limit_of: Optional[Callable[[int], Optional[int]]] = None,
+ logger: Any = None,
+) -> Optional[SdCppCardSplit]:
+ """Split for this load, else None. Only without offload flags; balanced / low_vram / cpu_offload stay one card."""
+ from .diffusion_multigpu import current_selection, multigpu_all_enabled
+
+ selection = current_selection()
+ if selection is None or not multigpu_all_enabled() or not selection.secondaries:
+ return None
+ if primary is None and selection.automatic:
+ primary = selection.primary
+ if primary is None or int(primary) != int(selection.primary):
+ return None
+ limit_of = limit_of or card_limit_mib
+ if list(offload):
+ if logger is not None:
+ logger.info("sd_cpp.multigpu: one card (explicit offload %s)", " ".join(offload))
+ return None
+ from .diffusion_memory import vram_budget_scope
+
+ limits = vram_budget_scope("image", load = True)(
+ lambda: {int(o): limit_of(int(o)) for o in (int(primary), *selection.secondaries)}
+ )()
+ split, why = plan_sd_cpp_split(
+ diffusion_mib = file_mib(getattr(files, "diffusion_model", None)),
+ text_encoder_mib = sum(file_mib(getattr(files, f, None)) for f in TEXT_ENCODER_FIELDS),
+ vae_mib = file_mib(getattr(files, "vae", None)),
+ primary = int(primary),
+ primary_limit_mib = limits.get(int(primary)),
+ secondary_limits_mib = {int(o): limits.get(int(o)) for o in selection.secondaries},
+ layer_ok = layer_ok,
+ )
+ if logger is not None:
+ logger.info(
+ "sd_cpp.multigpu: %s", split.reason if split is not None else f"one card ({why})"
+ )
+ return split
diff --git a/studio/backend/tests/test_sd_cpp_multigpu.py b/studio/backend/tests/test_sd_cpp_multigpu.py
new file mode 100644
index 00000000000..6d5002c35f3
--- /dev/null
+++ b/studio/backend/tests/test_sd_cpp_multigpu.py
@@ -0,0 +1,414 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The native (sd.cpp) engine on two CUDA cards, CPU only: begin_load carries the multi-card selection, the split
+decision, the two-card ``--backend`` assignment the server gets, status, and every way it stays one card (kill
+switch, one card, explicit offload, a load that fits one card)."""
+
+from __future__ import annotations
+
+import os
+import sys
+import time
+import types
+from pathlib import Path
+
+import pytest
+
+from core.inference import sd_cpp_backend as bk
+from core.inference.diffusion_families import detect_family
+from core.inference.sd_cpp_backend import SdCppDiffusionBackend
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+from test_sd_cpp_backend import _FakeServer # noqa: E402
+
+GIB = 1024
+
+
+def _m():
+ from core.inference import sd_cpp_multigpu
+ return sd_cpp_multigpu
+
+
+# An idle Kaggle T4: 14,900 MiB free, less the 1.5 GiB reserve.
+T4_LIMIT = 13_364
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for name in ("UNSLOTH_DIFFUSION_MULTIGPU", "UNSLOTH_DIFFUSION_MULTIGPU_ALL"):
+ monkeypatch.delenv(name, raising = False)
+
+
+def _plan(**kw):
+ args = dict(
+ diffusion_mib = 12_247,
+ text_encoder_mib = 4_465,
+ vae_mib = 242,
+ primary = 0,
+ primary_limit_mib = T4_LIMIT,
+ secondary_limits_mib = {1: T4_LIMIT},
+ )
+ args.update(kw)
+ return _m().plan_sd_cpp_split(**args)
+
+
+def test_a_load_that_does_not_fit_one_card_moves_the_encoders_to_the_second():
+ # Qwen-Image Q4_K_M on 2x T4: 12.0 GiB DiT + 4.4 GiB Qwen2.5-VL + VAE does not fit one card resident.
+ split, why = _plan()
+ assert split is not None and why == ""
+ assert (split.primary, split.secondary, split.vae_on_secondary) == (0, 1, True)
+ assert split.describe()["placement"] == {
+ "diffusion_model": "cuda:0",
+ "text_encoders": "cuda:1",
+ "vae": "cuda:1",
+ }
+
+
+def test_the_vae_stays_with_the_diffusion_model_when_the_second_card_has_no_decode_room():
+ split, _ = _plan(secondary_limits_mib = {1: 5_600})
+ assert split is not None and split.vae_on_secondary is False
+
+
+def test_one_card_whenever_one_card_holds_everything_or_nothing_fits():
+ assert _plan(primary_limit_mib = 40_000)[0] is None
+ assert "not needed" in _plan(primary_limit_mib = 40_000)[1]
+ assert _plan(secondary_limits_mib = {1: 2_000})[0] is None
+ assert _plan(secondary_limits_mib = {1: None})[0] is None
+ assert _plan(primary_limit_mib = None)[0] is None
+ assert _plan(text_encoder_mib = 0)[0] is None
+ split, _ = _plan(vae_mib = 0)
+ assert split is not None and split.vae_on_secondary is False
+
+
+def test_the_roomiest_secondary_is_chosen():
+ split, _ = _plan(secondary_limits_mib = {1: 7_000, 2: 9_000, 3: 9_000})
+ assert split.secondary == 2
+
+
+def test_file_sizes_count_index_shards(tmp_path):
+ (tmp_path / "a.safetensors").write_bytes(b"\0" * (3 << 20))
+ (tmp_path / "b.safetensors").write_bytes(b"\0" * (2 << 20))
+ index = tmp_path / "model.safetensors.index.json"
+ index.write_text(
+ '{"weight_map": {"x": "a.safetensors", "y": "b.safetensors", "z": "a.safetensors"}}'
+ )
+ assert _m().file_mib(str(index)) == 5
+ assert _m().file_mib(str(tmp_path / "a.safetensors")) == 3
+ assert _m().file_mib(str(tmp_path / "missing.gguf")) == 0
+ assert _m().file_mib(None) == 0
+
+
+def _names(monkeypatch):
+ monkeypatch.setattr(
+ bk, "_sd_cpp_probe_output", lambda *_a: "CUDA0\tTesla T4\nCUDA1\tTesla T4\nCPU\tXeon\n"
+ )
+ monkeypatch.setattr(bk, "physical_card_name", lambda ordinal: (None, None))
+
+
+def test_the_pin_names_a_card_per_module(monkeypatch):
+ _names(monkeypatch)
+ split = _m().SdCppCardSplit(primary = 0, secondary = 1, vae_on_secondary = False, reason = "r")
+ assert bk._offload_with_device_pin_impl([], "/sd/sd-cli", 0, split) == [
+ "--backend",
+ "diffusion=CUDA0,te=CUDA1,vae=CUDA0",
+ ]
+ both = _m().SdCppCardSplit(primary = 0, secondary = 1, vae_on_secondary = True, reason = "r")
+ assert bk._offload_with_device_pin_impl([], "/sd/sd-cli", 0, both) == [
+ "--backend",
+ "diffusion=CUDA0,te=CUDA1,vae=CUDA1",
+ ]
+ # without a split (and for a split of another primary) the one-card pin is byte-identical to before
+ for extra in (
+ (),
+ (None,),
+ (_m().SdCppCardSplit(primary = 1, secondary = 0, vae_on_secondary = False, reason = "r"),),
+ ):
+ assert bk._offload_with_device_pin_impl([], "/sd/sd-cli", 0, *extra) == [
+ "--backend",
+ "diffusion=CUDA0,te=CUDA0,vae=CUDA0",
+ ]
+
+
+def test_a_two_card_server_is_not_charged_to_one_card(monkeypatch):
+ _names(monkeypatch)
+ monkeypatch.setattr("utils.hardware.get_parent_visible_gpu_ids", lambda: [0, 1])
+ one = ("--backend", "diffusion=CUDA0,te=CUDA0,vae=CUDA0")
+ two = ("--backend", "diffusion=CUDA0,te=CUDA1,vae=CUDA0")
+ assert bk._resolved_server_physical_gpu_id("/sd/sd-server", "cuda", 0, one) == 0
+ assert bk._resolved_server_physical_gpu_id("/sd/sd-server", "cuda", 0, two) is None
+ assert bk._backend_devices(two) == {"cuda0", "cuda1"}
+ assert bk._backend_devices(("--backend", "diffusion=CUDA0&CUDA1,te=cpu,vae=CUDA0")) == {
+ "cuda0",
+ "cuda1",
+ }
+
+
+def _begin(monkeypatch, gpu_ids):
+ seen: dict = {}
+ monkeypatch.setattr(
+ "core.inference.diffusion_engine_router.off_torch_sd_cpp_device", lambda *a, **k: None
+ )
+ monkeypatch.setattr(
+ bk, "resolve_diffusion_device_target", lambda **kw: types.SimpleNamespace(device = "cuda")
+ )
+ monkeypatch.setattr(bk, "resolve_selected_cuda_ordinal", lambda ids: min(ids))
+ monkeypatch.setattr(bk, "resolve_selected_cuda_ordinals", lambda ids: sorted(set(ids)))
+ monkeypatch.setattr(bk, "auto_cuda_ordinal_for_load", lambda: 0)
+ b = SdCppDiffusionBackend()
+ monkeypatch.setattr(b, "_run_load", lambda **kw: seen.update(kw, done = True))
+ b.begin_load(
+ repo_id = "unsloth/Qwen-Image-GGUF", gguf_filename = "qwen-image-Q4_K_M.gguf", gpu_ids = gpu_ids
+ )
+ deadline = time.monotonic() + 5
+ while "done" not in seen and time.monotonic() < deadline:
+ time.sleep(0.01)
+ return seen
+
+
+def test_begin_load_carries_the_multi_card_selection(monkeypatch):
+ assert _begin(monkeypatch, [0, 1])["_gpu_ordinals"] == [0, 1]
+ assert _begin(monkeypatch, None)["_gpu_ordinals"] == "automatic"
+
+
+def test_one_card_and_the_kill_switch_carry_no_selection(monkeypatch):
+ assert "_gpu_ordinals" not in _begin(monkeypatch, [1])
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MULTIGPU_ALL", "0")
+ assert "_gpu_ordinals" not in _begin(monkeypatch, [0, 1])
+ assert "_gpu_ordinals" not in _begin(monkeypatch, None)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MULTIGPU_ALL", "1")
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MULTIGPU", "0")
+ assert "_gpu_ordinals" not in _begin(monkeypatch, [0, 1])
+
+
+def _sparse(path: Path, mib: int) -> str:
+ with open(path, "wb") as f:
+ f.truncate(mib << 20)
+ return str(path)
+
+
+def _server_load(
+ monkeypatch,
+ tmp_path,
+ *,
+ gpu_ordinals,
+ memory_mode = None,
+ limits = None,
+ gpu_ordinal = 0,
+):
+ _names(monkeypatch)
+ cards = limits or {0: (14_900, 15_360), 1: (14_900, 15_360)}
+ monkeypatch.setattr(
+ "core.inference.diffusion_device.smi_free_total_mib",
+ lambda o: cards.get(int(o), (None, None)),
+ )
+ fam = detect_family("qwen-image")
+ servers: list = []
+ monkeypatch.setattr(bk, "find_sd_server_binary", lambda: "/x/sd-server")
+ monkeypatch.setattr(bk, "_server_binary_runnable", lambda *_a, **_k: True)
+ monkeypatch.setattr(
+ bk, "SdCppServer", lambda binary: (servers.append(_FakeServer(binary)), servers[-1])[1]
+ )
+ paths = {
+ "diffusion_model": _sparse(tmp_path / "qwen-image-Q4_K_M.gguf", 12 * GIB),
+ "vae": _sparse(tmp_path / "vae.safetensors", 250),
+ "qwen2vl": _sparse(tmp_path / "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", 4 * GIB + 400),
+ }
+ b = SdCppDiffusionBackend()
+ monkeypatch.setattr(b, "_asset_specs", lambda *a, **k: [])
+ monkeypatch.setattr(b, "_set_expected_bytes", lambda *a, **k: None)
+ monkeypatch.setattr(b, "_fetch_assets", lambda *a, **k: dict(paths))
+ monkeypatch.setattr(
+ bk, "resolve_diffusion_device_target", lambda: types.SimpleNamespace(device = "cuda")
+ )
+ b._load_token = 1
+ kwargs = dict(
+ repo_id = "unsloth/Qwen-Image-GGUF",
+ gguf_filename = "qwen-image-Q4_K_M.gguf",
+ base = fam.base_repo,
+ fam = fam,
+ hf_token = None,
+ _load_token = 1,
+ gpu_ordinal = gpu_ordinal,
+ memory_mode = memory_mode,
+ )
+ if gpu_ordinals is not None:
+ kwargs["_gpu_ordinals"] = gpu_ordinals
+ b._run_load(**kwargs)
+ return b, servers
+
+
+def _backend_arg(flags):
+ flags = list(flags or ())
+ return flags[flags.index("--backend") + 1] if "--backend" in flags else None
+
+
+def test_a_two_card_load_starts_the_server_with_the_encoders_on_the_second_card(
+ monkeypatch, tmp_path
+):
+ b, servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = [0, 1])
+ assert _backend_arg(servers[-1].started.get("offload")) == "diffusion=CUDA0,te=CUDA1,vae=CUDA1"
+ assert _backend_arg(b._state.offload_flags) == "diffusion=CUDA0,te=CUDA1,vae=CUDA1"
+ assert b._state.physical_gpu_id is None
+ status = b.status()
+ assert status["multi_gpu"]["placement"]["text_encoders"] == "cuda:1"
+ assert status["offload_policy"] == "none"
+
+
+ "case",
+ ["one_card", "kill_switch", "explicit_offload", "fits_one_card"],
+)
+def test_every_one_card_case_keeps_main_argv(monkeypatch, tmp_path, case):
+ ordinals = [0, 1]
+ memory_mode, limits = None, None
+ if case == "one_card":
+ ordinals = None
+ elif case == "kill_switch":
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MULTIGPU_ALL", "0")
+ elif case == "explicit_offload":
+ memory_mode = "balanced"
+ elif case == "fits_one_card":
+ limits = {0: (48_000, 49_152), 1: (48_000, 49_152)}
+ b, servers = _server_load(
+ monkeypatch, tmp_path, gpu_ordinals = ordinals, memory_mode = memory_mode, limits = limits
+ )
+ assert _backend_arg(servers[-1].started.get("offload")) == "diffusion=CUDA0,te=CUDA0,vae=CUDA0"
+ assert "multi_gpu" not in b.status()
+ if case != "explicit_offload":
+ ref, ref_servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = None)
+ assert servers[-1].started.get("offload") == ref_servers[-1].started.get("offload")
+ assert b._state.offload_flags == ref._state.offload_flags
+
+
+def test_keep_multiple_models_loaded_keeps_one_card(monkeypatch, tmp_path):
+ monkeypatch.setattr("core.inference.diffusion_multigpu.models_kept_loaded", lambda: True)
+ b, servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = [0, 1])
+ assert _backend_arg(servers[-1].started.get("offload")) == "diffusion=CUDA0,te=CUDA0,vae=CUDA0"
+ assert os.environ.get("UNSLOTH_DIFFUSION_MULTIGPU_ALL") is None
+
+
+def test_automatic_with_an_unpinned_card_still_splits_and_pins_the_primary(monkeypatch, tmp_path):
+ # identical idle cards: Automatic's ranker leaves the load unpinned, the selection's primary is cuda:0
+ monkeypatch.setattr("core.inference.diffusion_device.auto_cuda_ranking_possible", lambda: True)
+ monkeypatch.setattr(
+ "core.inference.diffusion_device.resolve_diffusion_device_target",
+ lambda **kw: types.SimpleNamespace(device = "cuda"),
+ )
+ monkeypatch.setattr("torch.cuda.current_device", lambda: 0)
+ monkeypatch.setattr("torch.cuda.device_count", lambda: 2)
+ b, servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = "automatic", gpu_ordinal = None)
+ assert _backend_arg(servers[-1].started.get("offload")) == "diffusion=CUDA0,te=CUDA1,vae=CUDA1"
+ assert b.status()["multi_gpu"]["primary"] == "cuda:0"
+
+
+def test_helpers_keep_to_the_split_diffusion_card_and_status_follows_a_cpu_restart(
+ monkeypatch, tmp_path
+):
+ monkeypatch.setattr("utils.hardware.get_parent_visible_gpu_ids", lambda: [2, 3])
+ b, _servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = [0, 1])
+ assert b._state.physical_gpu_id is None
+ assert b._state.placed_ordinal == 0
+ assert "multi_gpu" in b.status()
+ b._cpu_backend_forced = True
+ assert "multi_gpu" not in b.status()
+
+
+def test_a_diffusion_model_no_card_holds_is_cut_across_both(monkeypatch):
+ split, _ = _plan(layer_ok = True)
+ assert split.layer is True and split.vae_on_secondary is True
+ assert split.budgets_mib == (T4_LIMIT, T4_LIMIT)
+ assert split.describe()["placement"]["diffusion_model"] == "cuda:0+cuda:1"
+ # a model that fits the primary keeps the companion split; without a capable build there is no block split
+ assert _plan(diffusion_mib = 9_000, layer_ok = True)[0].layer is False
+ assert _plan(layer_ok = False)[0].layer is False
+ assert _plan(diffusion_mib = 22_000, layer_ok = True)[0].layer is False
+ # the secondary's room is what is left beside the encoders and the VAE: no blocks fit there, no block split
+ crowded, _ = _plan(
+ primary_limit_mib = 20_000,
+ secondary_limits_mib = {1: 14_000},
+ diffusion_mib = 23_000,
+ text_encoder_mib = 10_000,
+ vae_mib = 250,
+ layer_ok = True,
+ )
+ assert crowded is None or crowded.layer is False
+
+
+def test_the_block_split_pin_lists_both_cards_and_their_budgets(monkeypatch):
+ monkeypatch.setattr(
+ bk,
+ "_sd_cpp_probe_output",
+ lambda _b, *args: (
+ " --max-vram budget\n --auto-fit on|off\n"
+ if args and args[0] == "--help"
+ else "CUDA0\tTesla T4\nCUDA1\tTesla T4\n"
+ ),
+ )
+ monkeypatch.setattr(bk, "physical_card_name", lambda ordinal: (None, None))
+ split = _m().SdCppCardSplit(
+ primary = 0,
+ secondary = 1,
+ vae_on_secondary = True,
+ reason = "r",
+ layer = True,
+ budgets_mib = (13_876, 13_876),
+ )
+ flags = bk._offload_with_device_pin_impl([], "/sd/sd-server", 0, split)
+ assert flags == [
+ "--backend",
+ "diffusion=CUDA0&CUDA1,te=CUDA1,vae=CUDA1",
+ "--auto-fit",
+ "off",
+ "--max-vram",
+ "CUDA0=13.55,CUDA1=13.55",
+ ]
+ from core.inference.sd_cpp_args import without_device_backend_flags
+
+ # the CPU restart and status drop the whole pin; a one-card pin keeps any other --max-vram
+ assert without_device_backend_flags(["--diffusion-fa", *flags]) == ["--diffusion-fa"]
+ assert without_device_backend_flags(
+ ["--max-vram", "-1", "--backend", "diffusion=CUDA0,te=CUDA0,vae=CUDA0"]
+ ) == ["--max-vram", "-1"]
+ monkeypatch.setattr(
+ bk, "_sd_cpp_probe_output", lambda _b, *args: "CUDA0\tTesla T4\nCUDA1\tTesla T4\n"
+ )
+ assert bk._offload_with_device_pin_impl([], "/sd/sd-server", 0, split) == [
+ "--backend",
+ "diffusion=CUDA0,te=CUDA1,vae=CUDA1",
+ ]
+
+
+def test_a_two_card_load_cuts_the_model_across_both_cards_on_a_capable_build(monkeypatch, tmp_path):
+ monkeypatch.setattr(bk, "_layer_split_supported", lambda binary: True)
+ monkeypatch.setattr("utils.hardware.get_parent_visible_gpu_ids", lambda: [0, 1])
+ b, servers = _server_load(monkeypatch, tmp_path, gpu_ordinals = [0, 1])
+ started = list(servers[-1].started.get("offload"))
+ assert _backend_arg(started) == "diffusion=CUDA0&CUDA1,te=CUDA1,vae=CUDA1"
+ assert started[started.index("--auto-fit") + 1] == "off"
+ assert "--max-vram" in started
+ status = b.status()
+ assert status["multi_gpu"]["chosen"] == "layer_split"
+ assert status["offload_policy"] == "none" and status["cpu_offload"] is False
+ assert b._state.physical_gpu_id is None and b._state.placed_ordinal == 0
+
+
+def test_a_model_sd_cpp_would_stream_on_one_card_is_cut_across_both(monkeypatch, tmp_path):
+ # 2x T4 under Automatic: nvidia-smi shows 15,360 MiB free, the sd-server's CUDA context leaves 14,806 and auto-fit
+ # budgets 14,294, 1 MiB short of the 12,247 MiB model plus its 2 GiB reserve: priced as one card, it streams.
+ monkeypatch.setattr(bk, "_layer_split_supported", lambda binary: True)
+ monkeypatch.setattr("utils.hardware.get_parent_visible_gpu_ids", lambda: [0, 1])
+ b, servers = _server_load(
+ monkeypatch,
+ tmp_path,
+ gpu_ordinals = [0, 1],
+ limits = {0: (15_360, 15_360), 1: (15_360, 15_360)},
+ )
+ assert (
+ _backend_arg(servers[-1].started.get("offload"))
+ == "diffusion=CUDA0&CUDA1,te=CUDA1,vae=CUDA1"
+ )
+ assert b.status()["multi_gpu"]["chosen"] == "layer_split"