Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fit an oversized DiffusionGemma GGUF to the GPU in Auto mode (#13312)

Daniel Hanfcee6485fe44
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 8b61e5f74c1..f998caaab3d 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -4913,6 +4913,46 @@ def _scale_diffusion_required_gb(required_gb: float, ngl: int, n_layers: Optiona
return required_gb * (ngl / n_layers)
+# The runner's own CUDA context, which the free reading taken before spawn cannot see.
+_DIFFUSION_RUNNER_OVERHEAD_MIB = 512.0
+# Context the fitted split leaves room for when the runner auto-sizes (maxtok 0).
+_DIFFUSION_FIT_CTX = 4096
+
+
+def _diffusion_auto_ngl(
+ file_bytes: int,
+ n_layers: Optional[int],
+ n_head: Optional[int],
+ maxtok: int,
+ free_mib: float,
+ total_mib: float,
+ *,
+ sysmem_fallback: bool = False,
+) -> Optional[int]:
+ """GPU layers for an Auto-mode diffusion load, or None to keep every layer on the GPU.
+
+ Only a GGUF that cannot fit moves: all layers die in cudaMalloc, or on Windows spill into
+ shared system memory until RAM and the page file run out (#6456). The split leaves room
+ for the compute buffer (measured 1.75x the [n_head, N, N] fp32 scores) and the 8% of VRAM
+ the runner's context probe keeps free (diffusion-gemma-visual-server.cpp)."""
+ if not file_bytes or not n_layers or n_layers <= 0 or (free_mib <= 0 and total_mib <= 0):
+ return None
+ usable = _vram_usable_mib(
+ free_mib, total_mib, _active_vram_fraction(), sysmem_fallback = sysmem_fallback
+ )
+ usable -= _DIFFUSION_RUNNER_OVERHEAD_MIB
+ file_mib = file_bytes / (1024 * 1024)
+ if file_mib <= usable:
+ return None
+ # Mirrors unsloth_zoo's _canvas_maxtok: larger requests are sent as 0 (auto-size).
+ n_ctx = maxtok if 0 < maxtok <= 8192 else _DIFFUSION_FIT_CTX
+ scores_mib = max(1, n_head or 1) * n_ctx * n_ctx * 4 / (1024 * 1024)
+ ctx_reserve = 1.75 * scores_mib + 0.08 * (total_mib if total_mib > 0 else free_mib)
+ # The whole file over the block count also charges token_embd, which stays on the CPU.
+ per_layer_mib = file_mib / n_layers
+ return max(0, min(n_layers - 1, int((usable - ctx_reserve) // per_layer_mib)))
+
+
def _source_declares_option(source: str, option: str) -> bool:
"""True iff ``source`` really registers ``option`` on an argparse parser.
@@ -9019,6 +9059,10 @@ class LlamaCppBackend:
never competed with."""
if self._arch_gate_forced_cpu:
return True
+ # A diffusion runner reports offload False only when its GPU token was masked: a
+ # manual zero split, or an Auto fit that found no room on the card.
+ if self._is_diffusion and self._gpu_offload_active is False:
+ return True
return (
self._gpu_memory_mode == "manual"
and self._gpu_layers == 0
@@ -19429,6 +19473,47 @@ class LlamaCppBackend:
return first
return "0"
+ def _diffusion_fit_ngl(
+ self, model_path: str, gpu: str, maxtok: int, *, pinned: bool
+ ) -> Optional[int]:
+ """Auto-mode ``--ngl`` for the runner's device ``gpu``, or None to keep its default."""
+ # An NGL in the environment is the zoo runner's own override: leave it in charge.
+ if not gpu.isdigit() or "NGL" in os.environ:
+ return None
+ try:
+ binary = self._find_llama_server_binary()
+ # Vulkan ordinals are not the CUDA ids the runner's mask takes.
+ if binary and self._is_vulkan_backend(binary):
+ return None
+ rows = self._get_gpu_memory(binary)
+ file_bytes = self._get_gguf_size_bytes(model_path)
+ sysmem_fallback = self._sysmem_fallback_risk(binary)
+ except Exception as e:
+ logger.debug(f"DiffusionGemma layer fit skipped: {e}")
+ return None
+ row = next((r for r in rows if r[0] == int(gpu)), None)
+ # Unpinned, the child's "0" is CUDA's fastest-first device unless PCI order is set,
+ # not nvidia-smi's index 0: size against the roomiest card so no load that fits
+ # today is cut down.
+ if not pinned and len(rows) > 1 and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID":
+ row = max(
+ rows,
+ key = lambda r: _vram_usable_mib(
+ r[1], r[2], _active_vram_fraction(), sysmem_fallback = sysmem_fallback
+ ),
+ )
+ if row is None:
+ return None
+ return _diffusion_auto_ngl(
+ file_bytes,
+ self._n_layers,
+ self._n_heads,
+ maxtok,
+ row[1],
+ row[2],
+ sysmem_fallback = sysmem_fallback,
+ )
+
def _start_diffusion_server(
self,
*,
@@ -19449,7 +19534,8 @@ class LlamaCppBackend:
interface as llama-server, so the rest of Unsloth is unchanged.
Manual mode forwards gpu_layers as --ngl so a GGUF larger than VRAM can be
- split across GPU and RAM; auto leaves the runner's all-layers default."""
+ split across GPU and RAM; auto keeps every layer on the GPU unless the GGUF
+ cannot fit it."""
assets = self._find_diffusion_assets()
if assets is None:
raise RuntimeError(
@@ -19498,6 +19584,32 @@ class LlamaCppBackend:
# the device used == the echoed gpu_ids[0]). With no pick, fall back to the
# CPU-only mask, else DG_GPU / 0.
gpu = self._diffusion_gpu_arg(gpu_ids, cpu_only = cpu_only, force_cpu = holds_no_gpu)
+ # Auto: a GGUF too large for that device gets the split that fits it (#6456).
+ fit_ngl = None
+ if manual_ngl is None:
+ fit_ngl = self._diffusion_fit_ngl(model_path, gpu, maxtok, pinned = bool(gpu_ids))
+ # The probe runs under the load lock and can be slow: honour a Stop that came in.
+ if cancelled is not None and cancelled():
+ logger.info("Load cancelled before diffusion server start")
+ self._health_wait_cancelled = True
+ return False
+ if fit_ngl is not None and not _shim_supports_ngl(shim_cmd):
+ logger.warning(
+ "DiffusionGemma does not fit GPU %s and the installed unsloth_zoo shim "
+ "has no --ngl to split it. Run 'unsloth studio update'.",
+ gpu,
+ )
+ fit_ngl = None
+ # Pin the context the split was sized for. Left to auto-size, the runner's second
+ # probe pass sizes against host RAM, which on Windows the driver backs with
+ # shared memory instead of refusing.
+ if fit_ngl is not None and not 0 < maxtok <= 8192:
+ maxtok = _DIFFUSION_FIT_CTX
+ # Nothing fits: run on the CPU like a manual zero-layer split, not on a
+ # GPU context the card may not have room for.
+ if fit_ngl == 0:
+ holds_no_gpu = True
+ gpu = self._diffusion_gpu_arg(gpu_ids, cpu_only = cpu_only, force_cpu = True)
cmd = list(shim_cmd) + [
"--gguf",
@@ -19522,6 +19634,16 @@ class LlamaCppBackend:
"quant over a large CPU split if throughput matters.",
manual_ngl,
)
+ elif fit_ngl is not None:
+ cmd += ["--ngl", str(fit_ngl)]
+ logger.warning(
+ "DiffusionGemma does not fit GPU %s: keeping %d of %s layers on it and the "
+ "rest on CPU, which is much slower for block diffusion. A smaller quant "
+ "that fits the GPU is faster.",
+ gpu,
+ fit_ngl,
+ self._n_layers,
+ )
env = child_env_without_native_path_secret()
# `python -m unsloth_zoo.diffusion_studio.shim` imports unsloth_zoo, which
@@ -25171,6 +25293,9 @@ class LlamaCppBackend:
# diffusion runner ignores the budget, so a previous llama-server's
# fraction would relaunch a healthy diffusion server on every Apply.
self._vram_fraction_launched = None
+ # Outside the lock, as for llama-server: the Auto layer fit reads free VRAM,
+ # which a just-killed runner still holds until the driver reclaims it.
+ self._wait_for_vram_settle(since_kill = self._last_kill_monotonic)
with self._lock:
if _load_cancelled():
logger.info("Load cancelled before diffusion server start")
diff --git a/studio/backend/tests/test_diffusion_auto_fit_ngl.py b/studio/backend/tests/test_diffusion_auto_fit_ngl.py
new file mode 100644
index 00000000000..6783b65032a
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_auto_fit_ngl.py
@@ -0,0 +1,302 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Auto-mode DiffusionGemma loads split a GGUF the GPU cannot hold (#6456).
+
+The runner's default puts every layer on the GPU, so a GGUF larger than VRAM died in
+cudaMalloc on Linux and spilled into shared system memory on Windows until RAM and the
+page file filled. Studio hides the GPU-layer control for diffusion models, so Auto is
+the only path a UI user has.
+"""
+
+from __future__ import annotations
+
+import sys
+import types as _types
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+_loggers_stub = _types.ModuleType("loggers")
+_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
+sys.modules.setdefault("loggers", _loggers_stub)
+
+_structlog_stub = _types.ModuleType("structlog")
+_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
+sys.modules.setdefault("structlog", _structlog_stub)
+
+import httpx # noqa: F401
+
+from core.inference import llama_cpp as llama_cpp_module
+from core.inference.llama_cpp import LlamaCppBackend
+
+# diffusiongemma-26B-A4B-it Q4_K_M: 16,806,810,208 bytes, 30 blocks, 16 heads.
+_Q4 = 16_806_810_208
+_GIB = 1024**3
+
+
[email protected](autouse = True)
+def _default_budget(monkeypatch):
+ monkeypatch.setattr(llama_cpp_module, "_active_vram_fraction", lambda: 0.97)
+
+
+ ("free", "total", "sysmem", "expected"),
+ [
+ (7_400, 8_188, True, 6), # RTX 4060 on Windows (the report)
+ (7_800, 8_188, False, 8), # RTX 4060 on Linux
+ (14_900, 15_360, False, 20), # T4
+ ],
+)
+def test_small_card_gets_a_partial_split(free, total, sysmem, expected):
+ assert (
+ llama_cpp_module._diffusion_auto_ngl(_Q4, 30, 16, 4096, free, total, sysmem_fallback = sysmem)
+ == expected
+ )
+
+
+ ("free", "total"), [(19_500, 20_475), (23_500, 24_576), (178_000, 183_359)]
+)
+def test_a_gguf_that_fits_keeps_every_layer(free, total):
+ assert llama_cpp_module._diffusion_auto_ngl(_Q4, 30, 16, 4096, free, total) is None
+
+
+def test_larger_context_leaves_fewer_layers_and_auto_size_uses_the_default():
+ small = llama_cpp_module._diffusion_auto_ngl(_Q4, 30, 16, 2048, 14_900, 15_360)
+ default = llama_cpp_module._diffusion_auto_ngl(_Q4, 30, 16, 4096, 14_900, 15_360)
+ big = llama_cpp_module._diffusion_auto_ngl(_Q4, 30, 16, 8192, 14_900, 15_360)
+ assert small > default > big
+ # The zoo sends anything past 8192 as 0 (auto-size): priced at the default context.
+ assert llama_cpp_module._diffusion_auto_ngl(_Q4, 30, 16, 0, 14_900, 15_360) == default
+ assert llama_cpp_module._diffusion_auto_ngl(_Q4, 30, 16, 32768, 14_900, 15_360) == default
+
+
+def test_split_never_goes_negative_and_unknowns_keep_the_default():
+ assert llama_cpp_module._diffusion_auto_ngl(_Q4, 30, 16, 8192, 2_000, 2_048) == 0
+ assert llama_cpp_module._diffusion_auto_ngl(_Q4, None, 16, 4096, 7_800, 8_188) is None
+ assert llama_cpp_module._diffusion_auto_ngl(0, 30, 16, 4096, 7_800, 8_188) is None
+ assert llama_cpp_module._diffusion_auto_ngl(_Q4, 30, 16, 4096, 0, 0) is None
+ # A full card is a reading, not a missing one.
+ assert llama_cpp_module._diffusion_auto_ngl(_Q4, 30, 16, 4096, 0, 8_188) == 0
+
+
+def _fit_host(
+ monkeypatch,
+ gpus,
+ *,
+ vulkan = False,
+):
+ monkeypatch.setattr(
+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "/bin/ls")
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda _b = None: vulkan))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_sysmem_fallback_risk", staticmethod(lambda _b = None: False)
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_get_gpu_memory", staticmethod(lambda _b = None, **_k: list(gpus))
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_get_gguf_size_bytes", staticmethod(lambda _p: _Q4))
+ monkeypatch.delenv("CUDA_DEVICE_ORDER", raising = False)
+ monkeypatch.delenv("NGL", raising = False)
+ backend = LlamaCppBackend()
+ backend._n_layers, backend._n_heads = 30, 16
+ return backend
+
+
+def test_fit_reads_the_runner_device(monkeypatch):
+ backend = _fit_host(monkeypatch, [(0, 23_500, 24_576), (1, 7_800, 8_188)])
+ assert backend._diffusion_fit_ngl("/m.gguf", "1", 4096, pinned = True) == 8
+ assert backend._diffusion_fit_ngl("/m.gguf", "0", 4096, pinned = True) is None
+ # A token that is not a physical index (UUID, CPU mask) is left alone.
+ assert backend._diffusion_fit_ngl("/m.gguf", "GPU-abc", 4096, pinned = False) is None
+ assert backend._diffusion_fit_ngl("/m.gguf", "", 4096, pinned = False) is None
+
+
+def test_fit_leaves_an_ngl_override_alone(monkeypatch):
+ backend = _fit_host(monkeypatch, [(0, 7_800, 8_188)])
+ assert backend._diffusion_fit_ngl("/m.gguf", "0", 4096, pinned = False) == 8
+ monkeypatch.setenv("NGL", "0")
+ assert backend._diffusion_fit_ngl("/m.gguf", "0", 4096, pinned = False) is None
+
+
+def test_diffusion_load_settles_vram_outside_the_lock():
+ # The fit reads free VRAM; a just-killed runner still holds it. Waiting under the
+ # lock would block /unload for the whole settle.
+ import inspect
+
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ branch = src[src.index("if self._is_diffusion:") :]
+ settle = branch.index("self._wait_for_vram_settle(since_kill = self._last_kill_monotonic)")
+ assert settle < branch.index("with self._lock:") < branch.index("_start_diffusion_server(")
+
+
+def test_unpinned_multi_gpu_never_cuts_a_load_that_fits_a_card(monkeypatch):
+ # Without PCI order the child's "0" is CUDA's fastest card, not nvidia-smi's index 0.
+ backend = _fit_host(monkeypatch, [(0, 7_800, 8_188), (1, 23_500, 24_576)])
+ assert backend._diffusion_fit_ngl("/m.gguf", "0", 4096, pinned = False) is None
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ assert backend._diffusion_fit_ngl("/m.gguf", "0", 4096, pinned = False) == 8
+
+
+def test_fit_skips_vulkan_and_a_failed_probe(monkeypatch):
+ backend = _fit_host(monkeypatch, [(0, 7_800, 8_188)], vulkan = True)
+ assert backend._diffusion_fit_ngl("/m.gguf", "0", 4096, pinned = False) is None
+ backend = _fit_host(monkeypatch, [(0, 7_800, 8_188)])
+
+ def _boom(_b = None, **_k):
+ raise OSError("nvidia-smi hung")
+
+ monkeypatch.setattr(LlamaCppBackend, "_get_gpu_memory", staticmethod(_boom))
+ assert backend._diffusion_fit_ngl("/m.gguf", "0", 4096, pinned = False) is None
+
+
+class _StopAtHealth(Exception):
+ pass
+
+
+def _launch(
+ monkeypatch,
+ tmp_path,
+ gpus,
+ *,
+ shim_has_ngl = True,
+ n_ctx = 4096,
+ **kwargs,
+):
+ """Drive the real _start_diffusion_server to the health wait; return (argv, backend)."""
+ backend = _fit_host(monkeypatch, gpus)
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ monkeypatch.delenv("DG_GPU", raising = False)
+ seen = {}
+
+ class _FakePopen:
+ pid = 999_999_999
+ stdout = iter(())
+
+ def __init__(self, cmd, **kw):
+ seen["cmd"] = list(cmd)
+
+ def poll(self):
+ return None
+
+ monkeypatch.setattr(llama_cpp_module, "_shim_supports_ngl", lambda _cmd: shim_has_ngl)
+ monkeypatch.setattr(
+ backend, "_find_diffusion_assets", lambda: (["python", "-m", "shim"], "/bin/vs", None)
+ )
+ monkeypatch.setattr(backend, "_kill_process", lambda: None)
+ monkeypatch.setattr(backend, "_find_free_port", lambda: 18_123)
+ monkeypatch.setattr(
+ LlamaCppBackend, "_effective_gpu_count", staticmethod(lambda *_a: len(gpus))
+ )
+ monkeypatch.setattr(llama_cpp_module, "_swa_cache_path", lambda: tmp_path / "swa.json")
+ monkeypatch.setattr(llama_cpp_module.subprocess, "Popen", _FakePopen)
+ monkeypatch.setattr(
+ backend, "_wait_for_health", lambda **_kw: (_ for _ in ()).throw(_StopAtHealth())
+ )
+ with pytest.raises(_StopAtHealth):
+ backend._start_diffusion_server(
+ model_path = "/m.gguf",
+ gguf_path = "/m.gguf",
+ hf_repo = None,
+ hf_variant = None,
+ model_identifier = "unsloth/diffusiongemma-26B-A4B-it-GGUF",
+ n_ctx = n_ctx,
+ extra_args = None,
+ **kwargs,
+ )
+ return seen["cmd"], backend
+
+
+def _ngl(cmd):
+ return cmd[cmd.index("--ngl") + 1] if "--ngl" in cmd else None
+
+
+def test_auto_launch_on_a_small_gpu_passes_the_fitted_split(monkeypatch, tmp_path):
+ cmd, backend = _launch(monkeypatch, tmp_path, [(0, 7_800, 8_188)])
+ assert _ngl(cmd) == "8"
+ # Still an Auto load: status and the reload dedup see the request, not the fit.
+ assert backend.gpu_memory_mode == "auto"
+ assert backend._gpu_layers == -1
+ assert backend.diffusion_requested_ngl is None
+
+
+def _maxtok(cmd):
+ return cmd[cmd.index("--maxtok") + 1]
+
+
+def test_fitted_launch_pins_the_context_it_was_sized_for(monkeypatch, tmp_path):
+ # Auto-sized (0 / past the zoo's 8192 clamp): the split priced 4096, so send that.
+ for n_ctx in (0, 32768):
+ cmd, backend = _launch(monkeypatch, tmp_path, [(0, 7_800, 8_188)], n_ctx = n_ctx)
+ assert (_ngl(cmd), _maxtok(cmd)) == ("8", "4096")
+ # Recorded too, for a runner that never reports the MAXTOK it resolved.
+ assert backend._effective_context_length == 4096
+ cmd, _backend = _launch(monkeypatch, tmp_path, [(0, 7_800, 8_188)], n_ctx = 2048)
+ assert (_ngl(cmd), _maxtok(cmd)) == ("11", "2048")
+
+
+def test_zero_layer_fit_runs_on_the_cpu(monkeypatch, tmp_path):
+ cmd, backend = _launch(monkeypatch, tmp_path, [(0, 0, 8_188)])
+ assert (_ngl(cmd), cmd[cmd.index("--gpu") + 1]) == ("0", "")
+ assert backend._gpu_offload_active is False
+ # The arbiter must see a CPU runner, though the request stays Auto.
+ backend._is_diffusion = True
+ assert backend.gpu_memory_mode == "auto"
+ assert backend.holds_no_vram is True
+
+
+def test_stop_during_the_fit_probe_spawns_nothing(monkeypatch, tmp_path):
+ backend = _fit_host(monkeypatch, [(0, 7_800, 8_188)])
+ monkeypatch.setattr(
+ backend, "_find_diffusion_assets", lambda: (["python", "-m", "shim"], "/bin/vs", None)
+ )
+ monkeypatch.setattr(backend, "_kill_process", lambda: None)
+ monkeypatch.setattr(backend, "_find_free_port", lambda: 18_123)
+ monkeypatch.setattr(LlamaCppBackend, "_effective_gpu_count", staticmethod(lambda *_a: 1))
+
+ def _no_spawn(*_a, **_k):
+ raise AssertionError("spawned after Stop")
+
+ monkeypatch.setattr(llama_cpp_module.subprocess, "Popen", _no_spawn)
+ stopped = backend._start_diffusion_server(
+ model_path = "/m.gguf",
+ gguf_path = "/m.gguf",
+ hf_repo = None,
+ hf_variant = None,
+ model_identifier = "unsloth/diffusiongemma-26B-A4B-it-GGUF",
+ n_ctx = 4096,
+ extra_args = None,
+ cancelled = lambda: True,
+ )
+ assert stopped is False
+
+
+def test_auto_launch_that_fits_is_unchanged(monkeypatch, tmp_path):
+ for n_ctx in (0, 4096, 32768):
+ cmd, _backend = _launch(monkeypatch, tmp_path, [(0, 23_500, 24_576)], n_ctx = n_ctx)
+ assert _ngl(cmd) is None
+ assert _maxtok(cmd) == str(n_ctx)
+ _backend._is_diffusion = True
+ assert _backend.holds_no_vram is False
+
+
+def test_auto_launch_with_an_old_shim_keeps_the_default(monkeypatch, tmp_path):
+ cmd, _backend = _launch(monkeypatch, tmp_path, [(0, 7_800, 8_188)], shim_has_ngl = False)
+ assert _ngl(cmd) is None
+
+
+def test_manual_split_is_not_refitted(monkeypatch, tmp_path):
+ cmd, _backend = _launch(
+ monkeypatch,
+ tmp_path,
+ [(0, 7_800, 8_188)],
+ gpu_memory_mode = "manual",
+ gpu_layers = 20,
+ )
+ assert _ngl(cmd) == "20"
diff --git a/studio/backend/tests/test_llama_cpp_wait_for_health.py b/studio/backend/tests/test_llama_cpp_wait_for_health.py
index 66191e753c9..a01d25da1c7 100644
--- a/studio/backend/tests/test_llama_cpp_wait_for_health.py
+++ b/studio/backend/tests/test_llama_cpp_wait_for_health.py
@@ -842,7 +842,8 @@ def test_a_cancelled_diffusion_start_reaps_the_runner():
model_identifier = "o/m",
n_ctx = 4096,
extra_args = None,
- cancelled = lambda: True,
+ # Cancelled during the health wait: a Stop before the spawn launches nothing.
+ cancelled = lambda: getattr(b, "_health_wait_cancelled", False),
)
is False
)