Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: MiniMax-H3 GGUF uses the BF16 cuBLAS matmul path by default on sm80+ (1.2 to 2.3x per step, closer to an F32 reference) (#12527)

Daniel Hane640be3c33e5
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index e94b068d6ad..1b7fdfce757 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -382,6 +382,7 @@ def sd_cpp_accelerator_device_verdict(binary: str) -> Optional[bool]:
indistinguishable from a real accelerator, so an unreadable re-probe would read as a build
that changed underneath the load and refuse it."""
text = _sd_cpp_probe_output(binary, "--list-devices")
+ _remember_device_listing(binary, text)
if text is None:
return None
names = [line.split("\t", 1)[0].strip() for line in text.splitlines() if "\t" in line]
@@ -424,6 +425,67 @@ def sd_cpp_device_name_for_ordinal(binary: Optional[str], ordinal: Optional[int]
return None
+# ggml-cuda's init log on stderr; a HIP build says "ROCm devices", so only CUDA builds match.
+_CUDA_INIT_RE = re.compile(r"ggml_cuda_init: found \d+ CUDA devices")
+_CUDA_DEVICE_CC_RE = re.compile(
+ r"^\s*Device (\d+): .*?, compute capability (\d+)\.(\d+)", re.MULTILINE
+)
+
+
+# Last --list-devices answer per binary + (size, mtime_ns); only the capability read reuses it, never the verdict.
+_LAST_DEVICE_LISTING: dict = {}
+
+
+def _binary_stat_identity(binary: str) -> Optional[tuple[int, int]]:
+ try:
+ st = os.stat(binary)
+ except OSError:
+ return None
+ return (st.st_size, st.st_mtime_ns)
+
+
+def _remember_device_listing(binary: Optional[str], text: Optional[str]) -> None:
+ if not binary:
+ return
+ if text is None:
+ _LAST_DEVICE_LISTING.pop(binary, None)
+ return
+ _LAST_DEVICE_LISTING[binary] = (_binary_stat_identity(binary), text)
+
+
+def sd_cpp_cuda_compute_capability(
+ binary: Optional[str],
+ device_name: Optional[str],
+ *,
+ probe: bool = True,
+) -> Optional[tuple[int, int]]:
+ """Compute capability the CUDA build reports for ``device_name`` (``CUDA<i>``); the lowest card
+ when unpinned. None when unsure (unreadable, non-CUDA build, unlisted device). ``probe = False``
+ only reads the listing the last accelerator verdict took of this file."""
+ if not binary:
+ return None
+ text = None
+ seen = _LAST_DEVICE_LISTING.get(binary)
+ if seen is not None and seen[0] == _binary_stat_identity(binary):
+ text = seen[1]
+ elif probe:
+ text = _sd_cpp_probe_output(binary, "--list-devices")
+ if text is None or not _CUDA_INIT_RE.search(text):
+ return None
+ caps = {
+ int(m.group(1)): (int(m.group(2)), int(m.group(3)))
+ for m in _CUDA_DEVICE_CC_RE.finditer(text)
+ }
+ if not caps:
+ return None
+ if device_name is None:
+ return min(caps.values())
+ head = device_name.rstrip("0123456789")
+ if head.upper() != "CUDA" or not device_name[len(head) :]:
+ return None
+ return caps.get(int(device_name[len(head) :]))
+
+
# Every namespace ggml names a device in; the narrower list above is physical-index schemes only.
_GGML_DEVICE_PREFIXES: tuple[str, ...] = ("CUDA", "ROCM", "VULKAN", "SYCL", "METAL", "OPENCL")
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 44b1ec385a2..d573b98500c 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3527,6 +3527,14 @@ class VideoBackend:
native_device_name = sd_cpp_device_named(
binary, selected_name, position = selected_position
)
+ # Same number the fork's dispatch reads, off the re-check's listing (no extra subprocess).
+ from .sd_cpp_backend import sd_cpp_cuda_compute_capability
+
+ native_cuda_cc = (
+ sd_cpp_cuda_compute_capability(binary, native_device_name, probe = False)
+ if native_device != "cpu" and listed_accelerator is not None
+ else None
+ )
requested_mode = normalize_memory_mode(memory_mode) or "auto"
policy = {
"auto": "none" if native_device == "cpu" else "group",
@@ -3562,10 +3570,20 @@ class VideoBackend:
native_env: tuple[tuple[str, str], ...] = ()
if h3_sage:
native_offload += ("--sage-attn",)
- # Same opt-in takes the fork's BF16 cuBLAS path; a user-exported value (0 included) wins.
- from .video_minimax_h3 import H3_QUANT_CUBLAS_ENV, H3_QUANT_CUBLAS_MIN_BATCH
- if H3_QUANT_CUBLAS_ENV not in os.environ:
- native_env += ((H3_QUANT_CUBLAS_ENV, H3_QUANT_CUBLAS_MIN_BATCH),)
+ if native_device != "cpu":
+ from .video_minimax_h3 import h3_quant_cublas_env
+ native_env += h3_quant_cublas_env(native_cuda_cc, sage = h3_sage)
+ from .video_minimax_h3 import H3_QUANT_CUBLAS_ENV, H3_QUANT_CUBLAS_MIN_CC
+
+ # The fork ignores the env below sm80, so a user value there is not the route.
+ h3_quant_cublas = (
+ native_device != "cpu"
+ and (native_cuda_cc is None or tuple(native_cuda_cc) >= H3_QUANT_CUBLAS_MIN_CC)
+ and dict(native_env)
+ .get(H3_QUANT_CUBLAS_ENV, os.environ.get(H3_QUANT_CUBLAS_ENV, ""))
+ .strip()
+ not in ("", "0")
+ )
# After the policy, so the pin can see which modules it left on the CPU; without it sd.cpp uses ordinal 0
# whatever was selected.
native_offload += tuple(device_backend_flags(native_device_name, list(native_offload)))
@@ -3646,9 +3664,12 @@ class VideoBackend:
"speed_mode": (
speed_mode,
SPEED_MAX if h3_sage else SPEED_OFF,
- "sd.cpp SageAttention + BF16 cuBLAS"
- if h3_sage
- else "sd.cpp exact kernels",
+ {
+ (True, True): "sd.cpp SageAttention + BF16 cuBLAS",
+ (True, False): "sd.cpp SageAttention",
+ (False, True): "sd.cpp BF16 cuBLAS matmuls",
+ (False, False): "sd.cpp exact kernels",
+ }[(bool(h3_sage), bool(h3_quant_cublas))],
),
"attention_backend": (
None,
diff --git a/studio/backend/core/inference/video_minimax_h3.py b/studio/backend/core/inference/video_minimax_h3.py
index 32d8d9ccf68..8112fd8fc28 100644
--- a/studio/backend/core/inference/video_minimax_h3.py
+++ b/studio/backend/core/inference/video_minimax_h3.py
@@ -1267,6 +1267,28 @@ H3_NATIVE_RESIDENT_ENV = "UNSLOTH_H3_NATIVE_RESIDENT"
# Unsloth sd.cpp fork: quantized matmuls with >= this many rows run BF16 cuBLAS instead of int8 MMQ. Unset/0 = MMQ.
H3_QUANT_CUBLAS_ENV = "GGML_CUDA_QUANT_CUBLAS_MIN_BATCH"
H3_QUANT_CUBLAS_MIN_BATCH = "1024"
+# The fork's own floor (bf16_mma_hardware_available); below it the env changes nothing.
+H3_QUANT_CUBLAS_MIN_CC = (8, 0)
+
+
+def h3_quant_cublas_env(
+ cuda_cc: "Optional[tuple[int, int]]",
+ *,
+ sage: bool,
+ environ: Optional[dict] = None,
+) -> tuple[tuple[str, str], ...]:
+ """sd-cli env for the BF16 cuBLAS route: known sm80+ always, known older never, unknown
+ (``cuda_cc`` None) only with sage. A user-exported value (0 included) always wins."""
+ environ = os.environ if environ is None else environ
+ if H3_QUANT_CUBLAS_ENV in environ:
+ return ()
+ if cuda_cc is not None:
+ take = tuple(cuda_cc) >= H3_QUANT_CUBLAS_MIN_CC
+ else:
+ take = sage
+ return ((H3_QUANT_CUBLAS_ENV, H3_QUANT_CUBLAS_MIN_BATCH),) if take else ()
+
+
# Denoiser compute buffer at 960x544x124, from sd-cli's log; scaled by pixel volume.
H3_NATIVE_DIT_COMPUTE_BYTES_H1 = int(5.4 * 1024**3)
H3_NATIVE_H1_PIXEL_VOLUME = 960 * 544 * 124
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index fa21e258088..b13bf51aa97 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -4033,6 +4033,7 @@ def _load_h3_native_offload(
accelerator = True,
memory_mode = None,
speed_mode = None,
+ devices = None,
):
"""Run the native H3 load against a stubbed sd-cli and hand back its committed offload flags.
@@ -4060,7 +4061,8 @@ def _load_h3_native_offload(
lambda *, allow_install, accelerator: "/existing/sd-cli",
)
# One probe helper serves both --list-devices and --help, so the ggml device line rides along or the CPU case is unreachable.
- devices = "CUDA0\tNVIDIA GeForce RTX 4070 Ti\n" if accelerator else "CPU\tAMD Ryzen 9\n"
+ if devices is None:
+ devices = "CUDA0\tNVIDIA GeForce RTX 4070 Ti\n" if accelerator else "CPU\tAMD Ryzen 9\n"
monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", lambda *_a: devices + help_text)
class _Engine:
@@ -4169,14 +4171,29 @@ def test_h3_native_sage_attention_needs_the_flag_and_honours_the_veto(monkeypatc
assert "--sage-attn" not in offload
+def _cuda_devices(*caps):
+ """sd-cli --list-devices on a CUDA build: ggml_cuda_init's stderr lines, then the device table."""
+ lines = [f"ggml_cuda_init: found {len(caps)} CUDA devices (Total VRAM: 1 MiB):"]
+ lines += [
+ f" Device {i}: NVIDIA Card {i}, compute capability {cc}, VMM: yes, VRAM: 1 MiB"
+ for i, cc in enumerate(caps)
+ ]
+ lines += [f"CUDA{i}\tNVIDIA Card {i}" for i in range(len(caps))]
+ return "\n".join(lines) + "\n"
+
+
+_BF16 = {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}
+
+
def test_h3_native_speed_max_takes_the_bf16_cublas_path(monkeypatch, tmp_path):
- """speed_mode=max also hands sd-cli GGML_CUDA_QUANT_CUBLAS_MIN_BATCH; every other mode launches with no extra env."""
+ """With the capability unreadable (no ggml_cuda_init lines), the older rule holds: speed_mode=max
+ hands sd-cli GGML_CUDA_QUANT_CUBLAS_MIN_BATCH, every other mode launches with no extra env."""
monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)
monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
state, _offload = _load_h3_native_offload(
monkeypatch, tmp_path, help_text = _SAGE_HELP, speed_mode = "max"
)
- assert dict(state.pipe.env) == {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}
+ assert dict(state.pipe.env) == _BF16
for mode in (None, "default", "off"):
state, _offload = _load_h3_native_offload(
monkeypatch, tmp_path, help_text = _SAGE_HELP, speed_mode = mode
@@ -4196,6 +4213,129 @@ def test_h3_native_speed_max_takes_the_bf16_cublas_path(monkeypatch, tmp_path):
assert state.pipe.env == ()
[email protected]("cc", ["8.0", "8.6", "8.9", "9.0", "10.0", "12.0"])
+def test_h3_native_bf16_cublas_is_the_default_on_sm80_plus(monkeypatch, tmp_path, cc):
+ """A CUDA card the build reports as sm80+ takes the BF16 cuBLAS path in every speed mode, sage or not."""
+ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)
+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
+ for mode in (None, "default", "off", "max"):
+ state, offload = _load_h3_native_offload(
+ monkeypatch, tmp_path, help_text = _SAGE_HELP, speed_mode = mode, devices = _cuda_devices(cc)
+ )
+ assert dict(state.pipe.env) == _BF16, (cc, mode)
+ assert ("--sage-attn" in offload) == (mode == "max")
+ # Vetoing sage no longer drops the matmul path: they are separate levers now.
+ monkeypatch.setenv("UNSLOTH_H3_SAGE_ATTN", "0")
+ state, offload = _load_h3_native_offload(
+ monkeypatch, tmp_path, help_text = _SAGE_HELP, speed_mode = "max", devices = _cuda_devices(cc)
+ )
+ assert "--sage-attn" not in offload and dict(state.pipe.env) == _BF16
+
+
[email protected]("value", ["0", "1024", "4096"])
+def test_h3_native_bf16_cublas_user_value_wins(monkeypatch, tmp_path, value):
+ """An exported GGML_CUDA_QUANT_CUBLAS_MIN_BATCH (0 = MMQ) is inherited untouched, never overridden."""
+ monkeypatch.setenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", value)
+ for mode in (None, "max"):
+ state, _offload = _load_h3_native_offload(
+ monkeypatch,
+ tmp_path,
+ help_text = _SAGE_HELP,
+ speed_mode = mode,
+ devices = _cuda_devices("10.0"),
+ )
+ assert state.pipe.env == (), (value, mode)
+
+
[email protected]("cc", ["7.5", "7.0", "6.1"])
+def test_h3_native_bf16_cublas_skipped_below_sm80(monkeypatch, tmp_path, cc):
+ """The fork never takes the route below sm80 (no BF16 tensor cores), so those cards launch unchanged in
+ every mode, max included."""
+ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)
+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
+ for mode in (None, "default", "max"):
+ state, _offload = _load_h3_native_offload(
+ monkeypatch, tmp_path, help_text = _SAGE_HELP, speed_mode = mode, devices = _cuda_devices(cc)
+ )
+ assert state.pipe.env == (), (cc, mode)
+
+
+def test_h3_native_bf16_cublas_reads_the_pinned_card(monkeypatch, tmp_path):
+ """Mixed host: the gate reads the card the load is pinned to, not the first or the best one."""
+ from core.inference import sd_cpp_backend
+
+ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)
+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
+ text = _cuda_devices("12.0", "7.5")
+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", lambda *_a: text)
+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", "CUDA0") == (12, 0)
+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", "CUDA1") == (7, 5)
+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", "CUDA7") is None
+ # No pin: sd.cpp picks a card itself, so the lowest decides.
+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", None) == (7, 5)
+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", "Vulkan0") is None
+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability(None, "CUDA0") is None
+
+
+ "devices",
+ [
+ # HIP build: ggml_cuda_init says ROCm, device names are ROCm<i>.
+ "ggml_cuda_init: found 1 ROCm devices:\n Device 0: AMD Radeon Graphics, gfx1151 (0x1151), "
+ "compute capability 11.5, VMM: no\nROCm0\tAMD Radeon Graphics\n",
+ "Vulkan0\tAMD Radeon RX 7900 XTX (RADV NAVI31)\n",
+ "MTL0\tApple M3 Max\n",
+ ],
+)
+def test_h3_native_bf16_cublas_never_on_non_cuda_builds(monkeypatch, tmp_path, devices):
+ from core.inference import sd_cpp_backend
+
+ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)
+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", lambda *_a: devices)
+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", None) is None
+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", "ROCm0") is None
+ state, _offload = _load_h3_native_offload(
+ monkeypatch, tmp_path, help_text = _SAGE_HELP, devices = devices
+ )
+ assert state.pipe.env == ()
+
+
+def test_h3_native_bf16_cublas_never_on_the_cpu_build(monkeypatch, tmp_path):
+ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)
+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
+ for mode in (None, "max"):
+ state, _offload = _load_h3_native_offload(
+ monkeypatch, tmp_path, help_text = _SAGE_HELP, accelerator = False, speed_mode = mode
+ )
+ assert state.pipe.env == (), mode
+
+
+def test_h3_native_status_names_the_matmul_route(monkeypatch, tmp_path):
+ """The speed_mode reason says BF16 cuBLAS exactly when sd-cli gets a nonzero value for it."""
+ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)
+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
+
+ def reason(**kw):
+ state, _offload = _load_h3_native_offload(monkeypatch, tmp_path, help_text = _SAGE_HELP, **kw)
+ return state.resolved["speed_mode"]["reason"]
+
+ assert reason(devices = _cuda_devices("10.0")) == "sd.cpp BF16 cuBLAS matmuls"
+ assert (
+ reason(devices = _cuda_devices("10.0"), speed_mode = "max")
+ == "sd.cpp SageAttention + BF16 cuBLAS"
+ )
+ assert reason(devices = _cuda_devices("7.5")) == "sd.cpp exact kernels"
+ assert reason(devices = _cuda_devices("7.5"), speed_mode = "max") == "sd.cpp SageAttention"
+ assert reason(accelerator = False) == "sd.cpp exact kernels"
+ monkeypatch.setenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", "0")
+ assert reason(devices = _cuda_devices("10.0"), speed_mode = "max") == "sd.cpp SageAttention"
+ monkeypatch.setenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", "2048")
+ assert reason(devices = _cuda_devices("8.6")) == "sd.cpp BF16 cuBLAS matmuls"
+ # The fork never takes the route below sm80, whatever the user exported.
+ assert reason(devices = _cuda_devices("7.5")) == "sd.cpp exact kernels"
+
+
def test_h3_native_generate_hands_the_runtime_env_to_sd_cli(monkeypatch):
import dataclasses