Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: MiniMax-H3 GGUF uses the BF16 cuBLAS matmul path by default on sm80+ (1.2 to 2.3x per step, closer to an F32 reference) (#12527)
Daniel Hane640be3c33e5
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex e94b068d6ad..1b7fdfce757 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -382,6 +382,7 @@ def sd_cpp_accelerator_device_verdict(binary: str) -> Optional[bool]:indistinguishable from a real accelerator, so an unreadable re-probe would read as a buildthat changed underneath the load and refuse it."""text = _sd_cpp_probe_output(binary, "--list-devices")+ _remember_device_listing(binary, text)if text is None:return Nonenames = [line.split("\t", 1)[0].strip() for line in text.splitlines() if "\t" in line]@@ -424,6 +425,67 @@ def sd_cpp_device_name_for_ordinal(binary: Optional[str], ordinal: Optional[int]return None+# ggml-cuda's init log on stderr; a HIP build says "ROCm devices", so only CUDA builds match.+_CUDA_INIT_RE = re.compile(r"ggml_cuda_init: found \d+ CUDA devices")+_CUDA_DEVICE_CC_RE = re.compile(+ r"^\s*Device (\d+): .*?, compute capability (\d+)\.(\d+)", re.MULTILINE+)+++# Last --list-devices answer per binary + (size, mtime_ns); only the capability read reuses it, never the verdict.+_LAST_DEVICE_LISTING: dict = {}+++def _binary_stat_identity(binary: str) -> Optional[tuple[int, int]]:+ try:+ st = os.stat(binary)+ except OSError:+ return None+ return (st.st_size, st.st_mtime_ns)+++def _remember_device_listing(binary: Optional[str], text: Optional[str]) -> None:+ if not binary:+ return+ if text is None:+ _LAST_DEVICE_LISTING.pop(binary, None)+ return+ _LAST_DEVICE_LISTING[binary] = (_binary_stat_identity(binary), text)+++def sd_cpp_cuda_compute_capability(+ binary: Optional[str],+ device_name: Optional[str],+ *,+ probe: bool = True,+) -> Optional[tuple[int, int]]:+ """Compute capability the CUDA build reports for ``device_name`` (``CUDA<i>``); the lowest card+ when unpinned. None when unsure (unreadable, non-CUDA build, unlisted device). ``probe = False``+ only reads the listing the last accelerator verdict took of this file."""+ if not binary:+ return None+ text = None+ seen = _LAST_DEVICE_LISTING.get(binary)+ if seen is not None and seen[0] == _binary_stat_identity(binary):+ text = seen[1]+ elif probe:+ text = _sd_cpp_probe_output(binary, "--list-devices")+ if text is None or not _CUDA_INIT_RE.search(text):+ return None+ caps = {+ int(m.group(1)): (int(m.group(2)), int(m.group(3)))+ for m in _CUDA_DEVICE_CC_RE.finditer(text)+ }+ if not caps:+ return None+ if device_name is None:+ return min(caps.values())+ head = device_name.rstrip("0123456789")+ if head.upper() != "CUDA" or not device_name[len(head) :]:+ return None+ return caps.get(int(device_name[len(head) :]))++# Every namespace ggml names a device in; the narrower list above is physical-index schemes only._GGML_DEVICE_PREFIXES: tuple[str, ...] = ("CUDA", "ROCM", "VULKAN", "SYCL", "METAL", "OPENCL")diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 44b1ec385a2..d573b98500c 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -3527,6 +3527,14 @@ class VideoBackend:native_device_name = sd_cpp_device_named(binary, selected_name, position = selected_position)+ # Same number the fork's dispatch reads, off the re-check's listing (no extra subprocess).+ from .sd_cpp_backend import sd_cpp_cuda_compute_capability++ native_cuda_cc = (+ sd_cpp_cuda_compute_capability(binary, native_device_name, probe = False)+ if native_device != "cpu" and listed_accelerator is not None+ else None+ )requested_mode = normalize_memory_mode(memory_mode) or "auto"policy = {"auto": "none" if native_device == "cpu" else "group",@@ -3562,10 +3570,20 @@ class VideoBackend:native_env: tuple[tuple[str, str], ...] = ()if h3_sage:native_offload += ("--sage-attn",)- # Same opt-in takes the fork's BF16 cuBLAS path; a user-exported value (0 included) wins.- from .video_minimax_h3 import H3_QUANT_CUBLAS_ENV, H3_QUANT_CUBLAS_MIN_BATCH- if H3_QUANT_CUBLAS_ENV not in os.environ:- native_env += ((H3_QUANT_CUBLAS_ENV, H3_QUANT_CUBLAS_MIN_BATCH),)+ if native_device != "cpu":+ from .video_minimax_h3 import h3_quant_cublas_env+ native_env += h3_quant_cublas_env(native_cuda_cc, sage = h3_sage)+ from .video_minimax_h3 import H3_QUANT_CUBLAS_ENV, H3_QUANT_CUBLAS_MIN_CC++ # The fork ignores the env below sm80, so a user value there is not the route.+ h3_quant_cublas = (+ native_device != "cpu"+ and (native_cuda_cc is None or tuple(native_cuda_cc) >= H3_QUANT_CUBLAS_MIN_CC)+ and dict(native_env)+ .get(H3_QUANT_CUBLAS_ENV, os.environ.get(H3_QUANT_CUBLAS_ENV, ""))+ .strip()+ not in ("", "0")+ )# After the policy, so the pin can see which modules it left on the CPU; without it sd.cpp uses ordinal 0# whatever was selected.native_offload += tuple(device_backend_flags(native_device_name, list(native_offload)))@@ -3646,9 +3664,12 @@ class VideoBackend:"speed_mode": (speed_mode,SPEED_MAX if h3_sage else SPEED_OFF,- "sd.cpp SageAttention + BF16 cuBLAS"- if h3_sage- else "sd.cpp exact kernels",+ {+ (True, True): "sd.cpp SageAttention + BF16 cuBLAS",+ (True, False): "sd.cpp SageAttention",+ (False, True): "sd.cpp BF16 cuBLAS matmuls",+ (False, False): "sd.cpp exact kernels",+ }[(bool(h3_sage), bool(h3_quant_cublas))],),"attention_backend": (None,diff --git a/studio/backend/core/inference/video_minimax_h3.py b/studio/backend/core/inference/video_minimax_h3.pyindex 32d8d9ccf68..8112fd8fc28 100644--- a/studio/backend/core/inference/video_minimax_h3.py+++ b/studio/backend/core/inference/video_minimax_h3.py@@ -1267,6 +1267,28 @@ H3_NATIVE_RESIDENT_ENV = "UNSLOTH_H3_NATIVE_RESIDENT"# Unsloth sd.cpp fork: quantized matmuls with >= this many rows run BF16 cuBLAS instead of int8 MMQ. Unset/0 = MMQ.H3_QUANT_CUBLAS_ENV = "GGML_CUDA_QUANT_CUBLAS_MIN_BATCH"H3_QUANT_CUBLAS_MIN_BATCH = "1024"+# The fork's own floor (bf16_mma_hardware_available); below it the env changes nothing.+H3_QUANT_CUBLAS_MIN_CC = (8, 0)+++def h3_quant_cublas_env(+ cuda_cc: "Optional[tuple[int, int]]",+ *,+ sage: bool,+ environ: Optional[dict] = None,+) -> tuple[tuple[str, str], ...]:+ """sd-cli env for the BF16 cuBLAS route: known sm80+ always, known older never, unknown+ (``cuda_cc`` None) only with sage. A user-exported value (0 included) always wins."""+ environ = os.environ if environ is None else environ+ if H3_QUANT_CUBLAS_ENV in environ:+ return ()+ if cuda_cc is not None:+ take = tuple(cuda_cc) >= H3_QUANT_CUBLAS_MIN_CC+ else:+ take = sage+ return ((H3_QUANT_CUBLAS_ENV, H3_QUANT_CUBLAS_MIN_BATCH),) if take else ()++# Denoiser compute buffer at 960x544x124, from sd-cli's log; scaled by pixel volume.H3_NATIVE_DIT_COMPUTE_BYTES_H1 = int(5.4 * 1024**3)H3_NATIVE_H1_PIXEL_VOLUME = 960 * 544 * 124diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex fa21e258088..b13bf51aa97 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -4033,6 +4033,7 @@ def _load_h3_native_offload(accelerator = True,memory_mode = None,speed_mode = None,+ devices = None,):"""Run the native H3 load against a stubbed sd-cli and hand back its committed offload flags.@@ -4060,7 +4061,8 @@ def _load_h3_native_offload(lambda *, allow_install, accelerator: "/existing/sd-cli",)# One probe helper serves both --list-devices and --help, so the ggml device line rides along or the CPU case is unreachable.- devices = "CUDA0\tNVIDIA GeForce RTX 4070 Ti\n" if accelerator else "CPU\tAMD Ryzen 9\n"+ if devices is None:+ devices = "CUDA0\tNVIDIA GeForce RTX 4070 Ti\n" if accelerator else "CPU\tAMD Ryzen 9\n"monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", lambda *_a: devices + help_text)class _Engine:@@ -4169,14 +4171,29 @@ def test_h3_native_sage_attention_needs_the_flag_and_honours_the_veto(monkeypatcassert "--sage-attn" not in offload+def _cuda_devices(*caps):+ """sd-cli --list-devices on a CUDA build: ggml_cuda_init's stderr lines, then the device table."""+ lines = [f"ggml_cuda_init: found {len(caps)} CUDA devices (Total VRAM: 1 MiB):"]+ lines += [+ f" Device {i}: NVIDIA Card {i}, compute capability {cc}, VMM: yes, VRAM: 1 MiB"+ for i, cc in enumerate(caps)+ ]+ lines += [f"CUDA{i}\tNVIDIA Card {i}" for i in range(len(caps))]+ return "\n".join(lines) + "\n"+++_BF16 = {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}++def test_h3_native_speed_max_takes_the_bf16_cublas_path(monkeypatch, tmp_path):- """speed_mode=max also hands sd-cli GGML_CUDA_QUANT_CUBLAS_MIN_BATCH; every other mode launches with no extra env."""+ """With the capability unreadable (no ggml_cuda_init lines), the older rule holds: speed_mode=max+ hands sd-cli GGML_CUDA_QUANT_CUBLAS_MIN_BATCH, every other mode launches with no extra env."""monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)state, _offload = _load_h3_native_offload(monkeypatch, tmp_path, help_text = _SAGE_HELP, speed_mode = "max")- assert dict(state.pipe.env) == {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}+ assert dict(state.pipe.env) == _BF16for mode in (None, "default", "off"):state, _offload = _load_h3_native_offload(monkeypatch, tmp_path, help_text = _SAGE_HELP, speed_mode = mode@@ -4196,6 +4213,129 @@ def test_h3_native_speed_max_takes_the_bf16_cublas_path(monkeypatch, tmp_path):assert state.pipe.env == ()[email protected]("cc", ["8.0", "8.6", "8.9", "9.0", "10.0", "12.0"])+def test_h3_native_bf16_cublas_is_the_default_on_sm80_plus(monkeypatch, tmp_path, cc):+ """A CUDA card the build reports as sm80+ takes the BF16 cuBLAS path in every speed mode, sage or not."""+ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)+ for mode in (None, "default", "off", "max"):+ state, offload = _load_h3_native_offload(+ monkeypatch, tmp_path, help_text = _SAGE_HELP, speed_mode = mode, devices = _cuda_devices(cc)+ )+ assert dict(state.pipe.env) == _BF16, (cc, mode)+ assert ("--sage-attn" in offload) == (mode == "max")+ # Vetoing sage no longer drops the matmul path: they are separate levers now.+ monkeypatch.setenv("UNSLOTH_H3_SAGE_ATTN", "0")+ state, offload = _load_h3_native_offload(+ monkeypatch, tmp_path, help_text = _SAGE_HELP, speed_mode = "max", devices = _cuda_devices(cc)+ )+ assert "--sage-attn" not in offload and dict(state.pipe.env) == _BF16++[email protected]("value", ["0", "1024", "4096"])+def test_h3_native_bf16_cublas_user_value_wins(monkeypatch, tmp_path, value):+ """An exported GGML_CUDA_QUANT_CUBLAS_MIN_BATCH (0 = MMQ) is inherited untouched, never overridden."""+ monkeypatch.setenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", value)+ for mode in (None, "max"):+ state, _offload = _load_h3_native_offload(+ monkeypatch,+ tmp_path,+ help_text = _SAGE_HELP,+ speed_mode = mode,+ devices = _cuda_devices("10.0"),+ )+ assert state.pipe.env == (), (value, mode)++[email protected]("cc", ["7.5", "7.0", "6.1"])+def test_h3_native_bf16_cublas_skipped_below_sm80(monkeypatch, tmp_path, cc):+ """The fork never takes the route below sm80 (no BF16 tensor cores), so those cards launch unchanged in+ every mode, max included."""+ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)+ for mode in (None, "default", "max"):+ state, _offload = _load_h3_native_offload(+ monkeypatch, tmp_path, help_text = _SAGE_HELP, speed_mode = mode, devices = _cuda_devices(cc)+ )+ assert state.pipe.env == (), (cc, mode)+++def test_h3_native_bf16_cublas_reads_the_pinned_card(monkeypatch, tmp_path):+ """Mixed host: the gate reads the card the load is pinned to, not the first or the best one."""+ from core.inference import sd_cpp_backend++ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)+ text = _cuda_devices("12.0", "7.5")+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", lambda *_a: text)+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", "CUDA0") == (12, 0)+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", "CUDA1") == (7, 5)+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", "CUDA7") is None+ # No pin: sd.cpp picks a card itself, so the lowest decides.+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", None) == (7, 5)+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", "Vulkan0") is None+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability(None, "CUDA0") is None+++ "devices",+ [+ # HIP build: ggml_cuda_init says ROCm, device names are ROCm<i>.+ "ggml_cuda_init: found 1 ROCm devices:\n Device 0: AMD Radeon Graphics, gfx1151 (0x1151), "+ "compute capability 11.5, VMM: no\nROCm0\tAMD Radeon Graphics\n",+ "Vulkan0\tAMD Radeon RX 7900 XTX (RADV NAVI31)\n",+ "MTL0\tApple M3 Max\n",+ ],+)+def test_h3_native_bf16_cublas_never_on_non_cuda_builds(monkeypatch, tmp_path, devices):+ from core.inference import sd_cpp_backend++ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", lambda *_a: devices)+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", None) is None+ assert sd_cpp_backend.sd_cpp_cuda_compute_capability("/x/sd-cli", "ROCm0") is None+ state, _offload = _load_h3_native_offload(+ monkeypatch, tmp_path, help_text = _SAGE_HELP, devices = devices+ )+ assert state.pipe.env == ()+++def test_h3_native_bf16_cublas_never_on_the_cpu_build(monkeypatch, tmp_path):+ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)+ for mode in (None, "max"):+ state, _offload = _load_h3_native_offload(+ monkeypatch, tmp_path, help_text = _SAGE_HELP, accelerator = False, speed_mode = mode+ )+ assert state.pipe.env == (), mode+++def test_h3_native_status_names_the_matmul_route(monkeypatch, tmp_path):+ """The speed_mode reason says BF16 cuBLAS exactly when sd-cli gets a nonzero value for it."""+ monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)++ def reason(**kw):+ state, _offload = _load_h3_native_offload(monkeypatch, tmp_path, help_text = _SAGE_HELP, **kw)+ return state.resolved["speed_mode"]["reason"]++ assert reason(devices = _cuda_devices("10.0")) == "sd.cpp BF16 cuBLAS matmuls"+ assert (+ reason(devices = _cuda_devices("10.0"), speed_mode = "max")+ == "sd.cpp SageAttention + BF16 cuBLAS"+ )+ assert reason(devices = _cuda_devices("7.5")) == "sd.cpp exact kernels"+ assert reason(devices = _cuda_devices("7.5"), speed_mode = "max") == "sd.cpp SageAttention"+ assert reason(accelerator = False) == "sd.cpp exact kernels"+ monkeypatch.setenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", "0")+ assert reason(devices = _cuda_devices("10.0"), speed_mode = "max") == "sd.cpp SageAttention"+ monkeypatch.setenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", "2048")+ assert reason(devices = _cuda_devices("8.6")) == "sd.cpp BF16 cuBLAS matmuls"+ # The fork never takes the route below sm80, whatever the user exported.+ assert reason(devices = _cuda_devices("7.5")) == "sd.cpp exact kernels"++def test_h3_native_generate_hands_the_runtime_env_to_sd_cli(monkeypatch):import dataclasses