Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio sd.cpp: route all-Q8_0 image GGUFs to BF16 cuBLAS, keep k-quants on MMQ (#14983)

Daniel Han0a34710f580f
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.py
index 173026b8b7a..08dcded3990 100644
--- a/studio/backend/core/inference/sd_cpp_args.py
+++ b/studio/backend/core/inference/sd_cpp_args.py
@@ -141,6 +141,54 @@ class SdCppUpscaleParams:
tile_size: Optional[int] = None
+# Fork patch (ggml-patches/0001): quantized matmuls with >= N rows use BF16 dequant + cuBLAS instead of int8 MMQ.
+# Q8_0 only (Z-Image Turbo vs BF16, B200: LPIPS 0.0148 -> 0.0052); k-quants are weight-error bound. Ada/Ampere/Orin keep MMQ (2x faster).
+QUANT_CUBLAS_ENV = "GGML_CUDA_QUANT_CUBLAS_MIN_BATCH"
+QUANT_CUBLAS_WEIGHT_TYPES = frozenset({"Q8_0"})
+_UNQUANTIZED_GGUF_TYPES = frozenset({"F32", "F16", "BF16", "F64", "I8", "I16", "I32", "I64"})
+
+
+def quant_cublas_min_batch(cuda_cc: tuple[int, int]) -> Optional[int]:
+ """Rows from which ``cuda_cc`` takes the BF16 cuBLAS route, or None to keep MMQ."""
+ major, minor = int(cuda_cc[0]), int(cuda_cc[1])
+ if (major, minor) == (8, 0) or major in (9, 10):
+ return 1024
+ if major == 12:
+ return 4096
+ return None
+
+
+def gguf_quantized_weight_types(path: Optional[str]) -> Optional[frozenset[str]]:
+ """Quantized tensor types in the GGUF at ``path``, or None if unreadable. Reads only the mmapped tensor table."""
+ if not path or not str(path).lower().endswith(".gguf"):
+ return None
+ try:
+ from gguf import GGUFReader
+ reader = GGUFReader(str(path), "r")
+ return frozenset(t.tensor_type.name for t in reader.tensors) - _UNQUANTIZED_GGUF_TYPES
+ except Exception: # noqa: BLE001 -- unknown keeps the build's default (MMQ)
+ return None
+
+
+def quant_cublas_env(
+ cuda_ccs: Sequence[tuple[int, int]],
+ environ: Optional[dict] = None,
+ weight_types: Optional[frozenset[str]] = None,
+) -> dict[str, str]:
+ """Set only if every card sd.cpp may pick agrees and all quantized tensors are Q8_0; a user value wins."""
+ import os
+
+ environ = os.environ if environ is None else environ
+ if QUANT_CUBLAS_ENV in environ or not cuda_ccs:
+ return {}
+ if not weight_types or not weight_types <= QUANT_CUBLAS_WEIGHT_TYPES:
+ return {}
+ rows = {quant_cublas_min_batch(cc) for cc in cuda_ccs}
+ if len(rows) != 1 or None in rows:
+ return {}
+ return {QUANT_CUBLAS_ENV: str(rows.pop())}
+
+
# Native speed profiles. default/max: --diffusion-fa + --diffusion-conv-direct (exact); Metal/Vulkan/ROCm get fa only.
NATIVE_SPEED_OFF = "off"
NATIVE_SPEED_DEFAULT = "default"
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index 742c3611ad6..b5143546d41 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -756,6 +756,63 @@ def torch_cuda_compute_capability(
return result[0] if result else None
+def _torch_cuda_capabilities(
+ ordinal: Optional[int], *, timeout_s: Optional[float] = None
+) -> list[tuple[int, int]]:
+ """Capability of ``ordinal`` (None = every visible card); a stalled driver call reads as unknown, not a hang."""
+ import threading
+
+ result: list = []
+ worker = threading.Thread(
+ target = lambda: result.append(_read_torch_cuda_capabilities(ordinal)),
+ daemon = True,
+ name = "sd-cpp-torch-ccs",
+ )
+ worker.start()
+ worker.join(TORCH_CC_TIMEOUT_S if timeout_s is None else timeout_s)
+ return result[0] if result else []
+
+
+def _read_torch_cuda_capabilities(ordinal: Optional[int]) -> list[tuple[int, int]]:
+ try:
+ import torch
+
+ if getattr(torch.version, "hip", None) or not torch.cuda.is_available():
+ return []
+ cards = [int(ordinal)] if ordinal is not None else list(range(torch.cuda.device_count()))
+ return [tuple(int(v) for v in torch.cuda.get_device_capability(i)) for i in cards]
+ except Exception: # noqa: BLE001 -- unknown keeps the build's default
+ return []
+
+
+def _sd_cpp_quant_cublas_env(
+ ordinal: Optional[int], weight_types: Optional[frozenset[str]]
+) -> dict[str, str]:
+ if not weight_types or not weight_types <= sd_cpp_args_mod.QUANT_CUBLAS_WEIGHT_TYPES:
+ return {}
+ return sd_cpp_args_mod.quant_cublas_env(
+ _torch_cuda_capabilities(ordinal), weight_types = weight_types
+ )
+
+
+def _sd_cpp_spawn_env(
+ off_torch: Any,
+ device: str,
+ ordinal: Optional[int],
+ offload: "tuple[str, ...] | list[str]",
+ pinned: "tuple[str, ...] | list[str]",
+ weight_types: Any,
+) -> Optional[dict[str, str]]:
+ """The child env for the FINAL placement and argv. The ordinal counts only when the argv carries its pin;
+ without one sd.cpp picks its own default card, so every visible card must qualify."""
+ if off_torch is not None:
+ return off_torch.child_env()
+ if device != "cuda":
+ return None
+ pin_applied = len(pinned) > len(offload)
+ return _sd_cpp_quant_cublas_env(ordinal if pin_applied else None, weight_types()) or None
+
+
def _torch_cuda_compute_capability(ordinal: Optional[int]) -> Optional[tuple[int, int]]:
try:
import torch
@@ -3619,6 +3676,11 @@ class SdCppDiffusionBackend:
device, offload = _placement(*_running())
spawn_env = off_torch.child_env() if off_torch is not None else None
+
+ @functools.lru_cache(maxsize = 1)
+ def _diffusion_weight_types() -> Optional[frozenset[str]]:
+ return sd_cpp_args_mod.gguf_quantized_weight_types(paths.get("diffusion_model"))
+
# The device pin is NOT folded in here: the binary can still change below, through the deferred
# accelerator install, the post-download re-resolve, or a server start that falls back to one-shot, and
# the ggml device names come from whichever build ends up running. It is added at each point the flags are
@@ -3742,12 +3804,21 @@ class SdCppDiffusionBackend:
# install that later overwrites the executable underneath the live server.
started_ok = False
try:
+ server_offload = _offload_with_device_pin_impl(
+ offload, server_binary, _ordinal()
+ )
+ spawn_env = _sd_cpp_spawn_env(
+ off_torch,
+ device,
+ _ordinal(),
+ offload,
+ server_offload,
+ _diffusion_weight_types,
+ )
server.start(
files,
vae_format = fam.sd_cpp_vae_format,
- offload = _offload_with_device_pin_impl(
- offload, server_binary, _ordinal()
- ),
+ offload = server_offload,
native_speed = native_speed,
threads = _default_threads(),
env = spawn_env,
@@ -3826,6 +3897,15 @@ class SdCppDiffusionBackend:
_ordinal(),
)
)
+ # Recomputed for what is committed: a deferred install or the one-shot fallback can move placement.
+ spawn_env = _sd_cpp_spawn_env(
+ off_torch,
+ device,
+ _ordinal(),
+ offload,
+ committed_offload_flags,
+ _diffusion_weight_types,
+ )
state = _SdState(
repo_id = repo_id,
display_repo_id = display_repo_id,
diff --git a/studio/backend/tests/test_sd_cpp_args.py b/studio/backend/tests/test_sd_cpp_args.py
index a97bbf1edb6..9bb3aea1c68 100644
--- a/studio/backend/tests/test_sd_cpp_args.py
+++ b/studio/backend/tests/test_sd_cpp_args.py
@@ -1101,3 +1101,90 @@ def test_mmap_env_override_and_no_duplicate(monkeypatch, tmp_path):
monkeypatch.delenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP")
for cmd in _all_builders(new, extra_args = ["--mmap"]):
assert cmd.count("--mmap") == 1
+
+
+ "ccs,expected",
+ [
+ ([(8, 0)], "1024"), # A100
+ ([(9, 0)], "1024"), # H100
+ ([(10, 0)], "1024"), # B200
+ ([(10, 3)], "1024"),
+ ([(12, 0)], "4096"), # RTX PRO 6000 / RTX 50
+ ([(8, 9)], None), # L4 / RTX 40: MMQ stays faster
+ ([(8, 6)], None),
+ ([(8, 7)], None),
+ ([(7, 5)], None),
+ ([(11, 0)], None),
+ ([(8, 0), (10, 0)], "1024"),
+ ([(8, 0), (8, 9)], None), # sd.cpp may land on either card
+ ([(10, 0), (12, 0)], None),
+ ([], None),
+ ],
+)
+def test_quant_cublas_env_follows_the_measured_architectures(ccs, expected):
+ env = _sd_cpp_args_module.quant_cublas_env(ccs, environ = {}, weight_types = frozenset({"Q8_0"}))
+ assert env == ({} if expected is None else {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": expected})
+
+
[email protected]("user_value", ["0", "512"])
+def test_quant_cublas_env_never_overrides_the_user(user_value):
+ environ = {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": user_value}
+ assert (
+ _sd_cpp_args_module.quant_cublas_env(
+ [(10, 0)], environ = environ, weight_types = frozenset({"Q8_0"})
+ )
+ == {}
+ )
+
+
+ "weight_types,expected",
+ [
+ (frozenset({"Q8_0"}), "1024"),
+ (frozenset({"Q4_K"}), None), # k-quants keep MMQ
+ (frozenset({"Q4_K", "Q5_K"}), None),
+ (frozenset({"Q8_0", "Q4_K"}), None), # a mixed file would route its k-quants too
+ (frozenset({"Q4_0"}), None),
+ (frozenset(), None), # unquantized: nothing to route
+ (None, None), # unknown (safetensors, unreadable)
+ ],
+)
+def test_quant_cublas_env_only_for_q8_0_diffusion_ggufs(weight_types, expected):
+ env = _sd_cpp_args_module.quant_cublas_env([(10, 0)], environ = {}, weight_types = weight_types)
+ assert env == ({} if expected is None else {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": expected})
+
+
+def test_gguf_quantized_weight_types_reads_the_tensor_table(tmp_path):
+ gguf = pytest.importorskip("gguf")
+ import numpy as np
+
+ def write(name, tensors):
+ path = tmp_path / name
+ w = gguf.GGUFWriter(str(path), "test")
+ for tname, arr, qtype in tensors:
+ if qtype is None:
+ w.add_tensor(tname, arr)
+ else:
+ w.add_tensor(tname, gguf.quants.quantize(arr, qtype), raw_dtype = qtype)
+ w.write_header_to_file()
+ w.write_kv_data_to_file()
+ w.write_tensors_to_file()
+ w.close()
+ return str(path)
+
+ x = np.ones((4, 256), dtype = np.float32)
+ q8 = write(
+ "q8.gguf",
+ [("a", x, gguf.GGMLQuantizationType.Q8_0), ("n", np.ones(4, dtype = np.float32), None)],
+ )
+ mixed = write(
+ "mixed.gguf",
+ [("a", x, gguf.GGMLQuantizationType.Q8_0), ("b", x, gguf.GGMLQuantizationType.Q4_0)],
+ )
+ types_of = _sd_cpp_args_module.gguf_quantized_weight_types
+ assert types_of(q8) == frozenset({"Q8_0"})
+ assert types_of(mixed) == frozenset({"Q8_0", "Q4_0"})
+ assert types_of(str(tmp_path / "missing.gguf")) is None
+ assert types_of(str(tmp_path / "model.safetensors")) is None
+ assert types_of(None) is None
diff --git a/studio/backend/tests/test_sd_cpp_backend.py b/studio/backend/tests/test_sd_cpp_backend.py
index baadacc9fa5..eba69d59842 100644
--- a/studio/backend/tests/test_sd_cpp_backend.py
+++ b/studio/backend/tests/test_sd_cpp_backend.py
@@ -1459,6 +1459,7 @@ def _run_server_load(
device = "cpu",
gguf_filename = "z.gguf",
family_override = None,
+ gpu_ordinal = None,
):
fam = detect_family(fam_name)
monkeypatch.setattr(bk, "find_sd_server_binary", lambda: "/x/sd-server")
@@ -1489,6 +1490,7 @@ def _run_server_load(
family_override = family_override,
hf_token = None,
_load_token = 1,
+ gpu_ordinal = gpu_ordinal,
)
@@ -2965,3 +2967,123 @@ def test_the_native_progress_total_is_sized_where_the_assets_are_fetched(monkeyp
b._set_expected_bytes(assets, None)
assert asked == ["unsloth/FLUX.1-schnell", "unsloth/FLUX.1-schnell"]
assert b._loading.expected_bytes == 2_000
+
+
+ "device,ccs,expected",
+ [
+ ("cuda", [(10, 0)], {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}),
+ ("cuda", [(8, 9)], None),
+ ("cuda", [], None),
+ ("cpu", [(10, 0)], None),
+ ],
+)
+def test_cuda_sd_server_gets_the_bf16_cublas_route_by_compute_capability(
+ monkeypatch, device, ccs, expected
+):
+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
+ monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: list(ccs))
+ monkeypatch.setattr(
+ bk.sd_cpp_args_mod, "gguf_quantized_weight_types", lambda path: frozenset({"Q8_0"})
+ )
+ b = SdCppDiffusionBackend()
+ servers: list = []
+ _run_server_load(monkeypatch, b, servers, device = device)
+ assert servers and servers[-1].env == expected
+ assert dict(b._state.child_env) == (expected or {})
+
+
+ "weight_types", [frozenset({"Q4_K", "Q5_K"}), frozenset({"Q8_0", "Q4_K"}), None]
+)
+def test_a_k_quant_diffusion_gguf_keeps_mmq(monkeypatch, weight_types):
+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
+ monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: [(10, 0)])
+ seen: list = []
+ monkeypatch.setattr(
+ bk.sd_cpp_args_mod,
+ "gguf_quantized_weight_types",
+ lambda path: seen.append(path) or weight_types,
+ )
+ b = SdCppDiffusionBackend()
+ servers: list = []
+ _run_server_load(monkeypatch, b, servers, device = "cuda")
+ assert seen == ["/m/z.gguf"] # the diffusion model, not the encoder or VAE
+ assert servers[-1].env is None
+ assert dict(b._state.child_env) == {}
+
+
+def test_a_user_quant_cublas_value_reaches_sd_server_unchanged(monkeypatch):
+ monkeypatch.setenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", "0")
+ monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: [(10, 0)])
+ monkeypatch.setattr(
+ bk.sd_cpp_args_mod, "gguf_quantized_weight_types", lambda path: frozenset({"Q8_0"})
+ )
+ b = SdCppDiffusionBackend()
+ servers: list = []
+ _run_server_load(monkeypatch, b, servers, device = "cuda")
+ assert servers[-1].env is None
+
+
+def test_the_capability_probe_is_bounded_when_the_driver_stalls(monkeypatch):
+ import sys
+ import time
+
+ release = threading.Event()
+ cuda = types.SimpleNamespace(is_available = lambda: release.wait(30) or True)
+ fake_torch = types.SimpleNamespace(version = types.SimpleNamespace(hip = None), cuda = cuda)
+ monkeypatch.setitem(sys.modules, "torch", fake_torch)
+ start = time.monotonic()
+ try:
+ assert bk._torch_cuda_capabilities(None, timeout_s = 0.2) == []
+ finally:
+ release.set()
+ assert time.monotonic() - start < 5
+
+
+def test_the_cublas_route_follows_a_deferred_install_onto_cuda(monkeypatch):
+ # Placed on the CPU bundle first; the deferred upgrade lands the CUDA build, so the committed env must follow.
+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
+ monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: [(10, 0)])
+ monkeypatch.setattr(
+ bk.sd_cpp_args_mod, "gguf_quantized_weight_types", lambda path: frozenset({"Q8_0"})
+ )
+ installed = {"accelerator": "cpu"}
+ monkeypatch.setattr(bk, "_installed_accelerator_of", lambda binary: installed["accelerator"])
+
+ def _upgrade(self, server_binary, **_kw):
+ installed["accelerator"] = "cuda"
+ return server_binary
+
+ monkeypatch.setattr(SdCppDiffusionBackend, "_upgraded_or_refused", _upgrade)
+ monkeypatch.setattr(
+ SdCppDiffusionBackend, "_resolved_accelerator", staticmethod(lambda card = None: "cuda")
+ )
+ monkeypatch.setattr(bk, "_managed_tree_in_use", lambda: True)
+ b = SdCppDiffusionBackend()
+ servers: list = []
+ _run_server_load(monkeypatch, b, servers, device = "cuda")
+ assert b._state.device == "cuda"
+ assert servers[-1].env == {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}
+ assert dict(b._state.child_env) == {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}
+
+
+ "resolved,expected", [("CUDA1", {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}), (None, None)]
+)
+def test_an_unresolved_pin_judges_every_visible_card(monkeypatch, resolved, expected):
+ # Card 1 is an A100, card 0 (sd.cpp's default) an Ada card. Without the pin sd.cpp runs on card 0.
+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
+ caps = {None: [(8, 9), (8, 0)], 0: [(8, 9)], 1: [(8, 0)]}
+ monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: list(caps[ordinal]))
+ monkeypatch.setattr(
+ bk.sd_cpp_args_mod, "gguf_quantized_weight_types", lambda path: frozenset({"Q8_0"})
+ )
+ monkeypatch.setattr(bk, "sd_cpp_device_name_for_ordinal", lambda binary, ordinal: resolved)
+ monkeypatch.setattr(bk, "physical_card_name", lambda ordinal: (None, None))
+ monkeypatch.setattr(bk, "sd_cpp_device_named", lambda binary, name, position = None: None)
+ b = SdCppDiffusionBackend()
+ servers: list = []
+ _run_server_load(monkeypatch, b, servers, device = "cuda", gpu_ordinal = 1)
+ assert servers[-1].env == expected
+ assert dict(b._state.child_env) == (expected or {})