Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio sd.cpp: route all-Q8_0 image GGUFs to BF16 cuBLAS, keep k-quants on MMQ (#14983)
Daniel Han0a34710f580f
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.pyindex 173026b8b7a..08dcded3990 100644--- a/studio/backend/core/inference/sd_cpp_args.py+++ b/studio/backend/core/inference/sd_cpp_args.py@@ -141,6 +141,54 @@ class SdCppUpscaleParams:tile_size: Optional[int] = None+# Fork patch (ggml-patches/0001): quantized matmuls with >= N rows use BF16 dequant + cuBLAS instead of int8 MMQ.+# Q8_0 only (Z-Image Turbo vs BF16, B200: LPIPS 0.0148 -> 0.0052); k-quants are weight-error bound. Ada/Ampere/Orin keep MMQ (2x faster).+QUANT_CUBLAS_ENV = "GGML_CUDA_QUANT_CUBLAS_MIN_BATCH"+QUANT_CUBLAS_WEIGHT_TYPES = frozenset({"Q8_0"})+_UNQUANTIZED_GGUF_TYPES = frozenset({"F32", "F16", "BF16", "F64", "I8", "I16", "I32", "I64"})+++def quant_cublas_min_batch(cuda_cc: tuple[int, int]) -> Optional[int]:+ """Rows from which ``cuda_cc`` takes the BF16 cuBLAS route, or None to keep MMQ."""+ major, minor = int(cuda_cc[0]), int(cuda_cc[1])+ if (major, minor) == (8, 0) or major in (9, 10):+ return 1024+ if major == 12:+ return 4096+ return None+++def gguf_quantized_weight_types(path: Optional[str]) -> Optional[frozenset[str]]:+ """Quantized tensor types in the GGUF at ``path``, or None if unreadable. Reads only the mmapped tensor table."""+ if not path or not str(path).lower().endswith(".gguf"):+ return None+ try:+ from gguf import GGUFReader+ reader = GGUFReader(str(path), "r")+ return frozenset(t.tensor_type.name for t in reader.tensors) - _UNQUANTIZED_GGUF_TYPES+ except Exception: # noqa: BLE001 -- unknown keeps the build's default (MMQ)+ return None+++def quant_cublas_env(+ cuda_ccs: Sequence[tuple[int, int]],+ environ: Optional[dict] = None,+ weight_types: Optional[frozenset[str]] = None,+) -> dict[str, str]:+ """Set only if every card sd.cpp may pick agrees and all quantized tensors are Q8_0; a user value wins."""+ import os++ environ = os.environ if environ is None else environ+ if QUANT_CUBLAS_ENV in environ or not cuda_ccs:+ return {}+ if not weight_types or not weight_types <= QUANT_CUBLAS_WEIGHT_TYPES:+ return {}+ rows = {quant_cublas_min_batch(cc) for cc in cuda_ccs}+ if len(rows) != 1 or None in rows:+ return {}+ return {QUANT_CUBLAS_ENV: str(rows.pop())}++# Native speed profiles. default/max: --diffusion-fa + --diffusion-conv-direct (exact); Metal/Vulkan/ROCm get fa only.NATIVE_SPEED_OFF = "off"NATIVE_SPEED_DEFAULT = "default"diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex 742c3611ad6..b5143546d41 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -756,6 +756,63 @@ def torch_cuda_compute_capability(return result[0] if result else None+def _torch_cuda_capabilities(+ ordinal: Optional[int], *, timeout_s: Optional[float] = None+) -> list[tuple[int, int]]:+ """Capability of ``ordinal`` (None = every visible card); a stalled driver call reads as unknown, not a hang."""+ import threading++ result: list = []+ worker = threading.Thread(+ target = lambda: result.append(_read_torch_cuda_capabilities(ordinal)),+ daemon = True,+ name = "sd-cpp-torch-ccs",+ )+ worker.start()+ worker.join(TORCH_CC_TIMEOUT_S if timeout_s is None else timeout_s)+ return result[0] if result else []+++def _read_torch_cuda_capabilities(ordinal: Optional[int]) -> list[tuple[int, int]]:+ try:+ import torch++ if getattr(torch.version, "hip", None) or not torch.cuda.is_available():+ return []+ cards = [int(ordinal)] if ordinal is not None else list(range(torch.cuda.device_count()))+ return [tuple(int(v) for v in torch.cuda.get_device_capability(i)) for i in cards]+ except Exception: # noqa: BLE001 -- unknown keeps the build's default+ return []+++def _sd_cpp_quant_cublas_env(+ ordinal: Optional[int], weight_types: Optional[frozenset[str]]+) -> dict[str, str]:+ if not weight_types or not weight_types <= sd_cpp_args_mod.QUANT_CUBLAS_WEIGHT_TYPES:+ return {}+ return sd_cpp_args_mod.quant_cublas_env(+ _torch_cuda_capabilities(ordinal), weight_types = weight_types+ )+++def _sd_cpp_spawn_env(+ off_torch: Any,+ device: str,+ ordinal: Optional[int],+ offload: "tuple[str, ...] | list[str]",+ pinned: "tuple[str, ...] | list[str]",+ weight_types: Any,+) -> Optional[dict[str, str]]:+ """The child env for the FINAL placement and argv. The ordinal counts only when the argv carries its pin;+ without one sd.cpp picks its own default card, so every visible card must qualify."""+ if off_torch is not None:+ return off_torch.child_env()+ if device != "cuda":+ return None+ pin_applied = len(pinned) > len(offload)+ return _sd_cpp_quant_cublas_env(ordinal if pin_applied else None, weight_types()) or None++def _torch_cuda_compute_capability(ordinal: Optional[int]) -> Optional[tuple[int, int]]:try:import torch@@ -3619,6 +3676,11 @@ class SdCppDiffusionBackend:device, offload = _placement(*_running())spawn_env = off_torch.child_env() if off_torch is not None else None++ @functools.lru_cache(maxsize = 1)+ def _diffusion_weight_types() -> Optional[frozenset[str]]:+ return sd_cpp_args_mod.gguf_quantized_weight_types(paths.get("diffusion_model"))+# The device pin is NOT folded in here: the binary can still change below, through the deferred# accelerator install, the post-download re-resolve, or a server start that falls back to one-shot, and# the ggml device names come from whichever build ends up running. It is added at each point the flags are@@ -3742,12 +3804,21 @@ class SdCppDiffusionBackend:# install that later overwrites the executable underneath the live server.started_ok = Falsetry:+ server_offload = _offload_with_device_pin_impl(+ offload, server_binary, _ordinal()+ )+ spawn_env = _sd_cpp_spawn_env(+ off_torch,+ device,+ _ordinal(),+ offload,+ server_offload,+ _diffusion_weight_types,+ )server.start(files,vae_format = fam.sd_cpp_vae_format,- offload = _offload_with_device_pin_impl(- offload, server_binary, _ordinal()- ),+ offload = server_offload,native_speed = native_speed,threads = _default_threads(),env = spawn_env,@@ -3826,6 +3897,15 @@ class SdCppDiffusionBackend:_ordinal(),))+ # Recomputed for what is committed: a deferred install or the one-shot fallback can move placement.+ spawn_env = _sd_cpp_spawn_env(+ off_torch,+ device,+ _ordinal(),+ offload,+ committed_offload_flags,+ _diffusion_weight_types,+ )state = _SdState(repo_id = repo_id,display_repo_id = display_repo_id,diff --git a/studio/backend/tests/test_sd_cpp_args.py b/studio/backend/tests/test_sd_cpp_args.pyindex a97bbf1edb6..9bb3aea1c68 100644--- a/studio/backend/tests/test_sd_cpp_args.py+++ b/studio/backend/tests/test_sd_cpp_args.py@@ -1101,3 +1101,90 @@ def test_mmap_env_override_and_no_duplicate(monkeypatch, tmp_path):monkeypatch.delenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP")for cmd in _all_builders(new, extra_args = ["--mmap"]):assert cmd.count("--mmap") == 1+++ "ccs,expected",+ [+ ([(8, 0)], "1024"), # A100+ ([(9, 0)], "1024"), # H100+ ([(10, 0)], "1024"), # B200+ ([(10, 3)], "1024"),+ ([(12, 0)], "4096"), # RTX PRO 6000 / RTX 50+ ([(8, 9)], None), # L4 / RTX 40: MMQ stays faster+ ([(8, 6)], None),+ ([(8, 7)], None),+ ([(7, 5)], None),+ ([(11, 0)], None),+ ([(8, 0), (10, 0)], "1024"),+ ([(8, 0), (8, 9)], None), # sd.cpp may land on either card+ ([(10, 0), (12, 0)], None),+ ([], None),+ ],+)+def test_quant_cublas_env_follows_the_measured_architectures(ccs, expected):+ env = _sd_cpp_args_module.quant_cublas_env(ccs, environ = {}, weight_types = frozenset({"Q8_0"}))+ assert env == ({} if expected is None else {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": expected})++[email protected]("user_value", ["0", "512"])+def test_quant_cublas_env_never_overrides_the_user(user_value):+ environ = {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": user_value}+ assert (+ _sd_cpp_args_module.quant_cublas_env(+ [(10, 0)], environ = environ, weight_types = frozenset({"Q8_0"})+ )+ == {}+ )+++ "weight_types,expected",+ [+ (frozenset({"Q8_0"}), "1024"),+ (frozenset({"Q4_K"}), None), # k-quants keep MMQ+ (frozenset({"Q4_K", "Q5_K"}), None),+ (frozenset({"Q8_0", "Q4_K"}), None), # a mixed file would route its k-quants too+ (frozenset({"Q4_0"}), None),+ (frozenset(), None), # unquantized: nothing to route+ (None, None), # unknown (safetensors, unreadable)+ ],+)+def test_quant_cublas_env_only_for_q8_0_diffusion_ggufs(weight_types, expected):+ env = _sd_cpp_args_module.quant_cublas_env([(10, 0)], environ = {}, weight_types = weight_types)+ assert env == ({} if expected is None else {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": expected})+++def test_gguf_quantized_weight_types_reads_the_tensor_table(tmp_path):+ gguf = pytest.importorskip("gguf")+ import numpy as np++ def write(name, tensors):+ path = tmp_path / name+ w = gguf.GGUFWriter(str(path), "test")+ for tname, arr, qtype in tensors:+ if qtype is None:+ w.add_tensor(tname, arr)+ else:+ w.add_tensor(tname, gguf.quants.quantize(arr, qtype), raw_dtype = qtype)+ w.write_header_to_file()+ w.write_kv_data_to_file()+ w.write_tensors_to_file()+ w.close()+ return str(path)++ x = np.ones((4, 256), dtype = np.float32)+ q8 = write(+ "q8.gguf",+ [("a", x, gguf.GGMLQuantizationType.Q8_0), ("n", np.ones(4, dtype = np.float32), None)],+ )+ mixed = write(+ "mixed.gguf",+ [("a", x, gguf.GGMLQuantizationType.Q8_0), ("b", x, gguf.GGMLQuantizationType.Q4_0)],+ )+ types_of = _sd_cpp_args_module.gguf_quantized_weight_types+ assert types_of(q8) == frozenset({"Q8_0"})+ assert types_of(mixed) == frozenset({"Q8_0", "Q4_0"})+ assert types_of(str(tmp_path / "missing.gguf")) is None+ assert types_of(str(tmp_path / "model.safetensors")) is None+ assert types_of(None) is Nonediff --git a/studio/backend/tests/test_sd_cpp_backend.py b/studio/backend/tests/test_sd_cpp_backend.pyindex baadacc9fa5..eba69d59842 100644--- a/studio/backend/tests/test_sd_cpp_backend.py+++ b/studio/backend/tests/test_sd_cpp_backend.py@@ -1459,6 +1459,7 @@ def _run_server_load(device = "cpu",gguf_filename = "z.gguf",family_override = None,+ gpu_ordinal = None,):fam = detect_family(fam_name)monkeypatch.setattr(bk, "find_sd_server_binary", lambda: "/x/sd-server")@@ -1489,6 +1490,7 @@ def _run_server_load(family_override = family_override,hf_token = None,_load_token = 1,+ gpu_ordinal = gpu_ordinal,)@@ -2965,3 +2967,123 @@ def test_the_native_progress_total_is_sized_where_the_assets_are_fetched(monkeypb._set_expected_bytes(assets, None)assert asked == ["unsloth/FLUX.1-schnell", "unsloth/FLUX.1-schnell"]assert b._loading.expected_bytes == 2_000+++ "device,ccs,expected",+ [+ ("cuda", [(10, 0)], {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}),+ ("cuda", [(8, 9)], None),+ ("cuda", [], None),+ ("cpu", [(10, 0)], None),+ ],+)+def test_cuda_sd_server_gets_the_bf16_cublas_route_by_compute_capability(+ monkeypatch, device, ccs, expected+):+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)+ monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: list(ccs))+ monkeypatch.setattr(+ bk.sd_cpp_args_mod, "gguf_quantized_weight_types", lambda path: frozenset({"Q8_0"})+ )+ b = SdCppDiffusionBackend()+ servers: list = []+ _run_server_load(monkeypatch, b, servers, device = device)+ assert servers and servers[-1].env == expected+ assert dict(b._state.child_env) == (expected or {})+++ "weight_types", [frozenset({"Q4_K", "Q5_K"}), frozenset({"Q8_0", "Q4_K"}), None]+)+def test_a_k_quant_diffusion_gguf_keeps_mmq(monkeypatch, weight_types):+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)+ monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: [(10, 0)])+ seen: list = []+ monkeypatch.setattr(+ bk.sd_cpp_args_mod,+ "gguf_quantized_weight_types",+ lambda path: seen.append(path) or weight_types,+ )+ b = SdCppDiffusionBackend()+ servers: list = []+ _run_server_load(monkeypatch, b, servers, device = "cuda")+ assert seen == ["/m/z.gguf"] # the diffusion model, not the encoder or VAE+ assert servers[-1].env is None+ assert dict(b._state.child_env) == {}+++def test_a_user_quant_cublas_value_reaches_sd_server_unchanged(monkeypatch):+ monkeypatch.setenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", "0")+ monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: [(10, 0)])+ monkeypatch.setattr(+ bk.sd_cpp_args_mod, "gguf_quantized_weight_types", lambda path: frozenset({"Q8_0"})+ )+ b = SdCppDiffusionBackend()+ servers: list = []+ _run_server_load(monkeypatch, b, servers, device = "cuda")+ assert servers[-1].env is None+++def test_the_capability_probe_is_bounded_when_the_driver_stalls(monkeypatch):+ import sys+ import time++ release = threading.Event()+ cuda = types.SimpleNamespace(is_available = lambda: release.wait(30) or True)+ fake_torch = types.SimpleNamespace(version = types.SimpleNamespace(hip = None), cuda = cuda)+ monkeypatch.setitem(sys.modules, "torch", fake_torch)+ start = time.monotonic()+ try:+ assert bk._torch_cuda_capabilities(None, timeout_s = 0.2) == []+ finally:+ release.set()+ assert time.monotonic() - start < 5+++def test_the_cublas_route_follows_a_deferred_install_onto_cuda(monkeypatch):+ # Placed on the CPU bundle first; the deferred upgrade lands the CUDA build, so the committed env must follow.+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)+ monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: [(10, 0)])+ monkeypatch.setattr(+ bk.sd_cpp_args_mod, "gguf_quantized_weight_types", lambda path: frozenset({"Q8_0"})+ )+ installed = {"accelerator": "cpu"}+ monkeypatch.setattr(bk, "_installed_accelerator_of", lambda binary: installed["accelerator"])++ def _upgrade(self, server_binary, **_kw):+ installed["accelerator"] = "cuda"+ return server_binary++ monkeypatch.setattr(SdCppDiffusionBackend, "_upgraded_or_refused", _upgrade)+ monkeypatch.setattr(+ SdCppDiffusionBackend, "_resolved_accelerator", staticmethod(lambda card = None: "cuda")+ )+ monkeypatch.setattr(bk, "_managed_tree_in_use", lambda: True)+ b = SdCppDiffusionBackend()+ servers: list = []+ _run_server_load(monkeypatch, b, servers, device = "cuda")+ assert b._state.device == "cuda"+ assert servers[-1].env == {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}+ assert dict(b._state.child_env) == {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}+++ "resolved,expected", [("CUDA1", {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}), (None, None)]+)+def test_an_unresolved_pin_judges_every_visible_card(monkeypatch, resolved, expected):+ # Card 1 is an A100, card 0 (sd.cpp's default) an Ada card. Without the pin sd.cpp runs on card 0.+ monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)+ caps = {None: [(8, 9), (8, 0)], 0: [(8, 9)], 1: [(8, 0)]}+ monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: list(caps[ordinal]))+ monkeypatch.setattr(+ bk.sd_cpp_args_mod, "gguf_quantized_weight_types", lambda path: frozenset({"Q8_0"})+ )+ monkeypatch.setattr(bk, "sd_cpp_device_name_for_ordinal", lambda binary, ordinal: resolved)+ monkeypatch.setattr(bk, "physical_card_name", lambda ordinal: (None, None))+ monkeypatch.setattr(bk, "sd_cpp_device_named", lambda binary, name, position = None: None)+ b = SdCppDiffusionBackend()+ servers: list = []+ _run_server_load(monkeypatch, b, servers, device = "cuda", gpu_ordinal = 1)+ assert servers[-1].env == expected+ assert dict(b._state.child_env) == (expected or {})