Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio sd.cpp: pick the Q8_0 BF16 cuBLAS route from the diffusion GGUF, not its companions (1.57x on A100) (#17098)
Daniel Hanba22bb20519a
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.pyindex 0bf59a0efc9..eaaa230cc29 100644--- a/studio/backend/core/inference/sd_cpp_args.py+++ b/studio/backend/core/inference/sd_cpp_args.py@@ -135,6 +135,8 @@ class SdCppUpscaleParams:# Fork patch (ggml-patches/0001): quantized matmuls with >= N rows use BF16 dequant + cuBLAS instead of int8 MMQ.# Q8_0 only (Z-Image Turbo vs BF16, B200: LPIPS 0.0148 -> 0.0052); k-quants are weight-error bound. Ada/Ampere/Orin keep MMQ (2x faster).+# Read from the diffusion GGUF only (A100, Q8_0 DiT + Q4_K_M encoder: 1.57x); the env is process-wide, but prompt rows+# stay below it. Drop the env once the pin carries fork patch 0016, the build's own Q8_0-only default.QUANT_CUBLAS_ENV = "GGML_CUDA_QUANT_CUBLAS_MIN_BATCH"QUANT_CUBLAS_WEIGHT_TYPES = frozenset({"Q8_0"})_UNQUANTIZED_GGUF_TYPES = frozenset({"F32", "F16", "BF16", "F64", "I8", "I16", "I32", "I64"})@@ -162,36 +164,12 @@ def gguf_quantized_weight_types(path: Optional[str]) -> Optional[frozenset[str]]return None-def model_files_quantized_weight_types(files: "SdCppModelFiles") -> Optional[frozenset[str]]:- """Quantized types across every loaded file (the env override is process-wide), or None if any GGUF is unreadable."""- types = gguf_quantized_weight_types(files.diffusion_model)- if types is None:- return None- for path in (- files.vae,- files.audio_vae,- files.clip_l,- files.clip_g,- files.t5xxl,- files.llm,- files.llm_vision,- files.qwen2vl,- ):- if not path or not str(path).lower().endswith(".gguf"):- continue- extra = gguf_quantized_weight_types(path)- if extra is None:- return None- types |= extra- return types--def quant_cublas_env(cuda_ccs: Sequence[tuple[int, int]],environ: Optional[dict] = None,weight_types: Optional[frozenset[str]] = None,) -> dict[str, str]:- """Set only if every card sd.cpp may pick agrees and all quantized tensors are Q8_0; a user value wins."""+ """Set only if every card sd.cpp may pick agrees and the diffusion GGUF is all Q8_0; a user value wins."""import osenviron = os.environ if environ is None else environdiff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex cc00e5d4e9f..025e475ce72 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -3779,7 +3779,7 @@ class SdCppDiffusionBackend:@functools.lru_cache(maxsize = 1)def _diffusion_weight_types() -> Optional[frozenset[str]]:- return sd_cpp_args_mod.model_files_quantized_weight_types(files)+ return sd_cpp_args_mod.gguf_quantized_weight_types(files.diffusion_model)# The device pin is NOT folded in here: the binary can still change below, through the deferred# accelerator install, the post-download re-resolve, or a server start that falls back to one-shot, anddiff --git a/studio/backend/tests/test_sd_cpp_args.py b/studio/backend/tests/test_sd_cpp_args.pyindex ad25c1b09c0..0b381a5da11 100644--- a/studio/backend/tests/test_sd_cpp_args.py+++ b/studio/backend/tests/test_sd_cpp_args.py@@ -1275,22 +1275,14 @@ def test_gguf_quantized_weight_types_reads_the_tensor_table(tmp_path):assert types_of(None) is Nonek_quant = write("te_q4k.gguf", [("w", None, gguf.GGMLQuantizationType.Q4_K)])- dense = write("vae_f16.gguf", [("w", x.astype(np.float16), None)])- all_types = _sd_cpp_args_module.model_files_quantized_weight_types+ assert types_of(k_quant) == frozenset({"Q4_K"})- def route(files):+ def route(path):return _sd_cpp_args_module.quant_cublas_env(- [(10, 0)], environ = {}, weight_types = all_types(files)+ [(10, 0)], environ = {}, weight_types = types_of(path))- with_k_te = SdCppModelFiles(diffusion_model = q8, llm = k_quant)- assert all_types(with_k_te) == frozenset({"Q8_0", "Q4_K"})- assert route(with_k_te) == {}- assert route(SdCppModelFiles(diffusion_model = q8, t5xxl = k_quant, vae = dense)) == {}- q8_only = SdCppModelFiles(- diffusion_model = q8, llm = q8, vae = dense, clip_l = str(tmp_path / "c.safetensors")- )- assert all_types(q8_only) == frozenset({"Q8_0"})- assert route(q8_only) == {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}- assert all_types(SdCppModelFiles(diffusion_model = q8, llm = str(tmp_path / "x.gguf"))) is None- assert all_types(SdCppModelFiles(diffusion_model = str(tmp_path / "m.safetensors"))) is None+ assert route(q8) == {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}+ assert route(mixed) == {}+ assert route(k_quant) == {}+ assert route(str(tmp_path / "missing.gguf")) == {}diff --git a/studio/backend/tests/test_sd_cpp_backend.py b/studio/backend/tests/test_sd_cpp_backend.pyindex 24a1e059c21..93ad15e9d7f 100644--- a/studio/backend/tests/test_sd_cpp_backend.py+++ b/studio/backend/tests/test_sd_cpp_backend.py@@ -3530,26 +3530,18 @@ def test_a_k_quant_diffusion_gguf_keeps_mmq(monkeypatch, weight_types):@pytest.mark.parametrize(- "companions,expected",+ "companions",[- ({"llm": ("/m/llm.gguf", frozenset({"Q4_K", "Q6_K"}))}, None),- (- {- "llm": ("/m/llm.gguf", frozenset({"Q8_0"})),- "vae": ("/m/vae.gguf", frozenset({"Q5_K"})),- },- None,- ),- ({"llm": ("/m/llm.gguf", None)}, None),- (- {"llm": ("/m/llm.gguf", frozenset({"Q8_0"})), "vae": ("/m/vae.gguf", frozenset())},- "1024",- ),+ {"llm": ("/m/llm.gguf", frozenset({"Q4_K", "Q6_K"}))},+ {"llm": ("/m/llm.gguf", frozenset({"Q8_0"})), "vae": ("/m/vae.gguf", frozenset({"Q5_K"}))},+ {"llm": ("/m/llm.gguf", None)},+ {"qwen2vl": ("/m/te.gguf", frozenset({"Q4_K"})), "llm_vision": ("/m/mm.gguf", frozenset())},],)-def test_a_q8_0_diffusion_gguf_with_a_k_quant_companion_keeps_mmq(- monkeypatch, companions, expected+def test_a_q8_0_diffusion_gguf_takes_the_route_whatever_its_companions_hold(+ monkeypatch, companions):+ # The route follows the diffusion GGUF: a k-quant / unreadable encoder or VAE GGUF no longer keeps it on MMQ.monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: [(10, 0)])types_by_path = {"/m/z.gguf": frozenset({"Q8_0"})}@@ -3565,10 +3557,10 @@ def test_a_q8_0_diffusion_gguf_with_a_k_quant_companion_keeps_mmq(b = SdCppDiffusionBackend()servers: list = []_run_server_load(monkeypatch, b, servers, device = "cuda", asset_paths = paths)- want = None if expected is None else {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": expected}+ want = {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}assert servers[-1].env == want- assert dict(b._state.child_env) == (want or {})- assert "/m/llm.gguf" in seen+ assert dict(b._state.child_env) == want+ assert seen == ["/m/z.gguf"]def test_a_user_quant_cublas_value_reaches_sd_server_unchanged(monkeypatch):