Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio sd.cpp: pick the Q8_0 BF16 cuBLAS route from the diffusion GGUF, not its companions (1.57x on A100) (#17098)

Daniel Hanba22bb20519a
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.py
index 0bf59a0efc9..eaaa230cc29 100644
--- a/studio/backend/core/inference/sd_cpp_args.py
+++ b/studio/backend/core/inference/sd_cpp_args.py
@@ -135,6 +135,8 @@ class SdCppUpscaleParams:
# Fork patch (ggml-patches/0001): quantized matmuls with >= N rows use BF16 dequant + cuBLAS instead of int8 MMQ.
# Q8_0 only (Z-Image Turbo vs BF16, B200: LPIPS 0.0148 -> 0.0052); k-quants are weight-error bound. Ada/Ampere/Orin keep MMQ (2x faster).
+# Read from the diffusion GGUF only (A100, Q8_0 DiT + Q4_K_M encoder: 1.57x); the env is process-wide, but prompt rows
+# stay below it. Drop the env once the pin carries fork patch 0016, the build's own Q8_0-only default.
QUANT_CUBLAS_ENV = "GGML_CUDA_QUANT_CUBLAS_MIN_BATCH"
QUANT_CUBLAS_WEIGHT_TYPES = frozenset({"Q8_0"})
_UNQUANTIZED_GGUF_TYPES = frozenset({"F32", "F16", "BF16", "F64", "I8", "I16", "I32", "I64"})
@@ -162,36 +164,12 @@ def gguf_quantized_weight_types(path: Optional[str]) -> Optional[frozenset[str]]
return None
-def model_files_quantized_weight_types(files: "SdCppModelFiles") -> Optional[frozenset[str]]:
- """Quantized types across every loaded file (the env override is process-wide), or None if any GGUF is unreadable."""
- types = gguf_quantized_weight_types(files.diffusion_model)
- if types is None:
- return None
- for path in (
- files.vae,
- files.audio_vae,
- files.clip_l,
- files.clip_g,
- files.t5xxl,
- files.llm,
- files.llm_vision,
- files.qwen2vl,
- ):
- if not path or not str(path).lower().endswith(".gguf"):
- continue
- extra = gguf_quantized_weight_types(path)
- if extra is None:
- return None
- types |= extra
- return types
-
-
def quant_cublas_env(
cuda_ccs: Sequence[tuple[int, int]],
environ: Optional[dict] = None,
weight_types: Optional[frozenset[str]] = None,
) -> dict[str, str]:
- """Set only if every card sd.cpp may pick agrees and all quantized tensors are Q8_0; a user value wins."""
+ """Set only if every card sd.cpp may pick agrees and the diffusion GGUF is all Q8_0; a user value wins."""
import os
environ = os.environ if environ is None else environ
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index cc00e5d4e9f..025e475ce72 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -3779,7 +3779,7 @@ class SdCppDiffusionBackend:
@functools.lru_cache(maxsize = 1)
def _diffusion_weight_types() -> Optional[frozenset[str]]:
- return sd_cpp_args_mod.model_files_quantized_weight_types(files)
+ return sd_cpp_args_mod.gguf_quantized_weight_types(files.diffusion_model)
# The device pin is NOT folded in here: the binary can still change below, through the deferred
# accelerator install, the post-download re-resolve, or a server start that falls back to one-shot, and
diff --git a/studio/backend/tests/test_sd_cpp_args.py b/studio/backend/tests/test_sd_cpp_args.py
index ad25c1b09c0..0b381a5da11 100644
--- a/studio/backend/tests/test_sd_cpp_args.py
+++ b/studio/backend/tests/test_sd_cpp_args.py
@@ -1275,22 +1275,14 @@ def test_gguf_quantized_weight_types_reads_the_tensor_table(tmp_path):
assert types_of(None) is None
k_quant = write("te_q4k.gguf", [("w", None, gguf.GGMLQuantizationType.Q4_K)])
- dense = write("vae_f16.gguf", [("w", x.astype(np.float16), None)])
- all_types = _sd_cpp_args_module.model_files_quantized_weight_types
+ assert types_of(k_quant) == frozenset({"Q4_K"})
- def route(files):
+ def route(path):
return _sd_cpp_args_module.quant_cublas_env(
- [(10, 0)], environ = {}, weight_types = all_types(files)
+ [(10, 0)], environ = {}, weight_types = types_of(path)
)
- with_k_te = SdCppModelFiles(diffusion_model = q8, llm = k_quant)
- assert all_types(with_k_te) == frozenset({"Q8_0", "Q4_K"})
- assert route(with_k_te) == {}
- assert route(SdCppModelFiles(diffusion_model = q8, t5xxl = k_quant, vae = dense)) == {}
- q8_only = SdCppModelFiles(
- diffusion_model = q8, llm = q8, vae = dense, clip_l = str(tmp_path / "c.safetensors")
- )
- assert all_types(q8_only) == frozenset({"Q8_0"})
- assert route(q8_only) == {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}
- assert all_types(SdCppModelFiles(diffusion_model = q8, llm = str(tmp_path / "x.gguf"))) is None
- assert all_types(SdCppModelFiles(diffusion_model = str(tmp_path / "m.safetensors"))) is None
+ assert route(q8) == {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}
+ assert route(mixed) == {}
+ assert route(k_quant) == {}
+ assert route(str(tmp_path / "missing.gguf")) == {}
diff --git a/studio/backend/tests/test_sd_cpp_backend.py b/studio/backend/tests/test_sd_cpp_backend.py
index 24a1e059c21..93ad15e9d7f 100644
--- a/studio/backend/tests/test_sd_cpp_backend.py
+++ b/studio/backend/tests/test_sd_cpp_backend.py
@@ -3530,26 +3530,18 @@ def test_a_k_quant_diffusion_gguf_keeps_mmq(monkeypatch, weight_types):
@pytest.mark.parametrize(
- "companions,expected",
+ "companions",
[
- ({"llm": ("/m/llm.gguf", frozenset({"Q4_K", "Q6_K"}))}, None),
- (
- {
- "llm": ("/m/llm.gguf", frozenset({"Q8_0"})),
- "vae": ("/m/vae.gguf", frozenset({"Q5_K"})),
- },
- None,
- ),
- ({"llm": ("/m/llm.gguf", None)}, None),
- (
- {"llm": ("/m/llm.gguf", frozenset({"Q8_0"})), "vae": ("/m/vae.gguf", frozenset())},
- "1024",
- ),
+ {"llm": ("/m/llm.gguf", frozenset({"Q4_K", "Q6_K"}))},
+ {"llm": ("/m/llm.gguf", frozenset({"Q8_0"})), "vae": ("/m/vae.gguf", frozenset({"Q5_K"}))},
+ {"llm": ("/m/llm.gguf", None)},
+ {"qwen2vl": ("/m/te.gguf", frozenset({"Q4_K"})), "llm_vision": ("/m/mm.gguf", frozenset())},
],
)
-def test_a_q8_0_diffusion_gguf_with_a_k_quant_companion_keeps_mmq(
- monkeypatch, companions, expected
+def test_a_q8_0_diffusion_gguf_takes_the_route_whatever_its_companions_hold(
+ monkeypatch, companions
):
+ # The route follows the diffusion GGUF: a k-quant / unreadable encoder or VAE GGUF no longer keeps it on MMQ.
monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
monkeypatch.setattr(bk, "_torch_cuda_capabilities", lambda ordinal: [(10, 0)])
types_by_path = {"/m/z.gguf": frozenset({"Q8_0"})}
@@ -3565,10 +3557,10 @@ def test_a_q8_0_diffusion_gguf_with_a_k_quant_companion_keeps_mmq(
b = SdCppDiffusionBackend()
servers: list = []
_run_server_load(monkeypatch, b, servers, device = "cuda", asset_paths = paths)
- want = None if expected is None else {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": expected}
+ want = {"GGML_CUDA_QUANT_CUBLAS_MIN_BATCH": "1024"}
assert servers[-1].env == want
- assert dict(b._state.child_env) == (want or {})
- assert "/m/llm.gguf" in seen
+ assert dict(b._state.child_env) == want
+ assert seen == ["/m/z.gguf"]
def test_a_user_quant_cublas_value_reaches_sd_server_unchanged(monkeypatch):