Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio sd.cpp: BF16 text encoders by default (Q8_0 / Q4_K_M changed pixels), opt-in lower precision (#14697)
Daniel Han08cd6a1598a9
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.pyindex 66e7a6ceafc..9ba51e2fb78 100644--- a/studio/backend/core/inference/diffusion_families.py+++ b/studio/backend/core/inference/diffusion_families.py@@ -530,8 +530,10 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (sd_cpp_vae_format = "flux2",# Klein 4B's text encoder IS Qwen3-4B (byte-identical shards to unsloth/Qwen3-4B; the BF16 GGUF matches it# tensor for tensor and renders pixel-identical to the old single-file encoder), so sd-cli reads unsloth's- # own GGUF of it. Q8_0, not Q4_K_M: on klein 4B the Q4 encoder moved renders by LPIPS 0.15, Q8 by 0.06.- sd_cpp_text_encoders = (("unsloth/Qwen3-4B-GGUF", "Qwen3-4B-Q8_0.gguf", "llm"),),+ # own GGUF of it. BF16, the precision the bf16 encoder had and ComfyUI's klein template loads: Q8_0 moved+ # renders by LPIPS 0.06 and Q4_K_M by 0.15 against an A/A of exactly 0. UNSLOTH_SD_CPP_TEXT_ENCODER_QUANT+ # trades it down.+ sd_cpp_text_encoders = (("unsloth/Qwen3-4B-GGUF", "Qwen3-4B-BF16.gguf", "llm"),),),# FLUX.3 Image Klein: a 5.6B flow-map DiT with a Qwen3-VL-4B encoder (layers 4..32 stacked) and the FLUX.2 VAE.# Diffusers ships it only as modular blocks, so diffusion_flux3 assembles it per component behind a classic@@ -677,11 +679,14 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (sd_cpp_vae = ("black-forest-labs/FLUX.2-dev", "vae/diffusion_pytorch_model.safetensors"),sd_cpp_vae_format = "flux2",# FLUX.2-dev's text encoder is Mistral-Small-3.2-24B-Instruct-2506 (byte-identical shards to the unsloth- # copy; the BF16 GGUF matches every text tensor), so sd-cli reads unsloth's GGUF of it.+ # copy; the BF16 GGUF matches every text tensor), so sd-cli reads unsloth's GGUF of it. Q8_0, not Q4_K_M: at+ # least as precise as ComfyUI's fp8 template (its default is a 36 GB bf16 file) and the int8 the diffusers route+ # takes, at 25 GB. The GGUF's q/k rows are in llama.cpp's interleaved RoPE order, which is the order sd.cpp's+ # GGML_ROPE_TYPE_NORMAL Mistral path expects; the HF-order bf16 safetensors this replaced did not match it.sd_cpp_text_encoders = (("unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF",- "Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf",+ "Mistral-Small-3.2-24B-Instruct-2506-Q8_0.gguf","llm",),),@@ -1122,9 +1127,9 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (# Z-Image decodes with the FLUX.1 autoencoder: BFL's ae.safetensors, byte-identical to the repack's file.sd_cpp_vae = ("black-forest-labs/FLUX.1-schnell", "ae.safetensors"),# Z-Image's encoder differs from stock Qwen3-4B only in the LAST layer's MLP, which neither diffusers- # (hidden_states[-2]) nor sd.cpp (out_layers {35}) reads, so the conditioning is the same. Same Q8_0 file+ # (hidden_states[-2]) nor sd.cpp (out_layers {35}) reads, so the conditioning is the same. Same BF16 file# klein 4B reads, so one download serves both.- sd_cpp_text_encoders = (("unsloth/Qwen3-4B-GGUF", "Qwen3-4B-Q8_0.gguf", "llm"),),+ sd_cpp_text_encoders = (("unsloth/Qwen3-4B-GGUF", "Qwen3-4B-BF16.gguf", "llm"),),),DiffusionFamily(name = "krea-2",@@ -2958,8 +2963,42 @@ def family_sd_cpp_supported(fam: DiffusionFamily) -> bool:# FLUX.2-klein 9B pairs with Qwen3-8B, the 4B with the family-default Qwen3-4B (a mismatched encoder fails deep in-# sd-cli), so pick per variant. Klein 9B's text encoder is byte-identical to unsloth/Qwen3-8B.-_FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS = (("unsloth/Qwen3-8B-GGUF", "Qwen3-8B-Q4_K_M.gguf", "llm"),)+# sd-cli), so pick per variant. Klein 9B's text encoder is byte-identical to unsloth/Qwen3-8B; BF16 is the precision+# the released bf16 encoder had (and renders pixel-identical to it).+_FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS = (("unsloth/Qwen3-8B-GGUF", "Qwen3-8B-BF16.gguf", "llm"),)++# Opt-in lower-precision sd.cpp text encoder for hosts short on RAM / VRAM: bf16, q8_0 or q4_k_m. Unset keeps each+# family's table entry. Only the unsloth LLM GGUF encoders follow it (every one of those repos ships all three files);+# official safetensors encoders (CLIP / T5 / vision projectors) are left alone.+SD_CPP_TEXT_ENCODER_QUANT_ENV = "UNSLOTH_SD_CPP_TEXT_ENCODER_QUANT"+_SD_CPP_TEXT_ENCODER_QUANTS = {"bf16": "BF16", "q8_0": "Q8_0", "q4_k_m": "Q4_K_M"}+_SD_CPP_LLM_GGUF_QUANT_RE = re.compile(r"-(BF16|F16|Q8_0|Q4_K_M)\.gguf$")+_SD_CPP_REQUANTABLE_TE_REPOS = frozenset(+ {+ "unsloth/qwen3-4b-gguf",+ "unsloth/qwen3-8b-gguf",+ "unsloth/mistral-small-3.2-24b-instruct-2506-gguf",+ }+)+++def sd_cpp_text_encoder_quant() -> Optional[str]:+ """The ``UNSLOTH_SD_CPP_TEXT_ENCODER_QUANT`` file tag (``BF16`` / ``Q8_0`` / ``Q4_K_M``), else None."""+ raw = (os.environ.get(SD_CPP_TEXT_ENCODER_QUANT_ENV) or "").strip().lower()+ return _SD_CPP_TEXT_ENCODER_QUANTS.get(raw)+++def _with_text_encoder_quant(+ encoders: tuple[tuple[str, str, str], ...], tag: Optional[str]+) -> tuple[tuple[str, str, str], ...]:+ if not tag:+ return encoders+ out = []+ for repo, filename, kind in encoders:+ if kind == "llm" and repo.lower() in _SD_CPP_REQUANTABLE_TE_REPOS:+ filename = _SD_CPP_LLM_GGUF_QUANT_RE.sub(f"-{tag}.gguf", filename)+ out.append((repo, filename, kind))+ return tuple(out)# sd-cli companion repos released Studio builds fetched and this one no longer reads: the unsloth copies of the@@ -3098,6 +3137,12 @@ def sd_cpp_text_encoder_candidates(fam: DiffusionFamily) -> tuple[tuple[str, strsets = [fam.sd_cpp_text_encoders]if fam.name == "flux.2-klein":sets.append(_FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS)+ # Every precision UNSLOTH_SD_CPP_TEXT_ENCODER_QUANT can pick: a load made under another setting fetched that file.+ sets = [+ _with_text_encoder_quant(group or (), tag)+ for group in sets+ for tag in (None, *_SD_CPP_TEXT_ENCODER_QUANTS.values())+ ]return tuple(dict.fromkeys(entry for group in sets for entry in group or ()))@@ -3117,16 +3162,17 @@ def sd_cpp_text_encoders_for(if fam.name == "flux.2-klein":# A dim this family does not have (a FLUX.2-dev file, a misread) falls through to the strings rather than# guessing, exactly as the base-size guard fails open on one.+ tag = sd_cpp_text_encoder_quant()if inner_dim == _FLUX2_KLEIN_9B_INNER_DIM:- return _FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS+ return _with_text_encoder_quant(_FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS, tag)if inner_dim == _FLUX2_KLEIN_4B_INNER_DIM:- return fam.sd_cpp_text_encoders+ return _with_text_encoder_quant(fam.sd_cpp_text_encoders, tag)identity = f"{repo_id or ''}/{gguf_filename or ''}".lower()# Match the size token on its own: klein-BASE-9B is 9B too, and matching "klein-9b" literally handed it the 4B# text encoder.if _token_in_needle("9b", identity) or "klein9b" in identity:- return _FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS- return fam.sd_cpp_text_encoders+ return _with_text_encoder_quant(_FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS, tag)+ return _with_text_encoder_quant(fam.sd_cpp_text_encoders, sd_cpp_text_encoder_quant())# FLUX.2 sizes the adaLN modulation projection as (6 * inner_dim, inner_dim), and inner_dim is the only thing thatdiff --git a/studio/backend/tests/test_sd_cpp_text_encoder_precision.py b/studio/backend/tests/test_sd_cpp_text_encoder_precision.pynew file mode 100644index 00000000000..93f0f41e670--- /dev/null+++ b/studio/backend/tests/test_sd_cpp_text_encoder_precision.py@@ -0,0 +1,79 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""sd.cpp text-encoder precision: the default keeps the precision the released bf16 encoders had (and ComfyUI's+templates load) wherever a quant measurably moves the render, and UNSLOTH_SD_CPP_TEXT_ENCODER_QUANT trades it down."""++import pytest++from core.inference import diffusion_families as df+++def _fam(name):+ return next(f for f in df._FAMILIES if f.name == name)+++def _llm(encoders):+ return [(repo, filename) for repo, filename, kind in encoders if kind == "llm"]++[email protected](autouse = True)+def _no_env(monkeypatch):+ monkeypatch.delenv("UNSLOTH_SD_CPP_TEXT_ENCODER_QUANT", raising = False)+++def test_klein_4b_and_z_image_default_to_the_bf16_encoder():+ for name in ("flux.2-klein", "z-image"):+ fam = _fam(name)+ assert _llm(+ df.sd_cpp_text_encoders_for(fam, "unsloth/FLUX.2-klein-4B-GGUF", "x-4b-Q8_0.gguf")+ ) == [("unsloth/Qwen3-4B-GGUF", "Qwen3-4B-BF16.gguf")], name+++def test_klein_9b_defaults_to_the_bf16_encoder():+ fam = _fam("flux.2-klein")+ for kwargs in ({"inner_dim": 4096}, {"gguf_filename": "flux-2-klein-9b-Q4_K_M.gguf"}):+ got = df.sd_cpp_text_encoders_for(fam, "unsloth/FLUX.2-klein-9B-GGUF", **kwargs)+ assert _llm(got) == [("unsloth/Qwen3-8B-GGUF", "Qwen3-8B-BF16.gguf")], kwargs+++def test_flux2_dev_never_defaults_below_8_bit():+ fam = _fam("flux.2-dev")+ files = [f for _r, f in _llm(df.sd_cpp_text_encoders_for(fam))]+ assert files == ["Mistral-Small-3.2-24B-Instruct-2506-Q8_0.gguf"]+++ "value, tag",+ [("q8_0", "Q8_0"), ("Q4_K_M", "Q4_K_M"), ("bf16", "BF16"), ("bogus", None), ("", None)],+)+def test_env_trades_the_llm_encoder_down(monkeypatch, value, tag):+ monkeypatch.setenv("UNSLOTH_SD_CPP_TEXT_ENCODER_QUANT", value)+ fam = _fam("flux.2-klein")+ want4 = f"Qwen3-4B-{tag or 'BF16'}.gguf"+ want9 = f"Qwen3-8B-{tag or 'BF16'}.gguf"+ assert _llm(df.sd_cpp_text_encoders_for(fam, inner_dim = 3072)) == [+ ("unsloth/Qwen3-4B-GGUF", want4)+ ]+ assert _llm(df.sd_cpp_text_encoders_for(fam, inner_dim = 4096)) == [+ ("unsloth/Qwen3-8B-GGUF", want9)+ ]+ dev = [f for _r, f in _llm(df.sd_cpp_text_encoders_for(_fam("flux.2-dev")))]+ assert dev == [f"Mistral-Small-3.2-24B-Instruct-2506-{tag or 'Q8_0'}.gguf"]+++def test_env_leaves_other_encoders_alone(monkeypatch):+ """Only the unsloth LLM GGUFs that ship every precision are rewritten: the FLUX.1 CLIP / T5 and the Qwen-Image+ VL encoder with its projector keep their table entries."""+ monkeypatch.setenv("UNSLOTH_SD_CPP_TEXT_ENCODER_QUANT", "q4_k_m")+ for fam in df._FAMILIES:+ if fam.name in ("flux.2-klein", "flux.2-dev", "z-image"):+ continue+ assert df.sd_cpp_text_encoders_for(fam) == tuple(fam.sd_cpp_text_encoders or ()), fam.name+++def test_delete_guard_candidates_cover_every_precision():+ """A load made under another UNSLOTH_SD_CPP_TEXT_ENCODER_QUANT fetched that file, so the guard names them all."""+ cands = {f for _r, f, _k in df.sd_cpp_text_encoder_candidates(_fam("flux.2-klein"))}+ for tag in ("BF16", "Q8_0", "Q4_K_M"):+ assert f"Qwen3-4B-{tag}.gguf" in cands and f"Qwen3-8B-{tag}.gguf" in cands