Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: launch q4_0 instead of iq4_nl KV cache on CUDA/HIP GPUs (#13330)
Daniel Hancaa9fbd0acc1
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex d221a8bf640..8b61e5f74c1 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -4355,7 +4355,8 @@ def _kv_bytes_per_elem(cache_type: Optional[str]) -> float:# Not a CUDA/HIP FlashAttention KV type (ggml_cuda_fattn_kv_type_supported), so attention runs# on the CPU. q4_1 / q5_0 / q5_1 did too until ggml-org/llama.cpp#28079 added an f16 fallback.-_GPU_UNACCELERATED_CACHE_TYPES = frozenset({"iq4_nl"})+# Mapped to a same-size type that has the kernel, so the KV budget is unchanged (#10336).+_GPU_UNACCELERATED_CACHE_TYPES = {"iq4_nl": "q4_0"}def _kv_cache_gpu_fallback_warning(@@ -4373,8 +4374,8 @@ def _kv_cache_gpu_fallback_warning(return Nonereturn (f"KV cache type {cache_type_kv} has no CUDA/HIP flash-attention kernel, so attention "- "falls back to CPU, causing high CPU load and slow generation. "- "q8_0 or q4_0 is a GPU-accelerated quantized option."+ "would fall back to CPU, causing high CPU load and slow generation. Using "+ f"{_GPU_UNACCELERATED_CACHE_TYPES[cache_type_kv]} instead, the same size on the GPU.")@@ -29092,15 +29093,12 @@ class LlamaCppBackend:and cache_type_kv in _valid_cache_typesand not _cache_type_from_env):- if not _extra_args_set_any_flag(extra_args, _CACHE_TYPE_K_FLAGS):- cmd.extend(["--cache-type-k", cache_type_kv])- if not _extra_args_set_any_flag(extra_args, _CACHE_TYPE_V_FLAGS):- cmd.extend(["--cache-type-v", cache_type_kv])- self._cache_type_kv = cache_type_kv- logger.info(f"KV cache type: {cache_type_kv}")gpu_fallback_warning = (None- if intent.cpu_fallback or _extras_cache is not None+ if intent.cpu_fallback+ or _arch_gate_forced_cpu+ or not _detected_gpus+ or _extras_cache is not Noneelse _kv_cache_gpu_fallback_warning(cache_type_kv,gpu_memory_mode,@@ -29110,6 +29108,13 @@ class LlamaCppBackend:)if gpu_fallback_warning:logger.warning(gpu_fallback_warning)+ cache_type_kv = _GPU_UNACCELERATED_CACHE_TYPES[cache_type_kv]+ if not _extra_args_set_any_flag(extra_args, _CACHE_TYPE_K_FLAGS):+ cmd.extend(["--cache-type-k", cache_type_kv])+ if not _extra_args_set_any_flag(extra_args, _CACHE_TYPE_V_FLAGS):+ cmd.extend(["--cache-type-v", cache_type_kv])+ self._cache_type_kv = cache_type_kv+ logger.info(f"KV cache type: {cache_type_kv}")else:# An env-only type is left inherited (untouched) so an# asymmetric K/V env reaches the child as set.diff --git a/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py b/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.pyindex 0489f8ff556..65a13f4a319 100644--- a/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py+++ b/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py@@ -58,21 +58,41 @@ def _write_minimal_gguf(path: Path) -> str:@pytest.mark.parametrize(- "cache_type,extra_args,env_cache,expected_warning",+ "cache_type,extra_args,env_cache,expected_warning,launched",[- ("iq4_nl", None, None, ("iq4_nl", "auto", -1, True)),- ("IQ4_NL", None, None, ("iq4_nl", "auto", -1, True)),- ("q8_0", None, None, ("q8_0", "auto", -1, False)),- ("iq4_nl", ["--cache-type-k", "q8_0"], None, None),- (None, None, "f32", None),+ # A ROCm build with no kernels for the GPU runs on the CPU, where iq4_nl is fine.+ ("iq4_nl", "arch_gated", None, None, ("iq4_nl", "iq4_nl")),+ # A CUDA build that sees no GPU (masked, driver down) also runs on the CPU.+ ("iq4_nl", "no_gpu", None, None, ("iq4_nl", "iq4_nl")),+ ("iq4_nl", None, None, ("iq4_nl", "auto", -1, True), ("q4_0", "q4_0")),+ ("IQ4_NL", None, None, ("iq4_nl", "auto", -1, True), ("q4_0", "q4_0")),+ ("q8_0", None, None, ("q8_0", "auto", -1, False), ("q8_0", "q8_0")),+ ("iq4_nl", ["--cache-type-k", "q8_0"], None, None, ("q8_0", "q8_0")),+ (None, None, "f32", None, (None, None)),],)def test_load_path_warns_only_for_its_own_cache_type(- monkeypatch, tmp_path, cache_type, extra_args, env_cache, expected_warning+ monkeypatch, tmp_path, cache_type, extra_args, env_cache, expected_warning, launched):backend = llama_cpp.LlamaCppBackend()gguf = _write_minimal_gguf(tmp_path / "model.gguf")backend._get_gpu_memory = lambda _binary = None, **_kw: [(0, 10_000, 16_000)]+ arch_gated = extra_args == "arch_gated"+ if extra_args == "no_gpu":+ extra_args = None+ backend._get_gpu_memory = lambda _binary = None, **_kw: []+ if arch_gated:+ extra_args = None+ backend._get_gpu_memory = lambda _binary = None, for_llama_server = False, **_kw: (+ [] if for_llama_server else [(0, 10_000, 16_000)]+ )+ backend._host_torch_is_rocm = lambda: True+ monkeypatch.setattr(+ llama_cpp.LlamaCppBackend, "_installed_llama_gfx_archs", lambda *_a: ("gfx1100",)+ )+ monkeypatch.setattr(+ llama_cpp.LlamaCppBackend, "_rocm_arch_by_physical_id", lambda *_a: {0: "gfx906"}+ )backend._get_gpu_free_memory = lambda _binary = None, **_kw: [(0, 10_000)]backend._read_gguf_metadata = lambda _path: Nonebackend._can_estimate_kv = lambda: False@@ -84,7 +104,9 @@ def test_load_path_warns_only_for_its_own_cache_type(backend._amd_apu_wants_unified_memory = lambda *args, **kwargs: Falsebackend._find_llama_server_binary = lambda **_kwargs: "/fake/llama-server"backend._is_vulkan_backend = lambda _binary = None: False- backend._installed_ggml_backends = lambda _binary = None: frozenset({"cuda"})+ backend._installed_ggml_backends = lambda _binary = None: frozenset(+ {"hip"} if arch_gated else {"cuda"}+ )backend._wait_for_health = lambda *_a, **_kw: Truebackend._detect_audio_type_strict = lambda: Nonebackend._apply_detected_audio = lambda _detected: True@@ -104,10 +126,12 @@ def test_load_path_warns_only_for_its_own_cache_type(monkeypatch.setattr(llama_cpp, "_kv_cache_gpu_fallback_warning", record)real_popen = subprocess.Popen+ argv = []def fake_popen(cmd, **kwargs):if not cmd or str(cmd[0]) != "/fake/llama-server":return real_popen(cmd, **kwargs)+ argv[:] = [str(a) for a in cmd]return type("Process",(),@@ -138,3 +162,9 @@ def test_load_path_warns_only_for_its_own_cache_type(assert warnings == []else:assert warnings == [expected_warning]++ def last(flag):+ idx = [i for i, a in enumerate(argv) if a == flag]+ return argv[idx[-1] + 1] if idx else None++ assert (last("--cache-type-k"), last("--cache-type-v")) == launched