Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: launch q4_0 instead of iq4_nl KV cache on CUDA/HIP GPUs (#13330)

Daniel Hancaa9fbd0acc1
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index d221a8bf640..8b61e5f74c1 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -4355,7 +4355,8 @@ def _kv_bytes_per_elem(cache_type: Optional[str]) -> float:
# Not a CUDA/HIP FlashAttention KV type (ggml_cuda_fattn_kv_type_supported), so attention runs
# on the CPU. q4_1 / q5_0 / q5_1 did too until ggml-org/llama.cpp#28079 added an f16 fallback.
-_GPU_UNACCELERATED_CACHE_TYPES = frozenset({"iq4_nl"})
+# Mapped to a same-size type that has the kernel, so the KV budget is unchanged (#10336).
+_GPU_UNACCELERATED_CACHE_TYPES = {"iq4_nl": "q4_0"}
def _kv_cache_gpu_fallback_warning(
@@ -4373,8 +4374,8 @@ def _kv_cache_gpu_fallback_warning(
return None
return (
f"KV cache type {cache_type_kv} has no CUDA/HIP flash-attention kernel, so attention "
- "falls back to CPU, causing high CPU load and slow generation. "
- "q8_0 or q4_0 is a GPU-accelerated quantized option."
+ "would fall back to CPU, causing high CPU load and slow generation. Using "
+ f"{_GPU_UNACCELERATED_CACHE_TYPES[cache_type_kv]} instead, the same size on the GPU."
)
@@ -29092,15 +29093,12 @@ class LlamaCppBackend:
and cache_type_kv in _valid_cache_types
and not _cache_type_from_env
):
- if not _extra_args_set_any_flag(extra_args, _CACHE_TYPE_K_FLAGS):
- cmd.extend(["--cache-type-k", cache_type_kv])
- if not _extra_args_set_any_flag(extra_args, _CACHE_TYPE_V_FLAGS):
- cmd.extend(["--cache-type-v", cache_type_kv])
- self._cache_type_kv = cache_type_kv
- logger.info(f"KV cache type: {cache_type_kv}")
gpu_fallback_warning = (
None
- if intent.cpu_fallback or _extras_cache is not None
+ if intent.cpu_fallback
+ or _arch_gate_forced_cpu
+ or not _detected_gpus
+ or _extras_cache is not None
else _kv_cache_gpu_fallback_warning(
cache_type_kv,
gpu_memory_mode,
@@ -29110,6 +29108,13 @@ class LlamaCppBackend:
)
if gpu_fallback_warning:
logger.warning(gpu_fallback_warning)
+ cache_type_kv = _GPU_UNACCELERATED_CACHE_TYPES[cache_type_kv]
+ if not _extra_args_set_any_flag(extra_args, _CACHE_TYPE_K_FLAGS):
+ cmd.extend(["--cache-type-k", cache_type_kv])
+ if not _extra_args_set_any_flag(extra_args, _CACHE_TYPE_V_FLAGS):
+ cmd.extend(["--cache-type-v", cache_type_kv])
+ self._cache_type_kv = cache_type_kv
+ logger.info(f"KV cache type: {cache_type_kv}")
else:
# An env-only type is left inherited (untouched) so an
# asymmetric K/V env reaches the child as set.
diff --git a/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py b/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py
index 0489f8ff556..65a13f4a319 100644
--- a/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py
+++ b/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py
@@ -58,21 +58,41 @@ def _write_minimal_gguf(path: Path) -> str:
@pytest.mark.parametrize(
- "cache_type,extra_args,env_cache,expected_warning",
+ "cache_type,extra_args,env_cache,expected_warning,launched",
[
- ("iq4_nl", None, None, ("iq4_nl", "auto", -1, True)),
- ("IQ4_NL", None, None, ("iq4_nl", "auto", -1, True)),
- ("q8_0", None, None, ("q8_0", "auto", -1, False)),
- ("iq4_nl", ["--cache-type-k", "q8_0"], None, None),
- (None, None, "f32", None),
+ # A ROCm build with no kernels for the GPU runs on the CPU, where iq4_nl is fine.
+ ("iq4_nl", "arch_gated", None, None, ("iq4_nl", "iq4_nl")),
+ # A CUDA build that sees no GPU (masked, driver down) also runs on the CPU.
+ ("iq4_nl", "no_gpu", None, None, ("iq4_nl", "iq4_nl")),
+ ("iq4_nl", None, None, ("iq4_nl", "auto", -1, True), ("q4_0", "q4_0")),
+ ("IQ4_NL", None, None, ("iq4_nl", "auto", -1, True), ("q4_0", "q4_0")),
+ ("q8_0", None, None, ("q8_0", "auto", -1, False), ("q8_0", "q8_0")),
+ ("iq4_nl", ["--cache-type-k", "q8_0"], None, None, ("q8_0", "q8_0")),
+ (None, None, "f32", None, (None, None)),
],
)
def test_load_path_warns_only_for_its_own_cache_type(
- monkeypatch, tmp_path, cache_type, extra_args, env_cache, expected_warning
+ monkeypatch, tmp_path, cache_type, extra_args, env_cache, expected_warning, launched
):
backend = llama_cpp.LlamaCppBackend()
gguf = _write_minimal_gguf(tmp_path / "model.gguf")
backend._get_gpu_memory = lambda _binary = None, **_kw: [(0, 10_000, 16_000)]
+ arch_gated = extra_args == "arch_gated"
+ if extra_args == "no_gpu":
+ extra_args = None
+ backend._get_gpu_memory = lambda _binary = None, **_kw: []
+ if arch_gated:
+ extra_args = None
+ backend._get_gpu_memory = lambda _binary = None, for_llama_server = False, **_kw: (
+ [] if for_llama_server else [(0, 10_000, 16_000)]
+ )
+ backend._host_torch_is_rocm = lambda: True
+ monkeypatch.setattr(
+ llama_cpp.LlamaCppBackend, "_installed_llama_gfx_archs", lambda *_a: ("gfx1100",)
+ )
+ monkeypatch.setattr(
+ llama_cpp.LlamaCppBackend, "_rocm_arch_by_physical_id", lambda *_a: {0: "gfx906"}
+ )
backend._get_gpu_free_memory = lambda _binary = None, **_kw: [(0, 10_000)]
backend._read_gguf_metadata = lambda _path: None
backend._can_estimate_kv = lambda: False
@@ -84,7 +104,9 @@ def test_load_path_warns_only_for_its_own_cache_type(
backend._amd_apu_wants_unified_memory = lambda *args, **kwargs: False
backend._find_llama_server_binary = lambda **_kwargs: "/fake/llama-server"
backend._is_vulkan_backend = lambda _binary = None: False
- backend._installed_ggml_backends = lambda _binary = None: frozenset({"cuda"})
+ backend._installed_ggml_backends = lambda _binary = None: frozenset(
+ {"hip"} if arch_gated else {"cuda"}
+ )
backend._wait_for_health = lambda *_a, **_kw: True
backend._detect_audio_type_strict = lambda: None
backend._apply_detected_audio = lambda _detected: True
@@ -104,10 +126,12 @@ def test_load_path_warns_only_for_its_own_cache_type(
monkeypatch.setattr(llama_cpp, "_kv_cache_gpu_fallback_warning", record)
real_popen = subprocess.Popen
+ argv = []
def fake_popen(cmd, **kwargs):
if not cmd or str(cmd[0]) != "/fake/llama-server":
return real_popen(cmd, **kwargs)
+ argv[:] = [str(a) for a in cmd]
return type(
"Process",
(),
@@ -138,3 +162,9 @@ def test_load_path_warns_only_for_its_own_cache_type(
assert warnings == []
else:
assert warnings == [expected_warning]
+
+ def last(flag):
+ idx = [i for i, a in enumerate(argv) if a == flag]
+ return argv[idx[-1] + 1] if idx else None
+
+ assert (last("--cache-type-k"), last("--cache-type-v")) == launched