Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: hide iq4_nl KV cache where llama.cpp runs its attention on the CPU (#7008)

Rod Boev4ccad39a1938
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 52f3d8b6458..4f6073318c9 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -4312,6 +4312,31 @@ def _kv_bytes_per_elem(cache_type: Optional[str]) -> float:
}.get((cache_type or "f16").strip().lower(), 2.0)
+# Not a CUDA/HIP FlashAttention KV type (ggml_cuda_fattn_kv_type_supported), so attention runs
+# on the CPU. q4_1 / q5_0 / q5_1 did too until ggml-org/llama.cpp#28079 added an f16 fallback.
+_GPU_UNACCELERATED_CACHE_TYPES = frozenset({"iq4_nl"})
+
+
+def _kv_cache_gpu_fallback_warning(
+ cache_type_kv: Optional[str],
+ gpu_memory_mode: str,
+ gpu_layers: int,
+ installed_backends: Callable[[], frozenset[str]],
+) -> Optional[str]:
+ """Warning for a KV cache type that runs attention on the CPU of a CUDA/HIP launch."""
+ if cache_type_kv not in _GPU_UNACCELERATED_CACHE_TYPES:
+ return None
+ if gpu_memory_mode == "manual" and gpu_layers == 0:
+ return None
+ if not {"cuda", "hip"} & installed_backends():
+ return None
+ return (
+ f"KV cache type {cache_type_kv} has no CUDA/HIP flash-attention kernel, so attention "
+ "falls back to CPU, causing high CPU load and slow generation. "
+ "q8_0 or q4_0 is a GPU-accelerated quantized option."
+ )
+
+
# Upper bound on any current tokenizer, for output rows when a truncated header drops
# the token array. Above Llama 4 / Gemma 3 (256k), the widest shipping.
_ASSUMED_MAX_VOCAB = 262144
@@ -27980,6 +28005,18 @@ class LlamaCppBackend:
cmd.extend(["--cache-type-v", cache_type_kv])
self._cache_type_kv = cache_type_kv
logger.info(f"KV cache type: {cache_type_kv}")
+ gpu_fallback_warning = (
+ None
+ if intent.cpu_fallback or _extras_cache is not None
+ else _kv_cache_gpu_fallback_warning(
+ cache_type_kv,
+ gpu_memory_mode,
+ gpu_layers,
+ lambda: self._installed_ggml_backends(binary),
+ )
+ )
+ if gpu_fallback_warning:
+ logger.warning(gpu_fallback_warning)
else:
# An env-only type is left inherited (untouched) so an
# asymmetric K/V env reaches the child as set.
diff --git a/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py b/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py
new file mode 100644
index 00000000000..0489f8ff556
--- /dev/null
+++ b/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py
@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Production-path contract tests for the GGUF KV-cache advisory."""
+
+from __future__ import annotations
+
+import struct
+import subprocess
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from core.inference import llama_cpp
+
+
+# q4_1 / q5_0 / q5_1 run on the GPU since ggml-org/llama.cpp#28079.
+SAFE = ("f16", "bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "f32", None)
+GPU = (frozenset({"cpu", "cuda"}), frozenset({"cpu", "hip"}))
+NO_GPU = (frozenset(), frozenset({"cpu"}), frozenset({"cpu", "vulkan"}))
+
+
+def _warning(cache_type, mode, layers, backends):
+ return llama_cpp._kv_cache_gpu_fallback_warning(cache_type, mode, layers, lambda: backends)
+
+
[email protected]("backends", GPU)
[email protected]("mode,layers", [("auto", -1), ("manual", -1), ("manual", 1)])
+def test_iq4_nl_warns_on_gpu_placement(backends, mode, layers):
+ assert "iq4_nl" in _warning("iq4_nl", mode, layers, backends)
+
+
[email protected]("cache_type", SAFE)
[email protected]("mode,layers", [("auto", -1), ("manual", 0), ("manual", 1)])
+def test_other_types_are_silent_and_skip_the_probe(cache_type, mode, layers):
+ def probe():
+ raise AssertionError("backend probe ran for a type that cannot warn")
+
+ assert llama_cpp._kv_cache_gpu_fallback_warning(cache_type, mode, layers, probe) is None
+
+
[email protected]("backends", NO_GPU)
+def test_iq4_nl_is_silent_without_cuda_or_hip(backends):
+ assert _warning("iq4_nl", "auto", -1, backends) is None
+
+
[email protected]("backends", GPU)
+def test_manual_zero_layers_is_silent(backends):
+ assert _warning("iq4_nl", "manual", 0, backends) is None
+
+
+def _write_minimal_gguf(path: Path) -> str:
+ key = b"general.architecture"
+ value = b"llama"
+ metadata = struct.pack("<Q", len(key)) + key
+ metadata += struct.pack("<I", 8) + struct.pack("<Q", len(value)) + value
+ path.write_bytes(struct.pack("<IIQQ", 0x46554747, 3, 0, 1) + metadata)
+ return str(path)
+
+
+ "cache_type,extra_args,env_cache,expected_warning",
+ [
+ ("iq4_nl", None, None, ("iq4_nl", "auto", -1, True)),
+ ("IQ4_NL", None, None, ("iq4_nl", "auto", -1, True)),
+ ("q8_0", None, None, ("q8_0", "auto", -1, False)),
+ ("iq4_nl", ["--cache-type-k", "q8_0"], None, None),
+ (None, None, "f32", None),
+ ],
+)
+def test_load_path_warns_only_for_its_own_cache_type(
+ monkeypatch, tmp_path, cache_type, extra_args, env_cache, expected_warning
+):
+ backend = llama_cpp.LlamaCppBackend()
+ gguf = _write_minimal_gguf(tmp_path / "model.gguf")
+ backend._get_gpu_memory = lambda _binary = None, **_kw: [(0, 10_000, 16_000)]
+ backend._get_gpu_free_memory = lambda _binary = None, **_kw: [(0, 10_000)]
+ backend._read_gguf_metadata = lambda _path: None
+ backend._can_estimate_kv = lambda: False
+ backend._get_gguf_size_bytes = lambda _path: 1024
+ backend._mmproj_vram_bytes = lambda _path: 0
+ backend._resolve_launch_mmproj_path = lambda **_kwargs: None
+ backend._apu_ram_shortfall_message = lambda *args, **kwargs: None
+ backend._launch_host_shortfall_message = lambda *args, **kwargs: None
+ backend._amd_apu_wants_unified_memory = lambda *args, **kwargs: False
+ backend._find_llama_server_binary = lambda **_kwargs: "/fake/llama-server"
+ backend._is_vulkan_backend = lambda _binary = None: False
+ backend._installed_ggml_backends = lambda _binary = None: frozenset({"cuda"})
+ backend._wait_for_health = lambda *_a, **_kw: True
+ backend._detect_audio_type_strict = lambda: None
+ backend._apply_detected_audio = lambda _detected: True
+ if env_cache is None:
+ monkeypatch.delenv("LLAMA_ARG_CACHE_TYPE_K", raising = False)
+ else:
+ monkeypatch.setenv("LLAMA_ARG_CACHE_TYPE_K", env_cache)
+
+ warnings = []
+ real_warning = llama_cpp._kv_cache_gpu_fallback_warning
+
+ def record(cache_type_kv, mode, layers, installed_backends):
+ result = real_warning(cache_type_kv, mode, layers, installed_backends)
+ warnings.append((cache_type_kv, mode, layers, result is not None))
+ return result
+
+ monkeypatch.setattr(llama_cpp, "_kv_cache_gpu_fallback_warning", record)
+
+ real_popen = subprocess.Popen
+
+ def fake_popen(cmd, **kwargs):
+ if not cmd or str(cmd[0]) != "/fake/llama-server":
+ return real_popen(cmd, **kwargs)
+ return type(
+ "Process",
+ (),
+ {
+ "pid": 123,
+ "stdout": (),
+ "poll": lambda self: None,
+ "terminate": lambda self: None,
+ "wait": lambda self, timeout = None: 0,
+ "kill": lambda self: None,
+ },
+ )()
+
+ with patch.object(subprocess, "Popen", side_effect = fake_popen):
+ assert (
+ backend.load_model(
+ llama_cpp.GgufLoadIntent(
+ gguf_path = gguf,
+ model_identifier = "test",
+ cache_type_kv = cache_type,
+ extra_args = extra_args,
+ )
+ )
+ is True
+ )
+
+ if expected_warning is None:
+ assert warnings == []
+ else:
+ assert warnings == [expected_warning]
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index 22b1e92c47f..22c7b996030 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { InferenceEnginePicker } from "./inference-engines";
+import { useLlamaCppBackend } from "@/hooks/use-llama-backend";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { InfoHint } from "@/components/ui/info-hint";
@@ -147,7 +148,7 @@ import {
DEFAULT_MAX_SEQ_LENGTH,
DEFAULT_PER_MODEL_CONFIG,
DRAFT_N_MAX_SPEC_TYPES,
- KV_CACHE_DTYPES,
+ kvCacheDtypeOptions,
LOAD_MODES,
LOAD_MODE_DEFAULT,
MAX_SEQ_LENGTH_MAX,
@@ -1349,6 +1350,7 @@ function GgufAdvancedSettings({
}) {
const batchAdviceId = useId();
const ubatchAdviceId = useId();
+ const llamaBackend = useLlamaCppBackend();
// llama-server aborts below 2 and below the slot count, so the loader raises the emitted value
// to max(slots, 2). Surfaced so the number typed here is not silently different from the
// one that runs. With Slots blank only the hard floor of 2 is asserted.
@@ -1395,7 +1397,7 @@ function GgufAdvancedSettings({
<SelectItem value={KV_CACHE_DTYPE_DEFAULT}>
{KV_CACHE_DTYPE_DEFAULT}
</SelectItem>
- {KV_CACHE_DTYPES.map((dtype) => (
+ {kvCacheDtypeOptions(llamaBackend, config.kvCacheDtype).map((dtype) => (
<SelectItem key={dtype} value={dtype}>
{dtype}
</SelectItem>
@@ -1511,7 +1513,10 @@ function GgufAdvancedSettings({
<SelectItem value={KV_CACHE_DTYPE_DEFAULT}>
{KV_CACHE_DTYPE_DEFAULT}
</SelectItem>
- {KV_CACHE_DTYPES.map((dtype) => (
+ {kvCacheDtypeOptions(
+ llamaBackend,
+ config.specDraftCacheDtype,
+ ).map((dtype) => (
<SelectItem key={dtype} value={dtype}>
{dtype}
</SelectItem>
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index 4fd53e57b57..a24a4d37801 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -301,6 +301,16 @@ export const KV_CACHE_DTYPES = [
"f32",
] as const;
+/** Menu entries for a llama.cpp backend. CUDA, ROCm and Metal have no iq4_nl FlashAttention kernel, so attention
+ * runs on the CPU there; Vulkan and CPU builds run it. A selected value stays listed so the trigger can show it. */
+export function kvCacheDtypeOptions(
+ backend: string | null,
+ selected: string | null | undefined,
+): readonly string[] {
+ if (backend === "vulkan" || backend === "cpu") return KV_CACHE_DTYPES;
+ return KV_CACHE_DTYPES.filter((dtype) => dtype !== "iq4_nl" || dtype === selected);
+}
+
export const MLX_KV_QUANTS = [
"8",
"6",
diff --git a/studio/frontend/src/hooks/use-llama-backend.ts b/studio/frontend/src/hooks/use-llama-backend.ts
new file mode 100644
index 00000000000..3d162a894e2
--- /dev/null
+++ b/studio/frontend/src/hooks/use-llama-backend.ts
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { useEffect, useState } from "react";
+
+import { loadLlamaBackendStatus } from "@/features/settings/api/llama-backend";
+
+/** The llama.cpp backend the install runs on; null until known or when the status call fails. */
+export function useLlamaCppBackend(): string | null {
+ const [backend, setBackend] = useState<string | null>(null);
+ useEffect(() => {
+ let cancelled = false;
+ loadLlamaBackendStatus()
+ .then((status) => {
+ if (!cancelled) setBackend(status.backend);
+ })
+ .catch(() => {});
+ return () => {
+ cancelled = true;
+ };
+ }, []);
+ return backend;
+}
diff --git a/studio/frontend/tests/kv-cache-dtype-options.test.ts b/studio/frontend/tests/kv-cache-dtype-options.test.ts
new file mode 100644
index 00000000000..9f85c487da4
--- /dev/null
+++ b/studio/frontend/tests/kv-cache-dtype-options.test.ts
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+const { kvCacheDtypeOptions } = await import(
+ "../src/features/model-picker/model-config/per-model-config.ts"
+);
+
+test("iq4_nl is offered only where llama.cpp runs its FlashAttention", () => {
+ for (const backend of ["vulkan", "cpu"]) {
+ assert.ok(kvCacheDtypeOptions(backend, null).includes("iq4_nl"));
+ }
+ for (const backend of ["cuda", "rocm", "metal", null]) {
+ const options = kvCacheDtypeOptions(backend, null);
+ assert.ok(!options.includes("iq4_nl"));
+ assert.deepEqual(options, ["bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "f32"]);
+ }
+});
+
+test("a selected iq4_nl stays listed so the trigger can show it", () => {
+ assert.ok(kvCacheDtypeOptions("cuda", "iq4_nl").includes("iq4_nl"));
+});