Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: hide iq4_nl KV cache where llama.cpp runs its attention on the CPU (#7008)
Rod Boev4ccad39a1938
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 52f3d8b6458..4f6073318c9 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -4312,6 +4312,31 @@ def _kv_bytes_per_elem(cache_type: Optional[str]) -> float:}.get((cache_type or "f16").strip().lower(), 2.0)+# Not a CUDA/HIP FlashAttention KV type (ggml_cuda_fattn_kv_type_supported), so attention runs+# on the CPU. q4_1 / q5_0 / q5_1 did too until ggml-org/llama.cpp#28079 added an f16 fallback.+_GPU_UNACCELERATED_CACHE_TYPES = frozenset({"iq4_nl"})+++def _kv_cache_gpu_fallback_warning(+ cache_type_kv: Optional[str],+ gpu_memory_mode: str,+ gpu_layers: int,+ installed_backends: Callable[[], frozenset[str]],+) -> Optional[str]:+ """Warning for a KV cache type that runs attention on the CPU of a CUDA/HIP launch."""+ if cache_type_kv not in _GPU_UNACCELERATED_CACHE_TYPES:+ return None+ if gpu_memory_mode == "manual" and gpu_layers == 0:+ return None+ if not {"cuda", "hip"} & installed_backends():+ return None+ return (+ f"KV cache type {cache_type_kv} has no CUDA/HIP flash-attention kernel, so attention "+ "falls back to CPU, causing high CPU load and slow generation. "+ "q8_0 or q4_0 is a GPU-accelerated quantized option."+ )++# Upper bound on any current tokenizer, for output rows when a truncated header drops# the token array. Above Llama 4 / Gemma 3 (256k), the widest shipping._ASSUMED_MAX_VOCAB = 262144@@ -27980,6 +28005,18 @@ class LlamaCppBackend:cmd.extend(["--cache-type-v", cache_type_kv])self._cache_type_kv = cache_type_kvlogger.info(f"KV cache type: {cache_type_kv}")+ gpu_fallback_warning = (+ None+ if intent.cpu_fallback or _extras_cache is not None+ else _kv_cache_gpu_fallback_warning(+ cache_type_kv,+ gpu_memory_mode,+ gpu_layers,+ lambda: self._installed_ggml_backends(binary),+ )+ )+ if gpu_fallback_warning:+ logger.warning(gpu_fallback_warning)else:# An env-only type is left inherited (untouched) so an# asymmetric K/V env reaches the child as set.diff --git a/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py b/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.pynew file mode 100644index 00000000000..0489f8ff556--- /dev/null+++ b/studio/backend/tests/test_llama_cpp_kv_cache_gpu_warning.py@@ -0,0 +1,140 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Production-path contract tests for the GGUF KV-cache advisory."""++from __future__ import annotations++import struct+import subprocess+from pathlib import Path+from unittest.mock import patch++import pytest++from core.inference import llama_cpp+++# q4_1 / q5_0 / q5_1 run on the GPU since ggml-org/llama.cpp#28079.+SAFE = ("f16", "bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "f32", None)+GPU = (frozenset({"cpu", "cuda"}), frozenset({"cpu", "hip"}))+NO_GPU = (frozenset(), frozenset({"cpu"}), frozenset({"cpu", "vulkan"}))+++def _warning(cache_type, mode, layers, backends):+ return llama_cpp._kv_cache_gpu_fallback_warning(cache_type, mode, layers, lambda: backends)++[email protected]("backends", GPU)[email protected]("mode,layers", [("auto", -1), ("manual", -1), ("manual", 1)])+def test_iq4_nl_warns_on_gpu_placement(backends, mode, layers):+ assert "iq4_nl" in _warning("iq4_nl", mode, layers, backends)++[email protected]("cache_type", SAFE)[email protected]("mode,layers", [("auto", -1), ("manual", 0), ("manual", 1)])+def test_other_types_are_silent_and_skip_the_probe(cache_type, mode, layers):+ def probe():+ raise AssertionError("backend probe ran for a type that cannot warn")++ assert llama_cpp._kv_cache_gpu_fallback_warning(cache_type, mode, layers, probe) is None++[email protected]("backends", NO_GPU)+def test_iq4_nl_is_silent_without_cuda_or_hip(backends):+ assert _warning("iq4_nl", "auto", -1, backends) is None++[email protected]("backends", GPU)+def test_manual_zero_layers_is_silent(backends):+ assert _warning("iq4_nl", "manual", 0, backends) is None+++def _write_minimal_gguf(path: Path) -> str:+ key = b"general.architecture"+ value = b"llama"+ metadata = struct.pack("<Q", len(key)) + key+ metadata += struct.pack("<I", 8) + struct.pack("<Q", len(value)) + value+ path.write_bytes(struct.pack("<IIQQ", 0x46554747, 3, 0, 1) + metadata)+ return str(path)+++ "cache_type,extra_args,env_cache,expected_warning",+ [+ ("iq4_nl", None, None, ("iq4_nl", "auto", -1, True)),+ ("IQ4_NL", None, None, ("iq4_nl", "auto", -1, True)),+ ("q8_0", None, None, ("q8_0", "auto", -1, False)),+ ("iq4_nl", ["--cache-type-k", "q8_0"], None, None),+ (None, None, "f32", None),+ ],+)+def test_load_path_warns_only_for_its_own_cache_type(+ monkeypatch, tmp_path, cache_type, extra_args, env_cache, expected_warning+):+ backend = llama_cpp.LlamaCppBackend()+ gguf = _write_minimal_gguf(tmp_path / "model.gguf")+ backend._get_gpu_memory = lambda _binary = None, **_kw: [(0, 10_000, 16_000)]+ backend._get_gpu_free_memory = lambda _binary = None, **_kw: [(0, 10_000)]+ backend._read_gguf_metadata = lambda _path: None+ backend._can_estimate_kv = lambda: False+ backend._get_gguf_size_bytes = lambda _path: 1024+ backend._mmproj_vram_bytes = lambda _path: 0+ backend._resolve_launch_mmproj_path = lambda **_kwargs: None+ backend._apu_ram_shortfall_message = lambda *args, **kwargs: None+ backend._launch_host_shortfall_message = lambda *args, **kwargs: None+ backend._amd_apu_wants_unified_memory = lambda *args, **kwargs: False+ backend._find_llama_server_binary = lambda **_kwargs: "/fake/llama-server"+ backend._is_vulkan_backend = lambda _binary = None: False+ backend._installed_ggml_backends = lambda _binary = None: frozenset({"cuda"})+ backend._wait_for_health = lambda *_a, **_kw: True+ backend._detect_audio_type_strict = lambda: None+ backend._apply_detected_audio = lambda _detected: True+ if env_cache is None:+ monkeypatch.delenv("LLAMA_ARG_CACHE_TYPE_K", raising = False)+ else:+ monkeypatch.setenv("LLAMA_ARG_CACHE_TYPE_K", env_cache)++ warnings = []+ real_warning = llama_cpp._kv_cache_gpu_fallback_warning++ def record(cache_type_kv, mode, layers, installed_backends):+ result = real_warning(cache_type_kv, mode, layers, installed_backends)+ warnings.append((cache_type_kv, mode, layers, result is not None))+ return result++ monkeypatch.setattr(llama_cpp, "_kv_cache_gpu_fallback_warning", record)++ real_popen = subprocess.Popen++ def fake_popen(cmd, **kwargs):+ if not cmd or str(cmd[0]) != "/fake/llama-server":+ return real_popen(cmd, **kwargs)+ return type(+ "Process",+ (),+ {+ "pid": 123,+ "stdout": (),+ "poll": lambda self: None,+ "terminate": lambda self: None,+ "wait": lambda self, timeout = None: 0,+ "kill": lambda self: None,+ },+ )()++ with patch.object(subprocess, "Popen", side_effect = fake_popen):+ assert (+ backend.load_model(+ llama_cpp.GgufLoadIntent(+ gguf_path = gguf,+ model_identifier = "test",+ cache_type_kv = cache_type,+ extra_args = extra_args,+ )+ )+ is True+ )++ if expected_warning is None:+ assert warnings == []+ else:+ assert warnings == [expected_warning]diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex 22b1e92c47f..22c7b996030 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -2,6 +2,7 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import { InferenceEnginePicker } from "./inference-engines";+import { useLlamaCppBackend } from "@/hooks/use-llama-backend";import { Button } from "@/components/ui/button";import { Checkbox } from "@/components/ui/checkbox";import { InfoHint } from "@/components/ui/info-hint";@@ -147,7 +148,7 @@ import {DEFAULT_MAX_SEQ_LENGTH,DEFAULT_PER_MODEL_CONFIG,DRAFT_N_MAX_SPEC_TYPES,- KV_CACHE_DTYPES,+ kvCacheDtypeOptions,LOAD_MODES,LOAD_MODE_DEFAULT,MAX_SEQ_LENGTH_MAX,@@ -1349,6 +1350,7 @@ function GgufAdvancedSettings({}) {const batchAdviceId = useId();const ubatchAdviceId = useId();+ const llamaBackend = useLlamaCppBackend();// llama-server aborts below 2 and below the slot count, so the loader raises the emitted value// to max(slots, 2). Surfaced so the number typed here is not silently different from the// one that runs. With Slots blank only the hard floor of 2 is asserted.@@ -1395,7 +1397,7 @@ function GgufAdvancedSettings({<SelectItem value={KV_CACHE_DTYPE_DEFAULT}>{KV_CACHE_DTYPE_DEFAULT}</SelectItem>- {KV_CACHE_DTYPES.map((dtype) => (+ {kvCacheDtypeOptions(llamaBackend, config.kvCacheDtype).map((dtype) => (<SelectItem key={dtype} value={dtype}>{dtype}</SelectItem>@@ -1511,7 +1513,10 @@ function GgufAdvancedSettings({<SelectItem value={KV_CACHE_DTYPE_DEFAULT}>{KV_CACHE_DTYPE_DEFAULT}</SelectItem>- {KV_CACHE_DTYPES.map((dtype) => (+ {kvCacheDtypeOptions(+ llamaBackend,+ config.specDraftCacheDtype,+ ).map((dtype) => (<SelectItem key={dtype} value={dtype}>{dtype}</SelectItem>diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.tsindex 4fd53e57b57..a24a4d37801 100644--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts@@ -301,6 +301,16 @@ export const KV_CACHE_DTYPES = ["f32",] as const;+/** Menu entries for a llama.cpp backend. CUDA, ROCm and Metal have no iq4_nl FlashAttention kernel, so attention+ * runs on the CPU there; Vulkan and CPU builds run it. A selected value stays listed so the trigger can show it. */+export function kvCacheDtypeOptions(+ backend: string | null,+ selected: string | null | undefined,+): readonly string[] {+ if (backend === "vulkan" || backend === "cpu") return KV_CACHE_DTYPES;+ return KV_CACHE_DTYPES.filter((dtype) => dtype !== "iq4_nl" || dtype === selected);+}+export const MLX_KV_QUANTS = ["8","6",diff --git a/studio/frontend/src/hooks/use-llama-backend.ts b/studio/frontend/src/hooks/use-llama-backend.tsnew file mode 100644index 00000000000..3d162a894e2--- /dev/null+++ b/studio/frontend/src/hooks/use-llama-backend.ts@@ -0,0 +1,23 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { useEffect, useState } from "react";++import { loadLlamaBackendStatus } from "@/features/settings/api/llama-backend";++/** The llama.cpp backend the install runs on; null until known or when the status call fails. */+export function useLlamaCppBackend(): string | null {+ const [backend, setBackend] = useState<string | null>(null);+ useEffect(() => {+ let cancelled = false;+ loadLlamaBackendStatus()+ .then((status) => {+ if (!cancelled) setBackend(status.backend);+ })+ .catch(() => {});+ return () => {+ cancelled = true;+ };+ }, []);+ return backend;+}diff --git a/studio/frontend/tests/kv-cache-dtype-options.test.ts b/studio/frontend/tests/kv-cache-dtype-options.test.tsnew file mode 100644index 00000000000..9f85c487da4--- /dev/null+++ b/studio/frontend/tests/kv-cache-dtype-options.test.ts@@ -0,0 +1,27 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import { registerBundlerResolver } from "./helpers/kit.ts";++registerBundlerResolver();+const { kvCacheDtypeOptions } = await import(+ "../src/features/model-picker/model-config/per-model-config.ts"+);++test("iq4_nl is offered only where llama.cpp runs its FlashAttention", () => {+ for (const backend of ["vulkan", "cpu"]) {+ assert.ok(kvCacheDtypeOptions(backend, null).includes("iq4_nl"));+ }+ for (const backend of ["cuda", "rocm", "metal", null]) {+ const options = kvCacheDtypeOptions(backend, null);+ assert.ok(!options.includes("iq4_nl"));+ assert.deepEqual(options, ["bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "f32"]);+ }+});++test("a selected iq4_nl stays listed so the trigger can show it", () => {+ assert.ok(kvCacheDtypeOptions("cuda", "iq4_nl").includes("iq4_nl"));+});