Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: cap context checkpoints by host RAM for sliding-window and SSM models (#11918)

Long Yixinge8ec65fd4c50
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 9d4619e71a9..626c6b79b26 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -4914,6 +4914,12 @@ _TARGET_ROLLBACK_SPEC_TYPES = frozenset(
)
+# Mirrors llama.cpp is_recr_impl for SSM headers with no full_attention_interval.
+_SSM_EVERY_LAYER_ARCHS = frozenset({"mamba", "mamba2", "falcon-h1"})
+# Nemotron-H also zeroes head_count_kv on its MLP/MoE layers; its SSM layers have no FFN.
+_SSM_FFN_FREE_LAYER_ARCHS = frozenset({"nemotron_h", "nemotron_h_moe"})
+
+
# Architectures whose TARGET context leaves the embedded MTP/NextN blocks out of
# its KV cache, so block_count must lose them before the target is priced.
#
@@ -7261,6 +7267,7 @@ class LlamaCppBackend:
# For the compute-graph buffer estimate; vocab from the tokens array len.
# feed_forward_length is the widest layer when the GGUF stores one per layer.
self._feed_forward_length: Optional[int] = None
+ self._feed_forward_length_by_layer: Optional[list[int]] = None
self._expert_used_count: Optional[int] = None
self._expert_feed_forward_length: Optional[int] = None
self._expert_shared_feed_forward_length: Optional[int] = None
@@ -15851,14 +15858,41 @@ class LlamaCppBackend:
n_parallel
)
+ def _ctx_checkpoint_bytes(
+ self,
+ cache_type_kv: Optional[str] = None,
+ *,
+ swa_full: bool = False,
+ flash_attn: bool = True,
+ ) -> int:
+ """Host bytes of one slot's context checkpoint: the recurrent state, else the SWA window."""
+ recurrent = self._rollback_state_bytes(1)
+ if recurrent > 0:
+ return recurrent
+
+ def kv(checkpoints: int) -> int:
+ return self._estimate_kv_cache_bytes(
+ 1,
+ cache_type_kv,
+ swa_full = swa_full,
+ ctx_checkpoints = checkpoints,
+ flash_attn = flash_attn,
+ )
+
+ return max(0, kv(1) - kv(0))
+
def _bounded_ctx_checkpoints(
self,
n_parallel: int,
server_caps: Mapping[str, object],
extra_args: Optional[Iterable[str]] = None,
env: Optional[Mapping[str, str]] = None,
+ *,
+ cache_type_kv: Optional[str] = None,
+ swa_full: bool = False,
+ flash_attn: bool = True,
) -> Optional[int]:
- """Return an automatic recurrent-checkpoint cap, or None to preserve the argv."""
+ """Return an automatic context-checkpoint cap, or None to preserve the argv."""
flag = server_caps.get("ctx_checkpoints_flag")
if not flag:
return None
@@ -15867,7 +15901,9 @@ class LlamaCppBackend:
return None
if _env_ctx_checkpoints_override(env) is not None:
return None
- per_checkpoint = self._rollback_state_bytes(1)
+ per_checkpoint = self._ctx_checkpoint_bytes(
+ cache_type_kv, swa_full = swa_full, flash_attn = flash_attn
+ )
if per_checkpoint <= 0:
return None
total_ram_mib = self._host_memory_capacity_mib()
@@ -15883,7 +15919,7 @@ class LlamaCppBackend:
return None
logger.info(
"Capping llama-server context checkpoints at %d per slot (this build's default %d): "
- "each one snapshots this model's whole recurrent state (%.1f MiB), so the default "
+ "each one snapshots %.1f MiB of this model's state, so the default "
"would hold %.1f GiB of host RAM across %d slot(s).",
bounded,
upstream_default,
@@ -15915,6 +15951,25 @@ class LlamaCppBackend:
arch = getattr(self, "_architecture", None)
return bool(arch) and str(arch).strip().lower() in _TARGET_KV_EXCLUDES_NEXTN_ARCHS
+ def _ssm_recurrent_layer_count(self, n_layers: int) -> int:
+ """SSM layers of a header with no attention interval, by llama.cpp's is_recr rules."""
+ arch = getattr(self, "_architecture", None)
+ if arch in _SSM_EVERY_LAYER_ARCHS:
+ return n_layers
+ heads = getattr(self, "_n_kv_heads_by_layer", None)
+ if not heads or getattr(self, "_kda_head_dim", None):
+ return 0
+ ffn = (
+ getattr(self, "_feed_forward_length_by_layer", None)
+ if arch in _SSM_FFN_FREE_LAYER_ARCHS
+ else None
+ ) or []
+ return sum(
+ 1
+ for i, n_kv in enumerate(heads[:n_layers])
+ if n_kv == 0 and not (i < len(ffn) and ffn[i] > 0)
+ )
+
def _mamba_recurrent_state_bytes(
self,
n_parallel: int = 1,
@@ -15933,17 +15988,16 @@ class LlamaCppBackend:
n_group_raw = getattr(self, "_ssm_group_count", None)
d_conv_raw = getattr(self, "_ssm_conv_kernel", None)
fai_raw = getattr(self, "_full_attention_interval", None)
+ # Mamba1 (Jamba, Mamba) writes no group count; llama.cpp reads it as 0.
if not all(
value is not None
for value in (
n_layers_raw,
d_inner_raw,
d_state_raw,
- n_group_raw,
d_conv_raw,
- fai_raw,
)
- ):
+ ) or (fai_raw is not None and n_group_raw is None):
return 0
# Excludes the embedded MTP blocks, as _estimate_kv_cache_bytes does and
# for the same reason: llama.cpp keeps them out of the target's n_layer.
@@ -15951,9 +16005,12 @@ class LlamaCppBackend:
0,
int(n_layers_raw or 0) - int(getattr(self, "_nextn_predict_layers", None) or 0),
)
- fai = int(fai_raw or 0)
- n_attn = -(-n_layers // fai) if fai > 0 else n_layers
- n_recurrent = max(0, n_layers - n_attn)
+ if fai_raw is not None:
+ fai = int(fai_raw or 0)
+ n_attn = -(-n_layers // fai) if fai > 0 else n_layers
+ n_recurrent = max(0, n_layers - n_attn)
+ else:
+ n_recurrent = self._ssm_recurrent_layer_count(n_layers)
if n_recurrent == 0:
return 0
d_inner = int(d_inner_raw or 0)
@@ -16180,6 +16237,8 @@ class LlamaCppBackend:
)
return int(global_bytes + swa_bytes + slots * checkpoint_extra_per_slot)
+ ssm_state = self._mamba_recurrent_state_bytes(n_parallel) + recurrent_checkpoints
+
# Path 4: Standard GQA with explicit key/value dimensions
if key_len is not None and val_len is not None:
padded_v_width = None if flash_attn else self._max_kv_value_width(val_len)
@@ -16188,11 +16247,16 @@ class LlamaCppBackend:
layer_n_kv = self._kv_heads_for_layer(layer_idx, n_kv)
v_width = layer_n_kv * val_len if padded_v_width is None else padded_v_width
bytes_per_cell += layer_n_kv * key_len * bpe_k + v_width * bpe_v
- return int(total_cells * bytes_per_cell)
+ return int(total_cells * bytes_per_cell) + ssm_state
# Path 5: Legacy fallback (old GGUFs without explicit dimensions)
head_dim = self._legacy_head_dim()
- return int(2 * n_kv * head_dim * n_layers_kv * total_cells * bpe_k)
+ # llama.cpp defaults head_count_kv to head_count, so a pure SSM (head_count 0) has no KV.
+ layer_default = self._n_kv_heads if self._n_kv_heads is not None else (self._n_heads or 0)
+ heads = [self._kv_heads_for_layer(i, layer_default) for i in range(n_layers_kv)]
+ # Without flash attention V is padded to the widest layer, as on Path 4.
+ v_heads = sum(heads) if flash_attn else len(heads) * max(heads)
+ return int((sum(heads) * bpe_k + v_heads * bpe_v) * head_dim * total_cells) + ssm_state
def _draft_backend_for(self, drafter_path: str) -> Optional["LlamaCppBackend"]:
"""Lightweight backend with a drafter GGUF's metadata, to size its own KV
@@ -17541,6 +17605,7 @@ class LlamaCppBackend:
self._pooling_type = None
self._gguf_path = gguf_path
self._feed_forward_length = None
+ self._feed_forward_length_by_layer = None
self._expert_used_count = None
self._expert_feed_forward_length = None
self._expert_shared_feed_forward_length = None
@@ -17734,7 +17799,10 @@ class LlamaCppBackend:
self._sliding_window_pattern = [bool(x) for x in val_a]
sliding_window_pattern_period = None
elif attr == "feed_forward_length" and val_a:
- self._feed_forward_length = max(int(x) for x in val_a)
+ self._feed_forward_length_by_layer = [int(x) for x in val_a]
+ self._feed_forward_length = max(
+ self._feed_forward_length_by_layer
+ )
else:
self._gguf_skip_value(f, vtype)
else:
@@ -23378,7 +23446,9 @@ class LlamaCppBackend:
server_caps,
extra_args,
ctx_checkpoints,
- per_checkpoint_bytes = self._rollback_state_bytes(1),
+ per_checkpoint_bytes = self._ctx_checkpoint_bytes(
+ cache_type_kv, swa_full = swa_full, flash_attn = planned_flash_attn
+ ),
n_parallel = n_parallel,
total_host_bytes = ((self._host_memory_capacity_mib() or 0) * 1024 * 1024)
or None,
@@ -27334,7 +27404,14 @@ class LlamaCppBackend:
def _decide_auto_ctx_checkpoints() -> Optional[int]:
"""The cap for the slots n_parallel currently names, or None to stand down."""
return (
- self._bounded_ctx_checkpoints(n_parallel, server_caps, extra_args)
+ self._bounded_ctx_checkpoints(
+ n_parallel,
+ server_caps,
+ extra_args,
+ cache_type_kv = cache_type_kv,
+ swa_full = swa_full,
+ flash_attn = planned_flash_attn,
+ )
if ctx_checkpoints is None
else None
)
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index 6cc80435e68..caadada73d7 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -39,7 +39,7 @@ CACHE_RAM_MAX_MIB = 1024 * 1024
# llama.cpp allocates this default even when Studio emits no flag.
LLAMA_CTX_CHECKPOINTS_DEFAULT = 32
-# Recurrent checkpoints live in host RAM and can be much larger than SWA snapshots.
+# Checkpoints live in host RAM: a hybrid's whole recurrent state, or an SWA model's window.
CTX_CHECKPOINT_HOST_BUDGET_FRACTION = 0.05
CTX_CHECKPOINT_HOST_BUDGET_FLOOR_BYTES = 1024**3
# Keep rollback available; zero forces a full prompt re-ingest after divergence.
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index e7c87b66d4f..423e0ca445a 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -11495,23 +11495,6 @@ def _gguf_runtime_bytes(
llama_extra_args,
default = managed_kv_unified,
)
- # Resolve the same capability-dependent count that load_model emits.
- _cc_caps: dict = {}
- try:
- _cc_caps = LlamaCppBackend.probe_server_capabilities() or {}
- except Exception as _cc_exc:
- logger.debug("ctx-checkpoints capability probe failed: %s", _cc_exc)
- # Older lightweight probes may not provide the new sizing helpers.
- _per_checkpoint = getattr(probe, "_rollback_state_bytes", lambda _n: 0)(1)
- _host_mib = getattr(probe, "_host_memory_capacity_mib", lambda: None)()
- _resolved_checkpoints = effective_ctx_checkpoints_for_caps(
- _cc_caps,
- llama_extra_args,
- ctx_checkpoints,
- per_checkpoint_bytes = _per_checkpoint,
- n_parallel = slots,
- total_host_bytes = (_host_mib * 1024 * 1024) if _host_mib else None,
- )
# load_model appends --flash-attn on to every launch whose build has the flag,
# so the default here is ON, not off. The false arm pads variable-width V
# tensors to the model-wide maximum (_max_kv_value_width), which on an
@@ -11521,7 +11504,7 @@ def _gguf_runtime_bytes(
_fa_supported = True
try:
_fa_caps = LlamaCppBackend.probe_server_capabilities()
- # Same ``found`` test as the checkpoint probe above: the defaults dict
+ # Same ``found`` test as the checkpoint probe below: the defaults dict
# reports nothing supported, so keying on the flag alone would size every
# unprobed host as flash-attention-less.
if _fa_caps.get("found") and not _fa_caps.get("supports_flash_attn", True):
@@ -11552,6 +11535,23 @@ def _gguf_runtime_bytes(
tensor_parallel = bool(tensor_parallel),
env = os.environ,
)
+ _cc_caps: dict = {}
+ try:
+ _cc_caps = LlamaCppBackend.probe_server_capabilities() or {}
+ except Exception as _cc_exc:
+ logger.debug("ctx-checkpoints capability probe failed: %s", _cc_exc)
+ _per_checkpoint = getattr(probe, "_ctx_checkpoint_bytes", lambda *_a, **_k: 0)(
+ cache_type_for_budget, swa_full = swa_full, flash_attn = flash_attn
+ )
+ _host_mib = getattr(probe, "_host_memory_capacity_mib", lambda: None)()
+ _resolved_checkpoints = effective_ctx_checkpoints_for_caps(
+ _cc_caps,
+ llama_extra_args,
+ ctx_checkpoints,
+ per_checkpoint_bytes = _per_checkpoint,
+ n_parallel = slots,
+ total_host_bytes = (_host_mib * 1024 * 1024) if _host_mib else None,
+ )
kv = probe._estimate_kv_cache_bytes(
ctx,
cache_type_for_budget,
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index f0d8c1ba410..073cfbb4cc1 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -4115,7 +4115,11 @@ async def get_kv_cache_estimate(
_cc_caps,
None,
ctx_checkpoints,
- per_checkpoint_bytes = getattr(be, "_rollback_state_bytes", lambda _n: 0)(1),
+ per_checkpoint_bytes = getattr(be, "_ctx_checkpoint_bytes", lambda *_a, **_k: 0)(
+ _effective_cache_type,
+ swa_full = _plan_kwargs.get("swa_full", False),
+ flash_attn = _plan_kwargs.get("flash_attn", True),
+ ),
n_parallel = n_parallel,
total_host_bytes = (_total_ram_mib * 1024 * 1024) if _total_ram_mib else None,
)
diff --git a/studio/backend/tests/test_ctx_checkpoint_host_budget.py b/studio/backend/tests/test_ctx_checkpoint_host_budget.py
index 52398e1f4fe..3183a5b9ade 100644
--- a/studio/backend/tests/test_ctx_checkpoint_host_budget.py
+++ b/studio/backend/tests/test_ctx_checkpoint_host_budget.py
@@ -437,7 +437,17 @@ class TestCheckpointsNeverReachAVramFigure:
source = inspect.getsource(LlamaCppBackend.load_model)
# One rule, re-asked wherever the slot count changes, never a second spelling.
assert source.count("def _decide_auto_ctx_checkpoints(") == 1
- assert "self._bounded_ctx_checkpoints(n_parallel, server_caps, extra_args)" in source
+ compact = "".join(source.split())
+ sized = (
+ "self._ctx_checkpoint_bytes(cache_type_kv,swa_full=swa_full,"
+ "flash_attn=planned_flash_attn)"
+ )
+ assert f"per_checkpoint_bytes={sized}" in compact
+ assert (
+ "self._bounded_ctx_checkpoints(n_parallel,server_caps,extra_args,"
+ "cache_type_kv=cache_type_kv,swa_full=swa_full,flash_attn=planned_flash_attn,)"
+ ) in compact
+ assert "per_checkpoint_bytes=self._rollback_state_bytes" not in compact
assert source.count("self._bounded_ctx_checkpoints(") == 1
assert "str(_auto_ctx_checkpoints)" in source
@@ -1077,3 +1087,182 @@ class TestTheGuardOnlyDropsHostBytesFromADiscretePool:
discrete = inference_routes._estimate_gguf_kv_gb("x.gguf", 4096, shared_memory_pool = False)
assert shared == 11.0 and discrete == 7.0
assert shared > discrete, "the shared pool must not be credited the host share"
+
+
+_GEMMA4_31B_SWA = [(i + 1) % 6 != 0 for i in range(60)]
+_GEMMA4_31B = {
+ "context_length": 262144,
+ "block_count": 60,
+ "embedding_length": 5376,
+ "attention.head_count": 32,
+ "attention.head_count_kv": [16 if swa else 4 for swa in _GEMMA4_31B_SWA],
+ "attention.key_length": 512,
+ "attention.value_length": 512,
+ "attention.key_length_swa": 256,
+ "attention.value_length_swa": 256,
+ "attention.sliding_window": 1024,
+ "attention.sliding_window_pattern": _GEMMA4_31B_SWA,
+}
+GEMMA4_SNAPSHOT = 800 * MIB
+
+
+def gemma4_gguf(tmp_path, monkeypatch):
+ import importlib.util
+ from pathlib import Path
+
+ spec = importlib.util.spec_from_file_location(
+ "_kv_cache_estimation_for_checkpoints",
+ Path(__file__).parent / "test_kv_cache_estimation.py",
+ )
+ helpers = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(helpers)
+ kv = {"general.architecture": "gemma4"}
+ kv.update({f"gemma4.{k}": v for k, v in _GEMMA4_31B.items()})
+ path = tmp_path / "gemma-4-31B-it-Q6_K.gguf"
+ path.write_bytes(helpers._make_gguf_bytes("gemma4", kv))
+ monkeypatch.delenv("LLAMA_ARG_SWA_FULL", raising = False)
+ monkeypatch.delenv("LLAMA_ARG_CTX_SIZE", raising = False)
+ monkeypatch.setattr(
+ LlamaCppBackend, "_total_system_memory_mib", staticmethod(lambda: 94 * 1024)
+ )
+ caps = {**_caps(), "supports_kv_unified": True, "supports_flash_attn": True}
+ monkeypatch.setattr(
+ LlamaCppBackend, "probe_server_capabilities", classmethod(lambda cls, *a, **k: caps)
+ )
+ return path
+
+
+def _gemma4_backend(path):
+ b = LlamaCppBackend()
+ b._read_gguf_metadata(str(path))
+ return b
+
+
+def test_an_swa_snapshot_is_its_window(gemma4_gguf):
+ b = _gemma4_backend(gemma4_gguf)
+ assert b._rollback_state_bytes(1) == 0
+ assert b._ctx_checkpoint_bytes("f16") == GEMMA4_SNAPSHOT
+ assert b._ctx_checkpoint_bytes("q8_0") < GEMMA4_SNAPSHOT
+ assert b._ctx_checkpoint_bytes("f16", swa_full = True) == 0
+ assert _plain_attention_backend()._ctx_checkpoint_bytes("f16") == 0
+ assert _backend()._ctx_checkpoint_bytes("f16") == _backend()._rollback_state_bytes(1)
+
+
+def test_the_launcher_bounds_an_swa_load(gemma4_gguf):
+ b = _gemma4_backend(gemma4_gguf)
+ assert b._bounded_ctx_checkpoints(1, _caps(), cache_type_kv = "f16") == 6
+ assert b._bounded_ctx_checkpoints(4, _caps(), cache_type_kv = "f16") == (
+ CTX_CHECKPOINTS_MIN_USEFUL
+ )
+ assert b._bounded_ctx_checkpoints(4, _caps(), cache_type_kv = "f16", swa_full = True) is None
+ assert b._bounded_ctx_checkpoints(1, _caps(), cache_type_kv = "q8_0", flash_attn = False) < (
+ b._bounded_ctx_checkpoints(1, _caps(), cache_type_kv = "q8_0")
+ )
+
+
[email protected]("flash_attn", [None, False], ids = ["managed", "fa-off"])
+def test_both_estimates_price_the_count_the_launcher_emits(gemma4_gguf, monkeypatch, flash_attn):
+ """The reported load: Auto context, f16, four slots. Uncapped this was 100 GiB of host RAM."""
+ import asyncio
+
+ from routes import inference as inference_routes
+ from routes import models as models_routes
+
+ sized_with = []
+ snapshot = LlamaCppBackend._ctx_checkpoint_bytes
+
+ def spy(self, *args, **kwargs):
+ sized_with.append(kwargs.get("flash_attn"))
+ return snapshot(self, *args, **kwargs)
+
+ monkeypatch.setattr(LlamaCppBackend, "_ctx_checkpoint_bytes", spy)
+ emitted = _gemma4_backend(gemma4_gguf)._bounded_ctx_checkpoints(4, _caps(), cache_type_kv = "f16")
+ expected = 4 * emitted * GEMMA4_SNAPSHOT
+
+ extras = None if flash_attn is None else ["--flash-attn", "off"]
+ sized_with.clear()
+ panel = inference_routes._gguf_runtime_bytes(str(gemma4_gguf), 0, extras, 4, "f16", False)
+ assert panel.n_ctx == 262144
+ assert panel.kv_checkpoint_bytes == expected
+
+ monkeypatch.setattr(
+ models_routes, "_resolve_quant_gguf", lambda *_a: (str(gemma4_gguf), 25 * GIB)
+ )
+ hub = asyncio.run(
+ models_routes.get_kv_cache_estimate(
+ repo_id = "unsloth/gemma-4-31B-it-GGUF",
+ quant = "Q6_K",
+ n_ctx = 262144,
+ cache_type_kv = "f16",
+ n_parallel = 4,
+ speculative_type = None,
+ spec_draft_n_max = None,
+ spec_draft_cache_type = None,
+ ctx_checkpoints = None,
+ disable_vision = True,
+ n_batch = None,
+ n_ubatch = None,
+ tensor_parallel = False,
+ flash_attn = flash_attn,
+ kv_unified = None,
+ swa_full = None,
+ no_mmproj_offload = None,
+ request = None,
+ current_subject = "test",
+ )
+ )
+ assert hub["kv_checkpoint_bytes"] == expected
+ assert set(sized_with) == {flash_attn is None}
+
+
+_GRANITE_H_SMALL = {
+ "context_length": 1048576,
+ "block_count": 40,
+ "embedding_length": 4096,
+ "attention.head_count": 32,
+ "attention.head_count_kv": [8 if i % 10 == 5 else 0 for i in range(40)],
+ "ssm.conv_kernel": 4,
+ "ssm.state_size": 128,
+ "ssm.group_count": 1,
+ "ssm.inner_size": 8192,
+}
+GRANITE_SNAPSHOT = 36 * (3 * (8192 + 2 * 128) + 128 * 8192) * 4
+
+
+def granite_gguf(tmp_path, monkeypatch):
+ import importlib.util
+ from pathlib import Path
+
+ spec = importlib.util.spec_from_file_location(
+ "_kv_cache_estimation_for_checkpoints",
+ Path(__file__).parent / "test_kv_cache_estimation.py",
+ )
+ helpers = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(helpers)
+ kv = {"general.architecture": "granitehybrid"}
+ kv.update({f"granitehybrid.{k}": v for k, v in _GRANITE_H_SMALL.items()})
+ path = tmp_path / "granite-4.0-h-small-Q4_K_M.gguf"
+ path.write_bytes(helpers._make_gguf_bytes("granitehybrid", kv))
+ monkeypatch.delenv("LLAMA_ARG_CTX_SIZE", raising = False)
+ monkeypatch.setattr(
+ LlamaCppBackend, "_total_system_memory_mib", staticmethod(lambda: 94 * 1024)
+ )
+ caps = {**_caps(), "supports_kv_unified": True, "supports_flash_attn": True}
+ monkeypatch.setattr(
+ LlamaCppBackend, "probe_server_capabilities", classmethod(lambda cls, *a, **k: caps)
+ )
+ return path
+
+
+def test_an_interval_free_hybrid_is_capped_and_priced(granite_gguf):
+ from routes import inference as inference_routes
+
+ b = LlamaCppBackend()
+ b._read_gguf_metadata(str(granite_gguf))
+ assert b._rollback_state_bytes(1) == GRANITE_SNAPSHOT
+ emitted = b._bounded_ctx_checkpoints(4, _caps())
+ assert emitted == 8
+ panel = inference_routes._gguf_runtime_bytes(str(granite_gguf), 32768, None, 4, "f16", False)
+ assert panel.kv_checkpoint_bytes == 4 * emitted * GRANITE_SNAPSHOT
diff --git a/studio/backend/tests/test_kv_cache_estimation.py b/studio/backend/tests/test_kv_cache_estimation.py
index b23b8d45dc3..53553624edc 100644
--- a/studio/backend/tests/test_kv_cache_estimation.py
+++ b/studio/backend/tests/test_kv_cache_estimation.py
@@ -990,6 +990,104 @@ class TestHybridMambaEstimation:
assert b._estimate_kv_cache_bytes(4096, "f16", n_parallel = 4) == (kv_only + 4 * per_slot)
+class TestSsmStateWithoutAttentionInterval:
+ """SSM headers with no full_attention_interval, recurrent layers per llama.cpp's is_recr."""
+
+ _SSM = {"ssm.inner_size": 1024, "ssm.state_size": 16, "ssm.conv_kernel": 4}
+
+ @staticmethod
+ def _per_layer(group):
+ return ((4 - 1) * (1024 + 2 * group * 16) + 16 * 1024) * 4
+
+ @pytest.mark.parametrize(
+ "arch, fields, recurrent",
+ [
+ pytest.param(
+ "granitehybrid",
+ {"attention.head_count_kv": [0, 0, 8, 0], "ssm.group_count": 1},
+ 3,
+ id = "zero-kv-heads",
+ ),
+ pytest.param(
+ "nemotron_h",
+ {
+ "attention.head_count_kv": [0, 0, 8, 0],
+ "feed_forward_length": [0, 21504, 0, 0],
+ "ssm.group_count": 8,
+ },
+ 2,
+ id = "dense-nemotron-mlp-layers-are-not-ssm",
+ ),
+ pytest.param(
+ "nemotron_h_moe",
+ {
+ "attention.head_count_kv": [0, 0, 2, 0],
+ "feed_forward_length": [0, 1856, 0, 0],
+ "ssm.group_count": 8,
+ },
+ 2,
+ id = "nemotron-ffn-layers-are-not-ssm",
+ ),
+ pytest.param(
+ "jamba", {"attention.head_count_kv": [0, 8, 0, 0]}, 3, id = "mamba1-no-groups"
+ ),
+ pytest.param(
+ "falcon-h1",
+ {"attention.head_count_kv": 8, "ssm.group_count": 1},
+ 4,
+ id = "parallel-every-layer",
+ ),
+ pytest.param(
+ "mamba2", {"attention.head_count": 0, "ssm.group_count": 1}, 4, id = "pure-mamba2"
+ ),
+ pytest.param("mamba", {"attention.head_count": 0}, 4, id = "pure-mamba1"),
+ ],
+ )
+ def test_recurrent_layers_follow_llama_cpp(self, arch, fields, recurrent):
+ b = _backend_from_gguf(
+ arch,
+ {"block_count": 4, "attention.head_count": 32, "embedding_length": 2048}
+ | self._SSM
+ | fields,
+ )
+ group = fields.get("ssm.group_count", 0)
+ assert b._mamba_recurrent_state_bytes() == recurrent * self._per_layer(group)
+ assert b._rollback_state_bytes(1) == b._mamba_recurrent_state_bytes()
+
+ def test_kda_and_non_ssm_headers_are_not_sized_here(self):
+ kda = _backend_from_gguf(
+ "kimi_linear",
+ {"block_count": 4, "attention.head_count_kv": [0, 0, 8, 0], "kda.head_dim": 128}
+ | self._SSM,
+ )
+ assert kda._mamba_recurrent_state_bytes() == 0
+ plain = _backend_from_gguf(
+ "granitehybrid", {"block_count": 4, "attention.head_count_kv": [0, 8, 0, 8]}
+ )
+ assert plain._mamba_recurrent_state_bytes() == 0
+
+ @pytest.mark.parametrize("key_length", [128, None], ids = ["gqa-path", "legacy-path"])
+ def test_the_estimate_charges_live_state_and_snapshots(self, key_length):
+ dims = (
+ {}
+ if key_length is None
+ else {"attention.key_length": 128, "attention.value_length": 128}
+ )
+ fields = {
+ "block_count": 4,
+ "attention.head_count": 32,
+ "embedding_length": 4096,
+ "attention.head_count_kv": [0, 0, 8, 0],
+ "ssm.group_count": 1,
+ } | dims
+ ssm = _backend_from_gguf("granitehybrid", fields | self._SSM)
+ attention_only = _backend_from_gguf("granitehybrid", fields)
+ state = 3 * self._per_layer(1)
+ kv = lambda b, **k: b._estimate_kv_cache_bytes(4096, "f16", n_parallel = 4, **k) # noqa: E731
+ assert kv(ssm) - kv(attention_only) == 4 * state
+ assert kv(ssm, ctx_checkpoints = 5) - kv(ssm) == 4 * 5 * state
+
+
# E. Path 3: Sliding Window Estimation
@@ -1219,6 +1317,33 @@ class TestLegacyEstimation:
old_formula = int(2 * n_kv_heads * head_dim * n_layers * n_ctx * bpe)
assert b._estimate_kv_cache_bytes(n_ctx, "f16") == old_formula
+ @pytest.mark.parametrize(
+ "overrides, flash_attn, k_heads, v_heads",
+ [
+ pytest.param(
+ {"_n_layers": 4, "_n_kv_heads_by_layer": [0, 0, 8, 4]},
+ True,
+ 12,
+ 12,
+ id = "per-layer-heads",
+ ),
+ pytest.param(
+ {"_n_layers": 4, "_n_kv_heads_by_layer": [0, 0, 8, 4]},
+ False,
+ 12,
+ 4 * 8,
+ id = "per-layer-heads-padded-v",
+ ),
+ pytest.param({"_n_kv_heads": None, "_n_heads": 0}, False, 0, 0, id = "pure-ssm"),
+ ],
+ )
+ def test_only_kv_bearing_layers_are_charged(self, overrides, flash_attn, k_heads, v_heads):
+ b = self._legacy_backend(**overrides)
+ kv = b._estimate_kv_cache_bytes(4096, "q8_0", flash_attn = flash_attn)
+ q8 = lc._kv_bytes_per_elem("q8_0")
+ # The FA-off retry rewrites a quantized V cache to f16.
+ assert kv == int((k_heads * q8 + v_heads * (q8 if flash_attn else 2)) * 128 * 4096)
+
# H. Path Priority (selection order)