Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: cap context checkpoints by host RAM for sliding-window and SSM models (#11918)
Long Yixinge8ec65fd4c50
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 9d4619e71a9..626c6b79b26 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -4914,6 +4914,12 @@ _TARGET_ROLLBACK_SPEC_TYPES = frozenset()+# Mirrors llama.cpp is_recr_impl for SSM headers with no full_attention_interval.+_SSM_EVERY_LAYER_ARCHS = frozenset({"mamba", "mamba2", "falcon-h1"})+# Nemotron-H also zeroes head_count_kv on its MLP/MoE layers; its SSM layers have no FFN.+_SSM_FFN_FREE_LAYER_ARCHS = frozenset({"nemotron_h", "nemotron_h_moe"})++# Architectures whose TARGET context leaves the embedded MTP/NextN blocks out of# its KV cache, so block_count must lose them before the target is priced.#@@ -7261,6 +7267,7 @@ class LlamaCppBackend:# For the compute-graph buffer estimate; vocab from the tokens array len.# feed_forward_length is the widest layer when the GGUF stores one per layer.self._feed_forward_length: Optional[int] = None+ self._feed_forward_length_by_layer: Optional[list[int]] = Noneself._expert_used_count: Optional[int] = Noneself._expert_feed_forward_length: Optional[int] = Noneself._expert_shared_feed_forward_length: Optional[int] = None@@ -15851,14 +15858,41 @@ class LlamaCppBackend:n_parallel)+ def _ctx_checkpoint_bytes(+ self,+ cache_type_kv: Optional[str] = None,+ *,+ swa_full: bool = False,+ flash_attn: bool = True,+ ) -> int:+ """Host bytes of one slot's context checkpoint: the recurrent state, else the SWA window."""+ recurrent = self._rollback_state_bytes(1)+ if recurrent > 0:+ return recurrent++ def kv(checkpoints: int) -> int:+ return self._estimate_kv_cache_bytes(+ 1,+ cache_type_kv,+ swa_full = swa_full,+ ctx_checkpoints = checkpoints,+ flash_attn = flash_attn,+ )++ return max(0, kv(1) - kv(0))+def _bounded_ctx_checkpoints(self,n_parallel: int,server_caps: Mapping[str, object],extra_args: Optional[Iterable[str]] = None,env: Optional[Mapping[str, str]] = None,+ *,+ cache_type_kv: Optional[str] = None,+ swa_full: bool = False,+ flash_attn: bool = True,) -> Optional[int]:- """Return an automatic recurrent-checkpoint cap, or None to preserve the argv."""+ """Return an automatic context-checkpoint cap, or None to preserve the argv."""flag = server_caps.get("ctx_checkpoints_flag")if not flag:return None@@ -15867,7 +15901,9 @@ class LlamaCppBackend:return Noneif _env_ctx_checkpoints_override(env) is not None:return None- per_checkpoint = self._rollback_state_bytes(1)+ per_checkpoint = self._ctx_checkpoint_bytes(+ cache_type_kv, swa_full = swa_full, flash_attn = flash_attn+ )if per_checkpoint <= 0:return Nonetotal_ram_mib = self._host_memory_capacity_mib()@@ -15883,7 +15919,7 @@ class LlamaCppBackend:return Nonelogger.info("Capping llama-server context checkpoints at %d per slot (this build's default %d): "- "each one snapshots this model's whole recurrent state (%.1f MiB), so the default "+ "each one snapshots %.1f MiB of this model's state, so the default ""would hold %.1f GiB of host RAM across %d slot(s).",bounded,upstream_default,@@ -15915,6 +15951,25 @@ class LlamaCppBackend:arch = getattr(self, "_architecture", None)return bool(arch) and str(arch).strip().lower() in _TARGET_KV_EXCLUDES_NEXTN_ARCHS+ def _ssm_recurrent_layer_count(self, n_layers: int) -> int:+ """SSM layers of a header with no attention interval, by llama.cpp's is_recr rules."""+ arch = getattr(self, "_architecture", None)+ if arch in _SSM_EVERY_LAYER_ARCHS:+ return n_layers+ heads = getattr(self, "_n_kv_heads_by_layer", None)+ if not heads or getattr(self, "_kda_head_dim", None):+ return 0+ ffn = (+ getattr(self, "_feed_forward_length_by_layer", None)+ if arch in _SSM_FFN_FREE_LAYER_ARCHS+ else None+ ) or []+ return sum(+ 1+ for i, n_kv in enumerate(heads[:n_layers])+ if n_kv == 0 and not (i < len(ffn) and ffn[i] > 0)+ )+def _mamba_recurrent_state_bytes(self,n_parallel: int = 1,@@ -15933,17 +15988,16 @@ class LlamaCppBackend:n_group_raw = getattr(self, "_ssm_group_count", None)d_conv_raw = getattr(self, "_ssm_conv_kernel", None)fai_raw = getattr(self, "_full_attention_interval", None)+ # Mamba1 (Jamba, Mamba) writes no group count; llama.cpp reads it as 0.if not all(value is not Nonefor value in (n_layers_raw,d_inner_raw,d_state_raw,- n_group_raw,d_conv_raw,- fai_raw,)- ):+ ) or (fai_raw is not None and n_group_raw is None):return 0# Excludes the embedded MTP blocks, as _estimate_kv_cache_bytes does and# for the same reason: llama.cpp keeps them out of the target's n_layer.@@ -15951,9 +16005,12 @@ class LlamaCppBackend:0,int(n_layers_raw or 0) - int(getattr(self, "_nextn_predict_layers", None) or 0),)- fai = int(fai_raw or 0)- n_attn = -(-n_layers // fai) if fai > 0 else n_layers- n_recurrent = max(0, n_layers - n_attn)+ if fai_raw is not None:+ fai = int(fai_raw or 0)+ n_attn = -(-n_layers // fai) if fai > 0 else n_layers+ n_recurrent = max(0, n_layers - n_attn)+ else:+ n_recurrent = self._ssm_recurrent_layer_count(n_layers)if n_recurrent == 0:return 0d_inner = int(d_inner_raw or 0)@@ -16180,6 +16237,8 @@ class LlamaCppBackend:)return int(global_bytes + swa_bytes + slots * checkpoint_extra_per_slot)+ ssm_state = self._mamba_recurrent_state_bytes(n_parallel) + recurrent_checkpoints+# Path 4: Standard GQA with explicit key/value dimensionsif key_len is not None and val_len is not None:padded_v_width = None if flash_attn else self._max_kv_value_width(val_len)@@ -16188,11 +16247,16 @@ class LlamaCppBackend:layer_n_kv = self._kv_heads_for_layer(layer_idx, n_kv)v_width = layer_n_kv * val_len if padded_v_width is None else padded_v_widthbytes_per_cell += layer_n_kv * key_len * bpe_k + v_width * bpe_v- return int(total_cells * bytes_per_cell)+ return int(total_cells * bytes_per_cell) + ssm_state# Path 5: Legacy fallback (old GGUFs without explicit dimensions)head_dim = self._legacy_head_dim()- return int(2 * n_kv * head_dim * n_layers_kv * total_cells * bpe_k)+ # llama.cpp defaults head_count_kv to head_count, so a pure SSM (head_count 0) has no KV.+ layer_default = self._n_kv_heads if self._n_kv_heads is not None else (self._n_heads or 0)+ heads = [self._kv_heads_for_layer(i, layer_default) for i in range(n_layers_kv)]+ # Without flash attention V is padded to the widest layer, as on Path 4.+ v_heads = sum(heads) if flash_attn else len(heads) * max(heads)+ return int((sum(heads) * bpe_k + v_heads * bpe_v) * head_dim * total_cells) + ssm_statedef _draft_backend_for(self, drafter_path: str) -> Optional["LlamaCppBackend"]:"""Lightweight backend with a drafter GGUF's metadata, to size its own KV@@ -17541,6 +17605,7 @@ class LlamaCppBackend:self._pooling_type = Noneself._gguf_path = gguf_pathself._feed_forward_length = None+ self._feed_forward_length_by_layer = Noneself._expert_used_count = Noneself._expert_feed_forward_length = Noneself._expert_shared_feed_forward_length = None@@ -17734,7 +17799,10 @@ class LlamaCppBackend:self._sliding_window_pattern = [bool(x) for x in val_a]sliding_window_pattern_period = Noneelif attr == "feed_forward_length" and val_a:- self._feed_forward_length = max(int(x) for x in val_a)+ self._feed_forward_length_by_layer = [int(x) for x in val_a]+ self._feed_forward_length = max(+ self._feed_forward_length_by_layer+ )else:self._gguf_skip_value(f, vtype)else:@@ -23378,7 +23446,9 @@ class LlamaCppBackend:server_caps,extra_args,ctx_checkpoints,- per_checkpoint_bytes = self._rollback_state_bytes(1),+ per_checkpoint_bytes = self._ctx_checkpoint_bytes(+ cache_type_kv, swa_full = swa_full, flash_attn = planned_flash_attn+ ),n_parallel = n_parallel,total_host_bytes = ((self._host_memory_capacity_mib() or 0) * 1024 * 1024)or None,@@ -27334,7 +27404,14 @@ class LlamaCppBackend:def _decide_auto_ctx_checkpoints() -> Optional[int]:"""The cap for the slots n_parallel currently names, or None to stand down."""return (- self._bounded_ctx_checkpoints(n_parallel, server_caps, extra_args)+ self._bounded_ctx_checkpoints(+ n_parallel,+ server_caps,+ extra_args,+ cache_type_kv = cache_type_kv,+ swa_full = swa_full,+ flash_attn = planned_flash_attn,+ )if ctx_checkpoints is Noneelse None)diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex 6cc80435e68..caadada73d7 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -39,7 +39,7 @@ CACHE_RAM_MAX_MIB = 1024 * 1024# llama.cpp allocates this default even when Studio emits no flag.LLAMA_CTX_CHECKPOINTS_DEFAULT = 32-# Recurrent checkpoints live in host RAM and can be much larger than SWA snapshots.+# Checkpoints live in host RAM: a hybrid's whole recurrent state, or an SWA model's window.CTX_CHECKPOINT_HOST_BUDGET_FRACTION = 0.05CTX_CHECKPOINT_HOST_BUDGET_FLOOR_BYTES = 1024**3# Keep rollback available; zero forces a full prompt re-ingest after divergence.diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex e7c87b66d4f..423e0ca445a 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -11495,23 +11495,6 @@ def _gguf_runtime_bytes(llama_extra_args,default = managed_kv_unified,)- # Resolve the same capability-dependent count that load_model emits.- _cc_caps: dict = {}- try:- _cc_caps = LlamaCppBackend.probe_server_capabilities() or {}- except Exception as _cc_exc:- logger.debug("ctx-checkpoints capability probe failed: %s", _cc_exc)- # Older lightweight probes may not provide the new sizing helpers.- _per_checkpoint = getattr(probe, "_rollback_state_bytes", lambda _n: 0)(1)- _host_mib = getattr(probe, "_host_memory_capacity_mib", lambda: None)()- _resolved_checkpoints = effective_ctx_checkpoints_for_caps(- _cc_caps,- llama_extra_args,- ctx_checkpoints,- per_checkpoint_bytes = _per_checkpoint,- n_parallel = slots,- total_host_bytes = (_host_mib * 1024 * 1024) if _host_mib else None,- )# load_model appends --flash-attn on to every launch whose build has the flag,# so the default here is ON, not off. The false arm pads variable-width V# tensors to the model-wide maximum (_max_kv_value_width), which on an@@ -11521,7 +11504,7 @@ def _gguf_runtime_bytes(_fa_supported = Truetry:_fa_caps = LlamaCppBackend.probe_server_capabilities()- # Same ``found`` test as the checkpoint probe above: the defaults dict+ # Same ``found`` test as the checkpoint probe below: the defaults dict# reports nothing supported, so keying on the flag alone would size every# unprobed host as flash-attention-less.if _fa_caps.get("found") and not _fa_caps.get("supports_flash_attn", True):@@ -11552,6 +11535,23 @@ def _gguf_runtime_bytes(tensor_parallel = bool(tensor_parallel),env = os.environ,)+ _cc_caps: dict = {}+ try:+ _cc_caps = LlamaCppBackend.probe_server_capabilities() or {}+ except Exception as _cc_exc:+ logger.debug("ctx-checkpoints capability probe failed: %s", _cc_exc)+ _per_checkpoint = getattr(probe, "_ctx_checkpoint_bytes", lambda *_a, **_k: 0)(+ cache_type_for_budget, swa_full = swa_full, flash_attn = flash_attn+ )+ _host_mib = getattr(probe, "_host_memory_capacity_mib", lambda: None)()+ _resolved_checkpoints = effective_ctx_checkpoints_for_caps(+ _cc_caps,+ llama_extra_args,+ ctx_checkpoints,+ per_checkpoint_bytes = _per_checkpoint,+ n_parallel = slots,+ total_host_bytes = (_host_mib * 1024 * 1024) if _host_mib else None,+ )kv = probe._estimate_kv_cache_bytes(ctx,cache_type_for_budget,diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.pyindex f0d8c1ba410..073cfbb4cc1 100644--- a/studio/backend/routes/models.py+++ b/studio/backend/routes/models.py@@ -4115,7 +4115,11 @@ async def get_kv_cache_estimate(_cc_caps,None,ctx_checkpoints,- per_checkpoint_bytes = getattr(be, "_rollback_state_bytes", lambda _n: 0)(1),+ per_checkpoint_bytes = getattr(be, "_ctx_checkpoint_bytes", lambda *_a, **_k: 0)(+ _effective_cache_type,+ swa_full = _plan_kwargs.get("swa_full", False),+ flash_attn = _plan_kwargs.get("flash_attn", True),+ ),n_parallel = n_parallel,total_host_bytes = (_total_ram_mib * 1024 * 1024) if _total_ram_mib else None,)diff --git a/studio/backend/tests/test_ctx_checkpoint_host_budget.py b/studio/backend/tests/test_ctx_checkpoint_host_budget.pyindex 52398e1f4fe..3183a5b9ade 100644--- a/studio/backend/tests/test_ctx_checkpoint_host_budget.py+++ b/studio/backend/tests/test_ctx_checkpoint_host_budget.py@@ -437,7 +437,17 @@ class TestCheckpointsNeverReachAVramFigure:source = inspect.getsource(LlamaCppBackend.load_model)# One rule, re-asked wherever the slot count changes, never a second spelling.assert source.count("def _decide_auto_ctx_checkpoints(") == 1- assert "self._bounded_ctx_checkpoints(n_parallel, server_caps, extra_args)" in source+ compact = "".join(source.split())+ sized = (+ "self._ctx_checkpoint_bytes(cache_type_kv,swa_full=swa_full,"+ "flash_attn=planned_flash_attn)"+ )+ assert f"per_checkpoint_bytes={sized}" in compact+ assert (+ "self._bounded_ctx_checkpoints(n_parallel,server_caps,extra_args,"+ "cache_type_kv=cache_type_kv,swa_full=swa_full,flash_attn=planned_flash_attn,)"+ ) in compact+ assert "per_checkpoint_bytes=self._rollback_state_bytes" not in compactassert source.count("self._bounded_ctx_checkpoints(") == 1assert "str(_auto_ctx_checkpoints)" in source@@ -1077,3 +1087,182 @@ class TestTheGuardOnlyDropsHostBytesFromADiscretePool:discrete = inference_routes._estimate_gguf_kv_gb("x.gguf", 4096, shared_memory_pool = False)assert shared == 11.0 and discrete == 7.0assert shared > discrete, "the shared pool must not be credited the host share"+++_GEMMA4_31B_SWA = [(i + 1) % 6 != 0 for i in range(60)]+_GEMMA4_31B = {+ "context_length": 262144,+ "block_count": 60,+ "embedding_length": 5376,+ "attention.head_count": 32,+ "attention.head_count_kv": [16 if swa else 4 for swa in _GEMMA4_31B_SWA],+ "attention.key_length": 512,+ "attention.value_length": 512,+ "attention.key_length_swa": 256,+ "attention.value_length_swa": 256,+ "attention.sliding_window": 1024,+ "attention.sliding_window_pattern": _GEMMA4_31B_SWA,+}+GEMMA4_SNAPSHOT = 800 * MIB+++def gemma4_gguf(tmp_path, monkeypatch):+ import importlib.util+ from pathlib import Path++ spec = importlib.util.spec_from_file_location(+ "_kv_cache_estimation_for_checkpoints",+ Path(__file__).parent / "test_kv_cache_estimation.py",+ )+ helpers = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(helpers)+ kv = {"general.architecture": "gemma4"}+ kv.update({f"gemma4.{k}": v for k, v in _GEMMA4_31B.items()})+ path = tmp_path / "gemma-4-31B-it-Q6_K.gguf"+ path.write_bytes(helpers._make_gguf_bytes("gemma4", kv))+ monkeypatch.delenv("LLAMA_ARG_SWA_FULL", raising = False)+ monkeypatch.delenv("LLAMA_ARG_CTX_SIZE", raising = False)+ monkeypatch.setattr(+ LlamaCppBackend, "_total_system_memory_mib", staticmethod(lambda: 94 * 1024)+ )+ caps = {**_caps(), "supports_kv_unified": True, "supports_flash_attn": True}+ monkeypatch.setattr(+ LlamaCppBackend, "probe_server_capabilities", classmethod(lambda cls, *a, **k: caps)+ )+ return path+++def _gemma4_backend(path):+ b = LlamaCppBackend()+ b._read_gguf_metadata(str(path))+ return b+++def test_an_swa_snapshot_is_its_window(gemma4_gguf):+ b = _gemma4_backend(gemma4_gguf)+ assert b._rollback_state_bytes(1) == 0+ assert b._ctx_checkpoint_bytes("f16") == GEMMA4_SNAPSHOT+ assert b._ctx_checkpoint_bytes("q8_0") < GEMMA4_SNAPSHOT+ assert b._ctx_checkpoint_bytes("f16", swa_full = True) == 0+ assert _plain_attention_backend()._ctx_checkpoint_bytes("f16") == 0+ assert _backend()._ctx_checkpoint_bytes("f16") == _backend()._rollback_state_bytes(1)+++def test_the_launcher_bounds_an_swa_load(gemma4_gguf):+ b = _gemma4_backend(gemma4_gguf)+ assert b._bounded_ctx_checkpoints(1, _caps(), cache_type_kv = "f16") == 6+ assert b._bounded_ctx_checkpoints(4, _caps(), cache_type_kv = "f16") == (+ CTX_CHECKPOINTS_MIN_USEFUL+ )+ assert b._bounded_ctx_checkpoints(4, _caps(), cache_type_kv = "f16", swa_full = True) is None+ assert b._bounded_ctx_checkpoints(1, _caps(), cache_type_kv = "q8_0", flash_attn = False) < (+ b._bounded_ctx_checkpoints(1, _caps(), cache_type_kv = "q8_0")+ )++[email protected]("flash_attn", [None, False], ids = ["managed", "fa-off"])+def test_both_estimates_price_the_count_the_launcher_emits(gemma4_gguf, monkeypatch, flash_attn):+ """The reported load: Auto context, f16, four slots. Uncapped this was 100 GiB of host RAM."""+ import asyncio++ from routes import inference as inference_routes+ from routes import models as models_routes++ sized_with = []+ snapshot = LlamaCppBackend._ctx_checkpoint_bytes++ def spy(self, *args, **kwargs):+ sized_with.append(kwargs.get("flash_attn"))+ return snapshot(self, *args, **kwargs)++ monkeypatch.setattr(LlamaCppBackend, "_ctx_checkpoint_bytes", spy)+ emitted = _gemma4_backend(gemma4_gguf)._bounded_ctx_checkpoints(4, _caps(), cache_type_kv = "f16")+ expected = 4 * emitted * GEMMA4_SNAPSHOT++ extras = None if flash_attn is None else ["--flash-attn", "off"]+ sized_with.clear()+ panel = inference_routes._gguf_runtime_bytes(str(gemma4_gguf), 0, extras, 4, "f16", False)+ assert panel.n_ctx == 262144+ assert panel.kv_checkpoint_bytes == expected++ monkeypatch.setattr(+ models_routes, "_resolve_quant_gguf", lambda *_a: (str(gemma4_gguf), 25 * GIB)+ )+ hub = asyncio.run(+ models_routes.get_kv_cache_estimate(+ repo_id = "unsloth/gemma-4-31B-it-GGUF",+ quant = "Q6_K",+ n_ctx = 262144,+ cache_type_kv = "f16",+ n_parallel = 4,+ speculative_type = None,+ spec_draft_n_max = None,+ spec_draft_cache_type = None,+ ctx_checkpoints = None,+ disable_vision = True,+ n_batch = None,+ n_ubatch = None,+ tensor_parallel = False,+ flash_attn = flash_attn,+ kv_unified = None,+ swa_full = None,+ no_mmproj_offload = None,+ request = None,+ current_subject = "test",+ )+ )+ assert hub["kv_checkpoint_bytes"] == expected+ assert set(sized_with) == {flash_attn is None}+++_GRANITE_H_SMALL = {+ "context_length": 1048576,+ "block_count": 40,+ "embedding_length": 4096,+ "attention.head_count": 32,+ "attention.head_count_kv": [8 if i % 10 == 5 else 0 for i in range(40)],+ "ssm.conv_kernel": 4,+ "ssm.state_size": 128,+ "ssm.group_count": 1,+ "ssm.inner_size": 8192,+}+GRANITE_SNAPSHOT = 36 * (3 * (8192 + 2 * 128) + 128 * 8192) * 4+++def granite_gguf(tmp_path, monkeypatch):+ import importlib.util+ from pathlib import Path++ spec = importlib.util.spec_from_file_location(+ "_kv_cache_estimation_for_checkpoints",+ Path(__file__).parent / "test_kv_cache_estimation.py",+ )+ helpers = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(helpers)+ kv = {"general.architecture": "granitehybrid"}+ kv.update({f"granitehybrid.{k}": v for k, v in _GRANITE_H_SMALL.items()})+ path = tmp_path / "granite-4.0-h-small-Q4_K_M.gguf"+ path.write_bytes(helpers._make_gguf_bytes("granitehybrid", kv))+ monkeypatch.delenv("LLAMA_ARG_CTX_SIZE", raising = False)+ monkeypatch.setattr(+ LlamaCppBackend, "_total_system_memory_mib", staticmethod(lambda: 94 * 1024)+ )+ caps = {**_caps(), "supports_kv_unified": True, "supports_flash_attn": True}+ monkeypatch.setattr(+ LlamaCppBackend, "probe_server_capabilities", classmethod(lambda cls, *a, **k: caps)+ )+ return path+++def test_an_interval_free_hybrid_is_capped_and_priced(granite_gguf):+ from routes import inference as inference_routes++ b = LlamaCppBackend()+ b._read_gguf_metadata(str(granite_gguf))+ assert b._rollback_state_bytes(1) == GRANITE_SNAPSHOT+ emitted = b._bounded_ctx_checkpoints(4, _caps())+ assert emitted == 8+ panel = inference_routes._gguf_runtime_bytes(str(granite_gguf), 32768, None, 4, "f16", False)+ assert panel.kv_checkpoint_bytes == 4 * emitted * GRANITE_SNAPSHOTdiff --git a/studio/backend/tests/test_kv_cache_estimation.py b/studio/backend/tests/test_kv_cache_estimation.pyindex b23b8d45dc3..53553624edc 100644--- a/studio/backend/tests/test_kv_cache_estimation.py+++ b/studio/backend/tests/test_kv_cache_estimation.py@@ -990,6 +990,104 @@ class TestHybridMambaEstimation:assert b._estimate_kv_cache_bytes(4096, "f16", n_parallel = 4) == (kv_only + 4 * per_slot)+class TestSsmStateWithoutAttentionInterval:+ """SSM headers with no full_attention_interval, recurrent layers per llama.cpp's is_recr."""++ _SSM = {"ssm.inner_size": 1024, "ssm.state_size": 16, "ssm.conv_kernel": 4}++ @staticmethod+ def _per_layer(group):+ return ((4 - 1) * (1024 + 2 * group * 16) + 16 * 1024) * 4++ @pytest.mark.parametrize(+ "arch, fields, recurrent",+ [+ pytest.param(+ "granitehybrid",+ {"attention.head_count_kv": [0, 0, 8, 0], "ssm.group_count": 1},+ 3,+ id = "zero-kv-heads",+ ),+ pytest.param(+ "nemotron_h",+ {+ "attention.head_count_kv": [0, 0, 8, 0],+ "feed_forward_length": [0, 21504, 0, 0],+ "ssm.group_count": 8,+ },+ 2,+ id = "dense-nemotron-mlp-layers-are-not-ssm",+ ),+ pytest.param(+ "nemotron_h_moe",+ {+ "attention.head_count_kv": [0, 0, 2, 0],+ "feed_forward_length": [0, 1856, 0, 0],+ "ssm.group_count": 8,+ },+ 2,+ id = "nemotron-ffn-layers-are-not-ssm",+ ),+ pytest.param(+ "jamba", {"attention.head_count_kv": [0, 8, 0, 0]}, 3, id = "mamba1-no-groups"+ ),+ pytest.param(+ "falcon-h1",+ {"attention.head_count_kv": 8, "ssm.group_count": 1},+ 4,+ id = "parallel-every-layer",+ ),+ pytest.param(+ "mamba2", {"attention.head_count": 0, "ssm.group_count": 1}, 4, id = "pure-mamba2"+ ),+ pytest.param("mamba", {"attention.head_count": 0}, 4, id = "pure-mamba1"),+ ],+ )+ def test_recurrent_layers_follow_llama_cpp(self, arch, fields, recurrent):+ b = _backend_from_gguf(+ arch,+ {"block_count": 4, "attention.head_count": 32, "embedding_length": 2048}+ | self._SSM+ | fields,+ )+ group = fields.get("ssm.group_count", 0)+ assert b._mamba_recurrent_state_bytes() == recurrent * self._per_layer(group)+ assert b._rollback_state_bytes(1) == b._mamba_recurrent_state_bytes()++ def test_kda_and_non_ssm_headers_are_not_sized_here(self):+ kda = _backend_from_gguf(+ "kimi_linear",+ {"block_count": 4, "attention.head_count_kv": [0, 0, 8, 0], "kda.head_dim": 128}+ | self._SSM,+ )+ assert kda._mamba_recurrent_state_bytes() == 0+ plain = _backend_from_gguf(+ "granitehybrid", {"block_count": 4, "attention.head_count_kv": [0, 8, 0, 8]}+ )+ assert plain._mamba_recurrent_state_bytes() == 0++ @pytest.mark.parametrize("key_length", [128, None], ids = ["gqa-path", "legacy-path"])+ def test_the_estimate_charges_live_state_and_snapshots(self, key_length):+ dims = (+ {}+ if key_length is None+ else {"attention.key_length": 128, "attention.value_length": 128}+ )+ fields = {+ "block_count": 4,+ "attention.head_count": 32,+ "embedding_length": 4096,+ "attention.head_count_kv": [0, 0, 8, 0],+ "ssm.group_count": 1,+ } | dims+ ssm = _backend_from_gguf("granitehybrid", fields | self._SSM)+ attention_only = _backend_from_gguf("granitehybrid", fields)+ state = 3 * self._per_layer(1)+ kv = lambda b, **k: b._estimate_kv_cache_bytes(4096, "f16", n_parallel = 4, **k) # noqa: E731+ assert kv(ssm) - kv(attention_only) == 4 * state+ assert kv(ssm, ctx_checkpoints = 5) - kv(ssm) == 4 * 5 * state++# E. Path 3: Sliding Window Estimation@@ -1219,6 +1317,33 @@ class TestLegacyEstimation:old_formula = int(2 * n_kv_heads * head_dim * n_layers * n_ctx * bpe)assert b._estimate_kv_cache_bytes(n_ctx, "f16") == old_formula+ @pytest.mark.parametrize(+ "overrides, flash_attn, k_heads, v_heads",+ [+ pytest.param(+ {"_n_layers": 4, "_n_kv_heads_by_layer": [0, 0, 8, 4]},+ True,+ 12,+ 12,+ id = "per-layer-heads",+ ),+ pytest.param(+ {"_n_layers": 4, "_n_kv_heads_by_layer": [0, 0, 8, 4]},+ False,+ 12,+ 4 * 8,+ id = "per-layer-heads-padded-v",+ ),+ pytest.param({"_n_kv_heads": None, "_n_heads": 0}, False, 0, 0, id = "pure-ssm"),+ ],+ )+ def test_only_kv_bearing_layers_are_charged(self, overrides, flash_attn, k_heads, v_heads):+ b = self._legacy_backend(**overrides)+ kv = b._estimate_kv_cache_bytes(4096, "q8_0", flash_attn = flash_attn)+ q8 = lc._kv_bytes_per_elem("q8_0")+ # The FA-off retry rewrites a quantized V cache to f16.+ assert kv == int((k_heads * q8 + v_heads * (q8 if flash_attn else 2)) * 128 * 4096)+# H. Path Priority (selection order)