Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: honour an unmeasured context request and price the real attention plan (#11025)
Daniel Han6d938276e19d
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 126f6922c7b..f218d8d55fb 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -3787,6 +3787,9 @@ _FIT_FLOOR_MIN_CTX = 256# the ceiling that assumed one._LLAMA_FIT_MIN_CTX = 4096+# _plan_tensor_parallel's no-KV fallback. A GUESS: may shrink Auto, never a request (#9653).+_TP_UNMEASURED_CTX = 4096+# Auto only reaches this path when no discrete-GPU subset can hold the model.# llama.cpp must offload layers to host RAM either way, so prefer a context that# remains useful for chat. Separate from _FIT_MIN_CTX: that value is also a search@@ -4668,6 +4671,142 @@ def _kv_unified_from_args(return enabled+# llama_init_from_model forces flash attention off for LLM_ARCH_GROK whatever the launch+# asks for: Grok softcaps the KQ logits, which the fused kernel does not carry.+_FLASH_ATTN_INCOMPATIBLE_ARCHITECTURES = frozenset({"grok"})+++def _architecture_forces_flash_attn_off(architecture: Optional[str]) -> bool:+ """Whether llama.cpp will disable flash attention for this architecture on its own."""+ return str(architecture or "").strip().lower() in _FLASH_ATTN_INCOMPATIBLE_ARCHITECTURES+++def _planned_flash_attn_state(+ extra_args: Optional[Iterable[str]] = None,+ *,+ planned_cache_types: Optional[tuple[str, str]] = None,+ supports_flash_attn: bool = True,+ tensor_parallel: bool = False,+ architecture: Optional[str] = None,+ env: Optional[Mapping[str, str]] = None,+) -> bool:+ """One answer for the estimate and for the argv: the estimator floors V at f16 and pads+ it model-wide when flash attention is off, so resolving the two separately prices a+ different load, not a conservative one (#9697, #10489)."""+ if not supports_flash_attn:+ return False+ if _architecture_forces_flash_attn_off(architecture):+ # Ahead of every rule below, as llama.cpp puts it ahead of both its own upgrades.+ return False+ v_type = (planned_cache_types[1] if planned_cache_types else "f16") or "f16"+ quantized_v = str(v_type).strip().lower() not in {"f16", "bf16", "f32"}+ if quantized_v:+ # Not a choice: llama.cpp turns it on itself rather than refusing the load.+ return True+ # Prepended because ``load_model`` appends a managed ``--flash-attn on``: the env loses+ # to it and only the user's own extras beat it (#9697, #10489).+ effective_args = ["--flash-attn", "on", *(str(arg) for arg in extra_args or ())]+ if _asked_for_auto_flash_attn(effective_args, env = env):+ if _effective_tensor_parallel(extra_args, tensor_parallel, env):+ # Not undecided: llama.cpp upgrades AUTO to ENABLED under SPLIT_MODE_TENSOR.+ return True+ # ``auto`` is decided at load time and can come back no; size for the costlier answer.+ return False+ return _flash_attn_enabled_from_args(effective_args, default = True, env = env)+++def _user_fit_disabled(+ extra_args: Optional[Iterable[str]] = None, *, env: Optional[Mapping[str, str]] = None+) -> bool:+ """Whether the USER has turned llama.cpp's fitter off, last-wins: Unsloth's own token+ comes first and a retry rewrites it, so only a user's later value survives a re-place."""+ values = [str(arg) for arg in extra_args] if extra_args else []+ asked: Optional[str] = None+ inherited = (os.environ if env is None else env).get("LLAMA_ARG_FIT")+ if inherited is not None and str(inherited).strip():+ asked = str(inherited).strip()+ for i, raw in enumerate(values):+ if _flag_name(raw) not in {"-fit", "--fit"}:+ continue+ _, eq, inline = raw.partition("=")+ value = inline if eq else "on"+ if not eq and i + 1 < len(values):+ value = values[i + 1]+ asked = value+ return str(asked).strip().lower() in {"off", "0", "false", "no", "disabled"}+++def _placement_is_fitter_proof(+ extra_args: Optional[Iterable[str]] = None,+ *,+ gpu_layers: Optional[int] = None,+ env: Optional[Mapping[str, str]] = None,+) -> bool:+ """Whether a layer count llama.cpp's fitter refuses to move owns this placement:+ ``common_params_fit_impl`` throws "n_gpu_layers already set by user" (common/fit.cpp:377)+ for any value but -1, so re-enabling the fitter for a respawn buys nothing."""+ if gpu_layers is not None and gpu_layers >= 0:+ return True+ values = [str(arg) for arg in extra_args] if extra_args else []+ for i, raw in enumerate(values):+ if _flag_name(raw) not in _GPU_LAYER_FLAGS:+ continue+ _, eq, inline = raw.partition("=")+ value = inline if eq else ""+ if not eq and i + 1 < len(values):+ value = values[i + 1]+ value = value.strip().lower()+ if value and value not in _LLAMA_ARG_AUTO_VALUES:+ return True+ return bool(_env_fixes_gpu_layers(env))+++def _reserved_flash_attn_state(+ planned: bool,+ extra_args: Optional[Iterable[str]] = None,+ *,+ tensor_parallel: bool = False,+ gpu_layers: Optional[int] = None,+ env: Optional[Mapping[str, str]] = None,+) -> bool:+ """``planned``, held to the conservative reading where a respawn cannot be re-placed.++ A flash-attention crash respawns with it off, which pads V model-wide, so where the+ fitter cannot re-place that respawn the larger cache lands on a placement chosen for the+ smaller one. Tensor mode has no such respawn, and pricing it would refuse loads that fit.+ """+ if not planned:+ return planned+ if _effective_tensor_parallel(extra_args, tensor_parallel, env):+ return planned+ if _user_fit_disabled(extra_args, env = env):+ return False+ if _placement_is_fitter_proof(extra_args, gpu_layers = gpu_layers, env = env):+ return False+ return planned+++def _asked_for_auto_flash_attn(+ extra_args: Optional[Iterable[str]] = None, *, env: Optional[Mapping[str, str]] = None+) -> bool:+ """``_flash_attn_enabled_from_args`` folds auto into True because it answers what the+ argv will SAY; sizing needs to know nobody has decided yet."""+ asked: Optional[str] = None+ value = (os.environ if env is None else env).get("LLAMA_ARG_FLASH_ATTN")+ if value in _LLAMA_ARG_TRUE_FALSE_AUTO_VALUES:+ asked = value+ values = [str(arg) for arg in extra_args] if extra_args else []+ for i, raw in enumerate(values):+ if _flag_name(raw) not in {"-fa", "--flash-attn"}:+ continue+ _, eq, inline = raw.partition("=")+ value = inline if eq else "on"+ if not eq and i + 1 < len(values) and values[i + 1] in _LLAMA_ARG_TRUE_FALSE_AUTO_VALUES:+ value = values[i + 1]+ asked = value+ return asked in _LLAMA_ARG_AUTO_VALUES++def _flash_attn_enabled_from_args(args: Optional[Iterable[str]],default: bool = True,@@ -4697,24 +4836,6 @@ def _flash_attn_enabled_from_args(return enabled-def _planned_flash_attn(- extra_args: Optional[Iterable[str]],- supports_flash_attn: bool,- v_cache_type: Optional[str] = None,- architecture: Optional[str] = None,-) -> bool:- """Flash attention as the launch command will resolve it.-- The managed --flash-attn on follows the environment and precedes user extras, so- only extras can turn it off. A build without the flag gets neither the flag nor- the inherited LLAMA_ARG_FLASH_ATTN. A quantized V cache forces it on, except on- Grok, which llama.cpp always runs without it."""- if not supports_flash_attn or architecture == "grok":- return False- enabled = _flash_attn_enabled_from_args(extra_args, default = True, env = {})- return enabled or (v_cache_type or "f16").strip().lower() not in {"f16", "bf16", "f32"}--def _effective_spec_type(extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None) -> Optional[str]:@@ -14369,6 +14490,29 @@ class LlamaCppBackend:f"{LlamaCppBackend.METAL_CTX_OVERCOMMIT_ENV}=1 to load it anyway.")+ @staticmethod+ def _unmeasured_context_notice(+ requested_ctx: int, cache_type_kv: Optional[str] = None+ ) -> Optional[str]:+ """Advisory for a hand-set context with no KV estimate behind it: the short-context+ fallbacks used to cut a 262,144-token request to 4,096 and publish it (#9653). Not a+ refusal, which on a guess blocks live loads."""+ if requested_ctx <= 0:+ return None+ kv_hint = (+ " Setting the KV cache to q8_0 roughly halves what the context costs."+ if (cache_type_kv or "f16").strip().lower() in ("f16", "fp16", "")+ else ""+ )+ return (+ f"This model's GGUF does not carry the attention dimensions needed to size "+ f"the KV cache, so the {requested_ctx:,}-token context was launched as "+ f"requested without being checked against available memory. The reported "+ f"maximum context is that request, not a measured ceiling. If the load fails "+ f"or the machine runs short of memory, lower the context or leave it on "+ f"Auto.{kv_hint}"+ )+# Skip the wait when the last kill is older than this; the driver has# already reclaimed the prior process's allocations._VRAM_SETTLE_WINDOW_S: float = 15.0@@ -15080,11 +15224,9 @@ class LlamaCppBackend:# layer's V is padded to hparams.n_embd_v_gqa_max() over the WHOLE model,# which is what _estimate_kv_cache_bytes charges (_max_kv_value_width). The# V half goes constant while K stays per-layer, so an unpadded vector- # prices a ratio the total does not have. Not an edge case: load_model pins- # planned_flash_attn = False unconditionally (llama_cpp.py:16690), so the- # padded branch is the only one the total ever takes. bpe_v is floored at- # f16 for a quantised cache, and with V constant that asymmetry moves the- # ratio too, so carry both rather than cancelling one.+ # prices a ratio the total does not have. bpe_v is floored at f16 for a quantised+ # cache, and with V constant that asymmetry moves the ratio too, so carry both+ # rather than cancelling one.bpe_k = _kv_bytes_per_elem(cache_type_kv)bpe_v = bpe_k if flash_attn else max(bpe_k, _kv_bytes_per_elem("f16"))padded_v_width = None if flash_attn else self._max_kv_value_width(val_len, val_len_swa)@@ -20091,6 +20233,7 @@ class LlamaCppBackend:flash_attn: bool = True,vram_fraction: Optional[float] = None,scratch_cache_type_kv: Optional[str] = None,+ explicit_ctx: bool = False,) -> tuple[int, int, list[int], Optional[list[int]]]:"""Plan a ``--split-mode tensor`` load. Pure: no model or GPU needed.@@ -20122,6 +20265,9 @@ class LlamaCppBackend:``scratch_cache_type_kv`` is the LIGHTER one and prices the dequant scratch,so an asymmetric pair keeps both terms conservative. Defaults to``cache_type_kv`` when unset, which is the symmetric case.++ ``explicit_ctx`` only matters when the KV cache cannot be sized: the fallback is then+ a guess, so it may shrink Auto but never overrule a request (#9653)."""# Per-GPU usable budget: free - (1-frac)*total, else (unknown total, e.g. a@@ -20220,7 +20366,11 @@ class LlamaCppBackend:)def _cc_ctx(ctx: int) -> int:- return n_dev * self._compute_buffer_ctx_bytes(ctx, n_ubatch, cc_cache_type)+ # The same resolved state ``_kv_at`` prices with: tensor mode usually answers+ # True, but not always, and the mask this buffer carries is f32 when it does not.+ return n_dev * self._compute_buffer_ctx_bytes(+ ctx, n_ubatch, cc_cache_type, flash_attn = flash_attn+ )def _fit_ctx(ctx: int) -> int:# Largest context whose KV (+ MTP draft reserve + context-linear@@ -20248,7 +20398,7 @@ class LlamaCppBackend:hi = mid - 1return best# KV size unknown -> can't prove a safe cap; floor.- return min(4096, ctx) if ctx > 0 else 4096+ return min(_TP_UNMEASURED_CTX, ctx) if ctx > 0 else _TP_UNMEASURED_CTX# max_available_ctx is the hardware ceiling for the UI bound, sized from# the native context independent of an explicit small -c (which only@@ -20256,6 +20406,11 @@ class LlamaCppBackend:max_ctx_target = max_target_ctx if (max_target_ctx and max_target_ctx > 0) else target_ctxmax_available_ctx = _fit_ctx(max_ctx_target)effective_ctx = min(_fit_ctx(target_ctx), max_available_ctx)+ if not self._can_estimate_kv() and explicit_ctx and target_ctx > 0:+ # The min() above would apply a guess to a typed context: 262,144 requested came+ # back as 4,096, for the launch and for max_context_length (#9653).+ effective_ctx = target_ctx+ max_available_ctx = max(max_available_ctx, effective_ctx)min_usable_mib = min(usable_by_idx.values())kv_bytes = _kv_at(effective_ctx) if (self._can_estimate_kv() and effective_ctx > 0) else 0@@ -20378,7 +20533,11 @@ class LlamaCppBackend:# card more, so it both under-refuses and over-refuses. Mirror the# planner: flat per device, alongside the compute-graph reserve.cc_per_device_bytes = self._compute_buffer_ctx_bytes(- effective_ctx, n_ubatch, scratch_cache_type_kv or cache_type_kv+ effective_ctx,+ n_ubatch,+ scratch_cache_type_kv or cache_type_kv,+ # Same state as the KV above.+ flash_attn = flash_attn,)total_bytes = model_size + kv_bytes + mtp_bytes + max(0, soft_overhead_bytes)@@ -22695,9 +22854,6 @@ class LlamaCppBackend:_pipeline_parallel_off = _pipeline_parallel_disabled_by_args(extra_args, n_layers = self._n_layers)- # A hard-crash recovery may relaunch this same plan with FA off.- # Size that larger cache up front so the recovery cannot OOM.- planned_flash_attn = Falsecache_override = parse_cache_override(extra_args)# Budget the heavier of asymmetric --cache-type-k/-v extras (they# win per axis at launch, appended last); resolve_cache_type_kv only@@ -22741,13 +22897,57 @@ class LlamaCppBackend:None if _cache_type_from_env else cache_type_kv,extra_args,)- # Price compute for the launch; KV separately covers the no-flash retry.- _launch_flash_attn = _planned_flash_attn(+ # Auto leaves this negative, which is llama.cpp's default and no override.+ _manual_gpu_layers = gpu_layers if gpu_memory_mode == "manual" else None+ # After the cache pair, because a quantized V forces flash attention on. NOT+ # pinned False for the no-flash recovery: --fit re-places that respawn.+ planned_flash_attn = _reserved_flash_attn_state(+ _planned_flash_attn_state(+ extra_args,+ planned_cache_types = _planned_cache_pair,+ supports_flash_attn = bool(server_caps.get("supports_flash_attn", True)),+ tensor_parallel = tensor_parallel,+ architecture = self._architecture,+ ),extra_args,- bool(server_caps.get("supports_flash_attn", True)),- _planned_cache_pair[1],- self._architecture,+ tensor_parallel = tensor_parallel,+ gpu_layers = _manual_gpu_layers,)++ def _replan_env(_current_tp: bool) -> "Optional[Mapping[str, str]]":+ """The environment the CHILD will see: the launch clears a non-layer+ inherited LLAMA_ARG_SPLIT_MODE, which otherwise turns tensor mode back+ on inside the helper."""+ if _current_tp:+ return None+ inherited = (os.environ.get("LLAMA_ARG_SPLIT_MODE") or "").strip().lower()+ if not inherited or inherited == "layer":+ return None+ return {+ key: value+ for key, value in os.environ.items()+ if key not in ("LLAMA_ARG_SPLIT_MODE", "LLAMA_ARG_TENSOR_SPLIT")+ }++ def _replanned_flash_attn(_current_tp: bool) -> bool:+ """Re-resolve the planned attention after a tensor-mode downgrade: AUTO+ plans it ON under tensor, so a stale plan budgets a V layer never gets."""+ _env = _replan_env(_current_tp)+ return _reserved_flash_attn_state(+ _planned_flash_attn_state(+ extra_args,+ planned_cache_types = _planned_cache_pair,+ supports_flash_attn = bool(server_caps.get("supports_flash_attn", True)),+ tensor_parallel = _current_tp,+ architecture = self._architecture,+ env = _env,+ ),+ extra_args,+ tensor_parallel = _current_tp,+ gpu_layers = _manual_gpu_layers,+ env = _env,+ )+# A user --split-mode in extras last-wins-overrides the toggle, and# an inherited tensor LLAMA_ARG_SPLIT_MODE flips it on (the child# would run tensor unbudgeted otherwise). The duplicate-load matchers@@ -22766,6 +22966,9 @@ class LlamaCppBackend:)tensor_parallel = Falsetensor_split = None+ # The plan was made under tensor, the one state forcing AUTO on. Same+ # for every later downgrade.+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)# Record the requested strategy for /status and the load# response. 'manual' has no fallback, so the request value is the# value actually applied.@@ -22807,6 +23010,7 @@ class LlamaCppBackend:"than 2 GPUs are in use; ignoring (needs >= 2).")tensor_parallel = False+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)if tensor_parallel and gpu_memory_mode == "manual" and gpu_layers < 0:logger.info("Manual mode with Auto layers hands memory management to "@@ -22814,6 +23018,7 @@ class LlamaCppBackend:"parallelism; ignoring the tensor split.")tensor_parallel = False+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)if ctx_override is not None and ctx_override > 0:logger.info(f"User --ctx-size {ctx_override} honored; skipping auto-reduce")if cache_override is not None:@@ -22961,6 +23166,12 @@ class LlamaCppBackend:# a ``gpu_indices`` the arm rebuilt from a WIDER device set than the# planner's reserve filter admitted, and index a card that is not in it._tp_planned = False+ # Held rather than recorded where it is decided: _record_load_warning is+ # first-notice-wins, and both arms that raise an unmeasured ceiling run+ # long before the host-RAM and offload advisories, so recording there took+ # the slot from a memory warning on the one path that also raises the+ # launched context. Flushed past those two, appending if one spoke.+ _unmeasured_ctx_notice: Optional[str] = Nonetotal_by_idx: dict[int, int] = {}_gpu_mem: list[tuple[int, int, int]] = []model_size = None # set in the fit try; used by the APU RAM guard@@ -23194,6 +23405,7 @@ class LlamaCppBackend:# otherwise reach llama-server. Strip it like the TP# downgrade does.extra_args = strip_split_mode_only(extra_args)+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)elif gpu_memory_mode == "manual":# Manual offload (--gpu-layers + --fit off): no automatic# device masking (a gpu_ids pick still pins below) or@@ -23214,6 +23426,7 @@ class LlamaCppBackend:# those.if tensor_parallel or split_mode_override == "tensor":extra_args = strip_split_mode_only(extra_args)+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)# Will MTP engage? If so, auto-fit reserves draft-model VRAM.# Mirrors _build_speculative_flags: forced mtp/mtp+ngram always@@ -23485,7 +23698,6 @@ class LlamaCppBackend:_swa_full: bool = swa_full,_kv_unified: bool = planned_kv_unified,_n_ubatch: Optional[int] = _effective_ubatch,- _flash_attn: bool = planned_flash_attn,) -> int:v = self._estimate_mtp_overhead_bytes(ctx,@@ -23500,7 +23712,13 @@ class LlamaCppBackend:swa_full = _swa_full,kv_unified = _kv_unified,n_ubatch = _n_ubatch,- flash_attn = _flash_attn,+ # Read at call time, like the KV and compute terms+ # beside it: a downgrade re-plans the attention, and a+ # default argument would hold the tensor plan's True+ # while those price the layer split's padded V, which+ # short-changes the draft reserve 4.5x on a hybrid SWA+ # draft head. The other captures settle with the load.+ flash_attn = planned_flash_attn,)return v if v is not None else 0@@ -23555,12 +23773,13 @@ class LlamaCppBackend:# per-device rate also steps up once split, unless llama.cpp# declines the pipeline parallelism that causes the step.# A nonzero ``slots`` prices that candidate count and its micro-batch.+ # Read at call time: a tensor-mode downgrade re-plans it.return max(1, n_gpus) * self._compute_buffer_ctx_bytes(ctx,_ubatch_for_slots(slots) if slots else _effective_ubatch,_scratch_cache_type_kv,layer_split = n_gpus > 1 and not _pipeline_parallel_off,- flash_attn = _launch_flash_attn,+ flash_attn = planned_flash_attn,n_parallel = slots or n_parallel,)@@ -23651,6 +23870,7 @@ class LlamaCppBackend:# than let the layer planner settle on the first card it fits._layer_min_gpus = max(_layer_min_gpus, len(gpus))extra_args = strip_split_mode_only(extra_args)+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)if tensor_parallel and self._tensor_split_aborts(binary, model_identifier, _planned_cache_pair):@@ -23667,6 +23887,7 @@ class LlamaCppBackend:_layer_min_gpus = max(_layer_min_gpus, len(gpus))# An extras --split-mode tensor would re-engage tensor mode.extra_args = strip_split_mode_only(extra_args)+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)# Tensor mode replicates a compute buffer on every GPU, so drop# GPUs below that reserve from the set up front (gpu_indices@@ -23716,6 +23937,7 @@ class LlamaCppBackend:if len(gpus) >= 2:_layer_min_gpus = max(_layer_min_gpus, len(gpus))extra_args = strip_split_mode_only(extra_args)+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)# Normalize the speculative reserve BEFORE anything prices it.# _mtp_bytes reads mtp_overhead_fn at call time, so leaving this@@ -24292,6 +24514,7 @@ class LlamaCppBackend:if len(tp_gpus) >= 2:_layer_min_gpus = max(_layer_min_gpus, len(tp_gpus))extra_args = strip_split_mode_only(extra_args)+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)# Layer split now, so the withheld verdict applies -- but not# when a GPU drafter is still in play. The drafter-drop probe# is gated off under tensor parallelism, so it has not run;@@ -24363,9 +24586,15 @@ class LlamaCppBackend:# would otherwise size placement against a fraction the# ranking above never used.vram_fraction = _vram_frac,+ explicit_ctx = explicit_ctx,)use_fit = False_tp_planned = True+ if explicit_ctx and not self._can_estimate_kv():+ # Tensor mode emits --fit off, so nothing downstream catches it.+ _unmeasured_ctx_notice = self._unmeasured_context_notice(+ effective_ctx, cache_type_kv+ )elif gpus and self._can_estimate_kv() and effective_ctx > 0:# Compute the largest hardware-aware cap from the model's# native context across all usable GPU subsets (for UI@@ -24627,6 +24856,9 @@ class LlamaCppBackend:# The other measured verdict: nothing fits, so there is no ceiling# to name and every explicit request over-commits._apple_nothing_fits = False+ # No measurement: the floor below is a guess, so it may neither+ # refuse nor be published as a ceiling (#9653).+ _apple_ctx_unmeasured = False# Reserve the flat MTP fraction up front like the discrete# _pin_fraction, so an unsized MTP draft (e.g. Qwen3.6-MTP, #6529)# can't over-commit. No-op when MTP is off; exclusive with the@@ -24746,8 +24978,16 @@ class LlamaCppBackend:# and floor to _FIT_MIN_CTX rather than launch at native and# over-commit.max_available_ctx = min(_metal_floor_ctx, native_ctx_for_cap)+ _apple_ctx_unmeasured = Trueif not explicit_ctx:effective_ctx = max_available_ctx+ elif _apple_ctx_unmeasured:+ # This arm never overruled the request but published the floor:+ # max_context_length 4,096 for a load launched at 262,144 (#9653).+ max_available_ctx = max(max_available_ctx, effective_ctx)+ _unmeasured_ctx_notice = self._unmeasured_context_notice(+ effective_ctx, cache_type_kv+ )elif ((_apple_measured_ceiling is not None or _apple_nothing_fits)and not _caller_owns_budget@@ -27825,6 +28065,39 @@ class LlamaCppBackend:# fallback). A per-call flag left those successes unrecorded._did_rocm_retry = False+ def _enable_managed_fit_for_no_flash(fa_cmd: list) -> list:+ """Unsloth's own ``--fit off`` back to ``on`` for a --flash-attn off respawn.++ The reserve is only safe because the respawn gets re-placed, and it+ inherits a managed ``--fit off`` that `_fit_off_retry_eligible` will not+ override, so without this it OOMs on the smaller-cache placement. Skipped+ where the flip buys nothing: a user's own ``--fit``, or a fixed count.+ """+ if "--fit" not in fa_cmd:+ return fa_cmd+ if any(+ _flag_name(str(token)) in {"-fit", "--fit"} for token in (extra_args or ())+ ):+ return fa_cmd+ # An inherited LLAMA_ARG_FIT=off is the user's too, and only manual mode+ # scrubs it: the flip would override it on the CLI, and the reserve+ # _user_fit_disabled already took prices this respawn where it lands.+ if _user_fit_disabled(extra_args, env = env):+ return fa_cmd+ if _placement_is_fitter_proof(fa_cmd, gpu_layers = _manual_gpu_layers, env = env):+ return fa_cmd+ index = fa_cmd.index("--fit")+ if index + 1 >= len(fa_cmd) or str(fa_cmd[index + 1]).strip().lower() == "on":+ return fa_cmd+ flipped = list(fa_cmd)+ flipped[index + 1] = "on"+ logger.info(+ "Re-enabling the fitter for the --flash-attn off retry: the padded "+ "V cache the retry runs with is bigger than the placement the fit "+ "priced with flash attention on."+ )+ return flipped+def _drop_fit_load_mode_for_no_flash(fa_cmd: list) -> list:"""The fit's ``--load-mode none`` off a --flash-attn off respawn.@@ -28447,6 +28720,15 @@ class LlamaCppBackend:max_available_ctx if max_available_ctx > 0 else self._effective_context_length)+ # Past the host-RAM and offload advisories, so it can say the ceiling is+ # the request without costing the user a memory warning it does not+ # replace: appended when one of those spoke, recorded when none did.+ if _unmeasured_ctx_notice:+ if self._last_load_warning:+ self._amend_load_warning(" " + _unmeasured_ctx_notice)+ else:+ self._record_load_warning(_unmeasured_ctx_notice)+# LoadRequest carries this flag, so a stale rollback, an API caller or# a swapped-out runtime can ask for a replay that never happened. Hold# it to the crash path's bar (managed auto Vulkan, no explicit@@ -29037,6 +29319,7 @@ class LlamaCppBackend:"--flash-attn off retry.")_fa_cmd = _drop_fit_load_mode_for_no_flash(_fa_cmd)+ _fa_cmd = _enable_managed_fit_for_no_flash(_fa_cmd)_flash_attn_known_off = Truecmd = _fa_cmdhealthy = _spawn_and_wait(_fa_cmd, label = "-noflash")@@ -29111,6 +29394,7 @@ class LlamaCppBackend:"--flash-attn off retry.")_fa_cmd = _drop_fit_load_mode_for_no_flash(_fa_cmd)+ _fa_cmd = _enable_managed_fit_for_no_flash(_fa_cmd)_flash_attn_known_off = Truecmd = _fa_cmdhealthy = (@@ -29548,12 +29832,10 @@ class LlamaCppBackend:0,int(self._DEFAULT_N_UBATCH if _launched_ubatch is None else _launched_ubatch),)- self._flash_attn_enabled = (- _flash_attn_enabled_from_args(- _last_spawn_cmd, default = not _flash_attn_known_off, env = env- )- and self._architecture != "grok"- )+ # What the child is RUNNING: the same architecture rule the plan above used.+ self._flash_attn_enabled = _flash_attn_enabled_from_args(+ _last_spawn_cmd, default = not _flash_attn_known_off, env = env+ ) and not _architecture_forces_flash_attn_off(self._architecture)self._effective_cache_types = _effective_main_cache_types(_last_spawn_cmd,env,diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 3dfd8fefb0c..f22fcf49b1c 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1618,9 +1618,10 @@ try:_kv_bytes_per_elem,_kv_unified_from_args,_metal_device_is_paravirtual,- _planned_flash_attn,+ _planned_flash_attn_state,_planned_main_cache_types,_planned_scratch_cache_type,+ _reserved_flash_attn_state,_swa_full_from_args_or_env,detect_reasoning_flags,paravirtual_normalized_request,@@ -1680,9 +1681,10 @@ except ImportError:_kv_bytes_per_elem,_kv_unified_from_args,_metal_device_is_paravirtual,- _planned_flash_attn,+ _planned_flash_attn_state,_planned_main_cache_types,_planned_scratch_cache_type,+ _reserved_flash_attn_state,_swa_full_from_args_or_env,detect_reasoning_flags,paravirtual_normalized_request,@@ -10733,6 +10735,10 @@ def _gguf_runtime_bytes(ctx_last_wins: bool = False,model_identifier: Optional[str] = None,launch_required_ubatch: int = 0,+ # False prices the plan the launch runs, which is what the panel shows. True also+ # keeps load_model's reserve for a flash-attention-off respawn that the fitter cannot+ # re-place, which is what a guard admitting a load beside training has to hold.+ reserve_no_flash_respawn: bool = False,) -> _GgufRuntimeBytes:"""KV-cache and compute-buffer VRAM (bytes) at the larger of max_seq_length andany `--ctx-size`/`-c` override, over n_parallel slots at the effective@@ -10829,7 +10835,6 @@ def _gguf_runtime_bytes(# pads V to f16 and charges the whole quantized saving back. Measured on# llama-server b10632, Qwen3-0.6B at 32k with -ctk/-ctv q4_0: 2296 MiB reserved# against 1008 MiB allocated, identical on the CPU and Vulkan builds.- planned_v_type = planned_cache_types[1]# the loader raises --batch-size to max(slots, 2) before launch, and llama.cpp# caps the micro-batch against it, so budget from the emitted value. Diffusion# takes neither flag, and SWA metadata prices the KV against the micro-batch,@@ -10901,8 +10906,8 @@ def _gguf_runtime_bytes(# so the default here is ON, not off. The false arm pads variable-width V# tensors to the model-wide maximum (_max_kv_value_width), which on an# architecture whose global and SWA layers disagree about n_embd_v_gqa- # inflates the whole cache and can call a load that fits an overflow.- # Resolved the way the launch resolves it, narrowed by the capability probe.+ # inflates the whole cache and can call a load that fits an overflow. Resolved+ # the way the launch resolves it, Grok's forced-off exception first._fa_supported = Truetry:_fa_caps = LlamaCppBackend.probe_server_capabilities()@@ -10913,12 +10918,30 @@ def _gguf_runtime_bytes(_fa_supported = Falseexcept Exception as _fa_exc:logger.debug("flash-attention capability probe failed: %s", _fa_exc)- flash_attn = _planned_flash_attn(+ # Shared with load_model so the estimate and the launch cannot answer differently+ # (#9697, #10489).+ flash_attn = _planned_flash_attn_state(llama_extra_args,- _fa_supported,- planned_v_type,- getattr(probe, "_architecture", None),- )+ planned_cache_types = planned_cache_types,+ supports_flash_attn = _fa_supported,+ tensor_parallel = bool(tensor_parallel),+ architecture = getattr(probe, "_architecture", None),+ env = os.environ,+ )+ if reserve_no_flash_respawn:+ # load_model holds the reading down where a flash-attention-off respawn cannot+ # be re-placed, and the training guard is admitting a load against the worst+ # this placement can reach rather than what it opens with. Resolving only the+ # plan admitted it against the smaller cache that the same placement then+ # reserves the larger one for, so the respawn could take VRAM the guard never+ # admitted and take a training run with it. The panel keeps the plan: its total+ # is what the launch uses, and is placement-independent.+ flash_attn = _reserved_flash_attn_state(+ flash_attn,+ llama_extra_args,+ tensor_parallel = bool(tensor_parallel),+ env = os.environ,+ )kv = probe._estimate_kv_cache_bytes(ctx,cache_type_for_budget,@@ -11055,6 +11078,7 @@ def _estimate_gguf_kv_gb(is_diffusion = is_diffusion,model_identifier = model_identifier,launch_required_ubatch = launch_required_ubatch,+ reserve_no_flash_respawn = True,)gpu_kv_bytes = (runtime.kv_bytesdiff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.pyindex fb5190b2430..6862fe0683a 100644--- a/studio/backend/routes/models.py+++ b/studio/backend/routes/models.py@@ -2177,6 +2177,18 @@ async def _require_model_access_or_caller_token(await asyncio.to_thread(account_access.authorize_download, model_name, "model", hf_token)+def _tensor_split_can_launch(tensor_parallel, flash_attn) -> bool:+ """Whether a load asking for a tensor split can actually take one.++ llama.cpp returns nullptr for "SPLIT_MODE_TENSOR requires flash_attn to be enabled", so+ such a load falls back to a layer split. Only a RESOLVED False refuses; None is "not+ resolved", not evidence the child runs without flash attention.+ """+ if not tensor_parallel:+ return False+ return flash_attn is not False++@router.get("/config/{model_name:path}")async def get_model_config(model_name: str,@@ -3752,6 +3764,38 @@ async def get_kv_cache_estimate(False,description = "Load a vision GGUF without its mmproj, freeing the projector's VRAM",),+ flash_attn: Optional[bool] = Query(+ None,+ description = (+ "Flash attention state to price. Omit to resolve it the way the launch does "+ "(the build's capability, LLAMA_ARG_FLASH_ATTN, a last-wins -fa in the extra "+ "arguments, and a quantized V cache which forces it on). It is not a detail: "+ "with flash attention off llama.cpp floors the V axis at f16 and pads "+ "variable-width V tensors, which is a 1.44x KV cache at q8_0 and 2.28x at q4_0"+ ),+ ),+ kv_unified: Optional[bool] = Query(+ None,+ description = (+ "--kv-unified state to price; omit to resolve it as the launch does. Changes "+ "the sliding-window allowance on an SWA model, which is per slot when unified"+ ),+ ),+ swa_full: Optional[bool] = Query(+ None,+ description = (+ "--swa-full state to price; omit to resolve it as the launch does. Collapses "+ "an SWA model's two cache sizes into one full-context cache"+ ),+ ),+ no_mmproj_offload: Optional[bool] = Query(+ None,+ description = (+ "--no-mmproj-offload state to price; omit to resolve it as the launch does. "+ "A projector on the host is not VRAM, so pricing the wrong side moves the "+ "total by the whole mmproj"+ ),+ ),request: Request = None, # type: ignore[assignment]current_subject: str = Depends(get_current_subject),):@@ -3888,6 +3932,65 @@ async def get_kv_cache_estimate(except Exception as e:logger.debug(f"cache type resolution failed for '{repo_id}': {e}")+ # Taking the estimator's defaults while the loader resolved the same knobs+ # differently is why one model and cache type reported two KV caches (#10489).+ # An asked-for value is spelled as the extra argument a load would carry and+ # re-resolved by the launch's own helpers, never taken verbatim.+ # isinstance(..., bool): called in process, an omitted argument arrives as the+ # ``Query`` default object, which is truthy.+ _asked_flash_attn = flash_attn if isinstance(flash_attn, bool) else None+ _asked_kv_unified = kv_unified if isinstance(kv_unified, bool) else None+ _asked_swa_full = swa_full if isinstance(swa_full, bool) else None+ _asked_no_mmproj = no_mmproj_offload if isinstance(no_mmproj_offload, bool) else None+ _plan_extra_args: list[str] = []+ if _asked_flash_attn is not None:+ _plan_extra_args += ["--flash-attn", "on" if _asked_flash_attn else "off"]+ if _asked_kv_unified is not None:+ _plan_extra_args += ["--kv-unified" if _asked_kv_unified else "--no-kv-unified"]+ if _asked_swa_full:+ # Enable-only: llama.cpp has no --no-swa-full, so a false leaves the env to+ # answer.+ _plan_extra_args += ["--swa-full"]+ if _asked_no_mmproj is not None:+ _plan_extra_args += [+ "--no-mmproj-offload" if _asked_no_mmproj else "--mmproj-offload"+ ]+ _planner_extras = _plan_extra_args or None++ _plan_kwargs: dict = {}+ try:+ from core.inference.llama_cpp import (+ _kv_unified_from_args,+ _planned_flash_attn_state,+ _planned_main_cache_types as _plan_cache_types,+ _swa_full_from_args_or_env,+ )++ _plan_caps = {}+ try:+ _plan_caps = LlamaCppBackend.probe_server_capabilities() or {}+ except Exception as e:+ logger.debug(f"capability probe failed for '{repo_id}': {e}")+ _plan_kwargs = {+ "flash_attn": _planned_flash_attn_state(+ _planner_extras,+ planned_cache_types = _plan_cache_types(cache_type_kv, _planner_extras),+ # An unreadable probe keeps the managed default.+ supports_flash_attn = bool(_plan_caps.get("supports_flash_attn", True)),+ tensor_parallel = bool(tensor_parallel),+ architecture = getattr(be, "_architecture", None),+ ),+ # The loader's own default: unified only for >1 slot, only if supported.+ "kv_unified": _kv_unified_from_args(+ _planner_extras,+ default = (n_parallel or 1) > 1+ and bool(_plan_caps.get("supports_kv_unified", False)),+ ),+ "swa_full": _swa_full_from_args_or_env(_planner_extras),+ }+ except Exception as e:+ logger.debug(f"attention plan resolution failed for '{repo_id}': {e}")+# Probe failures keep the unflagged default rather than assuming zero._cc_caps: dict = {}_total_ram_mib: Optional[int] = None@@ -3914,6 +4017,7 @@ async def get_kv_cache_estimate(n_parallel = n_parallel,ctx_checkpoints = _effective_checkpoints,n_ubatch = n_ubatch,+ **_plan_kwargs,)# Report the host-resident checkpoint share separately from GPU cache bytes.@@ -3925,6 +4029,7 @@ async def get_kv_cache_estimate(n_parallel = n_parallel,ctx_checkpoints = 0,n_ubatch = n_ubatch,+ **_plan_kwargs,)kv_checkpoint = max(0, int(kv) - int(_kv_without))@@ -4000,6 +4105,18 @@ async def get_kv_cache_estimate(projector = int(_Be._get_gguf_size_bytes(mmproj) * _Be._MMPROJ_VRAM_SAFETY)except Exception as e:logger.debug(f"mmproj estimate failed for '{repo_id}' {quant}: {e}")+ # The RESOLVED placement, not the query value: the env alone can put the+ # projector on the host, and the frontend adds projectorBytes onto its GPU+ # weights segment, so the bar was charged for memory that never reaches the card.+ try:+ from core.inference.llama_cpp import _resolved_mmproj_offload+ _mmproj_offloaded = _resolved_mmproj_offload(_planner_extras)+ except Exception as e: # noqa: BLE001 -- cannot resolve -> the asked value stands+ logger.debug(f"could not resolve the mmproj placement: {e}")+ _mmproj_offloaded = None if _asked_no_mmproj is None else not _asked_no_mmproj+ if _mmproj_offloaded is False:+ # The projector is in HOST memory, with vision still on.+ projector = None# Only the MTP modes reserve memory; ngram is free. "auto" may or may not resolve to MTP, and the estimator# returns None when it does not. Guarded separately: the MTP path reads more metadata than the KV path, and@@ -4091,6 +4208,8 @@ async def get_kv_cache_estimate(# 16. Blank is not zero: _build_speculative_flags emits its own default when the field is unset (2 with a# GPU, 3 without) and the rollback state is multiplied by it. An explicit 0 is still honoured.spec_draft_n_max = _effective_draft_n_max,+ # Same estimator, so it must get the same resolved plan.+ **_plan_kwargs,)# Plus the draft decode graph's floor, which the helper leaves to# the loader's soft overhead.@@ -4131,6 +4250,18 @@ async def get_kv_cache_estimate(_effective_tp = _effective_tensor_parallel(None, bool(tensor_parallel))except Exception as e:logger.debug(f"tensor mode resolution failed for '{repo_id}': {e}")+ if _effective_tp and not _tensor_split_can_launch(+ _effective_tp, _plan_kwargs.get("flash_attn")+ ):+ logger.debug(+ f"'{repo_id}': flash attention is off, so the launch cannot take a tensor "+ "split; pricing the layer split it would fall back to"+ )+ _effective_tp = False+ # Into the extras as well, not only the boolean: the breakdown re-resolves+ # the split through a helper that reads an inherited+ # LLAMA_ARG_SPLIT_MODE=tensor, which would turn a bare False back on.+ _planner_extras = list(_planner_extras or []) + ["--split-mode", "layer"]_planner_devices = 1if _effective_tp:from routes.inference import (@@ -4143,6 +4274,9 @@ async def get_kv_cache_estimate(None, _cached_inference_devices(), tensor_parallel = True),)+ # The planner resolves its plan from extra arguments, which is why the plan+ # was built in that vocabulary: without it, gpu_bytes and kv_bytes in ONE+ # response describe two different loads._cfg = _cached_estimate_config(repo_id, quant, None, False)if _cfg is not None and _cfg is not _ESTIMATE_NOT_ON_DISK:_cfg = _localized_estimate_config(_cfg, path)@@ -4159,8 +4293,11 @@ async def get_kv_cache_estimate(spec_draft_cache_type = spec_draft_cache_type,n_batch = n_batch,n_ubatch = n_ubatch,- tensor_parallel = tensor_parallel,+ # The RESOLVED split: _gguf_memory_breakdown re-resolves it, so the+ # raw toggle turned tensor mode straight back on.+ tensor_parallel = _effective_tp,n_devices = _planner_devices,+ llama_extra_args = _planner_extras,)if _b is not None:# `or None` would fold a real zero into "no answer". Zero is meaningful: inherited placement such as@@ -4187,8 +4324,10 @@ async def get_kv_cache_estimate(spec_draft_cache_type = spec_draft_cache_type,n_batch = n_batch,n_ubatch = n_ubatch,- tensor_parallel = tensor_parallel,+ # A floor priced for an impossible placement is not a floor.+ tensor_parallel = _effective_tp,n_devices = _planner_devices,+ llama_extra_args = _planner_extras,)if _floor is not None:planner_floor = min(int(_floor.gpu_bytes), planner_gpu)diff --git a/studio/backend/tests/test_estimator_flash_attn_parity.py b/studio/backend/tests/test_estimator_flash_attn_parity.pyindex 0d6be183208..810315c016e 100644--- a/studio/backend/tests/test_estimator_flash_attn_parity.py+++ b/studio/backend/tests/test_estimator_flash_attn_parity.py@@ -90,6 +90,33 @@ def _clear_flash_attn_parity_caches():ri._estimate_config_cache.clear()+_RAGGED_SWA_FIELDS = {+ "context_length": 131072,+ "block_count": 30,+ "attention.head_count": 16,+ "attention.head_count_kv": 8,+ "embedding_length": 2816,+ "attention.key_length": 512,+ "attention.value_length": 512,+ "attention.key_length_swa": 256,+ "attention.value_length_swa": 256,+ "attention.sliding_window": 1024,+ "attention.sliding_window_pattern": 6,+}+++def ragged_swa_gguf(tmp_path):+ """Gemma-class: the sliding-window layers carry a narrower V, padded to the model-wide+ maximum when flash attention is off, so even an f16 cache moves with the state. A+ quantized cache cannot serve here, because a quantized V forces flash attention on."""+ fields = {"general.architecture": "gemma3"}+ fields.update({f"gemma3.{k}": v for k, v in _RAGGED_SWA_FIELDS.items()})+ path = tmp_path / "ragged-swa.gguf"+ path.write_bytes(_make_gguf_bytes("gemma3", fields))+ return str(path)++@pytest.fixturedef qwen3_shaped_gguf(tmp_path):# general.architecture goes in FIRST. The reader takes it before it knows which@@ -383,3 +410,61 @@ def test_grok_is_priced_without_flash_attention(monkeypatch, qwen3_shaped_gguf):monkeypatch.setattr(ri.LlamaCppBackend, "_read_gguf_metadata", read_as_grok)assert _runtime(qwen3_shaped_gguf) == off+++def test_the_admission_estimate_keeps_the_no_flash_reserve(monkeypatch, ragged_swa_gguf):+ """``_estimate_gguf_kv_gb`` feeds the active-training admission guard, and ``load_model``+ holds the reading down through ``_reserved_flash_attn_state`` where a flash-attention-off+ respawn cannot be re-placed. Resolving only the plan here admitted a load against the+ smaller cache that the same placement then reserves the larger one for, so the respawn+ could take VRAM the guard never admitted and take a training run with it.++ A ragged-SWA shape with an f16 cache: a quantized V forces flash attention on, and an+ equal-width f16 cache does not move with the state, so on either of those the control+ below could not fail."""+ monkeypatch.delenv("LLAMA_ARG_FLASH_ATTN", raising = False)+ monkeypatch.delenv("LLAMA_ARG_FIT", raising = False)+ monkeypatch.delenv("LLAMA_ARG_N_GPU_LAYERS", raising = False)+ _flash_attn_caps(monkeypatch, True)++ def kv(extras = None, tensor_parallel = False):+ return ri._gguf_runtime_bytes(+ ragged_swa_gguf,+ 32768,+ extras,+ 1,+ "f16",+ tensor_parallel,+ reserve_no_flash_respawn = True,+ ).kv_bytes++ def panel(extras = None):+ """What the panel shows: the plan, not the reserve."""+ return ri._gguf_runtime_bytes(ragged_swa_gguf, 32768, extras, 1, "f16", False).kv_bytes++ managed = kv()+ priced_off = kv(["--flash-attn", "off"])+ # The control: the two readings really do differ on this shape, or nothing below holds.+ assert priced_off > managed++ # A user's own --fit off, and the environment spelling of it: both take away the+ # re-placement, so both keep the conservative reserve.+ assert kv(["--fit", "off"]) == priced_off+ monkeypatch.setenv("LLAMA_ARG_FIT", "off")+ assert kv() == priced_off+ monkeypatch.delenv("LLAMA_ARG_FIT")++ # A fixed layer count is the other fitter-proof placement.+ assert kv(["-ngl", "99"]) == priced_off++ # An ordinary managed placement keeps the launch price: the fitter re-places the+ # respawn, so reserving for it would refuse contexts that fit.+ assert kv() == managed++ # Tensor mode is exempt, because llama.cpp has no flash-attention-off respawn there.+ assert kv(["--fit", "off"], tensor_parallel = True) == managed++ # And the panel is untouched: its total is what the launch uses, and an existing pin+ # holds it placement-independent (test_memory_estimate's extras -ngl case).+ assert panel(["--fit", "off"]) == panel()+ assert panel(["-ngl", "99"]) == panel()diff --git a/studio/backend/tests/test_grok_flash_attn_exception.py b/studio/backend/tests/test_grok_flash_attn_exception.pynew file mode 100644index 00000000000..bf126502170--- /dev/null+++ b/studio/backend/tests/test_grok_flash_attn_exception.py@@ -0,0 +1,282 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Grok is priced without flash attention, whatever else the launch resolves.++``llama_init_from_model`` forces it off for LLM_ARCH_GROK above both later upgrades, so+answering "on" would publish a context the child cannot hold. The rule is easy to lose in a+merge (#11043 against #9697, #10489), so these tests hold both ends: the rule itself, and+the architecture reaching the resolver at every sizing seam.+"""++from __future__ import annotations++import ast+import importlib.util+import inspect+import sys+import textwrap+from pathlib import Path++import pytest++_TESTS_DIR = Path(__file__).resolve().parent+_BACKEND_DIR = str(_TESTS_DIR.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)+if str(_TESTS_DIR) not in sys.path:+ sys.path.insert(0, str(_TESTS_DIR))+++def _load(module_name: str, file_name: str):+ spec = importlib.util.spec_from_file_location(module_name, _TESTS_DIR / file_name)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++_placement = _load("_placement_for_grok_flash_attn", "test_llama_cpp_placement.py")+_plan = _load("_plan_for_grok_flash_attn", "test_resolved_flash_attn_plan.py")++from core.inference.llama_cpp import ( # noqa: E402+ LlamaCppBackend,+ _architecture_forces_flash_attn_off,+ _planned_flash_attn_state,+ _reserved_flash_attn_state,+)++NATIVE = _plan.NATIVE+++class TestTheRule:+ def test_grok_is_forced_off(self):+ assert _planned_flash_attn_state(architecture = "grok") is False++ def test_the_architecture_is_read_as_the_gguf_stores_it(self):+ assert _architecture_forces_flash_attn_off("grok") is True+ assert _architecture_forces_flash_attn_off("Grok") is True+ assert _architecture_forces_flash_attn_off(" grok ") is True++ @pytest.mark.parametrize("architecture", [None, "", "qwen3", "llama", "grok2", "deepseek2"])+ def test_every_other_architecture_keeps_the_managed_default(self, architecture):+ """One architecture, not a new default: an unread header keeps the managed on."""+ assert _architecture_forces_flash_attn_off(architecture) is False+ assert _planned_flash_attn_state(architecture = architecture) is True+++class TestNothingPutsItBack:+ @pytest.mark.parametrize("v_type", ["q8_0", "q4_0", "q5_1", "iq4_nl"])+ def test_a_quantized_v_cache_does_not(self, v_type):+ """The quantized-V upgrade sits below the Grok branch, so it never runs on Grok."""+ assert (+ _planned_flash_attn_state(planned_cache_types = ("f16", v_type), architecture = "grok")+ is False+ )+ # The same pair elsewhere still forces it on: not asserting a constant.+ assert (+ _planned_flash_attn_state(planned_cache_types = ("f16", v_type), architecture = "qwen3")+ is True+ )++ def test_a_tensor_split_does_not(self):+ assert (+ _planned_flash_attn_state(["-fa", "auto"], tensor_parallel = True, architecture = "grok")+ is False+ )+ assert (+ _planned_flash_attn_state(["-fa", "auto"], tensor_parallel = True, architecture = "qwen3")+ is True+ )++ def test_an_explicit_on_in_the_extras_does_not(self):+ assert _planned_flash_attn_state(["--flash-attn", "on"], architecture = "grok") is False+ assert _planned_flash_attn_state(["-fa", "1"], architecture = "grok") is False++ def test_the_inherited_environment_does_not(self):+ assert (+ _planned_flash_attn_state(architecture = "grok", env = {"LLAMA_ARG_FLASH_ATTN": "1"})+ is False+ )++ def test_the_reserve_wrapper_keeps_it_off(self):+ """The wrapper only holds a reading DOWN, including in tensor mode, the one case+ it returns ``planned`` unread."""+ for tensor_parallel in (False, True):+ planned = _planned_flash_attn_state(+ ["-fa", "on"],+ planned_cache_types = ("q8_0", "q8_0"),+ tensor_parallel = tensor_parallel,+ architecture = "grok",+ )+ assert (+ _reserved_flash_attn_state(planned, ["-fa", "on"], tensor_parallel = tensor_parallel)+ is False+ )+++def _grok_backend(tmp_path, architecture: str):+ backend, gguf = _plan._tensor_backend(tmp_path, free_mib = 12000)+ backend._architecture = architecture+ return backend, gguf+++def _states_seen(tmp_path, architecture: str, method: str, **load_kwargs):+ backend, gguf = _grok_backend(tmp_path, architecture)+ seen: list[object] = []+ real = getattr(backend, method)++ def spy(*args, **kwargs):+ seen.append(kwargs.get("flash_attn"))+ return real(*args, **kwargs)++ setattr(backend, method, spy)+ _placement._launch(backend, gguf, **load_kwargs)+ assert seen, f"the load took no {method} reading"+ return seen+++class TestTheLoadPricesGrokWithoutIt:+ """The load end to end. A quantized V under a tensor split is the hardest case: both+ upgrades ask at once, and on Grok neither gets it."""++ def test_the_kv_cache_is_priced_without_it(self, tmp_path):+ seen = _states_seen(+ tmp_path,+ "grok",+ "_estimate_kv_cache_bytes",+ n_ctx = NATIVE,+ cache_type_kv = "q8_0",+ tensor_parallel = True,+ )+ assert all(state is False for state in seen), (+ f"the load priced its KV cache with flash_attn states "+ f"{sorted(set(map(str, seen)))} on an architecture llama.cpp will not run "+ f"with flash attention"+ )++ def test_the_same_load_on_another_architecture_is_priced_with_it(self, tmp_path):+ """The control arm: without it the test above passes on an always-False resolver."""+ seen = _states_seen(+ tmp_path,+ "qwen3",+ "_estimate_kv_cache_bytes",+ n_ctx = NATIVE,+ cache_type_kv = "q8_0",+ tensor_parallel = True,+ )+ assert all(state is True for state in seen)++ def test_the_compute_buffer_is_priced_without_it(self, tmp_path):+ """The KQ mask is f16 with flash attention and f32 without, so the compute reserve+ moves with the state the KV cache does."""+ seen = _states_seen(+ tmp_path,+ "grok",+ "_compute_buffer_ctx_bytes",+ n_ctx = NATIVE,+ cache_type_kv = "q8_0",+ tensor_parallel = True,+ )+ assert all(state is False for state in seen), (+ f"the load priced its compute buffers with flash_attn states "+ f"{sorted(set(map(str, seen)))} on Grok"+ )++ def test_the_compute_buffer_control_arm_is_priced_with_it(self, tmp_path):+ seen = _states_seen(+ tmp_path,+ "qwen3",+ "_compute_buffer_ctx_bytes",+ n_ctx = NATIVE,+ cache_type_kv = "q8_0",+ tensor_parallel = True,+ )+ assert all(state is True for state in seen)++ def test_the_padded_cache_actually_costs_more(self, tmp_path):+ """Reachability: the two states must price differently, or the above assert nothing."""+ backend, _ = _grok_backend(tmp_path, "grok")+ on = backend._estimate_kv_cache_bytes(NATIVE, "q8_0", flash_attn = True)+ off = backend._estimate_kv_cache_bytes(NATIVE, "q8_0", flash_attn = False)+ assert off > on+ assert backend._compute_buffer_ctx_bytes(+ NATIVE, 512, "q8_0", flash_attn = False+ ) > backend._compute_buffer_ctx_bytes(NATIVE, 512, "q8_0", flash_attn = True)+++def _calls_to(source: str, name: str) -> list[ast.Call]:+ tree = ast.parse(textwrap.dedent(source))+ return [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and (+ (isinstance(node.func, ast.Name) and node.func.id == name)+ or (isinstance(node.func, ast.Attribute) and node.func.attr == name)+ )+ ]+++class TestEverySeamHandsTheArchitectureOver:+ """Read by AST, not text: a comment naming the architecture would pass a string match+ and a reflow would fail one, and neither says what the call passes."""++ def test_the_loader_does(self):+ source = inspect.getsource(inspect.unwrap(LlamaCppBackend.load_model))+ calls = _calls_to(source, "_planned_flash_attn_state")+ assert calls, "the launch path no longer resolves an attention plan"+ for call in calls:+ passed = {kw.arg for kw in call.keywords if kw.arg}+ assert "architecture" in passed, (+ "load_model resolves its attention plan without telling the resolver "+ "which architecture it is, so the Grok exception cannot fire"+ )++ def test_the_memory_estimate_route_does(self):+ import routes.inference as ri++ source = inspect.getsource(ri._gguf_runtime_bytes)+ calls = _calls_to(source, "_planned_flash_attn_state")+ assert calls, "the estimate no longer resolves an attention plan"+ for call in calls:+ assert "architecture" in {kw.arg for kw in call.keywords if kw.arg}++ def test_the_kv_cache_estimate_route_does(self):+ """Read from the file: importing the route drags the whole app in."""+ source = (Path(_BACKEND_DIR) / "routes" / "models.py").read_text(encoding = "utf-8")+ calls = _calls_to(source, "_planned_flash_attn_state")+ assert calls, "the kv-cache-estimate route no longer resolves an attention plan"+ for call in calls:+ assert "architecture" in {kw.arg for kw in call.keywords if kw.arg}+++class TestOneStateNotTwo:+ """The rule can also be lost by RE-DERIVING the state beside the resolved one, which is+ how the compute buffers came to be priced from their own reading."""++ def test_both_sizing_seams_price_the_one_resolved_state(self):+ source = inspect.getsource(inspect.unwrap(LlamaCppBackend.load_model))+ for method in ("_estimate_kv_cache_bytes", "_compute_buffer_ctx_bytes"):+ calls = _calls_to(source, method)+ assert calls, f"the launch path no longer calls {method}"+ for call in calls:+ passed = {+ kw.arg: kw.value+ for kw in call.keywords+ if kw.arg and isinstance(kw.value, ast.Name)+ }+ assert getattr(passed.get("flash_attn"), "id", None) == "planned_flash_attn", (+ f"{method} is priced from something other than the resolved "+ f"planned_flash_attn, so the two can disagree about the same load"+ )++ def test_there_is_only_one_resolver(self):+ """A duplicate carrying only some of the rules is worse than none."""+ import core.inference.llama_cpp as llama_cpp++ resolvers = sorted(+ name+ for name in dir(llama_cpp)+ if name.startswith("_planned_flash_attn") or name.startswith("_reserved_flash_attn")+ )+ assert resolvers == ["_planned_flash_attn_state", "_reserved_flash_attn_state"]diff --git a/studio/backend/tests/test_kv_cache_estimate_attention_plan.py b/studio/backend/tests/test_kv_cache_estimate_attention_plan.pynew file mode 100644index 00000000000..e1a91d7e250--- /dev/null+++ b/studio/backend/tests/test_kv_cache_estimate_attention_plan.py@@ -0,0 +1,219 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""/api/models/kv-cache-estimate answers for a named attention plan, not for the defaults.++The route took the estimator's signature defaults while the loader resolved the same three+knobs differently, so one model at one cache type put three numbers on screen (#10489).+The arithmetic itself is covered by tests/test_kv_cache_estimation.py.+"""++from __future__ import annotations++import asyncio+import sys+from pathlib import Path++import pytest++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from test_kv_cache_estimation import _make_gguf_bytes # noqa: E402++import routes.models as models_routes # noqa: E402+from core.inference.llama_cpp import LlamaCppBackend # noqa: E402++N_CTX = 32768++# Equal K/V widths, so the only flash-attention term that can move the total is the f16+# floor on V.+_PLAIN_GQA = {+ "context_length": 131072,+ "block_count": 32,+ "attention.head_count": 32,+ "attention.head_count_kv": 8,+ "embedding_length": 4096,+ "attention.key_length": 128,+ "attention.value_length": 128,+}++# Gemma-class: narrower V on the sliding-window layers, padded to the model-wide maximum+# when flash attention is off, so even an f16 cache moves.+_RAGGED_SWA = {+ "context_length": 131072,+ "block_count": 30,+ "attention.head_count": 16,+ "attention.head_count_kv": 8,+ "embedding_length": 2816,+ "attention.key_length": 512,+ "attention.value_length": 512,+ "attention.key_length_swa": 256,+ "attention.value_length_swa": 256,+ "attention.sliding_window": 1024,+ "attention.sliding_window_pattern": 6,+}+++def _write_gguf(+ path: Path,+ fields: dict,+ arch: str = "testarch",+) -> Path:+ kv = {"general.architecture": arch}+ for key, value in fields.items():+ kv[f"{arch}.{key}"] = value+ path.parent.mkdir(parents = True, exist_ok = True)+ path.write_bytes(_make_gguf_bytes(arch, kv))+ return path+++def _call_route(+ monkeypatch,+ *,+ path: Path,+ caps: dict | None = None,+ **overrides,+):+ """All passed explicitly: called in process an omitted one arrives as its ``Query``+ object rather than the None FastAPI would have resolved."""+ monkeypatch.setattr(+ models_routes,+ "_resolve_quant_gguf",+ lambda _repo, _quant, _local: (str(path), 4096),+ )+ monkeypatch.setattr(models_routes, "is_local_path", lambda _p: False, raising = False)+ monkeypatch.setattr(+ LlamaCppBackend,+ "probe_server_capabilities",+ classmethod(lambda cls, *a, **k: dict(caps if caps is not None else {"mtp_token": True})),+ )+ kwargs = dict(+ repo_id = "org/repo",+ quant = "Q4_K_M",+ n_ctx = N_CTX,+ cache_type_kv = None,+ n_parallel = 1,+ speculative_type = None,+ spec_draft_n_max = None,+ spec_draft_cache_type = None,+ ctx_checkpoints = None,+ disable_vision = False,+ n_batch = None,+ n_ubatch = None,+ tensor_parallel = False,+ flash_attn = None,+ kv_unified = None,+ swa_full = None,+ no_mmproj_offload = None,+ request = None,+ current_subject = "test",+ )+ kwargs.update(overrides)+ return asyncio.run(models_routes.get_kv_cache_estimate(**kwargs))+++def gqa(tmp_path):+ return _write_gguf(tmp_path / "gqa-Q4_K_M.gguf", _PLAIN_GQA)+++def ragged(tmp_path):+ return _write_gguf(tmp_path / "ragged-Q4_K_M.gguf", _RAGGED_SWA)+++class TestFlashAttention:+ def test_an_explicit_off_prices_the_padded_cache(self, monkeypatch, ragged):+ on = _call_route(monkeypatch, path = ragged, flash_attn = True)+ off = _call_route(monkeypatch, path = ragged, flash_attn = False)+ assert off["kv_bytes"] > on["kv_bytes"]++ def test_omitting_it_resolves_to_what_the_launch_emits(self, monkeypatch, ragged):+ default = _call_route(monkeypatch, path = ragged)+ assert (+ default["kv_bytes"]+ == _call_route(monkeypatch, path = ragged, flash_attn = True)["kv_bytes"]+ )++ def test_a_build_without_the_flag_prices_the_padded_cache(self, monkeypatch, ragged):+ blind = _call_route(monkeypatch, path = ragged)+ unsupported = _call_route(+ monkeypatch,+ path = ragged,+ caps = {"found": True, "supports_flash_attn": False},+ )+ assert unsupported["kv_bytes"] > blind["kv_bytes"]++ def test_a_quantized_v_cache_is_not_priced_without_it(self, monkeypatch, gqa):+ """An explicit off is not a launch that can happen: llama.cpp forces it on."""+ forced = _call_route(monkeypatch, path = gqa, cache_type_kv = "q8_0", flash_attn = False)+ resolved = _call_route(monkeypatch, path = gqa, cache_type_kv = "q8_0")+ assert forced["kv_bytes"] == resolved["kv_bytes"]++ def test_the_quantized_v_answer_is_the_narrow_one(self, monkeypatch, gqa):+ """2.125 bytes per element, not the 3.0625 the loader used to contradict it with."""+ q8 = _call_route(monkeypatch, path = gqa, cache_type_kv = "q8_0")+ f16 = _call_route(monkeypatch, path = gqa, cache_type_kv = "f16")+ assert q8["kv_bytes"] == pytest.approx(f16["kv_bytes"] * (2.125 / 4.0), rel = 1e-6)+++class TestTheOtherTwoLayoutKnobs:+ def test_swa_full_collapses_the_two_cache_sizes(self, monkeypatch, ragged):+ # Net of checkpoints: --swa-full zeroes the checkpoint share, so the totals can move+ # the other way while the attention cache itself still grows.+ compact = _call_route(monkeypatch, path = ragged)+ full = _call_route(monkeypatch, path = ragged, swa_full = True)+ # The share is None, not 0, when nothing is reserved.+ compact_attn = compact["kv_bytes"] - (compact["kv_checkpoint_bytes"] or 0)+ full_attn = full["kv_bytes"] - (full["kv_checkpoint_bytes"] or 0)+ assert full_attn > compact_attn++ def test_swa_full_drops_the_checkpoint_share(self, monkeypatch, ragged):+ """No sliding window left to snapshot, so any reported share is memory nobody+ reserves."""+ compact = _call_route(monkeypatch, path = ragged, ctx_checkpoints = 8)+ full = _call_route(monkeypatch, path = ragged, ctx_checkpoints = 8, swa_full = True)+ assert compact["kv_checkpoint_bytes"]+ assert not full["kv_checkpoint_bytes"]++ def test_the_unified_cache_is_an_input(self, monkeypatch, ragged):+ unified = _call_route(monkeypatch, path = ragged, n_parallel = 4, kv_unified = True)+ split = _call_route(monkeypatch, path = ragged, n_parallel = 4, kv_unified = False)+ assert unified["kv_bytes"] != split["kv_bytes"]++ def test_a_single_slot_resolves_to_the_launch_default(self, monkeypatch, ragged):+ """Unsloth asks for a unified cache only to serve more than one slot."""+ default = _call_route(monkeypatch, path = ragged, n_parallel = 1)+ assert (+ default["kv_bytes"]+ == _call_route(monkeypatch, path = ragged, n_parallel = 1, kv_unified = False)["kv_bytes"]+ )+++class TestTheContract:+ def test_the_new_parameters_are_all_optional(self):+ import inspect++ signature = inspect.signature(models_routes.get_kv_cache_estimate)+ for name in ("flash_attn", "kv_unified", "swa_full", "no_mmproj_offload"):+ assert name in signature.parameters, f"{name} is not on the route"+ assert signature.parameters[name].default is not inspect.Parameter.empty++ def test_the_response_shape_did_not_change(self, monkeypatch, gqa):+ """No response_model, so a strict client breaks on a new key."""+ from test_memory_estimate_contract_freeze import _KV_CACHE_ESTIMATE_KEYS+ assert set(_call_route(monkeypatch, path = gqa)) == set(_KV_CACHE_ESTIMATE_KEYS)++ def test_an_omitted_plan_does_not_move_the_answer_for_a_plain_model(self, monkeypatch, gqa):+ answer = _call_route(monkeypatch, path = gqa)+ assert (+ answer["kv_bytes"]+ == _call_route(monkeypatch, path = gqa, flash_attn = True, kv_unified = True, swa_full = False)[+ "kv_bytes"+ ]+ )diff --git a/studio/backend/tests/test_kv_cache_estimate_route.py b/studio/backend/tests/test_kv_cache_estimate_route.pyindex 578d3b27879..78be37c23df 100644--- a/studio/backend/tests/test_kv_cache_estimate_route.py+++ b/studio/backend/tests/test_kv_cache_estimate_route.py@@ -3,31 +3,16 @@"""Route-level tests for ``GET /kv-cache-estimate``.-The estimate is only worth showing if it describes the load llama-server will-actually perform, so these drive the real handler and the real drafter resolver-rather than a replayed copy of their logic.--Two classes of defect are covered:--* The MTP reserve must follow the loader's own ``is_mtp_model`` precondition. A- model with no embedded head, no MTP name and no separate drafter cannot run- MTP, so the loader emits ``--spec-default`` and reserves nothing. Because- ``_estimate_mtp_overhead_bytes`` defaults ``mtp_keeps_target_ctx=True`` (so an- unsure caller over-reserves), skipping that precondition billed every MLA- model a second full f16 copy of its own KV.--* Drafter discovery must resolve the same file ``_download_mtp`` opens, on every- platform. The estimate runs on hosts whose filesystems disagree about case and- whose ``Path`` ordering differs, and it is reached for both HF snapshots and- plain local folders.--No GPU, no network. Cross-platform: the layout cases below construct real-directories under ``tmp_path`` and assert against the loader's own picker.+These drive the real handler and the real drafter resolver, not a replayed copy: the MTP+reserve must follow the loader's own ``is_mtp_model`` precondition, since+``_estimate_mtp_overhead_bytes`` over-reserves for an unsure caller, and drafter discovery+must resolve the same file ``_download_mtp`` opens on hosts that disagree about case."""from __future__ import annotationsimport asyncio+import importlib.machineryimport sysfrom pathlib import Path@@ -42,6 +27,7 @@ if _BACKEND_DIR not in sys.path:sys.path.insert(0, _BACKEND_DIR)# Installs the process-wide loggers/structlog/httpx stubs and the GGUF builder.+import test_kv_cache_estimation # noqa: E402from test_kv_cache_estimation import _backend_from_gguf, _make_gguf_bytes # noqa: E402import routes.models as models_routes # noqa: E402@@ -120,6 +106,11 @@ def _call_route(ctx_checkpoints: int | None = None,disable_vision: bool = False,n_ctx: int | None = 32768,+ flash_attn: bool | None = None,+ kv_unified: bool | None = None,+ swa_full: bool | None = None,+ no_mmproj_offload: bool | None = None,+ tensor_parallel: bool = False,):"""Drive the real handler with the quant already resolved to *path*."""monkeypatch.setattr(@@ -149,7 +140,13 @@ def _call_route(disable_vision = disable_vision,n_batch = None,n_ubatch = None,- tensor_parallel = False,+ tensor_parallel = tensor_parallel,+ # Real None, not left out: a direct call leaves omitted parameters holding+ # their fastapi Query sentinel.+ flash_attn = flash_attn,+ kv_unified = kv_unified,+ swa_full = swa_full,+ no_mmproj_offload = no_mmproj_offload,request = None,current_subject = "test",)@@ -268,10 +265,8 @@ class TestDrafterDiscoveryMatchesTheLoader:assert got != str(main)def test_nested_mtp_subdir_copy_is_used_when_no_root_mirror(self, tmp_path):- """A repo publishing heads only under MTP/, as Qwen3.8-Flash-Next and- Qwen3.8-27B do, still gets a drafter. _cached_repo_mtp_drafter already- reuses such a copy offline, so skipping it here gave a cached user- speculation and a fresh one none."""+ """_cached_repo_mtp_drafter reuses such a copy offline, so skipping it here gave a+ cached user speculation and a fresh one none."""snap = self._snapshot(tmp_path)main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")nested = _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)@@ -300,9 +295,7 @@ class TestDrafterDiscoveryMatchesTheLoader:assert got == str(full)def test_precision_outranks_the_shared_preference(self, tmp_path):- """Q8_0 first is the stronger rule: a shared bf16 head is both larger than- a full Q8_0 one and slower, since a draft step is dominated by the LM head- and that head is cheaper to execute at 8 bits."""+ """Q8_0 first: a shared bf16 head is larger and slower than a full Q8_0 one."""snap = self._snapshot(tmp_path)main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")q8 = _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)@@ -312,9 +305,7 @@ class TestDrafterDiscoveryMatchesTheLoader:assert got == str(q8)def test_nested_fallback_is_skipped_for_other_architectures(self, tmp_path):- """The fallback is qwen4exp only. Qwen3.8-27B bakes the head into 20 of- its 24 quants, and llama.cpp prefers a -md drafter over an embedded one, so- pricing the sidecar would reserve for a file the load will not open."""+ """qwen4exp only: elsewhere the sidecar reserves for a file the load never opens."""snap = self._snapshot(tmp_path)main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen35")_write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)@@ -323,9 +314,7 @@ class TestDrafterDiscoveryMatchesTheLoader:assert got is None, f"billed a sidecar the load will not fetch: {got} ({size} bytes)"def test_nested_fallback_is_skipped_for_a_qwen4exp_with_its_own_head(self, tmp_path):- """Architecture is not the whole gate: a qwen4exp GGUF converted with the- block kept needs no sidecar either. Nothing published does this, so this- pins the intent."""+ """Not the whole gate: a qwen4exp GGUF with the block kept needs no sidecar."""snap = self._snapshot(tmp_path)main = _write_gguf(snap / "model-Q4_K_M.gguf",@@ -373,9 +362,8 @@ class TestDrafterDiscoveryMatchesTheLoader:assert got == str(companion)def test_a_non_drafter_parked_under_mtp_is_not_launched(self, tmp_path):- """Everything under MTP/ classifies as a drafter, which keeps companions- out of variant menus and is too broad for choosing what to launch: an- mmproj or an imatrix would be handed to --model-draft."""+ """Everything under MTP/ classifies as a drafter, which is too broad for choosing+ what to launch: an mmproj or an imatrix would be handed to --model-draft."""snap = self._snapshot(tmp_path)main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")_write_gguf(snap / "MTP" / "mmproj-BF16.gguf", _MLA_NO_HEAD)@@ -396,10 +384,8 @@ class TestDrafterDiscoveryMatchesTheLoader:assert got == str(old)def test_an_incomplete_nested_split_does_not_shadow_a_complete_head(self, tmp_path):- """llama.cpp resolves sibling shards from the first one's directory, so- half a set is unusable and _download_companion_gguf answers None to it.- Ranked first and rejected after, it would disable speculation with a usable- lower-ranked head present."""+ """Half a shard set is unusable, and ranked first then rejected it would disable+ speculation with a usable lower-ranked head present."""snap = self._snapshot(tmp_path)main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")_write_gguf(snap / "MTP" / "mtp-model-Q8_0-00001-of-00002.gguf", _MLA_NO_HEAD)@@ -502,11 +488,8 @@ if __name__ == "__main__":class TestTheEstimateMatchesTheConfiguredLoad:- """Settings a user can save must reach the estimator that prices them.-- Each of these is a term the load planner forwards and this route did not, so- the bar reported a comfortable fit for a launch that reserves more.- """+ """Settings a user can save must reach the estimator: each is a term the load planner+ forwards and this route did not, so the bar reported a fit for a heavier launch."""def test_saved_context_checkpoints_are_priced(self, monkeypatch, tmp_path):"""Each checkpoint is an SWA snapshot per slot, so this is not small."""@@ -576,11 +559,8 @@ class TestTheEstimateMatchesTheConfiguredLoad:assert (deep["spec_bytes"] > zero["spec_bytes"] * 10), "spec_draft_n_max did not reach the MTP estimator"- # A blank field is NOT zero: _build_speculative_flags emits its own- # default (2 with a GPU, 3 without) and the rollback state is multiplied- # by it, so pricing an unset field as zero dropped the dominant- # allocation on this exact model shape. An explicit zero still means- # zero, which is what separates the two calls below.+ # A blank field is NOT zero: _build_speculative_flags emits its own default and the+ # rollback state is multiplied by it. An explicit zero still means zero.assert (none["spec_bytes"] > zero["spec_bytes"]), "an unset draft depth was priced as zero, dropping every rollback copy"@@ -646,6 +626,111 @@ class TestTheEstimateMatchesTheConfiguredLoad:)assert off["projector_bytes"] is None, "vision off must free the projector"+ def test_a_host_pinned_projector_is_not_charged_to_the_card(self, monkeypatch, tmp_path):+ """``--no-mmproj-offload`` keeps vision ON with the projector in HOST memory, and+ the frontend adds ``projectorBytes`` onto its GPU weights segment regardless."""+ gguf = _write_gguf(tmp_path / "vision-Q4_K_M.gguf", _MLA_NO_HEAD)+ (tmp_path / "mmproj-F16.gguf").write_bytes(b"\x00" * 800_000)++ offloaded = _call_std_route(+ monkeypatch,+ path = gguf,+ repo_id = str(tmp_path),+ is_local = True,+ no_mmproj_offload = False,+ )+ assert offloaded["projector_bytes"], "a resident projector must still be charged"++ pinned = _call_std_route(+ monkeypatch,+ path = gguf,+ repo_id = str(tmp_path),+ is_local = True,+ no_mmproj_offload = True,+ )+ assert (+ pinned["projector_bytes"] is None+ ), "a projector pinned to host memory must not be reported as VRAM"+ # Vision is still on: "no projector on the card" is not "no vision".+ assert pinned["kv_bytes"] and pinned["kv_bytes"] > 0++ def test_an_inherited_host_pin_is_not_charged_to_the_card_either(self, monkeypatch, tmp_path):+ """The environment alone puts the projector in host memory, so reading only what+ was ASKED left projectorBytes populated where nothing was asked."""+ gguf = _write_gguf(tmp_path / "vision-Q4_K_M.gguf", _MLA_NO_HEAD)+ (tmp_path / "mmproj-F16.gguf").write_bytes(b"\x00" * 800_000)++ for variable, value in (+ ("LLAMA_ARG_NO_MMPROJ_OFFLOAD", "1"),+ ("LLAMA_ARG_MMPROJ_OFFLOAD", "off"),+ ):+ monkeypatch.delenv("LLAMA_ARG_NO_MMPROJ_OFFLOAD", raising = False)+ monkeypatch.delenv("LLAMA_ARG_MMPROJ_OFFLOAD", raising = False)+ monkeypatch.setenv(variable, value)+ inherited = _call_std_route(+ monkeypatch, path = gguf, repo_id = str(tmp_path), is_local = True+ )+ assert inherited["projector_bytes"] is None, variable+ assert inherited["kv_bytes"] and inherited["kv_bytes"] > 0, variable++ # An explicit flag still wins over the environment, as the child resolves them.+ monkeypatch.setenv("LLAMA_ARG_NO_MMPROJ_OFFLOAD", "1")+ monkeypatch.delenv("LLAMA_ARG_MMPROJ_OFFLOAD", raising = False)+ asked_on = _call_std_route(+ monkeypatch,+ path = gguf,+ repo_id = str(tmp_path),+ is_local = True,+ no_mmproj_offload = False,+ )+ assert asked_on["projector_bytes"], "an explicit --mmproj-offload must still charge"++ # A clean environment and nothing asked is unchanged: the projector is resident.+ monkeypatch.delenv("LLAMA_ARG_NO_MMPROJ_OFFLOAD", raising = False)+ monkeypatch.delenv("LLAMA_ARG_MMPROJ_OFFLOAD", raising = False)+ plain = _call_std_route(monkeypatch, path = gguf, repo_id = str(tmp_path), is_local = True)+ assert plain["projector_bytes"], "an unasked, uninherited projector is on the card"++ def test_flash_attention_off_cannot_take_a_tensor_split(self):+ """llama.cpp refuses the pair, so the loader falls back to a layer split and+ pricing the tensor placement charges buffers to a card count nothing uses."""+ from routes.models import _tensor_split_can_launch++ assert _tensor_split_can_launch(True, False) is False+ assert _tensor_split_can_launch(True, True) is True+ # Not resolved is not evidence that the child runs without it.+ assert _tensor_split_can_launch(True, None) is True+ assert _tensor_split_can_launch(False, True) is False+ assert _tensor_split_can_launch(False, False) is False++ def test_the_downgrade_reaches_the_breakdown_and_the_floor(self):+ """The boolean alone does not reach them: the breakdown re-resolves the split+ through a helper that reads an inherited LLAMA_ARG_SPLIT_MODE=tensor too."""+ import inspect++ from routes import models as models_module++ body = inspect.getsource(models_module.get_kv_cache_estimate)+ decide = body.index("_tensor_split_can_launch(")+ assert '"--split-mode", "layer"' in body[decide : decide + 1200], body[+ decide : decide + 600+ ]+ assert body.count("tensor_parallel = _effective_tp,") == 2, body.count(+ "tensor_parallel = _effective_tp,"+ )+ assert "tensor_parallel = tensor_parallel,\n" not in body[decide:]++ def test_the_estimate_prices_the_split_the_launch_can_take(self):+ import inspect++ from routes import models as models_module++ body = inspect.getsource(models_module.get_kv_cache_estimate)+ decide = body.index("_tensor_split_can_launch(")+ devices = body.index("_planner_devices = max(")+ assert decide < devices, (decide, devices)+ assert "_effective_tp = False" in body[decide:devices]+def test_a_model_with_no_projector_reports_none(self, monkeypatch, tmp_path):gguf = _write_gguf(tmp_path / "text-Q4_K_M.gguf", _MLA_NO_HEAD)out = _call_std_route(monkeypatch, path = gguf, repo_id = str(tmp_path), is_local = True)@@ -653,10 +738,8 @@ class TestTheEstimateMatchesTheConfiguredLoad:class TestHostMemoryIsNotChargedToTheCard:- """Context checkpoints live in host heap, so a VRAM bar must be able to- exclude them. The load planner's own GPU figure is- ``kv_bytes - kv_checkpoint_bytes``; without the second term reported the bar- warns OOM over memory that never reaches the GPU."""+ """Context checkpoints live in host heap, so a VRAM bar must be able to exclude them:+ without ``kv_checkpoint_bytes`` it warns OOM over memory that never reaches the GPU."""def test_checkpoints_are_reported_as_their_own_share(self, monkeypatch, tmp_path):gguf = _write_gguf(tmp_path / "swa-model-Q4_K_M.gguf", _SWA_MODEL)@@ -677,9 +760,7 @@ class TestHostMemoryIsNotChargedToTheCard:# By difference against the same call with none, which is how the load# planner derives it -- asking one function twice cannot drift from it.assert share == with_checkpoints["kv_bytes"] - without["kv_bytes"]- # And it is a SHARE of kv_bytes, not a figure beside it: the field- # shipped meaning the whole cache and an existing caller still reads it- # that way.+ # A SHARE of kv_bytes, not a figure beside it: an existing caller reads it that way.assert share < with_checkpoints["kv_bytes"]def test_no_checkpoints_means_no_host_share(self, monkeypatch, tmp_path):@@ -699,22 +780,16 @@ class TestHostMemoryIsNotChargedToTheCard:class TestAutoAbstainsOverASidecar:- """Auto promotes to DSpark or DFlash when the model ships one, so the route- must say the reserve is unpriced rather than chart a total without it.-- These assert the OUTCOME, not that the block runs: the first version of this- guard referenced two helpers that were only in another function's scope, and- the resulting NameError was swallowed by the surrounding except, leaving- spec_unpriced false with nothing in the response to show it had failed.- """+ """Auto promotes to DSpark or DFlash when the model ships one, so the route must say+ the reserve is unpriced rather than chart a total without it. Asserted on the OUTCOME: an+ earlier guard raised NameError into the surrounding except and showed nothing."""def _call(self, monkeypatch, tmp_path, *, sidecar: str | None, supports: bool):model_dir = tmp_path / "local-model"gguf = _write_gguf(model_dir / "model-Q4_K_M.gguf", _MLA_NO_HEAD)if sidecar:- # A DFlash sidecar is identified by its header, not its name:- # detect_dflash_file confirms general.architecture = dflash, which- # settles the adversarial case of a real model merely CALLED DFlash.+ # By header, not by name: detect_dflash_file confirms+ # general.architecture = dflash, settling the model merely CALLED DFlash._write_gguf(model_dir / sidecar,_MLA_NO_HEAD,@@ -772,24 +847,16 @@ class TestAutoAbstainsOverASidecar:assert out["spec_unpriced"] is Falsedef test_a_binary_that_cannot_run_one_still_prices_normally(self, monkeypatch, tmp_path):- # The planner gates sidecar selection on the binary's own capability, so- # abstaining where the launch would never open one blanks the bar for- # nothing.+ # The planner gates sidecar selection on the binary's capability, so abstaining+ # where the launch opens none blanks the bar for nothing.out = self._call(monkeypatch, tmp_path, sidecar = "dspark-model-Q8_0.gguf", supports = False)assert out["spec_unpriced"] is Falseclass TestThePlannerFiguresArrive:- """The route delegates to the load planner for the terms it cannot derive- itself, inside a try/except so a planner that cannot size a model still- leaves the KV bar drawn.-- That except is the hazard: every earlier version of this delegation failed- into it silently and returned nulls that looked like "not sizeable" rather- than "the call is broken". These assert the figures are actually present for- a model the planner can size, so a signature drift or an unresolved- parameter fails the suite instead of quietly blanking the bar.- """+ """The route delegates to the load planner inside a try/except, which earlier versions+ failed into silently and returned nulls reading as "not sizeable" rather than "the call+ is broken", so these assert the figures are present for a model it can size."""def test_the_planner_terms_are_populated(self, monkeypatch, tmp_path):model_dir = tmp_path / "planner-model"@@ -866,9 +933,8 @@ class TestTheInheritedEnvironmentIsPriced:not what the request said."""def test_an_inherited_context_beats_the_native_length(self, monkeypatch, tmp_path):- # load_model drops an inherited context only when it is zero, so a- # positive one is the legitimate way to set the window. Falling through- # to native priced a 4k load at the header's length.+ # load_model drops an inherited context only when it is zero, so a positive one sets+ # the window and falling through to native priced a 4k load at the header's length.gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "4096")out = _call_std_route(monkeypatch, path = gguf, n_ctx = None)@@ -885,9 +951,8 @@ class TestTheInheritedEnvironmentIsPriced:assert out["n_ctx"] == _PLAIN_GQA["context_length"]def test_an_inherited_cache_type_sizes_the_cache(self, monkeypatch, tmp_path):- # A q8_0 cache is roughly half an f16 one. Pricing f16 while the child- # opens q8_0 makes the KV segment and the per-token readout contradict- # the planner total drawn beside them.+ # Pricing f16 while the child opens q8_0 makes the KV segment and the per-token+ # readout contradict the planner total drawn beside them.gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)monkeypatch.delenv("LLAMA_ARG_CACHE_TYPE_K", raising = False)monkeypatch.delenv("LLAMA_ARG_CACHE_TYPE_V", raising = False)@@ -901,10 +966,8 @@ class TestTheInheritedEnvironmentIsPriced:)def test_an_inherited_context_is_reported_as_pinned(self, monkeypatch, tmp_path):- # The loader keeps a positive inherited context rather than fitting it,- # so a caller must not soften its verdict for that launch. Saying- # "auto-fitted" there suppressed the overage AND drew only the- # irreducible floor, which is a comfortable fit for a load that OOMs.+ # The loader keeps a positive inherited context rather than fitting it, so+ # "auto-fitted" suppressed the overage and drew only the irreducible floor.gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "4096")assert _call_std_route(monkeypatch, path = gguf, n_ctx = None)["context_is_pinned"] is True@@ -920,19 +983,37 @@ class TestTheInheritedEnvironmentIsPriced:assert _call_std_route(monkeypatch, path = gguf, n_ctx = 8192)["context_is_pinned"] is Truedef test_an_inherited_device_pin_is_reported(self, monkeypatch, tmp_path):- # The child is confined to the cards LLAMA_ARG_DEVICE names and an- # automatic launch preserves the pin, so an aggregate VRAM budget- # describes a pool the launch will not open. The caller cannot see the- # environment, so the route has to say.+ # The child is confined to the cards LLAMA_ARG_DEVICE names, so an aggregate VRAM+ # budget describes a pool the launch will not open.gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)monkeypatch.setenv("LLAMA_ARG_DEVICE", "CUDA0")assert _call_std_route(monkeypatch, path = gguf)["inherited_device_pin"] is True@pytest.mark.parametrize("value", ["", "none", "NONE"])def test_no_usable_pin_is_not_reported_as_one(self, monkeypatch, tmp_path, value):- # "none" is a CPU-only launch, which the planner already answers with- # zero GPU bytes and which draws no bar on its own; reporting it as a pin- # would blank the row for a second, unrelated reason.+ # "none" is a CPU-only launch, already answered with zero GPU bytes; reporting it+ # as a pin would blank the row for a second, unrelated reason.gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)monkeypatch.setenv("LLAMA_ARG_DEVICE", value)assert _call_std_route(monkeypatch, path = gguf)["inherited_device_pin"] is False+++def test_the_shared_loggers_stub_is_still_a_package():+ """``routes.models`` reaches ``loggers.media_progress``, and ``test_kv_cache_estimation``+ installs a bare ``ModuleType`` stub that shadows the whole package, so without+ ``__path__`` this module cannot be collected at all (#11028). Asserted against the stub+ object, not the ``sys.modules`` slot, which the conftest guard usually wins.+ """+ stub = test_kv_cache_estimation._loggers_stub+ search_path = list(getattr(stub, "__path__", ()) or ())+ assert search_path, (+ "the shared 'loggers' stub has no __path__, so it is a module and not a package: "+ "whichever test module installs it first makes every importer of "+ "loggers.media_progress uncollectable"+ )+ # __path__ existing is not enough; it has to actually resolve the submodule.+ spec = importlib.machinery.PathFinder.find_spec("media_progress", search_path)+ assert spec is not None, (+ f"the stub's __path__ {search_path} does not contain media_progress; "+ "it must point at the real studio/backend/loggers directory"+ )diff --git a/studio/backend/tests/test_kv_cache_estimation.py b/studio/backend/tests/test_kv_cache_estimation.pyindex 871b027966e..b23b8d45dc3 100644--- a/studio/backend/tests/test_kv_cache_estimation.py+++ b/studio/backend/tests/test_kv_cache_estimation.py@@ -29,6 +29,9 @@ if _BACKEND_DIR not in sys.path:# loggers_loggers_stub = _types.ModuleType("loggers")_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)+# __path__ so `loggers.media_progress` still resolves: a bare ModuleType shadows the whole+# package, so the submodule import dies with "'loggers' is not a package" (#10995).+_loggers_stub.__path__ = [str(Path(_BACKEND_DIR) / "loggers")]sys.modules.setdefault("loggers", _loggers_stub)# structlog. Carries get_logger because this stub is process-wide: whichever testdiff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.pyindex b1cb2ae41e5..35c39951d5b 100644--- a/studio/backend/tests/test_llama_cpp_placement.py+++ b/studio/backend/tests/test_llama_cpp_placement.py@@ -3678,14 +3678,24 @@ def _recording_compute_backend(def test_the_loader_prices_the_attention_mode_it_launches(tmp_path, monkeypatch, extra_args, expected):- """Compute follows the launch's attention mode; KV covers the no-flash retry."""+ """Compute AND KV follow the launch's attention mode, off one resolved state.++ The KV cache used to be pinned to flash attention off whatever the argv said, which+ priced a load that was not going to happen (#9697, #10489). The reserve it stood in for+ is narrower now: ``_reserved_flash_attn_state`` only holds the reading down where a+ no-flash respawn cannot be re-placed, which is neither of the cases here.+ """backend, gguf, calls = _recording_compute_backend(tmp_path, monkeypatch)assert _launch(backend, gguf, n_ctx = 0, n_parallel = 4, extra_args = extra_args)["cmd"]assert calls["ctx"], "the fit never priced the context term"assert set(calls["ctx"]) == {expected}- assert False in calls["kv"]+ assert calls["kv"], "the fit never priced the KV cache"+ assert set(calls["kv"]) == {expected}, (+ f"the KV cache was priced with flash_attn {sorted(set(map(str, calls['kv'])))} "+ f"while the compute buffers were priced {expected}: one load, two answers"+ )@pytest.mark.parametrize("build,expected", [(9415, True), (10909, False), (None, False)])diff --git a/studio/backend/tests/test_load_mode_fit_matrix.py b/studio/backend/tests/test_load_mode_fit_matrix.pyindex 337f1acbbb7..3c89443f5a8 100644--- a/studio/backend/tests/test_load_mode_fit_matrix.py+++ b/studio/backend/tests/test_load_mode_fit_matrix.py@@ -2139,10 +2139,12 @@ def test_the_no_flash_rung_recomputes_the_memory_record():import inspectsrc = inspect.getsource(B.load_model)- helper = src[src.index("_drop_fit_load_mode_for_no_flash") :]+ # Anchored on the DEFINITION: the bare name also appears in comments earlier in+ # load_model, so slicing from the first mention sliced a comment.+ helper = src[src.index("def _drop_fit_load_mode_for_no_flash(") :]# Bounded at the next definition, so this proves the recompute is in the# helper and not merely somewhere later in load_model.- helper = helper[: helper.index("_spawn_and_wait")]+ helper = helper[: helper.index("def _spawn_and_wait")]assert "self._record_memory_state" in helpercompact = "".join(helper.split())assert "self._record_memory_state(stripped" in compact@@ -2270,3 +2272,140 @@ def test_the_replayed_cpu_fallback_recomputes_the_memory_record():arm = src[src.index("allow_manual_cpu=True") :]arm = arm[: arm.index("_apply_cpu_fallback_state")]assert "self._record_memory_state(cmd,env)" in arm+++def _no_flash_fit_rewriter(+ extra_args,+ *,+ manual_gpu_layers = None,+ env = None,+):+ """The nested `_enable_managed_fit_for_no_flash` as a callable: it closes over+ load_model's locals, so compiling its source is the only way to run the REAL rewrite."""+ from core.inference.llama_cpp import LlamaCppBackend as B+ from core.inference import llama_cpp as B_module+ from core.inference.llama_cpp import logger, _flag_name+ import inspect, textwrap++ src = inspect.getsource(B.load_model)+ start = src.index(" def _enable_managed_fit_for_no_flash(")+ end = src.index(" def ", start + 1)+ namespace = {+ "logger": logger,+ "_flag_name": _flag_name,+ "_placement_is_fitter_proof": B_module._placement_is_fitter_proof,+ "_user_fit_disabled": B_module._user_fit_disabled,+ "extra_args": extra_args,+ "_manual_gpu_layers": manual_gpu_layers,+ "env": {} if env is None else env,+ }+ exec(textwrap.dedent(src[start:end]), namespace)+ return namespace["_enable_managed_fit_for_no_flash"]+++def test_the_no_flash_retry_re_enables_unsloths_own_fitter():+ """A managed `--fit off` is flipped back on for the respawn.++ The reserve is only safe because the respawn is re-placed, and it inherits the auto+ placement's `--fit off` that `_fit_off_retry_eligible` will not override, so without the+ flip the retry re-lands on the smaller-cache placement and OOMs.+ """+ rewrite = _no_flash_fit_rewriter(None)+ out = rewrite(["llama-server", "--fit", "off", "--flash-attn", "off"])+ assert out[out.index("--fit") + 1] == "on"+ on = ["llama-server", "--fit", "on"]+ assert rewrite(on) == on+ bare = ["llama-server", "--flash-attn", "off"]+ assert rewrite(bare) == bare+++def test_the_no_flash_fit_flip_leaves_a_user_fit_alone():+ """The user's own `--fit off` survives the respawn, in both spellings: theirs wins by+ last-arg anyway, except where the only `--fit` present is theirs."""+ for user_tokens in (["--fit", "off"], ["--fit=off"], ["-fit", "off"]):+ rewrite = _no_flash_fit_rewriter(user_tokens)+ # Unsloth's token present as well: the last value is still the user's "off".+ both = rewrite(["llama-server", "--fit", "off", *user_tokens])+ assert both == ["llama-server", "--fit", "off", *user_tokens]+ theirs = rewrite(["llama-server", *user_tokens])+ assert theirs == ["llama-server", *user_tokens]+++def test_an_inherited_fit_off_survives_the_no_flash_retry():+ """LLAMA_ARG_FIT=off is the user's `--fit off`, and only manual mode scrubs it from the+ child env, so outside manual the flip would put `--fit on` on the CLI and beat it.++ `_user_fit_disabled` reads the same variable and already reserved the padded V cache for+ this load, so the respawn lands on a placement priced for it and needs no re-place.+ """+ off = ["llama-server", "--fit", "off", "--flash-attn", "off"]+ for value in ("off", "0", "false", "no", "disabled", " OFF "):+ rewrite = _no_flash_fit_rewriter(None, env = {"LLAMA_ARG_FIT": value})+ assert rewrite(off) == off, value++ # Only an off. An inherited "on" leaves the managed placement re-placeable as before,+ # and an empty value is not an answer.+ for value in ("on", "1", ""):+ rewrite = _no_flash_fit_rewriter(None, env = {"LLAMA_ARG_FIT": value})+ assert rewrite(off)[off.index("--fit") + 1] == "on", value++ # A user's own --fit token skips whatever the environment says, as it already did.+ theirs = _no_flash_fit_rewriter(["--fit", "on"], env = {"LLAMA_ARG_FIT": "off"})+ assert theirs(off) == off+++def test_the_no_flash_retry_re_places_at_both_respawns():+ """Both --flash-attn off arms re-enable the fitter, each before its own spawn. Checked+ at the source because these arms only run behind a real crash."""+ from core.inference.llama_cpp import LlamaCppBackend as B+ import inspect++ src = inspect.getsource(B.load_model)+ for label in ('label = "-noflash"', 'label = "-noflash-mtp"'):+ arm = src[: src.index(label)]+ arm = arm[arm.rindex("self._with_flash_attn_off(") :]+ assert "_enable_managed_fit_for_no_flash(_fa_cmd)" in arm+ assert src.count("_enable_managed_fit_for_no_flash(_fa_cmd)") == 2+++def test_a_fixed_layer_count_keeps_the_no_flash_retry_on_its_placement():+ """Manual mode's `--fit off` is not an auto placement's, and must not be flipped.++ Manual emits `--gpu-layers N --fit off`, token for token what an auto placement emits,+ but `common_params_fit_impl` throws "n_gpu_layers already set by user"+ (common/fit.cpp:377) for any count but -1, so the retry keeps the fixed placement and+ the flip would only buy a reserve priced for a re-placement that cannot happen.+ """+ rewrite = _no_flash_fit_rewriter(None, manual_gpu_layers = 20)+ fixed = ["llama-server", "--gpu-layers", "20", "--fit", "off"]+ assert rewrite(fixed) == fixed++ # The argv count decides on its own: llama.cpp reads the tokens, not this process.+ assert _no_flash_fit_rewriter(None)(fixed) == fixed+ # And an inherited count, which the fitting path emits no -ngl to lose to.+ inherited = _no_flash_fit_rewriter(None, env = {"LLAMA_ARG_N_GPU_LAYERS": "20"})+ auto = ["llama-server", "--fit", "off"]+ assert inherited(auto) == auto+ # llama.cpp's own default is not an override, so an auto placement still re-places.+ default_count = ["llama-server", "--gpu-layers", "-1", "--fit", "off"]+ assert _no_flash_fit_rewriter(None)(default_count)[-1] == "on"+++def test_the_reserve_holds_for_a_placement_the_fitter_will_not_move():+ """The other half of the same case, on the estimate rather than the argv: with no+ re-placement available the reserve prices the padded, f16-floored V."""+ from core.inference.llama_cpp import _placement_is_fitter_proof, _reserved_flash_attn_state++ assert _reserved_flash_attn_state(True, None, gpu_layers = 20, env = {}) is False+ assert _reserved_flash_attn_state(True, ["-ngl", "20"], env = {}) is False+ assert _reserved_flash_attn_state(True, None, env = {"LLAMA_ARG_N_GPU_LAYERS": "20"}) is False+ # Auto placements keep the planned answer: the retry really is re-placed there.+ assert _reserved_flash_attn_state(True, None, gpu_layers = -1, env = {}) is True+ assert _reserved_flash_attn_state(True, ["-ngl", "-1"], env = {}) is True+ assert _reserved_flash_attn_state(True, None, env = {}) is True++ assert _placement_is_fitter_proof(["--gpu-layers=20"], env = {}) is True+ assert _placement_is_fitter_proof(["--n-gpu-layers", "0"], env = {}) is True+ assert _placement_is_fitter_proof(["-ngl", "auto"], env = {}) is False+ assert _placement_is_fitter_proof(None, gpu_layers = 0, env = {}) is True+ assert _placement_is_fitter_proof(None, env = {}) is Falsediff --git a/studio/backend/tests/test_offload_planner_seam.py b/studio/backend/tests/test_offload_planner_seam.pyindex cdd6fd8ea3f..012de7b3dcf 100644--- a/studio/backend/tests/test_offload_planner_seam.py+++ b/studio/backend/tests/test_offload_planner_seam.py@@ -1558,9 +1558,8 @@ def test_flash_disabled_v_padding_reaches_the_layer_weights():layer's V is padded to hparams.n_embd_v_gqa_max() over the whole model, whichis what _estimate_kv_cache_bytes charges via _max_kv_value_width. V goesconstant while K stays per-layer, so an unpadded vector prices a ratio the- total does not have. Not an edge case: load_model pins planned_flash_attn =- False unconditionally (llama_cpp.py:16690), so the padded branch is the one- every spill plan's total is built from."""+ total does not have. Reached whenever the resolved launch runs without flash+ attention (_planned_flash_attn_state), which is exactly when llama.cpp pads."""b = _swa_backend()# SWA layers wider than global ones, so the model-wide max is the SWA width# and the padding actually moves: n_embd_v_gqa_max = 8 * 256.diff --git a/studio/backend/tests/test_resolved_flash_attn_plan.py b/studio/backend/tests/test_resolved_flash_attn_plan.pynew file mode 100644index 00000000000..5c693aba529--- /dev/null+++ b/studio/backend/tests/test_resolved_flash_attn_plan.py@@ -0,0 +1,544 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The estimate and the launch price the same flash-attention state.++``load_model`` used to pin ``planned_flash_attn = False`` while emitting ``--flash-attn on``,+so every placement figure described a load that was not going to happen (#9697, #10489). The+cushion that pin gave is not lost: tensor mode cannot take the FA-off recovery at all, and+elsewhere the respawn re-enters ``_spawn_and_wait``, which re-places it with ``--fit on``.+"""++from __future__ import annotations++import importlib.util+import sys+import textwrap+from pathlib import Path++import pytest++_TESTS_DIR = Path(__file__).resolve().parent+_BACKEND_DIR = str(_TESTS_DIR.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)+if str(_TESTS_DIR) not in sys.path:+ sys.path.insert(0, str(_TESTS_DIR))+++def _load(module_name: str, file_name: str):+ spec = importlib.util.spec_from_file_location(module_name, _TESTS_DIR / file_name)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++_placement = _load("_placement_for_flash_attn_plan", "test_llama_cpp_placement.py")++from core.inference.llama_cpp import ( # noqa: E402+ LlamaCppBackend,+ _planned_flash_attn_state,+)++GB = 1024**3+NATIVE = 262144+++class TestTheResolver:+ def test_the_managed_default_is_on(self):+ assert _planned_flash_attn_state() is True++ def test_a_build_without_the_flag_is_off(self):+ """No flag emitted, so the padded, f16-floored arm is the right price."""+ assert _planned_flash_attn_state(supports_flash_attn = False) is False++ @pytest.mark.parametrize(+ "extras",+ [+ ["--flash-attn", "off"],+ ["--flash-attn=off"],+ ["-fa", "off"],+ ["-fa=0"],+ ],+ )+ def test_an_explicit_off_in_the_extras_wins(self, extras):+ assert _planned_flash_attn_state(extras) is False++ def test_the_last_flag_wins_like_llama_cpp(self):+ assert _planned_flash_attn_state(["-fa", "off", "--flash-attn", "on"]) is True++ def test_the_environment_loses_to_the_managed_flag(self):+ """The env is read before argv (arg.cpp set_env), so the managed on wins."""+ assert _planned_flash_attn_state(env = {"LLAMA_ARG_FLASH_ATTN": "0"}) is True++ def test_a_user_off_still_beats_the_managed_flag(self):+ """Order of authority: environment, managed flag, then the user's extras."""+ assert _planned_flash_attn_state(["-fa", "off"], env = {"LLAMA_ARG_FLASH_ATTN": "0"}) is False+ assert _planned_flash_attn_state(["-fa", "off"], env = {}) is False++ def test_the_extras_beat_the_environment(self):+ assert (+ _planned_flash_attn_state(["--flash-attn", "on"], env = {"LLAMA_ARG_FLASH_ATTN": "0"})+ is True+ )++ @pytest.mark.parametrize("v_type", ["q8_0", "q4_0", "q5_1", "iq4_nl"])+ def test_a_quantized_v_cache_forces_it_on(self, v_type):+ """llama.cpp turns it on itself, so an explicit off does not survive."""+ assert (+ _planned_flash_attn_state(["--flash-attn", "off"], planned_cache_types = ("f16", v_type))+ is True+ )++ @pytest.mark.parametrize("v_type", ["f16", "bf16", "f32"])+ def test_an_unquantized_v_cache_forces_nothing(self, v_type):+ assert (+ _planned_flash_attn_state(["--flash-attn", "off"], planned_cache_types = ("f16", v_type))+ is False+ )++ def test_a_quantized_v_cannot_force_a_build_that_has_no_flag(self):+ """The launch rewrites V to f16 instead, so forcing "on" would under-reserve."""+ assert (+ _planned_flash_attn_state(+ planned_cache_types = ("q8_0", "q8_0"), supports_flash_attn = False+ )+ is False+ )+++def _tensor_backend(tmp_path, *, free_mib: int):+ backend, gguf = _placement._backend(+ tmp_path,+ vulkan = False,+ memory = [(0, free_mib, free_mib), (1, free_mib, free_mib)],+ )+ # The harness turns KV estimation off; seed a real shape (Qwen3-0.6B) for arithmetic.+ backend._can_estimate_kv = lambda: True+ backend._n_layers = 28+ backend._embedding_length = 1024+ backend._n_heads = 16+ backend._n_kv_heads = 8+ backend._kv_key_length = 128+ backend._kv_value_length = 128+ backend._context_length = NATIVE+ backend._get_gguf_size_bytes = lambda _path: 4 * GB+ return backend, gguf+++def _ctx_of(cmd) -> int:+ values = [cmd[i + 1] for i, token in enumerate(cmd) if token in ("-c", "--ctx-size")]+ assert values, f"no context in argv: {cmd}"+ return int(values[-1])+++def _flash_attn_in(cmd) -> bool:+ for i, token in enumerate(cmd):+ if token in ("--flash-attn", "-fa"):+ return i + 1 >= len(cmd) or cmd[i + 1] != "off"+ return False+++class TestTheTensorPlanPricesTheLaunch:+ """A pool too small at f16-priced V and large enough once V is priced as launched."""++ def test_the_planned_context_matches_the_emitted_flash_attention(self, tmp_path):+ backend, gguf = _tensor_backend(tmp_path, free_mib = 12000)+ captured = _placement._launch(+ backend, gguf, n_ctx = NATIVE, cache_type_kv = "q8_0", tensor_parallel = True+ )+ cmd = captured["cmd"]+ assert _flash_attn_in(cmd), "the launch emits flash attention"+ planned = _ctx_of(cmd)++ # The gap is the defect, so assert the side of it rather than a magic number.+ pessimistic = backend._plan_tensor_parallel(+ [(0, 12000), (1, 12000)],+ 4 * GB,+ NATIVE,+ cache_type_kv = "q8_0",+ max_target_ctx = NATIVE,+ flash_attn = False,+ )[0]+ optimistic = backend._plan_tensor_parallel(+ [(0, 12000), (1, 12000)],+ 4 * GB,+ NATIVE,+ cache_type_kv = "q8_0",+ max_target_ctx = NATIVE,+ flash_attn = True,+ )[0]+ assert pessimistic < optimistic, "harness no longer straddles the difference"+ assert planned > pessimistic, (+ f"the tensor plan emitted {planned} tokens, at or below the {pessimistic} a "+ f"flash-attention-off cache prices, while the launch runs with flash "+ f"attention on and fits {optimistic}"+ )++ def test_the_published_ceiling_is_the_same_number(self, tmp_path):+ """max_context_length is what the context warning compares against."""+ backend, gguf = _tensor_backend(tmp_path, free_mib = 12000)+ captured = _placement._launch(+ backend, gguf, n_ctx = NATIVE, cache_type_kv = "q8_0", tensor_parallel = True+ )+ assert backend.max_context_length == _ctx_of(captured["cmd"])++ def test_an_unquantized_cache_is_unaffected(self, tmp_path):+ """On the arithmetic, not a hand-built plan: the loader charges extra overheads."""+ backend, gguf = _tensor_backend(tmp_path, free_mib = 12000)+ captured = _placement._launch(+ backend, gguf, n_ctx = NATIVE, cache_type_kv = "f16", tensor_parallel = True+ )+ planned = _ctx_of(captured["cmd"])+ assert backend._estimate_kv_cache_bytes(+ planned, "f16", flash_attn = True+ ) == backend._estimate_kv_cache_bytes(planned, "f16", flash_attn = False)+++class TestTheSizingCallsSeeTheResolvedState:+ def _seen_flash_attn(self, tmp_path, **load_kwargs):+ backend, gguf = _tensor_backend(tmp_path, free_mib = 12000)+ seen: list[object] = []+ real = backend._estimate_kv_cache_bytes++ def spy(+ n_ctx,+ cache_type_kv = None,+ **kwargs,+ ):+ seen.append(kwargs.get("flash_attn"))+ return real(n_ctx, cache_type_kv, **kwargs)++ backend._estimate_kv_cache_bytes = spy+ _placement._launch(backend, gguf, **load_kwargs)+ assert seen, "no KV estimate was taken during the load"+ return seen++ def test_a_quantized_cache_is_priced_with_flash_attention_on(self, tmp_path):+ seen = self._seen_flash_attn(+ tmp_path, n_ctx = NATIVE, cache_type_kv = "q8_0", tensor_parallel = True+ )+ assert all(state is True for state in seen), (+ f"the load priced its KV cache with flash_attn states {sorted(set(map(str, seen)))} "+ f"while emitting --flash-attn on"+ )++ def test_an_explicit_off_in_the_extras_is_priced_off(self, tmp_path):+ seen = self._seen_flash_attn(+ tmp_path,+ n_ctx = NATIVE,+ cache_type_kv = "f16",+ extra_args = ["--flash-attn", "off"],+ )+ assert all(state is False for state in seen)+++def test_the_state_is_still_named_planned_flash_attn():+ """The offload-planner seam asserts on this identifier by AST."""+ import inspect++ source = inspect.getsource(inspect.unwrap(LlamaCppBackend.load_model))+ # Through the reserve wrapper now, which only holds a reading DOWN.+ assert "planned_flash_attn = _reserved_flash_attn_state(" in source+ assert "_planned_flash_attn_state(" in source+ assert "planned_flash_attn = False" not in source+++class TestAutoIsNotAnAnswer:+ """``auto`` is decided at load time and silently decided against whenever the backend,+ model or cache pair cannot take it, so reading it as "on" under-reserves."""++ def test_an_explicit_auto_prices_the_padded_cache(self):+ assert _planned_flash_attn_state(["--flash-attn", "auto"]) is False+ assert _planned_flash_attn_state(["-fa", "auto"]) is False+ assert _planned_flash_attn_state(["-fa=auto"]) is False+ # llama.cpp's numeric spelling of the same value.+ assert _planned_flash_attn_state(["-fa", "-1"]) is False++ def test_an_inherited_auto_is_overridden_by_the_managed_flag(self):+ """The managed flag is appended after the env is read, so only a USER auto survives."""+ assert _planned_flash_attn_state(None, env = {"LLAMA_ARG_FLASH_ATTN": "auto"}) is True+ assert (+ _planned_flash_attn_state(["-fa", "auto"], env = {"LLAMA_ARG_FLASH_ATTN": "1"}) is False+ )+ assert (+ _planned_flash_attn_state(["-fa", "on"], env = {"LLAMA_ARG_FLASH_ATTN": "auto"}) is True+ )+ assert (+ _planned_flash_attn_state(["-fa", "auto"], env = {"LLAMA_ARG_FLASH_ATTN": "1"}) is False+ )++ def test_the_last_flag_still_wins(self):+ assert _planned_flash_attn_state(["-fa", "auto", "-fa", "on"]) is True+ assert _planned_flash_attn_state(["-fa", "on", "-fa", "auto"]) is False++ def test_a_quantized_v_cache_still_forces_it_on(self):+ assert (+ _planned_flash_attn_state(["-fa", "auto"], planned_cache_types = ("q8_0", "q4_0")) is True+ )++ def test_tensor_split_decides_auto_rather_than_leaving_it_open(self):+ """llama.cpp upgrades AUTO to ENABLED under SPLIT_MODE_TENSOR, so pricing the+ padded V there charges a cache the child never allocates."""+ # The three ways the launch decides the mode: toggle, extras, inherited env.+ assert _planned_flash_attn_state(["-fa", "auto"], tensor_parallel = True) is True+ assert _planned_flash_attn_state(["-fa", "auto", "--split-mode", "tensor"], env = {}) is True+ assert (+ _planned_flash_attn_state(["-fa", "auto"], env = {"LLAMA_ARG_SPLIT_MODE": "tensor"})+ is True+ )+ # An extras split mode last-wins over the toggle, so layer stays undecided.+ assert (+ _planned_flash_attn_state(+ ["-fa", "auto", "--split-mode", "layer"], tensor_parallel = True, env = {}+ )+ is False+ )+ assert _planned_flash_attn_state(["-fa", "auto"], env = {}) is False+ # A user OFF is not silently flipped on: refusing the pair is the launch's job.+ assert _planned_flash_attn_state(["-fa", "off"], tensor_parallel = True) is False++ def test_the_managed_launch_is_unaffected(self):+ assert _planned_flash_attn_state(None) is True+ assert _planned_flash_attn_state([]) is True+++class TestTheDowngradesRePlanTheAttention:+ """The plan is made once near the top of the load and the split is decided later, so a+ stale True budgets an unpadded V the layer split will not get."""++ def _load_model_body(self):+ import ast+ import inspect++ from core.inference import llama_cpp as module++ source = inspect.getsource(module.LlamaCppBackend.load_model)+ return ast.parse(textwrap.dedent(source)).body[0]++ def test_every_tensor_downgrade_re_plans_the_attention(self):+ """Not only the ones that strip the extras: the three manual guards set the toggle+ to False on their own, and Manual placement runs with --fit off."""+ import ast++ func = self._load_model_body()+ # Only downgrades AFTER the plan is made; virtualised Metal drops it before.+ planned_at = min(+ node.lineno+ for node in ast.walk(func)+ if isinstance(node, ast.Call)+ and getattr(node.func, "id", None) == "_planned_flash_attn_state"+ )+ checked = 0+ for node in ast.walk(func):+ for attr in ("body", "orelse", "finalbody"):+ block = getattr(node, attr, None)+ if not isinstance(block, list):+ continue+ for index, statement in enumerate(block):+ if not (+ isinstance(statement, ast.Assign)+ and getattr(statement.targets[0], "id", None) == "tensor_parallel"+ and isinstance(statement.value, ast.Constant)+ and statement.value.value is False+ and statement.lineno > planned_at+ ):+ continue+ checked += 1+ following = [+ getattr(getattr(later, "targets", [None])[0], "id", None)+ for later in block[index + 1 : index + 4]+ if isinstance(later, ast.Assign)+ ]+ assert "planned_flash_attn" in following, (+ "a tensor downgrade at line "+ f"{statement.lineno} does not re-plan the attention"+ )+ assert checked >= 6, checked++ def test_every_split_mode_strip_re_plans_the_attention(self):+ import ast++ func = self._load_model_body()+ strips = 0+ for node in ast.walk(func):+ for attr in ("body", "orelse", "finalbody"):+ block = getattr(node, attr, None)+ if not isinstance(block, list):+ continue+ for index, statement in enumerate(block):+ if not isinstance(statement, ast.Assign):+ continue+ call = statement.value+ if not (+ isinstance(call, ast.Call)+ and getattr(call.func, "id", None) == "strip_split_mode_only"+ ):+ continue+ strips += 1+ following = block[index + 1] if index + 1 < len(block) else None+ assert isinstance(following, ast.Assign), ast.dump(statement)+ assert (+ getattr(following.targets[0], "id", None) == "planned_flash_attn"+ ), ast.dump(following)+ assert strips >= 6, strips++ def test_the_re_plan_reads_the_current_split(self):+ import ast++ func = self._load_model_body()+ helper = next(+ node+ for node in ast.walk(func)+ if isinstance(node, ast.FunctionDef) and node.name == "_replanned_flash_attn"+ )+ call = next(+ node+ for node in ast.walk(helper)+ if isinstance(node, ast.Call)+ and getattr(node.func, "id", None) == "_planned_flash_attn_state"+ )+ passed = {kw.arg: kw.value for kw in call.keywords}+ assert getattr(passed["tensor_parallel"], "id", None) == "_current_tp"+ assert getattr(call.args[0], "id", None) == "extra_args"++ def test_no_sizing_closure_freezes_the_plan_in_a_default_argument(self):+ """A default argument is evaluated where the closure is written, so it holds+ whatever the plan said there, and the downgrades below re-plan it. The MTP draft+ reserve did that: a hybrid SWA draft head prices 4.5x larger with flash attention+ off (262,144 tokens, 3.5 GB on a 4-layer shape), so a frozen tensor-mode True left+ the reserve short of what the layer split it downgraded to actually allocates,+ beside a KV cache that had already re-read the plan."""+ import ast++ func = self._load_model_body()+ frozen = [+ (node.name, default.id)+ for node in ast.walk(func)+ if isinstance(node, ast.FunctionDef)+ for default in node.args.defaults+ if isinstance(default, ast.Name) and default.id == "planned_flash_attn"+ ]+ assert frozen == [], frozen+ # And the term is still priced, rather than dropped to silence the pin.+ mtp = next(+ node+ for node in ast.walk(func)+ if isinstance(node, ast.FunctionDef) and node.name == "mtp_overhead_fn"+ )+ call = next(+ node+ for node in ast.walk(mtp)+ if isinstance(node, ast.Call)+ and getattr(node.func, "attr", None) == "_estimate_mtp_overhead_bytes"+ )+ passed = {kw.arg: getattr(kw.value, "id", None) for kw in call.keywords}+ assert passed.get("flash_attn") == "planned_flash_attn", passed+++class TestTheReserveWhenTheFitterIsOff:+ """The reserve has to survive the respawn too, and a user's own `--fit off` is what+ takes the re-placement away."""++ def test_a_user_fit_off_keeps_the_conservative_reserve(self):+ from core.inference.llama_cpp import _reserved_flash_attn_state++ # The ordinary managed launch is unchanged: fitting is on, so the respawn re-places.+ assert _reserved_flash_attn_state(True, None, env = {}) is True+ assert _reserved_flash_attn_state(True, ["--ctx-size", "4096"], env = {}) is True+ # With the fitter off, the respawn lands on the placement this reserve chose.+ assert _reserved_flash_attn_state(True, ["--fit", "off"], env = {}) is False+ assert _reserved_flash_attn_state(True, ["--fit=off"], env = {}) is False+ assert _reserved_flash_attn_state(True, None, env = {"LLAMA_ARG_FIT": "off"}) is False+ # Last-wins, like every other flag: a later --fit on is the state that runs.+ assert _reserved_flash_attn_state(True, ["--fit", "off", "--fit", "on"], env = {}) is True+ assert (+ _reserved_flash_attn_state(True, ["--fit", "on"], env = {"LLAMA_ARG_FIT": "off"}) is True+ )++ def test_tensor_mode_keeps_its_answer(self):+ """No no-flash respawn to reserve for under SPLIT_MODE_TENSOR, and pricing the+ padded layout would refuse loads that fit."""+ from core.inference.llama_cpp import _reserved_flash_attn_state++ assert (+ _reserved_flash_attn_state(True, ["--fit", "off"], tensor_parallel = True, env = {}) is True+ )+ assert (+ _reserved_flash_attn_state(True, ["--fit", "off", "--split-mode", "tensor"], env = {})+ is True+ )++ def test_a_false_plan_is_never_raised(self):+ """This only holds a reading DOWN; an off is already the conservative one."""+ from core.inference.llama_cpp import _reserved_flash_attn_state++ assert _reserved_flash_attn_state(False, ["--fit", "off"], env = {}) is False+ assert _reserved_flash_attn_state(False, None, env = {}) is False++ def test_the_load_reserves_through_it(self):+ import inspect++ from core.inference import llama_cpp as module++ body = inspect.getsource(module.LlamaCppBackend.load_model)+ assert "_reserved_flash_attn_state(" in body+ # Both the first plan and every re-plan, or a downgrade would put the raw reading back.+ assert body.count("_reserved_flash_attn_state(") == 2, body.count(+ "_reserved_flash_attn_state("+ )+++class TestTheReplanIsAuthoritative:+ """A downgrade strips the extras, never the env, but the helper also resolves an+ inherited LLAMA_ARG_SPLIT_MODE, so a re-plan carrying the downgraded False still met a+ tensor answer inside it. The launch clears that variable for exactly this reason."""++ def _load_model_body(self):+ import ast+ import inspect+ import textwrap++ from core.inference import llama_cpp as module++ source = inspect.getsource(module.LlamaCppBackend.load_model)+ return ast.parse(textwrap.dedent(source)).body[0]++ def test_the_replan_scrubs_the_inherited_split_mode(self):+ import ast++ func = self._load_model_body()+ helper = next(+ node+ for node in ast.walk(func)+ if isinstance(node, ast.FunctionDef) and node.name == "_replan_env"+ )+ text = ast.dump(helper)+ assert "LLAMA_ARG_SPLIT_MODE" in text+ assert "LLAMA_ARG_TENSOR_SPLIT" in text+ replan = next(+ node+ for node in ast.walk(func)+ if isinstance(node, ast.FunctionDef) and node.name == "_replanned_flash_attn"+ )+ calls = [+ node+ for node in ast.walk(replan)+ if isinstance(node, ast.Call)+ and getattr(node.func, "id", None)+ in {"_planned_flash_attn_state", "_reserved_flash_attn_state"}+ ]+ assert len(calls) == 2, [getattr(c.func, "id", None) for c in calls]+ for call in calls:+ passed = {kw.arg for kw in call.keywords}+ assert "env" in passed, ast.dump(call)++ def test_the_helper_answers_layer_once_the_variable_is_gone(self):+ from core.inference.llama_cpp import _planned_flash_attn_state++ inherited = {"LLAMA_ARG_SPLIT_MODE": "tensor"}+ # The state the re-plan used to resolve: a downgraded toggle, a tensor environment.+ assert (+ _planned_flash_attn_state(["-fa", "auto"], tensor_parallel = False, env = inherited) is True+ )+ assert _planned_flash_attn_state(["-fa", "auto"], tensor_parallel = False, env = {}) is Falsediff --git a/studio/backend/tests/test_unmeasured_context_ceiling.py b/studio/backend/tests/test_unmeasured_context_ceiling.pynew file mode 100644index 00000000000..ae592dfc15a--- /dev/null+++ b/studio/backend/tests/test_unmeasured_context_ceiling.py@@ -0,0 +1,256 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A context nobody could measure is not a ceiling, and must not overrule a request.++Two arms fall back to a short context when the GGUF carries no attention dimensions, and+both applied it to an explicit request and published it, so 256k came back as 4096 (#9653).+A guess will not refuse against that number, so it may not overrule one either. Auto keeps+the conservative fallback.+"""++from __future__ import annotations++import importlib.util+import sys+from pathlib import Path++import pytest++_TESTS_DIR = Path(__file__).resolve().parent+_BACKEND_DIR = str(_TESTS_DIR.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)+if str(_TESTS_DIR) not in sys.path:+ sys.path.insert(0, str(_TESTS_DIR))+++def _load(module_name: str, file_name: str):+ spec = importlib.util.spec_from_file_location(module_name, _TESTS_DIR / file_name)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++# The no-GPU probe in this harness IS the Metal condition.+_metal = _load("_metal_guard_for_unmeasured_ctx", "test_metal_explicit_context_guard.py")+_metal_launch = _metal._launch+_ctx_values = _metal._ctx_values++from core.inference.llama_cpp import ( # noqa: E402+ _FIT_MIN_CTX,+ _TP_UNMEASURED_CTX,+ LlamaCppBackend,+)++GB = 1024**3+NATIVE = 262144+TWO_80 = [(0, 80 * 1024), (1, 80 * 1024)]+++def _measurable() -> LlamaCppBackend:+ backend = LlamaCppBackend()+ backend._n_layers = 28+ backend._embedding_length = 1024+ backend._n_heads = 16+ backend._n_kv_heads = 8+ backend._kv_key_length = 128+ backend._kv_value_length = 128+ backend._context_length = NATIVE+ return backend+++def _unmeasurable() -> LlamaCppBackend:+ backend = LlamaCppBackend()+ backend._context_length = NATIVE+ assert not backend._can_estimate_kv()+ return backend+++class TestTheTensorParallelFallback:+ def test_an_explicit_request_is_honoured(self):+ effective, ceiling, *_ = _unmeasurable()._plan_tensor_parallel(+ TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE, explicit_ctx = True+ )+ assert effective == NATIVE+ # The pair the issue quotes from /v1/models: the ceiling may not contradict it.+ assert ceiling >= effective++ def test_auto_still_takes_the_conservative_fallback(self):+ effective, ceiling, *_ = _unmeasurable()._plan_tensor_parallel(+ TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE+ )+ assert effective == _TP_UNMEASURED_CTX+ assert ceiling == _TP_UNMEASURED_CTX++ def test_a_measured_cap_still_binds_an_explicit_request(self):+ """``--fit`` is a no-op in tensor mode, so a measured cap is all that stands between+ the request and a startup OOM."""+ tight = [(0, 6 * 1024), (1, 6 * 1024)]+ capped, _ceiling, *_ = _measurable()._plan_tensor_parallel(+ tight, 8 * GB, NATIVE, max_target_ctx = NATIVE, explicit_ctx = True+ )+ assert 0 < capped < NATIVE++ def test_a_measurable_plan_that_fits_is_unchanged_by_the_flag(self):+ with_flag = _measurable()._plan_tensor_parallel(+ TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE, explicit_ctx = True+ )+ without = _measurable()._plan_tensor_parallel(+ TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE+ )+ assert with_flag == without++ def test_the_flag_defaults_off_for_every_existing_caller(self):+ assert _unmeasurable()._plan_tensor_parallel(TWO_80, 20 * GB, NATIVE)[0] == (+ _TP_UNMEASURED_CTX+ )++ @pytest.mark.parametrize("target", [0, -1])+ def test_no_request_is_not_a_request(self, target):+ effective, _ceiling, *_ = _unmeasurable()._plan_tensor_parallel(+ TWO_80, 20 * GB, target, explicit_ctx = True+ )+ assert effective == _TP_UNMEASURED_CTX+++class TestTheNotice:+ def test_it_names_the_context_that_was_launched(self):+ message = LlamaCppBackend._unmeasured_context_notice(NATIVE)+ assert message is not None+ assert "262,144" in message++ def test_it_says_the_maximum_is_not_a_measurement(self):+ message = LlamaCppBackend._unmeasured_context_notice(NATIVE)+ assert "not a measured ceiling" in message++ def test_it_does_not_read_as_a_refusal(self):+ message = LlamaCppBackend._unmeasured_context_notice(NATIVE)+ assert "launched as requested" in message+ for blame in ("cannot", "refus", "not supported"):+ assert blame not in message.lower()++ @pytest.mark.parametrize("cache_type", [None, "", "f16", "fp16"])+ def test_the_kv_hint_is_offered_on_an_unquantized_cache(self, cache_type):+ assert "q8_0" in LlamaCppBackend._unmeasured_context_notice(NATIVE, cache_type)++ @pytest.mark.parametrize("cache_type", ["q8_0", "q4_0"])+ def test_it_is_not_offered_once_the_cache_is_quantized(self, cache_type):+ assert "q8_0" not in LlamaCppBackend._unmeasured_context_notice(NATIVE, cache_type)++ @pytest.mark.parametrize("requested", [0, -1])+ def test_nothing_to_report_without_a_request(self, requested):+ assert LlamaCppBackend._unmeasured_context_notice(requested) is None++ def test_it_does_not_cost_the_load_a_memory_warning(self):+ """``_record_load_warning`` is first-notice-wins with one slot, and both arms+ that raise an unmeasured ceiling decide it long before the host-RAM and offload+ advisories. Recording it there took the slot from a memory warning on the one+ path that simultaneously raises the launched context, so the user who is told+ their load will page from disk heard only that the ceiling was a guess. It is+ held in ``_unmeasured_ctx_notice`` and flushed past both, appending when one+ spoke."""+ import ast+ import inspect+ import textwrap++ source = inspect.getsource(LlamaCppBackend.load_model)+ func = ast.parse(textwrap.dedent(source)).body[0]++ # Nothing hands the notice straight to the slot where it is decided.+ for node in ast.walk(func):+ if not (+ isinstance(node, ast.Call)+ and getattr(node.func, "attr", None) == "_record_load_warning"+ ):+ continue+ for arg in node.args:+ assert not (+ isinstance(arg, ast.Call)+ and getattr(arg.func, "attr", None) == "_unmeasured_context_notice"+ ), f"the notice claims the warning slot at line {node.lineno}"++ held = [+ node.lineno+ for node in ast.walk(func)+ if isinstance(node, ast.Assign)+ and getattr(node.targets[0], "id", None) == "_unmeasured_ctx_notice"+ and isinstance(node.value, ast.Call)+ ]+ flushed = [+ node.lineno+ for node in ast.walk(func)+ if isinstance(node, ast.If)+ and isinstance(node.test, ast.Name)+ and node.test.id == "_unmeasured_ctx_notice"+ ]+ # Both arms hold it, and there is one flush.+ assert len(held) == 2, held+ assert len(flushed) == 1, flushed++ # The flush is past this attempt's memory advisories, or it displaces one again.+ # Named rather than "every _record_load_warning": the arch-crash retry's pair sits+ # textually below the publish point but belongs to a later attempt, which clears+ # the slot through _begin_load_warnings before recording anything.+ advisories = {+ arg.id: node.lineno+ for node in ast.walk(func)+ if isinstance(node, ast.Call)+ and getattr(node.func, "attr", None) == "_record_load_warning"+ for arg in node.args+ if isinstance(arg, ast.Name)+ }+ for name in ("_ram_msg", "_offload_msg"):+ assert name in advisories, (name, sorted(advisories))+ assert flushed[0] > advisories[name], (name, flushed, advisories)+++class TestTheMetalArm:+ """Simulated with an empty GPU probe; no Metal device is exercised."""++ def _launch_unmeasurable(self, tmp_path, monkeypatch, **kwargs):+ return _metal_launch(+ tmp_path,+ monkeypatch,+ metal = True,+ can_estimate_kv = False,+ real_fit = True,+ budget_bytes = 24 * GB,+ native = NATIVE,+ **kwargs,+ )++ def test_an_explicit_request_reaches_llama_server(self, tmp_path, monkeypatch):+ captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = NATIVE)+ assert _ctx_values(captured["cmd"])[-1] == str(NATIVE)++ def test_the_published_ceiling_is_not_below_what_launched(self, tmp_path, monkeypatch):+ """The floor used to be published while the child ran at the request."""+ captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = NATIVE)+ backend = captured["backend"]+ assert backend.max_context_length >= NATIVE+ assert backend.native_context_length == NATIVE++ def test_the_load_says_the_ceiling_was_never_measured(self, tmp_path, monkeypatch):+ captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = NATIVE)+ assert "not a measured ceiling" in (captured["backend"].last_load_warning or "")++ def test_auto_keeps_the_floor(self, tmp_path, monkeypatch):+ captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = 0)+ assert _ctx_values(captured["cmd"])[-1] == str(_FIT_MIN_CTX)+ assert captured["backend"].max_context_length == _FIT_MIN_CTX++ def test_auto_is_not_told_anything(self, tmp_path, monkeypatch):+ """Auto made no request; an advisory on every unreadable header is noise."""+ captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = 0)+ assert captured["backend"].last_load_warning is None++ def test_a_measured_ceiling_still_refuses(self, tmp_path, monkeypatch):+ with pytest.raises(RuntimeError, match = "unified"):+ _metal_launch(+ tmp_path,+ monkeypatch,+ n_ctx = 32768,+ metal = True,+ can_estimate_kv = True,+ )