Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: honour an unmeasured context request and price the real attention plan (#11025)

Daniel Han6d938276e19d
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 126f6922c7b..f218d8d55fb 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -3787,6 +3787,9 @@ _FIT_FLOOR_MIN_CTX = 256
# the ceiling that assumed one.
_LLAMA_FIT_MIN_CTX = 4096
+# _plan_tensor_parallel's no-KV fallback. A GUESS: may shrink Auto, never a request (#9653).
+_TP_UNMEASURED_CTX = 4096
+
# Auto only reaches this path when no discrete-GPU subset can hold the model.
# llama.cpp must offload layers to host RAM either way, so prefer a context that
# remains useful for chat. Separate from _FIT_MIN_CTX: that value is also a search
@@ -4668,6 +4671,142 @@ def _kv_unified_from_args(
return enabled
+# llama_init_from_model forces flash attention off for LLM_ARCH_GROK whatever the launch
+# asks for: Grok softcaps the KQ logits, which the fused kernel does not carry.
+_FLASH_ATTN_INCOMPATIBLE_ARCHITECTURES = frozenset({"grok"})
+
+
+def _architecture_forces_flash_attn_off(architecture: Optional[str]) -> bool:
+ """Whether llama.cpp will disable flash attention for this architecture on its own."""
+ return str(architecture or "").strip().lower() in _FLASH_ATTN_INCOMPATIBLE_ARCHITECTURES
+
+
+def _planned_flash_attn_state(
+ extra_args: Optional[Iterable[str]] = None,
+ *,
+ planned_cache_types: Optional[tuple[str, str]] = None,
+ supports_flash_attn: bool = True,
+ tensor_parallel: bool = False,
+ architecture: Optional[str] = None,
+ env: Optional[Mapping[str, str]] = None,
+) -> bool:
+ """One answer for the estimate and for the argv: the estimator floors V at f16 and pads
+ it model-wide when flash attention is off, so resolving the two separately prices a
+ different load, not a conservative one (#9697, #10489)."""
+ if not supports_flash_attn:
+ return False
+ if _architecture_forces_flash_attn_off(architecture):
+ # Ahead of every rule below, as llama.cpp puts it ahead of both its own upgrades.
+ return False
+ v_type = (planned_cache_types[1] if planned_cache_types else "f16") or "f16"
+ quantized_v = str(v_type).strip().lower() not in {"f16", "bf16", "f32"}
+ if quantized_v:
+ # Not a choice: llama.cpp turns it on itself rather than refusing the load.
+ return True
+ # Prepended because ``load_model`` appends a managed ``--flash-attn on``: the env loses
+ # to it and only the user's own extras beat it (#9697, #10489).
+ effective_args = ["--flash-attn", "on", *(str(arg) for arg in extra_args or ())]
+ if _asked_for_auto_flash_attn(effective_args, env = env):
+ if _effective_tensor_parallel(extra_args, tensor_parallel, env):
+ # Not undecided: llama.cpp upgrades AUTO to ENABLED under SPLIT_MODE_TENSOR.
+ return True
+ # ``auto`` is decided at load time and can come back no; size for the costlier answer.
+ return False
+ return _flash_attn_enabled_from_args(effective_args, default = True, env = env)
+
+
+def _user_fit_disabled(
+ extra_args: Optional[Iterable[str]] = None, *, env: Optional[Mapping[str, str]] = None
+) -> bool:
+ """Whether the USER has turned llama.cpp's fitter off, last-wins: Unsloth's own token
+ comes first and a retry rewrites it, so only a user's later value survives a re-place."""
+ values = [str(arg) for arg in extra_args] if extra_args else []
+ asked: Optional[str] = None
+ inherited = (os.environ if env is None else env).get("LLAMA_ARG_FIT")
+ if inherited is not None and str(inherited).strip():
+ asked = str(inherited).strip()
+ for i, raw in enumerate(values):
+ if _flag_name(raw) not in {"-fit", "--fit"}:
+ continue
+ _, eq, inline = raw.partition("=")
+ value = inline if eq else "on"
+ if not eq and i + 1 < len(values):
+ value = values[i + 1]
+ asked = value
+ return str(asked).strip().lower() in {"off", "0", "false", "no", "disabled"}
+
+
+def _placement_is_fitter_proof(
+ extra_args: Optional[Iterable[str]] = None,
+ *,
+ gpu_layers: Optional[int] = None,
+ env: Optional[Mapping[str, str]] = None,
+) -> bool:
+ """Whether a layer count llama.cpp's fitter refuses to move owns this placement:
+ ``common_params_fit_impl`` throws "n_gpu_layers already set by user" (common/fit.cpp:377)
+ for any value but -1, so re-enabling the fitter for a respawn buys nothing."""
+ if gpu_layers is not None and gpu_layers >= 0:
+ return True
+ values = [str(arg) for arg in extra_args] if extra_args else []
+ for i, raw in enumerate(values):
+ if _flag_name(raw) not in _GPU_LAYER_FLAGS:
+ continue
+ _, eq, inline = raw.partition("=")
+ value = inline if eq else ""
+ if not eq and i + 1 < len(values):
+ value = values[i + 1]
+ value = value.strip().lower()
+ if value and value not in _LLAMA_ARG_AUTO_VALUES:
+ return True
+ return bool(_env_fixes_gpu_layers(env))
+
+
+def _reserved_flash_attn_state(
+ planned: bool,
+ extra_args: Optional[Iterable[str]] = None,
+ *,
+ tensor_parallel: bool = False,
+ gpu_layers: Optional[int] = None,
+ env: Optional[Mapping[str, str]] = None,
+) -> bool:
+ """``planned``, held to the conservative reading where a respawn cannot be re-placed.
+
+ A flash-attention crash respawns with it off, which pads V model-wide, so where the
+ fitter cannot re-place that respawn the larger cache lands on a placement chosen for the
+ smaller one. Tensor mode has no such respawn, and pricing it would refuse loads that fit.
+ """
+ if not planned:
+ return planned
+ if _effective_tensor_parallel(extra_args, tensor_parallel, env):
+ return planned
+ if _user_fit_disabled(extra_args, env = env):
+ return False
+ if _placement_is_fitter_proof(extra_args, gpu_layers = gpu_layers, env = env):
+ return False
+ return planned
+
+
+def _asked_for_auto_flash_attn(
+ extra_args: Optional[Iterable[str]] = None, *, env: Optional[Mapping[str, str]] = None
+) -> bool:
+ """``_flash_attn_enabled_from_args`` folds auto into True because it answers what the
+ argv will SAY; sizing needs to know nobody has decided yet."""
+ asked: Optional[str] = None
+ value = (os.environ if env is None else env).get("LLAMA_ARG_FLASH_ATTN")
+ if value in _LLAMA_ARG_TRUE_FALSE_AUTO_VALUES:
+ asked = value
+ values = [str(arg) for arg in extra_args] if extra_args else []
+ for i, raw in enumerate(values):
+ if _flag_name(raw) not in {"-fa", "--flash-attn"}:
+ continue
+ _, eq, inline = raw.partition("=")
+ value = inline if eq else "on"
+ if not eq and i + 1 < len(values) and values[i + 1] in _LLAMA_ARG_TRUE_FALSE_AUTO_VALUES:
+ value = values[i + 1]
+ asked = value
+ return asked in _LLAMA_ARG_AUTO_VALUES
+
+
def _flash_attn_enabled_from_args(
args: Optional[Iterable[str]],
default: bool = True,
@@ -4697,24 +4836,6 @@ def _flash_attn_enabled_from_args(
return enabled
-def _planned_flash_attn(
- extra_args: Optional[Iterable[str]],
- supports_flash_attn: bool,
- v_cache_type: Optional[str] = None,
- architecture: Optional[str] = None,
-) -> bool:
- """Flash attention as the launch command will resolve it.
-
- The managed --flash-attn on follows the environment and precedes user extras, so
- only extras can turn it off. A build without the flag gets neither the flag nor
- the inherited LLAMA_ARG_FLASH_ATTN. A quantized V cache forces it on, except on
- Grok, which llama.cpp always runs without it."""
- if not supports_flash_attn or architecture == "grok":
- return False
- enabled = _flash_attn_enabled_from_args(extra_args, default = True, env = {})
- return enabled or (v_cache_type or "f16").strip().lower() not in {"f16", "bf16", "f32"}
-
-
def _effective_spec_type(
extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
) -> Optional[str]:
@@ -14369,6 +14490,29 @@ class LlamaCppBackend:
f"{LlamaCppBackend.METAL_CTX_OVERCOMMIT_ENV}=1 to load it anyway."
)
+ @staticmethod
+ def _unmeasured_context_notice(
+ requested_ctx: int, cache_type_kv: Optional[str] = None
+ ) -> Optional[str]:
+ """Advisory for a hand-set context with no KV estimate behind it: the short-context
+ fallbacks used to cut a 262,144-token request to 4,096 and publish it (#9653). Not a
+ refusal, which on a guess blocks live loads."""
+ if requested_ctx <= 0:
+ return None
+ kv_hint = (
+ " Setting the KV cache to q8_0 roughly halves what the context costs."
+ if (cache_type_kv or "f16").strip().lower() in ("f16", "fp16", "")
+ else ""
+ )
+ return (
+ f"This model's GGUF does not carry the attention dimensions needed to size "
+ f"the KV cache, so the {requested_ctx:,}-token context was launched as "
+ f"requested without being checked against available memory. The reported "
+ f"maximum context is that request, not a measured ceiling. If the load fails "
+ f"or the machine runs short of memory, lower the context or leave it on "
+ f"Auto.{kv_hint}"
+ )
+
# Skip the wait when the last kill is older than this; the driver has
# already reclaimed the prior process's allocations.
_VRAM_SETTLE_WINDOW_S: float = 15.0
@@ -15080,11 +15224,9 @@ class LlamaCppBackend:
# layer's V is padded to hparams.n_embd_v_gqa_max() over the WHOLE model,
# which is what _estimate_kv_cache_bytes charges (_max_kv_value_width). The
# V half goes constant while K stays per-layer, so an unpadded vector
- # prices a ratio the total does not have. Not an edge case: load_model pins
- # planned_flash_attn = False unconditionally (llama_cpp.py:16690), so the
- # padded branch is the only one the total ever takes. bpe_v is floored at
- # f16 for a quantised cache, and with V constant that asymmetry moves the
- # ratio too, so carry both rather than cancelling one.
+ # prices a ratio the total does not have. bpe_v is floored at f16 for a quantised
+ # cache, and with V constant that asymmetry moves the ratio too, so carry both
+ # rather than cancelling one.
bpe_k = _kv_bytes_per_elem(cache_type_kv)
bpe_v = bpe_k if flash_attn else max(bpe_k, _kv_bytes_per_elem("f16"))
padded_v_width = None if flash_attn else self._max_kv_value_width(val_len, val_len_swa)
@@ -20091,6 +20233,7 @@ class LlamaCppBackend:
flash_attn: bool = True,
vram_fraction: Optional[float] = None,
scratch_cache_type_kv: Optional[str] = None,
+ explicit_ctx: bool = False,
) -> tuple[int, int, list[int], Optional[list[int]]]:
"""Plan a ``--split-mode tensor`` load. Pure: no model or GPU needed.
@@ -20122,6 +20265,9 @@ class LlamaCppBackend:
``scratch_cache_type_kv`` is the LIGHTER one and prices the dequant scratch,
so an asymmetric pair keeps both terms conservative. Defaults to
``cache_type_kv`` when unset, which is the symmetric case.
+
+ ``explicit_ctx`` only matters when the KV cache cannot be sized: the fallback is then
+ a guess, so it may shrink Auto but never overrule a request (#9653).
"""
# Per-GPU usable budget: free - (1-frac)*total, else (unknown total, e.g. a
@@ -20220,7 +20366,11 @@ class LlamaCppBackend:
)
def _cc_ctx(ctx: int) -> int:
- return n_dev * self._compute_buffer_ctx_bytes(ctx, n_ubatch, cc_cache_type)
+ # The same resolved state ``_kv_at`` prices with: tensor mode usually answers
+ # True, but not always, and the mask this buffer carries is f32 when it does not.
+ return n_dev * self._compute_buffer_ctx_bytes(
+ ctx, n_ubatch, cc_cache_type, flash_attn = flash_attn
+ )
def _fit_ctx(ctx: int) -> int:
# Largest context whose KV (+ MTP draft reserve + context-linear
@@ -20248,7 +20398,7 @@ class LlamaCppBackend:
hi = mid - 1
return best
# KV size unknown -> can't prove a safe cap; floor.
- return min(4096, ctx) if ctx > 0 else 4096
+ return min(_TP_UNMEASURED_CTX, ctx) if ctx > 0 else _TP_UNMEASURED_CTX
# max_available_ctx is the hardware ceiling for the UI bound, sized from
# the native context independent of an explicit small -c (which only
@@ -20256,6 +20406,11 @@ class LlamaCppBackend:
max_ctx_target = max_target_ctx if (max_target_ctx and max_target_ctx > 0) else target_ctx
max_available_ctx = _fit_ctx(max_ctx_target)
effective_ctx = min(_fit_ctx(target_ctx), max_available_ctx)
+ if not self._can_estimate_kv() and explicit_ctx and target_ctx > 0:
+ # The min() above would apply a guess to a typed context: 262,144 requested came
+ # back as 4,096, for the launch and for max_context_length (#9653).
+ effective_ctx = target_ctx
+ max_available_ctx = max(max_available_ctx, effective_ctx)
min_usable_mib = min(usable_by_idx.values())
kv_bytes = _kv_at(effective_ctx) if (self._can_estimate_kv() and effective_ctx > 0) else 0
@@ -20378,7 +20533,11 @@ class LlamaCppBackend:
# card more, so it both under-refuses and over-refuses. Mirror the
# planner: flat per device, alongside the compute-graph reserve.
cc_per_device_bytes = self._compute_buffer_ctx_bytes(
- effective_ctx, n_ubatch, scratch_cache_type_kv or cache_type_kv
+ effective_ctx,
+ n_ubatch,
+ scratch_cache_type_kv or cache_type_kv,
+ # Same state as the KV above.
+ flash_attn = flash_attn,
)
total_bytes = model_size + kv_bytes + mtp_bytes + max(0, soft_overhead_bytes)
@@ -22695,9 +22854,6 @@ class LlamaCppBackend:
_pipeline_parallel_off = _pipeline_parallel_disabled_by_args(
extra_args, n_layers = self._n_layers
)
- # A hard-crash recovery may relaunch this same plan with FA off.
- # Size that larger cache up front so the recovery cannot OOM.
- planned_flash_attn = False
cache_override = parse_cache_override(extra_args)
# Budget the heavier of asymmetric --cache-type-k/-v extras (they
# win per axis at launch, appended last); resolve_cache_type_kv only
@@ -22741,13 +22897,57 @@ class LlamaCppBackend:
None if _cache_type_from_env else cache_type_kv,
extra_args,
)
- # Price compute for the launch; KV separately covers the no-flash retry.
- _launch_flash_attn = _planned_flash_attn(
+ # Auto leaves this negative, which is llama.cpp's default and no override.
+ _manual_gpu_layers = gpu_layers if gpu_memory_mode == "manual" else None
+ # After the cache pair, because a quantized V forces flash attention on. NOT
+ # pinned False for the no-flash recovery: --fit re-places that respawn.
+ planned_flash_attn = _reserved_flash_attn_state(
+ _planned_flash_attn_state(
+ extra_args,
+ planned_cache_types = _planned_cache_pair,
+ supports_flash_attn = bool(server_caps.get("supports_flash_attn", True)),
+ tensor_parallel = tensor_parallel,
+ architecture = self._architecture,
+ ),
extra_args,
- bool(server_caps.get("supports_flash_attn", True)),
- _planned_cache_pair[1],
- self._architecture,
+ tensor_parallel = tensor_parallel,
+ gpu_layers = _manual_gpu_layers,
)
+
+ def _replan_env(_current_tp: bool) -> "Optional[Mapping[str, str]]":
+ """The environment the CHILD will see: the launch clears a non-layer
+ inherited LLAMA_ARG_SPLIT_MODE, which otherwise turns tensor mode back
+ on inside the helper."""
+ if _current_tp:
+ return None
+ inherited = (os.environ.get("LLAMA_ARG_SPLIT_MODE") or "").strip().lower()
+ if not inherited or inherited == "layer":
+ return None
+ return {
+ key: value
+ for key, value in os.environ.items()
+ if key not in ("LLAMA_ARG_SPLIT_MODE", "LLAMA_ARG_TENSOR_SPLIT")
+ }
+
+ def _replanned_flash_attn(_current_tp: bool) -> bool:
+ """Re-resolve the planned attention after a tensor-mode downgrade: AUTO
+ plans it ON under tensor, so a stale plan budgets a V layer never gets."""
+ _env = _replan_env(_current_tp)
+ return _reserved_flash_attn_state(
+ _planned_flash_attn_state(
+ extra_args,
+ planned_cache_types = _planned_cache_pair,
+ supports_flash_attn = bool(server_caps.get("supports_flash_attn", True)),
+ tensor_parallel = _current_tp,
+ architecture = self._architecture,
+ env = _env,
+ ),
+ extra_args,
+ tensor_parallel = _current_tp,
+ gpu_layers = _manual_gpu_layers,
+ env = _env,
+ )
+
# A user --split-mode in extras last-wins-overrides the toggle, and
# an inherited tensor LLAMA_ARG_SPLIT_MODE flips it on (the child
# would run tensor unbudgeted otherwise). The duplicate-load matchers
@@ -22766,6 +22966,9 @@ class LlamaCppBackend:
)
tensor_parallel = False
tensor_split = None
+ # The plan was made under tensor, the one state forcing AUTO on. Same
+ # for every later downgrade.
+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)
# Record the requested strategy for /status and the load
# response. 'manual' has no fallback, so the request value is the
# value actually applied.
@@ -22807,6 +23010,7 @@ class LlamaCppBackend:
"than 2 GPUs are in use; ignoring (needs >= 2)."
)
tensor_parallel = False
+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)
if tensor_parallel and gpu_memory_mode == "manual" and gpu_layers < 0:
logger.info(
"Manual mode with Auto layers hands memory management to "
@@ -22814,6 +23018,7 @@ class LlamaCppBackend:
"parallelism; ignoring the tensor split."
)
tensor_parallel = False
+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)
if ctx_override is not None and ctx_override > 0:
logger.info(f"User --ctx-size {ctx_override} honored; skipping auto-reduce")
if cache_override is not None:
@@ -22961,6 +23166,12 @@ class LlamaCppBackend:
# a ``gpu_indices`` the arm rebuilt from a WIDER device set than the
# planner's reserve filter admitted, and index a card that is not in it.
_tp_planned = False
+ # Held rather than recorded where it is decided: _record_load_warning is
+ # first-notice-wins, and both arms that raise an unmeasured ceiling run
+ # long before the host-RAM and offload advisories, so recording there took
+ # the slot from a memory warning on the one path that also raises the
+ # launched context. Flushed past those two, appending if one spoke.
+ _unmeasured_ctx_notice: Optional[str] = None
total_by_idx: dict[int, int] = {}
_gpu_mem: list[tuple[int, int, int]] = []
model_size = None # set in the fit try; used by the APU RAM guard
@@ -23194,6 +23405,7 @@ class LlamaCppBackend:
# otherwise reach llama-server. Strip it like the TP
# downgrade does.
extra_args = strip_split_mode_only(extra_args)
+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)
elif gpu_memory_mode == "manual":
# Manual offload (--gpu-layers + --fit off): no automatic
# device masking (a gpu_ids pick still pins below) or
@@ -23214,6 +23426,7 @@ class LlamaCppBackend:
# those.
if tensor_parallel or split_mode_override == "tensor":
extra_args = strip_split_mode_only(extra_args)
+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)
# Will MTP engage? If so, auto-fit reserves draft-model VRAM.
# Mirrors _build_speculative_flags: forced mtp/mtp+ngram always
@@ -23485,7 +23698,6 @@ class LlamaCppBackend:
_swa_full: bool = swa_full,
_kv_unified: bool = planned_kv_unified,
_n_ubatch: Optional[int] = _effective_ubatch,
- _flash_attn: bool = planned_flash_attn,
) -> int:
v = self._estimate_mtp_overhead_bytes(
ctx,
@@ -23500,7 +23712,13 @@ class LlamaCppBackend:
swa_full = _swa_full,
kv_unified = _kv_unified,
n_ubatch = _n_ubatch,
- flash_attn = _flash_attn,
+ # Read at call time, like the KV and compute terms
+ # beside it: a downgrade re-plans the attention, and a
+ # default argument would hold the tensor plan's True
+ # while those price the layer split's padded V, which
+ # short-changes the draft reserve 4.5x on a hybrid SWA
+ # draft head. The other captures settle with the load.
+ flash_attn = planned_flash_attn,
)
return v if v is not None else 0
@@ -23555,12 +23773,13 @@ class LlamaCppBackend:
# per-device rate also steps up once split, unless llama.cpp
# declines the pipeline parallelism that causes the step.
# A nonzero ``slots`` prices that candidate count and its micro-batch.
+ # Read at call time: a tensor-mode downgrade re-plans it.
return max(1, n_gpus) * self._compute_buffer_ctx_bytes(
ctx,
_ubatch_for_slots(slots) if slots else _effective_ubatch,
_scratch_cache_type_kv,
layer_split = n_gpus > 1 and not _pipeline_parallel_off,
- flash_attn = _launch_flash_attn,
+ flash_attn = planned_flash_attn,
n_parallel = slots or n_parallel,
)
@@ -23651,6 +23870,7 @@ class LlamaCppBackend:
# than let the layer planner settle on the first card it fits.
_layer_min_gpus = max(_layer_min_gpus, len(gpus))
extra_args = strip_split_mode_only(extra_args)
+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)
if tensor_parallel and self._tensor_split_aborts(
binary, model_identifier, _planned_cache_pair
):
@@ -23667,6 +23887,7 @@ class LlamaCppBackend:
_layer_min_gpus = max(_layer_min_gpus, len(gpus))
# An extras --split-mode tensor would re-engage tensor mode.
extra_args = strip_split_mode_only(extra_args)
+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)
# Tensor mode replicates a compute buffer on every GPU, so drop
# GPUs below that reserve from the set up front (gpu_indices
@@ -23716,6 +23937,7 @@ class LlamaCppBackend:
if len(gpus) >= 2:
_layer_min_gpus = max(_layer_min_gpus, len(gpus))
extra_args = strip_split_mode_only(extra_args)
+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)
# Normalize the speculative reserve BEFORE anything prices it.
# _mtp_bytes reads mtp_overhead_fn at call time, so leaving this
@@ -24292,6 +24514,7 @@ class LlamaCppBackend:
if len(tp_gpus) >= 2:
_layer_min_gpus = max(_layer_min_gpus, len(tp_gpus))
extra_args = strip_split_mode_only(extra_args)
+ planned_flash_attn = _replanned_flash_attn(tensor_parallel)
# Layer split now, so the withheld verdict applies -- but not
# when a GPU drafter is still in play. The drafter-drop probe
# is gated off under tensor parallelism, so it has not run;
@@ -24363,9 +24586,15 @@ class LlamaCppBackend:
# would otherwise size placement against a fraction the
# ranking above never used.
vram_fraction = _vram_frac,
+ explicit_ctx = explicit_ctx,
)
use_fit = False
_tp_planned = True
+ if explicit_ctx and not self._can_estimate_kv():
+ # Tensor mode emits --fit off, so nothing downstream catches it.
+ _unmeasured_ctx_notice = self._unmeasured_context_notice(
+ effective_ctx, cache_type_kv
+ )
elif gpus and self._can_estimate_kv() and effective_ctx > 0:
# Compute the largest hardware-aware cap from the model's
# native context across all usable GPU subsets (for UI
@@ -24627,6 +24856,9 @@ class LlamaCppBackend:
# The other measured verdict: nothing fits, so there is no ceiling
# to name and every explicit request over-commits.
_apple_nothing_fits = False
+ # No measurement: the floor below is a guess, so it may neither
+ # refuse nor be published as a ceiling (#9653).
+ _apple_ctx_unmeasured = False
# Reserve the flat MTP fraction up front like the discrete
# _pin_fraction, so an unsized MTP draft (e.g. Qwen3.6-MTP, #6529)
# can't over-commit. No-op when MTP is off; exclusive with the
@@ -24746,8 +24978,16 @@ class LlamaCppBackend:
# and floor to _FIT_MIN_CTX rather than launch at native and
# over-commit.
max_available_ctx = min(_metal_floor_ctx, native_ctx_for_cap)
+ _apple_ctx_unmeasured = True
if not explicit_ctx:
effective_ctx = max_available_ctx
+ elif _apple_ctx_unmeasured:
+ # This arm never overruled the request but published the floor:
+ # max_context_length 4,096 for a load launched at 262,144 (#9653).
+ max_available_ctx = max(max_available_ctx, effective_ctx)
+ _unmeasured_ctx_notice = self._unmeasured_context_notice(
+ effective_ctx, cache_type_kv
+ )
elif (
(_apple_measured_ceiling is not None or _apple_nothing_fits)
and not _caller_owns_budget
@@ -27825,6 +28065,39 @@ class LlamaCppBackend:
# fallback). A per-call flag left those successes unrecorded.
_did_rocm_retry = False
+ def _enable_managed_fit_for_no_flash(fa_cmd: list) -> list:
+ """Unsloth's own ``--fit off`` back to ``on`` for a --flash-attn off respawn.
+
+ The reserve is only safe because the respawn gets re-placed, and it
+ inherits a managed ``--fit off`` that `_fit_off_retry_eligible` will not
+ override, so without this it OOMs on the smaller-cache placement. Skipped
+ where the flip buys nothing: a user's own ``--fit``, or a fixed count.
+ """
+ if "--fit" not in fa_cmd:
+ return fa_cmd
+ if any(
+ _flag_name(str(token)) in {"-fit", "--fit"} for token in (extra_args or ())
+ ):
+ return fa_cmd
+ # An inherited LLAMA_ARG_FIT=off is the user's too, and only manual mode
+ # scrubs it: the flip would override it on the CLI, and the reserve
+ # _user_fit_disabled already took prices this respawn where it lands.
+ if _user_fit_disabled(extra_args, env = env):
+ return fa_cmd
+ if _placement_is_fitter_proof(fa_cmd, gpu_layers = _manual_gpu_layers, env = env):
+ return fa_cmd
+ index = fa_cmd.index("--fit")
+ if index + 1 >= len(fa_cmd) or str(fa_cmd[index + 1]).strip().lower() == "on":
+ return fa_cmd
+ flipped = list(fa_cmd)
+ flipped[index + 1] = "on"
+ logger.info(
+ "Re-enabling the fitter for the --flash-attn off retry: the padded "
+ "V cache the retry runs with is bigger than the placement the fit "
+ "priced with flash attention on."
+ )
+ return flipped
+
def _drop_fit_load_mode_for_no_flash(fa_cmd: list) -> list:
"""The fit's ``--load-mode none`` off a --flash-attn off respawn.
@@ -28447,6 +28720,15 @@ class LlamaCppBackend:
max_available_ctx if max_available_ctx > 0 else self._effective_context_length
)
+ # Past the host-RAM and offload advisories, so it can say the ceiling is
+ # the request without costing the user a memory warning it does not
+ # replace: appended when one of those spoke, recorded when none did.
+ if _unmeasured_ctx_notice:
+ if self._last_load_warning:
+ self._amend_load_warning(" " + _unmeasured_ctx_notice)
+ else:
+ self._record_load_warning(_unmeasured_ctx_notice)
+
# LoadRequest carries this flag, so a stale rollback, an API caller or
# a swapped-out runtime can ask for a replay that never happened. Hold
# it to the crash path's bar (managed auto Vulkan, no explicit
@@ -29037,6 +29319,7 @@ class LlamaCppBackend:
"--flash-attn off retry."
)
_fa_cmd = _drop_fit_load_mode_for_no_flash(_fa_cmd)
+ _fa_cmd = _enable_managed_fit_for_no_flash(_fa_cmd)
_flash_attn_known_off = True
cmd = _fa_cmd
healthy = _spawn_and_wait(_fa_cmd, label = "-noflash")
@@ -29111,6 +29394,7 @@ class LlamaCppBackend:
"--flash-attn off retry."
)
_fa_cmd = _drop_fit_load_mode_for_no_flash(_fa_cmd)
+ _fa_cmd = _enable_managed_fit_for_no_flash(_fa_cmd)
_flash_attn_known_off = True
cmd = _fa_cmd
healthy = (
@@ -29548,12 +29832,10 @@ class LlamaCppBackend:
0,
int(self._DEFAULT_N_UBATCH if _launched_ubatch is None else _launched_ubatch),
)
- self._flash_attn_enabled = (
- _flash_attn_enabled_from_args(
- _last_spawn_cmd, default = not _flash_attn_known_off, env = env
- )
- and self._architecture != "grok"
- )
+ # What the child is RUNNING: the same architecture rule the plan above used.
+ self._flash_attn_enabled = _flash_attn_enabled_from_args(
+ _last_spawn_cmd, default = not _flash_attn_known_off, env = env
+ ) and not _architecture_forces_flash_attn_off(self._architecture)
self._effective_cache_types = _effective_main_cache_types(
_last_spawn_cmd,
env,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 3dfd8fefb0c..f22fcf49b1c 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1618,9 +1618,10 @@ try:
_kv_bytes_per_elem,
_kv_unified_from_args,
_metal_device_is_paravirtual,
- _planned_flash_attn,
+ _planned_flash_attn_state,
_planned_main_cache_types,
_planned_scratch_cache_type,
+ _reserved_flash_attn_state,
_swa_full_from_args_or_env,
detect_reasoning_flags,
paravirtual_normalized_request,
@@ -1680,9 +1681,10 @@ except ImportError:
_kv_bytes_per_elem,
_kv_unified_from_args,
_metal_device_is_paravirtual,
- _planned_flash_attn,
+ _planned_flash_attn_state,
_planned_main_cache_types,
_planned_scratch_cache_type,
+ _reserved_flash_attn_state,
_swa_full_from_args_or_env,
detect_reasoning_flags,
paravirtual_normalized_request,
@@ -10733,6 +10735,10 @@ def _gguf_runtime_bytes(
ctx_last_wins: bool = False,
model_identifier: Optional[str] = None,
launch_required_ubatch: int = 0,
+ # False prices the plan the launch runs, which is what the panel shows. True also
+ # keeps load_model's reserve for a flash-attention-off respawn that the fitter cannot
+ # re-place, which is what a guard admitting a load beside training has to hold.
+ reserve_no_flash_respawn: bool = False,
) -> _GgufRuntimeBytes:
"""KV-cache and compute-buffer VRAM (bytes) at the larger of max_seq_length and
any `--ctx-size`/`-c` override, over n_parallel slots at the effective
@@ -10829,7 +10835,6 @@ def _gguf_runtime_bytes(
# pads V to f16 and charges the whole quantized saving back. Measured on
# llama-server b10632, Qwen3-0.6B at 32k with -ctk/-ctv q4_0: 2296 MiB reserved
# against 1008 MiB allocated, identical on the CPU and Vulkan builds.
- planned_v_type = planned_cache_types[1]
# the loader raises --batch-size to max(slots, 2) before launch, and llama.cpp
# caps the micro-batch against it, so budget from the emitted value. Diffusion
# takes neither flag, and SWA metadata prices the KV against the micro-batch,
@@ -10901,8 +10906,8 @@ def _gguf_runtime_bytes(
# so the default here is ON, not off. The false arm pads variable-width V
# tensors to the model-wide maximum (_max_kv_value_width), which on an
# architecture whose global and SWA layers disagree about n_embd_v_gqa
- # inflates the whole cache and can call a load that fits an overflow.
- # Resolved the way the launch resolves it, narrowed by the capability probe.
+ # inflates the whole cache and can call a load that fits an overflow. Resolved
+ # the way the launch resolves it, Grok's forced-off exception first.
_fa_supported = True
try:
_fa_caps = LlamaCppBackend.probe_server_capabilities()
@@ -10913,12 +10918,30 @@ def _gguf_runtime_bytes(
_fa_supported = False
except Exception as _fa_exc:
logger.debug("flash-attention capability probe failed: %s", _fa_exc)
- flash_attn = _planned_flash_attn(
+ # Shared with load_model so the estimate and the launch cannot answer differently
+ # (#9697, #10489).
+ flash_attn = _planned_flash_attn_state(
llama_extra_args,
- _fa_supported,
- planned_v_type,
- getattr(probe, "_architecture", None),
- )
+ planned_cache_types = planned_cache_types,
+ supports_flash_attn = _fa_supported,
+ tensor_parallel = bool(tensor_parallel),
+ architecture = getattr(probe, "_architecture", None),
+ env = os.environ,
+ )
+ if reserve_no_flash_respawn:
+ # load_model holds the reading down where a flash-attention-off respawn cannot
+ # be re-placed, and the training guard is admitting a load against the worst
+ # this placement can reach rather than what it opens with. Resolving only the
+ # plan admitted it against the smaller cache that the same placement then
+ # reserves the larger one for, so the respawn could take VRAM the guard never
+ # admitted and take a training run with it. The panel keeps the plan: its total
+ # is what the launch uses, and is placement-independent.
+ flash_attn = _reserved_flash_attn_state(
+ flash_attn,
+ llama_extra_args,
+ tensor_parallel = bool(tensor_parallel),
+ env = os.environ,
+ )
kv = probe._estimate_kv_cache_bytes(
ctx,
cache_type_for_budget,
@@ -11055,6 +11078,7 @@ def _estimate_gguf_kv_gb(
is_diffusion = is_diffusion,
model_identifier = model_identifier,
launch_required_ubatch = launch_required_ubatch,
+ reserve_no_flash_respawn = True,
)
gpu_kv_bytes = (
runtime.kv_bytes
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index fb5190b2430..6862fe0683a 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -2177,6 +2177,18 @@ async def _require_model_access_or_caller_token(
await asyncio.to_thread(account_access.authorize_download, model_name, "model", hf_token)
+def _tensor_split_can_launch(tensor_parallel, flash_attn) -> bool:
+ """Whether a load asking for a tensor split can actually take one.
+
+ llama.cpp returns nullptr for "SPLIT_MODE_TENSOR requires flash_attn to be enabled", so
+ such a load falls back to a layer split. Only a RESOLVED False refuses; None is "not
+ resolved", not evidence the child runs without flash attention.
+ """
+ if not tensor_parallel:
+ return False
+ return flash_attn is not False
+
+
@router.get("/config/{model_name:path}")
async def get_model_config(
model_name: str,
@@ -3752,6 +3764,38 @@ async def get_kv_cache_estimate(
False,
description = "Load a vision GGUF without its mmproj, freeing the projector's VRAM",
),
+ flash_attn: Optional[bool] = Query(
+ None,
+ description = (
+ "Flash attention state to price. Omit to resolve it the way the launch does "
+ "(the build's capability, LLAMA_ARG_FLASH_ATTN, a last-wins -fa in the extra "
+ "arguments, and a quantized V cache which forces it on). It is not a detail: "
+ "with flash attention off llama.cpp floors the V axis at f16 and pads "
+ "variable-width V tensors, which is a 1.44x KV cache at q8_0 and 2.28x at q4_0"
+ ),
+ ),
+ kv_unified: Optional[bool] = Query(
+ None,
+ description = (
+ "--kv-unified state to price; omit to resolve it as the launch does. Changes "
+ "the sliding-window allowance on an SWA model, which is per slot when unified"
+ ),
+ ),
+ swa_full: Optional[bool] = Query(
+ None,
+ description = (
+ "--swa-full state to price; omit to resolve it as the launch does. Collapses "
+ "an SWA model's two cache sizes into one full-context cache"
+ ),
+ ),
+ no_mmproj_offload: Optional[bool] = Query(
+ None,
+ description = (
+ "--no-mmproj-offload state to price; omit to resolve it as the launch does. "
+ "A projector on the host is not VRAM, so pricing the wrong side moves the "
+ "total by the whole mmproj"
+ ),
+ ),
request: Request = None, # type: ignore[assignment]
current_subject: str = Depends(get_current_subject),
):
@@ -3888,6 +3932,65 @@ async def get_kv_cache_estimate(
except Exception as e:
logger.debug(f"cache type resolution failed for '{repo_id}': {e}")
+ # Taking the estimator's defaults while the loader resolved the same knobs
+ # differently is why one model and cache type reported two KV caches (#10489).
+ # An asked-for value is spelled as the extra argument a load would carry and
+ # re-resolved by the launch's own helpers, never taken verbatim.
+ # isinstance(..., bool): called in process, an omitted argument arrives as the
+ # ``Query`` default object, which is truthy.
+ _asked_flash_attn = flash_attn if isinstance(flash_attn, bool) else None
+ _asked_kv_unified = kv_unified if isinstance(kv_unified, bool) else None
+ _asked_swa_full = swa_full if isinstance(swa_full, bool) else None
+ _asked_no_mmproj = no_mmproj_offload if isinstance(no_mmproj_offload, bool) else None
+ _plan_extra_args: list[str] = []
+ if _asked_flash_attn is not None:
+ _plan_extra_args += ["--flash-attn", "on" if _asked_flash_attn else "off"]
+ if _asked_kv_unified is not None:
+ _plan_extra_args += ["--kv-unified" if _asked_kv_unified else "--no-kv-unified"]
+ if _asked_swa_full:
+ # Enable-only: llama.cpp has no --no-swa-full, so a false leaves the env to
+ # answer.
+ _plan_extra_args += ["--swa-full"]
+ if _asked_no_mmproj is not None:
+ _plan_extra_args += [
+ "--no-mmproj-offload" if _asked_no_mmproj else "--mmproj-offload"
+ ]
+ _planner_extras = _plan_extra_args or None
+
+ _plan_kwargs: dict = {}
+ try:
+ from core.inference.llama_cpp import (
+ _kv_unified_from_args,
+ _planned_flash_attn_state,
+ _planned_main_cache_types as _plan_cache_types,
+ _swa_full_from_args_or_env,
+ )
+
+ _plan_caps = {}
+ try:
+ _plan_caps = LlamaCppBackend.probe_server_capabilities() or {}
+ except Exception as e:
+ logger.debug(f"capability probe failed for '{repo_id}': {e}")
+ _plan_kwargs = {
+ "flash_attn": _planned_flash_attn_state(
+ _planner_extras,
+ planned_cache_types = _plan_cache_types(cache_type_kv, _planner_extras),
+ # An unreadable probe keeps the managed default.
+ supports_flash_attn = bool(_plan_caps.get("supports_flash_attn", True)),
+ tensor_parallel = bool(tensor_parallel),
+ architecture = getattr(be, "_architecture", None),
+ ),
+ # The loader's own default: unified only for >1 slot, only if supported.
+ "kv_unified": _kv_unified_from_args(
+ _planner_extras,
+ default = (n_parallel or 1) > 1
+ and bool(_plan_caps.get("supports_kv_unified", False)),
+ ),
+ "swa_full": _swa_full_from_args_or_env(_planner_extras),
+ }
+ except Exception as e:
+ logger.debug(f"attention plan resolution failed for '{repo_id}': {e}")
+
# Probe failures keep the unflagged default rather than assuming zero.
_cc_caps: dict = {}
_total_ram_mib: Optional[int] = None
@@ -3914,6 +4017,7 @@ async def get_kv_cache_estimate(
n_parallel = n_parallel,
ctx_checkpoints = _effective_checkpoints,
n_ubatch = n_ubatch,
+ **_plan_kwargs,
)
# Report the host-resident checkpoint share separately from GPU cache bytes.
@@ -3925,6 +4029,7 @@ async def get_kv_cache_estimate(
n_parallel = n_parallel,
ctx_checkpoints = 0,
n_ubatch = n_ubatch,
+ **_plan_kwargs,
)
kv_checkpoint = max(0, int(kv) - int(_kv_without))
@@ -4000,6 +4105,18 @@ async def get_kv_cache_estimate(
projector = int(_Be._get_gguf_size_bytes(mmproj) * _Be._MMPROJ_VRAM_SAFETY)
except Exception as e:
logger.debug(f"mmproj estimate failed for '{repo_id}' {quant}: {e}")
+ # The RESOLVED placement, not the query value: the env alone can put the
+ # projector on the host, and the frontend adds projectorBytes onto its GPU
+ # weights segment, so the bar was charged for memory that never reaches the card.
+ try:
+ from core.inference.llama_cpp import _resolved_mmproj_offload
+ _mmproj_offloaded = _resolved_mmproj_offload(_planner_extras)
+ except Exception as e: # noqa: BLE001 -- cannot resolve -> the asked value stands
+ logger.debug(f"could not resolve the mmproj placement: {e}")
+ _mmproj_offloaded = None if _asked_no_mmproj is None else not _asked_no_mmproj
+ if _mmproj_offloaded is False:
+ # The projector is in HOST memory, with vision still on.
+ projector = None
# Only the MTP modes reserve memory; ngram is free. "auto" may or may not resolve to MTP, and the estimator
# returns None when it does not. Guarded separately: the MTP path reads more metadata than the KV path, and
@@ -4091,6 +4208,8 @@ async def get_kv_cache_estimate(
# 16. Blank is not zero: _build_speculative_flags emits its own default when the field is unset (2 with a
# GPU, 3 without) and the rollback state is multiplied by it. An explicit 0 is still honoured.
spec_draft_n_max = _effective_draft_n_max,
+ # Same estimator, so it must get the same resolved plan.
+ **_plan_kwargs,
)
# Plus the draft decode graph's floor, which the helper leaves to
# the loader's soft overhead.
@@ -4131,6 +4250,18 @@ async def get_kv_cache_estimate(
_effective_tp = _effective_tensor_parallel(None, bool(tensor_parallel))
except Exception as e:
logger.debug(f"tensor mode resolution failed for '{repo_id}': {e}")
+ if _effective_tp and not _tensor_split_can_launch(
+ _effective_tp, _plan_kwargs.get("flash_attn")
+ ):
+ logger.debug(
+ f"'{repo_id}': flash attention is off, so the launch cannot take a tensor "
+ "split; pricing the layer split it would fall back to"
+ )
+ _effective_tp = False
+ # Into the extras as well, not only the boolean: the breakdown re-resolves
+ # the split through a helper that reads an inherited
+ # LLAMA_ARG_SPLIT_MODE=tensor, which would turn a bare False back on.
+ _planner_extras = list(_planner_extras or []) + ["--split-mode", "layer"]
_planner_devices = 1
if _effective_tp:
from routes.inference import (
@@ -4143,6 +4274,9 @@ async def get_kv_cache_estimate(
None, _cached_inference_devices(), tensor_parallel = True
),
)
+ # The planner resolves its plan from extra arguments, which is why the plan
+ # was built in that vocabulary: without it, gpu_bytes and kv_bytes in ONE
+ # response describe two different loads.
_cfg = _cached_estimate_config(repo_id, quant, None, False)
if _cfg is not None and _cfg is not _ESTIMATE_NOT_ON_DISK:
_cfg = _localized_estimate_config(_cfg, path)
@@ -4159,8 +4293,11 @@ async def get_kv_cache_estimate(
spec_draft_cache_type = spec_draft_cache_type,
n_batch = n_batch,
n_ubatch = n_ubatch,
- tensor_parallel = tensor_parallel,
+ # The RESOLVED split: _gguf_memory_breakdown re-resolves it, so the
+ # raw toggle turned tensor mode straight back on.
+ tensor_parallel = _effective_tp,
n_devices = _planner_devices,
+ llama_extra_args = _planner_extras,
)
if _b is not None:
# `or None` would fold a real zero into "no answer". Zero is meaningful: inherited placement such as
@@ -4187,8 +4324,10 @@ async def get_kv_cache_estimate(
spec_draft_cache_type = spec_draft_cache_type,
n_batch = n_batch,
n_ubatch = n_ubatch,
- tensor_parallel = tensor_parallel,
+ # A floor priced for an impossible placement is not a floor.
+ tensor_parallel = _effective_tp,
n_devices = _planner_devices,
+ llama_extra_args = _planner_extras,
)
if _floor is not None:
planner_floor = min(int(_floor.gpu_bytes), planner_gpu)
diff --git a/studio/backend/tests/test_estimator_flash_attn_parity.py b/studio/backend/tests/test_estimator_flash_attn_parity.py
index 0d6be183208..810315c016e 100644
--- a/studio/backend/tests/test_estimator_flash_attn_parity.py
+++ b/studio/backend/tests/test_estimator_flash_attn_parity.py
@@ -90,6 +90,33 @@ def _clear_flash_attn_parity_caches():
ri._estimate_config_cache.clear()
+_RAGGED_SWA_FIELDS = {
+ "context_length": 131072,
+ "block_count": 30,
+ "attention.head_count": 16,
+ "attention.head_count_kv": 8,
+ "embedding_length": 2816,
+ "attention.key_length": 512,
+ "attention.value_length": 512,
+ "attention.key_length_swa": 256,
+ "attention.value_length_swa": 256,
+ "attention.sliding_window": 1024,
+ "attention.sliding_window_pattern": 6,
+}
+
+
+def ragged_swa_gguf(tmp_path):
+ """Gemma-class: the sliding-window layers carry a narrower V, padded to the model-wide
+ maximum when flash attention is off, so even an f16 cache moves with the state. A
+ quantized cache cannot serve here, because a quantized V forces flash attention on."""
+ fields = {"general.architecture": "gemma3"}
+ fields.update({f"gemma3.{k}": v for k, v in _RAGGED_SWA_FIELDS.items()})
+ path = tmp_path / "ragged-swa.gguf"
+ path.write_bytes(_make_gguf_bytes("gemma3", fields))
+ return str(path)
+
+
@pytest.fixture
def qwen3_shaped_gguf(tmp_path):
# general.architecture goes in FIRST. The reader takes it before it knows which
@@ -383,3 +410,61 @@ def test_grok_is_priced_without_flash_attention(monkeypatch, qwen3_shaped_gguf):
monkeypatch.setattr(ri.LlamaCppBackend, "_read_gguf_metadata", read_as_grok)
assert _runtime(qwen3_shaped_gguf) == off
+
+
+def test_the_admission_estimate_keeps_the_no_flash_reserve(monkeypatch, ragged_swa_gguf):
+ """``_estimate_gguf_kv_gb`` feeds the active-training admission guard, and ``load_model``
+ holds the reading down through ``_reserved_flash_attn_state`` where a flash-attention-off
+ respawn cannot be re-placed. Resolving only the plan here admitted a load against the
+ smaller cache that the same placement then reserves the larger one for, so the respawn
+ could take VRAM the guard never admitted and take a training run with it.
+
+ A ragged-SWA shape with an f16 cache: a quantized V forces flash attention on, and an
+ equal-width f16 cache does not move with the state, so on either of those the control
+ below could not fail."""
+ monkeypatch.delenv("LLAMA_ARG_FLASH_ATTN", raising = False)
+ monkeypatch.delenv("LLAMA_ARG_FIT", raising = False)
+ monkeypatch.delenv("LLAMA_ARG_N_GPU_LAYERS", raising = False)
+ _flash_attn_caps(monkeypatch, True)
+
+ def kv(extras = None, tensor_parallel = False):
+ return ri._gguf_runtime_bytes(
+ ragged_swa_gguf,
+ 32768,
+ extras,
+ 1,
+ "f16",
+ tensor_parallel,
+ reserve_no_flash_respawn = True,
+ ).kv_bytes
+
+ def panel(extras = None):
+ """What the panel shows: the plan, not the reserve."""
+ return ri._gguf_runtime_bytes(ragged_swa_gguf, 32768, extras, 1, "f16", False).kv_bytes
+
+ managed = kv()
+ priced_off = kv(["--flash-attn", "off"])
+ # The control: the two readings really do differ on this shape, or nothing below holds.
+ assert priced_off > managed
+
+ # A user's own --fit off, and the environment spelling of it: both take away the
+ # re-placement, so both keep the conservative reserve.
+ assert kv(["--fit", "off"]) == priced_off
+ monkeypatch.setenv("LLAMA_ARG_FIT", "off")
+ assert kv() == priced_off
+ monkeypatch.delenv("LLAMA_ARG_FIT")
+
+ # A fixed layer count is the other fitter-proof placement.
+ assert kv(["-ngl", "99"]) == priced_off
+
+ # An ordinary managed placement keeps the launch price: the fitter re-places the
+ # respawn, so reserving for it would refuse contexts that fit.
+ assert kv() == managed
+
+ # Tensor mode is exempt, because llama.cpp has no flash-attention-off respawn there.
+ assert kv(["--fit", "off"], tensor_parallel = True) == managed
+
+ # And the panel is untouched: its total is what the launch uses, and an existing pin
+ # holds it placement-independent (test_memory_estimate's extras -ngl case).
+ assert panel(["--fit", "off"]) == panel()
+ assert panel(["-ngl", "99"]) == panel()
diff --git a/studio/backend/tests/test_grok_flash_attn_exception.py b/studio/backend/tests/test_grok_flash_attn_exception.py
new file mode 100644
index 00000000000..bf126502170
--- /dev/null
+++ b/studio/backend/tests/test_grok_flash_attn_exception.py
@@ -0,0 +1,282 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Grok is priced without flash attention, whatever else the launch resolves.
+
+``llama_init_from_model`` forces it off for LLM_ARCH_GROK above both later upgrades, so
+answering "on" would publish a context the child cannot hold. The rule is easy to lose in a
+merge (#11043 against #9697, #10489), so these tests hold both ends: the rule itself, and
+the architecture reaching the resolver at every sizing seam.
+"""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+import inspect
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+_TESTS_DIR = Path(__file__).resolve().parent
+_BACKEND_DIR = str(_TESTS_DIR.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+if str(_TESTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_TESTS_DIR))
+
+
+def _load(module_name: str, file_name: str):
+ spec = importlib.util.spec_from_file_location(module_name, _TESTS_DIR / file_name)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+_placement = _load("_placement_for_grok_flash_attn", "test_llama_cpp_placement.py")
+_plan = _load("_plan_for_grok_flash_attn", "test_resolved_flash_attn_plan.py")
+
+from core.inference.llama_cpp import ( # noqa: E402
+ LlamaCppBackend,
+ _architecture_forces_flash_attn_off,
+ _planned_flash_attn_state,
+ _reserved_flash_attn_state,
+)
+
+NATIVE = _plan.NATIVE
+
+
+class TestTheRule:
+ def test_grok_is_forced_off(self):
+ assert _planned_flash_attn_state(architecture = "grok") is False
+
+ def test_the_architecture_is_read_as_the_gguf_stores_it(self):
+ assert _architecture_forces_flash_attn_off("grok") is True
+ assert _architecture_forces_flash_attn_off("Grok") is True
+ assert _architecture_forces_flash_attn_off(" grok ") is True
+
+ @pytest.mark.parametrize("architecture", [None, "", "qwen3", "llama", "grok2", "deepseek2"])
+ def test_every_other_architecture_keeps_the_managed_default(self, architecture):
+ """One architecture, not a new default: an unread header keeps the managed on."""
+ assert _architecture_forces_flash_attn_off(architecture) is False
+ assert _planned_flash_attn_state(architecture = architecture) is True
+
+
+class TestNothingPutsItBack:
+ @pytest.mark.parametrize("v_type", ["q8_0", "q4_0", "q5_1", "iq4_nl"])
+ def test_a_quantized_v_cache_does_not(self, v_type):
+ """The quantized-V upgrade sits below the Grok branch, so it never runs on Grok."""
+ assert (
+ _planned_flash_attn_state(planned_cache_types = ("f16", v_type), architecture = "grok")
+ is False
+ )
+ # The same pair elsewhere still forces it on: not asserting a constant.
+ assert (
+ _planned_flash_attn_state(planned_cache_types = ("f16", v_type), architecture = "qwen3")
+ is True
+ )
+
+ def test_a_tensor_split_does_not(self):
+ assert (
+ _planned_flash_attn_state(["-fa", "auto"], tensor_parallel = True, architecture = "grok")
+ is False
+ )
+ assert (
+ _planned_flash_attn_state(["-fa", "auto"], tensor_parallel = True, architecture = "qwen3")
+ is True
+ )
+
+ def test_an_explicit_on_in_the_extras_does_not(self):
+ assert _planned_flash_attn_state(["--flash-attn", "on"], architecture = "grok") is False
+ assert _planned_flash_attn_state(["-fa", "1"], architecture = "grok") is False
+
+ def test_the_inherited_environment_does_not(self):
+ assert (
+ _planned_flash_attn_state(architecture = "grok", env = {"LLAMA_ARG_FLASH_ATTN": "1"})
+ is False
+ )
+
+ def test_the_reserve_wrapper_keeps_it_off(self):
+ """The wrapper only holds a reading DOWN, including in tensor mode, the one case
+ it returns ``planned`` unread."""
+ for tensor_parallel in (False, True):
+ planned = _planned_flash_attn_state(
+ ["-fa", "on"],
+ planned_cache_types = ("q8_0", "q8_0"),
+ tensor_parallel = tensor_parallel,
+ architecture = "grok",
+ )
+ assert (
+ _reserved_flash_attn_state(planned, ["-fa", "on"], tensor_parallel = tensor_parallel)
+ is False
+ )
+
+
+def _grok_backend(tmp_path, architecture: str):
+ backend, gguf = _plan._tensor_backend(tmp_path, free_mib = 12000)
+ backend._architecture = architecture
+ return backend, gguf
+
+
+def _states_seen(tmp_path, architecture: str, method: str, **load_kwargs):
+ backend, gguf = _grok_backend(tmp_path, architecture)
+ seen: list[object] = []
+ real = getattr(backend, method)
+
+ def spy(*args, **kwargs):
+ seen.append(kwargs.get("flash_attn"))
+ return real(*args, **kwargs)
+
+ setattr(backend, method, spy)
+ _placement._launch(backend, gguf, **load_kwargs)
+ assert seen, f"the load took no {method} reading"
+ return seen
+
+
+class TestTheLoadPricesGrokWithoutIt:
+ """The load end to end. A quantized V under a tensor split is the hardest case: both
+ upgrades ask at once, and on Grok neither gets it."""
+
+ def test_the_kv_cache_is_priced_without_it(self, tmp_path):
+ seen = _states_seen(
+ tmp_path,
+ "grok",
+ "_estimate_kv_cache_bytes",
+ n_ctx = NATIVE,
+ cache_type_kv = "q8_0",
+ tensor_parallel = True,
+ )
+ assert all(state is False for state in seen), (
+ f"the load priced its KV cache with flash_attn states "
+ f"{sorted(set(map(str, seen)))} on an architecture llama.cpp will not run "
+ f"with flash attention"
+ )
+
+ def test_the_same_load_on_another_architecture_is_priced_with_it(self, tmp_path):
+ """The control arm: without it the test above passes on an always-False resolver."""
+ seen = _states_seen(
+ tmp_path,
+ "qwen3",
+ "_estimate_kv_cache_bytes",
+ n_ctx = NATIVE,
+ cache_type_kv = "q8_0",
+ tensor_parallel = True,
+ )
+ assert all(state is True for state in seen)
+
+ def test_the_compute_buffer_is_priced_without_it(self, tmp_path):
+ """The KQ mask is f16 with flash attention and f32 without, so the compute reserve
+ moves with the state the KV cache does."""
+ seen = _states_seen(
+ tmp_path,
+ "grok",
+ "_compute_buffer_ctx_bytes",
+ n_ctx = NATIVE,
+ cache_type_kv = "q8_0",
+ tensor_parallel = True,
+ )
+ assert all(state is False for state in seen), (
+ f"the load priced its compute buffers with flash_attn states "
+ f"{sorted(set(map(str, seen)))} on Grok"
+ )
+
+ def test_the_compute_buffer_control_arm_is_priced_with_it(self, tmp_path):
+ seen = _states_seen(
+ tmp_path,
+ "qwen3",
+ "_compute_buffer_ctx_bytes",
+ n_ctx = NATIVE,
+ cache_type_kv = "q8_0",
+ tensor_parallel = True,
+ )
+ assert all(state is True for state in seen)
+
+ def test_the_padded_cache_actually_costs_more(self, tmp_path):
+ """Reachability: the two states must price differently, or the above assert nothing."""
+ backend, _ = _grok_backend(tmp_path, "grok")
+ on = backend._estimate_kv_cache_bytes(NATIVE, "q8_0", flash_attn = True)
+ off = backend._estimate_kv_cache_bytes(NATIVE, "q8_0", flash_attn = False)
+ assert off > on
+ assert backend._compute_buffer_ctx_bytes(
+ NATIVE, 512, "q8_0", flash_attn = False
+ ) > backend._compute_buffer_ctx_bytes(NATIVE, 512, "q8_0", flash_attn = True)
+
+
+def _calls_to(source: str, name: str) -> list[ast.Call]:
+ tree = ast.parse(textwrap.dedent(source))
+ return [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and (
+ (isinstance(node.func, ast.Name) and node.func.id == name)
+ or (isinstance(node.func, ast.Attribute) and node.func.attr == name)
+ )
+ ]
+
+
+class TestEverySeamHandsTheArchitectureOver:
+ """Read by AST, not text: a comment naming the architecture would pass a string match
+ and a reflow would fail one, and neither says what the call passes."""
+
+ def test_the_loader_does(self):
+ source = inspect.getsource(inspect.unwrap(LlamaCppBackend.load_model))
+ calls = _calls_to(source, "_planned_flash_attn_state")
+ assert calls, "the launch path no longer resolves an attention plan"
+ for call in calls:
+ passed = {kw.arg for kw in call.keywords if kw.arg}
+ assert "architecture" in passed, (
+ "load_model resolves its attention plan without telling the resolver "
+ "which architecture it is, so the Grok exception cannot fire"
+ )
+
+ def test_the_memory_estimate_route_does(self):
+ import routes.inference as ri
+
+ source = inspect.getsource(ri._gguf_runtime_bytes)
+ calls = _calls_to(source, "_planned_flash_attn_state")
+ assert calls, "the estimate no longer resolves an attention plan"
+ for call in calls:
+ assert "architecture" in {kw.arg for kw in call.keywords if kw.arg}
+
+ def test_the_kv_cache_estimate_route_does(self):
+ """Read from the file: importing the route drags the whole app in."""
+ source = (Path(_BACKEND_DIR) / "routes" / "models.py").read_text(encoding = "utf-8")
+ calls = _calls_to(source, "_planned_flash_attn_state")
+ assert calls, "the kv-cache-estimate route no longer resolves an attention plan"
+ for call in calls:
+ assert "architecture" in {kw.arg for kw in call.keywords if kw.arg}
+
+
+class TestOneStateNotTwo:
+ """The rule can also be lost by RE-DERIVING the state beside the resolved one, which is
+ how the compute buffers came to be priced from their own reading."""
+
+ def test_both_sizing_seams_price_the_one_resolved_state(self):
+ source = inspect.getsource(inspect.unwrap(LlamaCppBackend.load_model))
+ for method in ("_estimate_kv_cache_bytes", "_compute_buffer_ctx_bytes"):
+ calls = _calls_to(source, method)
+ assert calls, f"the launch path no longer calls {method}"
+ for call in calls:
+ passed = {
+ kw.arg: kw.value
+ for kw in call.keywords
+ if kw.arg and isinstance(kw.value, ast.Name)
+ }
+ assert getattr(passed.get("flash_attn"), "id", None) == "planned_flash_attn", (
+ f"{method} is priced from something other than the resolved "
+ f"planned_flash_attn, so the two can disagree about the same load"
+ )
+
+ def test_there_is_only_one_resolver(self):
+ """A duplicate carrying only some of the rules is worse than none."""
+ import core.inference.llama_cpp as llama_cpp
+
+ resolvers = sorted(
+ name
+ for name in dir(llama_cpp)
+ if name.startswith("_planned_flash_attn") or name.startswith("_reserved_flash_attn")
+ )
+ assert resolvers == ["_planned_flash_attn_state", "_reserved_flash_attn_state"]
diff --git a/studio/backend/tests/test_kv_cache_estimate_attention_plan.py b/studio/backend/tests/test_kv_cache_estimate_attention_plan.py
new file mode 100644
index 00000000000..e1a91d7e250
--- /dev/null
+++ b/studio/backend/tests/test_kv_cache_estimate_attention_plan.py
@@ -0,0 +1,219 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""/api/models/kv-cache-estimate answers for a named attention plan, not for the defaults.
+
+The route took the estimator's signature defaults while the loader resolved the same three
+knobs differently, so one model at one cache type put three numbers on screen (#10489).
+The arithmetic itself is covered by tests/test_kv_cache_estimation.py.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+from pathlib import Path
+
+import pytest
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from test_kv_cache_estimation import _make_gguf_bytes # noqa: E402
+
+import routes.models as models_routes # noqa: E402
+from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
+
+N_CTX = 32768
+
+# Equal K/V widths, so the only flash-attention term that can move the total is the f16
+# floor on V.
+_PLAIN_GQA = {
+ "context_length": 131072,
+ "block_count": 32,
+ "attention.head_count": 32,
+ "attention.head_count_kv": 8,
+ "embedding_length": 4096,
+ "attention.key_length": 128,
+ "attention.value_length": 128,
+}
+
+# Gemma-class: narrower V on the sliding-window layers, padded to the model-wide maximum
+# when flash attention is off, so even an f16 cache moves.
+_RAGGED_SWA = {
+ "context_length": 131072,
+ "block_count": 30,
+ "attention.head_count": 16,
+ "attention.head_count_kv": 8,
+ "embedding_length": 2816,
+ "attention.key_length": 512,
+ "attention.value_length": 512,
+ "attention.key_length_swa": 256,
+ "attention.value_length_swa": 256,
+ "attention.sliding_window": 1024,
+ "attention.sliding_window_pattern": 6,
+}
+
+
+def _write_gguf(
+ path: Path,
+ fields: dict,
+ arch: str = "testarch",
+) -> Path:
+ kv = {"general.architecture": arch}
+ for key, value in fields.items():
+ kv[f"{arch}.{key}"] = value
+ path.parent.mkdir(parents = True, exist_ok = True)
+ path.write_bytes(_make_gguf_bytes(arch, kv))
+ return path
+
+
+def _call_route(
+ monkeypatch,
+ *,
+ path: Path,
+ caps: dict | None = None,
+ **overrides,
+):
+ """All passed explicitly: called in process an omitted one arrives as its ``Query``
+ object rather than the None FastAPI would have resolved."""
+ monkeypatch.setattr(
+ models_routes,
+ "_resolve_quant_gguf",
+ lambda _repo, _quant, _local: (str(path), 4096),
+ )
+ monkeypatch.setattr(models_routes, "is_local_path", lambda _p: False, raising = False)
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "probe_server_capabilities",
+ classmethod(lambda cls, *a, **k: dict(caps if caps is not None else {"mtp_token": True})),
+ )
+ kwargs = dict(
+ repo_id = "org/repo",
+ quant = "Q4_K_M",
+ n_ctx = N_CTX,
+ cache_type_kv = None,
+ n_parallel = 1,
+ speculative_type = None,
+ spec_draft_n_max = None,
+ spec_draft_cache_type = None,
+ ctx_checkpoints = None,
+ disable_vision = False,
+ n_batch = None,
+ n_ubatch = None,
+ tensor_parallel = False,
+ flash_attn = None,
+ kv_unified = None,
+ swa_full = None,
+ no_mmproj_offload = None,
+ request = None,
+ current_subject = "test",
+ )
+ kwargs.update(overrides)
+ return asyncio.run(models_routes.get_kv_cache_estimate(**kwargs))
+
+
+def gqa(tmp_path):
+ return _write_gguf(tmp_path / "gqa-Q4_K_M.gguf", _PLAIN_GQA)
+
+
+def ragged(tmp_path):
+ return _write_gguf(tmp_path / "ragged-Q4_K_M.gguf", _RAGGED_SWA)
+
+
+class TestFlashAttention:
+ def test_an_explicit_off_prices_the_padded_cache(self, monkeypatch, ragged):
+ on = _call_route(monkeypatch, path = ragged, flash_attn = True)
+ off = _call_route(monkeypatch, path = ragged, flash_attn = False)
+ assert off["kv_bytes"] > on["kv_bytes"]
+
+ def test_omitting_it_resolves_to_what_the_launch_emits(self, monkeypatch, ragged):
+ default = _call_route(monkeypatch, path = ragged)
+ assert (
+ default["kv_bytes"]
+ == _call_route(monkeypatch, path = ragged, flash_attn = True)["kv_bytes"]
+ )
+
+ def test_a_build_without_the_flag_prices_the_padded_cache(self, monkeypatch, ragged):
+ blind = _call_route(monkeypatch, path = ragged)
+ unsupported = _call_route(
+ monkeypatch,
+ path = ragged,
+ caps = {"found": True, "supports_flash_attn": False},
+ )
+ assert unsupported["kv_bytes"] > blind["kv_bytes"]
+
+ def test_a_quantized_v_cache_is_not_priced_without_it(self, monkeypatch, gqa):
+ """An explicit off is not a launch that can happen: llama.cpp forces it on."""
+ forced = _call_route(monkeypatch, path = gqa, cache_type_kv = "q8_0", flash_attn = False)
+ resolved = _call_route(monkeypatch, path = gqa, cache_type_kv = "q8_0")
+ assert forced["kv_bytes"] == resolved["kv_bytes"]
+
+ def test_the_quantized_v_answer_is_the_narrow_one(self, monkeypatch, gqa):
+ """2.125 bytes per element, not the 3.0625 the loader used to contradict it with."""
+ q8 = _call_route(monkeypatch, path = gqa, cache_type_kv = "q8_0")
+ f16 = _call_route(monkeypatch, path = gqa, cache_type_kv = "f16")
+ assert q8["kv_bytes"] == pytest.approx(f16["kv_bytes"] * (2.125 / 4.0), rel = 1e-6)
+
+
+class TestTheOtherTwoLayoutKnobs:
+ def test_swa_full_collapses_the_two_cache_sizes(self, monkeypatch, ragged):
+ # Net of checkpoints: --swa-full zeroes the checkpoint share, so the totals can move
+ # the other way while the attention cache itself still grows.
+ compact = _call_route(monkeypatch, path = ragged)
+ full = _call_route(monkeypatch, path = ragged, swa_full = True)
+ # The share is None, not 0, when nothing is reserved.
+ compact_attn = compact["kv_bytes"] - (compact["kv_checkpoint_bytes"] or 0)
+ full_attn = full["kv_bytes"] - (full["kv_checkpoint_bytes"] or 0)
+ assert full_attn > compact_attn
+
+ def test_swa_full_drops_the_checkpoint_share(self, monkeypatch, ragged):
+ """No sliding window left to snapshot, so any reported share is memory nobody
+ reserves."""
+ compact = _call_route(monkeypatch, path = ragged, ctx_checkpoints = 8)
+ full = _call_route(monkeypatch, path = ragged, ctx_checkpoints = 8, swa_full = True)
+ assert compact["kv_checkpoint_bytes"]
+ assert not full["kv_checkpoint_bytes"]
+
+ def test_the_unified_cache_is_an_input(self, monkeypatch, ragged):
+ unified = _call_route(monkeypatch, path = ragged, n_parallel = 4, kv_unified = True)
+ split = _call_route(monkeypatch, path = ragged, n_parallel = 4, kv_unified = False)
+ assert unified["kv_bytes"] != split["kv_bytes"]
+
+ def test_a_single_slot_resolves_to_the_launch_default(self, monkeypatch, ragged):
+ """Unsloth asks for a unified cache only to serve more than one slot."""
+ default = _call_route(monkeypatch, path = ragged, n_parallel = 1)
+ assert (
+ default["kv_bytes"]
+ == _call_route(monkeypatch, path = ragged, n_parallel = 1, kv_unified = False)["kv_bytes"]
+ )
+
+
+class TestTheContract:
+ def test_the_new_parameters_are_all_optional(self):
+ import inspect
+
+ signature = inspect.signature(models_routes.get_kv_cache_estimate)
+ for name in ("flash_attn", "kv_unified", "swa_full", "no_mmproj_offload"):
+ assert name in signature.parameters, f"{name} is not on the route"
+ assert signature.parameters[name].default is not inspect.Parameter.empty
+
+ def test_the_response_shape_did_not_change(self, monkeypatch, gqa):
+ """No response_model, so a strict client breaks on a new key."""
+ from test_memory_estimate_contract_freeze import _KV_CACHE_ESTIMATE_KEYS
+ assert set(_call_route(monkeypatch, path = gqa)) == set(_KV_CACHE_ESTIMATE_KEYS)
+
+ def test_an_omitted_plan_does_not_move_the_answer_for_a_plain_model(self, monkeypatch, gqa):
+ answer = _call_route(monkeypatch, path = gqa)
+ assert (
+ answer["kv_bytes"]
+ == _call_route(monkeypatch, path = gqa, flash_attn = True, kv_unified = True, swa_full = False)[
+ "kv_bytes"
+ ]
+ )
diff --git a/studio/backend/tests/test_kv_cache_estimate_route.py b/studio/backend/tests/test_kv_cache_estimate_route.py
index 578d3b27879..78be37c23df 100644
--- a/studio/backend/tests/test_kv_cache_estimate_route.py
+++ b/studio/backend/tests/test_kv_cache_estimate_route.py
@@ -3,31 +3,16 @@
"""Route-level tests for ``GET /kv-cache-estimate``.
-The estimate is only worth showing if it describes the load llama-server will
-actually perform, so these drive the real handler and the real drafter resolver
-rather than a replayed copy of their logic.
-
-Two classes of defect are covered:
-
-* The MTP reserve must follow the loader's own ``is_mtp_model`` precondition. A
- model with no embedded head, no MTP name and no separate drafter cannot run
- MTP, so the loader emits ``--spec-default`` and reserves nothing. Because
- ``_estimate_mtp_overhead_bytes`` defaults ``mtp_keeps_target_ctx=True`` (so an
- unsure caller over-reserves), skipping that precondition billed every MLA
- model a second full f16 copy of its own KV.
-
-* Drafter discovery must resolve the same file ``_download_mtp`` opens, on every
- platform. The estimate runs on hosts whose filesystems disagree about case and
- whose ``Path`` ordering differs, and it is reached for both HF snapshots and
- plain local folders.
-
-No GPU, no network. Cross-platform: the layout cases below construct real
-directories under ``tmp_path`` and assert against the loader's own picker.
+These drive the real handler and the real drafter resolver, not a replayed copy: the MTP
+reserve must follow the loader's own ``is_mtp_model`` precondition, since
+``_estimate_mtp_overhead_bytes`` over-reserves for an unsure caller, and drafter discovery
+must resolve the same file ``_download_mtp`` opens on hosts that disagree about case.
"""
from __future__ import annotations
import asyncio
+import importlib.machinery
import sys
from pathlib import Path
@@ -42,6 +27,7 @@ if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
# Installs the process-wide loggers/structlog/httpx stubs and the GGUF builder.
+import test_kv_cache_estimation # noqa: E402
from test_kv_cache_estimation import _backend_from_gguf, _make_gguf_bytes # noqa: E402
import routes.models as models_routes # noqa: E402
@@ -120,6 +106,11 @@ def _call_route(
ctx_checkpoints: int | None = None,
disable_vision: bool = False,
n_ctx: int | None = 32768,
+ flash_attn: bool | None = None,
+ kv_unified: bool | None = None,
+ swa_full: bool | None = None,
+ no_mmproj_offload: bool | None = None,
+ tensor_parallel: bool = False,
):
"""Drive the real handler with the quant already resolved to *path*."""
monkeypatch.setattr(
@@ -149,7 +140,13 @@ def _call_route(
disable_vision = disable_vision,
n_batch = None,
n_ubatch = None,
- tensor_parallel = False,
+ tensor_parallel = tensor_parallel,
+ # Real None, not left out: a direct call leaves omitted parameters holding
+ # their fastapi Query sentinel.
+ flash_attn = flash_attn,
+ kv_unified = kv_unified,
+ swa_full = swa_full,
+ no_mmproj_offload = no_mmproj_offload,
request = None,
current_subject = "test",
)
@@ -268,10 +265,8 @@ class TestDrafterDiscoveryMatchesTheLoader:
assert got != str(main)
def test_nested_mtp_subdir_copy_is_used_when_no_root_mirror(self, tmp_path):
- """A repo publishing heads only under MTP/, as Qwen3.8-Flash-Next and
- Qwen3.8-27B do, still gets a drafter. _cached_repo_mtp_drafter already
- reuses such a copy offline, so skipping it here gave a cached user
- speculation and a fresh one none."""
+ """_cached_repo_mtp_drafter reuses such a copy offline, so skipping it here gave a
+ cached user speculation and a fresh one none."""
snap = self._snapshot(tmp_path)
main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
nested = _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
@@ -300,9 +295,7 @@ class TestDrafterDiscoveryMatchesTheLoader:
assert got == str(full)
def test_precision_outranks_the_shared_preference(self, tmp_path):
- """Q8_0 first is the stronger rule: a shared bf16 head is both larger than
- a full Q8_0 one and slower, since a draft step is dominated by the LM head
- and that head is cheaper to execute at 8 bits."""
+ """Q8_0 first: a shared bf16 head is larger and slower than a full Q8_0 one."""
snap = self._snapshot(tmp_path)
main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
q8 = _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
@@ -312,9 +305,7 @@ class TestDrafterDiscoveryMatchesTheLoader:
assert got == str(q8)
def test_nested_fallback_is_skipped_for_other_architectures(self, tmp_path):
- """The fallback is qwen4exp only. Qwen3.8-27B bakes the head into 20 of
- its 24 quants, and llama.cpp prefers a -md drafter over an embedded one, so
- pricing the sidecar would reserve for a file the load will not open."""
+ """qwen4exp only: elsewhere the sidecar reserves for a file the load never opens."""
snap = self._snapshot(tmp_path)
main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen35")
_write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
@@ -323,9 +314,7 @@ class TestDrafterDiscoveryMatchesTheLoader:
assert got is None, f"billed a sidecar the load will not fetch: {got} ({size} bytes)"
def test_nested_fallback_is_skipped_for_a_qwen4exp_with_its_own_head(self, tmp_path):
- """Architecture is not the whole gate: a qwen4exp GGUF converted with the
- block kept needs no sidecar either. Nothing published does this, so this
- pins the intent."""
+ """Not the whole gate: a qwen4exp GGUF with the block kept needs no sidecar."""
snap = self._snapshot(tmp_path)
main = _write_gguf(
snap / "model-Q4_K_M.gguf",
@@ -373,9 +362,8 @@ class TestDrafterDiscoveryMatchesTheLoader:
assert got == str(companion)
def test_a_non_drafter_parked_under_mtp_is_not_launched(self, tmp_path):
- """Everything under MTP/ classifies as a drafter, which keeps companions
- out of variant menus and is too broad for choosing what to launch: an
- mmproj or an imatrix would be handed to --model-draft."""
+ """Everything under MTP/ classifies as a drafter, which is too broad for choosing
+ what to launch: an mmproj or an imatrix would be handed to --model-draft."""
snap = self._snapshot(tmp_path)
main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
_write_gguf(snap / "MTP" / "mmproj-BF16.gguf", _MLA_NO_HEAD)
@@ -396,10 +384,8 @@ class TestDrafterDiscoveryMatchesTheLoader:
assert got == str(old)
def test_an_incomplete_nested_split_does_not_shadow_a_complete_head(self, tmp_path):
- """llama.cpp resolves sibling shards from the first one's directory, so
- half a set is unusable and _download_companion_gguf answers None to it.
- Ranked first and rejected after, it would disable speculation with a usable
- lower-ranked head present."""
+ """Half a shard set is unusable, and ranked first then rejected it would disable
+ speculation with a usable lower-ranked head present."""
snap = self._snapshot(tmp_path)
main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
_write_gguf(snap / "MTP" / "mtp-model-Q8_0-00001-of-00002.gguf", _MLA_NO_HEAD)
@@ -502,11 +488,8 @@ if __name__ == "__main__":
class TestTheEstimateMatchesTheConfiguredLoad:
- """Settings a user can save must reach the estimator that prices them.
-
- Each of these is a term the load planner forwards and this route did not, so
- the bar reported a comfortable fit for a launch that reserves more.
- """
+ """Settings a user can save must reach the estimator: each is a term the load planner
+ forwards and this route did not, so the bar reported a fit for a heavier launch."""
def test_saved_context_checkpoints_are_priced(self, monkeypatch, tmp_path):
"""Each checkpoint is an SWA snapshot per slot, so this is not small."""
@@ -576,11 +559,8 @@ class TestTheEstimateMatchesTheConfiguredLoad:
assert (
deep["spec_bytes"] > zero["spec_bytes"] * 10
), "spec_draft_n_max did not reach the MTP estimator"
- # A blank field is NOT zero: _build_speculative_flags emits its own
- # default (2 with a GPU, 3 without) and the rollback state is multiplied
- # by it, so pricing an unset field as zero dropped the dominant
- # allocation on this exact model shape. An explicit zero still means
- # zero, which is what separates the two calls below.
+ # A blank field is NOT zero: _build_speculative_flags emits its own default and the
+ # rollback state is multiplied by it. An explicit zero still means zero.
assert (
none["spec_bytes"] > zero["spec_bytes"]
), "an unset draft depth was priced as zero, dropping every rollback copy"
@@ -646,6 +626,111 @@ class TestTheEstimateMatchesTheConfiguredLoad:
)
assert off["projector_bytes"] is None, "vision off must free the projector"
+ def test_a_host_pinned_projector_is_not_charged_to_the_card(self, monkeypatch, tmp_path):
+ """``--no-mmproj-offload`` keeps vision ON with the projector in HOST memory, and
+ the frontend adds ``projectorBytes`` onto its GPU weights segment regardless."""
+ gguf = _write_gguf(tmp_path / "vision-Q4_K_M.gguf", _MLA_NO_HEAD)
+ (tmp_path / "mmproj-F16.gguf").write_bytes(b"\x00" * 800_000)
+
+ offloaded = _call_std_route(
+ monkeypatch,
+ path = gguf,
+ repo_id = str(tmp_path),
+ is_local = True,
+ no_mmproj_offload = False,
+ )
+ assert offloaded["projector_bytes"], "a resident projector must still be charged"
+
+ pinned = _call_std_route(
+ monkeypatch,
+ path = gguf,
+ repo_id = str(tmp_path),
+ is_local = True,
+ no_mmproj_offload = True,
+ )
+ assert (
+ pinned["projector_bytes"] is None
+ ), "a projector pinned to host memory must not be reported as VRAM"
+ # Vision is still on: "no projector on the card" is not "no vision".
+ assert pinned["kv_bytes"] and pinned["kv_bytes"] > 0
+
+ def test_an_inherited_host_pin_is_not_charged_to_the_card_either(self, monkeypatch, tmp_path):
+ """The environment alone puts the projector in host memory, so reading only what
+ was ASKED left projectorBytes populated where nothing was asked."""
+ gguf = _write_gguf(tmp_path / "vision-Q4_K_M.gguf", _MLA_NO_HEAD)
+ (tmp_path / "mmproj-F16.gguf").write_bytes(b"\x00" * 800_000)
+
+ for variable, value in (
+ ("LLAMA_ARG_NO_MMPROJ_OFFLOAD", "1"),
+ ("LLAMA_ARG_MMPROJ_OFFLOAD", "off"),
+ ):
+ monkeypatch.delenv("LLAMA_ARG_NO_MMPROJ_OFFLOAD", raising = False)
+ monkeypatch.delenv("LLAMA_ARG_MMPROJ_OFFLOAD", raising = False)
+ monkeypatch.setenv(variable, value)
+ inherited = _call_std_route(
+ monkeypatch, path = gguf, repo_id = str(tmp_path), is_local = True
+ )
+ assert inherited["projector_bytes"] is None, variable
+ assert inherited["kv_bytes"] and inherited["kv_bytes"] > 0, variable
+
+ # An explicit flag still wins over the environment, as the child resolves them.
+ monkeypatch.setenv("LLAMA_ARG_NO_MMPROJ_OFFLOAD", "1")
+ monkeypatch.delenv("LLAMA_ARG_MMPROJ_OFFLOAD", raising = False)
+ asked_on = _call_std_route(
+ monkeypatch,
+ path = gguf,
+ repo_id = str(tmp_path),
+ is_local = True,
+ no_mmproj_offload = False,
+ )
+ assert asked_on["projector_bytes"], "an explicit --mmproj-offload must still charge"
+
+ # A clean environment and nothing asked is unchanged: the projector is resident.
+ monkeypatch.delenv("LLAMA_ARG_NO_MMPROJ_OFFLOAD", raising = False)
+ monkeypatch.delenv("LLAMA_ARG_MMPROJ_OFFLOAD", raising = False)
+ plain = _call_std_route(monkeypatch, path = gguf, repo_id = str(tmp_path), is_local = True)
+ assert plain["projector_bytes"], "an unasked, uninherited projector is on the card"
+
+ def test_flash_attention_off_cannot_take_a_tensor_split(self):
+ """llama.cpp refuses the pair, so the loader falls back to a layer split and
+ pricing the tensor placement charges buffers to a card count nothing uses."""
+ from routes.models import _tensor_split_can_launch
+
+ assert _tensor_split_can_launch(True, False) is False
+ assert _tensor_split_can_launch(True, True) is True
+ # Not resolved is not evidence that the child runs without it.
+ assert _tensor_split_can_launch(True, None) is True
+ assert _tensor_split_can_launch(False, True) is False
+ assert _tensor_split_can_launch(False, False) is False
+
+ def test_the_downgrade_reaches_the_breakdown_and_the_floor(self):
+ """The boolean alone does not reach them: the breakdown re-resolves the split
+ through a helper that reads an inherited LLAMA_ARG_SPLIT_MODE=tensor too."""
+ import inspect
+
+ from routes import models as models_module
+
+ body = inspect.getsource(models_module.get_kv_cache_estimate)
+ decide = body.index("_tensor_split_can_launch(")
+ assert '"--split-mode", "layer"' in body[decide : decide + 1200], body[
+ decide : decide + 600
+ ]
+ assert body.count("tensor_parallel = _effective_tp,") == 2, body.count(
+ "tensor_parallel = _effective_tp,"
+ )
+ assert "tensor_parallel = tensor_parallel,\n" not in body[decide:]
+
+ def test_the_estimate_prices_the_split_the_launch_can_take(self):
+ import inspect
+
+ from routes import models as models_module
+
+ body = inspect.getsource(models_module.get_kv_cache_estimate)
+ decide = body.index("_tensor_split_can_launch(")
+ devices = body.index("_planner_devices = max(")
+ assert decide < devices, (decide, devices)
+ assert "_effective_tp = False" in body[decide:devices]
+
def test_a_model_with_no_projector_reports_none(self, monkeypatch, tmp_path):
gguf = _write_gguf(tmp_path / "text-Q4_K_M.gguf", _MLA_NO_HEAD)
out = _call_std_route(monkeypatch, path = gguf, repo_id = str(tmp_path), is_local = True)
@@ -653,10 +738,8 @@ class TestTheEstimateMatchesTheConfiguredLoad:
class TestHostMemoryIsNotChargedToTheCard:
- """Context checkpoints live in host heap, so a VRAM bar must be able to
- exclude them. The load planner's own GPU figure is
- ``kv_bytes - kv_checkpoint_bytes``; without the second term reported the bar
- warns OOM over memory that never reaches the GPU."""
+ """Context checkpoints live in host heap, so a VRAM bar must be able to exclude them:
+ without ``kv_checkpoint_bytes`` it warns OOM over memory that never reaches the GPU."""
def test_checkpoints_are_reported_as_their_own_share(self, monkeypatch, tmp_path):
gguf = _write_gguf(tmp_path / "swa-model-Q4_K_M.gguf", _SWA_MODEL)
@@ -677,9 +760,7 @@ class TestHostMemoryIsNotChargedToTheCard:
# By difference against the same call with none, which is how the load
# planner derives it -- asking one function twice cannot drift from it.
assert share == with_checkpoints["kv_bytes"] - without["kv_bytes"]
- # And it is a SHARE of kv_bytes, not a figure beside it: the field
- # shipped meaning the whole cache and an existing caller still reads it
- # that way.
+ # A SHARE of kv_bytes, not a figure beside it: an existing caller reads it that way.
assert share < with_checkpoints["kv_bytes"]
def test_no_checkpoints_means_no_host_share(self, monkeypatch, tmp_path):
@@ -699,22 +780,16 @@ class TestHostMemoryIsNotChargedToTheCard:
class TestAutoAbstainsOverASidecar:
- """Auto promotes to DSpark or DFlash when the model ships one, so the route
- must say the reserve is unpriced rather than chart a total without it.
-
- These assert the OUTCOME, not that the block runs: the first version of this
- guard referenced two helpers that were only in another function's scope, and
- the resulting NameError was swallowed by the surrounding except, leaving
- spec_unpriced false with nothing in the response to show it had failed.
- """
+ """Auto promotes to DSpark or DFlash when the model ships one, so the route must say
+ the reserve is unpriced rather than chart a total without it. Asserted on the OUTCOME: an
+ earlier guard raised NameError into the surrounding except and showed nothing."""
def _call(self, monkeypatch, tmp_path, *, sidecar: str | None, supports: bool):
model_dir = tmp_path / "local-model"
gguf = _write_gguf(model_dir / "model-Q4_K_M.gguf", _MLA_NO_HEAD)
if sidecar:
- # A DFlash sidecar is identified by its header, not its name:
- # detect_dflash_file confirms general.architecture = dflash, which
- # settles the adversarial case of a real model merely CALLED DFlash.
+ # By header, not by name: detect_dflash_file confirms
+ # general.architecture = dflash, settling the model merely CALLED DFlash.
_write_gguf(
model_dir / sidecar,
_MLA_NO_HEAD,
@@ -772,24 +847,16 @@ class TestAutoAbstainsOverASidecar:
assert out["spec_unpriced"] is False
def test_a_binary_that_cannot_run_one_still_prices_normally(self, monkeypatch, tmp_path):
- # The planner gates sidecar selection on the binary's own capability, so
- # abstaining where the launch would never open one blanks the bar for
- # nothing.
+ # The planner gates sidecar selection on the binary's capability, so abstaining
+ # where the launch opens none blanks the bar for nothing.
out = self._call(monkeypatch, tmp_path, sidecar = "dspark-model-Q8_0.gguf", supports = False)
assert out["spec_unpriced"] is False
class TestThePlannerFiguresArrive:
- """The route delegates to the load planner for the terms it cannot derive
- itself, inside a try/except so a planner that cannot size a model still
- leaves the KV bar drawn.
-
- That except is the hazard: every earlier version of this delegation failed
- into it silently and returned nulls that looked like "not sizeable" rather
- than "the call is broken". These assert the figures are actually present for
- a model the planner can size, so a signature drift or an unresolved
- parameter fails the suite instead of quietly blanking the bar.
- """
+ """The route delegates to the load planner inside a try/except, which earlier versions
+ failed into silently and returned nulls reading as "not sizeable" rather than "the call
+ is broken", so these assert the figures are present for a model it can size."""
def test_the_planner_terms_are_populated(self, monkeypatch, tmp_path):
model_dir = tmp_path / "planner-model"
@@ -866,9 +933,8 @@ class TestTheInheritedEnvironmentIsPriced:
not what the request said."""
def test_an_inherited_context_beats_the_native_length(self, monkeypatch, tmp_path):
- # load_model drops an inherited context only when it is zero, so a
- # positive one is the legitimate way to set the window. Falling through
- # to native priced a 4k load at the header's length.
+ # load_model drops an inherited context only when it is zero, so a positive one sets
+ # the window and falling through to native priced a 4k load at the header's length.
gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)
monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "4096")
out = _call_std_route(monkeypatch, path = gguf, n_ctx = None)
@@ -885,9 +951,8 @@ class TestTheInheritedEnvironmentIsPriced:
assert out["n_ctx"] == _PLAIN_GQA["context_length"]
def test_an_inherited_cache_type_sizes_the_cache(self, monkeypatch, tmp_path):
- # A q8_0 cache is roughly half an f16 one. Pricing f16 while the child
- # opens q8_0 makes the KV segment and the per-token readout contradict
- # the planner total drawn beside them.
+ # Pricing f16 while the child opens q8_0 makes the KV segment and the per-token
+ # readout contradict the planner total drawn beside them.
gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)
monkeypatch.delenv("LLAMA_ARG_CACHE_TYPE_K", raising = False)
monkeypatch.delenv("LLAMA_ARG_CACHE_TYPE_V", raising = False)
@@ -901,10 +966,8 @@ class TestTheInheritedEnvironmentIsPriced:
)
def test_an_inherited_context_is_reported_as_pinned(self, monkeypatch, tmp_path):
- # The loader keeps a positive inherited context rather than fitting it,
- # so a caller must not soften its verdict for that launch. Saying
- # "auto-fitted" there suppressed the overage AND drew only the
- # irreducible floor, which is a comfortable fit for a load that OOMs.
+ # The loader keeps a positive inherited context rather than fitting it, so
+ # "auto-fitted" suppressed the overage and drew only the irreducible floor.
gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)
monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "4096")
assert _call_std_route(monkeypatch, path = gguf, n_ctx = None)["context_is_pinned"] is True
@@ -920,19 +983,37 @@ class TestTheInheritedEnvironmentIsPriced:
assert _call_std_route(monkeypatch, path = gguf, n_ctx = 8192)["context_is_pinned"] is True
def test_an_inherited_device_pin_is_reported(self, monkeypatch, tmp_path):
- # The child is confined to the cards LLAMA_ARG_DEVICE names and an
- # automatic launch preserves the pin, so an aggregate VRAM budget
- # describes a pool the launch will not open. The caller cannot see the
- # environment, so the route has to say.
+ # The child is confined to the cards LLAMA_ARG_DEVICE names, so an aggregate VRAM
+ # budget describes a pool the launch will not open.
gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)
monkeypatch.setenv("LLAMA_ARG_DEVICE", "CUDA0")
assert _call_std_route(monkeypatch, path = gguf)["inherited_device_pin"] is True
@pytest.mark.parametrize("value", ["", "none", "NONE"])
def test_no_usable_pin_is_not_reported_as_one(self, monkeypatch, tmp_path, value):
- # "none" is a CPU-only launch, which the planner already answers with
- # zero GPU bytes and which draws no bar on its own; reporting it as a pin
- # would blank the row for a second, unrelated reason.
+ # "none" is a CPU-only launch, already answered with zero GPU bytes; reporting it
+ # as a pin would blank the row for a second, unrelated reason.
gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)
monkeypatch.setenv("LLAMA_ARG_DEVICE", value)
assert _call_std_route(monkeypatch, path = gguf)["inherited_device_pin"] is False
+
+
+def test_the_shared_loggers_stub_is_still_a_package():
+ """``routes.models`` reaches ``loggers.media_progress``, and ``test_kv_cache_estimation``
+ installs a bare ``ModuleType`` stub that shadows the whole package, so without
+ ``__path__`` this module cannot be collected at all (#11028). Asserted against the stub
+ object, not the ``sys.modules`` slot, which the conftest guard usually wins.
+ """
+ stub = test_kv_cache_estimation._loggers_stub
+ search_path = list(getattr(stub, "__path__", ()) or ())
+ assert search_path, (
+ "the shared 'loggers' stub has no __path__, so it is a module and not a package: "
+ "whichever test module installs it first makes every importer of "
+ "loggers.media_progress uncollectable"
+ )
+ # __path__ existing is not enough; it has to actually resolve the submodule.
+ spec = importlib.machinery.PathFinder.find_spec("media_progress", search_path)
+ assert spec is not None, (
+ f"the stub's __path__ {search_path} does not contain media_progress; "
+ "it must point at the real studio/backend/loggers directory"
+ )
diff --git a/studio/backend/tests/test_kv_cache_estimation.py b/studio/backend/tests/test_kv_cache_estimation.py
index 871b027966e..b23b8d45dc3 100644
--- a/studio/backend/tests/test_kv_cache_estimation.py
+++ b/studio/backend/tests/test_kv_cache_estimation.py
@@ -29,6 +29,9 @@ if _BACKEND_DIR not in sys.path:
# loggers
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
+# __path__ so `loggers.media_progress` still resolves: a bare ModuleType shadows the whole
+# package, so the submodule import dies with "'loggers' is not a package" (#10995).
+_loggers_stub.__path__ = [str(Path(_BACKEND_DIR) / "loggers")]
sys.modules.setdefault("loggers", _loggers_stub)
# structlog. Carries get_logger because this stub is process-wide: whichever test
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index b1cb2ae41e5..35c39951d5b 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -3678,14 +3678,24 @@ def _recording_compute_backend(
def test_the_loader_prices_the_attention_mode_it_launches(
tmp_path, monkeypatch, extra_args, expected
):
- """Compute follows the launch's attention mode; KV covers the no-flash retry."""
+ """Compute AND KV follow the launch's attention mode, off one resolved state.
+
+ The KV cache used to be pinned to flash attention off whatever the argv said, which
+ priced a load that was not going to happen (#9697, #10489). The reserve it stood in for
+ is narrower now: ``_reserved_flash_attn_state`` only holds the reading down where a
+ no-flash respawn cannot be re-placed, which is neither of the cases here.
+ """
backend, gguf, calls = _recording_compute_backend(tmp_path, monkeypatch)
assert _launch(backend, gguf, n_ctx = 0, n_parallel = 4, extra_args = extra_args)["cmd"]
assert calls["ctx"], "the fit never priced the context term"
assert set(calls["ctx"]) == {expected}
- assert False in calls["kv"]
+ assert calls["kv"], "the fit never priced the KV cache"
+ assert set(calls["kv"]) == {expected}, (
+ f"the KV cache was priced with flash_attn {sorted(set(map(str, calls['kv'])))} "
+ f"while the compute buffers were priced {expected}: one load, two answers"
+ )
@pytest.mark.parametrize("build,expected", [(9415, True), (10909, False), (None, False)])
diff --git a/studio/backend/tests/test_load_mode_fit_matrix.py b/studio/backend/tests/test_load_mode_fit_matrix.py
index 337f1acbbb7..3c89443f5a8 100644
--- a/studio/backend/tests/test_load_mode_fit_matrix.py
+++ b/studio/backend/tests/test_load_mode_fit_matrix.py
@@ -2139,10 +2139,12 @@ def test_the_no_flash_rung_recomputes_the_memory_record():
import inspect
src = inspect.getsource(B.load_model)
- helper = src[src.index("_drop_fit_load_mode_for_no_flash") :]
+ # Anchored on the DEFINITION: the bare name also appears in comments earlier in
+ # load_model, so slicing from the first mention sliced a comment.
+ helper = src[src.index("def _drop_fit_load_mode_for_no_flash(") :]
# Bounded at the next definition, so this proves the recompute is in the
# helper and not merely somewhere later in load_model.
- helper = helper[: helper.index("_spawn_and_wait")]
+ helper = helper[: helper.index("def _spawn_and_wait")]
assert "self._record_memory_state" in helper
compact = "".join(helper.split())
assert "self._record_memory_state(stripped" in compact
@@ -2270,3 +2272,140 @@ def test_the_replayed_cpu_fallback_recomputes_the_memory_record():
arm = src[src.index("allow_manual_cpu=True") :]
arm = arm[: arm.index("_apply_cpu_fallback_state")]
assert "self._record_memory_state(cmd,env)" in arm
+
+
+def _no_flash_fit_rewriter(
+ extra_args,
+ *,
+ manual_gpu_layers = None,
+ env = None,
+):
+ """The nested `_enable_managed_fit_for_no_flash` as a callable: it closes over
+ load_model's locals, so compiling its source is the only way to run the REAL rewrite."""
+ from core.inference.llama_cpp import LlamaCppBackend as B
+ from core.inference import llama_cpp as B_module
+ from core.inference.llama_cpp import logger, _flag_name
+ import inspect, textwrap
+
+ src = inspect.getsource(B.load_model)
+ start = src.index(" def _enable_managed_fit_for_no_flash(")
+ end = src.index(" def ", start + 1)
+ namespace = {
+ "logger": logger,
+ "_flag_name": _flag_name,
+ "_placement_is_fitter_proof": B_module._placement_is_fitter_proof,
+ "_user_fit_disabled": B_module._user_fit_disabled,
+ "extra_args": extra_args,
+ "_manual_gpu_layers": manual_gpu_layers,
+ "env": {} if env is None else env,
+ }
+ exec(textwrap.dedent(src[start:end]), namespace)
+ return namespace["_enable_managed_fit_for_no_flash"]
+
+
+def test_the_no_flash_retry_re_enables_unsloths_own_fitter():
+ """A managed `--fit off` is flipped back on for the respawn.
+
+ The reserve is only safe because the respawn is re-placed, and it inherits the auto
+ placement's `--fit off` that `_fit_off_retry_eligible` will not override, so without the
+ flip the retry re-lands on the smaller-cache placement and OOMs.
+ """
+ rewrite = _no_flash_fit_rewriter(None)
+ out = rewrite(["llama-server", "--fit", "off", "--flash-attn", "off"])
+ assert out[out.index("--fit") + 1] == "on"
+ on = ["llama-server", "--fit", "on"]
+ assert rewrite(on) == on
+ bare = ["llama-server", "--flash-attn", "off"]
+ assert rewrite(bare) == bare
+
+
+def test_the_no_flash_fit_flip_leaves_a_user_fit_alone():
+ """The user's own `--fit off` survives the respawn, in both spellings: theirs wins by
+ last-arg anyway, except where the only `--fit` present is theirs."""
+ for user_tokens in (["--fit", "off"], ["--fit=off"], ["-fit", "off"]):
+ rewrite = _no_flash_fit_rewriter(user_tokens)
+ # Unsloth's token present as well: the last value is still the user's "off".
+ both = rewrite(["llama-server", "--fit", "off", *user_tokens])
+ assert both == ["llama-server", "--fit", "off", *user_tokens]
+ theirs = rewrite(["llama-server", *user_tokens])
+ assert theirs == ["llama-server", *user_tokens]
+
+
+def test_an_inherited_fit_off_survives_the_no_flash_retry():
+ """LLAMA_ARG_FIT=off is the user's `--fit off`, and only manual mode scrubs it from the
+ child env, so outside manual the flip would put `--fit on` on the CLI and beat it.
+
+ `_user_fit_disabled` reads the same variable and already reserved the padded V cache for
+ this load, so the respawn lands on a placement priced for it and needs no re-place.
+ """
+ off = ["llama-server", "--fit", "off", "--flash-attn", "off"]
+ for value in ("off", "0", "false", "no", "disabled", " OFF "):
+ rewrite = _no_flash_fit_rewriter(None, env = {"LLAMA_ARG_FIT": value})
+ assert rewrite(off) == off, value
+
+ # Only an off. An inherited "on" leaves the managed placement re-placeable as before,
+ # and an empty value is not an answer.
+ for value in ("on", "1", ""):
+ rewrite = _no_flash_fit_rewriter(None, env = {"LLAMA_ARG_FIT": value})
+ assert rewrite(off)[off.index("--fit") + 1] == "on", value
+
+ # A user's own --fit token skips whatever the environment says, as it already did.
+ theirs = _no_flash_fit_rewriter(["--fit", "on"], env = {"LLAMA_ARG_FIT": "off"})
+ assert theirs(off) == off
+
+
+def test_the_no_flash_retry_re_places_at_both_respawns():
+ """Both --flash-attn off arms re-enable the fitter, each before its own spawn. Checked
+ at the source because these arms only run behind a real crash."""
+ from core.inference.llama_cpp import LlamaCppBackend as B
+ import inspect
+
+ src = inspect.getsource(B.load_model)
+ for label in ('label = "-noflash"', 'label = "-noflash-mtp"'):
+ arm = src[: src.index(label)]
+ arm = arm[arm.rindex("self._with_flash_attn_off(") :]
+ assert "_enable_managed_fit_for_no_flash(_fa_cmd)" in arm
+ assert src.count("_enable_managed_fit_for_no_flash(_fa_cmd)") == 2
+
+
+def test_a_fixed_layer_count_keeps_the_no_flash_retry_on_its_placement():
+ """Manual mode's `--fit off` is not an auto placement's, and must not be flipped.
+
+ Manual emits `--gpu-layers N --fit off`, token for token what an auto placement emits,
+ but `common_params_fit_impl` throws "n_gpu_layers already set by user"
+ (common/fit.cpp:377) for any count but -1, so the retry keeps the fixed placement and
+ the flip would only buy a reserve priced for a re-placement that cannot happen.
+ """
+ rewrite = _no_flash_fit_rewriter(None, manual_gpu_layers = 20)
+ fixed = ["llama-server", "--gpu-layers", "20", "--fit", "off"]
+ assert rewrite(fixed) == fixed
+
+ # The argv count decides on its own: llama.cpp reads the tokens, not this process.
+ assert _no_flash_fit_rewriter(None)(fixed) == fixed
+ # And an inherited count, which the fitting path emits no -ngl to lose to.
+ inherited = _no_flash_fit_rewriter(None, env = {"LLAMA_ARG_N_GPU_LAYERS": "20"})
+ auto = ["llama-server", "--fit", "off"]
+ assert inherited(auto) == auto
+ # llama.cpp's own default is not an override, so an auto placement still re-places.
+ default_count = ["llama-server", "--gpu-layers", "-1", "--fit", "off"]
+ assert _no_flash_fit_rewriter(None)(default_count)[-1] == "on"
+
+
+def test_the_reserve_holds_for_a_placement_the_fitter_will_not_move():
+ """The other half of the same case, on the estimate rather than the argv: with no
+ re-placement available the reserve prices the padded, f16-floored V."""
+ from core.inference.llama_cpp import _placement_is_fitter_proof, _reserved_flash_attn_state
+
+ assert _reserved_flash_attn_state(True, None, gpu_layers = 20, env = {}) is False
+ assert _reserved_flash_attn_state(True, ["-ngl", "20"], env = {}) is False
+ assert _reserved_flash_attn_state(True, None, env = {"LLAMA_ARG_N_GPU_LAYERS": "20"}) is False
+ # Auto placements keep the planned answer: the retry really is re-placed there.
+ assert _reserved_flash_attn_state(True, None, gpu_layers = -1, env = {}) is True
+ assert _reserved_flash_attn_state(True, ["-ngl", "-1"], env = {}) is True
+ assert _reserved_flash_attn_state(True, None, env = {}) is True
+
+ assert _placement_is_fitter_proof(["--gpu-layers=20"], env = {}) is True
+ assert _placement_is_fitter_proof(["--n-gpu-layers", "0"], env = {}) is True
+ assert _placement_is_fitter_proof(["-ngl", "auto"], env = {}) is False
+ assert _placement_is_fitter_proof(None, gpu_layers = 0, env = {}) is True
+ assert _placement_is_fitter_proof(None, env = {}) is False
diff --git a/studio/backend/tests/test_offload_planner_seam.py b/studio/backend/tests/test_offload_planner_seam.py
index cdd6fd8ea3f..012de7b3dcf 100644
--- a/studio/backend/tests/test_offload_planner_seam.py
+++ b/studio/backend/tests/test_offload_planner_seam.py
@@ -1558,9 +1558,8 @@ def test_flash_disabled_v_padding_reaches_the_layer_weights():
layer's V is padded to hparams.n_embd_v_gqa_max() over the whole model, which
is what _estimate_kv_cache_bytes charges via _max_kv_value_width. V goes
constant while K stays per-layer, so an unpadded vector prices a ratio the
- total does not have. Not an edge case: load_model pins planned_flash_attn =
- False unconditionally (llama_cpp.py:16690), so the padded branch is the one
- every spill plan's total is built from."""
+ total does not have. Reached whenever the resolved launch runs without flash
+ attention (_planned_flash_attn_state), which is exactly when llama.cpp pads."""
b = _swa_backend()
# SWA layers wider than global ones, so the model-wide max is the SWA width
# and the padding actually moves: n_embd_v_gqa_max = 8 * 256.
diff --git a/studio/backend/tests/test_resolved_flash_attn_plan.py b/studio/backend/tests/test_resolved_flash_attn_plan.py
new file mode 100644
index 00000000000..5c693aba529
--- /dev/null
+++ b/studio/backend/tests/test_resolved_flash_attn_plan.py
@@ -0,0 +1,544 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The estimate and the launch price the same flash-attention state.
+
+``load_model`` used to pin ``planned_flash_attn = False`` while emitting ``--flash-attn on``,
+so every placement figure described a load that was not going to happen (#9697, #10489). The
+cushion that pin gave is not lost: tensor mode cannot take the FA-off recovery at all, and
+elsewhere the respawn re-enters ``_spawn_and_wait``, which re-places it with ``--fit on``.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+_TESTS_DIR = Path(__file__).resolve().parent
+_BACKEND_DIR = str(_TESTS_DIR.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+if str(_TESTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_TESTS_DIR))
+
+
+def _load(module_name: str, file_name: str):
+ spec = importlib.util.spec_from_file_location(module_name, _TESTS_DIR / file_name)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+_placement = _load("_placement_for_flash_attn_plan", "test_llama_cpp_placement.py")
+
+from core.inference.llama_cpp import ( # noqa: E402
+ LlamaCppBackend,
+ _planned_flash_attn_state,
+)
+
+GB = 1024**3
+NATIVE = 262144
+
+
+class TestTheResolver:
+ def test_the_managed_default_is_on(self):
+ assert _planned_flash_attn_state() is True
+
+ def test_a_build_without_the_flag_is_off(self):
+ """No flag emitted, so the padded, f16-floored arm is the right price."""
+ assert _planned_flash_attn_state(supports_flash_attn = False) is False
+
+ @pytest.mark.parametrize(
+ "extras",
+ [
+ ["--flash-attn", "off"],
+ ["--flash-attn=off"],
+ ["-fa", "off"],
+ ["-fa=0"],
+ ],
+ )
+ def test_an_explicit_off_in_the_extras_wins(self, extras):
+ assert _planned_flash_attn_state(extras) is False
+
+ def test_the_last_flag_wins_like_llama_cpp(self):
+ assert _planned_flash_attn_state(["-fa", "off", "--flash-attn", "on"]) is True
+
+ def test_the_environment_loses_to_the_managed_flag(self):
+ """The env is read before argv (arg.cpp set_env), so the managed on wins."""
+ assert _planned_flash_attn_state(env = {"LLAMA_ARG_FLASH_ATTN": "0"}) is True
+
+ def test_a_user_off_still_beats_the_managed_flag(self):
+ """Order of authority: environment, managed flag, then the user's extras."""
+ assert _planned_flash_attn_state(["-fa", "off"], env = {"LLAMA_ARG_FLASH_ATTN": "0"}) is False
+ assert _planned_flash_attn_state(["-fa", "off"], env = {}) is False
+
+ def test_the_extras_beat_the_environment(self):
+ assert (
+ _planned_flash_attn_state(["--flash-attn", "on"], env = {"LLAMA_ARG_FLASH_ATTN": "0"})
+ is True
+ )
+
+ @pytest.mark.parametrize("v_type", ["q8_0", "q4_0", "q5_1", "iq4_nl"])
+ def test_a_quantized_v_cache_forces_it_on(self, v_type):
+ """llama.cpp turns it on itself, so an explicit off does not survive."""
+ assert (
+ _planned_flash_attn_state(["--flash-attn", "off"], planned_cache_types = ("f16", v_type))
+ is True
+ )
+
+ @pytest.mark.parametrize("v_type", ["f16", "bf16", "f32"])
+ def test_an_unquantized_v_cache_forces_nothing(self, v_type):
+ assert (
+ _planned_flash_attn_state(["--flash-attn", "off"], planned_cache_types = ("f16", v_type))
+ is False
+ )
+
+ def test_a_quantized_v_cannot_force_a_build_that_has_no_flag(self):
+ """The launch rewrites V to f16 instead, so forcing "on" would under-reserve."""
+ assert (
+ _planned_flash_attn_state(
+ planned_cache_types = ("q8_0", "q8_0"), supports_flash_attn = False
+ )
+ is False
+ )
+
+
+def _tensor_backend(tmp_path, *, free_mib: int):
+ backend, gguf = _placement._backend(
+ tmp_path,
+ vulkan = False,
+ memory = [(0, free_mib, free_mib), (1, free_mib, free_mib)],
+ )
+ # The harness turns KV estimation off; seed a real shape (Qwen3-0.6B) for arithmetic.
+ backend._can_estimate_kv = lambda: True
+ backend._n_layers = 28
+ backend._embedding_length = 1024
+ backend._n_heads = 16
+ backend._n_kv_heads = 8
+ backend._kv_key_length = 128
+ backend._kv_value_length = 128
+ backend._context_length = NATIVE
+ backend._get_gguf_size_bytes = lambda _path: 4 * GB
+ return backend, gguf
+
+
+def _ctx_of(cmd) -> int:
+ values = [cmd[i + 1] for i, token in enumerate(cmd) if token in ("-c", "--ctx-size")]
+ assert values, f"no context in argv: {cmd}"
+ return int(values[-1])
+
+
+def _flash_attn_in(cmd) -> bool:
+ for i, token in enumerate(cmd):
+ if token in ("--flash-attn", "-fa"):
+ return i + 1 >= len(cmd) or cmd[i + 1] != "off"
+ return False
+
+
+class TestTheTensorPlanPricesTheLaunch:
+ """A pool too small at f16-priced V and large enough once V is priced as launched."""
+
+ def test_the_planned_context_matches_the_emitted_flash_attention(self, tmp_path):
+ backend, gguf = _tensor_backend(tmp_path, free_mib = 12000)
+ captured = _placement._launch(
+ backend, gguf, n_ctx = NATIVE, cache_type_kv = "q8_0", tensor_parallel = True
+ )
+ cmd = captured["cmd"]
+ assert _flash_attn_in(cmd), "the launch emits flash attention"
+ planned = _ctx_of(cmd)
+
+ # The gap is the defect, so assert the side of it rather than a magic number.
+ pessimistic = backend._plan_tensor_parallel(
+ [(0, 12000), (1, 12000)],
+ 4 * GB,
+ NATIVE,
+ cache_type_kv = "q8_0",
+ max_target_ctx = NATIVE,
+ flash_attn = False,
+ )[0]
+ optimistic = backend._plan_tensor_parallel(
+ [(0, 12000), (1, 12000)],
+ 4 * GB,
+ NATIVE,
+ cache_type_kv = "q8_0",
+ max_target_ctx = NATIVE,
+ flash_attn = True,
+ )[0]
+ assert pessimistic < optimistic, "harness no longer straddles the difference"
+ assert planned > pessimistic, (
+ f"the tensor plan emitted {planned} tokens, at or below the {pessimistic} a "
+ f"flash-attention-off cache prices, while the launch runs with flash "
+ f"attention on and fits {optimistic}"
+ )
+
+ def test_the_published_ceiling_is_the_same_number(self, tmp_path):
+ """max_context_length is what the context warning compares against."""
+ backend, gguf = _tensor_backend(tmp_path, free_mib = 12000)
+ captured = _placement._launch(
+ backend, gguf, n_ctx = NATIVE, cache_type_kv = "q8_0", tensor_parallel = True
+ )
+ assert backend.max_context_length == _ctx_of(captured["cmd"])
+
+ def test_an_unquantized_cache_is_unaffected(self, tmp_path):
+ """On the arithmetic, not a hand-built plan: the loader charges extra overheads."""
+ backend, gguf = _tensor_backend(tmp_path, free_mib = 12000)
+ captured = _placement._launch(
+ backend, gguf, n_ctx = NATIVE, cache_type_kv = "f16", tensor_parallel = True
+ )
+ planned = _ctx_of(captured["cmd"])
+ assert backend._estimate_kv_cache_bytes(
+ planned, "f16", flash_attn = True
+ ) == backend._estimate_kv_cache_bytes(planned, "f16", flash_attn = False)
+
+
+class TestTheSizingCallsSeeTheResolvedState:
+ def _seen_flash_attn(self, tmp_path, **load_kwargs):
+ backend, gguf = _tensor_backend(tmp_path, free_mib = 12000)
+ seen: list[object] = []
+ real = backend._estimate_kv_cache_bytes
+
+ def spy(
+ n_ctx,
+ cache_type_kv = None,
+ **kwargs,
+ ):
+ seen.append(kwargs.get("flash_attn"))
+ return real(n_ctx, cache_type_kv, **kwargs)
+
+ backend._estimate_kv_cache_bytes = spy
+ _placement._launch(backend, gguf, **load_kwargs)
+ assert seen, "no KV estimate was taken during the load"
+ return seen
+
+ def test_a_quantized_cache_is_priced_with_flash_attention_on(self, tmp_path):
+ seen = self._seen_flash_attn(
+ tmp_path, n_ctx = NATIVE, cache_type_kv = "q8_0", tensor_parallel = True
+ )
+ assert all(state is True for state in seen), (
+ f"the load priced its KV cache with flash_attn states {sorted(set(map(str, seen)))} "
+ f"while emitting --flash-attn on"
+ )
+
+ def test_an_explicit_off_in_the_extras_is_priced_off(self, tmp_path):
+ seen = self._seen_flash_attn(
+ tmp_path,
+ n_ctx = NATIVE,
+ cache_type_kv = "f16",
+ extra_args = ["--flash-attn", "off"],
+ )
+ assert all(state is False for state in seen)
+
+
+def test_the_state_is_still_named_planned_flash_attn():
+ """The offload-planner seam asserts on this identifier by AST."""
+ import inspect
+
+ source = inspect.getsource(inspect.unwrap(LlamaCppBackend.load_model))
+ # Through the reserve wrapper now, which only holds a reading DOWN.
+ assert "planned_flash_attn = _reserved_flash_attn_state(" in source
+ assert "_planned_flash_attn_state(" in source
+ assert "planned_flash_attn = False" not in source
+
+
+class TestAutoIsNotAnAnswer:
+ """``auto`` is decided at load time and silently decided against whenever the backend,
+ model or cache pair cannot take it, so reading it as "on" under-reserves."""
+
+ def test_an_explicit_auto_prices_the_padded_cache(self):
+ assert _planned_flash_attn_state(["--flash-attn", "auto"]) is False
+ assert _planned_flash_attn_state(["-fa", "auto"]) is False
+ assert _planned_flash_attn_state(["-fa=auto"]) is False
+ # llama.cpp's numeric spelling of the same value.
+ assert _planned_flash_attn_state(["-fa", "-1"]) is False
+
+ def test_an_inherited_auto_is_overridden_by_the_managed_flag(self):
+ """The managed flag is appended after the env is read, so only a USER auto survives."""
+ assert _planned_flash_attn_state(None, env = {"LLAMA_ARG_FLASH_ATTN": "auto"}) is True
+ assert (
+ _planned_flash_attn_state(["-fa", "auto"], env = {"LLAMA_ARG_FLASH_ATTN": "1"}) is False
+ )
+ assert (
+ _planned_flash_attn_state(["-fa", "on"], env = {"LLAMA_ARG_FLASH_ATTN": "auto"}) is True
+ )
+ assert (
+ _planned_flash_attn_state(["-fa", "auto"], env = {"LLAMA_ARG_FLASH_ATTN": "1"}) is False
+ )
+
+ def test_the_last_flag_still_wins(self):
+ assert _planned_flash_attn_state(["-fa", "auto", "-fa", "on"]) is True
+ assert _planned_flash_attn_state(["-fa", "on", "-fa", "auto"]) is False
+
+ def test_a_quantized_v_cache_still_forces_it_on(self):
+ assert (
+ _planned_flash_attn_state(["-fa", "auto"], planned_cache_types = ("q8_0", "q4_0")) is True
+ )
+
+ def test_tensor_split_decides_auto_rather_than_leaving_it_open(self):
+ """llama.cpp upgrades AUTO to ENABLED under SPLIT_MODE_TENSOR, so pricing the
+ padded V there charges a cache the child never allocates."""
+ # The three ways the launch decides the mode: toggle, extras, inherited env.
+ assert _planned_flash_attn_state(["-fa", "auto"], tensor_parallel = True) is True
+ assert _planned_flash_attn_state(["-fa", "auto", "--split-mode", "tensor"], env = {}) is True
+ assert (
+ _planned_flash_attn_state(["-fa", "auto"], env = {"LLAMA_ARG_SPLIT_MODE": "tensor"})
+ is True
+ )
+ # An extras split mode last-wins over the toggle, so layer stays undecided.
+ assert (
+ _planned_flash_attn_state(
+ ["-fa", "auto", "--split-mode", "layer"], tensor_parallel = True, env = {}
+ )
+ is False
+ )
+ assert _planned_flash_attn_state(["-fa", "auto"], env = {}) is False
+ # A user OFF is not silently flipped on: refusing the pair is the launch's job.
+ assert _planned_flash_attn_state(["-fa", "off"], tensor_parallel = True) is False
+
+ def test_the_managed_launch_is_unaffected(self):
+ assert _planned_flash_attn_state(None) is True
+ assert _planned_flash_attn_state([]) is True
+
+
+class TestTheDowngradesRePlanTheAttention:
+ """The plan is made once near the top of the load and the split is decided later, so a
+ stale True budgets an unpadded V the layer split will not get."""
+
+ def _load_model_body(self):
+ import ast
+ import inspect
+
+ from core.inference import llama_cpp as module
+
+ source = inspect.getsource(module.LlamaCppBackend.load_model)
+ return ast.parse(textwrap.dedent(source)).body[0]
+
+ def test_every_tensor_downgrade_re_plans_the_attention(self):
+ """Not only the ones that strip the extras: the three manual guards set the toggle
+ to False on their own, and Manual placement runs with --fit off."""
+ import ast
+
+ func = self._load_model_body()
+ # Only downgrades AFTER the plan is made; virtualised Metal drops it before.
+ planned_at = min(
+ node.lineno
+ for node in ast.walk(func)
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", None) == "_planned_flash_attn_state"
+ )
+ checked = 0
+ for node in ast.walk(func):
+ for attr in ("body", "orelse", "finalbody"):
+ block = getattr(node, attr, None)
+ if not isinstance(block, list):
+ continue
+ for index, statement in enumerate(block):
+ if not (
+ isinstance(statement, ast.Assign)
+ and getattr(statement.targets[0], "id", None) == "tensor_parallel"
+ and isinstance(statement.value, ast.Constant)
+ and statement.value.value is False
+ and statement.lineno > planned_at
+ ):
+ continue
+ checked += 1
+ following = [
+ getattr(getattr(later, "targets", [None])[0], "id", None)
+ for later in block[index + 1 : index + 4]
+ if isinstance(later, ast.Assign)
+ ]
+ assert "planned_flash_attn" in following, (
+ "a tensor downgrade at line "
+ f"{statement.lineno} does not re-plan the attention"
+ )
+ assert checked >= 6, checked
+
+ def test_every_split_mode_strip_re_plans_the_attention(self):
+ import ast
+
+ func = self._load_model_body()
+ strips = 0
+ for node in ast.walk(func):
+ for attr in ("body", "orelse", "finalbody"):
+ block = getattr(node, attr, None)
+ if not isinstance(block, list):
+ continue
+ for index, statement in enumerate(block):
+ if not isinstance(statement, ast.Assign):
+ continue
+ call = statement.value
+ if not (
+ isinstance(call, ast.Call)
+ and getattr(call.func, "id", None) == "strip_split_mode_only"
+ ):
+ continue
+ strips += 1
+ following = block[index + 1] if index + 1 < len(block) else None
+ assert isinstance(following, ast.Assign), ast.dump(statement)
+ assert (
+ getattr(following.targets[0], "id", None) == "planned_flash_attn"
+ ), ast.dump(following)
+ assert strips >= 6, strips
+
+ def test_the_re_plan_reads_the_current_split(self):
+ import ast
+
+ func = self._load_model_body()
+ helper = next(
+ node
+ for node in ast.walk(func)
+ if isinstance(node, ast.FunctionDef) and node.name == "_replanned_flash_attn"
+ )
+ call = next(
+ node
+ for node in ast.walk(helper)
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", None) == "_planned_flash_attn_state"
+ )
+ passed = {kw.arg: kw.value for kw in call.keywords}
+ assert getattr(passed["tensor_parallel"], "id", None) == "_current_tp"
+ assert getattr(call.args[0], "id", None) == "extra_args"
+
+ def test_no_sizing_closure_freezes_the_plan_in_a_default_argument(self):
+ """A default argument is evaluated where the closure is written, so it holds
+ whatever the plan said there, and the downgrades below re-plan it. The MTP draft
+ reserve did that: a hybrid SWA draft head prices 4.5x larger with flash attention
+ off (262,144 tokens, 3.5 GB on a 4-layer shape), so a frozen tensor-mode True left
+ the reserve short of what the layer split it downgraded to actually allocates,
+ beside a KV cache that had already re-read the plan."""
+ import ast
+
+ func = self._load_model_body()
+ frozen = [
+ (node.name, default.id)
+ for node in ast.walk(func)
+ if isinstance(node, ast.FunctionDef)
+ for default in node.args.defaults
+ if isinstance(default, ast.Name) and default.id == "planned_flash_attn"
+ ]
+ assert frozen == [], frozen
+ # And the term is still priced, rather than dropped to silence the pin.
+ mtp = next(
+ node
+ for node in ast.walk(func)
+ if isinstance(node, ast.FunctionDef) and node.name == "mtp_overhead_fn"
+ )
+ call = next(
+ node
+ for node in ast.walk(mtp)
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "attr", None) == "_estimate_mtp_overhead_bytes"
+ )
+ passed = {kw.arg: getattr(kw.value, "id", None) for kw in call.keywords}
+ assert passed.get("flash_attn") == "planned_flash_attn", passed
+
+
+class TestTheReserveWhenTheFitterIsOff:
+ """The reserve has to survive the respawn too, and a user's own `--fit off` is what
+ takes the re-placement away."""
+
+ def test_a_user_fit_off_keeps_the_conservative_reserve(self):
+ from core.inference.llama_cpp import _reserved_flash_attn_state
+
+ # The ordinary managed launch is unchanged: fitting is on, so the respawn re-places.
+ assert _reserved_flash_attn_state(True, None, env = {}) is True
+ assert _reserved_flash_attn_state(True, ["--ctx-size", "4096"], env = {}) is True
+ # With the fitter off, the respawn lands on the placement this reserve chose.
+ assert _reserved_flash_attn_state(True, ["--fit", "off"], env = {}) is False
+ assert _reserved_flash_attn_state(True, ["--fit=off"], env = {}) is False
+ assert _reserved_flash_attn_state(True, None, env = {"LLAMA_ARG_FIT": "off"}) is False
+ # Last-wins, like every other flag: a later --fit on is the state that runs.
+ assert _reserved_flash_attn_state(True, ["--fit", "off", "--fit", "on"], env = {}) is True
+ assert (
+ _reserved_flash_attn_state(True, ["--fit", "on"], env = {"LLAMA_ARG_FIT": "off"}) is True
+ )
+
+ def test_tensor_mode_keeps_its_answer(self):
+ """No no-flash respawn to reserve for under SPLIT_MODE_TENSOR, and pricing the
+ padded layout would refuse loads that fit."""
+ from core.inference.llama_cpp import _reserved_flash_attn_state
+
+ assert (
+ _reserved_flash_attn_state(True, ["--fit", "off"], tensor_parallel = True, env = {}) is True
+ )
+ assert (
+ _reserved_flash_attn_state(True, ["--fit", "off", "--split-mode", "tensor"], env = {})
+ is True
+ )
+
+ def test_a_false_plan_is_never_raised(self):
+ """This only holds a reading DOWN; an off is already the conservative one."""
+ from core.inference.llama_cpp import _reserved_flash_attn_state
+
+ assert _reserved_flash_attn_state(False, ["--fit", "off"], env = {}) is False
+ assert _reserved_flash_attn_state(False, None, env = {}) is False
+
+ def test_the_load_reserves_through_it(self):
+ import inspect
+
+ from core.inference import llama_cpp as module
+
+ body = inspect.getsource(module.LlamaCppBackend.load_model)
+ assert "_reserved_flash_attn_state(" in body
+ # Both the first plan and every re-plan, or a downgrade would put the raw reading back.
+ assert body.count("_reserved_flash_attn_state(") == 2, body.count(
+ "_reserved_flash_attn_state("
+ )
+
+
+class TestTheReplanIsAuthoritative:
+ """A downgrade strips the extras, never the env, but the helper also resolves an
+ inherited LLAMA_ARG_SPLIT_MODE, so a re-plan carrying the downgraded False still met a
+ tensor answer inside it. The launch clears that variable for exactly this reason."""
+
+ def _load_model_body(self):
+ import ast
+ import inspect
+ import textwrap
+
+ from core.inference import llama_cpp as module
+
+ source = inspect.getsource(module.LlamaCppBackend.load_model)
+ return ast.parse(textwrap.dedent(source)).body[0]
+
+ def test_the_replan_scrubs_the_inherited_split_mode(self):
+ import ast
+
+ func = self._load_model_body()
+ helper = next(
+ node
+ for node in ast.walk(func)
+ if isinstance(node, ast.FunctionDef) and node.name == "_replan_env"
+ )
+ text = ast.dump(helper)
+ assert "LLAMA_ARG_SPLIT_MODE" in text
+ assert "LLAMA_ARG_TENSOR_SPLIT" in text
+ replan = next(
+ node
+ for node in ast.walk(func)
+ if isinstance(node, ast.FunctionDef) and node.name == "_replanned_flash_attn"
+ )
+ calls = [
+ node
+ for node in ast.walk(replan)
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", None)
+ in {"_planned_flash_attn_state", "_reserved_flash_attn_state"}
+ ]
+ assert len(calls) == 2, [getattr(c.func, "id", None) for c in calls]
+ for call in calls:
+ passed = {kw.arg for kw in call.keywords}
+ assert "env" in passed, ast.dump(call)
+
+ def test_the_helper_answers_layer_once_the_variable_is_gone(self):
+ from core.inference.llama_cpp import _planned_flash_attn_state
+
+ inherited = {"LLAMA_ARG_SPLIT_MODE": "tensor"}
+ # The state the re-plan used to resolve: a downgraded toggle, a tensor environment.
+ assert (
+ _planned_flash_attn_state(["-fa", "auto"], tensor_parallel = False, env = inherited) is True
+ )
+ assert _planned_flash_attn_state(["-fa", "auto"], tensor_parallel = False, env = {}) is False
diff --git a/studio/backend/tests/test_unmeasured_context_ceiling.py b/studio/backend/tests/test_unmeasured_context_ceiling.py
new file mode 100644
index 00000000000..ae592dfc15a
--- /dev/null
+++ b/studio/backend/tests/test_unmeasured_context_ceiling.py
@@ -0,0 +1,256 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A context nobody could measure is not a ceiling, and must not overrule a request.
+
+Two arms fall back to a short context when the GGUF carries no attention dimensions, and
+both applied it to an explicit request and published it, so 256k came back as 4096 (#9653).
+A guess will not refuse against that number, so it may not overrule one either. Auto keeps
+the conservative fallback.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+
+_TESTS_DIR = Path(__file__).resolve().parent
+_BACKEND_DIR = str(_TESTS_DIR.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+if str(_TESTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_TESTS_DIR))
+
+
+def _load(module_name: str, file_name: str):
+ spec = importlib.util.spec_from_file_location(module_name, _TESTS_DIR / file_name)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+# The no-GPU probe in this harness IS the Metal condition.
+_metal = _load("_metal_guard_for_unmeasured_ctx", "test_metal_explicit_context_guard.py")
+_metal_launch = _metal._launch
+_ctx_values = _metal._ctx_values
+
+from core.inference.llama_cpp import ( # noqa: E402
+ _FIT_MIN_CTX,
+ _TP_UNMEASURED_CTX,
+ LlamaCppBackend,
+)
+
+GB = 1024**3
+NATIVE = 262144
+TWO_80 = [(0, 80 * 1024), (1, 80 * 1024)]
+
+
+def _measurable() -> LlamaCppBackend:
+ backend = LlamaCppBackend()
+ backend._n_layers = 28
+ backend._embedding_length = 1024
+ backend._n_heads = 16
+ backend._n_kv_heads = 8
+ backend._kv_key_length = 128
+ backend._kv_value_length = 128
+ backend._context_length = NATIVE
+ return backend
+
+
+def _unmeasurable() -> LlamaCppBackend:
+ backend = LlamaCppBackend()
+ backend._context_length = NATIVE
+ assert not backend._can_estimate_kv()
+ return backend
+
+
+class TestTheTensorParallelFallback:
+ def test_an_explicit_request_is_honoured(self):
+ effective, ceiling, *_ = _unmeasurable()._plan_tensor_parallel(
+ TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE, explicit_ctx = True
+ )
+ assert effective == NATIVE
+ # The pair the issue quotes from /v1/models: the ceiling may not contradict it.
+ assert ceiling >= effective
+
+ def test_auto_still_takes_the_conservative_fallback(self):
+ effective, ceiling, *_ = _unmeasurable()._plan_tensor_parallel(
+ TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE
+ )
+ assert effective == _TP_UNMEASURED_CTX
+ assert ceiling == _TP_UNMEASURED_CTX
+
+ def test_a_measured_cap_still_binds_an_explicit_request(self):
+ """``--fit`` is a no-op in tensor mode, so a measured cap is all that stands between
+ the request and a startup OOM."""
+ tight = [(0, 6 * 1024), (1, 6 * 1024)]
+ capped, _ceiling, *_ = _measurable()._plan_tensor_parallel(
+ tight, 8 * GB, NATIVE, max_target_ctx = NATIVE, explicit_ctx = True
+ )
+ assert 0 < capped < NATIVE
+
+ def test_a_measurable_plan_that_fits_is_unchanged_by_the_flag(self):
+ with_flag = _measurable()._plan_tensor_parallel(
+ TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE, explicit_ctx = True
+ )
+ without = _measurable()._plan_tensor_parallel(
+ TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE
+ )
+ assert with_flag == without
+
+ def test_the_flag_defaults_off_for_every_existing_caller(self):
+ assert _unmeasurable()._plan_tensor_parallel(TWO_80, 20 * GB, NATIVE)[0] == (
+ _TP_UNMEASURED_CTX
+ )
+
+ @pytest.mark.parametrize("target", [0, -1])
+ def test_no_request_is_not_a_request(self, target):
+ effective, _ceiling, *_ = _unmeasurable()._plan_tensor_parallel(
+ TWO_80, 20 * GB, target, explicit_ctx = True
+ )
+ assert effective == _TP_UNMEASURED_CTX
+
+
+class TestTheNotice:
+ def test_it_names_the_context_that_was_launched(self):
+ message = LlamaCppBackend._unmeasured_context_notice(NATIVE)
+ assert message is not None
+ assert "262,144" in message
+
+ def test_it_says_the_maximum_is_not_a_measurement(self):
+ message = LlamaCppBackend._unmeasured_context_notice(NATIVE)
+ assert "not a measured ceiling" in message
+
+ def test_it_does_not_read_as_a_refusal(self):
+ message = LlamaCppBackend._unmeasured_context_notice(NATIVE)
+ assert "launched as requested" in message
+ for blame in ("cannot", "refus", "not supported"):
+ assert blame not in message.lower()
+
+ @pytest.mark.parametrize("cache_type", [None, "", "f16", "fp16"])
+ def test_the_kv_hint_is_offered_on_an_unquantized_cache(self, cache_type):
+ assert "q8_0" in LlamaCppBackend._unmeasured_context_notice(NATIVE, cache_type)
+
+ @pytest.mark.parametrize("cache_type", ["q8_0", "q4_0"])
+ def test_it_is_not_offered_once_the_cache_is_quantized(self, cache_type):
+ assert "q8_0" not in LlamaCppBackend._unmeasured_context_notice(NATIVE, cache_type)
+
+ @pytest.mark.parametrize("requested", [0, -1])
+ def test_nothing_to_report_without_a_request(self, requested):
+ assert LlamaCppBackend._unmeasured_context_notice(requested) is None
+
+ def test_it_does_not_cost_the_load_a_memory_warning(self):
+ """``_record_load_warning`` is first-notice-wins with one slot, and both arms
+ that raise an unmeasured ceiling decide it long before the host-RAM and offload
+ advisories. Recording it there took the slot from a memory warning on the one
+ path that simultaneously raises the launched context, so the user who is told
+ their load will page from disk heard only that the ceiling was a guess. It is
+ held in ``_unmeasured_ctx_notice`` and flushed past both, appending when one
+ spoke."""
+ import ast
+ import inspect
+ import textwrap
+
+ source = inspect.getsource(LlamaCppBackend.load_model)
+ func = ast.parse(textwrap.dedent(source)).body[0]
+
+ # Nothing hands the notice straight to the slot where it is decided.
+ for node in ast.walk(func):
+ if not (
+ isinstance(node, ast.Call)
+ and getattr(node.func, "attr", None) == "_record_load_warning"
+ ):
+ continue
+ for arg in node.args:
+ assert not (
+ isinstance(arg, ast.Call)
+ and getattr(arg.func, "attr", None) == "_unmeasured_context_notice"
+ ), f"the notice claims the warning slot at line {node.lineno}"
+
+ held = [
+ node.lineno
+ for node in ast.walk(func)
+ if isinstance(node, ast.Assign)
+ and getattr(node.targets[0], "id", None) == "_unmeasured_ctx_notice"
+ and isinstance(node.value, ast.Call)
+ ]
+ flushed = [
+ node.lineno
+ for node in ast.walk(func)
+ if isinstance(node, ast.If)
+ and isinstance(node.test, ast.Name)
+ and node.test.id == "_unmeasured_ctx_notice"
+ ]
+ # Both arms hold it, and there is one flush.
+ assert len(held) == 2, held
+ assert len(flushed) == 1, flushed
+
+ # The flush is past this attempt's memory advisories, or it displaces one again.
+ # Named rather than "every _record_load_warning": the arch-crash retry's pair sits
+ # textually below the publish point but belongs to a later attempt, which clears
+ # the slot through _begin_load_warnings before recording anything.
+ advisories = {
+ arg.id: node.lineno
+ for node in ast.walk(func)
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "attr", None) == "_record_load_warning"
+ for arg in node.args
+ if isinstance(arg, ast.Name)
+ }
+ for name in ("_ram_msg", "_offload_msg"):
+ assert name in advisories, (name, sorted(advisories))
+ assert flushed[0] > advisories[name], (name, flushed, advisories)
+
+
+class TestTheMetalArm:
+ """Simulated with an empty GPU probe; no Metal device is exercised."""
+
+ def _launch_unmeasurable(self, tmp_path, monkeypatch, **kwargs):
+ return _metal_launch(
+ tmp_path,
+ monkeypatch,
+ metal = True,
+ can_estimate_kv = False,
+ real_fit = True,
+ budget_bytes = 24 * GB,
+ native = NATIVE,
+ **kwargs,
+ )
+
+ def test_an_explicit_request_reaches_llama_server(self, tmp_path, monkeypatch):
+ captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = NATIVE)
+ assert _ctx_values(captured["cmd"])[-1] == str(NATIVE)
+
+ def test_the_published_ceiling_is_not_below_what_launched(self, tmp_path, monkeypatch):
+ """The floor used to be published while the child ran at the request."""
+ captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = NATIVE)
+ backend = captured["backend"]
+ assert backend.max_context_length >= NATIVE
+ assert backend.native_context_length == NATIVE
+
+ def test_the_load_says_the_ceiling_was_never_measured(self, tmp_path, monkeypatch):
+ captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = NATIVE)
+ assert "not a measured ceiling" in (captured["backend"].last_load_warning or "")
+
+ def test_auto_keeps_the_floor(self, tmp_path, monkeypatch):
+ captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = 0)
+ assert _ctx_values(captured["cmd"])[-1] == str(_FIT_MIN_CTX)
+ assert captured["backend"].max_context_length == _FIT_MIN_CTX
+
+ def test_auto_is_not_told_anything(self, tmp_path, monkeypatch):
+ """Auto made no request; an advisory on every unreadable header is noise."""
+ captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = 0)
+ assert captured["backend"].last_load_warning is None
+
+ def test_a_measured_ceiling_still_refuses(self, tmp_path, monkeypatch):
+ with pytest.raises(RuntimeError, match = "unified"):
+ _metal_launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 32768,
+ metal = True,
+ can_estimate_kv = True,
+ )