Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep a pinned context as a request limit instead of refusing KV cache quantization (#11084)

Long Yixing1759e64588d4
diff --git a/studio/backend/core/inference/context_refusal.py b/studio/backend/core/inference/context_refusal.py
index 6d33bfc8a7c..647f1940fb3 100644
--- a/studio/backend/core/inference/context_refusal.py
+++ b/studio/backend/core/inference/context_refusal.py
@@ -21,6 +21,7 @@ __all__ = [
"latest_refusal",
"describe_oversize",
"open_slot",
+ "ContextBudgetExceeded",
]
@@ -321,3 +322,12 @@ def describe_unservable_tool_call(
head + tried + "Nothing was written. Increase the Context Length in Model settings, "
f"or {lever}, then try again."
)
+
+
+class ContextBudgetExceeded(ValueError):
+ """Prompt refused before generation; carries the counts so nothing parses the message."""
+
+ def __init__(self, request_tokens: int, context_tokens: int):
+ self.request_tokens = int(request_tokens)
+ self.context_tokens = int(context_tokens)
+ super().__init__(describe_oversize(self.request_tokens, self.context_tokens))
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 20b3f560d48..d8d7c39cb84 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -14,6 +14,7 @@ import time
from collections import OrderedDict
from contextlib import ExitStack, closing, contextmanager, nullcontext
from typing import Optional, Generator
+from core.inference import context_refusal
from core.inference.message_content import content_to_text
from core.inference.native_tool_tokens import (
NativeToolTokenDecoder,
@@ -1385,12 +1386,6 @@ MLX_KV_QUANT_NO_REUSE = (
MLX_KV_QUANT_VLM_CACHE_NOTE = (
"On vision models, quantization starts once the cache reaches {start} tokens."
)
-# RotatingKVCache.to_quantized raises and mlx-lm converts from the first token, so the two are resolved here rather
-# than failing generation on its first step.
-MLX_KV_QUANT_PINNED_CONTEXT = (
- "Context Length is set for this model, which limits the KV cache, and the installed "
- "mlx-lm cannot quantize a limited cache. Reset it to quantize instead."
-)
def _kv_entry_nbytes(entry):
@@ -1974,7 +1969,7 @@ def _kv_quant_status(
requested_bits,
model,
is_vlm,
- context_pinned = False,
+ starts_at_first_token = False,
):
"""Resolve a requested bit width against this model into a status dict."""
status = {
@@ -1986,18 +1981,13 @@ def _kv_quant_status(
}
if requested_bits is None:
return status
- if context_pinned:
- status["eligibility"] = "refused"
- status["reason"] = MLX_KV_QUANT_PINNED_CONTEXT
- logger.info("MLX KV quantization not applied: %s", MLX_KV_QUANT_PINNED_CONTEXT)
- return status
verdict, reason, retainable = _kv_quant_eligibility(model, is_vlm, requested_bits)
status["eligibility"] = verdict
status["reason"] = reason
if verdict in ("full", "partial"):
status["kv_bits"] = requested_bits
notes = []
- if is_vlm:
+ if is_vlm and not starts_at_first_token:
notes.append(MLX_KV_QUANT_VLM_CACHE_NOTE.format(start = _vlm_quantized_kv_start()))
if not retainable:
notes.append(MLX_KV_QUANT_NO_REUSE)
@@ -2495,6 +2485,7 @@ class MLXInferenceBackend:
# Bound now so a load that fails before installing leaves readers a dict rather than raising.
self._kv_quant = _kv_quant_status(None, None, False)
self._kv_cache_window = None
+ self._kv_context_budget = None
self._served_context = None
self._template_override = _template_override_status(None, None, None)[1]
@@ -2679,25 +2670,95 @@ class MLXInferenceBackend:
self,
prompt,
prompt_tokens = None,
+ images = None,
+ audio = None,
+ videos = None,
):
"""Free context for an unset limit, or the default if the prompt cannot be counted."""
try:
prompt_n = (
len(prompt_tokens)
if prompt_tokens is not None
- else self._count_prompt_tokens(prompt)
+ else self._count_prompt_tokens(prompt, images, audio, videos)
)
except Exception as exc:
logger.debug("MLX prompt count for an unset budget failed: %s", exc)
return UNSET_GENERATION_BUDGET
return generation_budget_for_window(self._served_context, prompt_n, None)
+ def _generation_limit(
+ self,
+ prompt,
+ max_new_tokens,
+ prompt_tokens = None,
+ images = None,
+ audio = None,
+ videos = None,
+ cap = None,
+ ):
+ budget = getattr(self, "_kv_context_budget", None)
+ if not budget:
+ # No budget: size from the placeholder count as before, never preparing the media.
+ if max_new_tokens is None:
+ max_new_tokens = self._unset_generation_budget(prompt, prompt_tokens)
+ if cap is not None:
+ max_new_tokens = min(max_new_tokens, cap)
+ return max_new_tokens
+ # Max Tokens == window is the UI's "no cap"; admitting it as-is would refuse every prompt.
+ if max_new_tokens is not None and int(max_new_tokens) == int(budget):
+ max_new_tokens = None
+ if prompt_tokens is None:
+ try:
+ prompt_tokens = range(self._count_prompt_tokens(prompt, images, audio, videos))
+ except Exception as exc:
+ logger.debug("MLX prompt count for the context budget failed: %s", exc)
+ if max_new_tokens is None:
+ max_new_tokens = self._unset_generation_budget(prompt, prompt_tokens)
+ if cap is not None:
+ max_new_tokens = min(max_new_tokens, cap)
+ if prompt_tokens is not None:
+ # Refuse only a prompt that leaves no room; a longer ask is cut at the budget as
+ # llama-server does, since callers such as Deep Research size it from an estimate.
+ self._check_context_budget(prompt, 1, prompt_tokens)
+ max_new_tokens = max(1, min(int(max_new_tokens), int(budget) - len(prompt_tokens)))
+ return max_new_tokens
+
def _kv_quant_generate_kwargs(self):
"""Load-time runtime knobs for a generate call, empty when unset. quantized_kv_start is
- deliberately not passed: mlx-lm and mlx-vlm ship different defaults (0 and 5000) and each
- runtime keeps its own."""
+ otherwise left to each runtime, whose defaults differ (0 and 5000); under a budget
+ mlx-vlm's could sit above every request the budget admits, and would never be reached."""
kv_bits = (getattr(self, "_kv_quant", None) or {}).get("kv_bits")
- return {} if kv_bits is None else {"kv_bits": kv_bits}
+ if kv_bits is None:
+ return {}
+ if getattr(self, "_kv_context_budget", None):
+ return {"kv_bits": kv_bits, "quantized_kv_start": 0}
+ return {"kv_bits": kv_bits}
+
+ def _check_context_budget(
+ self,
+ prompt,
+ max_new_tokens = 0,
+ prompt_tokens = None,
+ images = None,
+ audio = None,
+ videos = None,
+ ):
+ """Refuse a request over the budget; an uncountable prompt is admitted."""
+ budget = getattr(self, "_kv_context_budget", None)
+ if not budget:
+ return
+ try:
+ prompt_n = (
+ len(prompt_tokens)
+ if prompt_tokens is not None
+ else self._count_prompt_tokens(prompt, images, audio, videos)
+ )
+ except Exception as exc:
+ logger.debug("MLX prompt count for the context budget failed: %s", exc)
+ return
+ needed = prompt_n + max(0, int(max_new_tokens or 0))
+ if needed > budget:
+ raise context_refusal.ContextBudgetExceeded(needed, int(budget))
def _kv_window_generate_kwargs(self):
"""The cache bound for a generation that builds its own cache, empty when unset. Both
@@ -2717,10 +2778,60 @@ class MLXInferenceBackend:
)
)
- def _count_prompt_tokens(self, prompt):
+ def _count_media_prompt_tokens(
+ self,
+ prompt,
+ images = None,
+ audio = None,
+ videos = None,
+ ):
+ """Prompt length as the processor expands media (as mlx-vlm's server does), else None."""
+ if not (images or audio or videos):
+ return None
+ try:
+ from mlx_vlm.utils import prepare_inputs
+
+ config = getattr(self._model, "config", None)
+ read = (
+ config.get if isinstance(config, dict) else lambda attr: getattr(config, attr, None)
+ )
+ kwargs = {}
+ if images:
+ kwargs["images"] = images
+ if audio:
+ kwargs["audio"] = audio
+ if videos:
+ # Same fps as generation, not the processor's default rate.
+ kwargs["videos"] = videos
+ fps = _video_frame_rate(videos[0], self._processor)
+ if fps is not None:
+ kwargs["fps"] = fps
+ inputs = prepare_inputs(
+ self._processor,
+ prompts = prompt,
+ image_token_index = read("image_token_index"),
+ add_special_tokens = _vlm_add_special_tokens(read("model_type"), self._processor),
+ **kwargs,
+ )
+ ids = inputs["input_ids"]
+ return int(getattr(ids, "size", 0)) or len(ids)
+ except Exception as exc:
+ logger.debug("MLX media prompt count failed: %s", exc)
+ return None
+
+ def _count_prompt_tokens(
+ self,
+ prompt,
+ images = None,
+ audio = None,
+ videos = None,
+ ):
"""Prompt length as generation tokenizes it; vision models follow mlx_vlm's marker rule."""
if not self._is_vlm:
return len(self._encode_prompt(prompt))
+ expanded = self._count_media_prompt_tokens(prompt, images, audio, videos)
+ if expanded is not None:
+ return expanded
model_type = getattr(getattr(self._model, "config", None), "model_type", None)
add_special = _vlm_add_special_tokens(model_type, self._processor)
return len(self._tokenizer.encode(prompt, add_special_tokens = add_special))
@@ -2795,17 +2906,9 @@ class MLXInferenceBackend:
return enforced
def _resolve_kv_policy(self, is_vlm, kv_bits, max_seq_length, served):
- """The quantization status and cache window this load will run with.
-
- Rotation is what keeps a long conversation inside the window, so nothing here can refuse a
- request; the model simply stops attending to the oldest tokens.
+ """The quantization status, cache window and per-request budget this load will run with.
- Quantization cannot coexist with a bound -- a rotating cache has no conversion, and mlx-lm
- converts from the first token -- so an enforceable pin, an explicit instruction about
- memory, outranks it. A window the backend chose for itself yields to an explicitly requested
- quantization, and so does a pin that cannot be enforced, which would otherwise spend the
- quantization and bound nothing.
- """
+ A rotating cache cannot be quantized, so with kv_bits an enforceable pin becomes a budget."""
pinned = _positive_int(max_seq_length) is not None
# Tri-state: True bounded, False confirmed unbounded, None unjudgeable. Only True installs a bound; the other
# two stay apart so a client can tell them apart.
@@ -2815,14 +2918,16 @@ class MLXInferenceBackend:
_normalize_mlx_kv_bits(kv_bits),
self._model,
is_vlm,
- pinned and enforceable,
+ starts_at_first_token = pinned and enforceable,
)
- if not enforceable or (quant["kv_bits"] is not None and not pinned):
- # No bound installed, so False wherever the probe answered at all; None only where nothing could be built
- # to judge.
- return quant, None, None if confirmed is None else False
- logger.info("MLX KV cache limited to %d tokens", int(served))
- return quant, int(served), True
+ if quant["kv_bits"] is None and enforceable:
+ logger.info("MLX KV cache limited to %d tokens", int(served))
+ return quant, int(served), True, None
+ # No bound installed: False wherever the probe answered, None where it could not.
+ budget = int(served) if quant["kv_bits"] is not None and pinned and enforceable else None
+ if budget:
+ logger.info("MLX context limited to %d tokens per request, not by the cache", budget)
+ return quant, None, (None if confirmed is None else False), budget
def load_model(
self,
@@ -2947,9 +3052,12 @@ class MLXInferenceBackend:
self._served_context = _served_ctx
# Classify before the first generation: an ineligible cache would otherwise raise inside
# maybe_quantize_kv_cache mid-stream, after converting the leading entries.
- self._kv_quant, self._kv_cache_window, _ctx_enforced = self._resolve_kv_policy(
- is_vision, kv_bits, max_seq_length, _served_ctx
- )
+ (
+ self._kv_quant,
+ self._kv_cache_window,
+ _ctx_enforced,
+ self._kv_context_budget,
+ ) = self._resolve_kv_policy(is_vision, kv_bits, max_seq_length, _served_ctx)
if self._kv_quant["kv_bits"] is not None:
logger.info(
"MLX KV cache quantization: %s-bit (%s eligibility)",
@@ -3029,6 +3137,7 @@ class MLXInferenceBackend:
# unbounded, None nothing could be built to judge. Without it the API reports a limit a client cannot tell
# from an enforced one.
"context_length_enforced": _ctx_enforced,
+ "mlx_context_budget": self._kv_context_budget,
"mlx_kv_bits": self._kv_quant["kv_bits"],
"mlx_kv_bits_requested": self._kv_quant["requested_kv_bits"],
"mlx_kv_quant_eligibility": self._kv_quant["eligibility"],
@@ -3575,8 +3684,7 @@ class MLXInferenceBackend:
prompt_tokens,
cached_n,
) = self._prepare_prompt_cache(prompt, _adapter_state)
- if max_new_tokens is None:
- max_new_tokens = self._unset_generation_budget(prompt, prompt_tokens)
+ max_new_tokens = self._generation_limit(prompt, max_new_tokens, prompt_tokens)
logger.info(
"Generating: prompt_len=%d, cached=%d, max_tokens=%d, model=%s, tokenizer=%s",
len(prompt),
@@ -3921,12 +4029,23 @@ class MLXInferenceBackend:
# Matched on the sampled text, for the reason _generate_text gives.
sequences = _mlx_stop_sequences(stop)
stopped = False
- if max_new_tokens is None:
- max_new_tokens = self._unset_generation_budget(prompt)
- if images or video is not None:
- # Media expands past its placeholder (a clip by a frame each), so the counted
- # prompt is short: cap at the default, under the cache window.
- max_new_tokens = min(max_new_tokens, UNSET_GENERATION_BUDGET)
+ # Counted from a temp file (base64 is unreadable to the processor), discarded here.
+ counted_clip = (
+ _write_video_clip(video)
+ if video is not None and getattr(self, "_kv_context_budget", None)
+ else None
+ )
+ try:
+ max_new_tokens = self._generation_limit(
+ prompt,
+ max_new_tokens,
+ images = images,
+ videos = [counted_clip] if counted_clip is not None else None,
+ cap = UNSET_GENERATION_BUDGET if images or video is not None else None,
+ )
+ finally:
+ if counted_clip is not None:
+ _discard_video_clip(counted_clip)
logger.info(
"VLM generating: prompt_len=%d, images=%d, has_video=%s",
len(prompt),
@@ -4231,11 +4350,12 @@ class MLXInferenceBackend:
"cannot build an audio prompt."
)
- if max_new_tokens is None:
- # Audio expands past its placeholder token exactly as an image does, so the counted
- # prompt is short of the real one: cap at the default, but stay under the window.
- max_new_tokens = min(self._unset_generation_budget(prompt), UNSET_GENERATION_BUDGET)
-
+ max_new_tokens = self._generation_limit(
+ prompt,
+ max_new_tokens,
+ audio = [audio_array],
+ cap = UNSET_GENERATION_BUDGET,
+ )
logger.info("MLX audio-input generating: prompt_len=%d", len(prompt))
markers = detect_reasoning_channel_markers(self._processor)
normalizer = make_reasoning_normalizer(markers) if markers is not None else None
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 815532208be..2424e5b6a79 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -22,6 +22,7 @@ from io import BytesIO
from pathlib import Path
from typing import Any, Callable, Generator, Mapping, Optional, Sequence, Tuple, Union
from core.inference.audio_device import audio_device_forces_cpu
+from core.inference.context_refusal import ContextBudgetExceeded
from core.inference.native_audio import NATIVE_AUDIO_TYPES, is_native_audio_model
from core.inference.audio_errors import (
AUDIO_UNSUPPORTED_CODE,
@@ -368,6 +369,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
"max_context_length": model_info.get("max_context_length"),
"requested_context_length": model_info.get("requested_context_length"),
"context_length_enforced": model_info.get("context_length_enforced"),
+ "mlx_context_budget": model_info.get("mlx_context_budget"),
}
@@ -1425,6 +1427,12 @@ class InferenceOrchestrator:
stats_holder["stats"] = resp.get("stats")
return
elif rtype == "gen_error":
+ _budget = resp.get("context_budget")
+ if _budget:
+ # Rebuilt rather than yielded as text: the route arms match on the type.
+ raise ContextBudgetExceeded(
+ _budget["request_tokens"], _budget["context_tokens"]
+ )
yield GenStreamError(
f"Error: {resp.get('error', 'Unknown error')}",
public = bool(resp.get("public", False)),
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index f45dd68e249..502ffc914f4 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -27,6 +27,7 @@ from typing import Any
logger = get_logger(__name__)
from core.inference.audio_errors import AUDIO_UNSUPPORTED_CODE
+from core.inference.context_refusal import ContextBudgetExceeded
from utils.hardware import apply_gpu_ids, is_apple_silicon
# Fresh spawned interpreter: re-apply the OS-trust-store injection.
@@ -541,6 +542,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"native_context_length",
"max_context_length",
"requested_context_length",
+ "mlx_context_budget",
):
try:
_ctx_value = _entry.get(_ctx_field)
@@ -815,18 +817,7 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
except Exception as exc:
logger.error("Generation error: %s", exc, exc_info = True)
- _send_response(
- resp_queue,
- {
- "type": "gen_error",
- "request_id": request_id,
- "error": str(exc),
- # Client-safe refusals would otherwise reach the caller as a generic 500.
- "public": bool(getattr(exc, "public", False)),
- "openai_param": getattr(exc, "openai_param", None),
- "stack": traceback.format_exc(limit = 20),
- },
- )
+ _send_response(resp_queue, _generation_error_payload(request_id, exc))
def _handle_count_tokens(backend, cmd: dict, resp_queue: Any) -> None:
@@ -1059,17 +1050,26 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_eve
except Exception as exc:
logger.error("Audio input generation error: %s", exc, exc_info = True)
- _send_response(
- resp_queue,
- {
- "type": "gen_error",
- "request_id": request_id,
- "error": str(exc),
- "public": bool(getattr(exc, "public", False)),
- "openai_param": getattr(exc, "openai_param", None),
- "stack": traceback.format_exc(limit = 20),
- },
- )
+ _send_response(resp_queue, _generation_error_payload(request_id, exc))
+
+
+def _generation_error_payload(request_id, exc) -> dict:
+ """Carries a context refusal's counts so the parent can rebuild the typed error."""
+ payload = {
+ "type": "gen_error",
+ "request_id": request_id,
+ "error": str(exc),
+ # Client-safe refusals would otherwise reach the caller as a generic 500.
+ "public": bool(getattr(exc, "public", False)),
+ "openai_param": getattr(exc, "openai_param", None),
+ "stack": traceback.format_exc(limit = 20),
+ }
+ if isinstance(exc, ContextBudgetExceeded):
+ payload["context_budget"] = {
+ "request_tokens": exc.request_tokens,
+ "context_tokens": exc.context_tokens,
+ }
+ return payload
def _handle_unload(backend, cmd: dict, resp_queue: Any) -> None:
@@ -1370,15 +1370,10 @@ def run_inference_process(
)
except Exception as exc:
logger.error("MLX command error (%s): %s", cmd_type, exc)
- _send_response(
- resp_queue,
- {
- "type": "gen_error" if cmd_type == "generate" else "error",
- "request_id": cmd.get("request_id"),
- "error": str(exc),
- "stack": traceback.format_exc(limit = 20),
- },
- )
+ _payload = _generation_error_payload(cmd.get("request_id"), exc)
+ if cmd_type != "generate":
+ _payload["type"] = "error"
+ _send_response(resp_queue, _payload)
return
# Windows Triton check, ahead of the torchao stub below, matching the training and export workers' gate-then-stub
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index fb2ac7c5305..9053d4a3ce9 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1226,6 +1226,14 @@ class _InferenceRuntimeFields(BaseModel):
"to check, since a model with its own make_cache ignores the requested size."
),
)
+ mlx_context_budget: Optional[int] = Field(
+ None,
+ description = (
+ "Token limit enforced per request rather than by the KV cache, set where a "
+ "requested cache width took the cache and context_length could not bound it. "
+ "A request over it is refused; null where context_length bounds the cache itself."
+ ),
+ )
supports_reasoning: bool = Field(
False,
description = "Whether model supports thinking/reasoning mode (enable_thinking or reasoning_effort)",
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 1e9a9053c2e..4dc130b2bd2 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -296,6 +296,8 @@ _OVERSIZE_TOKENS_RE = _re.compile(
def _friendly_error(exc: Exception) -> str:
"""Extract a user-friendly message from known llama-server errors."""
+ if isinstance(exc, context_refusal.ContextBudgetExceeded):
+ return str(exc)
if isinstance(exc, httpx.ReadTimeout):
if "stopped producing tokens" in str(exc).lower():
return (
@@ -811,6 +813,19 @@ def _openai_stream_error_chunk(exc) -> dict:
return openai_error_body(_friendly_error(exc), status = 500)
+def _context_budget_http_error(exc) -> "HTTPException":
+ """Context refusal as the same 400 llama.cpp returns (raised before any token)."""
+ return HTTPException(
+ status_code = 400,
+ detail = openai_error_body(
+ _friendly_error(exc),
+ status = 400,
+ code = "context_length_exceeded",
+ param = "messages",
+ ),
+ )
+
+
def _openai_stream_error_sse(error: dict) -> str:
# Every OpenAI-family streaming error flows through here after the 200 headers, so flag
# the response failed: the middleware must not claim a preview-owned model on a failed stream.
@@ -1569,6 +1584,8 @@ def _classify_llama_generation_error(exc: Exception) -> Optional[bool]:
# explanation says "context window" while making the point that the window is
# SHARED, so the heuristic below would read it as an overflow and set the
# client compacting a conversation that was never too long.
+ if isinstance(exc, context_refusal.ContextBudgetExceeded):
+ return True
if isinstance(exc, LlamaStreamError):
# Only an oversize refusal is an overflow. Everything else stays None, which
# keeps it a 500: KV starvation is server capacity exhaustion and an in-band
@@ -7575,6 +7592,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
mlx_kv_quant_eligibility = None,
mlx_kv_quant_reason = None,
mlx_kv_quant_note = None,
+ mlx_context_budget = None,
chat_template_override_reason = None,
# llama.cpp allocates the window it reports: bounded by construction.
context_length_enforced = True,
@@ -16596,6 +16614,7 @@ async def _load_model_impl(
),
max_context_length = _positive_int_or_none(_model_info.get("max_context_length")),
context_length_enforced = _model_info.get("context_length_enforced"),
+ mlx_context_budget = _model_info.get("mlx_context_budget"),
chat_template = _chat_template,
)
@@ -17404,6 +17423,7 @@ async def _load_model_impl(
native_context_length = _positive_int_or_none(_model_info.get("native_context_length")),
max_context_length = _positive_int_or_none(_model_info.get("max_context_length")),
context_length_enforced = _model_info.get("context_length_enforced"),
+ mlx_context_budget = _model_info.get("mlx_context_budget"),
chat_template = _chat_template,
)
@@ -19846,6 +19866,7 @@ async def get_status(current_subject: str):
native_context_length = _positive_int_or_none(model_info.get("native_context_length")),
max_context_length = _positive_int_or_none(model_info.get("max_context_length")),
context_length_enforced = model_info.get("context_length_enforced"),
+ mlx_context_budget = model_info.get("mlx_context_budget"),
# 0 is an answer (size it yourself); None means no request is recorded. Either
# spelling: the route stamps max_seq_length_requested on every non-GGUF load,
# and the MLX mirror carries requested_context_length.
@@ -26158,6 +26179,9 @@ async def produce_openai_chat_completions(
cancel_event.set()
api_monitor.finish(monitor_id, "cancelled")
raise
+ except context_refusal.ContextBudgetExceeded as e:
+ api_monitor.fail(monitor_id, _friendly_error(e))
+ yield _openai_stream_error_sse(_openai_stream_error_chunk(e))
except Exception as e:
logger.error(f"Error during audio input streaming: {e}", exc_info = True)
_msg = _friendly_error(e)
@@ -26214,6 +26238,9 @@ async def produce_openai_chat_completions(
raise
except HTTPException:
raise
+ except context_refusal.ContextBudgetExceeded as e:
+ api_monitor.fail(monitor_id, _friendly_error(e))
+ raise _context_budget_http_error(e)
except Exception as e:
api_monitor.fail(monitor_id, _friendly_error(e))
raise
@@ -29041,6 +29068,10 @@ async def produce_openai_chat_completions(
_msg = _friendly_gen_stream_error(exc)
api_monitor.fail(monitor_id, _msg)
yield _openai_stream_error_sse({"error": {"message": _msg, "type": "server_error"}})
+ except context_refusal.ContextBudgetExceeded as e:
+ backend.reset_generation_state(cancel_event)
+ api_monitor.fail(monitor_id, _friendly_error(e))
+ yield _openai_stream_error_sse(_openai_stream_error_chunk(e))
except Exception:
backend.reset_generation_state(cancel_event)
# Generic wire message; full trace stays in the log (CWE-209:
@@ -29184,6 +29215,10 @@ async def produce_openai_chat_completions(
backend.reset_generation_state(cancel_event)
api_monitor.fail(monitor_id, str(exc.detail))
raise
+ except context_refusal.ContextBudgetExceeded as e:
+ backend.reset_generation_state(cancel_event)
+ api_monitor.fail(monitor_id, _friendly_error(e))
+ raise _context_budget_http_error(e)
except Exception:
backend.reset_generation_state(cancel_event)
# CWE-209: generic detail; full trace in log.
@@ -29681,6 +29716,10 @@ async def produce_openai_chat_completions(
if _refused
else {"error": {"message": _msg, "type": "server_error"}}
)
+ except context_refusal.ContextBudgetExceeded as e:
+ backend.reset_generation_state(cancel_event)
+ api_monitor.fail(monitor_id, _friendly_error(e))
+ yield _openai_stream_error_sse(_openai_stream_error_chunk(e))
except Exception as e:
backend.reset_generation_state(cancel_event)
logger.error(f"Error during OpenAI streaming: {e}", exc_info = True)
@@ -29993,6 +30032,10 @@ async def produce_openai_chat_completions(
),
)
raise HTTPException(status_code = 500, detail = _msg)
+ except context_refusal.ContextBudgetExceeded as e:
+ backend.reset_generation_state(cancel_event)
+ api_monitor.fail(monitor_id, _friendly_error(e))
+ raise _context_budget_http_error(e)
except Exception as e:
backend.reset_generation_state(cancel_event)
logger.error(f"Error during OpenAI completion: {e}", exc_info = True)
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index 6b860d51ce2..5dfff397478 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1058,6 +1058,7 @@ class TestLoadHubDownloadExclusion:
"mlx_kv_quant_eligibility",
"mlx_kv_quant_reason",
"mlx_kv_quant_note",
+ "mlx_context_budget",
"chat_template_override_reason",
# Constant True: llama.cpp allocates the window it reports.
"context_length_enforced",
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 72034ea6559..f748e300798 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -4528,39 +4528,233 @@ def test_the_load_policy_bounds_a_pin_only_where_the_bound_can_be_enforced(monke
backend = MLXInferenceBackend()
backend._model = model
backend._is_vlm = False
- quant, window, enforced = backend._resolve_kv_policy(
+ quant, window, enforced, budget = backend._resolve_kv_policy(
False, kv_bits, requested, requested if served is unset else served
)
- return quant["kv_bits"], window, enforced
+ return quant["kv_bits"], window, enforced, budget
- assert policy(honours, 4, 8192) == (None, 8192, True) # enforceable pin outranks quant
- # Auto yields to quantization, and a window that bounds nothing says so.
- assert policy(honours, 4, 0, 262144) == (4, None, False)
- assert policy(honours, None, 8192) == (None, 8192, True)
- assert policy(honours, None, 0, 262144) == (None, 262144, True) # nothing to yield to
+ assert policy(honours, 4, 8192) == (4, None, False, 8192)
+ assert policy(honours, 4, 0, 262144) == (4, None, False, None)
+ assert policy(honours, None, 8192) == (None, 8192, True, None)
+ assert policy(honours, None, 0, 262144) == (None, 262144, True, None) # nothing to yield to
- # An unenforceable pin spends nothing, and the verdict still travels to the API.
- assert policy(owns, 4, 8192) == (4, None, False)
- assert policy(owns, None, 8192) == (None, None, False)
+ assert policy(owns, 4, 8192) == (4, None, False, None)
+ assert policy(owns, None, 8192) == (None, None, False, None)
# A window nobody could read bounds nothing rather than bounding at zero.
- assert policy(honours, None, 0, None) == (None, None, False) # unreadable
- assert policy(honours, None, 0, 0) == (None, None, False) # non-positive
+ assert policy(honours, None, 0, None) == (None, None, False, None) # unreadable
+ assert policy(honours, None, 0, 0) == (None, None, False, None) # non-positive
# A shape the probe cannot judge is null, not a confirmed "not enforced".
unreadable = SimpleNamespace(layers = [object()], make_cache = lambda: None)
- assert policy(unreadable, None, 8192) == (None, None, None)
+ assert policy(unreadable, None, 8192) == (None, None, None, None)
+
+ # Under a budget quantization starts at the first token, not mlx-vlm's later start.
+ backend = MLXInferenceBackend()
+ backend._model = honours
+ budgeted, _, _, budget = backend._resolve_kv_policy(True, 4, 4096, 4096)
+ unpinned, _, _, _ = backend._resolve_kv_policy(True, 4, 0, 262144)
+ assert budget == 4096 and budgeted["note"] == "" and unpinned["note"]
+ backend._kv_quant, backend._kv_context_budget = budgeted, budget
+ assert backend._kv_quant_generate_kwargs() == {"kv_bits": 4, "quantized_kv_start": 0}
+ backend._kv_context_budget = None
+ assert backend._kv_quant_generate_kwargs() == {"kv_bits": 4}
+
+
+def _drive_vlm_generation(
+ backend,
+ monkeypatch,
+ image = None,
+ max_new_tokens = 1,
+ session = None,
+ video = None,
+):
+ import sys
+ import types
+
+ seen = {}
+ mlx_vlm = types.ModuleType("mlx_vlm")
+ mlx_vlm.prompt_utils = SimpleNamespace(
+ MODEL_CONFIG = {}, apply_chat_template = lambda *_a, **_k: "prompt"
+ )
+
+ def _stream(*_a, **kwargs):
+ seen.clear()
+ seen.update(kwargs)
+ yield SimpleNamespace(text = "ok", prompt_tokens = 3, generation_tokens = 1)
+
+ mlx_vlm.stream_generate = _stream
+ monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm)
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",
+ lambda *_a, **_k: "prompt",
+ )
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.detect_think_prefill", lambda *_a, **_k: ""
+ )
+ backend._model = SimpleNamespace(config = {"model_type": "gemma3"})
+ backend._processor = SimpleNamespace(tokenizer = SimpleNamespace())
+ backend._is_vlm = True
+ monkeypatch.setattr(backend, "_release_vlm_snapshots", lambda: None)
+ monkeypatch.setattr(backend, "_vlm_prompt_cache_session", lambda *_a, **_k: session)
+ list(
+ backend.generate_chat_response(
+ messages = [{"role": "user", "content": "hi"}],
+ image = image,
+ video = video,
+ max_new_tokens = max_new_tokens,
+ )
+ )
+ return seen
+
+
+def _stub_prepare_inputs(monkeypatch, per_medium = 520):
+ import sys
+ import types
+
+ seen = {}
+
+ def _prepare_inputs(
+ _processor,
+ *,
+ prompts,
+ images = None,
+ audio = None,
+ videos = None,
+ **_kwargs,
+ ):
+ media = [*(images or ()), *(audio or ()), *(videos or ())]
+ seen.clear()
+ seen.update(_kwargs)
+ return {"input_ids": SimpleNamespace(size = len(prompts.split()) + per_medium * len(media))}
+
+ utils = types.ModuleType("mlx_vlm.utils")
+ utils.prepare_inputs = _prepare_inputs
+ utils.seen = seen
+ monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)
+ return utils
+
+
+def _vision_backend(budget = 1024, window = 1024):
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ backend = MLXInferenceBackend()
+ backend._model = SimpleNamespace(config = SimpleNamespace(model_type = "qwen2_5_vl"))
+ backend._processor = object()
+ backend._is_vlm = True
+ backend._tokenizer = SimpleNamespace(encode = lambda text, **_k: [0] * len(text.split()))
+ backend._served_context = window
+ backend._kv_context_budget = budget
+ return backend
+
+
+def test_the_context_budget_counts_the_request_in_tokens_the_processor_would_produce(monkeypatch):
+ """Media must be counted expanded: text alone is 7 tokens vs 582 for one image."""
+ from core.inference import context_refusal
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ utils = _stub_prepare_inputs(monkeypatch)
+ backend = _vision_backend()
+ prompt = "placeholder describe this picture"
+
+ assert backend._count_prompt_tokens(prompt) == 4
+ assert backend._count_prompt_tokens(prompt, [object()]) == 524
+ assert backend._count_prompt_tokens(prompt, [object(), object()]) == 1044
+
+ backend._check_context_budget(prompt, 16)
+ backend._check_context_budget(prompt, 16, images = [object()])
+ with pytest.raises(context_refusal.ContextBudgetExceeded):
+ backend._check_context_budget(prompt, 16, images = [object(), object()])
+ with pytest.raises(context_refusal.ContextBudgetExceeded):
+ backend._check_context_budget(prompt, 16, audio = [object(), object()])
+ monkeypatch.setattr("core.inference.mlx_inference._video_frame_rate", lambda *_a: 1.5)
+ with pytest.raises(context_refusal.ContextBudgetExceeded):
+ backend._check_context_budget(prompt, 16, videos = ["a.mp4", "b.mp4"])
+ with pytest.raises(context_refusal.ContextBudgetExceeded):
+ backend._check_context_budget(prompt, 16, images = [object()], videos = ["a.mp4"])
+ # Counted at generation's fps, not the processor's default rate.
+ backend._check_context_budget(prompt, 16, videos = ["a.mp4"])
+ assert utils.seen["fps"] == 1.5
+
+ sized = backend._generation_limit(prompt, None, images = [object()])
+ assert sized == 1024 - 524
+ backend._generation_limit(prompt, sized, images = [object()])
+ with pytest.raises(context_refusal.ContextBudgetExceeded):
+ backend._generation_limit(prompt, 16, images = [object(), object()])
+ with pytest.raises(context_refusal.ContextBudgetExceeded):
+ backend._generation_limit(prompt, 16, videos = ["a.mp4", "b.mp4"])
+
+ def _unpreparable(*_a, **_k):
+ raise RuntimeError("no processor")
+
+ utils.prepare_inputs = _unpreparable
+ assert backend._count_prompt_tokens(prompt, [object(), object()]) == 4
+ backend._check_context_budget(prompt, 16, images = [object(), object()])
+
+
+def test_the_vlm_entry_point_hands_the_limit_a_clip_the_processor_can_read(monkeypatch):
+ """A video is counted from a readable file, not the route's base64."""
+ import base64
+ import os
+
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ monkeypatch.setattr("core.inference.mlx_inference._mlx_vlm_decodes_video", lambda: True)
+ backend = MLXInferenceBackend()
+ backend._kv_context_budget = 1024
+ seen = {}
+
+ def _limit(*_a, **kwargs):
+ seen.update(kwargs)
+ seen["on_disk"] = [os.path.exists(clip) for clip in kwargs.get("videos") or ()]
+ return 1
+
+ monkeypatch.setattr(backend, "_generation_limit", _limit)
+ clip = base64.b64encode(b"not a readable clip").decode()
+ # Past the limit a stub path fails setup; the counted clip must still be discarded.
+ with contextlib.suppress(Exception):
+ _drive_vlm_generation(backend, monkeypatch, video = clip)
+ assert seen["videos"] != [clip]
+ assert seen["on_disk"] == [True]
+ assert not os.path.exists(seen["videos"][0])
+
+
+def test_media_is_prepared_only_under_a_budget_and_only_once(monkeypatch):
+ from core.inference.mlx_inference import UNSET_GENERATION_BUDGET
+
+ utils = _stub_prepare_inputs(monkeypatch)
+ calls = []
+ real = utils.prepare_inputs
+ utils.prepare_inputs = lambda *a, **k: calls.append(1) or real(*a, **k)
+ prompt = "placeholder describe this picture"
+
+ unbudgeted = _vision_backend(budget = None, window = 8192)
+ sized = unbudgeted._generation_limit(
+ prompt, None, images = [object()], cap = UNSET_GENERATION_BUDGET
+ )
+ assert calls == []
+ assert sized == min(8192 - 4, UNSET_GENERATION_BUDGET)
+
+ budgeted = _vision_backend()
+ assert budgeted._generation_limit(prompt, None, images = [object()]) == 1024 - 524
+ assert calls == [1]
-def test_quantization_is_refused_for_a_pinned_context_rather_than_raising_mid_stream():
- """Left to mlx-lm this surfaces as NotImplementedError on the first generated token."""
- from core.inference.mlx_inference import _kv_quant_status
+def test_a_max_tokens_at_the_whole_window_is_the_chat_saying_no_cap(monkeypatch):
+ """Max Tokens at the window means "as much as fits", not a request for that many."""
+ from core.inference import context_refusal
- status = _kv_quant_status(4, None, False, context_pinned = True)
+ _stub_prepare_inputs(monkeypatch)
+ backend = _vision_backend()
+ prompt = "placeholder describe this picture"
- assert status["kv_bits"] is None
- assert status["eligibility"] == "refused"
- assert "quantize a limited cache" in status["reason"]
+ assert backend._generation_limit(prompt, 1024) == 1024 - 4
+ assert backend._generation_limit(prompt, 1024, images = [object()]) == 1024 - 524
+ assert backend._generation_limit(prompt, 16) == 16
+ assert backend._generation_limit(prompt, 2048) == 1024 - 4
+ assert backend._generation_limit(prompt, 500, range(700)) == 1024 - 700
+ with pytest.raises(context_refusal.ContextBudgetExceeded):
+ backend._generation_limit(prompt, 1, range(1024))
def test_the_bound_is_checked_on_a_real_cache_at_the_size_that_was_asked_for():
diff --git a/studio/backend/tests/test_stream_errors.py b/studio/backend/tests/test_stream_errors.py
index cfbe75bcf89..d62c635c8ba 100644
--- a/studio/backend/tests/test_stream_errors.py
+++ b/studio/backend/tests/test_stream_errors.py
@@ -15,6 +15,8 @@ token context:
stream failed".
"""
+import pytest
+
from core.inference.stream_errors import (
KV_STARVATION_MESSAGE,
LlamaStreamError,
@@ -156,6 +158,136 @@ class TestSurvivesTheRouteLayer:
# An overflow genuinely is one, so the client should compact here.
assert routes._classify_llama_generation_error(error) is True
+ def test_a_pre_generation_context_refusal_is_a_client_error_not_a_server_failure(self):
+ from core.inference.context_refusal import ContextBudgetExceeded
+ from utils.utils import safe_error_detail
+
+ routes = self._routes()
+ refusal = ContextBudgetExceeded(8193, 8192)
+
+ assert routes._classify_llama_generation_error(refusal) is True
+ unworded = ContextBudgetExceeded(8193, 8192)
+ unworded.args = ("a phrasing the substring test cannot read",)
+ assert routes._classify_llama_generation_error(unworded) is True
+
+ assert routes._friendly_error(refusal) == str(refusal)
+ assert safe_error_detail(refusal) == str(refusal)
+
+ blocked = routes._context_budget_http_error(refusal)
+ assert blocked.status_code == 400
+ assert blocked.detail["error"]["code"] == "context_length_exceeded"
+ chunk = routes._openai_stream_error_chunk(refusal)
+ assert chunk["error"]["code"] == "context_length_exceeded"
+ assert "8192-token context window" in chunk["error"]["message"]
+
+ @staticmethod
+ def _raiser(exc):
+ def _generate(**_kwargs):
+ raise exc
+ yield # pragma: no cover -- generator form, never reached
+
+ return _generate
+
+ def test_a_context_refusal_survives_the_worker_boundary_as_itself(self):
+ from types import SimpleNamespace
+
+ from core.inference import orchestrator as orch
+ from core.inference import worker as worker_mod
+ from core.inference.context_refusal import ContextBudgetExceeded
+
+ _raiser = self._raiser
+
+ def emitted(exc):
+ sent = []
+ backend = SimpleNamespace(
+ generate_chat_response = _raiser(exc),
+ last_generation_stats = None,
+ )
+ worker_mod._handle_generate(
+ backend,
+ {"request_id": "r1", "messages": [{"role": "user", "content": "hi"}]},
+ SimpleNamespace(put = sent.append),
+ None,
+ )
+ return [r for r in sent if r.get("type") == "gen_error"]
+
+ payloads = emitted(ContextBudgetExceeded(9000, 8192))
+ assert len(payloads) == 1
+ payload = payloads[0]
+ assert payload["context_budget"] == {"request_tokens": 9000, "context_tokens": 8192}
+ assert "context_budget" not in emitted(ValueError("boom"))[0]
+
+ consume = orch.InferenceOrchestrator._consume_token_stream
+ stream = consume(
+ SimpleNamespace(_proc = None, _resp_queue = None, _mark_worker_started = lambda _event: None),
+ lambda _timeout: payload,
+ lambda: None,
+ crash_context = "test",
+ )
+ with pytest.raises(ContextBudgetExceeded) as excinfo:
+ list(stream)
+ assert excinfo.value.request_tokens == 9000
+ assert excinfo.value.context_tokens == 8192
+ assert self._routes()._classify_llama_generation_error(excinfo.value) is True
+
+ def test_the_admission_budget_survives_the_worker_boundary_too(self, monkeypatch):
+ from types import SimpleNamespace
+
+ from core.inference import orchestrator as orch
+ from core.inference import worker as worker_mod
+
+ monkeypatch.setattr(
+ worker_mod,
+ "_build_model_config",
+ lambda _config: SimpleNamespace(
+ identifier = "org/Model",
+ display_name = "Model",
+ is_vision = True,
+ is_lora = False,
+ base_model = None,
+ audio_type = None,
+ is_audio = False,
+ has_audio_input = False,
+ ),
+ )
+ monkeypatch.setattr(worker_mod, "_run_security_gates", lambda *_a, **_k: True)
+ monkeypatch.setattr(
+ "core.inference.native_audio.native_audio_security_targets", lambda *_a, **_k: []
+ )
+ monkeypatch.setattr(
+ "utils.hf_xet_fallback.start_watchdog",
+ lambda **_k: SimpleNamespace(set = lambda: None),
+ )
+ backend = SimpleNamespace(
+ device = "mlx",
+ active_model_name = "org/Model",
+ models = {"org/Model": {"context_length": 8192, "mlx_context_budget": 8192}},
+ load_model = lambda **_k: True,
+ )
+ sent = []
+ worker_mod._handle_load(
+ backend,
+ {
+ "model_name": "org/Model",
+ "load_in_4bit": False,
+ "trust_remote_code": True,
+ "max_seq_length": 8192,
+ },
+ SimpleNamespace(put = sent.append),
+ )
+ loaded = [r for r in sent if r.get("type") == "loaded"]
+ assert loaded and loaded[0]["success"] is True, sent
+ assert loaded[0]["model_info"]["mlx_context_budget"] == 8192
+
+ mirrored = orch._mirrored_model_entry(
+ {"context_length": 8192, "context_length_enforced": False, "mlx_context_budget": 8192},
+ "org/Model",
+ )
+ assert mirrored["mlx_context_budget"] == 8192
+ assert (
+ orch._mirrored_model_entry({"context_length": 8192}, "m")["mlx_context_budget"] is None
+ )
+
def test_an_unrelated_error_survives_verbatim(self):
routes = self._routes()
assert routes._friendly_error(self._error("tokenizer failed")) == "tokenizer failed"
diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py
index 3eb872b7f47..4e22b32cd6e 100644
--- a/studio/backend/utils/utils.py
+++ b/studio/backend/utils/utils.py
@@ -773,8 +773,14 @@ def safe_error_detail(error: Exception, fallback: str = "An internal error occur
# A mid-stream llama-server failure carries a message that was written to be shown; without this the non-streaming paths reduced it to the fallback while streaming clients got the cause. Imported lazily: utils is low level and must not depend on core.inference at import time.
try:
from core.inference.stream_errors import LlamaStreamError # noqa: PLC0415
+
if isinstance(error, LlamaStreamError) and error.friendly:
return error.friendly
+ # Same reason: a context refusal is built for the user and names what to change.
+ from core.inference.context_refusal import ContextBudgetExceeded # noqa: PLC0415
+
+ if isinstance(error, ContextBudgetExceeded):
+ return str(error)
except Exception: # noqa: BLE001 -- fall through to the generic mapping below
pass
text = str(error).lower()
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index d6568ceeaa9..1278c45a0a9 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -2403,6 +2403,7 @@ const VISIBLE_MODEL_RUNTIME_KEYS = [
"loadedMlxKvBitsRequested",
"mlxKvQuantReason",
"mlxKvQuantNote",
+ "loadedContextBudget",
"loadedIsMultimodal",
"loadedIsDiffusion",
"speculativeType",
diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx
index 1898efefc3b..70d49b7afc3 100644
--- a/studio/frontend/src/features/chat/chat-page.tsx
+++ b/studio/frontend/src/features/chat/chat-page.tsx
@@ -2303,6 +2303,9 @@ export function ChatPage({
);
const contextUsage = useChatRuntimeStore((state) => state.contextUsage);
const loadedIsGguf = useChatRuntimeStore((state) => state.loadedIsGguf);
+ const loadedContextBudget = useChatRuntimeStore(
+ (state) => state.loadedContextBudget,
+ );
const loadedContextEnforced = useChatRuntimeStore(
(state) => state.loadedContextEnforced,
);
@@ -4256,6 +4259,7 @@ export function ChatPage({
platformChatOnlyReason,
)}
contextEnforced={loadedContextEnforced}
+ contextBudget={loadedContextBudget}
className="h-[var(--studio-chat-control-height,34px)]"
/>
) : null}
diff --git a/studio/frontend/src/features/chat/components/context-usage-bar.tsx b/studio/frontend/src/features/chat/components/context-usage-bar.tsx
index 7748e4fb503..31cbb6af18a 100644
--- a/studio/frontend/src/features/chat/components/context-usage-bar.tsx
+++ b/studio/frontend/src/features/chat/components/context-usage-bar.tsx
@@ -33,6 +33,7 @@ export const ContextUsageBar: FC<
completionTokens,
isMlx,
contextEnforced,
+ contextBudget,
className,
}) => {
const state = deriveContextUsageBar({
@@ -44,6 +45,7 @@ export const ContextUsageBar: FC<
completionTokens,
isMlx,
contextEnforced,
+ contextBudget,
});
if (!state) return null;
@@ -144,6 +146,15 @@ export const ContextUsageBar: FC<
<span className="font-medium">Context Length</span> in the
chat Settings panel to fit the whole conversation.
</>
+ ) : advice === "mlx-refuses-past-limit" ? (
+ <>
+ Close to the context limit. This model quantizes its cache
+ instead of capping it, so{" "}
+ <span className="font-medium">Context Length</span> is applied
+ to each request rather than to the cache: past it the request
+ is refused instead of the chat slowing down. Increase it in
+ the chat Settings panel, or shorten the conversation.
+ </>
) : advice === "unenforced-limit" ? (
<>
This model builds its own cache, so{" "}
diff --git a/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts b/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts
index d607511ba4b..08bc92a68e8 100644
--- a/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts
+++ b/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts
@@ -27,6 +27,7 @@ export type ContextUsageBarInput = {
isMlx?: boolean;
/** context_length_enforced as the load reported it; null where it does not answer. */
contextEnforced?: boolean | null;
+ contextBudget?: number | null;
};
/**
@@ -43,6 +44,7 @@ export type ContextLimitAdvice =
| "stops-at-limit"
| "mlx-near-limit"
| "mlx-past-limit"
+ | "mlx-refuses-past-limit"
| "unenforced-limit";
function contextLimitAdvice(
@@ -50,8 +52,11 @@ function contextLimitAdvice(
total: number,
isMlx: boolean | undefined,
enforced: boolean | null | undefined,
+ budget: number | null | undefined,
): ContextLimitAdvice {
if ((used / total) * 100 <= 85) return "none";
+ // Budget wins over contextEnforced: the cache is unbounded but requests are refused.
+ if (budget) return "mlx-refuses-past-limit";
// A window the backend confirmed does not bound the cache is not a limit at all:
// nothing rotates and nothing stops, so neither of the other two is true of it. An
// unjudged MLX window says the same thing operationally: the probe could not build a
@@ -83,6 +88,7 @@ export function deriveContextUsageBar({
completionTokens,
isMlx,
contextEnforced,
+ contextBudget,
}: ContextUsageBarInput): ContextUsageBarState | null {
const limit = typeof total === "number" && total > 0 ? total : null;
const usedTokens =
@@ -127,6 +133,12 @@ export function deriveContextUsageBar({
totalRowValue: `${formatTokenCountFull(usedTokens)} / ${formatTokenCountFull(limit)}`,
percent: Math.min((usedTokens / limit) * 100, 100),
hasUsageDetails,
- advice: contextLimitAdvice(usedTokens, limit, isMlx, contextEnforced),
+ advice: contextLimitAdvice(
+ usedTokens,
+ limit,
+ isMlx,
+ contextEnforced,
+ contextBudget,
+ ),
};
}
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index acf69bd5cc7..4848a8c4248 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -2292,6 +2292,7 @@ type ChatRuntimeStore = {
/** Whether loadedContextLength actually bounds the cache. Null where the backend does
* not answer, which is not the same as a confirmed false. */
loadedContextEnforced: boolean | null;
+ loadedContextBudget: number | null;
modelRequiresTrustRemoteCode: boolean;
supportsReasoning: boolean;
reasoningAlwaysOn: boolean;
@@ -4102,6 +4103,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
loadedIsGguf: null,
loadedIsMlx: null,
loadedContextEnforced: null,
+ loadedContextBudget: null,
modelRequiresTrustRemoteCode: false,
supportsReasoning: false,
reasoningAlwaysOn: false,
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index 9fffeccf493..0141ea6f327 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -205,6 +205,7 @@ export function loadedContextFields(resp: {
native_context_length?: number | null;
max_context_length?: number | null;
context_length_enforced?: boolean | null;
+ mlx_context_budget?: number | null;
} | null): {
loadedContextLength: number | null;
maxContextLength: number | null;
@@ -212,6 +213,7 @@ export function loadedContextFields(resp: {
loadedIsGguf: boolean | null;
loadedIsMlx: boolean | null;
loadedContextEnforced: boolean | null;
+ loadedContextBudget: number | null;
} {
if (!resp) {
return {
@@ -221,6 +223,7 @@ export function loadedContextFields(resp: {
loadedIsGguf: null,
loadedIsMlx: null,
loadedContextEnforced: null,
+ loadedContextBudget: null,
};
}
const isGguf = resp.is_gguf ?? false;
@@ -234,6 +237,7 @@ export function loadedContextFields(resp: {
loadedIsGguf: false,
loadedIsMlx: resp.is_mlx ?? null,
loadedContextEnforced: null,
+ loadedContextBudget: null,
};
}
return {
@@ -247,6 +251,7 @@ export function loadedContextFields(resp: {
// llama.cpp allocates what it reports, so GGUF is enforced by construction.
// Everything else answers for itself, or says nothing.
loadedContextEnforced: isGguf ? true : (resp.context_length_enforced ?? null),
+ loadedContextBudget: isGguf ? null : (resp.mlx_context_budget ?? null),
};
}
diff --git a/studio/frontend/tests/mlx-context-helpers.test.ts b/studio/frontend/tests/mlx-context-helpers.test.ts
index 377c1a663d0..d0cd8bfd426 100644
--- a/studio/frontend/tests/mlx-context-helpers.test.ts
+++ b/studio/frontend/tests/mlx-context-helpers.test.ts
@@ -61,6 +61,7 @@ test("an MLX response carries a window without a native one", () => {
loadedIsGguf: false,
loadedIsMlx: true,
loadedContextEnforced: null,
+ loadedContextBudget: null,
});
// Transformers, which sizes nothing, still contributes no window.
assert.deepEqual(loadedContextFields({ is_gguf: false, context_length: 2048 }), {
@@ -70,6 +71,7 @@ test("an MLX response carries a window without a native one", () => {
loadedIsGguf: false,
loadedIsMlx: null,
loadedContextEnforced: null,
+ loadedContextBudget: null,
});
assert.equal(loadedContextFields(null).loadedIsGguf, null);
});
@@ -87,6 +89,31 @@ test("the enforcement verdict is a tri-state, and GGUF is true by construction",
);
});
+test("a limit kept as a per-request budget travels beside the enforcement verdict", () => {
+ const budgeted = loadedContextFields({
+ ...MLX,
+ context_length_enforced: false,
+ mlx_context_budget: 32768,
+ });
+ assert.equal(budgeted.loadedContextBudget, 32768);
+ assert.equal(budgeted.loadedContextEnforced, false);
+ assert.equal(loadedContextFields(MLX).loadedContextBudget, null);
+ assert.equal(loadedContextFields({ ...GGUF, mlx_context_budget: 8192 }).loadedContextBudget, null);
+});
+
+test("a budgeted limit is advised on as a refusal, not as a slowdown or a decoration", () => {
+ const advice = (extra: Record<string, unknown>) =>
+ deriveContextUsageBar({ used: 30000, total: 32768, isMlx: true, ...extra })?.advice;
+
+ assert.equal(advice({ contextEnforced: false, contextBudget: 32768 }), "mlx-refuses-past-limit");
+ assert.equal(advice({ contextEnforced: false }), "unenforced-limit");
+ assert.equal(advice({ contextEnforced: true }), "mlx-near-limit");
+ assert.equal(
+ deriveContextUsageBar({ used: 100, total: 32768, isMlx: true, contextBudget: 32768 })?.advice,
+ "none",
+ );
+});
+
test("a load that reported a non-GGUF backend outranks a stale variant", () => {
assert.equal(isServedByLlamaCpp({ loadedIsGguf: true }), true);
assert.equal(isServedByLlamaCpp({ activeGgufVariant: "Q4_K_M" }), true);