Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep a pinned context as a request limit instead of refusing KV cache quantization (#11084)
Long Yixing1759e64588d4
diff --git a/studio/backend/core/inference/context_refusal.py b/studio/backend/core/inference/context_refusal.pyindex 6d33bfc8a7c..647f1940fb3 100644--- a/studio/backend/core/inference/context_refusal.py+++ b/studio/backend/core/inference/context_refusal.py@@ -21,6 +21,7 @@ __all__ = ["latest_refusal","describe_oversize","open_slot",+ "ContextBudgetExceeded",]@@ -321,3 +322,12 @@ def describe_unservable_tool_call(head + tried + "Nothing was written. Increase the Context Length in Model settings, "f"or {lever}, then try again.")+++class ContextBudgetExceeded(ValueError):+ """Prompt refused before generation; carries the counts so nothing parses the message."""++ def __init__(self, request_tokens: int, context_tokens: int):+ self.request_tokens = int(request_tokens)+ self.context_tokens = int(context_tokens)+ super().__init__(describe_oversize(self.request_tokens, self.context_tokens))diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 20b3f560d48..d8d7c39cb84 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -14,6 +14,7 @@ import timefrom collections import OrderedDictfrom contextlib import ExitStack, closing, contextmanager, nullcontextfrom typing import Optional, Generator+from core.inference import context_refusalfrom core.inference.message_content import content_to_textfrom core.inference.native_tool_tokens import (NativeToolTokenDecoder,@@ -1385,12 +1386,6 @@ MLX_KV_QUANT_NO_REUSE = (MLX_KV_QUANT_VLM_CACHE_NOTE = ("On vision models, quantization starts once the cache reaches {start} tokens.")-# RotatingKVCache.to_quantized raises and mlx-lm converts from the first token, so the two are resolved here rather-# than failing generation on its first step.-MLX_KV_QUANT_PINNED_CONTEXT = (- "Context Length is set for this model, which limits the KV cache, and the installed "- "mlx-lm cannot quantize a limited cache. Reset it to quantize instead."-)def _kv_entry_nbytes(entry):@@ -1974,7 +1969,7 @@ def _kv_quant_status(requested_bits,model,is_vlm,- context_pinned = False,+ starts_at_first_token = False,):"""Resolve a requested bit width against this model into a status dict."""status = {@@ -1986,18 +1981,13 @@ def _kv_quant_status(}if requested_bits is None:return status- if context_pinned:- status["eligibility"] = "refused"- status["reason"] = MLX_KV_QUANT_PINNED_CONTEXT- logger.info("MLX KV quantization not applied: %s", MLX_KV_QUANT_PINNED_CONTEXT)- return statusverdict, reason, retainable = _kv_quant_eligibility(model, is_vlm, requested_bits)status["eligibility"] = verdictstatus["reason"] = reasonif verdict in ("full", "partial"):status["kv_bits"] = requested_bitsnotes = []- if is_vlm:+ if is_vlm and not starts_at_first_token:notes.append(MLX_KV_QUANT_VLM_CACHE_NOTE.format(start = _vlm_quantized_kv_start()))if not retainable:notes.append(MLX_KV_QUANT_NO_REUSE)@@ -2495,6 +2485,7 @@ class MLXInferenceBackend:# Bound now so a load that fails before installing leaves readers a dict rather than raising.self._kv_quant = _kv_quant_status(None, None, False)self._kv_cache_window = None+ self._kv_context_budget = Noneself._served_context = Noneself._template_override = _template_override_status(None, None, None)[1]@@ -2679,25 +2670,95 @@ class MLXInferenceBackend:self,prompt,prompt_tokens = None,+ images = None,+ audio = None,+ videos = None,):"""Free context for an unset limit, or the default if the prompt cannot be counted."""try:prompt_n = (len(prompt_tokens)if prompt_tokens is not None- else self._count_prompt_tokens(prompt)+ else self._count_prompt_tokens(prompt, images, audio, videos))except Exception as exc:logger.debug("MLX prompt count for an unset budget failed: %s", exc)return UNSET_GENERATION_BUDGETreturn generation_budget_for_window(self._served_context, prompt_n, None)+ def _generation_limit(+ self,+ prompt,+ max_new_tokens,+ prompt_tokens = None,+ images = None,+ audio = None,+ videos = None,+ cap = None,+ ):+ budget = getattr(self, "_kv_context_budget", None)+ if not budget:+ # No budget: size from the placeholder count as before, never preparing the media.+ if max_new_tokens is None:+ max_new_tokens = self._unset_generation_budget(prompt, prompt_tokens)+ if cap is not None:+ max_new_tokens = min(max_new_tokens, cap)+ return max_new_tokens+ # Max Tokens == window is the UI's "no cap"; admitting it as-is would refuse every prompt.+ if max_new_tokens is not None and int(max_new_tokens) == int(budget):+ max_new_tokens = None+ if prompt_tokens is None:+ try:+ prompt_tokens = range(self._count_prompt_tokens(prompt, images, audio, videos))+ except Exception as exc:+ logger.debug("MLX prompt count for the context budget failed: %s", exc)+ if max_new_tokens is None:+ max_new_tokens = self._unset_generation_budget(prompt, prompt_tokens)+ if cap is not None:+ max_new_tokens = min(max_new_tokens, cap)+ if prompt_tokens is not None:+ # Refuse only a prompt that leaves no room; a longer ask is cut at the budget as+ # llama-server does, since callers such as Deep Research size it from an estimate.+ self._check_context_budget(prompt, 1, prompt_tokens)+ max_new_tokens = max(1, min(int(max_new_tokens), int(budget) - len(prompt_tokens)))+ return max_new_tokens+def _kv_quant_generate_kwargs(self):"""Load-time runtime knobs for a generate call, empty when unset. quantized_kv_start is- deliberately not passed: mlx-lm and mlx-vlm ship different defaults (0 and 5000) and each- runtime keeps its own."""+ otherwise left to each runtime, whose defaults differ (0 and 5000); under a budget+ mlx-vlm's could sit above every request the budget admits, and would never be reached."""kv_bits = (getattr(self, "_kv_quant", None) or {}).get("kv_bits")- return {} if kv_bits is None else {"kv_bits": kv_bits}+ if kv_bits is None:+ return {}+ if getattr(self, "_kv_context_budget", None):+ return {"kv_bits": kv_bits, "quantized_kv_start": 0}+ return {"kv_bits": kv_bits}++ def _check_context_budget(+ self,+ prompt,+ max_new_tokens = 0,+ prompt_tokens = None,+ images = None,+ audio = None,+ videos = None,+ ):+ """Refuse a request over the budget; an uncountable prompt is admitted."""+ budget = getattr(self, "_kv_context_budget", None)+ if not budget:+ return+ try:+ prompt_n = (+ len(prompt_tokens)+ if prompt_tokens is not None+ else self._count_prompt_tokens(prompt, images, audio, videos)+ )+ except Exception as exc:+ logger.debug("MLX prompt count for the context budget failed: %s", exc)+ return+ needed = prompt_n + max(0, int(max_new_tokens or 0))+ if needed > budget:+ raise context_refusal.ContextBudgetExceeded(needed, int(budget))def _kv_window_generate_kwargs(self):"""The cache bound for a generation that builds its own cache, empty when unset. Both@@ -2717,10 +2778,60 @@ class MLXInferenceBackend:))- def _count_prompt_tokens(self, prompt):+ def _count_media_prompt_tokens(+ self,+ prompt,+ images = None,+ audio = None,+ videos = None,+ ):+ """Prompt length as the processor expands media (as mlx-vlm's server does), else None."""+ if not (images or audio or videos):+ return None+ try:+ from mlx_vlm.utils import prepare_inputs++ config = getattr(self._model, "config", None)+ read = (+ config.get if isinstance(config, dict) else lambda attr: getattr(config, attr, None)+ )+ kwargs = {}+ if images:+ kwargs["images"] = images+ if audio:+ kwargs["audio"] = audio+ if videos:+ # Same fps as generation, not the processor's default rate.+ kwargs["videos"] = videos+ fps = _video_frame_rate(videos[0], self._processor)+ if fps is not None:+ kwargs["fps"] = fps+ inputs = prepare_inputs(+ self._processor,+ prompts = prompt,+ image_token_index = read("image_token_index"),+ add_special_tokens = _vlm_add_special_tokens(read("model_type"), self._processor),+ **kwargs,+ )+ ids = inputs["input_ids"]+ return int(getattr(ids, "size", 0)) or len(ids)+ except Exception as exc:+ logger.debug("MLX media prompt count failed: %s", exc)+ return None++ def _count_prompt_tokens(+ self,+ prompt,+ images = None,+ audio = None,+ videos = None,+ ):"""Prompt length as generation tokenizes it; vision models follow mlx_vlm's marker rule."""if not self._is_vlm:return len(self._encode_prompt(prompt))+ expanded = self._count_media_prompt_tokens(prompt, images, audio, videos)+ if expanded is not None:+ return expandedmodel_type = getattr(getattr(self._model, "config", None), "model_type", None)add_special = _vlm_add_special_tokens(model_type, self._processor)return len(self._tokenizer.encode(prompt, add_special_tokens = add_special))@@ -2795,17 +2906,9 @@ class MLXInferenceBackend:return enforceddef _resolve_kv_policy(self, is_vlm, kv_bits, max_seq_length, served):- """The quantization status and cache window this load will run with.-- Rotation is what keeps a long conversation inside the window, so nothing here can refuse a- request; the model simply stops attending to the oldest tokens.+ """The quantization status, cache window and per-request budget this load will run with.- Quantization cannot coexist with a bound -- a rotating cache has no conversion, and mlx-lm- converts from the first token -- so an enforceable pin, an explicit instruction about- memory, outranks it. A window the backend chose for itself yields to an explicitly requested- quantization, and so does a pin that cannot be enforced, which would otherwise spend the- quantization and bound nothing.- """+ A rotating cache cannot be quantized, so with kv_bits an enforceable pin becomes a budget."""pinned = _positive_int(max_seq_length) is not None# Tri-state: True bounded, False confirmed unbounded, None unjudgeable. Only True installs a bound; the other# two stay apart so a client can tell them apart.@@ -2815,14 +2918,16 @@ class MLXInferenceBackend:_normalize_mlx_kv_bits(kv_bits),self._model,is_vlm,- pinned and enforceable,+ starts_at_first_token = pinned and enforceable,)- if not enforceable or (quant["kv_bits"] is not None and not pinned):- # No bound installed, so False wherever the probe answered at all; None only where nothing could be built- # to judge.- return quant, None, None if confirmed is None else False- logger.info("MLX KV cache limited to %d tokens", int(served))- return quant, int(served), True+ if quant["kv_bits"] is None and enforceable:+ logger.info("MLX KV cache limited to %d tokens", int(served))+ return quant, int(served), True, None+ # No bound installed: False wherever the probe answered, None where it could not.+ budget = int(served) if quant["kv_bits"] is not None and pinned and enforceable else None+ if budget:+ logger.info("MLX context limited to %d tokens per request, not by the cache", budget)+ return quant, None, (None if confirmed is None else False), budgetdef load_model(self,@@ -2947,9 +3052,12 @@ class MLXInferenceBackend:self._served_context = _served_ctx# Classify before the first generation: an ineligible cache would otherwise raise inside# maybe_quantize_kv_cache mid-stream, after converting the leading entries.- self._kv_quant, self._kv_cache_window, _ctx_enforced = self._resolve_kv_policy(- is_vision, kv_bits, max_seq_length, _served_ctx- )+ (+ self._kv_quant,+ self._kv_cache_window,+ _ctx_enforced,+ self._kv_context_budget,+ ) = self._resolve_kv_policy(is_vision, kv_bits, max_seq_length, _served_ctx)if self._kv_quant["kv_bits"] is not None:logger.info("MLX KV cache quantization: %s-bit (%s eligibility)",@@ -3029,6 +3137,7 @@ class MLXInferenceBackend:# unbounded, None nothing could be built to judge. Without it the API reports a limit a client cannot tell# from an enforced one."context_length_enforced": _ctx_enforced,+ "mlx_context_budget": self._kv_context_budget,"mlx_kv_bits": self._kv_quant["kv_bits"],"mlx_kv_bits_requested": self._kv_quant["requested_kv_bits"],"mlx_kv_quant_eligibility": self._kv_quant["eligibility"],@@ -3575,8 +3684,7 @@ class MLXInferenceBackend:prompt_tokens,cached_n,) = self._prepare_prompt_cache(prompt, _adapter_state)- if max_new_tokens is None:- max_new_tokens = self._unset_generation_budget(prompt, prompt_tokens)+ max_new_tokens = self._generation_limit(prompt, max_new_tokens, prompt_tokens)logger.info("Generating: prompt_len=%d, cached=%d, max_tokens=%d, model=%s, tokenizer=%s",len(prompt),@@ -3921,12 +4029,23 @@ class MLXInferenceBackend:# Matched on the sampled text, for the reason _generate_text gives.sequences = _mlx_stop_sequences(stop)stopped = False- if max_new_tokens is None:- max_new_tokens = self._unset_generation_budget(prompt)- if images or video is not None:- # Media expands past its placeholder (a clip by a frame each), so the counted- # prompt is short: cap at the default, under the cache window.- max_new_tokens = min(max_new_tokens, UNSET_GENERATION_BUDGET)+ # Counted from a temp file (base64 is unreadable to the processor), discarded here.+ counted_clip = (+ _write_video_clip(video)+ if video is not None and getattr(self, "_kv_context_budget", None)+ else None+ )+ try:+ max_new_tokens = self._generation_limit(+ prompt,+ max_new_tokens,+ images = images,+ videos = [counted_clip] if counted_clip is not None else None,+ cap = UNSET_GENERATION_BUDGET if images or video is not None else None,+ )+ finally:+ if counted_clip is not None:+ _discard_video_clip(counted_clip)logger.info("VLM generating: prompt_len=%d, images=%d, has_video=%s",len(prompt),@@ -4231,11 +4350,12 @@ class MLXInferenceBackend:"cannot build an audio prompt.")- if max_new_tokens is None:- # Audio expands past its placeholder token exactly as an image does, so the counted- # prompt is short of the real one: cap at the default, but stay under the window.- max_new_tokens = min(self._unset_generation_budget(prompt), UNSET_GENERATION_BUDGET)-+ max_new_tokens = self._generation_limit(+ prompt,+ max_new_tokens,+ audio = [audio_array],+ cap = UNSET_GENERATION_BUDGET,+ )logger.info("MLX audio-input generating: prompt_len=%d", len(prompt))markers = detect_reasoning_channel_markers(self._processor)normalizer = make_reasoning_normalizer(markers) if markers is not None else Nonediff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 815532208be..2424e5b6a79 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -22,6 +22,7 @@ from io import BytesIOfrom pathlib import Pathfrom typing import Any, Callable, Generator, Mapping, Optional, Sequence, Tuple, Unionfrom core.inference.audio_device import audio_device_forces_cpu+from core.inference.context_refusal import ContextBudgetExceededfrom core.inference.native_audio import NATIVE_AUDIO_TYPES, is_native_audio_modelfrom core.inference.audio_errors import (AUDIO_UNSUPPORTED_CODE,@@ -368,6 +369,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:"max_context_length": model_info.get("max_context_length"),"requested_context_length": model_info.get("requested_context_length"),"context_length_enforced": model_info.get("context_length_enforced"),+ "mlx_context_budget": model_info.get("mlx_context_budget"),}@@ -1425,6 +1427,12 @@ class InferenceOrchestrator:stats_holder["stats"] = resp.get("stats")returnelif rtype == "gen_error":+ _budget = resp.get("context_budget")+ if _budget:+ # Rebuilt rather than yielded as text: the route arms match on the type.+ raise ContextBudgetExceeded(+ _budget["request_tokens"], _budget["context_tokens"]+ )yield GenStreamError(f"Error: {resp.get('error', 'Unknown error')}",public = bool(resp.get("public", False)),diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex f45dd68e249..502ffc914f4 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -27,6 +27,7 @@ from typing import Anylogger = get_logger(__name__)from core.inference.audio_errors import AUDIO_UNSUPPORTED_CODE+from core.inference.context_refusal import ContextBudgetExceededfrom utils.hardware import apply_gpu_ids, is_apple_silicon# Fresh spawned interpreter: re-apply the OS-trust-store injection.@@ -541,6 +542,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:"native_context_length","max_context_length","requested_context_length",+ "mlx_context_budget",):try:_ctx_value = _entry.get(_ctx_field)@@ -815,18 +817,7 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:except Exception as exc:logger.error("Generation error: %s", exc, exc_info = True)- _send_response(- resp_queue,- {- "type": "gen_error",- "request_id": request_id,- "error": str(exc),- # Client-safe refusals would otherwise reach the caller as a generic 500.- "public": bool(getattr(exc, "public", False)),- "openai_param": getattr(exc, "openai_param", None),- "stack": traceback.format_exc(limit = 20),- },- )+ _send_response(resp_queue, _generation_error_payload(request_id, exc))def _handle_count_tokens(backend, cmd: dict, resp_queue: Any) -> None:@@ -1059,17 +1050,26 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_eveexcept Exception as exc:logger.error("Audio input generation error: %s", exc, exc_info = True)- _send_response(- resp_queue,- {- "type": "gen_error",- "request_id": request_id,- "error": str(exc),- "public": bool(getattr(exc, "public", False)),- "openai_param": getattr(exc, "openai_param", None),- "stack": traceback.format_exc(limit = 20),- },- )+ _send_response(resp_queue, _generation_error_payload(request_id, exc))+++def _generation_error_payload(request_id, exc) -> dict:+ """Carries a context refusal's counts so the parent can rebuild the typed error."""+ payload = {+ "type": "gen_error",+ "request_id": request_id,+ "error": str(exc),+ # Client-safe refusals would otherwise reach the caller as a generic 500.+ "public": bool(getattr(exc, "public", False)),+ "openai_param": getattr(exc, "openai_param", None),+ "stack": traceback.format_exc(limit = 20),+ }+ if isinstance(exc, ContextBudgetExceeded):+ payload["context_budget"] = {+ "request_tokens": exc.request_tokens,+ "context_tokens": exc.context_tokens,+ }+ return payloaddef _handle_unload(backend, cmd: dict, resp_queue: Any) -> None:@@ -1370,15 +1370,10 @@ def run_inference_process()except Exception as exc:logger.error("MLX command error (%s): %s", cmd_type, exc)- _send_response(- resp_queue,- {- "type": "gen_error" if cmd_type == "generate" else "error",- "request_id": cmd.get("request_id"),- "error": str(exc),- "stack": traceback.format_exc(limit = 20),- },- )+ _payload = _generation_error_payload(cmd.get("request_id"), exc)+ if cmd_type != "generate":+ _payload["type"] = "error"+ _send_response(resp_queue, _payload)return# Windows Triton check, ahead of the torchao stub below, matching the training and export workers' gate-then-stubdiff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex fb2ac7c5305..9053d4a3ce9 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -1226,6 +1226,14 @@ class _InferenceRuntimeFields(BaseModel):"to check, since a model with its own make_cache ignores the requested size."),)+ mlx_context_budget: Optional[int] = Field(+ None,+ description = (+ "Token limit enforced per request rather than by the KV cache, set where a "+ "requested cache width took the cache and context_length could not bound it. "+ "A request over it is refused; null where context_length bounds the cache itself."+ ),+ )supports_reasoning: bool = Field(False,description = "Whether model supports thinking/reasoning mode (enable_thinking or reasoning_effort)",diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 1e9a9053c2e..4dc130b2bd2 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -296,6 +296,8 @@ _OVERSIZE_TOKENS_RE = _re.compile(def _friendly_error(exc: Exception) -> str:"""Extract a user-friendly message from known llama-server errors."""+ if isinstance(exc, context_refusal.ContextBudgetExceeded):+ return str(exc)if isinstance(exc, httpx.ReadTimeout):if "stopped producing tokens" in str(exc).lower():return (@@ -811,6 +813,19 @@ def _openai_stream_error_chunk(exc) -> dict:return openai_error_body(_friendly_error(exc), status = 500)+def _context_budget_http_error(exc) -> "HTTPException":+ """Context refusal as the same 400 llama.cpp returns (raised before any token)."""+ return HTTPException(+ status_code = 400,+ detail = openai_error_body(+ _friendly_error(exc),+ status = 400,+ code = "context_length_exceeded",+ param = "messages",+ ),+ )++def _openai_stream_error_sse(error: dict) -> str:# Every OpenAI-family streaming error flows through here after the 200 headers, so flag# the response failed: the middleware must not claim a preview-owned model on a failed stream.@@ -1569,6 +1584,8 @@ def _classify_llama_generation_error(exc: Exception) -> Optional[bool]:# explanation says "context window" while making the point that the window is# SHARED, so the heuristic below would read it as an overflow and set the# client compacting a conversation that was never too long.+ if isinstance(exc, context_refusal.ContextBudgetExceeded):+ return Trueif isinstance(exc, LlamaStreamError):# Only an oversize refusal is an overflow. Everything else stays None, which# keeps it a 500: KV starvation is server capacity exhaustion and an in-band@@ -7575,6 +7592,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:mlx_kv_quant_eligibility = None,mlx_kv_quant_reason = None,mlx_kv_quant_note = None,+ mlx_context_budget = None,chat_template_override_reason = None,# llama.cpp allocates the window it reports: bounded by construction.context_length_enforced = True,@@ -16596,6 +16614,7 @@ async def _load_model_impl(),max_context_length = _positive_int_or_none(_model_info.get("max_context_length")),context_length_enforced = _model_info.get("context_length_enforced"),+ mlx_context_budget = _model_info.get("mlx_context_budget"),chat_template = _chat_template,)@@ -17404,6 +17423,7 @@ async def _load_model_impl(native_context_length = _positive_int_or_none(_model_info.get("native_context_length")),max_context_length = _positive_int_or_none(_model_info.get("max_context_length")),context_length_enforced = _model_info.get("context_length_enforced"),+ mlx_context_budget = _model_info.get("mlx_context_budget"),chat_template = _chat_template,)@@ -19846,6 +19866,7 @@ async def get_status(current_subject: str):native_context_length = _positive_int_or_none(model_info.get("native_context_length")),max_context_length = _positive_int_or_none(model_info.get("max_context_length")),context_length_enforced = model_info.get("context_length_enforced"),+ mlx_context_budget = model_info.get("mlx_context_budget"),# 0 is an answer (size it yourself); None means no request is recorded. Either# spelling: the route stamps max_seq_length_requested on every non-GGUF load,# and the MLX mirror carries requested_context_length.@@ -26158,6 +26179,9 @@ async def produce_openai_chat_completions(cancel_event.set()api_monitor.finish(monitor_id, "cancelled")raise+ except context_refusal.ContextBudgetExceeded as e:+ api_monitor.fail(monitor_id, _friendly_error(e))+ yield _openai_stream_error_sse(_openai_stream_error_chunk(e))except Exception as e:logger.error(f"Error during audio input streaming: {e}", exc_info = True)_msg = _friendly_error(e)@@ -26214,6 +26238,9 @@ async def produce_openai_chat_completions(raiseexcept HTTPException:raise+ except context_refusal.ContextBudgetExceeded as e:+ api_monitor.fail(monitor_id, _friendly_error(e))+ raise _context_budget_http_error(e)except Exception as e:api_monitor.fail(monitor_id, _friendly_error(e))raise@@ -29041,6 +29068,10 @@ async def produce_openai_chat_completions(_msg = _friendly_gen_stream_error(exc)api_monitor.fail(monitor_id, _msg)yield _openai_stream_error_sse({"error": {"message": _msg, "type": "server_error"}})+ except context_refusal.ContextBudgetExceeded as e:+ backend.reset_generation_state(cancel_event)+ api_monitor.fail(monitor_id, _friendly_error(e))+ yield _openai_stream_error_sse(_openai_stream_error_chunk(e))except Exception:backend.reset_generation_state(cancel_event)# Generic wire message; full trace stays in the log (CWE-209:@@ -29184,6 +29215,10 @@ async def produce_openai_chat_completions(backend.reset_generation_state(cancel_event)api_monitor.fail(monitor_id, str(exc.detail))raise+ except context_refusal.ContextBudgetExceeded as e:+ backend.reset_generation_state(cancel_event)+ api_monitor.fail(monitor_id, _friendly_error(e))+ raise _context_budget_http_error(e)except Exception:backend.reset_generation_state(cancel_event)# CWE-209: generic detail; full trace in log.@@ -29681,6 +29716,10 @@ async def produce_openai_chat_completions(if _refusedelse {"error": {"message": _msg, "type": "server_error"}})+ except context_refusal.ContextBudgetExceeded as e:+ backend.reset_generation_state(cancel_event)+ api_monitor.fail(monitor_id, _friendly_error(e))+ yield _openai_stream_error_sse(_openai_stream_error_chunk(e))except Exception as e:backend.reset_generation_state(cancel_event)logger.error(f"Error during OpenAI streaming: {e}", exc_info = True)@@ -29993,6 +30032,10 @@ async def produce_openai_chat_completions(),)raise HTTPException(status_code = 500, detail = _msg)+ except context_refusal.ContextBudgetExceeded as e:+ backend.reset_generation_state(cancel_event)+ api_monitor.fail(monitor_id, _friendly_error(e))+ raise _context_budget_http_error(e)except Exception as e:backend.reset_generation_state(cancel_event)logger.error(f"Error during OpenAI completion: {e}", exc_info = True)diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.pyindex 6b860d51ce2..5dfff397478 100644--- a/studio/backend/tests/test_gguf_load_cache_reuse.py+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py@@ -1058,6 +1058,7 @@ class TestLoadHubDownloadExclusion:"mlx_kv_quant_eligibility","mlx_kv_quant_reason","mlx_kv_quant_note",+ "mlx_context_budget","chat_template_override_reason",# Constant True: llama.cpp allocates the window it reports."context_length_enforced",diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 72034ea6559..f748e300798 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -4528,39 +4528,233 @@ def test_the_load_policy_bounds_a_pin_only_where_the_bound_can_be_enforced(monkebackend = MLXInferenceBackend()backend._model = modelbackend._is_vlm = False- quant, window, enforced = backend._resolve_kv_policy(+ quant, window, enforced, budget = backend._resolve_kv_policy(False, kv_bits, requested, requested if served is unset else served)- return quant["kv_bits"], window, enforced+ return quant["kv_bits"], window, enforced, budget- assert policy(honours, 4, 8192) == (None, 8192, True) # enforceable pin outranks quant- # Auto yields to quantization, and a window that bounds nothing says so.- assert policy(honours, 4, 0, 262144) == (4, None, False)- assert policy(honours, None, 8192) == (None, 8192, True)- assert policy(honours, None, 0, 262144) == (None, 262144, True) # nothing to yield to+ assert policy(honours, 4, 8192) == (4, None, False, 8192)+ assert policy(honours, 4, 0, 262144) == (4, None, False, None)+ assert policy(honours, None, 8192) == (None, 8192, True, None)+ assert policy(honours, None, 0, 262144) == (None, 262144, True, None) # nothing to yield to- # An unenforceable pin spends nothing, and the verdict still travels to the API.- assert policy(owns, 4, 8192) == (4, None, False)- assert policy(owns, None, 8192) == (None, None, False)+ assert policy(owns, 4, 8192) == (4, None, False, None)+ assert policy(owns, None, 8192) == (None, None, False, None)# A window nobody could read bounds nothing rather than bounding at zero.- assert policy(honours, None, 0, None) == (None, None, False) # unreadable- assert policy(honours, None, 0, 0) == (None, None, False) # non-positive+ assert policy(honours, None, 0, None) == (None, None, False, None) # unreadable+ assert policy(honours, None, 0, 0) == (None, None, False, None) # non-positive# A shape the probe cannot judge is null, not a confirmed "not enforced".unreadable = SimpleNamespace(layers = [object()], make_cache = lambda: None)- assert policy(unreadable, None, 8192) == (None, None, None)+ assert policy(unreadable, None, 8192) == (None, None, None, None)++ # Under a budget quantization starts at the first token, not mlx-vlm's later start.+ backend = MLXInferenceBackend()+ backend._model = honours+ budgeted, _, _, budget = backend._resolve_kv_policy(True, 4, 4096, 4096)+ unpinned, _, _, _ = backend._resolve_kv_policy(True, 4, 0, 262144)+ assert budget == 4096 and budgeted["note"] == "" and unpinned["note"]+ backend._kv_quant, backend._kv_context_budget = budgeted, budget+ assert backend._kv_quant_generate_kwargs() == {"kv_bits": 4, "quantized_kv_start": 0}+ backend._kv_context_budget = None+ assert backend._kv_quant_generate_kwargs() == {"kv_bits": 4}+++def _drive_vlm_generation(+ backend,+ monkeypatch,+ image = None,+ max_new_tokens = 1,+ session = None,+ video = None,+):+ import sys+ import types++ seen = {}+ mlx_vlm = types.ModuleType("mlx_vlm")+ mlx_vlm.prompt_utils = SimpleNamespace(+ MODEL_CONFIG = {}, apply_chat_template = lambda *_a, **_k: "prompt"+ )++ def _stream(*_a, **kwargs):+ seen.clear()+ seen.update(kwargs)+ yield SimpleNamespace(text = "ok", prompt_tokens = 3, generation_tokens = 1)++ mlx_vlm.stream_generate = _stream+ monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm)+ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",+ lambda *_a, **_k: "prompt",+ )+ monkeypatch.setattr(+ "core.inference.chat_template_helpers.detect_think_prefill", lambda *_a, **_k: ""+ )+ backend._model = SimpleNamespace(config = {"model_type": "gemma3"})+ backend._processor = SimpleNamespace(tokenizer = SimpleNamespace())+ backend._is_vlm = True+ monkeypatch.setattr(backend, "_release_vlm_snapshots", lambda: None)+ monkeypatch.setattr(backend, "_vlm_prompt_cache_session", lambda *_a, **_k: session)+ list(+ backend.generate_chat_response(+ messages = [{"role": "user", "content": "hi"}],+ image = image,+ video = video,+ max_new_tokens = max_new_tokens,+ )+ )+ return seen+++def _stub_prepare_inputs(monkeypatch, per_medium = 520):+ import sys+ import types++ seen = {}++ def _prepare_inputs(+ _processor,+ *,+ prompts,+ images = None,+ audio = None,+ videos = None,+ **_kwargs,+ ):+ media = [*(images or ()), *(audio or ()), *(videos or ())]+ seen.clear()+ seen.update(_kwargs)+ return {"input_ids": SimpleNamespace(size = len(prompts.split()) + per_medium * len(media))}++ utils = types.ModuleType("mlx_vlm.utils")+ utils.prepare_inputs = _prepare_inputs+ utils.seen = seen+ monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)+ return utils+++def _vision_backend(budget = 1024, window = 1024):+ from core.inference.mlx_inference import MLXInferenceBackend++ backend = MLXInferenceBackend()+ backend._model = SimpleNamespace(config = SimpleNamespace(model_type = "qwen2_5_vl"))+ backend._processor = object()+ backend._is_vlm = True+ backend._tokenizer = SimpleNamespace(encode = lambda text, **_k: [0] * len(text.split()))+ backend._served_context = window+ backend._kv_context_budget = budget+ return backend+++def test_the_context_budget_counts_the_request_in_tokens_the_processor_would_produce(monkeypatch):+ """Media must be counted expanded: text alone is 7 tokens vs 582 for one image."""+ from core.inference import context_refusal+ from core.inference.mlx_inference import MLXInferenceBackend++ utils = _stub_prepare_inputs(monkeypatch)+ backend = _vision_backend()+ prompt = "placeholder describe this picture"++ assert backend._count_prompt_tokens(prompt) == 4+ assert backend._count_prompt_tokens(prompt, [object()]) == 524+ assert backend._count_prompt_tokens(prompt, [object(), object()]) == 1044++ backend._check_context_budget(prompt, 16)+ backend._check_context_budget(prompt, 16, images = [object()])+ with pytest.raises(context_refusal.ContextBudgetExceeded):+ backend._check_context_budget(prompt, 16, images = [object(), object()])+ with pytest.raises(context_refusal.ContextBudgetExceeded):+ backend._check_context_budget(prompt, 16, audio = [object(), object()])+ monkeypatch.setattr("core.inference.mlx_inference._video_frame_rate", lambda *_a: 1.5)+ with pytest.raises(context_refusal.ContextBudgetExceeded):+ backend._check_context_budget(prompt, 16, videos = ["a.mp4", "b.mp4"])+ with pytest.raises(context_refusal.ContextBudgetExceeded):+ backend._check_context_budget(prompt, 16, images = [object()], videos = ["a.mp4"])+ # Counted at generation's fps, not the processor's default rate.+ backend._check_context_budget(prompt, 16, videos = ["a.mp4"])+ assert utils.seen["fps"] == 1.5++ sized = backend._generation_limit(prompt, None, images = [object()])+ assert sized == 1024 - 524+ backend._generation_limit(prompt, sized, images = [object()])+ with pytest.raises(context_refusal.ContextBudgetExceeded):+ backend._generation_limit(prompt, 16, images = [object(), object()])+ with pytest.raises(context_refusal.ContextBudgetExceeded):+ backend._generation_limit(prompt, 16, videos = ["a.mp4", "b.mp4"])++ def _unpreparable(*_a, **_k):+ raise RuntimeError("no processor")++ utils.prepare_inputs = _unpreparable+ assert backend._count_prompt_tokens(prompt, [object(), object()]) == 4+ backend._check_context_budget(prompt, 16, images = [object(), object()])+++def test_the_vlm_entry_point_hands_the_limit_a_clip_the_processor_can_read(monkeypatch):+ """A video is counted from a readable file, not the route's base64."""+ import base64+ import os++ from core.inference.mlx_inference import MLXInferenceBackend++ monkeypatch.setattr("core.inference.mlx_inference._mlx_vlm_decodes_video", lambda: True)+ backend = MLXInferenceBackend()+ backend._kv_context_budget = 1024+ seen = {}++ def _limit(*_a, **kwargs):+ seen.update(kwargs)+ seen["on_disk"] = [os.path.exists(clip) for clip in kwargs.get("videos") or ()]+ return 1++ monkeypatch.setattr(backend, "_generation_limit", _limit)+ clip = base64.b64encode(b"not a readable clip").decode()+ # Past the limit a stub path fails setup; the counted clip must still be discarded.+ with contextlib.suppress(Exception):+ _drive_vlm_generation(backend, monkeypatch, video = clip)+ assert seen["videos"] != [clip]+ assert seen["on_disk"] == [True]+ assert not os.path.exists(seen["videos"][0])+++def test_media_is_prepared_only_under_a_budget_and_only_once(monkeypatch):+ from core.inference.mlx_inference import UNSET_GENERATION_BUDGET++ utils = _stub_prepare_inputs(monkeypatch)+ calls = []+ real = utils.prepare_inputs+ utils.prepare_inputs = lambda *a, **k: calls.append(1) or real(*a, **k)+ prompt = "placeholder describe this picture"++ unbudgeted = _vision_backend(budget = None, window = 8192)+ sized = unbudgeted._generation_limit(+ prompt, None, images = [object()], cap = UNSET_GENERATION_BUDGET+ )+ assert calls == []+ assert sized == min(8192 - 4, UNSET_GENERATION_BUDGET)++ budgeted = _vision_backend()+ assert budgeted._generation_limit(prompt, None, images = [object()]) == 1024 - 524+ assert calls == [1]-def test_quantization_is_refused_for_a_pinned_context_rather_than_raising_mid_stream():- """Left to mlx-lm this surfaces as NotImplementedError on the first generated token."""- from core.inference.mlx_inference import _kv_quant_status+def test_a_max_tokens_at_the_whole_window_is_the_chat_saying_no_cap(monkeypatch):+ """Max Tokens at the window means "as much as fits", not a request for that many."""+ from core.inference import context_refusal- status = _kv_quant_status(4, None, False, context_pinned = True)+ _stub_prepare_inputs(monkeypatch)+ backend = _vision_backend()+ prompt = "placeholder describe this picture"- assert status["kv_bits"] is None- assert status["eligibility"] == "refused"- assert "quantize a limited cache" in status["reason"]+ assert backend._generation_limit(prompt, 1024) == 1024 - 4+ assert backend._generation_limit(prompt, 1024, images = [object()]) == 1024 - 524+ assert backend._generation_limit(prompt, 16) == 16+ assert backend._generation_limit(prompt, 2048) == 1024 - 4+ assert backend._generation_limit(prompt, 500, range(700)) == 1024 - 700+ with pytest.raises(context_refusal.ContextBudgetExceeded):+ backend._generation_limit(prompt, 1, range(1024))def test_the_bound_is_checked_on_a_real_cache_at_the_size_that_was_asked_for():diff --git a/studio/backend/tests/test_stream_errors.py b/studio/backend/tests/test_stream_errors.pyindex cfbe75bcf89..d62c635c8ba 100644--- a/studio/backend/tests/test_stream_errors.py+++ b/studio/backend/tests/test_stream_errors.py@@ -15,6 +15,8 @@ token context:stream failed"."""+import pytest+from core.inference.stream_errors import (KV_STARVATION_MESSAGE,LlamaStreamError,@@ -156,6 +158,136 @@ class TestSurvivesTheRouteLayer:# An overflow genuinely is one, so the client should compact here.assert routes._classify_llama_generation_error(error) is True+ def test_a_pre_generation_context_refusal_is_a_client_error_not_a_server_failure(self):+ from core.inference.context_refusal import ContextBudgetExceeded+ from utils.utils import safe_error_detail++ routes = self._routes()+ refusal = ContextBudgetExceeded(8193, 8192)++ assert routes._classify_llama_generation_error(refusal) is True+ unworded = ContextBudgetExceeded(8193, 8192)+ unworded.args = ("a phrasing the substring test cannot read",)+ assert routes._classify_llama_generation_error(unworded) is True++ assert routes._friendly_error(refusal) == str(refusal)+ assert safe_error_detail(refusal) == str(refusal)++ blocked = routes._context_budget_http_error(refusal)+ assert blocked.status_code == 400+ assert blocked.detail["error"]["code"] == "context_length_exceeded"+ chunk = routes._openai_stream_error_chunk(refusal)+ assert chunk["error"]["code"] == "context_length_exceeded"+ assert "8192-token context window" in chunk["error"]["message"]++ @staticmethod+ def _raiser(exc):+ def _generate(**_kwargs):+ raise exc+ yield # pragma: no cover -- generator form, never reached++ return _generate++ def test_a_context_refusal_survives_the_worker_boundary_as_itself(self):+ from types import SimpleNamespace++ from core.inference import orchestrator as orch+ from core.inference import worker as worker_mod+ from core.inference.context_refusal import ContextBudgetExceeded++ _raiser = self._raiser++ def emitted(exc):+ sent = []+ backend = SimpleNamespace(+ generate_chat_response = _raiser(exc),+ last_generation_stats = None,+ )+ worker_mod._handle_generate(+ backend,+ {"request_id": "r1", "messages": [{"role": "user", "content": "hi"}]},+ SimpleNamespace(put = sent.append),+ None,+ )+ return [r for r in sent if r.get("type") == "gen_error"]++ payloads = emitted(ContextBudgetExceeded(9000, 8192))+ assert len(payloads) == 1+ payload = payloads[0]+ assert payload["context_budget"] == {"request_tokens": 9000, "context_tokens": 8192}+ assert "context_budget" not in emitted(ValueError("boom"))[0]++ consume = orch.InferenceOrchestrator._consume_token_stream+ stream = consume(+ SimpleNamespace(_proc = None, _resp_queue = None, _mark_worker_started = lambda _event: None),+ lambda _timeout: payload,+ lambda: None,+ crash_context = "test",+ )+ with pytest.raises(ContextBudgetExceeded) as excinfo:+ list(stream)+ assert excinfo.value.request_tokens == 9000+ assert excinfo.value.context_tokens == 8192+ assert self._routes()._classify_llama_generation_error(excinfo.value) is True++ def test_the_admission_budget_survives_the_worker_boundary_too(self, monkeypatch):+ from types import SimpleNamespace++ from core.inference import orchestrator as orch+ from core.inference import worker as worker_mod++ monkeypatch.setattr(+ worker_mod,+ "_build_model_config",+ lambda _config: SimpleNamespace(+ identifier = "org/Model",+ display_name = "Model",+ is_vision = True,+ is_lora = False,+ base_model = None,+ audio_type = None,+ is_audio = False,+ has_audio_input = False,+ ),+ )+ monkeypatch.setattr(worker_mod, "_run_security_gates", lambda *_a, **_k: True)+ monkeypatch.setattr(+ "core.inference.native_audio.native_audio_security_targets", lambda *_a, **_k: []+ )+ monkeypatch.setattr(+ "utils.hf_xet_fallback.start_watchdog",+ lambda **_k: SimpleNamespace(set = lambda: None),+ )+ backend = SimpleNamespace(+ device = "mlx",+ active_model_name = "org/Model",+ models = {"org/Model": {"context_length": 8192, "mlx_context_budget": 8192}},+ load_model = lambda **_k: True,+ )+ sent = []+ worker_mod._handle_load(+ backend,+ {+ "model_name": "org/Model",+ "load_in_4bit": False,+ "trust_remote_code": True,+ "max_seq_length": 8192,+ },+ SimpleNamespace(put = sent.append),+ )+ loaded = [r for r in sent if r.get("type") == "loaded"]+ assert loaded and loaded[0]["success"] is True, sent+ assert loaded[0]["model_info"]["mlx_context_budget"] == 8192++ mirrored = orch._mirrored_model_entry(+ {"context_length": 8192, "context_length_enforced": False, "mlx_context_budget": 8192},+ "org/Model",+ )+ assert mirrored["mlx_context_budget"] == 8192+ assert (+ orch._mirrored_model_entry({"context_length": 8192}, "m")["mlx_context_budget"] is None+ )+def test_an_unrelated_error_survives_verbatim(self):routes = self._routes()assert routes._friendly_error(self._error("tokenizer failed")) == "tokenizer failed"diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.pyindex 3eb872b7f47..4e22b32cd6e 100644--- a/studio/backend/utils/utils.py+++ b/studio/backend/utils/utils.py@@ -773,8 +773,14 @@ def safe_error_detail(error: Exception, fallback: str = "An internal error occur# A mid-stream llama-server failure carries a message that was written to be shown; without this the non-streaming paths reduced it to the fallback while streaming clients got the cause. Imported lazily: utils is low level and must not depend on core.inference at import time.try:from core.inference.stream_errors import LlamaStreamError # noqa: PLC0415+if isinstance(error, LlamaStreamError) and error.friendly:return error.friendly+ # Same reason: a context refusal is built for the user and names what to change.+ from core.inference.context_refusal import ContextBudgetExceeded # noqa: PLC0415++ if isinstance(error, ContextBudgetExceeded):+ return str(error)except Exception: # noqa: BLE001 -- fall through to the generic mapping belowpasstext = str(error).lower()diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex d6568ceeaa9..1278c45a0a9 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -2403,6 +2403,7 @@ const VISIBLE_MODEL_RUNTIME_KEYS = ["loadedMlxKvBitsRequested","mlxKvQuantReason","mlxKvQuantNote",+ "loadedContextBudget","loadedIsMultimodal","loadedIsDiffusion","speculativeType",diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsxindex 1898efefc3b..70d49b7afc3 100644--- a/studio/frontend/src/features/chat/chat-page.tsx+++ b/studio/frontend/src/features/chat/chat-page.tsx@@ -2303,6 +2303,9 @@ export function ChatPage({);const contextUsage = useChatRuntimeStore((state) => state.contextUsage);const loadedIsGguf = useChatRuntimeStore((state) => state.loadedIsGguf);+ const loadedContextBudget = useChatRuntimeStore(+ (state) => state.loadedContextBudget,+ );const loadedContextEnforced = useChatRuntimeStore((state) => state.loadedContextEnforced,);@@ -4256,6 +4259,7 @@ export function ChatPage({platformChatOnlyReason,)}contextEnforced={loadedContextEnforced}+ contextBudget={loadedContextBudget}className="h-[var(--studio-chat-control-height,34px)]"/>) : null}diff --git a/studio/frontend/src/features/chat/components/context-usage-bar.tsx b/studio/frontend/src/features/chat/components/context-usage-bar.tsxindex 7748e4fb503..31cbb6af18a 100644--- a/studio/frontend/src/features/chat/components/context-usage-bar.tsx+++ b/studio/frontend/src/features/chat/components/context-usage-bar.tsx@@ -33,6 +33,7 @@ export const ContextUsageBar: FC<completionTokens,isMlx,contextEnforced,+ contextBudget,className,}) => {const state = deriveContextUsageBar({@@ -44,6 +45,7 @@ export const ContextUsageBar: FC<completionTokens,isMlx,contextEnforced,+ contextBudget,});if (!state) return null;@@ -144,6 +146,15 @@ export const ContextUsageBar: FC<<span className="font-medium">Context Length</span> in thechat Settings panel to fit the whole conversation.</>+ ) : advice === "mlx-refuses-past-limit" ? (+ <>+ Close to the context limit. This model quantizes its cache+ instead of capping it, so{" "}+ <span className="font-medium">Context Length</span> is applied+ to each request rather than to the cache: past it the request+ is refused instead of the chat slowing down. Increase it in+ the chat Settings panel, or shorten the conversation.+ </>) : advice === "unenforced-limit" ? (<>This model builds its own cache, so{" "}diff --git a/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts b/studio/frontend/src/features/chat/lib/context-usage-bar-state.tsindex d607511ba4b..08bc92a68e8 100644--- a/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts+++ b/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts@@ -27,6 +27,7 @@ export type ContextUsageBarInput = {isMlx?: boolean;/** context_length_enforced as the load reported it; null where it does not answer. */contextEnforced?: boolean | null;+ contextBudget?: number | null;};/**@@ -43,6 +44,7 @@ export type ContextLimitAdvice =| "stops-at-limit"| "mlx-near-limit"| "mlx-past-limit"+ | "mlx-refuses-past-limit"| "unenforced-limit";function contextLimitAdvice(@@ -50,8 +52,11 @@ function contextLimitAdvice(total: number,isMlx: boolean | undefined,enforced: boolean | null | undefined,+ budget: number | null | undefined,): ContextLimitAdvice {if ((used / total) * 100 <= 85) return "none";+ // Budget wins over contextEnforced: the cache is unbounded but requests are refused.+ if (budget) return "mlx-refuses-past-limit";// A window the backend confirmed does not bound the cache is not a limit at all:// nothing rotates and nothing stops, so neither of the other two is true of it. An// unjudged MLX window says the same thing operationally: the probe could not build a@@ -83,6 +88,7 @@ export function deriveContextUsageBar({completionTokens,isMlx,contextEnforced,+ contextBudget,}: ContextUsageBarInput): ContextUsageBarState | null {const limit = typeof total === "number" && total > 0 ? total : null;const usedTokens =@@ -127,6 +133,12 @@ export function deriveContextUsageBar({totalRowValue: `${formatTokenCountFull(usedTokens)} / ${formatTokenCountFull(limit)}`,percent: Math.min((usedTokens / limit) * 100, 100),hasUsageDetails,- advice: contextLimitAdvice(usedTokens, limit, isMlx, contextEnforced),+ advice: contextLimitAdvice(+ usedTokens,+ limit,+ isMlx,+ contextEnforced,+ contextBudget,+ ),};}diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.tsindex acf69bd5cc7..4848a8c4248 100644--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts@@ -2292,6 +2292,7 @@ type ChatRuntimeStore = {/** Whether loadedContextLength actually bounds the cache. Null where the backend does* not answer, which is not the same as a confirmed false. */loadedContextEnforced: boolean | null;+ loadedContextBudget: number | null;modelRequiresTrustRemoteCode: boolean;supportsReasoning: boolean;reasoningAlwaysOn: boolean;@@ -4102,6 +4103,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({loadedIsGguf: null,loadedIsMlx: null,loadedContextEnforced: null,+ loadedContextBudget: null,modelRequiresTrustRemoteCode: false,supportsReasoning: false,reasoningAlwaysOn: false,diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.tsindex 9fffeccf493..0141ea6f327 100644--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts@@ -205,6 +205,7 @@ export function loadedContextFields(resp: {native_context_length?: number | null;max_context_length?: number | null;context_length_enforced?: boolean | null;+ mlx_context_budget?: number | null;} | null): {loadedContextLength: number | null;maxContextLength: number | null;@@ -212,6 +213,7 @@ export function loadedContextFields(resp: {loadedIsGguf: boolean | null;loadedIsMlx: boolean | null;loadedContextEnforced: boolean | null;+ loadedContextBudget: number | null;} {if (!resp) {return {@@ -221,6 +223,7 @@ export function loadedContextFields(resp: {loadedIsGguf: null,loadedIsMlx: null,loadedContextEnforced: null,+ loadedContextBudget: null,};}const isGguf = resp.is_gguf ?? false;@@ -234,6 +237,7 @@ export function loadedContextFields(resp: {loadedIsGguf: false,loadedIsMlx: resp.is_mlx ?? null,loadedContextEnforced: null,+ loadedContextBudget: null,};}return {@@ -247,6 +251,7 @@ export function loadedContextFields(resp: {// llama.cpp allocates what it reports, so GGUF is enforced by construction.// Everything else answers for itself, or says nothing.loadedContextEnforced: isGguf ? true : (resp.context_length_enforced ?? null),+ loadedContextBudget: isGguf ? null : (resp.mlx_context_budget ?? null),};}diff --git a/studio/frontend/tests/mlx-context-helpers.test.ts b/studio/frontend/tests/mlx-context-helpers.test.tsindex 377c1a663d0..d0cd8bfd426 100644--- a/studio/frontend/tests/mlx-context-helpers.test.ts+++ b/studio/frontend/tests/mlx-context-helpers.test.ts@@ -61,6 +61,7 @@ test("an MLX response carries a window without a native one", () => {loadedIsGguf: false,loadedIsMlx: true,loadedContextEnforced: null,+ loadedContextBudget: null,});// Transformers, which sizes nothing, still contributes no window.assert.deepEqual(loadedContextFields({ is_gguf: false, context_length: 2048 }), {@@ -70,6 +71,7 @@ test("an MLX response carries a window without a native one", () => {loadedIsGguf: false,loadedIsMlx: null,loadedContextEnforced: null,+ loadedContextBudget: null,});assert.equal(loadedContextFields(null).loadedIsGguf, null);});@@ -87,6 +89,31 @@ test("the enforcement verdict is a tri-state, and GGUF is true by construction",);});+test("a limit kept as a per-request budget travels beside the enforcement verdict", () => {+ const budgeted = loadedContextFields({+ ...MLX,+ context_length_enforced: false,+ mlx_context_budget: 32768,+ });+ assert.equal(budgeted.loadedContextBudget, 32768);+ assert.equal(budgeted.loadedContextEnforced, false);+ assert.equal(loadedContextFields(MLX).loadedContextBudget, null);+ assert.equal(loadedContextFields({ ...GGUF, mlx_context_budget: 8192 }).loadedContextBudget, null);+});++test("a budgeted limit is advised on as a refusal, not as a slowdown or a decoration", () => {+ const advice = (extra: Record<string, unknown>) =>+ deriveContextUsageBar({ used: 30000, total: 32768, isMlx: true, ...extra })?.advice;++ assert.equal(advice({ contextEnforced: false, contextBudget: 32768 }), "mlx-refuses-past-limit");+ assert.equal(advice({ contextEnforced: false }), "unenforced-limit");+ assert.equal(advice({ contextEnforced: true }), "mlx-near-limit");+ assert.equal(+ deriveContextUsageBar({ used: 100, total: 32768, isMlx: true, contextBudget: 32768 })?.advice,+ "none",+ );+});+test("a load that reported a non-GGUF backend outranks a stale variant", () => {assert.equal(isServedByLlamaCpp({ loadedIsGguf: true }), true);assert.equal(isServedByLlamaCpp({ activeGgufVariant: "Q4_K_M" }), true);