Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Runtime context length helpers shared by inference backends."""
from __future__ import annotations
from collections.abc import Iterator, Mapping
from itertools import chain
from typing import Any, Optional
# LoadRequest.max_seq_length is bounded by this; wider native windows are not served.
MAX_REQUESTABLE_CONTEXT = 1048576
#: Budget for an unset limit when there is no free context to size it from.
UNSET_GENERATION_BUDGET = 2048
def _field(source: Any, name: str) -> Any:
"""Key or attribute, since mlx.nn.Module is a dict; a raiser is absent, never a failed load."""
try:
if isinstance(source, Mapping) and name in source:
return source[name]
return getattr(source, name, None)
except Exception:
return None
def _attached_window(model: Any) -> Any:
"""What Unsloth attached: getattr only, so a Mapping model's parameters cannot pose as it."""
try:
return getattr(model, "max_seq_length", None)
except Exception:
return None
def _declared_context_lengths(model: Any) -> Iterator[Any]:
"""Declared windows, best first. Yields, so an outer 0 / "n/a" cannot shadow a real one."""
holders = (
model,
# config / _config: the spread _mlx_config_field walks in mlx_inference.py.
_field(model, "config"),
_field(model, "_config"),
_field(model, "args"),
)
for holder in holders:
if holder is None:
continue
for source in (holder, _field(holder, "text_config")):
if source is None:
continue
value = _field(source, "max_position_embeddings")
if value is not None:
yield value
def runtime_context_length(model: Any, fallback: Optional[int] = None) -> Optional[int]:
"""Return the effective context length a loaded model runs with."""
# Lazy: a transformers load always has a requested length, so it never reads the config.
candidates = chain(
(_attached_window(model), fallback),
_declared_context_lengths(model),
)
for value in candidates:
if isinstance(value, bool):
continue
try:
value_int = int(value)
# OverflowError: json.loads turns a bare Infinity into float("inf"), which int() rejects.
except (TypeError, ValueError, OverflowError):
continue
if value_int > 0:
return value_int
return None
def generation_budget_for_window(
window: Optional[int], prompt_length: int, max_new_tokens: Optional[int]
) -> Optional[int]:
"""Explicit limits pass through, so an oversized request gets the backend's overflow error."""
if max_new_tokens is not None:
return max_new_tokens
if not window:
return UNSET_GENERATION_BUDGET
free = int(window) - int(prompt_length)
# A floor of 1 would pass the overflow check on a model loaded narrower than its checkpoint.
return free if free > 0 else UNSET_GENERATION_BUDGET
def generation_budget_within_context(
model: Any, prompt_length: int, max_new_tokens: Optional[int]
) -> Optional[int]:
"""``generation_budget_for_window`` against the window a loaded model declares."""
return generation_budget_for_window(
runtime_context_length(model), prompt_length, max_new_tokens
)