Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Maps mid-stream llama-server errors to KV-cache exhaustion from parallel slots, not a long prompt."""
from typing import Any, Optional
# KV starvation: concurrent generations exhausted the shared cache; this request is not too big.
_STARVATION_MARKERS = (
"context size has been exceeded",
"failed to find free space in the kv cache",
"failed to find a memory slot",
)
# Oversize text is kept verbatim, with both token counts; only the remedy is added.
_OVERSIZE_MARKERS = (
"exceeds the available context size",
"exceeds the context size",
)
# Must avoid the client's isContextLimitError substrings (e.g. "context window") in
# chat-adapter.ts, which would wrongly tell the user to start a new chat.
KV_STARVATION_MESSAGE = (
"The model ran out of context space while generating. This happens when several "
"chats or research runs generate at the same time, because they all draw on one "
"shared pool of context. Try again with fewer running at once, or raise the "
"Context Length in Model settings."
)
# llama-server's tool-call parser rejecting a reply (b11000+ PEG, older parser, non-JSON args); the text is dropped.
_TOOL_PARSE_REJECTIONS = (
"the model produced output that does not match the expected",
"failed to parse input at pos",
"failed to parse tool call arguments as json",
)
_OVERSIZE_HINT = "Raise the Context Length in Model settings, or shorten the request."
_GENERIC_MESSAGE = "The model stopped generating early."
class LlamaStreamError(RuntimeError):
"""A mid-stream llama-server failure, carrying a message already fit to show.
Typed rather than a bare ``RuntimeError`` for two reasons, both in
``routes/inference.py``:
- ``_friendly_error`` ends with ``return "An internal error occurred"`` for any
exception it does not recognise. A bare RuntimeError would have that message
swapped out on the chat path, so the cause would still never reach the user
even though it now survives the stream loop.
- ``_classify_llama_generation_error`` flags an overflow by substring, matching
"context" beside "length"/"window". The starvation text above says "context"
while explaining that the context is shared, and names the Context Length
setting as the remedy, so it would be classified as ``context_length_exceeded``
and set the client compacting a conversation that was never too long.
``kv_starvation`` lets that path opt out explicitly.
"""
def __init__(
self,
friendly: str,
*,
server_message: Optional[str] = None,
kv_starvation: bool = False,
context_oversize: bool = False,
):
# Keep the server text so _friendly_error's token-count regex still matches oversize refusals.
super().__init__(server_message or friendly)
self.friendly = friendly
self.server_message = server_message
self.kv_starvation = kv_starvation
self.context_oversize = context_oversize
def stream_error_from_chunk(chunk: Any) -> Optional[LlamaStreamError]:
"""A raisable error for a streamed chunk, or None when the chunk is not one."""
message = error_message_from_chunk(chunk)
if message is None:
return None
starvation = is_kv_starvation(message)
return LlamaStreamError(
describe_stream_error(message),
server_message = message or None,
kv_starvation = starvation,
context_oversize = is_context_oversize(message),
)
def error_message_from_chunk(chunk: Any) -> Optional[str]:
"""An error key of any other shape returns the empty string, not None, since it is still an error."""
if not isinstance(chunk, dict) or "error" not in chunk:
return None
error = chunk["error"]
if isinstance(error, str):
return error.strip()
if isinstance(error, dict):
message = error.get("message")
if isinstance(message, str):
return message.strip()
return ""
def is_kv_starvation(message: Optional[str]) -> bool:
"""Whether the error is the shared-KV starvation case rather than an oversize request."""
if not message:
return False
lowered = message.lower()
return any(marker in lowered for marker in _STARVATION_MARKERS)
def is_context_oversize(message: Optional[str]) -> bool:
"""Whether the server refused the request because it alone did not fit."""
if not message:
return False
lowered = message.lower()
return any(marker in lowered for marker in _OVERSIZE_MARKERS)
def is_tool_parse_rejection(chunk: Any) -> bool:
message = error_message_from_chunk(chunk)
return bool(message) and message.lower().startswith(_TOOL_PARSE_REJECTIONS)
def describe_stream_error(message: Optional[str], *, prefix: str = "") -> str:
"""Starvation gets its own text; oversize keeps server token counts; other errors pass through unchanged, since a fixed string made them undiagnosable."""
if is_kv_starvation(message):
body = KV_STARVATION_MESSAGE
elif is_context_oversize(message):
body = f"{message.rstrip('. ')}. {_OVERSIZE_HINT}"
elif message:
body = message
else:
body = _GENERIC_MESSAGE
return f"{prefix}: {body}" if prefix else body