Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

5.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Maps mid-stream llama-server errors to KV-cache exhaustion from parallel slots, not a long prompt."""

from typing import Any, Optional

# KV starvation: concurrent generations exhausted the shared cache; this request is not too big.
_STARVATION_MARKERS = (
    "context size has been exceeded",
    "failed to find free space in the kv cache",
    "failed to find a memory slot",
)

# Oversize text is kept verbatim, with both token counts; only the remedy is added.
_OVERSIZE_MARKERS = (
    "exceeds the available context size",
    "exceeds the context size",
)

# Must avoid the client's isContextLimitError substrings (e.g. "context window") in
# chat-adapter.ts, which would wrongly tell the user to start a new chat.
KV_STARVATION_MESSAGE = (
    "The model ran out of context space while generating. This happens when several "
    "chats or research runs generate at the same time, because they all draw on one "
    "shared pool of context. Try again with fewer running at once, or raise the "
    "Context Length in Model settings."
)

# llama-server's tool-call parser rejecting a reply (b11000+ PEG, older parser, non-JSON args); the text is dropped.
_TOOL_PARSE_REJECTIONS = (
    "the model produced output that does not match the expected",
    "failed to parse input at pos",
    "failed to parse tool call arguments as json",
)

_OVERSIZE_HINT = "Raise the Context Length in Model settings, or shorten the request."

_GENERIC_MESSAGE = "The model stopped generating early."


class LlamaStreamError(RuntimeError):
    """A mid-stream llama-server failure, carrying a message already fit to show.

    Typed rather than a bare ``RuntimeError`` for two reasons, both in
    ``routes/inference.py``:

    - ``_friendly_error`` ends with ``return "An internal error occurred"`` for any
      exception it does not recognise. A bare RuntimeError would have that message
      swapped out on the chat path, so the cause would still never reach the user
      even though it now survives the stream loop.
    - ``_classify_llama_generation_error`` flags an overflow by substring, matching
      "context" beside "length"/"window". The starvation text above says "context"
      while explaining that the context is shared, and names the Context Length
      setting as the remedy, so it would be classified as ``context_length_exceeded``
      and set the client compacting a conversation that was never too long.
      ``kv_starvation`` lets that path opt out explicitly.
    """

    def __init__(
        self,
        friendly: str,
        *,
        server_message: Optional[str] = None,
        kv_starvation: bool = False,
        context_oversize: bool = False,
    ):
        # Keep the server text so _friendly_error's token-count regex still matches oversize refusals.
        super().__init__(server_message or friendly)
        self.friendly = friendly
        self.server_message = server_message
        self.kv_starvation = kv_starvation
        self.context_oversize = context_oversize


def stream_error_from_chunk(chunk: Any) -> Optional[LlamaStreamError]:
    """A raisable error for a streamed chunk, or None when the chunk is not one."""
    message = error_message_from_chunk(chunk)
    if message is None:
        return None
    starvation = is_kv_starvation(message)
    return LlamaStreamError(
        describe_stream_error(message),
        server_message = message or None,
        kv_starvation = starvation,
        context_oversize = is_context_oversize(message),
    )


def error_message_from_chunk(chunk: Any) -> Optional[str]:
    """An error key of any other shape returns the empty string, not None, since it is still an error."""
    if not isinstance(chunk, dict) or "error" not in chunk:
        return None
    error = chunk["error"]
    if isinstance(error, str):
        return error.strip()
    if isinstance(error, dict):
        message = error.get("message")
        if isinstance(message, str):
            return message.strip()
    return ""


def is_kv_starvation(message: Optional[str]) -> bool:
    """Whether the error is the shared-KV starvation case rather than an oversize request."""
    if not message:
        return False
    lowered = message.lower()
    return any(marker in lowered for marker in _STARVATION_MARKERS)


def is_context_oversize(message: Optional[str]) -> bool:
    """Whether the server refused the request because it alone did not fit."""
    if not message:
        return False
    lowered = message.lower()
    return any(marker in lowered for marker in _OVERSIZE_MARKERS)


def is_tool_parse_rejection(chunk: Any) -> bool:
    message = error_message_from_chunk(chunk)
    return bool(message) and message.lower().startswith(_TOOL_PARSE_REJECTIONS)


def describe_stream_error(message: Optional[str], *, prefix: str = "") -> str:
    """Starvation gets its own text; oversize keeps server token counts; other errors pass through unchanged, since a fixed string made them undiagnosable."""
    if is_kv_starvation(message):
        body = KV_STARVATION_MESSAGE
    elif is_context_oversize(message):
        body = f"{message.rstrip('. ')}. {_OVERSIZE_HINT}"
    elif message:
        body = message
    else:
        body = _GENERIC_MESSAGE
    return f"{prefix}: {body}" if prefix else body