Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

23.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Private loopback llama-server at POST /v1/systemone; Clef answers rebuilt with the PyTorch
formatter so the public response does not depend on the backend; GGUF-only models are checked,
then passed through"""

from __future__ import annotations

import importlib.util
import math
import os
import re
import secrets
import subprocess
import sys
import threading
import time
from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from pathlib import Path
from typing import Any, Callable

import httpx

DEFAULT_CTX = 16384
LOAD_TIMEOUT_S = 1800.0
DECIDE_TIMEOUT_S = 300.0
CLOSE_TIMEOUT_S = 10.0
_POLL_S = 0.1
# llama.cpp server-context.cpp wording for inputs over -ub / -c / -b.
_OVERFLOW = (
    re.compile(
        r"input \((\d+) tokens\) is too large to process\. increase the physical batch size "
        r"\(current batch size: (\d+)\)"
    ),
    re.compile(r"input \((\d+) tokens\) is larger than the max context size \((\d+) tokens\)"),
    re.compile(r"request \((\d+) tokens\) exceeds the available context size \((\d+) tokens\)"),
    re.compile(
        r"the question and its options \((\d+) tokens\) are too large to process\. "
        r"increase the batch size \(current batch size: (\d+)\)"
    ),
)
# server-common.cpp: an image mtmd cannot decode, answered as HTTP 500 by a healthy server.
_UNDECODABLE_MEDIA = "Failed to load image or audio file"
_SERVER_LOG_LINES = 40


class NativeError(RuntimeError):
    """The server failed or is unusable: 503, and the server is retired."""

    status = 503
    error_type = "model_unavailable"
    retire = True


class NativeIncapable(NativeError):
    """This llama-server build cannot serve decision models."""


class NativeLoadError(NativeError):
    """llama-server exited before it finished loading the model."""


class NativeInputError(NativeError):
    """The server refused this request; the server itself is fine."""

    status = 422
    error_type = "invalid_request_error"
    retire = False


class NativeContextOverflow(NativeInputError):
    def __init__(self, tokens: int, limit: int):
        self.tokens, self.limit = tokens, limit
        super().__init__(
            f"State and questions are {tokens} tokens; the llama.cpp backend serves at most {limit}. "
            "Shorten the state or use fewer/shorter criteria."
        )


# A found binary, kept for one settings answer, which would otherwise search for it once per catalog entry.
_FOUND_BINARY: ContextVar = ContextVar("systemone_found_binary", default = None)


@contextmanager
def reuse_found_binary() -> Iterator[None]:
    token = _FOUND_BINARY.set([])
    try:
        yield
    finally:
        _FOUND_BINARY.reset(token)


def resolve_binary() -> str | None:
    found = _FOUND_BINARY.get()
    if found:
        return found[0]
    from core.inference.llama_cpp import LlamaCppBackend

    binary = LlamaCppBackend._find_llama_server_binary()
    binary = (LlamaCppBackend._exec_path_for_launch(binary) or binary) if binary else None
    # Only a hit: a miss may log why, and each lookup still logs it.
    if found is not None and binary is not None:
        found.append(binary)
    return binary


def request_gap(questions: Mapping[str, Any]) -> str | None:
    """Why llama.cpp cannot take these questions (its parser wants 2 to 10 score levels), else None."""
    for name, question in questions.items():
        if question.get("type") == "score" and len(question.get("criteria") or ()) < 2:
            return f'llama.cpp needs at least two levels for score "{name}".'
    return None


def _reference_formatter() -> Callable[[dict, dict], dict]:
    """``systemone_answer`` from the vendored Clef reference, the function the PyTorch path answers with."""
    if (module := sys.modules.get("_unsloth_clef_reference")) is not None:
        return module.systemone_answer
    spec = importlib.util.find_spec("unsloth")
    roots = [Path(p) for p in (spec.submodule_search_locations or ())] if spec else []
    roots.append(Path(__file__).resolve().parents[4] / "unsloth")
    source = next(
        (
            path
            for root in roots
            if (path := root / "_vendor" / "clef" / "joint_schema_model.py").is_file()
        ),
        None,
    )
    if source is None:
        raise NativeError("The vendored Clef reference (unsloth/_vendor/clef) is missing.")
    # By path: importing the unsloth package would patch transformers in the Studio process.
    module_spec = importlib.util.spec_from_file_location("_unsloth_clef_reference", source)
    module = importlib.util.module_from_spec(module_spec)
    sys.modules["_unsloth_clef_reference"] = module
    try:
        module_spec.loader.exec_module(module)
    except BaseException:
        sys.modules.pop("_unsloth_clef_reference", None)
        raise
    return module.systemone_answer


def _probabilities(question: Mapping[str, Any], answer: Any) -> dict[str, float]:
    if not isinstance(answer, Mapping):
        raise ValueError("missing answer")
    kind = question["type"]
    if kind == "noul":
        p = answer.get("noul")
        if isinstance(p, bool) or not isinstance(p, (int, float)):
            raise ValueError("missing noul probability")
        values = {"true": float(p), "false": 1.0 - float(p)}
    else:
        raw = answer.get("probabilities")
        keys = (
            [str(k) for k in question["criteria"]]
            if kind == "choice"
            else [str(i) for i in range(len(question["criteria"]))]
        )
        if not isinstance(raw, Mapping) or set(raw) != set(keys):
            raise ValueError("probabilities do not name the question's options")
        if any(isinstance(v, bool) or not isinstance(v, (int, float)) for v in raw.values()):
            raise ValueError("non-numeric probabilities")
        values = {key: float(raw[key]) for key in keys}
        if not math.isclose(sum(values.values()), 1.0, rel_tol = 1e-4, abs_tol = 1e-4):
            raise ValueError("probabilities do not sum to one")
    if any(not math.isfinite(v) or not -1e-6 <= v <= 1 + 1e-6 for v in values.values()):
        raise ValueError("probabilities out of range")
    return values


def _number(answer: Mapping[str, Any], key: str) -> float:
    value = answer.get(key)
    if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
        raise ValueError(f"missing {key}")
    return float(value)


def _llama_answer(question: Mapping[str, Any], answer: Any) -> dict[str, Any]:
    """llama.cpp's own answer in Studio's shape, after checking it names the question's options."""
    probabilities = _probabilities(question, answer)
    kind = question["type"]
    if kind == "noul":
        return {"type": "noul", "noul": probabilities["true"]}
    if kind == "choice":
        choice = answer.get("choice")
        if choice not in probabilities:
            raise ValueError("choice is not one of the options")
        return {
            "type": "choice",
            "choice": choice,
            "confidence": _number(answer, "confidence"),
            "probabilities": probabilities,
        }
    return {
        "type": "score",
        "score": _number(answer, "score"),
        "confidence": _number(answer, "confidence"),
        "legend": {str(i): c for i, c in enumerate(question["criteria"])},
        "probabilities": probabilities,
    }


def normalise(
    data: Any,
    questions: Mapping[str, Mapping[str, Any]],
    clef_answers: bool = True,
) -> dict[str, Any]:
    """Studio's decision result from a llama.cpp /v1/systemone body; Clef answers rebuilt by the PyTorch formatter."""
    if not isinstance(data, Mapping) or not isinstance(data.get("answers"), Mapping):
        raise NativeError("llama.cpp returned an invalid decision response.")
    systemone_answer = _reference_formatter() if clef_answers else None
    answers = {}
    for name, question in questions.items():
        answer = data["answers"].get(name)
        try:
            if systemone_answer is None:
                answers[name] = _llama_answer(question, answer)
            else:
                answers[name] = systemone_answer(dict(question), _probabilities(question, answer))
        except (TypeError, ValueError, KeyError) as exc:
            raise NativeError(f'llama.cpp returned an invalid answer for "{name}": {exc}') from None
    usage = data.get("usage") if isinstance(data.get("usage"), Mapping) else {}
    tokens = usage.get("input_tokens")
    return {
        "answers": answers,
        "input_tokens": int(tokens)
        if isinstance(tokens, int) and not isinstance(tokens, bool)
        else 0,
        # llama.cpp never cuts a state: an input over its batch is refused (NativeContextOverflow).
        "truncated": False,
    }


def _wire_questions(questions: Mapping[str, Mapping[str, Any]]) -> dict[str, dict[str, Any]]:
    wire = {}
    for name, question in questions.items():
        copied = {k: v for k, v in question.items() if v is not None}
        # The reference encoder reads a missing or empty instruction as the question id; llama.cpp refuses both.
        if question.get("instructions") in (None, ""):
            copied["instructions"] = str(name)
        wire[name] = copied
    return wire


def _detail(response: httpx.Response) -> str:
    try:
        body = response.json()
    except ValueError:
        body = None
    if isinstance(body, Mapping):
        value = body.get("error") or body.get("detail") or body.get("message")
        if isinstance(value, Mapping):
            value = value.get("message") or value.get("detail")
        if isinstance(value, str) and value:
            return value[:1000]
    return response.text.strip()[:1000] or f"HTTP {response.status_code}"


def map_error(status: int, detail: str) -> NativeError:
    """The Studio error for a non-2xx llama.cpp answer."""
    if status not in (401, 403):
        for pattern in _OVERFLOW:
            if match := pattern.search(detail):
                return NativeContextOverflow(int(match.group(1)), int(match.group(2)))
        if 400 <= status < 500 or _UNDECODABLE_MEDIA in detail:
            return NativeInputError(detail)
    return NativeError(f"llama.cpp answered HTTP {status}: {detail}")


def _pick_device(binary: str, env: dict[str, str]) -> str:
    from core.inference.llama_cpp import LlamaCppBackend

    devices = LlamaCppBackend._enumerated_gpu_devices(binary, env)
    if not devices:
        raise NativeError("llama.cpp sees no usable GPU under this Unsloth's device visibility.")
    if len(devices) > 1:
        try:
            free = LlamaCppBackend._get_gpu_free_memory(binary, for_llama_server = True)
        except Exception:
            free = []
        # llama.cpp lists CUDA / ROCm GPUs in mask order; free rows come sorted by physical index.
        # Vulkan ordinals ignore these masks, and HIP on top of ROCR composes, so both keep physical order.
        by_index = dict(free)
        mask = None
        if all(d.startswith("CUDA") for d in devices):
            mask = env.get("CUDA_VISIBLE_DEVICES")
        elif all(d.startswith("ROCm") for d in devices):
            rocr = env.get("ROCR_VISIBLE_DEVICES")
            # HIP falls back to CUDA_VISIBLE_DEVICES when its own mask is unset.
            hip = env.get("HIP_VISIBLE_DEVICES") or env.get("CUDA_VISIBLE_DEVICES")
            mask = None if rocr and hip else rocr or hip
        try:
            order = [int(x) for x in (mask or "").split(",") if x.strip()]
        except ValueError:
            order = []
            if mask and all(d.startswith("CUDA") for d in devices):
                from utils.hardware import nvidia
                order = nvidia.resolve_uuid_mask(mask.strip()) or []
        if len(order) != len(devices):
            order = sorted(by_index)
        if len(order) == len(devices) and all(i in by_index for i in order):
            return devices[max(range(len(order)), key = lambda i: by_index[order[i]])]
    return devices[0]


def _write_key(key: str) -> Path:
    """The server's API key, readable only by this user, under Studio's auth directory."""
    from utils.paths.storage_roots import auth_root

    directory = auth_root()
    directory.mkdir(parents = True, exist_ok = True)
    path = directory / f"decision_llama_api_key_{secrets.token_hex(8)}"
    fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
    with os.fdopen(fd, "w", encoding = "utf-8") as handle:
        handle.write(key)
    return path


class NativeClefAgent:
    """One llama-server serving one decision GGUF; the Decision API keeps it where it keeps a Clef agent."""

    backend = "llama.cpp"

    def __init__(
        self,
        model: Path,
        mmproj: Path | None,
        alias: str,
        *,
        gpu: bool,
        ctx: int = DEFAULT_CTX,
        cancelled: Callable[[], bool] | None = None,
        clef_answers: bool = True,
    ):
        from core.inference.llama_cpp import LlamaCppBackend, _llama_server_key_via_env
        from utils.process_lifetime import (
            adopt_pid,
            child_popen_kwargs,
            is_process_shutting_down,
            spawn_on_lifetime_thread,
        )
        from utils.subprocess_compat import windows_hidden_subprocess_kwargs

        self.alias, self.ctx, self.gpu = alias, int(ctx), gpu
        self.clef_answers = clef_answers
        self._lock = threading.Lock()
        self._process: subprocess.Popen | None = None
        self._client: httpx.Client | None = None
        self.input_modalities: tuple[str, ...] = ()
        binary = resolve_binary()
        if binary is None:
            raise NativeError("llama-server is not installed; Unsloth installs it with llama.cpp.")
        if not Path(model).is_file():
            raise NativeError(f"The decision GGUF is missing: {model}")
        env = LlamaCppBackend._llama_server_env_for_binary(binary)
        LlamaCppBackend._clear_device_placement_env(env)
        if gpu:
            self.device = _pick_device(binary, env)
            placement = ["-ngl", "-1", "--device", self.device]
        else:
            self.device = "cpu"
            env.update(CUDA_VISIBLE_DEVICES = "", HIP_VISIBLE_DEVICES = "-1", ROCR_VISIBLE_DEVICES = "-1")
            # The AMX CPU variant aborts loading Clef in graph_reserve unless weights stay unrepacked.
            placement = ["-ngl", "0", "--device", "none", "--no-repack"]
            if mmproj is not None:
                placement.append("--no-mmproj-offload")
        self.port = LlamaCppBackend._find_free_port()
        self._key = secrets.token_urlsafe(32)
        self._key_file: Path | None = None
        self._log_path: Path | None = None
        if is_process_shutting_down():
            raise NativeError("Unsloth is shutting down; llama.cpp was not started.")
        if _llama_server_key_via_env():
            key_argv = []
            env["LLAMA_API_KEY"] = self._key
        else:
            self._key_file = _write_key(self._key)
            # Through a file, not argv: a command line is readable by every process of this user.
            key_argv = ["--api-key-file", str(self._key_file)]
        self.command = [
            binary,
            "-m",
            str(model),
            *(["--mmproj", str(mmproj)] if mmproj is not None else []),
            "--alias",
            alias,
            "--host",
            "127.0.0.1",
            "--port",
            str(self.port),
            *key_argv,
            "--parallel",
            "1",
            # A decision reads every token in one ubatch: -ub bounds the longest state served.
            "-c",
            str(self.ctx),
            "-b",
            str(self.ctx),
            "-ub",
            str(self.ctx),
            *placement,
        ]
        try:
            self._log_path = self._server_log(self.port)
            log = open(self._log_path, "wb")
        except BaseException:
            self._remove_files()
            raise
        try:
            self._process = spawn_on_lifetime_thread(
                lambda: subprocess.Popen(
                    self.command,
                    stdin = subprocess.DEVNULL,
                    stdout = log,
                    stderr = subprocess.STDOUT,
                    env = env,
                    **windows_hidden_subprocess_kwargs(),
                    **child_popen_kwargs(),
                )
            )
        except BaseException:
            self._remove_files()
            raise
        finally:
            log.close()
        adopt_pid(self._process.pid)
        self._client = httpx.Client(trust_env = False, timeout = DECIDE_TIMEOUT_S)
        try:
            self._wait_ready(cancelled)
            self.input_modalities = self._capabilities()
        except BaseException:
            self.close()
            raise

    @staticmethod
    def _server_log(port: int) -> Path:
        import tempfile

        try:
            from utils.paths.storage_roots import studio_root
            root = studio_root() / "logs"
        except Exception:
            root = Path(tempfile.gettempdir())
        root.mkdir(parents = True, exist_ok = True)
        # Per server: a retired one still exiting must not write into its successor's log.
        return root / f"decision-llama-server-{os.getpid()}-{port}.log"

    def _remove_files(self, keep_log: bool = False) -> None:
        for path in (self._key_file, None if keep_log else self._log_path):
            if path is not None:
                try:
                    path.unlink(missing_ok = True)
                except OSError:
                    pass

    def _log_tail(self) -> str:
        if self._log_path is None:
            return ""
        try:
            lines = self._log_path.read_text(encoding = "utf-8", errors = "replace").splitlines()
        except OSError:
            return ""
        return "\n".join(lines[-_SERVER_LOG_LINES:])

    @property
    def _base(self) -> str:
        return f"http://127.0.0.1:{self.port}"

    @property
    def _headers(self) -> dict[str, str]:
        return {"Authorization": f"Bearer {self._key}"}

    def _wait_ready(self, cancelled: Callable[[], bool] | None) -> None:
        from core.inference.llama_cpp import LlamaCppBackend
        from utils.process_lifetime import is_process_shutting_down

        deadline = time.monotonic() + LOAD_TIMEOUT_S
        while time.monotonic() < deadline:
            if is_process_shutting_down() or (cancelled is not None and cancelled()):
                raise NativeError("The llama.cpp decision server was stopped while it loaded.")
            if not self.is_alive():
                tail = self._log_tail()
                raise NativeLoadError(
                    f"llama.cpp exited while loading {self.alias} (code {self._process.returncode})"
                    + (f":\n{tail}" if tail else ".")
                )
            try:
                response = self._client.get(f"{self._base}/health", timeout = 2.0)
                if response.status_code == 200:
                    if not LlamaCppBackend._health_listener_is_ours(self._process, self.port):
                        raise NativeError(f"Another process holds llama.cpp's port {self.port}.")
                    return
            except httpx.TransportError:
                pass
            time.sleep(_POLL_S)
        raise NativeError(f"llama.cpp did not load {self.alias} within {LOAD_TIMEOUT_S:.0f}s.")

    def _capabilities(self) -> tuple[str, ...]:
        try:
            response = self._client.get(
                f"{self._base}/v1/models", headers = self._headers, timeout = 10.0
            )
            entries = response.json().get("data") if response.status_code == 200 else None
        except (httpx.HTTPError, ValueError, AttributeError):
            entries = None
        for entry in entries if isinstance(entries, list) else ():
            if not isinstance(entry, Mapping):
                continue
            names = {entry.get("id"), *(entry.get("aliases") or ())}
            arch = (
                entry.get("architecture") if isinstance(entry.get("architecture"), Mapping) else {}
            )
            if self.alias in names and "decisions" in (arch.get("output_modalities") or ()):
                return tuple(str(m) for m in arch.get("input_modalities") or ("text",))
        raise NativeIncapable(
            'This llama.cpp build does not serve decision models (no "decisions" output in '
            "/v1/models). Update llama.cpp in Unsloth."
        )

    @property
    def accepts_images(self) -> bool:
        return "image" in self.input_modalities

    def is_alive(self) -> bool:
        return self._process is not None and self._process.poll() is None

    def decide(
        self,
        state: Any,
        questions: dict[str, dict[str, Any]],
        images: list[str] | None = None,
    ) -> dict[str, Any]:
        if gap := request_gap(questions):
            raise NativeInputError(gap)
        if images and not self.accepts_images:
            raise NativeInputError(
                f"{self.alias} has no vision projector, so it cannot read images."
            )
        body = {"model": self.alias, "state": state, "questions": _wire_questions(questions)}
        if images:
            body["images"] = list(images)
        with self._lock:
            client = self._client
            if client is None or not self.is_alive():
                raise NativeError("The llama.cpp decision server is not running.")
            try:
                response = client.post(
                    f"{self._base}/v1/systemone", json = body, headers = self._headers
                )
            except httpx.HTTPError as exc:
                alive = "" if self.is_alive() else " and exited"
                raise NativeError(
                    f"The llama.cpp decision server failed{alive}: {type(exc).__name__}"
                ) from None
        if not 200 <= response.status_code < 300:
            raise map_error(response.status_code, _detail(response))
        try:
            data = response.json()
        except ValueError:
            raise NativeError("llama.cpp returned a decision that is not JSON.") from None
        return normalise(data, questions, self.clef_answers)

    def close(self) -> None:
        """Terminate, then kill, then reap; the pid is forgotten only once it is gone."""
        from utils.process_lifetime import forget_pid

        client, self._client = self._client, None
        if client is not None:
            client.close()
        process = self._process
        if process is None:
            self._remove_files()
            return
        # A server that exited by itself keeps its log for diagnosis.
        crashed = process.poll() is not None
        try:
            if process.poll() is None:
                process.terminate()
                try:
                    process.wait(timeout = CLOSE_TIMEOUT_S)
                except subprocess.TimeoutExpired:
                    process.kill()
                    process.wait(timeout = CLOSE_TIMEOUT_S)
        except (OSError, subprocess.TimeoutExpired):
            pass
        if process.poll() is not None:
            forget_pid(process.pid)
            self._remove_files(keep_log = crashed)