Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

48.2 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Model loading and streaming shared by `inference` and `chat`."""

import json
import os
import re
import sys
from contextlib import contextmanager, redirect_stderr, redirect_stdout
from pathlib import Path
from typing import List, Literal, Optional

import typer

# Mirrors the backend's _CANONICAL_SPEC_MODES; typer validates --speculative-type with it.
SpeculativeType = Literal[
    "auto", "mtp", "dspark", "dflash", "ngram", "mtp+ngram", "off", "ngram-simple"
]

_THINK_OPEN = "<think>"
_THINK_BLOCK = re.compile(rf"{re.escape(_THINK_OPEN)}.*?</think>", re.DOTALL)
_STREAMED_ERROR_PREFIX = "Error: "

# Cloudflare 403s the default urllib User-Agent as a bot.
_USER_AGENT = "unsloth-cli"
_MPI_ENV_PAIRS = (
    ("OMPI_COMM_WORLD_RANK", "OMPI_COMM_WORLD_SIZE"),
    ("PMI_RANK", "PMI_SIZE"),
    ("PMIX_RANK", "PMIX_SIZE"),
    ("MPI_RANK", "MPI_WORLD_SIZE"),
    ("MV2_COMM_WORLD_RANK", "MV2_COMM_WORLD_SIZE"),
)

_openers: dict = {}


def _opener(follow_redirects: bool, loopback: bool):
    key = (follow_redirects, loopback)
    if key not in _openers:
        import urllib.error
        import urllib.request

        class _NoRedirect(urllib.request.HTTPRedirectHandler):
            def redirect_request(self, req, fp, code, msg, headers, newurl):
                raise urllib.error.HTTPError(
                    req.full_url, code, f"refusing redirect to {newurl}", headers, fp
                )

        # urllib proxies 127.0.0.1 unless no_proxy lists it, leaking the bearer key to http_proxy.
        handlers = [urllib.request.ProxyHandler({})] if loopback else []
        if not follow_redirects:
            handlers.append(_NoRedirect)
        _openers[key] = urllib.request.build_opener(*handlers)
    return _openers[key]


def urlopen_no_redirect(request, timeout):
    """urlopen that errors on any redirect: following a 3xx would send a bearer
    token (or accept an identity proof) to a base we never vetted, letting a port
    squatter relay a real Unsloth's response."""
    return _opener(False, is_loopback_url(request.full_url)).open(request, timeout = timeout)


def urlopen_local(request, timeout):
    """urllib.request.urlopen that never sends a loopback request through a proxy."""
    if not is_loopback_url(request.full_url):
        import urllib.request
        return urllib.request.urlopen(request, timeout = timeout)
    return _opener(True, True).open(request, timeout = timeout)


# load and unload pad the body and commit 200 early, so a late failure rides only in-band under this key; a bare 200 reads as success.
_DEFERRED_ERROR_KEY = "_deferred_error"


def raise_for_deferred_error(url: str, body):
    """Raise the late failure a padded 200 body carries; else return ``body``.

    ``urllib.error.HTTPError`` specifically: it is the class every CLI caller already
    handles for a plain HTTP failure, so existing ``except`` blocks, messages and exit
    codes keep working, and ``.read()`` yields the same ``{"detail": ...}`` shape.
    """
    if not isinstance(body, dict):
        return body
    deferred = body.get(_DEFERRED_ERROR_KEY)
    if not isinstance(deferred, dict):
        return body

    import email.message
    import io
    import urllib.error

    status = deferred.get("status_code")
    if not isinstance(status, int) or isinstance(status, bool):
        status = 500
    detail = deferred.get("detail")
    if not isinstance(detail, str) or not detail:
        detail = "unknown error" if detail is None else json.dumps(detail)
    headers = email.message.Message()
    headers["Content-Type"] = "application/json"
    raise urllib.error.HTTPError(
        url, status, detail, headers, io.BytesIO(json.dumps({"detail": detail}).encode())
    )


def require_completed_padded_body(url: str, body):
    """Return ``body``, or raise if it is not the payload a padded route promised.

    A proxy that gives up mid-pad leaves a 200 with an empty or truncated body, so
    accepting it reports an unfinished load or unload as completed. Only the two padded
    routes commit their status that early, so only they require a payload; ``{}`` is
    rejected too, since that is what a blank body decodes to here. Mirrored by
    ``assertCompletedPaddedBody`` in studio/frontend/src/features/chat/api/padded-response.ts.
    """
    if isinstance(body, dict) and body:
        return body
    raise RuntimeError(
        f"{url} did not report completion: the connection closed before the "
        "server's reply arrived. Check the model's status before retrying."
    )


def read_json_checking_deferred_error(url: str, response):
    """Drain ``response``, then raise any deferred error its body carries.

    Draining matters on its own: stopping at the headers of a padded /load leaves the
    load running, so the caller resumes too early. An incomplete JSON payload is a
    truncated padded reply, not a success (see ``require_completed_padded_body``).
    """
    try:
        raw = response.read()
    finally:
        response.close()
    try:
        body = json.loads(raw.decode(errors = "replace") or "{}")
    except ValueError:
        body = None
    return require_completed_padded_body(url, raise_for_deferred_error(url, body))


_cache_env_seeded = False


def _seed_cache_env() -> None:
    """Pin cache locations as the backend does: unsloth_zoo's relative UNSLOTH_COMPILE_LOCATION
    would resolve against the cwd, leaving unsloth_compiled_cache that cache_cleanup misses
    (issue 8865)."""
    global _cache_env_seeded
    if _cache_env_seeded:
        return
    _cache_env_seeded = True
    try:
        from utils.paths.storage_roots import setup_cache_env
        setup_cache_env()
    except Exception:  # noqa: BLE001 - never fail a command over cache placement
        pass


def ensure_studio_backend_path(*, seed_cache_env: bool = True) -> None:
    """Put studio/backend on sys.path, and by default pin the cache locations too.

    `seed_cache_env = False` is for callers that only want to IMPORT a path helper.
    setup_cache_env() creates every cache directory it pins, so seeding it from
    `unsloth start`'s Node discovery turned a read-only lookup into 18 mkdirs under a home
    that may have nothing to do with the command being run -- including one against a remote
    server. Seeding stays on for the ML entry points, where the pins are the point.
    """
    backend_dir = str(Path(__file__).resolve().parents[1] / "studio" / "backend")
    if backend_dir not in sys.path:
        sys.path.insert(0, backend_dir)
    if seed_cache_env:
        # After the path insert, before the caller's backend import pulls in unsloth_zoo.compiler.
        _seed_cache_env()


def configure_quiet_logging() -> None:
    import logging

    # The CLI never configures structlog; LOG_LEVEL is exported so the worker inherits it.
    level_name = os.environ.setdefault("LOG_LEVEL", "WARNING").upper()
    level = getattr(logging, level_name, logging.WARNING)
    os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")

    # Quieting logs must not fail a command before the import that really needs structlog gets to report itself.
    try:
        import structlog
    except ModuleNotFoundError:
        return
    structlog.configure(wrapper_class = structlog.make_filtering_bound_logger(level))


def _parse_nonnegative_int(value: Optional[str]) -> Optional[int]:
    if value is None:
        return None
    try:
        parsed = int(value)
    except (TypeError, ValueError):
        return None
    return parsed if parsed >= 0 else None


def _first_mpi_env_pair() -> tuple[Optional[int], Optional[int]]:
    for rank_name, size_name in _MPI_ENV_PAIRS:
        rank = _parse_nonnegative_int(os.environ.get(rank_name))
        world_size = _parse_nonnegative_int(os.environ.get(size_name))
        if rank is not None and world_size is not None and world_size > 1 and rank < world_size:
            return rank, world_size
    return None, None


def _json_rank_count_from_env(name: str) -> Optional[int]:
    value = os.environ.get(name)
    if not value:
        return None
    try:
        if value.lstrip().startswith(("[", "{")):
            data = json.loads(value)
        else:
            with open(value, "r", encoding = "utf-8") as f:
                data = json.load(f)
    except (json.JSONDecodeError, OSError, UnicodeDecodeError):
        return None
    if isinstance(data, list):
        return len(data)
    if isinstance(data, dict) and isinstance(data.get("hosts"), list):
        return len(data["hosts"])
    return None


def mlx_distributed_info() -> tuple[bool, int, Optional[int]]:
    """Return launch-context metadata without initializing MLX distributed."""
    rank = _parse_nonnegative_int(os.environ.get("MLX_RANK"))
    world_size = _parse_nonnegative_int(os.environ.get("MLX_WORLD_SIZE"))
    if rank is not None:
        if (
            world_size is not None
            and world_size > 1
            and rank < world_size
            and os.environ.get("NCCL_HOST_IP")
            and os.environ.get("NCCL_PORT")
        ):
            return True, rank, world_size
        inferred_size = _json_rank_count_from_env("MLX_HOSTFILE")
        if inferred_size is not None and inferred_size > 1 and rank < inferred_size:
            return True, rank, inferred_size
        inferred_size = _json_rank_count_from_env("MLX_IBV_DEVICES")
        if (
            inferred_size is not None
            and inferred_size > 1
            and rank < inferred_size
            and os.environ.get("MLX_JACCL_COORDINATOR")
        ):
            return True, rank, inferred_size
        return False, 0, None

    mpi_rank, mpi_world_size = _first_mpi_env_pair()
    return mpi_rank is not None, mpi_rank or 0, mpi_world_size


def mlx_distributed_uses_mpi() -> bool:
    """Whether the current distributed context was launched through MPI."""
    return (
        _parse_nonnegative_int(os.environ.get("MLX_RANK")) is None
        and _first_mpi_env_pair()[0] is not None
    )


@contextmanager
def quiet_if_nonzero_mlx_rank():
    """Silence parent and child-process stdout/stderr on nonzero ranks."""
    if mlx_distributed_info()[1] == 0:
        yield
        return

    sys.stdout.flush()
    sys.stderr.flush()
    saved_stdout_fd = os.dup(1)
    saved_stderr_fd = os.dup(2)
    with open(os.devnull, "w", encoding = "utf-8") as devnull:
        try:
            os.dup2(devnull.fileno(), 1)
            os.dup2(devnull.fileno(), 2)
            with redirect_stdout(devnull), redirect_stderr(devnull):
                yield
        finally:
            sys.stdout.flush()
            sys.stderr.flush()
            os.dup2(saved_stdout_fd, 1)
            os.dup2(saved_stderr_fd, 2)
            os.close(saved_stdout_fd)
            os.close(saved_stderr_fd)


def visible_text(
    text: str,
    show_thinking: bool,
    final: bool = False,
) -> str:
    if show_thinking:
        return text
    text = _THINK_BLOCK.sub("", text)
    # Hold back an unclosed trailing <think> so reasoning never leaks mid-stream.
    open_idx = text.find(_THINK_OPEN)
    if open_idx != -1:
        text = text[:open_idx]
    if final:
        # Ended stream: a trailing "<" or "<th" can no longer become <think>.
        return text
    max_prefix = min(len(text), len(_THINK_OPEN) - 1)
    for size in range(max_prefix, 0, -1):
        if _THINK_OPEN.startswith(text[-size:]):
            return text[:-size]
    return text


def stream_to_stdout(stream, show_thinking: bool) -> str:
    # Backends yield the full text-so-far each step (llama.cpp ends with a metadata dict, skipped).
    raw = ""
    shown = ""
    for chunk in stream:
        if not isinstance(chunk, str):
            continue
        raw = chunk
        rendered = visible_text(chunk, show_thinking)
        delta = rendered[len(shown) :]
        if delta:
            sys.stdout.write(delta)
            sys.stdout.flush()
            shown = rendered
    tail = visible_text(raw, show_thinking, final = True)[len(shown) :]
    if tail:
        sys.stdout.write(tail)
    sys.stdout.write("\n")
    sys.stdout.flush()
    return raw


class _LatestMarkdown:
    # visible_text() and Markdown() each scan the whole reply: run them per drawn frame, not per chunk.
    def __init__(self, show_thinking: bool):
        self.state = ("", False)  # (raw, final), swapped whole so the Live thread never sees a mix
        self._show_thinking = show_thinking
        self._built = None

    def __rich_console__(self, console, options):
        from rich.markdown import Markdown
        from rich.text import Text

        state = self.state
        if self._built is None or self._built[0] is not state:
            text = visible_text(state[0], self._show_thinking, final = state[1])
            self._built = (state, Markdown(text) if text.strip() else Text(""))
        yield self._built[1]


def stream_markdown(stream, show_thinking: bool, *, console) -> str:
    from rich.live import Live

    raw = ""
    view = _LatestMarkdown(show_thinking)
    with Live(view, console = console, refresh_per_second = 12, vertical_overflow = "visible"):
        for chunk in stream:
            if isinstance(chunk, str):
                raw = chunk
                view.state = (raw, False)
        view.state = (raw, True)
    return raw


def collect_stream(stream, show_thinking: bool) -> str:
    raw = ""
    for chunk in stream:
        if isinstance(chunk, str):
            raw = chunk
    return visible_text(raw, show_thinking, final = True)


def raise_on_streamed_error(stream):
    # Match backend errors by GenStreamError type, not the Error: text prefix, so completion text opening with Error: is not misread.
    try:
        ensure_studio_backend_path()
        from core.inference.orchestrator import GenStreamError
    except Exception:
        GenStreamError = None
    for chunk in stream:
        if GenStreamError is not None and isinstance(chunk, GenStreamError):
            raise RuntimeError(str(chunk)[len(_STREAMED_ERROR_PREFIX) :].strip() or "Unknown error")
        yield chunk


def render_columns(
    left_label: str,
    left_text: str,
    right_label: str,
    right_text: str,
    *,
    console = None,
) -> None:
    from rich import box
    from rich.console import Console
    from rich.table import Table

    table = Table(box = box.MINIMAL, expand = True, padding = (0, 1), pad_edge = False)
    table.add_column(left_label, header_style = "bold yellow", ratio = 1, overflow = "fold")
    table.add_column(right_label, header_style = "bold magenta", ratio = 1, overflow = "fold")
    table.add_row(left_text or "", right_text or "")
    (console or Console()).print(table)


def stats_hit_token_limit(stats) -> bool:
    """Whether end-of-generation stats show the reply ran out of budget: safetensors sets truncated;
    MLX and llama-server set finish_reason to length."""
    if not isinstance(stats, dict):
        return False
    return bool(stats.get("truncated")) or stats.get("finish_reason") == "length"


_IMAGE_MIME_TYPES = {
    "BMP": "image/bmp",
    "GIF": "image/gif",
    "JPEG": "image/jpeg",
    "MPO": "image/jpeg",
    "PNG": "image/png",
    "TIFF": "image/tiff",
    "WEBP": "image/webp",
}


def image_part(path) -> dict:
    """An OpenAI image_url part carrying the file as a data URL, the shape Studio's chat sends."""
    import base64
    import io

    file = Path(path).expanduser()
    try:
        data = file.read_bytes()
    except OSError as exc:
        raise ValueError(f"can't read image {file}: {exc.strerror or exc}") from None
    # Without Pillow nothing could tell a real picture from a renamed text file.
    try:
        from PIL import Image
    except ImportError:
        raise ValueError("reading images needs Pillow: pip install pillow") from None
    try:
        with Image.open(io.BytesIO(data)) as picture:
            image_format = picture.format
            picture.verify()
    except Exception:
        raise ValueError(f"{file} is not an image file Unsloth can read") from None
    mime = _IMAGE_MIME_TYPES.get(image_format) or Image.MIME.get(image_format, "image/png")
    encoded = base64.b64encode(data).decode("ascii")
    return {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{encoded}"}}


def user_turn(text: str, images: Optional[list] = None) -> dict:
    """A user message; attached images go ahead of the text, as Studio orders them."""
    if not images:
        return {"role": "user", "content": text}
    return {"role": "user", "content": [*images, {"type": "text", "text": text}]}


def _image_payload(part) -> Optional[str]:
    if not isinstance(part, dict) or part.get("type") != "image_url":
        return None
    url = (part.get("image_url") or {}).get("url") or ""
    if not url.startswith("data:"):
        return None
    return url.partition(",")[2] or None


def _local_image_request(messages: list, several: bool):
    """Studio's local route shape: an image marker where each image stood, plus the payloads in order.

    A model that reads one image per conversation gets only the newest turn's, like the app.
    """
    owners = []  # (message index, user ordinal, payloads)
    ordinal = 0
    for index, message in enumerate(messages):
        if message.get("role") != "user":
            continue
        content = message.get("content")
        if isinstance(content, list):
            payloads = [p for p in map(_image_payload, content) if p]
            if payloads:
                owners.append((index, ordinal, payloads))
        ordinal += 1
    if not owners:
        return messages, None, None
    if not several:
        if len(owners[-1][2]) > 1:
            raise ValueError("this model reads one image per message; attach one image at a time")
        owners = owners[-1:]
    kept = {index: len(payloads) for index, _, payloads in owners}
    rebuilt = []
    for index, message in enumerate(messages):
        content = message.get("content")
        if not isinstance(content, list):
            rebuilt.append(message)
            continue
        text = "\n".join(
            part.get("text") or ""
            for part in content
            if isinstance(part, dict) and part.get("type") == "text"
        )
        if index in kept:
            markers = [{"type": "image"} for _ in range(kept[index])]
            rebuilt.append({**message, "content": [*markers, {"type": "text", "text": text}]})
        else:
            rebuilt.append({**message, "content": text})
    images = [payload for _, _, payloads in owners for payload in payloads]
    return rebuilt, images, owners[-1][1]


class ChatBackend:
    """Uniform stream()/close() over the llama-server and Unsloth backends."""

    def __init__(self, kind: str, backend) -> None:
        self._kind = kind  # "gguf" | "unsloth"
        self._backend = backend
        self.reply_hit_token_limit = False

    def _model_entry(self) -> dict:
        backend = self._backend
        return (getattr(backend, "models", None) or {}).get(
            getattr(backend, "active_model_name", None)
        ) or {}

    @property
    def supports_images(self) -> bool:
        if self._kind == "gguf":
            return bool(getattr(self._backend, "is_vision", False))
        return bool(self._model_entry().get("is_vision"))

    def stream(
        self,
        messages: list,
        *,
        system_prompt: str,
        temperature: Optional[float],
        top_p: Optional[float],
        top_k: Optional[int],
        max_new_tokens: Optional[int],
        repetition_penalty: Optional[float],
        enable_thinking: bool,
        use_adapter: Optional[bool] = None,
    ):
        self.reply_hit_token_limit = False
        ensure_studio_backend_path(seed_cache_env = False)
        from utils.inference.inference_config import resolve_effective_sampling

        model_id = getattr(
            self._backend, "model_identifier" if self._kind == "gguf" else "active_model_name", None
        )
        if self._kind == "gguf":
            from core.inference.llama_cpp import _extra_args_set_any_flag

            # These pass-throughs render a template the flags were not read from.
            supports_reasoning = getattr(
                self._backend, "supports_reasoning", False
            ) and not _extra_args_set_any_flag(
                getattr(self._backend, "extra_args", None),
                {"--chat-template", "--chat-template-file", "--no-jinja"},
            )
            style = getattr(self._backend, "reasoning_style", None)
            always_on = getattr(self._backend, "reasoning_always_on", False)
        else:
            from core.inference.chat_template_helpers import _selected_template_strings_from_value
            from core.inference.llama_cpp import detect_reasoning_flags

            info = (getattr(self._backend, "models", {}).get(model_id) or {}).get(
                "chat_template_info"
            ) or {}
            selected = _selected_template_strings_from_value(info.get("template"))
            flags = detect_reasoning_flags(
                selected[0] if selected else None, model_id, log_level = "debug"
            )
            supports_reasoning = flags["supports_reasoning"]
            style = flags["reasoning_style"]
            always_on = flags["reasoning_always_on"]
        # An effort-only template has no off switch: thinking off still renders a low effort.
        always_on = always_on or style == "reasoning_effort"
        thinking = (always_on or enable_thinking) if supports_reasoning else None
        # Instruct-2507 never reasons (its template still parses <think>); -GGUF ids miss its defaults.
        if re.search(r"qwen3[-_].*instruct[-_]2507", (model_id or "").lower()):
            thinking = False
        sampling = resolve_effective_sampling(
            model_id,
            dict(
                temperature = temperature,
                top_p = top_p,
                top_k = top_k,
                repetition_penalty = repetition_penalty,
            ),
            thinking = thinking,
        )
        if self._kind == "gguf":
            # llama-server takes the system prompt as the first message.
            msgs = list(messages)
            if system_prompt:
                msgs = [{"role": "system", "content": system_prompt}, *msgs]
            return self._watch_metadata(
                self._backend.generate_chat_completion(
                    messages = msgs,
                    max_tokens = max_new_tokens,
                    enable_thinking = enable_thinking,
                    **sampling,
                )
            )
        holder: dict = {}
        several = bool(
            (self._model_entry().get("chat_template_info") or {}).get("accepts_multiple_images")
        )
        messages, images, image_ordinal = _local_image_request(messages, several)
        gen_kwargs = dict(
            messages = messages,
            system_prompt = system_prompt,
            max_new_tokens = max_new_tokens,
            enable_thinking = enable_thinking,
            stats_holder = holder,
            **sampling,
        )
        if images:
            gen_kwargs.update(images = images, image_ordinal = image_ordinal)
        if use_adapter is not None:
            stream = self._backend.generate_with_adapter_control(
                use_adapter = use_adapter, **gen_kwargs
            )
        else:
            stream = self._backend.generate_chat_response(**gen_kwargs)
        return self._watch_stats(stream, holder)

    def _watch_metadata(self, stream):
        """llama-server closes a turn with a metadata event carrying the finish reason."""
        for chunk in stream:
            if isinstance(chunk, dict) and chunk.get("type") == "metadata":
                self.reply_hit_token_limit = chunk.get("finish_reason") == "length"
            yield chunk

    def _watch_stats(self, stream, holder: dict):
        """The worker fills the holder once the turn is done, so read it at the end."""
        yield from stream
        self.reply_hit_token_limit = stats_hit_token_limit(holder.get("stats"))

    def close(self) -> None:
        # Kill the worker directly: unload_model waits for an ack compare mode can swallow.
        try:
            if self._kind == "gguf":
                self._backend.unload_model()
            else:
                self._backend._shutdown_subprocess(timeout = 2.0)
        except Exception:
            pass

    def share_distributed_object(
        self,
        obj,
        *,
        timeout = 300.0,
    ):
        if self._kind != "unsloth" or not hasattr(self._backend, "share_distributed_object"):
            raise RuntimeError(
                "Distributed MLX chat requires the Unsloth MLX backend; "
                f"backend '{self._kind}' cannot broadcast chat turns."
            )
        return self._backend.share_distributed_object(obj, timeout = timeout)


def resolve_model_config(model: str, *, hf_token: Optional[str]):
    ensure_studio_backend_path()
    from utils.models import ModelConfig

    model_config = ModelConfig.from_identifier(model_id = model, hf_token = hf_token)
    if not model_config:
        typer.echo("Could not resolve model config", err = True)
        raise typer.Exit(code = 1)
    return model_config


def _validate_llama_extra_args_or_exit(llama_extra_args: Optional[List[str]]) -> list[str]:
    from core.inference.llama_server_args import validate_extra_args
    try:
        return validate_extra_args(llama_extra_args)
    except ValueError as exc:
        typer.echo(f"Error: {exc}", err = True)
        raise typer.Exit(code = 1)


def _load_gguf_backend(
    model_config,
    *,
    hf_token,
    max_seq_length,
    tensor_parallel: bool = False,
    speculative_type: Optional[SpeculativeType] = None,
    spec_draft_n_max: Optional[int] = None,
    llama_extra_args: Optional[List[str]] = None,
):
    ensure_studio_backend_path()
    from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend
    from core.inference.tensor_fallback import load_with_tensor_fallback

    llama_backend = LlamaCppBackend()
    extra_args = _validate_llama_extra_args_or_exit(llama_extra_args)
    intent_fields = dict(
        hf_variant = model_config.gguf_variant,
        model_identifier = model_config.identifier,
        is_vision = model_config.is_vision,
        n_ctx = max_seq_length,
    )
    if model_config.gguf_hf_repo:
        intent_fields.update(hf_repo = model_config.gguf_hf_repo, hf_token = hf_token)
    else:
        intent_fields.update(
            gguf_path = model_config.gguf_file,
            mmproj_path = model_config.gguf_mmproj_file,
            mtp_draft_path = model_config.gguf_mtp_file,
            dspark_draft_path = model_config.gguf_dspark_file,
            dflash_draft_path = model_config.gguf_dflash_file,
        )
    if speculative_type is not None:
        intent_fields["speculative_type"] = speculative_type
    if spec_draft_n_max is not None:
        intent_fields["spec_draft_n_max"] = spec_draft_n_max

    async def _attempt_gguf_load(
        requested_tensor_parallel: bool, attempt_extra_args: Optional[List[str]]
    ) -> bool:
        return llama_backend.load_model(
            GgufLoadIntent(
                **intent_fields,
                tensor_parallel = requested_tensor_parallel,
                extra_args = attempt_extra_args,
            )
        )

    import asyncio

    loaded = asyncio.run(
        load_with_tensor_fallback(
            _attempt_gguf_load,
            requested_tensor = tensor_parallel,
            extra_args = extra_args,
            label = model_config.identifier,
        )
    )
    if not loaded:
        typer.echo("Model load failed", err = True)
        raise typer.Exit(code = 1)
    return ChatBackend("gguf", llama_backend)


def load_chat_backend(
    model: str,
    *,
    hf_token: Optional[str],
    max_seq_length: int,
    load_in_4bit: bool,
    tensor_parallel: bool = False,
    speculative_type: Optional[SpeculativeType] = None,
    spec_draft_n_max: Optional[int] = None,
    llama_extra_args: Optional[List[str]] = None,
    model_config = None,
    fresh_backend: bool = False,
):
    """Load model in-process, GGUF via llama-server else the orchestrator; fresh_backend uses a
    private orchestrator so a second model (compare's base column) runs alongside."""
    from unsloth_cli._studio_deps import studio_backend_imports

    with studio_backend_imports("unsloth inference", studio_only = True), quiet_if_nonzero_mlx_rank():
        is_mlx_distributed, rank, _world_size = mlx_distributed_info()
        if model_config is None:
            model_config = resolve_model_config(model, hf_token = hf_token)

        if is_mlx_distributed and model_config.is_gguf:
            if rank == 0:
                typer.echo(
                    "Distributed MLX inference does not support GGUF/llama.cpp models. "
                    "Use a non-GGUF MLX model under mlx.launch, or run GGUF without "
                    "mlx.launch.",
                    err = True,
                )
            raise typer.Exit(code = 1)

        if rank == 0:
            typer.echo(f"Loading {model}", err = True)

        if model_config.is_gguf:
            return _load_gguf_backend(
                model_config,
                hf_token = hf_token,
                max_seq_length = max_seq_length,
                tensor_parallel = tensor_parallel,
                speculative_type = speculative_type,
                spec_draft_n_max = spec_draft_n_max,
                llama_extra_args = llama_extra_args,
            )

        if fresh_backend:
            ensure_studio_backend_path()
            from core.inference import InferenceOrchestrator
            backend = InferenceOrchestrator()
        else:
            ensure_studio_backend_path()
            from core.inference import get_inference_backend
            backend = get_inference_backend()
        try:
            loaded = backend.load_model(
                config = model_config,
                max_seq_length = max_seq_length,
                load_in_4bit = load_in_4bit,
                hf_token = hf_token,
                tensor_parallel = tensor_parallel,
                mlx_distributed = is_mlx_distributed,
            )
        except Exception as exc:
            if not is_mlx_distributed:
                raise
            if rank == 0:
                typer.echo(str(exc) or "Model load failed", err = True)
            raise typer.Exit(code = 1)
        if not loaded:
            typer.echo("Model load failed", err = True)
            raise typer.Exit(code = 1)
    return ChatBackend("unsloth", backend)


def _loopback_candidate_bases(base: str) -> list:
    """For a bare ``localhost`` base, the concrete IP bases to try, IPv4
    127.0.0.1 first (where ``unsloth studio`` binds by default). Pinning to one
    address up front means discovery, the identity check, and the credential we
    then send all target the same endpoint instead of racing IPv4/IPv6
    resolution -- which would otherwise let the health probe land on one address
    and the identity check on another. A literal IP or remote name is unchanged.
    """
    from urllib.parse import urlparse

    parsed = urlparse(base)
    if (parsed.hostname or "").lower() != "localhost":
        return [base]
    import socket

    port = parsed.port or (443 if parsed.scheme == "https" else 80)
    try:
        ips = {
            ai[4][0] for ai in socket.getaddrinfo(parsed.hostname, port, type = socket.SOCK_STREAM)
        }
    except Exception:
        return [base]
    ordered = sorted(ips, key = lambda ip: (ip != "127.0.0.1", ip))
    bases = [
        f"{parsed.scheme}://" + (f"[{ip}]:{port}" if ":" in ip else f"{ip}:{port}")
        for ip in ordered
    ]
    return bases or [base]


_STUDIO_SERVICE_MARKER = "Unsloth UI Backend"


def _recorded_loopback_bases(address: Optional[str], port: str) -> list:
    """Loopback bases for a recorded address; the wrong family hits whoever holds that port."""
    import ipaddress

    loopback, parsed_any = set(), False
    for text in (address or "").split(","):
        try:
            ip = ipaddress.ip_address(text.strip())
        except ValueError:
            continue
        parsed_any = True
        if ip.is_unspecified:
            loopback.add(ipaddress.ip_address("::1" if ip.version == 6 else "127.0.0.1"))
        elif ip.is_loopback:
            loopback.add(ip)
    if not parsed_any:
        loopback.add(ipaddress.ip_address("127.0.0.1"))
    return [
        f"http://[{ip.compressed}]:{port}" if ip.version == 6 else f"http://{ip.compressed}:{port}"
        for ip in sorted(loopback, key = lambda ip: (ip.version, ip.compressed))
    ]


def _recorded_studio_bases(tried: list):
    from unsloth_cli.commands.studio import (
        PID_FILE_GLOB,
        STUDIO_HOME,
        _pid_alive,
        _pid_is_studio_server,
        _read_pid_record,
    )

    seen = set(tried)
    try:
        paths = sorted(STUDIO_HOME.glob(PID_FILE_GLOB))
    except OSError:
        return
    for path in paths:
        match = re.fullmatch(r"studio-(\d+)-\d+\.pid", path.name)
        record = _read_pid_record(path) if match else None
        if record is None:
            continue
        pid, created, address = record
        if not _pid_alive(pid) or not _pid_is_studio_server(pid, [created]):
            continue
        for candidate in _recorded_loopback_bases(address, match.group(1)):
            if candidate not in seen:
                seen.add(candidate)
                yield candidate


def _answers_as_studio(base: str, timeout: float) -> bool:
    import urllib.request
    request = urllib.request.Request(f"{base}/api/health", headers = {"User-Agent": _USER_AGENT})
    try:
        with urlopen_local(request, timeout = timeout) as response:
            # A live port is not Studio: a stranger answering every path would get our key.
            body = json.loads(response.read(65536).decode() or "{}")
            return body.get("service") == _STUDIO_SERVICE_MARKER
    except Exception:
        return False


def find_studio_server(timeout: float = 3.0) -> Optional[str]:
    base = os.environ.get("UNSLOTH_STUDIO_URL", "http://127.0.0.1:8888").rstrip("/")
    candidates = _loopback_candidate_bases(base)
    # Use the first loopback address that answers, so the flow talks to that exact address.
    found = next((c for c in candidates if _answers_as_studio(c, timeout)), None)
    if os.environ.get("UNSLOTH_STUDIO_URL"):
        return found
    recorded = list(_recorded_studio_bases(candidates))
    # Another install may own the default port: prefer the server this install recorded.
    if found is not None and (not recorded or verify_studio_identity(found, timeout)):
        return found
    return next((c for c in recorded if _answers_as_studio(c, timeout)), found)


def is_loopback_url(base: str) -> bool:
    """True only when *base* resolves to loopback. find_studio_server() trusts a
    base after only a health probe, so credentials are auto-sent only to loopback
    (a local Unsloth or an SSH tunnel on 127.0.0.1), the targets the auto flows mean."""
    from urllib.parse import urlparse

    host = (urlparse(base).hostname or "").lower()
    if host in ("localhost", "127.0.0.1", "::1"):
        return True
    try:
        import ipaddress
        return ipaddress.ip_address(host).is_loopback
    except ValueError:
        return False


def verify_studio_identity(base: str, timeout: float = 3.0) -> bool:
    """Confirm `base` is really this machine's Unsloth before sending a secret.

    Send a random nonce to /api/auth/identity and check the returned HMAC against
    the one computed from the local same-user secret; an endpoint without that
    secret (port squatter, remote/fake) can't match. Fails closed on any error."""
    import base64
    import hmac as _hmac
    import json
    import secrets as _secrets
    import socket
    import urllib.request
    from urllib.parse import urlparse

    try:
        import studio.backend.core  # noqa: F401  puts studio/backend on sys.path
        from studio.backend.auth import storage
    except Exception:
        return False

    parsed = urlparse(base)
    host = parsed.hostname or ""
    port = parsed.port or (443 if parsed.scheme == "https" else 80)
    # Resolve once and talk to that concrete address, binding the proof to (address, port): localhost may hit a squatter on ::1.
    try:
        ip = socket.getaddrinfo(host, port, type = socket.SOCK_STREAM)[0][4][0]
    except Exception:
        return False
    netloc = f"[{ip}]:{port}" if ":" in ip else f"{ip}:{port}"
    nonce = _secrets.token_bytes(32)
    query = base64.urlsafe_b64encode(nonce).decode()
    request = urllib.request.Request(
        f"{parsed.scheme}://{netloc}/api/auth/identity?nonce={query}",
        headers = {"User-Agent": _USER_AGENT, "Host": parsed.netloc},
    )
    try:
        # No redirects: a 302 could relay a real Unsloth's proof. Cap the read: still unverified.
        with urlopen_no_redirect(request, timeout = timeout) as response:
            proof = json.loads(response.read(65536).decode() or "{}").get("proof")
    except Exception:
        return False
    if not isinstance(proof, str):
        return False
    try:
        expected = storage.compute_identity_proof(nonce, ip, port)
    except Exception:
        return False
    return _hmac.compare_digest(proof, expected)


def _studio_token() -> Optional[str]:
    """Signs a JWT with the server's stored secret, since the CLI runs as the same OS user."""
    try:
        import studio.backend.core  # noqa: F401  puts studio/backend on sys.path

        from studio.backend.auth import storage
        from studio.backend.auth.authentication import create_access_token

        row = (
            storage.get_connection()
            .execute(
                "SELECT username FROM auth_user WHERE username = ?",
                (storage.DEFAULT_ADMIN_USERNAME,),
            )
            .fetchone()
        )
        return create_access_token(row[0], desktop = True) if row else None
    except Exception:
        return None


class HttpChatBackend:
    """Chat against a running Unsloth server over its OpenAI-compatible API.

    close() leaves the model loaded on purpose — the next session (or the
    UI) starts instantly.
    """

    def __init__(self, base_url: str, token: str) -> None:
        self._base = base_url
        self._token = token
        self.reply_hit_token_limit = False
        self.gguf_variant: Optional[str] = None
        self.supports_images = False

    def _request(
        self,
        method: str,
        path: str,
        payload = None,
        timeout = None,
    ):
        import json
        import urllib.request

        request = urllib.request.Request(
            self._base + path,
            data = None if payload is None else json.dumps(payload).encode(),
            headers = {
                "Authorization": f"Bearer {self._token}",
                "Content-Type": "application/json",
                "User-Agent": _USER_AGENT,
            },
            method = method,
        )
        # No redirects: this carries a bearer token (see urlopen_no_redirect).
        return urlopen_no_redirect(request, timeout = timeout)

    def ensure_loaded(
        self,
        model: str,
        *,
        hf_token,
        max_seq_length,
        load_in_4bit,
        tensor_parallel: bool = False,
        speculative_type: Optional[SpeculativeType] = None,
        spec_draft_n_max: Optional[int] = None,
        llama_extra_args: Optional[List[str]] = None,
    ) -> None:
        typer.echo(f"Loading {model} on the Unsloth server", err = True)
        payload = {
            "model_path": model,
            "hf_token": hf_token,
            "max_seq_length": max_seq_length,
            "tensor_parallel": tensor_parallel,
        }
        if load_in_4bit is not None:
            payload["load_in_4bit"] = load_in_4bit
        if llama_extra_args:
            payload["llama_extra_args"] = llama_extra_args
        if speculative_type is not None:
            payload["speculative_type"] = speculative_type
        if spec_draft_n_max is not None:
            payload["spec_draft_n_max"] = spec_draft_n_max
        resident_variant = self._resident_gguf_variant(model)
        if resident_variant:
            payload["gguf_variant"] = resident_variant
        self.gguf_variant = resident_variant
        try:
            # Read the whole body, not just headers: closing early discards the only in-band report of a late load failure.
            loaded = read_json_checking_deferred_error(
                self._base + "/api/inference/load",
                self._request("POST", "/api/inference/load", payload),
            )
        except Exception as exc:
            typer.echo(f"Model load failed: {exc}", err = True)
            raise typer.Exit(code = 1)
        self.supports_images = isinstance(loaded, dict) and bool(loaded.get("is_vision"))

    def _resident_gguf_variant(self, model: str) -> Optional[str]:
        if model.lower().endswith(".gguf"):
            return None
        try:
            with self._request("GET", "/api/inference/status", timeout = 30) as response:
                status = json.loads(response.read())
        except Exception:
            return None
        if not isinstance(status, dict) or not status.get("is_gguf"):
            return None
        # A local load's active_model is only its basename.
        loaded = status.get("model_identifier") or status.get("active_model")
        if not loaded:
            return None
        loaded = str(loaded)
        if model == status.get("model_identifier"):
            same = True
        elif os.path.exists(model):
            # Mirrors the server's _same_loaded_identifier.
            same = os.path.normcase(model) == os.path.normcase(loaded)
        else:
            # The server loads an ownerless id as unsloth/<id> (ModelConfig.from_identifier).
            requested = model if "/" in model else f"unsloth/{model}"
            same = requested.casefold() == loaded.casefold()
        return status.get("gguf_variant") if same else None

    def stream(
        self,
        messages: list,
        *,
        system_prompt: str,
        temperature: Optional[float],
        top_p: Optional[float],
        top_k: Optional[int],
        max_new_tokens: Optional[int],
        repetition_penalty: Optional[float],
        enable_thinking: bool,
        use_adapter: Optional[bool] = None,
    ):
        import json

        msgs = list(messages)
        if system_prompt:
            msgs = [{"role": "system", "content": system_prompt}, *msgs]
        body = {
            "model": "default",
            "messages": msgs,
            "stream": True,
            "enable_thinking": enable_thinking,
        }
        sampling = dict(
            temperature = temperature,
            top_p = top_p,
            top_k = top_k,
            repetition_penalty = repetition_penalty,
        )
        body.update({key: value for key, value in sampling.items() if value is not None})
        if max_new_tokens is not None:
            body["max_tokens"] = max_new_tokens
        resp = self._request("POST", "/v1/chat/completions", body)

        self.reply_hit_token_limit = False

        def cumulative():
            # Accumulate SSE deltas into the full-text-so-far form; re-wrap delta.reasoning_content in think tags for visible_text.
            text = ""
            thinking = False
            saw_finish = False
            with resp:
                for raw_line in resp:
                    line = raw_line.decode("utf-8", "replace").strip()
                    if not line.startswith("data:"):
                        continue
                    data = line[len("data:") :].strip()
                    if data == "[DONE]":
                        return
                    try:
                        parsed = json.loads(data)
                    except ValueError:
                        continue
                    if "error" in parsed:
                        raise RuntimeError(
                            f"Server error: {parsed['error'].get('message', 'Unknown server error')}"
                        )
                    try:
                        choice = parsed["choices"][0]
                    except (KeyError, IndexError):
                        continue
                    # Read before the delta: the finish-reason chunk may carry no content.
                    if choice.get("finish_reason") is not None:
                        saw_finish = True
                        self.reply_hit_token_limit = choice["finish_reason"] == "length"
                    try:
                        reasoning = choice["delta"].get("reasoning_content")
                        delta = choice["delta"].get("content")
                    except (KeyError, AttributeError, TypeError):
                        continue
                    if reasoning:
                        if not thinking:
                            text += _THINK_OPEN
                            thinking = True
                        text += reasoning
                    if delta and thinking:
                        text += "</think>"
                        thinking = False
                    if not (reasoning or delta):
                        continue
                    text += delta or ""
                    # An emoji can arrive as lone surrogate halves across deltas.
                    visible = text
                    if "\ud800" <= visible[-1] <= "\udbff":
                        visible = visible[:-1]
                    yield visible.encode("utf-16", "surrogatepass").decode("utf-16", "replace")
            if not saw_finish:
                raise RuntimeError("Stream ended before the reply completed.")

        return cumulative()

    def close(self) -> None:
        pass


def server_load_opts(ctx, load_opts: dict) -> dict:
    """Drop an untyped --load-in-4bit so the server can keep a resident model's precision."""
    opts = dict(load_opts)
    if ctx.get_parameter_source("load_in_4bit").name != "COMMANDLINE":
        opts["load_in_4bit"] = None
    return opts


def connect_studio_server(
    model: str,
    *,
    hf_token,
    max_seq_length,
    load_in_4bit,
    tensor_parallel: bool = False,
    speculative_type: Optional[SpeculativeType] = None,
    spec_draft_n_max: Optional[int] = None,
    llama_extra_args: Optional[List[str]] = None,
):
    """Backend on a running Unsloth server, or None (caller loads locally)."""
    base_url = find_studio_server()
    if not base_url:
        return None

    # Explicit UNSLOTH_STUDIO_URL we cannot safely attach to fails loudly; local discovery falls back.
    explicit = bool(os.environ.get("UNSLOTH_STUDIO_URL"))

    def _refuse(reason: str):
        if not explicit:
            return None
        typer.echo(
            f"Can't attach to the Unsloth server at {base_url}: {reason} Run Unsloth "
            "on this machine, or unset UNSLOTH_STUDIO_URL to load the model locally.",
            err = True,
        )
        raise typer.Exit(code = 1)

    # Only hand the self-issued JWT to loopback: a remote URL is unverified.
    if not is_loopback_url(base_url):
        return _refuse(
            "it isn't a local Unsloth, so a self-issued token can't "
            "authenticate to it and must not be sent to it."
        )
    # Confirm the loopback responder is really our Unsloth (not a port squatter).
    if not verify_studio_identity(base_url):
        return _refuse(
            "its identity couldn't be verified (it may be running as a "
            "different OS user, or another process took the port)."
        )
    token = _studio_token()
    if not token:
        return _refuse("couldn't self-issue an Unsloth token (is Unsloth set up here?).")
    backend = HttpChatBackend(base_url, token)
    from unsloth_cli._npu import npu_model_path, prepare_npu_model

    npu_path = npu_model_path(model)
    if npu_path:
        # `unsloth inference` passes the bare id the picker shows; load it by its model path.
        model = npu_path
        try:
            prepare_npu_model(
                lambda method, path, payload, timeout: backend._request(
                    method, path, payload, timeout = timeout
                ),
                model,
                lambda line: typer.echo(line, err = True),
            )
        except RuntimeError as exc:
            typer.echo(f"Error: {exc}", err = True)
            raise typer.Exit(code = 1)
    backend.ensure_loaded(
        model,
        hf_token = hf_token,
        max_seq_length = max_seq_length,
        load_in_4bit = load_in_4bit,
        tensor_parallel = tensor_parallel,
        speculative_type = speculative_type,
        spec_draft_n_max = spec_draft_n_max,
        llama_extra_args = llama_extra_args,
    )
    return backend