Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Model loading and streaming shared by `inference` and `chat`."""
import json
import os
import re
import sys
from contextlib import contextmanager, redirect_stderr, redirect_stdout
from pathlib import Path
from typing import List, Literal, Optional
import typer
# Mirrors the backend's _CANONICAL_SPEC_MODES; typer validates --speculative-type with it.
SpeculativeType = Literal[
"auto", "mtp", "dspark", "dflash", "ngram", "mtp+ngram", "off", "ngram-simple"
]
_THINK_OPEN = "<think>"
_THINK_BLOCK = re.compile(rf"{re.escape(_THINK_OPEN)}.*?</think>", re.DOTALL)
_STREAMED_ERROR_PREFIX = "Error: "
# Cloudflare 403s the default urllib User-Agent as a bot.
_USER_AGENT = "unsloth-cli"
_MPI_ENV_PAIRS = (
("OMPI_COMM_WORLD_RANK", "OMPI_COMM_WORLD_SIZE"),
("PMI_RANK", "PMI_SIZE"),
("PMIX_RANK", "PMIX_SIZE"),
("MPI_RANK", "MPI_WORLD_SIZE"),
("MV2_COMM_WORLD_RANK", "MV2_COMM_WORLD_SIZE"),
)
_openers: dict = {}
def _opener(follow_redirects: bool, loopback: bool):
key = (follow_redirects, loopback)
if key not in _openers:
import urllib.error
import urllib.request
class _NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
raise urllib.error.HTTPError(
req.full_url, code, f"refusing redirect to {newurl}", headers, fp
)
# urllib proxies 127.0.0.1 unless no_proxy lists it, leaking the bearer key to http_proxy.
handlers = [urllib.request.ProxyHandler({})] if loopback else []
if not follow_redirects:
handlers.append(_NoRedirect)
_openers[key] = urllib.request.build_opener(*handlers)
return _openers[key]
def urlopen_no_redirect(request, timeout):
"""urlopen that errors on any redirect: following a 3xx would send a bearer
token (or accept an identity proof) to a base we never vetted, letting a port
squatter relay a real Unsloth's response."""
return _opener(False, is_loopback_url(request.full_url)).open(request, timeout = timeout)
def urlopen_local(request, timeout):
"""urllib.request.urlopen that never sends a loopback request through a proxy."""
if not is_loopback_url(request.full_url):
import urllib.request
return urllib.request.urlopen(request, timeout = timeout)
return _opener(True, True).open(request, timeout = timeout)
# load and unload pad the body and commit 200 early, so a late failure rides only in-band under this key; a bare 200 reads as success.
_DEFERRED_ERROR_KEY = "_deferred_error"
def raise_for_deferred_error(url: str, body):
"""Raise the late failure a padded 200 body carries; else return ``body``.
``urllib.error.HTTPError`` specifically: it is the class every CLI caller already
handles for a plain HTTP failure, so existing ``except`` blocks, messages and exit
codes keep working, and ``.read()`` yields the same ``{"detail": ...}`` shape.
"""
if not isinstance(body, dict):
return body
deferred = body.get(_DEFERRED_ERROR_KEY)
if not isinstance(deferred, dict):
return body
import email.message
import io
import urllib.error
status = deferred.get("status_code")
if not isinstance(status, int) or isinstance(status, bool):
status = 500
detail = deferred.get("detail")
if not isinstance(detail, str) or not detail:
detail = "unknown error" if detail is None else json.dumps(detail)
headers = email.message.Message()
headers["Content-Type"] = "application/json"
raise urllib.error.HTTPError(
url, status, detail, headers, io.BytesIO(json.dumps({"detail": detail}).encode())
)
def require_completed_padded_body(url: str, body):
"""Return ``body``, or raise if it is not the payload a padded route promised.
A proxy that gives up mid-pad leaves a 200 with an empty or truncated body, so
accepting it reports an unfinished load or unload as completed. Only the two padded
routes commit their status that early, so only they require a payload; ``{}`` is
rejected too, since that is what a blank body decodes to here. Mirrored by
``assertCompletedPaddedBody`` in studio/frontend/src/features/chat/api/padded-response.ts.
"""
if isinstance(body, dict) and body:
return body
raise RuntimeError(
f"{url} did not report completion: the connection closed before the "
"server's reply arrived. Check the model's status before retrying."
)
def read_json_checking_deferred_error(url: str, response):
"""Drain ``response``, then raise any deferred error its body carries.
Draining matters on its own: stopping at the headers of a padded /load leaves the
load running, so the caller resumes too early. An incomplete JSON payload is a
truncated padded reply, not a success (see ``require_completed_padded_body``).
"""
try:
raw = response.read()
finally:
response.close()
try:
body = json.loads(raw.decode(errors = "replace") or "{}")
except ValueError:
body = None
return require_completed_padded_body(url, raise_for_deferred_error(url, body))
_cache_env_seeded = False
def _seed_cache_env() -> None:
"""Pin cache locations as the backend does: unsloth_zoo's relative UNSLOTH_COMPILE_LOCATION
would resolve against the cwd, leaving unsloth_compiled_cache that cache_cleanup misses
(issue 8865)."""
global _cache_env_seeded
if _cache_env_seeded:
return
_cache_env_seeded = True
try:
from utils.paths.storage_roots import setup_cache_env
setup_cache_env()
except Exception: # noqa: BLE001 - never fail a command over cache placement
pass
def ensure_studio_backend_path(*, seed_cache_env: bool = True) -> None:
"""Put studio/backend on sys.path, and by default pin the cache locations too.
`seed_cache_env = False` is for callers that only want to IMPORT a path helper.
setup_cache_env() creates every cache directory it pins, so seeding it from
`unsloth start`'s Node discovery turned a read-only lookup into 18 mkdirs under a home
that may have nothing to do with the command being run -- including one against a remote
server. Seeding stays on for the ML entry points, where the pins are the point.
"""
backend_dir = str(Path(__file__).resolve().parents[1] / "studio" / "backend")
if backend_dir not in sys.path:
sys.path.insert(0, backend_dir)
if seed_cache_env:
# After the path insert, before the caller's backend import pulls in unsloth_zoo.compiler.
_seed_cache_env()
def configure_quiet_logging() -> None:
import logging
# The CLI never configures structlog; LOG_LEVEL is exported so the worker inherits it.
level_name = os.environ.setdefault("LOG_LEVEL", "WARNING").upper()
level = getattr(logging, level_name, logging.WARNING)
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
# Quieting logs must not fail a command before the import that really needs structlog gets to report itself.
try:
import structlog
except ModuleNotFoundError:
return
structlog.configure(wrapper_class = structlog.make_filtering_bound_logger(level))
def _parse_nonnegative_int(value: Optional[str]) -> Optional[int]:
if value is None:
return None
try:
parsed = int(value)
except (TypeError, ValueError):
return None
return parsed if parsed >= 0 else None
def _first_mpi_env_pair() -> tuple[Optional[int], Optional[int]]:
for rank_name, size_name in _MPI_ENV_PAIRS:
rank = _parse_nonnegative_int(os.environ.get(rank_name))
world_size = _parse_nonnegative_int(os.environ.get(size_name))
if rank is not None and world_size is not None and world_size > 1 and rank < world_size:
return rank, world_size
return None, None
def _json_rank_count_from_env(name: str) -> Optional[int]:
value = os.environ.get(name)
if not value:
return None
try:
if value.lstrip().startswith(("[", "{")):
data = json.loads(value)
else:
with open(value, "r", encoding = "utf-8") as f:
data = json.load(f)
except (json.JSONDecodeError, OSError, UnicodeDecodeError):
return None
if isinstance(data, list):
return len(data)
if isinstance(data, dict) and isinstance(data.get("hosts"), list):
return len(data["hosts"])
return None
def mlx_distributed_info() -> tuple[bool, int, Optional[int]]:
"""Return launch-context metadata without initializing MLX distributed."""
rank = _parse_nonnegative_int(os.environ.get("MLX_RANK"))
world_size = _parse_nonnegative_int(os.environ.get("MLX_WORLD_SIZE"))
if rank is not None:
if (
world_size is not None
and world_size > 1
and rank < world_size
and os.environ.get("NCCL_HOST_IP")
and os.environ.get("NCCL_PORT")
):
return True, rank, world_size
inferred_size = _json_rank_count_from_env("MLX_HOSTFILE")
if inferred_size is not None and inferred_size > 1 and rank < inferred_size:
return True, rank, inferred_size
inferred_size = _json_rank_count_from_env("MLX_IBV_DEVICES")
if (
inferred_size is not None
and inferred_size > 1
and rank < inferred_size
and os.environ.get("MLX_JACCL_COORDINATOR")
):
return True, rank, inferred_size
return False, 0, None
mpi_rank, mpi_world_size = _first_mpi_env_pair()
return mpi_rank is not None, mpi_rank or 0, mpi_world_size
def mlx_distributed_uses_mpi() -> bool:
"""Whether the current distributed context was launched through MPI."""
return (
_parse_nonnegative_int(os.environ.get("MLX_RANK")) is None
and _first_mpi_env_pair()[0] is not None
)
@contextmanager
def quiet_if_nonzero_mlx_rank():
"""Silence parent and child-process stdout/stderr on nonzero ranks."""
if mlx_distributed_info()[1] == 0:
yield
return
sys.stdout.flush()
sys.stderr.flush()
saved_stdout_fd = os.dup(1)
saved_stderr_fd = os.dup(2)
with open(os.devnull, "w", encoding = "utf-8") as devnull:
try:
os.dup2(devnull.fileno(), 1)
os.dup2(devnull.fileno(), 2)
with redirect_stdout(devnull), redirect_stderr(devnull):
yield
finally:
sys.stdout.flush()
sys.stderr.flush()
os.dup2(saved_stdout_fd, 1)
os.dup2(saved_stderr_fd, 2)
os.close(saved_stdout_fd)
os.close(saved_stderr_fd)
def visible_text(
text: str,
show_thinking: bool,
final: bool = False,
) -> str:
if show_thinking:
return text
text = _THINK_BLOCK.sub("", text)
# Hold back an unclosed trailing <think> so reasoning never leaks mid-stream.
open_idx = text.find(_THINK_OPEN)
if open_idx != -1:
text = text[:open_idx]
if final:
# Ended stream: a trailing "<" or "<th" can no longer become <think>.
return text
max_prefix = min(len(text), len(_THINK_OPEN) - 1)
for size in range(max_prefix, 0, -1):
if _THINK_OPEN.startswith(text[-size:]):
return text[:-size]
return text
def stream_to_stdout(stream, show_thinking: bool) -> str:
# Backends yield the full text-so-far each step (llama.cpp ends with a metadata dict, skipped).
raw = ""
shown = ""
for chunk in stream:
if not isinstance(chunk, str):
continue
raw = chunk
rendered = visible_text(chunk, show_thinking)
delta = rendered[len(shown) :]
if delta:
sys.stdout.write(delta)
sys.stdout.flush()
shown = rendered
tail = visible_text(raw, show_thinking, final = True)[len(shown) :]
if tail:
sys.stdout.write(tail)
sys.stdout.write("\n")
sys.stdout.flush()
return raw
class _LatestMarkdown:
# visible_text() and Markdown() each scan the whole reply: run them per drawn frame, not per chunk.
def __init__(self, show_thinking: bool):
self.state = ("", False) # (raw, final), swapped whole so the Live thread never sees a mix
self._show_thinking = show_thinking
self._built = None
def __rich_console__(self, console, options):
from rich.markdown import Markdown
from rich.text import Text
state = self.state
if self._built is None or self._built[0] is not state:
text = visible_text(state[0], self._show_thinking, final = state[1])
self._built = (state, Markdown(text) if text.strip() else Text(""))
yield self._built[1]
def stream_markdown(stream, show_thinking: bool, *, console) -> str:
from rich.live import Live
raw = ""
view = _LatestMarkdown(show_thinking)
with Live(view, console = console, refresh_per_second = 12, vertical_overflow = "visible"):
for chunk in stream:
if isinstance(chunk, str):
raw = chunk
view.state = (raw, False)
view.state = (raw, True)
return raw
def collect_stream(stream, show_thinking: bool) -> str:
raw = ""
for chunk in stream:
if isinstance(chunk, str):
raw = chunk
return visible_text(raw, show_thinking, final = True)
def raise_on_streamed_error(stream):
# Match backend errors by GenStreamError type, not the Error: text prefix, so completion text opening with Error: is not misread.
try:
ensure_studio_backend_path()
from core.inference.orchestrator import GenStreamError
except Exception:
GenStreamError = None
for chunk in stream:
if GenStreamError is not None and isinstance(chunk, GenStreamError):
raise RuntimeError(str(chunk)[len(_STREAMED_ERROR_PREFIX) :].strip() or "Unknown error")
yield chunk
def render_columns(
left_label: str,
left_text: str,
right_label: str,
right_text: str,
*,
console = None,
) -> None:
from rich import box
from rich.console import Console
from rich.table import Table
table = Table(box = box.MINIMAL, expand = True, padding = (0, 1), pad_edge = False)
table.add_column(left_label, header_style = "bold yellow", ratio = 1, overflow = "fold")
table.add_column(right_label, header_style = "bold magenta", ratio = 1, overflow = "fold")
table.add_row(left_text or "", right_text or "")
(console or Console()).print(table)
def stats_hit_token_limit(stats) -> bool:
"""Whether end-of-generation stats show the reply ran out of budget: safetensors sets truncated;
MLX and llama-server set finish_reason to length."""
if not isinstance(stats, dict):
return False
return bool(stats.get("truncated")) or stats.get("finish_reason") == "length"
_IMAGE_MIME_TYPES = {
"BMP": "image/bmp",
"GIF": "image/gif",
"JPEG": "image/jpeg",
"MPO": "image/jpeg",
"PNG": "image/png",
"TIFF": "image/tiff",
"WEBP": "image/webp",
}
def image_part(path) -> dict:
"""An OpenAI image_url part carrying the file as a data URL, the shape Studio's chat sends."""
import base64
import io
file = Path(path).expanduser()
try:
data = file.read_bytes()
except OSError as exc:
raise ValueError(f"can't read image {file}: {exc.strerror or exc}") from None
# Without Pillow nothing could tell a real picture from a renamed text file.
try:
from PIL import Image
except ImportError:
raise ValueError("reading images needs Pillow: pip install pillow") from None
try:
with Image.open(io.BytesIO(data)) as picture:
image_format = picture.format
picture.verify()
except Exception:
raise ValueError(f"{file} is not an image file Unsloth can read") from None
mime = _IMAGE_MIME_TYPES.get(image_format) or Image.MIME.get(image_format, "image/png")
encoded = base64.b64encode(data).decode("ascii")
return {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{encoded}"}}
def user_turn(text: str, images: Optional[list] = None) -> dict:
"""A user message; attached images go ahead of the text, as Studio orders them."""
if not images:
return {"role": "user", "content": text}
return {"role": "user", "content": [*images, {"type": "text", "text": text}]}
def _image_payload(part) -> Optional[str]:
if not isinstance(part, dict) or part.get("type") != "image_url":
return None
url = (part.get("image_url") or {}).get("url") or ""
if not url.startswith("data:"):
return None
return url.partition(",")[2] or None
def _local_image_request(messages: list, several: bool):
"""Studio's local route shape: an image marker where each image stood, plus the payloads in order.
A model that reads one image per conversation gets only the newest turn's, like the app.
"""
owners = [] # (message index, user ordinal, payloads)
ordinal = 0
for index, message in enumerate(messages):
if message.get("role") != "user":
continue
content = message.get("content")
if isinstance(content, list):
payloads = [p for p in map(_image_payload, content) if p]
if payloads:
owners.append((index, ordinal, payloads))
ordinal += 1
if not owners:
return messages, None, None
if not several:
if len(owners[-1][2]) > 1:
raise ValueError("this model reads one image per message; attach one image at a time")
owners = owners[-1:]
kept = {index: len(payloads) for index, _, payloads in owners}
rebuilt = []
for index, message in enumerate(messages):
content = message.get("content")
if not isinstance(content, list):
rebuilt.append(message)
continue
text = "\n".join(
part.get("text") or ""
for part in content
if isinstance(part, dict) and part.get("type") == "text"
)
if index in kept:
markers = [{"type": "image"} for _ in range(kept[index])]
rebuilt.append({**message, "content": [*markers, {"type": "text", "text": text}]})
else:
rebuilt.append({**message, "content": text})
images = [payload for _, _, payloads in owners for payload in payloads]
return rebuilt, images, owners[-1][1]
class ChatBackend:
"""Uniform stream()/close() over the llama-server and Unsloth backends."""
def __init__(self, kind: str, backend) -> None:
self._kind = kind # "gguf" | "unsloth"
self._backend = backend
self.reply_hit_token_limit = False
def _model_entry(self) -> dict:
backend = self._backend
return (getattr(backend, "models", None) or {}).get(
getattr(backend, "active_model_name", None)
) or {}
@property
def supports_images(self) -> bool:
if self._kind == "gguf":
return bool(getattr(self._backend, "is_vision", False))
return bool(self._model_entry().get("is_vision"))
def stream(
self,
messages: list,
*,
system_prompt: str,
temperature: Optional[float],
top_p: Optional[float],
top_k: Optional[int],
max_new_tokens: Optional[int],
repetition_penalty: Optional[float],
enable_thinking: bool,
use_adapter: Optional[bool] = None,
):
self.reply_hit_token_limit = False
ensure_studio_backend_path(seed_cache_env = False)
from utils.inference.inference_config import resolve_effective_sampling
model_id = getattr(
self._backend, "model_identifier" if self._kind == "gguf" else "active_model_name", None
)
if self._kind == "gguf":
from core.inference.llama_cpp import _extra_args_set_any_flag
# These pass-throughs render a template the flags were not read from.
supports_reasoning = getattr(
self._backend, "supports_reasoning", False
) and not _extra_args_set_any_flag(
getattr(self._backend, "extra_args", None),
{"--chat-template", "--chat-template-file", "--no-jinja"},
)
style = getattr(self._backend, "reasoning_style", None)
always_on = getattr(self._backend, "reasoning_always_on", False)
else:
from core.inference.chat_template_helpers import _selected_template_strings_from_value
from core.inference.llama_cpp import detect_reasoning_flags
info = (getattr(self._backend, "models", {}).get(model_id) or {}).get(
"chat_template_info"
) or {}
selected = _selected_template_strings_from_value(info.get("template"))
flags = detect_reasoning_flags(
selected[0] if selected else None, model_id, log_level = "debug"
)
supports_reasoning = flags["supports_reasoning"]
style = flags["reasoning_style"]
always_on = flags["reasoning_always_on"]
# An effort-only template has no off switch: thinking off still renders a low effort.
always_on = always_on or style == "reasoning_effort"
thinking = (always_on or enable_thinking) if supports_reasoning else None
# Instruct-2507 never reasons (its template still parses <think>); -GGUF ids miss its defaults.
if re.search(r"qwen3[-_].*instruct[-_]2507", (model_id or "").lower()):
thinking = False
sampling = resolve_effective_sampling(
model_id,
dict(
temperature = temperature,
top_p = top_p,
top_k = top_k,
repetition_penalty = repetition_penalty,
),
thinking = thinking,
)
if self._kind == "gguf":
# llama-server takes the system prompt as the first message.
msgs = list(messages)
if system_prompt:
msgs = [{"role": "system", "content": system_prompt}, *msgs]
return self._watch_metadata(
self._backend.generate_chat_completion(
messages = msgs,
max_tokens = max_new_tokens,
enable_thinking = enable_thinking,
**sampling,
)
)
holder: dict = {}
several = bool(
(self._model_entry().get("chat_template_info") or {}).get("accepts_multiple_images")
)
messages, images, image_ordinal = _local_image_request(messages, several)
gen_kwargs = dict(
messages = messages,
system_prompt = system_prompt,
max_new_tokens = max_new_tokens,
enable_thinking = enable_thinking,
stats_holder = holder,
**sampling,
)
if images:
gen_kwargs.update(images = images, image_ordinal = image_ordinal)
if use_adapter is not None:
stream = self._backend.generate_with_adapter_control(
use_adapter = use_adapter, **gen_kwargs
)
else:
stream = self._backend.generate_chat_response(**gen_kwargs)
return self._watch_stats(stream, holder)
def _watch_metadata(self, stream):
"""llama-server closes a turn with a metadata event carrying the finish reason."""
for chunk in stream:
if isinstance(chunk, dict) and chunk.get("type") == "metadata":
self.reply_hit_token_limit = chunk.get("finish_reason") == "length"
yield chunk
def _watch_stats(self, stream, holder: dict):
"""The worker fills the holder once the turn is done, so read it at the end."""
yield from stream
self.reply_hit_token_limit = stats_hit_token_limit(holder.get("stats"))
def close(self) -> None:
# Kill the worker directly: unload_model waits for an ack compare mode can swallow.
try:
if self._kind == "gguf":
self._backend.unload_model()
else:
self._backend._shutdown_subprocess(timeout = 2.0)
except Exception:
pass
def share_distributed_object(
self,
obj,
*,
timeout = 300.0,
):
if self._kind != "unsloth" or not hasattr(self._backend, "share_distributed_object"):
raise RuntimeError(
"Distributed MLX chat requires the Unsloth MLX backend; "
f"backend '{self._kind}' cannot broadcast chat turns."
)
return self._backend.share_distributed_object(obj, timeout = timeout)
def resolve_model_config(model: str, *, hf_token: Optional[str]):
ensure_studio_backend_path()
from utils.models import ModelConfig
model_config = ModelConfig.from_identifier(model_id = model, hf_token = hf_token)
if not model_config:
typer.echo("Could not resolve model config", err = True)
raise typer.Exit(code = 1)
return model_config
def _validate_llama_extra_args_or_exit(llama_extra_args: Optional[List[str]]) -> list[str]:
from core.inference.llama_server_args import validate_extra_args
try:
return validate_extra_args(llama_extra_args)
except ValueError as exc:
typer.echo(f"Error: {exc}", err = True)
raise typer.Exit(code = 1)
def _load_gguf_backend(
model_config,
*,
hf_token,
max_seq_length,
tensor_parallel: bool = False,
speculative_type: Optional[SpeculativeType] = None,
spec_draft_n_max: Optional[int] = None,
llama_extra_args: Optional[List[str]] = None,
):
ensure_studio_backend_path()
from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend
from core.inference.tensor_fallback import load_with_tensor_fallback
llama_backend = LlamaCppBackend()
extra_args = _validate_llama_extra_args_or_exit(llama_extra_args)
intent_fields = dict(
hf_variant = model_config.gguf_variant,
model_identifier = model_config.identifier,
is_vision = model_config.is_vision,
n_ctx = max_seq_length,
)
if model_config.gguf_hf_repo:
intent_fields.update(hf_repo = model_config.gguf_hf_repo, hf_token = hf_token)
else:
intent_fields.update(
gguf_path = model_config.gguf_file,
mmproj_path = model_config.gguf_mmproj_file,
mtp_draft_path = model_config.gguf_mtp_file,
dspark_draft_path = model_config.gguf_dspark_file,
dflash_draft_path = model_config.gguf_dflash_file,
)
if speculative_type is not None:
intent_fields["speculative_type"] = speculative_type
if spec_draft_n_max is not None:
intent_fields["spec_draft_n_max"] = spec_draft_n_max
async def _attempt_gguf_load(
requested_tensor_parallel: bool, attempt_extra_args: Optional[List[str]]
) -> bool:
return llama_backend.load_model(
GgufLoadIntent(
**intent_fields,
tensor_parallel = requested_tensor_parallel,
extra_args = attempt_extra_args,
)
)
import asyncio
loaded = asyncio.run(
load_with_tensor_fallback(
_attempt_gguf_load,
requested_tensor = tensor_parallel,
extra_args = extra_args,
label = model_config.identifier,
)
)
if not loaded:
typer.echo("Model load failed", err = True)
raise typer.Exit(code = 1)
return ChatBackend("gguf", llama_backend)
def load_chat_backend(
model: str,
*,
hf_token: Optional[str],
max_seq_length: int,
load_in_4bit: bool,
tensor_parallel: bool = False,
speculative_type: Optional[SpeculativeType] = None,
spec_draft_n_max: Optional[int] = None,
llama_extra_args: Optional[List[str]] = None,
model_config = None,
fresh_backend: bool = False,
):
"""Load model in-process, GGUF via llama-server else the orchestrator; fresh_backend uses a
private orchestrator so a second model (compare's base column) runs alongside."""
from unsloth_cli._studio_deps import studio_backend_imports
with studio_backend_imports("unsloth inference", studio_only = True), quiet_if_nonzero_mlx_rank():
is_mlx_distributed, rank, _world_size = mlx_distributed_info()
if model_config is None:
model_config = resolve_model_config(model, hf_token = hf_token)
if is_mlx_distributed and model_config.is_gguf:
if rank == 0:
typer.echo(
"Distributed MLX inference does not support GGUF/llama.cpp models. "
"Use a non-GGUF MLX model under mlx.launch, or run GGUF without "
"mlx.launch.",
err = True,
)
raise typer.Exit(code = 1)
if rank == 0:
typer.echo(f"Loading {model}", err = True)
if model_config.is_gguf:
return _load_gguf_backend(
model_config,
hf_token = hf_token,
max_seq_length = max_seq_length,
tensor_parallel = tensor_parallel,
speculative_type = speculative_type,
spec_draft_n_max = spec_draft_n_max,
llama_extra_args = llama_extra_args,
)
if fresh_backend:
ensure_studio_backend_path()
from core.inference import InferenceOrchestrator
backend = InferenceOrchestrator()
else:
ensure_studio_backend_path()
from core.inference import get_inference_backend
backend = get_inference_backend()
try:
loaded = backend.load_model(
config = model_config,
max_seq_length = max_seq_length,
load_in_4bit = load_in_4bit,
hf_token = hf_token,
tensor_parallel = tensor_parallel,
mlx_distributed = is_mlx_distributed,
)
except Exception as exc:
if not is_mlx_distributed:
raise
if rank == 0:
typer.echo(str(exc) or "Model load failed", err = True)
raise typer.Exit(code = 1)
if not loaded:
typer.echo("Model load failed", err = True)
raise typer.Exit(code = 1)
return ChatBackend("unsloth", backend)
def _loopback_candidate_bases(base: str) -> list:
"""For a bare ``localhost`` base, the concrete IP bases to try, IPv4
127.0.0.1 first (where ``unsloth studio`` binds by default). Pinning to one
address up front means discovery, the identity check, and the credential we
then send all target the same endpoint instead of racing IPv4/IPv6
resolution -- which would otherwise let the health probe land on one address
and the identity check on another. A literal IP or remote name is unchanged.
"""
from urllib.parse import urlparse
parsed = urlparse(base)
if (parsed.hostname or "").lower() != "localhost":
return [base]
import socket
port = parsed.port or (443 if parsed.scheme == "https" else 80)
try:
ips = {
ai[4][0] for ai in socket.getaddrinfo(parsed.hostname, port, type = socket.SOCK_STREAM)
}
except Exception:
return [base]
ordered = sorted(ips, key = lambda ip: (ip != "127.0.0.1", ip))
bases = [
f"{parsed.scheme}://" + (f"[{ip}]:{port}" if ":" in ip else f"{ip}:{port}")
for ip in ordered
]
return bases or [base]
_STUDIO_SERVICE_MARKER = "Unsloth UI Backend"
def _recorded_loopback_bases(address: Optional[str], port: str) -> list:
"""Loopback bases for a recorded address; the wrong family hits whoever holds that port."""
import ipaddress
loopback, parsed_any = set(), False
for text in (address or "").split(","):
try:
ip = ipaddress.ip_address(text.strip())
except ValueError:
continue
parsed_any = True
if ip.is_unspecified:
loopback.add(ipaddress.ip_address("::1" if ip.version == 6 else "127.0.0.1"))
elif ip.is_loopback:
loopback.add(ip)
if not parsed_any:
loopback.add(ipaddress.ip_address("127.0.0.1"))
return [
f"http://[{ip.compressed}]:{port}" if ip.version == 6 else f"http://{ip.compressed}:{port}"
for ip in sorted(loopback, key = lambda ip: (ip.version, ip.compressed))
]
def _recorded_studio_bases(tried: list):
from unsloth_cli.commands.studio import (
PID_FILE_GLOB,
STUDIO_HOME,
_pid_alive,
_pid_is_studio_server,
_read_pid_record,
)
seen = set(tried)
try:
paths = sorted(STUDIO_HOME.glob(PID_FILE_GLOB))
except OSError:
return
for path in paths:
match = re.fullmatch(r"studio-(\d+)-\d+\.pid", path.name)
record = _read_pid_record(path) if match else None
if record is None:
continue
pid, created, address = record
if not _pid_alive(pid) or not _pid_is_studio_server(pid, [created]):
continue
for candidate in _recorded_loopback_bases(address, match.group(1)):
if candidate not in seen:
seen.add(candidate)
yield candidate
def _answers_as_studio(base: str, timeout: float) -> bool:
import urllib.request
request = urllib.request.Request(f"{base}/api/health", headers = {"User-Agent": _USER_AGENT})
try:
with urlopen_local(request, timeout = timeout) as response:
# A live port is not Studio: a stranger answering every path would get our key.
body = json.loads(response.read(65536).decode() or "{}")
return body.get("service") == _STUDIO_SERVICE_MARKER
except Exception:
return False
def find_studio_server(timeout: float = 3.0) -> Optional[str]:
base = os.environ.get("UNSLOTH_STUDIO_URL", "http://127.0.0.1:8888").rstrip("/")
candidates = _loopback_candidate_bases(base)
# Use the first loopback address that answers, so the flow talks to that exact address.
found = next((c for c in candidates if _answers_as_studio(c, timeout)), None)
if os.environ.get("UNSLOTH_STUDIO_URL"):
return found
recorded = list(_recorded_studio_bases(candidates))
# Another install may own the default port: prefer the server this install recorded.
if found is not None and (not recorded or verify_studio_identity(found, timeout)):
return found
return next((c for c in recorded if _answers_as_studio(c, timeout)), found)
def is_loopback_url(base: str) -> bool:
"""True only when *base* resolves to loopback. find_studio_server() trusts a
base after only a health probe, so credentials are auto-sent only to loopback
(a local Unsloth or an SSH tunnel on 127.0.0.1), the targets the auto flows mean."""
from urllib.parse import urlparse
host = (urlparse(base).hostname or "").lower()
if host in ("localhost", "127.0.0.1", "::1"):
return True
try:
import ipaddress
return ipaddress.ip_address(host).is_loopback
except ValueError:
return False
def verify_studio_identity(base: str, timeout: float = 3.0) -> bool:
"""Confirm `base` is really this machine's Unsloth before sending a secret.
Send a random nonce to /api/auth/identity and check the returned HMAC against
the one computed from the local same-user secret; an endpoint without that
secret (port squatter, remote/fake) can't match. Fails closed on any error."""
import base64
import hmac as _hmac
import json
import secrets as _secrets
import socket
import urllib.request
from urllib.parse import urlparse
try:
import studio.backend.core # noqa: F401 puts studio/backend on sys.path
from studio.backend.auth import storage
except Exception:
return False
parsed = urlparse(base)
host = parsed.hostname or ""
port = parsed.port or (443 if parsed.scheme == "https" else 80)
# Resolve once and talk to that concrete address, binding the proof to (address, port): localhost may hit a squatter on ::1.
try:
ip = socket.getaddrinfo(host, port, type = socket.SOCK_STREAM)[0][4][0]
except Exception:
return False
netloc = f"[{ip}]:{port}" if ":" in ip else f"{ip}:{port}"
nonce = _secrets.token_bytes(32)
query = base64.urlsafe_b64encode(nonce).decode()
request = urllib.request.Request(
f"{parsed.scheme}://{netloc}/api/auth/identity?nonce={query}",
headers = {"User-Agent": _USER_AGENT, "Host": parsed.netloc},
)
try:
# No redirects: a 302 could relay a real Unsloth's proof. Cap the read: still unverified.
with urlopen_no_redirect(request, timeout = timeout) as response:
proof = json.loads(response.read(65536).decode() or "{}").get("proof")
except Exception:
return False
if not isinstance(proof, str):
return False
try:
expected = storage.compute_identity_proof(nonce, ip, port)
except Exception:
return False
return _hmac.compare_digest(proof, expected)
def _studio_token() -> Optional[str]:
"""Signs a JWT with the server's stored secret, since the CLI runs as the same OS user."""
try:
import studio.backend.core # noqa: F401 puts studio/backend on sys.path
from studio.backend.auth import storage
from studio.backend.auth.authentication import create_access_token
row = (
storage.get_connection()
.execute(
"SELECT username FROM auth_user WHERE username = ?",
(storage.DEFAULT_ADMIN_USERNAME,),
)
.fetchone()
)
return create_access_token(row[0], desktop = True) if row else None
except Exception:
return None
class HttpChatBackend:
"""Chat against a running Unsloth server over its OpenAI-compatible API.
close() leaves the model loaded on purpose — the next session (or the
UI) starts instantly.
"""
def __init__(self, base_url: str, token: str) -> None:
self._base = base_url
self._token = token
self.reply_hit_token_limit = False
self.gguf_variant: Optional[str] = None
self.supports_images = False
def _request(
self,
method: str,
path: str,
payload = None,
timeout = None,
):
import json
import urllib.request
request = urllib.request.Request(
self._base + path,
data = None if payload is None else json.dumps(payload).encode(),
headers = {
"Authorization": f"Bearer {self._token}",
"Content-Type": "application/json",
"User-Agent": _USER_AGENT,
},
method = method,
)
# No redirects: this carries a bearer token (see urlopen_no_redirect).
return urlopen_no_redirect(request, timeout = timeout)
def ensure_loaded(
self,
model: str,
*,
hf_token,
max_seq_length,
load_in_4bit,
tensor_parallel: bool = False,
speculative_type: Optional[SpeculativeType] = None,
spec_draft_n_max: Optional[int] = None,
llama_extra_args: Optional[List[str]] = None,
) -> None:
typer.echo(f"Loading {model} on the Unsloth server", err = True)
payload = {
"model_path": model,
"hf_token": hf_token,
"max_seq_length": max_seq_length,
"tensor_parallel": tensor_parallel,
}
if load_in_4bit is not None:
payload["load_in_4bit"] = load_in_4bit
if llama_extra_args:
payload["llama_extra_args"] = llama_extra_args
if speculative_type is not None:
payload["speculative_type"] = speculative_type
if spec_draft_n_max is not None:
payload["spec_draft_n_max"] = spec_draft_n_max
resident_variant = self._resident_gguf_variant(model)
if resident_variant:
payload["gguf_variant"] = resident_variant
self.gguf_variant = resident_variant
try:
# Read the whole body, not just headers: closing early discards the only in-band report of a late load failure.
loaded = read_json_checking_deferred_error(
self._base + "/api/inference/load",
self._request("POST", "/api/inference/load", payload),
)
except Exception as exc:
typer.echo(f"Model load failed: {exc}", err = True)
raise typer.Exit(code = 1)
self.supports_images = isinstance(loaded, dict) and bool(loaded.get("is_vision"))
def _resident_gguf_variant(self, model: str) -> Optional[str]:
if model.lower().endswith(".gguf"):
return None
try:
with self._request("GET", "/api/inference/status", timeout = 30) as response:
status = json.loads(response.read())
except Exception:
return None
if not isinstance(status, dict) or not status.get("is_gguf"):
return None
# A local load's active_model is only its basename.
loaded = status.get("model_identifier") or status.get("active_model")
if not loaded:
return None
loaded = str(loaded)
if model == status.get("model_identifier"):
same = True
elif os.path.exists(model):
# Mirrors the server's _same_loaded_identifier.
same = os.path.normcase(model) == os.path.normcase(loaded)
else:
# The server loads an ownerless id as unsloth/<id> (ModelConfig.from_identifier).
requested = model if "/" in model else f"unsloth/{model}"
same = requested.casefold() == loaded.casefold()
return status.get("gguf_variant") if same else None
def stream(
self,
messages: list,
*,
system_prompt: str,
temperature: Optional[float],
top_p: Optional[float],
top_k: Optional[int],
max_new_tokens: Optional[int],
repetition_penalty: Optional[float],
enable_thinking: bool,
use_adapter: Optional[bool] = None,
):
import json
msgs = list(messages)
if system_prompt:
msgs = [{"role": "system", "content": system_prompt}, *msgs]
body = {
"model": "default",
"messages": msgs,
"stream": True,
"enable_thinking": enable_thinking,
}
sampling = dict(
temperature = temperature,
top_p = top_p,
top_k = top_k,
repetition_penalty = repetition_penalty,
)
body.update({key: value for key, value in sampling.items() if value is not None})
if max_new_tokens is not None:
body["max_tokens"] = max_new_tokens
resp = self._request("POST", "/v1/chat/completions", body)
self.reply_hit_token_limit = False
def cumulative():
# Accumulate SSE deltas into the full-text-so-far form; re-wrap delta.reasoning_content in think tags for visible_text.
text = ""
thinking = False
saw_finish = False
with resp:
for raw_line in resp:
line = raw_line.decode("utf-8", "replace").strip()
if not line.startswith("data:"):
continue
data = line[len("data:") :].strip()
if data == "[DONE]":
return
try:
parsed = json.loads(data)
except ValueError:
continue
if "error" in parsed:
raise RuntimeError(
f"Server error: {parsed['error'].get('message', 'Unknown server error')}"
)
try:
choice = parsed["choices"][0]
except (KeyError, IndexError):
continue
# Read before the delta: the finish-reason chunk may carry no content.
if choice.get("finish_reason") is not None:
saw_finish = True
self.reply_hit_token_limit = choice["finish_reason"] == "length"
try:
reasoning = choice["delta"].get("reasoning_content")
delta = choice["delta"].get("content")
except (KeyError, AttributeError, TypeError):
continue
if reasoning:
if not thinking:
text += _THINK_OPEN
thinking = True
text += reasoning
if delta and thinking:
text += "</think>"
thinking = False
if not (reasoning or delta):
continue
text += delta or ""
# An emoji can arrive as lone surrogate halves across deltas.
visible = text
if "\ud800" <= visible[-1] <= "\udbff":
visible = visible[:-1]
yield visible.encode("utf-16", "surrogatepass").decode("utf-16", "replace")
if not saw_finish:
raise RuntimeError("Stream ended before the reply completed.")
return cumulative()
def close(self) -> None:
pass
def server_load_opts(ctx, load_opts: dict) -> dict:
"""Drop an untyped --load-in-4bit so the server can keep a resident model's precision."""
opts = dict(load_opts)
if ctx.get_parameter_source("load_in_4bit").name != "COMMANDLINE":
opts["load_in_4bit"] = None
return opts
def connect_studio_server(
model: str,
*,
hf_token,
max_seq_length,
load_in_4bit,
tensor_parallel: bool = False,
speculative_type: Optional[SpeculativeType] = None,
spec_draft_n_max: Optional[int] = None,
llama_extra_args: Optional[List[str]] = None,
):
"""Backend on a running Unsloth server, or None (caller loads locally)."""
base_url = find_studio_server()
if not base_url:
return None
# Explicit UNSLOTH_STUDIO_URL we cannot safely attach to fails loudly; local discovery falls back.
explicit = bool(os.environ.get("UNSLOTH_STUDIO_URL"))
def _refuse(reason: str):
if not explicit:
return None
typer.echo(
f"Can't attach to the Unsloth server at {base_url}: {reason} Run Unsloth "
"on this machine, or unset UNSLOTH_STUDIO_URL to load the model locally.",
err = True,
)
raise typer.Exit(code = 1)
# Only hand the self-issued JWT to loopback: a remote URL is unverified.
if not is_loopback_url(base_url):
return _refuse(
"it isn't a local Unsloth, so a self-issued token can't "
"authenticate to it and must not be sent to it."
)
# Confirm the loopback responder is really our Unsloth (not a port squatter).
if not verify_studio_identity(base_url):
return _refuse(
"its identity couldn't be verified (it may be running as a "
"different OS user, or another process took the port)."
)
token = _studio_token()
if not token:
return _refuse("couldn't self-issue an Unsloth token (is Unsloth set up here?).")
backend = HttpChatBackend(base_url, token)
from unsloth_cli._npu import npu_model_path, prepare_npu_model
npu_path = npu_model_path(model)
if npu_path:
# `unsloth inference` passes the bare id the picker shows; load it by its model path.
model = npu_path
try:
prepare_npu_model(
lambda method, path, payload, timeout: backend._request(
method, path, payload, timeout = timeout
),
model,
lambda line: typer.echo(line, err = True),
)
except RuntimeError as exc:
typer.echo(f"Error: {exc}", err = True)
raise typer.Exit(code = 1)
backend.ensure_loaded(
model,
hf_token = hf_token,
max_seq_length = max_seq_length,
load_in_4bit = load_in_4bit,
tensor_parallel = tensor_parallel,
speculative_type = speculative_type,
spec_draft_n_max = spec_draft_n_max,
llama_extra_args = llama_extra_args,
)
return backend