Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Opt-in GGUF embedder (RAG_EMBED_BACKEND=llama-server, no torch): own llama-server --embedding
subprocess on /v1/embeddings and /tokenize; auto falls back to CPU if GPU fails, RAG_EMBED_DEVICE
forces; each request respawns it if dead; only llama_cpp static helpers, no LlamaCppBackend()."""
from __future__ import annotations
from core.training.account_jobs import account_path, managed_account
from utils.account_context import account_thread
from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_memory
import atexit
import logging
import os
import socket
import struct
import subprocess
import sys
import threading
import time
from collections import OrderedDict
from contextlib import contextmanager
from functools import lru_cache
from pathlib import Path
import httpx
import numpy as np
from utils.native_path_leases import child_env_without_native_path_secret
from utils.subprocess_compat import windows_hidden_subprocess_kwargs
from utils.process_lifetime import (
adopt_pid,
child_popen_kwargs,
forget_pid,
is_process_shutting_down,
)
from . import config
from utils.paths.path_utils import is_appledouble_metadata
logger = logging.getLogger(__name__)
# httpx transport errors meaning "the server is gone" -> trigger a respawn.
_TRANSPORT_ERRORS = (
httpx.ConnectError,
httpx.ReadError,
httpx.RemoteProtocolError,
httpx.WriteError,
)
_GGUF_SCALAR_WIDTHS = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8}
# Matches llama.cpp's -ub default; a longer embedding prompt is refused with a 500.
_UBATCH_SIZE = 512
_TOKEN_COUNT_CACHE_SIZE = 4096
_TOKENIZE_BATCH = 256
_TOKENIZE_BATCH_CHARS = 262144
def _skip_gguf_value(f, vtype: int) -> None:
if vtype == 8:
f.seek(struct.unpack("<Q", f.read(8))[0], 1)
elif vtype == 9:
elem_type, count = struct.unpack("<IQ", f.read(12))
if elem_type in _GGUF_SCALAR_WIDTHS:
f.seek(_GGUF_SCALAR_WIDTHS[elem_type] * count, 1)
else:
for _ in range(count):
_skip_gguf_value(f, elem_type)
else:
f.seek(_GGUF_SCALAR_WIDTHS[vtype], 1)
def _gguf_arch_uint(path: str, field: str) -> tuple[str | None, int | None]:
try:
with open(path, "rb") as f:
if f.read(4) != b"GGUF":
return None, None
f.read(4)
_, kv_count = struct.unpack("<QQ", f.read(16))
arch = None
values: dict[str, int] = {}
for _ in range(kv_count):
key = f.read(struct.unpack("<Q", f.read(8))[0]).decode("utf-8")
vtype = struct.unpack("<I", f.read(4))[0]
if key == "general.architecture" and vtype == 8:
arch = f.read(struct.unpack("<Q", f.read(8))[0]).decode("utf-8")
elif key.endswith(f".{field}") and vtype in (4, 10):
values[key] = int.from_bytes(f.read(_GGUF_SCALAR_WIDTHS[vtype]), "little")
else:
_skip_gguf_value(f, vtype)
if arch is not None and f"{arch}.{field}" in values:
return arch, values[f"{arch}.{field}"]
except (OSError, struct.error, UnicodeDecodeError, KeyError, ValueError):
return None, None
return arch, None
def _gguf_context_length(path: str) -> int | None:
return _gguf_arch_uint(path, "context_length")[1] or None
_GGUF_POOLING = {1: "mean", 2: "cls", 3: "last"}
_GGUF_POOLING_DEFAULT = {"nomic-bert": "mean", "nomic-bert-moe": "mean"}
@lru_cache(maxsize = 8)
def _embed_host(host: str) -> str:
# One address for bind, URL and owner check: a multi-address name can bind one and dial another.
try:
ips = [a[4][0] for a in socket.getaddrinfo(host, None, type = socket.SOCK_STREAM)]
except OSError:
return host
v4 = [ip for ip in ips if ":" not in ip]
return (v4 or ips or [host])[0]
@lru_cache(maxsize = 32)
def _gguf_pooling_at(path: str, mtime_ns: int, size: int) -> str:
arch, value = _gguf_arch_uint(path, "pooling_type")
if value in _GGUF_POOLING:
return _GGUF_POOLING[value]
if value is None:
# Older Nomic embedders omit pooling metadata: mean-pool that family; unknown archs keep legacy CLS.
return _GGUF_POOLING_DEFAULT.get((arch or "").lower(), "cls")
return "cls"
def _gguf_pooling(path: str) -> str:
try:
st = os.stat(path)
except OSError:
return "cls"
return _gguf_pooling_at(path, st.st_mtime_ns, st.st_size)
def _resolve_entrypoint(binary: str) -> str:
"""Resolves past a wrapper on macOS, since SIP purges DYLD_* when the wrapper's /bin/sh starts."""
try:
from core.inference.llama_cpp import LlamaCppBackend
return LlamaCppBackend._exec_path_for_launch(binary) or binary
except Exception: # noqa: BLE001 - launch what we were given
return binary
def _binary_lib_dir(binary: str) -> str:
"""Directory holding ``binary``'s sibling libraries, through an entrypoint."""
try:
from core.inference.llama_cpp import _llama_lib_dir
return str(_llama_lib_dir(binary))
except Exception: # noqa: BLE001 - fall back to the plain parent
return str(Path(binary).parent)
def _with_dyld_path(env: dict[str, str], lib_dir: str) -> dict[str, str]:
"""Returns a copy with lib_dir first on DYLD_LIBRARY_PATH, sharing the chat backend's prepend."""
out = dict(env)
try:
from core.inference.llama_cpp import _prepend_loader_dir
out["DYLD_LIBRARY_PATH"] = _prepend_loader_dir(out.get("DYLD_LIBRARY_PATH", ""), lib_dir)
except Exception: # noqa: BLE001 - a search path is better than none
existing = [p for p in out.get("DYLD_LIBRARY_PATH", "").split(os.pathsep) if p]
out["DYLD_LIBRARY_PATH"] = os.pathsep.join(dict.fromkeys([lib_dir, *existing]))
return out
class LlamaServerBackend:
"""Manages a llama.cpp embedding subprocess and talks to it over HTTP."""
def __init__(self) -> None:
# Lifecycle (spawn/restart/kill) is serialized; HTTP requests are not.
self._lifecycle_lock = threading.Lock()
# Unload waits for in-flight encode/tokenize/probe calls and bars restarts after unpublish.
self._operation_condition = threading.Condition()
self._active_operations = 0
self._operation_local = threading.local()
# Repo/pooling from this thread's last encode, since the shared backend may switch models meanwhile.
self._served_identity_local = threading.local()
self._closed = False
self._process: subprocess.Popen | None = None
self._port: int | None = None
self._stdout_lines: list[str] = []
self._stdout_thread: threading.Thread | None = None
# dim() takes _serve_lock like requests; reentrant because dim() calls encode() on the same thread.
self._dim: int | None = None
self._max_tokens: int | None = None
# Readiness and request must be atomic: a model swap between them misroutes the POST.
self._serve_lock = threading.RLock()
self._model_path: str | None = None
# Read when the path is adopted: the running server keeps this pooling even if the file later vanishes.
self._model_pooling: str | None = None
# A Settings change makes the cached path/dim stale, forcing a re-resolve and respawn (see _ensure_ready).
self._model_repo: str | None = None
self._binary: str | None = None
self._binary_path_revision: int | None = None
# Sticky after an auto GPU start fails or a GPU server crashes twice on one request.
self._force_cpu = False
self._on_gpu = False
# trust_env=False skips HTTP(S)_PROXY; full URLs per request survive a respawn.
self._client = httpx.Client(timeout = config.EMBED_REQUEST_TIMEOUT_S, trust_env = False)
self._api_key: str | None = None
atexit.register(self._shutdown)
@contextmanager
def _operation(self):
depth = getattr(self._operation_local, "depth", 0)
with self._operation_condition:
if self._closed and depth == 0:
raise RuntimeError("llama-server embedding backend was unloaded")
if depth == 0:
self._active_operations += 1
self._operation_local.depth = depth + 1
try:
yield
finally:
with self._operation_condition:
next_depth = self._operation_local.depth - 1
self._operation_local.depth = next_depth
if next_depth == 0:
self._active_operations -= 1
if self._active_operations == 0:
self._operation_condition.notify_all()
@property
def _base_url(self) -> str:
host = _embed_host(config.EMBED_HOST)
return f"http://{f'[{host}]' if ':' in host else host}:{self._port}"
def _auth_headers(self) -> dict[str, str]:
return {"Authorization": f"Bearer {self._api_key}"} if self._api_key else {}
def _resolve_binary(self) -> str:
"""Find llama-server, verify embeddings support, cache it. Raises if
missing/unsupported."""
if self._binary is not None:
return self._binary
from core.inference.llama_cpp import LlamaCppBackend
from utils.llama_cpp_path_settings import custom_llama_cpp_path_revision
path_revision = custom_llama_cpp_path_revision()
binary = LlamaCppBackend._find_llama_server_binary()
if not binary:
raise RuntimeError(
"llama-server binary not found; cannot use RAG_EMBED_BACKEND="
"llama-server. Install llama.cpp or set LLAMA_SERVER_PATH / "
"UNSLOTH_LLAMA_CPP_PATH."
)
# Resolve the real executable before the probe: an entrypoint loses DYLD_* to SIP.
binary = _resolve_entrypoint(binary)
self._assert_embedding_support(binary)
self._binary = binary
self._binary_path_revision = path_revision
return binary
@staticmethod
@lru_cache(maxsize = 8)
def _help_text(binary: str) -> str:
"""Runs under the real launch's loader environment on macOS, else a loader error reads as help."""
probe_env = None
if sys.platform == "darwin":
try:
from core.inference.llama_cpp import LlamaCppBackend
probe_env = LlamaCppBackend._llama_server_env_for_binary(binary)
except Exception: # noqa: BLE001 - probe with the inherited env
probe_env = None
try:
proc = subprocess.run(
[binary, "--help"],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 30,
env = probe_env,
**windows_hidden_subprocess_kwargs(),
)
return (proc.stdout or "") + (proc.stderr or "")
except Exception as e: # noqa: BLE001
logger.warning("could not run `llama-server --help`: %s", e)
return ""
def _assert_embedding_support(self, binary: str) -> None:
help_text = self._help_text(binary)
# Empty help -> assume capable (a missing flag still fails at spawn).
if help_text and "--embedding" not in help_text:
raise RuntimeError(
"the bundled llama-server build lacks --embedding support; "
"RAG_EMBED_BACKEND=llama-server requires an embeddings-capable build"
)
@staticmethod
def _resolve_local_gguf(model: str) -> str | None:
"""A custom model may be a local .gguf file or a directory holding one;
resolve it without the hub. None when the value is not a local path."""
from utils.models.model_config import _local_gguf_load_path, colocated_split_shards
from utils.paths import normalize_path
# Normalize like the other probes, else a WSL user's C: path GGUF goes to the Hub as a bad filename.
p = Path(normalize_path(model)).expanduser()
if p.is_file() and p.suffix.lower() == ".gguf":
return None if is_appledouble_metadata(p) else str(p)
if p.is_dir():
files = [f for f in p.iterdir() if not is_appledouble_metadata(f)]
# Same pick as a hub listing gets, and its name tiebreak resolves a directory of split shards to shard 1.
remaining = list(files)
while remaining:
picked = LlamaServerBackend._pick_gguf(remaining, key = lambda f: f.name)
if picked is None:
break
# llama-server opens split siblings implicitly, so an incomplete family is unusable.
if colocated_split_shards(picked)[1]:
# Enter a split family at shard 1 with its full symlink set, as the chat loader does for local GGUF.
return str(_local_gguf_load_path(picked))
remaining = [f for f in remaining if f != picked]
raise RuntimeError(f"no .gguf file found in local model dir {model!r}")
return None
@classmethod
def _pick_complete_gguf(cls, names):
"""Skips a torn GGUF family, so the loop tries the next repo instead of one it cannot finish."""
remaining = list(names)
while remaining:
picked = cls._pick_gguf(remaining)
if picked is None:
return None, []
family = cls._split_family(names, picked)
if family:
return picked, family
remaining = [name for name in remaining if name != picked]
return None, []
@staticmethod
def _split_family(names, picked: str):
"""A lone shard is not a plan: llama-server opens split siblings implicitly, so all must be
fetched."""
from pathlib import PurePosixPath
from utils.models.model_config import _GGUF_SPLIT_FILE_RE
path = PurePosixPath(picked)
match = _GGUF_SPLIT_FILE_RE.match(path.name)
if match is None:
return [picked]
total = int(match.group("total"))
if total < 1:
return None
found: dict[int, str] = {}
for name in names:
other = PurePosixPath(name)
other_match = _GGUF_SPLIT_FILE_RE.match(other.name)
if (
other.parent != path.parent
or other_match is None
or other_match.group("prefix").casefold() != match.group("prefix").casefold()
or other_match.group("total") != match.group("total")
):
continue
index = int(other_match.group("index"))
if 1 <= index <= total:
found[index] = name
if len(found) != total:
return None
return [found[index] for index in range(1, total + 1)]
@staticmethod
def _pick_gguf(
names,
*,
key = None,
require_variant = False,
):
"""Refuses a variant fallback: on a hub listing it means unpublished, in cache it may be stale."""
from utils.models.model_config import _is_mtp_drafter
name_of = key or (lambda n: n)
usable = [
n
for n in names
if name_of(n).lower().endswith(".gguf")
and "mmproj" not in name_of(n).lower()
# Drafters are companions like mmproj: a cache holding only one would read as holding the embedder.
and not _is_mtp_drafter(name_of(n))
]
if not usable:
return None
variant = config.EMBED_GGUF_VARIANT.lower()
match = [n for n in usable if variant in name_of(n).lower()]
if not match:
if require_variant:
return None
match = usable
# Name breaks a length tie: a directory scan yields no guaranteed order.
return sorted(match, key = lambda n: (len(name_of(n)), name_of(n)))[0]
@staticmethod
def _cached_snapshot_dir(repo_id: str) -> Path | None:
"""Pins the embedder to refs/main, since the stored identity records no revision to compare
against."""
from utils.hf_cache_settings import active_hf_hub_cache
from utils.paths import resolve_cached_repo_id_case
# Casing-differing repo ids still match the cache folder; a miss re-downloads or fails offline.
cased = resolve_cached_repo_id_case(repo_id)
repo_dir = Path(active_hf_hub_cache()) / f"models--{cased.replace('/', '--')}"
try:
rev = (repo_dir / "refs" / "main").read_text(encoding = "utf-8").strip()
except (OSError, ValueError):
return None
# A ref file holds a bare commit hash; a separator would join out of the cache.
if not rev or rev in (".", "..") or "/" in rev or "\\" in rev:
return None
snapshot = repo_dir / "snapshots" / rev
return snapshot if snapshot.is_dir() else None
@staticmethod
def _resolve_cached_gguf(repo_id: str, *, require_variant: bool = True) -> str | None:
"""Finds the GGUF on disk without the network, so restarts do not re-list a repo that may
blackhole."""
from utils.models.model_config import colocated_split_shards
snapshot = LlamaServerBackend._cached_snapshot_dir(repo_id)
if snapshot is None:
return None
try:
# Not a *.gguf glob: it is case-sensitive and misses quant-per-directory layouts.
files = [
p for p in snapshot.rglob("*") if p.is_file() and not is_appledouble_metadata(p)
]
except OSError:
return None
# Split sets serve only when whole; drop an unservable pick and retry, or it shadows a complete one.
while files:
pick = LlamaServerBackend._pick_gguf(
files,
key = lambda p: p.relative_to(snapshot).as_posix(),
require_variant = require_variant,
)
# exists() guards only the window since the scan; is_file() dropped evicted blobs.
if pick is None or not pick.exists():
return None
if colocated_split_shards(pick)[1]:
return str(pick)
files = [p for p in files if p != pick]
return None
@staticmethod
def cached_pooling(model: str) -> str | None:
"""Pooling of ``model``'s GGUF already on disk, found without the network; None when
the loader cannot yet know which file it will serve. Match its local, planned and configured-
variant order so identity prediction never reads metadata from a different cached family."""
try:
desired = config.effective_gguf_repo_for_embedding_model(model)
path = LlamaServerBackend._resolve_local_gguf(model)
path = path or LlamaServerBackend._planned_family_path(model, desired)
path = path or LlamaServerBackend._resolve_cached_gguf(desired)
if path is None:
try:
from utils.embedding_model_settings import get_stored_download_pending
download_pending = get_stored_download_pending(model)
except Exception: # noqa: BLE001 - old/unavailable settings store
download_pending = False
if download_pending:
path = LlamaServerBackend._resolve_cached_gguf(desired, require_variant = False)
except Exception: # noqa: BLE001 - identity prediction must not block ingestion
return None
return None if path is None else _gguf_pooling(path)
def _resolve_model_path(self, model_name: str | None = None) -> str:
"""Uses the caller's pinned model, not the live setting, which would resolve other weights."""
account_path(model_name, reference = True)
model = model_name or config.effective_embedding_model()
# Captured once; tagged with its repo, so a mid-download setting change reads as stale, respawning.
desired = config.effective_gguf_repo_for_embedding_model(model)
if self._model_path is not None and self._model_repo == desired:
if Path(self._model_path).is_file():
return self._model_path
# Process stopped here; a remembered cache path may be evicted, so re-resolve via cache/Hub.
self._model_path = None
self._model_pooling = None
self._dim = None
self._max_tokens = None
local = self._resolve_local_gguf(model)
if local is not None:
return self._adopt_model_path(local, desired)
# Identity carries the model and repo but not the file, so a preferred variant arriving later would
# change the weights under an existing index without changing its tag.
planned = self._planned_family_path(model, desired)
if planned is not None:
try:
from utils.embedding_model_settings import clear_stored_download_pending
clear_stored_download_pending(model)
except Exception: # noqa: BLE001 - a settings write must not fail a load
pass
return self._adopt_model_path(planned, desired)
# Companion repo, then the naming variants, then the model repo itself.
repo = desired
candidates = list(dict.fromkeys([repo, *config.gguf_repo_candidates(model)]))
# Only the preferred repo: a file cached under the fallback must not pre-empt a companion repo the
# hub can still resolve.
cached = self._resolve_cached_gguf(desired)
if cached is not None:
# Retire the pending marker here, or the model stays cache-only for the life of the install and a
# later eviction reads as "never downloaded".
try:
from utils.embedding_model_settings import clear_stored_download_pending
clear_stored_download_pending(model)
except Exception: # noqa: BLE001 - a settings write must not fail a load
pass
return self._adopt_model_path(cached, desired)
try:
from utils.embedding_model_settings import get_stored_download_pending
download_pending = get_stored_download_pending(model)
except Exception: # noqa: BLE001 - old/unavailable settings store
download_pending = False
if download_pending:
# A published repo may not carry the configured quant, so accept any complete cached quant; never
# fall through to the online loader while the transfer is pending or cancelled.
cached = self._resolve_cached_gguf(desired, require_variant = False)
if cached is not None:
# Reaching here means the configured variant is NOT cached (the strict pass returned if it
# were), which happens two ways that need opposite answers.
if self._is_planned_family(model, desired, cached):
# Serve a stand-in instead of failing the index; keep the marker or it pins the wrong quant.
try:
from utils.embedding_model_settings import clear_stored_download_pending
clear_stored_download_pending(model)
except Exception: # noqa: BLE001 - a settings write must not fail a load
pass
# Serve a stand-in rather than fail the index, but leave the marker: retiring on it would pin this
# model to the wrong quant.
return self._adopt_model_path(cached, desired)
# Typed: the same condition the ST path raises, and /v1/embeddings answers it
# with a 409. Bare, it fell into the catch-all as a 502 naming nothing.
from .embeddings import EmbeddingModelDownloadRequiredError, download_required_message
raise EmbeddingModelDownloadRequiredError(download_required_message(model))
return self._resolve_uncached_model_path(desired, candidates)
@staticmethod
def _planned_family_path(model: str, repo_id: str) -> str | None:
"""Whole family or nothing: a partial family is a torn transfer llama-server cannot open."""
try:
from pathlib import PurePosixPath
from utils.embedding_model_settings import get_stored_gguf_files
planned = get_stored_gguf_files(model)
if not planned:
return None
snapshot = LlamaServerBackend._cached_snapshot_dir(repo_id)
if snapshot is None:
return None
paths = []
for name in planned:
relative = PurePosixPath(name)
if relative.is_absolute() or ".." in relative.parts:
return None
path = snapshot.joinpath(*relative.parts)
if not path.is_file():
return None
paths.append(path)
# Ordered shard 1..N by the planner, so the first is where llama-server enters the family.
return str(paths[0]) if paths else None
except Exception: # noqa: BLE001 - an unreadable record answers nothing
return None
@staticmethod
def _is_planned_family(model: str, repo_id: str, path: str) -> bool:
"""Compares the path relative to the snapshot: base names collide when each quant has its
own folder."""
try:
from pathlib import PurePosixPath
from utils.embedding_model_settings import get_stored_gguf_files
planned = get_stored_gguf_files(model)
if not planned:
return False
snapshot = LlamaServerBackend._cached_snapshot_dir(repo_id)
if snapshot is None:
return False
relative = Path(path).relative_to(snapshot)
names = {PurePosixPath(name).as_posix().casefold() for name in planned}
return PurePosixPath(relative.as_posix()).as_posix().casefold() in names
except Exception: # noqa: BLE001 - an unreadable record or an outside path answers nothing
return False
def _adopt_model_path(self, path: str, desired: str) -> str:
"""Serve `path` for `desired`; the width is re-probed against whatever is adopted."""
self._model_path = path
self._model_repo = desired
self._model_pooling = _gguf_pooling(path) if path else None
self._dim = None
self._max_tokens = None
return self._model_path
# A listing, not a transfer: a working hub answers well inside a second.
_LIST_DEADLINE_S = 20.0
def _resolve_uncached_model_path(self, desired: str, candidates: list[str]) -> str:
"""list_repo_files has no timeout, so the listing needs a deadline; transfers are bounded
per read."""
from huggingface_hub import hf_hub_download, list_repo_files
from core.inference.llama_cpp import _hf_offline_if_unreachable
from utils.utils import call_with_deadline
token = False if managed_account() else os.environ.get("HF_TOKEN") or None
repo = desired
filename: str | None = None
family: list[str] = []
errors: list[str] = []
with _hf_offline_if_unreachable():
for candidate in candidates:
try:
names = call_with_deadline(
lambda c = candidate: list_repo_files(c, token = token),
self._LIST_DEADLINE_S,
name = "embed-gguf-listing",
)
filename, family = self._pick_complete_gguf(names)
except Exception as e: # noqa: BLE001 - missing/gated/stalled -> next candidate
errors.append(f"{candidate!r}: {e}")
continue
if filename is not None:
repo = candidate
break
errors.append(f"{candidate!r}: no .gguf files")
if filename is None:
# Only where the hub failed to NAME a file: a transfer failing on its own (no disk, bad checksum)
# must surface.
for candidate in candidates:
cached = self._resolve_cached_gguf(candidate, require_variant = False)
if cached is not None:
logger.warning(
"using cached GGUF embedder %s: the %s variant could not be "
"resolved (%s)",
cached,
config.EMBED_GGUF_VARIANT,
"; ".join(errors),
)
return self._adopt_model_path(cached, desired)
raise RuntimeError("no .gguf embedder found; tried " + "; ".join(errors))
logger.info("resolving GGUF embedder %s/%s", repo, filename)
from utils.hf_cache_settings import active_hf_hub_cache
cache_dir = active_hf_hub_cache()
# Fetch the whole split family, not just the picked shard: -m shard1 makes llama-server open the
# siblings itself.
downloaded = None
for name in family:
fetched = hf_hub_download(
repo_id = repo,
filename = name,
token = token,
cache_dir = cache_dir,
)
if name == filename:
downloaded = fetched
return self._adopt_model_path(downloaded, desired)
# Min free VRAM (MiB) for the embedder; below this, auto stays on CPU.
_MIN_GPU_FREE_MIB = 1024
def _use_gpu(self) -> bool:
"""A sticky CPU fallback overrides auto, so a restart does not pay the GPU startup timeout again."""
dev = config.embed_device_preference()
if dev == "gpu" and not self._force_cpu:
return True
if dev == "cpu" or self._force_cpu:
return False
return self._gpu_available()
@staticmethod
def _gpu_available() -> bool:
"""Smi-first GPU probe leaves no CUDA/HIP context behind; llama-server opts into the ROCm
arch gate."""
from utils.hardware import is_apple_silicon
if is_apple_silicon():
return True
from core.inference.llama_cpp import LlamaCppBackend
# [(idx, free_mib)], honors CVD
gpus = LlamaCppBackend._get_gpu_free_memory(for_llama_server = True)
return any(free >= LlamaServerBackend._MIN_GPU_FREE_MIB for _, free in gpus)
@staticmethod
def _arch_gated_gpu_ids(binary: str) -> list[int]:
"""Pins survivors, not just a supported device: an uncovered ROCm agent still crashes the child."""
from core.inference.llama_cpp import LlamaCppBackend
return LlamaCppBackend._arch_gate_survivors(binary)
def _build_cmd(
self,
binary: str,
model_path: str,
port: int,
*,
use_gpu: bool,
pooling: str | None = None,
) -> list[str]:
# No --embd-normalize (missing in some builds; normalized in Python); --fit off keeps ctx fixed.
cmd = [
binary,
"-m",
model_path,
"--host",
_embed_host(config.EMBED_HOST),
"--port",
str(port),
"--embedding",
"--pooling",
pooling or _gguf_pooling(model_path),
"--fit",
"off",
]
# Raising -ub preallocates n_vocab * n_ubatch * 4 bytes; /props does not report the batch.
cmd += ["-ub", str(_UBATCH_SIZE)]
# -1 offloads every layer (matches the chat server); 0 keeps it on CPU.
cmd += ["-ngl", "-1" if use_gpu else "0"]
return cmd
def _build_env(self, binary: str, *, use_gpu: bool) -> dict[str, str]:
env = child_env_without_native_path_secret()
# Strip inherited GGML_CUDA_P2P=0 here too: llama.cpp reads it as ON (#10613).
from core.inference.llama_cpp import LlamaCppBackend
LlamaCppBackend._sanitize_p2p_env(env)
env["LLAMA_SET_ROWS"] = "1"
if sys.platform == "darwin":
# Managed installs put an entrypoint in front; dylibs sit next to the real target (#8566).
env = _with_dyld_path(env, _binary_lib_dir(binary))
elif use_gpu:
if sys.platform == "win32":
# Chat's DLL search path: without it a venv-hosted cudart is never found and the CUDA build runs on the CPU.
from core.inference.llama_cpp import _llama_lib_dir
path_dirs = LlamaCppBackend._build_windows_path_dirs(
str(_llama_lib_dir(binary)), sys.prefix, os.environ.get("CUDA_PATH", "")
)
env["PATH"] = ";".join(path_dirs) + ";" + env.get("PATH", "")
else:
# Path(binary).parent, unresolved: resolving would move the first LD_LIBRARY_PATH entry of every symlinked install.
self._add_linux_cuda_libs(env, str(Path(binary).parent))
_pinned = self._arch_gated_gpu_ids(binary)
if _pinned:
from core.inference.llama_cpp import LlamaCppBackend
# prefer_rocr: a HIP-only mask still lets HSA enumerate, and die on, the unsupported agent.
LlamaCppBackend._emit_child_gpu_visibility(
env, ",".join(str(i) for i in _pinned), prefer_rocr = True
)
logger.info("pinning the embed server to arch-supported GPU(s) %s", _pinned)
# Not else: a macOS CPU start still has to blank the devices.
if not use_gpu:
# Blank devices so a CUDA build stays on CPU and reserves no VRAM.
env["CUDA_VISIBLE_DEVICES"] = ""
# HIP ignores CUDA_VISIBLE_DEVICES when HIP_VISIBLE_DEVICES is set; leave ROCR mask alone (#7624).
env["HIP_VISIBLE_DEVICES"] = "-1"
# A stale LLAMA_ARG_DEVICE/MAIN_GPU naming a hidden device makes llama.cpp exit, not fall back.
from core.inference.llama_cpp import LlamaCppBackend
LlamaCppBackend._clear_device_placement_env(env)
return env
@staticmethod
def _add_linux_cuda_libs(env: dict[str, str], binary_dir: str) -> None:
"""Best-effort LD_LIBRARY_PATH so the prebuilt binary finds CUDA libs."""
import glob
import platform
if sys.platform == "win32":
return
arch = platform.machine()
lib_dirs = [binary_dir]
# glob.escape: a prefix with [brackets] is otherwise read as a pattern.
site = os.path.join(glob.escape(sys.prefix), "lib", "python*", "site-packages")
pip_dirs = []
for pattern in (
os.path.join(site, "nvidia", "cu*", "lib"),
os.path.join(site, "nvidia", "cudnn", "lib"),
os.path.join(site, "torch", "lib"),
):
pip_dirs.extend(d for d in glob.glob(pattern) if os.path.isdir(d))
system_dirs = [
d
for d in (
"/usr/local/cuda/lib64",
f"/usr/local/cuda/targets/{arch}-linux/lib",
"/usr/local/cuda-12/lib64",
)
if os.path.isdir(d)
]
from utils.tegra import TEGRA_LIB_DIR, is_tegra
if is_tegra():
tegra = [TEGRA_LIB_DIR] if os.path.isdir(TEGRA_LIB_DIR) else []
lib_dirs.extend(tegra + system_dirs + pip_dirs)
else:
lib_dirs.extend(pip_dirs + system_dirs)
existing = env.get("LD_LIBRARY_PATH", "")
joined = ":".join(lib_dirs)
env["LD_LIBRARY_PATH"] = f"{joined}:{existing}" if existing else joined
def _drain_stdout(self, proc: subprocess.Popen) -> None:
"""Drain the child's stdout so its pipe buffer never deadlocks; keep the
tail for crash diagnostics."""
try:
for line in proc.stdout: # type: ignore[union-attr]
line = line.rstrip()
if line:
self._stdout_lines.append(line)
if len(self._stdout_lines) > 200:
del self._stdout_lines[:-200]
logger.debug("[llama-embed] %s", line)
except Exception: # noqa: BLE001 - drain thread must never raise
pass
@_invalidates_gpu_memory("embedding server start")
def _spawn(self, model_name: str | None = None) -> None:
"""Start the embed server (caller holds the lock). A failed GPU start falls
back to CPU once, unless ``RAG_EMBED_DEVICE`` is the literal ``gpu``."""
use_gpu = self._use_gpu()
try:
self._spawn_once(use_gpu, model_name)
except RuntimeError:
# A shutdown refusal is terminal: the CPU fallback would just spawn into the
# same latch and be refused again.
if is_process_shutting_down():
raise
if use_gpu and not config.embed_device_requires_gpu():
logger.warning("embed server GPU start failed; falling back to CPU")
self._force_cpu = True
self._spawn_once(False, model_name)
else:
raise
def _spawn_once(
self,
use_gpu: bool,
model_name: str | None = None,
) -> None:
binary = self._resolve_binary()
model_path = self._resolve_model_path(model_name)
# The cached path can be replaced in place between server lifetimes. Capture pooling once
# per spawn so the explicit llama-server flag and the vectors' recorded identity agree.
pooling = _gguf_pooling(model_path)
self._model_pooling = pooling
port = config.EMBED_PORT or self._find_free_port()
env = self._build_env(binary, use_gpu = use_gpu)
cmd = self._build_cmd(binary, model_path, port, use_gpu = use_gpu, pooling = pooling)
logger.info(
"starting llama-server embedder (%s): %s",
"gpu" if use_gpu else "cpu",
" ".join(cmd),
)
from core.inference.llama_cpp import (
LlamaCppBackend,
_llama_server_api_key_enabled,
_llama_server_key_launch,
)
self._api_key = key_file = None
if _llama_server_api_key_enabled() and LlamaCppBackend.probe_server_capabilities(
binary
).get("supports_api_key_file", True):
# Per-spawn key against permissive CORS; read once at startup, so deleted once healthy.
import secrets
self._api_key = secrets.token_urlsafe(32)
key_argv, key_env, key_file = _llama_server_key_launch(self._api_key)
cmd[1:1] = key_argv
env.update(key_env)
self._stdout_lines = []
# No graceful-shutdown step covers this backend, so gate every spawn on the shutdown latch.
if is_process_shutting_down():
raise RuntimeError("Unsloth is shutting down; not starting the embed server")
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
env = env,
**windows_hidden_subprocess_kwargs(),
**child_popen_kwargs(),
)
self._process = proc
self._on_gpu = use_gpu
# child_popen_kwargs() is empty on macOS, so the crash record is the only thing that can reap it
# after a force quit.
adopt_pid(proc.pid)
# Recheck after recording the pid: the latch may have been set after the gate above.
if is_process_shutting_down():
logger.info("shutdown began during the spawn; killing the new embed server")
self._kill_process()
raise RuntimeError("Unsloth is shutting down; not starting the embed server")
self._port = port
self._stdout_thread = account_thread(
target = self._drain_stdout,
args = (proc,),
daemon = True,
name = "llama-embed-stdout",
)
self._stdout_thread.start()
try:
healthy = self._wait_for_health(config.EMBED_STARTUP_TIMEOUT_S)
finally:
if key_file is not None:
try:
key_file.unlink()
except OSError:
pass
if not healthy:
tail = "\n".join(self._stdout_lines[-30:])
self._kill_process()
raise RuntimeError(
f"llama-server embedder failed to become healthy. Last output:\n{tail[:2000]}"
)
@staticmethod
def _find_free_port() -> int:
from core.inference.llama_cpp import LlamaCppBackend
return LlamaCppBackend._find_free_port()
def _wait_for_health(
self,
timeout: float,
interval: float = 0.5,
) -> bool:
"""Poll /health until 200; bail early if the process exits."""
from core.inference.llama_cpp import LlamaCppBackend
deadline = time.monotonic() + timeout
url = f"{self._base_url}/health"
while time.monotonic() < deadline:
if not self._process_alive():
code = None if self._process is None else self._process.returncode
logger.error("llama-server embedder exited early (code %s)", code)
return False
try:
# trust_env=False: a proxy that 503s 127.0.0.1 must not block this probe.
if httpx.get(url, timeout = 2.0, trust_env = False).status_code == 200:
return LlamaCppBackend._health_listener_is_ours(
self._process, self._port, _embed_host(config.EMBED_HOST)
)
except (*_TRANSPORT_ERRORS, httpx.TimeoutException):
pass
time.sleep(interval)
logger.error("llama-server embedder health check timed out after %ss", timeout)
return False
def _process_alive(self) -> bool:
return self._process is not None and self._process.poll() is None
def _current(self, model_name: str | None = None) -> bool:
"""Alive AND serving the repo for ``model_name`` (a Settings model change,
or a request pinned to a different model, makes a live server stale)."""
from utils.llama_cpp_path_settings import custom_llama_cpp_path_revision
desired = config.effective_gguf_repo_for_embedding_model(
model_name or config.effective_embedding_model()
)
return (
self._process_alive()
and self._model_repo == desired
and self._binary_path_revision == custom_llama_cpp_path_revision()
)
def pooling_identity_snapshot(self) -> tuple[str | None, str | None, str | None, bool]:
"""One coherent path/repo/pooling/process snapshot for pre-encode identity prediction."""
with self._serve_lock:
return self._model_path, self._model_repo, self._model_pooling, self._process_alive()
def _ensure_ready(self, model_name: str | None = None) -> None:
"""One subprocess serves one GGUF, so a request for another model respawns the server onto it."""
if self._current(model_name):
return
with self._lifecycle_lock:
if self._current(model_name):
return
self._kill_process()
from utils.llama_cpp_path_settings import custom_llama_cpp_path_revision
if self._binary_path_revision != custom_llama_cpp_path_revision():
self._binary = None
self._binary_path_revision = None
# Must be cleared here: _resolve_model_path fast-paths on an unchanged repo.
self._max_tokens = None
self._spawn(model_name)
def _restart(self, model_name: str | None = None) -> None:
with self._lifecycle_lock:
self._kill_process()
self._spawn(model_name)
@_invalidates_gpu_memory("embedding server stop")
def _kill_process(self) -> None:
proc = self._process
if proc is None:
return
try:
proc.terminate()
proc.wait(timeout = 5)
except subprocess.TimeoutExpired:
logger.warning("llama-server embedder did not exit on SIGTERM; killing")
proc.kill()
try:
proc.wait(timeout = 5)
except Exception: # noqa: BLE001
pass
except Exception as e: # noqa: BLE001
logger.warning("error killing llama-server embedder: %s", e)
finally:
# A survivor must stay recorded so the next startup sweep can reap it.
if proc.poll() is not None:
forget_pid(proc.pid)
self._process = None
if self._stdout_thread is not None:
self._stdout_thread.join(timeout = 2)
self._stdout_thread = None
def _shutdown(self) -> None:
with self._operation_condition:
self._closed = True
while self._active_operations:
self._operation_condition.wait()
try:
with self._lifecycle_lock:
self._kill_process()
finally:
try:
self._client.close()
except Exception: # noqa: BLE001
pass
# Embedder may swap mid-call, so report the backend actually used (see encode_with_identity).
try:
atexit.unregister(self._shutdown)
except Exception: # noqa: BLE001 - teardown must not raise
pass
def _post(
self,
path: str,
payload: dict,
model_name: str | None = None,
) -> dict:
"""POST to the server, restarting once and retrying on a dropped connection
(the reaper may have killed us) or a timeout (the bundled build sometimes
wedges a request); a fresh server unsticks both."""
with self._operation():
return self._post_active(path, payload, model_name)
def _post_active(
self,
path: str,
payload: dict,
model_name: str | None = None,
) -> dict:
last_exc: Exception | None = None
for attempt in range(2):
with self._serve_lock:
# Held across readiness AND the request, so a swap can only land between whole requests.
self._ensure_ready(model_name)
try:
resp = self._client.post(
f"{self._base_url}{path}", json = payload, headers = self._auth_headers()
)
resp.raise_for_status()
return resp.json()
except (*_TRANSPORT_ERRORS, httpx.TimeoutException) as e:
last_exc = e
crashed = not isinstance(e, httpx.TimeoutException) and self._server_exited()
if attempt == 0:
self._restart(model_name)
continue
# A second crash is the GPU build failing on this input, not the reaper (#13663).
if crashed and self._fall_back_to_cpu():
self._restart(model_name)
return self._post_active(path, payload, model_name)
except httpx.HTTPStatusError as e:
body = e.response.text[:500] if e.response is not None else ""
raise RuntimeError(
f"llama-server embedder POST {path} -> {e.response.status_code}: {body}"
) from e
raise RuntimeError(f"llama-server embedder POST {path} failed after retry") from last_exc
def _server_exited(self) -> bool:
"""Whether the server died under the request; logs its exit code and output tail."""
proc = self._process
if proc is None:
return False
try:
code = proc.wait(timeout = 2)
except subprocess.TimeoutExpired:
return False
if self._stdout_thread is not None:
self._stdout_thread.join(timeout = 2)
tail = "\n".join(self._stdout_lines[-30:])
logger.warning(
"llama-server embedder exited during a request (code %s). Last output:\n%s",
code,
tail[:2000],
)
return True
def _fall_back_to_cpu(self) -> bool:
if not self._on_gpu or self._force_cpu or config.embed_device_requires_gpu():
return False
logger.warning("llama-server embedder crashed twice on the GPU; continuing on CPU")
self._force_cpu = True
return True
def encode(
self,
texts,
*,
model_name = None,
normalize = True,
):
"""Embed texts -> (N, dim) float32. ``model_name`` pins which GGUF serves
the request, so a Settings change cannot answer it from another model.
Normalizes in Python to match the ST backend."""
self._served_identity_local.value = None
with self._operation(), self._serve_lock:
vectors = self._encode_active(texts, normalize = normalize, model_name = model_name)
self._served_identity_local.value = (self._model_repo, self._model_pooling)
return vectors
def served_embedding_identity(self) -> tuple[str | None, str | None] | None:
"""Repo and pooling captured by this thread's last completed encode."""
return getattr(self._served_identity_local, "value", None)
def _encode_active(
self,
texts,
*,
normalize = True,
model_name = None,
):
n = len(texts)
if n == 0:
return np.zeros((0, self.dim(model_name = model_name)), dtype = np.float32)
rows: list[list[float]] = []
batch = max(1, config.EMBED_BATCH)
for start in range(0, n, batch):
chunk = list(texts[start : start + batch])
data = self._post(
"/v1/embeddings",
{"input": chunk, "model": "embedding", "encoding_format": "float"},
model_name = model_name,
)
items = data.get("data", [])
if len(items) != len(chunk):
raise RuntimeError(
f"embedder returned {len(items)} vectors for {len(chunk)} inputs"
)
# OpenAI spec lets the server reorder; sort back by index.
items = sorted(items, key = lambda d: d.get("index", 0))
rows.extend(d["embedding"] for d in items)
arr = np.asarray(rows, dtype = np.float32)
if arr.ndim != 2:
raise RuntimeError(f"embedder returned ragged vectors: shape {arr.shape}")
if normalize:
norms = np.linalg.norm(arr, axis = 1, keepdims = True)
norms[norms == 0] = 1.0
arr = arr / norms
return arr
def dim(self, *, model_name = None) -> int:
"""Under _serve_lock: _dim is per subprocess, so a model switch must not cache another
model's width."""
with self._operation(), self._serve_lock:
self._ensure_ready(model_name)
cached = self._dim
if cached is not None:
return cached
vec = self.encode(["x"], normalize = False, model_name = model_name)
width = int(vec.shape[1])
self._dim = width
return width
def _server_props(self) -> dict | None:
"""Advisory only: any failure, including a malformed base URL on an unstarted server,
returns None."""
try:
data = httpx.get(
f"{self._base_url}/props",
headers = self._auth_headers(),
timeout = 2.0,
trust_env = False,
).json()
except Exception: # noqa: BLE001 - see docstring
return None
return data if isinstance(data, dict) else None
@staticmethod
def _positive(data: dict | None, key: str) -> int | None:
"""*key* from /props, whether the build reports it at the top level or under
``default_generation_settings``."""
for scope in (data, (data or {}).get("default_generation_settings")):
if isinstance(scope, dict):
value = scope.get(key)
if isinstance(value, int) and value > 0:
return value
return None
def _server_context(self) -> int | None:
return self._positive(self._server_props(), "n_ctx")
def _server_batch(self) -> int | None:
"""Caps advertised input: a non-causal embedding prompt longer than the physical batch is
refused."""
# Effective physical batch is min(n_batch, n_ubatch), so only n_ubatch can limit us.
found = self._positive(self._server_props(), "n_ubatch")
return min(found, _UBATCH_SIZE) if found else _UBATCH_SIZE
def max_tokens(self, *, model_name = None) -> int | None:
with self._operation(), self._serve_lock:
self._ensure_ready(model_name)
if self._max_tokens is None and self._model_path:
limit = _gguf_context_length(self._model_path)
running = self._server_context()
if limit and running:
limit = min(limit, running)
elif running:
limit = running
# Past one batch llama.cpp returns a 500, not the 400 this limit exists to give.
batch = self._server_batch()
if batch:
limit = min(limit, batch) if limit else batch
if limit:
data = self._post(
"/tokenize",
{"content": "", "add_special": True},
model_name = model_name,
)
answer = max(1, limit - len(data.get("tokens", [])))
if running is None or batch is None:
# Do not cache the guess, or a too-large limit outlives a later /props readback.
return answer
self._max_tokens = answer
return self._max_tokens
def warm(self, *, model_name = None) -> None:
"""Readiness is left to dim(), which takes _serve_lock; warming here must not respawn outside it."""
with self._operation():
self.dim(model_name = model_name)
def token_counter(self, *, model_name = None):
"""Count tokens via the GGUF's /tokenize, cached per text. ``many`` sends a content array:
tokenize_mixed tokenizes each string alone, so -1 markers split the reply per text."""
memo: OrderedDict[str, int] = OrderedDict()
memo_lock = threading.Lock()
batching = True
def _remember(text: str, n: int) -> None:
with memo_lock:
memo[text] = n
memo.move_to_end(text)
if len(memo) > _TOKEN_COUNT_CACHE_SIZE:
memo.popitem(last = False)
def _lookup(text: str) -> int | None:
with memo_lock:
n = memo.get(text)
if n is not None:
memo.move_to_end(text)
return n
def _count_one(text: str) -> int:
data = self._post(
"/tokenize",
{"content": text, "add_special": False},
model_name = model_name,
)
return len(data.get("tokens", []))
def _count(text: str) -> int:
n = _lookup(text)
if n is None:
n = _count_one(text)
_remember(text, n)
return n
def _count_batch(keys: list[str]) -> list[int] | None:
try:
data = self._post(
"/tokenize",
{"content": [x for key in keys for x in (-1, key)][1:], "add_special": False},
model_name = model_name,
)
except RuntimeError as e:
# A build that refuses the array answers with a status; anything else still raises.
if not isinstance(e.__cause__, httpx.HTTPStatusError):
raise
return None
counts, run = [], 0
for token in data.get("tokens", []):
if token == -1:
counts.append(run)
run = 0
else:
run += 1
counts.append(run)
return counts if len(counts) == len(keys) else None
def _count_many(texts) -> list[int]:
nonlocal batching
texts = list(texts)
out = [0] * len(texts)
misses: dict[str, list[int]] = {}
for i, text in enumerate(texts):
n = _lookup(text)
if n is None:
misses.setdefault(text, []).append(i)
else:
out[i] = n
keys = list(misses)
start = 0
while start < len(keys):
# Bounded batches, so a query encode waiting on _serve_lock never waits out a whole document.
end, chars = start, 0
while (
end < len(keys)
and end - start < _TOKENIZE_BATCH
and (end == start or chars + len(keys[end]) <= _TOKENIZE_BATCH_CHARS)
):
chars += len(keys[end])
end += 1
batch = keys[start:end]
counts = None
if batching and len(batch) > 1:
counts = _count_batch(batch)
batching = counts is not None
if counts is None:
counts = [_count_one(key) for key in batch]
for key, n in zip(batch, counts):
_remember(key, n)
for i in misses[key]:
out[i] = n
start = end
return out
_count.many = _count_many
return _count