Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

43.9 KB
Raw Blame
"""
Local LLM engine: manages a llama.cpp `llama-server` subprocess.

Everything lives under the agent directory (THREEWS_LLM_DIR, set by Electron to
the `agent` folder beside models/, extensions/, …, ~/.three-ws-forge/llm/ when the API
runs standalone):
  bin/         llama-server binary + DLLs (auto-downloaded from GitHub releases)
  models/      GGUF files (catalog downloads + any custom .gguf the user drops in)
  logs/        one log per llama-server slot
  config.json  pool settings (max_models)

Nothing is hardcoded to a machine: the binary variant is picked per-platform
(CUDA if an NVIDIA driver is present, otherwise Vulkan, otherwise CPU) and
models are chosen by the user from the catalog.
"""
import contextlib
import hashlib
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import time
import zipfile
from pathlib import Path
from typing import Callable, Optional
from urllib.request import Request, urlopen
import json as _json

LLM_DIR        = Path(os.environ.get("THREEWS_LLM_DIR") or Path.home() / ".three-ws-forge" / "llm")
BIN_DIR        = LLM_DIR / "bin"
LLM_MODELS_DIR = LLM_DIR / "models"
LOGS_DIR       = LLM_DIR / "logs"
SERVER_PORT    = int(os.environ.get("THREEWS_LLM_PORT", "8791"))

# three.ws Forge is first and foremost a 3D-generation app: the LLM must never sit on
# VRAM it isn't using. Idle models are evicted after this many seconds.
IDLE_TTL_SECONDS = int(os.environ.get("THREEWS_LLM_IDLE_TTL", "300"))

# Multi-model pool: each loaded model gets its own llama-server process on its
# own port (SERVER_PORT .. SERVER_PORT+MAX_SLOT_PORTS-1). How many may run at
# once is user-configurable ("auto" sizes it from the GPU's VRAM: small cards
# stay at 1, exactly like the old single-slot behavior).
MAX_SLOT_PORTS = 4
CONFIG_PATH    = LLM_DIR / "config.json"

LLM_MODELS_DIR.mkdir(parents=True, exist_ok=True)


def load_config() -> dict:
    try:
        return _json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
    except Exception:
        return {}


def save_config(cfg: dict) -> None:
    CONFIG_PATH.write_text(_json.dumps(cfg, indent=2), encoding="utf-8")


_vram_gb_cache: Optional[float] = None


def detect_vram_gb() -> float:
    """Total VRAM of the first NVIDIA GPU in GiB (0.0 if none/unknown)."""
    global _vram_gb_cache
    if _vram_gb_cache is not None:
        return _vram_gb_cache
    try:
        out = subprocess.run(
            ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
            capture_output=True, text=True, timeout=10,
        ).stdout.strip().splitlines()
        _vram_gb_cache = round(float(out[0]) / 1024.0, 1) if out else 0.0
    except Exception:
        _vram_gb_cache = 0.0
    return _vram_gb_cache


def resolve_max_models() -> int:
    """How many llama-server processes may run at once.

    Priority: THREEWS_LLM_MAX_MODELS env → user config → auto from VRAM.
    Auto is deliberately conservative: 8 GB cards keep the single-slot
    behavior, VRAM stays available for the 3D pipeline."""
    raw = os.environ.get("THREEWS_LLM_MAX_MODELS") or load_config().get("max_models") or "auto"
    if isinstance(raw, str) and raw.lower() == "auto":
        # Thresholds sit just under the marketing size on purpose: a card sold
        # as "12 GB" reports 12227 MiB = 11.9 GiB, so a `>= 12` test excluded
        # every 12 GB card there is (5070, 4070, 3060 12G…) from the 2-slot
        # tier it was written for. Same story at 20 for a 20/24 GB card.
        vram = detect_vram_gb()
        n = 3 if vram >= 19.5 else 2 if vram >= 11.5 else 1
    else:
        try:
            n = int(raw)
        except (TypeError, ValueError):
            n = 1
    return max(1, min(n, MAX_SLOT_PORTS))

_BINARY_NAME = "llama-server.exe" if sys.platform == "win32" else "llama-server"
_UA          = {"User-Agent": "forge-llm"}

# Pinned to a known-good tag by default so installs are reproducible and never
# silently pick up a broken/incompatible "latest" release. Set THREEWS_LLM_RELEASE
# to another llama.cpp tag (e.g. "b10075") to override, or to "latest" to track
# the newest release.
_RELEASE_TAG  = os.environ.get("THREEWS_LLM_RELEASE", "b10075")
_RELEASES_API = (
    "https://api.github.com/repos/ggml-org/llama.cpp/releases/latest"
    if _RELEASE_TAG.lower() == "latest"
    else f"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/{_RELEASE_TAG}"
)

CATALOG_PATH = Path(__file__).resolve().parent.parent / "resources" / "llm_catalog.json"

DEFAULT_CTX = 16384
DEFAULT_NGL = 99


def load_catalog() -> list[dict]:
    return _json.loads(CATALOG_PATH.read_text(encoding="utf-8"))


_FALLBACK_DEFAULT_MODEL = "qwen3-4b"

# What the agent sends when a catalog entry declares nothing of its own. Low
# temperature is what keeps small models from emitting malformed tool-call JSON.
_DEFAULT_SAMPLING = {"temperature": 0.2}


def sampling_for(model_id: str) -> dict:
    """Sampling parameters for one local model.

    Model families publish the settings they were tuned for, and ignoring them
    is not neutral: served at a flat temperature with no presence penalty, a
    Qwen 3.5 repeated the same lookup ten times in one turn and never acted.
    A catalog entry can therefore carry its own `sampling` block."""
    entry = next((e for e in load_catalog() if e["id"] == model_id), None)
    declared = (entry or {}).get("sampling") or {}
    return {**_DEFAULT_SAMPLING, **{k: v for k, v in declared.items() if v is not None}}


def default_model_id() -> str:
    """Catalog entry tagged "default", falling back to a known-good id if the
    catalog has no such tag (keeps old behavior if llm_catalog.json changes)."""
    for entry in load_catalog():
        if "default" in (entry.get("tags") or []):
            return entry["id"]
    return _FALLBACK_DEFAULT_MODEL


def mmproj_local_name(entry: dict) -> str:
    """Local filename for a vision projector. HF repos all name it mmproj-F16.gguf,
    which would collide across models in the shared models dir."""
    return f"mmproj-{entry['id']}.gguf"


def list_models() -> list[dict]:
    """Catalog entries with a `downloaded` flag + custom GGUFs found on disk."""
    catalog = load_catalog()
    known_files = set()
    for entry in catalog:
        path = LLM_MODELS_DIR / entry["hf_filename"]
        downloaded = path.exists()
        if entry.get("hf_mmproj_filename"):
            downloaded = downloaded and (LLM_MODELS_DIR / mmproj_local_name(entry)).exists()
            known_files.add(mmproj_local_name(entry))
            # Show the real download size (weights + vision projector)
            entry["size_bytes"] = (entry.get("size_bytes") or 0) + (entry.get("mmproj_size_bytes") or 0)
        entry["downloaded"] = downloaded
        entry["source"] = "catalog"
        known_files.add(entry["hf_filename"])

    custom = [
        {
            "id": f"custom:{f.name}",
            "name": f.stem,
            "description": "Custom model found in the models folder.",
            "hf_filename": f.name,
            "size_bytes": f.stat().st_size,
            "downloaded": True,
            "source": "custom",
            "ctx": DEFAULT_CTX,
            "ngl_suggestion": DEFAULT_NGL,
            "tags": ["custom"],
        }
        for f in sorted(LLM_MODELS_DIR.glob("*.gguf"))
        if f.name not in known_files and not f.name.lower().startswith("mmproj")
    ]
    return catalog + custom


def estimate_vram_mb(entry: dict) -> int:
    """How much VRAM this model is expected to hold, in MiB.

    Catalog entries declare it. A user's own GGUF doesn't, so it is derived from
    the file: weights land in VRAM about 1:1, plus the KV cache and compute
    buffers for a 16k context. Measured against the catalog's own numbers, whose
    declared/file ratio runs 1.22-1.32, so 1.25 + 500 MiB sits in the middle."""
    declared = entry.get("vram_estimate_mb")
    if isinstance(declared, (int, float)) and declared > 0:
        return int(declared)
    size_mb = (entry.get("size_bytes") or 0) / (1024 * 1024)
    return int(size_mb * 1.25 + 500) if size_mb else 0


def vram_budget_mb() -> int:
    """VRAM the LLM pool may fill, in MiB, total minus what the desktop needs.
    0 when no NVIDIA GPU is detected, which disables budgeting entirely."""
    total = detect_vram_gb() * 1024
    if total <= 0:
        return 0
    reserve = _env_int("THREEWS_LLM_VRAM_RESERVE_MB", 768)
    return max(0, int(total) - reserve)


def _env_int(name: str, default: int) -> int:
    try:
        return int(os.environ.get(name) or default)
    except ValueError:
        return default


def resolve_model(model_id: str) -> dict:
    """Return {gguf_path, mmproj_path, ngl, ctx, vision, vram_mb} for a catalog id
    or a custom:<file> id."""
    if model_id.startswith("custom:"):
        # The name is user/agent-supplied and this path is both loaded and
        # DELETEd (DELETE /llm/models/{model_id}). Confine it to the models dir:
        # on Windows a backslash is a separator too, so "custom:..\..\x.gguf"
        # would otherwise resolve: and unlink, outside it.
        name = model_id[len("custom:"):]
        path = (LLM_MODELS_DIR / name).resolve()
        root = LLM_MODELS_DIR.resolve()
        if path.parent != root or not path.exists() or path.suffix != ".gguf":
            raise KeyError(f"Custom model not found: {model_id}")
        return {
            "gguf_path": path, "mmproj_path": None, "ngl": DEFAULT_NGL,
            "ctx": DEFAULT_CTX, "vision": False,
            "vram_mb": estimate_vram_mb({"size_bytes": path.stat().st_size}),
        }
    for entry in load_catalog():
        if entry["id"] == model_id:
            has_mmproj = bool(entry.get("hf_mmproj_filename"))
            return {
                "gguf_path": LLM_MODELS_DIR / entry["hf_filename"],
                "mmproj_path": LLM_MODELS_DIR / mmproj_local_name(entry) if has_mmproj else None,
                "ngl": entry.get("ngl_suggestion", DEFAULT_NGL),
                "ctx": entry.get("ctx", DEFAULT_CTX),
                "vision": "vision" in (entry.get("tags") or []),
                "vram_mb": estimate_vram_mb(entry),
            }
    raise KeyError(f"Unknown model id: {model_id}")


def binary_path() -> Path:
    return BIN_DIR / _BINARY_NAME


def binary_installed() -> bool:
    return binary_path().exists()


def has_nvidia_gpu() -> bool:
    return shutil.which("nvidia-smi") is not None


# ─── Binary bootstrap ─────────────────────────────────────────────────────────

def _fetch_release() -> dict:
    with urlopen(Request(_RELEASES_API, headers=_UA), timeout=30) as r:
        return _json.loads(r.read())


def _cuda_ver(name: str) -> tuple[int, int]:
    m = re.search(r"cuda-(?:cu)?(\d+)[._](\d+)", name.lower())
    return (int(m.group(1)), int(m.group(2))) if m else (0, 0)


def _driver_cuda_version() -> tuple[int, int]:
    """Highest CUDA version the installed NVIDIA driver supports (0,0 if unknown)."""
    try:
        out = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10).stdout
        m = re.search(r"CUDA Version:\s*(\d+)\.(\d+)", out)
        return (int(m.group(1)), int(m.group(2))) if m else (0, 0)
    except Exception:
        return (0, 0)


def _pick_assets(assets: list[dict]) -> list[dict]:
    """Ordered list of assets to install for this machine.

    Windows + NVIDIA: the CUDA build (self-contained, includes CPU fallback)
    plus its matching `cudart-…` runtime package. Otherwise Vulkan, then CPU.
    macOS / Linux: the platform tarball (Vulkan variant preferred on Linux).
    """
    import platform as _platform

    def matching(ext: tuple[str, ...], *needles: str) -> list[dict]:
        return [
            a for a in assets
            if a["name"].endswith(ext) and all(n in a["name"].lower() for n in needles)
        ]

    if sys.platform == "win32":
        if has_nvidia_gpu():
            builds = matching((".zip",), "win", "x64", "cuda")
            builds = [b for b in builds if not b["name"].startswith("cudart")]
            if builds:
                driver = _driver_cuda_version()
                # Newest build the driver can run; if the driver version is
                # unknown, the oldest build has the widest compatibility.
                supported = [b for b in builds if _cuda_ver(b["name"]) <= driver]
                pool = supported or builds
                pick = max(pool, key=lambda b: _cuda_ver(b["name"])) if supported else \
                       min(pool, key=lambda b: _cuda_ver(b["name"]))
                ver = _cuda_ver(pick["name"])
                cudart = [
                    a for a in assets
                    if a["name"].startswith("cudart") and _cuda_ver(a["name"]) == ver
                ]
                return [pick, *cudart[:1]]
        return (matching((".zip",), "win", "x64", "vulkan")
                or matching((".zip",), "win", "x64", "cpu")
                or matching((".zip",), "win", "x64", "avx2"))[:1]

    if sys.platform == "darwin":
        arch = "arm64" if _platform.machine().lower() in ("arm64", "aarch64") else "x64"
        exts = (".zip", ".tar.gz")
        return (matching(exts, "macos", arch) or matching(exts, "macos"))[:1]

    exts = (".zip", ".tar.gz")
    return (matching(exts, "ubuntu", "vulkan", "x64")
            or matching(exts, "ubuntu", "x64")
            or matching(exts, "linux", "x64"))[:1]


def _download_asset(asset: dict, progress_cb: Callable[[dict], None], control_check: Callable[[], None], label: str) -> Path:
    suffix = ".tar.gz" if asset["name"].endswith(".tar.gz") else ".zip"
    fd, tmp_name = tempfile.mkstemp(suffix=suffix)
    os.close(fd)
    tmp = Path(tmp_name)
    sha256 = hashlib.sha256()
    try:
        with urlopen(Request(asset["browser_download_url"], headers=_UA), timeout=30) as resp:
            total = int(resp.headers.get("Content-Length", 0)) or asset.get("size", 0)
            done = 0
            last_emit = 0.0
            with open(tmp, "wb") as fh:
                while chunk := resp.read(1 << 20):
                    control_check()
                    fh.write(chunk)
                    sha256.update(chunk)
                    done += len(chunk)
                    now = time.monotonic()
                    if now - last_emit >= 0.5:
                        progress_cb({
                            "status": f"Downloading {label} ({asset['name']})",
                            "bytesDownloaded": done,
                            "totalBytes": total,
                            "percent": round(done / total * 100) if total else 0,
                        })
                        last_emit = now
        _verify_digest(asset, sha256.hexdigest())
    except BaseException:
        tmp.unlink(missing_ok=True)  # never leave a partial or rejected archive behind
        raise
    return tmp


def _verify_digest(asset: dict, actual_sha256: str) -> None:
    """Refuse an archive whose bytes differ from what GitHub published for it.

    The files extracted from it are executed (llama-server and its DLLs), so a
    corrupted or tampered download must not get that far. GitHub reports a
    `digest` ("sha256:<hex>") for every release asset; one without it (older
    uploads) cannot be checked and is accepted as before."""
    expected = asset.get("digest") or ""
    if not expected.startswith("sha256:"):
        return
    if actual_sha256.lower() != expected[len("sha256:"):].lower():
        raise RuntimeError(
            f"Checksum mismatch for {asset['name']}: the download does not match the "
            "published llama.cpp release. Try installing the engine again."
        )


_LIB_SUFFIXES = (".so", ".dylib", ".metal")


def _extract_archive(tmp: Path, asset_name: str) -> int:
    """Extract runtime files flat into BIN_DIR (exe/dll on Windows, bin/* + libs elsewhere)."""
    BIN_DIR.mkdir(parents=True, exist_ok=True)
    count = 0

    def wanted(member_path: str, fname_low: str) -> bool:
        if sys.platform == "win32":
            return fname_low.endswith((".exe", ".dll"))
        return "/bin/" in member_path.replace("\\", "/") or fname_low.endswith(_LIB_SUFFIXES)

    if asset_name.endswith(".tar.gz"):
        import tarfile
        with tarfile.open(tmp, "r:gz") as tf:
            for m in tf.getmembers():
                if not m.isfile():
                    continue
                fname = Path(m.name).name
                if not fname or not wanted(m.name, fname.lower()):
                    continue
                src = tf.extractfile(m)
                if src is None:
                    continue
                dest = BIN_DIR / fname
                with open(dest, "wb") as dst:
                    shutil.copyfileobj(src, dst)
                if not fname.lower().endswith(_LIB_SUFFIXES):
                    dest.chmod(0o755)
                count += 1
        return count

    with zipfile.ZipFile(tmp) as zf:
        for item in zf.infolist():
            if item.is_dir():
                continue
            fname = Path(item.filename).name
            if not fname or not wanted(item.filename, fname.lower()):
                continue
            dest = BIN_DIR / fname
            with zf.open(item) as src, open(dest, "wb") as dst:
                shutil.copyfileobj(src, dst)
            if sys.platform != "win32" and not fname.lower().endswith(_LIB_SUFFIXES):
                dest.chmod(0o755)
            count += 1
    return count


def install_binary(progress_cb: Callable[[dict], None], control_check: Callable[[], None]) -> None:
    """Download and install the best llama-server build for this machine."""
    progress_cb({"status": "Fetching latest llama.cpp release…", "percent": 0})
    release = _fetch_release()
    progress_cb({"status": f"Release {release['tag_name']}, selecting build for this machine…", "percent": 1})

    assets = _pick_assets(release["assets"])
    if not assets:
        raise RuntimeError("No compatible llama.cpp build found for this platform.")

    for i, asset in enumerate(assets):
        label = "engine" if i == 0 else "CUDA runtime"
        tmp = _download_asset(asset, progress_cb, control_check, label)
        try:
            count = _extract_archive(tmp, asset["name"])
            progress_cb({"status": f"Extracted {count} files from {asset['name']}"})
        finally:
            tmp.unlink(missing_ok=True)

    if not binary_installed():
        raise RuntimeError("Install finished but llama-server binary is missing.")
    progress_cb({"status": "done", "percent": 100})


# ─── Orphan prevention ────────────────────────────────────────────────────────
# If three.ws Forge is force-killed, a plain child process would keep its model in
# memory forever. On Windows we put llama-server in a Job object with
# KILL_ON_JOB_CLOSE (the OS kills it when three.ws Forge dies); on Linux we ask the
# kernel to deliver SIGKILL on parent death.

_win_job_handle = None


def _windows_job() -> Optional[int]:
    global _win_job_handle
    if _win_job_handle is not None:
        return _win_job_handle
    try:
        import ctypes
        from ctypes import wintypes

        class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure):
            _fields_ = [
                ("PerProcessUserTimeLimit", ctypes.c_int64),
                ("PerJobUserTimeLimit", ctypes.c_int64),
                ("LimitFlags", wintypes.DWORD),
                ("MinimumWorkingSetSize", ctypes.c_size_t),
                ("MaximumWorkingSetSize", ctypes.c_size_t),
                ("ActiveProcessLimit", wintypes.DWORD),
                ("Affinity", ctypes.c_size_t),
                ("PriorityClass", wintypes.DWORD),
                ("SchedulingClass", wintypes.DWORD),
            ]

        class IO_COUNTERS(ctypes.Structure):
            _fields_ = [(n, ctypes.c_uint64) for n in (
                "ReadOperationCount", "WriteOperationCount", "OtherOperationCount",
                "ReadTransferCount", "WriteTransferCount", "OtherTransferCount")]

        class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure):
            _fields_ = [
                ("BasicLimitInformation", JOBOBJECT_BASIC_LIMIT_INFORMATION),
                ("IoInfo", IO_COUNTERS),
                ("ProcessMemoryLimit", ctypes.c_size_t),
                ("JobMemoryLimit", ctypes.c_size_t),
                ("PeakProcessMemoryUsed", ctypes.c_size_t),
                ("PeakJobMemoryUsed", ctypes.c_size_t),
            ]

        kernel32 = ctypes.windll.kernel32
        job = kernel32.CreateJobObjectW(None, None)
        info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION()
        info.BasicLimitInformation.LimitFlags = 0x2000  # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
        kernel32.SetInformationJobObject(job, 9, ctypes.byref(info), ctypes.sizeof(info))  # JobObjectExtendedLimitInformation
        _win_job_handle = job
        return job
    except Exception:
        return None


def _tie_to_parent(process: subprocess.Popen) -> None:
    if sys.platform == "win32":
        job = _windows_job()
        if job is not None:
            try:
                import ctypes
                ctypes.windll.kernel32.AssignProcessToJobObject(job, int(process._handle))  # type: ignore[attr-defined]
            except Exception:
                pass


def _linux_preexec():
    try:
        import ctypes
        import signal as _signal
        libc = ctypes.CDLL("libc.so.6", use_errno=True)
        libc.prctl(1, _signal.SIGKILL)  # PR_SET_PDEATHSIG
    except Exception:
        pass


def _kill_stale_server(port: int = SERVER_PORT) -> None:
    """Kill a leftover llama-server holding `port` (e.g. after a force-kill of three.ws Forge)."""
    try:
        with urlopen(Request(f"http://127.0.0.1:{port}/health", headers=_UA), timeout=1) as r:
            if r.status != 200:
                return
    except Exception:
        return  # nothing listening, the normal case
    try:
        if sys.platform == "win32":
            out = subprocess.run(
                ["netstat", "-ano", "-p", "tcp"], capture_output=True, text=True, timeout=10,
            ).stdout
            for line in out.splitlines():
                if f":{port}" in line and "LISTENING" in line.upper():
                    pid = line.split()[-1]
                    name = subprocess.run(
                        ["tasklist", "/FI", f"PID eq {pid}", "/FO", "CSV", "/NH"],
                        capture_output=True, text=True, timeout=10,
                    ).stdout.lower()
                    if "llama-server" in name:
                        subprocess.run(["taskkill", "/PID", pid, "/F"], capture_output=True, timeout=10)
                    break
        else:
            out = subprocess.run(
                ["lsof", "-ti", f"tcp:{port}"], capture_output=True, text=True, timeout=10,
            ).stdout
            for pid in out.split():
                name = subprocess.run(
                    ["ps", "-p", pid, "-o", "comm="], capture_output=True, text=True, timeout=10,
                ).stdout.lower()
                if "llama-server" in name:
                    subprocess.run(["kill", "-9", pid], capture_output=True, timeout=10)
    except Exception:
        pass


# ─── Managed server ───────────────────────────────────────────────────────────

class LlamaServerManager:
    """One llama-server slot, a single loaded GGUF on a dedicated port.

    Slots are owned by LlamaPool, which decides how many run at once, evicts
    idle ones (IDLE_TTL_SECONDS), and unloads everything when the 3D pipeline
    needs the VRAM.
    """

    def __init__(self, port: int = SERVER_PORT) -> None:
        self.port = port
        self._lock = threading.Lock()
        self._process: Optional[subprocess.Popen] = None
        self._current_id: Optional[str] = None
        self._started_at: Optional[float] = None
        self._last_used: float = 0.0
        self._log_file = None
        # Requests currently being answered by this slot. Guarded by its own
        # lock: _lock is held for the whole of a cold start (up to 180 s), and a
        # request must never wait on that to declare itself in flight.
        self._inflight: int = 0
        self._inflight_lock = threading.Lock()
        # Expected VRAM of whatever this slot is serving, for the pool's budget.
        self.vram_mb: int = 0

    @property
    def base_url(self) -> str:
        return f"http://127.0.0.1:{self.port}/v1"

    def _alive(self) -> bool:
        return self._process is not None and self._process.poll() is None

    def ensure(self, model_id: str, spec: dict) -> None:
        """Blocking: make sure `model_id` is loaded, swapping the server if needed.
        `spec` comes from resolve_model()."""
        with self._lock:
            self._last_used = time.monotonic()
            if self._current_id == model_id and self._alive():
                return
            if not binary_installed():
                raise RuntimeError("llama-server is not installed. Install the engine in Settings → Agent.")
            if not spec["gguf_path"].exists():
                raise RuntimeError(f"Model file not found: {spec['gguf_path'].name}. Download it in Settings → Agent.")

            self._terminate_locked()
            _kill_stale_server(self.port)
            self._spawn(spec)
            self._current_id = model_id
            self._started_at = time.monotonic()
            self._last_used = time.monotonic()

    def touch(self) -> None:
        """Mark the server as recently used (postpones idle eviction)."""
        self._last_used = time.monotonic()

    @property
    def busy_count(self) -> int:
        with self._inflight_lock:
            return self._inflight

    def hold(self) -> None:
        """Claim the slot for a request that is about to start."""
        with self._inflight_lock:
            self._inflight += 1
            self._last_used = time.monotonic()

    def release(self) -> None:
        with self._inflight_lock:
            self._inflight -= 1
            self._last_used = time.monotonic()

    @contextlib.contextmanager
    def busy(self):
        """Hold the slot for the duration of one request.

        _last_used only moved when a completion *finished*, so a generation
        longer than IDLE_TTL_SECONDS, a Text-to-CAD node parked on /llm/chat,
        or an agent round on a 14B running on CPU, looked idle to the reaper,
        which terminated the server mid-answer: truncated stream, node failed
        with no message. Marking the request in flight covers the whole call,
        including the prompt-eval stall before the first token."""
        self.hold()
        try:
            yield self
        finally:
            self.release()

    def _spawn(self, spec: dict) -> None:
        n_threads = max(1, int((os.cpu_count() or 4) * 0.8))
        cmd = [
            str(binary_path()),
            "-m", str(spec["gguf_path"]),
            "--port", str(self.port),
            "--host", "127.0.0.1",
            "-c", str(spec["ctx"]),
            "-ngl", str(spec["ngl"]),
            "--jinja",
            "-np", "1",
            "--threads", str(n_threads),
            "--threads-batch", str(n_threads),
        ]
        if spec.get("mmproj_path"):
            cmd += ["--mmproj", str(spec["mmproj_path"])]
        if has_nvidia_gpu():
            # Halve KV-cache VRAM so the larger context still fits 12 GB cards.
            # V-cache quantization requires flash attention (fine on CUDA).
            cmd += ["-fa", "on", "-ctk", "q8_0", "-ctv", "q8_0"]
        # llama-server resolves its plugin DLLs (ggml-cuda, ggml-cpu-*) relative
        # to cwd, not the exe path: running from bin/ is required.
        env = {**os.environ, "PATH": str(BIN_DIR) + os.pathsep + os.environ.get("PATH", "")}
        kwargs: dict = {}
        if sys.platform.startswith("linux"):
            kwargs["preexec_fn"] = _linux_preexec

        # Overwritten on every spawn (one log per slot/port) so a crash at
        # startup (OOM, corrupt GGUF) can be diagnosed instead of vanishing
        # into DEVNULL.
        LOGS_DIR.mkdir(parents=True, exist_ok=True)
        self._log_file = open(LOGS_DIR / f"slot-{self.port}.log", "w", encoding="utf-8")

        self._process = subprocess.Popen(
            cmd,
            stdout=self._log_file,
            stderr=subprocess.STDOUT,
            cwd=str(BIN_DIR),
            env=env,
            **kwargs,
        )
        _tie_to_parent(self._process)
        self._wait_for_health()

    def _wait_for_health(self, timeout: float = 180.0, interval: float = 1.0) -> None:
        url = f"http://127.0.0.1:{self.port}/health"
        deadline = time.time() + timeout
        while time.time() < deadline:
            if not self._alive():
                raise RuntimeError("llama-server exited during startup (bad model file or out of memory?).")
            try:
                with urlopen(Request(url, headers=_UA), timeout=2) as r:
                    if r.status == 200:
                        return
            except Exception:
                pass
            time.sleep(interval)
        self._terminate_locked()
        raise TimeoutError(f"llama-server did not become ready within {timeout:.0f}s")

    def unload(self) -> None:
        with self._lock:
            self._terminate_locked()

    def _terminate_locked(self) -> None:
        if self._process is None:
            return
        try:
            self._process.terminate()
            try:
                self._process.wait(timeout=10)
            except subprocess.TimeoutExpired:
                self._process.kill()
        except Exception:
            pass
        finally:
            self._process = None
            self._current_id = None
            self._started_at = None
            if self._log_file is not None:
                self._log_file.close()
                self._log_file = None

    def snapshot(self) -> dict:
        alive = self._alive()
        return {
            "alive": alive,
            "model_id": self._current_id if alive else None,
            "port": self.port,
            "uptime_seconds": round(time.monotonic() - self._started_at, 1) if alive and self._started_at else None,
            "vram_mb": self.vram_mb if alive else None,
        }


class LlamaPool:
    """Pool of llama-server slots, one process per loaded model, each on its
    own port, capped by BOTH resolve_max_models() and vram_budget_mb()
    (LRU eviction).

    The idle reaper and unload_all() keep the old guarantees: models are
    evicted after IDLE_TTL_SECONDS unused, and the 3D pipeline can reclaim all
    VRAM at once.
    """

    def __init__(self) -> None:
        self._lock = threading.Lock()
        self._slots: dict[str, LlamaServerManager] = {}
        # Ports held by a load in flight. A reserved slot has no process yet, so
        # _alive() cannot tell it apart from a dead one; tracking the port rather
        # than the model_id keeps the reservation valid even if unload() drops
        # the slot from _slots while it is still starting.
        self._loading_ports: set[int] = set()
        # Signalled whenever a load lands, so callers held back by a concurrent
        # cold start can re-run the limit check instead of loading on top of it.
        self._cond = threading.Condition(self._lock)
        self._reaper_started = False

    def ensure(self, model_id: str, spec: dict, hold: bool = False) -> LlamaServerManager:
        """Blocking: return a ready slot serving `model_id`, starting it (and
        evicting the least-recently-used slots) if needed.

        With `hold`, the slot is returned already claimed for one request and the
        caller MUST call slot.release() when done (see busy()). Without it there
        is a window between "loaded" and "in flight" in which a competing load
        can evict the slot, and the caller then posts to a dead server: with
        max_models = 1, two nodes asking for different models had one of them
        answer 500 Internal Server Error.

        The slot is reserved under the pool lock but LOADED outside it. A cold
        start blocks in _wait_for_health for up to 180 s, and holding the pool
        lock that long froze every other caller: /llm/status, polled by an open
        Settings → Agent page, hung for the whole load, and switch_model's
        unload_all(), which exists to reclaim VRAM before a 3D model, queued
        behind the very LLM competing for it.

        Loading outside the lock means two callers can be starting different
        models at the same time (an agent turn plus a workflow LLM node). A slot
        being started holds no process, so it cannot be evicted to make room:
        when only such loads keep the pool over its limit, this waits for one to
        land, it then becomes an ordinary eviction candidate, instead of
        putting a second model on a card sized for one.
        """
        incoming_mb = spec.get("vram_mb") or 0
        # A last resort, not a policy: past this the thing we are waiting for is
        # stuck, and blocking forever would be worse than loading on top of it.
        deadline = time.monotonic() + 300.0
        with self._lock:
            while True:
                slot = self._slots.get(model_id)
                if slot is not None and slot._alive():
                    break
                if slot is not None and slot.port not in self._loading_ports:
                    # A server that crashed (OOM, CUDA error) leaves its slot
                    # behind, and its port is free for the next model. Respawning
                    # in place would then find that model's server healthy on
                    # "our" port and kill it. Start over on a fresh port instead.
                    self._drop_dead_locked(model_id, slot)
                    slot = None
                # Also for a dead slot respawned in place, so reviving one
                # cannot push the pool past the limit. Only alive slots that are
                # not answering a request are eviction candidates, so this never
                # evicts our own: nor anyone's live stream.
                self._enforce_limit_locked(reserve=1, incoming_mb=incoming_mb)
                remaining = deadline - time.monotonic()
                if (not self._over_capacity_locked(incoming_mb)
                        or not self._transient_blockers_locked()
                        or remaining <= 0):
                    break
                # Re-checked about once a second: capacity also frees up when a
                # request finishes, which is not a load event and so notifies
                # nothing.
                self._cond.wait(min(1.0, remaining))
            if slot is None:
                used_ports = {s.port for s in self._slots.values() if s._alive()} | self._loading_ports
                port = next(
                    (SERVER_PORT + i for i in range(MAX_SLOT_PORTS)
                     if SERVER_PORT + i not in used_ports),
                    None,
                )
                if port is None:
                    # A bare next() raised StopIteration here, which surfaced to
                    # the caller as "Could not start the local LLM: ": an empty
                    # message. Say what actually happened.
                    raise RuntimeError(
                        f"All {MAX_SLOT_PORTS} local-LLM slots are in use or starting up. "
                        "Wait for a run to finish, or lower the model limit in Settings → Agent."
                    )
                slot = LlamaServerManager(port)
                self._slots[model_id] = slot
            slot.vram_mb = incoming_mb
            self._loading_ports.add(slot.port)
            if hold:
                # Claimed BEFORE the load, not after: a caller woken by this load
                # landing would otherwise find the slot idle and evict it, and
                # two racing callers spent their time killing each other's
                # freshly loaded model until one gave up.
                slot.hold()
            self._start_reaper()

        # Outside the pool lock. LlamaServerManager holds its own, so concurrent
        # callers for the same model serialise here and the later ones no-op.
        try:
            slot.ensure(model_id, spec)
        except Exception:
            if hold:
                slot.release()  # nothing to answer with, do not pin the slot
            with self._lock:
                if self._slots.get(model_id) is slot and not slot._alive():
                    self._slots.pop(model_id, None)
            raise
        finally:
            with self._lock:
                self._loading_ports.discard(slot.port)
                self._cond.notify_all()  # waiters can re-run the limit check
        return slot

    def _evictable_locked(self, slot: LlamaServerManager) -> bool:
        """May this slot be unloaded to make room? Shared by the limit check and
        the idle reaper, which both unload while holding the pool lock, a rule
        applied to only one of them is a rule that does not hold.

        A slot answering a request is spared because terminating it truncates
        the caller's stream mid-generation. A slot another thread is loading is
        spared for a harder reason: from the moment Popen returns it is _alive()
        (the long wait is _wait_for_health, up to 180 s) while its own lock is
        held by the loader, so unload() would block on that lock and freeze the
        whole pool with it, /llm/status, every ensure(), and the unload_all()
        the 3D pipeline needs to reclaim VRAM. Its cost is still counted, it
        just cannot be the victim."""
        return slot.busy_count == 0 and slot.port not in self._loading_ports

    def _enforce_limit_locked(self, reserve: int = 0, incoming_mb: int = 0) -> None:
        """Evict LRU slots until the pool fits BOTH limits: the configured model
        count, and the VRAM budget.

        The count alone was not enough. `max_models: 2` let any two models in,
        so a 9 GB custom model plus a vision model on a 12 GB card oversubscribed
        the card; Windows spills to shared memory instead of failing, so nothing
        errored, a cold start just went from 5 s to 24 s with no explanation.
        Budgeting on declared estimates keeps the pairs that actually fit (a 4B
        and a 7B come to 10.4 of 11.5 GiB) and refuses the ones that never did.

        The incoming model is never rejected: if it does not fit even alone,
        everything else is evicted and it loads by itself."""
        alive = [(mid, s) for mid, s in self._slots.items() if s._alive()]
        alive.sort(key=lambda kv: kv[1]._last_used)  # oldest first
        # Slots reserved by a concurrent ensure() hold no process yet, so
        # _alive() cannot see them. Counting only live slots let two concurrent
        # ensure() calls: an agent turn plus a workflow LLM node, each
        # conclude the pool was empty and both load, which on an 8 GB card
        # (max_models = 1) is exactly the oversubscription this rule prevents.
        loading = [s for s in self._slots.values()
                   if s.port in self._loading_ports and not s._alive()]
        candidates = [kv for kv in alive if self._evictable_locked(kv[1])]

        def _evict_oldest() -> None:
            mid, slot = candidates.pop(0)
            alive.remove((mid, slot))
            slot.unload()
            self._slots.pop(mid, None)

        max_n = resolve_max_models()
        while len(alive) + len(loading) + reserve > max_n and candidates:
            _evict_oldest()

        budget = vram_budget_mb()
        if not budget or not incoming_mb:
            return  # no GPU detected, or an unknown estimate, count rule only

        def _committed_mb() -> int:
            return sum(s.vram_mb for _mid, s in alive) + sum(s.vram_mb for s in loading)

        while candidates and _committed_mb() + incoming_mb > budget:
            _evict_oldest()

    def _transient_blockers_locked(self) -> bool:
        """Is the pool full only of things that end on their own, a load in
        flight, or a slot answering a request? Those are worth waiting for; a
        merely idle slot is not (it gets evicted instead)."""
        return bool(self._loading_ports) or any(
            s.busy_count for s in self._slots.values() if s._alive()
        )

    def _over_capacity_locked(self, incoming_mb: int) -> bool:
        """Would loading one more model break either limit, counting the slots
        another thread is starting right now?"""
        alive   = [s for s in self._slots.values() if s._alive()]
        loading = [s for s in self._slots.values()
                   if s.port in self._loading_ports and not s._alive()]
        if len(alive) + len(loading) + 1 > resolve_max_models():
            return True
        budget = vram_budget_mb()
        if not budget or not incoming_mb:
            return False  # no GPU detected, or an unknown estimate
        committed = sum(s.vram_mb for s in alive) + sum(s.vram_mb for s in loading)
        return committed + incoming_mb > budget

    def enforce_limit(self) -> None:
        """Apply the configured limit right away (used when the user lowers it)."""
        with self._lock:
            self._enforce_limit_locked()

    def is_loaded(self, model_id: str) -> bool:
        with self._lock:
            slot = self._slots.get(model_id)
            return slot is not None and slot._alive()

    def touch(self, model_id: str) -> None:
        with self._lock:
            slot = self._slots.get(model_id)
            if slot is not None:
                slot.touch()

    def unload(self, model_id: str) -> None:
        with self._lock:
            slot = self._slots.pop(model_id, None)
        if slot is not None:
            slot.unload()

    def unload_all(self, force: bool = False) -> None:
        """Free the pool's VRAM. Slots answering a request or being started are
        spared (see _evictable_locked): killing one cut a Text-to-CAD stream off
        mid-answer when a 3D model loaded, and a cold start in another thread
        handed its caller a dead slot. `force` is for shutdown only."""
        if force:
            with self._lock:
                slots = list(self._slots.values())
                self._slots.clear()
            for slot in slots:
                slot.unload()
            return
        with self._lock:
            for mid, slot in list(self._slots.items()):
                if self._evictable_locked(slot):
                    slot.unload()
                    self._slots.pop(mid, None)

    def _drop_dead_locked(self, model_id: str, slot: LlamaServerManager) -> None:
        self._slots.pop(model_id, None)
        slot.unload()  # reaps the exited process and closes its log

    def _start_reaper(self) -> None:
        if self._reaper_started or IDLE_TTL_SECONDS <= 0:
            return
        self._reaper_started = True
        threading.Thread(target=self._reap_idle, daemon=True, name="llm-idle-reaper").start()

    def _reap_idle(self) -> None:
        while True:
            time.sleep(15)
            self._reap_once()

    def _reap_once(self, now: Optional[float] = None) -> None:
        now = time.monotonic() if now is None else now
        with self._lock:
            for mid, slot in list(self._slots.items()):
                if not self._evictable_locked(slot):
                    continue  # answering right now, or being loaded, never idle
                if not slot._alive():
                    self._drop_dead_locked(mid, slot)
                elif now - slot._last_used > IDLE_TTL_SECONDS:
                    slot.unload()
                    self._slots.pop(mid, None)

    def snapshot(self) -> dict:
        with self._lock:
            slots = list(self._slots.values())
        servers = [s.snapshot() for s in slots if s._alive()]
        first = servers[0] if servers else {"alive": False, "model_id": None, "port": SERVER_PORT, "uptime_seconds": None}
        return {
            **first,  # legacy single-server shape
            "servers": servers,
            "max_models": resolve_max_models(),
            "vram_gb": detect_vram_gb() or None,
            "vram_budget_mb": vram_budget_mb() or None,
            "vram_used_mb": sum(s.get("vram_mb") or 0 for s in servers) or None,
        }


llama_pool = LlamaPool()