Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

23.4 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""CUDA 12 cuDNN for sd.cpp, named via GGML_CUDA_CUDNN_LIB, installed with --target outside the
venv; never loaded into this process"""

from __future__ import annotations

import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, Optional

from loggers import get_logger

logger = get_logger(__name__)

CUDNN_INSTALL_ENV = "UNSLOTH_SD_CPP_CUDNN"
GGML_CUDNN_LIB_ENV = "GGML_CUDA_CUDNN_LIB"
GGML_CUDNN_ATTN_ENV = "GGML_CUDA_CUDNN_ATTN"
# The fork never opens cuDNN below cc 8.0.
CUDNN_MIN_CC = (8, 0)

_CUDNN_BUILD_MARKER = b"GGML_CUDA_CUDNN_LIB"
_CUDART_MARKERS = {12: b"libcudart.so.12\x00", 13: b"libcudart.so.13\x00"}


@dataclass(frozen = True)
class CudnnRuntime:
    """Exact pins, never a floor: another cuDNN release can pick other SDPA engines, and only this one was measured."""

    cuda_major: int
    cudnn_package: str
    cudnn_version: str
    nvrtc_package: str
    nvrtc_version: str

    @property
    def cudnn_numeric(self) -> int:
        major, minor, patch = (int(x) for x in self.cudnn_version.split(".")[:3])
        return major * 10000 + minor * 100 + patch

    @property
    def dirname(self) -> str:
        return f"cu{self.cuda_major}-cudnn{self.cudnn_version}-nvrtc{self.nvrtc_version}"

    def requirements(self) -> list[str]:
        return [
            f"{self.cudnn_package}=={self.cudnn_version}",
            f"{self.nvrtc_package}=={self.nvrtc_version}",
        ]


# cuDNN's runtime-compiled engines find NVRTC via RUNPATH ($ORIGIN/../../cuda_nvrtc/lib); cuBLAS is the prebuilt's own.
CUDNN_RUNTIMES: dict[int, CudnnRuntime] = {
    12: CudnnRuntime(12, "nvidia-cudnn-cu12", "9.27.0.42", "nvidia-cuda-nvrtc-cu12", "12.8.93"),
}

_LIB_RELPATH = Path("nvidia") / "cudnn" / "lib" / "libcudnn.so.9"
_NVRTC_RELPATH = Path("nvidia") / "cuda_nvrtc" / "lib" / "libnvrtc.so.12"
_MARKER_NAME = "unsloth-cudnn.json"
_MIN_FREE_BYTES = 4 << 30
_INSTALL_TIMEOUT_S = 1800
_VERIFY_TIMEOUT_S = 120
_LOCK_TIMEOUT_S = _INSTALL_TIMEOUT_S + _VERIFY_TIMEOUT_S + 120
_PYPI_PROBE_URL = "https://pypi.org/simple/nvidia-cudnn-cu12/"
_CANCELLED = "cancelled"

_SCAN_MEMO: dict[tuple[str, int, int], tuple[bool, Optional[int]]] = {}
_SCAN_LOCK = threading.Lock()
_INSTALL_LOCK = threading.Lock()
# Never retry a 0.85 GB download that just failed in this process.
_FAILED: dict[int, str] = {}

StatusCb = Optional[Callable[[str], None]]


class InstallCancelled(Exception):
    """Not memoised: the next load installs."""


class _Refused(str):
    """Disk / network preflight refusal: rechecked next load, never memoised."""


def _off(value: Optional[str]) -> bool:
    return str(value or "").strip().lower() in ("0", "false", "no", "off")


def install_enabled(environ: Optional[dict] = None) -> bool:
    environ = os.environ if environ is None else environ
    return not _off(environ.get(CUDNN_INSTALL_ENV))


def _scan_file(path: Path) -> tuple[bool, Optional[int]]:
    try:
        st = path.stat()
    except OSError:
        return False, None
    key = (str(path.resolve()), st.st_size, st.st_mtime_ns)
    with _SCAN_LOCK:
        if key in _SCAN_MEMO:
            return _SCAN_MEMO[key]
    needles = [_CUDNN_BUILD_MARKER, *_CUDART_MARKERS.values()]
    overlap = max(len(n) for n in needles)
    found: set[bytes] = set()
    try:
        with path.open("rb") as fh:
            tail = b""
            while True:
                chunk = fh.read(8 << 20)
                if not chunk:
                    break
                window = tail + chunk
                for needle in needles:
                    if needle not in found and needle in window:
                        found.add(needle)
                tail = window[-overlap:]
    except OSError:
        return False, None
    major = next((m for m, n in _CUDART_MARKERS.items() if n in found), None)
    result = (_CUDNN_BUILD_MARKER in found, major)
    with _SCAN_LOCK:
        _SCAN_MEMO[key] = result
    return result


def _elf_needed(path: Path) -> Optional[list[str]]:
    """DT_NEEDED names of a 64-bit little-endian ELF (None if not one, or unreadable). Never raises."""
    import struct
    try:
        with path.open("rb") as fh:
            ident = fh.read(64)
            if len(ident) < 64 or ident[:4] != b"\x7fELF" or ident[4] != 2 or ident[5] != 1:
                return None
            shoff = struct.unpack_from("<Q", ident, 0x28)[0]
            shentsize, shnum = struct.unpack_from("<HH", ident, 0x3A)
            if not shoff or shentsize < 64 or not shnum:
                return None
            fh.seek(shoff)
            table = fh.read(shentsize * shnum)
            sections = [
                struct.unpack_from("<IIQQQQIIQQ", table, i * shentsize) for i in range(shnum)
            ]
            names: list[str] = []
            for _name, sh_type, _fl, _addr, off, size, link, _info, _al, entsize in sections:
                if sh_type != 6 or link >= shnum:  # SHT_DYNAMIC; sh_link is its string table
                    continue
                _n, _t, _f, _a, str_off, str_size, *_rest = sections[link]
                fh.seek(str_off)
                strtab = fh.read(str_size)
                fh.seek(off)
                dyn = fh.read(size)
                step = entsize or 16
                for j in range(0, len(dyn) - 15, step):
                    tag, val = struct.unpack_from("<qQ", dyn, j)
                    if tag == 0:
                        break
                    if tag == 1 and val < len(strtab):  # DT_NEEDED
                        names.append(
                            strtab[val : strtab.index(b"\x00", val)].decode("utf-8", "replace")
                        )
            return names
    except (OSError, ValueError, struct.error):
        return None


def _linked_cudart_major(path: Path) -> Optional[int]:
    """The CUDA major ``path`` actually links (its DT_NEEDED libcudart), not one a fallback string merely names."""
    needed = _elf_needed(path) or []
    for major, marker in _CUDART_MARKERS.items():
        if marker.rstrip(b"\x00").decode() in needed:
            return major
    return None


def binary_cudnn_build(binary: Optional[str]) -> tuple[bool, Optional[int]]:
    """(built with cuDNN attention, linked CUDA major); ggml may be static or a ``libggml*.so`` beside the binary.

    The major comes from DT_NEEDED first: a CUDA 13 ``libggml-cuda`` also carries "libcudart.so.12" in a dlopen
    fallback list, and the string scan alone read it as a CUDA 12 build (then paired it with the cu12 cuDNN)."""
    if not binary:
        return False, None
    path = Path(binary)
    carries, major = _scan_file(path)
    linked = _linked_cudart_major(path)
    major = linked if linked is not None else major
    if carries and linked is not None:
        return carries, major
    try:
        siblings = sorted(path.resolve().parent.glob("libggml*.so*"))
    except OSError:
        siblings = []
    for lib in siblings:
        lib_carries, lib_major = _scan_file(lib)
        lib_linked = _linked_cudart_major(lib)
        carries = carries or lib_carries
        if lib_linked is not None and linked is None:
            linked = major = lib_linked
        major = major or lib_major
    if major is None:
        for m in _CUDART_MARKERS:
            if (path.resolve().parent / f"libcudart.so.{m}").exists():
                major = m
                break
    return carries, major


def managed_root() -> Path:
    from utils.paths.storage_roots import studio_bin_root
    return studio_bin_root() / "sd-cpp-cudnn"


def _verified_library(directory: Path, runtime: CudnnRuntime) -> Optional[str]:
    try:
        marker = json.loads((directory / _MARKER_NAME).read_text(encoding = "utf-8"))
    except (OSError, ValueError):
        return None
    if marker.get("requirements") != runtime.requirements():
        return None
    lib = directory / _LIB_RELPATH
    if not lib.is_file() or not (directory / _NVRTC_RELPATH).is_file():
        return None
    return str(lib)


def installed_library(runtime: CudnnRuntime, root: Optional[Path] = None) -> Optional[str]:
    return _verified_library((root or managed_root()) / runtime.dirname, runtime)


def _uv_executable() -> Optional[str]:
    try:
        from utils.mlx_repair import _uv_executable as find_uv
        return find_uv()
    except Exception:  # noqa: BLE001
        return shutil.which("uv")


def _child_env() -> dict[str, str]:
    try:
        from utils.child_stdio import utf8_child_env
        from utils.native_path_leases import child_env_without_native_path_secret
        return utf8_child_env(child_env_without_native_path_secret())
    except Exception:  # noqa: BLE001
        env = dict(os.environ)
        env["PYTHONIOENCODING"] = "utf-8"
        return env


def _reachable(url: str) -> bool:
    try:
        from .diffusion_nvfp4_install import _reachable as probe
        return probe(url)
    except Exception:  # noqa: BLE001 - an unanswerable probe means no network
        return False


def _installer_config(uv: Optional[str]) -> dict[str, Any]:
    try:
        from .diffusion_nvfp4_install import _installer_config as read
        return read(uv, subprocess.run)
    except Exception:  # noqa: BLE001 - unreadable config: let the installer decide
        return {"mirror": True, "unprobed": False}


def install_command(
    runtime: CudnnRuntime,
    target: Path,
    uv: Optional[str],
    *,
    own_index: bool = False,
) -> list[str]:
    try:
        # uv ignores PIP_INDEX_URL and pip ignores uv's; this carries each over.
        from .diffusion_nvfp4_install import _installer_prefix
        cmd = _installer_prefix(uv, own_index = own_index)
    except Exception:  # noqa: BLE001
        cmd = (
            [uv, "pip", "install", "--python", sys.executable]
            if uv
            else [sys.executable, "-m", "pip", "install", "--disable-pip-version-check"]
        )
    if not uv:
        cmd = [*cmd, "--no-input"]
    return [
        *cmd,
        "--target",
        str(target),
        "--no-deps",
        "--only-binary",
        ":all:",
        *runtime.requirements(),
    ]


def _verify_command(lib: Path) -> list[str]:
    # Fresh interpreter: a CUDA 12 cuDNN must never be mapped beside torch's own.
    code = (
        "import ctypes,sys;"
        "h=ctypes.CDLL(sys.argv[1],mode=ctypes.RTLD_LOCAL);"
        "f=h.cudnnGetVersion;f.restype=ctypes.c_size_t;print(f())"
    )
    return [sys.executable, "-c", code, str(lib)]


def _venv_snapshot() -> dict[str, str]:
    try:
        import importlib.metadata as md
    except Exception:  # noqa: BLE001
        return {}
    out: dict[str, str] = {}
    for dist in md.distributions():
        name = str(dist.metadata.get("Name") or "").lower()
        if name == "torch" or name.startswith("nvidia-"):
            out[name] = str(dist.version)
    return out


def _redact(text: str) -> str:
    # Index URLs in installer output can carry credentials.
    try:
        from utils.log_redaction import redact_log_text
        return redact_log_text(text)
    except Exception:  # noqa: BLE001
        return re.sub(r"://[^/\s@]+@", "://***@", text)


def _kill_tree(proc: subprocess.Popen) -> None:
    try:
        os.killpg(proc.pid, 9)
    except Exception:  # noqa: BLE001
        try:
            proc.kill()
        except Exception:  # noqa: BLE001
            pass
    try:
        proc.communicate(timeout = 10)
    except Exception:  # noqa: BLE001
        pass


def _run(
    cmd: list[str],
    timeout: int,
    cancel_event: Optional[threading.Event] = None,
) -> tuple[bool, str]:
    env = _child_env()
    # A linked target breaks when uv's cache is pruned.
    env["UV_LINK_MODE"] = "copy"
    try:
        proc = subprocess.Popen(
            cmd,
            stdout = subprocess.PIPE,
            stderr = subprocess.STDOUT,
            text = True,
            encoding = "utf-8",
            errors = "replace",
            env = env,
            start_new_session = True,
        )
    except Exception as exc:  # noqa: BLE001
        return False, f"{type(exc).__name__}: {exc}"
    deadline = time.monotonic() + timeout
    while True:
        try:
            out, _ = proc.communicate(timeout = 0.5)
            break
        except subprocess.TimeoutExpired:
            if cancel_event is not None and cancel_event.is_set():
                _kill_tree(proc)
                raise InstallCancelled()
            if time.monotonic() >= deadline:
                _kill_tree(proc)
                return False, f"timed out after {timeout}s"
    return proc.returncode == 0, _redact((out or "").strip()[-2000:])


def _file_lock(path: Path):
    try:
        from filelock import FileLock
    except ImportError:
        return None
    return FileLock(str(path))


def ensure_library(
    runtime: CudnnRuntime,
    *,
    allow_install: bool = True,
    status_cb: StatusCb = None,
    root: Optional[Path] = None,
    cancel_event: Optional[threading.Event] = None,
) -> tuple[Optional[str], Optional[str]]:
    """``(library, None)`` or ``(None, why not)``; a failure leaves sd.cpp on its ggml kernels."""
    root = root or managed_root()
    dest = root / runtime.dirname
    found = _verified_library(dest, runtime)
    if found:
        return found, None
    if not allow_install:
        return None, "the CUDA 12 cuDNN is not installed and this load does not download"
    if not install_enabled():
        return None, f"{CUDNN_INSTALL_ENV}=0"
    if runtime.cuda_major in _FAILED:
        return None, _FAILED[runtime.cuda_major]
    with _INSTALL_LOCK:
        found = _verified_library(dest, runtime)
        if found:
            return found, None
        if runtime.cuda_major in _FAILED:
            return None, _FAILED[runtime.cuda_major]
        try:
            root.mkdir(parents = True, exist_ok = True)
        except OSError as exc:
            return None, f"cannot create {root.name}: {exc}"
        lock = _file_lock(root / ".install.lock")
        if lock is not None:
            deadline = time.monotonic() + _LOCK_TIMEOUT_S
            while True:
                try:
                    lock.acquire(timeout = 1)
                    break
                except Exception as exc:  # noqa: BLE001
                    if type(exc).__name__ != "Timeout":
                        return None, _redact(f"cannot lock {root.name}: {exc}")
                    if cancel_event is not None and cancel_event.is_set():
                        return None, _CANCELLED
                    if time.monotonic() >= deadline:
                        return None, "another Unsloth process is installing cuDNN"
        try:
            found = _verified_library(dest, runtime)
            if found:
                return found, None
            try:
                reason = _install_locked(runtime, root, dest, status_cb, cancel_event)
            except InstallCancelled:
                return None, _CANCELLED
            found = _verified_library(dest, runtime)
            if found:
                return found, None
            if isinstance(reason, _Refused):
                return None, str(reason)
            reason = reason or "the installed cuDNN failed verification"
            _FAILED[runtime.cuda_major] = reason
            return None, reason
        finally:
            if lock is not None:
                try:
                    lock.release()
                except Exception:  # noqa: BLE001
                    pass


def _install_locked(
    runtime: CudnnRuntime,
    root: Path,
    dest: Path,
    status_cb: StatusCb,
    cancel_event: Optional[threading.Event] = None,
) -> Optional[str]:
    try:
        free = shutil.disk_usage(root).free
    except OSError:
        free = None
    if free is not None and free < _MIN_FREE_BYTES:
        return _Refused(
            f"needs {_MIN_FREE_BYTES >> 30} GiB free beside the Unsloth home, {free / 2**30:.1f} GiB free"
        )
    uv = _uv_executable()
    config = _installer_config(uv)
    if not (config.get("mirror") or config.get("unprobed")) and not _reachable(_PYPI_PROBE_URL):
        return _Refused("pypi.org is not reachable")
    staging = Path(tempfile.mkdtemp(prefix = ".staging-", dir = str(root)))
    try:
        before = _venv_snapshot()
        msg = f"Installing {', '.join(runtime.requirements())} for sd.cpp cuDNN attention"
        logger.info("sd_cpp.cudnn: %s into %s", msg, dest)
        if status_cb is not None:
            try:
                status_cb(msg)
            except Exception:  # noqa: BLE001
                pass
        cmd = install_command(runtime, staging, uv, own_index = bool(config.get("own_index")))
        ok, output = _run(cmd, _INSTALL_TIMEOUT_S, cancel_event)
        after = _venv_snapshot()
        if before != after:
            drift = {
                k: (before.get(k), after.get(k))
                for k in set(before) | set(after)
                if before.get(k) != after.get(k)
            }
            logger.error("sd_cpp.cudnn: the install changed this environment: %s", drift)
            return f"the install changed this environment ({sorted(drift)})"
        if not ok:
            return f"install failed: {output[-400:]}"
        lib = staging / _LIB_RELPATH
        if not lib.is_file() or not (staging / _NVRTC_RELPATH).is_file():
            return "install finished without libcudnn.so.9 and libnvrtc.so.12"
        ok, output = _run(_verify_command(lib), _VERIFY_TIMEOUT_S, cancel_event)
        version = output.strip().splitlines()[-1] if ok and output.strip() else ""
        if not ok or not version.isdigit() or int(version) != runtime.cudnn_numeric:
            return f"verification failed: libcudnn.so.9 reports {version or output[-200:]!r}, expected {runtime.cudnn_numeric}"
        (staging / _MARKER_NAME).write_text(
            json.dumps(
                {"requirements": runtime.requirements(), "cudnn_version": int(version)}, indent = 1
            ),
            encoding = "utf-8",
        )
        if dest.exists():
            shutil.rmtree(dest, ignore_errors = True)
        os.replace(staging, dest)
        logger.info("sd_cpp.cudnn: cuDNN %s ready at %s", version, dest)
        return None
    finally:
        if staging.exists():
            shutil.rmtree(staging, ignore_errors = True)


# The fork prints these once per process / shape.
_LOADED_RE = re.compile(r"cuDNN (\d+) loaded from \S+ for attention")
_PLAN_RE = re.compile(r"cuDNN SDPA plan b=\d+ hq=\d+ hk=\d+ sq=(\d+) skv=(\d+) d=(\d+)")
_NO_PLAN_RE = re.compile(r"no cuDNN SDPA plan for")
_EXEC_FAIL_RE = re.compile(r"cuDNN SDPA execute failed")
_TOO_OLD_RE = re.compile(r"reports cuDNN \d+, need 9\.0 or newer")

STATE_READY = "ready"  # library named to the child, no render seen yet
STATE_ENGAGED = "engaged"
STATE_FALLBACK = "fallback"
STATE_UNAVAILABLE = "unavailable"  # eligible, but no CUDA 12 cuDNN could be provided
STATE_OFF = "off"


@dataclass
class CudnnAttention:
    state: Optional[str] = None
    reason: Optional[str] = None
    env: tuple[tuple[str, str], ...] = ()
    cudnn_version: Optional[int] = None
    shapes: set = field(default_factory = set)
    _render: dict = field(default_factory = dict)
    _lock: threading.Lock = field(default_factory = threading.Lock, repr = False)

    def begin_render(self) -> None:
        with self._lock:
            self._render = {"loaded": False, "plans": 0, "fallbacks": 0}

    def feed(self, line: str) -> None:
        if (
            self.state is None
            or self.state in (STATE_UNAVAILABLE, STATE_OFF)
            or "cuDNN" not in line
        ):
            return
        with self._lock:
            m = _LOADED_RE.search(line)
            if m:
                self._render["loaded"] = True
                self.cudnn_version = int(m.group(1))
                return
            m = _PLAN_RE.search(line)
            if m:
                self._render["plans"] = self._render.get("plans", 0) + 1
                self.shapes.add(tuple(int(g) for g in m.groups()))
                self.state, self.reason = STATE_ENGAGED, None
                return
            if _EXEC_FAIL_RE.search(line):
                self._render["fallbacks"] = self._render.get("fallbacks", 0) + 1
                self.state, self.reason = (
                    STATE_FALLBACK,
                    "cuDNN attention failed to run; the ggml kernels ran",
                )
                return
            if _NO_PLAN_RE.search(line) or _TOO_OLD_RE.search(line):
                self._render["fallbacks"] = self._render.get("fallbacks", 0) + 1

    def end_render(self, ok: bool = True) -> None:
        """A reused sd-server prints nothing again, so only a still-``ready`` state settles here."""
        with self._lock:
            seen = self._render
            self._render = {}
            if not ok or self.state != STATE_READY:
                return
            if seen.get("plans"):
                self.state, self.reason = STATE_ENGAGED, None
            elif seen.get("fallbacks"):
                self.state, self.reason = (
                    STATE_FALLBACK,
                    "cuDNN built no attention plan; the ggml kernels ran",
                )
            elif not seen.get("loaded"):
                self.state, self.reason = STATE_FALLBACK, "sd.cpp did not load the cuDNN library"

    def status_fields(self) -> dict[str, Any]:
        return {"sd_cpp_cudnn_attention": self.state, "sd_cpp_cudnn_reason": self.reason}


def plan_cudnn_attention(
    binary: Optional[str],
    cuda_cc: "Optional[tuple[int, int]]",
    *,
    allow_install: bool = True,
    status_cb: StatusCb = None,
    platform: Optional[str] = None,
    environ: Optional[dict] = None,
    root: Optional[Path] = None,
    cancel_event: Optional[threading.Event] = None,
) -> CudnnAttention:
    """Inert (state None, no env) unless Linux, a cuDNN-attention build, a known cc >= 8.0 and a tested CUDA major."""
    platform = sys.platform if platform is None else platform
    environ = os.environ if environ is None else environ
    if not platform.startswith("linux") or not binary or cuda_cc is None:
        return CudnnAttention()
    if tuple(cuda_cc) < CUDNN_MIN_CC:
        return CudnnAttention()
    carries, major = binary_cudnn_build(binary)
    if not carries:
        return CudnnAttention()
    if _off(environ.get(GGML_CUDNN_ATTN_ENV)):
        return CudnnAttention(STATE_OFF, f"{GGML_CUDNN_ATTN_ENV}=0")
    user_lib = str(environ.get(GGML_CUDNN_LIB_ENV) or "").strip()
    if user_lib:
        return CudnnAttention(STATE_READY, f"{GGML_CUDNN_LIB_ENV} set by the user")
    runtime = CUDNN_RUNTIMES.get(major) if major is not None else None
    if runtime is None:
        return CudnnAttention(
            STATE_UNAVAILABLE,
            f"no tested cuDNN runtime for a CUDA {major if major is not None else 'unknown'} build",
        )
    if not install_enabled(environ):
        return CudnnAttention(STATE_OFF, f"{CUDNN_INSTALL_ENV}=0")
    try:
        lib, reason = ensure_library(
            runtime,
            allow_install = allow_install,
            status_cb = status_cb,
            root = root,
            cancel_event = cancel_event,
        )
    except Exception as exc:  # noqa: BLE001 - eligible, so report it rather than read as not applicable
        lib, reason = None, _redact(f"cuDNN install failed: {type(exc).__name__}: {exc}")
    if lib is None:
        logger.info("sd_cpp.cudnn: attention stays on the ggml kernels: %s", reason)
        return CudnnAttention(STATE_UNAVAILABLE, reason)
    return CudnnAttention(STATE_READY, None, env = ((GGML_CUDNN_LIB_ENV, lib),))