Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
#!/usr/bin/env python3
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Cross platform llama.cpp prebuilt installer for Unsloth Studio"""
from __future__ import annotations
import argparse
import atexit
import errno
import fnmatch
import functools
import glob
import hashlib
import importlib.util
import json
import os
import platform
import random
import re
import shutil
import signal
import site
import socket
import ssl
import stat
import struct
import subprocess
import sys
import sysconfig
import tempfile
import textwrap
import time
import urllib.error
import urllib.parse
import urllib.request
from collections.abc import Sequence
from contextlib import contextmanager
from dataclasses import dataclass, field, replace as dataclasses_replace
from pathlib import Path, PurePath
from typing import Any, Callable, Iterable, Iterator, Union
# Put studio/ on sys.path so prebuilt_core resolves when run as a script or via importlib.
_STUDIO_DIR = os.path.dirname(os.path.abspath(__file__))
if _STUDIO_DIR not in sys.path:
sys.path.insert(0, _STUDIO_DIR)
import prebuilt_core as _core # noqa: E402
try:
import nvidia_probe as _nvidia_probe # noqa: E402
except Exception: # an older checkout beside a newer setup script
_nvidia_probe = None
def nvidia_library_inventory():
"""The NVML / CUDA driver inventory, or None. A seam, so tests can hide the real host."""
return _nvidia_probe.probe() if _nvidia_probe is not None else None
def proc_driver_version() -> str:
if _nvidia_probe is None or not _nvidia_probe.enabled():
return ""
return _nvidia_probe.proc_driver_version()
def cuda_version_for_driver(driver_version: str) -> tuple[int, int] | None:
if _nvidia_probe is None:
return None
return _nvidia_probe.cuda_version_for_driver(driver_version)
from backend.utils.prebuilt.llama_backend import ( # noqa: E402
INSTALL_KIND_BACKENDS,
REQUESTABLE_BACKENDS,
backend_for_install_kind,
environment_backend_override,
install_kinds_for_backend,
is_requestable_backend,
marker_backend,
marker_backend_request,
normalize_backend_request,
)
# Core functions resolve collaborators through this module's globals at call time,
# so tests that monkeypatch names here keep working.
_OPS = _core.ModuleOps(globals())
USER_AGENT = "unsloth-studio-llama-prebuilt"
EXIT_SUCCESS = 0
EXIT_FALLBACK = 2
EXIT_ERROR = 1
EXIT_BUSY = 3
EXIT_NO_SPACE = 4
EXIT_BACKEND_UNAVAILABLE = 5
# Must stay a superset of the manifest's windows-rocm mapped_targets, else auto-Vulkan
# steals a host the fork builds for. Below this floor Vulkan is the Windows backend.
WINDOWS_HIP_PREBUILT_GFX_TARGETS = frozenset(
{
"gfx908",
"gfx90a",
"gfx1030",
"gfx1031",
"gfx1032",
"gfx1034",
"gfx1100",
"gfx1101",
"gfx1102",
"gfx1103",
"gfx1150",
"gfx1151",
"gfx1200",
"gfx1201",
}
)
WINDOWS_ROCM_FAMILY_GFX_LABELS = frozenset({"gfx103x", "gfx110x", "gfx120x"})
# Vulkan beats HIP on these APUs; an allowlist because CDNA ships no Vulkan ICD.
VULKAN_PREFERRED_GFX_TARGETS = frozenset({"gfx1150", "gfx1151"})
# No ROCm torch wheel and no linux-rocm bundle covers these, so a HIP source build has no
# rocBLAS kernels for them. Mirrors install_python_stack._LINUX_UNCOVERED_AMD_GFX.
LINUX_NO_ROCM_GFX_TARGETS = frozenset({"gfx1010", "gfx1011", "gfx1012", "gfx803", "gfx90c"})
# APUs that lead HIP enumeration and shadow a discrete card (#7776). Mirrors
# install_python_stack.py / setup.ps1 (TestShadowingIntegratedGfxParity keeps them in step);
# duplicated rather than imported because this module is vendored standalone into the backend.
SHADOWING_INTEGRATED_GFX = frozenset(
{
"gfx90c", # Renoir / Cezanne
"gfx1013", # Cyan Skillfish
"gfx1033", # Van Gogh
"gfx1035", # Rembrandt
"gfx1036", # Raphael / Mendocino
"gfx1103", # Phoenix / Hawk Point
"gfx1153", # Krackan Point 2
}
)
# Exactly ggml-org release.yml's windows-hip gpu_targets; fork-only archs are excluded.
UPSTREAM_WINDOWS_HIP_GFX_TARGETS = frozenset(
{
"gfx1030",
"gfx1031",
"gfx1032",
"gfx1100",
"gfx1101",
"gfx1102",
"gfx1150",
"gfx1151",
"gfx1200",
"gfx1201",
}
)
# Install kinds that really are a Vulkan bundle, since a Vulkan request can still end on a CPU
# bundle when its archive is missing or fails validation (issue 7357); derived from
# INSTALL_KIND_BACKENDS so a new Vulkan family is not missed.
VULKAN_INSTALL_KINDS = frozenset(
kind for kind, kind_backend in INSTALL_KIND_BACKENDS.items() if kind_backend == "vulkan"
)
CONCRETE_BACKENDS = tuple(backend for backend in REQUESTABLE_BACKENDS if backend != "auto")
# RunAsInvoker does NOT stop amd-smi's runtime elevation; the real guard is _amd_smi_allowed().
if platform.system() == "Windows":
os.environ.setdefault("__COMPAT_LAYER", "RunAsInvoker")
def _path_inside_venv(path: str) -> bool:
"""True if ``path`` is inside the active venv (sys.prefix).
The venv hipInfo.exe (AMD wheel, put on PATH by the bnb fix) is NOT a HIP SDK
(_amd_smi_allowed)."""
try:
_root = os.path.normcase(os.path.realpath(sys.prefix))
# Root-dir prefix: commonpath would match every path, and a venv is never at root.
if os.path.dirname(_root) == _root:
return False
return os.path.normcase(os.path.commonpath([os.path.realpath(path), _root])) == _root
except (ValueError, OSError):
return False
def _external_hipinfo_on_path() -> bool:
"""True if a hipinfo OUTSIDE the venv is on PATH.
shutil.which returns only the first hit, so the venv hipInfo could shadow a
real HIP SDK's; scan every PATH entry and skip the venv copy."""
for _dir in os.environ.get("PATH", "").split(os.pathsep):
_dir = _dir.strip('"') # PATH entries can be quoted on Windows
if not _dir:
continue
_candidate = os.path.join(_dir, "hipinfo.exe")
if os.path.isfile(_candidate) and not _path_inside_venv(_candidate):
return True
return False
def _amd_smi_allowed() -> bool:
"""Whether it is safe to spawn amd-smi here.
On Windows w/o a working HIP runtime, amd-smi elevates a child and pops a
UAC/DiskPart prompt RunAsInvoker can't suppress. Only call it on Windows
when a HIP SDK is detectable (hipinfo present) or UNSLOTH_ENABLE_AMD_SMI=1;
Linux/macOS always allowed. When skipped, the gfx arch still arrives via the
forwarded --rocm-gfx, so prebuilt selection is unaffected.
"""
if platform.system() != "Windows":
return True
flag = os.environ.get("UNSLOTH_ENABLE_AMD_SMI", "").strip().lower()
if flag in ("1", "true", "yes", "on"):
return True
if flag in ("0", "false", "no", "off"):
return False
# A real HIP SDK lets amd-smi run un-elevated; ignore the venv hipInfo.exe (not an SDK).
if _external_hipinfo_on_path():
return True
for _var in ("HIP_PATH", "HIP_PATH_57", "ROCM_PATH"):
_root = os.environ.get(_var)
if not _root:
continue
_candidate = os.path.join(_root, "bin", "hipinfo.exe")
if os.path.isfile(_candidate) and not _path_inside_venv(_candidate):
return True
return False
windows_hidden_subprocess_kwargs = _core.windows_hidden_subprocess_kwargs
def env_int(
name: str,
default: int,
*,
minimum: int | None = None,
) -> int:
raw = os.environ.get(name)
if raw is None:
value = default
else:
try:
value = int(str(raw).strip())
except (TypeError, ValueError):
value = default
if minimum is not None:
value = max(minimum, value)
return value
# Prefer "latest": "master" bypasses the prebuilt resolver and forces a source build.
DEFAULT_LLAMA_TAG = os.environ.get("UNSLOTH_LLAMA_TAG", "latest")
DEFAULT_PUBLISHED_REPO = "unslothai/llama.cpp"
DEFAULT_PUBLISHED_TAG = os.environ.get("UNSLOTH_LLAMA_RELEASE_TAG")
DEFAULT_PUBLISHED_MANIFEST_ASSET = os.environ.get(
"UNSLOTH_LLAMA_RELEASE_MANIFEST_ASSET", "llama-prebuilt-manifest.json"
)
DEFAULT_PUBLISHED_SHA256_ASSET = os.environ.get(
"UNSLOTH_LLAMA_RELEASE_SHA256_ASSET", "llama-prebuilt-sha256.json"
)
UPSTREAM_REPO = "ggml-org/llama.cpp"
UPSTREAM_RELEASES_API = f"https://api.github.com/repos/{UPSTREAM_REPO}/releases/latest"
TEST_MODEL_URL = "https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K.gguf"
TEST_MODEL_SHA256 = "270cba1bd5109f42d03350f60406024560464db173c0e387d91f0426d3bd256d"
VALIDATION_MODEL_CACHE_DIRNAME = ".cache"
VALIDATION_MODEL_CACHE_FILENAME = "stories260K.gguf"
# Disabled: the llama-server GPU forward pass JIT-compiles kernels and stalls installs by
# minutes on Blackwell. Set True or UNSLOTH_LLAMA_STAGED_VALIDATION=1 to re-enable.
_RUN_STAGED_PREBUILT_VALIDATION = False
def prebuilt_needs_functional_validation(choice: "AssetChoice") -> bool:
"""True when a staged prebuilt must run the smoke test: a digest proves only upstream bytes, so
only a manifest-approved bundle skips it, and that pass (minutes of cold CUDA JIT on
Blackwell sm_100) stays behind staged_validation_enabled() (issue 5854)."""
if choice.expected_sha256 is None:
return True
if choice.unmanifested_digest:
return True
return staged_validation_enabled()
def staged_validation_enabled() -> bool:
"""True when the expensive llama-server GPU smoke test should run.
Default off (Blackwell CUDA JIT stalls installs). Opt in via the module
constant or ``UNSLOTH_LLAMA_STAGED_VALIDATION`` (1/true/yes/on). Used by both
the prebuilt path and setup.sh's source-build post-check (#5854).
"""
if _RUN_STAGED_PREBUILT_VALIDATION:
return True
raw = os.environ.get("UNSLOTH_LLAMA_STAGED_VALIDATION", "").strip().lower()
return raw in ("1", "true", "yes", "on")
INSTALL_LOCK_TIMEOUT_SECONDS = 300
INSTALL_STAGING_ROOT_NAME = ".staging"
GITHUB_AUTH_HOSTS = {"api.github.com", "github.com"}
HF_AUTH_HOSTS = {"huggingface.co", "www.huggingface.co"}
RETRYABLE_HTTP_STATUS = {408, 429, 500, 502, 503, 504}
HTTP_FETCH_ATTEMPTS = 4
HTTP_FETCH_BASE_DELAY_SECONDS = 0.75
JSON_FETCH_ATTEMPTS = 3
DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES = env_int(
"UNSLOTH_LLAMA_GITHUB_RELEASE_SCAN_MAX_PAGES",
5,
minimum = 1,
)
SERVER_PORT_BIND_ATTEMPTS = 3
SERVER_BIND_RETRY_WINDOW_SECONDS = 5.0
TTY_PROGRESS_START_DELAY_SECONDS = 0.5
DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS = env_int(
"UNSLOTH_LLAMA_MAX_PREBUILT_RELEASE_FALLBACKS",
2,
minimum = 1,
)
# macOS only: upstream can ship a run of prebuilts built for a newer macOS than the host.
DEFAULT_MAX_MACOS_RELEASE_FALLBACKS = env_int(
"UNSLOTH_LLAMA_MAX_MACOS_RELEASE_FALLBACKS",
16,
minimum = 1,
)
# Last upstream macOS release before ggml-org moved macOS builds to Tahoe.
_PINNED_MACOS_FALLBACK_TAG = "b9415"
_PINNED_MACOS_LATEST_FLOOR = (26, 0)
FORCE_COMPILE_DEFAULT_REF = os.environ.get("UNSLOTH_LLAMA_FORCE_COMPILE_REF", "master")
# Per-major detection keeps cuda14+ working while llama.cpp keeps the cudart naming.
_MIN_CUDA_MAJOR = 12
_MAX_PROBE_CUDA_MAJOR = 19
# CUDA 13 dropped every target below sm_75.
_TURING_MIN_SM = 75
# Span of PyTorch's cu126 wheels; mirrors _CU126_SM_RANGE in install_python_stack.py.
_CU126_SM_RANGE = (50, 90)
# Blackwell floor is sm_100 (data-center parts sit below consumer sm_120).
_BLACKWELL_MIN_SM = 100
_BLACKWELL_MIN_TOOLKIT = (12, 8)
_BLACKWELL_SM_MIN_TOOLKIT = {103: (12, 9), 121: (12, 9)}
def _cuda_runtime_lines_for_major(major: int) -> list[str]:
"""Runtime lines a driver of this CUDA major can use, newest first down to the minimum shipped;
a driver runs its own major and any older one (backward compatibility)."""
return [f"cuda{m}" for m in range(major, _MIN_CUDA_MAJOR - 1, -1)]
@dataclass
class HostInfo:
system: str
machine: str
is_windows: bool
is_linux: bool
is_macos: bool
is_x86_64: bool
is_arm64: bool
nvidia_smi: str | None
driver_cuda_version: tuple[int, int] | None
compute_caps: list[str]
visible_cuda_devices: str | None
has_physical_nvidia: bool
has_usable_nvidia: bool
has_rocm: bool = False
# Caps before CUDA_VISIBLE_DEVICES filtering; compute_caps empties under a mask,
# which would skip min_sm/max_sm checks.
physical_compute_caps: list[str] = field(default_factory = list)
has_intel_gpu: bool = False
has_amd_gpu_without_rocm: bool = False
rocm_gfx_target: str | None = None
rocm_gfx_targets: list[str] = field(default_factory = list)
# rocm_gfx_target is a forwarded arch the probe never reported, not the probe's pick.
rocm_gfx_forwarded: bool = False
# (major, minor) from platform.mac_ver(); None off macOS or if unparseable.
# Skips a macos prebuilt whose minimum-OS exceeds this host.
macos_version: tuple[int, int] | None = None
# Set only by _vulkan_only_host. The Windows arm64 branches read it, not has_intel_gpu.
vulkan_routed: bool = False
# Set only by the probe route: the installed Vulkan bundle must itself list a Vulkan GPU.
vulkan_device_required: bool = False
@dataclass
class AssetChoice:
repo: str
tag: str
name: str
url: str
source_label: str
# Paired Windows cudart archive, overlaid on top of the main install.
runtime_name: str | None = None
runtime_url: str | None = None
runtime_sha256: str | None = None
is_ready_bundle: bool = False
install_kind: str = ""
bundle_profile: str | None = None
runtime_line: str | None = None
coverage_class: str | None = None
supported_sms: list[str] | None = None
min_sm: int | None = None
max_sm: int | None = None
selection_log: list[str] | None = None
expected_sha256: str | None = None
# Digest came from GitHub's release, not the approved manifest.
unmanifested_digest: bool = False
gfx_target: str | None = None
mapped_targets: list[str] | None = None
@dataclass(frozen = True)
class PublishedLlamaArtifact:
asset_name: str
install_kind: str
runtime_line: str | None
coverage_class: str | None
supported_sms: list[str]
min_sm: int | None
max_sm: int | None
bundle_profile: str | None
rank: int
gfx_target: str | None = None
mapped_targets: list[str] = field(default_factory = list)
@dataclass
class PublishedReleaseBundle:
repo: str
release_tag: str
upstream_tag: str
manifest_sha256: str | None = None
source_repo: str | None = None
source_repo_url: str | None = None
source_ref_kind: str | None = None
requested_source_ref: str | None = None
resolved_source_ref: str | None = None
source_commit: str | None = None
source_commit_short: str | None = None
assets: dict[str, str] = field(default_factory = dict)
manifest_asset_name: str = DEFAULT_PUBLISHED_MANIFEST_ASSET
artifacts: list[PublishedLlamaArtifact] = field(default_factory = list)
selection_log: list[str] = field(default_factory = list)
@dataclass
class LinuxCudaSelection:
attempts: list[AssetChoice]
selection_log: list[str]
@property
def primary(self) -> AssetChoice:
if not self.attempts:
raise RuntimeError("linux CUDA selection unexpectedly had no attempts")
return self.attempts[0]
CudaRuntimePreference = _core.CudaRuntimePreference
@dataclass(frozen = True)
class ApprovedArtifactHash:
asset_name: str
sha256: str
repo: str | None
kind: str | None
@dataclass
class ApprovedReleaseChecksums:
repo: str
release_tag: str
upstream_tag: str
source_repo: str | None = None
source_repo_url: str | None = None
source_ref_kind: str | None = None
requested_source_ref: str | None = None
resolved_source_ref: str | None = None
source_commit: str | None = None
source_commit_short: str | None = None
# git tree id of ggml/: the ABI key for slim whisper bundles.
ggml_tree: str | None = None
artifacts: dict[str, ApprovedArtifactHash] = field(default_factory = dict)
@dataclass(frozen = True)
class ResolvedPublishedRelease:
bundle: PublishedReleaseBundle
checksums: ApprovedReleaseChecksums
@dataclass(frozen = True)
class SourceBuildPlan:
source_url: str
source_ref: str
source_ref_kind: str
compatibility_upstream_tag: str
source_repo: str | None = None
source_repo_url: str | None = None
requested_source_ref: str | None = None
resolved_source_ref: str | None = None
source_commit: str | None = None
@dataclass(frozen = True)
class InstallReleasePlan:
requested_tag: str
llama_tag: str
release_tag: str
attempts: list[AssetChoice]
approved_checksums: ApprovedReleaseChecksums
walk_back: "_core.WalkBack | None" = None
PrebuiltFallback = _core.PrebuiltFallback
BusyInstallConflict = _core.BusyInstallConflict
class ExistingInstallSatisfied(RuntimeError):
def __init__(self, choice: AssetChoice, used_fallback: bool):
super().__init__(f"existing install already matches candidate {choice.name}")
self.choice = choice
self.used_fallback = used_fallback
class BackendUnavailable(PrebuiltFallback):
"""This host and these releases publish no bundle for the requested backend.
It shares the prebuilt rollback path, but remains distinguishable from transport
failures and from fallbacks where automatic source selection is safe.
"""
class UnknownBackendRequest(RuntimeError):
"""A newer installer recorded a backend this version cannot preserve."""
_os_error_messages = _core._os_error_messages
is_busy_lock_error = _core.is_busy_lock_error
def is_cross_device_error(exc: BaseException) -> bool:
"""True for an EXDEV "cross-device link" rename failure, e.g. inside a Docker
build where the staging tree and the install dir land on different overlayfs
layers. Unlike a busy/in-use error, such a move is safely completed by copy +
remove of the (idle) source."""
return isinstance(exc, OSError) and exc.errno == errno.EXDEV
# stderr by default so resolver modes keep stdout machine-readable (setup.sh json.load()s it);
# main() flips it since PowerShell renders stderr as NativeCommandError noise.
_LOG_TO_STDOUT = False
def log(message: str) -> None:
print(f"[llama-prebuilt] {message}", file = sys.stdout if _LOG_TO_STDOUT else sys.stderr)
def log_lines(lines: Iterable[str]) -> None:
for line in lines:
log(line)
parsed_hostname = _core.parsed_hostname
should_send_github_auth = _core.should_send_github_auth
should_send_hf_auth = _core.should_send_hf_auth
def auth_headers(url: str | None = None) -> dict[str, str]:
return _core.auth_headers(_OPS, url)
_CrossHostAuthStrippingRedirectHandler = _core._CrossHostAuthStrippingRedirectHandler
_URL_OPENER = _core._URL_OPENER
def github_api_headers(url: str | None = None) -> dict[str, str]:
return _core.github_api_headers(_OPS, url)
is_github_api_url = _core.is_github_api_url
is_retryable_url_error = _core.is_retryable_url_error
# Alias, not a copy: _http_error_retry_delay reads prebuilt_core's global.
_RATE_LIMIT_WAIT_CAP_SECONDS = _core._RATE_LIMIT_WAIT_CAP_SECONDS
_http_error_retry_delay = _core._http_error_retry_delay
sleep_backoff = _core.sleep_backoff
atomic_write_bytes = _core.atomic_write_bytes
atomic_replace_from_tempfile = _core.atomic_replace_from_tempfile
def source_archive_logical_name(upstream_tag: str) -> str:
return f"llama.cpp-source-{upstream_tag}.tar.gz"
def exact_source_archive_logical_name(source_commit: str) -> str:
return f"llama.cpp-source-commit-{source_commit}.tar.gz"
sha256_file = _core.sha256_file
sha256_bytes = _core.sha256_bytes
normalize_sha256_digest = _core.normalize_sha256_digest
def normalize_source_ref_kind(value: str | None) -> str | None:
if not isinstance(value, str):
return None
normalized = value.strip().lower()
if normalized in {"tag", "branch", "pull", "commit", "custom"}:
return normalized
return None
def normalize_source_commit(value: str | None) -> str | None:
if not isinstance(value, str):
return None
normalized = value.strip().lower()
if len(normalized) < 7 or len(normalized) > 40:
return None
if any(ch not in "0123456789abcdef" for ch in normalized):
return None
return normalized
def validate_schema_version(payload: dict[str, Any], *, label: str) -> None:
_core.validate_schema_version(payload, label = label)
def repo_slug_from_source(value: str | None) -> str | None:
if not isinstance(value, str):
return None
normalized = value.strip()
if not normalized:
return None
normalized = normalized.removesuffix(".git")
if normalized.startswith("https://github.com/"):
slug = normalized[len("https://github.com/") :]
elif normalized.startswith("http://github.com/"):
slug = normalized[len("http://github.com/") :]
elif normalized.startswith("[email protected]:"):
slug = normalized[len("[email protected]:") :]
else:
slug = normalized
slug = slug.strip("/")
parts = slug.split("/")
if len(parts) != 2 or not all(parts):
return None
return f"{parts[0]}/{parts[1]}"
def source_url_from_repo_slug(repo_slug: str | None) -> str | None:
if not isinstance(repo_slug, str) or not repo_slug:
return None
return f"https://github.com/{repo_slug}"
def source_repo_clone_url(repo: str | None, repo_url: str | None) -> str | None:
if isinstance(repo_url, str) and repo_url.strip():
return repo_url.strip().removesuffix(".git")
return source_url_from_repo_slug(repo_slug_from_source(repo))
def infer_source_ref_kind(ref: str | None) -> str:
if not isinstance(ref, str):
return "tag"
normalized = ref.strip()
lowered = normalized.lower()
if not normalized:
return "tag"
if lowered.startswith("refs/pull/") or lowered.startswith("pull/"):
return "pull"
if (
lowered.startswith("refs/heads/")
or lowered in {"main", "master", "head"}
or lowered.startswith("origin/")
):
return "branch"
normalized_commit = normalize_source_commit(normalized)
if normalized_commit is not None:
return "commit"
return "tag"
def normalized_ref_aliases(ref: str | None) -> set[str]:
if not isinstance(ref, str):
return set()
normalized = ref.strip()
if not normalized:
return set()
aliases = {normalized}
lowered = normalized.lower()
commit = normalize_source_commit(normalized)
if commit is not None:
aliases.add(commit)
if lowered.startswith("refs/heads/"):
aliases.add(normalized.split("/", 2)[2])
elif "/" not in normalized and infer_source_ref_kind(normalized) == "branch":
aliases.add(f"refs/heads/{normalized}")
if lowered.startswith("refs/pull/"):
aliases.add(normalized.removeprefix("refs/"))
elif lowered.startswith("pull/"):
aliases.add(f"refs/{normalized}")
return aliases
def refs_match(candidate_ref: str | None, requested_ref: str | None) -> bool:
candidate_aliases = normalized_ref_aliases(candidate_ref)
requested_aliases = normalized_ref_aliases(requested_ref)
if not candidate_aliases or not requested_aliases:
return False
if candidate_aliases & requested_aliases:
return True
candidate_commit = normalize_source_commit(candidate_ref)
requested_commit = normalize_source_commit(requested_ref)
if candidate_commit and requested_commit:
return candidate_commit.startswith(requested_commit) or requested_commit.startswith(
candidate_commit
)
return False
def checkout_friendly_ref(ref_kind: str | None, ref: str | None) -> str | None:
"""Normalize a source ref to a form that ``git clone --branch`` accepts.
Fully qualified branch refs like ``refs/heads/main`` are stripped to
``main``; tag refs like ``refs/tags/b8508`` are stripped to ``b8508``.
Pull refs like ``refs/pull/123/head`` are left as-is since they are
always fetched explicitly rather than cloned with ``--branch``.
"""
if not isinstance(ref, str) or not ref:
return ref
lowered = ref.lower()
if ref_kind == "branch" and lowered.startswith("refs/heads/"):
return ref.split("/", 2)[2]
if ref_kind == "tag" and lowered.startswith("refs/tags/"):
return ref.split("/", 2)[2]
return ref
WINDOWS_CUDA_ARCHS = ("x64", "arm64")
_WINDOWS_CUDA_INSTALL_KINDS = ("windows-cuda", "windows-arm64-cuda")
def _upstream_arm64_cuda_allowed() -> bool:
"""May a Windows ARM64 NVIDIA host install a CUDA llama.cpp bundle at all? Defaults
to yes, because the only other answer is a CPU-only build on a CUDA machine.
UNSLOTH_LLAMA_ARM64_CUDA=0/false/no/off opts out.
This governs every ARM64 CUDA bundle, published or upstream. It started narrower --
only the unverified upstream bundle, since that is the one with a provenance question
-- but what the flag promises the user is a CPU bundle, and scoping it to the
unverified case meant the escape hatch would switch itself off the day the fork
published an approved windows-arm64-cuda artifact. A setting that silently stops
working on someone else's release is worse than one that is merely blunt.
"""
raw = (os.environ.get("UNSLOTH_LLAMA_ARM64_CUDA") or "").strip().lower()
return raw not in {"0", "false", "no", "off"}
def _upstream_windows_hip_asset_names(release_tag: str, assets: "Iterable[str]") -> list[str]:
"""Legacy win-hip-radeon first, then win-rocm-<ver> (current upstream name), newest first."""
names = list(assets)
legacy = f"llama-{release_tag}-bin-win-hip-radeon-x64.zip"
pattern = re.compile(
rf"llama-{re.escape(release_tag)}-bin-win-rocm-([0-9]+(?:\.[0-9]+)*)-x64\.zip"
)
versioned = sorted(
(
(tuple(int(p) for p in m.group(1).split(".")), name)
for name in names
if (m := pattern.fullmatch(name)) is not None
),
reverse = True,
)
return ([legacy] if legacy in names else []) + [name for _version, name in versioned]
def _windows_rocm_vulkan_fallback_allowed(host: HostInfo) -> bool:
"""Vulkan behind HIP only with no HIP mask and no NVIDIA trace: it honours neither mask,
and following ROCm torch clears has_physical_nvidia while the card stays enumerable."""
return not (
host.has_physical_nvidia
or host.nvidia_smi
or host.physical_compute_caps
or host.compute_caps
or _hip_visible_device_mask_set()
)
def windows_cuda_arch_for_host(host: HostInfo) -> str:
"""CUDA prebuilt arch token for a Windows host (arm64 on Windows on ARM, else x64); is_arm64 is
also true on macOS and Linux aarch64, so the Windows test is part of the answer, not the
caller's job."""
return "arm64" if (host.is_windows and host.is_arm64) else "x64"
def windows_cuda_install_kind_for_arch(arch: str) -> str:
"""install_kind for a Windows CUDA prebuilt of `arch`. Kept separate from the x64
kind so the runtime-overlay, health-check and marker tables can tell the two apart
(their payloads differ: the arm64 bundle carries no cuBLASLt-for-x64 DLL set)."""
return "windows-arm64-cuda" if arch == "arm64" else "windows-cuda"
def windows_cuda_upstream_asset_names(
llama_tag: str,
runtime: str,
arch: str = "x64",
) -> list[str]:
return [
f"llama-{llama_tag}-bin-win-cuda-{runtime}-{arch}.zip",
f"cudart-llama-bin-win-cuda-{runtime}-{arch}.zip",
]
def windows_cuda_asset_aliases(
asset_name: str, *, compatibility_tag: str | None = None
) -> list[str]:
aliases: list[str] = []
_arch_re = "|".join(WINDOWS_CUDA_ARCHS)
legacy_match = re.fullmatch(
rf"llama-(?P<tag>[^/]+)-bin-win-cuda-(?P<runtime>\d+\.\d+)-(?P<arch>{_arch_re})\.zip",
asset_name,
)
if legacy_match:
runtime = legacy_match.group("runtime")
arch = legacy_match.group("arch")
aliases.append(f"cudart-llama-bin-win-cuda-{runtime}-{arch}.zip")
if compatibility_tag:
aliases.append(f"llama-{compatibility_tag}-bin-win-cuda-{runtime}-{arch}.zip")
return aliases
current_match = re.fullmatch(
rf"cudart-llama-bin-win-cuda-(?P<runtime>\d+\.\d+)-(?P<arch>{_arch_re})\.zip",
asset_name,
)
if current_match and compatibility_tag:
runtime = current_match.group("runtime")
arch = current_match.group("arch")
aliases.append(f"llama-{compatibility_tag}-bin-win-cuda-{runtime}-{arch}.zip")
return aliases
def _published_windows_cuda_runtime(
upstream_assets: dict[str, str],
major: int,
driver: tuple[int, int] | None,
arch: str = "x64",
) -> str | None:
"""Highest cuda-major.minor build published upstream that driver can run by default CUDA
compatibility, i.e. (major, minor) <= driver, else None; gating on the full version keeps a
13.3 build off a 13.1 driver that would rely on the unguaranteed minor-version path."""
if driver is None:
return None
best: int | None = None
for name in upstream_assets:
m = re.search(rf"-bin-win-cuda-(\d+)\.(\d+)-{re.escape(arch)}\.zip$", name)
if m and int(m.group(1)) == major:
minor = int(m.group(2))
if (major, minor) <= driver and (best is None or minor > best):
best = minor
return f"{major}.{best}" if best is not None else None
format_byte_count = _core.format_byte_count
_progress_percent_step = _core._progress_percent_step
DownloadProgress = _core.DownloadProgress
download_label_from_url = _core.download_label_from_url
def download_bytes(
url: str,
*,
timeout: int = 120,
attempts: int = HTTP_FETCH_ATTEMPTS,
headers: dict[str, str] | None = None,
progress_label: str | None = None,
) -> bytes:
def _download() -> bytes:
return _core.download_bytes(
_OPS,
url,
timeout = timeout,
attempts = attempts,
headers = headers,
progress_label = progress_label,
)
# Archive downloads must never be held in memory.
if url.endswith(".json"):
return _memoized_fetch(("bytes", url), _download)
return _download()
_METADATA_MEMO: dict[Any, Any] | None = None
@contextmanager
def _cached_metadata_fetches() -> Iterator[None]:
global _METADATA_MEMO
previous = _METADATA_MEMO
_METADATA_MEMO = {}
try:
yield
finally:
_METADATA_MEMO = previous
def _memoized_fetch(key: Any, produce: Callable[[], Any]) -> Any:
memo = _METADATA_MEMO
if memo is None:
return produce()
if key not in memo:
memo[key] = produce()
return memo[key]
def fetch_json(url: str) -> Any:
return _memoized_fetch(("json", url), lambda: _core.fetch_json(_OPS, url))
def download_file(url: str, destination: Path) -> None:
_core.download_file(_OPS, url, destination)
def download_file_verified(
url: str, destination: Path, *, expected_sha256: str | None, label: str
) -> None:
_core.download_file_verified(
_OPS, url, destination, expected_sha256 = expected_sha256, label = label
)
def upstream_source_archive_urls(tag: str) -> list[str]:
encoded_tag = urllib.parse.quote(tag, safe = "")
return [
f"https://codeload.github.com/{UPSTREAM_REPO}/tar.gz/refs/tags/{encoded_tag}",
f"https://github.com/{UPSTREAM_REPO}/archive/refs/tags/{encoded_tag}.tar.gz",
]
def commit_source_archive_urls(repo: str, source_commit: str) -> list[str]:
encoded_commit = urllib.parse.quote(source_commit, safe = "")
return [
f"https://codeload.github.com/{repo}/tar.gz/{encoded_commit}",
f"https://github.com/{repo}/archive/{encoded_commit}.tar.gz",
]
def release_asset_download_url(
repo: str | None, release_tag: str | None, asset_name: str | None
) -> str | None:
"""Direct download URL for a release asset, or None if any part is missing; a mix build's source
tree is reachable only as this asset, since codeload would 404 on its unpushed merge commit."""
if not repo or not release_tag or not asset_name:
return None
return (
f"https://github.com/{repo}/releases/download/"
f"{urllib.parse.quote(release_tag, safe = '')}/{urllib.parse.quote(asset_name, safe = '')}"
)
def github_release_assets(repo: str, tag: str) -> dict[str, str]:
return _core.github_release_assets(_OPS, repo, tag)
def github_release(repo: str, tag: str) -> dict[str, Any]:
return _core.github_release(_OPS, repo, tag)
def github_releases(
repo: str,
*,
per_page: int = 100,
max_pages: int = 0,
) -> list[dict[str, Any]]:
releases: list[dict[str, Any]] = []
page = 1
while True:
payload = fetch_json(
f"https://api.github.com/repos/{repo}/releases?per_page={per_page}&page={page}"
)
if not isinstance(payload, list):
raise RuntimeError(f"unexpected releases payload for {repo}")
page_items = [item for item in payload if isinstance(item, dict)]
releases.extend(page_items)
if len(payload) < per_page:
break
page += 1
if max_pages > 0 and page > max_pages:
break
return releases
def web_release_tags(repo: str, *, limit: int = 30) -> list[str]:
return _core.web_release_tags(_OPS, repo, limit = limit)
def web_release_payload(repo: str, tag: str) -> dict[str, Any]:
# Upstream publishes every bNNNN build as a prerelease, so the tag answers the label question.
default = True if repo == UPSTREAM_REPO and is_release_tag_like(tag) else None
return _core.web_release_payload(_OPS, repo, tag, prerelease_default = default)
def upstream_web_release_tags(repo: str, *, limit: int = 30) -> list[str]:
"""Recent upstream build tags, newest first, resolved without api.github.com.
Filtered to bNNNN: upstream's designated latest is a versioned pointer release
whose only asset is nightly-tag.txt, so it never names a release carrying binaries.
"""
return [tag for tag in web_release_tags(repo, limit = limit) if is_release_tag_like(tag)]
def latest_upstream_release_tag() -> str:
"""The newest upstream build tag, which the source-build fallback compiles.
Only a bNNNN REST answer is taken: /releases/latest resolves by make_latest, which
upstream points at a pointer release packaging no prebuilt, and the source build
must compile the version the prebuilt path would have installed.
"""
rest_tag = ""
try:
payload = fetch_json(UPSTREAM_RELEASES_API)
tag = payload.get("tag_name")
if isinstance(tag, str) and tag:
rest_tag = tag
if is_release_tag_like(tag):
return tag
reason: Exception = RuntimeError(
f"{UPSTREAM_RELEASES_API} named {tag}, which is not a build release"
)
else:
reason = RuntimeError(f"latest release tag was missing from {UPSTREAM_RELEASES_API}")
except RELEASE_LISTING_TRANSPORT_ERRORS as exc:
reason = exc
if not _web_fallback_eligible(UPSTREAM_REPO):
if rest_tag:
return rest_tag
raise reason
try:
tags = upstream_web_release_tags(UPSTREAM_REPO, limit = 10)
except Exception as exc: # noqa: BLE001 - the REST cause is the one worth reporting
if rest_tag:
log(f"could not resolve a build tag from the release feed ({exc}); using {rest_tag}")
return rest_tag
raise RuntimeError(f"{reason}; release feed fallback also failed: {exc}") from reason
if tags:
log(f"resolved the latest upstream build tag {tags[0]} from the release feed")
return tags[0]
if rest_tag:
log(f"the release feed listed no build tag; using {rest_tag}")
return rest_tag
raise reason
def is_release_tag_like(value: str | None) -> bool:
return isinstance(value, str) and bool(re.fullmatch(r"b\d+", value.strip()))
def release_time_sort_key(release: dict[str, Any]) -> tuple[str, int]:
published_at = release.get("published_at")
created_at = release.get("created_at")
release_id = release.get("id")
timestamp = (
published_at
if isinstance(published_at, str) and published_at
else created_at
if isinstance(created_at, str) and created_at
else ""
)
try:
normalized_id = int(release_id)
except (TypeError, ValueError):
normalized_id = 0
return (timestamp, normalized_id)
def release_is_selectable(repo: str, release: dict[str, Any]) -> bool:
"""A draft is never selectable, nor a prerelease except an upstream bNNNN build: ggml-org marks
all of them prerelease, so the plain rule would strand the upstream path hundreds of builds
back on non-prerelease releases that publish no prebuilt at all."""
if release.get("draft"):
return False
if repo == UPSTREAM_REPO:
# Upstream ships binaries only under bNNNN, all marked prerelease; versioned pointer releases
# carry no assets and must not be named newest.
return is_release_tag_like(release.get("tag_name"))
return not release.get("prerelease")
def _web_fallback_eligible(repo: str) -> bool:
"""Whether <repo> may be resolved through github.com when the REST API is down.
Upstream only. The fork already has its own API-free path through the download
host (_download_host_resolved_release), and any other repo could publish assets
this parser has never seen.
"""
return repo == UPSTREAM_REPO and _download_host_resolve_enabled()
def _web_release_or_raise(repo: str, tag: str, reason: Exception) -> dict[str, Any]:
try:
release = web_release_payload(repo, tag)
except Exception as exc: # noqa: BLE001 - report both causes, neither alone explains it
raise RuntimeError(f"{reason}; release page fallback also failed: {exc}") from reason
log(
f"GitHub REST release listing failed ({reason}); resolved {repo}@{tag} "
"from its release page instead"
)
return release
def _web_release_payloads(repo: str, reason: Exception) -> Iterable[dict[str, Any]]:
"""Recent upstream releases, newest first, with no api.github.com call.
Lazy, one page at a time, so the caller's older-release walk-back still works and
only the releases actually looked at cost a request.
"""
try:
tags = upstream_web_release_tags(repo, limit = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES * 4)
except Exception as exc: # noqa: BLE001 - report both causes
raise RuntimeError(f"{reason}; release feed fallback also failed: {exc}") from reason
if not tags:
raise RuntimeError(
f"{reason}; the release feed for {repo} listed no build tags"
) from reason
log(
f"GitHub REST release listing failed ({reason}); resolved {len(tags)} recent "
f"{repo} releases from the release feed instead"
)
for tag in tags:
try:
release = web_release_payload(repo, tag)
except Exception as exc: # noqa: BLE001 - one unreadable release is not the end of the walk
log(f"skipping {repo}@{tag}: {exc}")
continue
if not release_is_selectable(repo, release):
log(f"skipping {repo}@{tag}: not selectable (prerelease or draft)")
continue
yield release
def iter_release_payloads_by_time(
repo: str,
published_release_tag: str = "",
requested_tag: str = "",
) -> Iterable[dict[str, Any]]:
if published_release_tag:
try:
yield github_release(repo, published_release_tag)
except RELEASE_LISTING_TRANSPORT_ERRORS as exc:
if not _web_fallback_eligible(repo):
raise
yield _web_release_or_raise(repo, published_release_tag, exc)
return
if requested_tag and requested_tag != "latest" and is_release_tag_like(requested_tag):
try:
yield github_release(repo, requested_tag)
return
except urllib.error.HTTPError as exc:
# HTTPError subclasses URLError, so this clause must route the fallback itself.
if exc.code == 404:
log(f"release tag {requested_tag} not found in {repo}; scanning recent releases")
elif _web_fallback_eligible(repo):
yield _web_release_or_raise(repo, requested_tag, exc)
return
else:
raise
except RELEASE_LISTING_TRANSPORT_ERRORS as exc:
if not _web_fallback_eligible(repo):
raise
yield _web_release_or_raise(repo, requested_tag, exc)
return
except Exception:
raise
try:
listing = github_releases(repo, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES)
except RELEASE_LISTING_TRANSPORT_ERRORS as exc:
if not _web_fallback_eligible(repo):
raise
yield from _web_release_payloads(repo, exc)
return
releases = [
release
for release in listing
if isinstance(release, dict) and release_is_selectable(repo, release)
]
releases.sort(key = release_time_sort_key, reverse = True)
for release in releases:
yield release
def direct_release_matches_request(*, release_tag: str, llama_tag: str, requested_tag: str) -> bool:
if requested_tag == "latest":
return True
for candidate in (release_tag, llama_tag):
if refs_match(candidate, requested_tag):
return True
return False
def synthetic_checksums_for_release(
repo: str, release_tag: str, upstream_tag: str
) -> ApprovedReleaseChecksums:
return ApprovedReleaseChecksums(
repo = repo,
release_tag = release_tag,
upstream_tag = upstream_tag,
artifacts = {},
)
def _masked_nvidia_selection_host(host: HostInfo) -> HostInfo | None:
"""The host to select CUDA for when CUDA_VISIBLE_DEVICES hides its NVIDIA GPU, else None.
physical-without-usable means the GPU is there and the mask is empty or -1. The mask is
per process, the install tree is not, so one masked run left a CUDA machine on the CPU
bundle for good. Select CUDA against the PHYSICAL caps: the visible ones are empty, the
selector's unknown-SM path, which would let a masked sm_61 host accept an sm_70 floor.
A CUDA build under the mask runs on CPU exactly as the CPU bundle would.
None with usable ROCm (that host keeps ROCm) and for an explicit CPU request, which
never arrives here because _apply_host_overrides has cleared has_physical_nvidia.
"""
if host.has_usable_nvidia or not host.has_physical_nvidia or host.has_rocm:
return None
log(
"NVIDIA GPU present but hidden by CUDA_VISIBLE_DEVICES="
f"{host.visible_cuda_devices!r}; selecting the CUDA bundle for the hardware "
"rather than installing the CPU bundle over it"
)
if host.physical_compute_caps and not host.compute_caps:
log(
"selecting against the physical compute caps "
f"{','.join(host.physical_compute_caps)} the mask hid"
)
return dataclasses_replace(host, compute_caps = list(host.physical_compute_caps))
return host
def direct_upstream_release_plan(
release: dict[str, Any], host: HostInfo, repo: str, requested_tag: str
) -> InstallReleasePlan | None:
release_tag = release.get("tag_name")
if not isinstance(release_tag, str) or not release_tag:
return None
if not direct_release_matches_request(
release_tag = release_tag,
llama_tag = release_tag,
requested_tag = requested_tag,
):
return None
assets = release_asset_map(release)
attempts: list[AssetChoice] = []
if host.is_windows and host.is_x86_64:
masked_host = _masked_nvidia_selection_host(host)
if host.has_usable_nvidia or masked_host is not None:
selection_host = masked_host or host
torch_preference = detect_torch_cuda_runtime_preference(
selection_host, gpu_hidden_by_mask = masked_host is not None
)
attempts.extend(
windows_cuda_attempts(
selection_host,
release_tag,
assets,
torch_preference.runtime_line,
torch_preference.selection_log,
)
)
attempts[:] = _drop_blackwell_incapable_windows_cuda(selection_host, attempts)
elif host.has_rocm:
for hip_asset in _upstream_windows_hip_asset_names(release_tag, assets):
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = hip_asset,
url = assets[hip_asset],
source_label = "upstream",
install_kind = "windows-hip",
)
)
# Vulkan before CPU if HIP is missing or fails validation; never past a HIP mask, which Vulkan ignores.
if _windows_rocm_vulkan_fallback_allowed(host):
vulkan_asset = f"llama-{release_tag}-bin-win-vulkan-x64.zip"
vulkan_url = assets.get(vulkan_asset)
if vulkan_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = vulkan_asset,
url = vulkan_url,
source_label = "upstream",
install_kind = "windows-vulkan",
)
)
# Intel or AMD-without-usable-ROCm: use the Vulkan prebuilt, matching the published
# branch and the Linux x86_64 one here. Reached when the caller pins a non-default
# --published-repo, so leaving it Intel-only sent a Windows AMD host to the CPU
# attempt even when the release carries win-vulkan. Gate on no PHYSICAL NVIDIA (not
# just no usable one): a host that hid NVIDIA via CUDA_VISIBLE_DEVICES must not reach
# Vulkan, which ignores that mask and could enumerate the reserved card. Falls
# through to CPU below.
elif (host.has_intel_gpu or host.has_amd_gpu_without_rocm) and not host.has_physical_nvidia:
vulkan_asset = f"llama-{release_tag}-bin-win-vulkan-x64.zip"
vulkan_url = assets.get(vulkan_asset)
if vulkan_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = vulkan_asset,
url = vulkan_url,
source_label = "upstream",
install_kind = "windows-vulkan",
)
)
cpu_asset = f"llama-{release_tag}-bin-win-cpu-x64.zip"
cpu_url = assets.get(cpu_asset)
if cpu_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = cpu_asset,
url = cpu_url,
source_label = "upstream",
install_kind = "windows-cpu",
)
)
elif host.is_windows and host.is_arm64:
if host.has_usable_nvidia and _upstream_arm64_cuda_allowed():
torch_preference = detect_torch_cuda_runtime_preference(host)
attempts.extend(
windows_cuda_attempts(
host,
release_tag,
assets,
torch_preference.runtime_line,
torch_preference.selection_log,
arch = "arm64",
)
)
attempts[:] = _drop_blackwell_incapable_windows_cuda(host, attempts)
# Upstream ships win-vulkan-arm64 since b11396; CPU bundle still last.
if host.vulkan_routed:
vulkan_asset = f"llama-{release_tag}-bin-win-vulkan-arm64.zip"
vulkan_url = assets.get(vulkan_asset)
if vulkan_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = vulkan_asset,
url = vulkan_url,
source_label = "upstream",
install_kind = "windows-arm64-vulkan",
)
)
# Upstream ggml-org/llama.cpp ships llama-bNNNN-bin-win-cpu-arm64.zip
# (visible in the b9334 release manifest). Without this branch the
# selector returned 0 attempts and the installer fell back to a
# source build on every Windows ARM64 host.
cpu_asset = f"llama-{release_tag}-bin-win-cpu-arm64.zip"
cpu_url = assets.get(cpu_asset)
if cpu_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = cpu_asset,
url = cpu_url,
source_label = "upstream",
install_kind = "windows-arm64",
)
)
elif host.is_macos and host.is_arm64:
asset_name = f"llama-{release_tag}-bin-macos-arm64.tar.gz"
asset_url = assets.get(asset_name)
if asset_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = asset_name,
url = asset_url,
source_label = "upstream",
install_kind = "macos-arm64",
)
)
elif host.is_macos and host.is_x86_64:
asset_name = f"llama-{release_tag}-bin-macos-x64.tar.gz"
asset_url = assets.get(asset_name)
if asset_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = asset_name,
url = asset_url,
source_label = "upstream",
install_kind = "macos-x64",
)
)
elif host.is_linux and host.is_x86_64 and not host.has_usable_nvidia and not host.has_rocm:
# ROCm hosts are excluded: no per-gfx ROCm asset here, so they source-build rather than get CPU.
# Vulkan for Intel/AMD-without-ROCm falls back to CPU when no Vulkan device enumerates.
if (host.has_intel_gpu or host.has_amd_gpu_without_rocm) and not host.has_physical_nvidia:
vulkan_asset = f"llama-{release_tag}-bin-ubuntu-vulkan-x64.tar.gz"
vulkan_url = assets.get(vulkan_asset)
if vulkan_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = vulkan_asset,
url = vulkan_url,
source_label = "upstream",
install_kind = "linux-vulkan",
)
)
asset_name = f"llama-{release_tag}-bin-ubuntu-x64.tar.gz"
asset_url = assets.get(asset_name)
if asset_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = asset_name,
url = asset_url,
source_label = "upstream",
install_kind = "linux-cpu",
)
)
elif host.is_linux and host.is_arm64 and not host.has_usable_nvidia:
# Intel only, not has_amd_gpu_without_rocm as on x86_64: that widening was measured only there.
if host.has_intel_gpu and not host.has_physical_nvidia and not host.has_rocm:
vulkan_asset = f"llama-{release_tag}-bin-ubuntu-vulkan-arm64.tar.gz"
vulkan_url = assets.get(vulkan_asset)
if vulkan_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = vulkan_asset,
url = vulkan_url,
source_label = "upstream",
install_kind = "linux-vulkan",
)
)
asset_name = f"llama-{release_tag}-bin-ubuntu-arm64.tar.gz"
asset_url = assets.get(asset_name)
if asset_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = asset_name,
url = asset_url,
source_label = "upstream",
install_kind = "linux-arm64",
)
)
if not attempts:
raise PrebuiltFallback("no compatible upstream prebuilt asset was found")
# These archives are executed and a None digest passes verification, so bind every attempt
# to the release's per-asset digest and drop the ones it does not cover.
verified = _apply_release_digests(attempts, release_asset_digests(release))
if not verified:
raise PrebuiltFallback(
f"{repo}@{release_tag} publishes no asset digest for any compatible prebuilt; "
"refusing to install one unverified"
)
for attempt in verified:
attempt.unmanifested_digest = True
return InstallReleasePlan(
requested_tag = requested_tag,
llama_tag = release_tag,
release_tag = release_tag,
attempts = verified,
approved_checksums = synthetic_checksums_for_release(
repo,
release_tag,
release_tag,
),
)
def pinned_macos_release_tag(host: HostInfo, repo: str) -> str | None:
if repo != UPSTREAM_REPO or not host.is_macos or host.macos_version is None:
return None
if host.macos_version >= _PINNED_MACOS_LATEST_FLOOR:
return None
return _PINNED_MACOS_FALLBACK_TAG
def resolve_simple_install_release_plans(
llama_tag: str,
host: HostInfo,
published_repo: str,
published_release_tag: str,
*,
max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS,
) -> "tuple[str, Sequence[InstallReleasePlan]]":
repo = published_repo or DEFAULT_PUBLISHED_REPO
# The fork ships a coverage manifest; upstream is selected by asset filename below.
if repo == DEFAULT_PUBLISHED_REPO:
return _fork_manifest_release_plans(
llama_tag,
host,
published_repo,
published_release_tag,
max_release_fallbacks = max_release_fallbacks,
)
requested_tag = normalized_requested_llama_tag(llama_tag)
allow_older_release_fallback = requested_tag == "latest" and not published_release_tag
if allow_older_release_fallback:
pinned_macos = pinned_macos_release_tag(host, repo)
if pinned_macos is not None:
requested_tag = pinned_macos
allow_older_release_fallback = False
release_limit = max(1, max_release_fallbacks)
plans: list[InstallReleasePlan] = []
last_error: PrebuiltFallback | None = None
try:
releases = iter_release_payloads_by_time(repo, published_release_tag, requested_tag)
for release in releases:
try:
plan = direct_upstream_release_plan(release, host, repo, requested_tag)
if plan is None:
continue
except PrebuiltFallback as exc:
last_error = exc
if not allow_older_release_fallback:
raise
release_tag = release.get("tag_name") or "unknown"
log(f"published release skipped for install planning: {repo}@{release_tag} ({exc})")
continue
plans.append(plan)
if not allow_older_release_fallback or len(plans) >= release_limit:
break
except PrebuiltFallback:
raise
except Exception as exc:
raise PrebuiltFallback(f"failed to inspect published releases in {repo}: {exc}") from exc
if plans:
return requested_tag, plans
if last_error is not None:
raise last_error
raise PrebuiltFallback(f"no installable published llama.cpp releases were found in {repo}")
def normalized_requested_llama_tag(requested_tag: str | None) -> str:
if isinstance(requested_tag, str):
normalized = requested_tag.strip()
if normalized:
return normalized
return "latest"
normalize_compute_cap = _core.normalize_compute_cap
normalize_compute_caps = _core.normalize_compute_caps
parse_cuda_visible_devices = _core.parse_cuda_visible_devices
supports_explicit_visible_device_matching = _core.supports_explicit_visible_device_matching
select_visible_gpu_rows = _core.select_visible_gpu_rows
dir_provides_exact_library = _core.dir_provides_exact_library
def linux_runtime_dirs_for_required_libraries(required_libraries: Iterable[str]) -> list[str]:
return _core.linux_runtime_dirs_for_required_libraries(_OPS, required_libraries)
def detected_linux_runtime_lines() -> tuple[list[str], dict[str, list[str]]]:
return _core.detected_linux_runtime_lines(_OPS)
release_asset_map = _core.release_asset_map
release_asset_digests = _core.release_asset_digests
def github_release_asset_digests(repo: str, tag: str) -> dict[str, str]:
"""asset name -> sha256, from the release GitHub already serves us.
Separate call rather than a second return value from github_release_assets: that one is
mocked in a dozen suites and used on every platform, while this is read on one branch
only. Any failure is an empty mapping, which that branch reads as "refuse", so a rate
limit or an outage costs the CUDA bundle rather than installing it unchecked.
"""
try:
return release_asset_digests(github_release(repo, tag))
except Exception as exc:
log(f"could not read release asset digests for {repo}@{tag}: {exc}")
return {}
def parse_published_artifact(raw: Any) -> PublishedLlamaArtifact | None:
if not isinstance(raw, dict):
raise ValueError("artifact entry was not an object")
asset_name = raw.get("asset_name")
install_kind = raw.get("install_kind")
if not isinstance(asset_name, str) or not asset_name:
raise ValueError("artifact.asset_name was missing or not a string")
if not isinstance(install_kind, str) or not install_kind:
raise ValueError(f"artifact {asset_name} install_kind was missing or not a string")
supported_sms_raw = raw.get("supported_sms", [])
if not isinstance(supported_sms_raw, (list, tuple)):
raise ValueError(f"artifact {asset_name} supported_sms must be a list or tuple")
if any(not isinstance(value, (int, str)) for value in supported_sms_raw):
raise ValueError(f"artifact {asset_name} supported_sms entries must be ints or strings")
supported_sms = normalize_compute_caps(supported_sms_raw)
min_sm_raw = raw.get("min_sm")
max_sm_raw = raw.get("max_sm")
try:
min_sm = int(min_sm_raw) if min_sm_raw is not None else None
max_sm = int(max_sm_raw) if max_sm_raw is not None else None
except (TypeError, ValueError) as exc:
raise ValueError(f"artifact {asset_name} min_sm/max_sm were not integers") from exc
runtime_line = raw.get("runtime_line")
coverage_class = raw.get("coverage_class")
bundle_profile = raw.get("bundle_profile")
rank_raw = raw.get("rank", 1000)
if runtime_line is not None and not isinstance(runtime_line, str):
raise ValueError(f"artifact {asset_name} runtime_line was not a string")
if coverage_class is not None and not isinstance(coverage_class, str):
raise ValueError(f"artifact {asset_name} coverage_class was not a string")
if bundle_profile is not None and not isinstance(bundle_profile, str):
raise ValueError(f"artifact {asset_name} bundle_profile was not a string")
try:
rank = int(rank_raw)
except (TypeError, ValueError):
raise ValueError(f"artifact {asset_name} rank was not an integer")
gfx_target_raw = raw.get("gfx_target")
gfx_target = (
gfx_target_raw.strip()
if isinstance(gfx_target_raw, str) and gfx_target_raw.strip()
else None
)
mapped_raw = raw.get("mapped_targets", [])
mapped_targets = (
[value.strip() for value in mapped_raw if isinstance(value, str) and value.strip()]
if isinstance(mapped_raw, (list, tuple))
else []
)
return PublishedLlamaArtifact(
asset_name = asset_name,
install_kind = install_kind,
runtime_line = runtime_line if isinstance(runtime_line, str) and runtime_line else None,
coverage_class = coverage_class
if isinstance(coverage_class, str) and coverage_class
else None,
supported_sms = supported_sms,
min_sm = min_sm,
max_sm = max_sm,
bundle_profile = bundle_profile
if isinstance(bundle_profile, str) and bundle_profile
else None,
rank = rank,
gfx_target = gfx_target,
mapped_targets = mapped_targets,
)
def parse_published_release_bundle(
repo: str, release: dict[str, Any]
) -> PublishedReleaseBundle | None:
release_tag = release.get("tag_name")
if not isinstance(release_tag, str) or not release_tag:
return None
assets = release_asset_map(release)
manifest_url = assets.get(DEFAULT_PUBLISHED_MANIFEST_ASSET)
if not manifest_url:
return None
manifest_bytes = _memoized_fetch(
("bytes", manifest_url),
lambda: download_bytes(
manifest_url,
timeout = 30,
headers = auth_headers(manifest_url),
),
)
manifest_sha256 = sha256_bytes(manifest_bytes)
try:
manifest_payload = json.loads(manifest_bytes.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError(
f"published manifest {DEFAULT_PUBLISHED_MANIFEST_ASSET} was not valid JSON"
) from exc
if not isinstance(manifest_payload, dict):
raise RuntimeError(
f"published manifest {DEFAULT_PUBLISHED_MANIFEST_ASSET} was not a JSON object"
)
validate_schema_version(
manifest_payload,
label = f"published manifest {DEFAULT_PUBLISHED_MANIFEST_ASSET} in {repo}@{release_tag}",
)
component = manifest_payload.get("component")
upstream_tag = manifest_payload.get("upstream_tag")
source_repo = manifest_payload.get("source_repo")
source_repo_url = manifest_payload.get("source_repo_url")
source_ref_kind = normalize_source_ref_kind(manifest_payload.get("source_ref_kind"))
requested_source_ref = manifest_payload.get("requested_source_ref")
resolved_source_ref = manifest_payload.get("resolved_source_ref")
source_commit = normalize_source_commit(manifest_payload.get("source_commit"))
source_commit_short = manifest_payload.get("source_commit_short")
if component != "llama.cpp":
return None
if not isinstance(upstream_tag, str) or not upstream_tag:
raise RuntimeError(
f"published manifest {DEFAULT_PUBLISHED_MANIFEST_ASSET} in {repo}@{release_tag} omitted upstream_tag"
)
artifacts_payload = manifest_payload.get("artifacts")
if not isinstance(artifacts_payload, list):
raise RuntimeError(
f"published manifest {DEFAULT_PUBLISHED_MANIFEST_ASSET} in {repo}@{release_tag} omitted artifacts"
)
artifacts: list[PublishedLlamaArtifact] = []
for index, raw_artifact in enumerate(artifacts_payload):
try:
artifact = parse_published_artifact(raw_artifact)
except ValueError as exc:
log(f"published artifact ignored for {repo}@{release_tag} artifact[{index}]: {exc}")
continue
if artifact is not None:
artifacts.append(artifact)
selection_log = [
f"published_release: repo={repo}",
f"published_release: tag={release_tag}",
f"published_release: manifest={DEFAULT_PUBLISHED_MANIFEST_ASSET}",
f"published_release: upstream_tag={upstream_tag}",
]
if isinstance(source_repo, str) and source_repo:
selection_log.append(f"published_release: source_repo={source_repo}")
if source_commit:
selection_log.append(f"published_release: source_commit={source_commit}")
return PublishedReleaseBundle(
repo = repo,
release_tag = release_tag,
upstream_tag = upstream_tag,
manifest_sha256 = manifest_sha256,
source_repo = source_repo if isinstance(source_repo, str) and source_repo else None,
source_repo_url = source_repo_url
if isinstance(source_repo_url, str) and source_repo_url
else None,
source_ref_kind = source_ref_kind,
requested_source_ref = requested_source_ref
if isinstance(requested_source_ref, str) and requested_source_ref
else None,
resolved_source_ref = resolved_source_ref
if isinstance(resolved_source_ref, str) and resolved_source_ref
else None,
source_commit = source_commit,
source_commit_short = source_commit_short
if isinstance(source_commit_short, str) and source_commit_short
else None,
assets = assets,
manifest_asset_name = DEFAULT_PUBLISHED_MANIFEST_ASSET,
artifacts = artifacts,
selection_log = selection_log,
)
def parse_approved_release_checksums(
repo: str, release_tag: str, payload: Any
) -> ApprovedReleaseChecksums:
if not isinstance(payload, dict):
raise RuntimeError(
f"published checksum asset {DEFAULT_PUBLISHED_SHA256_ASSET} was not a JSON object"
)
validate_schema_version(
payload,
label = f"published checksum asset {DEFAULT_PUBLISHED_SHA256_ASSET}",
)
if payload.get("component") != "llama.cpp":
raise RuntimeError(
f"published checksum asset {DEFAULT_PUBLISHED_SHA256_ASSET} did not describe llama.cpp"
)
payload_release_tag = payload.get("release_tag")
if not isinstance(payload_release_tag, str) or not payload_release_tag:
raise RuntimeError(
f"published checksum asset {DEFAULT_PUBLISHED_SHA256_ASSET} omitted release_tag"
)
if payload_release_tag != release_tag:
raise RuntimeError(
f"published checksum asset {DEFAULT_PUBLISHED_SHA256_ASSET} release_tag={payload_release_tag} "
f"did not match pinned release tag {release_tag}"
)
upstream_tag = payload.get("upstream_tag")
if not isinstance(upstream_tag, str) or not upstream_tag:
raise RuntimeError(
f"published checksum asset {DEFAULT_PUBLISHED_SHA256_ASSET} omitted upstream_tag"
)
artifacts_payload = payload.get("artifacts")
if not isinstance(artifacts_payload, dict):
raise RuntimeError(
f"published checksum asset {DEFAULT_PUBLISHED_SHA256_ASSET} omitted artifacts"
)
artifacts: dict[str, ApprovedArtifactHash] = {}
for asset_name, raw_entry in artifacts_payload.items():
if not isinstance(asset_name, str) or not asset_name:
raise RuntimeError("published checksum asset used a non-string artifact key")
if not isinstance(raw_entry, dict):
raise RuntimeError(f"published checksum entry for {asset_name} was not an object")
digest = normalize_sha256_digest(raw_entry.get("sha256"))
if not digest:
raise RuntimeError(f"published checksum entry for {asset_name} omitted a valid sha256")
repo_value = raw_entry.get("repo")
kind_value = raw_entry.get("kind")
artifacts[asset_name] = ApprovedArtifactHash(
asset_name = asset_name,
sha256 = digest,
repo = repo_value if isinstance(repo_value, str) and repo_value else None,
kind = kind_value if isinstance(kind_value, str) and kind_value else None,
)
source_commit = normalize_source_commit(payload.get("source_commit"))
source_commit_short = payload.get("source_commit_short")
ggml_tree = payload.get("ggml_tree")
source_repo = payload.get("source_repo")
source_repo_url = payload.get("source_repo_url")
source_ref_kind = normalize_source_ref_kind(payload.get("source_ref_kind"))
requested_source_ref = payload.get("requested_source_ref")
resolved_source_ref = payload.get("resolved_source_ref")
return ApprovedReleaseChecksums(
repo = repo,
release_tag = release_tag,
upstream_tag = upstream_tag,
source_repo = source_repo if isinstance(source_repo, str) and source_repo else None,
source_repo_url = source_repo_url
if isinstance(source_repo_url, str) and source_repo_url
else None,
source_ref_kind = source_ref_kind,
requested_source_ref = requested_source_ref
if isinstance(requested_source_ref, str) and requested_source_ref
else None,
resolved_source_ref = resolved_source_ref
if isinstance(resolved_source_ref, str) and resolved_source_ref
else None,
source_commit = source_commit,
source_commit_short = source_commit_short
if isinstance(source_commit_short, str) and source_commit_short
else None,
ggml_tree = ggml_tree if isinstance(ggml_tree, str) and ggml_tree else None,
artifacts = artifacts,
)
def load_approved_release_checksums(
repo: str, release_tag: str, assets: dict[str, str]
) -> ApprovedReleaseChecksums:
checksum_url = assets.get(DEFAULT_PUBLISHED_SHA256_ASSET)
if not checksum_url:
raise PrebuiltFallback(
f"approved prebuilt release {repo}@{release_tag} did not expose {DEFAULT_PUBLISHED_SHA256_ASSET}"
)
try:
payload = fetch_json(checksum_url)
checksums = parse_approved_release_checksums(repo, release_tag, payload)
except PrebuiltFallback:
raise
except Exception as exc:
raise PrebuiltFallback(
f"approved checksum asset {DEFAULT_PUBLISHED_SHA256_ASSET} in {repo}@{release_tag} was invalid"
) from exc
return checksums
def iter_published_release_bundles(
repo: str, published_release_tag: str = ""
) -> Iterable[PublishedReleaseBundle]:
releases = (
[github_release(repo, published_release_tag)]
if published_release_tag
else github_releases(repo, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES)
)
for release in releases:
if not published_release_tag and (release.get("draft") or release.get("prerelease")):
continue
try:
bundle = parse_published_release_bundle(repo, release)
except Exception as exc:
release_tag = release.get("tag_name", "unknown")
log(f"published release metadata ignored for {repo}@{release_tag}: {exc}")
continue
if bundle is None:
continue
yield bundle
_artifact_covers_sms = _core.artifact_covers_sms
_sm_range = _core.sm_range
_blackwell_capable_linux_runtime_lines = _core.blackwell_capable_linux_runtime_lines
_UNCOVERED_CUDA_HOST_WARNINGS: set[tuple[tuple[str, ...], ...]] = set()
def _warn_uncovered_cuda_host(
host_sms: list[str],
detected_runtime_lines: list[str],
driver_runtime_lines: list[str],
artifacts: list[PublishedLlamaArtifact],
is_arm64: bool = False,
) -> None:
"""Log an uncovered CUDA host once, with cu126 advice only when it can help; selection logs
cover only viable attempts, and an explicit pin stays valid when llama.cpp visibility differs
from the physical inventory."""
# Decided before the dedupe so an unhelpful release cannot swallow the remedy.
advise_cu126 = (
not is_arm64
and any(int(sm) < _TURING_MIN_SM for sm in host_sms)
and all(_CU126_SM_RANGE[0] <= int(sm) <= _CU126_SM_RANGE[1] for sm in host_sms)
and "cuda12" in driver_runtime_lines
and "cuda12" not in detected_runtime_lines
and any(
artifact.runtime_line == "cuda12" and _artifact_covers_sms(artifact, host_sms)
for artifact in artifacts
)
)
reason = (
tuple(host_sms),
tuple(detected_runtime_lines),
tuple(driver_runtime_lines),
advise_cu126,
)
if reason in _UNCOVERED_CUDA_HOST_WARNINGS:
return
_UNCOVERED_CUDA_HOST_WARNINGS.add(reason)
message = (
"no published CUDA bundle covers this host "
f"(GPUs={','.join(f'sm_{sm}' for sm in host_sms) if host_sms else 'unknown'}"
f", CUDA runtimes on disk={','.join(detected_runtime_lines) or 'none'}"
f", runnable by this driver={','.join(driver_runtime_lines) or 'none'})"
" -- GGUF inference will fall back to a source build or the CPU"
)
if advise_cu126:
message += (
". CUDA 13 dropped pre-Turing GPUs, so the venv needs a CUDA 12 runtime:"
" re-run the Unsloth installer with UNSLOTH_TORCH_INDEX_FAMILY=cu126"
)
log(message)
def _mask_widened_sms(host: HostInfo) -> list[str] | None:
"""Every physical SM when CUDA_VISIBLE_DEVICES hides a card with an SM the visible ones lack.
The mask is per process, the install tree is not: a bundle cut to the visible cards dies
with "no kernel image" on the hidden one once the mask is gone, and torch / setup.sh
already target every physical GPU. Coverage only widens; the mask still picks the card
at runtime. None when nothing is hidden or the physical caps are unknown, and when the
visible caps are unknown: the visible-only path's unknown-GPU safeguard then decides.
"""
visible = normalize_compute_caps(host.compute_caps)
physical = normalize_compute_caps(host.physical_compute_caps)
if not visible or not physical or set(physical) <= set(visible):
return None
return normalize_compute_caps(physical + visible)
_TEGRA_FALLBACK_LOGGED: list[bool] = []
@functools.lru_cache(maxsize = 1)
def _is_tegra() -> bool:
"""Mirrors studio/backend/utils/tegra.py; never raises."""
try:
if os.path.exists("/etc/nv_tegra_release"):
return True
with open("/proc/device-tree/compatible", "rb") as fh:
return b"nvidia,tegra" in fh.read()
except OSError:
return False
def _tegra_blocks_arm64_cuda(host: HostInfo) -> bool:
"""Jetson below CUDA 13 cannot run the SBSA arm64 CUDA bundle; Thor (CUDA 13) and unknown drivers can."""
return (
host.is_linux
and host.is_arm64
and host.driver_cuda_version is not None
and host.driver_cuda_version[0] < 13
and _is_tegra()
)
def linux_cuda_choice_from_release(
host: HostInfo,
release: PublishedReleaseBundle,
preferred_runtime_line: str | None = None,
selection_preamble: Iterable[str] = (),
host_sms: list[str] | None = None,
) -> LinuxCudaSelection | None:
if _tegra_blocks_arm64_cuda(host):
if not _TEGRA_FALLBACK_LOGGED:
_TEGRA_FALLBACK_LOGGED.append(True)
log(
"NVIDIA Jetson (Tegra) with CUDA "
f"{host.driver_cuda_version[0]}.{host.driver_cuda_version[1]}: the arm64 CUDA "
"bundle targets server ARM64, building from source instead"
)
return None
selection_preamble = list(selection_preamble)
# An explicit host_sms is the widened try, whose miss falls back rather than warns.
warn_uncovered = host_sms is None
if host_sms is None:
widened_sms = _mask_widened_sms(host)
if widened_sms is not None:
widened = linux_cuda_choice_from_release(
host,
release,
preferred_runtime_line,
selection_preamble
+ [
"linux_cuda_selection: covering the GPUs CUDA_VISIBLE_DEVICES hides, "
f"physical_sms={','.join(widened_sms)}"
],
host_sms = widened_sms,
)
if widened is not None:
# The visible-only picks stay as later attempts, should the wider one fail.
visible = linux_cuda_choice_from_release(
host,
release,
preferred_runtime_line,
selection_preamble,
host_sms = normalize_compute_caps(host.compute_caps),
)
names = {a.name for a in widened.attempts}
widened.attempts.extend(
a for a in (visible.attempts if visible else []) if a.name not in names
)
return widened
# No bundle covers every physical card: keep the visible-only choice.
selection_preamble.append(
"linux_cuda_selection: no bundle covers physical_sms="
f"{','.join(widened_sms)}; selecting for the visible GPUs"
)
host_sms = normalize_compute_caps(host.compute_caps)
detected_runtime_lines, runtime_dirs = detected_linux_runtime_lines()
driver_runtime_lines = compatible_linux_runtime_lines(host)
runtime_lines = [
runtime_line
for runtime_line in detected_runtime_lines
if runtime_line in driver_runtime_lines
]
ordered_runtime_lines = list(runtime_lines)
selection_log = (
list(release.selection_log)
+ list(selection_preamble)
+ [
f"linux_cuda_selection: release={release.release_tag}",
f"linux_cuda_selection: detected_sms={','.join(host_sms) if host_sms else 'unknown'}",
"linux_cuda_selection: detected_runtime_lines="
+ (",".join(detected_runtime_lines) if detected_runtime_lines else "none"),
"linux_cuda_selection: driver_runtime_lines="
+ (",".join(driver_runtime_lines) if driver_runtime_lines else "none"),
"linux_cuda_selection: compatible_runtime_lines="
+ (",".join(runtime_lines) if runtime_lines else "none"),
]
)
for runtime_line in ("cuda13", "cuda12"):
selection_log.append(
"linux_cuda_selection: runtime_dirs "
f"{runtime_line}="
+ (
",".join(runtime_dirs.get(runtime_line, []))
if runtime_dirs.get(runtime_line)
else "none"
)
)
cuda_install_kind = "linux-arm64-cuda" if host.is_arm64 else "linux-cuda"
published_artifacts = [
artifact for artifact in release.artifacts if artifact.install_kind == cuda_install_kind
]
published_asset_names = sorted(artifact.asset_name for artifact in published_artifacts)
selection_log.append(
"linux_cuda_selection: published_assets="
+ (",".join(published_asset_names) if published_asset_names else "none")
)
if not host_sms:
selection_log.append(
"linux_cuda_selection: compute capability detection unavailable; prefer portable by runtime line"
)
if not runtime_lines:
selection_log.append(
"linux_cuda_selection: no Linux CUDA runtime line satisfied both runtime libraries and driver compatibility"
)
if not warn_uncovered:
return None
if driver_below_cuda_prebuilt_floor(host):
floor_message = cuda_driver_floor_message(host)
selection_log.append(f"linux_cuda_selection: {floor_message}")
log(floor_message)
_warn_uncovered_cuda_host(
host_sms,
detected_runtime_lines,
driver_runtime_lines,
published_artifacts,
host.is_arm64,
)
return None
blackwell_lines = (
[
line
for line in _blackwell_capable_linux_runtime_lines(host_sms, published_artifacts)
if line in ordered_runtime_lines
]
if _host_is_blackwell(host)
else []
)
if blackwell_lines:
ordered_runtime_lines = blackwell_lines + [
line for line in ordered_runtime_lines if line not in blackwell_lines
]
selection_log.append(
"linux_cuda_selection: blackwell_runtime_override prefer="
+ ",".join(blackwell_lines)
+ (f" over torch_preferred={preferred_runtime_line}" if preferred_runtime_line else "")
)
elif preferred_runtime_line:
if preferred_runtime_line in ordered_runtime_lines:
ordered_runtime_lines = [preferred_runtime_line] + [
runtime_line
for runtime_line in ordered_runtime_lines
if runtime_line != preferred_runtime_line
]
selection_log.append(
"linux_cuda_selection: torch_preferred_runtime_line="
f"{preferred_runtime_line} reordered_attempts={','.join(ordered_runtime_lines)}"
)
else:
selection_log.append(
"linux_cuda_selection: torch_preferred_runtime_line="
f"{preferred_runtime_line} unavailable_on_host"
)
attempts: list[AssetChoice] = []
seen_attempts: set[str] = set()
def add_attempt(artifact: PublishedLlamaArtifact, asset_url: str, reason: str) -> None:
asset_name = artifact.asset_name
if asset_name in seen_attempts:
return
seen_attempts.add(asset_name)
attempts.append(
AssetChoice(
repo = release.repo,
tag = release.release_tag,
name = asset_name,
url = asset_url,
source_label = "published",
is_ready_bundle = True,
install_kind = cuda_install_kind,
bundle_profile = artifact.bundle_profile,
runtime_line = artifact.runtime_line,
coverage_class = artifact.coverage_class,
supported_sms = artifact.supported_sms,
min_sm = artifact.min_sm,
max_sm = artifact.max_sm,
selection_log = list(selection_log)
+ [
"linux_cuda_selection: selected "
f"{asset_name} runtime_line={artifact.runtime_line} coverage_class={artifact.coverage_class} reason={reason}"
],
)
)
for runtime_line in ordered_runtime_lines:
coverage_candidates: list[tuple[PublishedLlamaArtifact, str]] = []
portable_candidate: tuple[PublishedLlamaArtifact, str] | None = None
for artifact in published_artifacts:
if artifact.runtime_line != runtime_line:
continue
asset_name = artifact.asset_name
asset_url = release.assets.get(asset_name)
if not asset_url:
selection_log.append(f"linux_cuda_selection: reject {asset_name} missing asset")
continue
if not host_sms and artifact.coverage_class != "portable":
selection_log.append(
"linux_cuda_selection: reject "
f"{asset_name} runtime_line={runtime_line} coverage_class={artifact.coverage_class} "
"reason=unknown_compute_caps_prefer_portable"
)
continue
if not artifact.supported_sms:
selection_log.append(
"linux_cuda_selection: reject "
f"{asset_name} runtime_line={runtime_line} coverage_class={artifact.coverage_class} "
"reason=artifact_missing_supported_sms"
)
continue
if artifact.min_sm is None or artifact.max_sm is None:
selection_log.append(
"linux_cuda_selection: reject "
f"{asset_name} runtime_line={runtime_line} coverage_class={artifact.coverage_class} "
"reason=artifact_missing_sm_bounds"
)
continue
supported_sms = {str(value) for value in artifact.supported_sms}
missing_sms = [sm for sm in host_sms if sm not in supported_sms]
out_of_range_sms = [
sm for sm in host_sms if not (artifact.min_sm <= int(sm) <= artifact.max_sm)
]
reasons: list[str] = []
if missing_sms:
reasons.append(f"missing_sms={','.join(missing_sms)}")
if out_of_range_sms:
reasons.append(f"out_of_range_sms={','.join(out_of_range_sms)}")
if reasons:
selection_log.append(
"linux_cuda_selection: reject "
f"{asset_name} runtime_line={runtime_line} coverage_class={artifact.coverage_class} "
f"coverage={artifact.min_sm}-{artifact.max_sm} supported={','.join(artifact.supported_sms)} "
f"reasons={' '.join(reasons)}"
)
continue
selection_log.append(
"linux_cuda_selection: accept "
f"{asset_name} runtime_line={runtime_line} coverage_class={artifact.coverage_class} "
f"coverage={artifact.min_sm}-{artifact.max_sm} supported={','.join(artifact.supported_sms)}"
)
if artifact.coverage_class == "portable":
portable_candidate = (artifact, asset_url)
else:
coverage_candidates.append((artifact, asset_url))
if coverage_candidates:
artifact, url = sorted(
coverage_candidates,
key = lambda item: (
_sm_range(item[0]),
item[0].rank,
item[0].max_sm or 0,
),
)[0]
add_attempt(artifact, url, "best coverage for runtime line")
if portable_candidate:
artifact, url = portable_candidate
add_attempt(artifact, url, "portable fallback for runtime line")
if not attempts:
if warn_uncovered:
_warn_uncovered_cuda_host(
host_sms,
detected_runtime_lines,
driver_runtime_lines,
published_artifacts,
host.is_arm64,
)
return None
selection_log.append(
"linux_cuda_selection: attempt_order=" + ",".join(choice.name for choice in attempts)
)
for attempt in attempts:
attempt.selection_log = list(selection_log) + [
"linux_cuda_selection: attempt "
f"{attempt.name} runtime_line={attempt.runtime_line} coverage_class={attempt.coverage_class}"
]
return LinuxCudaSelection(attempts = attempts, selection_log = selection_log)
def latest_published_linux_cuda_tag(host: HostInfo, published_repo: str) -> str | None:
for release in iter_published_release_bundles(published_repo):
if linux_cuda_choice_from_release(host, release):
return release.upstream_tag
return None
def iter_upstream_releases() -> Iterable[dict[str, Any]]:
for release in github_releases(UPSTREAM_REPO, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES):
if release.get("draft") or release.get("prerelease"):
continue
yield release
def pinned_published_release_bundle(
repo: str, published_release_tag: str
) -> PublishedReleaseBundle:
bundle = next(iter_published_release_bundles(repo, published_release_tag), None)
if bundle is None:
raise PrebuiltFallback(
f"published release {repo}@{published_release_tag} did not expose a usable llama.cpp manifest"
)
return bundle
def _validate_checksums_against_bundle(
repo: str, bundle: PublishedReleaseBundle, checksums: ApprovedReleaseChecksums
) -> ApprovedReleaseChecksums:
manifest_hash = checksums.artifacts.get(bundle.manifest_asset_name)
if manifest_hash is not None and bundle.manifest_sha256 is not None:
if manifest_hash.sha256 != bundle.manifest_sha256:
raise PrebuiltFallback(
"published manifest checksum did not match the approved checksum asset"
)
if exact_source_archive_hash(checksums) is None:
require_approved_source_hash(checksums, bundle.upstream_tag)
elif source_clone_url_for_release(checksums, bundle) is None:
# Fail closed: falling back to ggml-org source could pair the prebuilt with a mismatched tree.
raise PrebuiltFallback(
f"approved checksum asset for {repo}@{bundle.release_tag} declared an "
"exact source archive without a source repo to clone it from"
)
return checksums
def validated_checksums_for_bundle(
repo: str, bundle: PublishedReleaseBundle
) -> ApprovedReleaseChecksums:
checksums = load_approved_release_checksums(repo, bundle.release_tag, bundle.assets)
return _validate_checksums_against_bundle(repo, bundle, checksums)
def _download_host_resolve_enabled() -> bool:
"""Escape hatch to force the legacy GitHub API path instead of the
download-host fast path (which avoids the api.github.com rate limit)."""
return os.environ.get(
"UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE", ""
).strip().lower() not in {"1", "true", "yes", "on"}
def _release_asset_download_url(repo: str, tag: str, asset_name: str) -> str:
return _core.release_asset_download_url(repo, tag, asset_name)
def _download_host_latest_release_tag(repo: str) -> str | None:
return _core.download_host_latest_release_tag(_OPS, repo)
def _fetch_download_host_json(url: str) -> Any:
return _memoized_fetch(
("download-host-json", url), lambda: _core.fetch_download_host_json(_OPS, url)
)
def _download_host_resolved_release(
repo: str, published_release_tag: str = ""
) -> ResolvedPublishedRelease | None:
"""Resolve a fork release from the download host with zero api.github.com calls,
reusing the API path's parsing and validation.
When *published_release_tag* is set, that tag is fetched directly. Otherwise the
latest tag comes from the /releases/latest redirect and the checksum asset's
self-reported release_tag is cross-checked against it. Returns None (caller falls
back to the API) on a missing JSON asset or a tag mismatch."""
release_tag = (published_release_tag or "").strip() or _download_host_latest_release_tag(repo)
if not release_tag:
return None
sha_url = _release_asset_download_url(repo, release_tag, DEFAULT_PUBLISHED_SHA256_ASSET)
try:
sha_payload = _fetch_download_host_json(sha_url)
except urllib.error.HTTPError as exc:
if exc.code == 404:
return None
raise
if not isinstance(sha_payload, dict):
return None
checksums = parse_approved_release_checksums(repo, release_tag, sha_payload)
asset_names = set(checksums.artifacts) | {
DEFAULT_PUBLISHED_MANIFEST_ASSET,
DEFAULT_PUBLISHED_SHA256_ASSET,
}
synthetic_release: dict[str, Any] = {
"tag_name": release_tag,
"draft": False,
"prerelease": False,
"assets": [
{
"name": name,
"browser_download_url": _release_asset_download_url(repo, release_tag, name),
}
for name in sorted(asset_names)
],
}
try:
bundle = parse_published_release_bundle(repo, synthetic_release)
except urllib.error.HTTPError as exc:
# In-progress release: the manifest can land after the checksum asset.
if exc.code == 404:
return None
raise
if bundle is None:
return None
# A manifest artifact can be keyed under an upstream-tag alias; sha256 is still verified.
for artifact in bundle.artifacts:
bundle.assets.setdefault(
artifact.asset_name,
_release_asset_download_url(repo, release_tag, artifact.asset_name),
)
_validate_checksums_against_bundle(repo, bundle, checksums)
return ResolvedPublishedRelease(bundle = bundle, checksums = checksums)
def published_release_matches_request(bundle: PublishedReleaseBundle, requested_ref: str) -> bool:
if requested_ref == "latest":
return True
for candidate in (
bundle.upstream_tag,
bundle.requested_source_ref,
bundle.resolved_source_ref,
bundle.source_commit,
):
if refs_match(candidate, requested_ref):
return True
return False
def resolve_published_release(
requested_tag: str | None,
published_repo: str,
published_release_tag: str = "",
) -> ResolvedPublishedRelease:
repo = published_repo or DEFAULT_PUBLISHED_REPO
normalized_requested = normalized_requested_llama_tag(requested_tag)
if published_release_tag:
bundle = pinned_published_release_bundle(repo, published_release_tag)
if not published_release_matches_request(bundle, normalized_requested):
raise PrebuiltFallback(
"published release "
f"{repo}@{published_release_tag} targeted upstream tag {bundle.upstream_tag}, "
f"but requested {normalized_requested}"
)
return ResolvedPublishedRelease(
bundle = bundle,
checksums = validated_checksums_for_bundle(repo, bundle),
)
skipped_invalid = 0
for bundle in iter_published_release_bundles(repo):
if not published_release_matches_request(bundle, normalized_requested):
continue
try:
checksums = validated_checksums_for_bundle(repo, bundle)
except PrebuiltFallback as exc:
skipped_invalid += 1
log(
"published release ignored for install resolution: "
f"{repo}@{bundle.release_tag} ({exc})"
)
continue
return ResolvedPublishedRelease(bundle = bundle, checksums = checksums)
if normalized_requested == "latest":
if skipped_invalid:
raise PrebuiltFallback(
f"no usable published llama.cpp releases were available in {repo}"
)
raise PrebuiltFallback(f"no published llama.cpp releases were available in {repo}")
raise PrebuiltFallback(
f"no published prebuilt release in {repo} matched upstream tag {normalized_requested}"
)
def iter_resolved_published_releases(
requested_tag: str | None,
published_repo: str,
published_release_tag: str = "",
*,
allow_download_host_fast_path: bool = True,
continue_after_fast_path: bool = False,
) -> Iterable[ResolvedPublishedRelease]:
repo = published_repo or DEFAULT_PUBLISHED_REPO
normalized_requested = normalized_requested_llama_tag(requested_tag)
# Fast path resolves via the download host to avoid api.github.com rate limits;
# any rejection or network error falls through to the API.
fast_path_tag: str | None = None
if published_release_tag:
fast_path_tag = published_release_tag
elif normalized_requested == "latest":
fast_path_tag = ""
fast_path_release_tag: str | None = None
if (
fast_path_tag is not None
and allow_download_host_fast_path
and repo == DEFAULT_PUBLISHED_REPO
and _download_host_resolve_enabled()
):
tag_label = fast_path_tag or "latest"
try:
resolved = _download_host_resolved_release(repo, fast_path_tag)
except PrebuiltFallback as exc:
log(f"download-host release rejected for {repo}@{tag_label} ({exc}); trying GitHub API")
resolved = None
except Exception as exc:
log(
f"download-host resolve unavailable for {repo}@{tag_label} ({exc}); trying GitHub API"
)
resolved = None
if resolved is not None:
if published_release_tag and not published_release_matches_request(
resolved.bundle, normalized_requested
):
raise PrebuiltFallback(
"published release "
f"{repo}@{resolved.bundle.release_tag} targeted upstream tag "
f"{resolved.bundle.upstream_tag}, but requested {normalized_requested}"
)
yield resolved
# A pin names exactly one release, so re-resolving it via the API would be wasted.
if not continue_after_fast_path or published_release_tag:
return
fast_path_release_tag = resolved.bundle.release_tag
if published_release_tag:
bundle = pinned_published_release_bundle(repo, published_release_tag)
if not published_release_matches_request(bundle, normalized_requested):
raise PrebuiltFallback(
"published release "
f"{repo}@{published_release_tag} targeted upstream tag {bundle.upstream_tag}, "
f"but requested {normalized_requested}"
)
yield ResolvedPublishedRelease(
bundle = bundle,
checksums = validated_checksums_for_bundle(repo, bundle),
)
return
matched_any = fast_path_release_tag is not None
skipped_invalid = 0
yielded_valid = fast_path_release_tag is not None
for bundle in iter_published_release_bundles(repo):
if not published_release_matches_request(bundle, normalized_requested):
continue
if fast_path_release_tag is not None and bundle.release_tag == fast_path_release_tag:
continue
matched_any = True
try:
checksums = validated_checksums_for_bundle(repo, bundle)
except PrebuiltFallback as exc:
skipped_invalid += 1
log(
"published release ignored for install resolution: "
f"{repo}@{bundle.release_tag} ({exc})"
)
continue
yielded_valid = True
yield ResolvedPublishedRelease(bundle = bundle, checksums = checksums)
if yielded_valid:
return
if matched_any:
if skipped_invalid:
raise PrebuiltFallback(
f"no usable published llama.cpp releases were available in {repo}"
)
return
if normalized_requested == "latest":
raise PrebuiltFallback(f"no published llama.cpp releases were available in {repo}")
raise PrebuiltFallback(
f"no published prebuilt release in {repo} matched upstream tag {normalized_requested}"
)
def resolve_requested_llama_tag(
requested_tag: str | None,
published_repo: str = "",
published_release_tag: str = "",
) -> str:
"""Resolve a llama.cpp tag for source-build fallback.
Resolution order:
1. Concrete tag (e.g. "b8508") -- returned as-is.
2. "latest" with published_repo -- resolve the latest usable Unsloth
published release bundle and return its upstream_tag. This is the
preferred version that matches the published prebuilt metadata.
3. "latest" without published_repo or if (2) fails -- query the upstream
ggml-org/llama.cpp repo. This may return a newer, untested tag.
The Unsloth repo is preferred because its releases are pinned to specific
upstream tags that have been validated with Unsloth Studio. Using the
upstream bleeding-edge tag risks API/ABI incompatibilities.
"""
normalized_requested = normalized_requested_llama_tag(requested_tag)
if normalized_requested != "latest":
return normalized_requested
# Build source at the fork's tag so it matches what the prebuilt path would install.
if published_repo:
try:
return resolve_published_release(
"latest",
published_repo,
published_release_tag,
).bundle.upstream_tag
except Exception:
pass
return latest_upstream_release_tag()
def resolve_requested_install_tag(
requested_tag: str | None,
published_release_tag: str = "",
published_repo: str = DEFAULT_PUBLISHED_REPO,
) -> str:
return resolve_published_release(
requested_tag,
published_repo,
published_release_tag,
).bundle.upstream_tag
def exact_source_archive_hash(checksums: ApprovedReleaseChecksums) -> ApprovedArtifactHash | None:
if not checksums.source_commit:
return None
return checksums.artifacts.get(exact_source_archive_logical_name(checksums.source_commit))
def source_clone_url_for_release(
checksums: ApprovedReleaseChecksums, bundle: PublishedReleaseBundle
) -> str | None:
return source_repo_clone_url(
checksums.source_repo or bundle.source_repo,
checksums.source_repo_url or bundle.source_repo_url,
)
def source_build_plan_for_release(release: ResolvedPublishedRelease) -> SourceBuildPlan:
checksums = release.checksums
exact_source = exact_source_archive_hash(checksums)
source_repo = checksums.source_repo or release.bundle.source_repo
source_repo_url = checksums.source_repo_url or release.bundle.source_repo_url
requested_source_ref = checksums.requested_source_ref or release.bundle.requested_source_ref
resolved_source_ref = checksums.resolved_source_ref or release.bundle.resolved_source_ref
source_commit = checksums.source_commit or release.bundle.source_commit
source_ref_kind = checksums.source_ref_kind or release.bundle.source_ref_kind
source_url = source_clone_url_for_release(checksums, release.bundle)
if exact_source is not None and source_url and source_commit:
return SourceBuildPlan(
source_url = source_url,
source_ref = source_commit,
source_ref_kind = "commit",
compatibility_upstream_tag = release.bundle.upstream_tag,
source_repo = source_repo,
source_repo_url = source_repo_url,
requested_source_ref = requested_source_ref,
resolved_source_ref = resolved_source_ref,
source_commit = source_commit,
)
source_ref = checkout_friendly_ref(source_ref_kind, resolved_source_ref or requested_source_ref)
if source_url and source_ref and source_ref_kind in {"tag", "branch", "pull", "commit"}:
return SourceBuildPlan(
source_url = source_url,
source_ref = source_ref,
source_ref_kind = source_ref_kind,
compatibility_upstream_tag = release.bundle.upstream_tag,
source_repo = source_repo,
source_repo_url = source_repo_url,
requested_source_ref = requested_source_ref,
resolved_source_ref = resolved_source_ref,
source_commit = source_commit,
)
return SourceBuildPlan(
source_url = source_url_from_repo_slug(UPSTREAM_REPO)
or "https://github.com/ggml-org/llama.cpp",
source_ref = release.bundle.upstream_tag,
source_ref_kind = "tag",
compatibility_upstream_tag = release.bundle.upstream_tag,
source_repo = source_repo,
source_repo_url = source_repo_url,
requested_source_ref = requested_source_ref,
resolved_source_ref = resolved_source_ref,
source_commit = source_commit,
)
def resolve_source_build_plan(
requested_tag: str | None,
published_repo: str,
published_release_tag: str = "",
) -> SourceBuildPlan:
normalized_requested = normalized_requested_llama_tag(requested_tag)
if normalized_requested != "latest":
try:
release = resolve_published_release(
normalized_requested,
published_repo,
published_release_tag,
)
return source_build_plan_for_release(release)
except Exception:
pass
inferred_kind = infer_source_ref_kind(normalized_requested)
return SourceBuildPlan(
source_url = "https://github.com/ggml-org/llama.cpp",
source_ref = checkout_friendly_ref(inferred_kind, normalized_requested)
or normalized_requested,
source_ref_kind = inferred_kind,
compatibility_upstream_tag = normalized_requested,
)
if published_repo:
try:
release = resolve_published_release(
"latest",
published_repo,
published_release_tag,
)
return source_build_plan_for_release(release)
except Exception:
pass
latest_tag = latest_upstream_release_tag()
return SourceBuildPlan(
source_url = "https://github.com/ggml-org/llama.cpp",
source_ref = latest_tag,
source_ref_kind = "tag",
compatibility_upstream_tag = latest_tag,
)
def run_capture(
command: list[str],
*,
timeout: int = 30,
check: bool = False,
env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess[str]:
# amd-smi on Windows auto-elevates and pops a UAC/DiskPart prompt; RunAsInvoker prevents it.
if (
command
and platform.system() == "Windows"
and os.path.basename(command[0]).lower().startswith("amd-smi")
):
env = {**(os.environ if env is None else env), "__COMPAT_LAYER": "RunAsInvoker"}
result = subprocess.run(
command,
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = timeout,
env = env,
**windows_hidden_subprocess_kwargs(),
)
if check and result.returncode != 0:
raise subprocess.CalledProcessError(
result.returncode, command, result.stdout, result.stderr
)
return result
def _list_rocm_gfx_targets(out: str) -> list[str]:
"""Lists gfx targets from rocminfo or hipinfo, one per physical GPU; both repeat the token per
GPU, so split on per-GPU section headers to keep two entries on a dual same-arch host, while
flat strings and test stubs fall back to insertion-order dedup."""
_sections = re.split(
r"(?mi)^\s*\*+\s*$\s*agent\s+\d+\s*$|\bdevice\s*#\s*\d+\b",
out,
)
if len(_sections) > 1:
_tokens: list[str] = []
for _sec in _sections[1:]:
_m = re.search(r"gfx[1-9][0-9a-z]{2,3}", _sec.lower())
if _m:
_tokens.append(_m.group(0))
else:
_raw = re.findall(r"gfx[1-9][0-9a-z]{2,3}", out.lower())
_tokens = list(dict.fromkeys(_raw))
return _tokens
def _pick_rocm_gfx_target(
out: str,
rocr_filtered: bool = False,
hip_filtered: bool = False,
) -> str | None:
"""Choose the gfx target rocminfo / hipinfo report for the active GPU.
A bare first-match picked the wrong device on mixed APU + dGPU hosts (Strix Halo gfx1151
+ RX 7900 gfx1100), so honour HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES /
CUDA_VISIBLE_DEVICES; no env var means the first GPU, empty / "-1" means none (None).
Unmasked, the first GPU is the wrong one when an integrated GPU leads enumeration (#7776,
#11143: gfx1036 ahead of an RX 9060 XT gfx1200), since one arch picks the bundle for the
whole host. So prefer the first arch neither in SHADOWING_INTEGRATED_GFX nor gfx906, the
same rule and gfx906 exclusion as _amd_prefer_discrete_gfx in install.sh / studio/setup.sh;
an all-integrated host keeps its own arch. A mask is never second-guessed at ANY value,
including one this cannot resolve: HIP_VISIBLE_DEVICES=N is the #7624 / #7669 workaround.
hip_filtered: the output came from a HIP application (Windows hipinfo), which HIP_VISIBLE_DEVICES
/ CUDA_VISIBLE_DEVICES already filtered and renumbered, so the selected device is survivor 0.
"""
_tokens = _list_rocm_gfx_targets(out)
if not _tokens:
return None
# hipinfo is already filtered by HIP_VISIBLE_DEVICES / CUDA_VISIBLE_DEVICES (never ROCR on
# Windows): with either set, the selected device is survivor 0.
if hip_filtered:
_hip_vis = os.environ.get("HIP_VISIBLE_DEVICES")
if _hip_vis is None:
# An all-hiding ROCR mask still means no GPU, as before survivor indexing.
_rocr_vis = os.environ.get("ROCR_VISIBLE_DEVICES")
if _rocr_vis is not None and _rocr_vis.strip() in ("", "-1"):
return None
_hip_vis = os.environ.get("CUDA_VISIBLE_DEVICES")
if _hip_vis is not None:
return None if _hip_vis.strip() in ("", "-1") else _tokens[0]
_vis_raw = None
# rocminfo output is already ROCr-filtered and renumbered: only HIP-layer masks index it.
_masks = (
("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
if rocr_filtered
else ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
)
for _env in _masks:
_val = os.environ.get(_env)
if _val is not None:
_vis_raw = _val
break
if _vis_raw is None and rocr_filtered and os.environ.get("ROCR_VISIBLE_DEVICES") is not None:
_vis_raw = "0" if os.environ["ROCR_VISIBLE_DEVICES"].strip() not in ("", "-1") else ""
if _vis_raw is not None:
_vis = _vis_raw.strip()
if _vis == "" or _vis == "-1":
return None
_first = _vis.split(",")[0].strip()
try:
_idx = int(_first)
if 0 <= _idx < len(_tokens):
return _tokens[_idx]
except ValueError:
pass
return _tokens[0]
_pick = _tokens[0]
if _pick not in SHADOWING_INTEGRATED_GFX:
return _pick
for _candidate in _tokens:
if _candidate in SHADOWING_INTEGRATED_GFX or _candidate == "gfx906":
continue
return _candidate
return _pick
_WINDOWS_DISPLAY_CLASS_KEY = (
r"SYSTEM\CurrentControlSet\Control\Class\{4d36e968-e325-11ce-bfc1-08002be10318}"
)
def windows_intel_gpu_in_registry() -> bool:
"""Whether the Windows registry lists an Intel display adapter.
In-process Windows counterpart of the Linux DRM vendor-id check (0x8086),
with weaker semantics: the class key lists installed display-driver
configs, which can outlive removed hardware, where sysfs lists present
devices. A stale Intel entry at worst routes to the upstream Vulkan
prebuilt instead of the fork CPU bundle: inference still works (the
Vulkan build runs on CPU when no Vulkan device exists), at the cost of
fork-only extras such as the DiffusionGemma visual server. detect_host's
PowerShell + WMI probe can silently miss a real Intel GPU: a cold
powershell.exe start plus the first CIM query routinely exceeds the 15s
budget on hosts with slow AV scanning or a degraded WMI repository, and
the probe swallows the timeout (#4452, Arc A770 routed to the CPU
prebuilt). Reading the display-adapter class key needs no subprocess and
answers in microseconds. Matches the PCI vendor id in MatchingDeviceId
(ven_8086) or an Intel DriverDesc.
"""
return _windows_display_adapter_in_registry(
(("MatchingDeviceId", "ven_8086"), ("DriverDesc", "intel"))
)
def windows_amd_gpu_in_registry() -> bool:
"""Whether the Windows registry lists an AMD display adapter.
Same probe and same caveats as windows_intel_gpu_in_registry, against PCI
vendor id 0x1002. Two DriverDesc needles because AMD ships adapters named
"AMD Radeon(TM) 8060S Graphics" and others named only "Radeon (TM) ...".
Only consulted with no usable NVIDIA and no usable ROCm, so a host reaching
it is one the ROCm branch has already declined.
"""
return _windows_display_adapter_in_registry(
(("MatchingDeviceId", "ven_1002"), ("DriverDesc", "amd"), ("DriverDesc", "radeon"))
)
def _windows_display_adapter_in_registry(needles: tuple[tuple[str, str], ...]) -> bool:
"""Whether any display-adapter class key matches one of (value_name, needle)."""
try:
import winreg
except ImportError:
return False
try:
with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, _WINDOWS_DISPLAY_CLASS_KEY) as class_key:
for index in range(winreg.QueryInfoKey(class_key)[0]):
try:
name = winreg.EnumKey(class_key, index)
if not name.isdigit():
continue
with winreg.OpenKey(class_key, name) as adapter_key:
for value_name, needle in needles:
try:
value, _ = winreg.QueryValueEx(adapter_key, value_name)
except OSError:
continue
if needle in str(value).lower():
return True
except OSError:
continue
except Exception:
# Advisory probe: any failure must degrade to the CIM fallback, never crash the installer.
return False
return False
@functools.lru_cache(maxsize = 1)
def _darwin_hw_optional_arm64() -> bool:
"""sysctl hw.optional.arm64 read in-process (no subprocess on the launch preflight)."""
try:
import ctypes
import ctypes.util
libc = ctypes.CDLL(ctypes.util.find_library("c"))
value = ctypes.c_int(0)
size = ctypes.c_size_t(ctypes.sizeof(value))
if (
libc.sysctlbyname(
b"hw.optional.arm64", ctypes.byref(value), ctypes.byref(size), None, 0
)
== 0
):
return value.value == 1
except (OSError, AttributeError, TypeError, ValueError):
pass
return False
def _host_machine(system: str) -> str:
"""platform.machine(), but arm64 under Rosetta: macos-x64 is built without Metal."""
machine = platform.machine().lower()
if system != "Darwin" or machine != "x86_64":
return machine
return "arm64" if _darwin_hw_optional_arm64() else machine
def detect_host(*, probe_rocm_with_nvidia: bool = False) -> HostInfo:
system = platform.system()
machine = _host_machine(system)
is_windows = system == "Windows"
is_linux = system == "Linux"
is_macos = system == "Darwin"
is_x86_64 = machine in {"x86_64", "amd64"}
is_arm64 = machine in {"arm64", "aarch64"}
macos_version = parse_macos_version(platform.mac_ver()[0]) if is_macos else None
nvidia_smi = shutil.which("nvidia-smi")
driver_cuda_version = None
compute_caps: list[str] = []
physical_compute_caps: list[str] = []
visible_cuda_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
visible_device_tokens = parse_cuda_visible_devices(visible_cuda_devices)
has_physical_nvidia = False
has_usable_nvidia = False
if nvidia_smi:
# Require `nvidia-smi -L` to list a GPU: the banner prints even when the driver is unreachable.
try:
listing = run_capture([nvidia_smi, "-L"], timeout = 20)
gpu_lines = [line for line in listing.stdout.splitlines() if line.startswith("GPU ")]
if gpu_lines:
has_physical_nvidia = True
has_usable_nvidia = visible_device_tokens != []
except Exception:
pass
try:
result = run_capture([nvidia_smi], timeout = 20)
merged = "\n".join(part for part in (result.stdout, result.stderr) if part)
# Newer drivers print "CUDA UMD Version: X.Y" instead of "CUDA Version: X.Y".
cuda_match = re.search(
r"CUDA(?: UMD)? Version:\s*(\d+)\.(\d+)",
merged,
)
if cuda_match is not None:
driver_cuda_version = (
int(cuda_match.group(1)),
int(cuda_match.group(2)),
)
except Exception as exc:
log(f"nvidia-smi driver probe failed: {exc}")
try:
caps = run_capture(
[
nvidia_smi,
"--query-gpu=index,uuid,compute_cap",
"--format=csv,noheader",
],
timeout = 20,
)
visible_gpu_rows: list[tuple[str, str, str]] = []
for raw in caps.stdout.splitlines():
parts = [part.strip() for part in raw.split(",")]
if len(parts) != 3:
continue
index, uuid, cap = parts
normalized_cap = normalize_compute_cap(cap)
if normalized_cap is not None and normalized_cap not in physical_compute_caps:
physical_compute_caps.append(normalized_cap)
visible_gpu_row = select_visible_gpu_rows(
[(index, uuid, cap)],
visible_device_tokens,
)
if not visible_gpu_row:
continue
visible_gpu_rows.extend(visible_gpu_row)
if normalized_cap is None:
continue
if normalized_cap not in compute_caps:
compute_caps.append(normalized_cap)
if visible_gpu_rows:
has_usable_nvidia = True
if not has_physical_nvidia:
has_physical_nvidia = True
elif visible_device_tokens == []:
has_usable_nvidia = False
elif supports_explicit_visible_device_matching(visible_device_tokens):
has_usable_nvidia = False
elif has_physical_nvidia:
has_usable_nvidia = True
except Exception:
pass
# The driver exposes one subdir per GPU here even when nvidia-smi is absent or wedged.
if is_linux and not has_physical_nvidia:
try:
proc_gpu_dir = "/proc/driver/nvidia/gpus"
if os.path.isdir(proc_gpu_dir) and os.listdir(proc_gpu_dir):
has_physical_nvidia = True
has_usable_nvidia = visible_device_tokens != []
except OSError:
pass
if (is_linux or is_windows) and (
not has_physical_nvidia or driver_cuda_version is None or not physical_compute_caps
):
inventory = nvidia_library_inventory()
if inventory is not None and inventory.devices:
log(
f"NVIDIA inventory read through {inventory.source}: "
f"{len(inventory.devices)} GPU(s), CUDA driver {inventory.cuda_driver_version}"
)
rows = [(d["index"], d["uuid"], d["compute_cap"]) for d in inventory.devices]
if inventory.source == "nvml":
visible_rows = select_visible_gpu_rows(rows, visible_device_tokens)
else:
visible_rows = rows
if not has_physical_nvidia:
has_physical_nvidia = True
has_usable_nvidia = bool(visible_rows) or (
visible_device_tokens is not None
and visible_device_tokens != []
and not supports_explicit_visible_device_matching(visible_device_tokens)
)
for _index, _uuid, cap in rows:
normalized_cap = normalize_compute_cap(cap)
if normalized_cap is not None and normalized_cap not in physical_compute_caps:
physical_compute_caps.append(normalized_cap)
if not compute_caps:
for _index, _uuid, cap in visible_rows:
normalized_cap = normalize_compute_cap(cap)
if normalized_cap is not None and normalized_cap not in compute_caps:
compute_caps.append(normalized_cap)
if inventory is not None and driver_cuda_version is None:
driver_cuda_version = inventory.cuda_driver_version
if is_linux and has_physical_nvidia and driver_cuda_version is None:
# The kernel module reports its release even when every library call fails.
driver_cuda_version = cuda_version_for_driver(proc_driver_version())
if driver_cuda_version is not None:
log(
f"CUDA driver version {driver_cuda_version} inferred from /proc/driver/nvidia/version"
)
# Skip ROCm probing when NVIDIA is usable so co-installed ROCm tools cannot misroute the host.
def _amd_smi_has_gpu(stdout: str) -> bool:
"""Check for 'GPU: <number>' data rows, not just a table header."""
return bool(re.search(r"(?im)^gpu\s*[:\[]\s*\d", stdout))
has_rocm = False
rocm_gfx_target: str | None = None
rocm_gfx_targets: list[str] = []
if is_linux and (probe_rocm_with_nvidia or not has_usable_nvidia):
# WSL2 ROCDXG: rocminfo needs HSA_ENABLE_DXG_DETECTION=1 and may live only in /opt/rocm/bin.
_dxg_probe_env = {**os.environ}
_dxg_probe_env.setdefault("HSA_ENABLE_DXG_DETECTION", "1")
for _cmd, _check in (
# gfx000 is the CPU agent; generic ISA lines like "gfx11-generic" are not real GPUs.
(
["rocminfo"],
lambda out: bool(re.search(r"gfx[1-9][0-9a-z]{2,3}", out.lower())),
),
(["amd-smi", "list"], _amd_smi_has_gpu),
):
_exe = shutil.which(_cmd[0])
if not _exe and _cmd[0] == "rocminfo":
_opt_rocminfo = "/opt/rocm/bin/rocminfo"
if os.access(_opt_rocminfo, os.X_OK):
_exe = _opt_rocminfo
if not _exe:
continue
try:
_result = run_capture(
[_exe, *_cmd[1:]],
timeout = 10,
env = _dxg_probe_env if _cmd[0] == "rocminfo" else None,
)
except Exception:
continue
if _result.returncode == 0 and _result.stdout.strip():
if _check(_result.stdout):
has_rocm = True
rocm_gfx_targets = _list_rocm_gfx_targets(_result.stdout)
rocm_gfx_target = _pick_rocm_gfx_target(
_result.stdout, rocr_filtered = _cmd[0] == "rocminfo"
)
break
elif is_windows and (probe_rocm_with_nvidia or not has_usable_nvidia):
# The HIP SDK sets HIP_PATH / ROCM_PATH but may not add its bin dir to PATH.
def _resolve_exe(name: str) -> str | None:
"""Return full path to `name`, checking PATH then HIP_PATH/ROCM_PATH bin."""
found = shutil.which(name)
if found:
return found
for _env in ("HIP_PATH", "ROCM_PATH"):
_root = os.environ.get(_env)
if _root:
_candidate = os.path.join(_root, "bin", f"{name}.exe")
if os.path.isfile(_candidate):
return _candidate
# AMD torch wheels ship hipInfo.exe beside python.exe, usable on driver-only hosts.
_venv_candidate = os.path.join(os.path.dirname(sys.executable), f"{name}.exe")
if os.path.isfile(_venv_candidate):
return _venv_candidate
return None
_win_probes = [(["hipinfo"], lambda out: "gcnarchname" in out.lower())]
if _amd_smi_allowed():
_win_probes.append((["amd-smi", "list"], _amd_smi_has_gpu))
for _cmd, _check in _win_probes:
_exe = _resolve_exe(_cmd[0])
if not _exe:
continue
try:
_result = run_capture([_exe, *_cmd[1:]], timeout = 10)
except Exception:
continue
if _result.returncode == 0 and _result.stdout.strip():
if _check(_result.stdout):
has_rocm = True
rocm_gfx_targets = _list_rocm_gfx_targets(_result.stdout)
rocm_gfx_target = _pick_rocm_gfx_target(
_result.stdout, hip_filtered = _cmd[0] == "hipinfo"
)
break
# amdhip64.dll alone is not GPU evidence: the HIP SDK installs without an AMD GPU.
# Only probed with no usable NVIDIA and no ROCm, so "without_rocm" holds by construction.
has_intel_gpu = False
has_amd_gpu_without_rocm = False
if not has_usable_nvidia and not has_rocm:
# Vulkan honours no HIP mask, so a caller-hidden GPU must not be auto-routed to it. Linux reads
# ROCR only (sysfs probe); Windows reads all three since its probe is hipinfo, a HIP app.
_amd_hidden_by_mask = (
_hip_visible_device_mask_set()
if is_windows
else os.environ.get("ROCR_VISIBLE_DEVICES") is not None
)
if is_linux:
# No early break: a laptop can pair an Intel iGPU with an AMD dGPU.
for _vendor_file in glob.glob("/sys/class/drm/card*/device/vendor"):
try:
with open(_vendor_file, encoding = "utf-8") as _vf:
_vendor_id = _vf.read().strip().lower()
except (OSError, UnicodeDecodeError):
continue
if _vendor_id == "0x8086":
has_intel_gpu = True
elif _vendor_id == "0x1002" and not _amd_hidden_by_mask:
has_amd_gpu_without_rocm = True
elif is_windows:
has_intel_gpu = windows_intel_gpu_in_registry()
if not _amd_hidden_by_mask:
has_amd_gpu_without_rocm = windows_amd_gpu_in_registry()
# `and`, not `or`: `or` made single-vendor hosts pay the CIM probe's 15s timeout.
if not has_intel_gpu and not has_amd_gpu_without_rocm:
_ps = shutil.which("powershell") or shutil.which("pwsh")
if _ps:
try:
_result = run_capture(
[
_ps,
"-NoProfile",
"-Command",
"Get-CimInstance Win32_VideoController | "
"Select-Object -ExpandProperty Name",
],
timeout = 15,
)
if _result.returncode == 0:
_names = _result.stdout.lower()
if "intel" in _names:
has_intel_gpu = True
if not _amd_hidden_by_mask and ("amd" in _names or "radeon" in _names):
has_amd_gpu_without_rocm = True
except Exception:
pass
return HostInfo(
system = system,
machine = machine,
is_windows = is_windows,
is_linux = is_linux,
is_macos = is_macos,
is_x86_64 = is_x86_64,
is_arm64 = is_arm64,
nvidia_smi = nvidia_smi,
driver_cuda_version = driver_cuda_version,
compute_caps = compute_caps,
physical_compute_caps = physical_compute_caps,
visible_cuda_devices = visible_cuda_devices,
has_physical_nvidia = has_physical_nvidia,
has_usable_nvidia = has_usable_nvidia,
has_rocm = has_rocm,
has_intel_gpu = has_intel_gpu,
has_amd_gpu_without_rocm = has_amd_gpu_without_rocm,
rocm_gfx_target = rocm_gfx_target,
rocm_gfx_targets = rocm_gfx_targets,
macos_version = macos_version,
)
def _normalize_forwarded_gfx(value: str | None) -> str | None:
"""Extract a single gfx token from a forwarded --rocm-gfx / env value.
setup.sh/setup.ps1 already picked the active GPU, so take the token as-is
without re-applying visible-device selection. Ignore anything malformed."""
if not value:
return None
m = re.search(r"gfx[1-9][0-9a-z]{2,3}", value.lower())
return m.group(0) if m else None
def _apply_host_overrides(
host: HostInfo,
*,
override_has_rocm: bool = False,
override_rocm_gfx: str | None = None,
force_cpu: bool = False,
) -> HostInfo:
"""Fold setup.sh/setup.ps1's forwarded detection into the host profile.
A forwarded gfx (--rocm-gfx or UNSLOTH_ROCM_GFX_ARCH) implies ROCm and fills the gap
where our own hipinfo/amd-smi probe misses the arch (amd-smi-only hosts, or setup
inferring it from the GPU name), leaving no per-gfx ROCm prebuilt selected; it stays
authoritative except for the two advisory shapes narrowed below. force_cpu is the
opposite explicit signal (arm64 Linux GPU host whose source build failed): drop GPU
attributes so the CPU prebuilt for this OS/arch is selected."""
if force_cpu:
return dataclasses_replace(
host,
has_usable_nvidia = False,
has_physical_nvidia = False,
has_rocm = False,
rocm_gfx_target = None,
rocm_gfx_targets = [],
has_intel_gpu = False,
has_amd_gpu_without_rocm = False,
)
gfx = _normalize_forwarded_gfx(override_rocm_gfx)
if gfx:
# Keep a probed active arch when the forwarded one is only advisory (another GPU on this host,
# or a family label like gfx110X); any other forward is an operator override and wins.
_manual = _normalize_forwarded_gfx(os.environ.get("UNSLOTH_ROCM_GFX_ARCH"))
_physical = _host_rocm_gfx_targets(host)
_active = _active_rocm_gfx_target(host)
_advisory = gfx in _physical or gfx in WINDOWS_ROCM_FAMILY_GFX_LABELS
# Unless setup skipped a shadowing APU for the dGPU; never override a pin.
if (
_advisory
and _active in SHADOWING_INTEGRATED_GFX
and gfx in _physical
and gfx not in SHADOWING_INTEGRATED_GFX
and not _hip_visible_device_mask_set()
):
_advisory = False
if gfx != _manual and _active and gfx != _active and _advisory:
return dataclasses_replace(host, has_rocm = True)
return dataclasses_replace(
host,
has_rocm = True,
rocm_gfx_target = gfx,
# ADD the forwarded arch, never replace: the list is the physical inventory the Vulkan gate reads.
rocm_gfx_targets = list(dict.fromkeys([*_physical, gfx])),
rocm_gfx_forwarded = gfx not in _physical,
)
# A remembered probe result, not an override, so it only fills a gap.
remembered = _normalize_forwarded_gfx(os.environ.get("UNSLOTH_ROCM_GFX_REMEMBERED"))
if remembered and not _active_rocm_gfx_target(host):
return dataclasses_replace(
host,
has_rocm = True,
rocm_gfx_target = remembered,
rocm_gfx_targets = list(dict.fromkeys([*_host_rocm_gfx_targets(host), remembered])),
)
if override_has_rocm and not host.has_rocm:
return dataclasses_replace(host, has_rocm = True)
return host
def pick_windows_cuda_runtime(host: HostInfo) -> str | None:
if not host.driver_cuda_version:
return None
major, minor = host.driver_cuda_version
if major > 13 or (major == 13): # and minor >= 1):
return "13.1"
if major > 12 or (major == 12 and minor >= 4):
return "12.4"
return None
compatible_linux_runtime_lines = _core.compatible_linux_runtime_lines
windows_runtime_line_info = _core.windows_runtime_line_info
def detected_windows_runtime_lines() -> tuple[list[str], dict[str, list[str]]]:
return _core.detected_windows_runtime_lines(_OPS)
driver_below_cuda_prebuilt_floor = _core.driver_below_cuda_prebuilt_floor
cuda_driver_floor_message = _core.cuda_driver_floor_message
def compatible_windows_runtime_lines(host: HostInfo) -> list[str]:
if not host.driver_cuda_version:
return []
major, _minor = host.driver_cuda_version
# cuda12 app bundles are toolkit-12.8 builds with bundled runtime libs; CUDA
# minor-version compatibility runs them on a 12.x driver from 12.4 on (#12842).
if major < _MIN_CUDA_MAJOR or driver_below_cuda_prebuilt_floor(host):
return []
return _cuda_runtime_lines_for_major(major)
runtime_line_from_cuda_version = _core.runtime_line_from_cuda_version
detect_torch_cuda_runtime_preference = _core.detect_torch_cuda_runtime_preference
def windows_cuda_attempts(
host: HostInfo,
llama_tag: str,
upstream_assets: dict[str, str],
preferred_runtime_line: str | None,
selection_preamble: Iterable[str] = (),
arch: str | None = None,
) -> list[AssetChoice]:
if arch is None:
arch = windows_cuda_arch_for_host(host)
install_kind = windows_cuda_install_kind_for_arch(arch)
selection_log = list(selection_preamble)
if arch != "x64":
selection_log.append(f"windows_cuda_selection: arch={arch}")
driver_runtime = pick_windows_cuda_runtime(host)
detected_runtime_lines, runtime_dirs = detected_windows_runtime_lines()
compatible_runtime_lines = compatible_windows_runtime_lines(host)
normal_runtime_lines: list[str]
if detected_runtime_lines:
normal_runtime_lines = [
line for line in compatible_runtime_lines if line in detected_runtime_lines
]
else:
normal_runtime_lines = compatible_runtime_lines
selection_log.append(
"windows_cuda_selection: driver_runtime="
+ (driver_runtime if driver_runtime else "unknown")
)
selection_log.append(
"windows_cuda_selection: detected_runtime_lines="
+ (",".join(detected_runtime_lines) if detected_runtime_lines else "none")
)
for runtime_line in ("cuda13", "cuda12"):
selection_log.append(
"windows_cuda_selection: runtime_dirs "
f"{runtime_line}="
+ (
",".join(runtime_dirs.get(runtime_line, []))
if runtime_dirs.get(runtime_line)
else "none"
)
)
if detected_runtime_lines:
selection_log.append(
"windows_cuda_selection: host_runtime_order="
+ (",".join(normal_runtime_lines) if normal_runtime_lines else "none")
)
else:
selection_log.append(
"windows_cuda_selection: no CUDA runtime DLL line detected; falling back to driver order"
)
if not normal_runtime_lines:
if detected_runtime_lines:
selection_log.append(
"windows_cuda_selection: detected CUDA runtime DLLs were incompatible with the reported driver"
)
normal_runtime_lines = compatible_runtime_lines
runtime_order: list[str] = []
if preferred_runtime_line and preferred_runtime_line in normal_runtime_lines:
runtime_order.append(preferred_runtime_line)
selection_log.append(
"windows_cuda_selection: torch_preferred_runtime_line="
f"{preferred_runtime_line} reordered_attempts"
)
elif preferred_runtime_line:
selection_log.append(
"windows_cuda_selection: torch_preferred_runtime_line="
f"{preferred_runtime_line} unavailable_or_incompatible"
)
else:
selection_log.append("windows_cuda_selection: no Torch runtime preference available")
runtime_order.extend(
runtime_line for runtime_line in normal_runtime_lines if runtime_line not in runtime_order
)
runtime_order.extend(
runtime_line
for runtime_line in compatible_runtime_lines
if runtime_line not in runtime_order
)
selection_log.append(
"windows_cuda_selection: normal_runtime_order="
+ (",".join(normal_runtime_lines) if normal_runtime_lines else "none")
)
selection_log.append(
"windows_cuda_selection: attempt_runtime_order="
+ (",".join(runtime_order) if runtime_order else "none")
)
attempts: list[AssetChoice] = []
for runtime_line in runtime_order:
major = int(runtime_line.removeprefix("cuda"))
runtime = _published_windows_cuda_runtime(
upstream_assets, major, host.driver_cuda_version, arch
)
if runtime is None:
selection_log.append(
f"windows_cuda_selection: no driver-supported asset for {runtime_line}"
)
continue
selected_name = None
asset_url = None
for candidate_name in windows_cuda_upstream_asset_names(llama_tag, runtime, arch):
asset_url = upstream_assets.get(candidate_name)
if asset_url:
selected_name = candidate_name
break
if not asset_url or not selected_name:
selection_log.append(
"windows_cuda_selection: skip missing assets "
+ ",".join(windows_cuda_upstream_asset_names(llama_tag, runtime, arch))
)
continue
# Pair the cudart bundle when shipped, else the binary needs a system CUDA toolkit on PATH.
runtime_archive_name: str | None = None
runtime_archive_url: str | None = None
if selected_name.startswith("llama-"):
cudart_name = f"cudart-llama-bin-win-cuda-{runtime}-{arch}.zip"
cudart_url = upstream_assets.get(cudart_name)
if cudart_url and cudart_url != asset_url:
runtime_archive_name = cudart_name
runtime_archive_url = cudart_url
attempt_log = list(selection_log) + [
f"windows_cuda_selection: selected {selected_name} runtime={runtime}"
]
if runtime_archive_name:
attempt_log.append(
f"windows_cuda_selection: paired runtime archive {runtime_archive_name}"
)
else:
attempt_log.append(
"windows_cuda_selection: no paired runtime archive found; "
"binary will rely on a system CUDA toolkit at runtime"
)
attempts.append(
AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = selected_name,
url = asset_url,
source_label = "upstream",
install_kind = install_kind,
runtime_line = runtime_line,
runtime_name = runtime_archive_name,
runtime_url = runtime_archive_url,
selection_log = attempt_log,
)
)
return attempts
def _windows_cuda_attempt_covers_blackwell(
attempt: AssetChoice, min_toolkit: tuple[int, int] = _BLACKWELL_MIN_TOOLKIT
) -> bool:
"""True if a windows CUDA attempt is Blackwell-capable (app bundles via
declared SMs; legacy upstream bundles via toolkit minor >= min_toolkit:
12.8 for the family, 12.9 for sm_103/sm_121)."""
if attempt.install_kind not in _WINDOWS_CUDA_INSTALL_KINDS:
return False
# Legacy bundle: the toolkit minor binds (12.4 cannot offload Blackwell).
m = re.search(
rf"-bin-win-cuda-(\d+)\.(\d+)-(?:{'|'.join(WINDOWS_CUDA_ARCHS)})\.zip$", attempt.name
)
if m is not None:
return (int(m.group(1)), int(m.group(2))) >= min_toolkit
return attempt.max_sm is not None and attempt.max_sm >= _BLACKWELL_MIN_SM
_host_is_blackwell = _core.host_is_blackwell
_blackwell_min_toolkit_for_host = _core.blackwell_min_toolkit_for_host
def _drop_blackwell_incapable_windows_cuda(
host: HostInfo, attempts: list[AssetChoice]
) -> list[AssetChoice]:
"""On Blackwell hosts, drop windows CUDA attempts that cannot offload Blackwell (e.g.
cuda-12.4), which load but run a slow non-native path; non-CUDA attempts pass through.
Windows on ARM: its NVIDIA parts are Blackwell and ship only cuda-13.x bundles, so nothing
legitimate is dropped."""
if not _host_is_blackwell(host):
return attempts
min_toolkit = _blackwell_min_toolkit_for_host(host)
return [
attempt
for attempt in attempts
if attempt.install_kind not in _WINDOWS_CUDA_INSTALL_KINDS
or _windows_cuda_attempt_covers_blackwell(attempt, min_toolkit)
]
def published_windows_cuda_attempts(
host: HostInfo,
release: PublishedReleaseBundle,
preferred_runtime_line: str | None,
selection_preamble: Iterable[str] = (),
arch: str | None = None,
host_sms: list[str] | None = None,
) -> list[AssetChoice]:
selection_preamble = list(selection_preamble)
# The widened try logs nothing a visible-only retry would repeat.
log_floor = host_sms is None
if host_sms is None:
# As linux_cuda_choice_from_release: cover the cards the mask hides when a bundle can.
widened_sms = _mask_widened_sms(host)
if widened_sms is not None:
widened = published_windows_cuda_attempts(
host,
release,
preferred_runtime_line,
selection_preamble
+ [
"windows_cuda_selection: covering the GPUs CUDA_VISIBLE_DEVICES hides, "
f"physical_sms={','.join(widened_sms)}"
],
arch = arch,
host_sms = widened_sms,
)
# Only bundles whose SM metadata proves they cover every card count as widening.
widened = [
a
for a in widened
if a.supported_sms and a.min_sm is not None and a.max_sm is not None
]
if widened:
names = {a.name for a in widened}
visible = published_windows_cuda_attempts(
host,
release,
preferred_runtime_line,
selection_preamble,
arch = arch,
host_sms = normalize_compute_caps(host.compute_caps),
)
return widened + [a for a in visible if a.name not in names]
selection_preamble.append(
"windows_cuda_selection: no bundle covers physical_sms="
f"{','.join(widened_sms)}; selecting for the visible GPUs"
)
host_sms = normalize_compute_caps(host.compute_caps)
if arch is None:
arch = windows_cuda_arch_for_host(host)
install_kind = windows_cuda_install_kind_for_arch(arch)
selection_log = list(release.selection_log) + list(selection_preamble)
published_artifacts = [
artifact for artifact in release.artifacts if artifact.install_kind == install_kind
]
artifacts_by_runtime: dict[str, list[PublishedLlamaArtifact]] = {}
for artifact in published_artifacts:
if not artifact.runtime_line:
continue
artifacts_by_runtime.setdefault(artifact.runtime_line, []).append(artifact)
# Legacy bundles encode a CUDA minor and are driver-gated per minor; the fork's app-named
# bundles are gated at the major level and by torch's runtime line.
legacy_minors: list[str] = []
_legacy_pattern = rf"-bin-win-cuda-(\d+\.\d+)-{re.escape(arch)}\.zip$"
for artifact in published_artifacts:
m = re.search(_legacy_pattern, artifact.asset_name)
if m:
legacy_minors.append(m.group(1))
if legacy_minors:
ordered_lines = [
attempt.runtime_line
for attempt in windows_cuda_attempts(
host,
release.upstream_tag,
{
f"llama-{release.upstream_tag}-bin-win-cuda-{minor}-{arch}.zip": "published"
for minor in legacy_minors
},
preferred_runtime_line,
selection_log,
arch = arch,
)
if attempt.runtime_line
]
else:
detected, _ = detected_windows_runtime_lines()
compatible = compatible_windows_runtime_lines(host)
# The bundle ships its own runtime, and a Pascal card beside CUDA 13 DLLs needs cuda12-legacy.
ordered_lines = [line for line in compatible if line in detected] + [
line for line in compatible if line not in detected
]
if preferred_runtime_line and preferred_runtime_line in ordered_lines:
ordered_lines = [preferred_runtime_line] + [
line for line in ordered_lines if line != preferred_runtime_line
]
selection_log.append(
"windows_cuda_selection: app-bundle runtime lines (major-gated)="
+ (",".join(ordered_lines) if ordered_lines else "none")
)
if driver_below_cuda_prebuilt_floor(host):
floor_message = cuda_driver_floor_message(host)
selection_log.append(f"windows_cuda_selection: {floor_message}")
if log_floor:
log(floor_message)
attempts: list[AssetChoice] = []
for runtime_line in ordered_lines:
if not runtime_line:
continue
# Filter by SM coverage, else app-named bundles skip it and break newer GPUs.
targeted: list[tuple[PublishedLlamaArtifact, str, re.Match[str] | None]] = []
portable: tuple[PublishedLlamaArtifact, str, re.Match[str] | None] | None = None
for artifact in artifacts_by_runtime.get(runtime_line, []):
asset_url = release.assets.get(artifact.asset_name)
if not asset_url:
continue
am = re.search(
rf"-bin-win-cuda-(\d+)\.(\d+)-{re.escape(arch)}\.zip$", artifact.asset_name
)
if (
am is not None
and host.driver_cuda_version is not None
and (int(am.group(1)), int(am.group(2))) > host.driver_cuda_version
):
continue
has_sm_info = (
bool(artifact.supported_sms)
and artifact.min_sm is not None
and artifact.max_sm is not None
)
if host_sms and has_sm_info and not _artifact_covers_sms(artifact, host_sms):
continue
if not host_sms and has_sm_info and artifact.coverage_class != "portable":
continue
if artifact.coverage_class == "portable":
portable = (artifact, asset_url, am)
else:
targeted.append((artifact, asset_url, am))
picks: list[tuple[PublishedLlamaArtifact, str, re.Match[str] | None]] = []
if targeted:
picks.append(
sorted(
targeted,
key = lambda item: (
_sm_range(item[0]),
item[0].rank,
item[0].max_sm or 0,
),
)[0]
)
if portable is not None:
picks.append(portable)
for artifact, asset_url, am in picks:
runtime_archive_name: str | None = None
runtime_archive_url: str | None = None
if am is not None and artifact.asset_name.startswith("llama-"):
runtime = f"{am.group(1)}.{am.group(2)}"
cudart_name = f"cudart-llama-bin-win-cuda-{runtime}-{arch}.zip"
cudart_url = release.assets.get(cudart_name)
if cudart_url and cudart_url != asset_url:
runtime_archive_name = cudart_name
runtime_archive_url = cudart_url
attempt_log = list(selection_log) + [
"windows_cuda_selection: selected published asset "
f"{artifact.asset_name} for runtime_line={runtime_line}"
]
if runtime_archive_name:
attempt_log.append(
f"windows_cuda_selection: paired published runtime archive {runtime_archive_name}"
)
attempts.append(
AssetChoice(
repo = release.repo,
tag = release.release_tag,
name = artifact.asset_name,
url = asset_url,
source_label = "published",
install_kind = install_kind,
runtime_line = runtime_line,
runtime_name = runtime_archive_name,
runtime_url = runtime_archive_url,
bundle_profile = artifact.bundle_profile,
coverage_class = artifact.coverage_class,
supported_sms = artifact.supported_sms,
min_sm = artifact.min_sm,
max_sm = artifact.max_sm,
selection_log = attempt_log,
)
)
return attempts
def resolve_windows_cuda_choices(
host: HostInfo,
llama_tag: str,
upstream_assets: dict[str, str],
arch: str | None = None,
) -> list[AssetChoice]:
torch_preference = detect_torch_cuda_runtime_preference(host)
attempts = windows_cuda_attempts(
host,
llama_tag,
upstream_assets,
torch_preference.runtime_line,
torch_preference.selection_log,
arch = arch,
)
return attempts
def resolve_linux_cuda_choice(
host: HostInfo, release: PublishedReleaseBundle
) -> LinuxCudaSelection:
torch_preference = detect_torch_cuda_runtime_preference(host)
selection = linux_cuda_choice_from_release(
host,
release,
preferred_runtime_line = torch_preference.runtime_line,
selection_preamble = torch_preference.selection_log,
)
if selection is not None:
return selection
raise PrebuiltFallback("no compatible published Linux CUDA bundle was found")
def published_asset_choice_for_kind(
release: PublishedReleaseBundle,
install_kind: str,
*,
host: HostInfo | None = None,
) -> AssetChoice | None:
candidates = sorted(
(artifact for artifact in release.artifacts if artifact.install_kind == install_kind),
key = lambda artifact: (artifact.rank, artifact.asset_name),
)
for artifact in candidates:
if host is not None and install_kind in VULKAN_INSTALL_KINDS:
identity = f"{artifact.bundle_profile or ''} {artifact.asset_name}".lower()
if host.is_arm64:
if not any(token in identity for token in ("arm64", "aarch64")):
continue
elif host.is_x86_64:
if not any(token in identity for token in ("x64", "x86_64", "amd64")):
continue
else:
continue
asset_url = release.assets.get(artifact.asset_name)
if not asset_url:
continue
return AssetChoice(
repo = release.repo,
tag = release.release_tag,
name = artifact.asset_name,
url = asset_url,
source_label = "published",
install_kind = install_kind,
bundle_profile = artifact.bundle_profile,
runtime_line = artifact.runtime_line,
selection_log = list(release.selection_log)
+ [f"published_selection: selected {artifact.asset_name} install_kind={install_kind}"],
)
return None
def _detect_host_rocm_version() -> tuple[int, int] | None:
"""Return (major, minor) of the installed ROCm runtime, or None.
Best-effort read from /opt/rocm/.info/version, amd-smi version, and
hipconfig --version. Used to pick a compatible upstream llama.cpp
ROCm prebuilt rather than always taking the numerically newest one
(which can be newer than the host runtime).
"""
rocm_root = os.environ.get("ROCM_PATH") or "/opt/rocm"
for path in (
os.path.join(rocm_root, ".info", "version"),
os.path.join(rocm_root, "lib", "rocm_version"),
):
try:
with open(path, encoding = "utf-8") as fh:
parts = fh.read().strip().split("-")[0].split(".")
if len(parts) >= 2:
return int(parts[0]), int(parts[1])
except Exception:
pass
amd_smi = shutil.which("amd-smi") if _amd_smi_allowed() else None
if amd_smi:
try:
result = run_capture([amd_smi, "version"], timeout = 5)
if result.returncode == 0:
m = re.search(r"ROCm version:\s*(\d+)\.(\d+)", result.stdout)
if m:
return int(m.group(1)), int(m.group(2))
except Exception:
pass
hipconfig = shutil.which("hipconfig")
if hipconfig:
try:
result = subprocess.run(
[hipconfig, "--version"],
stdout = subprocess.PIPE,
stderr = subprocess.DEVNULL,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 5,
)
if result.returncode == 0:
raw = (result.stdout or "").strip().split("\n")[0]
parts = raw.split(".")
if len(parts) >= 2 and parts[0].isdigit() and parts[1].split("-")[0].isdigit():
return int(parts[0]), int(parts[1].split("-")[0])
except Exception:
pass
# Deliberately diverges from install.sh: stops at the first answer, no status-word gate.
for _cmd in (
["dpkg-query", "-W", "-f=${Version}\n", "rocm-core"],
["rpm", "-q", "--qf", "%{VERSION}\n", "rocm-core"],
):
_exe = shutil.which(_cmd[0])
if not _exe:
continue
try:
_result = subprocess.run(
[_exe, *_cmd[1:]],
stdout = subprocess.PIPE,
stderr = subprocess.DEVNULL,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 5,
)
except Exception:
continue
if _result.returncode != 0 or not _result.stdout.strip():
continue
_raw = _result.stdout.strip()
_raw = re.sub(r"^\d+:", "", _raw)
_m = re.match(r"(\d+)[.-](\d+)", _raw)
if _m:
return int(_m.group(1)), int(_m.group(2))
return None
def published_rocm_choice_for_host(
release: PublishedReleaseBundle, host: HostInfo, install_kind: str
) -> AssetChoice | None:
"""Select the published ROCm bundle whose gfx target covers the host GPU.
The manifest's gfx_target uses umbrella family labels (gfx110X, gfx120X,
...). A host's detected gfx is matched against the bundle's concrete
mapped_targets list, or against the family label itself. Returns None when no
published bundle covers the GPU, so the caller falls back to a HIP source
build."""
if not host.rocm_gfx_target:
return None
gfx = host.rocm_gfx_target.lower().strip()
for artifact in release.artifacts:
if artifact.install_kind != install_kind:
continue
# Match concrete built archs so an unbuilt in-family arch (gfx1033) is not served; also accept
# the family label, which the update path forwards.
mapped = {target.lower() for target in artifact.mapped_targets}
if gfx not in mapped and gfx != (artifact.gfx_target or "").lower():
continue
asset_url = release.assets.get(artifact.asset_name)
if not asset_url:
continue
return AssetChoice(
repo = release.repo,
tag = release.release_tag,
name = artifact.asset_name,
url = asset_url,
source_label = "published",
install_kind = install_kind,
gfx_target = artifact.gfx_target,
mapped_targets = list(artifact.mapped_targets),
selection_log = list(release.selection_log)
+ [
f"rocm_selection: gpu={host.rocm_gfx_target} "
f"selected published {artifact.asset_name}"
],
)
return None
def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice:
upstream_assets = github_release_assets(UPSTREAM_REPO, llama_tag)
if host.is_linux and host.is_x86_64:
if host.has_rocm and not host.has_usable_nvidia:
# Pick the newest ROCm tarball not newer than the host runtime, else the newest overall.
_rocm_pattern = re.compile(
rf"llama-{re.escape(llama_tag)}-bin-ubuntu-rocm-([0-9]+(?:\.[0-9]+)*)-x64\.tar\.gz"
)
rocm_candidates: list[tuple[tuple[int, ...], str]] = []
for _name in upstream_assets:
_m = _rocm_pattern.match(_name)
if _m is None:
continue
_parts = tuple(int(p) for p in _m.group(1).split("."))
rocm_candidates.append((_parts, _name))
rocm_candidates.sort(reverse = True)
_host_rocm_version = _detect_host_rocm_version()
_compatible: list[tuple[tuple[int, ...], str]] = rocm_candidates
if _host_rocm_version is not None:
_compatible = [
item for item in rocm_candidates if item[0][:2] <= _host_rocm_version
]
if rocm_candidates and not _compatible:
_compatible = rocm_candidates[:1]
if _compatible:
rocm_name = _compatible[0][1]
if _host_rocm_version is not None:
log(
f"AMD ROCm {_host_rocm_version[0]}.{_host_rocm_version[1]} "
f"detected -- trying upstream prebuilt {rocm_name}"
)
else:
log(f"AMD ROCm detected -- trying upstream prebuilt {rocm_name}")
log(
"Note: if your ROCm runtime version differs significantly, "
"this may fail preflight and fall back to a source build (safe)"
)
return AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = rocm_name,
url = upstream_assets[rocm_name],
source_label = "upstream",
install_kind = "linux-rocm",
)
raise PrebuiltFallback(
"AMD ROCm detected but no upstream ROCm prebuilt found; "
"falling back to source build with HIP support"
)
# No PHYSICAL NVIDIA, since Vulkan ignores CUDA_VISIBLE_DEVICES.
if (
(host.has_intel_gpu or host.has_amd_gpu_without_rocm)
and not host.has_physical_nvidia
and not host.has_rocm
):
vendor = "Intel GPU" if host.has_intel_gpu else "AMD GPU without usable ROCm"
vulkan_name = f"llama-{llama_tag}-bin-ubuntu-vulkan-x64.tar.gz"
if vulkan_name in upstream_assets:
log(f"{vendor} detected -- using upstream Vulkan prebuilt {vulkan_name}")
return AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = vulkan_name,
url = upstream_assets[vulkan_name],
source_label = "upstream",
install_kind = "linux-vulkan",
)
log(f"{vendor} detected but no Vulkan prebuilt found -- falling back to CPU")
upstream_name = f"llama-{llama_tag}-bin-ubuntu-x64.tar.gz"
if upstream_name not in upstream_assets:
raise PrebuiltFallback("upstream Linux CPU asset was not found")
return AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = upstream_name,
url = upstream_assets[upstream_name],
source_label = "upstream",
install_kind = "linux-cpu",
)
if host.is_windows and host.is_x86_64:
if host.has_usable_nvidia:
attempts = _drop_blackwell_incapable_windows_cuda(
host, resolve_windows_cuda_choices(host, llama_tag, upstream_assets)
)
if attempts:
return attempts[0]
# A Blackwell host with only an sm_120-incapable cuda-12.4 build falls through to CPU.
if not _host_is_blackwell(host):
raise PrebuiltFallback("no compatible Windows CUDA asset was found")
# AMD ROCm on Windows: try upstream HIP prebuilt, then Vulkan, then CPU
if host.has_rocm:
for hip_name in _upstream_windows_hip_asset_names(llama_tag, upstream_assets):
log(f"AMD ROCm detected on Windows -- trying upstream HIP prebuilt {hip_name}")
return AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = hip_name,
url = upstream_assets[hip_name],
source_label = "upstream",
install_kind = "windows-hip",
)
log("AMD ROCm detected on Windows but no HIP prebuilt found -- trying Vulkan, then CPU")
# Mirrors the Linux branch: without the AMD half a Strix Halo box with no ROCm
# tooling took windows-cpu, where the same silicon takes Vulkan on Linux. ROCm
# hosts reach here only without a HIP bundle.
if (
_windows_rocm_vulkan_fallback_allowed(host)
if host.has_rocm
else host.has_intel_gpu or host.has_amd_gpu_without_rocm
) and not host.has_physical_nvidia:
vendor = (
"AMD ROCm"
if host.has_rocm
else "Intel GPU"
if host.has_intel_gpu
else "AMD GPU without usable ROCm"
)
vulkan_name = f"llama-{llama_tag}-bin-win-vulkan-x64.zip"
if vulkan_name in upstream_assets:
log(f"{vendor} detected on Windows -- using upstream Vulkan prebuilt {vulkan_name}")
return AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = vulkan_name,
url = upstream_assets[vulkan_name],
source_label = "upstream",
install_kind = "windows-vulkan",
)
log(f"{vendor} detected on Windows but no Vulkan prebuilt found -- falling back to CPU")
upstream_name = f"llama-{llama_tag}-bin-win-cpu-x64.zip"
if upstream_name not in upstream_assets:
raise PrebuiltFallback("upstream Windows CPU asset was not found")
return AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = upstream_name,
url = upstream_assets[upstream_name],
source_label = "upstream",
install_kind = "windows-cpu",
)
if host.is_windows and host.is_arm64:
if host.has_usable_nvidia and _upstream_arm64_cuda_allowed():
attempts = _drop_blackwell_incapable_windows_cuda(
host,
resolve_windows_cuda_choices(host, llama_tag, upstream_assets, arch = "arm64"),
)
if attempts:
return attempts[0]
log("no compatible Windows ARM64 CUDA asset was found -- falling back to CPU")
if host.vulkan_routed:
vulkan_name = f"llama-{llama_tag}-bin-win-vulkan-arm64.zip"
if vulkan_name in upstream_assets:
log(
f"Vulkan requested on Windows ARM64 -- using upstream Vulkan prebuilt {vulkan_name}"
)
return AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = vulkan_name,
url = upstream_assets[vulkan_name],
source_label = "upstream",
install_kind = "windows-arm64-vulkan",
)
log(
"Vulkan requested on Windows ARM64 but no Vulkan prebuilt found -- falling back to CPU"
)
upstream_name = f"llama-{llama_tag}-bin-win-cpu-arm64.zip"
if upstream_name not in upstream_assets:
raise PrebuiltFallback("upstream Windows ARM64 CPU asset was not found")
return AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = upstream_name,
url = upstream_assets[upstream_name],
source_label = "upstream",
install_kind = "windows-arm64",
)
if host.is_macos and host.is_arm64:
upstream_name = f"llama-{llama_tag}-bin-macos-arm64.tar.gz"
if upstream_name not in upstream_assets:
raise PrebuiltFallback("upstream macOS arm64 asset was not found")
return AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = upstream_name,
url = upstream_assets[upstream_name],
source_label = "upstream",
install_kind = "macos-arm64",
)
if host.is_macos and host.is_x86_64:
upstream_name = f"llama-{llama_tag}-bin-macos-x64.tar.gz"
if upstream_name not in upstream_assets:
raise PrebuiltFallback("upstream macOS x64 asset was not found")
return AssetChoice(
repo = UPSTREAM_REPO,
tag = llama_tag,
name = upstream_name,
url = upstream_assets[upstream_name],
source_label = "upstream",
install_kind = "macos-x64",
)
raise PrebuiltFallback(f"no prebuilt policy exists for {host.system} {host.machine}")
def resolve_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice:
if host.is_linux and host.is_x86_64 and host.has_usable_nvidia:
raise PrebuiltFallback(
"Linux CUDA installs require a compatible published bundle; upstream fallback is not available"
)
return resolve_upstream_asset_choice(host, llama_tag)
def resolve_release_asset_choice(
host: HostInfo,
llama_tag: str,
release: PublishedReleaseBundle,
checksums: ApprovedReleaseChecksums,
) -> list[AssetChoice]:
# A masked NVIDIA host with no CUDA match source-builds with CUDA, never CPU.
masked_host = (
_masked_nvidia_selection_host(host) if host.is_windows and host.is_x86_64 else None
)
if host.is_windows and host.is_x86_64 and (host.has_usable_nvidia or masked_host is not None):
selection_host = masked_host or host
if driver_below_cuda_prebuilt_floor(selection_host):
# #12842, #13662: the source fallback needs a toolkit this driver runs, which winget
# rarely offers. Vulkan still runs on the NVIDIA GPU; CPU after it. Not under a CUDA
# mask: Vulkan ignores it, so it could take a card the caller hid.
vulkan_choice = (
published_asset_choice_for_kind(release, "windows-vulkan", host = host)
if masked_host is None
else None
)
choices = [
choice
for choice in (
vulkan_choice,
published_asset_choice_for_kind(release, "windows-cpu"),
)
if choice is not None
]
if choices:
bundle = "Vulkan" if vulkan_choice is not None else "CPU"
log(
f"{cuda_driver_floor_message(selection_host)} Installing the {bundle} "
"bundle for now."
)
return apply_approved_hashes(choices, checksums)
torch_preference = detect_torch_cuda_runtime_preference(
selection_host, gpu_hidden_by_mask = masked_host is not None
)
published_attempts = published_windows_cuda_attempts(
selection_host,
release,
torch_preference.runtime_line,
torch_preference.selection_log,
)
if published_attempts:
pin_attempts = _drop_blackwell_incapable_windows_cuda(
selection_host, published_attempts
)
try:
return apply_approved_hashes(pin_attempts, checksums)
except PrebuiltFallback as exc:
log(
"published Windows CUDA assets ignored for install planning: "
f"{release.repo}@{release.release_tag} ({exc})"
)
if release.repo == DEFAULT_PUBLISHED_REPO:
# Prebuilts come only from the fork's own release, never upstream ggml-org's.
raise PrebuiltFallback(
f"no published Windows CUDA bundle in {release.repo}@{release.release_tag} "
"covers this host"
)
upstream_assets = github_release_assets(UPSTREAM_REPO, llama_tag)
upstream_attempts = _drop_blackwell_incapable_windows_cuda(
selection_host,
resolve_windows_cuda_choices(selection_host, llama_tag, upstream_assets),
)
return apply_approved_hashes(upstream_attempts, checksums)
published_choice: AssetChoice | None = None
if host.is_windows and host.is_x86_64:
if (
(host.has_intel_gpu or host.has_amd_gpu_without_rocm)
and not host.has_physical_nvidia
and not host.has_rocm
):
choices = [
choice
for choice in (
published_asset_choice_for_kind(release, "windows-vulkan", host = host),
published_asset_choice_for_kind(release, "windows-cpu"),
)
if choice is not None
]
if choices:
return apply_approved_hashes(choices, checksums)
# Do not hard-pin windows-cpu here so a CPU bundle never shadows the ROCm path.
if host.has_rocm:
published_choice = published_rocm_choice_for_host(release, host, "windows-rocm")
else:
published_choice = published_asset_choice_for_kind(release, "windows-cpu")
elif host.is_windows and host.is_arm64:
if host.vulkan_routed:
vulkan_choice = published_asset_choice_for_kind(
release, "windows-arm64-vulkan", host = host
)
if vulkan_choice is None:
log(
f"no published Windows ARM64 Vulkan bundle in {release.repo}@"
f"{release.release_tag}; falling through to the ARM64 CPU bundle."
)
choices = [
choice
for choice in (
vulkan_choice,
published_asset_choice_for_kind(release, "windows-arm64"),
)
if choice is not None
]
if choices:
return apply_approved_hashes(choices, checksums)
# A host the fork's windows-arm64-cuda bundle does not cover takes its ARM64 CPU bundle.
# The opt-out gates the WHOLE branch.
if host.has_usable_nvidia and _upstream_arm64_cuda_allowed():
torch_preference = detect_torch_cuda_runtime_preference(host)
published_arm64_cuda = _drop_blackwell_incapable_windows_cuda(
host,
published_windows_cuda_attempts(
host,
release,
torch_preference.runtime_line,
torch_preference.selection_log,
arch = "arm64",
),
)
if published_arm64_cuda:
try:
return apply_approved_hashes(published_arm64_cuda, checksums)
except PrebuiltFallback as exc:
log(
"published Windows ARM64 CUDA assets ignored for install planning: "
f"{release.repo}@{release.release_tag} ({exc})"
)
log(
f"no published Windows ARM64 CUDA bundle in {release.repo}@{release.release_tag} "
"covers this GPU; falling through to the ARM64 CPU bundle."
)
published_choice = published_asset_choice_for_kind(release, "windows-arm64")
elif host.is_macos and host.is_arm64:
published_choice = published_asset_choice_for_kind(release, "macos-arm64")
elif host.is_macos and host.is_x86_64:
published_choice = published_asset_choice_for_kind(release, "macos-x64")
if published_choice is not None:
try:
return apply_approved_hashes([published_choice], checksums)
except PrebuiltFallback as exc:
log(
"published platform asset ignored for install planning: "
f"{release.repo}@{release.release_tag} {published_choice.name} ({exc})"
)
if release.repo == DEFAULT_PUBLISHED_REPO:
# The upstream filename resolver below lists ggml-org's release; the fork never uses it.
raise PrebuiltFallback(
f"no compatible published prebuilt in {release.repo}@{release.release_tag} "
f"for {host.system} {host.machine}"
)
return apply_approved_hashes(
[resolve_asset_choice(host, llama_tag)],
checksums,
)
extract_archive = _core.extract_archive
def copy_globs(
source_dir: Path,
destination: Path,
patterns: list[str],
*,
required: bool = True,
) -> None:
destination.mkdir(parents = True, exist_ok = True)
matched_sources: dict[str, Path] = {}
for path in sorted(
(candidate for candidate in source_dir.rglob("*") if candidate.is_file()),
key = lambda candidate: (
len(candidate.relative_to(source_dir).parts),
str(candidate),
),
):
for pattern in patterns:
if fnmatch.fnmatch(path.name, pattern):
previous = matched_sources.get(path.name)
if previous is not None and previous != path:
raise PrebuiltFallback(
f"ambiguous archive layout for {path.name}: "
f"{previous.relative_to(source_dir)} and {path.relative_to(source_dir)}"
)
matched_sources[path.name] = path
break
if required and not matched_sources:
raise PrebuiltFallback(f"required files missing from {source_dir}: {patterns}")
for name, path in matched_sources.items():
shutil.copy2(path, destination / name)
def ensure_converter_scripts(
install_dir: Path,
llama_tag: str,
*,
repo: str | None = None,
release_tag: str | None = None,
) -> None:
canonical = install_dir / "convert_hf_to_gguf.py"
if not canonical.exists():
# Hydration normally placed it; else the fork at the installed mix tag (bare bNNNN tags are not pushed).
source_repo = repo or DEFAULT_PUBLISHED_REPO
ref = release_tag or llama_tag
raw_base = (
f"https://raw.githubusercontent.com/{source_repo}/{urllib.parse.quote(ref, safe = '')}"
)
source_url = f"{raw_base}/convert_hf_to_gguf.py"
data = download_bytes(
source_url,
progress_label = f"Downloading {download_label_from_url(source_url)}",
)
if not data:
raise RuntimeError(f"downloaded empty converter script from {source_url}")
if b"import " not in data and b"def " not in data and b"#!/" not in data:
raise RuntimeError(
f"downloaded converter script did not look like Python source: {source_url}"
)
atomic_write_bytes(canonical, data)
legacy = install_dir / "convert-hf-to-gguf.py"
if legacy.exists() or legacy.is_symlink():
legacy.unlink()
try:
legacy.symlink_to("convert_hf_to_gguf.py")
except OSError:
shutil.copy2(canonical, legacy)
def ensure_diffusion_visual_server(
install_dir: Path,
host: HostInfo,
release_tag: str | None,
approved_checksums: ApprovedReleaseChecksums,
) -> None:
"""Best-effort placement of the DiffusionGemma visual-server binary next to
llama-server in the install tree, so Unsloth can serve DiffusionGemma GGUFs
without any DG_* env. This is an Unsloth artifact (not a ggml-org one), so it
is optional: if it is already present we just make it executable, otherwise we
try the published release and quietly skip on absence. A source build
(setup.sh / setup.ps1) copies it from build/bin directly. Users can always
build it from llama.cpp PR #24423 and point DG_VISUAL_BIN at it.
"""
name = "llama-diffusion-gemma-visual-server" + (".exe" if host.is_windows else "")
bin_dir = install_dir / "build" / ("bin/Release" if host.is_windows else "bin")
target = bin_dir / name
if target.exists():
if not host.is_windows:
try:
target.chmod(0o755)
except OSError:
pass
return
if not release_tag:
log(
"diffusion visual server not bundled (no release tag); build it from llama.cpp "
"PR #24423 and set DG_VISUAL_BIN if you want native DiffusionGemma serving"
)
return
try:
match = None
for asset_name, approved in approved_checksums.artifacts.items():
low = asset_name.lower()
if "llama-diffusion-gemma-visual-server" not in low:
continue
if host.is_windows and not low.endswith(".exe"):
continue
if (not host.is_windows) and low.endswith(".exe"):
continue
if approved.repo and approved.repo != DEFAULT_PUBLISHED_REPO:
continue
url = release_asset_download_url(DEFAULT_PUBLISHED_REPO, release_tag, asset_name)
if not url:
continue
match = (asset_name, url, approved.sha256)
break
if match is None:
log(
"diffusion visual server not in the approved checksum manifest for this "
"release; native DiffusionGemma serving needs DG_VISUAL_BIN or a source build"
)
return
bin_dir.mkdir(parents = True, exist_ok = True)
download_file_verified(
match[1],
target,
expected_sha256 = match[2],
label = f"diffusion visual server {match[0]}",
)
if not host.is_windows:
target.chmod(0o755)
log(f"installed verified diffusion visual server: {match[0]}")
except Exception as exc:
log(
"diffusion visual server fetch skipped "
f"({textwrap.shorten(str(exc), width = 160, placeholder = '...')}); "
"set DG_VISUAL_BIN or build from llama.cpp PR #24423 for native serving"
)
def extracted_archive_root(extract_dir: Path) -> Path:
children = [path for path in extract_dir.iterdir()]
if len(children) == 1 and children[0].is_dir():
return children[0]
return extract_dir
def copy_directory_contents(source_dir: Path, destination: Path) -> None:
destination.mkdir(parents = True, exist_ok = True)
for item in source_dir.iterdir():
target = destination / item.name
if item.is_dir():
shutil.copytree(item, target, dirs_exist_ok = True)
else:
shutil.copy2(item, target)
def _is_link_or_junction(path: Path) -> bool:
"""Return whether ``path`` redirects to another filesystem location."""
if os.name == "nt":
try:
attributes = getattr(path.lstat(), "st_file_attributes", 0)
except OSError:
return True
return bool(attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT)
try:
return path.is_symlink()
except OSError:
return True
def remove_agent_instruction_files(root: Path) -> int:
"""Best-effort removal inside a managed tree without following links."""
if _is_link_or_junction(root) or not root.is_dir():
return 0
removed = 0
for current_dir, dirnames, filenames in os.walk(root, topdown = True, followlinks = False):
current_path = Path(current_dir)
# followlinks=False still follows Windows junctions.
if current_path != root and _is_link_or_junction(current_path):
dirnames.clear()
continue
dirnames[:] = [
dirname for dirname in dirnames if not _is_link_or_junction(current_path / dirname)
]
for filename in sorted({"AGENTS.md", "CLAUDE.md"}.intersection(filenames)):
candidate = current_path / filename
try:
candidate.unlink()
except FileNotFoundError:
continue
except OSError as exc:
log(f"could not remove contributor-only instruction {candidate}: {exc}")
else:
removed += 1
return removed
def hydrate_source_tree(
source_ref: str,
install_dir: Path,
work_dir: Path,
*,
source_repo: str = UPSTREAM_REPO,
expected_sha256: str | None,
source_label: str | None = None,
exact_source: bool = False,
asset_url: str | None = None,
) -> None:
archive_path = work_dir / f"llama.cpp-source-{source_ref}.tar.gz"
repo_urls = (
commit_source_archive_urls(source_repo, source_ref)
if exact_source
else upstream_source_archive_urls(source_ref)
)
# The release asset is the only copy of a mix build's merged tree.
source_urls = ([asset_url] if asset_url else []) + repo_urls
label = source_label or f"llama.cpp source tree for {source_ref}"
extract_dir = Path(tempfile.mkdtemp(prefix = "source-extract-", dir = work_dir))
try:
log(f"downloading {label}")
last_exc: Exception | None = None
downloaded = False
for index, source_url in enumerate(source_urls):
try:
if index > 0:
log(f"retrying source tree download from fallback URL: {source_url}")
download_file_verified(
source_url,
archive_path,
expected_sha256 = expected_sha256,
label = label,
)
downloaded = True
break
except Exception as exc:
last_exc = exc
# A full disk fails every mirror; stop so a later 404 cannot mask it.
if _environment_fatal_reason(exc) or index == len(source_urls) - 1:
raise
log(f"source tree download failed from {source_url}: {exc}")
if not downloaded:
assert last_exc is not None
raise last_exc
extract_archive(archive_path, extract_dir)
source_root = extracted_archive_root(extract_dir)
required_paths = [
source_root / "CMakeLists.txt",
source_root / "convert_hf_to_gguf.py",
source_root / "gguf-py",
]
missing = [
str(path.relative_to(source_root)) for path in required_paths if not path.exists()
]
if missing:
raise PrebuiltFallback(
"upstream source archive was missing required repo files: " + ", ".join(missing)
)
copy_directory_contents(source_root, install_dir)
removed = remove_agent_instruction_files(install_dir)
if removed:
log(f"removed {removed} contributor-only agent instruction file(s) from staged source")
except PrebuiltFallback:
raise
except Exception as exc:
raise PrebuiltFallback(f"failed to hydrate {label}: {exc}") from exc
finally:
remove_tree(extract_dir)
def normalize_install_layout(install_dir: Path, host: HostInfo) -> tuple[Path, Path]:
build_bin = install_dir / "build" / "bin"
if host.is_windows:
exec_dir = build_bin / "Release"
exec_dir.mkdir(parents = True, exist_ok = True)
return exec_dir / "llama-server.exe", exec_dir / "llama-quantize.exe"
install_dir.mkdir(parents = True, exist_ok = True)
build_bin.mkdir(parents = True, exist_ok = True)
return install_dir / "llama-server", install_dir / "llama-quantize"
def discover_installed_executable(install_dir: Path, executable_name: str) -> Path:
direct = install_dir / executable_name
if direct.exists() and direct.is_file():
return direct
candidate = next((path for path in install_dir.rglob(executable_name) if path.is_file()), None)
if candidate is None:
raise PrebuiltFallback(f"{executable_name} was not installed")
return candidate
def write_exec_wrapper(entrypoint: Path, target: Path) -> None:
relative_target = os.path.relpath(target, entrypoint.parent)
script = "\n".join(
[
"#!/bin/sh",
f'exec "$(dirname "$0")/{relative_target}" "$@"',
"",
]
)
atomic_write_bytes(entrypoint, script.encode("utf-8"))
os.chmod(entrypoint, 0o755)
def create_exec_entrypoint(entrypoint: Path, target: Path) -> None:
if entrypoint == target:
return
if entrypoint.exists() or entrypoint.is_symlink():
entrypoint.unlink()
try:
entrypoint.symlink_to(os.path.relpath(target, entrypoint.parent))
except Exception:
write_exec_wrapper(entrypoint, target)
def overlay_directory_for_choice(install_dir: Path, choice: AssetChoice, host: HostInfo) -> Path:
if host.is_windows or choice.install_kind.startswith("windows"):
path = install_dir / "build" / "bin" / "Release"
else:
path = install_dir / "build" / "bin"
path.mkdir(parents = True, exist_ok = True)
return path
def paired_runtime_dll_patterns(choice: AssetChoice) -> list[str]:
"""Filename patterns the paired runtime archive is allowed to drop
into the install. Used for the second copy_globs pass in
install_from_archives, narrower than runtime_patterns_for_choice so
the runtime archive cannot overwrite main-archive payload like
llama-server.exe. Only Windows CUDA has paired runtimes today (x64 and arm64
alike: upstream ships a cudart-...-arm64.zip beside the arm64 binary bundle,
carrying the same DLL names built for ARM64)."""
if choice.install_kind in _WINDOWS_CUDA_INSTALL_KINDS:
return ["cudart64_*.dll", "cublas64_*.dll", "cublasLt64_*.dll"]
return []
def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]:
return runtime_patterns_for_install_kind(choice.install_kind, choice.source_label)
def runtime_patterns_for_install_kind(
install_kind: str, source_label: "str | None" = None
) -> list[str]:
# Broad shared-library glob so upstream can repackage the SO/DLL set without re-enumerating.
if install_kind in {
"linux-cpu",
"linux-cuda",
"linux-arm64-cuda",
"linux-rocm",
"linux-arm64",
"linux-vulkan",
}:
return [
"llama-server",
"llama-quantize",
"llama-diffusion-gemma-visual-server",
"llama-bench",
"lib*.so*",
]
if install_kind in {"macos-arm64", "macos-x64"}:
return [
"llama-server",
"llama-quantize",
"llama-diffusion-gemma-visual-server",
"llama-bench",
"llama-fit-params",
"lib*.dylib",
]
if install_kind in {
"windows-cpu",
"windows-cuda",
"windows-arm64-cuda",
"windows-hip",
"windows-vulkan",
"windows-arm64-vulkan",
"windows-rocm",
"windows-arm64",
}:
return [
"llama-server.exe",
"llama-quantize.exe",
"llama-diffusion-gemma-visual-server.exe",
"llama-bench.exe",
"*.dll",
]
raise PrebuiltFallback(f"unsupported install kind for runtime overlay: {install_kind}")
def runtime_subdirs_for_choice(choice: AssetChoice) -> list[str]:
"""Archive subdirectories copied into the overlay beside the flat libraries: hipBLASLt and
rocBLAS need their Tensile kernel trees (hipblaslt/library/<gfx>, rocblas/library/<gfx>)
beside their libraries; copy_globs matches filenames and copies flat, so it cannot carry
multi-level trees."""
if choice.install_kind in {"linux-rocm", "windows-rocm", "windows-hip"}:
return ["hipblaslt", "rocblas"]
return []
def metadata_patterns_for_choice(choice: AssetChoice) -> list[str]:
patterns = ["BUILD_INFO.txt", "THIRD_PARTY_LICENSES.txt"]
if choice.install_kind.startswith("windows"):
patterns.append("LICENSE.txt")
else:
patterns.append("LICENSE")
return patterns
install_lock = _core.install_lock
install_lock_path = _core.install_lock_path
def install_staging_root(install_dir: Path) -> Path:
root = install_dir.parent / INSTALL_STAGING_ROOT_NAME
root.mkdir(parents = True, exist_ok = True)
return root
def prune_install_staging_root(install_dir: Path) -> None:
root = install_dir.parent / INSTALL_STAGING_ROOT_NAME
try:
root.rmdir()
except OSError:
pass
def create_install_staging_dir(install_dir: Path) -> Path:
staging_dir = _core.make_staging_dir(
install_staging_root(install_dir), f"{install_dir.name}.staging-"
)
log(f"created install staging dir {staging_dir}")
return staging_dir
def unique_install_side_path(install_dir: Path, label: str) -> Path:
root = install_staging_root(install_dir)
timestamp = time.strftime("%Y%m%d%H%M%S", time.gmtime())
prefix = f"{install_dir.name}.{label}-{timestamp}-{os.getpid()}"
candidate = root / prefix
counter = 0
while candidate.exists():
counter += 1
candidate = root / f"{prefix}-{counter}"
return candidate
def blocked_replace_hint(winerror: object) -> str:
if winerror == 5:
return (
"access is denied -- a scanner or running process may still hold a handle, "
"or the ACLs are broken"
)
if winerror == 145:
return "the directory is not empty yet -- an earlier copy is still being removed"
return "a scanner is likely still holding the install open"
def _tree_link_state(root: Path) -> str:
"""Return "link", "unreadable" (root or a directory not readable) or "clean"."""
try:
root.lstat()
except OSError:
return "unreadable"
if _is_link_or_junction(root):
return "link"
unreadable: list[OSError] = []
for current_dir, dirnames, filenames in os.walk(
root, followlinks = False, onerror = unreadable.append
):
current_path = type(root)(current_dir)
if any(_is_link_or_junction(current_path / name) for name in (*dirnames, *filenames)):
return "link"
return "unreadable" if unreadable else "clean"
def log_acl_repair(path: Path) -> None:
# Printed, never run. takeown /R and icacls /T follow links, so recurse only on a link-free tree.
state = _tree_link_state(path)
if state == "link":
log(f"rename still denied after retrying; {path} contains a link, check its permissions")
else:
log(
"rename still denied after retrying; if the permissions on this tree are broken, run in an elevated PowerShell:"
)
recursive = state == "clean"
log(f'takeown /F "{path}"' + (" /R /D Y" if recursive else ""))
# /L: if an unreadable root is a link after all, reset the link, not its target.
log(f'icacls "{path}" /reset' + (" /T" if recursive else " /L"))
if not recursive:
log("then run the install again")
log(
"if access stays denied, Controlled folder access or antivirus may be blocking "
"the path: allow or exclude it there"
)
def replace_with_busy_retry(
src: Path,
dst: Path,
*,
attempts: int = 8,
acl_repair: bool = True,
) -> None:
"""``os.replace``, retried against transient Windows sharing violations.
WinError 5/32/145 usually means a scanner holds a handle inside the tree,
which clears in a second or two; without a backoff that turns an update
into a failure, and on the aside-move of the *existing* install that is the
failure this installer most needs to avoid. Mirrors the Node installer's
``_replace_with_retry``. Other errors raise at once, and POSIX never
retries because EACCES/EBUSY there mean a permission or mount problem no
amount of waiting clears.
"""
if attempts < 1:
raise ValueError("replace_with_busy_retry needs at least one attempt")
delay = 0.25
for attempt in range(attempts):
try:
os.replace(src, dst)
return
except OSError as exc:
winerror = getattr(exc, "winerror", None)
transient = os.name == "nt" and winerror in (5, 32, 145)
if not transient or attempt == attempts - 1:
if acl_repair and transient and winerror == 5:
log_acl_repair(src)
raise
log(
f"rename {src.name} -> {dst.name} blocked ({winerror}), retrying in "
f"{delay:.2f}s -- {blocked_replace_hint(winerror)}"
)
time.sleep(delay)
delay = min(delay * 2, 4.0)
def remove_tree(path: Path | None) -> None:
if path and path.exists():
shutil.rmtree(path, ignore_errors = True)
def _clear_readonly_and_retry(func: Callable[[str], Any], path: str, excinfo: Any) -> None:
"""``shutil.rmtree`` handler that clears the read-only bit and retries.
Straight out of the shutil docs' rmtree example. Windows refuses to unlink
a read-only file, and a llama.cpp tree picks that attribute up from its
archive or a security product, so one read-only DLL would abort a cleanup
that leaves the install neither replaced nor restored. Two cases are
refused rather than retried, because the retry does more damage than the
original error:
- ``rmtree`` routes ``lstat``/``open``/``scandir``/``islink``/``close``
failures through this hook too, and none takes a lone path: ``func(path)``
would raise ``TypeError`` over the real error, or for ``scandir`` return
an unconsumed iterator and report success for a subtree still on disk.
- ``os.chmod`` follows symlinks and junctions, so chmod-ing a linked root
(whose refusal ``rmtree`` also reports here) would repermission a tree
this installer does not own.
"""
error = excinfo[1] if isinstance(excinfo, tuple) else excinfo
if func not in (os.unlink, os.remove, os.rmdir):
raise error
if _is_link_or_junction(Path(path)):
raise error
os.chmod(path, stat.S_IWRITE)
func(path)
def _rmtree_handler_kwargs() -> dict[str, Any]:
# Windows only: on POSIX unlink permission lives on the parent, and S_IWRITE would leave 0o200.
if os.name != "nt":
return {}
if sys.version_info >= (3, 12):
return {"onexc": _clear_readonly_and_retry}
return {"onerror": _clear_readonly_and_retry}
def remove_tree_logged(path: Path | None, label: str) -> None:
if not path:
return
if not path.exists():
log(f"{label} already absent at {path}")
return
log(f"removing {label} at {path}")
try:
shutil.rmtree(path, **_rmtree_handler_kwargs())
except Exception as exc:
log(f"failed to remove {label} at {path}: {exc}")
raise
def cleanup_install_side_paths(
install_dir: Path,
*,
staging_dir: Path | None = None,
rollback_dir: Path | None = None,
failed_dir: Path | None = None,
active_dir: Path | None = None,
) -> None:
cleanup_failures: list[str] = []
for label, path in (
("failed install path", failed_dir),
("rollback path", rollback_dir),
("active install path", active_dir),
("staging dir", staging_dir),
):
if not path:
continue
try:
remove_tree_logged(path, label)
except Exception as exc:
cleanup_failures.append(f"{label} ({path}): {exc}")
prune_install_staging_root(install_dir)
if cleanup_failures:
raise RuntimeError("cleanup failed for " + "; ".join(cleanup_failures))
_INSTALL_SIDE_PATH_TAIL = re.compile(r"\d{14}-\d+(?:-\d+)?")
def _install_side_path_candidates(install_dir: Path, label: str) -> list[Path]:
root = install_dir.parent / INSTALL_STAGING_ROOT_NAME
if not root.is_dir():
return []
# glob.escape: the name comes from UNSLOTH_LLAMA_CPP_PATH and could match another install's paths.
prefix = f"{install_dir.name}.{label}-"
pattern = f"{glob.escape(install_dir.name)}.{label}-*"
# The trailing * can run into a sibling "<name>.rollback-<sfx>"; the tail check excludes it.
# Link roots are excluded: rmtree refuses them.
return sorted(
path
for path in root.glob(pattern)
if _INSTALL_SIDE_PATH_TAIL.fullmatch(path.name[len(prefix) :])
and path.is_dir()
and not _is_link_or_junction(path)
)
def prune_stale_install_side_paths(install_dir: Path, *, keep: Iterable[Path | None] = ()) -> int:
"""Drop rollback and failed trees earlier updates parked under the staging root.
``activate_install_tree`` retains an unrestored rollback tree on purpose:
when it gives up, that tree is the only llama.cpp left. Nothing else ever
removes those paths, so without a sweep every failed update parks another
multi-gigabyte copy for good. They are only safe to drop where the caller
can show they are no longer the last copy, hence the explicit ``keep``.
"""
kept: set[Path] = set()
for path in keep:
if not path:
continue
try:
kept.add(path.resolve())
except OSError:
continue
removed = 0
for label in ("rollback", "failed"):
for candidate in _install_side_path_candidates(install_dir, label):
try:
if candidate.resolve() in kept:
continue
except OSError:
continue
try:
remove_tree_logged(candidate, f"stale {label} path from an earlier update")
removed += 1
except Exception as exc:
log(f"could not remove stale {label} path {candidate}: {exc}")
return removed
def confirm_install_tree(install_dir: Path, host: HostInfo) -> None:
if host.is_windows:
expected = [
install_dir / "build" / "bin" / "Release" / "llama-server.exe",
install_dir / "build" / "bin" / "Release" / "llama-quantize.exe",
install_dir / "convert_hf_to_gguf.py",
install_dir / "gguf-py",
]
else:
expected = [
install_dir / "llama-server",
install_dir / "llama-quantize",
install_dir / "build" / "bin" / "llama-server",
install_dir / "build" / "bin" / "llama-quantize",
install_dir / "convert_hf_to_gguf.py",
install_dir / "gguf-py",
]
expected.append(install_dir / "UNSLOTH_PREBUILT_INFO.json")
missing = [str(path) for path in expected if not path.exists()]
if missing:
raise RuntimeError("activated install was missing expected files: " + ", ".join(missing))
def _install_tree_is_usable(path: Path, host: HostInfo) -> bool:
"""Whether ``path`` holds a tree ``confirm_install_tree`` would accept.
Structural only (a handful of ``exists`` checks), so it is cheap enough to
run on the failure path where the alternative is guessing.
"""
try:
confirm_install_tree(path, host)
except Exception:
return False
return True
def newest_usable_install_side_path(
install_dir: Path,
host: HostInfo,
*,
exclude: Path | None = None,
) -> Path | None:
"""The most recent retained rollback tree that is still a usable install.
Candidates sort by the timestamp ``unique_install_side_path`` embeds, so the
last usable one is the newest. ``failed`` trees are never considered: they
are the trees that failed confirmation in the first place.
"""
skip: set[Path] = set()
if exclude:
try:
skip.add(exclude.resolve())
except OSError:
pass
for candidate in reversed(_install_side_path_candidates(install_dir, "rollback")):
try:
if candidate.resolve() in skip:
continue
except OSError:
continue
if _install_tree_is_usable(candidate, host):
return candidate
return None
def activate_staged_dir(staging_dir: Path, dst: Path) -> None:
"""Move a freshly extracted ``staging_dir`` onto ``dst``.
``os.replace`` is attempted first as the fast path. On Windows ARM64 the
antivirus scanner can transiently hold a freshly extracted DLL open at the
moment ``MoveFileEx`` runs, surfacing as ``[WinError 5] Access is denied``;
a file-by-file copy bypasses the rename entirely.
This fallback is intentionally limited to staging trees we just extracted.
It must not be used to move an existing/active install aside: there an
``os.replace`` failure means the directory is genuinely in use, and a
silent copy + ``rmtree`` could partially delete a live install.
Only busy/lock errors (``is_busy_lock_error``) and cross-device links
(``is_cross_device_error``, hit when a Docker build lands the staging tree
and the install dir on different overlay layers) trigger the copy; anything
else (disk full, missing path) re-raises so it cannot leave a partially
copied install behind. A copy is preferred over retrying the rename because
antivirus scans of large DLLs can outlast any reasonable retry window.
"""
try:
os.replace(staging_dir, dst)
except OSError as exc:
if not (is_busy_lock_error(exc) or is_cross_device_error(exc)):
raise
log(f"os.replace failed ({exc!r}); falling back to file-by-file copy of staging tree")
# symlinks=True keeps soname links; a leftover non-empty dst fails in os.symlink, not a merge.
shutil.copytree(staging_dir, dst, dirs_exist_ok = True, symlinks = True)
remove_tree(staging_dir)
def move_install_dir_aside(
src: Path,
dst: Path,
*,
busy_retry: bool = False,
) -> None:
"""Moves a tree to a non-existent sibling with os.replace (copy then remove on EXDEV); busy
failures are never copy-faked; busy_retry is opt-in for the live aside-move; dst.exists()
proves a complete tree, so the copy is staged then atomically renamed."""
try:
if busy_retry:
replace_with_busy_retry(src, dst)
else:
os.replace(src, dst)
except OSError as exc:
if not is_cross_device_error(exc):
raise
if _is_link_or_junction(src):
# copytree follows the ROOT, so a linked install would duplicate a checkout we do not own.
log("install path is a link; not copying it aside")
raise
copy_tmp = dst.with_name(dst.name + ".copying")
counter = 0
while copy_tmp.exists():
counter += 1
copy_tmp = dst.with_name(f"{dst.name}.copying-{counter}")
log(f"os.replace cross-device ({exc!r}); copy+publish {src} -> {dst}")
try:
# symlinks=True is safe here only because copy_tmp is fresh; merges into populated trees cannot use it.
shutil.copytree(src, copy_tmp, symlinks = True)
os.replace(copy_tmp, dst)
except BaseException:
remove_tree(copy_tmp)
raise
remove_tree(src)
def activate_install_tree(staging_dir: Path, install_dir: Path, host: HostInfo) -> None:
rollback_dir: Path | None = None
failed_dir: Path | None = None
had_existing = False
moved_aside = False
try:
if install_dir.exists():
had_existing = True
rollback_dir = unique_install_side_path(install_dir, "rollback")
log(f"moving existing install to rollback path {rollback_dir}")
move_install_dir_aside(install_dir, rollback_dir, busy_retry = True)
moved_aside = True
log(f"moved existing install to rollback path {rollback_dir.name}")
log(f"activating staged install {staging_dir} -> {install_dir}")
activate_staged_dir(staging_dir, install_dir)
log(f"activated staged install at {install_dir}")
log(f"confirming activated install tree at {install_dir}")
confirm_install_tree(install_dir, host)
log(f"activated install tree confirmed at {install_dir}")
except Exception as exc:
log(f"activation failed for staged install: {exc}")
if had_existing and not moved_aside:
log("existing install could not be moved aside; leaving it in place")
if is_busy_lock_error(exc):
raise BusyInstallConflict(
"staged prebuilt validation passed but the existing install could not be "
"moved aside because llama.cpp appears to still be in use; previous install "
f"left in place ({textwrap.shorten(str(exc), width = 200, placeholder = '...')})"
) from exc
raise PrebuiltFallback(
"staged prebuilt validation passed but the existing install could not be "
"moved aside; previous install left in place "
f"({textwrap.shorten(str(exc), width = 200, placeholder = '...')})"
) from exc
restore_attempted = False
restored = False
recovery_error: BaseException | None = None
try:
if install_dir.exists():
failed_dir = unique_install_side_path(install_dir, "failed")
log(f"moving failed active install to {failed_dir}")
try:
# Not os.replace: in Docker the trees can sit on different overlay layers (EXDEV).
move_install_dir_aside(install_dir, failed_dir)
except Exception as failed_move_exc:
failed_dir = None
log(f"failed active install could not be moved aside: {failed_move_exc}")
if rollback_dir is None:
raise
remove_tree_logged(install_dir, "failed active install path")
elif staging_dir.exists():
failed_dir = staging_dir
staging_dir = None
log(f"retaining failed staging tree at {failed_dir}")
if rollback_dir and rollback_dir.exists():
log(f"restoring rollback path {rollback_dir} -> {install_dir}")
restore_attempted = True
try:
replace_with_busy_retry(rollback_dir, install_dir, acl_repair = False)
except OSError as restore_exc:
# The rollback tree is the sole remaining llama.cpp, so a copy beats an empty install path.
if _is_link_or_junction(rollback_dir):
# copytree follows the root link, which would duplicate a user's --with-llama-cpp-dir checkout.
log("previous install is a link; not copying it back")
raise
log(
f"rollback rename failed ({restore_exc}); copying the previous install "
f"back to {install_dir}"
)
try:
shutil.copytree(rollback_dir, install_dir, symlinks = True)
except Exception as copy_exc:
log(f"copying the previous install back also failed: {copy_exc}")
raise restore_exc from copy_exc
log(f"copied the previous install back to {install_dir}")
try:
remove_tree_logged(rollback_dir, "copied rollback path")
except Exception as duplicate_exc:
log(
f"non-fatal: the copied rollback path could not be removed: {duplicate_exc}"
)
restored = True
log(f"restored previous install from rollback path {rollback_dir.name}")
if is_busy_lock_error(exc):
raise BusyInstallConflict(
"staged prebuilt validation passed but the existing install could not be replaced "
"because llama.cpp appears to still be in use; restored previous install "
f"({textwrap.shorten(str(exc), width = 200, placeholder = '...')})"
) from exc
raise PrebuiltFallback(
"staged prebuilt validation passed but activation failed; restored previous install "
f"({textwrap.shorten(str(exc), width = 200, placeholder = '...')})"
) from exc
except (BusyInstallConflict, PrebuiltFallback):
raise
except Exception as rollback_exc:
# `except` unbinds the name; keep it so the recovery failure stays in the causal chain.
recovery_error = rollback_exc
if restore_attempted:
log(f"rollback after failed activation also failed: {rollback_exc}")
else:
log(f"recovering from the failed activation also failed: {rollback_exc}")
# An unrestored rollback is the only copy of the previous install, so keep it.
keep_rollback = not restored and rollback_dir is not None and rollback_dir.exists()
retained_dir = rollback_dir
if keep_rollback:
# An older rollback that passes confirm_install_tree wins over a newer unconfirmed one.
if not _install_tree_is_usable(rollback_dir, host):
known_good = newest_usable_install_side_path(
install_dir, host, exclude = rollback_dir
)
if known_good is not None:
log(
f"rollback path {rollback_dir.name} is not a usable install; keeping the "
f"known-good {known_good.name} from an earlier failed update instead"
)
retained_dir = known_good
log(f"previous install kept at rollback path {retained_dir}")
superseded = prune_stale_install_side_paths(
install_dir, keep = (retained_dir, failed_dir)
)
if superseded:
log(f"removed {superseded} superseded install tree(s) from earlier failed updates")
log(
"rollback restoration failed; cleaning staging, install, and failed paths before source build fallback"
)
else:
log(
"rollback restoration failed; cleaning staging, install, and rollback paths before source build fallback"
)
cleanup_error: Exception | None = None
try:
cleanup_install_side_paths(
install_dir,
staging_dir = staging_dir,
rollback_dir = None if keep_rollback else rollback_dir,
failed_dir = failed_dir,
active_dir = install_dir,
)
except Exception as cleanup_exc:
cleanup_error = cleanup_exc
log(f"cleanup after rollback failure also failed: {cleanup_exc}")
details = textwrap.shorten(str(exc), width = 200, placeholder = "...")
kept = f"; previous install kept at {retained_dir}" if keep_rollback else ""
# A disk-full hit by recovery may be the only sign of a full disk; chain from it so the caller
# does not start a source build needing more room.
cause: BaseException = exc
if (
recovery_error is not None
and _environment_fatal_reason(exc) is None
and _environment_fatal_reason(recovery_error) is not None
):
cause = recovery_error
if cleanup_error is not None:
raise PrebuiltFallback(
"staged prebuilt validation passed but activation and rollback failed; "
f"cleanup also reported errors ({details}; cleanup={cleanup_error}){kept}"
) from cause
raise PrebuiltFallback(
"staged prebuilt validation passed but activation and rollback failed; "
f"cleaned install state for fresh source build ({details}){kept}"
) from cause
else:
if rollback_dir:
try:
remove_tree_logged(rollback_dir, "rollback path")
except Exception as cleanup_exc:
log(
f"non-fatal: rollback cleanup failed after successful activation: {cleanup_exc}"
)
# Only safe point: confirm_install_tree just passed, so nothing retained is the last copy.
stale = prune_stale_install_side_paths(install_dir, keep = (rollback_dir,))
if stale:
log(f"removed {stale} install tree(s) left behind by earlier failed updates")
finally:
remove_tree(failed_dir)
remove_tree(staging_dir)
prune_install_staging_root(install_dir)
def ensure_fit_params_executable(install_dir: Path) -> None:
"""The guarded extractor leaves the optional Metal probe 0644 (#12901); reuse paths repair old installs."""
helper = install_dir / "build" / "bin" / "llama-fit-params"
try:
if helper.is_file() and not helper.is_symlink():
if stat.S_IMODE(helper.stat().st_mode) & 0o111 != 0o111:
os.chmod(helper, 0o755)
except OSError:
pass
def install_from_archives(
choice: AssetChoice, host: HostInfo, install_dir: Path, work_dir: Path
) -> tuple[Path, Path]:
main_archive = work_dir / choice.name
log(f"downloading {choice.name} from {choice.source_label} release")
download_file_verified(
choice.url,
main_archive,
expected_sha256 = choice.expected_sha256,
label = f"prebuilt archive {choice.name}",
)
install_dir.mkdir(parents = True, exist_ok = True)
extract_dir = Path(tempfile.mkdtemp(prefix = "extract-", dir = work_dir))
runtime_extract_dir: Path | None = None
try:
extract_archive(main_archive, extract_dir)
# Separate temp dir avoids copy_globs's ambiguous-layout guard on shared names (LICENSE.txt).
if choice.runtime_url and choice.runtime_name:
runtime_archive = work_dir / choice.runtime_name
log(
f"downloading paired runtime archive {choice.runtime_name} "
f"from {choice.source_label} release"
)
download_file_verified(
choice.runtime_url,
runtime_archive,
expected_sha256 = choice.runtime_sha256,
label = f"prebuilt runtime archive {choice.runtime_name}",
)
runtime_extract_dir = Path(tempfile.mkdtemp(prefix = "extract-runtime-", dir = work_dir))
extract_archive(runtime_archive, runtime_extract_dir)
source_dir = extract_dir
overlay_dir = overlay_directory_for_choice(install_dir, choice, host)
copy_globs(source_dir, overlay_dir, runtime_patterns_for_choice(choice), required = True)
for _subdir in runtime_subdirs_for_choice(choice):
_src_subdir = source_dir / _subdir
if _src_subdir.is_dir():
shutil.copytree(_src_subdir, overlay_dir / _subdir, dirs_exist_ok = True)
if runtime_extract_dir is not None:
# Restrict to the cudart DLLs so a malformed runtime archive cannot overwrite llama-server.exe.
copy_globs(
runtime_extract_dir,
overlay_dir,
paired_runtime_dll_patterns(choice),
required = False,
)
copy_globs(
source_dir,
install_dir,
metadata_patterns_for_choice(choice),
required = False,
)
finally:
remove_tree(extract_dir)
if runtime_extract_dir is not None:
remove_tree(runtime_extract_dir)
if host.is_windows:
exec_dir = install_dir / "build" / "bin" / "Release"
server_src = next(exec_dir.glob("llama-server.exe"), None)
quantize_src = next(exec_dir.glob("llama-quantize.exe"), None)
if server_src is None or quantize_src is None:
raise PrebuiltFallback("windows executables were not installed correctly")
return server_src, quantize_src
build_bin = install_dir / "build" / "bin"
source_server = build_bin / "llama-server"
source_quantize = build_bin / "llama-quantize"
if not source_server.exists() or not source_quantize.exists():
raise PrebuiltFallback("unix executables were not installed correctly into build/bin")
os.chmod(source_server, 0o755)
os.chmod(source_quantize, 0o755)
# Optional; bundles ship it non-executable.
source_bench = build_bin / "llama-bench"
if source_bench.is_file():
os.chmod(source_bench, 0o755)
ensure_fit_params_executable(install_dir)
root_server = install_dir / "llama-server"
root_quantize = install_dir / "llama-quantize"
if source_server != root_server:
create_exec_entrypoint(root_server, source_server)
if source_quantize != root_quantize:
create_exec_entrypoint(root_quantize, source_quantize)
build_server = build_bin / "llama-server"
build_quantize = build_bin / "llama-quantize"
if source_server != build_server:
create_exec_entrypoint(build_server, source_server)
if source_quantize != build_quantize:
create_exec_entrypoint(build_quantize, source_quantize)
return source_server, source_quantize
def ensure_repo_shape(install_dir: Path) -> None:
required = [
install_dir / "CMakeLists.txt",
install_dir / "convert_hf_to_gguf.py",
install_dir / "gguf-py",
]
missing = [str(path.relative_to(install_dir)) for path in required if not path.exists()]
if missing:
raise PrebuiltFallback("hydrated llama.cpp source tree was missing: " + ", ".join(missing))
def validation_model_cache_path(install_dir: Path) -> Path:
cache_dir = install_dir.parent / VALIDATION_MODEL_CACHE_DIRNAME
cache_dir.mkdir(parents = True, exist_ok = True)
return cache_dir / VALIDATION_MODEL_CACHE_FILENAME
def validated_validation_model_bytes(data: bytes) -> bytes:
if not data:
raise RuntimeError(f"downloaded empty validation model from {TEST_MODEL_URL}")
digest = hashlib.sha256(data).hexdigest()
if digest != TEST_MODEL_SHA256:
raise RuntimeError(
f"validation model checksum mismatch: expected={TEST_MODEL_SHA256} actual={digest}"
)
return data
def _hf_resolve_url_parts(url: str) -> tuple[str, str, str] | None:
"""Parse a huggingface.co .../resolve/<rev>/<path> URL into
(repo_id, revision, filename); None if it is not such a URL."""
try:
parsed = urllib.parse.urlparse(url)
except Exception:
return None
if (parsed.netloc or "").lower() not in ("huggingface.co", "www.huggingface.co"):
return None
parts = parsed.path.strip("/").split("/")
# <owner>/<name>/resolve/<rev>/<path...>
if len(parts) >= 5 and parts[2] == "resolve":
return f"{parts[0]}/{parts[1]}", parts[3], "/".join(parts[4:])
return None
def _fetch_validation_model_bytes() -> bytes:
"""Fetch the tiny GGUF validation model. Prefer huggingface_hub (completes
TLS chains via AIA fetching that bare urllib can't on some Windows/proxy
setups); fall back to the direct URL when hf_hub is unavailable or fails."""
parts = _hf_resolve_url_parts(TEST_MODEL_URL)
if parts is not None:
repo_id, revision, filename = parts
try:
from huggingface_hub import hf_hub_download
local = hf_hub_download(repo_id = repo_id, filename = filename, revision = revision)
return validated_validation_model_bytes(Path(local).read_bytes())
except Exception as exc:
log(
f"huggingface_hub fetch of validation model failed ({exc}); "
"falling back to direct URL"
)
return validated_validation_model_bytes(
download_bytes(
TEST_MODEL_URL,
progress_label = f"Downloading {download_label_from_url(TEST_MODEL_URL)}",
)
)
def download_validation_model(path: Path, cache_path: Path | None = None) -> None:
try:
data: bytes | None = None
if cache_path and cache_path.exists():
try:
data = validated_validation_model_bytes(cache_path.read_bytes())
log(f"using cached tiny GGUF validation model from {cache_path}")
except Exception as exc:
log(f"cached tiny GGUF validation model was invalid; refreshing cache ({exc})")
data = None
if data is None:
log("downloading tiny GGUF validation model")
data = _fetch_validation_model_bytes()
if cache_path is not None:
atomic_write_bytes(cache_path, data)
atomic_write_bytes(path, data)
except Exception as exc:
raise PrebuiltFallback(f"validation model unavailable: {exc}") from exc
ValidationModelProvider = Union[Path, Callable[[], Path]]
def lazy_validation_model(path: Path, cache_path: Path | None = None) -> Callable[[], Path]:
"""Fetch the tiny GGUF probe on first use, at most once.
An approved bundle proves its integrity by sha256 and so skips the staged smoke
test, leaving the probe unread on the default install path. Fetching it up front
still made huggingface.co a hard gate: a 429 there raised PrebuiltFallback and
forced a multi-minute source build over a file nothing went on to open.
"""
fetched = False
def ensure() -> Path:
nonlocal fetched
if not fetched:
download_validation_model(path, cache_path)
fetched = True
return path
return ensure
def resolve_validation_model(probe: ValidationModelProvider) -> Path:
"""Materialize a probe model that may have been supplied lazily."""
return probe() if callable(probe) else probe
def free_local_port() -> int:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("127.0.0.1", 0))
_, port = sock.getsockname()
sock.close()
return int(port)
def read_log_excerpt(log_path: Path, *, max_lines: int = 60) -> str:
try:
content = log_path.read_text(encoding = "utf-8", errors = "replace")
except (FileNotFoundError, UnicodeDecodeError):
return ""
return "\n".join(content.splitlines()[-max_lines:])
def is_retryable_server_bind_error(
exc: Exception | None,
output: str = "",
*,
exited_quickly: bool = False,
) -> bool:
haystack = output.lower()
bind_markers = (
"address already in use",
"only one usage of each socket address",
"failed to bind",
"bind failed",
"failed to listen",
"errno 98",
"errno 10048",
)
if any(marker in haystack for marker in bind_markers):
return True
if isinstance(exc, urllib.error.URLError):
reason = exc.reason
if exited_quickly and isinstance(reason, ConnectionRefusedError):
return True
if isinstance(reason, OSError) and reason.errno in {
98,
99,
111,
10048,
10049,
10061,
}:
return exited_quickly
if exited_quickly and isinstance(exc, ConnectionRefusedError):
return True
if isinstance(exc, OSError) and exc.errno in {98, 99, 111, 10048, 10049, 10061}:
return exited_quickly
return False
def dedupe_existing_dirs(paths: Iterable[str | Path], *, skip_unusable: bool = False) -> list[str]:
unique: list[str] = []
seen: set[str] = set()
for raw in paths:
if not raw:
continue
try:
path = Path(raw).expanduser()
if not path.is_dir():
continue
resolved = str(path.resolve())
except (OSError, ValueError):
if not skip_unusable:
raise
continue
if resolved in seen:
continue
seen.add(resolved)
unique.append(resolved)
return unique
def linux_missing_libraries(binary_path: Path, *, env: dict[str, str] | None = None) -> list[str]:
try:
result = run_capture(["ldd", str(binary_path)], timeout = 20, env = env)
except Exception:
return []
missing: list[str] = []
for line in (result.stdout + result.stderr).splitlines():
line = line.strip()
if "=> not found" not in line:
continue
library = line.split("=>", 1)[0].strip()
if library and library not in missing:
missing.append(library)
return missing
def python_runtime_dirs() -> list[str]:
candidates: list[Path] = []
search_roots = [Path(entry) for entry in sys.path if entry]
try:
search_roots.extend(Path(path) for path in site.getsitepackages())
except Exception:
pass
try:
user_site = site.getusersitepackages()
if user_site:
search_roots.append(Path(user_site))
except Exception:
pass
for root in search_roots:
# is_dir() re-raises PermissionError, which would defeat skip_unusable discovery.
try:
if not root.is_dir():
continue
except (OSError, ValueError):
continue
candidates.extend(root.glob("nvidia/*/lib"))
candidates.extend(root.glob("nvidia/*/bin"))
# CUDA 13 Windows wheels use nvidia/cu13/bin/x86_64. Keep in sync with
# llama_cpp.LlamaCppBackend._windows_pip_nvidia_dll_dirs.
candidates.extend(root.glob("nvidia/*/bin/x86_64"))
candidates.extend(root.glob("nvidia/*/bin/x64"))
candidates.extend(root.glob("nvidia/*/Library/bin"))
candidates.extend(root.glob("nvidia/*/Library/bin/x86_64"))
candidates.extend(root.glob("nvidia/*/Library/bin/x64"))
candidates.extend(root.glob("torch/lib"))
# Optional dirs: a denied child must not abort discovery.
return dedupe_existing_dirs(candidates, skip_unusable = True)
def ldconfig_runtime_dirs(required_libraries: Iterable[str]) -> list[str]:
try:
result = run_capture(["ldconfig", "-p"], timeout = 20)
except Exception:
return []
required = set(required_libraries)
candidates: list[str] = []
for line in result.stdout.splitlines():
if "=>" not in line:
continue
library, _, location = line.partition("=>")
library = library.strip().split()[0]
if required and library not in required:
continue
path = Path(location.strip()).parent
candidates.append(str(path))
return dedupe_existing_dirs(candidates)
def linux_runtime_dirs(binary_path: Path) -> list[str]:
# ldd may execute the binary, so probe it with a secret-free env.
missing = linux_missing_libraries(binary_path, env = scrubbed_environ())
if not missing:
return []
return linux_runtime_dirs_for_required_libraries(missing)
# Upstream macOS prebuilts built on a newer macOS fail dyld load on older hosts; read minos to skip.
_MACHO_FAT_MAGICS = {0xCAFEBABE, 0xCAFEBABF} # universal binary (32/64-bit fat)
_LC_VERSION_MIN_MACOSX = 0x24 # legacy min-macOS load command
_LC_BUILD_VERSION = 0x32 # modern platform+minos+sdk load command
_MACHO_PLATFORM_MACOS = 1 # LC_BUILD_VERSION platform id for macOS (iOS=2, ...)
_CPU_TYPE_X86_64 = 0x01000007
_CPU_TYPE_ARM64 = 0x0100000C
parse_macos_version = _core.parse_macos_version
def host_supports_macos_minos(host: HostInfo, minos: tuple[int, int] | None) -> bool:
"""True if a prebuilt requiring minos can load here; an unknown host version or minos yields
True so runtime validation decides instead."""
if minos is None or host.macos_version is None:
return True
return host.macos_version >= minos
def _macho_slice_minos(data: bytes, offset: int) -> tuple[int, int] | None:
"""Minimum macOS for a single thin Mach-O at `offset`, via LC_BUILD_VERSION
(platform macOS) or the legacy LC_VERSION_MIN_MACOSX. None if absent."""
if offset + 4 > len(data):
return None
magic = struct.unpack_from(">I", data, offset)[0]
if magic in (0xFEEDFACE, 0xFEEDFACF):
endian, is64 = ">", magic == 0xFEEDFACF
elif magic in (0xCEFAEDFE, 0xCFFAEDFE):
endian, is64 = "<", magic == 0xCFFAEDFE
else:
return None
header_size = 32 if is64 else 28
if offset + header_size > len(data):
return None
ncmds = struct.unpack_from(endian + "I", data, offset + 16)[0]
cursor = offset + header_size
for _ in range(ncmds):
if cursor + 8 > len(data):
break
cmd, cmdsize = struct.unpack_from(endian + "II", data, cursor)
if cmdsize < 8:
break
if cmd == _LC_BUILD_VERSION and cursor + 16 <= len(data):
platform_id, minos = struct.unpack_from(endian + "II", data, cursor + 8)
if platform_id == _MACHO_PLATFORM_MACOS:
return (minos >> 16) & 0xFFFF, (minos >> 8) & 0xFF
elif cmd == _LC_VERSION_MIN_MACOSX and cursor + 12 <= len(data):
version = struct.unpack_from(endian + "I", data, cursor + 8)[0]
return (version >> 16) & 0xFFFF, (version >> 8) & 0xFF
cursor += cmdsize
return None
def macho_minimum_macos(path: Path, host: HostInfo | None = None) -> tuple[int, int] | None:
"""Minimum macOS (major, minor) a Mach-O binary requires, parsed in pure Python so it works
without Xcode command line tools (otool/vtool); universal files use the host-arch slice, else
the highest minos; None if absent."""
try:
data = path.read_bytes()
except Exception:
return None
if len(data) < 8:
return None
magic = struct.unpack_from(">I", data, 0)[0]
if magic in _MACHO_FAT_MAGICS:
is64 = magic == 0xCAFEBABF
nfat = struct.unpack_from(">I", data, 4)[0]
entry = 8
slices: list[tuple[int, tuple[int, int]]] = []
for _ in range(nfat):
if is64:
if entry + 32 > len(data):
break
cputype = struct.unpack_from(">I", data, entry)[0]
slice_offset = struct.unpack_from(">Q", data, entry + 8)[0]
entry += 32
else:
if entry + 20 > len(data):
break
cputype = struct.unpack_from(">I", data, entry)[0]
slice_offset = struct.unpack_from(">I", data, entry + 8)[0]
entry += 20
minos = _macho_slice_minos(data, slice_offset)
if minos is not None:
slices.append((cputype, minos))
if not slices:
return None
if host is not None:
want = (
_CPU_TYPE_ARM64 if host.is_arm64 else (_CPU_TYPE_X86_64 if host.is_x86_64 else None)
)
for cputype, minos in slices:
if cputype == want:
return minos
return max(minos for _cputype, minos in slices)
return _macho_slice_minos(data, 0)
def looks_like_macos_incompatibility(text: str) -> bool:
"""True when dyld output means a prebuilt needs a newer macOS than the host
(the runtime backstop for cases the static minos scan cannot read)."""
if not text:
return False
if "built for macOS" in text and "newer than running OS" in text:
return True
return "Symbol not found" in text and "MTLResidency" in text
_MACOS_LOADER_FAILURE_MARKERS = (
"library not loaded",
"symbol not found",
"image not found",
"no suitable image found",
"incompatible library version",
"code signature",
)
def looks_like_macos_loader_failure(text: str) -> bool:
"""True when output is dyld refusing to load the image, rather than a program
that started and exited non-zero.
The distinction is the whole point: llama-quantize answers --version by
printing its quantization table and exiting non-zero, so an exit code cannot
separate "never reached main" from "ran and disagreed". dyld failures are
recognisable by what they say, and they say it before any program output.
Deliberately narrow -- a false positive here rejects a working prebuilt and
spends a source build, so anything unrecognised is treated as healthy and
left to the runtime validation that follows.
"""
if not text:
return False
lowered = text.lower()
if any(marker in lowered for marker in _MACOS_LOADER_FAILURE_MARKERS):
return True
# No bare "dyld[pid]:" rule: DYLD_PRINT_LIBRARIES narrates a healthy load with that prefix.
return looks_like_macos_incompatibility(text)
def macos_binary_minos_issues(
binaries: Iterable[Path], install_dir: Path, host: HostInfo
) -> list[str]:
"""Issue strings for every installed Mach-O whose minimum macOS exceeds the
host. Scans the given executables plus every bundled .dylib next to them --
the dyld failure originates in libggml-metal.dylib, not the executable."""
candidates: list[Path] = list(binaries)
bin_dir = install_dir / "build" / "bin"
if bin_dir.is_dir():
candidates.extend(sorted(bin_dir.rglob("*.dylib")))
issues: list[str] = []
seen: set[Path] = set()
for path in candidates:
try:
resolved = path.resolve()
except Exception:
resolved = path
if resolved in seen or not path.exists():
continue
seen.add(resolved)
minos = macho_minimum_macos(path, host)
if minos is not None and not host_supports_macos_minos(host, minos):
issues.append(
f"{path.name}: built for macOS {minos[0]}.{minos[1]} > "
f"host macOS {host.macos_version[0]}.{host.macos_version[1]}"
)
return issues
def macos_dyld_load_issues(
binaries: Iterable[Path],
install_dir: Path,
host: HostInfo,
*,
loaded: "set[str] | None" = None,
) -> list[str]:
"""Issue strings for every installed executable dyld refuses to load.
*loaded*, when given, collects the binaries dyld actually resolved: this probe
fails OPEN, so "no issues" alone cannot tell a clean load from one that never
ran. Only a name in here is evidence of a load.
`--version` costs a process spawn and still makes dyld resolve the whole link
graph, so it catches a missing dylib, a missing symbol or a too-new slice --
unlike validate_server, which loads a model and is off for checksummed bundles
(#5854). Executables only: a broken dylib surfaces through whatever links it."""
issues: list[str] = []
seen: set[Path] = set()
for binary_path in binaries:
try:
resolved = binary_path.resolve()
except Exception:
resolved = binary_path
if resolved in seen or not binary_path.is_file():
continue
seen.add(resolved)
env = binary_env(binary_path, install_dir, host)
for noisy in [k for k in env if k.startswith("DYLD_PRINT_")]:
env.pop(noisy, None)
env.pop("DYLD_INSERT_LIBRARIES", None)
try:
result = run_capture([str(binary_path), "--version"], timeout = 60, env = env)
except Exception as exc:
# A timeout or spawn refusal is not evidence of a bad link.
log(f"macos load probe could not run {binary_path.name}: {exc}")
continue
if result.returncode == 0:
if loaded is not None:
loaded.add(binary_path.name)
continue
output = (result.stdout + result.stderr).strip()
# A non-zero exit is not evidence: llama-quantize --version exits 1. Require dyld's signature.
if not looks_like_macos_loader_failure(output):
if loaded is not None:
loaded.add(binary_path.name)
continue
detail = " | ".join(output.splitlines()[-5:]) or f"exit {result.returncode}"
issues.append(f"{binary_path.name}: {detail}")
return issues
def preflight_macos_installed_binaries(
binaries: Iterable[Path],
install_dir: Path,
host: HostInfo,
*,
load_probe: bool = True,
) -> bool:
"""Reject a macos prebuilt whose minimum-OS is newer than the host, or (with
*load_probe*) that dyld will not load at all. The upstream selector pins a loadable release up
front, so here this is the post-download backstop; the published/fork path
also uses it to advance the walk-back.
Returns True only when the load probe actually ran and dyld resolved every
binary. False means "no evidence", not "bad": the probe is skipped, or it
timed out or could not spawn. Callers record the skip only on True, so an
inconclusive probe costs another probe next time instead of being cached as
a pass.
The load probe is the macOS counterpart of preflight_linux_installed_binaries'
ldd sweep, which this side went without: a bundle whose libggml-rpc.0.dylib
links a /usr/lib/librdma.dylib that exists on the builder and nowhere else
passed the minos scan, was logged "prebuilt installed and validated", and
then died on first launch. Raising PrebuiltFallback here spends a source
build instead of installing something that cannot start.
Only the static comparison needs the host version; dyld does not. Skipping
both on an unparseable ``platform.mac_ver()`` left that host with no check at
all, since the runtime validation it deferred to is off by default (#5854)."""
if not host.is_macos:
return False
binaries = list(binaries)
if host.macos_version is not None:
issues = macos_binary_minos_issues(binaries, install_dir, host)
if issues:
raise PrebuiltFallback(
"macos prebuilt requires a newer macOS than this host:\n" + "\n".join(issues)
)
if not load_probe:
log("installed binaries match their recorded digests; skipping the dyld load probe")
return False
loaded: set[str] = set()
load_issues = macos_dyld_load_issues(binaries, install_dir, host, loaded = loaded)
if load_issues:
raise PrebuiltFallback(
"macos prebuilt does not load on this host:\n" + "\n".join(load_issues)
)
expected = {path.name for path in binaries if path.is_file()}
if not expected or not expected.issubset(loaded):
# The probe fails open, which is right for installing but wrong to record as a load.
log("macos load probe did not run for every binary; not recording it as a pass")
return False
return True
def preflight_linux_installed_binaries(
binaries: Iterable[Path], install_dir: Path, host: HostInfo
) -> None:
if not host.is_linux:
return
issues: list[str] = []
for binary_path in binaries:
env = binary_env(binary_path, install_dir, host)
missing = linux_missing_libraries(binary_path, env = env)
if not missing:
continue
runtime_dirs = [part for part in env.get("LD_LIBRARY_PATH", "").split(os.pathsep) if part]
issues.append(
f"{binary_path.name}: missing={','.join(missing)} "
f"ld_library_path={','.join(runtime_dirs) if runtime_dirs else 'none'}"
)
if issues:
raise PrebuiltFallback("linux extracted binary preflight failed:\n" + "\n".join(issues))
def glob_paths(*patterns: str) -> list[str]:
matches: list[str] = []
for pattern in patterns:
if any(char in pattern for char in "*?[]"):
matches.extend(str(path) for path in Path("/").glob(pattern.lstrip("/")))
else:
matches.append(pattern)
return matches
def windows_runtime_dirs() -> list[str]:
candidates: list[str | Path] = []
env_dirs = os.environ.get("CUDA_RUNTIME_DLL_DIR", "")
if env_dirs:
candidates.extend(part for part in env_dirs.split(os.pathsep) if part)
path_dirs = os.environ.get("PATH", "")
if path_dirs:
candidates.extend(part for part in path_dirs.split(os.pathsep) if part)
cuda_roots: list[Path] = []
for name in ("CUDA_PATH", "CUDA_HOME", "CUDA_ROOT"):
value = os.environ.get(name)
if value:
cuda_roots.append(Path(value))
for root in cuda_roots:
candidates.extend([root / "bin", root / "lib" / "x64"])
program_files = os.environ.get("ProgramFiles", r"C:\Program Files")
toolkit_base = Path(program_files) / "NVIDIA GPU Computing Toolkit" / "CUDA"
# %ProgramFiles% is user-controllable, so this stat can be denied.
try:
toolkit_is_dir = toolkit_base.is_dir()
except (OSError, ValueError):
toolkit_is_dir = False
if toolkit_is_dir:
candidates.extend(toolkit_base.glob("v*/bin"))
candidates.extend(toolkit_base.glob("v*/lib/x64"))
candidates.extend(Path(path) for path in python_runtime_dirs())
return dedupe_existing_dirs(candidates, skip_unusable = True)
def windows_runtime_dirs_for_patterns(
required_patterns: Iterable[str], candidate_dirs: Iterable[str] | None = None
) -> list[str]:
directories = list(candidate_dirs) if candidate_dirs is not None else windows_runtime_dirs()
matching_dirs: list[str] = []
for pattern in required_patterns:
matched_dirs = [
directory for directory in directories if any(Path(directory).glob(pattern))
]
if not matched_dirs:
return []
for directory in matched_dirs:
if directory not in matching_dirs:
matching_dirs.append(directory)
return matching_dirs
def windows_runtime_dirs_for_runtime_line(runtime_line: str | None) -> list[str]:
if not runtime_line:
return []
patterns = windows_runtime_line_info().get(runtime_line)
if not patterns:
return []
return windows_runtime_dirs_for_patterns(patterns)
def _wsl_system_rocm_lib_dirs() -> list[str]:
"""System ROCm lib dirs go first so binary_env loads the WSL-capable HIP (libamdhip64,
librocdxg), since a bundled bare-metal HIP cannot drive WSL and segfaults; the bundle
supplies libggml-hip and librocblas. No-op off WSL, which needs /dev/dxg, microsoft in
/proc/version and librocdxg."""
try:
if not os.path.exists("/dev/dxg"):
return []
with open("/proc/version", encoding = "utf-8", errors = "replace") as fh:
if "microsoft" not in fh.read().lower():
return []
except (OSError, UnicodeDecodeError):
return []
out: list[str] = []
for d in ("/opt/rocm/lib", "/opt/rocm/lib64"):
if os.path.exists(os.path.join(d, "librocdxg.so")) or os.path.exists(
os.path.join(d, "librocdxg.so.1")
):
out.append(d)
return out
def _bundled_hip_present(binary_dir: str) -> bool:
if not binary_dir:
return False
try:
return any(Path(str(binary_dir)).glob("libggml-hip.so*"))
except OSError:
return False
def _native_linux_system_rocm_lib_dirs(binary_dir: str = "") -> list[str]:
if os.environ.get("UNSLOTH_LLAMA_NO_SYSTEM_ROCM") == "1":
return []
if sys.platform != "linux" or os.path.exists("/dev/dxg"):
return []
if not os.path.exists("/dev/kfd"):
return []
if not _bundled_hip_present(binary_dir):
return []
# Env-configured ROCm root first so a stale /opt/rocm does not shadow it.
candidates = []
for var in ("HIP_PATH", "HIP_PATH_57", "ROCM_PATH"):
val = os.environ.get(var)
if val:
candidates.append(val)
candidates.append("/opt/rocm")
out: list[str] = []
seen: set[str] = set()
for base in candidates:
for lib_sub in ("lib", "lib64"):
d = os.path.join(base, lib_sub)
if d in seen:
continue
seen.add(d)
if os.path.exists(os.path.join(d, "libhsa-runtime64.so")) or os.path.exists(
os.path.join(d, "libhsa-runtime64.so.1")
):
out.append(d)
# Keep LLVM dirs ahead of the bundle, else system libamd_comgr binds the bundle's libLLVM.
for _sub in (lib_sub, "lib"):
llvm_lib = os.path.join(base, _sub, "llvm", "lib")
if llvm_lib not in seen and os.path.isdir(llvm_lib):
seen.add(llvm_lib)
out.append(llvm_lib)
return out
# Secrets a downloaded binary never needs; the installer's own API calls read os.environ.
_SECRET_ENV_EXACT_NAMES = frozenset(
{
"HF_TOKEN",
"HUGGING_FACE_HUB_TOKEN",
"GH_TOKEN",
"GITHUB_TOKEN",
"WANDB_API_KEY",
"OPENAI_API_KEY",
"ANTHROPIC_API_KEY",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
"GOOGLE_APPLICATION_CREDENTIALS",
"AZURE_CLIENT_SECRET",
"ACTIONS_ID_TOKEN_REQUEST_TOKEN",
"ACTIONS_ID_TOKEN_REQUEST_URL",
"ACTIONS_RUNTIME_TOKEN",
"KUBECONFIG",
"SSH_AUTH_SOCK",
}
)
# No bare "KEY": it would hit benign runtime vars.
_SECRET_ENV_MARKERS = (
"TOKEN",
"SECRET",
"PASSWORD",
"PASSWD",
"PASSPHRASE",
"CREDENTIAL",
"PRIVATE_KEY",
"API_KEY",
)
_SECRET_ENV_URL_NAMES = frozenset(
{
"HTTP_PROXY",
"HTTPS_PROXY",
"ALL_PROXY",
"FTP_PROXY",
"RSYNC_PROXY",
"PIP_INDEX_URL",
"PIP_EXTRA_INDEX_URL",
"UV_INDEX_URL",
"UV_DEFAULT_INDEX",
"UV_EXTRA_INDEX_URL",
}
)
_URL_USERINFO_CREDENTIAL_RE = re.compile(r"://[^/@\s]+@")
def is_secret_env_name(name: str) -> bool:
upper = name.upper()
return (
upper in _SECRET_ENV_EXACT_NAMES
or upper in _SECRET_ENV_URL_NAMES
or any(marker in upper for marker in _SECRET_ENV_MARKERS)
)
def scrub_env(env: dict[str, str]) -> dict[str, str]:
"""Drop secret-bearing variables before handing an env to a downloaded binary."""
return {
key: value
for key, value in env.items()
if not is_secret_env_name(key) and not _URL_USERINFO_CREDENTIAL_RE.search(value or "")
}
# Point home/cache vars at an empty home so the binary cannot read on-disk token stores.
_RUNTIME_HOME_POINTER_VARS = (
"HOME",
"USERPROFILE",
"APPDATA",
"LOCALAPPDATA",
"XDG_CACHE_HOME",
"XDG_CONFIG_HOME",
"XDG_DATA_HOME",
"HF_HOME",
"HUGGINGFACE_HUB_CACHE",
"HF_HUB_CACHE",
)
_CREDENTIAL_FILE_POINTER_VARS = (
"NETRC",
"PIP_CONFIG_FILE",
"DOCKER_CONFIG",
"GIT_CONFIG_GLOBAL",
)
# GitHub Actions command files: appending to these injects PATH/env into later steps.
_CI_COMMAND_FILE_VARS = (
"GITHUB_ENV",
"GITHUB_PATH",
"GITHUB_OUTPUT",
"GITHUB_STEP_SUMMARY",
"BASH_ENV",
)
_isolated_runtime_home_dir: str | None = None
def isolated_runtime_home() -> str:
# A binary resolving the real home via getpwuid is out of scope (needs OS sandboxing).
global _isolated_runtime_home_dir
if _isolated_runtime_home_dir is None:
path = tempfile.mkdtemp(prefix = "unsloth-prebuilt-home-")
atexit.register(shutil.rmtree, path, ignore_errors = True)
_isolated_runtime_home_dir = path
return _isolated_runtime_home_dir
def scrubbed_environ() -> dict[str, str]:
env = scrub_env(os.environ.copy())
runtime_home = isolated_runtime_home()
for pointer in _RUNTIME_HOME_POINTER_VARS:
env[pointer] = runtime_home
drive, tail = os.path.splitdrive(runtime_home)
env["HOMEDRIVE"], env["HOMEPATH"] = drive, tail or runtime_home
for pointer in (*_CREDENTIAL_FILE_POINTER_VARS, *_CI_COMMAND_FILE_VARS):
env.pop(pointer, None)
return env
def binary_env(
binary_path: Path,
install_dir: Path,
host: HostInfo,
*,
runtime_line: str | None = None,
) -> dict[str, str]:
env = scrubbed_environ()
if host.is_windows:
path_dirs = [
str(binary_path.parent),
*windows_runtime_dirs_for_runtime_line(runtime_line),
]
existing = [part for part in env.get("PATH", "").split(os.pathsep) if part]
required = dedupe_existing_dirs(path_dirs)
inherited = dedupe_existing_dirs(existing, skip_unusable = True)
env["PATH"] = os.pathsep.join(dict.fromkeys([*required, *inherited]))
elif host.is_linux:
ld_dirs = [
str(binary_path.parent),
str(install_dir),
*linux_runtime_dirs(binary_path),
]
# WSL: system HIP before the bundle's (which segfaults on /dev/dxg).
_wsl_rocm = _wsl_system_rocm_lib_dirs()
if _wsl_rocm:
ld_dirs = [*_wsl_rocm, *ld_dirs]
env.setdefault("HSA_ENABLE_DXG_DETECTION", "1")
# The bundled HIP runtime can be incompatible with the host amdkfd driver.
_native_rocm = _native_linux_system_rocm_lib_dirs(str(binary_path.parent))
if _native_rocm:
ld_dirs = [*_native_rocm, *ld_dirs]
existing = [part for part in env.get("LD_LIBRARY_PATH", "").split(os.pathsep) if part]
required = dedupe_existing_dirs(ld_dirs)
inherited = dedupe_existing_dirs(existing, skip_unusable = True)
env["LD_LIBRARY_PATH"] = os.pathsep.join(dict.fromkeys([*required, *inherited]))
elif host.is_macos:
dyld_dirs = [str(binary_path.parent), str(install_dir)]
existing = [part for part in env.get("DYLD_LIBRARY_PATH", "").split(os.pathsep) if part]
required = dedupe_existing_dirs(dyld_dirs)
inherited = dedupe_existing_dirs(existing, skip_unusable = True)
env["DYLD_LIBRARY_PATH"] = os.pathsep.join(dict.fromkeys([*required, *inherited]))
return env
def validate_quantize(
quantize_path: Path,
probe_path: Path,
quantized_path: Path,
install_dir: Path,
host: HostInfo,
*,
runtime_line: str | None = None,
) -> None:
command = [str(quantize_path), str(probe_path), str(quantized_path), "Q6_K", "2"]
result = subprocess.run(
command,
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 120,
env = binary_env(quantize_path, install_dir, host, runtime_line = runtime_line),
**windows_hidden_subprocess_kwargs(),
)
if result.returncode != 0 or not quantized_path.exists() or quantized_path.stat().st_size == 0:
combined = result.stdout + ("\n" + result.stderr if result.stderr else "")
prefix = (
"macos prebuilt requires a newer macOS than this host: "
if looks_like_macos_incompatibility(combined)
else ""
)
raise PrebuiltFallback(prefix + "llama-quantize validation failed:\n" + combined)
def validate_server(
server_path: Path,
probe_path: Path,
host: HostInfo,
install_dir: Path,
*,
runtime_line: str | None = None,
install_kind: str | None = None,
) -> None:
last_failure: PrebuiltFallback | None = None
for port_attempt in range(1, SERVER_PORT_BIND_ATTEMPTS + 1):
port = free_local_port()
command = [
str(server_path),
"-m",
str(probe_path),
"--host",
"127.0.0.1",
"--port",
str(port),
"-c",
"32",
"--parallel",
"1",
"--threads",
"1",
"--ubatch-size",
"32",
"--batch-size",
"32",
]
# Gate GPU offload on install_kind, not host.has_rocm: a CPU fallback bundle on AMD Windows
# would fail validation with --n-gpu-layers 1.
_gpu_kinds = {
"linux-cuda",
"linux-arm64-cuda",
"linux-rocm",
"linux-vulkan",
"windows-cuda",
"windows-arm64-cuda",
"windows-hip",
"windows-vulkan",
"windows-arm64-vulkan",
"windows-rocm",
"macos-arm64",
}
if install_kind is not None:
_enable_gpu_layers = install_kind in _gpu_kinds
else:
_enable_gpu_layers = (
host.has_usable_nvidia or host.has_rocm or (host.is_macos and host.is_arm64)
)
if _enable_gpu_layers:
command.extend(["--n-gpu-layers", "1"])
log_fd, log_name = tempfile.mkstemp(prefix = "llama-server-", suffix = ".log")
os.close(log_fd)
log_path = Path(log_name)
process: subprocess.Popen[str] | None = None
try:
with log_path.open("w", encoding = "utf-8", errors = "replace") as log_handle:
process = subprocess.Popen(
command,
stdout = log_handle,
stderr = subprocess.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
env = binary_env(server_path, install_dir, host, runtime_line = runtime_line),
**_validation_server_kwargs(),
**windows_hidden_subprocess_kwargs(),
)
# The validation server is the caller's grandchild; announce it so a crash cannot orphan it.
_announce_child("started", process.pid)
deadline = time.time() + 60
startup_started = time.time()
response_body = ""
last_error: Exception | None = None
while time.time() < deadline:
if process.poll() is not None:
process.wait(timeout = 5)
log_handle.flush()
output = read_log_excerpt(log_path)
exited_quickly = (
time.time() - startup_started
) <= SERVER_BIND_RETRY_WINDOW_SECONDS
failure = PrebuiltFallback("llama-server exited during startup:\n" + output)
if (
port_attempt < SERVER_PORT_BIND_ATTEMPTS
and is_retryable_server_bind_error(
last_error,
output,
exited_quickly = exited_quickly,
)
):
log(
f"llama-server startup hit a port race on {port}; retrying with a fresh port "
f"({port_attempt}/{SERVER_PORT_BIND_ATTEMPTS})"
)
last_failure = failure
break
raise failure
payload = json.dumps({"prompt": "a", "n_predict": 1}).encode("utf-8")
request = urllib.request.Request(
f"http://127.0.0.1:{port}/completion",
data = payload,
headers = {"Content-Type": "application/json"},
)
try:
with urllib.request.urlopen(request, timeout = 5) as response:
status_code = response.status
response_body = response.read().decode("utf-8", "replace")
if status_code == 200:
return
last_error = RuntimeError(f"unexpected HTTP status {status_code}")
except urllib.error.HTTPError as exc:
response_body = exc.read().decode("utf-8", "replace")
last_error = exc
except Exception as exc:
last_error = exc
time.sleep(0.5)
else:
log_handle.flush()
output = read_log_excerpt(log_path)
raise PrebuiltFallback(
"llama-server completion validation timed out"
+ (f" ({last_error})" if last_error else "")
+ ":\n"
+ output
+ ("\n" + response_body if response_body else "")
)
finally:
if process is not None:
# Announcing the stop drops the record, the only handle on a survivor.
if _terminate_validation_server(process):
_announce_child("stopped", process.pid)
try:
log_path.unlink(missing_ok = True)
except Exception:
pass
if last_failure is not None:
raise last_failure
raise PrebuiltFallback("llama-server validation failed unexpectedly")
_PR_SET_PDEATHSIG = 1
def _validation_server_leads_group() -> bool:
"""Whether the validation server starts as its own group leader; true only on Linux, where the
parent-death signal can be armed alongside, so the two always agree and killpg only hits a
led group."""
return os.name == "posix" and sys.platform.startswith("linux")
def _validation_server_kwargs() -> "dict[str, Any]":
"""Popen kwargs for the validation server: on Linux it leads its own group with a parent-death
signal, so a killed installer cannot leave it holding the GPU; macOS has no such signal and
keeps the inherited group, since a session there would leave an unreachable window."""
if not _validation_server_leads_group():
return {}
kwargs: "dict[str, Any]" = {"start_new_session": True}
owner_pid = os.getpid() # read pre-fork, so the child can tell reparenting apart
def _arm_parent_death() -> None:
# fork clears the setting and setsid does not undo it; getppid closes the reparent race.
try:
import ctypes
ctypes.CDLL("libc.so.6", use_errno = True).prctl(_PR_SET_PDEATHSIG, signal.SIGTERM)
if os.getppid() != owner_pid:
os._exit(1)
except Exception:
pass
kwargs["preexec_fn"] = _arm_parent_death
return kwargs
def _announce_child(state: str, pid: int) -> None:
"""Prints the UNSLOTH_INSTALLER_CHILD line so Unsloth can adopt the pid for its sweep; run by
hand the line is just noise on stdout."""
print(f"UNSLOTH_INSTALLER_CHILD {state} {pid}", flush = True)
def _group_is_running(pgid: "int | None") -> bool:
"""Whether anything is left in that process group."""
if pgid is None or os.name != "posix" or not hasattr(os, "killpg"):
return False
try:
os.killpg(pgid, 0)
except OSError:
return False
return True
def _wait_for_group(pgid: "int | None", grace: float) -> None:
"""Give a signalled group its moment to go."""
deadline = time.time() + grace
while _group_is_running(pgid) and time.time() < deadline:
time.sleep(0.05)
def _terminate_validation_server(process: "subprocess.Popen", grace: float = 5.0) -> bool:
"""Stops the validation server and any children; True when gone. A group leader's whole group is
signalled, as a SIGTERM-ignoring child would outlive it; False keeps the pid announced for a
later sweep. Otherwise only the server is signalled, since killpg would hit the installer."""
pgid = None
if _validation_server_leads_group() and hasattr(os, "getpgid"):
try:
pgid = os.getpgid(process.pid)
except OSError:
# Already reaped, but the kernel holds its pid as pgid while any group member remains.
pgid = process.pid
if pgid != process.pid:
pgid = None # not the leader after all; never signal a shared group
if pgid is not None:
try:
os.killpg(pgid, signal.SIGTERM)
except OSError:
process.terminate()
else:
process.terminate()
try:
process.wait(timeout = grace)
except subprocess.TimeoutExpired:
pass
# A member that ignored SIGTERM can still hold the GPU and staged files.
_wait_for_group(pgid, grace)
if pgid is not None and _group_is_running(pgid):
try:
os.killpg(pgid, signal.SIGKILL)
except OSError:
pass
_wait_for_group(pgid, grace)
elif pgid is None and process.poll() is None:
process.kill()
try:
process.wait(timeout = grace)
except subprocess.TimeoutExpired:
pass
return not _group_is_running(pgid) and process.poll() is not None
def collect_system_report(host: HostInfo, choice: AssetChoice | None, install_dir: Path) -> str:
lines = [
f"platform={host.system} machine={host.machine}",
f"driver_cuda_version={host.driver_cuda_version}",
f"compute_caps={','.join(host.compute_caps) if host.compute_caps else 'unknown'}",
f"cuda_visible_devices={host.visible_cuda_devices if host.visible_cuda_devices is not None else 'unset'}",
f"has_physical_nvidia={host.has_physical_nvidia}",
f"has_usable_nvidia={host.has_usable_nvidia}",
f"chosen_asset={(choice.name if choice else 'none')}",
f"asset_source={(choice.source_label if choice else 'none')}",
]
if host.is_linux and host.has_physical_nvidia:
runtime_lines, runtime_dirs = detected_linux_runtime_lines()
lines.append(
"linux_runtime_lines=" + (",".join(runtime_lines) if runtime_lines else "none")
)
for runtime_line in ("cuda13", "cuda12"):
lines.append(
f"linux_runtime_dirs_{runtime_line}="
+ (
",".join(runtime_dirs.get(runtime_line, []))
if runtime_dirs.get(runtime_line)
else "none"
)
)
if choice and choice.selection_log:
lines.append("selection_log:")
lines.extend(choice.selection_log)
if host.nvidia_smi:
try:
smi = run_capture([host.nvidia_smi], timeout = 20)
excerpt = "\n".join((smi.stdout + smi.stderr).splitlines()[:20])
lines.append("nvidia-smi:")
lines.append(excerpt)
except Exception as exc:
lines.append(f"nvidia-smi error: {exc}")
if host.is_linux:
server_binary = install_dir / "llama-server"
if server_binary.exists():
server_env = binary_env(server_binary, install_dir, host)
lines.append(
"linux_missing_libs="
+ (",".join(linux_missing_libraries(server_binary, env = server_env)) or "none")
)
lines.append(
"linux_runtime_dirs="
+ (
",".join(
[
part
for part in server_env.get("LD_LIBRARY_PATH", "").split(os.pathsep)
if part
]
)
or "none"
)
)
try:
ldd = run_capture(["ldd", str(server_binary)], timeout = 20, env = server_env)
lines.append("ldd llama-server:")
lines.append((ldd.stdout + ldd.stderr).strip())
except Exception as exc:
lines.append(f"ldd error: {exc}")
elif host.is_windows:
lines.append("windows_runtime_dirs=" + (",".join(windows_runtime_dirs()) or "none"))
runtime_lines, runtime_dirs = detected_windows_runtime_lines()
lines.append(
"windows_runtime_lines=" + (",".join(runtime_lines) if runtime_lines else "none")
)
for runtime_line in ("cuda13", "cuda12"):
lines.append(
f"windows_runtime_dirs_{runtime_line}="
+ (
",".join(runtime_dirs.get(runtime_line, []))
if runtime_dirs.get(runtime_line)
else "none"
)
)
elif host.is_macos:
server_binary = install_dir / "llama-server"
if server_binary.exists():
try:
otool = run_capture(["otool", "-L", str(server_binary)], timeout = 20)
lines.append("otool -L llama-server:")
lines.append((otool.stdout + otool.stderr).strip())
except Exception as exc:
lines.append(f"otool error: {exc}")
return "\n".join(lines)
def _apply_release_digests(
attempts: Iterable[AssetChoice], digests: dict[str, str]
) -> list[AssetChoice]:
"""apply_approved_hashes against GitHub's per-asset digests instead of our manifest.
Same two rules, because they are the ones that matter: an attempt with no digest is
dropped rather than installed unverified, and a paired runtime archive with no digest
unpairs rather than riding along unchecked. Returns [] when nothing survives, which the
one caller reads as "refuse and fall through", not as "install anyway".
"""
kept: list[AssetChoice] = []
for attempt in attempts:
digest = digests.get(attempt.name)
if digest is None:
continue
attempt.expected_sha256 = digest
if attempt.runtime_name and attempt.runtime_url:
runtime_digest = digests.get(attempt.runtime_name)
if runtime_digest is None:
attempt.runtime_name = None
attempt.runtime_url = None
attempt.runtime_sha256 = None
else:
attempt.runtime_sha256 = runtime_digest
kept.append(attempt)
return kept
def apply_approved_hashes(
attempts: Iterable[AssetChoice], checksums: ApprovedReleaseChecksums
) -> list[AssetChoice]:
def approved_hash_for_attempt(attempt: AssetChoice) -> ApprovedArtifactHash | None:
candidate_names = [attempt.name]
if (
isinstance(attempt.tag, str)
and attempt.tag
and attempt.tag != checksums.upstream_tag
and attempt.name.startswith("llama-")
):
legacy_prefix = f"llama-{attempt.tag}-"
compatibility_prefix = f"llama-{checksums.upstream_tag}-"
compatibility_name = (
attempt.name.replace(legacy_prefix, compatibility_prefix, 1)
if attempt.name.startswith(legacy_prefix)
else attempt.name
)
candidate_names.append(compatibility_name)
candidate_names.extend(
windows_cuda_asset_aliases(
attempt.name,
compatibility_tag = checksums.upstream_tag,
)
)
seen_names: set[str] = set()
for candidate_name in candidate_names:
if candidate_name in seen_names:
continue
seen_names.add(candidate_name)
approved = checksums.artifacts.get(candidate_name)
if approved is not None:
return approved
return None
approved_attempts: list[AssetChoice] = []
missing_assets: list[str] = []
for attempt in attempts:
approved = approved_hash_for_attempt(attempt)
if approved is None:
missing_assets.append(attempt.name)
continue
attempt.expected_sha256 = approved.sha256
# Drop the pair if the manifest does not list it: never install an unverified archive.
if attempt.runtime_name and attempt.runtime_url:
runtime_approved = checksums.artifacts.get(attempt.runtime_name)
if runtime_approved is None:
attempt.runtime_name = None
attempt.runtime_url = None
attempt.runtime_sha256 = None
else:
attempt.runtime_sha256 = runtime_approved.sha256
approved_attempts.append(attempt)
if not approved_attempts:
missing_text = ", ".join(missing_assets) if missing_assets else "none"
raise PrebuiltFallback(
"approved checksum asset did not contain the selected prebuilt archive(s): "
f"{missing_text}"
)
return approved_attempts
def require_approved_source_hash(
checksums: ApprovedReleaseChecksums, llama_tag: str
) -> ApprovedArtifactHash:
source_asset_name = source_archive_logical_name(llama_tag)
approved_source = checksums.artifacts.get(source_asset_name)
if approved_source is None:
raise PrebuiltFallback(
f"approved checksum asset did not contain source archive {source_asset_name}"
)
return approved_source
def preferred_source_archive(
checksums: ApprovedReleaseChecksums, llama_tag: str
) -> tuple[str, str, ApprovedArtifactHash | None, bool]:
exact_source = exact_source_archive_hash(checksums)
exact_repo = repo_slug_from_source(checksums.source_repo) or repo_slug_from_source(
checksums.source_repo_url
)
if exact_source is not None and exact_repo and checksums.source_commit:
return (
exact_repo,
checksums.source_commit,
exact_source,
True,
)
legacy = checksums.artifacts.get(source_archive_logical_name(llama_tag))
return (
UPSTREAM_REPO,
llama_tag,
legacy,
False,
)
def exact_source_asset_url(
approved_checksums: ApprovedReleaseChecksums,
source_repo: str,
source_archive: ApprovedArtifactHash | None,
exact_source: bool,
release_tag: str,
) -> str | None:
"""Release-asset URL for a mix build's merged source tree, or None.
A mix build's merge commit is never pushed, so its codeload/archive URLs
404; the only durable copy of the merged tree is the
``llama.cpp-source-commit-<sha>.tar.gz`` asset published alongside the
prebuilt. Resolve its host and tag defensively so a manifest that omits the
top-level ``repo``/``release_tag`` still reaches the asset: prefer the
artifact's own repo, then the manifest repo, then the source repo; and prefer
the manifest release tag, then the tag we actually installed the prebuilt
from (its sibling on the same release). Without this the empty fields drop the
only working URL and hydration falls through to the 404-ing commit archive.
"""
if not exact_source or source_archive is None:
return None
return release_asset_download_url(
source_archive.repo or approved_checksums.repo or source_repo,
approved_checksums.release_tag or release_tag,
source_archive.asset_name,
)
def selected_source_archive_metadata(
checksums: ApprovedReleaseChecksums, llama_tag: str
) -> tuple[str, str | None]:
_source_repo, _source_ref, source_archive, _exact_source = preferred_source_archive(
checksums, llama_tag
)
if source_archive is None:
return source_archive_logical_name(llama_tag), None
return source_archive.asset_name, source_archive.sha256
def resolve_install_attempts(
llama_tag: str, host: HostInfo, published_repo: str, published_release_tag: str
) -> tuple[str, str, list[AssetChoice], ApprovedReleaseChecksums]:
requested_tag, plans = _fork_manifest_release_plans(
llama_tag,
host,
published_repo,
published_release_tag,
)
if not plans:
raise PrebuiltFallback("no prebuilt release plans were available")
plan = plans[0]
return requested_tag, plan.llama_tag, plan.attempts, plan.approved_checksums
def _hipcc_available() -> bool:
"""hipcc where setup.sh's HIP source build looks: PATH, /opt/rocm/bin, /opt/rocm-*/bin."""
return bool(
shutil.which("hipcc")
or os.access("/opt/rocm/bin/hipcc", os.X_OK)
or glob.glob("/opt/rocm-*/bin/hipcc")
)
def _uncovered_rocm_gpu_takes_vulkan(host: HostInfo) -> bool:
"""No hipcc means a CPU-only source build; Vulkan ignores HIP masks and would reach an NVIDIA card."""
return (
host.is_linux
and host.is_x86_64
and not host.has_physical_nvidia
# Following ROCm torch clears has_physical_nvidia but keeps the card's evidence.
and not (host.nvidia_smi or host.physical_compute_caps or host.driver_cuda_version)
and not _hip_visible_device_mask_set()
and not _vulkan_visible_device_mask_set()
and not _hipcc_available()
)
def _linux_published_attempts(host: HostInfo, bundle: PublishedReleaseBundle) -> list[AssetChoice]:
"""Builds install attempts for a fork Linux host from a manifest-described bundle (CUDA, per-gfx
ROCm, Vulkan or CPU), selecting as the upstream filename path does but from the manifest."""
attempts: list[AssetChoice] = []
if host.has_usable_nvidia:
# Prefer torch's cudart major, else a stray cuda13 runtime outranks the cuda12 torch links.
torch_preference = detect_torch_cuda_runtime_preference(host)
selection = linux_cuda_choice_from_release(
host,
bundle,
preferred_runtime_line = torch_preference.runtime_line,
selection_preamble = torch_preference.selection_log,
)
if selection is not None:
attempts.extend(selection.attempts)
elif host.has_rocm:
# No CPU asset for ROCm-only hosts: an uncovered GPU must trigger the HIP source build.
published_rocm = published_rocm_choice_for_host(bundle, host, "linux-rocm")
if published_rocm is not None:
attempts.append(published_rocm)
elif _uncovered_rocm_gpu_takes_vulkan(host):
vulkan_choice = published_asset_choice_for_kind(bundle, "linux-vulkan", host = host)
if vulkan_choice is not None:
log(
f"no ROCm prebuilt covers {_active_rocm_gfx_target(host) or 'this AMD GPU'} "
"and hipcc is not installed -- using the Vulkan prebuilt"
)
attempts.append(vulkan_choice)
else:
if (host.has_intel_gpu or host.has_amd_gpu_without_rocm) and not host.has_physical_nvidia:
vulkan_choice = published_asset_choice_for_kind(bundle, "linux-vulkan", host = host)
if vulkan_choice is not None:
attempts.append(vulkan_choice)
# Below ROCm and Vulkan on purpose: a masked NVIDIA host with usable ROCm keeps ROCm.
selection_host = _masked_nvidia_selection_host(host)
if selection_host is not None:
torch_preference = detect_torch_cuda_runtime_preference(
selection_host, gpu_hidden_by_mask = True
)
masked_selection = linux_cuda_choice_from_release(
selection_host,
bundle,
preferred_runtime_line = torch_preference.runtime_line,
selection_preamble = torch_preference.selection_log,
)
if masked_selection is not None:
attempts.extend(masked_selection.attempts)
return attempts
# A usable-NVIDIA host never reaches here. Other Linux archs get no bundle: the preflight checks
# libraries, not ELF arch, so a wrong-arch binary would not be caught.
kind = "linux-cpu" if host.is_x86_64 else "linux-arm64" if host.is_arm64 else None
cpu_choice = published_asset_choice_for_kind(bundle, kind) if kind else None
if cpu_choice is not None:
attempts.append(cpu_choice)
return attempts
RELEASE_LISTING_TRANSPORT_ERRORS = (
urllib.error.URLError,
ssl.SSLError,
ConnectionError,
TimeoutError,
RuntimeError,
ValueError,
)
def release_listing_fallback(exc: BaseException, published_repo: str) -> PrebuiltFallback:
# A resolver probe must fail hard, or update_flow caches "no prebuilt".
return PrebuiltFallback(
f"failed to inspect published releases in "
f"{published_repo or DEFAULT_PUBLISHED_REPO}: {exc}"
)
def iter_release_plans(
plans: "Sequence[InstallReleasePlan]", published_repo: str
) -> "Iterator[InstallReleasePlan]":
try:
yield from plans
except (BusyInstallConflict, PrebuiltFallback):
raise
except RELEASE_LISTING_TRANSPORT_ERRORS as exc:
raise release_listing_fallback(exc, published_repo) from exc
class LazyReleasePlans(Sequence):
"""Release plans resolved on first access; ``len()`` resolves them all."""
def __init__(self, plans: "Iterable[InstallReleasePlan]") -> None:
self._source: "Iterator[InstallReleasePlan] | None" = iter(plans)
self._resolved: list[InstallReleasePlan] = []
self._failure: "BaseException | None" = None
def _resolve_through(self, index: int) -> bool:
while len(self._resolved) <= index:
# A raised generator is closed; remember the failure so re-traversal answers the same.
if self._failure is not None:
raise self._failure
if self._source is None:
return False
try:
self._resolved.append(next(self._source))
except StopIteration:
self._source = None
return False
except BaseException as exc:
self._source = None
self._failure = exc
raise
return True
def __getitem__(self, index):
if isinstance(index, slice) or index < 0:
self._resolve_through(sys.maxsize)
return self._resolved[index]
if not self._resolve_through(index):
raise IndexError(index)
return self._resolved[index]
def __len__(self) -> int:
self._resolve_through(sys.maxsize)
return len(self._resolved)
def __iter__(self):
index = 0
while self._resolve_through(index):
yield self._resolved[index]
index += 1
def __bool__(self) -> bool:
return self._resolve_through(0)
def _fork_manifest_release_plans(
llama_tag: str,
host: HostInfo,
published_repo: str,
published_release_tag: str,
*,
max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS,
) -> "tuple[str, Sequence[InstallReleasePlan]]":
"""Manifest-reading branch of resolve_simple_install_release_plans, used for
every fork host: all of the fork's bundles describe their GPU/arch coverage
in llama-prebuilt-manifest.json rather than in the asset filename (CPU,
x64/arm64 CUDA, Windows CUDA, per-gfx ROCm, and macOS)."""
requested_tag = normalized_requested_llama_tag(llama_tag)
allow_older_release_fallback = requested_tag == "latest" and not published_release_tag
release_limit = max(1, max_release_fallbacks)
if host.is_macos and allow_older_release_fallback and host.macos_version is not None:
release_limit = max(release_limit, DEFAULT_MAX_MACOS_RELEASE_FALLBACKS)
def _plans() -> "Iterator[InstallReleasePlan]":
resolved_count = 0
last_error: PrebuiltFallback | None = None
skipped_newest: str | None = None
for resolved_release in iter_resolved_published_releases(
llama_tag,
published_repo,
published_release_tag,
continue_after_fast_path = host.is_macos,
):
bundle = resolved_release.bundle
checksums = resolved_release.checksums
resolved_tag = bundle.upstream_tag
try:
if host.is_linux:
linux_attempts = _linux_published_attempts(host, bundle)
if not linux_attempts:
raise PrebuiltFallback("no compatible Linux prebuilt asset was found")
attempts = apply_approved_hashes(linux_attempts, checksums)
if not attempts:
raise PrebuiltFallback("no compatible Linux prebuilt asset was found")
if attempts[0].selection_log:
log_lines(attempts[0].selection_log)
else:
attempts = resolve_release_asset_choice(
host,
resolved_tag,
bundle,
checksums,
)
if not attempts:
raise PrebuiltFallback("no compatible prebuilt asset was found")
if attempts[0].selection_log:
log_lines(attempts[0].selection_log)
except PrebuiltFallback as exc:
last_error = exc
if not allow_older_release_fallback:
raise
# #12842: the floor is the host's, so every older release fails alike.
if (
(host.is_linux or host.is_windows)
and host.has_physical_nvidia
and driver_below_cuda_prebuilt_floor(host)
):
raise PrebuiltFallback(cuda_driver_floor_message(host)) from exc
# Same for Jetson: no release's arm64 CUDA bundle serves it.
if host.has_physical_nvidia and _tegra_blocks_arm64_cuda(host):
raise
log(
"published release skipped for install planning: "
f"{bundle.repo}@{bundle.release_tag} upstream_tag={resolved_tag} ({exc})"
)
if resolved_count == 0 and skipped_newest is None:
skipped_newest = bundle.release_tag
continue
yield InstallReleasePlan(
requested_tag = requested_tag,
llama_tag = resolved_tag,
release_tag = bundle.release_tag,
attempts = attempts,
approved_checksums = checksums,
walk_back = (
_core.walk_back_for(host, skipped_newest) if resolved_count == 0 else None
),
)
resolved_count += 1
if not allow_older_release_fallback or resolved_count >= release_limit:
return
if resolved_count:
return
if last_error is not None:
raise last_error
raise PrebuiltFallback("no installable published llama.cpp releases were found")
plans = LazyReleasePlans(_plans())
if not plans:
raise PrebuiltFallback("no installable published llama.cpp releases were found")
return requested_tag, plans
def persisted_llama_backend(llama_backend: str | None, choice: AssetChoice) -> str | None:
"""Backend to record for an install that landed choice: a Vulkan request can end on a non-Vulkan
bundle (no upstream Vulkan archive for Windows arm64; x64 falls to win-cpu-x64), and
recording vulkan would make the updater re-assert it; persist the real outcome."""
if llama_backend == "vulkan" and choice.install_kind not in VULKAN_INSTALL_KINDS:
return None
return llama_backend
def persisted_marker_backend_request(backend_request: str | None, choice: AssetChoice) -> str:
"""The backend choice to record for an install that landed ``choice``.
The request VERBATIM, "auto" only when that is what was asked. "auto" is stored
explicitly rather than omitted: absent means "written before this field existed",
which reads back as the derived legacy choice, not as detection.
It used to store "auto" whenever the request and the bundle disagreed, which erased
the choice permanently: every later update re-detected, and on AMD that means ROCm, so
a configured Vulkan kept coming back as "automatic" (#11143). The miss is now recorded
alongside by marker_backend_request_was_satisfied instead.
"""
if backend_request in (None, "auto"):
return "auto"
effective = backend_for_install_kind(choice.install_kind)
if effective is not None and not is_requestable_backend(effective):
return "auto"
return backend_request
def marker_backend_request_was_satisfied(backend_request: str | None, choice: AssetChoice) -> bool:
"""Whether the request this marker records is the backend that actually landed.
A bundle whose kind maps to no backend cannot contradict the request, so it counts as
satisfied: the same "None cannot disagree" rule the erasing version used.
"""
if backend_request in (None, "auto"):
return True
effective = backend_for_install_kind(choice.install_kind)
if effective is None:
return True
# A single-build platform cannot honour a named backend, so "metal" is not requestable.
if not is_requestable_backend(effective):
return True
return effective == backend_request
# Absent means satisfied, which is how every older marker reads.
MARKER_BACKEND_REQUEST_UNSATISFIED = "backend_request_unsatisfied"
def marker_records_unsatisfied_backend_request(marker: "dict[str, Any] | None") -> bool:
"""Whether ``marker`` says its recorded request was not the backend installed."""
if not marker:
return False
return bool(marker.get(MARKER_BACKEND_REQUEST_UNSATISFIED))
def host_profile(host: HostInfo) -> dict[str, Any]:
"""The hardware facts the backend routing reads, in a shape a marker can hold.
Recorded so a later run can ask the one question the recorded request cannot answer:
is this still the box the bundle was chosen for. `backend_request` records what was
ASKED ("auto" on every automatic install), not what detection replied, and the
release tag does not move when the hardware does -- so without this an automatic
install keeps its CPU bundle after a GPU is added, keeps its CUDA bundle after the
driver or the card is removed, and keeps a coverage bundle after a card outside its
supported_sms replaces the old one, in every case until the fork happens to publish
a new release.
Only fields the selectors branch on, and only JSON-native types: a tuple reads back
from JSON as a list, and a profile that never compares equal to itself would take
the full path on every update forever. Computed from the ROUTED host, so the
comparison is made against the same HostInfo the selection was made from.
"""
def _sorted_labels(values: "Iterable[Any] | None") -> list[str]:
return sorted({str(value).strip() for value in (values or []) if str(value).strip()})
return {
"machine": str(host.machine or "").strip().lower(),
"has_usable_nvidia": bool(host.has_usable_nvidia),
"has_physical_nvidia": bool(host.has_physical_nvidia),
"driver_cuda_version": (
list(host.driver_cuda_version) if host.driver_cuda_version else None
),
"compute_caps": _sorted_labels(host.compute_caps),
# Only when a mask hides another SM (the selectors then widen), so other hosts keep their profile.
**(
{"physical_compute_caps": _sorted_labels(host.physical_compute_caps)}
if _mask_widened_sms(host) is not None
else {}
),
"has_rocm": bool(host.has_rocm),
"rocm_runtime": _rocm_runtime_for_profile(host),
"rocm_gfx_target": host.rocm_gfx_target or None,
"rocm_gfx_targets": _sorted_labels(host.rocm_gfx_targets),
"has_intel_gpu": bool(host.has_intel_gpu),
"has_amd_gpu_without_rocm": bool(host.has_amd_gpu_without_rocm),
"macos_version": list(host.macos_version) if host.macos_version else None,
"cuda_runtime_lines": _detected_cuda_runtime_lines(host),
}
def _rocm_runtime_for_profile(host: HostInfo) -> "list[int] | None":
if not host.has_rocm:
return None
try:
version = _detect_host_rocm_version()
except Exception: # noqa: BLE001 - an unreadable runtime is "cannot tell", recorded as such
return None
return list(version) if version else None
def _detected_cuda_runtime_lines(host: HostInfo) -> "list[str] | None":
if not (host.has_usable_nvidia and (host.is_linux or host.is_windows)):
return None
try:
detected = (
detected_linux_runtime_lines() if host.is_linux else detected_windows_runtime_lines()
)
except Exception: # noqa: BLE001 - an unreadable scan is "cannot tell", which takes the full path
return None
lines = detected[0] if isinstance(detected, tuple) else detected
return sorted({str(line) for line in lines})
def write_prebuilt_metadata(
install_dir: Path,
*,
host: HostInfo | None = None,
requested_tag: str,
llama_tag: str,
release_tag: str,
choice: AssetChoice,
approved_checksums: ApprovedReleaseChecksums,
prebuilt_fallback_used: bool,
force_cpu: bool = False,
llama_backend: str | None = None,
backend_request: str | None = None,
rocm_gfx: str | None = None,
walk_back: "_core.WalkBack | None" = None,
macos_load_probe_passed: bool = False,
) -> None:
source_asset_name, source_sha256 = selected_source_archive_metadata(
approved_checksums,
llama_tag,
)
# Reuse expected_install_fingerprint so the two payloads cannot drift and look stale.
fingerprint = expected_install_fingerprint(
llama_tag = llama_tag,
release_tag = release_tag,
choice = choice,
approved_checksums = approved_checksums,
)
if fingerprint is None:
raise PrebuiltFallback(f"cannot compute install fingerprint for {choice.name}")
_persisted_backend = persisted_llama_backend(llama_backend, choice)
_load_probe = (
macos_load_probe_record(host) if macos_load_probe_passed and host is not None else None
)
try:
_runtime_patterns: list[str] | None = runtime_patterns_for_choice(choice)
except PrebuiltFallback:
_runtime_patterns = None
metadata = {
"requested_tag": requested_tag,
"tag": llama_tag,
"release_tag": release_tag,
"published_repo": approved_checksums.repo,
"asset": choice.name,
# Only for --force-cpu, so the updater does not re-route a forced CPU install; --cpu-fallback can heal.
"force_cpu": force_cpu,
# Kept for older Unsloth versions that only understand Vulkan overrides.
"llama_backend": _persisted_backend,
"backend": backend_for_install_kind(choice.install_kind),
"backend_request": persisted_marker_backend_request(backend_request, choice),
**(
{}
if marker_backend_request_was_satisfied(backend_request, choice)
else {MARKER_BACKEND_REQUEST_UNSATISFIED: True}
),
# The AMD gfx an AUTOMATIC route used: Vulkan asset names carry no arch, and driver-only hosts
# have no probe to re-detect it.
**({"rocm_gfx": rocm_gfx} if rocm_gfx and _persisted_backend == "auto" else {}),
"asset_sha256": choice.expected_sha256,
"runtime_asset": choice.runtime_name,
"source": choice.source_label,
# Where the binaries came from; published_repo/release_tag refer to the source tree.
"binary_repo": choice.repo,
"binary_release_tag": choice.tag,
"source_asset": source_asset_name,
"source_sha256": source_sha256,
"source_commit": approved_checksums.source_commit,
"source_commit_short": approved_checksums.source_commit_short,
"source_repo": approved_checksums.source_repo,
"source_repo_url": approved_checksums.source_repo_url,
"source_ref_kind": approved_checksums.source_ref_kind,
"requested_source_ref": approved_checksums.requested_source_ref,
"resolved_source_ref": approved_checksums.resolved_source_ref,
# Read back by install_whisper_prebuilt.py to pair slim bundles.
"ggml_tree": recorded_ggml_tree(approved_checksums, choice),
"bundle_profile": choice.bundle_profile,
"runtime_line": choice.runtime_line,
"torch_runtime_preference": _torch_runtime_preference_for_marker(host),
"coverage_class": choice.coverage_class,
# Not fingerprinted, so existing installs stay valid and fail open until refresh.
"gfx_target": choice.gfx_target,
"mapped_targets": list(choice.mapped_targets or []),
"supported_sms": [str(s) for s in (choice.supported_sms or [])],
"runtime_sha256": choice.runtime_sha256,
"install_fingerprint": fingerprint,
**(walk_back.marker_fields() if walk_back is not None else {}),
"runtime_files": runtime_file_records(install_dir, host, _runtime_patterns),
**({"host_profile": host_profile(host)} if host is not None else {}),
# Evidence dyld loaded these bytes on this macOS build; not fingerprinted.
**({MACOS_LOAD_PROBE_KEY: _load_probe} if _load_probe is not None else {}),
"prebuilt_fallback_used": prebuilt_fallback_used,
"installed_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
}
(install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text(
json.dumps(metadata, indent = 2) + "\n", encoding = "utf-8"
)
def _write_marker(marker_path: Path, marker: dict) -> bool:
"""Best-effort marker rewrite for the reuse path. Returns False if it stuck.
Atomic only, deliberately. os.replace swaps in a sibling temp file, so it
succeeds on a read-only marker in a writable dir (a shared or admin-owned
install) where a plain write_text would not, and it can never leave a
half-written marker behind: an in-place retry would truncate a valid file
first, and an ENOSPC or I/O error mid-write would strand a partial
UNSLOTH_PREBUILT_INFO.json that later updates no longer recognise. The whole
install is worth more than this one refreshed field.
Never raises -- the reuse path runs after the install is already valid, and
an unexpected exit no longer falls back to a source build, so failing here
would abort setup over a metadata refresh.
"""
data = (json.dumps(marker, indent = 2) + "\n").encode("utf-8")
try:
original = marker_path.stat()
except OSError:
original = None
# Outside the try so a raising write (ENOSPC) cannot strand a partial .tmp-*.
tmp_path: Path | None = None
try:
# Own temp file so the mode is fixed BEFORE the swap: os.replace keeps NamedTemporaryFile's 0600.
with tempfile.NamedTemporaryFile(
prefix = marker_path.name + ".tmp-",
dir = marker_path.parent,
delete = False,
) as handle:
tmp_path = Path(handle.name)
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
if original is not None:
os.chmod(tmp_path, stat.S_IMODE(original.st_mode))
try:
os.chown(tmp_path, original.st_uid, original.st_gid)
except (OSError, AttributeError):
try:
os.chown(tmp_path, -1, original.st_gid)
except (OSError, AttributeError):
pass
atomic_replace_from_tempfile(tmp_path, marker_path)
return True
except OSError:
if tmp_path is not None:
try:
tmp_path.unlink()
except OSError:
pass
return False
def recorded_ggml_tree(
approved_checksums: ApprovedReleaseChecksums, choice: AssetChoice
) -> str | None:
"""The ggml tree to record, or None when it does not describe this install: the tree comes from
the fork release's checkout, so an approved ggml-org archive (choice.repo differs) must not
record it."""
tree = approved_checksums.ggml_tree
if not isinstance(tree, str) or not tree:
return None
return tree if choice.repo == approved_checksums.repo else None
def _marker_selection_patch(
marker: dict,
*,
choice: AssetChoice,
backend_request: str | None,
persist_force_cpu: bool,
persist_llama_backend: str | None,
ggml_tree: str | None,
rocm_gfx: str | None,
install_dir: Path | None = None,
host: HostInfo | None = None,
prebuilt_fallback_used: bool | None = None,
walk_back: "_core.WalkBack | None" = None,
) -> dict:
"""The fields a reused install must still take from this run.
Everything write_prebuilt_metadata records about *how* the install was chosen
belongs here too, because "the bundle is already correct" does not mean the
choice behind it is unchanged: someone can switch a CUDA install to CPU and land
on the very bundle detection would have picked, and only these fields stop the
next update from routing it straight back.
It is also the only place that ever re-examines a bundle without reinstalling it,
which makes it the only place that can catch an OLD marker up to the evidence the
no-network re-check needs. Hence *install_dir* and *host*: see the backfill at the
end.
"""
patch: dict = {}
if bool(marker.get("force_cpu")) != persist_force_cpu:
patch["force_cpu"] = persist_force_cpu
if marker.get("llama_backend") != persist_llama_backend:
patch["llama_backend"] = persist_llama_backend
effective = backend_for_install_kind(choice.install_kind)
if effective is not None and marker.get("backend") != effective:
patch["backend"] = effective
recorded_request = persisted_marker_backend_request(backend_request, choice)
if marker.get("backend_request") != recorded_request:
patch["backend_request"] = recorded_request
# None pops the key: absence is the only spelling of satisfied.
unsatisfied = not marker_backend_request_was_satisfied(backend_request, choice)
if bool(marker.get(MARKER_BACKEND_REQUEST_UNSATISFIED)) != unsatisfied or (
not unsatisfied and MARKER_BACKEND_REQUEST_UNSATISFIED in marker
):
patch[MARKER_BACKEND_REQUEST_UNSATISFIED] = True if unsatisfied else None
# None here means "this release declares none", not "clear it".
if ggml_tree and marker.get("ggml_tree") != ggml_tree:
patch["ggml_tree"] = ggml_tree
automatic = patch.get("llama_backend", marker.get("llama_backend")) == "auto"
if rocm_gfx and automatic and marker.get("rocm_gfx") != rocm_gfx:
patch["rocm_gfx"] = rocm_gfx
# Absent targets mean "declares none", not "clear it"; outside the fingerprint, so no reinstall.
targets = [str(t).strip() for t in (choice.mapped_targets or []) if str(t).strip()]
if targets and (
marker.get("mapped_targets") != targets or marker.get("gfx_target") != choice.gfx_target
):
patch["gfx_target"] = choice.gfx_target
patch["mapped_targets"] = targets
sms = [str(s).strip() for s in (choice.supported_sms or []) if str(s).strip()]
if sms and marker.get("supported_sms") != sms:
patch["supported_sms"] = sms
if choice.runtime_name and marker.get("runtime_asset") != choice.runtime_name:
patch["runtime_asset"] = choice.runtime_name
# ADDED only, never corrected, so reuse cannot re-bless unhashed bytes; a stale
# prebuilt_fallback_used False would block retrying the preferred bundle.
if prebuilt_fallback_used is not None and marker.get("prebuilt_fallback_used") is not (
prebuilt_fallback_used
):
patch["prebuilt_fallback_used"] = prebuilt_fallback_used
if "runtime_sha256" not in marker:
patch["runtime_sha256"] = choice.runtime_sha256
patch.update(_core.walk_back_patch(marker, walk_back))
# Empty counts as absent, else the marker fails closed on every update.
if install_dir is not None and not marker.get("runtime_files"):
try:
backfill_patterns: list[str] | None = runtime_patterns_for_choice(choice)
except PrebuiltFallback:
backfill_patterns = None
records = runtime_file_records(install_dir, host, backfill_patterns)
if records:
patch["runtime_files"] = records
if host is not None and marker.get("host_profile") != host_profile(host):
patch["host_profile"] = host_profile(host)
if host is not None and (host.is_linux or host.is_windows):
preference = _torch_runtime_preference_for_marker(host)
if (
"torch_runtime_preference" not in marker
or marker.get("torch_runtime_preference") != preference
):
patch["torch_runtime_preference"] = preference
return patch
def sync_marker_selection(
install_dir: Path,
*,
choice: AssetChoice,
backend_request: str | None,
persist_force_cpu: bool = False,
persist_llama_backend: str | None = None,
ggml_tree: str | None = None,
rocm_gfx: str | None = None,
host: HostInfo | None = None,
prebuilt_fallback_used: bool | None = None,
walk_back: "_core.WalkBack | None" = None,
) -> None:
"""Record this run's selection on a marker whose bundle was reused unchanged.
One write for every field rather than one per field: these are read together by
the updater, and a partial refresh (say force_cpu recorded but backend_request
not) describes an install nobody asked for. Best-effort throughout -- the
install is already valid, so a read-only or full disk warns instead of failing
setup over metadata.
"""
marker_path = install_dir / "UNSLOTH_PREBUILT_INFO.json"
try:
marker = json.loads(marker_path.read_text(encoding = "utf-8"))
except (OSError, ValueError):
return
if not isinstance(marker, dict):
return
patch = _marker_selection_patch(
marker,
choice = choice,
backend_request = backend_request,
persist_force_cpu = persist_force_cpu,
persist_llama_backend = persist_llama_backend,
ggml_tree = ggml_tree,
rocm_gfx = rocm_gfx,
install_dir = install_dir,
host = host,
prebuilt_fallback_used = prebuilt_fallback_used,
walk_back = walk_back,
)
if not patch:
return
for key, value in patch.items():
if value is None and key in (
"llama_backend",
MARKER_BACKEND_REQUEST_UNSATISFIED,
*_core.WALK_BACK_KEYS,
):
marker.pop(key, None)
else:
marker[key] = value
if not _write_marker(marker_path, marker):
# Losing this lets the next update re-route a deliberate choice; warn, do not fail.
log(
f"WARNING: could not record {sorted(patch)} in {marker_path}; "
"a later update may not re-assert this choice"
)
return
log(f"existing install reused; recorded {sorted(patch)} from this run")
def expected_install_fingerprint(
*,
llama_tag: str,
release_tag: str,
choice: AssetChoice,
approved_checksums: ApprovedReleaseChecksums,
) -> str | None:
source_asset_name, source_sha256 = selected_source_archive_metadata(
approved_checksums,
llama_tag,
)
payload = {
"published_repo": approved_checksums.repo,
"release_tag": release_tag,
"upstream_tag": llama_tag,
"asset": choice.name,
"asset_sha256": choice.expected_sha256,
"source": choice.source_label,
"source_asset": source_asset_name,
"source_sha256": source_sha256,
"runtime_line": choice.runtime_line,
# Fingerprinting the cudart pair forces older cudart-less installs to refresh.
"runtime_asset": choice.runtime_name,
"runtime_sha256": choice.runtime_sha256,
"bundle_profile": choice.bundle_profile,
"coverage_class": choice.coverage_class,
}
return hashlib.sha256(
json.dumps(payload, sort_keys = True, separators = (",", ":")).encode("utf-8")
).hexdigest()
def load_prebuilt_metadata(install_dir: Path) -> dict[str, Any] | None:
metadata_path = install_dir / "UNSLOTH_PREBUILT_INFO.json"
if not metadata_path.is_file():
return None
try:
payload = json.loads(metadata_path.read_text(encoding = "utf-8"))
except Exception:
return None
if not isinstance(payload, dict):
return None
return payload
def default_managed_llama_dir() -> Path:
"""The managed llama.cpp install root, mirroring setup.sh and the updater:
the UNSLOTH_LLAMA_CPP_PATH override, else <custom studio home>/llama.cpp,
else the legacy ~/.unsloth/llama.cpp."""
override = (os.environ.get("UNSLOTH_LLAMA_CPP_PATH") or "").strip()
if override:
return Path(override).expanduser()
home = (os.environ.get("UNSLOTH_STUDIO_HOME") or os.environ.get("STUDIO_HOME") or "").strip()
if home:
root = Path(home).expanduser()
legacy_studio = Path.home() / ".unsloth" / "studio"
try:
is_legacy = root.resolve() == legacy_studio.resolve()
except (OSError, ValueError):
is_legacy = root == legacy_studio
if not is_legacy:
return root / "llama.cpp"
return Path.home() / ".unsloth" / "llama.cpp"
def installed_llama_runtime(install_dir: Path | None = None) -> tuple[Path, str, str] | None:
"""The completed managed llama.cpp prebuilt install, or None.
Returns (runtime bin dir holding the ggml libraries, installed release tag,
bundle profile). install_whisper_prebuilt.py reads this to pair slim whisper
bundles with the ggml runtime the llama install already provides; tests pass
an explicit ``install_dir``.
"""
root = install_dir if install_dir is not None else default_managed_llama_dir()
metadata = load_prebuilt_metadata(root)
if metadata is None:
return None
release_tag = metadata.get("release_tag")
if not isinstance(release_tag, str) or not release_tag:
return None
bin_dir = root / "build" / "bin"
if os.name == "nt":
bin_dir = bin_dir / "Release"
if not bin_dir.is_dir():
return None
profile = metadata.get("bundle_profile")
return bin_dir, release_tag, profile if isinstance(profile, str) else ""
def installed_llama_ggml_tree(install_dir: Path | None = None) -> str | None:
"""ggml tree id of the managed llama.cpp install; None for installs predating
it. Kept out of installed_llama_runtime() to keep that tuple's shape."""
root = install_dir if install_dir is not None else default_managed_llama_dir()
metadata = load_prebuilt_metadata(root)
if metadata is None:
return None
tree = metadata.get("ggml_tree")
return tree if isinstance(tree, str) and tree else None
# Older archives are monolithic and healthy without llama-server-impl.dll.
LLAMA_SERVER_IMPL_SPLIT_BUILD = 9283
def _release_build_number(tag: str | None) -> int | None:
"""Build number of a ``bNNNN`` tag, also matching the fork's ``bNNNN-mix-<sha>``.
None for a branch or commit pin, which callers treat as "assume current"."""
if not isinstance(tag, str):
return None
match = re.match(r"b(\d+)(?:[-.]|$)", tag.strip())
return int(match.group(1)) if match else None
def _windows_shared_groups(source_label: str | None, tag: str | None = None) -> list[list[str]]:
"""Runtime files every Windows install kind owes before its backend DLL; requiring only
llama.dll let a truncated extract validate then fail at exec. Prebuilt sources only:
setup.ps1 links statically and ships none of these."""
groups: list[list[str]] = [["llama.dll"]]
if source_label in {"published", "upstream"}:
groups.append(["llama-common.dll"])
groups.append(["llama-server.exe"])
build = _release_build_number(tag)
if build is None or build >= LLAMA_SERVER_IMPL_SPLIT_BUILD:
# Both halves: llama-quantize.exe links llama-quantize-impl.dll like the server does.
groups.append(["llama-server-impl.dll"])
groups.append(["llama-quantize-impl.dll"])
groups.append(["ggml.dll"])
groups.append(["ggml-base.dll"])
groups.append(["ggml-cpu*.dll"])
groups.append(["mtmd.dll"])
return groups
"""The CUDA runtime a windows-cuda bundle pairs with, installed and removed together."""
_CUDA_RUNTIME_TRIO = ("cudart64_*.dll", "cublas64_*.dll", "cublasLt64_*.dll")
def _has_a_paired_cuda_runtime(install_dir: Path) -> bool:
"""Whether a windows-cuda tree carries any member of that trio.
A marker written before ``runtime_asset`` existed names no paired archive, so the
trio was dropped from the table entirely and losing one member read as healthy while
llama-server.exe died in the loader with no repair offered and no marker backfilled.
The three arrive and go together, so one of them still being there is what says this
install was paired; a machine running on a system CUDA toolkit has none and is asked
for none.
"""
runtime_dir = install_dir / "build" / "bin" / "Release"
return any(
any(_payload_match_is_loadable(match) for match in runtime_dir.glob(pattern))
for pattern in _CUDA_RUNTIME_TRIO
)
def _linux_split_entrypoint_groups(
source_label: str | None, tag: str | None = None
) -> list[list[str]]:
"""The impl libraries a Linux entrypoint links against, when the release has them.
The same upstream split that gave Windows ``llama-server-impl.dll`` gives Linux
``libllama-server-impl.so``: ``llama-server`` and ``llama-quantize`` carry no
entry code of their own any more and load these by DT_NEEDED. The library
groups above name only the shared libraries, so quarantining one of these left
every group satisfied while ``llama-server`` died in the loader and
``_existing_install_runs`` returned false, which is the disagreement this whole
probe exists to prevent. Measured on a b10360 managed install: removing either
one leaves ``installed_runtime_health`` answering ``(True, "")`` and
``_existing_install_runs`` answering false.
Gated exactly as the Windows side is, and on the same build for the same
reason: an older monolithic archive is healthy without them, and requiring one
a bundle does not carry would reinstall on every check forever.
"""
if source_label not in {"published", "upstream"}:
return []
build = _release_build_number(tag)
if build is not None and build < LLAMA_SERVER_IMPL_SPLIT_BUILD:
return []
return [["libllama-server-impl.so*"], ["libllama-quantize-impl.so*"]]
def _macos_split_entrypoint_groups(
source_label: str | None, tag: str | None = None
) -> list[list[str]]:
"""The macOS half of the same upstream split, gated by the same rule.
The bundles carry them: b10840 and b11007 macos-arm64 both ship
libllama-server-impl.dylib and libllama-quantize-impl.dylib, and
_PREBUILT_TREE_EVIDENCE already reads the first as proof of a prebuilt macOS tree.
Without them here every group and both executable checks passed while dyld could not
start the entrypoint. Delegates the gate so the build floor cannot drift from Linux.
"""
if not _linux_split_entrypoint_groups(source_label, tag):
return []
return [["libllama-server-impl*.dylib"], ["libllama-quantize-impl*.dylib"]]
def runtime_payload_health_groups(
install_kind: str,
*,
source_label: str | None = None,
runtime_name: str | None = None,
tag: str | None = None,
install_dir: Path | None = None,
) -> list[list[str]]:
"""Required runtime file groups for an install kind; install_dir is read only where the marker
cannot answer alone, today the windows-cuda trio a legacy marker does not name."""
if install_kind in {"linux-cpu", "linux-arm64"}:
return [
["libllama-common.so*"],
["libllama.so*"],
["libggml.so*"],
["libggml-base.so*"],
["libggml-cpu*.so*"],
["libmtmd.so*"],
] + _linux_split_entrypoint_groups(source_label, tag)
if install_kind in {"linux-cuda", "linux-arm64-cuda"}:
return [
["libllama-common.so*"],
["libllama.so*"],
["libggml.so*"],
["libggml-base.so*"],
["libggml-cpu*.so*"],
["libmtmd.so*"],
["libggml-cuda.so*"],
] + _linux_split_entrypoint_groups(source_label, tag)
if install_kind in {"macos-arm64", "macos-x64"}:
# One group per library: a broad libggml*.dylib group stays satisfied by siblings after the needed
# one is quarantined. blas, metal and rpc are optional backends, so not required.
return [
["libllama-common.dylib", "libllama-common.*.dylib"],
["libllama.dylib", "libllama.*.dylib"],
["libggml.dylib", "libggml.*.dylib"],
["libggml-base.dylib", "libggml-base.*.dylib"],
["libggml-cpu.dylib", "libggml-cpu.*.dylib"],
["libmtmd.dylib", "libmtmd.*.dylib"],
] + _macos_split_entrypoint_groups(source_label, tag)
if install_kind == "linux-rocm":
return [
["libllama-common.so*"],
["libllama.so*"],
["libggml.so*"],
["libggml-base.so*"],
["libggml-cpu*.so*"],
["libmtmd.so*"],
["libggml-hip.so*"],
] + _linux_split_entrypoint_groups(source_label, tag)
if install_kind == "linux-vulkan":
groups = [
["libllama-common.so*"],
["libllama.so*"],
["libggml.so*"],
["libggml-base.so*"],
# arm64 may ship a bare libggml-cpu.so; the '-' form missed it.
["libggml-cpu*.so*"],
["libmtmd.so*"],
["libggml-vulkan.so*"],
]
if source_label == "published":
groups.append(["llama-diffusion-gemma-visual-server"])
return groups + _linux_split_entrypoint_groups(source_label, tag)
if install_kind in {"windows-cpu", "windows-arm64"}:
return _windows_shared_groups(source_label, tag)
if install_kind in {"windows-cuda", "windows-arm64-cuda"}:
groups = _windows_shared_groups(source_label, tag) + [["ggml-cuda.dll"]]
if runtime_name or (install_dir is not None and _has_a_paired_cuda_runtime(install_dir)):
groups.append(["cudart64_*.dll"])
groups.append(["cublas64_*.dll"])
groups.append(["cublasLt64_*.dll"])
return groups
if install_kind in {"windows-hip", "windows-rocm"}:
# ggml-hip.dll by name: other ROCm libs would satisfy a "*hip*.dll" group without it.
return _windows_shared_groups(source_label, tag) + [["ggml-hip*.dll"]]
if install_kind in {"windows-vulkan", "windows-arm64-vulkan"}:
groups = _windows_shared_groups(source_label, tag) + [["ggml-vulkan.dll"]]
if source_label == "published":
groups.append(["llama-diffusion-gemma-visual-server.exe"])
return groups
return []
def install_runtime_dir(install_dir: Path, host: HostInfo) -> Path:
if host.is_windows:
return install_dir / "build" / "bin" / "Release"
return install_dir / "build" / "bin"
"""``libfoo.so``, or ``libfoo.so.0``, but not ``libfoo.so.0.0.10360``."""
_LINKER_NAME_RE = re.compile(r"^.+\.so(?P<version>(?:\.\d+)*)$")
"""``libfoo.dylib``, or ``libfoo.0.dylib``, but not ``libfoo.0.23.0.dylib``."""
_DYLIB_NAME_RE = re.compile(r"^.+?(?P<version>(?:\.\d+)*)\.dylib$")
def _family_base(name: str) -> str | None:
"""``libllama`` for every spelling of the libllama family, or None if not one."""
if name.endswith(".dylib"):
stem = name[: -len(".dylib")]
return re.sub(r"(?:\.\d+)+$", "", stem) or None
at = name.find(".so")
if at <= 0:
return None
if name[at + 3 :] and not re.fullmatch(r"(?:\.\d+)+", name[at + 3 :]):
return None
return name[:at]
def _has_versioned_siblings(path: Path) -> bool:
"""Whether a versionless library sits beside versioned copies of itself.
A family that only ever ships one unversioned file (``libggml-cpu-x64.so``) is
loadable under that name, and the versionless member of a versioned family is
not: the loader asks for the SONAME, one component deep. Reading the directory
is the only way to tell those apart, since both are the same name.
"""
base = _family_base(path.name)
if base is None:
return False
try:
siblings = list(path.parent.iterdir())
except OSError:
return False
for sibling in siblings:
if sibling.name == path.name or _family_base(sibling.name) != base:
continue
match = _LINKER_NAME_RE.match(sibling.name) or _DYLIB_NAME_RE.match(sibling.name)
if match is not None and match.group("version"):
return True
return False
def _is_nonempty_file(path: Path) -> bool:
"""A regular file with something in it.
Length is the one property of a file's contents this probe may read: it executes
nothing, and a zero-length library or entrypoint is not a thing any loader can use.
An interrupted extraction and security software that empties a file in place both
leave the directory entry, so ``is_file()`` and the execute bit stayed true while
``_existing_install_runs`` rejected the tree on ENOEXEC or a loader failure. No real
payload file is empty, so nothing shipped is refused by this.
"""
try:
status = path.stat()
except OSError:
return False
return stat.S_ISREG(status.st_mode) and status.st_size > 0
def _payload_match_is_loadable(path: Path) -> bool:
"""Whether a glob match is a file the loader would actually resolve.
``Path.glob`` does not follow links, so a dangling link (or a directory)
still matches the pattern; ``is_file()`` drops both.
A release ships ``libllama.so.0`` (the SONAME the binary asks for) beside
``libllama.so.0.0.10360``, and ``libllama.so*`` matches both, so quarantining
the SONAME left the group satisfied by the twin while ``llama-server
--version`` exited 127. A name with more version components than a SONAME can
only be the twin, so it does not count on its own.
macOS names the same pair the other way round, and needs the same rule: the
shipped bundle carries ``libggml.dylib -> libggml.0.dylib ->
libggml.0.23.0.dylib``, and ``llama-server``'s LC_LOAD_DYLIB entry is
``@rpath/libggml.0.dylib`` (the install name recorded in the terminal file's
own LC_ID_DYLIB), so losing the middle link is fatal to dyld while
``libggml.*.dylib`` stays satisfied by the terminal file. ``.dll`` names and
bare executables are unaffected.
"""
if not _is_nonempty_file(path):
return False
match = _LINKER_NAME_RE.match(path.name) or _DYLIB_NAME_RE.match(path.name)
if match is None:
# A ".so" name with a non-version tail (a renamed quarantine victim) is not loadable.
return ".so" not in path.name
depth = match.group("version").count(".")
if depth > 1:
return False
if depth == 1:
return True
# A versionless name is loadable only with no versioned siblings: copy_globs flattens links,
# so libllama.so can survive after libllama.so.0 is quarantined.
return not _has_versioned_siblings(path)
def _runtime_payload_has(install_dir: Path, host: HostInfo, groups: list[list[str]]) -> bool:
runtime_dir = install_runtime_dir(install_dir, host)
if not runtime_dir.exists():
return False
for pattern_group in groups:
matched = False
for pattern in pattern_group:
if any(_payload_match_is_loadable(path) for path in runtime_dir.glob(pattern)):
matched = True
break
if not matched:
return False
return True
def runtime_payload_is_healthy(install_dir: Path, host: HostInfo, choice: AssetChoice) -> bool:
return _runtime_payload_has(
install_dir,
host,
runtime_payload_health_groups(
choice.install_kind,
source_label = choice.source_label,
runtime_name = choice.runtime_name,
tag = choice.tag,
install_dir = install_dir,
),
)
"""Files only a published bundle ships, per platform.
A source build links these into its binaries: ``setup.ps1`` builds statically, and
no source tree produces a per-binary ``-impl`` library. So finding one is evidence
that the tree came from a release even when the marker cannot say so.
"""
_PREBUILT_TREE_EVIDENCE = {
"windows": ["llama-common.dll", "mtmd.dll", "llama-server-impl.dll"],
"linux": ["libllama-server-impl.so*", "libmtmd.so*"],
"macos": ["libllama-server-impl*.dylib", "libmtmd*.dylib"],
}
def _tree_looks_prebuilt(install_dir: Path, host: HostInfo) -> bool:
"""Whether the tree holds a file only a published bundle ships: an unparseable marker must not
drop the source-gated groups (a quarantined ggml-base.dll would then pass health), and static
source builds must stay healthy; with no evidence left the lenient answer stands."""
key = "windows" if host.is_windows else "macos" if host.is_macos else "linux"
runtime_dir = install_runtime_dir(install_dir, host)
return any(
any(_payload_match_is_loadable(match) for match in runtime_dir.glob(pattern))
for pattern in _PREBUILT_TREE_EVIDENCE[key]
)
_RUNTIME_RECORD_NAMES = (
"llama-server",
"llama-quantize",
"llama-diffusion-gemma-visual-server",
)
def _runtime_record_paths(install_dir: Path, host: "HostInfo | None" = None) -> "list[Path]":
"""Both copies of each binary, on every layout this installer produces.
_find_llama_server_binary reaches the root copy first and, without a symlink, the
two can rot independently. Both the Windows Release/ layout and the POSIX one are
walked whatever the host says: a record is only ever taken of a file that is
actually there, and a host argument that has to be threaded through every caller is
how this would go stale on the one platform nobody tests it on.
"""
directories = [
install_dir,
install_dir / "build" / "bin",
install_dir / "build" / "bin" / "Release",
]
if host is not None:
directories.insert(1, install_runtime_dir(install_dir, host))
paths: list[Path] = []
for directory in directories:
for name in _RUNTIME_RECORD_NAMES:
for candidate in (directory / name, directory / f"{name}.exe"):
if candidate not in paths:
paths.append(candidate)
return paths
def _stat_record(path: Path) -> "dict[str, Any] | None":
"""size + mtime_ns for one regular file, or None when it is neither."""
try:
if not path.is_file():
return None
info = path.stat()
except OSError:
return None
return {"size": info.st_size, "mtime_ns": info.st_mtime_ns}
def runtime_file_records(
install_dir: Path,
host: "HostInfo | None" = None,
patterns: "list[str] | None" = None,
) -> "dict[str, dict[str, Any]]":
"""What the runtime is made of, in a form a later run can re-check without running it.
Two tiers, because the two halves of a bundle fail differently:
* the binaries a reuse decision would otherwise START get size + sha256. Their
bytes decide whether the install runs at all, and three files is ~100 ms.
* every other file the bundle's own copy allowlist matches gets size + mtime_ns,
stat only. _runtime_payload_has globs for EXISTENCE, so a truncated
libggml-cuda.so, ggml-cuda.dll or cudart DLL passes it -- and those shared
libraries are where most of a CUDA bundle's bytes live, which is exactly the
shape a full disk or an interrupted extract leaves behind. A size comparison
catches it for the price of a stat; hashing 300 MB of kernels on every update
would not be worth it. macOS hashes both tiers, since its record stands in for the
dyld probe and a Metal bundle is tens of MB.
*patterns* comes from runtime_patterns_for_install_kind for the bundle being
installed; without it only the binary tier is recorded, which is what the callers
that have no bundle in hand (a bare re-record) can honestly say.
Call only after the platform preflights pass: the macOS probe skip trusts the record.
"""
records: dict[str, dict[str, Any]] = {}
runtime_dir = install_runtime_dir(install_dir, host) if host is not None else None
hash_payload = host is not None and host.is_macos
if patterns and runtime_dir is not None:
for pattern in patterns:
try:
matches = sorted(runtime_dir.glob(pattern))
except OSError:
continue
for candidate in matches:
record = _stat_record(candidate)
if record is None:
continue
try:
relative = candidate.relative_to(install_dir).as_posix()
except ValueError:
continue
if hash_payload:
try:
record["sha256"] = sha256_file(candidate)
except (OSError, MemoryError) as exc:
log(
f"could not hash {relative} for the runtime record ({exc}); "
"not recording"
)
return {}
records[relative] = record
for candidate in _runtime_record_paths(install_dir, host):
try:
relative = candidate.relative_to(install_dir).as_posix()
except ValueError:
continue
record = _stat_record(candidate)
if record is None:
continue
try:
record["sha256"] = sha256_file(candidate)
except (OSError, MemoryError) as exc:
# No record at all: the size-only tier would trust a same-size corrupt binary.
log(f"could not hash {relative} for the runtime record ({exc}); not recording")
return {}
records[relative] = record
return records
MACOS_LOAD_PROBE_KEY = "macos_load_probe"
def macos_product_version() -> str:
"""The macOS identity the dyld shared cache actually tracks, as one string.
host_profile records parse_macos_version, which is (major, minor) by design --
right for the minos comparison, wrong for "can this image still load": since Big
Sur the system libraries are a dyld shared cache blob, and Apple replaces that
blob in point releases and in Rapid Security Responses.
A Rapid Security Response does NOT move ProductVersion, which is all
platform.mac_ver() reads: 13.3.1 (a) reports ProductVersion 13.3.1 with the "(a)"
in ProductVersionExtra and the build at 22E772610a instead of 22E261. Reading
SystemVersion.plist has the same blind spot and is additionally stale for an RSR
(osquery/osquery#8008), so ask sw_vers -- bare, since it documents single-dash
options and a misspelt flag would silently cost the field this exists to read.
"""
try:
done = subprocess.run(
["/usr/bin/sw_vers"],
capture_output = True,
text = True,
timeout = 10,
check = False,
)
if done.returncode == 0:
values: "dict[str, str]" = {}
for line in done.stdout.splitlines():
key, sep, value = line.partition(":")
if sep:
values[key.strip()] = value.strip()
fields = [
values.get(key, "")
for key in ("ProductVersion", "ProductVersionExtra", "BuildVersion")
]
if fields[0]:
return " ".join(f for f in fields if f)
except Exception: # noqa: BLE001
pass
try:
return str(platform.mac_ver()[0] or "").strip()
except Exception: # noqa: BLE001
return ""
def macos_load_probe_record(host: HostInfo) -> "dict[str, Any] | None":
"""The evidence a completed dyld probe leaves for later runs, or None when the
host cannot describe itself finely enough to be worth recording."""
version = macos_product_version()
if not host.is_macos or not version:
return None
return {"passed": True, "macos_product_version": version}
def persist_macos_load_probe(install_dir: Path, host: HostInfo) -> bool:
"""Remember a reuse-path probe that just passed, so the next update can skip it.
Without this the skip is unreachable for everyone it was not born with: a marker
written before the record existed, an install whose first probe timed out, and
every install after a macOS point release all probe again on EVERY later update
and never stop, since nothing writes the evidence outside a fresh install.
Re-recording the runtime files is safe HERE and nowhere else: the minos check, the
digests the marker did carry and dyld itself all passed on these exact bytes
moments ago, which is the same evidence the install path records on.
The record covers the ROOT copies too, so only a caller that has started every one
of them may write it. That is _existing_install_runs alone; the two reuse fast paths
probe build/bin and never reach a root wrapper, which can be a separate file rather
than a symlink.
"""
record = macos_load_probe_record(host)
if record is None:
return False
try:
marker = load_prebuilt_metadata(install_dir)
if not isinstance(marker, dict) or not marker:
return False
patch: "dict[str, Any]" = {MACOS_LOAD_PROBE_KEY: record}
recorded = marker.get("runtime_files")
if not (
isinstance(recorded, dict)
and recorded
and all(isinstance(e, dict) and e.get("sha256") for e in recorded.values())
):
kind = "macos-arm64" if str(host.machine or "").lower() == "arm64" else "macos-x64"
records = runtime_file_records(
install_dir, host, runtime_patterns_for_install_kind(kind)
)
if not records:
return False
patch["runtime_files"] = records
marker.update(patch)
return _write_marker(install_dir / "UNSLOTH_PREBUILT_INFO.json", marker)
except Exception: # noqa: BLE001
return False
def _macos_load_record_is_current(marker: "dict[str, Any] | None", host: HostInfo) -> bool:
"""Whether the recorded dyld probe still holds, only with positive MACOS_LOAD_PROBE_KEY evidence
of a pass on this exact macOS product version; callers must also check _runtime_files_match,
since size-only entries miss same-size rewrites."""
if not host.is_macos or host.macos_version is None or prebuilt_full_check_requested():
return False
marker = marker or {}
recorded = marker.get("runtime_files")
if not isinstance(recorded, dict) or not recorded:
return False
if not all(isinstance(entry, dict) and entry.get("sha256") for entry in recorded.values()):
return False
probe = marker.get(MACOS_LOAD_PROBE_KEY)
if not isinstance(probe, dict) or probe.get("passed") is not True:
return False
version = macos_product_version()
if not version or probe.get("macos_product_version") != version:
return False
return marker.get("host_profile") == host_profile(host)
def _runtime_files_match(install_dir: Path, host: HostInfo, marker: "dict[str, Any]") -> bool:
"""Whether every recorded runtime file is still the file that was installed.
Sizes for everything, digests for the binaries that carry one (see
runtime_file_records for why the split). mtime_ns is recorded but deliberately NOT
compared: a restore from backup, an rsync or a container layer rewrites it without
changing a byte, and the answer to a mismatch here is a 200-400 MB re-download.
Fails CLOSED, unlike the payload scans: this is the evidence that replaces actually
starting the binaries, so an empty or unreadable record is not proof of anything.
A marker written before this existed has none, and takes the full path once.
"""
recorded = marker.get("runtime_files")
if not isinstance(recorded, dict) or not recorded:
return False
for relative, expected in recorded.items():
if not isinstance(expected, dict):
return False
candidate = install_dir / relative
try:
info = candidate.stat()
if info.st_size != expected.get("size"):
log(f"kept install rejected: {relative} is {info.st_size} bytes")
return False
digest = expected.get("sha256")
if digest is not None and sha256_file(candidate) != digest:
log(f"kept install rejected: {relative} does not match the recorded digest")
return False
except OSError as exc:
log(f"kept install rejected: {relative} is unreadable ({exc})")
return False
return True
def _marker_install_fingerprint(marker: "dict[str, Any]") -> "str | None":
"""Recompute the fingerprint from the marker's own recorded fields.
Self-consistency, not a comparison against a fresh plan: it proves the marker was
written whole by this installer and has not been hand-edited or truncated, which is
what lets the rest of the no-network check trust the fields it reads. A marker
missing any input -- every pre-PR one, which never recorded runtime_sha256 -- gets
None and takes the full path once.
"""
keys = (
"published_repo",
"release_tag",
"asset",
"asset_sha256",
"source",
"source_asset",
"source_sha256",
"runtime_line",
"runtime_asset",
"runtime_sha256",
"bundle_profile",
"coverage_class",
)
if "runtime_sha256" not in marker or "tag" not in marker:
return None
payload = {key: marker.get(key) for key in keys}
payload["upstream_tag"] = marker.get("tag")
return hashlib.sha256(
json.dumps(payload, sort_keys = True, separators = (",", ":")).encode("utf-8")
).hexdigest()
_PREBUILT_FULL_CHECK_ENV = "UNSLOTH_PREBUILT_FULL_CHECK"
def prebuilt_full_check_requested() -> bool:
"""The escape hatch for the no-network re-check. A skip nobody can turn off is a bug
nobody can work around."""
return os.environ.get(_PREBUILT_FULL_CHECK_ENV, "").strip().lower() in (
"1",
"true",
"yes",
"on",
)
def _newest_release_tag_from_releases(repo: str, releases: "Iterable[Any]") -> "str | None":
"""The newest published release tag by published_at, the ordering _select uses.
Mirrors iter_release_payloads_by_time's sort and its repo-aware selectability rule,
hence the repo: answering differently here reports the freshly installed build as
stale and reinstalls it on every update run.
"""
published = [
release
for release in releases
if isinstance(release, dict)
and release_is_selectable(repo, release)
and isinstance(release.get("tag_name"), str)
and release.get("tag_name")
]
if not published:
return None
return max(published, key = release_time_sort_key)["tag_name"]
def _api_newest_release_tag(repo: str) -> "str | None":
"""The API's notion of latest, over the pages the full selector scans.
published_at ordering, so a release drafted early and published late can sit on a
later page of a repository with more than a hundred releases; one page would report
an older marker current where iter_release_payloads_by_time would install the newer one.
"""
try:
return _newest_release_tag_from_releases(
repo, github_releases(repo, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES)
)
except Exception as exc: # noqa: BLE001 - unreachable is a reason to do the work
log(f"could not resolve the latest release from the GitHub API ({exc})")
return None
def _api_newest_release_tag_for_upstream(
repo: str, upstream_tag: str, recorded_release: str
) -> "str | None":
"""The newest published fork release packaging *upstream_tag*.
The fork names its releases after the build (bNNNN, bNNNN-mix-<sha>); the release
the marker records is known to package it whatever it is called, so it is a
candidate too, and the newest of them all is what the selector would install.
"""
try:
releases = github_releases(repo, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES)
except Exception as exc: # noqa: BLE001 - unreachable is a reason to do the work
log(f"could not list the releases packaging {upstream_tag} ({exc})")
return None
matching = [
release
for release in releases
if isinstance(release, dict)
and isinstance(release.get("tag_name"), str)
and (
release["tag_name"] == upstream_tag
or release["tag_name"].startswith(upstream_tag + "-")
or release["tag_name"] == recorded_release
)
]
return _newest_release_tag_from_releases(repo, matching)
def _memoized_api_newest_release_tag(repo: str) -> "str | None":
"""The API's notion of latest, but only when this process already has the payload.
_METADATA_MEMO is populated only inside _cached_metadata_fetches (--resolve-backends
planning several options from one snapshot); on the install path it is None and this
answers None without touching the network. Free when it is there, and there is no
reason to prefer the cheaper, staler answer over one already in hand.
"""
memo = _METADATA_MEMO
if not memo:
return None
prefix = f"https://api.github.com/repos/{repo}/releases?"
releases: list[Any] = []
for key, payload in memo.items():
if not (isinstance(key, tuple) and len(key) == 2 and key[0] == "json"):
continue
url = key[1]
if isinstance(url, str) and url.startswith(prefix) and isinstance(payload, list):
releases.extend(payload)
return _newest_release_tag_from_releases(repo, releases) if releases else None
def _runtime_preference_moved(marker: "dict[str, Any]", host: HostInfo) -> bool:
"""Whether torch now prefers another CUDA runtime line than the one installed.
The selectors order the CUDA bundles around detect_torch_cuda_runtime_preference
(torch.version.cuda), which the host profile does not see: a torch upgrade from a
CUDA 12 to a CUDA 13 build leaves the GPU and driver as they were. A marker recorded
under the old preference is then not what this run would choose. Only a CUDA install
on a CUDA host is asked; a preference torch cannot state keeps the fast path.
"""
if not (host.has_usable_nvidia and (host.is_linux or host.is_windows)):
return False
recorded_line = marker.get("runtime_line")
if not isinstance(recorded_line, str) or not recorded_line.startswith("cuda"):
return False
preferred = detect_torch_cuda_runtime_preference(host).runtime_line
if "torch_runtime_preference" in marker:
recorded_preference = marker.get("torch_runtime_preference")
if preferred == recorded_preference:
return False
if not preferred:
fallback = _fallback_runtime_line(host)
if fallback is None or fallback == recorded_line:
return False
log(
f"kept install rejected: torch no longer states a CUDA preference and the "
f"host's runtime order starts at {fallback}, the install is {recorded_line}"
)
return True
if not _runtime_line_selectable(host, preferred):
return False
log(
f"kept install rejected: torch now prefers the {preferred} runtime line, "
f"the install was chosen under {recorded_preference or 'no preference'}"
)
return True
if not preferred or preferred == recorded_line:
return False
# A preference the selectors cannot act on must be ignored, or every update is full.
if not _runtime_line_selectable(host, preferred):
return False
log(
f"kept install rejected: torch now prefers the {preferred} runtime line, "
f"the install is {recorded_line}"
)
return True
def _fallback_runtime_line(host: HostInfo) -> "str | None":
"""None if undecidable; mirrors linux_cuda_choice_from_release and windows_cuda_attempts without
a release (Linux: detected lines the driver can run, in detection order; Windows:
driver-compatible lines narrowed to detected, or all if none); Blackwell routes by its own
rule before any preference."""
try:
if _host_is_blackwell(host):
return None
if host.is_linux:
detected = detected_linux_runtime_lines()[0]
compatible = compatible_linux_runtime_lines(host)
order = [line for line in detected if line in compatible]
else:
detected = detected_windows_runtime_lines()[0]
compatible = compatible_windows_runtime_lines(host)
order = [line for line in compatible if line in detected] or list(compatible)
except Exception: # noqa: BLE001 - an unreadable host answers "cannot tell"
return None
return order[0] if order else None
def _torch_runtime_preference_for_marker(host: "HostInfo | None") -> "str | None":
"""Returns the torch CUDA preference to record, or None off CUDA Linux or Windows hosts, where
the selectors never read it; never raises, since the marker is metadata rather than the
install."""
if host is None:
return None
try:
if not (host.has_usable_nvidia and (host.is_linux or host.is_windows)):
return None
return detect_torch_cuda_runtime_preference(host).runtime_line
except Exception: # noqa: BLE001 - metadata only
return None
def _runtime_line_selectable(host: HostInfo, line: str) -> bool:
"""Whether the CUDA selectors could pick *line* here. Linux filters on the runtime being
on disk and the driver running it; the Windows selector only orders by detected DLLs (a
bundle carries its own runtime), so every driver-compatible line is selectable there."""
try:
if host.is_linux:
detected = detected_linux_runtime_lines()[0]
return line in detected and line in compatible_linux_runtime_lines(host)
return line in compatible_windows_runtime_lines(host)
except Exception: # noqa: BLE001 - an unreadable host answers "cannot tell", which is not movement
return False
def _expected_release_tag_without_plan(
marker: "dict[str, Any]",
llama_tag: str,
published_repo: str,
published_release_tag: str,
host: "HostInfo | None" = None,
) -> "str | None":
"""The release this run would install, worked out without listing anything.
None means "cannot say", and every caller treats that as a reason to take the full
path. Three shapes:
* a pinned UNSLOTH_LLAMA_RELEASE_TAG names the answer outright;
* an upstream bNNNN pin: the marker must record that upstream tag, and on the
fork the newest published release packaging that build is the answer (the fork
can republish a build, b9596-mix-aaa then b9596-mix-bbb, and the selector takes
the newest, as does test_llama_cpp_freshness), so this lists the releases; on the
upstream repo the release tag is the upstream tag itself;
* "latest" costs ONE HEAD on github.com/<repo>/releases/latest -- no
api.github.com call, so no rate limit, and no manifest or checksum download.
KNOWN AND ACCEPTED LAG, for "latest" only. The HEAD follows /releases/latest, which
GitHub resolves by make_latest / created_at. _select reads the same pointer through
the download host, but falls back to the API listing, ordered by published_at, when
that host is unavailable. The two disagree when a release is created before, but
published after, another -- e.g. a re-published or back-dated release.
In that window this check can report "current" for an install _select would have
moved off. The consequence is one deferred update, never a wrong install: the very
next run whose pointer has caught up does the move, and every OTHER guard here
(fingerprint, host profile, payload) still holds. Paying an api.github.com call on
every update to close it would reintroduce the rate limit this path exists to avoid.
Two things narrow it anyway, below: an explicit request for the API path gets the
API's answer, and a payload this process already fetched is compared for free.
A SECOND DEFERRAL of the same class, in the macOS walk-back rather than the pointer.
iter_resolved_published_releases skips a release whose checksum index it cannot fetch
and logs it, and those skips are invisible to the planner, so a walk-back records only
that the NEWEST release was unusable. If a middle release was passed over for a
transient fetch failure rather than for its OS floor, walk_back_stands keeps answering
"current" for the older install while the newest is unchanged, and the middle one is
not reconsidered until something newer publishes. The pre-PR path re-planned every
update and would have taken it on the next run. Same consequence as above and left
alone for the same reason: both releases are compatible, so this is one deferred
update and never a wrong install, and it clears itself on the next publish. Suppressing
the walk-back whenever any release was skipped transiently is the fix if this is ever
worth closing; it costs a flag threaded out of the iterator.
"""
pinned = (published_release_tag or "").strip()
requested = normalized_requested_llama_tag(llama_tag)
if pinned:
if requested != "latest":
recorded_upstream = marker.get("tag")
if not isinstance(recorded_upstream, str) or recorded_upstream != requested:
return None
return pinned
repo = published_repo or DEFAULT_PUBLISHED_REPO
if requested != "latest":
recorded_upstream = marker.get("tag")
if not isinstance(recorded_upstream, str) or recorded_upstream != requested:
return None
recorded_release = marker.get("release_tag")
if not isinstance(recorded_release, str) or not recorded_release:
return None
if repo == UPSTREAM_REPO:
return recorded_release
if repo != DEFAULT_PUBLISHED_REPO and is_release_tag_like(requested):
return requested
# The fork can package one build more than once, so a pinned build still asks the API.
return _api_newest_release_tag_for_upstream(repo, requested, recorded_release)
if host is not None:
pinned_macos = pinned_macos_release_tag(host, repo)
if pinned_macos is not None:
return pinned_macos
if not _download_host_resolve_enabled() or repo != DEFAULT_PUBLISHED_REPO:
return _api_newest_release_tag(repo)
try:
resolved = _download_host_latest_release_tag(repo)
except Exception as exc: # noqa: BLE001 - unreachable is a reason to do the work
log(f"could not resolve the latest release without the API ({exc})")
return None
if not resolved:
return None
cached = _memoized_api_newest_release_tag(repo)
if cached is not None and cached != resolved:
log(
f"/releases/latest points at {resolved} but the newest published release in "
f"{repo} is {cached}; using the published_at answer this run already has"
)
return cached
return resolved
def _diffusion_visual_server_missing_for_marker(
install_dir: Path, host: HostInfo, marker: "dict[str, Any]"
) -> bool:
"""The marker-only twin of diffusion_visual_server_backfill_needed.
An install made before the visual-server entered the copy allowlist matches on tag
and is missing the binary, so a tag-match skip never backfills it and DiffusionGemma
fails with "runner not found". The plan-based check reads the chosen bundle's
pattern list; here the marker's own recorded install kind stands in for it. Gated to
the fork ("published") bundles that carry it, so upstream installs -- which never
ship it -- cannot thrash on repeated updates.
"""
if marker.get("source") != "published":
return False
backend = marker_backend(marker)
platform_prefix = "windows-" if host.is_windows else "macos-" if host.is_macos else "linux-"
kinds = [
kind for kind in install_kinds_for_backend(backend) if kind.startswith(platform_prefix)
]
if not kinds:
return False
name = "llama-diffusion-gemma-visual-server" + (".exe" if host.is_windows else "")
if not any(
name in runtime_patterns_for_install_kind(kind, marker.get("source")) for kind in kinds
):
return False
for candidate in (
install_dir / name,
install_dir / "build" / "bin" / name,
install_dir / "build" / "bin" / "Release" / name,
):
if candidate.is_file():
return False
return True
def _marker_backend_fits_host(marker: "dict[str, Any]", host: HostInfo) -> bool:
"""Whether the marker's own backend/request pair is one this platform can hold.
Two assertions, both cheap and both about the MARKER rather than the disk:
* self-consistency. A marker naming a concrete request must name the same backend,
or it was not written by this installer -- and the rest of this check reads those
two fields to decide it need do no work. Except when this installer recorded the
disagreement on purpose (#11143): then the disagreement IS the record. Without
that flag it is still a marker from somewhere else.
* platform fit. backend_for_install_kind maps kinds to backends; a backend with no
kind on this platform (a "cuda" marker on macOS, a copied install directory)
cannot describe a bundle this run would produce, and _kept_install_payload_is_healthy
answers True vacuously for it because there is no shared payload to require.
"""
recorded_backend = marker_backend(marker)
if recorded_backend is None:
return False
recorded_request = marker.get("backend_request")
if (
isinstance(recorded_request, str)
and recorded_request not in ("", "auto")
and recorded_request != recorded_backend
and not marker_records_unsatisfied_backend_request(marker)
):
return False
platform_prefix = "windows-" if host.is_windows else "macos-" if host.is_macos else "linux-"
return any(
kind.startswith(platform_prefix) for kind in install_kinds_for_backend(recorded_backend)
)
def existing_install_current_without_plan(
install_dir: Path,
*,
llama_tag: str,
published_repo: str,
published_release_tag: str,
backend_request: str,
force_cpu: bool,
override_has_rocm: bool = False,
override_rocm_gfx: str | None = None,
route: "BackendRoute | None" = None,
backend_request_mandatory: bool = False,
) -> bool:
"""Whether the install on disk is already the one this run would produce.
Runs BEFORE the release listing, the manifest fetch and the checksum download, so a
no-op update costs one HEAD instead of the whole selection plus a re-validation that
starts llama-server and loads the CUDA runtime -- 13-63 s on macOS, ~5 s on Windows,
every update, to arrive back where it started.
Every check is on-disk evidence, a local hardware probe, or a tag comparison. Nothing
here reasons about what a bundle SHOULD contain: that is existing_install_matches_choice's
job, and it needs the plan. This one asserts that the recorded install is intact, that
the HOST is still the host it was chosen for, and that the release it names is the
release this run would ask for.
The host half is not optional. The recorded request is "auto" on every automatic
install, so on its own it says a CPU bundle is still right after a GPU is added, a
CUDA bundle is still right after the card or the driver is gone, and a coverage
bundle is still right for a card outside its supported_sms -- and since the tag does
not move when hardware does, that verdict would stand until the fork published a new
release. So the host is re-derived here through route_backend_request, the same
no-network routing _select uses (identical overrides, identical cpu mechanism), and
compared against the profile the install recorded. A marker with no host_profile --
every one written before this existed -- cannot answer and takes the full path.
*backend_request_mandatory* says the request was named by THIS run rather than read back
off the marker; it only matters for a request the install could not honour, see below.
See _expected_release_tag_without_plan for the one thing this deliberately does NOT
close: the documented "latest" pointer lag.
"""
if prebuilt_full_check_requested():
return False
marker = load_prebuilt_metadata(install_dir)
if not marker:
return False
# (1) the marker describes an install this run would have made.
if not isinstance(marker.get("install_fingerprint"), str):
return False
recorded_request = marker.get("backend_request")
if not isinstance(recorded_request, str) or recorded_request != backend_request:
return False
# An unsatisfied request is retried only once something moved, else every update pays the full
# listing; judge a marker-read install as the automatic one it is.
unsatisfied = marker_records_unsatisfied_backend_request(marker)
if unsatisfied and backend_request_mandatory:
return False
if bool(marker.get("force_cpu")) != bool(force_cpu):
return False
if marker.get("prebuilt_fallback_used") is True:
log("kept install rejected: it is a fallback bundle; the preferred one is retried")
return False
# (2) the hardware. Local probes only, and the caller's route, so they run once per update.
if unsatisfied and (route is None or route.backend != "auto"):
route = route_backend_request(
backend = "auto",
published_repo = published_repo,
published_release_tag = published_release_tag,
override_has_rocm = override_has_rocm,
override_rocm_gfx = override_rocm_gfx,
cpu_mechanism = force_cpu,
)
if route is None:
route = route_backend_request(
backend = backend_request,
published_repo = published_repo,
published_release_tag = published_release_tag,
override_has_rocm = override_has_rocm,
override_rocm_gfx = override_rocm_gfx,
cpu_mechanism = force_cpu,
)
host = route.host
if (marker.get("published_repo") or "") != (route.published_repo or DEFAULT_PUBLISHED_REPO):
return False
if not _marker_backend_fits_host(marker, host):
return False
recorded_profile = marker.get("host_profile")
if not isinstance(recorded_profile, dict):
return False
if recorded_profile != host_profile(host):
log("kept install rejected: this host no longer matches the one it was installed for")
return False
if not _kept_install_covers_host(marker, host):
log("kept install rejected: the bundle no longer covers this card or driver")
return False
if _runtime_preference_moved(marker, host):
return False
# (3) the release this run would ask for is the release that is installed.
expected_release = _expected_release_tag_without_plan(
marker, llama_tag, route.published_repo, route.published_release_tag, host = host
)
if not _release_expectation_met(
marker,
expected_release,
host,
pinned = bool((route.published_release_tag or "").strip()),
):
return False
# (4) the marker was written whole by this installer, so its fields can be trusted.
if _marker_install_fingerprint(marker) != marker.get("install_fingerprint"):
return False
# (5) the tree is the shape the marker's backend implies, and is executable.
if not _install_tree_is_usable(install_dir, host):
return False
if not _kept_install_payload_is_healthy(install_dir, host):
return False
extension = ".exe" if host.is_windows else ""
binaries = [
install_runtime_dir(install_dir, host) / f"llama-{name}{extension}"
for name in ("server", "quantize")
]
# _damaged_entrypoint also covers the install root's copies, which _find_llama_server_binary
# reaches first.
if _damaged_entrypoint(install_dir, host) is not None:
return False
# (6) the bytes are the ones installed; before the preflight, whose probe skip trusts them.
if not _runtime_files_match(install_dir, host, marker):
return False
try:
preflight_linux_installed_binaries(binaries, install_dir, host)
preflight_macos_installed_binaries(
binaries,
install_dir,
host,
load_probe = not _macos_load_record_is_current(marker, host),
)
except Exception: # noqa: BLE001
return False
if _diffusion_visual_server_missing_for_marker(install_dir, host, marker):
return False
# "already matches" is what setup.sh and setup.ps1 grep for to report the install up to date.
log(
"existing llama.cpp install already matches selected release "
f"{expected_release} upstream_tag={marker.get('tag')}; skipping download and install"
)
return True
def _release_expectation_met(
marker: "dict[str, Any]",
expected_release: "str | None",
host: HostInfo,
*,
pinned: bool = False,
) -> bool:
"""Whether the release this run would ask for is the one installed.
On a Mac below the newest bundle's OS floor the planner walks back to an older
release and records the one it skipped and the host version that skipped it
(prebuilt_core.WalkBack). While the newest published release is still that one and
the host is still that macOS version, the walk-back stands and the install is
current; a newer release or an OS upgrade takes the full path, which re-decides it.
*pinned* is the caller naming a release explicitly, and it disables that: the walk-back
explains why AUTOMATIC selection settled on an older release, and an explicit request is
not automatic selection. Without this, a Mac holding b9998 after walking back from b9999
answers "already matches selected release b9999" to a run that asked for b9999, which is
both a release the user did not get and a sentence that is not true. The full path
disables older-release fallback for a pinned request for the same reason
(allow_older_release_fallback in _fork_manifest_release_plans), so it reports the
incompatibility instead of quietly serving a different build.
"""
if not expected_release:
return False
if expected_release == marker.get("release_tag"):
return True
if pinned:
return False
return _core.walk_back_stands(marker, host, expected_release)
def _kept_install_payload_is_healthy(install_dir: Path, host: HostInfo) -> bool:
"""Check the payload shared by install kinds allowed by the tree's marker."""
marker = load_prebuilt_metadata(install_dir)
backend = marker_backend(marker)
platform_prefix = "windows-" if host.is_windows else "macos-" if host.is_macos else "linux-"
kinds = sorted(
kind for kind in install_kinds_for_backend(backend) if kind.startswith(platform_prefix)
)
if not kinds:
kinds = sorted(kind for kind in INSTALL_KIND_BACKENDS if kind.startswith(platform_prefix))
if not kinds:
return True
runtime_asset = (marker or {}).get("runtime_asset")
source_label = (marker or {}).get("source")
if not isinstance(source_label, str) or not source_label:
source_label = "published" if _tree_looks_prebuilt(install_dir, host) else None
marker_tag = (marker or {}).get("tag")
shared = set.intersection(
*(
{
tuple(group)
for group in runtime_payload_health_groups(
kind,
source_label = source_label,
runtime_name = runtime_asset,
tag = marker_tag if isinstance(marker_tag, str) else None,
install_dir = install_dir,
)
}
for kind in kinds
)
)
return _runtime_payload_has(install_dir, host, [list(group) for group in sorted(shared)])
def platform_only_host() -> HostInfo:
"""A HostInfo carrying platform facts and nothing probed.
detect_host() costs over a second shelling out to nvidia-smi and friends. The
payload health checks read only the platform booleans and the marker's own
backend, never a probed GPU field, so they should not pay for that. A parity
test against detect_host() holds this to it. macos_version is derived from
platform.mac_ver() rather than probed.
"""
system = platform.system()
machine = _host_machine(system)
is_macos = system == "Darwin"
return HostInfo(
system = system,
machine = machine,
is_windows = system == "Windows",
is_linux = system == "Linux",
is_macos = is_macos,
is_x86_64 = machine in {"x86_64", "amd64"},
is_arm64 = machine in {"arm64", "aarch64"},
nvidia_smi = None,
driver_cuda_version = None,
compute_caps = [],
visible_cuda_devices = None,
has_physical_nvidia = False,
has_usable_nvidia = False,
macos_version = parse_macos_version(platform.mac_ver()[0]) if is_macos else None,
)
def installed_runtime_health(
install_dir: Path | None = None, *, host: HostInfo | None = None
) -> tuple[bool, str] | None:
"""(ok, reason) for the managed llama.cpp runtime, or None when none is installed.
Smart App Control and antivirus quarantine individual files out of a tree
that is otherwise present, so "the marker says installed" is not "the binaries
are still there". Launch preflight asks this so such a runtime is offered for
repair instead of failing later at model load.
Must stay no stricter than the setup scripts' own keep-or-reinstall decision:
a tree rejected here but kept by ``_existing_install_runs`` would be repaired,
left unchanged, and rejected again next launch, a loop with no way out. Every
check below has a counterpart there. Nothing is executed, only looked for,
since preflight is on the launch path.
"""
root = install_dir if install_dir is not None else default_managed_llama_dir()
if load_prebuilt_metadata(root) is None:
# A present but unparseable marker is a real tree: grade it rather than answer None.
if not (root / "UNSLOTH_PREBUILT_INFO.json").is_file():
return None
host = host if host is not None else platform_only_host()
runtime_dir = install_runtime_dir(root, host)
if not runtime_dir.is_dir():
return False, "llama_runtime_dir_missing"
if not _kept_install_payload_is_healthy(root, host):
return False, "llama_runtime_payload_incomplete"
# Ask for the execute bit and a regular file like the finder does: cleared bits or a
# same-named directory would otherwise read as Ready.
if _damaged_entrypoint(root, host, selected_root_only = True) is not None:
return False, "llama_runtime_binaries_missing"
return True, ""
def _damaged_entrypoint(
install_dir: Path,
host: HostInfo,
*,
selected_root_only: bool = False,
) -> Path | None:
"""The first runtime entrypoint the loader would not start, or None.
build/bin, and the install root's own copy: ``create_exec_entrypoint`` writes a
real wrapper there when it cannot make a symlink, and it can rot on its own.
``selected_root_only`` is the launch probe's rule, and the two questions differ.
The installer asks whether it can reuse this tree as it stands, and a rotten root
wrapper is worth replacing. The launch probe asks whether the runtime the backend
will actually start is broken, and ``resolve_llama_server_binary`` returns the
first USABLE candidate rather than the first that exists (``_usable_binary`` is
``is_file()`` plus ``os.access(X_OK)`` off Windows), so it walks past a wrapper
whose execute bit is gone and runs build/bin. Grading that tree damaged marked a
working runtime stale and sent it through a repair, and where repair is
unavailable it blocked the launch outright. Pinned from the other side by
``test_runtime_skips_non_executable_root_entrypoint_for_valid_build_layout``.
Never the reverse: the launch verdict stays no stricter than the keep decision,
which is what stops a tree being repaired by changing nothing and rejected again.
One owner for the question, so the launch verdict and both keep decisions cannot
answer it differently: a tree one rejects and another keeps is repaired by
changing nothing and rejected again on the next launch.
"""
runtime_dir = install_runtime_dir(install_dir, host)
ext = ".exe" if host.is_windows else ""
for name in ("llama-server", "llama-quantize"):
binary = runtime_dir / f"{name}{ext}"
if not _entrypoint_is_runnable(binary, host):
return binary
root_binary = install_dir / f"{name}{ext}"
present = (
_discovery_would_select(root_binary, host)
if selected_root_only
else root_binary.exists()
)
if present and not _entrypoint_is_runnable(root_binary, host):
return root_binary
return None
def _discovery_would_select(binary: Path, host: HostInfo) -> bool:
"""Whether the backend's resolver would stop at this candidate.
Mirrors ``_usable_binary`` in studio/backend/utils/llama_cpp_path_settings.py,
which is what decides the question at runtime. Kept in step with it deliberately:
a candidate the resolver walks past cannot break a launch, so grading it would
repair a runtime that works.
"""
try:
if not binary.is_file():
return False
except OSError:
return False
return host.is_windows or os.access(binary, os.X_OK)
def _entrypoint_is_runnable(binary: Path, host: HostInfo) -> bool:
"""Whether a runtime entrypoint is a file the loader would start.
Shared with ``_existing_install_runs`` so the keep-or-reinstall decision and
the launch-time verdict cannot disagree: a tree this rejects but that one
keeps would be repaired, left unchanged and rejected again next launch.
Empty is not runnable, whatever its mode bits say. A truncated entrypoint keeps
its execute bit, so os.access answered true while the exec of it dies on ENOEXEC,
which is what _binary_image_runs sees over in the keep decision.
"""
if not _is_nonempty_file(binary):
return False
return True if host.is_windows else os.access(binary, os.X_OK)
# SIGKILL is absent: that is an OOM, not a broken image.
_BROKEN_IMAGE_SIGNALS = frozenset(
{signal.SIGSEGV, signal.SIGILL, signal.SIGFPE}
| ({signal.SIGBUS} if hasattr(signal, "SIGBUS") else set())
)
_ERROR_BAD_EXE_FORMAT = 193
# A Windows loader failure is not a signal: it exits the NTSTATUS (0xC0000135) positive.
_NTSTATUS_FAILURE_FLOOR = 0xC0000000
def _binary_image_runs(
path: Path,
install_dir: Path,
host: HostInfo,
runtime_line: str | None = None,
) -> bool:
"""Whether the OS will actually start ``path``. ``os.access`` asks "may I execute this", not
"is this an executable": a truncated file keeps its mode bits and ``ldd`` on a non-ELF only
says it is not a dynamic executable, while execve raises ENOEXEC (measured on Linux, for
zero-byte and non-ELF alike). The exit code is NOT the verdict: llama-quantize answers
``--version`` by printing its table and exiting non-zero, the same reason
``macos_dyld_load_issues`` reads output. So a timeout, a failed spawn or an ordinary
non-zero exit all read as healthy, since a false positive spends a source build on a
working install."""
try:
result = run_capture(
[str(path), "--version"],
timeout = 60,
env = binary_env(path, install_dir, host, runtime_line = runtime_line),
)
except OSError as exc:
if exc.errno == errno.ENOEXEC or getattr(exc, "winerror", None) == _ERROR_BAD_EXE_FORMAT:
log(f"kept install rejected: {path.name} is not an executable image ({exc})")
return False
return True
except Exception:
return True
if result.returncode < 0 and -result.returncode in _BROKEN_IMAGE_SIGNALS:
log(f"kept install rejected: {path.name} died on signal {-result.returncode}")
return False
if result.returncode >= _NTSTATUS_FAILURE_FLOOR:
log(f"kept install rejected: {path.name} exited 0x{result.returncode:08X} (loader failure)")
return False
return True
def _kept_install_covers_host(marker: "dict[str, Any] | None", host: HostInfo) -> bool:
"""Whether the bundle's recorded GPU coverage still includes this host's GPU.
A same-vendor card swap or a driver downgrade passes the vendor checks, and `--version`
runs no kernels, so the recorded supported_sms / mapped_targets / runtime_line are the
only record of what the bundle was built for. A marker without them (CPU, Vulkan,
older) cannot tell and passes, as does a host whose driver or SMs are unknown.
"""
marker = marker or {}
backend = marker_backend(marker)
if backend == "cuda":
if _tegra_blocks_arm64_cuda(host):
return False
line = marker.get("runtime_line")
if isinstance(line, str) and line.startswith("cuda") and host.driver_cuda_version:
lines = (
compatible_windows_runtime_lines(host)
if host.is_windows
else compatible_linux_runtime_lines(host)
)
if line not in lines:
return False
supported = set(normalize_compute_caps(marker.get("supported_sms") or []))
host_sms = normalize_compute_caps(host.compute_caps or host.physical_compute_caps or [])
return not supported or not host_sms or all(sm in supported for sm in host_sms)
if backend == "rocm":
mapped = {
str(t).strip().lower() for t in marker.get("mapped_targets") or [] if str(t).strip()
}
gfx = (host.rocm_gfx_target or "").strip().lower()
family = str(marker.get("gfx_target") or "").strip().lower()
return not mapped or not gfx or gfx in mapped or gfx == family
return True
def _existing_install_runs(install_dir: Path, host: HostInfo) -> bool:
"""Check whether the setup scripts could reuse and run this install."""
if not _install_tree_is_usable(install_dir, host):
return False
if not _kept_install_payload_is_healthy(install_dir, host):
return False
marker = load_prebuilt_metadata(install_dir) or {}
recorded_runtime_line = marker.get("runtime_line")
if not isinstance(recorded_runtime_line, str):
recorded_runtime_line = None
runtime_dir = install_runtime_dir(install_dir, host)
ext = ".exe" if host.is_windows else ""
binaries = [runtime_dir / f"llama-{name}{ext}" for name in ("server", "quantize")]
if _damaged_entrypoint(install_dir, host) is not None:
return False
recorded_bytes_intact = _macos_load_record_is_current(marker, host) and _runtime_files_match(
install_dir, host, marker
)
try:
preflight_linux_installed_binaries(binaries, install_dir, host)
probed = preflight_macos_installed_binaries(
binaries, install_dir, host, load_probe = not recorded_bytes_intact
)
except Exception:
return False
if recorded_bytes_intact:
return True
# Root copies first: _find_llama_server_binary reaches them first.
probes: list[Path] = []
seen: set[Path] = set()
for binary in [install_dir / p.name for p in binaries] + binaries:
if not binary.exists():
continue
try:
key = binary.resolve()
except OSError:
key = binary
if key in seen:
continue
seen.add(key)
probes.append(binary)
if not all(
_binary_image_runs(binary, install_dir, host, recorded_runtime_line) for binary in probes
):
return False
# Record last, or a corrupt root wrapper is blessed and never started on later runs.
if probed:
persist_macos_load_probe(install_dir, host)
return True
def reusable_existing_install(install_dir: Path, host: HostInfo) -> bool:
"""Reached only after the prebuilt path failed; the old both-entrypoints-executable test missed
a quarantine that removed a library, so a tree that cannot load a model is not kept;
markerless source builds, which ship no prebuilt payload, keep the old test."""
if not (install_dir / "UNSLOTH_PREBUILT_INFO.json").is_file():
return True
# #12842: below the floor a CUDA prebuilt still answers --version but loads no kernel.
marker = load_prebuilt_metadata(install_dir) or {}
if marker_backend(marker) == "cuda" and (
driver_below_cuda_prebuilt_floor(host) or _tegra_blocks_arm64_cuda(host)
):
return False
return _existing_install_runs(install_dir, host)
def existing_install_matches_choice(
install_dir: Path,
host: HostInfo,
*,
llama_tag: str,
release_tag: str,
choice: AssetChoice,
approved_checksums: ApprovedReleaseChecksums,
) -> bool:
if not install_dir.exists():
return False
metadata = load_prebuilt_metadata(install_dir)
if metadata is None:
return False
try:
confirm_install_tree(install_dir, host)
except Exception:
return False
if not runtime_payload_is_healthy(install_dir, host, choice):
return False
# Same startable test as installed_runtime_health, else the repair downloads nothing and the
# tree is rejected again on next launch.
if _damaged_entrypoint(install_dir, host) is not None:
return False
runtime_dir = install_runtime_dir(install_dir, host)
ext = ".exe" if host.is_windows else ""
for binary in ("llama-server", "llama-quantize"):
try:
if (runtime_dir / f"{binary}{ext}").stat().st_size == 0:
return False
except OSError:
return False
# Windows has no image-reading preflight, so hold markers carrying the record to it.
recorded_files = metadata.get("runtime_files")
runtime_files_matched = False
if isinstance(recorded_files, dict) and recorded_files:
if not _runtime_files_match(install_dir, host, metadata):
return False
runtime_files_matched = True
elif not (host.is_linux or host.is_macos):
# Pre-record marker on Windows: start the binaries once so damaged bytes do not become the reference.
if not all(
_binary_image_runs(
runtime_dir / f"{binary}{ext}",
install_dir,
host,
metadata.get("runtime_line")
if isinstance(metadata.get("runtime_line"), str)
else None,
)
for binary in ("llama-server", "llama-quantize")
):
return False
if host.is_linux:
try:
preflight_linux_installed_binaries(
[runtime_dir / "llama-server", runtime_dir / "llama-quantize"],
install_dir,
host,
)
except Exception:
return False
elif host.is_macos:
try:
preflight_macos_installed_binaries(
[runtime_dir / "llama-server", runtime_dir / "llama-quantize"],
install_dir,
host,
load_probe = not (
runtime_files_matched and _macos_load_record_is_current(metadata, host)
),
)
except Exception:
return False
expected_fingerprint = expected_install_fingerprint(
llama_tag = llama_tag,
release_tag = release_tag,
choice = choice,
approved_checksums = approved_checksums,
)
if not expected_fingerprint:
return False
recorded_fingerprint = metadata.get("install_fingerprint")
if not isinstance(recorded_fingerprint, str) or not recorded_fingerprint:
return False
if recorded_fingerprint != expected_fingerprint:
return False
expected_pairs = {
"release_tag": release_tag,
"published_repo": approved_checksums.repo,
"tag": llama_tag,
"asset": choice.name,
"asset_sha256": choice.expected_sha256,
"source": choice.source_label,
"runtime_line": choice.runtime_line,
"bundle_profile": choice.bundle_profile,
"coverage_class": choice.coverage_class,
}
for key, expected in expected_pairs.items():
if metadata.get(key) != expected:
return False
return True
def existing_install_matches_plan(
install_dir: Path, host: HostInfo, plan: InstallReleasePlan
) -> bool:
if not plan.attempts:
return False
return existing_install_matches_choice(
install_dir,
host,
llama_tag = plan.llama_tag,
release_tag = plan.release_tag,
choice = plan.attempts[0],
approved_checksums = plan.approved_checksums,
)
_LISTED_VULKAN_DEVICE_RE = re.compile(r"^\s*Vulkan\d+:", re.MULTILINE)
def bundle_lists_vulkan_device(server_path: Path, install_dir: Path, host: HostInfo) -> bool:
"""Whether llama.cpp itself, from the installed bundle, enumerates a Vulkan GPU."""
try:
result = subprocess.run(
[str(server_path), "--list-devices"],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
# Device init compiles every shader pipeline, over a minute on a loaded host.
timeout = 180,
env = binary_env(server_path, install_dir, host),
**windows_hidden_subprocess_kwargs(),
)
except Exception as exc:
log(f"llama-server --list-devices failed: {exc}")
return False
return result.returncode == 0 and bool(_LISTED_VULKAN_DEVICE_RE.search(result.stdout or ""))
def validate_prebuilt_choice(
choice: AssetChoice,
host: HostInfo,
install_dir: Path,
work_dir: Path,
probe: ValidationModelProvider,
*,
requested_tag: str,
llama_tag: str,
release_tag: str,
approved_checksums: ApprovedReleaseChecksums,
prebuilt_fallback_used: bool,
quantized_path: Path,
force_cpu: bool = False,
llama_backend: str | None = None,
backend_request: str | None = None,
rocm_gfx: str | None = None,
walk_back: "_core.WalkBack | None" = None,
) -> tuple[Path, Path]:
source_repo, source_ref, source_archive, exact_source = preferred_source_archive(
approved_checksums, llama_tag
)
# For a mix source the merge commit lives only in the release asset.
asset_url = exact_source_asset_url(
approved_checksums, source_repo, source_archive, exact_source, release_tag
)
if exact_source:
log(f"hydrating exact llama.cpp source for {source_repo}@{source_ref} into {install_dir}")
else:
log(f"hydrating upstream llama.cpp source for {llama_tag} into {install_dir}")
hydrate_source_tree(
source_ref,
install_dir,
work_dir,
source_repo = source_repo,
expected_sha256 = source_archive.sha256 if source_archive is not None else None,
source_label = (
f"llama.cpp source tree for {source_repo}@{source_ref}"
if exact_source
else f"llama.cpp source tree for {llama_tag}"
),
exact_source = exact_source,
asset_url = asset_url,
)
log(f"overlaying prebuilt bundle {choice.name} into {install_dir}")
server_path, quantize_path = install_from_archives(choice, host, install_dir, work_dir)
# Every Windows kind: gating only Vulkan accepted a tree missing llama-common.dll.
if (
choice.install_kind in VULKAN_INSTALL_KINDS or choice.install_kind.startswith("windows-")
) and not runtime_payload_is_healthy(install_dir, host, choice):
raise PrebuiltFallback(
f"{choice.install_kind} bundle {choice.name} omitted a required runtime component"
)
preflight_linux_installed_binaries((server_path, quantize_path), install_dir, host)
macos_load_probe_passed = preflight_macos_installed_binaries(
(server_path, quantize_path), install_dir, host
)
ensure_repo_shape(install_dir)
if (
host.vulkan_device_required
and choice.install_kind in VULKAN_INSTALL_KINDS
and not bundle_lists_vulkan_device(server_path, install_dir, host)
):
# The bundle runs on CPU without a device, so validation alone would keep it.
raise PrebuiltFallback(
f"{choice.name} lists no Vulkan GPU on this host; using the CPU bundle instead"
)
write_prebuilt_metadata(
install_dir,
host = host,
requested_tag = requested_tag,
llama_tag = llama_tag,
release_tag = release_tag,
choice = choice,
approved_checksums = approved_checksums,
prebuilt_fallback_used = prebuilt_fallback_used,
force_cpu = force_cpu,
llama_backend = llama_backend,
backend_request = backend_request,
rocm_gfx = rocm_gfx,
walk_back = walk_back,
macos_load_probe_passed = macos_load_probe_passed,
)
if prebuilt_needs_functional_validation(choice):
probe_path = resolve_validation_model(probe)
validate_quantize(
quantize_path,
probe_path,
quantized_path,
install_dir,
host,
runtime_line = choice.runtime_line,
)
validate_server(
server_path,
probe_path,
host,
install_dir,
runtime_line = choice.runtime_line,
install_kind = choice.install_kind,
)
log(f"staged prebuilt validation succeeded for {choice.name}")
return server_path, quantize_path
def validate_existing_install(
install_dir: Path,
*,
install_kind: str | None = None,
host: HostInfo | None = None,
) -> None:
"""Run the staged smoke test against an already-built llama.cpp tree (#5854).
Used by setup.sh after a GPU source build when ``UNSLOTH_LLAMA_STAGED_VALIDATION``
is set. Raises ``PrebuiltFallback`` on failure so the caller can retry CPU.
"""
host = host or detect_host()
bin_dir = install_dir / "build" / "bin"
server_name = "llama-server.exe" if host.is_windows else "llama-server"
quantize_name = "llama-quantize.exe" if host.is_windows else "llama-quantize"
server_path = bin_dir / server_name
quantize_path = bin_dir / quantize_name
if not server_path.is_file():
raise PrebuiltFallback(f"llama-server not found at {server_path}")
with tempfile.TemporaryDirectory(prefix = "unsloth-llama-source-validate-") as tmp:
work_dir = Path(tmp)
probe_path = work_dir / "stories260K.gguf"
quantized_path = work_dir / "stories260K-q4.gguf"
download_validation_model(probe_path, validation_model_cache_path(install_dir))
if quantize_path.is_file():
validate_quantize(
quantize_path,
probe_path,
quantized_path,
install_dir,
host,
)
validate_server(
server_path,
probe_path,
host,
install_dir,
install_kind = install_kind,
)
log(f"staged source-build validation succeeded for {install_dir}")
def validate_prebuilt_attempts(
attempts: Iterable[AssetChoice],
host: HostInfo,
install_dir: Path,
work_dir: Path,
probe: ValidationModelProvider,
*,
requested_tag: str,
llama_tag: str,
release_tag: str,
approved_checksums: ApprovedReleaseChecksums,
initial_fallback_used: bool = False,
existing_install_dir: Path | None = None,
force_cpu: bool = False,
llama_backend: str | None = None,
backend_request: str | None = None,
rocm_gfx: str | None = None,
walk_back: "_core.WalkBack | None" = None,
) -> tuple[AssetChoice, Path, bool]:
attempt_list = list(attempts)
if not attempt_list:
raise PrebuiltFallback("no prebuilt bundle attempts were available")
# Resolve the probe up front: a probe download failure inside the per-candidate handler would
# read as a bad bundle and demote the pick to CPU.
if any(prebuilt_needs_functional_validation(a) for a in attempt_list):
probe = resolve_validation_model(probe)
tried_fallback = initial_fallback_used
for index, attempt in enumerate(attempt_list):
if index > 0:
tried_fallback = True
log(
"retrying CUDA prebuilt "
f"{attempt.name} install_kind={attempt.install_kind} "
f"runtime_line={attempt.runtime_line} coverage_class={attempt.coverage_class}"
)
if (
existing_install_dir is not None
and existing_install_matches_choice(
existing_install_dir,
host,
llama_tag = llama_tag,
release_tag = release_tag,
choice = attempt,
approved_checksums = approved_checksums,
)
and not diffusion_visual_server_backfill_needed(existing_install_dir, host, attempt)
):
log(
"existing llama.cpp install already matches fallback candidate "
f"{attempt.name}; skipping reinstall"
)
raise ExistingInstallSatisfied(attempt, tried_fallback)
# Advisory: rejecting here would also skip the source-build fallback.
if index == 0:
low_disk = _low_disk_warning(install_dir)
if low_disk is not None:
log(low_disk)
_log_disk_space_help()
staging_dir = create_install_staging_dir(install_dir)
quantized_path = work_dir / f"stories260K-q4-{index}.gguf"
if quantized_path.exists():
quantized_path.unlink()
try:
validate_prebuilt_choice(
attempt,
host,
staging_dir,
work_dir,
probe,
requested_tag = requested_tag,
llama_tag = llama_tag,
release_tag = release_tag,
approved_checksums = approved_checksums,
prebuilt_fallback_used = tried_fallback,
quantized_path = quantized_path,
force_cpu = force_cpu,
llama_backend = llama_backend,
backend_request = backend_request,
rocm_gfx = rocm_gfx,
walk_back = walk_back,
)
except Exception as exc:
remove_tree(staging_dir)
prune_install_staging_root(install_dir)
if isinstance(exc, PrebuiltFallback):
attempt_error = exc
else:
attempt_error = PrebuiltFallback(
f"candidate attempt failed before activation for {attempt.name}: {exc}"
)
if _environment_fatal_reason(exc) or index == len(attempt_list) - 1:
if attempt_error is exc:
raise
raise attempt_error from exc
log(
"selected CUDA bundle failed before activation; trying next prebuilt fallback "
f"({textwrap.shorten(str(attempt_error), width = 200, placeholder = '...')})"
)
continue
return attempt, staging_dir, tried_fallback
raise PrebuiltFallback("no prebuilt bundle passed validation")
def llama_backend_from_env() -> str | None:
"""Read the backend preference from UNSLOTH_LLAMA_CPP_BACKEND."""
return normalize_backend_request(os.environ.get("UNSLOTH_LLAMA_CPP_BACKEND"))
def resolved_llama_backend(llama_backend: str | None = None) -> str | None:
"""The backend this run was explicitly told to use: --llama-backend, else the
env var. None when neither named one we know -- which is distinct from "auto",
a request to detect that outranks both a stored choice and the legacy
UNSLOTH_FORCE_VULKAN flag."""
return normalize_backend_request(llama_backend) or llama_backend_from_env()
def persisted_backend_request(install_dir: Path | None) -> str:
"""Return the recorded backend choice, with legacy marker compatibility."""
if install_dir is None:
return "auto"
return marker_backend_request(load_prebuilt_metadata(install_dir))
def effective_backend_request(
llama_backend: str | None = None, *, install_dir: Path | None = None
) -> tuple[str, bool]:
"""Returns (backend, mandatory) by CLI, then env, then marker precedence; explicit requests are
mandatory, while a stored choice is advisory and dropped when hardware or published bundles
no longer offer it."""
explicit = normalize_backend_request(llama_backend)
if explicit is None:
explicit = environment_backend_override(
os.environ.get("UNSLOTH_LLAMA_CPP_BACKEND"),
os.environ.get("UNSLOTH_FORCE_VULKAN"),
)
if explicit is not None:
return explicit, True
stored = persisted_backend_request(install_dir)
if not is_requestable_backend(stored):
# Written by a newer Unsloth; detection would silently rewrite it to "auto", so refuse.
raise UnknownBackendRequest(
f"this install records an unsupported llama.cpp backend choice ({stored!r}); "
"update Unsloth before replacing it"
)
return stored, False
def force_vulkan_requested(llama_backend: str | None = None) -> bool:
"""Whether this run should install the Vulkan llama.cpp prebuilt.
Triggered by ``UNSLOTH_LLAMA_CPP_BACKEND=vulkan``, legacy ``UNSLOTH_FORCE_VULKAN``,
or ``--llama-backend vulkan``. Scoped to the
llama.cpp backend; the torch/training stack installs separately and still sees
the real GPU.
"""
backend = normalize_backend_request(llama_backend)
if backend is None:
backend = environment_backend_override(
os.environ.get("UNSLOTH_LLAMA_CPP_BACKEND"),
os.environ.get("UNSLOTH_FORCE_VULKAN"),
)
return backend == "vulkan"
def _host_rocm_gfx_targets(host: HostInfo) -> list[str]:
if host.rocm_gfx_targets:
return [target.lower() for target in host.rocm_gfx_targets]
if host.rocm_gfx_target:
return [host.rocm_gfx_target.lower()]
return []
def _active_rocm_gfx_target(host: HostInfo) -> str | None:
"""The gfx HIP will run on (visible-device aware), not every physical GPU."""
if host.rocm_gfx_target:
return host.rocm_gfx_target.lower().strip()
return None
def _hip_visible_device_mask_set() -> bool:
"""Whether a HIP visible-device mask is in force for this process.
The Windows arch probe is hipinfo, itself a HIP application, so under a mask it
enumerates the VISIBLE devices, not the physical ones. Presence is the whole test: a
partial mask leaves the inventory unknowable, and an all-hiding "" / "-1" is the
strongest form of that, not an exemption, since --rocm-gfx can still supply an arch
(setup infers it from the display adapter, which no HIP mask touches) and would
auto-route a host on which the user hid every AMD GPU. Reads the same three vars as
_pick_rocm_gfx_target, so the two cannot disagree about the host."""
return any(
os.environ.get(_env) is not None
for _env in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
)
def _vulkan_visible_device_mask_set() -> bool:
"""Whether GGML_VK_VISIBLE_DEVICES is set, which can leave the Vulkan bundle with
nothing to enumerate. Presence is the whole test, as for the HIP masks: the ordinals
are Vulkan's own, so which device an index names is unknowable here."""
return os.environ.get("GGML_VK_VISIBLE_DEVICES") is not None
# The checks ggml_vk_instance_init applies before it takes a device: a Vulkan 1.2 loader,
# then a discrete or integrated GPU with 16-bit storage buffers (ggml_vk_device_is_supported).
_VULKAN_DEVICE_PROBE = r"""
import ctypes as c, json, os, sys
if sys.platform == "win32":
root = os.environ.get("SystemRoot") or r"C:\Windows"
lib = c.WinDLL(os.path.join(root, "System32", "vulkan-1.dll"))
else:
lib = c.CDLL("libvulkan.so.1")
P, U32 = c.c_void_p, c.c_uint32
def fn(name, *args):
f = getattr(lib, name)
f.argtypes, f.restype = list(args), c.c_int32
return f
class App(c.Structure):
_fields_ = [("sType", c.c_int32), ("pNext", P), ("name", c.c_char_p), ("ver", U32),
("engine", c.c_char_p), ("engineVer", U32), ("api", U32)]
class Create(c.Structure):
_fields_ = [("sType", c.c_int32), ("pNext", P), ("flags", U32), ("app", P),
("nLayers", U32), ("layers", P), ("nExts", U32), ("exts", P)]
class Driver(c.Structure):
_fields_ = [("sType", c.c_int32), ("pNext", P), ("id", c.c_int32),
("slack", c.c_char * 1024)]
class Props(c.Structure):
_fields_ = [("sType", c.c_int32), ("pNext", P), ("api", U32), ("driverVersion", U32),
("vendor", U32), ("device", U32), ("type", c.c_int32),
("name", c.c_char * 256), ("slack", c.c_uint64 * 256)]
class Vk11(c.Structure):
_fields_ = [("sType", c.c_int32), ("pNext", P), ("bits", U32 * 32)]
class Features(c.Structure):
_fields_ = [("sType", c.c_int32), ("pNext", P), ("bits", U32 * 128)]
V12 = (1 << 22) | (2 << 12)
version = U32(1 << 22)
if hasattr(lib, "vkEnumerateInstanceVersion"):
fn("vkEnumerateInstanceVersion", c.POINTER(U32))(c.byref(version))
devices = []
if version.value >= V12:
app = App(0, None, b"unsloth", 1, None, 0, version.value)
instance = P()
if fn("vkCreateInstance", P, P, P)(c.byref(Create(1, None, 0, c.addressof(app))), None, c.byref(instance)) == 0:
enum = fn("vkEnumeratePhysicalDevices", P, c.POINTER(U32), P)
count = U32(0)
enum(instance, c.byref(count), None)
handles = (P * count.value)()
enum(instance, c.byref(count), handles)
get_props = fn("vkGetPhysicalDeviceProperties2", P, P)
get_features = fn("vkGetPhysicalDeviceFeatures2", P, P)
for handle in handles[: count.value]:
driver = Driver(1000196000)
props = Props(1000059001, c.addressof(driver))
get_props(handle, c.byref(props))
fp16 = False
if props.api >= V12:
vk11 = Vk11(49)
get_features(handle, c.byref(Features(1000059000, c.addressof(vk11))))
fp16 = bool(vk11.bits[0])
devices.append({"vendor": props.vendor, "type": props.type, "api": props.api,
"driver": driver.id, "fp16": fp16,
"name": props.name.decode("utf-8", "replace")})
fn("vkDestroyInstance", P, P)(instance, None)
print(json.dumps(devices))
"""
# VK_DRIVER_ID_GOOGLE_SWIFTSHADER, _MESA_LLVMPIPE (software), _MESA_DOZEN (Vulkan over D3D12).
_VULKAN_SKIPPED_DRIVER_IDS = frozenset({10, 13, 23})
# VK_DRIVER_ID_QUALCOMM_PROPRIETARY (Snapdragon X, Windows): garbage output (ggml-org/llama.cpp#28290), so
# explicit Vulkan requests only until verified there. Mesa Turnip on Adreno is a different driver.
_VULKAN_AUTO_OPT_IN_DRIVER_IDS = frozenset({8})
class _VulkanGpuName(str):
"""A Vulkan device name that also carries its vendor and driver ids (None when unknown)."""
vendor = None
driver = None
def __new__(
cls,
name,
vendor = None,
driver = None,
):
self = super().__new__(cls, name)
self.vendor = vendor
self.driver = driver
return self
@functools.lru_cache(maxsize = 1)
def _vulkan_gpu_devices() -> tuple[str, ...]:
"""Names of the GPUs the Vulkan bundle would run on; empty when none or the probe fails.
A child process, since loading a broken driver can take its caller down with it.
UNSLOTH_VULKAN_DEVICE_PROBE=0 turns it off.
"""
if os.environ.get("UNSLOTH_VULKAN_DEVICE_PROBE", "1") == "0":
return ()
try:
result = run_capture([sys.executable, "-I", "-c", _VULKAN_DEVICE_PROBE], timeout = 30)
devices = json.loads(result.stdout.strip().splitlines()[-1])
except Exception:
return ()
if result.returncode != 0 or not isinstance(devices, list):
return ()
# Vulkan ignores the CUDA and HIP masks, so a vendor the caller hid stays hidden.
hidden_vendors = set()
if os.environ.get("CUDA_VISIBLE_DEVICES") is not None:
hidden_vendors.add(0x10DE)
if (
_hip_visible_device_mask_set()
if sys.platform == "win32"
else os.environ.get("ROCR_VISIBLE_DEVICES") is not None
):
hidden_vendors.add(0x1002)
names = []
for device in devices:
try:
if (
device["type"] in (1, 2)
and device["api"] >= (1 << 22) | (2 << 12)
and device["fp16"]
and device["driver"] not in _VULKAN_SKIPPED_DRIVER_IDS
and device["vendor"] not in hidden_vendors
):
names.append(
_VulkanGpuName(str(device["name"]), device["vendor"], device["driver"])
)
except (KeyError, TypeError):
continue
return tuple(names)
def _vulkan_auto_gpu_devices() -> tuple[str, ...]:
"""The GPUs the automatic route may pick: every accepted one except opt-in-only drivers."""
devices = _vulkan_gpu_devices()
auto = tuple(
d for d in devices if getattr(d, "driver", None) not in _VULKAN_AUTO_OPT_IN_DRIVER_IDS
)
if devices and not auto:
log(
f"Vulkan GPU found ({', '.join(dict.fromkeys(devices))}) but llama.cpp's Vulkan backend "
"is unverified there, so the CPU llama.cpp prebuilt is used. Set "
"UNSLOTH_LLAMA_CPP_BACKEND=vulkan to install the Vulkan prebuilt instead."
)
return auto
def _windows_hip_gfx_targets(published_repo: str | None) -> frozenset[str]:
"""gfx targets the Windows HIP bundle of ``published_repo`` is actually built for.
The combined floor above includes the FORK's windows-rocm bundles, and only the fork is
planned from a manifest: resolve_simple_install_release_plans() sends every other
--published-repo (ggml-org, but equally any mirror of upstream-standard assets) to
direct_upstream_release_plan(), whose AMD branch offers the upstream HIP bundle first and
Vulkan only behind it. Answering "supported" there for a fork-only arch would silently land
it on HIP instead of the Vulkan bundle that would actually run, so gate on the fork rather
than exempting one repo, mirroring that dispatch exactly, spelling included: an empty
value defaults to the fork, and a differently cased repo really does take the upstream
path and must be answered with upstream coverage."""
if (published_repo or DEFAULT_PUBLISHED_REPO) == DEFAULT_PUBLISHED_REPO:
return WINDOWS_HIP_PREBUILT_GFX_TARGETS
return UPSTREAM_WINDOWS_HIP_GFX_TARGETS
def _gfx_is_windows_hip_supported(gfx: str, published_repo: str | None = None) -> bool:
token = gfx.lower().strip()
if token in WINDOWS_ROCM_FAMILY_GFX_LABELS:
# A family label, not an arch: HIP serves it either way rather than move working members to Vulkan.
return True
return token in _windows_hip_gfx_targets(published_repo)
def _host_has_windows_hip_prebuilt_gfx(host: HostInfo, published_repo: str | None = None) -> bool:
active = _active_rocm_gfx_target(host)
if not active:
return False
return _gfx_is_windows_hip_supported(active, published_repo)
def _should_auto_vulkan_for_amd_windows(host: HostInfo, published_repo: str | None = None) -> bool:
"""True when no AMD GPU on the host reaches the Windows HIP prebuilt floor, or the
unmasked probe's own pick has no Windows HIP bundle."""
active = _active_rocm_gfx_target(host)
if not active:
return False
if not (
host.is_windows
and host.has_rocm
# PHYSICAL, not usable: Vulkan ignores CUDA_VISIBLE_DEVICES and would reach a hidden card.
and not host.has_physical_nvidia
):
return False
# Judge every PHYSICAL AMD gfx, not just the active one. The visible-device vars choose
# `active`, but the Vulkan runtime honours none of them (it enumerates through
# GGML_VK_VISIBLE_DEVICES and Vulkan ordinals), so masking down to a below-floor card
# must not route the install to Vulkan: the installed backend would happily enumerate
# the HIP-capable card the user deliberately hid, possibly one reserved for another
# workload. Under a mask, auto-fall back only when no AMD device on the box can be exposed
# to HIP; an explicit vulkan opt-in is unaffected.
#
# Under a mask the probe cannot supply that inventory at all (hipinfo sees only visible
# devices), so "no AMD GPU here reaches the floor" is unprovable and guessing wrong is
# the same reserved-card handover. An all-hiding "" / "-1" is included: a forwarded
# --rocm-gfx still reconstructs an arch there, and auto-routing would then hand Vulkan
# every AMD GPU the user hid. A mask is only ever set deliberately, and the driver-only
# single-GPU host this fallback exists for does not set one.
if _hip_visible_device_mask_set():
return False
targets = list(dict.fromkeys([*_host_rocm_gfx_targets(host), active]))
# Mixed host whose active card has no HIP bundle (gfx1010 at device 0 beside a gfx1034):
# no HIP prebuilt can serve the card HIP runs on, while Vulkan runs on both. Only when the
# active arch is the unmasked probe's own pick: a forwarded arch the probe never reported,
# or UNSLOTH_ROCM_GFX_ARCH, keeps the all-cards rule. Any NVIDIA trace also keeps it:
# following ROCm torch clears has_physical_nvidia, but Vulkan would still see that card.
if (
not os.environ.get("UNSLOTH_ROCM_GFX_ARCH")
and not host.rocm_gfx_forwarded
and not (host.nvidia_smi or host.physical_compute_caps or host.compute_caps)
and active == _pick_rocm_gfx_target(" ".join(targets))
and not _gfx_is_windows_hip_supported(active, published_repo)
):
return True
return not any(_gfx_is_windows_hip_supported(target, published_repo) for target in targets)
# 64-bit only: WOW6432Node holds 32-bit registrations windows-x64-vulkan cannot load.
_VULKAN_ICD_REGISTRY_KEYS = (r"SOFTWARE\Khronos\Vulkan\Drivers",)
_WINDOWS_VULKAN_DEVICE_CLASS_KEYS = (
_WINDOWS_DISPLAY_CLASS_KEY,
r"SYSTEM\CurrentControlSet\Control\Class\{5c4c3332-344d-483c-8739-259e934c9cc8}",
)
_WINDOWS_VULKAN_DRIVER_VALUE = "VulkanDriverName"
# Matched on the basename ("amd" names directories too) with "-" folded to "_".
_AMD_VULKAN_ICD_NEEDLES = ("radeon", "radv", "amdvlk", "amd_icd", "amd_pro", "amd_vulkan")
_AMD_VULKAN_ICD_32_BIT_NEEDLES = ("i686", "i386")
def _vulkan_glob_matches(pattern: str, name: str) -> bool:
"""The loader's four driver-filter globs, case-insensitively: "s", "s*", "*s", "*s*"."""
pattern, name = pattern.lower(), name.lower()
starts, ends = pattern.startswith("*"), pattern.endswith("*")
core = pattern[1 if starts else 0 : len(pattern) - 1 if ends else len(pattern)]
if starts and ends:
return core in name
if starts:
return name.endswith(core)
if ends:
return name.startswith(core)
return name == core
def _vulkan_loader_allows(path: str) -> bool:
"""Whether the loader's own driver filters leave this manifest loadable.
They apply to every driver the loader knows, a force list included, and match the
manifest's basename. A manifest filtered out is registered and never loaded, so
counting it hands an integrated host a Vulkan build with no AMD device.
Select is an allowlist and disable is a denylist that WINS over it, rather than being
skipped when select is set. The loader documents the order the other way round from the
way this once read it: "The values from the disable environment variable will be
considered before the enable or select environment variable", and
VK_LOADER_DRIVERS_DISABLE is "also checked before other driver environment variables
(such as VK_LOADER_DRIVERS_SELECT)" (Vulkan-Loader, LoaderInterfaceArchitecture.md).
Drivers have no VK_LOADER_LAYERS_ALLOW counterpart to name one back, so selecting
radeon* while also disabling radeon* leaves the real loader with no driver, where this
counted Radeon as usable and reported only the device-node repair.
"""
def _globs(env_name: str) -> list[str]:
value = os.environ.get(env_name) or ""
return [entry.strip() for entry in value.split(",") if entry.strip()]
name = PurePath(path).name
disable = _globs("VK_LOADER_DRIVERS_DISABLE")
if any(_vulkan_glob_matches(pattern, name) for pattern in disable):
return False
select = _globs("VK_LOADER_DRIVERS_SELECT")
return any(_vulkan_glob_matches(pattern, name) for pattern in select) if select else True
# Per call, not at import: Path.home() raises with no USERPROFILE.
def _vulkan_icd_search_dirs() -> list[Path]:
"""Lists the icd.d directories the loader searches, in its own order, from the XDG variables
(defaults only when one is unset); reading the defaults regardless misses a custom layout's
only AMD manifest."""
def _paths(var: str, default: str) -> list[Path]:
value = os.environ.get(var)
raw = value if (value or "").strip() else default
return [Path(entry) for entry in raw.split(os.pathsep) if entry.strip()]
def _home(var: str, default: str) -> list[Path]:
value = os.environ.get(var)
if (value or "").strip():
return [Path(value)]
try:
return [Path.home() / default]
except Exception:
return []
dirs: list[Path] = []
# Loader order: XDG_CONFIG_HOME, XDG_CONFIG_DIRS, then XDG_DATA_HOME, XDG_DATA_DIRS.
for base in (
*_home("XDG_CONFIG_HOME", ".config"),
*_paths("XDG_CONFIG_DIRS", "/etc/xdg"),
):
dirs.append(base / "vulkan/icd.d")
dirs.append(Path("/etc/vulkan/icd.d"))
for base in (
*_home("XDG_DATA_HOME", ".local/share"),
*_paths("XDG_DATA_DIRS", "/usr/local/share" + os.pathsep + "/usr/share"),
):
dirs.append(base / "vulkan/icd.d")
seen: set[str] = set()
unique: list[Path] = []
for directory in dirs:
key = str(directory)
if key not in seen:
seen.add(key)
unique.append(directory)
return unique
_CM_LOCATE_DEVNODE_NORMAL = 0x00000000
_CM_GETIDLIST_FILTER_PRESENT = 0x00000100
_CM_GETIDLIST_FILTER_CLASS = 0x00000200
_CM_DRP_DRIVER = 0x0000000A
_CR_SUCCESS = 0x00000000
_CR_BUFFER_SMALL = 0x0000001A
_DN_HAS_PROBLEM = 0x00000400
_CM_PROB_NEED_RESTART = 0x0000000E
# A status bit in pulStatus (cfg.h DN_LIAR), never a pulProblemNumber code.
_DN_NEED_RESTART = 0x00000100
_CM_DEVICE_LIST_ATTEMPTS = 4
def _windows_present_class_instances(class_key_path: str) -> set[str] | None:
"""Return present class instances ("0000"), or None if enumeration fails.
CM_DRP_DRIVER maps each device to "{class guid}\\NNNN"; an empty set means none
are present. Every devnode filter here is the loader's own
(windows_get_device_registry_files), but its traversal is NOT: the loader reaches
SoftwareComponents only as children of a present adapter, and the caller
enumerates that class directly, so discovery there is wider.
"""
guid = class_key_path.rsplit("\\", 1)[-1]
try:
import ctypes
from ctypes import wintypes
cfgmgr = ctypes.WinDLL("cfgmgr32.dll")
except Exception:
return None
try:
length = wintypes.ULONG(0)
flags = _CM_GETIDLIST_FILTER_CLASS | _CM_GETIDLIST_FILTER_PRESENT
# A device arriving between the two calls makes the API refuse; retry rather than drop the scan.
for _attempt in range(_CM_DEVICE_LIST_ATTEMPTS):
if (
cfgmgr.CM_Get_Device_ID_List_SizeW(ctypes.pointer(length), guid, flags)
!= _CR_SUCCESS
):
return None
buffer = ctypes.create_unicode_buffer(length.value)
listed = cfgmgr.CM_Get_Device_ID_ListW(guid, buffer, length.value, flags)
if listed == _CR_SUCCESS:
break
if listed != _CR_BUFFER_SMALL:
return None
else:
return None
device_ids = [entry for entry in buffer[: length.value].split("\0") if entry]
instances: set[str] = set()
for device_id in device_ids:
devinst = wintypes.DWORD(0)
if (
cfgmgr.CM_Locate_DevNodeW(
ctypes.pointer(devinst), device_id, _CM_LOCATE_DEVNODE_NORMAL
)
!= _CR_SUCCESS
):
continue
if not _windows_devnode_is_usable(cfgmgr, ctypes, wintypes, devinst):
continue
size = wintypes.ULONG(0)
cfgmgr.CM_Get_DevNode_Registry_PropertyW(
devinst, _CM_DRP_DRIVER, None, None, ctypes.pointer(size), 0
)
if not size.value:
continue
value = ctypes.create_unicode_buffer(size.value // ctypes.sizeof(ctypes.c_wchar) + 1)
if (
cfgmgr.CM_Get_DevNode_Registry_PropertyW(
devinst, _CM_DRP_DRIVER, None, value, ctypes.pointer(size), 0
)
!= _CR_SUCCESS
):
continue
driver = (value.value or "").strip()
if "\\" in driver:
instances.add(driver.rsplit("\\", 1)[-1])
return instances
except Exception:
return None
def _windows_devnode_is_usable(cfgmgr: Any, ctypes: Any, wintypes: Any, devinst: Any) -> bool:
"""Whether this devnode is one the Vulkan loader would read, not merely present.
A driver update registers VulkanDriverName and drops its manifest before the reboot
that binds it: until then the adapter is present, the file is on disk, and the driver
cannot load. Counting that window hands the host a Vulkan bundle and lets llama-server
fall back to CPU, which is what _amd_vulkan_icd_present exists to prevent. An
unreadable status is skipped too, since guessing wrong replaces a working ROCm install.
"""
status = wintypes.ULONG(0)
problem = wintypes.ULONG(0)
if (
cfgmgr.CM_Get_DevNode_Status(ctypes.pointer(status), ctypes.pointer(problem), devinst, 0)
!= _CR_SUCCESS
):
return False
# Checked independently of DN_HAS_PROBLEM: the restart bit can be set without a problem.
if status.value & _DN_NEED_RESTART:
return False
if not status.value & _DN_HAS_PROBLEM:
return True
return problem.value != _CM_PROB_NEED_RESTART
def _windows_device_icd_manifest_paths(winreg: Any) -> list[str]:
"""Reads Vulkan manifests only from present adapters and SoftwareComponents: a removed device
keeps its registration and files, so presence decides and unknown-presence classes are
skipped."""
paths: list[str] = []
for class_key_path in _WINDOWS_VULKAN_DEVICE_CLASS_KEYS:
present = _windows_present_class_instances(class_key_path)
if present is None:
continue
try:
with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, class_key_path) as class_key:
for index in range(winreg.QueryInfoKey(class_key)[0]):
try:
name = winreg.EnumKey(class_key, index)
if not name.isdigit() or name not in present:
continue
with winreg.OpenKey(class_key, name) as device_key:
value, kind = winreg.QueryValueEx(
device_key, _WINDOWS_VULKAN_DRIVER_VALUE
)
paths.extend(_windows_vulkan_driver_value_paths(winreg, value, kind))
except Exception:
continue
except OSError:
continue
except Exception:
continue
return paths
def _windows_vulkan_driver_value_paths(winreg: Any, value: Any, kind: Any) -> list[str]:
"""Existing absolute manifest paths from REG_SZ or REG_MULTI_SZ values; relative paths need the
device's DriverStore directory, which is unreachable here, so they are skipped."""
if kind == winreg.REG_SZ:
entries = [value]
elif kind == winreg.REG_MULTI_SZ:
entries = list(value or [])
else:
return []
found = []
for entry in entries:
if not isinstance(entry, str) or not entry.strip():
continue
entry = entry.strip()
if not os.path.isabs(entry):
continue
try:
if os.path.isfile(entry):
found.append(entry)
except OSError:
continue
return found
def _amd_vulkan_icd_manifest_paths() -> list[str]:
"""Vulkan ICD manifests registered on this host, in loader search order, read in-process from
file lists or registry keys, since vulkaninfo is absent on most Windows hosts and libvulkan
on a headless ROCm box."""
# Force lists, not hints; VK_DRIVER_FILES supersedes VK_ICD_FILENAMES rather than joining it.
for env_name in ("VK_DRIVER_FILES", "VK_ICD_FILENAMES"):
value = os.environ.get(env_name)
if not (value or "").strip():
continue
forced = []
for entry in value.split(os.pathsep):
if not entry:
continue
try:
if os.path.isfile(entry):
forced.append(entry)
except OSError:
continue
return forced
if sys.platform == "win32":
try:
import winreg
except ImportError:
return []
paths: list[str] = _windows_device_icd_manifest_paths(winreg)
for key_path in _VULKAN_ICD_REGISTRY_KEYS:
try:
with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key_path) as key:
for index in range(winreg.QueryInfoKey(key)[1]):
try:
name, value, kind = winreg.EnumValue(key, index)
except OSError:
continue
# Name is the path, DWORD data the enable flag, 0 loads.
if kind != winreg.REG_DWORD or value != 0:
continue
try:
if not os.path.isfile(name):
continue
except OSError:
continue
paths.append(name)
except OSError:
continue
return list(dict.fromkeys(paths))
paths = []
for directory in _vulkan_icd_search_dirs():
try:
paths.extend(str(entry) for entry in sorted(directory.glob("*.json")))
except OSError:
continue
return paths
def _amd_vulkan_icd_usable(path: str) -> bool:
"""Whether a manifest still points at a driver library that is there: a leftover JSON
is a registration with no device behind it. A bare name is accepted, since the loader
resolves it through a search path this cannot see."""
try:
with open(path, "r", encoding = "utf-8") as handle:
manifest = json.load(handle)
library = (manifest.get("ICD") or {}).get("library_path")
except Exception:
return False
if not isinstance(library, str) or not library.strip():
return False
library = library.strip()
if not (os.path.isabs(library) or "/" in library or "\\" in library):
return True
if not os.path.isabs(library):
library = os.path.join(os.path.dirname(path), library)
try:
return os.path.isfile(library)
except OSError:
return False
def _amd_vulkan_icd_present() -> bool:
"""Whether an AMD Vulkan driver is installed so the Vulkan bundle has a device; with no AMD ICD,
ROCm is kept rather than silently running on CPU, failing towards the status quo."""
def _is_amd_64_bit(name: str) -> bool:
stem = PurePath(name).stem.lower().replace("-", "_")
if stem.endswith("32") or any(n in stem for n in _AMD_VULKAN_ICD_32_BIT_NEEDLES):
return False
return any(needle in stem for needle in _AMD_VULKAN_ICD_NEEDLES)
try:
return any(
_is_amd_64_bit(PurePath(path).name)
and _vulkan_loader_allows(path)
and _amd_vulkan_icd_usable(path)
for path in _amd_vulkan_icd_manifest_paths()
)
except Exception:
return False
def _should_prefer_vulkan_for_amd_igpu(host: HostInfo) -> bool:
"""True when every AMD GPU here is an integrated part Vulkan serves better.
A preference, not a fallback: unlike _should_auto_vulkan_for_amd_windows these archs
have a working HIP prebuilt, just slower. Both platforms, the Linux fault being worse.
Every guard that fallback applies is applied here for the same reasons, over every
PHYSICAL gfx so a masked-off discrete card is not moved.
"""
active = _active_rocm_gfx_target(host)
if not active:
return False
if not (host.has_rocm and not host.has_physical_nvidia):
return False
if _hip_visible_device_mask_set() or _vulkan_visible_device_mask_set():
return False
targets = list(dict.fromkeys([*_host_rocm_gfx_targets(host), active]))
if not all(target in VULKAN_PREFERRED_GFX_TARGETS for target in targets):
return False
return _amd_vulkan_icd_present()
def _should_auto_vulkan_for_amd_linux(host: HostInfo) -> bool:
"""Linux counterpart of _should_auto_vulkan_for_amd_windows: ROCm userspace is installed
but no AMD GPU here has ROCm kernels (RDNA 1, Polaris, gfx90c), so HIP would source-build
into a binary that cannot run. Same guards as the integrated-GPU preference, plus an explicit
HSA_OVERRIDE_GFX_VERSION, which means the user runs this card on ROCm on purpose."""
active = _active_rocm_gfx_target(host)
if not active:
return False
if not (host.is_linux and host.has_rocm and not host.has_physical_nvidia):
return False
if _hip_visible_device_mask_set() or _vulkan_visible_device_mask_set():
return False
if os.environ.get("HSA_OVERRIDE_GFX_VERSION"):
return False
targets = list(dict.fromkeys([*_host_rocm_gfx_targets(host), active]))
if not all(target in LINUX_NO_ROCM_GFX_TARGETS for target in targets):
return False
return _amd_vulkan_icd_present()
def _vulkan_only_host(host: HostInfo) -> HostInfo:
"""Rewrite ``host`` so the asset selectors take their Vulkan branch.
That branch fires on ``has_intel_gpu and not nvidia and not rocm``, so clear
the CUDA/ROCm flags and raise the integrated-GPU flag. The synthetic flag
never leaves install planning -- it only routes the llama.cpp prebuilt
choice, not the torch/training stack.
"""
return dataclasses_replace(
host,
has_usable_nvidia = False,
has_physical_nvidia = False,
has_rocm = False,
rocm_gfx_target = None,
rocm_gfx_targets = [],
has_intel_gpu = True,
vulkan_routed = True,
)
def _backend_only_attempts(attempts: Iterable[AssetChoice], backend: str) -> list[AssetChoice]:
"""Drop the generic fallbacks a plan appends, keeping only ``backend`` bundles."""
kinds = install_kinds_for_backend(backend)
return [attempt for attempt in attempts if attempt.install_kind in kinds]
def _backend_only_release_plans(
plans: Iterable[InstallReleasePlan], backend: str
) -> list[InstallReleasePlan]:
"""Keeps plans with a bundle for backend; a named request must not quietly fall back to the
generic CPU tail, so unmet requests fail here and the caller surfaces them, or a stored one
falls back to detection."""
filtered = []
for plan in plans:
attempts = _backend_only_attempts(plan.attempts, backend)
if attempts:
filtered.append(dataclasses_replace(plan, attempts = attempts))
if not filtered:
raise BackendUnavailable(f"no {backend} prebuilt bundle attempts were available")
return filtered
def _route_to_vulkan_prebuilt(
host: HostInfo,
published_repo: str,
published_release_tag: str,
*,
force_cpu: bool,
llama_backend: str | None = None,
) -> tuple[HostInfo, str, str, str | None]:
"""Point a Vulkan-capable host at the selected repository's Vulkan prebuilt.
The default Unsloth release manifest includes Vulkan app bundles, including the
DiffusionGemma visual server. These triggers route here, all suppressed when a CPU
flag (--cpu-fallback or --force-cpu, folded into force_cpu) wins:
* ``UNSLOTH_LLAMA_CPP_BACKEND=vulkan`` / ``UNSLOTH_FORCE_VULKAN`` /
``--llama-backend vulkan`` forces Vulkan over the detected CUDA/ROCm backend;
* Windows AMD with no HIP-prebuilt gfx arch auto-falls back to Vulkan (#7357), and
so does Linux AMD whose every gfx has no ROCm kernels;
* an AMD host whose every GPU is an integrated part Vulkan serves better prefers
Vulkan over the HIP bundle it could otherwise run;
* an auto-detected Intel GPU with NO physical NVIDIA/ROCm, the purpose of the
has_intel_gpu probe;
* any other non-macOS host that would get the CPU bundle, when the Vulkan loader
enumerates a GPU llama.cpp accepts (Linux arm64 GPUs, nouveau); Qualcomm's proprietary
driver (Snapdragon X) stays behind the explicit request above.
Applied by BOTH the install path and the --resolve-prebuilt probe so the "is a prebuilt
available" answer matches what actually gets installed.
Returns the (possibly rewritten) host, repo, release tag, and a backend to persist in
the install marker when updates must re-assert Vulkan.
"""
forced = force_vulkan_requested(llama_backend)
explicit_backend = resolved_llama_backend(llama_backend)
# Host-only test, so a forced Vulkan run can still tell this box would have
# routed itself to Vulkan anyway.
amd_no_hip_host = _should_auto_vulkan_for_amd_windows(
host, published_repo
) or _should_auto_vulkan_for_amd_linux(host)
amd_igpu_host = _should_prefer_vulkan_for_amd_igpu(host)
detecting = explicit_backend in (None, "auto")
auto_no_hip = detecting and amd_no_hip_host
auto_igpu = detecting and amd_igpu_host
# No PHYSICAL NVIDIA, not merely no usable one: Vulkan ignores CUDA_VISIBLE_DEVICES, so
# auto-routing a host that hides its NVIDIA card would let it grab the reserved GPU.
# Not Windows arm64: a stale Intel adapter entry there names no GPU; the probe below decides.
auto_intel = (
detecting
and host.has_intel_gpu
and not host.has_physical_nvidia
and not host.has_rocm
and not (host.is_windows and host.is_arm64)
)
# Probe last so only hosts otherwise headed for the CPU bundle pay for it; CPU stays the fallback.
auto_device = (
detecting
and not (forced or force_cpu or auto_intel or auto_no_hip or auto_igpu)
and not host.is_macos
and not host.has_physical_nvidia
and not host.has_rocm
and not (host.is_x86_64 and host.has_amd_gpu_without_rocm)
and not _vulkan_visible_device_mask_set()
and bool(_vulkan_auto_gpu_devices())
)
if force_cpu or not (forced or auto_intel or auto_no_hip or auto_igpu or auto_device):
return host, published_repo, published_release_tag, None
if host.is_macos:
if forced:
log(
"UNSLOTH_LLAMA_CPP_BACKEND=vulkan is set but ignored on macOS "
"(Metal is used; there is no Vulkan prebuilt)"
)
return host, published_repo, published_release_tag, None
if auto_no_hip:
active = _active_rocm_gfx_target(host) or "unknown"
if host.is_linux:
log(
f"AMD GPU arch {active} has no ROCm kernels; installing the Vulkan "
"llama.cpp prebuilt instead"
)
else:
log(
"Active AMD GPU arch is not supported by the Windows HIP prebuilt "
f"({active}); installing the Vulkan llama.cpp prebuilt, which runs on "
"every AMD GPU here"
)
host = _vulkan_only_host(host)
persist_backend = "auto"
elif auto_igpu:
log(
"AMD integrated GPU detected "
f"({_active_rocm_gfx_target(host) or 'unknown'}); installing the Vulkan "
"llama.cpp prebuilt, which is faster on these parts than ROCm"
)
host = _vulkan_only_host(host)
persist_backend = "auto"
elif forced:
log(
"Vulkan llama.cpp backend requested; installing the Vulkan "
"prebuilt instead of the detected GPU backend"
)
host = _vulkan_only_host(host)
# Persist as automatic so these installs stay eligible for the CPU crash recovery.
persist_backend = "auto" if (amd_no_hip_host or amd_igpu_host) else "vulkan"
elif auto_intel:
log("Intel GPU detected; installing the Vulkan llama.cpp prebuilt")
persist_backend = None
else:
names = ", ".join(dict.fromkeys(_vulkan_auto_gpu_devices()))
log(
f"Vulkan GPU found ({names}); installing the Vulkan llama.cpp prebuilt, "
"with the CPU bundle as fallback"
)
host = dataclasses_replace(_vulkan_only_host(host), vulkan_device_required = True)
# Detected, like the Intel route: updates re-detect and the CPU crash recovery holds.
persist_backend = None
# A fork release without a linux-vulkan-arm64 bundle source-builds; no upstream prebuilt.
return host, published_repo, published_release_tag, persist_backend
def diffusion_visual_server_backfill_needed(
install_dir: Path, host: HostInfo, choice: AssetChoice
) -> bool:
"""True when an existing install matches the tag but lacks the DiffusionGemma
visual-server the chosen bundle ships. An install made before the visual-server
entered the copy allowlist matches on tag yet is missing the binary, so the
tag-match skip never backfills it (DiffusionGemma then fails with "runner not
found"). Gated to the fork ("published") bundles that actually carry it, so
upstream installs -- which never ship it -- can't thrash on repeated updates.
Once a re-extract lands the binary this returns False, so it self-limits."""
if choice.source_label != "published":
return False
name = "llama-diffusion-gemma-visual-server" + (".exe" if host.is_windows else "")
if name not in runtime_patterns_for_choice(choice):
return False
for cand in (
install_dir / name,
install_dir / "build" / "bin" / name,
install_dir / "build" / "bin" / "Release" / name,
):
if cand.is_file():
return False
return True
def _causal_chain(exc: BaseException) -> Iterable[BaseException]:
seen: set[int] = set()
current: BaseException | None = exc
while current is not None and id(current) not in seen:
seen.add(id(current))
yield current
# `raise X from None` sets __suppress_context__; do not follow __context__ then.
if current.__cause__ is not None:
current = current.__cause__
elif current.__suppress_context__:
current = None
else:
current = current.__context__
# CPython maps 112 to ENOSPC but 39 arrives as EINVAL, so check winerror too.
_WINDOWS_DISK_FULL = (39, 112)
# A quota dooms the bigger source build too. Guarded: Windows aliases EDQUOT to WSAEDQUOT.
_DISK_FULL_ERRNOS = {errno.ENOSPC: "no space left on device"}
if hasattr(errno, "EDQUOT"):
_DISK_FULL_ERRNOS[errno.EDQUOT] = "disk quota exceeded"
def _winerror_of(exc: OSError) -> Any:
"""Reads exc.winerror defensively, not via getattr: on Python 3.9 urllib's HTTPError proxies
unknown attributes and raises KeyError, which getattr will not swallow; a mirror's 404 must
not crash this."""
try:
return exc.winerror
except Exception:
return None
def _out_of_space_reason(exc: BaseException) -> str | None:
"""Why `exc` means the install cannot fit, or None if it means something else."""
if isinstance(exc, OSError):
reason = _DISK_FULL_ERRNOS.get(exc.errno)
if reason is not None:
return reason
if _winerror_of(exc) in _WINDOWS_DISK_FULL:
return "no space left on device"
# shutil.Error keeps only text; Windows reads "[WinError 112]", brackets so 1120 cannot match.
if isinstance(exc, shutil.Error):
text = str(exc)
for code, reason in _DISK_FULL_ERRNOS.items():
if f"[Errno {code}]" in text:
return reason
if any(f"[WinError {code}]" in text for code in _WINDOWS_DISK_FULL):
return "no space left on device"
return None
def _environment_fatal_reason(exc: BaseException) -> str | None:
for cause in _causal_chain(exc):
reason = _out_of_space_reason(cause)
if reason is not None:
return reason
return None
def _log_disk_space_help() -> None:
log(
"free up space or point TMPDIR and UNSLOTH_STUDIO_HOME at a larger "
"volume (e.g. /workspace), then re-run"
)
@contextmanager
def scratch_dir(prefix: str) -> Iterator[Path]:
"""Temp dir whose cleanup never raises, so an rmtree failure cannot replace EXIT_NO_SPACE; not
TemporaryDirectory(ignore_cleanup_errors = True), which needs 3.10+ while setup.sh runs this
on the host python and 3.9 is supported."""
path = Path(tempfile.mkdtemp(prefix = prefix))
try:
yield path
finally:
shutil.rmtree(path, ignore_errors = True)
def _first_existing_ancestor(path: Path) -> Path:
current = path
while current != current.parent and not current.exists():
current = current.parent
return current
def _low_disk_warning(install_dir: Path, *, advised_gb: float = 5.0) -> str | None:
"""Advisory only, never fatal: prebuilt installs peak under 1 GB, so a fixed threshold cannot
decide whether there is room and a real ENOSPC does; advised_gb is the headroom a
source-build fallback would want."""
advised = int(advised_gb * (1024**3))
targets = {
"build/download scratch (TMPDIR)": Path(tempfile.gettempdir()),
"llama.cpp install dir": _first_existing_ancestor(install_dir),
}
for label, path in targets.items():
try:
free = shutil.disk_usage(path).free
except OSError:
continue
if free < advised:
return (
f"low disk space for llama.cpp: {label} at {path} has "
f"{free / (1024**3):.1f} GB free (~{advised_gb:.0f} GB recommended)"
)
return None
def _fail_no_space(reason: str) -> None:
log(reason)
_log_disk_space_help()
raise SystemExit(EXIT_NO_SPACE)
@dataclass
class BackendSelection:
"""The install plan for one backend request."""
backend: str | None
host: HostInfo
published_repo: str
published_release_tag: str
requested_tag: str
release_plans: "Sequence[InstallReleasePlan]"
persist_llama_backend: str | None
persist_rocm_gfx: str | None
@property
def choice(self) -> AssetChoice | None:
"""The bundle this plan installs first, or None when it found none."""
plans = self.release_plans
return plans[0].attempts[0] if plans and plans[0].attempts else None
@dataclass
class BackendRoute:
"""Where a backend request points this host, before any network call."""
backend: str | None
host: HostInfo
published_repo: str
published_release_tag: str
persist_llama_backend: str | None
persist_rocm_gfx: str | None
rocm_fallback_host: HostInfo | None = None
# Quoted values only: non-ROCm builds write `hip: Optional[str] = None`.
_TORCH_VERSION_PY_HIP_RE = re.compile(r"""^hip\s*(?::[^=]*)?=\s*['"]([^'"]*)['"]""", re.MULTILINE)
# AMD's Radeon SDK wheels leave hip None and carry the tag here (2.9.0+rocmsdk20251116).
_TORCH_VERSION_PY_VERSION_RE = re.compile(
r"""^__version__\s*(?::[^=]*)?=\s*['"]([^'"]*)['"]""", re.MULTILINE
)
def _torch_version_py_is_rocm(version_py: Path) -> bool | None:
try:
text = version_py.read_text(encoding = "utf-8", errors = "replace")
except OSError:
return None
hip = _TORCH_VERSION_PY_HIP_RE.search(text)
version = _TORCH_VERSION_PY_VERSION_RE.search(text)
return bool((hip and hip.group(1)) or (version and "rocm" in version.group(1).lower()))
def _installed_torch_is_rocm() -> bool | None:
"""Read off disk: importing torch here is slow and can fail."""
try:
spec = importlib.util.find_spec("torch")
if spec is None or not spec.origin:
return None
return _torch_version_py_is_rocm(Path(spec.origin).with_name("version.py"))
except Exception:
return None
def _rocm_torch_preferred() -> bool:
if (os.environ.get("UNSLOTH_FORCE_ROCM_TORCH") or "").strip().lower() in (
"1",
"true",
"yes",
"on",
):
return True
return _installed_torch_is_rocm() is True
def _auto_host_following_rocm_torch(host: HostInfo) -> HostInfo:
"""Only moves off CUDA on live AMD evidence (stale ROCm torch keeps CUDA)."""
if not host.has_usable_nvidia:
return host
probed = host if host.has_rocm else detect_host(probe_rocm_with_nvidia = True)
if not (probed.has_rocm or _normalize_forwarded_gfx(os.environ.get("UNSLOTH_ROCM_GFX_ARCH"))):
return host
log(
"ROCm torch is installed or requested; Automatic prefers the ROCm llama.cpp build "
"over CUDA on this mixed NVIDIA+AMD host"
)
return dataclasses_replace(probed, has_physical_nvidia = False, has_usable_nvidia = False)
def route_backend_request(
*,
backend: str | None,
published_repo: str,
published_release_tag: str,
override_has_rocm: bool = False,
override_rocm_gfx: str | None = None,
cpu_mechanism: bool = False,
host: HostInfo | None = None,
) -> BackendRoute:
"""Apply a backend request to the host profile without fetching releases."""
if host is None:
detected_host = (
detect_host(probe_rocm_with_nvidia = True) if backend == "rocm" else detect_host()
)
elif backend == "rocm" and host.has_usable_nvidia and not host.has_rocm:
# The automatic profile skips AMD probes once CUDA is usable; re-probe for explicit ROCm.
detected_host = detect_host(probe_rocm_with_nvidia = True)
else:
detected_host = host
if backend == "rocm":
detected_host = dataclasses_replace(
detected_host,
has_physical_nvidia = False,
has_usable_nvidia = False,
)
elif backend in (None, "auto") and _rocm_torch_preferred():
detected_host = _auto_host_following_rocm_torch(detected_host)
force_cpu = cpu_mechanism or backend == "cpu"
resolved_host = _apply_host_overrides(
detected_host,
override_has_rocm = override_has_rocm,
override_rocm_gfx = override_rocm_gfx,
force_cpu = force_cpu,
)
# Read before the route, which rewrites the host to Vulkan-only and drops the AMD arch.
persist_rocm_gfx = _active_rocm_gfx_target(resolved_host)
routed_host, repo, release_tag, persist_llama_backend = _route_to_vulkan_prebuilt(
resolved_host,
published_repo,
published_release_tag,
force_cpu = force_cpu,
llama_backend = backend,
)
return BackendRoute(
backend = backend,
host = routed_host,
published_repo = repo,
published_release_tag = release_tag,
persist_llama_backend = persist_llama_backend,
persist_rocm_gfx = persist_rocm_gfx,
rocm_fallback_host = (
resolved_host
if (
resolved_host.has_rocm
and not routed_host.has_rocm
and _should_prefer_vulkan_for_amd_igpu(resolved_host)
)
else None
),
)
def _with_rocm_behind_vulkan(
plans: "Sequence[InstallReleasePlan]",
llama_tag: str,
rocm_host: HostInfo,
published_repo: str,
published_release_tag: str,
) -> list[InstallReleasePlan]:
"""Put this host's ROCm bundle behind Vulkan and ahead of the generic CPU fallback.
_vulkan_only_host clears has_rocm, so the selectors read the box as Intel/CPU and end
the plan in a CPU attempt: a Vulkan asset that fails validation would then replace a
working ROCm install with CPU inference. A named backend request is filtered
afterwards, so this cannot smuggle ROCm into an explicit --llama-backend vulkan.
"""
# A source build is slow and a CPU prebuilt is silent, so fail towards the visible one.
_NO_GPU_BUNDLE = (
"no published release carries a Vulkan or matching ROCm bundle for this host, "
"and a CPU prebuilt would replace a working ROCm install"
)
cpu_kinds = install_kinds_for_backend("cpu")
def _without_cpu(plan: InstallReleasePlan) -> InstallReleasePlan | None:
"""Drop the CPU tail from a plan with no ROCm attempt behind Vulkan: a failed
Vulkan install is recoverable, a silent CPU one is "it got slow". A release that
published neither bundle is only its CPU fallback, so it drops out whole."""
attempts = [attempt for attempt in plan.attempts if attempt.install_kind not in cpu_kinds]
if len(attempts) == len(plan.attempts):
return plan
return dataclasses_replace(plan, attempts = attempts) if attempts else None
def _drop_cpu_only(candidates: "Sequence[InstallReleasePlan]") -> list[InstallReleasePlan]:
"""Apply _without_cpu across releases, or fail rather than install CPU."""
kept = [narrowed for plan in candidates if (narrowed := _without_cpu(plan)) is not None]
if candidates and not kept:
raise PrebuiltFallback(_NO_GPU_BUNDLE)
return kept
try:
_tag, rocm_plans = resolve_simple_install_release_plans(
llama_tag, rocm_host, published_repo, published_release_tag
)
rocm_attempts = {plan.release_tag: plan.attempts for plan in rocm_plans}
except Exception:
return _drop_cpu_only(plans)
out: list[InstallReleasePlan] = []
for plan in plans:
present = {attempt.install_kind for attempt in plan.attempts}
candidates = rocm_attempts.get(plan.release_tag, [])
extra = [
attempt
for attempt in candidates
if attempt.install_kind not in present and attempt.install_kind not in cpu_kinds
]
if not extra:
if any(attempt.install_kind in present for attempt in candidates):
out.append(plan)
else:
narrowed = _without_cpu(plan)
if narrowed is not None:
out.append(narrowed)
continue
at = next(
(i for i, a in enumerate(plan.attempts) if a.install_kind in cpu_kinds),
len(plan.attempts),
)
out.append(
dataclasses_replace(plan, attempts = [*plan.attempts[:at], *extra, *plan.attempts[at:]])
)
if plans and not out:
raise PrebuiltFallback(_NO_GPU_BUNDLE)
return out
def select_backend_install(
*,
backend: str | None,
llama_tag: str,
published_repo: str,
published_release_tag: str,
override_has_rocm: bool = False,
override_rocm_gfx: str | None = None,
cpu_mechanism: bool = False,
host: HostInfo | None = None,
route: BackendRoute | None = None,
) -> BackendSelection:
"""Resolve bundles for a backend, or raise BackendUnavailable if none match."""
if route is None:
route = route_backend_request(
backend = backend,
published_repo = published_repo,
published_release_tag = published_release_tag,
override_has_rocm = override_has_rocm,
override_rocm_gfx = override_rocm_gfx,
cpu_mechanism = cpu_mechanism,
host = host,
)
# Up front: the fork's plans are lazy, so their Jetson rejection would surface outside the try below (#4856).
if route.backend == "cuda" and _tegra_blocks_arm64_cuda(route.host):
raise BackendUnavailable(
"no CUDA prebuilt runs on a Jetson before CUDA 13; source-building instead"
)
try:
requested_tag, release_plans = resolve_simple_install_release_plans(
llama_tag, route.host, route.published_repo, route.published_release_tag
)
except PrebuiltFallback as exc:
# #12842: a stored cuda choice must fall back to detection, not fail the update.
if route.backend == "cuda" and (
driver_below_cuda_prebuilt_floor(route.host) or _tegra_blocks_arm64_cuda(route.host)
):
raise BackendUnavailable(str(exc)) from exc
raise
if route.rocm_fallback_host is not None:
release_plans = _with_rocm_behind_vulkan(
release_plans,
llama_tag,
route.rocm_fallback_host,
route.published_repo,
route.published_release_tag,
)
if route.backend in CONCRETE_BACKENDS and not route.host.is_macos:
release_plans = _backend_only_release_plans(release_plans, route.backend)
return BackendSelection(
backend = route.backend,
host = route.host,
published_repo = route.published_repo,
published_release_tag = route.published_release_tag,
requested_tag = requested_tag,
release_plans = release_plans,
persist_llama_backend = route.persist_llama_backend,
persist_rocm_gfx = route.persist_rocm_gfx,
)
def install_prebuilt(
install_dir: Path,
llama_tag: str,
published_repo: str,
published_release_tag: str,
*,
override_has_rocm: bool = False,
override_rocm_gfx: str | None = None,
force_cpu: bool = False,
persist_force_cpu: bool = False,
llama_backend: str | None = None,
instruction_cleanup_root: Path | None = None,
) -> None:
choice: AssetChoice | None = None
# Read before the body mutates force_cpu, which it sets True for a stored "cpu" choice.
explicit_version_request = (
normalized_requested_llama_tag(llama_tag) != "latest"
or bool((published_release_tag or "").strip())
or (bool((published_repo or "").strip()) and published_repo != DEFAULT_PUBLISHED_REPO)
# --has-rocm and --rocm-gfx do not count: both entrypoints forward a DETECTED GPU on every AMD host.
or force_cpu
)
host: HostInfo | None = None
backend = "auto"
backend_mandatory = False
cleanup_root = install_dir if instruction_cleanup_root is None else instruction_cleanup_root
try:
with install_lock(install_lock_path(install_dir)):
# Under the install lock so a waiting update cannot undo a concurrent backend switch.
backend, backend_mandatory = effective_backend_request(
llama_backend, install_dir = install_dir
)
if backend != "auto" and not backend_mandatory:
log(f"honouring the backend recorded by this install: {backend}")
caller_force_cpu, caller_persist_force_cpu = force_cpu, persist_force_cpu
if backend == "cpu":
force_cpu = True
persist_force_cpu = True
if (install_dir / "UNSLOTH_PREBUILT_INFO.json").is_file():
removed = remove_agent_instruction_files(cleanup_root)
if removed:
log(
f"removed {removed} contributor-only agent instruction file(s) from install"
)
if install_dir.exists():
log(
f"existing llama.cpp install detected at {install_dir}; validating staged prebuilt update before replacement"
)
else:
log(
f"no existing llama.cpp install detected at {install_dir}; performing fresh prebuilt install"
)
# Listing failures become PrebuiltFallback here, not in the resolver: --resolve-prebuilt turns
# that into a cached exit-0 "no prebuilt", while nonzero exits are never cached. Plain OSError
# (EMFILE, ENOMEM, EACCES) stays EXIT_ERROR since a source build only makes it worse.
def _select(
requested_backend: str | None, route: BackendRoute | None = None
) -> BackendSelection:
nonlocal host
# Routing stays outside the conversion so a host-detection bug is not read as a listing failure.
if route is None:
route = route_backend_request(
backend = requested_backend,
published_repo = published_repo,
published_release_tag = published_release_tag,
override_has_rocm = override_has_rocm,
override_rocm_gfx = override_rocm_gfx,
cpu_mechanism = force_cpu,
)
host = route.host
try:
return select_backend_install(
backend = route.backend,
llama_tag = llama_tag,
published_repo = route.published_repo,
published_release_tag = route.published_release_tag,
route = route,
)
except (BusyInstallConflict, PrebuiltFallback):
raise
except RELEASE_LISTING_TRANSPORT_ERRORS as exc:
raise release_listing_fallback(exc, published_repo) from exc
initial_route = route_backend_request(
backend = backend,
published_repo = published_repo,
published_release_tag = published_release_tag,
override_has_rocm = override_has_rocm,
override_rocm_gfx = override_rocm_gfx,
cpu_mechanism = force_cpu,
)
# Before the kept-install check and selection, which both read the runtime on disk.
ensure_linux_cuda_runtime_before_selection(initial_route)
if existing_install_current_without_plan(
install_dir,
llama_tag = llama_tag,
published_repo = published_repo,
published_release_tag = published_release_tag,
backend_request = backend,
force_cpu = force_cpu,
# Same forwarded detection as initial_route, else the "auto" profile can never match.
override_has_rocm = override_has_rocm,
override_rocm_gfx = override_rocm_gfx,
route = initial_route,
backend_request_mandatory = backend_mandatory,
):
ensure_fit_params_executable(install_dir)
return
unhonoured_request: str | None = None
try:
selection = _select(backend, initial_route)
except BackendUnavailable:
if backend_mandatory:
raise
log(
f"the {backend} backend recorded by this install is not available here; "
"falling back to hardware detection"
)
unhonoured_request = backend
backend = "auto"
force_cpu, persist_force_cpu = caller_force_cpu, caller_persist_force_cpu
selection = _select(backend)
host = selection.host
published_repo = selection.published_repo
published_release_tag = selection.published_release_tag
requested_tag = selection.requested_tag
release_plans = selection.release_plans
persist_llama_backend = selection.persist_llama_backend
persist_rocm_gfx = selection.persist_rocm_gfx
# The CHOICE, not detection's replacement: persisting "auto" destroyed Vulkan choices on AMD.
persist_backend_request = unhonoured_request or backend
def _record_reused_selection(
plan: InstallReleasePlan, reused: AssetChoice, used_fallback: bool
) -> None:
"""Update selection fields when the existing bundle is reused."""
ensure_fit_params_executable(install_dir)
sync_marker_selection(
install_dir,
choice = reused,
prebuilt_fallback_used = used_fallback,
backend_request = persist_backend_request,
persist_force_cpu = persist_force_cpu,
persist_llama_backend = persisted_llama_backend(persist_llama_backend, reused),
ggml_tree = recorded_ggml_tree(plan.approved_checksums, reused),
rocm_gfx = persist_rocm_gfx,
walk_back = plan.walk_back,
host = host,
)
if release_plans and existing_install_matches_plan(install_dir, host, release_plans[0]):
current = release_plans[0]
if diffusion_visual_server_backfill_needed(install_dir, host, current.attempts[0]):
log(
f"existing install matches {current.release_tag} but is missing the "
"DiffusionGemma visual-server; re-extracting the bundle to backfill it"
)
else:
log(
"existing llama.cpp install already matches selected release "
f"{current.release_tag} upstream_tag={current.llama_tag}; skipping download and install"
)
_record_reused_selection(current, current.attempts[0], False)
return
with scratch_dir("unsloth-llama-prebuilt-") as work_dir:
probe = lazy_validation_model(
work_dir / "stories260K.gguf",
validation_model_cache_path(install_dir),
)
probe_resolved = False
last_failure: PrebuiltFallback | None = None
for release_index, plan in enumerate(
iter_release_plans(release_plans, published_repo)
):
choice = plan.attempts[0]
backfill = diffusion_visual_server_backfill_needed(install_dir, host, choice)
if existing_install_matches_plan(install_dir, host, plan):
if backfill:
log(
f"existing install matches fallback {plan.release_tag} but is missing "
"the DiffusionGemma visual-server; re-extracting to backfill it"
)
else:
log(
"existing llama.cpp install already matches fallback release "
f"{plan.release_tag} upstream_tag={plan.llama_tag}; skipping reinstall"
)
_record_reused_selection(plan, choice, release_index > 0)
return
log(
"selected "
f"{choice.name} ({choice.source_label}) from published release "
f"{plan.release_tag} for {host.system} {host.machine}"
)
# Outside the handler, so a transient failure cannot demote to an older release.
if not probe_resolved and any(
prebuilt_needs_functional_validation(attempt) for attempt in plan.attempts
):
probe = resolve_validation_model(probe)
probe_resolved = True
try:
choice, selected_staging_dir, _ = validate_prebuilt_attempts(
plan.attempts,
host,
install_dir,
work_dir,
probe,
requested_tag = requested_tag,
llama_tag = plan.llama_tag,
release_tag = plan.release_tag,
approved_checksums = plan.approved_checksums,
initial_fallback_used = release_index > 0,
existing_install_dir = install_dir,
force_cpu = persist_force_cpu,
llama_backend = persist_llama_backend,
backend_request = persist_backend_request,
rocm_gfx = persist_rocm_gfx,
walk_back = plan.walk_back,
)
except ExistingInstallSatisfied as satisfied:
_record_reused_selection(plan, satisfied.choice, satisfied.used_fallback)
return
except PrebuiltFallback as exc:
if _environment_fatal_reason(exc):
raise
last_failure = exc
log(
"published release "
f"{plan.release_tag} upstream_tag={plan.llama_tag} failed; "
"trying an older published prebuilt if one remains "
f"({textwrap.shorten(str(exc), width = 200, placeholder = '...')})"
)
continue
activate_install_tree(selected_staging_dir, install_dir, host)
try:
ensure_converter_scripts(
install_dir,
plan.llama_tag,
repo = published_repo or DEFAULT_PUBLISHED_REPO,
release_tag = plan.release_tag,
)
except Exception as exc:
log(
"converter script fetch failed after activation; install remains valid "
f"({textwrap.shorten(str(exc), width = 200, placeholder = '...')})"
)
try:
ensure_diffusion_visual_server(
install_dir, host, plan.release_tag, plan.approved_checksums
)
except Exception as exc:
log(
"diffusion visual server step skipped; install remains valid "
f"({textwrap.shorten(str(exc), width = 200, placeholder = '...')})"
)
return
if last_failure is not None:
raise last_failure
except BusyInstallConflict as exc:
log("prebuilt install path is blocked by an in-use llama.cpp install")
log(f"prebuilt busy reason: {exc}")
raise SystemExit(EXIT_BUSY) from exc
except UnknownBackendRequest as exc:
# EXIT_ERROR, never the source fallback: detection would replace a choice this build cannot read.
log(f"prebuilt install refused: {exc}")
raise SystemExit(EXIT_ERROR) from exc
except PrebuiltFallback as exc:
fatal = _environment_fatal_reason(exc)
if fatal:
log(f"prebuilt install failed: {fatal}")
_log_disk_space_help()
raise SystemExit(EXIT_NO_SPACE) from exc
preserve_backend = backend in CONCRETE_BACKENDS and host is not None and not host.is_macos
satisfied_stored_backend = (
preserve_backend
and not backend_mandatory
and marker_backend(load_prebuilt_metadata(install_dir)) == backend
)
if (
(not preserve_backend or satisfied_stored_backend)
and not explicit_version_request
and not backend_mandatory
# An untrustworthy release must not be reported as a routine "update unavailable".
and not isinstance(exc, _core.ReleaseIntegrityError)
and host is not None
and _existing_install_runs(install_dir, host)
and _kept_install_covers_host(load_prebuilt_metadata(install_dir), host)
):
log("prebuilt update unavailable; keeping the existing complete install")
log(f"prebuilt update reason: {exc}")
ensure_fit_params_executable(install_dir)
return
log(
"prebuilt install failed; preserving the selected backend"
if preserve_backend
else "prebuilt install path failed; falling back to source build"
)
log_lines(f"prebuilt fallback reason: {exc}".splitlines())
# Diagnostics must never change the verdict: EXIT_ERROR never source builds.
try:
if host is not None:
print(collect_system_report(host, choice, install_dir))
except Exception as report_exc:
log(f"system report unavailable: {report_exc}")
if preserve_backend:
log(f"the {backend} backend selection was preserved; source fallback was not started")
raise SystemExit(EXIT_BACKEND_UNAVAILABLE) from exc
raise SystemExit(EXIT_FALLBACK) from exc
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description = "Install and validate a prebuilt llama.cpp bundle for Unsloth Studio."
)
parser.add_argument("--install-dir", help = "Target ~/.unsloth/llama.cpp directory")
parser.add_argument(
"--llama-tag",
default = DEFAULT_LLAMA_TAG,
help = (
"llama.cpp release tag. Defaults to the latest usable published Unsloth "
"release unless UNSLOTH_LLAMA_TAG overrides it."
),
)
parser.add_argument(
"--published-repo",
default = DEFAULT_PUBLISHED_REPO,
help = "Published bundle repository",
)
parser.add_argument(
"--published-release-tag",
default = DEFAULT_PUBLISHED_TAG,
help = (
"Published GitHub release tag to pin. By default, scan releases "
"until a usable published llama.cpp release bundle is found."
),
)
parser.add_argument(
"--has-rocm",
action = "store_true",
default = False,
help = (
"Assert that an AMD ROCm GPU is present. When set, skips the internal "
"hipinfo/amd-smi probe and forces has_rocm=True in the host profile. "
"Used by setup.ps1/setup.sh to forward their own ROCm detection result "
"so the HIP llama.cpp prebuilt is selected even when hipinfo is not on PATH."
),
)
parser.add_argument(
"--rocm-gfx",
default = os.environ.get("UNSLOTH_ROCM_GFX_ARCH"),
help = (
"Forward the AMD gfx target (e.g. gfx1151) that setup.ps1/setup.sh "
"resolved, so the per-gfx ROCm prebuilt is selected even when the "
"installer's own hipinfo/amd-smi probe cannot report it. Implies "
"--has-rocm. Defaults to the UNSLOTH_ROCM_GFX_ARCH environment variable."
),
)
parser.add_argument(
"--cpu-fallback",
action = "store_true",
default = False,
help = (
"Select the CPU prebuilt for this OS/arch even when a GPU is present. "
"Automatic/transient: setup.sh uses this as a last resort for arm64 Linux "
"GPU hosts whose source build failed. Does NOT persist, so a later update "
"heals back to a GPU bundle once one is available (#6097). Use --force-cpu "
"for a deliberate CPU-only choice that survives updates."
),
)
parser.add_argument(
"--force-cpu",
action = "store_true",
default = False,
help = (
"Deliberate CPU-only install (UNSLOTH_LLAMA_CPP_BACKEND=cpu). Drops GPU "
"detection like --cpu-fallback but also records force_cpu in the marker, so "
"the in-app updater re-asserts CPU and never re-routes to a GPU/Vulkan "
"bundle that would revive the Intel iGPU crash (#7213)."
),
)
parser.add_argument(
"--llama-backend",
choices = (*REQUESTABLE_BACKENDS, "hip"),
help = (
"Install the llama.cpp prebuilt for this backend and record the choice, "
"so later updates keep it instead of re-detecting. cuda and rocm (alias "
"hip) also opt out of the automatic Vulkan routes. A backend with no "
"bundle for this host fails rather than installing a different one; "
"auto restores hardware detection and clears a recorded choice. Ignored "
"on macOS, which has only the universal Metal build. Same effect as "
"UNSLOTH_LLAMA_CPP_BACKEND."
),
)
resolve_group = parser.add_mutually_exclusive_group()
resolve_group.add_argument(
"--resolve-llama-tag",
nargs = "?",
const = "latest",
help = "Resolve a llama.cpp tag such as 'latest' to the logical upstream release tag.",
)
resolve_group.add_argument(
"--resolve-install-tag",
nargs = "?",
const = "latest",
help = (
"Resolve a llama.cpp tag such as 'latest' to the concrete upstream tag "
"selected by the current published-release policy."
),
)
resolve_group.add_argument(
"--resolve-source-build",
nargs = "?",
const = "latest",
help = ("Resolve the source-build fallback plan."),
)
resolve_group.add_argument(
"--resolve-prebuilt",
nargs = "?",
const = "latest",
help = (
"Report whether an official prebuilt exists for this host without "
"downloading. Plans against --published-repo (defaults to the "
"fork). Use --output-format json."
),
)
resolve_group.add_argument(
"--resolve-backends",
nargs = "?",
const = "latest",
help = (
"Report every llama.cpp backend installable on this host, plus what "
"--install-dir currently runs, without downloading. Feeds the Unsloth "
"backend picker. Use --output-format json."
),
)
resolve_group.add_argument(
"--validate-install",
metavar = "DIR",
help = (
"Run the staged llama-server smoke test against an existing build "
"tree (setup.sh source-build post-check, #5854). Exit 2 on failure. "
"Normally gated by UNSLOTH_LLAMA_STAGED_VALIDATION; this flag always "
"runs the check."
),
)
resolve_group.add_argument(
"--check-installed",
metavar = "DIR",
help = (
"Exit 0 when the install at DIR is complete and its binaries load (the same "
"network-free check the updater uses before keeping an install), else 2."
),
)
parser.add_argument(
"--install-kind",
default = None,
help = (
"Install kind for --validate-install GPU offload (e.g. linux-cuda, "
"linux-rocm, macos-arm64). When omitted, host detection decides."
),
)
parser.add_argument(
"--check-existing-install",
default = None,
metavar = "DIR",
help = (
"Exit 0 when setup.sh may reuse DIR instead of building from source, "
"1 when it must not. Prints nothing."
),
)
parser.add_argument(
"--output-format",
choices = ("plain", "json"),
default = "plain",
help = "Resolver output format. Defaults to plain.",
)
return parser.parse_args()
def emit_resolver_output(payload: dict[str, Any], *, output_format: str) -> None:
if output_format == "json":
print(json.dumps(payload, sort_keys = True))
return
if "llama_tag" in payload:
print(payload["llama_tag"])
return
if {
"source_url",
"source_ref_kind",
"source_ref",
}.issubset(payload):
print(
"\t".join(
(
str(payload["source_url"]),
str(payload["source_ref_kind"]),
str(payload["source_ref"]),
)
)
)
return
print(json.dumps(payload, sort_keys = True))
def requested_backend_arg(args: argparse.Namespace) -> str | None:
"""Return the requested backend, treating --force-cpu as an explicit choice."""
if args.llama_backend:
return args.llama_backend
return "cpu" if args.force_cpu else None
def _selection_payload(selection: BackendSelection) -> dict[str, Any]:
choice = selection.choice
if choice is None:
return {"prebuilt_available": False, "repo": selection.published_repo}
plan = selection.release_plans[0]
return {
"prebuilt_available": True,
"repo": selection.published_repo,
"release_tag": plan.release_tag,
"llama_tag": plan.llama_tag,
"asset": choice.name,
"install_kind": choice.install_kind,
"backend": backend_for_install_kind(choice.install_kind),
}
def _resolve_prebuilt_payload(
requested_tag: str, *, backend: str | None, args: argparse.Namespace
) -> dict[str, Any]:
"""--resolve-prebuilt for one backend, through the install path's own selector."""
route = route_backend_request(
backend = resolved_llama_backend(backend),
published_repo = args.published_repo,
published_release_tag = args.published_release_tag or "",
override_has_rocm = args.has_rocm,
override_rocm_gfx = args.rocm_gfx,
cpu_mechanism = args.cpu_fallback,
)
try:
return _selection_payload(
select_backend_install(
backend = route.backend,
llama_tag = requested_tag,
published_repo = route.published_repo,
published_release_tag = route.published_release_tag,
route = route,
)
)
except PrebuiltFallback:
return {"prebuilt_available": False, "repo": route.published_repo}
def resolve_backends_payload(
requested_tag: str,
*,
args: argparse.Namespace,
install_dir: Path | None = None,
) -> dict[str, Any]:
"""Resolves every requestable backend from one metadata snapshot, so the picker's five answers
cannot disagree and the release is not re-listed five times."""
host = detect_host()
marker = load_prebuilt_metadata(install_dir) if install_dir is not None else None
installed_repo = (marker or {}).get("published_repo")
installed_release = (marker or {}).get("release_tag") or (marker or {}).get("tag")
pinned_to = args.published_release_tag or installed_release or ""
entries: list[dict[str, Any]] = []
with _cached_metadata_fetches():
for backend in ("auto",) if host.is_macos else REQUESTABLE_BACKENDS:
entry: dict[str, Any] = {"backend": backend}
try:
selection = select_backend_install(
backend = backend,
llama_tag = requested_tag,
published_repo = args.published_repo,
published_release_tag = pinned_to,
override_has_rocm = args.has_rocm,
override_rocm_gfx = args.rocm_gfx,
host = host,
)
except BackendUnavailable as exc:
entry.update(available = False, reason = "unavailable", detail = str(exc))
except PrebuiltFallback as exc:
entry.update(available = False, reason = "no_prebuilt", detail = str(exc))
except Exception as exc: # noqa: BLE001 - one backend must not sink the list
entry.update(available = False, reason = "error", detail = str(exc))
else:
payload = _selection_payload(selection)
entry.update(
{
key: payload.get(key)
for key in ("repo", "release_tag", "llama_tag", "asset", "install_kind")
}
)
entry["resolved_backend"] = payload.get("backend")
entry["available"] = bool(payload.get("prebuilt_available"))
if entry["available"] and (
(installed_repo and entry["repo"] != installed_repo)
or (installed_release and entry["release_tag"] != installed_release)
):
# Slim whisper.cpp bundles are paired to the installed release, so a switch is an update.
entry["available"] = False
entry["detail"] = "switching this backend would leave the installed release"
if not entry["available"]:
entry["reason"] = "no_prebuilt"
entries.append(entry)
if entries and all(entry.get("reason") == "error" for entry in entries):
raise RuntimeError("could not resolve any llama.cpp backend")
return {
"requested_tag": normalized_requested_llama_tag(requested_tag),
"pinned_release_tag": pinned_to or None,
"platform": {"system": host.system, "machine": host.machine, "is_macos": host.is_macos},
"backends": entries,
}
# The fork's x64 CUDA bundles ship no cudart/cublas; full installs borrow torch's, chat-only installs
# have none and llama-server silently runs on the CPU. NVIDIA's pip wheels fill the gap, once per CUDA line.
_CUDA_RUNTIME_WHEELS = {
("windows-cuda", 12): ("nvidia-cuda-runtime-cu12", "nvidia-cublas-cu12==12.9.2.10"),
("windows-cuda", 13): ("nvidia-cuda-runtime", "nvidia-cublas"),
("linux-cuda", 12): ("nvidia-cuda-runtime-cu12", "nvidia-cublas-cu12"),
("linux-cuda", 13): ("nvidia-cuda-runtime", "nvidia-cublas"),
}
def _x64_cuda_install_kind(asset: str) -> str | None:
"""windows-cuda or linux-cuda for an x64 CUDA asset name. The marker records the asset but no
install kind, so the name is what tells an install's platform and architecture."""
if not re.search(r"cuda-?\d+", asset) or not re.search(r"-x64[-.]", asset):
return None
if re.search(r"-win(dows)?-", asset):
return "windows-cuda"
return "linux-cuda" if "-linux-" in asset else None
def _missing_cuda_runtime_wheels(install_dir: Path) -> tuple[str, ...]:
"""The wheels to install when this CUDA install lacks cudart or cublas of its major, else ()."""
asset = str((load_prebuilt_metadata(install_dir) or {}).get("asset") or "")
match = re.search(r"cuda-?(\d+)", asset)
kind = _x64_cuda_install_kind(asset)
wheels = _CUDA_RUNTIME_WHEELS.get((kind, int(match.group(1)) if match else 0))
if not wheels:
return ()
major = match.group(1)
if kind == "linux-cuda":
# The detector selection uses: a second, narrower scan would call a runtime that
# selection found (an Ollama or versioned toolkit dir) missing.
return () if f"cuda{major}" in detected_linux_runtime_lines()[0] else wheels
site_packages = Path(sysconfig.get_paths()["purelib"])
roots = [install_dir, site_packages / "torch" / "lib", site_packages / "nvidia"]
cuda_home = os.environ.get("CUDA_PATH")
if cuda_home:
roots.append(Path(cuda_home))
# The loader's own dirs (PATH, Program Files toolkit): checked flat, PATH holds System32.
flat_dirs = [Path(d) for d in windows_runtime_dirs()]
def _found(name: str) -> bool:
if any((d / name).is_file() for d in flat_dirs):
return True
for root in roots:
try:
if root.is_dir() and next(root.rglob(name), None) is not None:
return True
except OSError:
continue
return False
# ggml-cuda needs all three, as windows_runtime_line_info does; a runtime-only toolkit has cudart alone.
if all(_found(f"{lib}64_{major}.dll") for lib in ("cudart", "cublas", "cublasLt")):
return ()
return wheels
def _install_cuda_runtime_wheels(wheels: tuple[str, ...]) -> bool:
"""Installs these wheels into this env; False when the install failed."""
uv = shutil.which("uv")
cmd = (
[uv, "pip", "install", "--python", sys.executable]
if uv
else [sys.executable, "-m", "pip", "install", "--disable-pip-version-check"]
)
log(f"no CUDA runtime found for llama.cpp; installing {', '.join(wheels)}")
result = subprocess.run([*cmd, "--only-binary", ":all:", *wheels], timeout = 1800)
if result.returncode != 0:
log(
f"CUDA runtime wheel install failed (exit {result.returncode}); llama.cpp will run on the CPU"
)
return result.returncode == 0
def _linux_cuda_runtime_wheels_to_supply(host: HostInfo) -> tuple[str, ...]:
"""The wheels a Linux NVIDIA host needs before selection can pick a CUDA bundle, else ().
linux_cuda_choice_from_release only takes a CUDA bundle whose runtime line is on disk, so a
host with the driver and nothing else selects none and falls back to a source build. The
line supplied is the newest one the driver runs; CUDA 13 dropped pre-Turing GPUs, which
take CUDA 12.
"""
if not (host.is_linux and host.is_x86_64 and host.has_usable_nvidia):
return ()
pre_turing = any(int(sm) < _TURING_MIN_SM for sm in normalize_compute_caps(host.compute_caps))
# A CUDA 13 runtime cannot serve a pre-Turing GPU, so it does not count as present.
driver_lines = [
line
for line in compatible_linux_runtime_lines(host)
if not (pre_turing and int(line.removeprefix("cuda")) >= 13)
]
if any(line in driver_lines for line in detected_linux_runtime_lines()[0]):
return ()
for line in driver_lines:
major = int(line.removeprefix("cuda"))
wheels = _CUDA_RUNTIME_WHEELS.get(("linux-cuda", major))
if wheels:
return wheels
return ()
def ensure_linux_cuda_runtime_before_selection(route: BackendRoute) -> None:
"""Installs the CUDA runtime wheels a chat-only Linux NVIDIA host lacks; never raises."""
try:
if route.backend not in (None, "auto", "cuda"):
return
wheels = _linux_cuda_runtime_wheels_to_supply(route.host)
if wheels:
_install_cuda_runtime_wheels(wheels)
except Exception as exc:
log(f"CUDA runtime wheel step skipped: {exc}")
def ensure_cuda_runtime_wheels(install_dir: Path) -> None:
"""Installs NVIDIA's runtime wheels into this env when the CUDA install has none; never raises."""
try:
wheels = _missing_cuda_runtime_wheels(install_dir)
if wheels:
_install_cuda_runtime_wheels(wheels)
except Exception as exc:
log(f"CUDA runtime wheel step skipped: {exc}")
def main() -> int:
args = parse_args()
if args.check_existing_install is not None:
install_dir = Path(args.check_existing_install)
return EXIT_SUCCESS if reusable_existing_install(install_dir, detect_host()) else 1
if args.check_installed is not None:
install_dir = Path(args.check_installed)
try:
host = detect_host()
runs = _existing_install_runs(install_dir, host) and _kept_install_covers_host(
load_prebuilt_metadata(install_dir), host
)
# A CUDA install with no runtime falls back to the CPU, so let setup reinstall and fill it.
runs = runs and not _missing_cuda_runtime_wheels(install_dir)
except Exception as exc:
print(f"install check failed: {exc}", file = sys.stderr)
runs = False
return EXIT_SUCCESS if runs else EXIT_FALLBACK
if args.validate_install is not None:
try:
validate_existing_install(
Path(args.validate_install),
install_kind = args.install_kind,
)
except PrebuiltFallback as exc:
fatal = _environment_fatal_reason(exc)
if fatal:
_fail_no_space(f"install validation failed: {fatal}")
print(str(exc), file = sys.stderr)
raise SystemExit(EXIT_FALLBACK) from exc
return EXIT_SUCCESS
if args.resolve_llama_tag is not None:
resolved = resolve_requested_llama_tag(
args.resolve_llama_tag,
args.published_repo,
args.published_release_tag or "",
)
emit_resolver_output(
{
"requested_tag": normalized_requested_llama_tag(args.resolve_llama_tag),
"llama_tag": resolved,
},
output_format = args.output_format,
)
return EXIT_SUCCESS
if args.resolve_install_tag is not None:
resolved = resolve_requested_install_tag(
args.resolve_install_tag,
args.published_release_tag or "",
args.published_repo,
)
emit_resolver_output(
{
"requested_tag": normalized_requested_llama_tag(args.resolve_install_tag),
"llama_tag": resolved,
},
output_format = args.output_format,
)
return EXIT_SUCCESS
if args.resolve_source_build is not None:
plan = resolve_source_build_plan(
args.resolve_source_build,
args.published_repo,
args.published_release_tag or "",
)
emit_resolver_output(
{
"requested_tag": normalized_requested_llama_tag(args.resolve_source_build),
"source_url": plan.source_url,
"source_ref_kind": plan.source_ref_kind,
"source_ref": plan.source_ref,
"compatibility_upstream_tag": plan.compatibility_upstream_tag,
},
output_format = args.output_format,
)
return EXIT_SUCCESS
if args.resolve_prebuilt is not None:
payload = _resolve_prebuilt_payload(
args.resolve_prebuilt,
backend = requested_backend_arg(args),
args = args,
)
emit_resolver_output(payload, output_format = args.output_format)
return EXIT_SUCCESS
if args.resolve_backends is not None:
emit_resolver_output(
resolve_backends_payload(
args.resolve_backends,
args = args,
install_dir = (Path(args.install_dir).expanduser() if args.install_dir else None),
),
output_format = args.output_format,
)
return EXIT_SUCCESS
if not args.install_dir:
raise SystemExit(
"install_llama_prebuilt.py: --install-dir is required unless --resolve-llama-tag, --resolve-install-tag, or --resolve-source-build is used"
)
global _LOG_TO_STDOUT
_LOG_TO_STDOUT = True
install_arg = Path(args.install_dir).expanduser()
install_prebuilt(
install_dir = install_arg.resolve(),
llama_tag = args.llama_tag,
published_repo = args.published_repo,
published_release_tag = args.published_release_tag or "",
override_has_rocm = args.has_rocm,
override_rocm_gfx = args.rocm_gfx,
# A deliberate CPU choice comes via the backend request and persists; --cpu-fallback heals to GPU.
force_cpu = args.cpu_fallback,
llama_backend = requested_backend_arg(args),
instruction_cleanup_root = install_arg.absolute(),
)
ensure_cuda_runtime_wheels(install_arg.resolve())
return EXIT_SUCCESS
if __name__ == "__main__":
try:
raise SystemExit(main())
except SystemExit:
raise
except BusyInstallConflict as exc:
log(
f"fatal helper busy conflict: {textwrap.shorten(str(exc), width = 400, placeholder = '...')}"
)
raise SystemExit(EXIT_BUSY)
except PrebuiltFallback as exc:
# Expected when the repo has no manifest: exit quietly with EXIT_FALLBACK.
fatal = _environment_fatal_reason(exc)
if fatal:
_fail_no_space(f"prebuilt install failed: {fatal}")
log(
f"prebuilt install failed: {textwrap.shorten(str(exc), width = 400, placeholder = '...')}"
)
raise SystemExit(EXIT_FALLBACK)
except Exception as exc:
fatal = _environment_fatal_reason(exc)
if fatal:
_fail_no_space(f"prebuilt install failed: {fatal}")
message = textwrap.shorten(str(exc), width = 400, placeholder = "...")
log(f"fatal helper error: {message}")
raise SystemExit(EXIT_ERROR)