Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""A datacenter name is not NVLink evidence; P2P needs confirmed NVLink, failing closed."""
from __future__ import annotations
import ctypes
import os
import subprocess
import sys
import types
import pytest
from core.inference.llama_cpp import LlamaCppBackend
def _fake_torch(
names,
*,
hip = None,
cuda_ok = True,
):
"""torch stub: version.hip, cuda.*, get_device_properties(i).name."""
t = types.ModuleType("torch")
t.version = types.SimpleNamespace(hip = hip)
t.cuda = types.SimpleNamespace(
is_available = lambda: cuda_ok,
device_count = lambda: len(names),
get_device_properties = lambda i: types.SimpleNamespace(name = names[i]),
)
return t
# Real `nvidia-smi topo -m` from an 8x B200 host: ANSI header, NIC rows, affinity columns, legend.
TOPO_NVLINK_8X = (
"\t\x1b[4mGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tNIC0\tNIC1\t"
"CPU Affinity\tNUMA Affinity\tGPU NUMA ID\x1b[0m\n"
"GPU0\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"
"GPU1\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"
"GPU2\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"
"GPU3\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"
"GPU4\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tPIX\tPIX\t48-95,144-191\t1\t\tN/A\n"
"GPU5\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tPIX\tPIX\t48-95,144-191\t1\t\tN/A\n"
"GPU6\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNODE\tNODE\t48-95,144-191\t1\t\tN/A\n"
"GPU7\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNODE\tNODE\t48-95,144-191\t1\t\tN/A\n"
"NIC0\tSYS\tSYS\tSYS\tSYS\tPIX\tPIX\tNODE\tNODE\t X \tPIX\t\t\t\n"
"NIC1\tSYS\tSYS\tSYS\tSYS\tPIX\tPIX\tNODE\tNODE\tPIX\t X \t\t\t\n"
"\n"
"Legend:\n"
"\n"
" X = Self\n"
" SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes\n"
" NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges\n"
" NV# = Connection traversing a bonded set of # NVLinks\n"
)
# The reporter's host: 2x RTX 6000 Ada over PCIe (NODE), no NVLink, P2P still advertised.
TOPO_PCIE_2X = (
"\t\x1b[4mGPU0\tGPU1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\x1b[0m\n"
"GPU0\t X \tNODE\t0-23\t0\t\tN/A\n"
"GPU1\tNODE\t X \t0-23\t0\t\tN/A\n"
"\n"
"Legend:\n"
"\n"
" X = Self\n"
" NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges\n"
)
@pytest.fixture(autouse = True)
def _clear_cuda_visible_devices(monkeypatch):
"""Run unmasked by default (physical id == ordinal); masked tests set it."""
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
monkeypatch.delenv("CUDA_DEVICE_ORDER", raising = False)
# Captured before the autouse fixture stubs them, so the probes can run for real.
_REAL_IOMMU_IS_TRANSLATING = LlamaCppBackend.__dict__["_iommu_is_translating"].__func__
_REAL_NVML_LIBRARY = LlamaCppBackend.__dict__["_nvml_library"].__func__
def _no_nvidia_smi(*a, **k):
raise FileNotFoundError("nvidia-smi")
@pytest.fixture(autouse = True)
def _isolate_host_topology(monkeypatch):
"""Only nvidia-smi output is stubbed; NVML is absent so the topo parser runs, not a real driver."""
LlamaCppBackend._NVLINK_TOPO_CACHE = None
LlamaCppBackend._NVLINK_TOPO_GENERATION = 0
LlamaCppBackend._IOMMU_CACHE = None
monkeypatch.setattr(subprocess, "run", _no_nvidia_smi)
monkeypatch.setattr(LlamaCppBackend, "_nvml_library", staticmethod(lambda: None))
monkeypatch.delenv("UNSLOTH_P2P_TOPO_CROSSCHECK", raising = False)
monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(lambda *a: False))
monkeypatch.setattr(LlamaCppBackend, "_running_virtualized", staticmethod(lambda: False))
monkeypatch.delenv("UNSLOTH_DISABLE_DC_P2P", raising = False)
monkeypatch.delenv("UNSLOTH_FORCE_DC_P2P", raising = False)
# Default to a pinned order; the unpinned-path tests delete this themselves.
monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
LlamaCppBackend._warned_no_nvlink = False
yield
LlamaCppBackend._NVLINK_TOPO_CACHE = None
LlamaCppBackend._NVLINK_TOPO_GENERATION = 0
LlamaCppBackend._IOMMU_CACHE = None
LlamaCppBackend._warned_no_nvlink = False
def _use_topo(
monkeypatch,
text,
returncode = 0,
):
"""Feed the real parser canned `nvidia-smi topo -m` output."""
monkeypatch.setattr(
subprocess,
"run",
lambda *a, **k: types.SimpleNamespace(returncode = returncode, stdout = text, stderr = ""),
)
LlamaCppBackend._NVLINK_TOPO_CACHE = None
@pytest.mark.parametrize(
"names,expected",
[
(["NVIDIA A100-SXM4-80GB"], True),
(["NVIDIA A30"], True),
(["NVIDIA H100 80GB HBM3"], True),
(["NVIDIA H200"], True),
(["NVIDIA H800"], True),
(["NVIDIA GH200 480GB"], True),
(["NVIDIA B200"], True),
(["NVIDIA GB200"], True),
(["NVIDIA L40S"], True),
(["NVIDIA L4"], True),
(["NVIDIA RTX PRO 6000 Blackwell Server Edition"], True),
(["NVIDIA RTX 6000 Ada Generation"], True),
(["NVIDIA GeForce RTX 4090"], False),
(["NVIDIA GeForce RTX 5090"], False),
(["NVIDIA GeForce RTX 3090"], False),
(["NVIDIA GeForce RTX 2080 Ti"], False),
(["NVIDIA GeForce GTX 1080"], False),
# Short markers must not match as substrings ("a100" in "A1000").
(["NVIDIA RTX A1000 Laptop GPU"], False),
(["NVIDIA RTX A1000 6GB Laptop GPU"], False),
(["NVIDIA RTX A3000 Laptop GPU"], False),
(["NVIDIA B200", "NVIDIA B200"], True),
(["NVIDIA H100 80GB HBM3", "NVIDIA H100 80GB HBM3"], True),
(["NVIDIA B200", "NVIDIA GeForce RTX 4090"], False),
(["NVIDIA GeForce RTX 4090", "NVIDIA B200"], False),
],
)
def test_is_datacenter_gpu(monkeypatch, names, expected):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(names))
assert LlamaCppBackend._is_datacenter_gpu() is expected
def test_is_datacenter_gpu_respects_selection(monkeypatch):
monkeypatch.setitem(
sys.modules,
"torch",
_fake_torch(["NVIDIA B200", "NVIDIA GeForce RTX 4090"]),
)
assert LlamaCppBackend._is_datacenter_gpu([0]) is True
assert LlamaCppBackend._is_datacenter_gpu([1]) is False
assert LlamaCppBackend._is_datacenter_gpu([0, 1]) is False
def test_is_datacenter_gpu_out_of_range_indices_skipped(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"]))
assert LlamaCppBackend._is_datacenter_gpu([0, 5, -1]) is True
assert LlamaCppBackend._is_datacenter_gpu([5, 9]) is False
def test_is_datacenter_gpu_masked_host_physical_ids(monkeypatch):
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "4,5,6,7")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
assert LlamaCppBackend._is_datacenter_gpu([4, 5]) is True
assert LlamaCppBackend._is_datacenter_gpu([4, 5, 6, 7]) is True
assert LlamaCppBackend._is_datacenter_gpu(None) is True
assert LlamaCppBackend._is_datacenter_gpu([0, 1]) is False
def test_is_datacenter_gpu_masked_host_reordered(monkeypatch):
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "7,4,5,6")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100 80GB HBM3"] * 4))
assert LlamaCppBackend._is_datacenter_gpu([7, 4]) is True
def test_is_datacenter_gpu_masked_host_mixed_class(monkeypatch):
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "4,5")
monkeypatch.setitem(
sys.modules,
"torch",
_fake_torch(["NVIDIA GeForce RTX 4090", "NVIDIA B200"]),
)
assert LlamaCppBackend._is_datacenter_gpu([4]) is False
assert LlamaCppBackend._is_datacenter_gpu([5]) is True
assert LlamaCppBackend._is_datacenter_gpu([4, 5]) is False
def test_is_datacenter_gpu_unparsable_mask_falls_back(monkeypatch):
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "GPU-abcdef12")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"]))
assert LlamaCppBackend._is_datacenter_gpu([0]) is True
def test_is_datacenter_gpu_rocm_is_false(monkeypatch):
monkeypatch.setitem(
sys.modules,
"torch",
_fake_torch(["AMD Instinct MI300X"], hip = "6.2.0"),
)
assert LlamaCppBackend._is_datacenter_gpu() is False
def test_is_datacenter_gpu_no_cuda_is_false(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch([], cuda_ok = False))
assert LlamaCppBackend._is_datacenter_gpu() is False
def test_is_datacenter_gpu_missing_torch_is_false(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", None)
assert LlamaCppBackend._is_datacenter_gpu() is False
def test_effective_gpu_count_explicit_selection(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
assert LlamaCppBackend._effective_gpu_count([0]) == 1
assert LlamaCppBackend._effective_gpu_count([0, 1, 2]) == 3
def test_effective_gpu_count_none_uses_visible(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
assert LlamaCppBackend._effective_gpu_count(None) == 4
def test_effective_gpu_count_no_cuda_is_zero(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch([], cuda_ok = False))
assert LlamaCppBackend._effective_gpu_count(None) == 0
def test_effective_gpu_count_missing_torch_is_zero(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", None)
assert LlamaCppBackend._effective_gpu_count(None) == 0
def test_apply_env_single_dc_gpu_sets_only_fp32(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"]))
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0]) is True
assert env == {"GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "1"}
assert "GGML_CUDA_P2P" not in env
assert "CUDA_SCALE_LAUNCH_QUEUES" not in env
def test_apply_env_multi_dc_gpu_sets_all(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
assert env["GGML_CUDA_P2P"] == "1"
assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"
def test_apply_env_none_indices_uses_visible_count(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100", "NVIDIA H100"]))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, None) is True
assert env["GGML_CUDA_P2P"] == "1"
assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"
def test_apply_env_consumer_gpu_is_noop(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA GeForce RTX 4090"] * 2))
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is False
assert env == {}
def test_apply_env_user_value_wins(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
env = {
"GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "0",
"CUDA_SCALE_LAUNCH_QUEUES": "8x",
}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "0"
assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "8x"
assert env["GGML_CUDA_P2P"] == "1"
def test_apply_env_disable_flag_respected(monkeypatch):
monkeypatch.setenv("UNSLOTH_DISABLE_DC_TUNING", "1")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is False
assert env == {}
def test_apply_env_fail_open_on_detection_error(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", None)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0]) is False
assert env == {}
def test_apply_env_masked_host_multi_dc(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "4,5,6,7")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [4, 5]) is True
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
assert env["GGML_CUDA_P2P"] == "1"
assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"
def test_topo_parses_real_nvlink_table(monkeypatch):
_use_topo(monkeypatch, TOPO_NVLINK_8X)
matrix = LlamaCppBackend._nvlink_topology()
assert len(matrix) == 56
assert set(matrix.values()) == {"NV18"}
assert (0, 0) not in matrix
assert matrix[(6, 7)] == "NV18"
def test_topo_parses_pcie_table(monkeypatch):
_use_topo(monkeypatch, TOPO_PCIE_2X)
assert LlamaCppBackend._nvlink_topology() == {(0, 1): "NODE", (1, 0): "NODE"}
def test_topo_is_cached_then_refreshable(monkeypatch):
calls = []
def _once(*a, **k):
calls.append(1)
return types.SimpleNamespace(returncode = 0, stdout = TOPO_PCIE_2X, stderr = "")
monkeypatch.setattr(subprocess, "run", _once)
LlamaCppBackend._NVLINK_TOPO_CACHE = None
LlamaCppBackend._nvlink_topology()
LlamaCppBackend._nvlink_topology()
assert len(calls) == 1
LlamaCppBackend._nvlink_topology(refresh = True)
assert len(calls) == 2
def test_topo_unavailable_paths_are_none(monkeypatch):
assert LlamaCppBackend._nvlink_topology() is None
_use_topo(monkeypatch, TOPO_NVLINK_8X, returncode = 9)
assert LlamaCppBackend._nvlink_topology() is None
_use_topo(monkeypatch, "Legend:\n X = Self\n")
assert LlamaCppBackend._nvlink_topology() is None
def test_topo_truncated_row_refuses_to_guess(monkeypatch):
_use_topo(
monkeypatch,
"\tGPU0\tGPU1\tCPU Affinity\nGPU0\t X \tNV18\t0-23\nGPU1\tNV18\n",
)
assert LlamaCppBackend._nvlink_topology() is None
def test_p2p_vetoed_on_rtx_6000_ada(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
_use_topo(monkeypatch, TOPO_PCIE_2X)
reason = LlamaCppBackend._p2p_veto_reason([0, 1])
assert reason is not None and "NVLink-capable" in reason
@pytest.mark.parametrize(
"name",
["NVIDIA L40S", "NVIDIA L40", "NVIDIA L4", "NVIDIA RTX PRO 6000 Blackwell Server Edition"],
)
def test_p2p_vetoed_on_other_connectorless_parts(monkeypatch, name):
monkeypatch.setitem(sys.modules, "torch", _fake_torch([name] * 2))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
def test_p2p_allowed_on_confirmed_nvlink(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
def test_p2p_vetoed_when_topology_unknown(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
reason = LlamaCppBackend._p2p_veto_reason([0, 1])
assert reason is not None and "interconnect matrix" in reason
def test_p2p_vetoed_on_pcie_pair_between_nvlink_parts(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 2))
_use_topo(monkeypatch, TOPO_PCIE_2X)
reason = LlamaCppBackend._p2p_veto_reason([0, 1])
assert reason is not None and "NODE" in reason
def test_p2p_veto_names_the_iommu_on_bare_metal(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 2))
_use_topo(monkeypatch, TOPO_PCIE_2X)
monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(lambda *a: True))
reason = LlamaCppBackend._p2p_veto_reason([0, 1])
assert "translating IOMMU" in reason
monkeypatch.setattr(LlamaCppBackend, "_running_virtualized", staticmethod(lambda: True))
assert "IOMMU" not in LlamaCppBackend._p2p_veto_reason([0, 1])
def test_p2p_exact_mapping_consults_only_the_selected_pair(monkeypatch):
monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100"] * 3))
_use_topo(
monkeypatch,
"\tGPU0\tGPU1\tGPU2\tCPU Affinity\n"
"GPU0\t X \tNV18\tNODE\t0-23\n"
"GPU1\tNV18\t X \tNODE\t0-23\n"
"GPU2\tNODE\tNODE\t X \t0-23\n",
)
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None
assert LlamaCppBackend._p2p_veto_reason([0, 1, 2]) is not None
def test_p2p_checks_the_selected_pair_on_a_partially_linked_box(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100"] * 3))
_use_topo(
monkeypatch,
"\tGPU0\tGPU1\tGPU2\tCPU Affinity\n"
"GPU0\t X \tNV18\tNODE\t0-23\n"
"GPU1\tNV18\t X \tNODE\t0-23\n"
"GPU2\tNODE\tNODE\t X \t0-23\n",
)
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None
assert LlamaCppBackend._p2p_veto_reason(None) is not None
_use_topo(monkeypatch, TOPO_NVLINK_8X)
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
def test_apply_env_rtx_6000_ada_gets_fp32_but_not_p2p(monkeypatch):
"""10613: FP32 accum is harmless on 2x RTX 6000 Ada, but the P2P garbled every model."""
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
_use_topo(monkeypatch, TOPO_PCIE_2X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
assert "GGML_CUDA_P2P" not in env
assert env == {"GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "1", "CUDA_SCALE_LAUNCH_QUEUES": "4x"}
def test_apply_env_l40s_multi_gpu_gets_fp32_but_not_p2p(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA L40S"] * 4))
_use_topo(monkeypatch, TOPO_PCIE_2X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1, 2, 3]) is True
assert "GGML_CUDA_P2P" not in env
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
def test_sanitize_falsy_user_p2p_is_removed():
"""ggml tests GGML_CUDA_P2P for presence, so passing a user's "0" through ENABLES
peer copies; the opt-out has to unset it."""
for value in ("0", "false", "OFF", "no", "", " 0 "):
env = {"GGML_CUDA_P2P": value, "OTHER": "kept"}
assert LlamaCppBackend._sanitize_p2p_env(env) == value
assert env == {"OTHER": "kept"}, value
def test_sanitize_leaves_a_truthy_user_p2p_alone():
env = {"GGML_CUDA_P2P": "1"}
assert LlamaCppBackend._sanitize_p2p_env(env) is None
assert env == {"GGML_CUDA_P2P": "1"}
def test_opted_out_p2p_is_not_reintroduced_by_the_default(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1], p2p_opted_out = True) is True
assert "GGML_CUDA_P2P" not in env
assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
def test_disable_dc_p2p_drops_peer_flag_but_keeps_fp32(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
assert "GGML_CUDA_P2P" not in env
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
def test_force_dc_p2p_opts_back_in_over_an_unreadable_topology(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.delenv("UNSLOTH_DISABLE_DC_P2P", raising = False)
monkeypatch.setenv("UNSLOTH_FORCE_DC_P2P", "1")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
assert env["GGML_CUDA_P2P"] == "1"
def test_disable_dc_p2p_beats_force_dc_p2p(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")
monkeypatch.setenv("UNSLOTH_FORCE_DC_P2P", "1")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
assert "GGML_CUDA_P2P" not in env
def test_sanitize_applies_to_a_consumer_box_that_never_reaches_the_dc_gate():
# A 2x RTX 3090 never matches the allowlist, so only the call-site strip helps.
env = {"GGML_CUDA_P2P": "0"}
assert LlamaCppBackend._sanitize_p2p_env(env) == "0"
assert env == {}
def test_apply_env_truthy_user_p2p_still_wins(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
_use_topo(monkeypatch, TOPO_PCIE_2X)
env = {"GGML_CUDA_P2P": "1"}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
assert env["GGML_CUDA_P2P"] == "1"
def test_apply_env_single_nvlink_gpu_still_skips_p2p(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0]) is True
assert env == {"GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "1"}
def test_apply_env_multi_dc_without_nvidia_smi_withholds_p2p(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
assert "GGML_CUDA_P2P" not in env
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
def _iommu_tree(tmp_path, types_by_group):
root = tmp_path / "iommu_groups"
for group, kind in types_by_group.items():
(root / str(group)).mkdir(parents = True)
if kind is not None:
(root / str(group) / "type").write_text(f"{kind}\n")
return str(root)
def test_iommu_identity_groups_are_not_translating(tmp_path):
root = _iommu_tree(tmp_path, {0: "identity", 1: "identity"})
assert _REAL_IOMMU_IS_TRANSLATING(root) is False
def test_iommu_dma_group_is_translating(tmp_path):
root = _iommu_tree(tmp_path, {0: "identity", 1: "DMA-FQ"})
assert _REAL_IOMMU_IS_TRANSLATING(root) is True
def test_iommu_absent_or_empty_is_not_translating(tmp_path):
assert _REAL_IOMMU_IS_TRANSLATING(str(tmp_path / "nope")) is False
(tmp_path / "empty").mkdir()
assert _REAL_IOMMU_IS_TRANSLATING(str(tmp_path / "empty")) is False
def test_iommu_unreadable_types_are_unknown(tmp_path):
root = _iommu_tree(tmp_path, {0: None, 1: None})
assert _REAL_IOMMU_IS_TRANSLATING(root) is None
def test_disable_dc_p2p_also_drops_an_inherited_truthy_value(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
env = {"GGML_CUDA_P2P": "1"}
LlamaCppBackend._apply_datacenter_env(env, [0, 1])
assert "GGML_CUDA_P2P" not in env
def test_apply_env_does_not_leave_a_falsy_value_present_when_called_directly(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
env = {"GGML_CUDA_P2P": "0"}
LlamaCppBackend._apply_datacenter_env(env, [0, 1])
assert "GGML_CUDA_P2P" not in env
def test_shared_llama_server_env_builder_sanitizes_p2p(monkeypatch, tmp_path):
"""STT sidecar and embedding probe build from this, so stripping here covers every child."""
monkeypatch.setenv("GGML_CUDA_P2P", "0")
binary = tmp_path / "llama-server"
binary.write_text("", encoding = "utf-8")
env = LlamaCppBackend._llama_server_env_for_binary(str(binary))
assert "GGML_CUDA_P2P" not in env
def test_stt_sidecar_env_sanitizes_p2p(monkeypatch, tmp_path):
from core.inference import stt_mtmd_sidecar
monkeypatch.setenv("GGML_CUDA_P2P", "0")
binary = tmp_path / "llama-server"
binary.write_text("", encoding = "utf-8")
assert "GGML_CUDA_P2P" not in stt_mtmd_sidecar._llama_server_child_env(str(binary))
def test_embedding_server_env_sanitizes_p2p(monkeypatch, tmp_path):
"""Bypasses the shared builder, and a corrupt embedding would degrade retrieval silently."""
from core.rag.embed_llama_server import LlamaServerBackend
monkeypatch.setenv("GGML_CUDA_P2P", "0")
binary = tmp_path / "llama-server"
binary.write_text("", encoding = "utf-8")
server = LlamaServerBackend.__new__(LlamaServerBackend)
env = server._build_env(str(binary), use_gpu = False)
assert "GGML_CUDA_P2P" not in env
TOPO_BRIDGED_4X = (
"\t\x1b[4mGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\tNUMA Affinity\x1b[0m\n"
"GPU0\t X \tNV12\tSYS\tSYS\t0-23\t0\n"
"GPU1\tNV12\t X \tSYS\tSYS\t0-23\t0\n"
"GPU2\tSYS\tSYS\t X \tNV12\t24-47\t1\n"
"GPU3\tSYS\tSYS\tNV12\t X \t24-47\t1\n"
"\nLegend:\n\n X = Self\n"
)
def test_bridged_pair_keeps_p2p_on_a_partially_bridged_box(monkeypatch):
"""A genuinely NVLinked pair on a partially bridged box keeps P2P: gpu_indices
are already nvidia-smi indices, so the pair checked is the pair selected."""
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))
_use_topo(monkeypatch, TOPO_BRIDGED_4X)
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
assert LlamaCppBackend._p2p_veto_reason([2, 3]) is None
assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None
assert LlamaCppBackend._p2p_veto_reason([1, 3]) is not None
def test_selection_is_not_remapped_out_of_the_nvidia_smi_index_space(monkeypatch):
"""Indexes must stay in nvidia-smi space; FASTEST_FIRST remapping could enable P2P across PCIe."""
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))
_use_topo(monkeypatch, TOPO_BRIDGED_4X)
for pair in ([0, 1], [2, 3]):
assert LlamaCppBackend._p2p_veto_reason(pair) is None, pair
for pair in ([0, 2], [0, 3], [1, 2], [1, 3]):
assert LlamaCppBackend._p2p_veto_reason(pair) is not None, pair
monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None
def test_name_gate_veto_still_names_the_iommu(monkeypatch):
"""The NAME gate vetoes first, so the IOMMU diagnosis would never reach them (#10613)."""
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(lambda *a: True))
LlamaCppBackend._IOMMU_CACHE = None
_use_topo(monkeypatch, TOPO_PCIE_2X)
reason = LlamaCppBackend._p2p_veto_reason([0, 1])
assert "translating IOMMU" in reason
def test_a_raising_probe_never_fails_the_model_load(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
def boom(*a, **k):
raise OSError("sysfs exploded")
monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(boom))
monkeypatch.setattr(LlamaCppBackend, "_nvlink_topology", classmethod(boom))
LlamaCppBackend._IOMMU_CACHE = None
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
assert "GGML_CUDA_P2P" not in env
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
def test_iommu_scan_is_cached_across_loads(monkeypatch):
calls = []
def counted(*a, **k):
calls.append(1)
return True
monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(counted))
LlamaCppBackend._IOMMU_CACHE = None
for _ in range(5):
LlamaCppBackend._iommu_is_translating_cached()
assert len(calls) == 1
def _capture_warnings(monkeypatch):
"""caplog does not see structlog here, so this collects logger.warning calls instead."""
from core.inference import llama_cpp as _mod
seen: list[str] = []
real = _mod.logger.warning
monkeypatch.setattr(
_mod.logger,
"warning",
lambda msg, *a, **k: (seen.append(str(msg)), real(msg, *a, **k))[0],
)
return seen
def test_no_pcie_warning_when_the_user_opts_in_on_a_verified_fabric(monkeypatch):
"""Deliberate GGML_CUDA_P2P=1 on a confirmed NV# pair is benchmarked, so it must not warn."""
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
seen = _capture_warnings(monkeypatch)
env = {"GGML_CUDA_P2P": "1"}
LlamaCppBackend._apply_datacenter_env(env, [0, 1])
assert env["GGML_CUDA_P2P"] == "1"
assert not [m for m in seen if "peer copies" in m], seen
def test_pcie_warning_still_fires_on_an_unverified_fabric(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
_use_topo(monkeypatch, TOPO_PCIE_2X)
seen = _capture_warnings(monkeypatch)
env = {"GGML_CUDA_P2P": "1"}
LlamaCppBackend._apply_datacenter_env(env, [0, 1])
assert [m for m in seen if "peer copies stay" in m], seen
def test_masked_visible_devices_filter_needs_a_shared_index_space(monkeypatch):
"""Filtering nvidia-smi rows by CUDA_VISIBLE_DEVICES needs PCI_BUS_ID; FASTEST_FIRST can misnumber."""
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 2))
_use_topo(monkeypatch, TOPO_BRIDGED_4X)
monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")
assert LlamaCppBackend._p2p_veto_reason(None) is None
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,2")
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._p2p_veto_reason(None) is not None
monkeypatch.delenv("CUDA_DEVICE_ORDER")
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._p2p_veto_reason(None) is not None
def test_explicit_p2p_opt_out_does_not_warn_about_corruption(monkeypatch):
"""UNSLOTH_DISABLE_DC_P2P=1 returns before the topology is inspected, so calling
the fabric unconfirmed would warn about corruption on a healthy box."""
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
seen = _capture_warnings(monkeypatch)
env: dict = {}
LlamaCppBackend._apply_datacenter_env(env, [0, 1])
assert "GGML_CUDA_P2P" not in env
assert not [m for m in seen if "without a confirmed NVLink" in m], seen
def test_auto_fit_selection_without_a_pinned_order_refuses_p2p(monkeypatch):
"""Without a pinned CUDA_DEVICE_ORDER the child may use cards nvidia-smi never checked; refuse P2P."""
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))
_use_topo(monkeypatch, TOPO_BRIDGED_4X)
monkeypatch.delenv("CUDA_DEVICE_ORDER")
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
assert LlamaCppBackend._p2p_veto_reason([0, 1], launch_order_pinned = True) is None
monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
def test_auto_fit_launch_does_not_rewrite_an_inherited_device_order(monkeypatch):
"""Forcing PCI_BUS_ID for an auto-fit launch would re-read an inherited numeric
mask: a scheduler's CUDA_VISIBLE_DEVICES=0,1 meaning physical 2,0 would become
physical 0,1, running on a GPU that was hidden on purpose."""
import inspect
src = inspect.getsource(LlamaCppBackend.load_model)
branch = src[src.index("elif gpu_indices is not None and not is_vulkan_backend") :]
branch = branch[: branch.index("_launch_pinned_ids")]
pin = 'env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"'
assert pin in branch
assert "if _p2p_launch_order_pinned:" in branch[: branch.index(pin)]
# The guard must require an ABSENT inherited mask: an inherited numeric one must not be re-read.
start = src.index("_p2p_launch_order_pinned = ")
guard = src[start : src.index("\n\n", start)]
assert 'os.environ.get("CUDA_VISIBLE_DEVICES") is None' in guard, guard
def test_datacenter_box_warns_once_not_twice(monkeypatch):
"""Datacenter boxes reach _apply_datacenter_env and warn once from its veto branch."""
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
_use_topo(monkeypatch, TOPO_PCIE_2X)
assert LlamaCppBackend._is_datacenter_gpu([0, 1]) is True
seen = _capture_warnings(monkeypatch)
env = {"GGML_CUDA_P2P": "1"}
LlamaCppBackend._apply_datacenter_env(env, [0, 1])
assert len([m for m in seen if "peer copies stay" in m]) == 1, seen
def test_uniform_nvlink_box_keeps_p2p_without_a_pinned_order(monkeypatch):
"""The NVSwitch case PR #6098 benchmarked: with every pair NV#, index mapping is irrelevant as
any resolved cards are linked; pinning would strip P2P from every DGX/HGX auto-fit load."""
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
monkeypatch.delenv("CUDA_DEVICE_ORDER")
for sel in ([0, 1], [0, 7], list(range(8)), None):
assert LlamaCppBackend._p2p_veto_reason(sel) is None, sel
def test_partially_bridged_box_still_needs_a_pinned_order(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))
_use_topo(monkeypatch, TOPO_BRIDGED_4X)
monkeypatch.delenv("CUDA_DEVICE_ORDER")
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
def test_unmasked_auto_fit_keeps_p2p(monkeypatch):
"""An Auto load that fits on a subset sets gpu_indices with gpu_ids empty. With no
inherited CUDA_VISIBLE_DEVICES there is nothing to reinterpret, so the launch pins
PCI_BUS_ID and the flag is kept, sparing ordinary Auto loads on NVLink boxes."""
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))
_use_topo(monkeypatch, TOPO_BRIDGED_4X)
monkeypatch.delenv("CUDA_DEVICE_ORDER")
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
# Not uniform, so this only passes because the launch can pin the order.
assert LlamaCppBackend._p2p_veto_reason([0, 1], launch_order_pinned = True) is None
def test_p2p_opt_out_skips_the_topology_probe(monkeypatch):
"""A user who turned P2P off should not pay the nvidia-smi probe, up to a 10s timeout."""
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
probed = []
monkeypatch.setattr(
LlamaCppBackend,
"_nvlink_topology",
classmethod(lambda cls, *a, **k: (probed.append(1), None)[1]),
)
env: dict = {}
LlamaCppBackend._apply_datacenter_env(env, [0, 1], p2p_opted_out = True)
assert "GGML_CUDA_P2P" not in env
assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"
assert probed == [], "topology probed despite an explicit opt-out"
def test_row_truncated_matrix_is_rejected(monkeypatch):
"""Row-truncated output still exits 0 and parses cleanly, and the rows that
arrived can be uniformly NV#, which would read as "the whole box is NVLinked" and
enable P2P on cards whose links were never seen. Partial must mean none."""
header, *rows = TOPO_NVLINK_8X.splitlines()
truncated = "\n".join([header] + rows[:4]) + "\n"
_use_topo(monkeypatch, truncated)
assert LlamaCppBackend._nvlink_topology() is None
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))
monkeypatch.delenv("CUDA_DEVICE_ORDER")
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
def test_complete_matrix_is_still_accepted(monkeypatch):
_use_topo(monkeypatch, TOPO_NVLINK_8X)
assert len(LlamaCppBackend._nvlink_topology()) == 8 * 7
_use_topo(monkeypatch, TOPO_BRIDGED_4X)
assert len(LlamaCppBackend._nvlink_topology()) == 4 * 3
_use_topo(monkeypatch, TOPO_PCIE_2X)
assert len(LlamaCppBackend._nvlink_topology()) == 2 * 1
def test_pin_requires_ids_to_be_pci_indices(monkeypatch):
"""When the nvidia-smi query fails, _get_gpu_memory falls back to torch CUDA ordinals, and
pinning PCI order re-emits them as other cards, so Auto could run on unmeasured GPUs."""
import inspect
src = inspect.getsource(LlamaCppBackend.load_model)
start = src.index("_p2p_launch_order_pinned = ")
guard = src[start : src.index("\n\n", start)]
assert "_GPU_IDS_ARE_PCI_INDICES is True" in guard, guard
assert 'os.environ.get("CUDA_VISIBLE_DEVICES") is None' in guard, guard
def test_gpu_id_provenance_is_recorded(monkeypatch):
monkeypatch.setattr(
subprocess,
"run",
lambda *a, **k: types.SimpleNamespace(
returncode = 0,
stdout = "0, 1000, 2000\n1, 1000, 2000\n",
stderr = "",
),
)
monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b: False))
monkeypatch.setattr(
LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "llama-server")
)
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = None
assert LlamaCppBackend._get_gpu_memory("llama-server")
assert LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is True
monkeypatch.setattr(subprocess, "run", _no_nvidia_smi)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = None
LlamaCppBackend._get_gpu_memory("llama-server")
assert LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not True
_NVML_OK = 0
_NVML_ERROR = 999
class _FakeNvml:
"""Stand-in for libnvidia-ml.so.1 with real calling convention: rc out, values via byref."""
def __init__(
self,
count = 2,
linked_pairs = None,
active_links = None,
status_rc = _NVML_OK,
status_value = None,
handle_rc = _NVML_OK,
init_rc = _NVML_OK,
count_rc = _NVML_OK,
missing = (),
fail_status_after = None,
):
self.count = count
self.linked_pairs = linked_pairs # None = every pair NVLinked
self.active_links = active_links # None = every device has links
self.status_rc = status_rc
self.status_value = status_value
self.handle_rc = handle_rc
self.init_rc = init_rc
self.count_rc = count_rc
self.missing = set(missing)
self.fail_status_after = fail_status_after
self.status_calls = 0
self.shutdown_calls = 0
def _fn(self, name, impl):
if name in self.missing:
raise AttributeError(name)
return impl
@property
def nvmlInit_v2(self):
return self._fn("nvmlInit_v2", lambda: self.init_rc)
@property
def nvmlShutdown(self):
def _shutdown():
self.shutdown_calls += 1
return _NVML_OK
return _shutdown
@property
def nvmlDeviceGetCount_v2(self):
def _count(ref):
ref._obj.value = self.count
return self.count_rc
return self._fn("nvmlDeviceGetCount_v2", _count)
@property
def nvmlDeviceGetHandleByIndex_v2(self):
def _handle(index, ref):
ref._obj.value = 1000 + int(getattr(index, "value", index))
return self.handle_rc
return self._fn("nvmlDeviceGetHandleByIndex_v2", _handle)
@property
def nvmlDeviceGetNvLinkState(self):
def _state(handle, link, ref):
device = int(handle.value) - 1000
links = 4 if self.active_links is None else self.active_links[device]
if int(getattr(link, "value", link)) >= links:
return _NVML_ERROR
ref._obj.value = 1 # NVML_FEATURE_ENABLED
return _NVML_OK
return self._fn("nvmlDeviceGetNvLinkState", _state)
@property
def nvmlDeviceGetP2PStatus(self):
def _status(a, b, index, ref):
self.status_calls += 1
if self.fail_status_after is not None and self.status_calls > self.fail_status_after:
return _NVML_ERROR
if self.status_rc != _NVML_OK:
return self.status_rc
pair = (int(a.value) - 1000, int(b.value) - 1000)
linked = self.linked_pairs is None or pair in self.linked_pairs
if self.status_value is not None:
ref._obj.value = self.status_value
else:
ref._obj.value = 0 if linked else 5 # OK / NOT_SUPPORTED
return _NVML_OK
return self._fn("nvmlDeviceGetP2PStatus", _status)
def _use_nvml(monkeypatch, fake):
monkeypatch.setattr(LlamaCppBackend, "_nvml_library", staticmethod(lambda: fake))
LlamaCppBackend._NVLINK_TOPO_CACHE = None
return fake
def test_nvml_all_pairs_nvlinked_is_a_complete_matrix(monkeypatch):
_use_nvml(monkeypatch, _FakeNvml(count = 4))
matrix = LlamaCppBackend._probe_nvml_nvlink_topology()
assert set(matrix) == {(a, b) for a in range(4) for b in range(4) if a != b}
assert all(LlamaCppBackend._label_is_nvlink(v) for v in matrix.values())
def test_nvml_is_preferred_over_the_shell_out(monkeypatch):
"""The whole point: on a host where both answer, nvidia-smi is never spawned."""
_use_nvml(monkeypatch, _FakeNvml(count = 2))
calls = []
monkeypatch.setattr(
subprocess,
"run",
lambda *a, **k: calls.append(a)
or types.SimpleNamespace(returncode = 0, stdout = TOPO_PCIE_2X, stderr = ""),
)
matrix = LlamaCppBackend._nvlink_topology()
assert LlamaCppBackend._matrix_is_nvml(matrix)
assert calls == []
def test_nvml_absent_falls_back_to_topo(monkeypatch):
monkeypatch.setattr(LlamaCppBackend, "_nvml_library", staticmethod(lambda: None))
_use_topo(monkeypatch, TOPO_NVLINK_8X)
matrix = LlamaCppBackend._nvlink_topology()
assert matrix and not LlamaCppBackend._matrix_is_nvml(matrix)
@pytest.mark.parametrize(
"kwargs",
[
{"init_rc": _NVML_ERROR},
{"count_rc": _NVML_ERROR},
{"handle_rc": _NVML_ERROR},
{"status_rc": _NVML_ERROR},
{"missing": ("nvmlDeviceGetP2PStatus",)},
{"missing": ("nvmlDeviceGetNvLinkState",)},
{"count": 1},
],
)
def test_nvml_unknown_falls_through_to_topo(monkeypatch, kwargs):
"""Every way NVML can fail to answer must reach the shell-out, not a verdict."""
_use_nvml(monkeypatch, _FakeNvml(**kwargs))
assert LlamaCppBackend._probe_nvml_nvlink_topology() is None
_use_topo_after_nvml(monkeypatch, _FakeNvml(**kwargs), TOPO_NVLINK_8X)
matrix = LlamaCppBackend._nvlink_topology()
assert matrix and not LlamaCppBackend._matrix_is_nvml(matrix)
def _use_topo_after_nvml(monkeypatch, fake, text):
monkeypatch.setattr(LlamaCppBackend, "_nvml_library", staticmethod(lambda: fake))
monkeypatch.setattr(
subprocess,
"run",
lambda *a, **k: types.SimpleNamespace(returncode = 0, stdout = text, stderr = ""),
)
LlamaCppBackend._NVLINK_TOPO_CACHE = None
def test_nvml_partial_walk_is_not_a_partial_matrix(monkeypatch):
"""A mid-walk failure must discard the successes, not keep a partial matrix."""
_use_nvml(monkeypatch, _FakeNvml(count = 4, fail_status_after = 3))
assert LlamaCppBackend._probe_nvml_nvlink_topology() is None
def test_nvml_positive_without_active_links_is_a_contradiction(monkeypatch):
"""An OK on a box with no live NVLinks is unknown, not a green light."""
_use_nvml(monkeypatch, _FakeNvml(count = 2, active_links = [0, 0]))
assert LlamaCppBackend._probe_nvml_nvlink_topology() is None
def test_nvml_unlinked_pair_vetoes_p2p(monkeypatch):
"""Two islands, 0-1 and 2-3. A cross-island selection must not get P2P."""
linked = {(0, 1), (1, 0), (2, 3), (3, 2)}
_use_nvml(monkeypatch, _FakeNvml(count = 4, linked_pairs = linked))
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = True
assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
def test_nvml_matrix_needs_nvidia_smi_provenance(monkeypatch):
"""NVML alone does not prove torch ordinals were enumerated; the matrix needs nvidia-smi provenance."""
_use_nvml(
monkeypatch,
_FakeNvml(count = 4, linked_pairs = {(0, 1), (1, 0), (2, 3), (3, 2)}),
)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = False
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
LlamaCppBackend._NVLINK_TOPO_CACHE = None
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = True
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
def test_nvml_shuts_down_what_it_started(monkeypatch):
fake = _use_nvml(monkeypatch, _FakeNvml(count = 2))
LlamaCppBackend._probe_nvml_nvlink_topology()
assert fake.shutdown_calls == 1
failed = _use_nvml(monkeypatch, _FakeNvml(count = 2, init_rc = _NVML_ERROR))
LlamaCppBackend._probe_nvml_nvlink_topology()
assert failed.shutdown_calls == 0
def test_nvml_single_gpu_gives_no_verdict(monkeypatch):
"""One device cannot make a pair, and an empty matrix must never read as
'everything is NVLinked' through a vacuous all()."""
_use_nvml(monkeypatch, _FakeNvml(count = 1))
assert LlamaCppBackend._probe_nvml_nvlink_topology() is None
def test_crosscheck_prefers_topo_on_disagreement(monkeypatch):
"""When both answer and differ, the slow one wins and it is logged."""
monkeypatch.setenv("UNSLOTH_P2P_TOPO_CROSSCHECK", "1")
_use_topo_after_nvml(monkeypatch, _FakeNvml(count = 2), TOPO_PCIE_2X)
records = _capture_warnings(monkeypatch)
matrix = LlamaCppBackend._probe_interconnect_matrix()
assert not LlamaCppBackend._matrix_is_nvml(matrix)
assert any("cross-check disagreement" in r for r in records)
def test_nvml_unknown_status_is_not_a_denial(monkeypatch):
"""NVML UNKNOWN (6) or a status from a newer driver is not evidence of no NVLink; recording
NO-NVLINK would deny topo -m the chance to confirm a real fabric."""
for status in (6, 42):
_use_nvml(monkeypatch, _FakeNvml(count = 2, status_value = status))
assert LlamaCppBackend._probe_nvml_nvlink_topology() is None
for status in (1, 2, 3, 4, 5):
_use_nvml(monkeypatch, _FakeNvml(count = 2, status_value = status))
matrix = LlamaCppBackend._probe_nvml_nvlink_topology()
assert matrix and not any(LlamaCppBackend._label_is_nvlink(v) for v in matrix.values())
def test_a_failed_prime_does_not_poison_the_cache(monkeypatch):
"""The prime runs early, possibly mid driver init, so a cached miss would keep P2P off."""
monkeypatch.setattr(
LlamaCppBackend, "_probe_interconnect_matrix", classmethod(lambda cls: None)
)
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._nvlink_topology(cache_failure = False) is None
assert LlamaCppBackend._NVLINK_TOPO_CACHE is None, "a failed prime cached its miss"
assert LlamaCppBackend._nvlink_topology() is None
assert LlamaCppBackend._NVLINK_TOPO_CACHE == (None,)
def test_a_successful_prime_is_still_cached(monkeypatch):
monkeypatch.setattr(
LlamaCppBackend,
"_probe_interconnect_matrix",
classmethod(lambda cls: {(0, 1): "NVLINK", (1, 0): "NVLINK"}),
)
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._nvlink_topology(cache_failure = False) is not None
assert LlamaCppBackend._NVLINK_TOPO_CACHE is not None
def test_windows_nvml_candidates_include_the_nvsmi_directory(monkeypatch):
"""nvml.dll may sit in NVSMI rather than a DLL search path, as nvidia-smi.exe does."""
tried = []
monkeypatch.setattr(LlamaCppBackend, "_nvml_library", staticmethod(_REAL_NVML_LIBRARY))
monkeypatch.setattr(os, "name", "nt")
monkeypatch.setenv("ProgramFiles", r"C:\Program Files")
monkeypatch.setenv("SystemRoot", r"C:\Windows")
def _fake_cdll(name):
tried.append(name)
raise OSError("not here")
monkeypatch.setattr(ctypes, "CDLL", _fake_cdll)
assert LlamaCppBackend._nvml_library() is None
assert any("NVSMI" in t for t in tried), tried
assert tried[0] == "nvml.dll", "the search path should still be tried first"
def test_an_explicit_pick_keeps_its_pci_provenance(monkeypatch):
"""A torch fallback in the memory query must not clear PCI provenance for an explicit NVLinked pick."""
_use_nvml(
monkeypatch,
_FakeNvml(count = 4, linked_pairs = {(0, 1), (1, 0), (2, 3), (3, 2)}),
)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = False
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._p2p_veto_reason([0, 1], True, ids_are_pci_indices = True) is None
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._p2p_veto_reason([0, 1], True, ids_are_pci_indices = False) is not None
def test_a_stalled_nvml_call_cannot_hang_the_load(monkeypatch):
"""ctypes has no timeout, unlike the old shell-out; a wedged driver must cost a fallback."""
import threading as _threading
release = _threading.Event()
def _hang(cls):
release.wait(30)
return {(0, 1): "NVLINK", (1, 0): "NVLINK"}
monkeypatch.setattr(LlamaCppBackend, "_probe_nvml_nvlink_topology_inner", classmethod(_hang))
monkeypatch.setattr(LlamaCppBackend, "_NVML_PROBE_TIMEOUT_SECONDS", 0.2)
try:
assert LlamaCppBackend._probe_nvml_nvlink_topology() is None
_use_topo(monkeypatch, TOPO_NVLINK_8X)
matrix = LlamaCppBackend._probe_interconnect_matrix()
assert matrix and not LlamaCppBackend._matrix_is_nvml(matrix)
finally:
release.set()
def test_a_racing_failure_does_not_erase_a_published_matrix(monkeypatch):
"""A failed probe in one load must not overwrite the prime's good matrix, or P2P stays off."""
good = {(0, 1): "NVLINK", (1, 0): "NVLINK"}
LlamaCppBackend._NVLINK_TOPO_CACHE = None
monkeypatch.setattr(
LlamaCppBackend, "_probe_interconnect_matrix", classmethod(lambda cls: good)
)
assert LlamaCppBackend._nvlink_topology() == good
monkeypatch.setattr(
LlamaCppBackend, "_probe_interconnect_matrix", classmethod(lambda cls: None)
)
assert LlamaCppBackend._nvlink_topology(refresh = True) == good
assert LlamaCppBackend._NVLINK_TOPO_CACHE == (good,)
def test_the_prime_skips_work_the_opt_outs_make_useless(monkeypatch):
"""UNSLOTH_DISABLE_DC_TUNING=1, UNSLOTH_DISABLE_DC_P2P=1 or a falsy GGML_CUDA_P2P
all mean the answer can never be used, so the prime must not pay for it."""
from utils import torch_warmup
probed = []
monkeypatch.setattr(
LlamaCppBackend,
"_probe_nvml_nvlink_topology",
classmethod(lambda cls: (probed.append(1), {(0, 1): "NVLINK", (1, 0): "NVLINK"})[1]),
)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))
for var, value in (
("UNSLOTH_DISABLE_DC_TUNING", "1"),
("UNSLOTH_DISABLE_DC_P2P", "1"),
("GGML_CUDA_P2P", "0"),
):
monkeypatch.setenv(var, value)
LlamaCppBackend._NVLINK_TOPO_CACHE = None
probed.clear()
torch_warmup._prime_nvlink_topology().join(10)
assert probed == [], f"{var}={value} still primed"
monkeypatch.delenv(var, raising = False)
LlamaCppBackend._NVLINK_TOPO_CACHE = None
probed.clear()
torch_warmup._prime_nvlink_topology().join(10)
assert probed == [1]
def test_the_prime_never_blocks_the_warm_sequence(monkeypatch):
"""The probe can spend its NVML bound plus the shell-out timeout, which would delay every warm
stage behind it and tip timing tests on a loaded CI worker."""
from utils import torch_warmup
import threading as _threading
import time as _time
release = _threading.Event()
def _slow(cls, **kw):
release.wait(30)
return {(0, 1): "NVLINK", (1, 0): "NVLINK"}
# The prime calls _probe_nvml_nvlink_topology, not _nvlink_topology, so patch that.
monkeypatch.setattr(LlamaCppBackend, "_probe_nvml_nvlink_topology", classmethod(_slow))
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))
try:
started = _time.perf_counter()
worker = torch_warmup._prime_nvlink_topology()
elapsed = _time.perf_counter() - started
assert elapsed < 1.0, f"the prime blocked its caller for {elapsed:.2f}s"
assert worker.is_alive()
finally:
release.set()
worker.join(10)
def test_a_uniform_fabric_does_not_need_pci_provenance(monkeypatch):
"""Where every pair is NVLinked, which ids name which cards cannot change the verdict, so
torch-ordinal provenance must not veto, as for an unpinned device order."""
_use_nvml(monkeypatch, _FakeNvml(count = 4))
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = False
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
linked = {(0, 1), (1, 0), (2, 3), (3, 2)}
_use_nvml(monkeypatch, _FakeNvml(count = 4, linked_pairs = linked))
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = False
assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
LlamaCppBackend._NVLINK_TOPO_CACHE = None
assert LlamaCppBackend._p2p_veto_reason([0, 1], True, ids_are_pci_indices = True) is None
def test_ids_outside_a_uniform_matrix_still_veto(monkeypatch):
"""The uniform escape must not be a blanket pass: uncovered ids stay unknown."""
_use_nvml(monkeypatch, _FakeNvml(count = 2))
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = False
assert LlamaCppBackend._p2p_veto_reason([0, 5]) is not None
def test_explicitness_is_not_evidence_of_pci_indexing(monkeypatch):
"""Explicitness is not PCI evidence: a FASTEST_FIRST override makes the picker's ids torch ordinals."""
linked = {(0, 1), (1, 0), (2, 3), (3, 2)}
_use_nvml(monkeypatch, _FakeNvml(count = 4, linked_pairs = linked))
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = False
monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
assert LlamaCppBackend._p2p_veto_reason([0, 1], True, ids_are_pci_indices = False) is not None
LlamaCppBackend._NVLINK_TOPO_CACHE = None
monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
assert LlamaCppBackend._p2p_veto_reason([0, 1], True, ids_are_pci_indices = True) is None
def test_the_prime_never_spawns_the_shell_out(monkeypatch):
"""The prime must not run the nvidia-smi shell-out, which can stall other timing-sensitive tests."""
calls = []
monkeypatch.setattr(
subprocess,
"run",
lambda *a, **k: calls.append(a)
or types.SimpleNamespace(returncode = 0, stdout = TOPO_NVLINK_8X, stderr = ""),
)
monkeypatch.setattr(
LlamaCppBackend, "_probe_nvml_nvlink_topology", classmethod(lambda cls: None)
)
LlamaCppBackend._NVLINK_TOPO_CACHE = None
LlamaCppBackend.prime_nvlink_topology()
assert calls == [], "the prime reached the nvidia-smi fallback"
assert LlamaCppBackend._NVLINK_TOPO_CACHE is None, "a failed prime cached its miss"
assert LlamaCppBackend._nvlink_topology() is not None
assert calls
def test_the_prime_publishes_an_nvml_success(monkeypatch):
good = {(0, 1): "NVLINK", (1, 0): "NVLINK"}
monkeypatch.setattr(
LlamaCppBackend, "_probe_nvml_nvlink_topology", classmethod(lambda cls: good)
)
LlamaCppBackend._NVLINK_TOPO_CACHE = None
LlamaCppBackend.prime_nvlink_topology()
assert LlamaCppBackend._NVLINK_TOPO_CACHE == (good,)
def test_the_prime_does_not_retire_the_cross_check(monkeypatch):
"""UNSLOTH_P2P_TOPO_CROSSCHECK=1 must survive the prime, which otherwise answers from NVML alone."""
monkeypatch.setenv("UNSLOTH_P2P_TOPO_CROSSCHECK", "1")
nvml = {(0, 1): "NVLINK", (1, 0): "NVLINK"}
topo = {(0, 1): "SYS", (1, 0): "SYS"}
monkeypatch.setattr(
LlamaCppBackend, "_probe_nvml_nvlink_topology", classmethod(lambda cls: nvml)
)
monkeypatch.setattr(LlamaCppBackend, "_probe_nvlink_topology", staticmethod(lambda: topo))
LlamaCppBackend._NVLINK_TOPO_CACHE = None
LlamaCppBackend.prime_nvlink_topology()
assert LlamaCppBackend._NVLINK_TOPO_CACHE is None, (
"the prime published under cross-check, so the load path will read the cache "
"and the two tiers will never be compared"
)
assert LlamaCppBackend._nvlink_topology() == topo
monkeypatch.delenv("UNSLOTH_P2P_TOPO_CROSSCHECK", raising = False)
LlamaCppBackend._NVLINK_TOPO_CACHE = None
LlamaCppBackend.prime_nvlink_topology()
assert LlamaCppBackend._NVLINK_TOPO_CACHE == (nvml,)
def test_cross_check_survives_the_full_warm_then_load_sequence(monkeypatch):
"""Warm stage primes on its own thread, then a load hits the gate; before the prime read
UNSLOTH_P2P_TOPO_CROSSCHECK, zero nvidia-smi ran and NVML went unchallenged."""
from utils import torch_warmup
monkeypatch.setenv("UNSLOTH_P2P_TOPO_CROSSCHECK", "1")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))
nvml = {(a, b): "NVLINK" for a in range(8) for b in range(8) if a != b}
topo_calls = []
def _topo():
topo_calls.append(1)
return {(a, b): "SYS" for a in range(8) for b in range(8) if a != b}
monkeypatch.setattr(
LlamaCppBackend, "_probe_nvml_nvlink_topology", classmethod(lambda cls: nvml)
)
monkeypatch.setattr(LlamaCppBackend, "_probe_nvlink_topology", staticmethod(_topo))
LlamaCppBackend._NVLINK_TOPO_CACHE = None
torch_warmup._prime_nvlink_topology().join(10)
assert LlamaCppBackend._p2p_veto_reason([0, 1], True, True) is not None
assert topo_calls, "the cross-check never ran the slow tier"